{"action":"created","comment":{"url":"https://api.github.com/repos/Mondego/SourcererCC/pulls/comments/65788852","id":65788852,"diff_hunk":"@@ -0,0 +1,88 @@\n+const esprima = require('esprima')\n+const escodegen = require('escodegen')\n+const fs = require('fs-extra-promise')\n+const tokenizer = require('./tokenizer')\n+\n+const immutable = require('immutable')\n+const walk = require('esprima-walk')\n+\n+const { base64FileName } = require('./util')\n+\n+const estools = require('estools')\n+\n+const TOKENIZER_SCOPE_FILE = 'file-scope'\n+const TOKENIZER_SCOPE_FUNCTION = 'function-scope'\n+\n+const TOKENIZER_SCOPE = TOKENIZER_SCOPE_FILE\n+\n+// TODO: estools map / filter / traverse (instead of walk)\n+// - filter subfunctions from fuction asts somehow\n+// - test on SCC\n+\n+// TODO: get rid of the function block and indentation\n+const regenerateFunctionCode = function(functionAst) {\n+  codegenOptions = { // NOTE: doesn't help\n+    format: {\n+      parentheses: false\n+    }\n+  }\n+\n+  // NOTE: functionAst.body ommits the function signature (returns block only)\n+  return escodegen.generate(functionAst.body, {})\n+}\n+\n+const processFile = function(fileName, data) {\n+  //let parentId = base64FileName(fileName) // TODO: incorporate repo name / hash\n+  let parentId = fileName\n+  let blockId = 1\n+\n+  if (TOKENIZER_SCOPE === TOKENIZER_SCOPE_FILE) {\n+    return immutable.List.of(tokenizer(data, parentId, blockId))","path":"tokenizers/javascript/index.js","position":40,"original_position":40,"commit_id":"457ff40ed8ef42a6de6e20d4a614fbd4a014d9b8","original_commit_id":"457ff40ed8ef42a6de6e20d4a614fbd4a014d9b8","user":{"login":"saini","id":1272237,"avatar_url":"https://avatars.githubusercontent.com/u/1272237?v=3","gravatar_id":"","url":"https://api.github.com/users/saini","html_url":"https://github.com/saini","followers_url":"https://api.github.com/users/saini/followers","following_url":"https://api.github.com/users/saini/following{/other_user}","gists_url":"https://api.github.com/users/saini/gists{/gist_id}","starred_url":"https://api.github.com/users/saini/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/saini/subscriptions","organizations_url":"https://api.github.com/users/saini/orgs","repos_url":"https://api.github.com/users/saini/repos","events_url":"https://api.github.com/users/saini/events{/privacy}","received_events_url":"https://api.github.com/users/saini/received_events","type":"User","site_admin":false},"body":"let's increment the blockId before we call the tokenizer. Or do you increment it somewhere else?","created_at":"2016-06-03T23:14:14Z","updated_at":"2016-06-03T23:14:14Z","html_url":"https://github.com/Mondego/SourcererCC/pull/4#discussion_r65788852","pull_request_url":"https://api.github.com/repos/Mondego/SourcererCC/pulls/4","_links":{"self":{"href":"https://api.github.com/repos/Mondego/SourcererCC/pulls/comments/65788852"},"html":{"href":"https://github.com/Mondego/SourcererCC/pull/4#discussion_r65788852"},"pull_request":{"href":"https://api.github.com/repos/Mondego/SourcererCC/pulls/4"}}},"pull_request":{"url":"https://api.github.com/repos/Mondego/SourcererCC/pulls/4","id":72530306,"html_url":"https://github.com/Mondego/SourcererCC/pull/4","diff_url":"https://github.com/Mondego/SourcererCC/pull/4.diff","patch_url":"https://github.com/Mondego/SourcererCC/pull/4.patch","issue_url":"https://api.github.com/repos/Mondego/SourcererCC/issues/4","number":4,"state":"open","locked":false,"title":"WIP: add javascript tokenizer","user":{"login":"jakubzitny","id":3315662,"avatar_url":"https://avatars.githubusercontent.com/u/3315662?v=3","gravatar_id":"","url":"https://api.github.com/users/jakubzitny","html_url":"https://github.com/jakubzitny","followers_url":"https://api.github.com/users/jakubzitny/followers","following_url":"https://api.github.com/users/jakubzitny/following{/other_user}","gists_url":"https://api.github.com/users/jakubzitny/gists{/gist_id}","starred_url":"https://api.github.com/users/jakubzitny/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakubzitny/subscriptions","organizations_url":"https://api.github.com/users/jakubzitny/orgs","repos_url":"https://api.github.com/users/jakubzitny/repos","events_url":"https://api.github.com/users/jakubzitny/events{/privacy}","received_events_url":"https://api.github.com/users/jakubzitny/received_events","type":"User","site_admin":false},"body":"Hey,\r\n\r\nI am [working](https://git.io/voeEf) (in coordination with @crista) on a javascript tokenizer for SourcererCC. This is the first working concept written in `node.js`, so we can test it out and communicate better.\r\n\r\nThis is a Work In Progress.\r\nI don't think this is good enough to be merged yet, we should think about better way to manage the tokenizer \"modules\" in SourcererCC. It would be great to be able to switch between tokenizers in the the `.properties` config and to have (at least roughly) the same api. What do you think? Maybe \r\n\r\nI will post updated version with better preprocessing of JS files and a script for generically running the tokenization.","created_at":"2016-06-03T15:59:54Z","updated_at":"2016-06-03T23:14:14Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a69c4e38a5870d8dce84df24e0e32374db04a7c3","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/Mondego/SourcererCC/pulls/4/commits","review_comments_url":"https://api.github.com/repos/Mondego/SourcererCC/pulls/4/comments","review_comment_url":"https://api.github.com/repos/Mondego/SourcererCC/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Mondego/SourcererCC/issues/4/comments","statuses_url":"https://api.github.com/repos/Mondego/SourcererCC/statuses/457ff40ed8ef42a6de6e20d4a614fbd4a014d9b8","head":{"label":"jakubzitny:javascript-tokenizer","ref":"javascript-tokenizer","sha":"457ff40ed8ef42a6de6e20d4a614fbd4a014d9b8","user":{"login":"jakubzitny","id":3315662,"avatar_url":"https://avatars.githubusercontent.com/u/3315662?v=3","gravatar_id":"","url":"https://api.github.com/users/jakubzitny","html_url":"https://github.com/jakubzitny","followers_url":"https://api.github.com/users/jakubzitny/followers","following_url":"https://api.github.com/users/jakubzitny/following{/other_user}","gists_url":"https://api.github.com/users/jakubzitny/gists{/gist_id}","starred_url":"https://api.github.com/users/jakubzitny/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakubzitny/subscriptions","organizations_url":"https://api.github.com/users/jakubzitny/orgs","repos_url":"https://api.github.com/users/jakubzitny/repos","events_url":"https://api.github.com/users/jakubzitny/events{/privacy}","received_events_url":"https://api.github.com/users/jakubzitny/received_events","type":"User","site_admin":false},"repo":{"id":58219241,"name":"SourcererCC","full_name":"jakubzitny/SourcererCC","owner":{"login":"jakubzitny","id":3315662,"avatar_url":"https://avatars.githubusercontent.com/u/3315662?v=3","gravatar_id":"","url":"https://api.github.com/users/jakubzitny","html_url":"https://github.com/jakubzitny","followers_url":"https://api.github.com/users/jakubzitny/followers","following_url":"https://api.github.com/users/jakubzitny/following{/other_user}","gists_url":"https://api.github.com/users/jakubzitny/gists{/gist_id}","starred_url":"https://api.github.com/users/jakubzitny/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakubzitny/subscriptions","organizations_url":"https://api.github.com/users/jakubzitny/orgs","repos_url":"https://api.github.com/users/jakubzitny/repos","events_url":"https://api.github.com/users/jakubzitny/events{/privacy}","received_events_url":"https://api.github.com/users/jakubzitny/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jakubzitny/SourcererCC","description":"Sourcerer's Code Clone project","fork":true,"url":"https://api.github.com/repos/jakubzitny/SourcererCC","forks_url":"https://api.github.com/repos/jakubzitny/SourcererCC/forks","keys_url":"https://api.github.com/repos/jakubzitny/SourcererCC/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jakubzitny/SourcererCC/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jakubzitny/SourcererCC/teams","hooks_url":"https://api.github.com/repos/jakubzitny/SourcererCC/hooks","issue_events_url":"https://api.github.com/repos/jakubzitny/SourcererCC/issues/events{/number}","events_url":"https://api.github.com/repos/jakubzitny/SourcererCC/events","assignees_url":"https://api.github.com/repos/jakubzitny/SourcererCC/assignees{/user}","branches_url":"https://api.github.com/repos/jakubzitny/SourcererCC/branches{/branch}","tags_url":"https://api.github.com/repos/jakubzitny/SourcererCC/tags","blobs_url":"https://api.github.com/repos/jakubzitny/SourcererCC/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jakubzitny/SourcererCC/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jakubzitny/SourcererCC/git/refs{/sha}","trees_url":"https://api.github.com/repos/jakubzitny/SourcererCC/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jakubzitny/SourcererCC/statuses/{sha}","languages_url":"https://api.github.com/repos/jakubzitny/SourcererCC/languages","stargazers_url":"https://api.github.com/repos/jakubzitny/SourcererCC/stargazers","contributors_url":"https://api.github.com/repos/jakubzitny/SourcererCC/contributors","subscribers_url":"https://api.github.com/repos/jakubzitny/SourcererCC/subscribers","subscription_url":"https://api.github.com/repos/jakubzitny/SourcererCC/subscription","commits_url":"https://api.github.com/repos/jakubzitny/SourcererCC/commits{/sha}","git_commits_url":"https://api.github.com/repos/jakubzitny/SourcererCC/git/commits{/sha}","comments_url":"https://api.github.com/repos/jakubzitny/SourcererCC/comments{/number}","issue_comment_url":"https://api.github.com/repos/jakubzitny/SourcererCC/issues/comments{/number}","contents_url":"https://api.github.com/repos/jakubzitny/SourcererCC/contents/{+path}","compare_url":"https://api.github.com/repos/jakubzitny/SourcererCC/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jakubzitny/SourcererCC/merges","archive_url":"https://api.github.com/repos/jakubzitny/SourcererCC/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jakubzitny/SourcererCC/downloads","issues_url":"https://api.github.com/repos/jakubzitny/SourcererCC/issues{/number}","pulls_url":"https://api.github.com/repos/jakubzitny/SourcererCC/pulls{/number}","milestones_url":"https://api.github.com/repos/jakubzitny/SourcererCC/milestones{/number}","notifications_url":"https://api.github.com/repos/jakubzitny/SourcererCC/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jakubzitny/SourcererCC/labels{/name}","releases_url":"https://api.github.com/repos/jakubzitny/SourcererCC/releases{/id}","deployments_url":"https://api.github.com/repos/jakubzitny/SourcererCC/deployments","created_at":"2016-05-06T16:09:17Z","updated_at":"2016-05-06T16:09:18Z","pushed_at":"2016-06-03T15:51:51Z","git_url":"git://github.com/jakubzitny/SourcererCC.git","ssh_url":"git@github.com:jakubzitny/SourcererCC.git","clone_url":"https://github.com/jakubzitny/SourcererCC.git","svn_url":"https://github.com/jakubzitny/SourcererCC","homepage":null,"size":52580,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"Mondego:master","ref":"master","sha":"3c4473903963c7e4fddef1da37b2a239100f610d","user":{"login":"Mondego","id":3903688,"avatar_url":"https://avatars.githubusercontent.com/u/3903688?v=3","gravatar_id":"","url":"https://api.github.com/users/Mondego","html_url":"https://github.com/Mondego","followers_url":"https://api.github.com/users/Mondego/followers","following_url":"https://api.github.com/users/Mondego/following{/other_user}","gists_url":"https://api.github.com/users/Mondego/gists{/gist_id}","starred_url":"https://api.github.com/users/Mondego/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Mondego/subscriptions","organizations_url":"https://api.github.com/users/Mondego/orgs","repos_url":"https://api.github.com/users/Mondego/repos","events_url":"https://api.github.com/users/Mondego/events{/privacy}","received_events_url":"https://api.github.com/users/Mondego/received_events","type":"Organization","site_admin":false},"repo":{"id":41764752,"name":"SourcererCC","full_name":"Mondego/SourcererCC","owner":{"login":"Mondego","id":3903688,"avatar_url":"https://avatars.githubusercontent.com/u/3903688?v=3","gravatar_id":"","url":"https://api.github.com/users/Mondego","html_url":"https://github.com/Mondego","followers_url":"https://api.github.com/users/Mondego/followers","following_url":"https://api.github.com/users/Mondego/following{/other_user}","gists_url":"https://api.github.com/users/Mondego/gists{/gist_id}","starred_url":"https://api.github.com/users/Mondego/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Mondego/subscriptions","organizations_url":"https://api.github.com/users/Mondego/orgs","repos_url":"https://api.github.com/users/Mondego/repos","events_url":"https://api.github.com/users/Mondego/events{/privacy}","received_events_url":"https://api.github.com/users/Mondego/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Mondego/SourcererCC","description":"Sourcerer's Code Clone project","fork":false,"url":"https://api.github.com/repos/Mondego/SourcererCC","forks_url":"https://api.github.com/repos/Mondego/SourcererCC/forks","keys_url":"https://api.github.com/repos/Mondego/SourcererCC/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Mondego/SourcererCC/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Mondego/SourcererCC/teams","hooks_url":"https://api.github.com/repos/Mondego/SourcererCC/hooks","issue_events_url":"https://api.github.com/repos/Mondego/SourcererCC/issues/events{/number}","events_url":"https://api.github.com/repos/Mondego/SourcererCC/events","assignees_url":"https://api.github.com/repos/Mondego/SourcererCC/assignees{/user}","branches_url":"https://api.github.com/repos/Mondego/SourcererCC/branches{/branch}","tags_url":"https://api.github.com/repos/Mondego/SourcererCC/tags","blobs_url":"https://api.github.com/repos/Mondego/SourcererCC/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Mondego/SourcererCC/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Mondego/SourcererCC/git/refs{/sha}","trees_url":"https://api.github.com/repos/Mondego/SourcererCC/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Mondego/SourcererCC/statuses/{sha}","languages_url":"https://api.github.com/repos/Mondego/SourcererCC/languages","stargazers_url":"https://api.github.com/repos/Mondego/SourcererCC/stargazers","contributors_url":"https://api.github.com/repos/Mondego/SourcererCC/contributors","subscribers_url":"https://api.github.com/repos/Mondego/SourcererCC/subscribers","subscription_url":"https://api.github.com/repos/Mondego/SourcererCC/subscription","commits_url":"https://api.github.com/repos/Mondego/SourcererCC/commits{/sha}","git_commits_url":"https://api.github.com/repos/Mondego/SourcererCC/git/commits{/sha}","comments_url":"https://api.github.com/repos/Mondego/SourcererCC/comments{/number}","issue_comment_url":"https://api.github.com/repos/Mondego/SourcererCC/issues/comments{/number}","contents_url":"https://api.github.com/repos/Mondego/SourcererCC/contents/{+path}","compare_url":"https://api.github.com/repos/Mondego/SourcererCC/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Mondego/SourcererCC/merges","archive_url":"https://api.github.com/repos/Mondego/SourcererCC/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Mondego/SourcererCC/downloads","issues_url":"https://api.github.com/repos/Mondego/SourcererCC/issues{/number}","pulls_url":"https://api.github.com/repos/Mondego/SourcererCC/pulls{/number}","milestones_url":"https://api.github.com/repos/Mondego/SourcererCC/milestones{/number}","notifications_url":"https://api.github.com/repos/Mondego/SourcererCC/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Mondego/SourcererCC/labels{/name}","releases_url":"https://api.github.com/repos/Mondego/SourcererCC/releases{/id}","deployments_url":"https://api.github.com/repos/Mondego/SourcererCC/deployments","created_at":"2015-09-01T21:36:58Z","updated_at":"2016-05-31T12:30:24Z","pushed_at":"2016-06-03T15:59:54Z","git_url":"git://github.com/Mondego/SourcererCC.git","ssh_url":"git@github.com:Mondego/SourcererCC.git","clone_url":"https://github.com/Mondego/SourcererCC.git","svn_url":"https://github.com/Mondego/SourcererCC","homepage":null,"size":52602,"stargazers_count":12,"watchers_count":12,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":3,"forks":3,"open_issues":3,"watchers":12,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Mondego/SourcererCC/pulls/4"},"html":{"href":"https://github.com/Mondego/SourcererCC/pull/4"},"issue":{"href":"https://api.github.com/repos/Mondego/SourcererCC/issues/4"},"comments":{"href":"https://api.github.com/repos/Mondego/SourcererCC/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/Mondego/SourcererCC/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/Mondego/SourcererCC/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Mondego/SourcererCC/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/Mondego/SourcererCC/statuses/457ff40ed8ef42a6de6e20d4a614fbd4a014d9b8"}}}}
{"actor":{"display_login":"saini"}}