{"action":"created","comment":{"url":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/comments/85631874","pull_request_review_id":6340050,"id":85631874,"diff_hunk":"@@ -1,19 +1,21 @@\n-function tokenizer(string, delimiters, punctuation) {\n-  var arrayOfStrings= string.split(delimiters, punctuation);\n+function tokenize(string, delimiters, punctuation) {\n \n-new regExp('\\ ', 'i','g');\n-var delimitersArray = new regExp('\\ ', 'i','g');\n-   str.join()\n+  //validation\n+  //check that string is a string\n+  //check that delimiters and punctuation is an array\n+  //throw an error if any of those are false\n \n+  var delimiterPattern = '[' + delimiters.join('') + ']';\n+  var punctuationPattern = '[' + punctuation.join('') + ']';\n \n-new regExp('\\?', '\\.', '\\!', 'i','g');\n-var punctuationArray = new regExp('\\?', '\\.', '\\!', 'i','g');\n-  str.replace('\\?', '\\.', '\\!', '\\ ');\n-  str.join()\n+  var delimiterRegExp = new RegExp(delimiterPattern);\n+  var punctuationRegExp = new RegExp(punctuationPattern);\n \n-return arrayOfStrings\n+  // strip string of punctuation (uses .replace(..., ''))\n+  // split the string based on delimiters (uses .split())\n+  // return the array of words\n \n+  var stripPunctuation= string.replace(punctuationRegExp, '');","path":"tokenize.js","position":null,"original_position":28,"commit_id":"d4b691f2fcada036cab72f69ec0976d1c87f5b84","original_commit_id":"d4b691f2fcada036cab72f69ec0976d1c87f5b84","user":{"login":"dwhieb","id":5892997,"avatar_url":"https://avatars.githubusercontent.com/u/5892997?v=3","gravatar_id":"","url":"https://api.github.com/users/dwhieb","html_url":"https://github.com/dwhieb","followers_url":"https://api.github.com/users/dwhieb/followers","following_url":"https://api.github.com/users/dwhieb/following{/other_user}","gists_url":"https://api.github.com/users/dwhieb/gists{/gist_id}","starred_url":"https://api.github.com/users/dwhieb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwhieb/subscriptions","organizations_url":"https://api.github.com/users/dwhieb/orgs","repos_url":"https://api.github.com/users/dwhieb/repos","events_url":"https://api.github.com/users/dwhieb/events{/privacy}","received_events_url":"https://api.github.com/users/dwhieb/received_events","type":"User","site_admin":false},"body":"Let's call this something like `withoutPunctuation` or `noPunctuation`. Oh! How about, `depunctuated` or `unpunctuated`. You decide. You generally want to give static variables names that are nouns or adjectives, and give functions names that are verbs.","created_at":"2016-10-29T02:12:28Z","updated_at":"2016-10-29T02:26:26Z","html_url":"https://github.com/digitallinguistics/dlx-js/pull/30#discussion_r85631874","pull_request_url":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/30","_links":{"self":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/comments/85631874"},"html":{"href":"https://github.com/digitallinguistics/dlx-js/pull/30#discussion_r85631874"},"pull_request":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/30"}}},"pull_request":{"url":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/30","id":90496861,"html_url":"https://github.com/digitallinguistics/dlx-js/pull/30","diff_url":"https://github.com/digitallinguistics/dlx-js/pull/30.diff","patch_url":"https://github.com/digitallinguistics/dlx-js/pull/30.patch","issue_url":"https://api.github.com/repos/digitallinguistics/dlx-js/issues/30","number":30,"state":"open","locked":false,"title":"Tokenize","user":{"login":"dwhieb","id":5892997,"avatar_url":"https://avatars.githubusercontent.com/u/5892997?v=3","gravatar_id":"","url":"https://api.github.com/users/dwhieb","html_url":"https://github.com/dwhieb","followers_url":"https://api.github.com/users/dwhieb/followers","following_url":"https://api.github.com/users/dwhieb/following{/other_user}","gists_url":"https://api.github.com/users/dwhieb/gists{/gist_id}","starred_url":"https://api.github.com/users/dwhieb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwhieb/subscriptions","organizations_url":"https://api.github.com/users/dwhieb/orgs","repos_url":"https://api.github.com/users/dwhieb/repos","events_url":"https://api.github.com/users/dwhieb/events{/privacy}","received_events_url":"https://api.github.com/users/dwhieb/received_events","type":"User","site_admin":false},"body":"@majaalexis11 Awesome job! I'm going to make some comments directly in the code, and some in this thread as well.","created_at":"2016-10-22T21:53:33Z","updated_at":"2016-10-29T02:26:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f5a0e192011387389ac760547fe4361abdd20043","assignee":{"login":"dwhieb","id":5892997,"avatar_url":"https://avatars.githubusercontent.com/u/5892997?v=3","gravatar_id":"","url":"https://api.github.com/users/dwhieb","html_url":"https://github.com/dwhieb","followers_url":"https://api.github.com/users/dwhieb/followers","following_url":"https://api.github.com/users/dwhieb/following{/other_user}","gists_url":"https://api.github.com/users/dwhieb/gists{/gist_id}","starred_url":"https://api.github.com/users/dwhieb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwhieb/subscriptions","organizations_url":"https://api.github.com/users/dwhieb/orgs","repos_url":"https://api.github.com/users/dwhieb/repos","events_url":"https://api.github.com/users/dwhieb/events{/privacy}","received_events_url":"https://api.github.com/users/dwhieb/received_events","type":"User","site_admin":false},"assignees":[{"login":"dwhieb","id":5892997,"avatar_url":"https://avatars.githubusercontent.com/u/5892997?v=3","gravatar_id":"","url":"https://api.github.com/users/dwhieb","html_url":"https://github.com/dwhieb","followers_url":"https://api.github.com/users/dwhieb/followers","following_url":"https://api.github.com/users/dwhieb/following{/other_user}","gists_url":"https://api.github.com/users/dwhieb/gists{/gist_id}","starred_url":"https://api.github.com/users/dwhieb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwhieb/subscriptions","organizations_url":"https://api.github.com/users/dwhieb/orgs","repos_url":"https://api.github.com/users/dwhieb/repos","events_url":"https://api.github.com/users/dwhieb/events{/privacy}","received_events_url":"https://api.github.com/users/dwhieb/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/30/commits","review_comments_url":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/30/comments","review_comment_url":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/comments{/number}","comments_url":"https://api.github.com/repos/digitallinguistics/dlx-js/issues/30/comments","statuses_url":"https://api.github.com/repos/digitallinguistics/dlx-js/statuses/d4b691f2fcada036cab72f69ec0976d1c87f5b84","head":{"label":"digitallinguistics:tokenize","ref":"tokenize","sha":"d4b691f2fcada036cab72f69ec0976d1c87f5b84","user":{"login":"digitallinguistics","id":5896943,"avatar_url":"https://avatars.githubusercontent.com/u/5896943?v=3","gravatar_id":"","url":"https://api.github.com/users/digitallinguistics","html_url":"https://github.com/digitallinguistics","followers_url":"https://api.github.com/users/digitallinguistics/followers","following_url":"https://api.github.com/users/digitallinguistics/following{/other_user}","gists_url":"https://api.github.com/users/digitallinguistics/gists{/gist_id}","starred_url":"https://api.github.com/users/digitallinguistics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/digitallinguistics/subscriptions","organizations_url":"https://api.github.com/users/digitallinguistics/orgs","repos_url":"https://api.github.com/users/digitallinguistics/repos","events_url":"https://api.github.com/users/digitallinguistics/events{/privacy}","received_events_url":"https://api.github.com/users/digitallinguistics/received_events","type":"Organization","site_admin":false},"repo":{"id":47965106,"name":"dlx-js","full_name":"digitallinguistics/dlx-js","owner":{"login":"digitallinguistics","id":5896943,"avatar_url":"https://avatars.githubusercontent.com/u/5896943?v=3","gravatar_id":"","url":"https://api.github.com/users/digitallinguistics","html_url":"https://github.com/digitallinguistics","followers_url":"https://api.github.com/users/digitallinguistics/followers","following_url":"https://api.github.com/users/digitallinguistics/following{/other_user}","gists_url":"https://api.github.com/users/digitallinguistics/gists{/gist_id}","starred_url":"https://api.github.com/users/digitallinguistics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/digitallinguistics/subscriptions","organizations_url":"https://api.github.com/users/digitallinguistics/orgs","repos_url":"https://api.github.com/users/digitallinguistics/repos","events_url":"https://api.github.com/users/digitallinguistics/events{/privacy}","received_events_url":"https://api.github.com/users/digitallinguistics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/digitallinguistics/dlx-js","description":"A JavaScript library for working with linguistic data in DLx format. For browser and Node.","fork":false,"url":"https://api.github.com/repos/digitallinguistics/dlx-js","forks_url":"https://api.github.com/repos/digitallinguistics/dlx-js/forks","keys_url":"https://api.github.com/repos/digitallinguistics/dlx-js/keys{/key_id}","collaborators_url":"https://api.github.com/repos/digitallinguistics/dlx-js/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/digitallinguistics/dlx-js/teams","hooks_url":"https://api.github.com/repos/digitallinguistics/dlx-js/hooks","issue_events_url":"https://api.github.com/repos/digitallinguistics/dlx-js/issues/events{/number}","events_url":"https://api.github.com/repos/digitallinguistics/dlx-js/events","assignees_url":"https://api.github.com/repos/digitallinguistics/dlx-js/assignees{/user}","branches_url":"https://api.github.com/repos/digitallinguistics/dlx-js/branches{/branch}","tags_url":"https://api.github.com/repos/digitallinguistics/dlx-js/tags","blobs_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/refs{/sha}","trees_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/trees{/sha}","statuses_url":"https://api.github.com/repos/digitallinguistics/dlx-js/statuses/{sha}","languages_url":"https://api.github.com/repos/digitallinguistics/dlx-js/languages","stargazers_url":"https://api.github.com/repos/digitallinguistics/dlx-js/stargazers","contributors_url":"https://api.github.com/repos/digitallinguistics/dlx-js/contributors","subscribers_url":"https://api.github.com/repos/digitallinguistics/dlx-js/subscribers","subscription_url":"https://api.github.com/repos/digitallinguistics/dlx-js/subscription","commits_url":"https://api.github.com/repos/digitallinguistics/dlx-js/commits{/sha}","git_commits_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/commits{/sha}","comments_url":"https://api.github.com/repos/digitallinguistics/dlx-js/comments{/number}","issue_comment_url":"https://api.github.com/repos/digitallinguistics/dlx-js/issues/comments{/number}","contents_url":"https://api.github.com/repos/digitallinguistics/dlx-js/contents/{+path}","compare_url":"https://api.github.com/repos/digitallinguistics/dlx-js/compare/{base}...{head}","merges_url":"https://api.github.com/repos/digitallinguistics/dlx-js/merges","archive_url":"https://api.github.com/repos/digitallinguistics/dlx-js/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/digitallinguistics/dlx-js/downloads","issues_url":"https://api.github.com/repos/digitallinguistics/dlx-js/issues{/number}","pulls_url":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls{/number}","milestones_url":"https://api.github.com/repos/digitallinguistics/dlx-js/milestones{/number}","notifications_url":"https://api.github.com/repos/digitallinguistics/dlx-js/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/digitallinguistics/dlx-js/labels{/name}","releases_url":"https://api.github.com/repos/digitallinguistics/dlx-js/releases{/id}","deployments_url":"https://api.github.com/repos/digitallinguistics/dlx-js/deployments","created_at":"2015-12-14T09:33:42Z","updated_at":"2016-10-22T22:03:27Z","pushed_at":"2016-10-25T07:29:18Z","git_url":"git://github.com/digitallinguistics/dlx-js.git","ssh_url":"git@github.com:digitallinguistics/dlx-js.git","clone_url":"https://github.com/digitallinguistics/dlx-js.git","svn_url":"https://github.com/digitallinguistics/dlx-js","homepage":"","size":442,"stargazers_count":1,"watchers_count":1,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":10,"forks":1,"open_issues":10,"watchers":1,"default_branch":"master"}},"base":{"label":"digitallinguistics:master","ref":"master","sha":"33d09cf1f4301837a0506cf058b34e1756a167b3","user":{"login":"digitallinguistics","id":5896943,"avatar_url":"https://avatars.githubusercontent.com/u/5896943?v=3","gravatar_id":"","url":"https://api.github.com/users/digitallinguistics","html_url":"https://github.com/digitallinguistics","followers_url":"https://api.github.com/users/digitallinguistics/followers","following_url":"https://api.github.com/users/digitallinguistics/following{/other_user}","gists_url":"https://api.github.com/users/digitallinguistics/gists{/gist_id}","starred_url":"https://api.github.com/users/digitallinguistics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/digitallinguistics/subscriptions","organizations_url":"https://api.github.com/users/digitallinguistics/orgs","repos_url":"https://api.github.com/users/digitallinguistics/repos","events_url":"https://api.github.com/users/digitallinguistics/events{/privacy}","received_events_url":"https://api.github.com/users/digitallinguistics/received_events","type":"Organization","site_admin":false},"repo":{"id":47965106,"name":"dlx-js","full_name":"digitallinguistics/dlx-js","owner":{"login":"digitallinguistics","id":5896943,"avatar_url":"https://avatars.githubusercontent.com/u/5896943?v=3","gravatar_id":"","url":"https://api.github.com/users/digitallinguistics","html_url":"https://github.com/digitallinguistics","followers_url":"https://api.github.com/users/digitallinguistics/followers","following_url":"https://api.github.com/users/digitallinguistics/following{/other_user}","gists_url":"https://api.github.com/users/digitallinguistics/gists{/gist_id}","starred_url":"https://api.github.com/users/digitallinguistics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/digitallinguistics/subscriptions","organizations_url":"https://api.github.com/users/digitallinguistics/orgs","repos_url":"https://api.github.com/users/digitallinguistics/repos","events_url":"https://api.github.com/users/digitallinguistics/events{/privacy}","received_events_url":"https://api.github.com/users/digitallinguistics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/digitallinguistics/dlx-js","description":"A JavaScript library for working with linguistic data in DLx format. For browser and Node.","fork":false,"url":"https://api.github.com/repos/digitallinguistics/dlx-js","forks_url":"https://api.github.com/repos/digitallinguistics/dlx-js/forks","keys_url":"https://api.github.com/repos/digitallinguistics/dlx-js/keys{/key_id}","collaborators_url":"https://api.github.com/repos/digitallinguistics/dlx-js/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/digitallinguistics/dlx-js/teams","hooks_url":"https://api.github.com/repos/digitallinguistics/dlx-js/hooks","issue_events_url":"https://api.github.com/repos/digitallinguistics/dlx-js/issues/events{/number}","events_url":"https://api.github.com/repos/digitallinguistics/dlx-js/events","assignees_url":"https://api.github.com/repos/digitallinguistics/dlx-js/assignees{/user}","branches_url":"https://api.github.com/repos/digitallinguistics/dlx-js/branches{/branch}","tags_url":"https://api.github.com/repos/digitallinguistics/dlx-js/tags","blobs_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/refs{/sha}","trees_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/trees{/sha}","statuses_url":"https://api.github.com/repos/digitallinguistics/dlx-js/statuses/{sha}","languages_url":"https://api.github.com/repos/digitallinguistics/dlx-js/languages","stargazers_url":"https://api.github.com/repos/digitallinguistics/dlx-js/stargazers","contributors_url":"https://api.github.com/repos/digitallinguistics/dlx-js/contributors","subscribers_url":"https://api.github.com/repos/digitallinguistics/dlx-js/subscribers","subscription_url":"https://api.github.com/repos/digitallinguistics/dlx-js/subscription","commits_url":"https://api.github.com/repos/digitallinguistics/dlx-js/commits{/sha}","git_commits_url":"https://api.github.com/repos/digitallinguistics/dlx-js/git/commits{/sha}","comments_url":"https://api.github.com/repos/digitallinguistics/dlx-js/comments{/number}","issue_comment_url":"https://api.github.com/repos/digitallinguistics/dlx-js/issues/comments{/number}","contents_url":"https://api.github.com/repos/digitallinguistics/dlx-js/contents/{+path}","compare_url":"https://api.github.com/repos/digitallinguistics/dlx-js/compare/{base}...{head}","merges_url":"https://api.github.com/repos/digitallinguistics/dlx-js/merges","archive_url":"https://api.github.com/repos/digitallinguistics/dlx-js/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/digitallinguistics/dlx-js/downloads","issues_url":"https://api.github.com/repos/digitallinguistics/dlx-js/issues{/number}","pulls_url":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls{/number}","milestones_url":"https://api.github.com/repos/digitallinguistics/dlx-js/milestones{/number}","notifications_url":"https://api.github.com/repos/digitallinguistics/dlx-js/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/digitallinguistics/dlx-js/labels{/name}","releases_url":"https://api.github.com/repos/digitallinguistics/dlx-js/releases{/id}","deployments_url":"https://api.github.com/repos/digitallinguistics/dlx-js/deployments","created_at":"2015-12-14T09:33:42Z","updated_at":"2016-10-22T22:03:27Z","pushed_at":"2016-10-25T07:29:18Z","git_url":"git://github.com/digitallinguistics/dlx-js.git","ssh_url":"git@github.com:digitallinguistics/dlx-js.git","clone_url":"https://github.com/digitallinguistics/dlx-js.git","svn_url":"https://github.com/digitallinguistics/dlx-js","homepage":"","size":442,"stargazers_count":1,"watchers_count":1,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":10,"forks":1,"open_issues":10,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/30"},"html":{"href":"https://github.com/digitallinguistics/dlx-js/pull/30"},"issue":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/issues/30"},"comments":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/issues/30/comments"},"review_comments":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/30/comments"},"review_comment":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/pulls/30/commits"},"statuses":{"href":"https://api.github.com/repos/digitallinguistics/dlx-js/statuses/d4b691f2fcada036cab72f69ec0976d1c87f5b84"}}}}
{"actor":{"display_login":"dwhieb"}}