{"action":"created","comment":{"url":"https://api.github.com/repos/westlab/door-api/pulls/comments/64763627","id":64763627,"diff_hunk":"@@ -4,13 +4,66 @@ import (\n \t\"io/ioutil\"\n \t\"log\"\n \t\"net/http\"\n+\t\"os\"\n+\t\"path/filepath\"\n+\t\"regexp\"\n+\t\"strings\"\n \n \t\"github.com/ikawaha/kagome/tokenizer\"\n \t\"github.com/kennygrant/sanitize\"\n \n+\t\"github.com/westlab/door-api/common\"\n+\t\"github.com/westlab/door-api/context\"\n \t\"github.com/westlab/door-api/model\"\n )\n \n+// wikipedia dictionary file\n+const (\n+\tuserDicPath = \"./job/userdic.txt\" // TODO: should be changed\n+)\n+\n+// HTMLAnalyzer is analyzer for HTML page words\n+type HTMLAnalyzer struct {\n+\ttokenizer *Tokenizer\n+}\n+\n+// NewHTMLAnalyzer creates HTMLAnalyzer\n+func NewHTMLAnalyzer() *HTMLAnalyzer {\n+\ttokenizer := NewTokenizer(userDicPath)\n+\treturn &HTMLAnalyzer{&tokenizer}\n+}\n+\n+// Manage manages HTMLAnalyzer\n+func (htmlAnalyzer *HTMLAnalyzer) Manage(b *model.Browsing) {\n+\turl := b.URL\n+\thashed := common.GetMD5Hash(url)\n+\tcxt := context.GetContext()\n+\tok := common.IsFileExist(hashed, cxt.GetConf().WordsPath)\n+\tif ok { // If the file is existed, do nothing\n+\t\treturn\n+\t}\n+\n+\thtml, err := DonwloadHTML(url)\n+\tif err != nil {\n+\t\treturn // else to write?\n+\t}\n+\thtml = RemoveAnyTagData(html, \"script\") // remove scirpt tag data\n+\tsanitizedText := RemoveHTMLTags(html)\n+\twordList := htmlAnalyzer.tokenizer.GetNouns(sanitizedText, true)\n+\tSaveWordsText(hashed, wordList)\n+\n+\t// save words to Word table\n+\tcounts := WordCount(wordList)\n+\twords := make([]*model.Word, len(counts))\n+\tfor i := 0; i < len(words); i++ {","path":"job/html_analyzer.go","position":55,"original_position":55,"commit_id":"bfb0f52bcf2f57b4e7e715687b18cca4f07c8dac","original_commit_id":"bfb0f52bcf2f57b4e7e715687b18cca4f07c8dac","user":{"login":"momotaro98","id":9294726,"avatar_url":"https://avatars.githubusercontent.com/u/9294726?v=3","gravatar_id":"","url":"https://api.github.com/users/momotaro98","html_url":"https://github.com/momotaro98","followers_url":"https://api.github.com/users/momotaro98/followers","following_url":"https://api.github.com/users/momotaro98/following{/other_user}","gists_url":"https://api.github.com/users/momotaro98/gists{/gist_id}","starred_url":"https://api.github.com/users/momotaro98/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/momotaro98/subscriptions","organizations_url":"https://api.github.com/users/momotaro98/orgs","repos_url":"https://api.github.com/users/momotaro98/repos","events_url":"https://api.github.com/users/momotaro98/events{/privacy}","received_events_url":"https://api.github.com/users/momotaro98/received_events","type":"User","site_admin":false},"body":"@giwa \r\n修正しました。何度もすみません。","created_at":"2016-05-26T15:11:27Z","updated_at":"2016-05-26T15:11:27Z","html_url":"https://github.com/westlab/door-api/pull/144#discussion_r64763627","pull_request_url":"https://api.github.com/repos/westlab/door-api/pulls/144","_links":{"self":{"href":"https://api.github.com/repos/westlab/door-api/pulls/comments/64763627"},"html":{"href":"https://github.com/westlab/door-api/pull/144#discussion_r64763627"},"pull_request":{"href":"https://api.github.com/repos/westlab/door-api/pulls/144"}}},"pull_request":{"url":"https://api.github.com/repos/westlab/door-api/pulls/144","id":71444612,"html_url":"https://github.com/westlab/door-api/pull/144","diff_url":"https://github.com/westlab/door-api/pull/144.diff","patch_url":"https://github.com/westlab/door-api/pull/144.patch","issue_url":"https://api.github.com/repos/westlab/door-api/issues/144","number":144,"state":"open","locked":false,"title":"Fix add html analyzer job","user":{"login":"momotaro98","id":9294726,"avatar_url":"https://avatars.githubusercontent.com/u/9294726?v=3","gravatar_id":"","url":"https://api.github.com/users/momotaro98","html_url":"https://github.com/momotaro98","followers_url":"https://api.github.com/users/momotaro98/followers","following_url":"https://api.github.com/users/momotaro98/following{/other_user}","gists_url":"https://api.github.com/users/momotaro98/gists{/gist_id}","starred_url":"https://api.github.com/users/momotaro98/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/momotaro98/subscriptions","organizations_url":"https://api.github.com/users/momotaro98/orgs","repos_url":"https://api.github.com/users/momotaro98/repos","events_url":"https://api.github.com/users/momotaro98/events{/privacy}","received_events_url":"https://api.github.com/users/momotaro98/received_events","type":"User","site_admin":false},"body":"https://github.com/westlab/door-api/pull/140\r\nからの引き継ぎです。\r\n\r\n一応、指摘部分をすべて修正しました。","created_at":"2016-05-25T18:57:25Z","updated_at":"2016-05-26T15:11:27Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e5de727a8f718d1492d748347a2d6e8e7cf6125d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/westlab/door-api/pulls/144/commits","review_comments_url":"https://api.github.com/repos/westlab/door-api/pulls/144/comments","review_comment_url":"https://api.github.com/repos/westlab/door-api/pulls/comments{/number}","comments_url":"https://api.github.com/repos/westlab/door-api/issues/144/comments","statuses_url":"https://api.github.com/repos/westlab/door-api/statuses/bfb0f52bcf2f57b4e7e715687b18cca4f07c8dac","head":{"label":"westlab:fix-add-html-analyzer-job","ref":"fix-add-html-analyzer-job","sha":"bfb0f52bcf2f57b4e7e715687b18cca4f07c8dac","user":{"login":"westlab","id":7532472,"avatar_url":"https://avatars.githubusercontent.com/u/7532472?v=3","gravatar_id":"","url":"https://api.github.com/users/westlab","html_url":"https://github.com/westlab","followers_url":"https://api.github.com/users/westlab/followers","following_url":"https://api.github.com/users/westlab/following{/other_user}","gists_url":"https://api.github.com/users/westlab/gists{/gist_id}","starred_url":"https://api.github.com/users/westlab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/westlab/subscriptions","organizations_url":"https://api.github.com/users/westlab/orgs","repos_url":"https://api.github.com/users/westlab/repos","events_url":"https://api.github.com/users/westlab/events{/privacy}","received_events_url":"https://api.github.com/users/westlab/received_events","type":"Organization","site_admin":false},"repo":{"id":56182161,"name":"door-api","full_name":"westlab/door-api","owner":{"login":"westlab","id":7532472,"avatar_url":"https://avatars.githubusercontent.com/u/7532472?v=3","gravatar_id":"","url":"https://api.github.com/users/westlab","html_url":"https://github.com/westlab","followers_url":"https://api.github.com/users/westlab/followers","following_url":"https://api.github.com/users/westlab/following{/other_user}","gists_url":"https://api.github.com/users/westlab/gists{/gist_id}","starred_url":"https://api.github.com/users/westlab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/westlab/subscriptions","organizations_url":"https://api.github.com/users/westlab/orgs","repos_url":"https://api.github.com/users/westlab/repos","events_url":"https://api.github.com/users/westlab/events{/privacy}","received_events_url":"https://api.github.com/users/westlab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/westlab/door-api","description":"","fork":false,"url":"https://api.github.com/repos/westlab/door-api","forks_url":"https://api.github.com/repos/westlab/door-api/forks","keys_url":"https://api.github.com/repos/westlab/door-api/keys{/key_id}","collaborators_url":"https://api.github.com/repos/westlab/door-api/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/westlab/door-api/teams","hooks_url":"https://api.github.com/repos/westlab/door-api/hooks","issue_events_url":"https://api.github.com/repos/westlab/door-api/issues/events{/number}","events_url":"https://api.github.com/repos/westlab/door-api/events","assignees_url":"https://api.github.com/repos/westlab/door-api/assignees{/user}","branches_url":"https://api.github.com/repos/westlab/door-api/branches{/branch}","tags_url":"https://api.github.com/repos/westlab/door-api/tags","blobs_url":"https://api.github.com/repos/westlab/door-api/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/westlab/door-api/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/westlab/door-api/git/refs{/sha}","trees_url":"https://api.github.com/repos/westlab/door-api/git/trees{/sha}","statuses_url":"https://api.github.com/repos/westlab/door-api/statuses/{sha}","languages_url":"https://api.github.com/repos/westlab/door-api/languages","stargazers_url":"https://api.github.com/repos/westlab/door-api/stargazers","contributors_url":"https://api.github.com/repos/westlab/door-api/contributors","subscribers_url":"https://api.github.com/repos/westlab/door-api/subscribers","subscription_url":"https://api.github.com/repos/westlab/door-api/subscription","commits_url":"https://api.github.com/repos/westlab/door-api/commits{/sha}","git_commits_url":"https://api.github.com/repos/westlab/door-api/git/commits{/sha}","comments_url":"https://api.github.com/repos/westlab/door-api/comments{/number}","issue_comment_url":"https://api.github.com/repos/westlab/door-api/issues/comments{/number}","contents_url":"https://api.github.com/repos/westlab/door-api/contents/{+path}","compare_url":"https://api.github.com/repos/westlab/door-api/compare/{base}...{head}","merges_url":"https://api.github.com/repos/westlab/door-api/merges","archive_url":"https://api.github.com/repos/westlab/door-api/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/westlab/door-api/downloads","issues_url":"https://api.github.com/repos/westlab/door-api/issues{/number}","pulls_url":"https://api.github.com/repos/westlab/door-api/pulls{/number}","milestones_url":"https://api.github.com/repos/westlab/door-api/milestones{/number}","notifications_url":"https://api.github.com/repos/westlab/door-api/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/westlab/door-api/labels{/name}","releases_url":"https://api.github.com/repos/westlab/door-api/releases{/id}","deployments_url":"https://api.github.com/repos/westlab/door-api/deployments","created_at":"2016-04-13T20:04:29Z","updated_at":"2016-04-15T09:28:22Z","pushed_at":"2016-05-26T15:09:30Z","git_url":"git://github.com/westlab/door-api.git","ssh_url":"git@github.com:westlab/door-api.git","clone_url":"https://github.com/westlab/door-api.git","svn_url":"https://github.com/westlab/door-api","homepage":null,"size":11684,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":21,"forks":0,"open_issues":21,"watchers":0,"default_branch":"develop"}},"base":{"label":"westlab:develop","ref":"develop","sha":"97bef0510b13bc31dd746e7ca907b3b1081e485f","user":{"login":"westlab","id":7532472,"avatar_url":"https://avatars.githubusercontent.com/u/7532472?v=3","gravatar_id":"","url":"https://api.github.com/users/westlab","html_url":"https://github.com/westlab","followers_url":"https://api.github.com/users/westlab/followers","following_url":"https://api.github.com/users/westlab/following{/other_user}","gists_url":"https://api.github.com/users/westlab/gists{/gist_id}","starred_url":"https://api.github.com/users/westlab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/westlab/subscriptions","organizations_url":"https://api.github.com/users/westlab/orgs","repos_url":"https://api.github.com/users/westlab/repos","events_url":"https://api.github.com/users/westlab/events{/privacy}","received_events_url":"https://api.github.com/users/westlab/received_events","type":"Organization","site_admin":false},"repo":{"id":56182161,"name":"door-api","full_name":"westlab/door-api","owner":{"login":"westlab","id":7532472,"avatar_url":"https://avatars.githubusercontent.com/u/7532472?v=3","gravatar_id":"","url":"https://api.github.com/users/westlab","html_url":"https://github.com/westlab","followers_url":"https://api.github.com/users/westlab/followers","following_url":"https://api.github.com/users/westlab/following{/other_user}","gists_url":"https://api.github.com/users/westlab/gists{/gist_id}","starred_url":"https://api.github.com/users/westlab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/westlab/subscriptions","organizations_url":"https://api.github.com/users/westlab/orgs","repos_url":"https://api.github.com/users/westlab/repos","events_url":"https://api.github.com/users/westlab/events{/privacy}","received_events_url":"https://api.github.com/users/westlab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/westlab/door-api","description":"","fork":false,"url":"https://api.github.com/repos/westlab/door-api","forks_url":"https://api.github.com/repos/westlab/door-api/forks","keys_url":"https://api.github.com/repos/westlab/door-api/keys{/key_id}","collaborators_url":"https://api.github.com/repos/westlab/door-api/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/westlab/door-api/teams","hooks_url":"https://api.github.com/repos/westlab/door-api/hooks","issue_events_url":"https://api.github.com/repos/westlab/door-api/issues/events{/number}","events_url":"https://api.github.com/repos/westlab/door-api/events","assignees_url":"https://api.github.com/repos/westlab/door-api/assignees{/user}","branches_url":"https://api.github.com/repos/westlab/door-api/branches{/branch}","tags_url":"https://api.github.com/repos/westlab/door-api/tags","blobs_url":"https://api.github.com/repos/westlab/door-api/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/westlab/door-api/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/westlab/door-api/git/refs{/sha}","trees_url":"https://api.github.com/repos/westlab/door-api/git/trees{/sha}","statuses_url":"https://api.github.com/repos/westlab/door-api/statuses/{sha}","languages_url":"https://api.github.com/repos/westlab/door-api/languages","stargazers_url":"https://api.github.com/repos/westlab/door-api/stargazers","contributors_url":"https://api.github.com/repos/westlab/door-api/contributors","subscribers_url":"https://api.github.com/repos/westlab/door-api/subscribers","subscription_url":"https://api.github.com/repos/westlab/door-api/subscription","commits_url":"https://api.github.com/repos/westlab/door-api/commits{/sha}","git_commits_url":"https://api.github.com/repos/westlab/door-api/git/commits{/sha}","comments_url":"https://api.github.com/repos/westlab/door-api/comments{/number}","issue_comment_url":"https://api.github.com/repos/westlab/door-api/issues/comments{/number}","contents_url":"https://api.github.com/repos/westlab/door-api/contents/{+path}","compare_url":"https://api.github.com/repos/westlab/door-api/compare/{base}...{head}","merges_url":"https://api.github.com/repos/westlab/door-api/merges","archive_url":"https://api.github.com/repos/westlab/door-api/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/westlab/door-api/downloads","issues_url":"https://api.github.com/repos/westlab/door-api/issues{/number}","pulls_url":"https://api.github.com/repos/westlab/door-api/pulls{/number}","milestones_url":"https://api.github.com/repos/westlab/door-api/milestones{/number}","notifications_url":"https://api.github.com/repos/westlab/door-api/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/westlab/door-api/labels{/name}","releases_url":"https://api.github.com/repos/westlab/door-api/releases{/id}","deployments_url":"https://api.github.com/repos/westlab/door-api/deployments","created_at":"2016-04-13T20:04:29Z","updated_at":"2016-04-15T09:28:22Z","pushed_at":"2016-05-26T15:09:30Z","git_url":"git://github.com/westlab/door-api.git","ssh_url":"git@github.com:westlab/door-api.git","clone_url":"https://github.com/westlab/door-api.git","svn_url":"https://github.com/westlab/door-api","homepage":null,"size":11684,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":21,"forks":0,"open_issues":21,"watchers":0,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/westlab/door-api/pulls/144"},"html":{"href":"https://github.com/westlab/door-api/pull/144"},"issue":{"href":"https://api.github.com/repos/westlab/door-api/issues/144"},"comments":{"href":"https://api.github.com/repos/westlab/door-api/issues/144/comments"},"review_comments":{"href":"https://api.github.com/repos/westlab/door-api/pulls/144/comments"},"review_comment":{"href":"https://api.github.com/repos/westlab/door-api/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/westlab/door-api/pulls/144/commits"},"statuses":{"href":"https://api.github.com/repos/westlab/door-api/statuses/bfb0f52bcf2f57b4e7e715687b18cca4f07c8dac"}}}}
{"actor":{"display_login":"momotaro98"}}