{"action":"created","comment":{"url":"https://api.github.com/repos/idio/wiki2vec/pulls/comments/53795333","id":53795333,"diff_hunk":"@@ -0,0 +1,69 @@\n+__author__ = 'dowling'\n+import datetime\n+import logging\n+\n+logFormatter = logging.Formatter(\"%(asctime)s %(levelname)-8s %(name)-18s: %(message)s\")\n+\n+ln = logging.getLogger()\n+\n+fileHandler = logging.FileHandler(\"fix_corpus_log%s.txt\" % datetime.datetime.now().isoformat())\n+\n+fileHandler.setFormatter(logFormatter)\n+ln.addHandler(fileHandler)\n+\n+consoleHandler = logging.StreamHandler()\n+consoleHandler.setFormatter(logFormatter)\n+ln.addHandler(consoleHandler)\n+\n+ln.setLevel(logging.DEBUG)\n+\n+import re\n+import string\n+from gensim import utils\n+\n+replace_punctuation = string.maketrans(string.punctuation, ' '*len(string.punctuation))\n+\n+class PreprocessingLineSentence():\n+    def __init__(self, path_to_corpus):\n+        self.path = path_to_corpus\n+\n+    def __iter__(self):\n+        with utils.smart_open(self.path) as fin:\n+            for line_no, line in enumerate(fin):\n+                if line_no % 10000 == 0:\n+                    ln.debug(\"Processed %s lines\" % line_no)\n+\n+                # replace all punctuation with a space, unless it's inside a DBPEDIA_ID\n+                line_parts = []\n+                start_at = 0\n+                for list_idx, match in enumerate(re.finditer(r\"DBPEDIA_ID/\\S+\", line)):\n+\n+                    edited = line[start_at: match.start()].translate(replace_punctuation)\n+\n+                    line_parts.append(edited)\n+                    line_parts.append(match.group(0))\n+                    start_at = match.end()\n+\n+                edited = line[start_at: -1].translate(replace_punctuation)\n+                line_parts.append(edited)\n+\n+                line = \"\".join(line_parts)\n+\n+                #line = utils.to_unicode(line)","path":"resources/fix_corpus.py","position":52,"original_position":52,"commit_id":"48148c0e3ed1dcb21bc242e6cd894a55e5fafee8","original_commit_id":"48148c0e3ed1dcb21bc242e6cd894a55e5fafee8","user":{"login":"dav009","id":1659415,"avatar_url":"https://avatars.githubusercontent.com/u/1659415?v=3","gravatar_id":"","url":"https://api.github.com/users/dav009","html_url":"https://github.com/dav009","followers_url":"https://api.github.com/users/dav009/followers","following_url":"https://api.github.com/users/dav009/following{/other_user}","gists_url":"https://api.github.com/users/dav009/gists{/gist_id}","starred_url":"https://api.github.com/users/dav009/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dav009/subscriptions","organizations_url":"https://api.github.com/users/dav009/orgs","repos_url":"https://api.github.com/users/dav009/repos","events_url":"https://api.github.com/users/dav009/events{/privacy}","received_events_url":"https://api.github.com/users/dav009/received_events","type":"User","site_admin":false},"body":"comment","created_at":"2016-02-23T15:29:19Z","updated_at":"2016-02-23T15:29:19Z","html_url":"https://github.com/idio/wiki2vec/pull/8#discussion_r53795333","pull_request_url":"https://api.github.com/repos/idio/wiki2vec/pulls/8","_links":{"self":{"href":"https://api.github.com/repos/idio/wiki2vec/pulls/comments/53795333"},"html":{"href":"https://github.com/idio/wiki2vec/pull/8#discussion_r53795333"},"pull_request":{"href":"https://api.github.com/repos/idio/wiki2vec/pulls/8"}}},"pull_request":{"url":"https://api.github.com/repos/idio/wiki2vec/pulls/8","id":43040793,"html_url":"https://github.com/idio/wiki2vec/pull/8","diff_url":"https://github.com/idio/wiki2vec/pull/8.diff","patch_url":"https://github.com/idio/wiki2vec/pull/8.patch","issue_url":"https://api.github.com/repos/idio/wiki2vec/issues/8","number":8,"state":"open","locked":false,"title":"Add post-processing for cleaner output corpora, support for entity min_count in gensim model creation, redirects resolution","user":{"login":"phdowling","id":876919,"avatar_url":"https://avatars.githubusercontent.com/u/876919?v=3","gravatar_id":"","url":"https://api.github.com/users/phdowling","html_url":"https://github.com/phdowling","followers_url":"https://api.github.com/users/phdowling/followers","following_url":"https://api.github.com/users/phdowling/following{/other_user}","gists_url":"https://api.github.com/users/phdowling/gists{/gist_id}","starred_url":"https://api.github.com/users/phdowling/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/phdowling/subscriptions","organizations_url":"https://api.github.com/users/phdowling/orgs","repos_url":"https://api.github.com/users/phdowling/repos","events_url":"https://api.github.com/users/phdowling/events{/privacy}","received_events_url":"https://api.github.com/users/phdowling/received_events","type":"User","site_admin":false},"body":"These are some changes I made that seem to improve corpus and model quality. I also added a script that converts the vectors to a csv format that can be used by Spotlight to create a vector store.","created_at":"2015-08-21T14:02:39Z","updated_at":"2016-02-23T15:29:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d5b3e579c52ec2f75b1452dd1b3d9401f463ceae","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/idio/wiki2vec/pulls/8/commits","review_comments_url":"https://api.github.com/repos/idio/wiki2vec/pulls/8/comments","review_comment_url":"https://api.github.com/repos/idio/wiki2vec/pulls/comments{/number}","comments_url":"https://api.github.com/repos/idio/wiki2vec/issues/8/comments","statuses_url":"https://api.github.com/repos/idio/wiki2vec/statuses/48148c0e3ed1dcb21bc242e6cd894a55e5fafee8","head":{"label":"phdowling:master","ref":"master","sha":"48148c0e3ed1dcb21bc242e6cd894a55e5fafee8","user":{"login":"phdowling","id":876919,"avatar_url":"https://avatars.githubusercontent.com/u/876919?v=3","gravatar_id":"","url":"https://api.github.com/users/phdowling","html_url":"https://github.com/phdowling","followers_url":"https://api.github.com/users/phdowling/followers","following_url":"https://api.github.com/users/phdowling/following{/other_user}","gists_url":"https://api.github.com/users/phdowling/gists{/gist_id}","starred_url":"https://api.github.com/users/phdowling/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/phdowling/subscriptions","organizations_url":"https://api.github.com/users/phdowling/orgs","repos_url":"https://api.github.com/users/phdowling/repos","events_url":"https://api.github.com/users/phdowling/events{/privacy}","received_events_url":"https://api.github.com/users/phdowling/received_events","type":"User","site_admin":false},"repo":{"id":36064652,"name":"wiki2vec","full_name":"phdowling/wiki2vec","owner":{"login":"phdowling","id":876919,"avatar_url":"https://avatars.githubusercontent.com/u/876919?v=3","gravatar_id":"","url":"https://api.github.com/users/phdowling","html_url":"https://github.com/phdowling","followers_url":"https://api.github.com/users/phdowling/followers","following_url":"https://api.github.com/users/phdowling/following{/other_user}","gists_url":"https://api.github.com/users/phdowling/gists{/gist_id}","starred_url":"https://api.github.com/users/phdowling/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/phdowling/subscriptions","organizations_url":"https://api.github.com/users/phdowling/orgs","repos_url":"https://api.github.com/users/phdowling/repos","events_url":"https://api.github.com/users/phdowling/events{/privacy}","received_events_url":"https://api.github.com/users/phdowling/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/phdowling/wiki2vec","description":"Generating Vectors for DBpedia Entities via Word2Vec and Wikipedia Dumps","fork":true,"url":"https://api.github.com/repos/phdowling/wiki2vec","forks_url":"https://api.github.com/repos/phdowling/wiki2vec/forks","keys_url":"https://api.github.com/repos/phdowling/wiki2vec/keys{/key_id}","collaborators_url":"https://api.github.com/repos/phdowling/wiki2vec/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/phdowling/wiki2vec/teams","hooks_url":"https://api.github.com/repos/phdowling/wiki2vec/hooks","issue_events_url":"https://api.github.com/repos/phdowling/wiki2vec/issues/events{/number}","events_url":"https://api.github.com/repos/phdowling/wiki2vec/events","assignees_url":"https://api.github.com/repos/phdowling/wiki2vec/assignees{/user}","branches_url":"https://api.github.com/repos/phdowling/wiki2vec/branches{/branch}","tags_url":"https://api.github.com/repos/phdowling/wiki2vec/tags","blobs_url":"https://api.github.com/repos/phdowling/wiki2vec/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/phdowling/wiki2vec/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/phdowling/wiki2vec/git/refs{/sha}","trees_url":"https://api.github.com/repos/phdowling/wiki2vec/git/trees{/sha}","statuses_url":"https://api.github.com/repos/phdowling/wiki2vec/statuses/{sha}","languages_url":"https://api.github.com/repos/phdowling/wiki2vec/languages","stargazers_url":"https://api.github.com/repos/phdowling/wiki2vec/stargazers","contributors_url":"https://api.github.com/repos/phdowling/wiki2vec/contributors","subscribers_url":"https://api.github.com/repos/phdowling/wiki2vec/subscribers","subscription_url":"https://api.github.com/repos/phdowling/wiki2vec/subscription","commits_url":"https://api.github.com/repos/phdowling/wiki2vec/commits{/sha}","git_commits_url":"https://api.github.com/repos/phdowling/wiki2vec/git/commits{/sha}","comments_url":"https://api.github.com/repos/phdowling/wiki2vec/comments{/number}","issue_comment_url":"https://api.github.com/repos/phdowling/wiki2vec/issues/comments{/number}","contents_url":"https://api.github.com/repos/phdowling/wiki2vec/contents/{+path}","compare_url":"https://api.github.com/repos/phdowling/wiki2vec/compare/{base}...{head}","merges_url":"https://api.github.com/repos/phdowling/wiki2vec/merges","archive_url":"https://api.github.com/repos/phdowling/wiki2vec/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/phdowling/wiki2vec/downloads","issues_url":"https://api.github.com/repos/phdowling/wiki2vec/issues{/number}","pulls_url":"https://api.github.com/repos/phdowling/wiki2vec/pulls{/number}","milestones_url":"https://api.github.com/repos/phdowling/wiki2vec/milestones{/number}","notifications_url":"https://api.github.com/repos/phdowling/wiki2vec/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/phdowling/wiki2vec/labels{/name}","releases_url":"https://api.github.com/repos/phdowling/wiki2vec/releases{/id}","deployments_url":"https://api.github.com/repos/phdowling/wiki2vec/deployments","created_at":"2015-05-22T09:42:25Z","updated_at":"2015-05-22T09:42:25Z","pushed_at":"2015-08-21T14:25:28Z","git_url":"git://github.com/phdowling/wiki2vec.git","ssh_url":"git@github.com:phdowling/wiki2vec.git","clone_url":"https://github.com/phdowling/wiki2vec.git","svn_url":"https://github.com/phdowling/wiki2vec","homepage":"","size":704,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"idio:master","ref":"master","sha":"85fcc7888b0aca263c6dabc79b6b27034b3aea5d","user":{"login":"idio","id":394469,"avatar_url":"https://avatars.githubusercontent.com/u/394469?v=3","gravatar_id":"","url":"https://api.github.com/users/idio","html_url":"https://github.com/idio","followers_url":"https://api.github.com/users/idio/followers","following_url":"https://api.github.com/users/idio/following{/other_user}","gists_url":"https://api.github.com/users/idio/gists{/gist_id}","starred_url":"https://api.github.com/users/idio/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/idio/subscriptions","organizations_url":"https://api.github.com/users/idio/orgs","repos_url":"https://api.github.com/users/idio/repos","events_url":"https://api.github.com/users/idio/events{/privacy}","received_events_url":"https://api.github.com/users/idio/received_events","type":"Organization","site_admin":false},"repo":{"id":30615139,"name":"wiki2vec","full_name":"idio/wiki2vec","owner":{"login":"idio","id":394469,"avatar_url":"https://avatars.githubusercontent.com/u/394469?v=3","gravatar_id":"","url":"https://api.github.com/users/idio","html_url":"https://github.com/idio","followers_url":"https://api.github.com/users/idio/followers","following_url":"https://api.github.com/users/idio/following{/other_user}","gists_url":"https://api.github.com/users/idio/gists{/gist_id}","starred_url":"https://api.github.com/users/idio/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/idio/subscriptions","organizations_url":"https://api.github.com/users/idio/orgs","repos_url":"https://api.github.com/users/idio/repos","events_url":"https://api.github.com/users/idio/events{/privacy}","received_events_url":"https://api.github.com/users/idio/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/idio/wiki2vec","description":"Generating Vectors for DBpedia Entities via Word2Vec and Wikipedia Dumps","fork":false,"url":"https://api.github.com/repos/idio/wiki2vec","forks_url":"https://api.github.com/repos/idio/wiki2vec/forks","keys_url":"https://api.github.com/repos/idio/wiki2vec/keys{/key_id}","collaborators_url":"https://api.github.com/repos/idio/wiki2vec/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/idio/wiki2vec/teams","hooks_url":"https://api.github.com/repos/idio/wiki2vec/hooks","issue_events_url":"https://api.github.com/repos/idio/wiki2vec/issues/events{/number}","events_url":"https://api.github.com/repos/idio/wiki2vec/events","assignees_url":"https://api.github.com/repos/idio/wiki2vec/assignees{/user}","branches_url":"https://api.github.com/repos/idio/wiki2vec/branches{/branch}","tags_url":"https://api.github.com/repos/idio/wiki2vec/tags","blobs_url":"https://api.github.com/repos/idio/wiki2vec/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/idio/wiki2vec/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/idio/wiki2vec/git/refs{/sha}","trees_url":"https://api.github.com/repos/idio/wiki2vec/git/trees{/sha}","statuses_url":"https://api.github.com/repos/idio/wiki2vec/statuses/{sha}","languages_url":"https://api.github.com/repos/idio/wiki2vec/languages","stargazers_url":"https://api.github.com/repos/idio/wiki2vec/stargazers","contributors_url":"https://api.github.com/repos/idio/wiki2vec/contributors","subscribers_url":"https://api.github.com/repos/idio/wiki2vec/subscribers","subscription_url":"https://api.github.com/repos/idio/wiki2vec/subscription","commits_url":"https://api.github.com/repos/idio/wiki2vec/commits{/sha}","git_commits_url":"https://api.github.com/repos/idio/wiki2vec/git/commits{/sha}","comments_url":"https://api.github.com/repos/idio/wiki2vec/comments{/number}","issue_comment_url":"https://api.github.com/repos/idio/wiki2vec/issues/comments{/number}","contents_url":"https://api.github.com/repos/idio/wiki2vec/contents/{+path}","compare_url":"https://api.github.com/repos/idio/wiki2vec/compare/{base}...{head}","merges_url":"https://api.github.com/repos/idio/wiki2vec/merges","archive_url":"https://api.github.com/repos/idio/wiki2vec/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/idio/wiki2vec/downloads","issues_url":"https://api.github.com/repos/idio/wiki2vec/issues{/number}","pulls_url":"https://api.github.com/repos/idio/wiki2vec/pulls{/number}","milestones_url":"https://api.github.com/repos/idio/wiki2vec/milestones{/number}","notifications_url":"https://api.github.com/repos/idio/wiki2vec/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/idio/wiki2vec/labels{/name}","releases_url":"https://api.github.com/repos/idio/wiki2vec/releases{/id}","deployments_url":"https://api.github.com/repos/idio/wiki2vec/deployments","created_at":"2015-02-10T21:20:32Z","updated_at":"2016-02-16T06:56:08Z","pushed_at":"2016-02-06T11:59:22Z","git_url":"git://github.com/idio/wiki2vec.git","ssh_url":"git@github.com:idio/wiki2vec.git","clone_url":"https://github.com/idio/wiki2vec.git","svn_url":"https://github.com/idio/wiki2vec","homepage":"","size":646,"stargazers_count":150,"watchers_count":150,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":44,"mirror_url":null,"open_issues_count":6,"forks":44,"open_issues":6,"watchers":150,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/idio/wiki2vec/pulls/8"},"html":{"href":"https://github.com/idio/wiki2vec/pull/8"},"issue":{"href":"https://api.github.com/repos/idio/wiki2vec/issues/8"},"comments":{"href":"https://api.github.com/repos/idio/wiki2vec/issues/8/comments"},"review_comments":{"href":"https://api.github.com/repos/idio/wiki2vec/pulls/8/comments"},"review_comment":{"href":"https://api.github.com/repos/idio/wiki2vec/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/idio/wiki2vec/pulls/8/commits"},"statuses":{"href":"https://api.github.com/repos/idio/wiki2vec/statuses/48148c0e3ed1dcb21bc242e6cd894a55e5fafee8"}}}}