{"action":"created","comment":{"url":"https://api.github.com/repos/wellcometrust/policytool/pulls/comments/265950185","pull_request_review_id":214993419,"id":265950185,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDI2NTk1MDE4NQ==","diff_hunk":"@@ -0,0 +1,74 @@\n+import pandas as pd\n+import re\n+\n+class HardTextSearch:\n+\tdef __init__(self, ref_file, min_chars):\n+\t\t#Cleans up the ref file, changing from pandas DF to dict to reduce run time\n+\t\tref_file['title'] = [self.clean_text(x) for x in ref_file['title']]\n+\t\tref_file = ref_file.loc[ref_file['title'].str.len() >= min_chars]\n+\t\tref_file = ref_file.drop_duplicates(subset = 'title')\n+\t\tref_file = ref_file.to_dict(orient = 'list')\n+\n+\t\tself.ref_file = ref_file\n+\n+\n+\tdef clean_text(self, string):\n+\t\t\"\"\"\n+\t\tInput:\n+\t\t-A string\n+\t\tOutput:\n+\t\t-A string, with white space normalised and\n+\t\t non-alphanumeric characters removed\n+\t\tCleans up text such that it can easily be searched\n+\t\t\"\"\"\n+\n+\t\tstring = re.sub(\"\\\\n\", \" \", string)\n+\t\tstring = re.sub(\"\\s{1,}\", \" \", string)\n+\t\tstring = re.sub(\"[^A-Za-z0-9 ]\", \"\", string)\n+\n+\t\tstring = string.lower()\n+\n+\t\treturn (string)\n+\n+\n+\tdef hard_text_search(self, scraped_text, refs_to_exclude):\n+\t\t\"\"\"\n+\t\tInput:\n+\t\t-Raw scraped text (named tuple, SectionedDocument)\n+\t\t-IDs of matches already found by fuzzy matcher (pandas Series)\n+\t\tOutput:\n+\t\t-Extension to matches, with hard text search results concatenated\n+\t\t\"\"\"\n+\n+\t\t#Removes refs already found by fuzzy matcher\n+\t\tindices_to_exclude = [i for i,x in enumerate(self.ref_file['uber_id']) if x in refs_to_exclude]\n+\t\tfiltered_ref_file = self.ref_file\n+\t\tfor i in sorted(indices_to_exclude, reverse = True):\n+\t\t\tdel filtered_ref_file['title'][i]\n+\t\t\tdel filtered_ref_file['uber_id'][i]\n+\n+\t\tclean_scraped_text = self.clean_text(scraped_text.section)\n+\n+\t\tmatches = pd.DataFrame()\n+\n+\t\tfor i in range(len(filtered_ref_file['title'])):\n+\n+\t\t\ttitle = filtered_ref_file['title'][i]\n+\t\t\tuber_id = filtered_ref_file['uber_id'][i]\n+\n+\t\t\t#If there are new matches, append all relevant columns to the matched_refs DataFrame\n+\t\t\tif title in clean_scraped_text:\n+\n+\t\t\t\trefs_matched_with_title = {\n+\t\t\t\t\t'Document id'       : scraped_text.id,\n+\t\t\t\t\t'Reference id'      : hash(title),","path":"utils/hard_text_search.py","position":64,"original_position":64,"commit_id":"98d5a5ece169e959481474b00cc2228571a25dc7","original_commit_id":"98d5a5ece169e959481474b00cc2228571a25dc7","user":{"login":"SamDepardieu","id":6536364,"node_id":"MDQ6VXNlcjY1MzYzNjQ=","avatar_url":"https://avatars2.githubusercontent.com/u/6536364?v=4","gravatar_id":"","url":"https://api.github.com/users/SamDepardieu","html_url":"https://github.com/SamDepardieu","followers_url":"https://api.github.com/users/SamDepardieu/followers","following_url":"https://api.github.com/users/SamDepardieu/following{/other_user}","gists_url":"https://api.github.com/users/SamDepardieu/gists{/gist_id}","starred_url":"https://api.github.com/users/SamDepardieu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SamDepardieu/subscriptions","organizations_url":"https://api.github.com/users/SamDepardieu/orgs","repos_url":"https://api.github.com/users/SamDepardieu/repos","events_url":"https://api.github.com/users/SamDepardieu/events{/privacy}","received_events_url":"https://api.github.com/users/SamDepardieu/received_events","type":"User","site_admin":false},"body":"We're hashing the full text on our side, so this should be fine 👍 ","created_at":"2019-03-15T11:52:35Z","updated_at":"2019-03-15T11:52:43Z","html_url":"https://github.com/wellcometrust/policytool/pull/100#discussion_r265950185","pull_request_url":"https://api.github.com/repos/wellcometrust/policytool/pulls/100","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/wellcometrust/policytool/pulls/comments/265950185"},"html":{"href":"https://github.com/wellcometrust/policytool/pull/100#discussion_r265950185"},"pull_request":{"href":"https://api.github.com/repos/wellcometrust/policytool/pulls/100"}},"in_reply_to_id":265925739},"pull_request":{"url":"https://api.github.com/repos/wellcometrust/policytool/pulls/100","id":259796465,"node_id":"MDExOlB1bGxSZXF1ZXN0MjU5Nzk2NDY1","html_url":"https://github.com/wellcometrust/policytool/pull/100","diff_url":"https://github.com/wellcometrust/policytool/pull/100.diff","patch_url":"https://github.com/wellcometrust/policytool/pull/100.patch","issue_url":"https://api.github.com/repos/wellcometrust/policytool/issues/100","number":100,"state":"open","locked":false,"title":"Add hard text search to Policy Tool","user":{"login":"peter337","id":4973632,"node_id":"MDQ6VXNlcjQ5NzM2MzI=","avatar_url":"https://avatars1.githubusercontent.com/u/4973632?v=4","gravatar_id":"","url":"https://api.github.com/users/peter337","html_url":"https://github.com/peter337","followers_url":"https://api.github.com/users/peter337/followers","following_url":"https://api.github.com/users/peter337/following{/other_user}","gists_url":"https://api.github.com/users/peter337/gists{/gist_id}","starred_url":"https://api.github.com/users/peter337/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/peter337/subscriptions","organizations_url":"https://api.github.com/users/peter337/orgs","repos_url":"https://api.github.com/users/peter337/repos","events_url":"https://api.github.com/users/peter337/events{/privacy}","received_events_url":"https://api.github.com/users/peter337/received_events","type":"User","site_admin":false},"body":"Won't be available Monday morning so adding this now. Currently runs, but has a few bugs.\r\n\r\n@nsorros, I've encountered a bug while adding rows to `matched_references` which seems to be caused by its duplicate columns name - three of the columns are called 'index'. I currently work around that by removing columns with duplicate names on line 60 of hard_text_search.py.\r\nDo you know what the role if these multiple index columns is/is there any issue in removing them?","created_at":"2019-03-10T22:49:26Z","updated_at":"2019-03-15T11:52:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"87c7e4d0cce9a1981419ba4af3112339da00d23b","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/wellcometrust/policytool/pulls/100/commits","review_comments_url":"https://api.github.com/repos/wellcometrust/policytool/pulls/100/comments","review_comment_url":"https://api.github.com/repos/wellcometrust/policytool/pulls/comments{/number}","comments_url":"https://api.github.com/repos/wellcometrust/policytool/issues/100/comments","statuses_url":"https://api.github.com/repos/wellcometrust/policytool/statuses/98d5a5ece169e959481474b00cc2228571a25dc7","head":{"label":"wellcometrust:add-hard-text","ref":"add-hard-text","sha":"98d5a5ece169e959481474b00cc2228571a25dc7","user":{"login":"wellcometrust","id":2814213,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI4MTQyMTM=","avatar_url":"https://avatars3.githubusercontent.com/u/2814213?v=4","gravatar_id":"","url":"https://api.github.com/users/wellcometrust","html_url":"https://github.com/wellcometrust","followers_url":"https://api.github.com/users/wellcometrust/followers","following_url":"https://api.github.com/users/wellcometrust/following{/other_user}","gists_url":"https://api.github.com/users/wellcometrust/gists{/gist_id}","starred_url":"https://api.github.com/users/wellcometrust/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wellcometrust/subscriptions","organizations_url":"https://api.github.com/users/wellcometrust/orgs","repos_url":"https://api.github.com/users/wellcometrust/repos","events_url":"https://api.github.com/users/wellcometrust/events{/privacy}","received_events_url":"https://api.github.com/users/wellcometrust/received_events","type":"Organization","site_admin":false},"repo":{"id":144130297,"node_id":"MDEwOlJlcG9zaXRvcnkxNDQxMzAyOTc=","name":"policytool","full_name":"wellcometrust/policytool","private":false,"owner":{"login":"wellcometrust","id":2814213,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI4MTQyMTM=","avatar_url":"https://avatars3.githubusercontent.com/u/2814213?v=4","gravatar_id":"","url":"https://api.github.com/users/wellcometrust","html_url":"https://github.com/wellcometrust","followers_url":"https://api.github.com/users/wellcometrust/followers","following_url":"https://api.github.com/users/wellcometrust/following{/other_user}","gists_url":"https://api.github.com/users/wellcometrust/gists{/gist_id}","starred_url":"https://api.github.com/users/wellcometrust/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wellcometrust/subscriptions","organizations_url":"https://api.github.com/users/wellcometrust/orgs","repos_url":"https://api.github.com/users/wellcometrust/repos","events_url":"https://api.github.com/users/wellcometrust/events{/privacy}","received_events_url":"https://api.github.com/users/wellcometrust/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/wellcometrust/policytool","description":"Wellcome tool to parse references scraped from policy documents using machine learning","fork":false,"url":"https://api.github.com/repos/wellcometrust/policytool","forks_url":"https://api.github.com/repos/wellcometrust/policytool/forks","keys_url":"https://api.github.com/repos/wellcometrust/policytool/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wellcometrust/policytool/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wellcometrust/policytool/teams","hooks_url":"https://api.github.com/repos/wellcometrust/policytool/hooks","issue_events_url":"https://api.github.com/repos/wellcometrust/policytool/issues/events{/number}","events_url":"https://api.github.com/repos/wellcometrust/policytool/events","assignees_url":"https://api.github.com/repos/wellcometrust/policytool/assignees{/user}","branches_url":"https://api.github.com/repos/wellcometrust/policytool/branches{/branch}","tags_url":"https://api.github.com/repos/wellcometrust/policytool/tags","blobs_url":"https://api.github.com/repos/wellcometrust/policytool/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wellcometrust/policytool/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wellcometrust/policytool/git/refs{/sha}","trees_url":"https://api.github.com/repos/wellcometrust/policytool/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wellcometrust/policytool/statuses/{sha}","languages_url":"https://api.github.com/repos/wellcometrust/policytool/languages","stargazers_url":"https://api.github.com/repos/wellcometrust/policytool/stargazers","contributors_url":"https://api.github.com/repos/wellcometrust/policytool/contributors","subscribers_url":"https://api.github.com/repos/wellcometrust/policytool/subscribers","subscription_url":"https://api.github.com/repos/wellcometrust/policytool/subscription","commits_url":"https://api.github.com/repos/wellcometrust/policytool/commits{/sha}","git_commits_url":"https://api.github.com/repos/wellcometrust/policytool/git/commits{/sha}","comments_url":"https://api.github.com/repos/wellcometrust/policytool/comments{/number}","issue_comment_url":"https://api.github.com/repos/wellcometrust/policytool/issues/comments{/number}","contents_url":"https://api.github.com/repos/wellcometrust/policytool/contents/{+path}","compare_url":"https://api.github.com/repos/wellcometrust/policytool/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wellcometrust/policytool/merges","archive_url":"https://api.github.com/repos/wellcometrust/policytool/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wellcometrust/policytool/downloads","issues_url":"https://api.github.com/repos/wellcometrust/policytool/issues{/number}","pulls_url":"https://api.github.com/repos/wellcometrust/policytool/pulls{/number}","milestones_url":"https://api.github.com/repos/wellcometrust/policytool/milestones{/number}","notifications_url":"https://api.github.com/repos/wellcometrust/policytool/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wellcometrust/policytool/labels{/name}","releases_url":"https://api.github.com/repos/wellcometrust/policytool/releases{/id}","deployments_url":"https://api.github.com/repos/wellcometrust/policytool/deployments","created_at":"2018-08-09T09:23:51Z","updated_at":"2019-03-14T10:21:17Z","pushed_at":"2019-03-14T17:09:06Z","git_url":"git://github.com/wellcometrust/policytool.git","ssh_url":"git@github.com:wellcometrust/policytool.git","clone_url":"https://github.com/wellcometrust/policytool.git","svn_url":"https://github.com/wellcometrust/policytool","homepage":"","size":2175,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":24,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":24,"watchers":6,"default_branch":"master"}},"base":{"label":"wellcometrust:master","ref":"master","sha":"d46c43b93233adcbce87795334601e656f0b0896","user":{"login":"wellcometrust","id":2814213,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI4MTQyMTM=","avatar_url":"https://avatars3.githubusercontent.com/u/2814213?v=4","gravatar_id":"","url":"https://api.github.com/users/wellcometrust","html_url":"https://github.com/wellcometrust","followers_url":"https://api.github.com/users/wellcometrust/followers","following_url":"https://api.github.com/users/wellcometrust/following{/other_user}","gists_url":"https://api.github.com/users/wellcometrust/gists{/gist_id}","starred_url":"https://api.github.com/users/wellcometrust/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wellcometrust/subscriptions","organizations_url":"https://api.github.com/users/wellcometrust/orgs","repos_url":"https://api.github.com/users/wellcometrust/repos","events_url":"https://api.github.com/users/wellcometrust/events{/privacy}","received_events_url":"https://api.github.com/users/wellcometrust/received_events","type":"Organization","site_admin":false},"repo":{"id":144130297,"node_id":"MDEwOlJlcG9zaXRvcnkxNDQxMzAyOTc=","name":"policytool","full_name":"wellcometrust/policytool","private":false,"owner":{"login":"wellcometrust","id":2814213,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI4MTQyMTM=","avatar_url":"https://avatars3.githubusercontent.com/u/2814213?v=4","gravatar_id":"","url":"https://api.github.com/users/wellcometrust","html_url":"https://github.com/wellcometrust","followers_url":"https://api.github.com/users/wellcometrust/followers","following_url":"https://api.github.com/users/wellcometrust/following{/other_user}","gists_url":"https://api.github.com/users/wellcometrust/gists{/gist_id}","starred_url":"https://api.github.com/users/wellcometrust/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wellcometrust/subscriptions","organizations_url":"https://api.github.com/users/wellcometrust/orgs","repos_url":"https://api.github.com/users/wellcometrust/repos","events_url":"https://api.github.com/users/wellcometrust/events{/privacy}","received_events_url":"https://api.github.com/users/wellcometrust/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/wellcometrust/policytool","description":"Wellcome tool to parse references scraped from policy documents using machine learning","fork":false,"url":"https://api.github.com/repos/wellcometrust/policytool","forks_url":"https://api.github.com/repos/wellcometrust/policytool/forks","keys_url":"https://api.github.com/repos/wellcometrust/policytool/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wellcometrust/policytool/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wellcometrust/policytool/teams","hooks_url":"https://api.github.com/repos/wellcometrust/policytool/hooks","issue_events_url":"https://api.github.com/repos/wellcometrust/policytool/issues/events{/number}","events_url":"https://api.github.com/repos/wellcometrust/policytool/events","assignees_url":"https://api.github.com/repos/wellcometrust/policytool/assignees{/user}","branches_url":"https://api.github.com/repos/wellcometrust/policytool/branches{/branch}","tags_url":"https://api.github.com/repos/wellcometrust/policytool/tags","blobs_url":"https://api.github.com/repos/wellcometrust/policytool/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wellcometrust/policytool/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wellcometrust/policytool/git/refs{/sha}","trees_url":"https://api.github.com/repos/wellcometrust/policytool/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wellcometrust/policytool/statuses/{sha}","languages_url":"https://api.github.com/repos/wellcometrust/policytool/languages","stargazers_url":"https://api.github.com/repos/wellcometrust/policytool/stargazers","contributors_url":"https://api.github.com/repos/wellcometrust/policytool/contributors","subscribers_url":"https://api.github.com/repos/wellcometrust/policytool/subscribers","subscription_url":"https://api.github.com/repos/wellcometrust/policytool/subscription","commits_url":"https://api.github.com/repos/wellcometrust/policytool/commits{/sha}","git_commits_url":"https://api.github.com/repos/wellcometrust/policytool/git/commits{/sha}","comments_url":"https://api.github.com/repos/wellcometrust/policytool/comments{/number}","issue_comment_url":"https://api.github.com/repos/wellcometrust/policytool/issues/comments{/number}","contents_url":"https://api.github.com/repos/wellcometrust/policytool/contents/{+path}","compare_url":"https://api.github.com/repos/wellcometrust/policytool/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wellcometrust/policytool/merges","archive_url":"https://api.github.com/repos/wellcometrust/policytool/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wellcometrust/policytool/downloads","issues_url":"https://api.github.com/repos/wellcometrust/policytool/issues{/number}","pulls_url":"https://api.github.com/repos/wellcometrust/policytool/pulls{/number}","milestones_url":"https://api.github.com/repos/wellcometrust/policytool/milestones{/number}","notifications_url":"https://api.github.com/repos/wellcometrust/policytool/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wellcometrust/policytool/labels{/name}","releases_url":"https://api.github.com/repos/wellcometrust/policytool/releases{/id}","deployments_url":"https://api.github.com/repos/wellcometrust/policytool/deployments","created_at":"2018-08-09T09:23:51Z","updated_at":"2019-03-14T10:21:17Z","pushed_at":"2019-03-14T17:09:06Z","git_url":"git://github.com/wellcometrust/policytool.git","ssh_url":"git@github.com:wellcometrust/policytool.git","clone_url":"https://github.com/wellcometrust/policytool.git","svn_url":"https://github.com/wellcometrust/policytool","homepage":"","size":2175,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":24,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":24,"watchers":6,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/wellcometrust/policytool/pulls/100"},"html":{"href":"https://github.com/wellcometrust/policytool/pull/100"},"issue":{"href":"https://api.github.com/repos/wellcometrust/policytool/issues/100"},"comments":{"href":"https://api.github.com/repos/wellcometrust/policytool/issues/100/comments"},"review_comments":{"href":"https://api.github.com/repos/wellcometrust/policytool/pulls/100/comments"},"review_comment":{"href":"https://api.github.com/repos/wellcometrust/policytool/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/wellcometrust/policytool/pulls/100/commits"},"statuses":{"href":"https://api.github.com/repos/wellcometrust/policytool/statuses/98d5a5ece169e959481474b00cc2228571a25dc7"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"SamDepardieu"}}