{"action":"created","comment":{"url":"https://api.github.com/repos/opentrials/processors/pulls/comments/71360200","id":71360200,"diff_hunk":"@@ -70,3 +76,27 @@ def write_location(conn, location, source_id, trial_id=None):\n             'created' if create else 'updated', name)\n \n     return object['id']\n+\n+#----helper function to get the canonical name of the locations\n+def extract_CanonicalName(location):\n+    location_name = location.lower().strip().replace(\".\", \"\")\n+    try:\n+        canonical_name = countries.get(location_name).name\n+        logger.debug('Location - %s: %s','normalized', canonical_name)\n+    except KeyError:\n+        canonical_name = location\n+        # This part is for looking for matching cases (suggestion), uses the \n+        # fuzzy logic to get the country name who get 80% of match rate.\n+        with open(os.path.join(os.path.dirname(__file__),'countries.csv'), 'r') as f:\n+            countries_data = csv.reader(f, delimiter= str(u','))\n+            logger.debug('Location - %s: %s','not normalized', canonical_name)\n+            best_similarity = 0\n+            for country in countries_data:\n+                unicode_row = [x.decode('utf8') for x in country]\n+                country_name = unicode_row[0].lower().strip().replace(\".\", \"\")\n+                if ((fuzz.ratio(location_name, country_name) >= 80) and \n+                    (fuzz.ratio(location_name, country_name) > best_similarity)):\n+                    canonical_name = countries.get(unicode_row[3]).name\n+                    logger.debug('Location - %s: %s','not normalized with the most close result', canonical_name)","path":"processors/base/writers/location.py","position":47,"original_position":47,"commit_id":"68bb41fd98e441fb6fcf87b716244c011fc98158","original_commit_id":"68bb41fd98e441fb6fcf87b716244c011fc98158","user":{"login":"vitorbaptista","id":76945,"avatar_url":"https://avatars.githubusercontent.com/u/76945?v=3","gravatar_id":"","url":"https://api.github.com/users/vitorbaptista","html_url":"https://github.com/vitorbaptista","followers_url":"https://api.github.com/users/vitorbaptista/followers","following_url":"https://api.github.com/users/vitorbaptista/following{/other_user}","gists_url":"https://api.github.com/users/vitorbaptista/gists{/gist_id}","starred_url":"https://api.github.com/users/vitorbaptista/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vitorbaptista/subscriptions","organizations_url":"https://api.github.com/users/vitorbaptista/orgs","repos_url":"https://api.github.com/users/vitorbaptista/repos","events_url":"https://api.github.com/users/vitorbaptista/events{/privacy}","received_events_url":"https://api.github.com/users/vitorbaptista/received_events","type":"User","site_admin":false},"body":"The `best_similarity` is never updated. To avoid doing these two `fuzz.ratio` calls, you could simply set `best_similarity = 80` and then just test if `fuzz.ratio(...) >= best_similarity`, updating the `best_similarity` value.","created_at":"2016-07-19T15:19:53Z","updated_at":"2016-07-19T15:19:53Z","html_url":"https://github.com/opentrials/processors/pull/34#discussion_r71360200","pull_request_url":"https://api.github.com/repos/opentrials/processors/pulls/34","_links":{"self":{"href":"https://api.github.com/repos/opentrials/processors/pulls/comments/71360200"},"html":{"href":"https://github.com/opentrials/processors/pull/34#discussion_r71360200"},"pull_request":{"href":"https://api.github.com/repos/opentrials/processors/pulls/34"}}},"pull_request":{"url":"https://api.github.com/repos/opentrials/processors/pulls/34","id":77796174,"html_url":"https://github.com/opentrials/processors/pull/34","diff_url":"https://github.com/opentrials/processors/pull/34.diff","patch_url":"https://github.com/opentrials/processors/pull/34.patch","issue_url":"https://api.github.com/repos/opentrials/processors/issues/34","number":34,"state":"open","locked":false,"title":"First version of the code with a country normalization","user":{"login":"Iararibeiro","id":3472291,"avatar_url":"https://avatars.githubusercontent.com/u/3472291?v=3","gravatar_id":"","url":"https://api.github.com/users/Iararibeiro","html_url":"https://github.com/Iararibeiro","followers_url":"https://api.github.com/users/Iararibeiro/followers","following_url":"https://api.github.com/users/Iararibeiro/following{/other_user}","gists_url":"https://api.github.com/users/Iararibeiro/gists{/gist_id}","starred_url":"https://api.github.com/users/Iararibeiro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Iararibeiro/subscriptions","organizations_url":"https://api.github.com/users/Iararibeiro/orgs","repos_url":"https://api.github.com/users/Iararibeiro/repos","events_url":"https://api.github.com/users/Iararibeiro/events{/privacy}","received_events_url":"https://api.github.com/users/Iararibeiro/received_events","type":"User","site_admin":false},"body":"Some details about it, after testing all the suggested libs for @vitorbaptista  I decided use the one that @roll suggested combine with the csv table (is not a beautiful solution, but I gonna explain the decisions).\r\nFirst the lib that I use resolve more problems than the ones suggested, it works if all the \"normal\" normalization cases and resolve some cases like synonyms (\"US\", 'USA\" case). Another good point is because the lib offers a  search and results for more options, so if in the future you guys decide works with the ISO name, alpha or numeric that still will works.\r\nI thought that I good option to deal with the corner cases or some problems that can show in the future was using some Fuzzy in the code, so if you can't find a name of the place using the lib, the code check in a table if exist a country that match in 80%. That works in cases like \"United States of America\" and common error like \"Brasil\", and in the future if you guys decided deal with the other corners cases in a different way is just add lines in that table. Again I thought that was a good improvement, and can be changed easily. ","created_at":"2016-07-18T11:53:21Z","updated_at":"2016-07-19T15:19:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bec6c8ca52d1ae8b9b85e98bee9c556ec07e1e4e","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/opentrials/processors/pulls/34/commits","review_comments_url":"https://api.github.com/repos/opentrials/processors/pulls/34/comments","review_comment_url":"https://api.github.com/repos/opentrials/processors/pulls/comments{/number}","comments_url":"https://api.github.com/repos/opentrials/processors/issues/34/comments","statuses_url":"https://api.github.com/repos/opentrials/processors/statuses/68bb41fd98e441fb6fcf87b716244c011fc98158","head":{"label":"Iararibeiro:master","ref":"master","sha":"68bb41fd98e441fb6fcf87b716244c011fc98158","user":{"login":"Iararibeiro","id":3472291,"avatar_url":"https://avatars.githubusercontent.com/u/3472291?v=3","gravatar_id":"","url":"https://api.github.com/users/Iararibeiro","html_url":"https://github.com/Iararibeiro","followers_url":"https://api.github.com/users/Iararibeiro/followers","following_url":"https://api.github.com/users/Iararibeiro/following{/other_user}","gists_url":"https://api.github.com/users/Iararibeiro/gists{/gist_id}","starred_url":"https://api.github.com/users/Iararibeiro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Iararibeiro/subscriptions","organizations_url":"https://api.github.com/users/Iararibeiro/orgs","repos_url":"https://api.github.com/users/Iararibeiro/repos","events_url":"https://api.github.com/users/Iararibeiro/events{/privacy}","received_events_url":"https://api.github.com/users/Iararibeiro/received_events","type":"User","site_admin":false},"repo":{"id":63597673,"name":"processors","full_name":"Iararibeiro/processors","owner":{"login":"Iararibeiro","id":3472291,"avatar_url":"https://avatars.githubusercontent.com/u/3472291?v=3","gravatar_id":"","url":"https://api.github.com/users/Iararibeiro","html_url":"https://github.com/Iararibeiro","followers_url":"https://api.github.com/users/Iararibeiro/followers","following_url":"https://api.github.com/users/Iararibeiro/following{/other_user}","gists_url":"https://api.github.com/users/Iararibeiro/gists{/gist_id}","starred_url":"https://api.github.com/users/Iararibeiro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Iararibeiro/subscriptions","organizations_url":"https://api.github.com/users/Iararibeiro/orgs","repos_url":"https://api.github.com/users/Iararibeiro/repos","events_url":"https://api.github.com/users/Iararibeiro/events{/privacy}","received_events_url":"https://api.github.com/users/Iararibeiro/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/Iararibeiro/processors","description":"The OpenTrials data processors.","fork":true,"url":"https://api.github.com/repos/Iararibeiro/processors","forks_url":"https://api.github.com/repos/Iararibeiro/processors/forks","keys_url":"https://api.github.com/repos/Iararibeiro/processors/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Iararibeiro/processors/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Iararibeiro/processors/teams","hooks_url":"https://api.github.com/repos/Iararibeiro/processors/hooks","issue_events_url":"https://api.github.com/repos/Iararibeiro/processors/issues/events{/number}","events_url":"https://api.github.com/repos/Iararibeiro/processors/events","assignees_url":"https://api.github.com/repos/Iararibeiro/processors/assignees{/user}","branches_url":"https://api.github.com/repos/Iararibeiro/processors/branches{/branch}","tags_url":"https://api.github.com/repos/Iararibeiro/processors/tags","blobs_url":"https://api.github.com/repos/Iararibeiro/processors/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Iararibeiro/processors/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Iararibeiro/processors/git/refs{/sha}","trees_url":"https://api.github.com/repos/Iararibeiro/processors/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Iararibeiro/processors/statuses/{sha}","languages_url":"https://api.github.com/repos/Iararibeiro/processors/languages","stargazers_url":"https://api.github.com/repos/Iararibeiro/processors/stargazers","contributors_url":"https://api.github.com/repos/Iararibeiro/processors/contributors","subscribers_url":"https://api.github.com/repos/Iararibeiro/processors/subscribers","subscription_url":"https://api.github.com/repos/Iararibeiro/processors/subscription","commits_url":"https://api.github.com/repos/Iararibeiro/processors/commits{/sha}","git_commits_url":"https://api.github.com/repos/Iararibeiro/processors/git/commits{/sha}","comments_url":"https://api.github.com/repos/Iararibeiro/processors/comments{/number}","issue_comment_url":"https://api.github.com/repos/Iararibeiro/processors/issues/comments{/number}","contents_url":"https://api.github.com/repos/Iararibeiro/processors/contents/{+path}","compare_url":"https://api.github.com/repos/Iararibeiro/processors/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Iararibeiro/processors/merges","archive_url":"https://api.github.com/repos/Iararibeiro/processors/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Iararibeiro/processors/downloads","issues_url":"https://api.github.com/repos/Iararibeiro/processors/issues{/number}","pulls_url":"https://api.github.com/repos/Iararibeiro/processors/pulls{/number}","milestones_url":"https://api.github.com/repos/Iararibeiro/processors/milestones{/number}","notifications_url":"https://api.github.com/repos/Iararibeiro/processors/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Iararibeiro/processors/labels{/name}","releases_url":"https://api.github.com/repos/Iararibeiro/processors/releases{/id}","deployments_url":"https://api.github.com/repos/Iararibeiro/processors/deployments","created_at":"2016-07-18T11:32:57Z","updated_at":"2016-07-18T11:32:59Z","pushed_at":"2016-07-18T22:08:05Z","git_url":"git://github.com/Iararibeiro/processors.git","ssh_url":"git@github.com:Iararibeiro/processors.git","clone_url":"https://github.com/Iararibeiro/processors.git","svn_url":"https://github.com/Iararibeiro/processors","homepage":"","size":461,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"opentrials:master","ref":"master","sha":"5e61805589958921f33542a28b42b7e2a53ed0ad","user":{"login":"opentrials","id":13900836,"avatar_url":"https://avatars.githubusercontent.com/u/13900836?v=3","gravatar_id":"","url":"https://api.github.com/users/opentrials","html_url":"https://github.com/opentrials","followers_url":"https://api.github.com/users/opentrials/followers","following_url":"https://api.github.com/users/opentrials/following{/other_user}","gists_url":"https://api.github.com/users/opentrials/gists{/gist_id}","starred_url":"https://api.github.com/users/opentrials/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/opentrials/subscriptions","organizations_url":"https://api.github.com/users/opentrials/orgs","repos_url":"https://api.github.com/users/opentrials/repos","events_url":"https://api.github.com/users/opentrials/events{/privacy}","received_events_url":"https://api.github.com/users/opentrials/received_events","type":"Organization","site_admin":false},"repo":{"id":50779358,"name":"processors","full_name":"opentrials/processors","owner":{"login":"opentrials","id":13900836,"avatar_url":"https://avatars.githubusercontent.com/u/13900836?v=3","gravatar_id":"","url":"https://api.github.com/users/opentrials","html_url":"https://github.com/opentrials","followers_url":"https://api.github.com/users/opentrials/followers","following_url":"https://api.github.com/users/opentrials/following{/other_user}","gists_url":"https://api.github.com/users/opentrials/gists{/gist_id}","starred_url":"https://api.github.com/users/opentrials/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/opentrials/subscriptions","organizations_url":"https://api.github.com/users/opentrials/orgs","repos_url":"https://api.github.com/users/opentrials/repos","events_url":"https://api.github.com/users/opentrials/events{/privacy}","received_events_url":"https://api.github.com/users/opentrials/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/opentrials/processors","description":"The OpenTrials data processors.","fork":false,"url":"https://api.github.com/repos/opentrials/processors","forks_url":"https://api.github.com/repos/opentrials/processors/forks","keys_url":"https://api.github.com/repos/opentrials/processors/keys{/key_id}","collaborators_url":"https://api.github.com/repos/opentrials/processors/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/opentrials/processors/teams","hooks_url":"https://api.github.com/repos/opentrials/processors/hooks","issue_events_url":"https://api.github.com/repos/opentrials/processors/issues/events{/number}","events_url":"https://api.github.com/repos/opentrials/processors/events","assignees_url":"https://api.github.com/repos/opentrials/processors/assignees{/user}","branches_url":"https://api.github.com/repos/opentrials/processors/branches{/branch}","tags_url":"https://api.github.com/repos/opentrials/processors/tags","blobs_url":"https://api.github.com/repos/opentrials/processors/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/opentrials/processors/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/opentrials/processors/git/refs{/sha}","trees_url":"https://api.github.com/repos/opentrials/processors/git/trees{/sha}","statuses_url":"https://api.github.com/repos/opentrials/processors/statuses/{sha}","languages_url":"https://api.github.com/repos/opentrials/processors/languages","stargazers_url":"https://api.github.com/repos/opentrials/processors/stargazers","contributors_url":"https://api.github.com/repos/opentrials/processors/contributors","subscribers_url":"https://api.github.com/repos/opentrials/processors/subscribers","subscription_url":"https://api.github.com/repos/opentrials/processors/subscription","commits_url":"https://api.github.com/repos/opentrials/processors/commits{/sha}","git_commits_url":"https://api.github.com/repos/opentrials/processors/git/commits{/sha}","comments_url":"https://api.github.com/repos/opentrials/processors/comments{/number}","issue_comment_url":"https://api.github.com/repos/opentrials/processors/issues/comments{/number}","contents_url":"https://api.github.com/repos/opentrials/processors/contents/{+path}","compare_url":"https://api.github.com/repos/opentrials/processors/compare/{base}...{head}","merges_url":"https://api.github.com/repos/opentrials/processors/merges","archive_url":"https://api.github.com/repos/opentrials/processors/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/opentrials/processors/downloads","issues_url":"https://api.github.com/repos/opentrials/processors/issues{/number}","pulls_url":"https://api.github.com/repos/opentrials/processors/pulls{/number}","milestones_url":"https://api.github.com/repos/opentrials/processors/milestones{/number}","notifications_url":"https://api.github.com/repos/opentrials/processors/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/opentrials/processors/labels{/name}","releases_url":"https://api.github.com/repos/opentrials/processors/releases{/id}","deployments_url":"https://api.github.com/repos/opentrials/processors/deployments","created_at":"2016-01-31T14:53:31Z","updated_at":"2016-06-28T21:19:57Z","pushed_at":"2016-07-18T22:08:05Z","git_url":"git://github.com/opentrials/processors.git","ssh_url":"git@github.com:opentrials/processors.git","clone_url":"https://github.com/opentrials/processors.git","svn_url":"https://github.com/opentrials/processors","homepage":"","size":483,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":2,"forks":4,"open_issues":2,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/opentrials/processors/pulls/34"},"html":{"href":"https://github.com/opentrials/processors/pull/34"},"issue":{"href":"https://api.github.com/repos/opentrials/processors/issues/34"},"comments":{"href":"https://api.github.com/repos/opentrials/processors/issues/34/comments"},"review_comments":{"href":"https://api.github.com/repos/opentrials/processors/pulls/34/comments"},"review_comment":{"href":"https://api.github.com/repos/opentrials/processors/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/opentrials/processors/pulls/34/commits"},"statuses":{"href":"https://api.github.com/repos/opentrials/processors/statuses/68bb41fd98e441fb6fcf87b716244c011fc98158"}}}}
{"actor":{"display_login":"vitorbaptista"}}