{"action":"created","comment":{"url":"https://api.github.com/repos/zar777/exercises/pulls/comments/68511297","id":68511297,"diff_hunk":"@@ -0,0 +1,76 @@\n+\"\"\"\n+Create a JSON file used to indexing all words in a config_file.yaml\n+\"\"\"\n+import argparse\n+import codecs\n+import psycopg2\n+import yaml\n+\n+from collections import defaultdict\n+import clean_file\n+\n+\n+def index(config_path):\n+    \"\"\"\n+    Build a dictionary, given a configuration files\n+    :param config_path: Configuration path which contains a list of files that will be cleaned\n+    :return: Index object\n+    \"\"\"\n+    with open(config_path) as config_file:\n+        data = yaml.load(config_file)\n+    if data is not None:\n+        index_words = defaultdict(lambda: defaultdict(list))\n+        for file_dirty in data['files_path']:\n+            with codecs.open(file_dirty.get('file'), 'r', 'utf-8') as source_file:\n+                # Used 1-based indexing for showing line numbers in output representation\n+                for i, line in enumerate(source_file, 1):\n+                    cleaned_line = clean_file.sanitize(line)\n+                    for word in cleaned_line.split():\n+                        index_words[word.lower()][source_file.name].append(i)\n+    else:\n+        raise ValueError('Empty configuration file')\n+    return index_words\n+\n+\n+def build_index(index_words):\n+    \"\"\"\n+    Insert the index in a specific database for store all data\n+    :param index_words: Object which contains the built index\n+    \"\"\"\n+    connection = None\n+    try:\n+        connection = psycopg2.connect(database='index_db', user='gianluca', password='password')\n+        cursor = connection.cursor()\n+        query_update = \"UPDATE index SET occurrences=%s WHERE word=%s AND file=%s;\"\n+\n+        query_insert = (\"INSERT INTO index (word, file, occurrences)\"\n+                        \"SELECT %s, %s, %s\"\n+                        \"WHERE NOT EXISTS (SELECT 1 FROM index WHERE word=%s AND file=%s);\")\n+        for word in index_words:\n+            for file_path in index_words[word]:\n+                data_update = (index_words[word].get(file_path), word, file_path)\n+                cursor.execute(query_update, data_update)\n+                data_insert = (word, file_path, index_words[word].get(file_path), word, file_path)\n+                cursor.execute(query_insert, data_insert)\n+                connection.commit()\n+\n+    except psycopg2.DatabaseError, e:\n+\n+        if connection:\n+            connection.rollback()\n+        print 'Error %s' % e\n+\n+    finally:\n+\n+        if connection:\n+            connection.close()","path":"clean_project/index.py","position":66,"original_position":66,"commit_id":"3df7427cd26ab7442f0e40e7d416bc783aee0d70","original_commit_id":"3df7427cd26ab7442f0e40e7d416bc783aee0d70","user":{"login":"rvitale","id":351499,"avatar_url":"https://avatars.githubusercontent.com/u/351499?v=3","gravatar_id":"","url":"https://api.github.com/users/rvitale","html_url":"https://github.com/rvitale","followers_url":"https://api.github.com/users/rvitale/followers","following_url":"https://api.github.com/users/rvitale/following{/other_user}","gists_url":"https://api.github.com/users/rvitale/gists{/gist_id}","starred_url":"https://api.github.com/users/rvitale/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rvitale/subscriptions","organizations_url":"https://api.github.com/users/rvitale/orgs","repos_url":"https://api.github.com/users/rvitale/repos","events_url":"https://api.github.com/users/rvitale/events{/privacy}","received_events_url":"https://api.github.com/users/rvitale/received_events","type":"User","site_admin":false},"body":"ah ok, but you need to close the cursor here","created_at":"2016-06-26T22:35:40Z","updated_at":"2016-06-26T22:35:40Z","html_url":"https://github.com/zar777/exercises/pull/9#discussion_r68511297","pull_request_url":"https://api.github.com/repos/zar777/exercises/pulls/9","_links":{"self":{"href":"https://api.github.com/repos/zar777/exercises/pulls/comments/68511297"},"html":{"href":"https://github.com/zar777/exercises/pull/9#discussion_r68511297"},"pull_request":{"href":"https://api.github.com/repos/zar777/exercises/pulls/9"}}},"pull_request":{"url":"https://api.github.com/repos/zar777/exercises/pulls/9","id":74453887,"html_url":"https://github.com/zar777/exercises/pull/9","diff_url":"https://github.com/zar777/exercises/pull/9.diff","patch_url":"https://github.com/zar777/exercises/pull/9.patch","issue_url":"https://api.github.com/repos/zar777/exercises/issues/9","number":9,"state":"open","locked":false,"title":"Initial commit of search project","user":{"login":"zar777","id":2695741,"avatar_url":"https://avatars.githubusercontent.com/u/2695741?v=3","gravatar_id":"","url":"https://api.github.com/users/zar777","html_url":"https://github.com/zar777","followers_url":"https://api.github.com/users/zar777/followers","following_url":"https://api.github.com/users/zar777/following{/other_user}","gists_url":"https://api.github.com/users/zar777/gists{/gist_id}","starred_url":"https://api.github.com/users/zar777/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zar777/subscriptions","organizations_url":"https://api.github.com/users/zar777/orgs","repos_url":"https://api.github.com/users/zar777/repos","events_url":"https://api.github.com/users/zar777/events{/privacy}","received_events_url":"https://api.github.com/users/zar777/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-06-20T14:37:02Z","updated_at":"2016-06-26T22:35:40Z","closed_at":null,"merged_at":null,"merge_commit_sha":"178e4410bd2dad0b7d79b863792d1de54c69f10b","assignee":{"login":"llucherini","id":351420,"avatar_url":"https://avatars.githubusercontent.com/u/351420?v=3","gravatar_id":"","url":"https://api.github.com/users/llucherini","html_url":"https://github.com/llucherini","followers_url":"https://api.github.com/users/llucherini/followers","following_url":"https://api.github.com/users/llucherini/following{/other_user}","gists_url":"https://api.github.com/users/llucherini/gists{/gist_id}","starred_url":"https://api.github.com/users/llucherini/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/llucherini/subscriptions","organizations_url":"https://api.github.com/users/llucherini/orgs","repos_url":"https://api.github.com/users/llucherini/repos","events_url":"https://api.github.com/users/llucherini/events{/privacy}","received_events_url":"https://api.github.com/users/llucherini/received_events","type":"User","site_admin":false},"assignees":[{"login":"llucherini","id":351420,"avatar_url":"https://avatars.githubusercontent.com/u/351420?v=3","gravatar_id":"","url":"https://api.github.com/users/llucherini","html_url":"https://github.com/llucherini","followers_url":"https://api.github.com/users/llucherini/followers","following_url":"https://api.github.com/users/llucherini/following{/other_user}","gists_url":"https://api.github.com/users/llucherini/gists{/gist_id}","starred_url":"https://api.github.com/users/llucherini/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/llucherini/subscriptions","organizations_url":"https://api.github.com/users/llucherini/orgs","repos_url":"https://api.github.com/users/llucherini/repos","events_url":"https://api.github.com/users/llucherini/events{/privacy}","received_events_url":"https://api.github.com/users/llucherini/received_events","type":"User","site_admin":false},{"login":"rvitale","id":351499,"avatar_url":"https://avatars.githubusercontent.com/u/351499?v=3","gravatar_id":"","url":"https://api.github.com/users/rvitale","html_url":"https://github.com/rvitale","followers_url":"https://api.github.com/users/rvitale/followers","following_url":"https://api.github.com/users/rvitale/following{/other_user}","gists_url":"https://api.github.com/users/rvitale/gists{/gist_id}","starred_url":"https://api.github.com/users/rvitale/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rvitale/subscriptions","organizations_url":"https://api.github.com/users/rvitale/orgs","repos_url":"https://api.github.com/users/rvitale/repos","events_url":"https://api.github.com/users/rvitale/events{/privacy}","received_events_url":"https://api.github.com/users/rvitale/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/zar777/exercises/pulls/9/commits","review_comments_url":"https://api.github.com/repos/zar777/exercises/pulls/9/comments","review_comment_url":"https://api.github.com/repos/zar777/exercises/pulls/comments{/number}","comments_url":"https://api.github.com/repos/zar777/exercises/issues/9/comments","statuses_url":"https://api.github.com/repos/zar777/exercises/statuses/3df7427cd26ab7442f0e40e7d416bc783aee0d70","head":{"label":"zar777:search-project","ref":"search-project","sha":"3df7427cd26ab7442f0e40e7d416bc783aee0d70","user":{"login":"zar777","id":2695741,"avatar_url":"https://avatars.githubusercontent.com/u/2695741?v=3","gravatar_id":"","url":"https://api.github.com/users/zar777","html_url":"https://github.com/zar777","followers_url":"https://api.github.com/users/zar777/followers","following_url":"https://api.github.com/users/zar777/following{/other_user}","gists_url":"https://api.github.com/users/zar777/gists{/gist_id}","starred_url":"https://api.github.com/users/zar777/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zar777/subscriptions","organizations_url":"https://api.github.com/users/zar777/orgs","repos_url":"https://api.github.com/users/zar777/repos","events_url":"https://api.github.com/users/zar777/events{/privacy}","received_events_url":"https://api.github.com/users/zar777/received_events","type":"User","site_admin":false},"repo":{"id":57226948,"name":"exercises","full_name":"zar777/exercises","owner":{"login":"zar777","id":2695741,"avatar_url":"https://avatars.githubusercontent.com/u/2695741?v=3","gravatar_id":"","url":"https://api.github.com/users/zar777","html_url":"https://github.com/zar777","followers_url":"https://api.github.com/users/zar777/followers","following_url":"https://api.github.com/users/zar777/following{/other_user}","gists_url":"https://api.github.com/users/zar777/gists{/gist_id}","starred_url":"https://api.github.com/users/zar777/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zar777/subscriptions","organizations_url":"https://api.github.com/users/zar777/orgs","repos_url":"https://api.github.com/users/zar777/repos","events_url":"https://api.github.com/users/zar777/events{/privacy}","received_events_url":"https://api.github.com/users/zar777/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/zar777/exercises","description":"","fork":false,"url":"https://api.github.com/repos/zar777/exercises","forks_url":"https://api.github.com/repos/zar777/exercises/forks","keys_url":"https://api.github.com/repos/zar777/exercises/keys{/key_id}","collaborators_url":"https://api.github.com/repos/zar777/exercises/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/zar777/exercises/teams","hooks_url":"https://api.github.com/repos/zar777/exercises/hooks","issue_events_url":"https://api.github.com/repos/zar777/exercises/issues/events{/number}","events_url":"https://api.github.com/repos/zar777/exercises/events","assignees_url":"https://api.github.com/repos/zar777/exercises/assignees{/user}","branches_url":"https://api.github.com/repos/zar777/exercises/branches{/branch}","tags_url":"https://api.github.com/repos/zar777/exercises/tags","blobs_url":"https://api.github.com/repos/zar777/exercises/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/zar777/exercises/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/zar777/exercises/git/refs{/sha}","trees_url":"https://api.github.com/repos/zar777/exercises/git/trees{/sha}","statuses_url":"https://api.github.com/repos/zar777/exercises/statuses/{sha}","languages_url":"https://api.github.com/repos/zar777/exercises/languages","stargazers_url":"https://api.github.com/repos/zar777/exercises/stargazers","contributors_url":"https://api.github.com/repos/zar777/exercises/contributors","subscribers_url":"https://api.github.com/repos/zar777/exercises/subscribers","subscription_url":"https://api.github.com/repos/zar777/exercises/subscription","commits_url":"https://api.github.com/repos/zar777/exercises/commits{/sha}","git_commits_url":"https://api.github.com/repos/zar777/exercises/git/commits{/sha}","comments_url":"https://api.github.com/repos/zar777/exercises/comments{/number}","issue_comment_url":"https://api.github.com/repos/zar777/exercises/issues/comments{/number}","contents_url":"https://api.github.com/repos/zar777/exercises/contents/{+path}","compare_url":"https://api.github.com/repos/zar777/exercises/compare/{base}...{head}","merges_url":"https://api.github.com/repos/zar777/exercises/merges","archive_url":"https://api.github.com/repos/zar777/exercises/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/zar777/exercises/downloads","issues_url":"https://api.github.com/repos/zar777/exercises/issues{/number}","pulls_url":"https://api.github.com/repos/zar777/exercises/pulls{/number}","milestones_url":"https://api.github.com/repos/zar777/exercises/milestones{/number}","notifications_url":"https://api.github.com/repos/zar777/exercises/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/zar777/exercises/labels{/name}","releases_url":"https://api.github.com/repos/zar777/exercises/releases{/id}","deployments_url":"https://api.github.com/repos/zar777/exercises/deployments","created_at":"2016-04-27T16:02:48Z","updated_at":"2016-04-27T16:07:40Z","pushed_at":"2016-06-26T13:38:54Z","git_url":"git://github.com/zar777/exercises.git","ssh_url":"git@github.com:zar777/exercises.git","clone_url":"https://github.com/zar777/exercises.git","svn_url":"https://github.com/zar777/exercises","homepage":null,"size":100,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":7,"forks":0,"open_issues":7,"watchers":0,"default_branch":"master"}},"base":{"label":"zar777:master","ref":"master","sha":"054aaddb0b0b5f8c44bdd6e839473e0c163057d3","user":{"login":"zar777","id":2695741,"avatar_url":"https://avatars.githubusercontent.com/u/2695741?v=3","gravatar_id":"","url":"https://api.github.com/users/zar777","html_url":"https://github.com/zar777","followers_url":"https://api.github.com/users/zar777/followers","following_url":"https://api.github.com/users/zar777/following{/other_user}","gists_url":"https://api.github.com/users/zar777/gists{/gist_id}","starred_url":"https://api.github.com/users/zar777/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zar777/subscriptions","organizations_url":"https://api.github.com/users/zar777/orgs","repos_url":"https://api.github.com/users/zar777/repos","events_url":"https://api.github.com/users/zar777/events{/privacy}","received_events_url":"https://api.github.com/users/zar777/received_events","type":"User","site_admin":false},"repo":{"id":57226948,"name":"exercises","full_name":"zar777/exercises","owner":{"login":"zar777","id":2695741,"avatar_url":"https://avatars.githubusercontent.com/u/2695741?v=3","gravatar_id":"","url":"https://api.github.com/users/zar777","html_url":"https://github.com/zar777","followers_url":"https://api.github.com/users/zar777/followers","following_url":"https://api.github.com/users/zar777/following{/other_user}","gists_url":"https://api.github.com/users/zar777/gists{/gist_id}","starred_url":"https://api.github.com/users/zar777/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zar777/subscriptions","organizations_url":"https://api.github.com/users/zar777/orgs","repos_url":"https://api.github.com/users/zar777/repos","events_url":"https://api.github.com/users/zar777/events{/privacy}","received_events_url":"https://api.github.com/users/zar777/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/zar777/exercises","description":"","fork":false,"url":"https://api.github.com/repos/zar777/exercises","forks_url":"https://api.github.com/repos/zar777/exercises/forks","keys_url":"https://api.github.com/repos/zar777/exercises/keys{/key_id}","collaborators_url":"https://api.github.com/repos/zar777/exercises/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/zar777/exercises/teams","hooks_url":"https://api.github.com/repos/zar777/exercises/hooks","issue_events_url":"https://api.github.com/repos/zar777/exercises/issues/events{/number}","events_url":"https://api.github.com/repos/zar777/exercises/events","assignees_url":"https://api.github.com/repos/zar777/exercises/assignees{/user}","branches_url":"https://api.github.com/repos/zar777/exercises/branches{/branch}","tags_url":"https://api.github.com/repos/zar777/exercises/tags","blobs_url":"https://api.github.com/repos/zar777/exercises/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/zar777/exercises/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/zar777/exercises/git/refs{/sha}","trees_url":"https://api.github.com/repos/zar777/exercises/git/trees{/sha}","statuses_url":"https://api.github.com/repos/zar777/exercises/statuses/{sha}","languages_url":"https://api.github.com/repos/zar777/exercises/languages","stargazers_url":"https://api.github.com/repos/zar777/exercises/stargazers","contributors_url":"https://api.github.com/repos/zar777/exercises/contributors","subscribers_url":"https://api.github.com/repos/zar777/exercises/subscribers","subscription_url":"https://api.github.com/repos/zar777/exercises/subscription","commits_url":"https://api.github.com/repos/zar777/exercises/commits{/sha}","git_commits_url":"https://api.github.com/repos/zar777/exercises/git/commits{/sha}","comments_url":"https://api.github.com/repos/zar777/exercises/comments{/number}","issue_comment_url":"https://api.github.com/repos/zar777/exercises/issues/comments{/number}","contents_url":"https://api.github.com/repos/zar777/exercises/contents/{+path}","compare_url":"https://api.github.com/repos/zar777/exercises/compare/{base}...{head}","merges_url":"https://api.github.com/repos/zar777/exercises/merges","archive_url":"https://api.github.com/repos/zar777/exercises/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/zar777/exercises/downloads","issues_url":"https://api.github.com/repos/zar777/exercises/issues{/number}","pulls_url":"https://api.github.com/repos/zar777/exercises/pulls{/number}","milestones_url":"https://api.github.com/repos/zar777/exercises/milestones{/number}","notifications_url":"https://api.github.com/repos/zar777/exercises/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/zar777/exercises/labels{/name}","releases_url":"https://api.github.com/repos/zar777/exercises/releases{/id}","deployments_url":"https://api.github.com/repos/zar777/exercises/deployments","created_at":"2016-04-27T16:02:48Z","updated_at":"2016-04-27T16:07:40Z","pushed_at":"2016-06-26T13:38:54Z","git_url":"git://github.com/zar777/exercises.git","ssh_url":"git@github.com:zar777/exercises.git","clone_url":"https://github.com/zar777/exercises.git","svn_url":"https://github.com/zar777/exercises","homepage":null,"size":100,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":7,"forks":0,"open_issues":7,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/zar777/exercises/pulls/9"},"html":{"href":"https://github.com/zar777/exercises/pull/9"},"issue":{"href":"https://api.github.com/repos/zar777/exercises/issues/9"},"comments":{"href":"https://api.github.com/repos/zar777/exercises/issues/9/comments"},"review_comments":{"href":"https://api.github.com/repos/zar777/exercises/pulls/9/comments"},"review_comment":{"href":"https://api.github.com/repos/zar777/exercises/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/zar777/exercises/pulls/9/commits"},"statuses":{"href":"https://api.github.com/repos/zar777/exercises/statuses/3df7427cd26ab7442f0e40e7d416bc783aee0d70"}}}}
{"actor":{"display_login":"rvitale"}}