{"action":"created","comment":{"url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/comments/357860094","pull_request_review_id":332146094,"id":357860094,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDM1Nzg2MDA5NA==","diff_hunk":"@@ -5,59 +5,79 @@\n import os.path\n import json\n from ..utils import instance_path\n+from ..scrape import hae_dataa\n \n instancepath = instance_path() / \"yle_scrape\"\n instancepath.mkdir(exist_ok=True)\n \n-IMAGE_BASE_URL = \"https://ehdokaskone.yle.webscale.fi/\"\n+#instancepath = os.path.join(instance_path(), \"scraped\")\n+\n+# load in parties and constituents data\n+# if data is missing ask if scraping is wanted to be performed\n+while True:\n+    try:\n+        with open(os.path.join(instancepath, f\"parties.json\"), \"r\", encoding='utf-8') as json_file:\n+            parties = json.load(json_file)\n+\n+        with open(os.path.join(instancepath, f\"constituencies.json\"), \"r\", encoding='utf-8') as json_file:\n+            constituencies = json.load(json_file)\n+        break\n+    except FileNotFoundError:\n+        text = input(\"Seems like you are missing some data. Do you want to scrape now, THIS MIGHT TAKE UP TO 35 minutes Y/N? (case sensitive)\") \n+        if (text.upper() == \"Y\"):\n+            hae_dataa()\n+        else:\n+            raise Exception(\"You choose to not scrape: Missing data can't build\")\n+\n+# load the scraped data to its own data frame\n+df_scraped = pd.DataFrame(columns=['first_name', 'last_name', 'election_number', 'image', 'election_promise_1', 'party', 'constituency'])\n+i = 1\n+while i <= 3000:\n+    try:\n+        with open(os.path.join(instancepath, f\"candidate\" + str(i) + \".json\"), \"r\", encoding='utf-8') as json_file:\n+            candidate = json.load(json_file)\n+            party_name = None\n+            constituency_name = None\n+            for part in parties:\n+                if part['id'] == candidate[\"party_id\"]:\n+                    party_name = part['name_fi']\n+\n+            for consti in constituencies:\n+                if consti['id'] == candidate[\"constituency_id\"]:\n+                    constituency_name = consti['name_fi']\n+\n+            df_scraped = df_scraped.append({'first_name': candidate['first_name'], \n+                'last_name': candidate['last_name'], \n+                'election_number': candidate['election_number'], \n+                'image': candidate['image'], \n+                'election_promise_1': candidate['info']['election_promise_1'],\n+                'party': party_name,\n+                'constituency': constituency_name}, \n+                ignore_index = True)\n+    except (FileNotFoundError, KeyError):\n+        pass","path":"agora_analytica/data/interpolation/combine.py","position":55,"original_position":55,"commit_id":"55e5d6bd9cbea74a917823a3bffa535ed06615a3","original_commit_id":"55e5d6bd9cbea74a917823a3bffa535ed06615a3","user":{"login":"isoteemu","id":281298,"node_id":"MDQ6VXNlcjI4MTI5OA==","avatar_url":"https://avatars0.githubusercontent.com/u/281298?v=4","gravatar_id":"","url":"https://api.github.com/users/isoteemu","html_url":"https://github.com/isoteemu","followers_url":"https://api.github.com/users/isoteemu/followers","following_url":"https://api.github.com/users/isoteemu/following{/other_user}","gists_url":"https://api.github.com/users/isoteemu/gists{/gist_id}","starred_url":"https://api.github.com/users/isoteemu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/isoteemu/subscriptions","organizations_url":"https://api.github.com/users/isoteemu/orgs","repos_url":"https://api.github.com/users/isoteemu/repos","events_url":"https://api.github.com/users/isoteemu/events{/privacy}","received_events_url":"https://api.github.com/users/isoteemu/received_events","type":"User","site_admin":false},"body":"tyhjät exceptionit on vahvasti suositeltuja vältettäväksi. Mutta ymmärrettävästi monasti haluttuja. Jos haluaa jotkin exceptionit skipata, voi käyttää [context manageria](https://docs.python.org/3/library/contextlib.html#contextlib.suppress) (samaa joka huolehtii tiedostojen avauksesta, sulkemisesta):\r\n```python\r\ntry: \r\n    # tee jotain\r\nexcept KeyError:\r\n    pass\r\n```\r\nmuutetaan:\r\n```python\r\nfrom contextlib import suppress\r\nwith supress(KeyError):\r\n    # tee jotain\r\n```","created_at":"2019-12-13T22:39:48Z","updated_at":"2019-12-13T22:56:52Z","html_url":"https://github.com/Agora-Analytica/prototyyppi/pull/52#discussion_r357860094","pull_request_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/52","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/comments/357860094"},"html":{"href":"https://github.com/Agora-Analytica/prototyyppi/pull/52#discussion_r357860094"},"pull_request":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/52"}}},"pull_request":{"url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/52","id":352898542,"node_id":"MDExOlB1bGxSZXF1ZXN0MzUyODk4NTQy","html_url":"https://github.com/Agora-Analytica/prototyyppi/pull/52","diff_url":"https://github.com/Agora-Analytica/prototyyppi/pull/52.diff","patch_url":"https://github.com/Agora-Analytica/prototyyppi/pull/52.patch","issue_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/issues/52","number":52,"state":"open","locked":false,"title":"scrape and combine finishing touches","user":{"login":"jokukayttajanimi","id":56018797,"node_id":"MDQ6VXNlcjU2MDE4Nzk3","avatar_url":"https://avatars3.githubusercontent.com/u/56018797?v=4","gravatar_id":"","url":"https://api.github.com/users/jokukayttajanimi","html_url":"https://github.com/jokukayttajanimi","followers_url":"https://api.github.com/users/jokukayttajanimi/followers","following_url":"https://api.github.com/users/jokukayttajanimi/following{/other_user}","gists_url":"https://api.github.com/users/jokukayttajanimi/gists{/gist_id}","starred_url":"https://api.github.com/users/jokukayttajanimi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jokukayttajanimi/subscriptions","organizations_url":"https://api.github.com/users/jokukayttajanimi/orgs","repos_url":"https://api.github.com/users/jokukayttajanimi/repos","events_url":"https://api.github.com/users/jokukayttajanimi/events{/privacy}","received_events_url":"https://api.github.com/users/jokukayttajanimi/received_events","type":"User","site_admin":false},"body":"","created_at":"2019-12-13T14:02:56Z","updated_at":"2019-12-13T22:56:52Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b3046e9c6ba9bc893c996366ac1dc9b0aa58ca6c","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/52/commits","review_comments_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/52/comments","review_comment_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/issues/52/comments","statuses_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/statuses/55e5d6bd9cbea74a917823a3bffa535ed06615a3","head":{"label":"jokukayttajanimi:master","ref":"master","sha":"55e5d6bd9cbea74a917823a3bffa535ed06615a3","user":{"login":"jokukayttajanimi","id":56018797,"node_id":"MDQ6VXNlcjU2MDE4Nzk3","avatar_url":"https://avatars3.githubusercontent.com/u/56018797?v=4","gravatar_id":"","url":"https://api.github.com/users/jokukayttajanimi","html_url":"https://github.com/jokukayttajanimi","followers_url":"https://api.github.com/users/jokukayttajanimi/followers","following_url":"https://api.github.com/users/jokukayttajanimi/following{/other_user}","gists_url":"https://api.github.com/users/jokukayttajanimi/gists{/gist_id}","starred_url":"https://api.github.com/users/jokukayttajanimi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jokukayttajanimi/subscriptions","organizations_url":"https://api.github.com/users/jokukayttajanimi/orgs","repos_url":"https://api.github.com/users/jokukayttajanimi/repos","events_url":"https://api.github.com/users/jokukayttajanimi/events{/privacy}","received_events_url":"https://api.github.com/users/jokukayttajanimi/received_events","type":"User","site_admin":false},"repo":{"id":215774084,"node_id":"MDEwOlJlcG9zaXRvcnkyMTU3NzQwODQ=","name":"prototyyppi","full_name":"jokukayttajanimi/prototyyppi","private":false,"owner":{"login":"jokukayttajanimi","id":56018797,"node_id":"MDQ6VXNlcjU2MDE4Nzk3","avatar_url":"https://avatars3.githubusercontent.com/u/56018797?v=4","gravatar_id":"","url":"https://api.github.com/users/jokukayttajanimi","html_url":"https://github.com/jokukayttajanimi","followers_url":"https://api.github.com/users/jokukayttajanimi/followers","following_url":"https://api.github.com/users/jokukayttajanimi/following{/other_user}","gists_url":"https://api.github.com/users/jokukayttajanimi/gists{/gist_id}","starred_url":"https://api.github.com/users/jokukayttajanimi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jokukayttajanimi/subscriptions","organizations_url":"https://api.github.com/users/jokukayttajanimi/orgs","repos_url":"https://api.github.com/users/jokukayttajanimi/repos","events_url":"https://api.github.com/users/jokukayttajanimi/events{/privacy}","received_events_url":"https://api.github.com/users/jokukayttajanimi/received_events","type":"User","site_admin":false},"html_url":"https://github.com/jokukayttajanimi/prototyyppi","description":null,"fork":true,"url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi","forks_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/forks","keys_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/teams","hooks_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/hooks","issue_events_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/issues/events{/number}","events_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/events","assignees_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/assignees{/user}","branches_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/branches{/branch}","tags_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/tags","blobs_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/git/refs{/sha}","trees_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/statuses/{sha}","languages_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/languages","stargazers_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/stargazers","contributors_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/contributors","subscribers_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/subscribers","subscription_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/subscription","commits_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/commits{/sha}","git_commits_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/git/commits{/sha}","comments_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/comments{/number}","issue_comment_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/issues/comments{/number}","contents_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/contents/{+path}","compare_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/merges","archive_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/downloads","issues_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/issues{/number}","pulls_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/pulls{/number}","milestones_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/milestones{/number}","notifications_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/labels{/name}","releases_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/releases{/id}","deployments_url":"https://api.github.com/repos/jokukayttajanimi/prototyyppi/deployments","created_at":"2019-10-17T11:18:09Z","updated_at":"2019-12-13T13:56:45Z","pushed_at":"2019-12-13T13:56:41Z","git_url":"git://github.com/jokukayttajanimi/prototyyppi.git","ssh_url":"git@github.com:jokukayttajanimi/prototyyppi.git","clone_url":"https://github.com/jokukayttajanimi/prototyyppi.git","svn_url":"https://github.com/jokukayttajanimi/prototyyppi","homepage":null,"size":403,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"Agora-Analytica:master","ref":"master","sha":"9522f70783d8d5e8e63d35a52eda1382857fa43d","user":{"login":"Agora-Analytica","id":56018592,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2MDE4NTky","avatar_url":"https://avatars2.githubusercontent.com/u/56018592?v=4","gravatar_id":"","url":"https://api.github.com/users/Agora-Analytica","html_url":"https://github.com/Agora-Analytica","followers_url":"https://api.github.com/users/Agora-Analytica/followers","following_url":"https://api.github.com/users/Agora-Analytica/following{/other_user}","gists_url":"https://api.github.com/users/Agora-Analytica/gists{/gist_id}","starred_url":"https://api.github.com/users/Agora-Analytica/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Agora-Analytica/subscriptions","organizations_url":"https://api.github.com/users/Agora-Analytica/orgs","repos_url":"https://api.github.com/users/Agora-Analytica/repos","events_url":"https://api.github.com/users/Agora-Analytica/events{/privacy}","received_events_url":"https://api.github.com/users/Agora-Analytica/received_events","type":"Organization","site_admin":false},"repo":{"id":212039257,"node_id":"MDEwOlJlcG9zaXRvcnkyMTIwMzkyNTc=","name":"prototyyppi","full_name":"Agora-Analytica/prototyyppi","private":false,"owner":{"login":"Agora-Analytica","id":56018592,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2MDE4NTky","avatar_url":"https://avatars2.githubusercontent.com/u/56018592?v=4","gravatar_id":"","url":"https://api.github.com/users/Agora-Analytica","html_url":"https://github.com/Agora-Analytica","followers_url":"https://api.github.com/users/Agora-Analytica/followers","following_url":"https://api.github.com/users/Agora-Analytica/following{/other_user}","gists_url":"https://api.github.com/users/Agora-Analytica/gists{/gist_id}","starred_url":"https://api.github.com/users/Agora-Analytica/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Agora-Analytica/subscriptions","organizations_url":"https://api.github.com/users/Agora-Analytica/orgs","repos_url":"https://api.github.com/users/Agora-Analytica/repos","events_url":"https://api.github.com/users/Agora-Analytica/events{/privacy}","received_events_url":"https://api.github.com/users/Agora-Analytica/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/Agora-Analytica/prototyyppi","description":null,"fork":false,"url":"https://api.github.com/repos/Agora-Analytica/prototyyppi","forks_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/forks","keys_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/teams","hooks_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/hooks","issue_events_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/issues/events{/number}","events_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/events","assignees_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/assignees{/user}","branches_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/branches{/branch}","tags_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/tags","blobs_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/git/refs{/sha}","trees_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/statuses/{sha}","languages_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/languages","stargazers_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/stargazers","contributors_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/contributors","subscribers_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/subscribers","subscription_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/subscription","commits_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/commits{/sha}","git_commits_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/git/commits{/sha}","comments_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/comments{/number}","issue_comment_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/issues/comments{/number}","contents_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/contents/{+path}","compare_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/merges","archive_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/downloads","issues_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/issues{/number}","pulls_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls{/number}","milestones_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/milestones{/number}","notifications_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/labels{/name}","releases_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/releases{/id}","deployments_url":"https://api.github.com/repos/Agora-Analytica/prototyyppi/deployments","created_at":"2019-10-01T07:43:15Z","updated_at":"2019-12-12T01:38:29Z","pushed_at":"2019-12-13T14:02:57Z","git_url":"git://github.com/Agora-Analytica/prototyyppi.git","ssh_url":"git@github.com:Agora-Analytica/prototyyppi.git","clone_url":"https://github.com/Agora-Analytica/prototyyppi.git","svn_url":"https://github.com/Agora-Analytica/prototyyppi","homepage":"https://agora-analytica.herokuapp.com/","size":404,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":15,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":4,"open_issues":15,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/52"},"html":{"href":"https://github.com/Agora-Analytica/prototyyppi/pull/52"},"issue":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/issues/52"},"comments":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/issues/52/comments"},"review_comments":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/52/comments"},"review_comment":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/pulls/52/commits"},"statuses":{"href":"https://api.github.com/repos/Agora-Analytica/prototyyppi/statuses/55e5d6bd9cbea74a917823a3bffa535ed06615a3"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"isoteemu"}}