{"action":"created","comment":{"url":"https://api.github.com/repos/weecology/retriever/pulls/comments/169392045","pull_request_review_id":97911372,"id":169392045,"diff_hunk":"@@ -0,0 +1,97 @@\n+from retriever import install_sqlite\n+from retriever.lib.engine_tools import getmd5\n+from retriever import datasets\n+from retriever.engines import engine_list\n+from tempfile import mkdtemp\n+from shutil import rmtree\n+import os\n+import urllib.request\n+import json\n+from datetime import datetime\n+\n+mysql_engine, postgres_engine, sqlite_engine, msaccess_engine, csv_engine, download_engine, json_engine, xml_engine = engine_list\n+\n+file_location = os.path.dirname(os.path.realpath(__file__))\n+\n+example_datasets = ['bird-size', 'mammal-masses', 'airports', 'portal']\n+precalculated_md5 = {'airports': 'f16a0bb21d347e2ba180cde26f35c87a',\n+                     'bird-size': '98dcfdca19d729c90ee1c6db5221b775',\n+                     'mammal-masses': '6fec0fc63007a4040d9bbc5cfcd9953e',\n+                     'portal': 'aaa2b66a3a77efe416e3555c444f4e2e'}\n+\n+\n+def check_urls_exist(dataset):\n+    \"\"\"Check to see if all resources of a dataset\n+     exist at given urls and return a dictionary of failing ones.\"\"\"\n+    failed = {}\n+    for (url_key, url) in dataset.tables.items():\n+        request = urllib.request.Request(url)\n+        request.get_method = lambda: 'HEAD'\n+        try:\n+            urllib.request.urlopen(request)\n+        except urllib.request.HTTPError as e:\n+            failed[url_key] = (url, e.code)\n+    return failed\n+\n+\n+def get_last_modified(dataset):\n+    \"\"\"Returns a dictionary of tables with the last modified date as","path":"status_dashboard.py","position":38,"original_position":38,"commit_id":"340ad4d63acc30119fcc4184dea16f6dd11dc319","original_commit_id":"340ad4d63acc30119fcc4184dea16f6dd11dc319","user":{"login":"apoorvaeternity","id":21103831,"avatar_url":"https://avatars3.githubusercontent.com/u/21103831?v=4","gravatar_id":"","url":"https://api.github.com/users/apoorvaeternity","html_url":"https://github.com/apoorvaeternity","followers_url":"https://api.github.com/users/apoorvaeternity/followers","following_url":"https://api.github.com/users/apoorvaeternity/following{/other_user}","gists_url":"https://api.github.com/users/apoorvaeternity/gists{/gist_id}","starred_url":"https://api.github.com/users/apoorvaeternity/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apoorvaeternity/subscriptions","organizations_url":"https://api.github.com/users/apoorvaeternity/orgs","repos_url":"https://api.github.com/users/apoorvaeternity/repos","events_url":"https://api.github.com/users/apoorvaeternity/events{/privacy}","received_events_url":"https://api.github.com/users/apoorvaeternity/received_events","type":"User","site_admin":false},"body":"@henrykironde I will update the comments so it becomes clearer what each of the functions are doing.","created_at":"2018-02-20T17:12:39Z","updated_at":"2018-02-20T17:12:39Z","html_url":"https://github.com/weecology/retriever/pull/1068#discussion_r169392045","pull_request_url":"https://api.github.com/repos/weecology/retriever/pulls/1068","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/weecology/retriever/pulls/comments/169392045"},"html":{"href":"https://github.com/weecology/retriever/pull/1068#discussion_r169392045"},"pull_request":{"href":"https://api.github.com/repos/weecology/retriever/pulls/1068"}},"in_reply_to_id":169385710},"pull_request":{"url":"https://api.github.com/repos/weecology/retriever/pulls/1068","id":167016219,"html_url":"https://github.com/weecology/retriever/pull/1068","diff_url":"https://github.com/weecology/retriever/pull/1068.diff","patch_url":"https://github.com/weecology/retriever/pull/1068.patch","issue_url":"https://api.github.com/repos/weecology/retriever/issues/1068","number":1068,"state":"open","locked":false,"title":"[WIP] Create status_dashboard.py","user":{"login":"apoorvaeternity","id":21103831,"avatar_url":"https://avatars3.githubusercontent.com/u/21103831?v=4","gravatar_id":"","url":"https://api.github.com/users/apoorvaeternity","html_url":"https://github.com/apoorvaeternity","followers_url":"https://api.github.com/users/apoorvaeternity/followers","following_url":"https://api.github.com/users/apoorvaeternity/following{/other_user}","gists_url":"https://api.github.com/users/apoorvaeternity/gists{/gist_id}","starred_url":"https://api.github.com/users/apoorvaeternity/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apoorvaeternity/subscriptions","organizations_url":"https://api.github.com/users/apoorvaeternity/orgs","repos_url":"https://api.github.com/users/apoorvaeternity/repos","events_url":"https://api.github.com/users/apoorvaeternity/events{/privacy}","received_events_url":"https://api.github.com/users/apoorvaeternity/received_events","type":"User","site_admin":false},"body":"Initial script to check whether datasets are installing. ","created_at":"2018-02-04T15:55:58Z","updated_at":"2018-02-20T17:12:39Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1466c3f960e29d2fe82a2b4ed3b839d73efbe28a","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":557227948,"url":"https://api.github.com/repos/weecology/retriever/labels/Under%20Review%20and%20Tests","name":"Under Review and Tests","color":"d93f0b","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/weecology/retriever/pulls/1068/commits","review_comments_url":"https://api.github.com/repos/weecology/retriever/pulls/1068/comments","review_comment_url":"https://api.github.com/repos/weecology/retriever/pulls/comments{/number}","comments_url":"https://api.github.com/repos/weecology/retriever/issues/1068/comments","statuses_url":"https://api.github.com/repos/weecology/retriever/statuses/340ad4d63acc30119fcc4184dea16f6dd11dc319","head":{"label":"apoorvaeternity:dashboard","ref":"dashboard","sha":"340ad4d63acc30119fcc4184dea16f6dd11dc319","user":{"login":"apoorvaeternity","id":21103831,"avatar_url":"https://avatars3.githubusercontent.com/u/21103831?v=4","gravatar_id":"","url":"https://api.github.com/users/apoorvaeternity","html_url":"https://github.com/apoorvaeternity","followers_url":"https://api.github.com/users/apoorvaeternity/followers","following_url":"https://api.github.com/users/apoorvaeternity/following{/other_user}","gists_url":"https://api.github.com/users/apoorvaeternity/gists{/gist_id}","starred_url":"https://api.github.com/users/apoorvaeternity/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apoorvaeternity/subscriptions","organizations_url":"https://api.github.com/users/apoorvaeternity/orgs","repos_url":"https://api.github.com/users/apoorvaeternity/repos","events_url":"https://api.github.com/users/apoorvaeternity/events{/privacy}","received_events_url":"https://api.github.com/users/apoorvaeternity/received_events","type":"User","site_admin":false},"repo":{"id":111313311,"name":"retriever","full_name":"apoorvaeternity/retriever","owner":{"login":"apoorvaeternity","id":21103831,"avatar_url":"https://avatars3.githubusercontent.com/u/21103831?v=4","gravatar_id":"","url":"https://api.github.com/users/apoorvaeternity","html_url":"https://github.com/apoorvaeternity","followers_url":"https://api.github.com/users/apoorvaeternity/followers","following_url":"https://api.github.com/users/apoorvaeternity/following{/other_user}","gists_url":"https://api.github.com/users/apoorvaeternity/gists{/gist_id}","starred_url":"https://api.github.com/users/apoorvaeternity/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apoorvaeternity/subscriptions","organizations_url":"https://api.github.com/users/apoorvaeternity/orgs","repos_url":"https://api.github.com/users/apoorvaeternity/repos","events_url":"https://api.github.com/users/apoorvaeternity/events{/privacy}","received_events_url":"https://api.github.com/users/apoorvaeternity/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/apoorvaeternity/retriever","description":"Quickly download, clean up, and install ecological datasets into a database management system","fork":true,"url":"https://api.github.com/repos/apoorvaeternity/retriever","forks_url":"https://api.github.com/repos/apoorvaeternity/retriever/forks","keys_url":"https://api.github.com/repos/apoorvaeternity/retriever/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apoorvaeternity/retriever/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apoorvaeternity/retriever/teams","hooks_url":"https://api.github.com/repos/apoorvaeternity/retriever/hooks","issue_events_url":"https://api.github.com/repos/apoorvaeternity/retriever/issues/events{/number}","events_url":"https://api.github.com/repos/apoorvaeternity/retriever/events","assignees_url":"https://api.github.com/repos/apoorvaeternity/retriever/assignees{/user}","branches_url":"https://api.github.com/repos/apoorvaeternity/retriever/branches{/branch}","tags_url":"https://api.github.com/repos/apoorvaeternity/retriever/tags","blobs_url":"https://api.github.com/repos/apoorvaeternity/retriever/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apoorvaeternity/retriever/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apoorvaeternity/retriever/git/refs{/sha}","trees_url":"https://api.github.com/repos/apoorvaeternity/retriever/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apoorvaeternity/retriever/statuses/{sha}","languages_url":"https://api.github.com/repos/apoorvaeternity/retriever/languages","stargazers_url":"https://api.github.com/repos/apoorvaeternity/retriever/stargazers","contributors_url":"https://api.github.com/repos/apoorvaeternity/retriever/contributors","subscribers_url":"https://api.github.com/repos/apoorvaeternity/retriever/subscribers","subscription_url":"https://api.github.com/repos/apoorvaeternity/retriever/subscription","commits_url":"https://api.github.com/repos/apoorvaeternity/retriever/commits{/sha}","git_commits_url":"https://api.github.com/repos/apoorvaeternity/retriever/git/commits{/sha}","comments_url":"https://api.github.com/repos/apoorvaeternity/retriever/comments{/number}","issue_comment_url":"https://api.github.com/repos/apoorvaeternity/retriever/issues/comments{/number}","contents_url":"https://api.github.com/repos/apoorvaeternity/retriever/contents/{+path}","compare_url":"https://api.github.com/repos/apoorvaeternity/retriever/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apoorvaeternity/retriever/merges","archive_url":"https://api.github.com/repos/apoorvaeternity/retriever/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apoorvaeternity/retriever/downloads","issues_url":"https://api.github.com/repos/apoorvaeternity/retriever/issues{/number}","pulls_url":"https://api.github.com/repos/apoorvaeternity/retriever/pulls{/number}","milestones_url":"https://api.github.com/repos/apoorvaeternity/retriever/milestones{/number}","notifications_url":"https://api.github.com/repos/apoorvaeternity/retriever/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apoorvaeternity/retriever/labels{/name}","releases_url":"https://api.github.com/repos/apoorvaeternity/retriever/releases{/id}","deployments_url":"https://api.github.com/repos/apoorvaeternity/retriever/deployments","created_at":"2017-11-19T16:41:47Z","updated_at":"2017-11-19T16:41:49Z","pushed_at":"2018-02-18T14:19:07Z","git_url":"git://github.com/apoorvaeternity/retriever.git","ssh_url":"git@github.com:apoorvaeternity/retriever.git","clone_url":"https://github.com/apoorvaeternity/retriever.git","svn_url":"https://github.com/apoorvaeternity/retriever","homepage":"http://data-retriever.org","size":3287,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"weecology:develop","ref":"develop","sha":"280052a7b4df490194aebe98256e49c4cd5e39f7","user":{"login":"weecology","id":1156696,"avatar_url":"https://avatars3.githubusercontent.com/u/1156696?v=4","gravatar_id":"","url":"https://api.github.com/users/weecology","html_url":"https://github.com/weecology","followers_url":"https://api.github.com/users/weecology/followers","following_url":"https://api.github.com/users/weecology/following{/other_user}","gists_url":"https://api.github.com/users/weecology/gists{/gist_id}","starred_url":"https://api.github.com/users/weecology/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/weecology/subscriptions","organizations_url":"https://api.github.com/users/weecology/orgs","repos_url":"https://api.github.com/users/weecology/repos","events_url":"https://api.github.com/users/weecology/events{/privacy}","received_events_url":"https://api.github.com/users/weecology/received_events","type":"Organization","site_admin":false},"repo":{"id":1968190,"name":"retriever","full_name":"weecology/retriever","owner":{"login":"weecology","id":1156696,"avatar_url":"https://avatars3.githubusercontent.com/u/1156696?v=4","gravatar_id":"","url":"https://api.github.com/users/weecology","html_url":"https://github.com/weecology","followers_url":"https://api.github.com/users/weecology/followers","following_url":"https://api.github.com/users/weecology/following{/other_user}","gists_url":"https://api.github.com/users/weecology/gists{/gist_id}","starred_url":"https://api.github.com/users/weecology/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/weecology/subscriptions","organizations_url":"https://api.github.com/users/weecology/orgs","repos_url":"https://api.github.com/users/weecology/repos","events_url":"https://api.github.com/users/weecology/events{/privacy}","received_events_url":"https://api.github.com/users/weecology/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/weecology/retriever","description":"Quickly download, clean up, and install ecological datasets into a database management system","fork":false,"url":"https://api.github.com/repos/weecology/retriever","forks_url":"https://api.github.com/repos/weecology/retriever/forks","keys_url":"https://api.github.com/repos/weecology/retriever/keys{/key_id}","collaborators_url":"https://api.github.com/repos/weecology/retriever/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/weecology/retriever/teams","hooks_url":"https://api.github.com/repos/weecology/retriever/hooks","issue_events_url":"https://api.github.com/repos/weecology/retriever/issues/events{/number}","events_url":"https://api.github.com/repos/weecology/retriever/events","assignees_url":"https://api.github.com/repos/weecology/retriever/assignees{/user}","branches_url":"https://api.github.com/repos/weecology/retriever/branches{/branch}","tags_url":"https://api.github.com/repos/weecology/retriever/tags","blobs_url":"https://api.github.com/repos/weecology/retriever/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/weecology/retriever/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/weecology/retriever/git/refs{/sha}","trees_url":"https://api.github.com/repos/weecology/retriever/git/trees{/sha}","statuses_url":"https://api.github.com/repos/weecology/retriever/statuses/{sha}","languages_url":"https://api.github.com/repos/weecology/retriever/languages","stargazers_url":"https://api.github.com/repos/weecology/retriever/stargazers","contributors_url":"https://api.github.com/repos/weecology/retriever/contributors","subscribers_url":"https://api.github.com/repos/weecology/retriever/subscribers","subscription_url":"https://api.github.com/repos/weecology/retriever/subscription","commits_url":"https://api.github.com/repos/weecology/retriever/commits{/sha}","git_commits_url":"https://api.github.com/repos/weecology/retriever/git/commits{/sha}","comments_url":"https://api.github.com/repos/weecology/retriever/comments{/number}","issue_comment_url":"https://api.github.com/repos/weecology/retriever/issues/comments{/number}","contents_url":"https://api.github.com/repos/weecology/retriever/contents/{+path}","compare_url":"https://api.github.com/repos/weecology/retriever/compare/{base}...{head}","merges_url":"https://api.github.com/repos/weecology/retriever/merges","archive_url":"https://api.github.com/repos/weecology/retriever/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/weecology/retriever/downloads","issues_url":"https://api.github.com/repos/weecology/retriever/issues{/number}","pulls_url":"https://api.github.com/repos/weecology/retriever/pulls{/number}","milestones_url":"https://api.github.com/repos/weecology/retriever/milestones{/number}","notifications_url":"https://api.github.com/repos/weecology/retriever/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/weecology/retriever/labels{/name}","releases_url":"https://api.github.com/repos/weecology/retriever/releases{/id}","deployments_url":"https://api.github.com/repos/weecology/retriever/deployments","created_at":"2011-06-28T19:01:15Z","updated_at":"2018-02-16T17:37:57Z","pushed_at":"2018-02-20T14:43:45Z","git_url":"git://github.com/weecology/retriever.git","ssh_url":"git@github.com:weecology/retriever.git","clone_url":"https://github.com/weecology/retriever.git","svn_url":"https://github.com/weecology/retriever","homepage":"http://data-retriever.org","size":46915,"stargazers_count":147,"watchers_count":147,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":62,"mirror_url":null,"archived":false,"open_issues_count":54,"license":{"key":"other","name":"Other","spdx_id":null,"url":null},"forks":62,"open_issues":54,"watchers":147,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/weecology/retriever/pulls/1068"},"html":{"href":"https://github.com/weecology/retriever/pull/1068"},"issue":{"href":"https://api.github.com/repos/weecology/retriever/issues/1068"},"comments":{"href":"https://api.github.com/repos/weecology/retriever/issues/1068/comments"},"review_comments":{"href":"https://api.github.com/repos/weecology/retriever/pulls/1068/comments"},"review_comment":{"href":"https://api.github.com/repos/weecology/retriever/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/weecology/retriever/pulls/1068/commits"},"statuses":{"href":"https://api.github.com/repos/weecology/retriever/statuses/340ad4d63acc30119fcc4184dea16f6dd11dc319"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"apoorvaeternity"}}