{"action":"created","comment":{"url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/comments/67609499","id":67609499,"diff_hunk":"@@ -3,6 +3,29 @@\n # Usage: pgtsv_to_json COLUMN_NAME1:TYPE1 COLUMN_NAME2:TYPE2 ...\n \n import json, sys, csv, re\n+from datetime import datetime\n+\n+def timestamp(timestamp_str):\n+    \"\"\"Given a timestamp string, return a timestamp string in ISO 8601 format to emulate\n+    Postgres 9.5's to_json timestamp formatting. The given timestamp is assumed UTC.\n+\n+    Time zones are not supported. http://bugs.python.org/issue6641\n+\n+    Examples:\n+\n+        >>> timestamp('2016-06-17 20:10:38')\n+        '2016-06-17T20:10:38+00:00'\n+\n+        >>> timestamp('2016-06-17 20:10:37.9293')\n+        '2016-06-17T20:10:37.929300+00:00'\n+\n+    \"\"\"\n+\n+    try:\n+        parsed = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f')\n+    except ValueError:\n+        parsed = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')\n+    return parsed.isoformat() + '+00:00'","path":"database/pgtsv_to_json","position":26,"original_position":26,"commit_id":"a3c6404e0044dd2c0e72b9c1c0c1f298218fc042","original_commit_id":"a3c6404e0044dd2c0e72b9c1c0c1f298218fc042","user":{"login":"alldefector","id":5607717,"avatar_url":"https://avatars.githubusercontent.com/u/5607717?v=3","gravatar_id":"","url":"https://api.github.com/users/alldefector","html_url":"https://github.com/alldefector","followers_url":"https://api.github.com/users/alldefector/followers","following_url":"https://api.github.com/users/alldefector/following{/other_user}","gists_url":"https://api.github.com/users/alldefector/gists{/gist_id}","starred_url":"https://api.github.com/users/alldefector/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alldefector/subscriptions","organizations_url":"https://api.github.com/users/alldefector/orgs","repos_url":"https://api.github.com/users/alldefector/repos","events_url":"https://api.github.com/users/alldefector/events{/privacy}","received_events_url":"https://api.github.com/users/alldefector/received_events","type":"User","site_admin":false},"body":"1 sounds better. But we are probably over-engineering here. I'd simply output verbatim whatever psql feeds us. Chances are the downstream app would actually prefer verbatim. For the current main consumer of ES, it supports a variety of formats already: https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html","created_at":"2016-06-19T02:55:22Z","updated_at":"2016-06-19T02:55:22Z","html_url":"https://github.com/HazyResearch/deepdive/pull/542#discussion_r67609499","pull_request_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/542","_links":{"self":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/comments/67609499"},"html":{"href":"https://github.com/HazyResearch/deepdive/pull/542#discussion_r67609499"},"pull_request":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/542"}}},"pull_request":{"url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/542","id":74323813,"html_url":"https://github.com/HazyResearch/deepdive/pull/542","diff_url":"https://github.com/HazyResearch/deepdive/pull/542.diff","patch_url":"https://github.com/HazyResearch/deepdive/pull/542.patch","issue_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/542","number":542,"state":"open","locked":false,"title":"Add timestamp conversion from Postgres dump to JSON","user":{"login":"shahin","id":20877,"avatar_url":"https://avatars.githubusercontent.com/u/20877?v=3","gravatar_id":"","url":"https://api.github.com/users/shahin","html_url":"https://github.com/shahin","followers_url":"https://api.github.com/users/shahin/followers","following_url":"https://api.github.com/users/shahin/following{/other_user}","gists_url":"https://api.github.com/users/shahin/gists{/gist_id}","starred_url":"https://api.github.com/users/shahin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shahin/subscriptions","organizations_url":"https://api.github.com/users/shahin/orgs","repos_url":"https://api.github.com/users/shahin/repos","events_url":"https://api.github.com/users/shahin/events{/privacy}","received_events_url":"https://api.github.com/users/shahin/received_events","type":"User","site_admin":false},"body":"The `pgtsv_to_json` module is the fallback for database systems that don't support `to_json()`. It's called when I run on Greenplum, but I have timestamps in my table. Without this commit, the module terminates on a ValueError. With this commit, timestamp input columns are included in the JSON output.","created_at":"2016-06-18T01:30:23Z","updated_at":"2016-06-19T02:55:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"76fccc3f3a05006297122f5267f415ccc491f15b","assignee":{"login":"zifeishan","id":1811701,"avatar_url":"https://avatars.githubusercontent.com/u/1811701?v=3","gravatar_id":"","url":"https://api.github.com/users/zifeishan","html_url":"https://github.com/zifeishan","followers_url":"https://api.github.com/users/zifeishan/followers","following_url":"https://api.github.com/users/zifeishan/following{/other_user}","gists_url":"https://api.github.com/users/zifeishan/gists{/gist_id}","starred_url":"https://api.github.com/users/zifeishan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zifeishan/subscriptions","organizations_url":"https://api.github.com/users/zifeishan/orgs","repos_url":"https://api.github.com/users/zifeishan/repos","events_url":"https://api.github.com/users/zifeishan/events{/privacy}","received_events_url":"https://api.github.com/users/zifeishan/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/542/commits","review_comments_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/542/comments","review_comment_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/comments{/number}","comments_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/542/comments","statuses_url":"https://api.github.com/repos/HazyResearch/deepdive/statuses/a3c6404e0044dd2c0e72b9c1c0c1f298218fc042","head":{"label":"shahin:master","ref":"master","sha":"a3c6404e0044dd2c0e72b9c1c0c1f298218fc042","user":{"login":"shahin","id":20877,"avatar_url":"https://avatars.githubusercontent.com/u/20877?v=3","gravatar_id":"","url":"https://api.github.com/users/shahin","html_url":"https://github.com/shahin","followers_url":"https://api.github.com/users/shahin/followers","following_url":"https://api.github.com/users/shahin/following{/other_user}","gists_url":"https://api.github.com/users/shahin/gists{/gist_id}","starred_url":"https://api.github.com/users/shahin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shahin/subscriptions","organizations_url":"https://api.github.com/users/shahin/orgs","repos_url":"https://api.github.com/users/shahin/repos","events_url":"https://api.github.com/users/shahin/events{/privacy}","received_events_url":"https://api.github.com/users/shahin/received_events","type":"User","site_admin":false},"repo":{"id":61400216,"name":"deepdive","full_name":"shahin/deepdive","owner":{"login":"shahin","id":20877,"avatar_url":"https://avatars.githubusercontent.com/u/20877?v=3","gravatar_id":"","url":"https://api.github.com/users/shahin","html_url":"https://github.com/shahin","followers_url":"https://api.github.com/users/shahin/followers","following_url":"https://api.github.com/users/shahin/following{/other_user}","gists_url":"https://api.github.com/users/shahin/gists{/gist_id}","starred_url":"https://api.github.com/users/shahin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shahin/subscriptions","organizations_url":"https://api.github.com/users/shahin/orgs","repos_url":"https://api.github.com/users/shahin/repos","events_url":"https://api.github.com/users/shahin/events{/privacy}","received_events_url":"https://api.github.com/users/shahin/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/shahin/deepdive","description":"DeepDive","fork":true,"url":"https://api.github.com/repos/shahin/deepdive","forks_url":"https://api.github.com/repos/shahin/deepdive/forks","keys_url":"https://api.github.com/repos/shahin/deepdive/keys{/key_id}","collaborators_url":"https://api.github.com/repos/shahin/deepdive/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/shahin/deepdive/teams","hooks_url":"https://api.github.com/repos/shahin/deepdive/hooks","issue_events_url":"https://api.github.com/repos/shahin/deepdive/issues/events{/number}","events_url":"https://api.github.com/repos/shahin/deepdive/events","assignees_url":"https://api.github.com/repos/shahin/deepdive/assignees{/user}","branches_url":"https://api.github.com/repos/shahin/deepdive/branches{/branch}","tags_url":"https://api.github.com/repos/shahin/deepdive/tags","blobs_url":"https://api.github.com/repos/shahin/deepdive/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/shahin/deepdive/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/shahin/deepdive/git/refs{/sha}","trees_url":"https://api.github.com/repos/shahin/deepdive/git/trees{/sha}","statuses_url":"https://api.github.com/repos/shahin/deepdive/statuses/{sha}","languages_url":"https://api.github.com/repos/shahin/deepdive/languages","stargazers_url":"https://api.github.com/repos/shahin/deepdive/stargazers","contributors_url":"https://api.github.com/repos/shahin/deepdive/contributors","subscribers_url":"https://api.github.com/repos/shahin/deepdive/subscribers","subscription_url":"https://api.github.com/repos/shahin/deepdive/subscription","commits_url":"https://api.github.com/repos/shahin/deepdive/commits{/sha}","git_commits_url":"https://api.github.com/repos/shahin/deepdive/git/commits{/sha}","comments_url":"https://api.github.com/repos/shahin/deepdive/comments{/number}","issue_comment_url":"https://api.github.com/repos/shahin/deepdive/issues/comments{/number}","contents_url":"https://api.github.com/repos/shahin/deepdive/contents/{+path}","compare_url":"https://api.github.com/repos/shahin/deepdive/compare/{base}...{head}","merges_url":"https://api.github.com/repos/shahin/deepdive/merges","archive_url":"https://api.github.com/repos/shahin/deepdive/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/shahin/deepdive/downloads","issues_url":"https://api.github.com/repos/shahin/deepdive/issues{/number}","pulls_url":"https://api.github.com/repos/shahin/deepdive/pulls{/number}","milestones_url":"https://api.github.com/repos/shahin/deepdive/milestones{/number}","notifications_url":"https://api.github.com/repos/shahin/deepdive/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/shahin/deepdive/labels{/name}","releases_url":"https://api.github.com/repos/shahin/deepdive/releases{/id}","deployments_url":"https://api.github.com/repos/shahin/deepdive/deployments","created_at":"2016-06-17T20:23:51Z","updated_at":"2016-06-17T20:23:53Z","pushed_at":"2016-06-18T00:57:02Z","git_url":"git://github.com/shahin/deepdive.git","ssh_url":"git@github.com:shahin/deepdive.git","clone_url":"https://github.com/shahin/deepdive.git","svn_url":"https://github.com/shahin/deepdive","homepage":"deepdive.stanford.edu","size":469328,"stargazers_count":0,"watchers_count":0,"language":"Shell","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"HazyResearch:master","ref":"master","sha":"c303c9c7ac9d130ba60dcaab52e74c0d1937ebe4","user":{"login":"HazyResearch","id":2165246,"avatar_url":"https://avatars.githubusercontent.com/u/2165246?v=3","gravatar_id":"","url":"https://api.github.com/users/HazyResearch","html_url":"https://github.com/HazyResearch","followers_url":"https://api.github.com/users/HazyResearch/followers","following_url":"https://api.github.com/users/HazyResearch/following{/other_user}","gists_url":"https://api.github.com/users/HazyResearch/gists{/gist_id}","starred_url":"https://api.github.com/users/HazyResearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HazyResearch/subscriptions","organizations_url":"https://api.github.com/users/HazyResearch/orgs","repos_url":"https://api.github.com/users/HazyResearch/repos","events_url":"https://api.github.com/users/HazyResearch/events{/privacy}","received_events_url":"https://api.github.com/users/HazyResearch/received_events","type":"Organization","site_admin":false},"repo":{"id":17907002,"name":"deepdive","full_name":"HazyResearch/deepdive","owner":{"login":"HazyResearch","id":2165246,"avatar_url":"https://avatars.githubusercontent.com/u/2165246?v=3","gravatar_id":"","url":"https://api.github.com/users/HazyResearch","html_url":"https://github.com/HazyResearch","followers_url":"https://api.github.com/users/HazyResearch/followers","following_url":"https://api.github.com/users/HazyResearch/following{/other_user}","gists_url":"https://api.github.com/users/HazyResearch/gists{/gist_id}","starred_url":"https://api.github.com/users/HazyResearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HazyResearch/subscriptions","organizations_url":"https://api.github.com/users/HazyResearch/orgs","repos_url":"https://api.github.com/users/HazyResearch/repos","events_url":"https://api.github.com/users/HazyResearch/events{/privacy}","received_events_url":"https://api.github.com/users/HazyResearch/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/HazyResearch/deepdive","description":"DeepDive","fork":false,"url":"https://api.github.com/repos/HazyResearch/deepdive","forks_url":"https://api.github.com/repos/HazyResearch/deepdive/forks","keys_url":"https://api.github.com/repos/HazyResearch/deepdive/keys{/key_id}","collaborators_url":"https://api.github.com/repos/HazyResearch/deepdive/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/HazyResearch/deepdive/teams","hooks_url":"https://api.github.com/repos/HazyResearch/deepdive/hooks","issue_events_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/events{/number}","events_url":"https://api.github.com/repos/HazyResearch/deepdive/events","assignees_url":"https://api.github.com/repos/HazyResearch/deepdive/assignees{/user}","branches_url":"https://api.github.com/repos/HazyResearch/deepdive/branches{/branch}","tags_url":"https://api.github.com/repos/HazyResearch/deepdive/tags","blobs_url":"https://api.github.com/repos/HazyResearch/deepdive/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/HazyResearch/deepdive/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/HazyResearch/deepdive/git/refs{/sha}","trees_url":"https://api.github.com/repos/HazyResearch/deepdive/git/trees{/sha}","statuses_url":"https://api.github.com/repos/HazyResearch/deepdive/statuses/{sha}","languages_url":"https://api.github.com/repos/HazyResearch/deepdive/languages","stargazers_url":"https://api.github.com/repos/HazyResearch/deepdive/stargazers","contributors_url":"https://api.github.com/repos/HazyResearch/deepdive/contributors","subscribers_url":"https://api.github.com/repos/HazyResearch/deepdive/subscribers","subscription_url":"https://api.github.com/repos/HazyResearch/deepdive/subscription","commits_url":"https://api.github.com/repos/HazyResearch/deepdive/commits{/sha}","git_commits_url":"https://api.github.com/repos/HazyResearch/deepdive/git/commits{/sha}","comments_url":"https://api.github.com/repos/HazyResearch/deepdive/comments{/number}","issue_comment_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/comments{/number}","contents_url":"https://api.github.com/repos/HazyResearch/deepdive/contents/{+path}","compare_url":"https://api.github.com/repos/HazyResearch/deepdive/compare/{base}...{head}","merges_url":"https://api.github.com/repos/HazyResearch/deepdive/merges","archive_url":"https://api.github.com/repos/HazyResearch/deepdive/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/HazyResearch/deepdive/downloads","issues_url":"https://api.github.com/repos/HazyResearch/deepdive/issues{/number}","pulls_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls{/number}","milestones_url":"https://api.github.com/repos/HazyResearch/deepdive/milestones{/number}","notifications_url":"https://api.github.com/repos/HazyResearch/deepdive/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/HazyResearch/deepdive/labels{/name}","releases_url":"https://api.github.com/repos/HazyResearch/deepdive/releases{/id}","deployments_url":"https://api.github.com/repos/HazyResearch/deepdive/deployments","created_at":"2014-03-19T14:23:50Z","updated_at":"2016-06-18T18:15:17Z","pushed_at":"2016-06-18T03:41:31Z","git_url":"git://github.com/HazyResearch/deepdive.git","ssh_url":"git@github.com:HazyResearch/deepdive.git","clone_url":"https://github.com/HazyResearch/deepdive.git","svn_url":"https://github.com/HazyResearch/deepdive","homepage":"deepdive.stanford.edu","size":488705,"stargazers_count":896,"watchers_count":896,"language":"Shell","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":244,"mirror_url":null,"open_issues_count":55,"forks":244,"open_issues":55,"watchers":896,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/542"},"html":{"href":"https://github.com/HazyResearch/deepdive/pull/542"},"issue":{"href":"https://api.github.com/repos/HazyResearch/deepdive/issues/542"},"comments":{"href":"https://api.github.com/repos/HazyResearch/deepdive/issues/542/comments"},"review_comments":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/542/comments"},"review_comment":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/542/commits"},"statuses":{"href":"https://api.github.com/repos/HazyResearch/deepdive/statuses/a3c6404e0044dd2c0e72b9c1c0c1f298218fc042"}}}}
{"actor":{"display_login":"alldefector"}}