{"action":"created","comment":{"url":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/comments/228703190","pull_request_review_id":169045709,"id":228703190,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyODcwMzE5MA==","diff_hunk":"@@ -0,0 +1,260 @@\n+# *****************************************************************************\n+# *                      c s v 2 p q _ s c h e m a . p y                      *\n+# *****************************************************************************\n+#\n+# This file is part of parquet_tools.\n+#\n+# Developed for the LSST Data Management System.\n+# This product includes software developed by the LSST Project\n+# (http://www.lsst.org).\n+# See the COPYRIGHT file at the top-level directory of this distribution\n+# for details of code ownership.\n+#\n+# This program is free software: you can redistribute it and/or modify\n+# it under the terms of the GNU General Public License as published by\n+# the Free Software Foundation, either version 3 of the License, or\n+# (at your option) any later version.\n+#\n+# This program is distributed in the hope that it will be useful,\n+# but WITHOUT ANY WARRANTY; without even the implied warranty of\n+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+# GNU General Public License for more details.\n+#\n+# You should have received a copy of the GNU General Public License\n+# along with this program.  If not, see <http://www.gnu.org/licenses/>.\n+#\n+\"\"\"\n+This module encapsulates all the code to process schema files and provide\n+support for schemas that allow null values for non-floating point values.\n+\n+This module augments schemas that allow null values for data types which\n+pandas does not support to be null. Specifically, for each such column is\n+adds a  column with a name of '<cname>_ISNULL', where <cname> is the name\n+of the column that may contain an unsupported null value. The associated\n+column will contain true if the original column contains a null value or\n+false, otherwise. Such a column will contain a well defined value or a\n+value specified by the command issuer. While this module sets up the\n+mechanics, the relealiztion of this process is handled by csv2pq_handler.py.\n+\"\"\"\n+\n+__all__ = ['chkSchema', 'getSchema']\n+\n+import csv\n+import sys\n+import numpy as np\n+\n+from csv2pq_cmdinfo import CmdInfo\n+from csv2pq_utils import Fatal\n+\n+\n+# *****************************************************************************\n+# *           T y p e   C o n v e r s i o n   I n f o r m a t i o n           *\n+# *****************************************************************************\n+\n+class TypeInfo:\n+\n+    table = {'int':     np.int32,   'short':    np.int16, 'long':  np.int64,\n+             'bigint':  np.int64,   'smallint': np.int16,\n+             'int8':    np.int8,    'int32':    np.int32, 'int64': np.int64,\n+             'float':   np.float32, 'double':   np.float64,\n+             'float32': np.float32, 'float64':  np.float64,\n+             'char':    np.bytes_,  'bool':     np.bool_\n+             }\n+\n+    isInt = {'int':     True,       'short':    True,     'long':  True,\n+             'bigint':  True,       'smallint': True,\n+             'int8':    True,       'int32':    True,     'int64': True\n+             }\n+\n+\n+# *****************************************************************************\n+# *                                a d d C o l                                *\n+# *****************************************************************************\n+\n+def addCol(cName):\n+    \"Add a column to the column list and record size of longest name.\"\n+\n+    CmdInfo.colNames.append(cName)\n+    if len(cName) > CmdInfo.cNameMax: CmdInfo.cNameMax = len(cName)\n+\n+\n+# *****************************************************************************\n+# *                               g e t D e c T                               *\n+# *****************************************************************************\n+\n+def getDecT(dArgs):\n+    \"Convert decimal type to appropriate numpy type.\"\n+\n+    # Find the closing paren a strip off the remianing text\n+    #\n+    n = dArgs.find(')')\n+    if n < 1: return ''\n+    xArgs = dArgs[0:n]\n+\n+    # Split this into one or two args and convert\n+    #\n+    pSpec = xArgs.split(',')\n+    try:\n+        pVal1 = int(pSpec[0])\n+    except Exception:  # We don't care what he exception is\n+        return ''\n+    if len(pSpec) == 1:\n+        pVal2 = 0\n+    else:\n+        try:\n+            pVal2 = int(pSpec[1])\n+        except Exception:  # We don't care what gthe exception is","path":"python/parquet_tools/csv2pq_schema.py","position":106,"original_position":106,"commit_id":"4a042a56b77a7376b9d8291037ae81a1c9bf32b3","original_commit_id":"4a042a56b77a7376b9d8291037ae81a1c9bf32b3","user":{"login":"abh3","id":2657143,"node_id":"MDQ6VXNlcjI2NTcxNDM=","avatar_url":"https://avatars1.githubusercontent.com/u/2657143?v=4","gravatar_id":"","url":"https://api.github.com/users/abh3","html_url":"https://github.com/abh3","followers_url":"https://api.github.com/users/abh3/followers","following_url":"https://api.github.com/users/abh3/following{/other_user}","gists_url":"https://api.github.com/users/abh3/gists{/gist_id}","starred_url":"https://api.github.com/users/abh3/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abh3/subscriptions","organizations_url":"https://api.github.com/users/abh3/orgs","repos_url":"https://api.github.com/users/abh3/repos","events_url":"https://api.github.com/users/abh3/events{/privacy}","received_events_url":"https://api.github.com/users/abh3/received_events","type":"User","site_admin":false},"body":"Ya know, I should use a ide that checks for spelling in the comments :-)","created_at":"2018-10-27T06:31:55Z","updated_at":"2018-10-27T06:31:55Z","html_url":"https://github.com/lsst-dm/parquet_tools/pull/1#discussion_r228703190","pull_request_url":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/1","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/comments/228703190"},"html":{"href":"https://github.com/lsst-dm/parquet_tools/pull/1#discussion_r228703190"},"pull_request":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/1"}},"in_reply_to_id":227932835},"pull_request":{"url":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/1","id":224894466,"node_id":"MDExOlB1bGxSZXF1ZXN0MjI0ODk0NDY2","html_url":"https://github.com/lsst-dm/parquet_tools/pull/1","diff_url":"https://github.com/lsst-dm/parquet_tools/pull/1.diff","patch_url":"https://github.com/lsst-dm/parquet_tools/pull/1.patch","issue_url":"https://api.github.com/repos/lsst-dm/parquet_tools/issues/1","number":1,"state":"open","locked":false,"title":"Initial version of csv2pq in the parquet_tools package.","user":{"login":"abh3","id":2657143,"node_id":"MDQ6VXNlcjI2NTcxNDM=","avatar_url":"https://avatars1.githubusercontent.com/u/2657143?v=4","gravatar_id":"","url":"https://api.github.com/users/abh3","html_url":"https://github.com/abh3","followers_url":"https://api.github.com/users/abh3/followers","following_url":"https://api.github.com/users/abh3/following{/other_user}","gists_url":"https://api.github.com/users/abh3/gists{/gist_id}","starred_url":"https://api.github.com/users/abh3/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abh3/subscriptions","organizations_url":"https://api.github.com/users/abh3/orgs","repos_url":"https://api.github.com/users/abh3/repos","events_url":"https://api.github.com/users/abh3/events{/privacy}","received_events_url":"https://api.github.com/users/abh3/received_events","type":"User","site_admin":false},"body":"Ready for review. All globals have been eliminated, The setup.py and README likely need some work.","created_at":"2018-10-23T04:48:43Z","updated_at":"2018-10-27T06:31:55Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ba0bcb5911a0f4c368b7f7090bd63c85f43e93f3","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/1/commits","review_comments_url":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/1/comments","review_comment_url":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/comments{/number}","comments_url":"https://api.github.com/repos/lsst-dm/parquet_tools/issues/1/comments","statuses_url":"https://api.github.com/repos/lsst-dm/parquet_tools/statuses/4a042a56b77a7376b9d8291037ae81a1c9bf32b3","head":{"label":"lsst-dm:tickets/DM-14709","ref":"tickets/DM-14709","sha":"4a042a56b77a7376b9d8291037ae81a1c9bf32b3","user":{"login":"lsst-dm","id":10968870,"node_id":"MDEyOk9yZ2FuaXphdGlvbjEwOTY4ODcw","avatar_url":"https://avatars1.githubusercontent.com/u/10968870?v=4","gravatar_id":"","url":"https://api.github.com/users/lsst-dm","html_url":"https://github.com/lsst-dm","followers_url":"https://api.github.com/users/lsst-dm/followers","following_url":"https://api.github.com/users/lsst-dm/following{/other_user}","gists_url":"https://api.github.com/users/lsst-dm/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst-dm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst-dm/subscriptions","organizations_url":"https://api.github.com/users/lsst-dm/orgs","repos_url":"https://api.github.com/users/lsst-dm/repos","events_url":"https://api.github.com/users/lsst-dm/events{/privacy}","received_events_url":"https://api.github.com/users/lsst-dm/received_events","type":"Organization","site_admin":false},"repo":{"id":153518469,"node_id":"MDEwOlJlcG9zaXRvcnkxNTM1MTg0Njk=","name":"parquet_tools","full_name":"lsst-dm/parquet_tools","private":false,"owner":{"login":"lsst-dm","id":10968870,"node_id":"MDEyOk9yZ2FuaXphdGlvbjEwOTY4ODcw","avatar_url":"https://avatars1.githubusercontent.com/u/10968870?v=4","gravatar_id":"","url":"https://api.github.com/users/lsst-dm","html_url":"https://github.com/lsst-dm","followers_url":"https://api.github.com/users/lsst-dm/followers","following_url":"https://api.github.com/users/lsst-dm/following{/other_user}","gists_url":"https://api.github.com/users/lsst-dm/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst-dm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst-dm/subscriptions","organizations_url":"https://api.github.com/users/lsst-dm/orgs","repos_url":"https://api.github.com/users/lsst-dm/repos","events_url":"https://api.github.com/users/lsst-dm/events{/privacy}","received_events_url":"https://api.github.com/users/lsst-dm/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/lsst-dm/parquet_tools","description":null,"fork":false,"url":"https://api.github.com/repos/lsst-dm/parquet_tools","forks_url":"https://api.github.com/repos/lsst-dm/parquet_tools/forks","keys_url":"https://api.github.com/repos/lsst-dm/parquet_tools/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst-dm/parquet_tools/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst-dm/parquet_tools/teams","hooks_url":"https://api.github.com/repos/lsst-dm/parquet_tools/hooks","issue_events_url":"https://api.github.com/repos/lsst-dm/parquet_tools/issues/events{/number}","events_url":"https://api.github.com/repos/lsst-dm/parquet_tools/events","assignees_url":"https://api.github.com/repos/lsst-dm/parquet_tools/assignees{/user}","branches_url":"https://api.github.com/repos/lsst-dm/parquet_tools/branches{/branch}","tags_url":"https://api.github.com/repos/lsst-dm/parquet_tools/tags","blobs_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst-dm/parquet_tools/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst-dm/parquet_tools/languages","stargazers_url":"https://api.github.com/repos/lsst-dm/parquet_tools/stargazers","contributors_url":"https://api.github.com/repos/lsst-dm/parquet_tools/contributors","subscribers_url":"https://api.github.com/repos/lsst-dm/parquet_tools/subscribers","subscription_url":"https://api.github.com/repos/lsst-dm/parquet_tools/subscription","commits_url":"https://api.github.com/repos/lsst-dm/parquet_tools/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst-dm/parquet_tools/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst-dm/parquet_tools/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst-dm/parquet_tools/contents/{+path}","compare_url":"https://api.github.com/repos/lsst-dm/parquet_tools/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst-dm/parquet_tools/merges","archive_url":"https://api.github.com/repos/lsst-dm/parquet_tools/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst-dm/parquet_tools/downloads","issues_url":"https://api.github.com/repos/lsst-dm/parquet_tools/issues{/number}","pulls_url":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst-dm/parquet_tools/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst-dm/parquet_tools/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst-dm/parquet_tools/labels{/name}","releases_url":"https://api.github.com/repos/lsst-dm/parquet_tools/releases{/id}","deployments_url":"https://api.github.com/repos/lsst-dm/parquet_tools/deployments","created_at":"2018-10-17T20:19:48Z","updated_at":"2018-10-17T22:43:47Z","pushed_at":"2018-10-23T04:48:44Z","git_url":"git://github.com/lsst-dm/parquet_tools.git","ssh_url":"git@github.com:lsst-dm/parquet_tools.git","clone_url":"https://github.com/lsst-dm/parquet_tools.git","svn_url":"https://github.com/lsst-dm/parquet_tools","homepage":null,"size":130,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"lsst-dm:master","ref":"master","sha":"7f6d32db09c7e9f45443e12fef25f944e0c4cda1","user":{"login":"lsst-dm","id":10968870,"node_id":"MDEyOk9yZ2FuaXphdGlvbjEwOTY4ODcw","avatar_url":"https://avatars1.githubusercontent.com/u/10968870?v=4","gravatar_id":"","url":"https://api.github.com/users/lsst-dm","html_url":"https://github.com/lsst-dm","followers_url":"https://api.github.com/users/lsst-dm/followers","following_url":"https://api.github.com/users/lsst-dm/following{/other_user}","gists_url":"https://api.github.com/users/lsst-dm/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst-dm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst-dm/subscriptions","organizations_url":"https://api.github.com/users/lsst-dm/orgs","repos_url":"https://api.github.com/users/lsst-dm/repos","events_url":"https://api.github.com/users/lsst-dm/events{/privacy}","received_events_url":"https://api.github.com/users/lsst-dm/received_events","type":"Organization","site_admin":false},"repo":{"id":153518469,"node_id":"MDEwOlJlcG9zaXRvcnkxNTM1MTg0Njk=","name":"parquet_tools","full_name":"lsst-dm/parquet_tools","private":false,"owner":{"login":"lsst-dm","id":10968870,"node_id":"MDEyOk9yZ2FuaXphdGlvbjEwOTY4ODcw","avatar_url":"https://avatars1.githubusercontent.com/u/10968870?v=4","gravatar_id":"","url":"https://api.github.com/users/lsst-dm","html_url":"https://github.com/lsst-dm","followers_url":"https://api.github.com/users/lsst-dm/followers","following_url":"https://api.github.com/users/lsst-dm/following{/other_user}","gists_url":"https://api.github.com/users/lsst-dm/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst-dm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst-dm/subscriptions","organizations_url":"https://api.github.com/users/lsst-dm/orgs","repos_url":"https://api.github.com/users/lsst-dm/repos","events_url":"https://api.github.com/users/lsst-dm/events{/privacy}","received_events_url":"https://api.github.com/users/lsst-dm/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/lsst-dm/parquet_tools","description":null,"fork":false,"url":"https://api.github.com/repos/lsst-dm/parquet_tools","forks_url":"https://api.github.com/repos/lsst-dm/parquet_tools/forks","keys_url":"https://api.github.com/repos/lsst-dm/parquet_tools/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst-dm/parquet_tools/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst-dm/parquet_tools/teams","hooks_url":"https://api.github.com/repos/lsst-dm/parquet_tools/hooks","issue_events_url":"https://api.github.com/repos/lsst-dm/parquet_tools/issues/events{/number}","events_url":"https://api.github.com/repos/lsst-dm/parquet_tools/events","assignees_url":"https://api.github.com/repos/lsst-dm/parquet_tools/assignees{/user}","branches_url":"https://api.github.com/repos/lsst-dm/parquet_tools/branches{/branch}","tags_url":"https://api.github.com/repos/lsst-dm/parquet_tools/tags","blobs_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst-dm/parquet_tools/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst-dm/parquet_tools/languages","stargazers_url":"https://api.github.com/repos/lsst-dm/parquet_tools/stargazers","contributors_url":"https://api.github.com/repos/lsst-dm/parquet_tools/contributors","subscribers_url":"https://api.github.com/repos/lsst-dm/parquet_tools/subscribers","subscription_url":"https://api.github.com/repos/lsst-dm/parquet_tools/subscription","commits_url":"https://api.github.com/repos/lsst-dm/parquet_tools/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst-dm/parquet_tools/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst-dm/parquet_tools/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst-dm/parquet_tools/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst-dm/parquet_tools/contents/{+path}","compare_url":"https://api.github.com/repos/lsst-dm/parquet_tools/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst-dm/parquet_tools/merges","archive_url":"https://api.github.com/repos/lsst-dm/parquet_tools/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst-dm/parquet_tools/downloads","issues_url":"https://api.github.com/repos/lsst-dm/parquet_tools/issues{/number}","pulls_url":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst-dm/parquet_tools/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst-dm/parquet_tools/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst-dm/parquet_tools/labels{/name}","releases_url":"https://api.github.com/repos/lsst-dm/parquet_tools/releases{/id}","deployments_url":"https://api.github.com/repos/lsst-dm/parquet_tools/deployments","created_at":"2018-10-17T20:19:48Z","updated_at":"2018-10-17T22:43:47Z","pushed_at":"2018-10-23T04:48:44Z","git_url":"git://github.com/lsst-dm/parquet_tools.git","ssh_url":"git@github.com:lsst-dm/parquet_tools.git","clone_url":"https://github.com/lsst-dm/parquet_tools.git","svn_url":"https://github.com/lsst-dm/parquet_tools","homepage":null,"size":130,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/1"},"html":{"href":"https://github.com/lsst-dm/parquet_tools/pull/1"},"issue":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/issues/1"},"comments":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/lsst-dm/parquet_tools/statuses/4a042a56b77a7376b9d8291037ae81a1c9bf32b3"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"abh3"}}