{"action":"created","comment":{"url":"https://api.github.com/repos/quiltdata/t4/pulls/comments/227612362","pull_request_review_id":167693867,"id":227612362,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyNzYxMjM2Mg==","diff_hunk":"@@ -131,10 +139,350 @@ def download_file_from_snapshot(src, dst, snapshothash):\n     snapshot_data = read_snapshot_by_hash(bucket, snapshothash)\n     obj_rec = snapshot_data['contents'][key]\n     return download_file(src, dst, version=obj_rec['VersionId'])\n-    \n+\n \n def download_bytes_from_snapshot(src, snapshothash):\n     bucket, key = split_path(src)\n     snapshot_data = read_snapshot_by_hash(bucket, snapshothash)\n     obj_rec = snapshot_data['contents'][key]\n     return download_bytes(src, version=obj_rec['VersionId'])\n+\n+class PhysicalKeyType(Enum):\n+    LOCAL = 1\n+    S3 = 2\n+\n+def hash_file(path):\n+    \"\"\" Returns SHA256 hash of file at path. \"\"\"\n+    with open(path, 'rb') as f:\n+        buf = f.read(4096)\n+        hasher = hashlib.sha256()\n+        while buf:\n+            hasher.update(buf)\n+            buf = f.read(4096)\n+\n+        return hasher.hexdigest()\n+\n+def dereference_physical_key(physical_key):\n+    ty = physical_key['type']\n+    if ty == PhysicalKeyType.LOCAL.name:\n+        return open(physical_key['path'])\n+\n+    raise NotImplementedError\n+\n+def get_package_registry(path):\n+    \"\"\" Returns the package registry for a given path \"\"\"\n+    if path.startswith('s3://'):\n+        bucket = path[5:].partition('/')[0]\n+        return \"s3://{}/.quilt/packages/\".format(bucket)\n+    else:\n+        # local path\n+        return \"appdirs/.quilt/packages\" # TODO: replace with actual logic\n+\n+class PackageEntry(object):\n+    \"\"\"\n+    Represents an entry at a logical key inside a package.\n+    \"\"\"\n+    __slots__ = ['physical_keys', 'size', 'hash_obj', 'meta']\n+    def __init__(self, physical_keys, size, hash_obj, meta):\n+        \"\"\"\n+        Creates an entry.\n+\n+        Args:\n+            physical_keys([physical_key]): nonempty list of physical keys\n+            size(number): size of object in bytes\n+            hash({'type': string, 'value': string}): hash object\n+            meta(dict): metadata dictionary\n+\n+        Returns:\n+            a PackageEntry\n+        \"\"\"\n+        self.physical_keys = physical_keys\n+        self.size = size\n+        self.hash = hash_obj\n+        self.meta = meta\n+\n+    def as_dict(self):\n+        \"\"\"\n+        Returns dict representation of entry.\n+        \"\"\"\n+        ret = {\n+            'physical_keys': self.physical_keys,\n+            'size': self.size,\n+            'hash': self.hash_obj,\n+            'meta': self.meta\n+        }\n+        return copy.deepcopy(ret)\n+\n+class Package(object):\n+    \"\"\" In-memory representation of a package \"\"\"\n+\n+    def __init__(self, data=None, meta=None):\n+        \"\"\"\n+        _data is of the form {logical_key: PackageEntry}\n+        physical_keys is a list of objects of the form {\n+            schema_version: string\n+            type: string\n+            uri: string\n+        }\n+        hash is of the form {\n+            type: string (e.g. \"SHA256\")\n+            value: string\n+        }\n+        size is the length of the object in bytes\n+        \"\"\"\n+        self._data = data or {}\n+        self._meta = meta or {}\n+\n+    def _clone(self):\n+        \"\"\"\n+        Returns clone of this package.\n+        \"\"\"\n+        return Package(copy.deepcopy(self._data), copy.deepcopy(self._meta))\n+\n+    def __contains__(self, logical_key):\n+        \"\"\"\n+        Checks whether the package contains a specified logical_key.\n+\n+        Returns:\n+            True or False\n+        \"\"\"\n+        return logical_key in self._data\n+\n+    @staticmethod\n+    def load(path):\n+        \"\"\"\n+        Loads a package from a path.\n+\n+        Args:\n+            path(string): the location to load the package from\n+\n+        Returns:\n+            a new package object\n+\n+        Raises:\n+            file not found\n+            json decode error\n+            invalid package exception\n+        \"\"\"\n+        data = {}\n+        with jsonlines.open(path) as reader:\n+            meta = reader.read()\n+            for obj in reader:\n+                lk = obj.pop('logical_key')\n+                if lk in data:\n+                    raise PackageException(\"Duplicate logical key while loading package\")\n+                data[lk] = PackageEntry(\n+                    obj['physical_keys'],\n+                    obj['size'],\n+                    obj['hash'],\n+                    obj['meta']\n+                )\n+\n+        return Package(data, meta)\n+\n+    @staticmethod\n+    def create_package(path):\n+        \"\"\"\n+        Takes a package of a provided path.\n+\n+        Recursively enumerates every file in path, and returns a new\n+        package that contains all those files.\n+\n+        Args:\n+            path(string): path to package\n+\n+        Returns:\n+            A new Package of that path\n+\n+        Raises:\n+            when path doesn't exist\n+        \"\"\"\n+        # TODO: anything but local paths\n+        # TODO: deserialization metadata\n+        data = {}\n+        meta = {}\n+        src_path = pathlib.Path(path)\n+        files = src_path.rglob('*')\n+        for f in files:\n+            if not f.is_file():\n+                continue\n+\n+            hash_obj = {\n+                'type': 'SHA256',\n+                'value': hash_file(f)\n+            }\n+            size = os.path.getsize(f)\n+            physical_keys = [{\n+                'type': PhysicalKeyType.LOCAL.name,\n+                'path': os.path.abspath(f)\n+            }]\n+            entry = PackageEntry(physical_keys, size, hash_obj, {})\n+            data[str(f)] = entry","path":"api/python/t4/snapshots.py","position":224,"original_position":229,"commit_id":"eac5dfdcb06b78829ede2ca77bfadfa75c3dcfb0","original_commit_id":"6dfeb935e56444f81d97a29e04e35d033bf83b6c","user":{"login":"meffij","id":8787479,"node_id":"MDQ6VXNlcjg3ODc0Nzk=","avatar_url":"https://avatars0.githubusercontent.com/u/8787479?v=4","gravatar_id":"","url":"https://api.github.com/users/meffij","html_url":"https://github.com/meffij","followers_url":"https://api.github.com/users/meffij/followers","following_url":"https://api.github.com/users/meffij/following{/other_user}","gists_url":"https://api.github.com/users/meffij/gists{/gist_id}","starred_url":"https://api.github.com/users/meffij/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/meffij/subscriptions","organizations_url":"https://api.github.com/users/meffij/orgs","repos_url":"https://api.github.com/users/meffij/repos","events_url":"https://api.github.com/users/meffij/events{/privacy}","received_events_url":"https://api.github.com/users/meffij/received_events","type":"User","site_admin":false},"body":"oh I see what you're saying","created_at":"2018-10-24T00:49:30Z","updated_at":"2018-10-24T00:49:30Z","html_url":"https://github.com/quiltdata/t4/pull/25#discussion_r227612362","pull_request_url":"https://api.github.com/repos/quiltdata/t4/pulls/25","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/quiltdata/t4/pulls/comments/227612362"},"html":{"href":"https://github.com/quiltdata/t4/pull/25#discussion_r227612362"},"pull_request":{"href":"https://api.github.com/repos/quiltdata/t4/pulls/25"}},"in_reply_to_id":227514182},"pull_request":{"url":"https://api.github.com/repos/quiltdata/t4/pulls/25","id":223808221,"node_id":"MDExOlB1bGxSZXF1ZXN0MjIzODA4MjIx","html_url":"https://github.com/quiltdata/t4/pull/25","diff_url":"https://github.com/quiltdata/t4/pull/25.diff","patch_url":"https://github.com/quiltdata/t4/pull/25.patch","issue_url":"https://api.github.com/repos/quiltdata/t4/issues/25","number":25,"state":"open","locked":false,"title":"Snapshots prototype","user":{"login":"meffij","id":8787479,"node_id":"MDQ6VXNlcjg3ODc0Nzk=","avatar_url":"https://avatars0.githubusercontent.com/u/8787479?v=4","gravatar_id":"","url":"https://api.github.com/users/meffij","html_url":"https://github.com/meffij","followers_url":"https://api.github.com/users/meffij/followers","following_url":"https://api.github.com/users/meffij/following{/other_user}","gists_url":"https://api.github.com/users/meffij/gists{/gist_id}","starred_url":"https://api.github.com/users/meffij/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/meffij/subscriptions","organizations_url":"https://api.github.com/users/meffij/orgs","repos_url":"https://api.github.com/users/meffij/repos","events_url":"https://api.github.com/users/meffij/events{/privacy}","received_events_url":"https://api.github.com/users/meffij/received_events","type":"User","site_admin":false},"body":"Strawman API to be implemented by the 26th\r\n\r\nTODO:\r\n- [x] add todos\r\n- [x] ~\"snapshot\"~ => \"package\"","created_at":"2018-10-18T02:38:21Z","updated_at":"2018-10-24T00:49:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"809ceda782144a24269d0f5b5a82ff36d8d8bd9b","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/quiltdata/t4/pulls/25/commits","review_comments_url":"https://api.github.com/repos/quiltdata/t4/pulls/25/comments","review_comment_url":"https://api.github.com/repos/quiltdata/t4/pulls/comments{/number}","comments_url":"https://api.github.com/repos/quiltdata/t4/issues/25/comments","statuses_url":"https://api.github.com/repos/quiltdata/t4/statuses/eac5dfdcb06b78829ede2ca77bfadfa75c3dcfb0","head":{"label":"quiltdata:snapshots-prototype","ref":"snapshots-prototype","sha":"eac5dfdcb06b78829ede2ca77bfadfa75c3dcfb0","user":{"login":"quiltdata","id":17112125,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE3MTEyMTI1","avatar_url":"https://avatars3.githubusercontent.com/u/17112125?v=4","gravatar_id":"","url":"https://api.github.com/users/quiltdata","html_url":"https://github.com/quiltdata","followers_url":"https://api.github.com/users/quiltdata/followers","following_url":"https://api.github.com/users/quiltdata/following{/other_user}","gists_url":"https://api.github.com/users/quiltdata/gists{/gist_id}","starred_url":"https://api.github.com/users/quiltdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quiltdata/subscriptions","organizations_url":"https://api.github.com/users/quiltdata/orgs","repos_url":"https://api.github.com/users/quiltdata/repos","events_url":"https://api.github.com/users/quiltdata/events{/privacy}","received_events_url":"https://api.github.com/users/quiltdata/received_events","type":"Organization","site_admin":false},"repo":{"id":152173979,"node_id":"MDEwOlJlcG9zaXRvcnkxNTIxNzM5Nzk=","name":"t4","full_name":"quiltdata/t4","private":false,"owner":{"login":"quiltdata","id":17112125,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE3MTEyMTI1","avatar_url":"https://avatars3.githubusercontent.com/u/17112125?v=4","gravatar_id":"","url":"https://api.github.com/users/quiltdata","html_url":"https://github.com/quiltdata","followers_url":"https://api.github.com/users/quiltdata/followers","following_url":"https://api.github.com/users/quiltdata/following{/other_user}","gists_url":"https://api.github.com/users/quiltdata/gists{/gist_id}","starred_url":"https://api.github.com/users/quiltdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quiltdata/subscriptions","organizations_url":"https://api.github.com/users/quiltdata/orgs","repos_url":"https://api.github.com/users/quiltdata/repos","events_url":"https://api.github.com/users/quiltdata/events{/privacy}","received_events_url":"https://api.github.com/users/quiltdata/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/quiltdata/t4","description":"Dropbox for data science, built on S3","fork":false,"url":"https://api.github.com/repos/quiltdata/t4","forks_url":"https://api.github.com/repos/quiltdata/t4/forks","keys_url":"https://api.github.com/repos/quiltdata/t4/keys{/key_id}","collaborators_url":"https://api.github.com/repos/quiltdata/t4/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/quiltdata/t4/teams","hooks_url":"https://api.github.com/repos/quiltdata/t4/hooks","issue_events_url":"https://api.github.com/repos/quiltdata/t4/issues/events{/number}","events_url":"https://api.github.com/repos/quiltdata/t4/events","assignees_url":"https://api.github.com/repos/quiltdata/t4/assignees{/user}","branches_url":"https://api.github.com/repos/quiltdata/t4/branches{/branch}","tags_url":"https://api.github.com/repos/quiltdata/t4/tags","blobs_url":"https://api.github.com/repos/quiltdata/t4/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/quiltdata/t4/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/quiltdata/t4/git/refs{/sha}","trees_url":"https://api.github.com/repos/quiltdata/t4/git/trees{/sha}","statuses_url":"https://api.github.com/repos/quiltdata/t4/statuses/{sha}","languages_url":"https://api.github.com/repos/quiltdata/t4/languages","stargazers_url":"https://api.github.com/repos/quiltdata/t4/stargazers","contributors_url":"https://api.github.com/repos/quiltdata/t4/contributors","subscribers_url":"https://api.github.com/repos/quiltdata/t4/subscribers","subscription_url":"https://api.github.com/repos/quiltdata/t4/subscription","commits_url":"https://api.github.com/repos/quiltdata/t4/commits{/sha}","git_commits_url":"https://api.github.com/repos/quiltdata/t4/git/commits{/sha}","comments_url":"https://api.github.com/repos/quiltdata/t4/comments{/number}","issue_comment_url":"https://api.github.com/repos/quiltdata/t4/issues/comments{/number}","contents_url":"https://api.github.com/repos/quiltdata/t4/contents/{+path}","compare_url":"https://api.github.com/repos/quiltdata/t4/compare/{base}...{head}","merges_url":"https://api.github.com/repos/quiltdata/t4/merges","archive_url":"https://api.github.com/repos/quiltdata/t4/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/quiltdata/t4/downloads","issues_url":"https://api.github.com/repos/quiltdata/t4/issues{/number}","pulls_url":"https://api.github.com/repos/quiltdata/t4/pulls{/number}","milestones_url":"https://api.github.com/repos/quiltdata/t4/milestones{/number}","notifications_url":"https://api.github.com/repos/quiltdata/t4/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/quiltdata/t4/labels{/name}","releases_url":"https://api.github.com/repos/quiltdata/t4/releases{/id}","deployments_url":"https://api.github.com/repos/quiltdata/t4/deployments","created_at":"2018-10-09T02:15:10Z","updated_at":"2018-10-23T17:16:35Z","pushed_at":"2018-10-23T22:23:45Z","git_url":"git://github.com/quiltdata/t4.git","ssh_url":"git@github.com:quiltdata/t4.git","clone_url":"https://github.com/quiltdata/t4.git","svn_url":"https://github.com/quiltdata/t4","homepage":"","size":7067,"stargazers_count":23,"watchers_count":23,"language":"JavaScript","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"archived":false,"open_issues_count":4,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":1,"open_issues":4,"watchers":23,"default_branch":"master"}},"base":{"label":"quiltdata:master","ref":"master","sha":"06d25212cfc15c2c7dbc86b23e1451bf9c1057c2","user":{"login":"quiltdata","id":17112125,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE3MTEyMTI1","avatar_url":"https://avatars3.githubusercontent.com/u/17112125?v=4","gravatar_id":"","url":"https://api.github.com/users/quiltdata","html_url":"https://github.com/quiltdata","followers_url":"https://api.github.com/users/quiltdata/followers","following_url":"https://api.github.com/users/quiltdata/following{/other_user}","gists_url":"https://api.github.com/users/quiltdata/gists{/gist_id}","starred_url":"https://api.github.com/users/quiltdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quiltdata/subscriptions","organizations_url":"https://api.github.com/users/quiltdata/orgs","repos_url":"https://api.github.com/users/quiltdata/repos","events_url":"https://api.github.com/users/quiltdata/events{/privacy}","received_events_url":"https://api.github.com/users/quiltdata/received_events","type":"Organization","site_admin":false},"repo":{"id":152173979,"node_id":"MDEwOlJlcG9zaXRvcnkxNTIxNzM5Nzk=","name":"t4","full_name":"quiltdata/t4","private":false,"owner":{"login":"quiltdata","id":17112125,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE3MTEyMTI1","avatar_url":"https://avatars3.githubusercontent.com/u/17112125?v=4","gravatar_id":"","url":"https://api.github.com/users/quiltdata","html_url":"https://github.com/quiltdata","followers_url":"https://api.github.com/users/quiltdata/followers","following_url":"https://api.github.com/users/quiltdata/following{/other_user}","gists_url":"https://api.github.com/users/quiltdata/gists{/gist_id}","starred_url":"https://api.github.com/users/quiltdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quiltdata/subscriptions","organizations_url":"https://api.github.com/users/quiltdata/orgs","repos_url":"https://api.github.com/users/quiltdata/repos","events_url":"https://api.github.com/users/quiltdata/events{/privacy}","received_events_url":"https://api.github.com/users/quiltdata/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/quiltdata/t4","description":"Dropbox for data science, built on S3","fork":false,"url":"https://api.github.com/repos/quiltdata/t4","forks_url":"https://api.github.com/repos/quiltdata/t4/forks","keys_url":"https://api.github.com/repos/quiltdata/t4/keys{/key_id}","collaborators_url":"https://api.github.com/repos/quiltdata/t4/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/quiltdata/t4/teams","hooks_url":"https://api.github.com/repos/quiltdata/t4/hooks","issue_events_url":"https://api.github.com/repos/quiltdata/t4/issues/events{/number}","events_url":"https://api.github.com/repos/quiltdata/t4/events","assignees_url":"https://api.github.com/repos/quiltdata/t4/assignees{/user}","branches_url":"https://api.github.com/repos/quiltdata/t4/branches{/branch}","tags_url":"https://api.github.com/repos/quiltdata/t4/tags","blobs_url":"https://api.github.com/repos/quiltdata/t4/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/quiltdata/t4/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/quiltdata/t4/git/refs{/sha}","trees_url":"https://api.github.com/repos/quiltdata/t4/git/trees{/sha}","statuses_url":"https://api.github.com/repos/quiltdata/t4/statuses/{sha}","languages_url":"https://api.github.com/repos/quiltdata/t4/languages","stargazers_url":"https://api.github.com/repos/quiltdata/t4/stargazers","contributors_url":"https://api.github.com/repos/quiltdata/t4/contributors","subscribers_url":"https://api.github.com/repos/quiltdata/t4/subscribers","subscription_url":"https://api.github.com/repos/quiltdata/t4/subscription","commits_url":"https://api.github.com/repos/quiltdata/t4/commits{/sha}","git_commits_url":"https://api.github.com/repos/quiltdata/t4/git/commits{/sha}","comments_url":"https://api.github.com/repos/quiltdata/t4/comments{/number}","issue_comment_url":"https://api.github.com/repos/quiltdata/t4/issues/comments{/number}","contents_url":"https://api.github.com/repos/quiltdata/t4/contents/{+path}","compare_url":"https://api.github.com/repos/quiltdata/t4/compare/{base}...{head}","merges_url":"https://api.github.com/repos/quiltdata/t4/merges","archive_url":"https://api.github.com/repos/quiltdata/t4/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/quiltdata/t4/downloads","issues_url":"https://api.github.com/repos/quiltdata/t4/issues{/number}","pulls_url":"https://api.github.com/repos/quiltdata/t4/pulls{/number}","milestones_url":"https://api.github.com/repos/quiltdata/t4/milestones{/number}","notifications_url":"https://api.github.com/repos/quiltdata/t4/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/quiltdata/t4/labels{/name}","releases_url":"https://api.github.com/repos/quiltdata/t4/releases{/id}","deployments_url":"https://api.github.com/repos/quiltdata/t4/deployments","created_at":"2018-10-09T02:15:10Z","updated_at":"2018-10-23T17:16:35Z","pushed_at":"2018-10-23T22:23:45Z","git_url":"git://github.com/quiltdata/t4.git","ssh_url":"git@github.com:quiltdata/t4.git","clone_url":"https://github.com/quiltdata/t4.git","svn_url":"https://github.com/quiltdata/t4","homepage":"","size":7067,"stargazers_count":23,"watchers_count":23,"language":"JavaScript","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"archived":false,"open_issues_count":4,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":1,"open_issues":4,"watchers":23,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/quiltdata/t4/pulls/25"},"html":{"href":"https://github.com/quiltdata/t4/pull/25"},"issue":{"href":"https://api.github.com/repos/quiltdata/t4/issues/25"},"comments":{"href":"https://api.github.com/repos/quiltdata/t4/issues/25/comments"},"review_comments":{"href":"https://api.github.com/repos/quiltdata/t4/pulls/25/comments"},"review_comment":{"href":"https://api.github.com/repos/quiltdata/t4/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/quiltdata/t4/pulls/25/commits"},"statuses":{"href":"https://api.github.com/repos/quiltdata/t4/statuses/eac5dfdcb06b78829ede2ca77bfadfa75c3dcfb0"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"meffij"}}