{"action":"created","comment":{"url":"https://api.github.com/repos/ucbrise/opaque/pulls/comments/232747566","pull_request_review_id":174013182,"id":232747566,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIzMjc0NzU2Ng==","diff_hunk":"@@ -17,25 +17,46 @@\n \n package edu.berkeley.cs.rise.opaque\n \n+import java.io.File\n+import java.io.FileInputStream\n+import java.io.FileOutputStream\n+import java.io.ObjectInputStream\n+import java.io.ObjectOutputStream\n+\n import org.apache.spark.rdd.RDD\n+import org.apache.spark.sql.DataFrame\n import org.apache.spark.sql.SQLContext\n import org.apache.spark.sql.SaveMode\n import org.apache.spark.sql.SparkSession\n-import org.apache.spark.sql.DataFrame\n import org.apache.spark.sql.sources.BaseRelation\n import org.apache.spark.sql.sources.CreatableRelationProvider\n+import org.apache.spark.sql.sources.RelationProvider\n import org.apache.spark.sql.sources.SchemaRelationProvider\n import org.apache.spark.sql.types.StructType\n \n import edu.berkeley.cs.rise.opaque.execution.Block\n import edu.berkeley.cs.rise.opaque.execution.OpaqueOperatorExec\n \n-class EncryptedSource extends SchemaRelationProvider with CreatableRelationProvider {\n+class EncryptedSource\n+    extends RelationProvider\n+    with SchemaRelationProvider\n+    with CreatableRelationProvider {\n+\n+  override def createRelation(\n+    sqlContext: SQLContext,\n+    parameters: Map[String, String]): BaseRelation = {\n+    val schemaFile = new File(parameters(\"path\"), \"schema\")\n+    val schema = new ObjectInputStream(new FileInputStream(schemaFile))","path":"src/main/scala/edu/berkeley/cs/rise/opaque/sources.scala","position":35,"original_position":35,"commit_id":"9999b119928f31671b9844da0118ea58ce8462bd","original_commit_id":"9999b119928f31671b9844da0118ea58ce8462bd","user":{"login":"ankurdave","id":408432,"node_id":"MDQ6VXNlcjQwODQzMg==","avatar_url":"https://avatars1.githubusercontent.com/u/408432?v=4","gravatar_id":"","url":"https://api.github.com/users/ankurdave","html_url":"https://github.com/ankurdave","followers_url":"https://api.github.com/users/ankurdave/followers","following_url":"https://api.github.com/users/ankurdave/following{/other_user}","gists_url":"https://api.github.com/users/ankurdave/gists{/gist_id}","starred_url":"https://api.github.com/users/ankurdave/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ankurdave/subscriptions","organizations_url":"https://api.github.com/users/ankurdave/orgs","repos_url":"https://api.github.com/users/ankurdave/repos","events_url":"https://api.github.com/users/ankurdave/events{/privacy}","received_events_url":"https://api.github.com/users/ankurdave/received_events","type":"User","site_admin":false},"body":"We should instead use the Hadoop APIs for writing the schema, since `parameters(\"path\")` may be a Hadoop or S3 path.","created_at":"2018-11-12T17:28:48Z","updated_at":"2018-11-12T17:29:12Z","html_url":"https://github.com/ucbrise/opaque/pull/60#discussion_r232747566","pull_request_url":"https://api.github.com/repos/ucbrise/opaque/pulls/60","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/ucbrise/opaque/pulls/comments/232747566"},"html":{"href":"https://github.com/ucbrise/opaque/pull/60#discussion_r232747566"},"pull_request":{"href":"https://api.github.com/repos/ucbrise/opaque/pulls/60"}}},"pull_request":{"url":"https://api.github.com/repos/ucbrise/opaque/pulls/60","id":230003473,"node_id":"MDExOlB1bGxSZXF1ZXN0MjMwMDAzNDcz","html_url":"https://github.com/ucbrise/opaque/pull/60","diff_url":"https://github.com/ucbrise/opaque/pull/60.diff","patch_url":"https://github.com/ucbrise/opaque/pull/60.patch","issue_url":"https://api.github.com/repos/ucbrise/opaque/issues/60","number":60,"state":"open","locked":false,"title":"Support schema inference when loading encrypted data","user":{"login":"ankurdave","id":408432,"node_id":"MDQ6VXNlcjQwODQzMg==","avatar_url":"https://avatars1.githubusercontent.com/u/408432?v=4","gravatar_id":"","url":"https://api.github.com/users/ankurdave","html_url":"https://github.com/ankurdave","followers_url":"https://api.github.com/users/ankurdave/followers","following_url":"https://api.github.com/users/ankurdave/following{/other_user}","gists_url":"https://api.github.com/users/ankurdave/gists{/gist_id}","starred_url":"https://api.github.com/users/ankurdave/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ankurdave/subscriptions","organizations_url":"https://api.github.com/users/ankurdave/orgs","repos_url":"https://api.github.com/users/ankurdave/repos","events_url":"https://api.github.com/users/ankurdave/events{/privacy}","received_events_url":"https://api.github.com/users/ankurdave/received_events","type":"User","site_admin":false},"body":"Schema inference is implemented by serializing the schema alongside the encrypted data. Schemas are saved to `${path}/schema`, while data is saved to `${path}/data`.\r\n\r\nFixes #59.\r\n\r\nCo-authored-by: Chester Leung <chester.leung@berkeley.edu>","created_at":"2018-11-12T02:49:39Z","updated_at":"2018-11-12T17:29:12Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4854f0fe4ce98dc9d57d28a67110ac4c160f4852","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/ucbrise/opaque/pulls/60/commits","review_comments_url":"https://api.github.com/repos/ucbrise/opaque/pulls/60/comments","review_comment_url":"https://api.github.com/repos/ucbrise/opaque/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ucbrise/opaque/issues/60/comments","statuses_url":"https://api.github.com/repos/ucbrise/opaque/statuses/9999b119928f31671b9844da0118ea58ce8462bd","head":{"label":"ankurdave:schema-inference","ref":"schema-inference","sha":"9999b119928f31671b9844da0118ea58ce8462bd","user":{"login":"ankurdave","id":408432,"node_id":"MDQ6VXNlcjQwODQzMg==","avatar_url":"https://avatars1.githubusercontent.com/u/408432?v=4","gravatar_id":"","url":"https://api.github.com/users/ankurdave","html_url":"https://github.com/ankurdave","followers_url":"https://api.github.com/users/ankurdave/followers","following_url":"https://api.github.com/users/ankurdave/following{/other_user}","gists_url":"https://api.github.com/users/ankurdave/gists{/gist_id}","starred_url":"https://api.github.com/users/ankurdave/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ankurdave/subscriptions","organizations_url":"https://api.github.com/users/ankurdave/orgs","repos_url":"https://api.github.com/users/ankurdave/repos","events_url":"https://api.github.com/users/ankurdave/events{/privacy}","received_events_url":"https://api.github.com/users/ankurdave/received_events","type":"User","site_admin":false},"repo":{"id":78589192,"node_id":"MDEwOlJlcG9zaXRvcnk3ODU4OTE5Mg==","name":"opaque","full_name":"ankurdave/opaque","private":false,"owner":{"login":"ankurdave","id":408432,"node_id":"MDQ6VXNlcjQwODQzMg==","avatar_url":"https://avatars1.githubusercontent.com/u/408432?v=4","gravatar_id":"","url":"https://api.github.com/users/ankurdave","html_url":"https://github.com/ankurdave","followers_url":"https://api.github.com/users/ankurdave/followers","following_url":"https://api.github.com/users/ankurdave/following{/other_user}","gists_url":"https://api.github.com/users/ankurdave/gists{/gist_id}","starred_url":"https://api.github.com/users/ankurdave/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ankurdave/subscriptions","organizations_url":"https://api.github.com/users/ankurdave/orgs","repos_url":"https://api.github.com/users/ankurdave/repos","events_url":"https://api.github.com/users/ankurdave/events{/privacy}","received_events_url":"https://api.github.com/users/ankurdave/received_events","type":"User","site_admin":false},"html_url":"https://github.com/ankurdave/opaque","description":"A data analytics platform with strong security","fork":true,"url":"https://api.github.com/repos/ankurdave/opaque","forks_url":"https://api.github.com/repos/ankurdave/opaque/forks","keys_url":"https://api.github.com/repos/ankurdave/opaque/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ankurdave/opaque/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ankurdave/opaque/teams","hooks_url":"https://api.github.com/repos/ankurdave/opaque/hooks","issue_events_url":"https://api.github.com/repos/ankurdave/opaque/issues/events{/number}","events_url":"https://api.github.com/repos/ankurdave/opaque/events","assignees_url":"https://api.github.com/repos/ankurdave/opaque/assignees{/user}","branches_url":"https://api.github.com/repos/ankurdave/opaque/branches{/branch}","tags_url":"https://api.github.com/repos/ankurdave/opaque/tags","blobs_url":"https://api.github.com/repos/ankurdave/opaque/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ankurdave/opaque/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ankurdave/opaque/git/refs{/sha}","trees_url":"https://api.github.com/repos/ankurdave/opaque/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ankurdave/opaque/statuses/{sha}","languages_url":"https://api.github.com/repos/ankurdave/opaque/languages","stargazers_url":"https://api.github.com/repos/ankurdave/opaque/stargazers","contributors_url":"https://api.github.com/repos/ankurdave/opaque/contributors","subscribers_url":"https://api.github.com/repos/ankurdave/opaque/subscribers","subscription_url":"https://api.github.com/repos/ankurdave/opaque/subscription","commits_url":"https://api.github.com/repos/ankurdave/opaque/commits{/sha}","git_commits_url":"https://api.github.com/repos/ankurdave/opaque/git/commits{/sha}","comments_url":"https://api.github.com/repos/ankurdave/opaque/comments{/number}","issue_comment_url":"https://api.github.com/repos/ankurdave/opaque/issues/comments{/number}","contents_url":"https://api.github.com/repos/ankurdave/opaque/contents/{+path}","compare_url":"https://api.github.com/repos/ankurdave/opaque/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ankurdave/opaque/merges","archive_url":"https://api.github.com/repos/ankurdave/opaque/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ankurdave/opaque/downloads","issues_url":"https://api.github.com/repos/ankurdave/opaque/issues{/number}","pulls_url":"https://api.github.com/repos/ankurdave/opaque/pulls{/number}","milestones_url":"https://api.github.com/repos/ankurdave/opaque/milestones{/number}","notifications_url":"https://api.github.com/repos/ankurdave/opaque/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ankurdave/opaque/labels{/name}","releases_url":"https://api.github.com/repos/ankurdave/opaque/releases{/id}","deployments_url":"https://api.github.com/repos/ankurdave/opaque/deployments","created_at":"2017-01-11T01:00:33Z","updated_at":"2018-11-12T02:48:16Z","pushed_at":"2018-11-12T02:56:31Z","git_url":"git://github.com/ankurdave/opaque.git","ssh_url":"git@github.com:ankurdave/opaque.git","clone_url":"https://github.com/ankurdave/opaque.git","svn_url":"https://github.com/ankurdave/opaque","homepage":null,"size":15935,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"ucbrise:master","ref":"master","sha":"7407d5218304d714d389dd00bd1584b9621ef838","user":{"login":"ucbrise","id":20959024,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIwOTU5MDI0","avatar_url":"https://avatars2.githubusercontent.com/u/20959024?v=4","gravatar_id":"","url":"https://api.github.com/users/ucbrise","html_url":"https://github.com/ucbrise","followers_url":"https://api.github.com/users/ucbrise/followers","following_url":"https://api.github.com/users/ucbrise/following{/other_user}","gists_url":"https://api.github.com/users/ucbrise/gists{/gist_id}","starred_url":"https://api.github.com/users/ucbrise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ucbrise/subscriptions","organizations_url":"https://api.github.com/users/ucbrise/orgs","repos_url":"https://api.github.com/users/ucbrise/repos","events_url":"https://api.github.com/users/ucbrise/events{/privacy}","received_events_url":"https://api.github.com/users/ucbrise/received_events","type":"Organization","site_admin":false},"repo":{"id":72469352,"node_id":"MDEwOlJlcG9zaXRvcnk3MjQ2OTM1Mg==","name":"opaque","full_name":"ucbrise/opaque","private":false,"owner":{"login":"ucbrise","id":20959024,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIwOTU5MDI0","avatar_url":"https://avatars2.githubusercontent.com/u/20959024?v=4","gravatar_id":"","url":"https://api.github.com/users/ucbrise","html_url":"https://github.com/ucbrise","followers_url":"https://api.github.com/users/ucbrise/followers","following_url":"https://api.github.com/users/ucbrise/following{/other_user}","gists_url":"https://api.github.com/users/ucbrise/gists{/gist_id}","starred_url":"https://api.github.com/users/ucbrise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ucbrise/subscriptions","organizations_url":"https://api.github.com/users/ucbrise/orgs","repos_url":"https://api.github.com/users/ucbrise/repos","events_url":"https://api.github.com/users/ucbrise/events{/privacy}","received_events_url":"https://api.github.com/users/ucbrise/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/ucbrise/opaque","description":"A data analytics platform with strong security","fork":false,"url":"https://api.github.com/repos/ucbrise/opaque","forks_url":"https://api.github.com/repos/ucbrise/opaque/forks","keys_url":"https://api.github.com/repos/ucbrise/opaque/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ucbrise/opaque/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ucbrise/opaque/teams","hooks_url":"https://api.github.com/repos/ucbrise/opaque/hooks","issue_events_url":"https://api.github.com/repos/ucbrise/opaque/issues/events{/number}","events_url":"https://api.github.com/repos/ucbrise/opaque/events","assignees_url":"https://api.github.com/repos/ucbrise/opaque/assignees{/user}","branches_url":"https://api.github.com/repos/ucbrise/opaque/branches{/branch}","tags_url":"https://api.github.com/repos/ucbrise/opaque/tags","blobs_url":"https://api.github.com/repos/ucbrise/opaque/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ucbrise/opaque/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ucbrise/opaque/git/refs{/sha}","trees_url":"https://api.github.com/repos/ucbrise/opaque/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ucbrise/opaque/statuses/{sha}","languages_url":"https://api.github.com/repos/ucbrise/opaque/languages","stargazers_url":"https://api.github.com/repos/ucbrise/opaque/stargazers","contributors_url":"https://api.github.com/repos/ucbrise/opaque/contributors","subscribers_url":"https://api.github.com/repos/ucbrise/opaque/subscribers","subscription_url":"https://api.github.com/repos/ucbrise/opaque/subscription","commits_url":"https://api.github.com/repos/ucbrise/opaque/commits{/sha}","git_commits_url":"https://api.github.com/repos/ucbrise/opaque/git/commits{/sha}","comments_url":"https://api.github.com/repos/ucbrise/opaque/comments{/number}","issue_comment_url":"https://api.github.com/repos/ucbrise/opaque/issues/comments{/number}","contents_url":"https://api.github.com/repos/ucbrise/opaque/contents/{+path}","compare_url":"https://api.github.com/repos/ucbrise/opaque/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ucbrise/opaque/merges","archive_url":"https://api.github.com/repos/ucbrise/opaque/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ucbrise/opaque/downloads","issues_url":"https://api.github.com/repos/ucbrise/opaque/issues{/number}","pulls_url":"https://api.github.com/repos/ucbrise/opaque/pulls{/number}","milestones_url":"https://api.github.com/repos/ucbrise/opaque/milestones{/number}","notifications_url":"https://api.github.com/repos/ucbrise/opaque/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ucbrise/opaque/labels{/name}","releases_url":"https://api.github.com/repos/ucbrise/opaque/releases{/id}","deployments_url":"https://api.github.com/repos/ucbrise/opaque/deployments","created_at":"2016-10-31T19:09:35Z","updated_at":"2018-11-12T09:42:36Z","pushed_at":"2018-11-12T02:56:32Z","git_url":"git://github.com/ucbrise/opaque.git","ssh_url":"git@github.com:ucbrise/opaque.git","clone_url":"https://github.com/ucbrise/opaque.git","svn_url":"https://github.com/ucbrise/opaque","homepage":"","size":15921,"stargazers_count":70,"watchers_count":70,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":34,"mirror_url":null,"archived":false,"open_issues_count":10,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":34,"open_issues":10,"watchers":70,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ucbrise/opaque/pulls/60"},"html":{"href":"https://github.com/ucbrise/opaque/pull/60"},"issue":{"href":"https://api.github.com/repos/ucbrise/opaque/issues/60"},"comments":{"href":"https://api.github.com/repos/ucbrise/opaque/issues/60/comments"},"review_comments":{"href":"https://api.github.com/repos/ucbrise/opaque/pulls/60/comments"},"review_comment":{"href":"https://api.github.com/repos/ucbrise/opaque/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ucbrise/opaque/pulls/60/commits"},"statuses":{"href":"https://api.github.com/repos/ucbrise/opaque/statuses/9999b119928f31671b9844da0118ea58ce8462bd"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"ankurdave"}}