{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/63059450","id":63059450,"diff_hunk":"@@ -0,0 +1,137 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.hive.execution\n+\n+import java.io.File\n+import java.util.Properties\n+\n+import scala.language.existentials\n+\n+import antlr.SemanticException\n+import org.apache.hadoop.fs.{FileSystem, Path}\n+import org.apache.hadoop.hive.ql.io.{HiveIgnoreKeyTextOutputFormat, RCFileOutputFormat}\n+import org.apache.hadoop.hive.ql.io.orc.{OrcOutputFormat, OrcSerde}\n+import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat\n+import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe\n+import org.apache.hadoop.hive.ql.plan.TableDesc\n+import org.apache.hadoop.hive.serde.serdeConstants\n+import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe\n+import org.apache.hadoop.io.Text\n+import org.apache.hadoop.mapred.{FileOutputFormat, JobConf, SequenceFileOutputFormat, TextInputFormat}\n+\n+import org.apache.spark.rdd.RDD\n+import org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat\n+import org.apache.spark.sql.catalyst.InternalRow\n+import org.apache.spark.sql.catalyst.expressions.Attribute\n+import org.apache.spark.sql.execution.SparkPlan\n+import org.apache.spark.sql.hive._\n+import org.apache.spark.sql.hive.HiveShim.{ShimFileSinkDesc => FileSinkDesc}\n+import org.apache.spark.util.{SerializableJobConf, Utils}\n+\n+case class InsertIntoDir(\n+    path: String,\n+    isLocal: Boolean,\n+    fileFormat: String,\n+    rowFormat: CatalogStorageFormat,\n+    child: SparkPlan) extends SaveAsHiveFile {\n+\n+  @transient private val sessionState = sqlContext.sessionState.asInstanceOf[HiveSessionState]\n+  def output: Seq[Attribute] = Seq.empty\n+\n+  protected[sql] lazy val sideEffectResult: Seq[InternalRow] = {\n+    val hadoopConf = sessionState.newHadoopConf()\n+\n+    val properties = new Properties()\n+\n+    val Array(cols, types) = child.output.foldLeft(Array(\"\", \"\"))((r, a) => {\n+      r(0) = r(0) + a.name + \",\"\n+      r(1) = r(1) + a.dataType.typeName + \":\"\n+      r\n+    })\n+\n+    properties.put(\"columns\", cols.dropRight(1))\n+    properties.put(\"columns.types\", types.dropRight(1))\n+\n+    val fileFormatMap = Map(\n+      \"orc\" -> (classOf[OrcOutputFormat], classOf[OrcSerde]),\n+      \"parquet\" -> (classOf[MapredParquetOutputFormat], classOf[ParquetHiveSerDe]),\n+//      \"rcfile\" -> (classOf[RCFileOutputFormat], classOf[LazySimpleSerDe]),\n+      \"textfile\" -> (classOf[HiveIgnoreKeyTextOutputFormat[Text, Text]], classOf[LazySimpleSerDe]),\n+      \"sequencefile\" -> (classOf[SequenceFileOutputFormat[Any, Any]], classOf[LazySimpleSerDe])\n+    )\n+\n+    val (ouputFormatClass, serdeClass) = fileFormatMap.getOrElse(fileFormat.toLowerCase,\n+      throw new SemanticException(s\"Unrecognized file format in STORED AS clause: $fileFormat,\" +\n+        s\" expected one of ${fileFormatMap.keys.mkString(\",\")}\"))\n+\n+    properties.put(serdeConstants.SERIALIZATION_LIB, serdeClass.getName)\n+    import scala.collection.JavaConverters._\n+    properties.putAll(rowFormat.serdeProperties.asJava)\n+\n+    // if user specified a serde in the ROW FORMAT, use that.\n+    rowFormat.serde.map(properties.put(serdeConstants.SERIALIZATION_LIB, _))\n+\n+    val tableDesc = new TableDesc(\n+      classOf[TextInputFormat],\n+      ouputFormatClass,\n+      properties\n+    )\n+\n+    val isCompressed =\n+      sessionState.conf.getConfString(\"hive.exec.compress.output\", \"false\").toBoolean\n+\n+    val targetPath = new Path(path)\n+\n+    val fileSinkConf = new FileSinkDesc(targetPath.toString, tableDesc, isCompressed)\n+\n+    val jobConf = new JobConf(hadoopConf)\n+    jobConf.set(\"mapreduce.fileoutputcommitter.marksuccessfuljobs\", \"false\")\n+\n+    val jobConfSer = new SerializableJobConf(jobConf)\n+\n+    val writerContainer = new SparkHiveWriterContainer(\n+        jobConf,\n+        fileSinkConf,\n+        child.output)\n+\n+    if( !isLocal ) {\n+        FileSystem.get(jobConf).delete(targetPath, true)","path":"sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoDir.scala","position":null,"original_position":113,"commit_id":"326c27f0d28b40552c6d70f94fe130bf7e8f6923","original_commit_id":"4708c802393dbec77d2b078b109dcc7bbabb0632","user":{"login":"Parth-Brahmbhatt","id":6914358,"avatar_url":"https://avatars.githubusercontent.com/u/6914358?v=3","gravatar_id":"","url":"https://api.github.com/users/Parth-Brahmbhatt","html_url":"https://github.com/Parth-Brahmbhatt","followers_url":"https://api.github.com/users/Parth-Brahmbhatt/followers","following_url":"https://api.github.com/users/Parth-Brahmbhatt/following{/other_user}","gists_url":"https://api.github.com/users/Parth-Brahmbhatt/gists{/gist_id}","starred_url":"https://api.github.com/users/Parth-Brahmbhatt/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Parth-Brahmbhatt/subscriptions","organizations_url":"https://api.github.com/users/Parth-Brahmbhatt/orgs","repos_url":"https://api.github.com/users/Parth-Brahmbhatt/repos","events_url":"https://api.github.com/users/Parth-Brahmbhatt/events{/privacy}","received_events_url":"https://api.github.com/users/Parth-Brahmbhatt/received_events","type":"User","site_admin":false},"body":"Fixed.","created_at":"2016-05-12T17:14:07Z","updated_at":"2016-05-12T17:14:07Z","html_url":"https://github.com/apache/spark/pull/13067#discussion_r63059450","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/13067","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/63059450"},"html":{"href":"https://github.com/apache/spark/pull/13067#discussion_r63059450"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/13067"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/13067","id":69756500,"html_url":"https://github.com/apache/spark/pull/13067","diff_url":"https://github.com/apache/spark/pull/13067.diff","patch_url":"https://github.com/apache/spark/pull/13067.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/13067","number":13067,"state":"open","locked":false,"title":"[SPARK-4131] [SQL] Support INSERT OVERWRITE [LOCAL] DIRECTORY '/path/…","user":{"login":"Parth-Brahmbhatt","id":6914358,"avatar_url":"https://avatars.githubusercontent.com/u/6914358?v=3","gravatar_id":"","url":"https://api.github.com/users/Parth-Brahmbhatt","html_url":"https://github.com/Parth-Brahmbhatt","followers_url":"https://api.github.com/users/Parth-Brahmbhatt/followers","following_url":"https://api.github.com/users/Parth-Brahmbhatt/following{/other_user}","gists_url":"https://api.github.com/users/Parth-Brahmbhatt/gists{/gist_id}","starred_url":"https://api.github.com/users/Parth-Brahmbhatt/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Parth-Brahmbhatt/subscriptions","organizations_url":"https://api.github.com/users/Parth-Brahmbhatt/orgs","repos_url":"https://api.github.com/users/Parth-Brahmbhatt/repos","events_url":"https://api.github.com/users/Parth-Brahmbhatt/events{/privacy}","received_events_url":"https://api.github.com/users/Parth-Brahmbhatt/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nSupport INSERT OVERWRITE [LOCAL] DIRECTORY '/path/to/dir' [ROW FORMAT row_format] [STORED AS file_format] query\r\n\r\n## How was this patch tested?\r\nI have tested the patch locally and I have some unit tests that I am finishing up.\r\n","created_at":"2016-05-11T23:03:41Z","updated_at":"2016-05-12T17:14:07Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a4c707c0af0b01746eebc1487c98c1b8e4362fbc","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/13067/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/13067/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/13067/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/326c27f0d28b40552c6d70f94fe130bf7e8f6923","head":{"label":"Parth-Brahmbhatt:SPARK-4131","ref":"SPARK-4131","sha":"326c27f0d28b40552c6d70f94fe130bf7e8f6923","user":{"login":"Parth-Brahmbhatt","id":6914358,"avatar_url":"https://avatars.githubusercontent.com/u/6914358?v=3","gravatar_id":"","url":"https://api.github.com/users/Parth-Brahmbhatt","html_url":"https://github.com/Parth-Brahmbhatt","followers_url":"https://api.github.com/users/Parth-Brahmbhatt/followers","following_url":"https://api.github.com/users/Parth-Brahmbhatt/following{/other_user}","gists_url":"https://api.github.com/users/Parth-Brahmbhatt/gists{/gist_id}","starred_url":"https://api.github.com/users/Parth-Brahmbhatt/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Parth-Brahmbhatt/subscriptions","organizations_url":"https://api.github.com/users/Parth-Brahmbhatt/orgs","repos_url":"https://api.github.com/users/Parth-Brahmbhatt/repos","events_url":"https://api.github.com/users/Parth-Brahmbhatt/events{/privacy}","received_events_url":"https://api.github.com/users/Parth-Brahmbhatt/received_events","type":"User","site_admin":false},"repo":{"id":50132357,"name":"spark","full_name":"Parth-Brahmbhatt/spark","owner":{"login":"Parth-Brahmbhatt","id":6914358,"avatar_url":"https://avatars.githubusercontent.com/u/6914358?v=3","gravatar_id":"","url":"https://api.github.com/users/Parth-Brahmbhatt","html_url":"https://github.com/Parth-Brahmbhatt","followers_url":"https://api.github.com/users/Parth-Brahmbhatt/followers","following_url":"https://api.github.com/users/Parth-Brahmbhatt/following{/other_user}","gists_url":"https://api.github.com/users/Parth-Brahmbhatt/gists{/gist_id}","starred_url":"https://api.github.com/users/Parth-Brahmbhatt/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Parth-Brahmbhatt/subscriptions","organizations_url":"https://api.github.com/users/Parth-Brahmbhatt/orgs","repos_url":"https://api.github.com/users/Parth-Brahmbhatt/repos","events_url":"https://api.github.com/users/Parth-Brahmbhatt/events{/privacy}","received_events_url":"https://api.github.com/users/Parth-Brahmbhatt/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/Parth-Brahmbhatt/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/Parth-Brahmbhatt/spark","forks_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/forks","keys_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/teams","hooks_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/hooks","issue_events_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/issues/events{/number}","events_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/events","assignees_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/assignees{/user}","branches_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/branches{/branch}","tags_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/tags","blobs_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/languages","stargazers_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/stargazers","contributors_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/contributors","subscribers_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/subscribers","subscription_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/subscription","commits_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/contents/{+path}","compare_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/merges","archive_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/downloads","issues_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/issues{/number}","pulls_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/labels{/name}","releases_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/releases{/id}","deployments_url":"https://api.github.com/repos/Parth-Brahmbhatt/spark/deployments","created_at":"2016-01-21T19:47:30Z","updated_at":"2016-01-21T19:47:54Z","pushed_at":"2016-05-12T17:13:24Z","git_url":"git://github.com/Parth-Brahmbhatt/spark.git","ssh_url":"git@github.com:Parth-Brahmbhatt/spark.git","clone_url":"https://github.com/Parth-Brahmbhatt/spark.git","svn_url":"https://github.com/Parth-Brahmbhatt/spark","homepage":null,"size":178638,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:branch-2.0","ref":"branch-2.0","sha":"f9ea54575659f489750a37e9a8c1105a825c94f8","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2016-05-12T15:11:52Z","pushed_at":"2016-05-12T17:13:26Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":492124,"stargazers_count":8485,"watchers_count":8485,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":7882,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":403,"forks":7882,"open_issues":403,"watchers":8485,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/13067"},"html":{"href":"https://github.com/apache/spark/pull/13067"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/13067"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/13067/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/13067/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/13067/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/326c27f0d28b40552c6d70f94fe130bf7e8f6923"}}}}