title: Exception when using collect on Spark Dataframe that is a union of Selects of another dataframe
body: <p>My code:</p>
  
  <pre><code>import org.apache.spark.sql._
  import org.apache.spark.sql.types._
  
  def yearFrame(x: String) : org.apache.spark.sql.DataFrame = {
      val csv0 = sc.textFile("Data/Casos_Notificados_Dengue_01_"+x+".csv")
  
      val csv = sc.textFile("Data/*"+x+".csv")
  
      val rdd = csv.mapPartitionsWithIndex(
          ((i,iterator) =&gt; if (i == 0 &amp;&amp; iterator.hasNext){
            iterator.next
            iterator.next
            iterator
          }else iterator), true)
  
      var schemaArray = csv0.collect()(1).split(",")
      schemaArray(0) = "NU_NOTIF" //Corrigindo mudança de header de 2011 para 2012
      val schema =
        StructType(
          schemaArray.map(fieldName =&gt; 
          if(fieldName == "NU_NOTIF") StructField(fieldName, StringType, false))
          else StructField(fieldName, StringType, true))      
          )
  
      val rowRDD = rdd.map(_.split(",(?=([^\"]*\"[^\"]*\")*[^\"]*$)")).map(p =&gt; Row.fromSeq(p))
  
      // Apply the schema to the RDD.
      val schemaRDD = sqlContext.applySchema(rowRDD, schema)
  
      // Register the SchemaRDD as a table.
      schemaRDD.registerTempTable("casos")
  
      // SQL statements can be run by using the sql methods provided by sqlContext.
      val r = sqlContext.sql("SELECT NU_NOTIF,NU_ANO,Long_WGS84,Lat_WGS84 FROM casos")
      return r
  }
  
  val years = List("2010","2011","2012","2013","2014")
  val allTables = years.map(x =&gt; yearFrame(x))
  val finalTables = allTables.reduce(_.unionAll(_))
  </code></pre>
  
  <p>This executes without a problem, so let's say i want now to get all rows with NU_ANO = 2014:</p>
  
  <pre><code>scala&gt; val a = finalTables.filter("NU_ANO = 2014")
  a: org.apache.spark.sql.DataFrame = [NU_NOTIF: string, NU_ANO: string, Long_WGS84: string, Lat_WGS84: string]
  
  scala&gt; a.first
  15/05/28 11:42:59 ERROR Executor: Exception in task 0.0 in stage 91.0 (TID 287)
  java.lang.ArrayIndexOutOfBoundsException
  15/05/28 11:42:59 ERROR TaskSetManager: Task 0 in stage 91.0 failed 1 times; aborting job
  15/05/28 11:42:59 ERROR Executor: Exception in task 1.0 in stage 91.0 (TID 288)
  java.lang.ArrayIndexOutOfBoundsException
  15/05/28 11:42:59 ERROR Executor: Exception in task 3.0 in stage 91.0 (TID 290)
  org.apache.spark.TaskKilledException
      at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:194)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
      at java.lang.Thread.run(Thread.java:745)
  org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 91.0 failed 1 times, most recent failure: Lost task 0.0 in stage 91.0 (TID 287, localhost): java.lang.ArrayIndexOutOfBoundsException
  
  Driver stacktrace:
    at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1204)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1193)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1192)
    at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
    at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
    at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1192)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)
    at scala.Option.foreach(Option.scala:245)
    at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:693)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1393)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1354)
    at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
  
  scala&gt; a.schema
  res116: org.apache.spark.sql.types.StructType = StructType(StructField(NU_NOTIF,StringType,true), StructField(NU_ANO,StringType,true), StructField(Long_WGS84,StringType,true), StructField(Lat_WGS84,StringType,true))
  
  scala&gt; a.count
  15/05/28 11:43:13 ERROR Executor: Exception in task 1.0 in stage 92.0 (TID 293)
  java.lang.ArrayIndexOutOfBoundsException
  15/05/28 11:43:13 ERROR TaskSetManager: Task 1 in stage 92.0 failed 1 times; aborting job
  15/05/28 11:43:13 ERROR Executor: Exception in task 2.0 in stage 92.0 (TID 294)
  java.lang.ArrayIndexOutOfBoundsException
  15/05/28 11:43:13 ERROR Executor: Exception in task 3.0 in stage 92.0 (TID 295)
  java.lang.ArrayIndexOutOfBoundsException
  15/05/28 11:43:13 ERROR Executor: Exception in task 4.0 in stage 92.0 (TID 296)
  org.apache.spark.TaskKilledException
      at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:194)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
      at java.lang.Thread.run(Thread.java:745)
  org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 92.0 failed 1 times, most recent failure: Lost task 1.0 in stage 92.0 (TID 293, localhost): java.lang.ArrayIndexOutOfBoundsException
  
  Driver stacktrace:
    at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1204)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1193)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1192)
    at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
    at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
    at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1192)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)
    at scala.Option.foreach(Option.scala:245)
    at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:693)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1393)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1354)
    at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
  </code></pre>
  
  <p>So, it didn't work, let's try collecting the data and iterating in a map:</p>
  
  <pre><code>scala&gt; finalTable
  finalTable   finalTables
  
  scala&gt; finalTables.count
  res118: Long = 226570                                                           
  
  scala&gt; finalTables.collect()
  15/05/28 11:45:59 ERROR Executor: Exception in task 1.0 in stage 96.0 (TID 351)
  java.lang.ArrayIndexOutOfBoundsException
  15/05/28 11:45:59 ERROR TaskSetManager: Task 1 in stage 96.0 failed 1 times; aborting job
  org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 96.0 failed 1 times, most recent failure: Lost task 1.0 in stage 96.0 (TID 351, localhost): java.lang.ArrayIndexOutOfBoundsException
  
  Driver stacktrace:
    at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1204)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1193)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1192)
    at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
    at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
    at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1192)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)
    at scala.Option.foreach(Option.scala:245)
    at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:693)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1393)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1354)
    at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
  </code></pre>
  
  <p>Now, i do know i did something wrong and in case someone wonders, the error is the same if i try to pick only "r" or any other Dataframe, with the exception of "SchemaRDD". Anyone encountered a similar problem and/or has a suggestion?</p>
  
  <p>I am using Spark 1.3.1 by the way.</p>