title: Spark AnalysisException when "flattening" DataFrame in Spark SQL
body: <p>I'm using the approach given <a href="https://stackoverflow.com/a/37473765/7648">here</a> to flatten a DataFrame in Spark SQL.   Here is my code:</p>
  
  <pre class="lang-scala prettyprint-override"><code>package com.acme.etl.xml
  
  import org.apache.spark.sql.types._ 
  import org.apache.spark.sql.{Column, SparkSession}
  
  object RuntimeError {   def main(args: Array[String]): Unit = {
      val spark = SparkSession.builder().appName("FlattenSchema").getOrCreate()
      val rowTag = "idocData"
      val dataFrameReader =
          spark.read
            .option("rowTag", rowTag)
      val xmlUri = "bad_011_1.xml"
      val df =
          dataFrameReader
            .format("xml")
            .load(xmlUri)
      val schema: StructType = df.schema
      val columns: Array[Column] = flattenSchema(schema)
      val df2 = df.select(columns: _*)
  
    }
  
    def flattenSchema(schema: StructType, prefix: String = null) : Array[Column] = {
      schema.fields.flatMap(f =&gt; {
        val colName: String = if (prefix == null) f.name else prefix + "." + f.name
        val dataType = f.dataType
        dataType match {
          case st: StructType =&gt; flattenSchema(st, colName)
          case _: StringType =&gt; Array(new org.apache.spark.sql.Column(colName))
          case _: LongType =&gt; Array(new org.apache.spark.sql.Column(colName))
          case _: DoubleType =&gt; Array(new org.apache.spark.sql.Column(colName))
          case arrayType: ArrayType =&gt; arrayType.elementType match {
            case structType: StructType =&gt; flattenSchema(structType, colName)
          }
          case _ =&gt; Array(new org.apache.spark.sql.Column(colName))
        }
      })
    }
  }
  </code></pre>
  
  <p>Much of the time, this works fine.  But for the XML given below:</p>
  
  <pre class="lang-xml prettyprint-override"><code>&lt;Receive xmlns="http://Microsoft.LobServices.Sap/2007/03/Idoc/3/ORDERS05/ZORDERS5/702/Receive"&gt;
      &lt;idocData&gt;
          &lt;E2EDP01008GRP xmlns="http://Microsoft.LobServices.Sap/2007/03/Types/Idoc/3/ORDERS05/ZORDERS5/702"&gt;
              &lt;E2EDPT1001GRP&gt;
                  &lt;E2EDPT2001&gt;
                      &lt;DATAHEADERCOLUMN_DOCNUM&gt;0000000141036013&lt;/DATAHEADERCOLUMN_DOCNUM&gt;
                  &lt;/E2EDPT2001&gt;
                  &lt;E2EDPT2001&gt;
                      &lt;DATAHEADERCOLUMN_DOCNUM&gt;0000000141036013&lt;/DATAHEADERCOLUMN_DOCNUM&gt;
                  &lt;/E2EDPT2001&gt;
              &lt;/E2EDPT1001GRP&gt;
          &lt;/E2EDP01008GRP&gt;
          &lt;E2EDP01008GRP xmlns="http://Microsoft.LobServices.Sap/2007/03/Types/Idoc/3/ORDERS05/ZORDERS5/702"&gt;
          &lt;/E2EDP01008GRP&gt;
      &lt;/idocData&gt;
  &lt;/Receive&gt;
  </code></pre>
  
  <p>this exception occurs:</p>
  
  <pre><code>Exception in thread "main" org.apache.spark.sql.AnalysisException: cannot resolve '`E2EDP01008GRP`.`E2EDPT1001GRP`.`E2EDPT2001`['DATAHEADERCOLUMN_DOCNUM']' due to data type mismatch: argument 2 requires integral type, however, ''DATAHEADERCOLUMN_DOCNUM'' is of string type.;;
  'Project [E2EDP01008GRP#0.E2EDPT1001GRP.E2EDPT2001[DATAHEADERCOLUMN_DOCNUM] AS DATAHEADERCOLUMN_DOCNUM#3, E2EDP01008GRP#0._VALUE AS _VALUE#4, E2EDP01008GRP#0._xmlns AS _xmlns#5]
  +- Relation[E2EDP01008GRP#0] XmlRelation(&lt;function0&gt;,Some(/Users/paulreiners/s3/cdi-events-partition-staging/content_acme_purchase_order_json_v1/bad_011_1.xml),Map(rowtag -&gt; idocData, path -&gt; /Users/paulreiners/s3/cdi-events-partition-staging/content_acme_purchase_order_json_v1/bad_011_1.xml),null)
  </code></pre>
  
  <p>What is causing this?</p>