title: Cannot merge two DataFrames in Scala Spark
body: <p>I've been trying to append 1 DataFrame to another DF in Scala. The append operation in this case is simply adding a new column of the same size to the existing column - no key matching is involved. Both DataFrames are of the same shape (5 rows and 1 column only). </p>
  
  <pre><code>scala&gt; val coefficients = lrModel.coefficients.toArray.toSeq.toDF("coefficients")
  coefficients: org.apache.spark.sql.DataFrame = [coefficients: double]
  
  scala&gt; coefficients.show()
  +--------------------+
  |        coefficients|
  +--------------------+
  |   -59525.0697785032|
  |   6957.836000531959|
  |   314.2998010755629|
  |-0.37884289844065666|
  |  -1758.154438149325|
  +--------------------+
  scala&gt; val tvalues = trainingSummary.tValues.toArray.drop(1).toSeq.toDF("t-values")
  tvalues: org.apache.spark.sql.DataFrame = [t-values: double]
  
  scala&gt; tvalues.show()
  +-------------------+
  |           t-values|
  +-------------------+
  | 1.8267249911295418|
  | 100.35507390273406|
  | -8.768588605222108|
  |-0.4656738230173362|
  |  10.48091833711012|
  +-------------------+
  </code></pre>
  
  <p>The <code>join()</code> function runs and I can even get the schema, but when I want to display all values of the new DF I'm getting the error:</p>
  
  <pre><code>scala&gt; val outputModelDF1 = coefficients.join(tvalues)
  outputModelDF1: org.apache.spark.sql.DataFrame = [coefficients: double, t-values: double]
  scala&gt; outputModelDF1.printSchema()
  root
   |-- coefficients: double (nullable = false)
   |-- t-values: double (nullable = false)
  
  scala&gt; outputModelDF1.show()
  org.apache.spark.sql.AnalysisException: Detected cartesian product for INNER join between logical plans
  Project [value#359 AS coefficients#361]
  +- LocalRelation [value#359]
  and
  Project [value#368 AS t-values#370]
  +- LocalRelation [value#368]
  Join condition is missing or trivial.
  Use the CROSS JOIN syntax to allow cartesian products between these relations.;
    at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$20.applyOrElse(Optimizer.scala:1080)
    at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$20.applyOrElse(Optimizer.scala:1077)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267)
    at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:266)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:256)
    at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts.apply(Optimizer.scala:1077)
    at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts.apply(Optimizer.scala:1062)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:85)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:82)
    at scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:57)
    at scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:66)
    at scala.collection.mutable.WrappedArray.foldLeft(WrappedArray.scala:35)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:82)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:74)
    at scala.collection.immutable.List.foreach(List.scala:381)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:74)
    at org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:78)
    at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:78)
    at org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute(QueryExecution.scala:84)
    at org.apache.spark.sql.execution.QueryExecution.sparkPlan(QueryExecution.scala:80)
    at org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:89)
    at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:89)
    at org.apache.spark.sql.Dataset.withAction(Dataset.scala:2832)
    at org.apache.spark.sql.Dataset.head(Dataset.scala:2153)
    at org.apache.spark.sql.Dataset.take(Dataset.scala:2366)
    at org.apache.spark.sql.Dataset.showString(Dataset.scala:245)
    at org.apache.spark.sql.Dataset.show(Dataset.scala:644)
    at org.apache.spark.sql.Dataset.show(Dataset.scala:603)
    at org.apache.spark.sql.Dataset.show(Dataset.scala:612)
    ... 52 elided
  </code></pre>
  
  <p>Any idea how to deal with it and how to simply merge these two DFs together?</p>
  
  <p>UPDATE 1</p>
  
  <p>I should have stated the desired format of the output that I want to achieve. Please see below:</p>
  
  <pre><code>    +--------------------+--------------------+
      |        coefficients|            t-values|
      +--------------------+--------------------+
      |   -59525.0697785032|  1.8267249911295418|
      |   6957.836000531959|  100.35507390273406|
      |   314.2998010755629|  -8.768588605222108|
      |-0.37884289844065666| -0.4656738230173362|
      |  -1758.154438149325|  -1758.154438149325|
      +--------------------+--------------------+
  </code></pre>
  
  <p>UPDATE 2</p>
  
  <p>Unfortunately, the following approach using <code>withColumn()</code> didn't work. </p>
  
  <pre><code>scala&gt; val outputModelDF1 = coefficients.withColumn("t-values", tvalues("t-values"))
  org.apache.spark.sql.AnalysisException: resolved attribute(s) t-values#119 missing from coefficients#113 in operator !Project [coefficients#113, t-values#119 AS t-values#130];;
  !Project [coefficients#113, t-values#119 AS t-values#130]
  +- Project [value#111 AS coefficients#113]
     +- LocalRelation [value#111]
  
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:39)
    at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:91)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:347)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:78)
    at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:127)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:78)
    at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:91)
    at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:52)
    at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:66)
    at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withPlan(Dataset.scala:2872)
    at org.apache.spark.sql.Dataset.select(Dataset.scala:1153)
    at org.apache.spark.sql.Dataset.withColumn(Dataset.scala:1908)
    ... 52 elided
  </code></pre>