title: NoSuchMethodError Spark internal logging
body: <p>I have packaged my application into a jar file, however, when I try to execute it, the application fails with this error:</p>
  
  <pre><code>Exception in thread "main" java.lang.NoSuchMethodError: org.apache.spark.internal.Logging.$init$(Lorg/apache/spark/internal/Logging;)V
      at org.apache.spark.streaming.kafka010.KafkaUtils$.&lt;init&gt;(KafkaUtils.scala:39)
      at org.apache.spark.streaming.kafka010.KafkaUtils$.&lt;clinit&gt;(KafkaUtils.scala)
      at jobs.SparkJobExample$.main(SparkJobExample.scala:56)
      at jobs.SparkJobExample.main(SparkJobExample.scala)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:498)
      at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
      at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:846)
      at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167)
      at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:194)
      at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
      at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:921)
      at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:932)
      at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
  </code></pre>
  
  <p><code>SparkJobExample.scala:56</code> line referred in this error looks like this:</p>
  
  <pre><code> val dataRDD = KafkaUtils.createRDD(sparkContext,kafkaParams(),offsetRanges,LocationStrategies.PreferConsistent)
  </code></pre>
  
  <p>As I understood it might be dependencies version problem I checked them but everything seems to be fine. What could be the reason of this error?</p>
  
  <p>My build.sbt file contents:</p>
  
  <pre><code>name := "SparkK8s"
  version := "0.1"
  scalaVersion := "2.12.8"
  val sparkVersion = "2.4.0"
  val circeVersion = "0.11.0"
  
  dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-core" % "2.9.8"
  dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.8"
  dependencyOverrides += "com.fasterxml.jackson.module" % "jackson-module-scala" % "2.9.8"
  
  resolvers += "Spark Packages Repo" at "http://dl.bintray.com/spark-packages/maven"
  resolvers += "confluent" at "http://packages.confluent.io/maven/"
  
  libraryDependencies ++= Seq(
    "org.apache.spark" %% "spark-core" % sparkVersion,
    "org.apache.spark" %% "spark-sql" % sparkVersion,
    "org.apache.spark" %% "spark-streaming" % sparkVersion,
    "org.apache.spark" %% "spark-hive" % sparkVersion,
    "org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVersion,
    "org.apache.spark" %% "spark-streaming-kafka-0-10_2.12" % sparkVersion,
    "org.apache.kafka" %% "kafka" % "2.1.0",
    "org.apache.kafka" % "kafka-log4j-appender" % "2.1.0",
    "org.scalatest" %% "scalatest" % "3.2.0-SNAP10" % "it, test",
    "org.scalacheck" %% "scalacheck" % "1.14.0" % "it, test",
    "io.kubernetes" % "client-java" % "3.0.0" % "it",
    "org.json" % "json" % "20180813",
    "io.circe" %% "circe-core" % circeVersion,
    "io.circe" %% "circe-generic" % circeVersion,
    "io.circe" %% "circe-parser" % circeVersion,
    "org.apache.avro" % "avro" % "1.8.2",
    "io.confluent" % "kafka-avro-serializer" % "5.0.1"
  )
  </code></pre>
  
  <p>UPDATE: I deployed jar file both with Kubernetes and locally. The result was the same</p>
  
  <pre><code>$SPARK_HOME/bin/spark-submit \
     --master local \
     --name spark-etl-job \
     --class jobs.SparkJobExample \
     --conf spark.executor.instances=1 \
  http://192.168.99.100:8089/spark_hive_job.jar &lt;params_list&gt;
  </code></pre>
  
  <p>Sbt assembly settings:</p>
  
  <pre><code>assemblyJarName in assembly := "spark_hive_job.jar"
  
  mainClass in assembly := Some("jobs.SparkJobExample")
  
  test in assembly := {}
  
  assemblyMergeStrategy in assembly := {
    case x if x.endsWith(".conf") =&gt; MergeStrategy.discard
    case PathList("org", "apache", "spark", "unused", "UnusedStubClass.class") =&gt; MergeStrategy.first
    case PathList("org", "apache", "commons", "logging", _*) =&gt; MergeStrategy.first
    case PathList("org", "apache", "commons", "beanutils", _*) =&gt; MergeStrategy.first
    case PathList("org", "apache", "commons", "collections", _*) =&gt; MergeStrategy.first
    case PathList("org", "apache", "hadoop", "yarn", _*) =&gt; MergeStrategy.first
    case PathList("org", "aopalliance", _*) =&gt; MergeStrategy.first
    case PathList("org", "objenesis", _*) =&gt; MergeStrategy.first
    case PathList("com", "sun", "jersey", _*) =&gt; MergeStrategy.first
    case PathList("org", "apache", "hadoop", "yarn", _*) =&gt; MergeStrategy.first
    case PathList("org", "slf4j", "impl", _*) =&gt; MergeStrategy.first
    case PathList("com", "codahale", "metrics", _*) =&gt; MergeStrategy.first
    case PathList("javax", "transaction", _*) =&gt; MergeStrategy.first
    case PathList("javax", "inject", _*) =&gt; MergeStrategy.first
    case PathList("javax", "xml", _*) =&gt; MergeStrategy.first
    case PathList("META-INF", "jersey-module-version") =&gt; MergeStrategy.first
    case PathList("example",  _*) =&gt; MergeStrategy.discard
    case PathList("utils",  _*) =&gt; MergeStrategy.discard
    case PathList("data_generation",  _*) =&gt; MergeStrategy.discard
    case PathList("example_tests",  _*) =&gt; MergeStrategy.discard
    case PathList("plugin.xml") =&gt; MergeStrategy.first
    case PathList("parquet.thrift") =&gt; MergeStrategy.first
    case PathList("git.properties") =&gt; MergeStrategy.first
    case PathList("codegen", "config.fmpp") =&gt; MergeStrategy.first
    case PathList("git.properties") =&gt; MergeStrategy.first
    case PathList("overview.html") =&gt; MergeStrategy.discard
    case x =&gt; (assemblyMergeStrategy in assembly).value(x)
  }
  </code></pre>