title: Spark 2.3.0 Failed to find data source: kafka
body: <p>I am attempting to setup a Kafka stream using a CSV so that I can stream it into Spark. However, I keep getting</p>
  
  <pre><code>Exception in thread "main" java.lang.ClassNotFoundException: Failed to find data source: kafka. Please find packages at http://spark.apache.org/third-party-projects.html
  </code></pre>
  
  <p>My code looks like this</p>
  
  <pre><code>import org.apache.spark.sql.SparkSession
  import org.apache.spark.sql.functions._
  import org.apache.log4j.{Level, Logger}
  import org.apache.spark.sql.execution.streaming.FileStreamSource.Timestamp
  import org.apache.spark.sql.types._
  
  object SpeedTester {
    def main(args: Array[String]): Unit = {
  
    val spark = SparkSession.builder.master("local[4]").appName("SpeedTester").config("spark.driver.memory", "8g").getOrCreate()
    val rootLogger = Logger.getRootLogger()
    rootLogger.setLevel(Level.ERROR)
    import spark.implicits._
    val mySchema = StructType(Array(
      StructField("incident_id", IntegerType),
      StructField("date", StringType),
      StructField("state", StringType),
      StructField("city_or_county", StringType),
      StructField("n_killed", IntegerType),
      StructField("n_injured", IntegerType)
    ))
  
    val streamingDataFrame = spark.readStream.schema(mySchema).csv("C:/Users/zoldham/IdeaProjects/flinkpoc/Data/test")
    streamingDataFrame.selectExpr("CAST(incident_id AS STRING) AS key",
    "to_json(struct(*)) AS value").writeStream
      .format("kafka")
      .option("topic", "testTopic")
      .option("kafka.bootstrap.servers", "localhost:9092")
      .option("checkpointLocation", "C:/Users/zoldham/IdeaProjects/flinkpoc/Data")
      .start()
  
    val df = spark.readStream.format("kafka").option("kafka.bootstrap.servers", "localhost:9092")
      .option("subscribe", "testTopic").load()
    val df1 = df.selectExpr("CAST(value AS STRING)", "CAST(timestamp AS TIMESTAMP)").as[(String, Timestamp)]
      .select(from_json(col("value"), mySchema).as("data"), col("timestamp"))
      .select("data.*", "timestamp")
    df1.writeStream
      .format("console")
      .option("truncate","false")
      .start()
      .awaitTermination()
    }
  }
  </code></pre>
  
  <p>And my <code>build.sbt</code> file looks like this</p>
  
  <pre><code>name := "Spark POC"
  
  version := "0.1"
  
  scalaVersion := "2.11.12"
  
  libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.3.0"
  
  libraryDependencies += "com.microsoft.sqlserver" % "mssql-jdbc" % "6.2.1.jre8"
  
  libraryDependencies += "org.scalafx" %% "scalafx" % "8.0.144-R12"
  
  libraryDependencies += "org.apache.ignite" % "ignite-core" % "2.5.0"
  
  libraryDependencies += "org.apache.ignite" % "ignite-spring" % "2.5.0"
  
  libraryDependencies += "org.apache.ignite" % "ignite-indexing" % "2.5.0"
  
  libraryDependencies += "org.apache.spark" %% "spark-streaming-kafka-0-10_2.11" % "2.3.0"
  
  libraryDependencies += "org.apache.kafka" % "kafka-clients" % "0.11.0.1"
  </code></pre>
  
  <p>What is causing that error? As you can see, I plainly included Kafka in the library dependencies, and even followed the <a href="https://spark.apache.org/docs/2.2.0/streaming-kafka-0-10-integration.html" rel="nofollow noreferrer">official guide</a>. Here is the stack trace:</p>
  
  <pre class="lang-none prettyprint-override"><code>Exception in thread "main" java.lang.ClassNotFoundException: Failed to find data source: kafka. Please find packages at http://spark.apache.org/third-party-projects.html
      at org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:635)
      at org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:283)
      at SpeedTester$.main(SpeedTester.scala:61)
      at SpeedTester.main(SpeedTester.scala)
  Caused by: java.lang.ClassNotFoundException: kafka.DefaultSource
      at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
      at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
      at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$23$$anonfun$apply$15.apply(DataSource.scala:618)
      at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$23$$anonfun$apply$15.apply(DataSource.scala:618)
      at scala.util.Try$.apply(Try.scala:192)
      at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$23.apply(DataSource.scala:618)
      at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$23.apply(DataSource.scala:618)
      at scala.util.Try.orElse(Try.scala:84)
      at org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:618)
      ... 3 more
  </code></pre>