title: About a java.lang.NoClassDefFoundError: Could not initialize class org.xerial.snappy.Snappy
body: <p>i have an error when I try to compile, test and run a junit test.</p>
  
  <p>I want to load a local Avro file using DataFrames but I am getting an exception:</p>
  
  <pre><code>org.xerial.snappy.SnappyError: [FAILED_TO_LOAD_NATIVE_LIBRARY] null
  </code></pre>
  
  <p>I am not using Cassandra at all, the version of involved jars are:</p>
  
  <pre><code>&lt;properties&gt;
      &lt;!-- Generic properties --&gt;
      &lt;java.version&gt;1.7&lt;/java.version&gt;
      &lt;project.build.sourceEncoding&gt;UTF-8&lt;/project.build.sourceEncoding&gt;
      &lt;project.reporting.outputEncoding&gt;UTF-8&lt;/project.reporting.outputEncoding&gt;
      &lt;!--  Dependency versions --&gt;
      &lt;maven.compiler.source&gt;1.7&lt;/maven.compiler.source&gt;
      &lt;maven.compiler.target&gt;1.7&lt;/maven.compiler.target&gt;
      &lt;scala.version&gt;2.10.4&lt;/scala.version&gt;
      &lt;junit.version&gt;4.11&lt;/junit.version&gt;
      &lt;slf4j.version&gt;1.7.12&lt;/slf4j.version&gt;
      &lt;spark.version&gt;1.5.0-cdh5.5.2&lt;/spark.version&gt;
      &lt;databricks.version&gt;1.5.0&lt;/databricks.version&gt;
      &lt;json4s-native.version&gt;3.5.0&lt;/json4s-native.version&gt;
      &lt;spark-avro.version&gt;2.0.1&lt;/spark-avro.version&gt;
  &lt;/properties&gt;
  </code></pre>
  
  <p>and these are the dependencies:</p>
  
  <pre><code>&lt;dependencies&gt;
  
      &lt;dependency&gt;
          &lt;groupId&gt;org.json4s&lt;/groupId&gt;
          &lt;artifactId&gt;json4s-native_2.10&lt;/artifactId&gt;
          &lt;version&gt;${json4s-native.version}&lt;/version&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;junit&lt;/groupId&gt;
          &lt;artifactId&gt;junit&lt;/artifactId&gt;
          &lt;version&gt;${junit.version}&lt;/version&gt;
          &lt;scope&gt;test&lt;/scope&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;org.apache.spark&lt;/groupId&gt;
          &lt;artifactId&gt;spark-sql_2.10&lt;/artifactId&gt;
          &lt;version&gt;${spark.version}&lt;/version&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;com.databricks&lt;/groupId&gt;
          &lt;artifactId&gt;spark-csv_2.10&lt;/artifactId&gt;
          &lt;version&gt;${databricks.version}&lt;/version&gt;
          &lt;exclusions&gt;
              &lt;exclusion&gt;
                  &lt;groupId&gt;org.xerial.snappy&lt;/groupId&gt;
                  &lt;artifactId&gt;snappy-java&lt;/artifactId&gt;
              &lt;/exclusion&gt;
          &lt;/exclusions&gt;
      &lt;/dependency&gt;
  
      &lt;dependency&gt;
          &lt;groupId&gt;org.xerial.snappy&lt;/groupId&gt;
          &lt;artifactId&gt;snappy-java&lt;/artifactId&gt;
          &lt;version&gt;1.0.4.1&lt;/version&gt;
          &lt;scope&gt;compile&lt;/scope&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;com.databricks&lt;/groupId&gt;
          &lt;artifactId&gt;spark-avro_2.10&lt;/artifactId&gt;
          &lt;version&gt;${spark-avro.version}&lt;/version&gt;
      &lt;/dependency&gt;
      &lt;!-- https://mvnrepository.com/artifact/log4j/log4j --&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;log4j&lt;/groupId&gt;
          &lt;artifactId&gt;log4j&lt;/artifactId&gt;
          &lt;version&gt;1.2.17&lt;/version&gt;
      &lt;/dependency&gt;
  &lt;/dependencies&gt;
  </code></pre>
  
  <p>I have tried to compile the project with </p>
  
  <p><code>mvn clean install -Dorg.xerial.snappy.lib.name=libsnappyjava.jnlib -Dorg.xerial.snappy.tempdir=/tmp</code></p>
  
  <p>before copying the jar within /tmp, with no luck.</p>
  
  <pre><code>$ ls -lt /tmp/
  total 1944
  ...27 dic 13:01 snappy-java-1.0.4.jar
  </code></pre>
  
  <p>This is the code:</p>
  
  <pre><code>import org.apache.spark.rdd.RDD
  import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
  import org.apache.spark.{SparkConf, SparkContext}
  import com.databricks.spark.avro._
  import java.io._
  
  //auxiliary function
  def readRawData(pathToResources: String, sqlContext: SQLContext, rawFormat: String = "json"): DataFrame = {
  val a: DataFrame = rawFormat match {
    case "avro" =&gt; sqlContext.read.avro(pathToResources)
    case "json" =&gt; sqlContext.read.json(pathToResources)
    case _ =&gt; throw new Exception("Format not supported, use AVRO or JSON instead.")
  }
  val b: DataFrame = a.filter("extraData.type = 'data'")
  val c: DataFrame = a.select("extraData.topic", "extraData.timestamp",
    "extraData.sha1Hex", "extraData.filePath", "extraData.fileName",
    "extraData.lineNumber", "extraData.type",
    "message")
  
  val indexForMessage: Int = c.schema.fieldIndex("message")
  val result: RDD[Row] = c.rdd.filter(r =&gt;
    !r.anyNull match {
      case true =&gt; true
      case false =&gt; false
    }
  ).flatMap(r =&gt; {
    val metadata: String = r.toSeq.slice(0, indexForMessage).mkString(",")
    val lines = r.getString(indexForMessage).split("\n")
    lines.map(l =&gt; Row.fromSeq(metadata.split(",").toSeq ++ Seq(l)))
  })
  sqlContext.createDataFrame(result, c.schema)
  }//readRawData
  
  
  def validate(rawFlumeData : String = "FlumeData.1482407196579",fileNamesToBeDigested : String = "fileNames-to-be-digested.txt", sqlContext: SQLContext,sc:SparkContext) : Boolean = {
  
  val result : Boolean = true
  
    sqlContext.sparkContext.hadoopConfiguration.set("avro.mapred.ignore.inputs.without.extension", "false")
  
  val rawDF : DataFrame = readRawData(rawFlumeData, sqlContext, rawFormat = "avro")
  
  rawDF.registerTempTable("RAW")
  //this line provokes the exception! cannot load snappy jar file!
  val arrayRows : Array[org.apache.spark.sql.Row] = sqlContext.sql("SELECT distinct fileName as filenames FROM RAW GROUP BY fileName").collect()
  
  val arrayFileNames : Array[String] = arrayRows.map(row=&gt;row.getString(0))
  
  val fileNamesDigested = "fileNames-AVRO-1482407196579.txt"
  
  val pw = new PrintWriter(new File(fileNamesDigested))
  
  for (filename &lt;-arrayFileNames) pw.write(filename + "\n")
  
  pw.close
  
  val searchListToBeDigested : org.apache.spark.rdd.RDD[String] = sc.textFile(fileNamesToBeDigested)
  
  //creo un map con valores como éstos: Map(EUR_BACK_SWVOL_SMILE_GBP_20160930.csv -&gt; 0, UK_SC_equities_20160930.csv -&gt; 14,...
  //val mapFileNamesToBeDigested: Map[String, Long] = searchListToBeDigested.zipWithUniqueId().collect().toMap
  
  val searchFilesAVRODigested = sc.textFile(fileNamesDigested)
  
  val mapFileNamesAVRODigested: Map[String, Long] = searchFilesAVRODigested.zipWithUniqueId().collect().toMap
  
  val pwResults = new PrintWriter(new File("validation-results.txt"))
  
  //Hay que guardar el resultado en un fichero de texto, en algún lado...
  val buffer = StringBuilder.newBuilder
  
  //Me traigo los resultados al Driver.
  val listFilesToBeDigested = searchListToBeDigested.map {line =&gt;
    val resultTemp = mapFileNamesAVRODigested.getOrElse(line,"NOT INGESTED!")
    var resul = ""
    if (resultTemp == "NOT INGESTED!"){
      resul = "File " + line + " " + resultTemp + "\n"
    }
    else{
      resul = "File " + line + " " + " is INGESTED!" + "\n"
    }
    resul
  }.collect()
  
  //añado los datos al buffer
  listFilesToBeDigested.foreach(buffer.append(_))
  //guardo el contenido del buffer en el fichero de texto de salida.
  pwResults.write(buffer.toString)
  pwResults.close
  //this boolean must return false in case of a exception or error...
  result
  }//
  </code></pre>
  
  <p>This is the unit test code:</p>
  
  <pre><code>private[validation] class ValidateInputCSVFilesTest {
  
  //AS YOU CAN SEE, I do not WANT to use snappy at all!
  val conf = new SparkConf()
  .setAppName("ValidateInputCSVFilesTest")
  .setMaster("local[2]")
  .set("spark.driver.allowMultipleContexts", "true")
  .set("spark.driver.host", "127.0.0.1")
  .set("spark.io.compression.codec", "lzf")
  
  val sc = new SparkContext(conf)
  val sqlContext = new org.apache.spark.sql.SQLContext(sc)
  
  val properties : Properties = new Properties()
   properties.setProperty("frtb.input.csv.validation.avro","./src/test/resources/avro/FlumeData.1482407196579")
  properties.setProperty("frtb.input.csv.validation.list.files","./src/test/resources/fileNames-to-be-digested.txt")
   import sqlContext.implicits._
  
  sqlContext.sparkContext.hadoopConfiguration.set("avro.mapred.ignore.inputs.without.extension", "false")
  
  @Test
  def testValidateInputFiles() = {
  
  //def validate(rawFlumeData : String = "FlumeData.1482407196579",fileNamesToBeDigested : String = "fileNames-to-be-digested.txt", sqlContext: SQLContext)
  val rawFlumeData = properties.getProperty("frtb.input.csv.validation.avro")
  val fileNamesToBeDigested = properties.getProperty("frtb.input.csv.validation.list.files")
  println("rawFlumeData  is " + rawFlumeData )
  println("fileNamesToBeDigested is " + fileNamesToBeDigested )
  val result : Boolean = ValidateInputCSVFiles.validate(rawFlumeData ,fileNamesToBeDigested ,sqlContext,sc)
  
  Assert.assertTrue("Must be true...",result)
  
  }//end of test method
  
  }//end of unit class
  </code></pre>
  
  <p>I can run perfectly the same code in a local spark-shell, using this command:</p>
  
  <pre><code>$ bin/spark-shell --packages org.json4s:json4s-native_2.10:3.5.0 --packages com.databricks:spark-csv_2.10:1.5.0 --packages com.databricks:spark-avro_2.10:2.0.1 
  </code></pre>
  
  <p>What else can I do? </p>
  
  <p>Thanks in advance.</p>