title: How to run simple Spark app from Eclipse/Intellij IDE?
body: <p>To ease the development of my map reduce tasks running on Hadoop prior to actually deploying the tasks to Hadoop I test using a simple map reducer I wrote : </p>
  
  <pre><code>object mapreduce {
    import scala.collection.JavaConversions._
  
    val intermediate = new java.util.HashMap[String, java.util.List[Int]]
                                                    //&gt; intermediate  : java.util.HashMap[String,java.util.List[Int]] = {}
    val result = new java.util.ArrayList[Int]       //&gt; result  : java.util.ArrayList[Int] = []
  
    def emitIntermediate(key: String, value: Int) {
      if (!intermediate.containsKey(key)) {
        intermediate.put(key, new java.util.ArrayList)
      }
      intermediate.get(key).add(value)
    }                                               //&gt; emitIntermediate: (key: String, value: Int)Unit
  
    def emit(value: Int) {
      println("value is " + value)
      result.add(value)
    }                                               //&gt; emit: (value: Int)Unit
  
    def execute(data: java.util.List[String], mapper: String =&gt; Unit, reducer: (String, java.util.List[Int]) =&gt; Unit) {
  
      for (line &lt;- data) {
        mapper(line)
      }
  
      for (keyVal &lt;- intermediate) {
        reducer(keyVal._1, intermediate.get(keyVal._1))
      }
  
      for (item &lt;- result) {
        println(item)
      }
    }                                               //&gt; execute: (data: java.util.List[String], mapper: String =&gt; Unit, reducer: (St
                                                    //| ring, java.util.List[Int]) =&gt; Unit)Unit
  
    def mapper(record: String) {
      var jsonAttributes = com.nebhale.jsonpath.JsonPath.read("$", record, classOf[java.util.ArrayList[String]])
      println("jsonAttributes are " + jsonAttributes)
      var key = jsonAttributes.get(0)
      var value = jsonAttributes.get(1)
  
      println("key is " + key)
      var delims = "[ ]+";
      var words = value.split(delims);
      for (w &lt;- words) {
        emitIntermediate(w, 1)
      }
    }                                               //&gt; mapper: (record: String)Unit
  
    def reducer(key: String, listOfValues: java.util.List[Int]) = {
      var total = 0
      for (value &lt;- listOfValues) {
        total += value;
      }
  
      emit(total)
    }                                               //&gt; reducer: (key: String, listOfValues: java.util.List[Int])Unit
    var dataToProcess = new java.util.ArrayList[String]
                                                    //&gt; dataToProcess  : java.util.ArrayList[String] = []
    dataToProcess.add("[\"test1\" , \"test1 here is another test1 test1 \"]")
                                                    //&gt; res0: Boolean = true
    dataToProcess.add("[\"test2\" , \"test2 here is another test2 test1 \"]")
                                                    //&gt; res1: Boolean = true
  
    execute(dataToProcess, mapper, reducer)         //&gt; jsonAttributes are [test1, test1 here is another test1 test1 ]
                                                    //| key is test1
                                                    //| jsonAttributes are [test2, test2 here is another test2 test1 ]
                                                    //| key is test2
                                                    //| value is 2
                                                    //| value is 2
                                                    //| value is 4
                                                    //| value is 2
                                                    //| value is 2
                                                    //| 2
                                                    //| 2
                                                    //| 4
                                                    //| 2
                                                    //| 2
  
  
    for (keyValue &lt;- intermediate) {
        println(keyValue._1 + "-&gt;"+keyValue._2.size)//&gt; another-&gt;2
                                                    //| is-&gt;2
                                                    //| test1-&gt;4
                                                    //| here-&gt;2
                                                    //| test2-&gt;2
     }
  
  
  }
  </code></pre>
  
  <p>This allows me to run my mapreduce tasks within my Eclipse IDE on Windows before deploying to the actual Hadoop cluster. I would like to perform something similar for Spark or have the ability to write Spark code from within Eclipse to test prior to deploying to Spark cluster. Is this possible with Spark ? Since Spark runs on top of Hadoop does this mean I cannot run Spark without first having Hadoop installed ? So in other words can I run the code using just the Spark libraries ? : </p>
  
  <pre><code>import org.apache.spark.SparkContext
  import org.apache.spark.SparkContext._
  
  object SimpleApp {
    def main(args: Array[String]) {
      val logFile = "$YOUR_SPARK_HOME/README.md" // Should be some file on your system
      val sc = new SparkContext("local", "Simple App", "YOUR_SPARK_HOME",
        List("target/scala-2.10/simple-project_2.10-1.0.jar"))
      val logData = sc.textFile(logFile, 2).cache()
      val numAs = logData.filter(line =&gt; line.contains("a")).count()
      val numBs = logData.filter(line =&gt; line.contains("b")).count()
      println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
    }
  }
  </code></pre>
  
  <p>taken from <a href="https://spark.apache.org/docs/0.9.0/quick-start.html#a-standalone-app-in-scala" rel="noreferrer">https://spark.apache.org/docs/0.9.0/quick-start.html#a-standalone-app-in-scala</a></p>
  
  <p>If so what are the Spark libraries I need to include within my project ?</p>