title: How to use environment variables in spark which deployed on cluster mode?
body: <p>When I set environment variable using Intellij below code works, but when i deploy code with spark-submit it does not work since environment variables are not exits on entire cluster.</p>
  <pre><code>import com.hepsiburada.util.KafkaUtil
  import org.apache.spark.SparkConf
  import org.apache.spark.sql.functions.{col, date_format, to_date}
  import org.apache.spark.sql.{SaveMode, SparkSession}
  import org.apache.spark.streaming.{Minutes, StreamingContext}
  
  object StreamingApplication {
  
    private val appName: String = sys.env(&quot;APPNAME&quot;)
    private val bootstrapServers: String = sys.env(&quot;BOOTSTRAPSERVERS&quot;)
    private val topic: String = sys.env(&quot;TOPIC&quot;)
    private val consumerGroupId: String = sys.env(&quot;CONSUMERGROUPID&quot;)
    private val batchIntervalMinutes = sys.env(&quot;BATCHINTERVALMIN&quot;).toLong
    private val outputPathHdfs: String = sys.env(&quot;OUTPUTPATHHDFS&quot;)
  
    def main(args: Array[String]): Unit = {
  
      val conf = new SparkConf()
        .set(&quot;spark.app.name&quot;, appName)
        .set(&quot;spark.streaming.stopGracefullyOnShutdown&quot;, &quot;true&quot;)
        .set(&quot;spark.sql.legacy.timeParserPolicy&quot;, &quot;LEGACY&quot;)
        .set(&quot;spark.sql.session.timeZone&quot;, &quot;UTC&quot;)
  
      val streamingContext = new StreamingContext(conf, Minutes(batchIntervalMinutes))
      streamingContext.sparkContext.setLogLevel(&quot;WARN&quot;)
  
      val kafkaStream = KafkaUtil.getKafkaStream(streamingContext, topic, bootstrapServers, consumerGroupId)
  
      val spark = SparkSession.builder.config(streamingContext.sparkContext.getConf).getOrCreate()
  
      kafkaStream.foreachRDD { rdd =&gt;
        val rdd_value = rdd.map(_.value().toString)
        if (!rdd_value.isEmpty()) {
          val df_init = spark.read.json(rdd_value)
          val df = df_init.withColumn(&quot;dy&quot;, to_date(col(&quot;receivedat&quot;))).withColumn(&quot;hr&quot;, date_format(col(&quot;receivedat&quot;), &quot;HH&quot;))
          df.coalesce(1)
            .write.mode(SaveMode.Append)
            .format(&quot;json&quot;)
            .partitionBy(&quot;dy&quot;, &quot;hr&quot;)
            .save(outputPathHdfs)
        }
      }
      streamingContext.start()
      streamingContext.awaitTermination()
    }
  }
  </code></pre>
  <p><strong>What i have tried so far:</strong></p>
  <p>spark.yarn.appMasterEnv, spark.yarn.appExecutorEnv configuration to pass environment variables to the cluster but that does not works.</p>
  <p>I also tried to set variables as conf like --conf spark.Variable=x and spark.get(&quot;Variable&quot;) instead of sys.env(&quot;Variable&quot;) but that does not worked as well</p>
  <p><strong>Here is the error:</strong></p>
  <pre><code>22/04/17 17:08:40 ERROR ApplicationMaster: User class threw exception: java.util.NoSuchElementException: APPNAME
  java.util.NoSuchElementException: APPNAME
      at org.apache.spark.SparkConf.$anonfun$get$1(SparkConf.scala:245)
      at scala.Option.getOrElse(Option.scala:189)
      at org.apache.spark.SparkConf.get(SparkConf.scala:245)
      at com.hepsiburada.StreamingApplication$.main(StreamingApplication.scala:18)
      at com.hepsiburada.StreamingApplication.main(StreamingApplication.scala)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:498)
      at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:732)
  </code></pre>