title: StackOverflowError during class deserialization by Spark Executor
body: <p>A Spark job has started failing all of a sudden in the code where it's folding some historical data from files of a total size of around 1 GB. I will followup on the actual RDD size.</p>
  
  <pre><code>2016-08-03_09:43:21.00177 ERROR executor.Executor: Exception in task 49.0 in stage 0.0 (TID 32)
  java.lang.StackOverflowError
      at java.lang.ClassLoader.defineClass1(Native Method)
      at java.lang.ClassLoader.defineClass(ClassLoader.java:760)
      at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
      at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
      at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
      at java.security.AccessController.doPrivileged(Native Method)
      at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
      at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
      at java.lang.ClassLoader.defineClass1(Native Method)
      at java.lang.ClassLoader.defineClass(ClassLoader.java:760)
      at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
      at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
      at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
      at java.security.AccessController.doPrivileged(Native Method)
      at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
      at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
      at java.lang.ClassLoader.defineClass1(Native Method)
      at java.lang.ClassLoader.defineClass(ClassLoader.java:760)
      at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
      at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
      at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
      at java.security.AccessController.doPrivileged(Native Method)
      at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
      at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:411)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
      at java.lang.ClassLoader.defineClass1(Native Method)
      at java.lang.ClassLoader.defineClass(ClassLoader.java:760)
      at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
      at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
      at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
      at java.security.AccessController.doPrivileged(Native Method)
      at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
      at java.lang.Class.forName0(Native Method)
      at java.lang.Class.forName(Class.java:348)
      at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:66)
      at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1613)
      at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1518)
      at java.io.ObjectInputStream.readClass(ObjectInputStream.java:1484)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1334)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371)
      at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:497)
      at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1896)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371)
      at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:497)
      at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1896)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.readArray(ObjectInputStream.java:1707)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1345)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
  ...
  </code></pre>
  
  <p>The same job seems to be working fine again if the folding is done on a smaller dataset. However, it seems unusual that this amount of data would cause it to fail like this.</p>