{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/74393971","id":74393971,"diff_hunk":"@@ -35,184 +34,303 @@ import org.apache.spark.util.Utils\n \n private[json] class SparkSQLJsonProcessingException(msg: String) extends RuntimeException(msg)\n \n-object JacksonParser extends Logging {\n+class JacksonParser(\n+    schema: StructType,\n+    columnNameOfCorruptRecord: String,\n+    options: JSONOptions) extends Logging {\n \n-  def parse(\n-      input: RDD[String],\n-      schema: StructType,\n-      columnNameOfCorruptRecords: String,\n-      configOptions: JSONOptions): RDD[InternalRow] = {\n+  import com.fasterxml.jackson.core.JsonToken._\n+\n+  // A `ValueConverter` is responsible for converting a value from `JsonParser`\n+  // to a value in a field for `InternalRow`.\n+  private type ValueConverter = (JsonParser) => Any\n+\n+  // `ValueConverter`s for the root schema for all fields in the schema\n+  private val rootConverter: ValueConverter = makeRootConverter(schema)\n \n-    input.mapPartitions { iter =>\n-      parseJson(iter, schema, columnNameOfCorruptRecords, configOptions)\n+  private val factory = new JsonFactory()\n+  options.setJacksonOptions(factory)\n+\n+  /**\n+   * This function deals with the cases it fails to parse. This function will be called\n+   * when exceptions are caught during converting. This functions also deals with `mode` option.\n+   */\n+  private def failedRecord(record: String): Seq[InternalRow] = {\n+    // create a row even if no corrupt record column is present\n+    if (options.failFast) {\n+      throw new RuntimeException(s\"Malformed line in FAILFAST mode: $record\")\n+    }\n+    if (options.dropMalformed) {\n+      logWarning(s\"Dropping malformed line: $record\")\n+      Nil\n+    } else {\n+      val row = new GenericMutableRow(schema.length)\n+      for (corruptIndex <- schema.getFieldIndex(columnNameOfCorruptRecord)) {\n+        require(schema(corruptIndex).dataType == StringType)\n+        row.update(corruptIndex, UTF8String.fromString(record))\n+      }\n+      Seq(row)\n     }\n   }\n \n   /**\n-   * Parse the current token (and related children) according to a desired schema\n-   * This is a wrapper for the method `convertField()` to handle a row wrapped\n-   * with an array.\n+   * Create a converter which converts the JSON documents held by the `JsonParser`\n+   * to a value according to a desired schema. This is a wrapper for the method\n+   * `makeConverter()` to handle a row wrapped with an array.\n    */\n-  def convertRootField(\n-      factory: JsonFactory,\n-      parser: JsonParser,\n-      schema: DataType): Any = {\n-    import com.fasterxml.jackson.core.JsonToken._\n-    (parser.getCurrentToken, schema) match {\n-      case (START_ARRAY, st: StructType) =>\n-        // SPARK-3308: support reading top level JSON arrays and take every element\n-        // in such an array as a row\n-        convertArray(factory, parser, st)\n-\n-      case (START_OBJECT, ArrayType(st, _)) =>\n+  def makeRootConverter(dataType: DataType): ValueConverter = dataType match {\n+    case st: StructType =>\n+      val elementConverter = makeConverter(st)\n+      val fieldConverters = st.map(_.dataType).map(makeConverter)\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case START_OBJECT => convertObject(parser, st, fieldConverters)\n+          // SPARK-3308: support reading top level JSON arrays and take every element\n+          // in such an array as a row\n+          //\n+          // For example, we support, the JSON data as below:\n+          //\n+          // [{\"a\":\"str_a_1\"}]\n+          // [{\"a\":\"str_a_2\"}, {\"b\":\"str_b_3\"}]\n+          //\n+          // resulting in:\n+          //\n+          // List([str_a_1,null])\n+          // List([str_a_2,null], [null,str_b_3])\n+          //\n+        case START_ARRAY => convertArray(parser, elementConverter)\n+      }\n+\n+    case ArrayType(st: StructType, _) =>\n+      val elementConverter = makeConverter(st)\n+      val fieldConverters = st.map(_.dataType).map(makeConverter)\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n         // the business end of SPARK-3308:\n-        // when an object is found but an array is requested just wrap it in a list\n-        convertField(factory, parser, st) :: Nil\n+        // when an object is found but an array is requested just wrap it in a list.\n+        // This is being wrapped in `JacksonParser.parse`.\n+        case START_OBJECT => convertObject(parser, st, fieldConverters)\n+        case START_ARRAY => convertArray(parser, elementConverter)\n+      }\n \n-      case _ =>\n-        convertField(factory, parser, schema)\n-    }\n+    case _ => makeConverter(dataType)\n   }\n \n-  private def convertField(\n-      factory: JsonFactory,\n-      parser: JsonParser,\n-      schema: DataType): Any = {\n-    import com.fasterxml.jackson.core.JsonToken._\n-    (parser.getCurrentToken, schema) match {\n-      case (null | VALUE_NULL, _) =>\n-        null\n+  /**\n+   * Create a converter which converts the JSON documents held by the `JsonParser`\n+   * to a value according to a desired schema.\n+   */\n+  private def makeConverter(dataType: DataType): ValueConverter = dataType match {\n+    case BooleanType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_TRUE => true\n+        case VALUE_FALSE => false\n+      }\n \n-      case (FIELD_NAME, _) =>\n-        parser.nextToken()\n-        convertField(factory, parser, schema)\n-\n-      case (VALUE_STRING, StringType) =>\n-        UTF8String.fromString(parser.getText)\n-\n-      case (VALUE_STRING, _) if parser.getTextLength < 1 =>\n-        // guard the non string type\n-        null\n-\n-      case (VALUE_STRING, BinaryType) =>\n-        parser.getBinaryValue\n-\n-      case (VALUE_STRING, DateType) =>\n-        val stringValue = parser.getText\n-        if (stringValue.contains(\"-\")) {\n-          // The format of this string will probably be \"yyyy-mm-dd\".\n-          DateTimeUtils.millisToDays(DateTimeUtils.stringToTime(parser.getText).getTime)\n-        } else {\n-          // In Spark 1.5.0, we store the data as number of days since epoch in string.\n-          // So, we just convert it to Int.\n-          stringValue.toInt\n-        }\n+    case ByteType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_NUMBER_INT => parser.getByteValue\n+      }\n \n-      case (VALUE_STRING, TimestampType) =>\n-        // This one will lose microseconds parts.\n-        // See https://issues.apache.org/jira/browse/SPARK-10681.\n-        DateTimeUtils.stringToTime(parser.getText).getTime * 1000L\n+    case ShortType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_NUMBER_INT => parser.getShortValue\n+      }\n \n-      case (VALUE_NUMBER_INT, TimestampType) =>\n-        parser.getLongValue * 1000000L\n+    case IntegerType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_NUMBER_INT => parser.getIntValue\n+      }\n \n-      case (_, StringType) =>\n-        val writer = new ByteArrayOutputStream()\n-        Utils.tryWithResource(factory.createGenerator(writer, JsonEncoding.UTF8)) {\n-          generator => generator.copyCurrentStructure(parser)\n-        }\n-        UTF8String.fromBytes(writer.toByteArray)\n-\n-      case (VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT, FloatType) =>\n-        parser.getFloatValue\n-\n-      case (VALUE_STRING, FloatType) =>\n-        // Special case handling for NaN and Infinity.\n-        val value = parser.getText\n-        val lowerCaseValue = value.toLowerCase()\n-        if (lowerCaseValue.equals(\"nan\") ||\n-          lowerCaseValue.equals(\"infinity\") ||\n-          lowerCaseValue.equals(\"-infinity\") ||\n-          lowerCaseValue.equals(\"inf\") ||\n-          lowerCaseValue.equals(\"-inf\")) {\n-          value.toFloat\n-        } else {\n-          throw new SparkSQLJsonProcessingException(s\"Cannot parse $value as FloatType.\")\n-        }\n+    case LongType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_NUMBER_INT => parser.getLongValue\n+      }\n \n-      case (VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT, DoubleType) =>\n-        parser.getDoubleValue\n-\n-      case (VALUE_STRING, DoubleType) =>\n-        // Special case handling for NaN and Infinity.\n-        val value = parser.getText\n-        val lowerCaseValue = value.toLowerCase()\n-        if (lowerCaseValue.equals(\"nan\") ||\n-          lowerCaseValue.equals(\"infinity\") ||\n-          lowerCaseValue.equals(\"-infinity\") ||\n-          lowerCaseValue.equals(\"inf\") ||\n-          lowerCaseValue.equals(\"-inf\")) {\n-          value.toDouble\n-        } else {\n-          throw new SparkSQLJsonProcessingException(s\"Cannot parse $value as DoubleType.\")\n-        }\n+    case FloatType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT =>\n+          parser.getFloatValue\n+\n+        case VALUE_STRING =>\n+          // Special case handling for NaN and Infinity.\n+          val value = parser.getText\n+          val lowerCaseValue = value.toLowerCase\n+          if (lowerCaseValue.equals(\"nan\") ||\n+            lowerCaseValue.equals(\"infinity\") ||\n+            lowerCaseValue.equals(\"-infinity\") ||\n+            lowerCaseValue.equals(\"inf\") ||\n+            lowerCaseValue.equals(\"-inf\")) {\n+            value.toFloat\n+          } else {\n+            throw new SparkSQLJsonProcessingException(s\"Cannot parse $value as FloatType.\")\n+          }\n+      }\n+\n+    case DoubleType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT =>\n+          parser.getDoubleValue\n+\n+        case VALUE_STRING =>\n+          // Special case handling for NaN and Infinity.\n+          val value = parser.getText\n+          val lowerCaseValue = value.toLowerCase\n+          if (lowerCaseValue.equals(\"nan\") ||\n+            lowerCaseValue.equals(\"infinity\") ||\n+            lowerCaseValue.equals(\"-infinity\") ||\n+            lowerCaseValue.equals(\"inf\") ||\n+            lowerCaseValue.equals(\"-inf\")) {\n+            value.toDouble\n+          } else {\n+            throw new SparkSQLJsonProcessingException(s\"Cannot parse $value as DoubleType.\")\n+          }\n+      }\n+\n+    case StringType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_STRING =>\n+          UTF8String.fromString(parser.getText)\n+\n+        case _ =>\n+          // Note that it always tries to convert the data as string without the case of failure.\n+          val writer = new ByteArrayOutputStream()\n+          Utils.tryWithResource(factory.createGenerator(writer, JsonEncoding.UTF8)) {\n+            generator => generator.copyCurrentStructure(parser)\n+          }\n+          UTF8String.fromBytes(writer.toByteArray)\n+      }\n+\n+    case TimestampType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_STRING =>\n+          // This one will lose microseconds parts.\n+          // See https://issues.apache.org/jira/browse/SPARK-10681.\n+          DateTimeUtils.stringToTime(parser.getText).getTime * 1000L\n \n-      case (VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT, dt: DecimalType) =>\n-        Decimal(parser.getDecimalValue, dt.precision, dt.scale)\n+        case VALUE_NUMBER_INT =>\n+          parser.getLongValue * 1000000L\n+      }\n \n-      case (VALUE_NUMBER_INT, ByteType) =>\n-        parser.getByteValue\n+    case DateType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_STRING =>\n+          val stringValue = parser.getText\n+          if (stringValue.contains(\"-\")) {\n+            // The format of this string will probably be \"yyyy-mm-dd\".\n+            DateTimeUtils.millisToDays(DateTimeUtils.stringToTime(parser.getText).getTime)\n+          } else {\n+            // In Spark 1.5.0, we store the data as number of days since epoch in string.\n+            // So, we just convert it to Int.\n+            stringValue.toInt\n+          }\n+      }\n \n-      case (VALUE_NUMBER_INT, ShortType) =>\n-        parser.getShortValue\n+    case BinaryType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case VALUE_STRING => parser.getBinaryValue\n+      }\n \n-      case (VALUE_NUMBER_INT, IntegerType) =>\n-        parser.getIntValue\n+    case dt: DecimalType =>\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case (VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT) =>\n+          Decimal(parser.getDecimalValue, dt.precision, dt.scale)\n+      }\n \n-      case (VALUE_NUMBER_INT, LongType) =>\n-        parser.getLongValue\n+    case st: StructType =>\n+      val fieldConverters = st.map(_.dataType).map(makeConverter)\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case START_OBJECT => convertObject(parser, st, fieldConverters)\n+      }\n \n-      case (VALUE_TRUE, BooleanType) =>\n-        true\n+    case at: ArrayType =>\n+      val elementConverter = makeConverter(at.elementType)\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case START_ARRAY => convertArray(parser, elementConverter)\n+      }\n \n-      case (VALUE_FALSE, BooleanType) =>\n-        false\n+    case mt: MapType =>\n+      val valueConverter = makeConverter(mt.valueType)\n+      (parser: JsonParser) => parseJsonToken(parser, dataType) {\n+        case START_OBJECT => convertMap(parser, valueConverter)\n+      }\n \n-      case (START_OBJECT, st: StructType) =>\n-        convertObject(factory, parser, st)\n+    case udt: UserDefinedType[_] =>\n+      makeConverter(udt.sqlType)\n \n-      case (START_ARRAY, ArrayType(st, _)) =>\n-        convertArray(factory, parser, st)\n+    case _ =>\n+      (parser: JsonParser) =>\n+        // Here, we pass empty `PartialFunction` so that this case can be\n+        // handled as a failed conversion. It will throw an exception as\n+        // long as the value is not null.\n+        parseJsonToken(parser, dataType)(PartialFunction.empty[JsonToken, Any])\n+  }\n \n-      case (START_OBJECT, MapType(StringType, kt, _)) =>\n-        convertMap(factory, parser, kt)\n+  /**\n+   * This handles nulls ahead before trying to check the tokens, and applies the conversion\n+   * function and then checks failed the conversion afterward if the `f` fails to convert\n+   * the value.\n+   *\n+   * In more details, it checks `FIELD_NAME` if exists and then skip. If this is called after\n+   * `START_OBJECT`, then, the next token can be `FIELD_NAME`. Since the names are kept in\n+   * `JacksonParser.convertObject`, this `FIELD_NAME` token can be skipped as below. When this\n+   * is called after `START_ARRAY`, the tokens are consecutive tokens for values without\n+   * `FIELD_NAME` until `END_ARRAY`. In this case, we don't have to skip.","path":"sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala","position":376,"original_position":376,"commit_id":"a8b6a04aa753773aaf7d5e582cc4657f69bce9f9","original_commit_id":"a8b6a04aa753773aaf7d5e582cc4657f69bce9f9","user":{"login":"cloud-fan","id":3182036,"avatar_url":"https://avatars.githubusercontent.com/u/3182036?v=3","gravatar_id":"","url":"https://api.github.com/users/cloud-fan","html_url":"https://github.com/cloud-fan","followers_url":"https://api.github.com/users/cloud-fan/followers","following_url":"https://api.github.com/users/cloud-fan/following{/other_user}","gists_url":"https://api.github.com/users/cloud-fan/gists{/gist_id}","starred_url":"https://api.github.com/users/cloud-fan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloud-fan/subscriptions","organizations_url":"https://api.github.com/users/cloud-fan/orgs","repos_url":"https://api.github.com/users/cloud-fan/repos","events_url":"https://api.github.com/users/cloud-fan/events{/privacy}","received_events_url":"https://api.github.com/users/cloud-fan/received_events","type":"User","site_admin":false},"body":"it's good to know this, but do we really need to put it in comments? It seems not that helpful for people to understand the code. We just need some comments `case FIELD_NAME =>` to explain why we need to skip, i.e. there are useless `FIELD_NAME`s between `START_OBJECT` and `END_OBJECT` tokens.","created_at":"2016-08-11T09:43:03Z","updated_at":"2016-08-11T09:43:03Z","html_url":"https://github.com/apache/spark/pull/14102#discussion_r74393971","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/14102","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/74393971"},"html":{"href":"https://github.com/apache/spark/pull/14102#discussion_r74393971"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/14102"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/14102","id":76689413,"html_url":"https://github.com/apache/spark/pull/14102","diff_url":"https://github.com/apache/spark/pull/14102.diff","patch_url":"https://github.com/apache/spark/pull/14102.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/14102","number":14102,"state":"open","locked":false,"title":"[SPARK-16434][SQL] Avoid per-record type dispatch in JSON when reading","user":{"login":"HyukjinKwon","id":6477701,"avatar_url":"https://avatars.githubusercontent.com/u/6477701?v=3","gravatar_id":"","url":"https://api.github.com/users/HyukjinKwon","html_url":"https://github.com/HyukjinKwon","followers_url":"https://api.github.com/users/HyukjinKwon/followers","following_url":"https://api.github.com/users/HyukjinKwon/following{/other_user}","gists_url":"https://api.github.com/users/HyukjinKwon/gists{/gist_id}","starred_url":"https://api.github.com/users/HyukjinKwon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HyukjinKwon/subscriptions","organizations_url":"https://api.github.com/users/HyukjinKwon/orgs","repos_url":"https://api.github.com/users/HyukjinKwon/repos","events_url":"https://api.github.com/users/HyukjinKwon/events{/privacy}","received_events_url":"https://api.github.com/users/HyukjinKwon/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nCurrently, `JacksonParser.parse` is doing type-based dispatch for each row to convert the tokens to appropriate values for Spark.\r\nIt might not have to be done like this because the schema is already kept.\r\n\r\nSo, appropriate converters can be created first according to the schema once, and then apply them to each row. \r\n\r\nThis PR corrects `JacksonParser` so that it creates all converters for the schema once and then applies them to each row rather than type dispatching for every row.\r\n\r\nBenchmark was proceeded with the codes below:\r\n\r\n#### Parser tests\r\n\r\n**Before**\r\n\r\n```scala\r\ntest(\"Benchmark for JSON converter\") {\r\n  val N = 500 << 8\r\n  val row =\r\n    \"\"\"{\"struct\":{\"field1\": true, \"field2\": 92233720368547758070},\r\n    \"structWithArrayFields\":{\"field1\":[4, 5, 6], \"field2\":[\"str1\", \"str2\"]},\r\n    \"arrayOfString\":[\"str1\", \"str2\"],\r\n    \"arrayOfInteger\":[1, 2147483647, -2147483648],\r\n    \"arrayOfLong\":[21474836470, 9223372036854775807, -9223372036854775808],\r\n    \"arrayOfBigInteger\":[922337203685477580700, -922337203685477580800],\r\n    \"arrayOfDouble\":[1.2, 1.7976931348623157E308, 4.9E-324, 2.2250738585072014E-308],\r\n    \"arrayOfBoolean\":[true, false, true],\r\n    \"arrayOfNull\":[null, null, null, null],\r\n    \"arrayOfStruct\":[{\"field1\": true, \"field2\": \"str1\"}, {\"field1\": false}, {\"field3\": null}],\r\n    \"arrayOfArray1\":[[1, 2, 3], [\"str1\", \"str2\"]],\r\n    \"arrayOfArray2\":[[1, 2, 3], [1.1, 2.1, 3.1]]\r\n   }\"\"\"\r\n  val data = List.fill(N)(row)\r\n  val dummyOption = new JSONOptions(Map.empty[String, String])\r\n  val schema =\r\n    InferSchema.infer(spark.sparkContext.parallelize(Seq(row)), \"\", dummyOption)\r\n  val factory = new JsonFactory()\r\n\r\n  val benchmark = new Benchmark(\"JSON converter\", N)\r\n  benchmark.addCase(\"convert JSON file\", 10) { _ =>\r\n    data.foreach { input =>\r\n      val parser = factory.createParser(input)\r\n      parser.nextToken()\r\n      JacksonParser.convertRootField(factory, parser, schema)\r\n    }\r\n  }\r\n  benchmark.run()\r\n}\r\n```\r\n\r\n```\r\nJSON converter:                          Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative\r\n------------------------------------------------------------------------------------------------\r\nconvert JSON file                             1697 / 1807          0.1       13256.9       1.0X\r\n```\r\n\r\n\r\n**After**\r\n\r\n```scala\r\ntest(\"Benchmark for JSON converter\") {\r\n  val N = 500 << 8\r\n  val row =\r\n    \"\"\"{\"struct\":{\"field1\": true, \"field2\": 92233720368547758070},\r\n    \"structWithArrayFields\":{\"field1\":[4, 5, 6], \"field2\":[\"str1\", \"str2\"]},\r\n    \"arrayOfString\":[\"str1\", \"str2\"],\r\n    \"arrayOfInteger\":[1, 2147483647, -2147483648],\r\n    \"arrayOfLong\":[21474836470, 9223372036854775807, -9223372036854775808],\r\n    \"arrayOfBigInteger\":[922337203685477580700, -922337203685477580800],\r\n    \"arrayOfDouble\":[1.2, 1.7976931348623157E308, 4.9E-324, 2.2250738585072014E-308],\r\n    \"arrayOfBoolean\":[true, false, true],\r\n    \"arrayOfNull\":[null, null, null, null],\r\n    \"arrayOfStruct\":[{\"field1\": true, \"field2\": \"str1\"}, {\"field1\": false}, {\"field3\": null}],\r\n    \"arrayOfArray1\":[[1, 2, 3], [\"str1\", \"str2\"]],\r\n    \"arrayOfArray2\":[[1, 2, 3], [1.1, 2.1, 3.1]]\r\n   }\"\"\"\r\n  val data = List.fill(N)(row)\r\n  val dummyOption = new JSONOptions(Map.empty[String, String], new SQLConf())\r\n  val schema =\r\n    InferSchema.infer(spark.sparkContext.parallelize(Seq(row)), dummyOption)\r\n\r\n  val benchmark = new Benchmark(\"JSON converter\", N)\r\n  benchmark.addCase(\"convert JSON file\", 10) { _ =>\r\n    val parser = new JacksonParser(schema, dummyOption)\r\n    data.foreach { input =>\r\n      parser.parse(input)\r\n    }\r\n  }\r\n  benchmark.run()\r\n}\r\n```\r\n\r\n```\r\nJSON converter:                          Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative\r\n------------------------------------------------------------------------------------------------\r\nconvert JSON file                             1401 / 1461          0.1       10947.4       1.0X\r\n```\r\n\r\nIt seems parsing time is improved by roughly ~20%\r\n\r\n\r\n#### End-to-End test\r\n\r\n```scala\r\ntest(\"Benchmark for JSON reader\") {\r\n  val N = 500 << 8\r\n  val row =\r\n    \"\"\"{\"struct\":{\"field1\": true, \"field2\": 92233720368547758070},\r\n    \"structWithArrayFields\":{\"field1\":[4, 5, 6], \"field2\":[\"str1\", \"str2\"]},\r\n    \"arrayOfString\":[\"str1\", \"str2\"],\r\n    \"arrayOfInteger\":[1, 2147483647, -2147483648],\r\n    \"arrayOfLong\":[21474836470, 9223372036854775807, -9223372036854775808],\r\n    \"arrayOfBigInteger\":[922337203685477580700, -922337203685477580800],\r\n    \"arrayOfDouble\":[1.2, 1.7976931348623157E308, 4.9E-324, 2.2250738585072014E-308],\r\n    \"arrayOfBoolean\":[true, false, true],\r\n    \"arrayOfNull\":[null, null, null, null],\r\n    \"arrayOfStruct\":[{\"field1\": true, \"field2\": \"str1\"}, {\"field1\": false}, {\"field3\": null}],\r\n    \"arrayOfArray1\":[[1, 2, 3], [\"str1\", \"str2\"]],\r\n    \"arrayOfArray2\":[[1, 2, 3], [1.1, 2.1, 3.1]]\r\n   }\"\"\"\r\n  val df = spark.sqlContext.read.json(spark.sparkContext.parallelize(List.fill(N)(row)))\r\n  withTempPath { path =>\r\n    df.write.format(\"json\").save(path.getCanonicalPath)\r\n\r\n    val benchmark = new Benchmark(\"JSON reader\", N)\r\n    benchmark.addCase(\"reading JSON file\", 10) { _ =>\r\n      spark.read.format(\"json\").load(path.getCanonicalPath).collect()\r\n    }\r\n    benchmark.run()\r\n  }\r\n}\r\n```\r\n\r\n**Before**\r\n\r\n```\r\nJSON reader:                             Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative\r\n------------------------------------------------------------------------------------------------\r\nreading JSON file                             6485 / 6924          0.0       50665.0       1.0X\r\n```\r\n\r\n**After**\r\n\r\n```\r\nJSON reader:                             Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)   Relative\r\n------------------------------------------------------------------------------------------------\r\nreading JSON file                             6350 / 6529          0.0       49609.3       1.0X\r\n```\r\n\r\n\r\n## How was this patch tested?\r\n\r\nExisting test cases should cover this.\r\n","created_at":"2016-07-08T01:15:14Z","updated_at":"2016-08-11T09:43:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f8acc74c12234af3ed917e0f17ef2febf9a064c1","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/14102/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/14102/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/14102/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/a8b6a04aa753773aaf7d5e582cc4657f69bce9f9","head":{"label":"HyukjinKwon:SPARK-16434","ref":"SPARK-16434","sha":"a8b6a04aa753773aaf7d5e582cc4657f69bce9f9","user":{"login":"HyukjinKwon","id":6477701,"avatar_url":"https://avatars.githubusercontent.com/u/6477701?v=3","gravatar_id":"","url":"https://api.github.com/users/HyukjinKwon","html_url":"https://github.com/HyukjinKwon","followers_url":"https://api.github.com/users/HyukjinKwon/followers","following_url":"https://api.github.com/users/HyukjinKwon/following{/other_user}","gists_url":"https://api.github.com/users/HyukjinKwon/gists{/gist_id}","starred_url":"https://api.github.com/users/HyukjinKwon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HyukjinKwon/subscriptions","organizations_url":"https://api.github.com/users/HyukjinKwon/orgs","repos_url":"https://api.github.com/users/HyukjinKwon/repos","events_url":"https://api.github.com/users/HyukjinKwon/events{/privacy}","received_events_url":"https://api.github.com/users/HyukjinKwon/received_events","type":"User","site_admin":false},"repo":{"id":42300713,"name":"spark","full_name":"HyukjinKwon/spark","owner":{"login":"HyukjinKwon","id":6477701,"avatar_url":"https://avatars.githubusercontent.com/u/6477701?v=3","gravatar_id":"","url":"https://api.github.com/users/HyukjinKwon","html_url":"https://github.com/HyukjinKwon","followers_url":"https://api.github.com/users/HyukjinKwon/followers","following_url":"https://api.github.com/users/HyukjinKwon/following{/other_user}","gists_url":"https://api.github.com/users/HyukjinKwon/gists{/gist_id}","starred_url":"https://api.github.com/users/HyukjinKwon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HyukjinKwon/subscriptions","organizations_url":"https://api.github.com/users/HyukjinKwon/orgs","repos_url":"https://api.github.com/users/HyukjinKwon/repos","events_url":"https://api.github.com/users/HyukjinKwon/events{/privacy}","received_events_url":"https://api.github.com/users/HyukjinKwon/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/HyukjinKwon/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/HyukjinKwon/spark","forks_url":"https://api.github.com/repos/HyukjinKwon/spark/forks","keys_url":"https://api.github.com/repos/HyukjinKwon/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/HyukjinKwon/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/HyukjinKwon/spark/teams","hooks_url":"https://api.github.com/repos/HyukjinKwon/spark/hooks","issue_events_url":"https://api.github.com/repos/HyukjinKwon/spark/issues/events{/number}","events_url":"https://api.github.com/repos/HyukjinKwon/spark/events","assignees_url":"https://api.github.com/repos/HyukjinKwon/spark/assignees{/user}","branches_url":"https://api.github.com/repos/HyukjinKwon/spark/branches{/branch}","tags_url":"https://api.github.com/repos/HyukjinKwon/spark/tags","blobs_url":"https://api.github.com/repos/HyukjinKwon/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/HyukjinKwon/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/HyukjinKwon/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/HyukjinKwon/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/HyukjinKwon/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/HyukjinKwon/spark/languages","stargazers_url":"https://api.github.com/repos/HyukjinKwon/spark/stargazers","contributors_url":"https://api.github.com/repos/HyukjinKwon/spark/contributors","subscribers_url":"https://api.github.com/repos/HyukjinKwon/spark/subscribers","subscription_url":"https://api.github.com/repos/HyukjinKwon/spark/subscription","commits_url":"https://api.github.com/repos/HyukjinKwon/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/HyukjinKwon/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/HyukjinKwon/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/HyukjinKwon/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/HyukjinKwon/spark/contents/{+path}","compare_url":"https://api.github.com/repos/HyukjinKwon/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/HyukjinKwon/spark/merges","archive_url":"https://api.github.com/repos/HyukjinKwon/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/HyukjinKwon/spark/downloads","issues_url":"https://api.github.com/repos/HyukjinKwon/spark/issues{/number}","pulls_url":"https://api.github.com/repos/HyukjinKwon/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/HyukjinKwon/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/HyukjinKwon/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/HyukjinKwon/spark/labels{/name}","releases_url":"https://api.github.com/repos/HyukjinKwon/spark/releases{/id}","deployments_url":"https://api.github.com/repos/HyukjinKwon/spark/deployments","created_at":"2015-09-11T09:47:43Z","updated_at":"2016-04-11T01:30:54Z","pushed_at":"2016-08-11T08:40:02Z","git_url":"git://github.com/HyukjinKwon/spark.git","ssh_url":"git@github.com:HyukjinKwon/spark.git","clone_url":"https://github.com/HyukjinKwon/spark.git","svn_url":"https://github.com/HyukjinKwon/spark","homepage":null,"size":176233,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"41a7dbdd34d2641d42eb00828f16285089356aa9","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2016-08-11T09:37:54Z","pushed_at":"2016-08-11T08:52:02Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":510165,"stargazers_count":9625,"watchers_count":9625,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":8952,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":457,"forks":8952,"open_issues":457,"watchers":9625,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/14102"},"html":{"href":"https://github.com/apache/spark/pull/14102"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/14102"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/14102/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/14102/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/14102/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/a8b6a04aa753773aaf7d5e582cc4657f69bce9f9"}}}}
{"actor":{"display_login":"cloud-fan"}}