{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/302306932","pull_request_review_id":260395767,"id":302306932,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwMjMwNjkzMg==","diff_hunk":"@@ -445,197 +529,68 @@ private[kafka010] case class InternalKafkaConsumer(\n    * Throw an exception or log a warning as per `failOnDataLoss`.\n    */\n   private def reportDataLoss(\n+      topicPartition: TopicPartition,\n+      groupId: String,\n       failOnDataLoss: Boolean,\n       message: String,\n       cause: Throwable = null): Unit = {\n-    val finalMessage = s\"$message ${additionalMessage(failOnDataLoss)}\"\n+    val finalMessage = s\"$message ${additionalMessage(topicPartition, groupId, failOnDataLoss)}\"\n     reportDataLoss0(failOnDataLoss, finalMessage, cause)\n   }\n \n-  def close(): Unit = consumer.close()\n-\n-  private def seek(offset: Long): Unit = {\n-    logDebug(s\"Seeking to $groupId $topicPartition $offset\")\n-    consumer.seek(topicPartition, offset)\n-  }\n-\n-  /**\n-   * Poll messages from Kafka starting from `offset` and update `fetchedData`. `fetchedData` may be\n-   * empty if the Kafka consumer fetches some messages but all of them are not visible messages\n-   * (either transaction messages, or aborted messages when `isolation.level` is `read_committed`).\n-   *\n-   * @throws OffsetOutOfRangeException if `offset` is out of range.\n-   * @throws TimeoutException if the consumer position is not changed after polling. It means the\n-   *                          consumer polls nothing before timeout.\n-   */\n-  private def fetchData(offset: Long, pollTimeoutMs: Long): Unit = {\n-    // Seek to the offset because we may call seekToBeginning or seekToEnd before this.\n-    seek(offset)\n-    val p = consumer.poll(pollTimeoutMs)\n-    val r = p.records(topicPartition)\n-    logDebug(s\"Polled $groupId ${p.partitions()}  ${r.size}\")\n-    val offsetAfterPoll = consumer.position(topicPartition)\n-    logDebug(s\"Offset changed from $offset to $offsetAfterPoll after polling\")\n-    fetchedData.withNewPoll(r.listIterator, offsetAfterPoll)\n-    if (!fetchedData.hasNext) {\n-      // We cannot fetch anything after `poll`. Two possible cases:\n-      // - `offset` is out of range so that Kafka returns nothing. `OffsetOutOfRangeException` will\n-      //   be thrown.\n-      // - Cannot fetch any data before timeout. `TimeoutException` will be thrown.\n-      // - Fetched something but all of them are not invisible. This is a valid case and let the\n-      //   caller handles this.\n-      val range = getAvailableOffsetRange()\n-      if (offset < range.earliest || offset >= range.latest) {\n-        throw new OffsetOutOfRangeException(\n-          Map(topicPartition -> java.lang.Long.valueOf(offset)).asJava)\n-      } else if (offset == offsetAfterPoll) {\n-        throw new TimeoutException(\n-          s\"Cannot fetch record for offset $offset in $pollTimeoutMs milliseconds\")\n-      }\n-    }\n+  private def runUninterruptiblyIfPossible[T](body: => T): T = Thread.currentThread match {\n+    case ut: UninterruptibleThread =>\n+      ut.runUninterruptibly(body)\n+    case _ =>\n+      logWarning(\"KafkaDataConsumer is not running in UninterruptibleThread. \" +\n+        \"It may hang when KafkaDataConsumer's methods are interrupted because of KAFKA-1894\")\n+      body\n   }\n }\n \n-\n private[kafka010] object KafkaDataConsumer extends Logging {\n+  val UNKNOWN_OFFSET = -2L\n \n   case class AvailableOffsetRange(earliest: Long, latest: Long)\n \n-  private case class CachedKafkaDataConsumer(internalConsumer: InternalKafkaConsumer)\n-    extends KafkaDataConsumer {\n-    assert(internalConsumer.inUse) // make sure this has been set to true\n-    override def release(): Unit = { KafkaDataConsumer.release(internalConsumer) }\n-  }\n-\n-  private case class NonCachedKafkaDataConsumer(internalConsumer: InternalKafkaConsumer)\n-    extends KafkaDataConsumer {\n-    override def release(): Unit = { internalConsumer.close() }\n-  }\n-\n-  private case class CacheKey(groupId: String, topicPartition: TopicPartition) {\n+  case class CacheKey(groupId: String, topicPartition: TopicPartition) {\n     def this(topicPartition: TopicPartition, kafkaParams: ju.Map[String, Object]) =\n       this(kafkaParams.get(ConsumerConfig.GROUP_ID_CONFIG).asInstanceOf[String], topicPartition)\n   }\n \n-  // This cache has the following important properties.\n-  // - We make a best-effort attempt to maintain the max size of the cache as configured capacity.\n-  //   The capacity is not guaranteed to be maintained, especially when there are more active\n-  //   tasks simultaneously using consumers than the capacity.\n-  private lazy val cache = {\n-    val conf = SparkEnv.get.conf\n-    val capacity = conf.get(CONSUMER_CACHE_CAPACITY)\n-    new ju.LinkedHashMap[CacheKey, InternalKafkaConsumer](capacity, 0.75f, true) {\n-      override def removeEldestEntry(\n-        entry: ju.Map.Entry[CacheKey, InternalKafkaConsumer]): Boolean = {\n-\n-        // Try to remove the least-used entry if its currently not in use.\n-        //\n-        // If you cannot remove it, then the cache will keep growing. In the worst case,\n-        // the cache will grow to the max number of concurrent tasks that can run in the executor,\n-        // (that is, number of tasks slots) after which it will never reduce. This is unlikely to\n-        // be a serious problem because an executor with more than 64 (default) tasks slots is\n-        // likely running on a beefy machine that can handle a large number of simultaneously\n-        // active consumers.\n-\n-        if (!entry.getValue.inUse && this.size > capacity) {\n-          logWarning(\n-            s\"KafkaConsumer cache hitting max capacity of $capacity, \" +\n-              s\"removing consumer for ${entry.getKey}\")\n-          try {\n-            entry.getValue.close()\n-          } catch {\n-            case e: SparkException =>\n-              logError(s\"Error closing earliest Kafka consumer for ${entry.getKey}\", e)\n-          }\n-          true\n-        } else {\n-          false\n-        }\n-      }\n+  private val consumerPool = InternalKafkaConsumerPool.build\n+  private val fetchedDataPool = FetchedDataPool.build\n+\n+  ShutdownHookManager.addShutdownHook { () =>","path":"external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaDataConsumer.scala","position":770,"original_position":770,"commit_id":"40f10d8ec1c8ff9446c535a53e40c34db980aabe","original_commit_id":"40f10d8ec1c8ff9446c535a53e40c34db980aabe","user":{"login":"vanzin","id":1694083,"node_id":"MDQ6VXNlcjE2OTQwODM=","avatar_url":"https://avatars0.githubusercontent.com/u/1694083?v=4","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"body":"Hmm... probably doesn't matter much, but the use of shutdown hooks is sort of a \"code smell\".\r\n\r\ne.g. if you have two Spark applications running in the same JVM, one after another, the second one will see the cached bits from the first one.\r\n\r\nif possible it would be better to tie this to the lifecycle of the application itself (`SparkContext` instance), although I'm not sure how easy that is.","created_at":"2019-07-10T23:05:34Z","updated_at":"2019-07-10T23:11:25Z","html_url":"https://github.com/apache/spark/pull/22138#discussion_r302306932","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/22138","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/302306932"},"html":{"href":"https://github.com/apache/spark/pull/22138#discussion_r302306932"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/22138"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/22138","id":209278078,"node_id":"MDExOlB1bGxSZXF1ZXN0MjA5Mjc4MDc4","html_url":"https://github.com/apache/spark/pull/22138","diff_url":"https://github.com/apache/spark/pull/22138.diff","patch_url":"https://github.com/apache/spark/pull/22138.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/22138","number":22138,"state":"open","locked":false,"title":"[SPARK-25151][SS] Apply Apache Commons Pool to KafkaDataConsumer","user":{"login":"HeartSaVioR","id":1317309,"node_id":"MDQ6VXNlcjEzMTczMDk=","avatar_url":"https://avatars2.githubusercontent.com/u/1317309?v=4","gravatar_id":"","url":"https://api.github.com/users/HeartSaVioR","html_url":"https://github.com/HeartSaVioR","followers_url":"https://api.github.com/users/HeartSaVioR/followers","following_url":"https://api.github.com/users/HeartSaVioR/following{/other_user}","gists_url":"https://api.github.com/users/HeartSaVioR/gists{/gist_id}","starred_url":"https://api.github.com/users/HeartSaVioR/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HeartSaVioR/subscriptions","organizations_url":"https://api.github.com/users/HeartSaVioR/orgs","repos_url":"https://api.github.com/users/HeartSaVioR/repos","events_url":"https://api.github.com/users/HeartSaVioR/events{/privacy}","received_events_url":"https://api.github.com/users/HeartSaVioR/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nThis patch does pooling for both kafka consumers as well as fetched data. The overall benefits of the patch are following:\r\n\r\n* Both pools support eviction on idle objects, which will help closing invalid idle objects which topic or partition are no longer be assigned to any tasks.\r\n* It also enables applying different policies on pool, which helps optimization of pooling for each pool.\r\n* We concerned about multiple tasks pointing same topic partition as well as same group id, and existing code can't handle this hence excess seek and fetch could happen. This patch properly handles the case.\r\n* It also makes the code always safe to leverage cache, hence no need to maintain reuseCache parameter.\r\n\r\nMoreover, pooling kafka consumers is implemented based on Apache Commons Pool, which also gives couple of benefits:\r\n\r\n* We can get rid of synchronization of KafkaDataConsumer object while acquiring and returning InternalKafkaConsumer.\r\n* We can extract the feature of object pool to outside of the class, so that the behaviors of the pool can be tested easily.\r\n* We can get various statistics for the object pool, and also be able to enable JMX for the pool.\r\n\r\nThis patch brings additional dependency, Apache Commons Pool 2.6.0 into `spark-sql-kafka-0-10` module.\r\n\r\n## How was this patch tested?\r\n\r\nExisting unit tests as well as new tests for object pool.\r\n\r\nAlso did some experiment regarding proving concurrent access of consumers for same topic partition.\r\n\r\n* Made change on both sides (master and patch) to log when creating Kafka consumer or fetching records from Kafka is happening.\r\n* branches\r\n  * master: https://github.com/HeartSaVioR/spark/tree/SPARK-25151-master-ref-debugging\r\n  * patch: https://github.com/HeartSaVioR/spark/tree/SPARK-25151-debugging\r\n* Test query (doing self-join)\r\n  * https://gist.github.com/HeartSaVioR/d831974c3f25c02846f4b15b8d232cc2\r\n* Ran query from spark-shell, with using `local[*]` to maximize the chance to have concurrent access\r\n* Collected the count of fetch requests on Kafka via command: `grep \"creating new Kafka consumer\" logfile | wc -l`\r\n* Collected the count of creating Kafka consumers via command: `grep \"fetching data from Kafka consumer\" logfile | wc -l`\r\n\r\nTopic and data distribution is follow:\r\n\r\n```\r\ntruck_speed_events_stream_spark_25151_v1:0:99440\r\ntruck_speed_events_stream_spark_25151_v1:1:99489\r\ntruck_speed_events_stream_spark_25151_v1:2:397759\r\ntruck_speed_events_stream_spark_25151_v1:3:198917\r\ntruck_speed_events_stream_spark_25151_v1:4:99484\r\ntruck_speed_events_stream_spark_25151_v1:5:497320\r\ntruck_speed_events_stream_spark_25151_v1:6:99430\r\ntruck_speed_events_stream_spark_25151_v1:7:397887\r\ntruck_speed_events_stream_spark_25151_v1:8:397813\r\ntruck_speed_events_stream_spark_25151_v1:9:0\r\n```\r\n\r\nThe experiment only used smallest 4 partitions (0, 1, 4, 6) from these partitions to finish the query earlier.\r\n\r\nThe result of experiment is below:\r\n\r\nbranch | create Kafka consumer | fetch request\r\n-- | -- | --\r\nmaster | 1986 | 2837\r\npatch | 8 | 1706\r\n\r\n","created_at":"2018-08-17T23:20:27Z","updated_at":"2019-07-10T23:11:24Z","closed_at":null,"merged_at":null,"merge_commit_sha":"62569524ca361b41c07c7221fac62f08a1bf6d4c","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":1406587328,"node_id":"MDU6TGFiZWwxNDA2NTg3MzI4","url":"https://api.github.com/repos/apache/spark/labels/STRUCTURED%20STREAMING","name":"STRUCTURED STREAMING","color":"ededed","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/22138/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/22138/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/22138/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/40f10d8ec1c8ff9446c535a53e40c34db980aabe","head":{"label":"HeartSaVioR:SPARK-25151","ref":"SPARK-25151","sha":"40f10d8ec1c8ff9446c535a53e40c34db980aabe","user":{"login":"HeartSaVioR","id":1317309,"node_id":"MDQ6VXNlcjEzMTczMDk=","avatar_url":"https://avatars2.githubusercontent.com/u/1317309?v=4","gravatar_id":"","url":"https://api.github.com/users/HeartSaVioR","html_url":"https://github.com/HeartSaVioR","followers_url":"https://api.github.com/users/HeartSaVioR/followers","following_url":"https://api.github.com/users/HeartSaVioR/following{/other_user}","gists_url":"https://api.github.com/users/HeartSaVioR/gists{/gist_id}","starred_url":"https://api.github.com/users/HeartSaVioR/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HeartSaVioR/subscriptions","organizations_url":"https://api.github.com/users/HeartSaVioR/orgs","repos_url":"https://api.github.com/users/HeartSaVioR/repos","events_url":"https://api.github.com/users/HeartSaVioR/events{/privacy}","received_events_url":"https://api.github.com/users/HeartSaVioR/received_events","type":"User","site_admin":false},"repo":{"id":27994296,"node_id":"MDEwOlJlcG9zaXRvcnkyNzk5NDI5Ng==","name":"spark","full_name":"HeartSaVioR/spark","private":false,"owner":{"login":"HeartSaVioR","id":1317309,"node_id":"MDQ6VXNlcjEzMTczMDk=","avatar_url":"https://avatars2.githubusercontent.com/u/1317309?v=4","gravatar_id":"","url":"https://api.github.com/users/HeartSaVioR","html_url":"https://github.com/HeartSaVioR","followers_url":"https://api.github.com/users/HeartSaVioR/followers","following_url":"https://api.github.com/users/HeartSaVioR/following{/other_user}","gists_url":"https://api.github.com/users/HeartSaVioR/gists{/gist_id}","starred_url":"https://api.github.com/users/HeartSaVioR/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HeartSaVioR/subscriptions","organizations_url":"https://api.github.com/users/HeartSaVioR/orgs","repos_url":"https://api.github.com/users/HeartSaVioR/repos","events_url":"https://api.github.com/users/HeartSaVioR/events{/privacy}","received_events_url":"https://api.github.com/users/HeartSaVioR/received_events","type":"User","site_admin":false},"html_url":"https://github.com/HeartSaVioR/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/HeartSaVioR/spark","forks_url":"https://api.github.com/repos/HeartSaVioR/spark/forks","keys_url":"https://api.github.com/repos/HeartSaVioR/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/HeartSaVioR/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/HeartSaVioR/spark/teams","hooks_url":"https://api.github.com/repos/HeartSaVioR/spark/hooks","issue_events_url":"https://api.github.com/repos/HeartSaVioR/spark/issues/events{/number}","events_url":"https://api.github.com/repos/HeartSaVioR/spark/events","assignees_url":"https://api.github.com/repos/HeartSaVioR/spark/assignees{/user}","branches_url":"https://api.github.com/repos/HeartSaVioR/spark/branches{/branch}","tags_url":"https://api.github.com/repos/HeartSaVioR/spark/tags","blobs_url":"https://api.github.com/repos/HeartSaVioR/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/HeartSaVioR/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/HeartSaVioR/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/HeartSaVioR/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/HeartSaVioR/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/HeartSaVioR/spark/languages","stargazers_url":"https://api.github.com/repos/HeartSaVioR/spark/stargazers","contributors_url":"https://api.github.com/repos/HeartSaVioR/spark/contributors","subscribers_url":"https://api.github.com/repos/HeartSaVioR/spark/subscribers","subscription_url":"https://api.github.com/repos/HeartSaVioR/spark/subscription","commits_url":"https://api.github.com/repos/HeartSaVioR/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/HeartSaVioR/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/HeartSaVioR/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/HeartSaVioR/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/HeartSaVioR/spark/contents/{+path}","compare_url":"https://api.github.com/repos/HeartSaVioR/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/HeartSaVioR/spark/merges","archive_url":"https://api.github.com/repos/HeartSaVioR/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/HeartSaVioR/spark/downloads","issues_url":"https://api.github.com/repos/HeartSaVioR/spark/issues{/number}","pulls_url":"https://api.github.com/repos/HeartSaVioR/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/HeartSaVioR/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/HeartSaVioR/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/HeartSaVioR/spark/labels{/name}","releases_url":"https://api.github.com/repos/HeartSaVioR/spark/releases{/id}","deployments_url":"https://api.github.com/repos/HeartSaVioR/spark/deployments","created_at":"2014-12-14T12:56:04Z","updated_at":"2014-12-14T12:56:20Z","pushed_at":"2019-07-10T18:52:02Z","git_url":"git://github.com/HeartSaVioR/spark.git","ssh_url":"git@github.com:HeartSaVioR/spark.git","clone_url":"https://github.com/HeartSaVioR/spark.git","svn_url":"https://github.com/HeartSaVioR/spark","homepage":null,"size":279447,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"f1486742fa032d838c0730fcb968e42ac145acc8","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"node_id":"MDEwOlJlcG9zaXRvcnkxNzE2NTY1OA==","name":"spark","full_name":"apache/spark","private":false,"owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/apache/spark","description":"Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2019-07-10T20:16:35Z","pushed_at":"2019-07-10T22:48:58Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":"","size":300208,"stargazers_count":22561,"watchers_count":22561,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":19449,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":531,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":19449,"open_issues":531,"watchers":22561,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/22138"},"html":{"href":"https://github.com/apache/spark/pull/22138"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/22138"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/22138/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/22138/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/22138/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/40f10d8ec1c8ff9446c535a53e40c34db980aabe"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"vanzin"}}