{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/207743563","pull_request_review_id":143418142,"id":207743563,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwNzc0MzU2Mw==","diff_hunk":"@@ -0,0 +1,230 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark\n+\n+import java.util.{Timer, TimerTask}\n+import java.util.concurrent.ConcurrentHashMap\n+import java.util.function.Consumer\n+\n+import scala.collection.mutable.ArrayBuffer\n+\n+import org.apache.spark.internal.Logging\n+import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}\n+import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}\n+\n+/**\n+ * For each barrier stage attempt, only at most one barrier() call can be active at any time, thus\n+ * we can use (stageId, stageAttemptId) to identify the stage attempt where the barrier() call is\n+ * from.\n+ */\n+private case class ContextBarrierId(stageId: Int, stageAttemptId: Int) {\n+  override def toString: String = s\"Stage $stageId (Attempt $stageAttemptId)\"\n+}\n+\n+/**\n+ * A coordinator that handles all global sync requests from BarrierTaskContext. Each global sync\n+ * request is generated by `BarrierTaskContext.barrier()`, and identified by\n+ * stageId + stageAttemptId + barrierEpoch. Reply all the blocking global sync requests upon\n+ * all the requests for a group of `barrier()` calls are received. If the coordinator is unable to\n+ * collect enough global sync requests within a configured time, fail all the requests and return\n+ * an Exception with timeout message.\n+ */\n+private[spark] class BarrierCoordinator(\n+    timeoutInSecs: Long,\n+    listenerBus: LiveListenerBus,\n+    override val rpcEnv: RpcEnv) extends ThreadSafeRpcEndpoint with Logging {\n+\n+  private val timer = new Timer(\"BarrierCoordinator barrier epoch increment timer\")\n+\n+  // Listen to StageCompleted event, clear corresponding ContextBarrierState.\n+  private val listener = new SparkListener {\n+    override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = {\n+      val stageInfo = stageCompleted.stageInfo\n+      val barrierId = ContextBarrierId(stageInfo.stageId, stageInfo.attemptNumber)\n+      // Clear ContextBarrierState from a finished stage attempt.\n+      cleanupBarrierStage(barrierId)\n+    }\n+  }\n+\n+  // Record all active stage attempts that make barrier() call(s), and the corresponding internal\n+  // state.\n+  private val states = new ConcurrentHashMap[ContextBarrierId, ContextBarrierState]\n+\n+  override def onStart(): Unit = {\n+    super.onStart()\n+    listenerBus.addToStatusQueue(listener)\n+  }\n+\n+  override def onStop(): Unit = {\n+    try {\n+      states.forEachValue(1, clearStateConsumer)\n+      states.clear()\n+      listenerBus.removeListener(listener)\n+    } finally {\n+      super.onStop()\n+    }\n+  }\n+\n+  /**\n+   * Provide the current state of a barrier() call. A state is created when a new stage attempt\n+   * sends out a barrier() call, and recycled on stage completed.\n+   *\n+   * @param barrierId Identifier of the barrier stage that make a barrier() call.\n+   * @param numTasks Number of tasks of the barrier stage, all barrier() calls from the stage shall\n+   *                 collect `numTasks` requests to succeed.\n+   */\n+  private class ContextBarrierState(\n+      val barrierId: ContextBarrierId,\n+      val numTasks: Int) {\n+\n+    // There may be multiple barrier() calls from a barrier stage attempt, `barrierEpoch` is used\n+    // to identify each barrier() call. It shall get increased when a barrier() call succeed, or\n+    // reset when a barrier() call fail due to timeout.\n+    private var barrierEpoch: Int = 0\n+\n+    // An array of RPCCallContexts for barrier tasks that are waiting for reply of a barrier()\n+    // call.\n+    private val requesters: ArrayBuffer[RpcCallContext] = new ArrayBuffer[RpcCallContext](numTasks)\n+\n+    // A timer task that ensures we may timeout for a barrier() call.\n+    private var timerTask: TimerTask = null\n+\n+    // Init a TimerTask for a barrier() call.\n+    private def initTimerTask(): Unit = {\n+      timerTask = new TimerTask {\n+        override def run(): Unit = synchronized {\n+          // Timeout current barrier() call, fail all the sync requests.\n+          requesters.foreach(_.sendFailure(new SparkException(\"The coordinator didn't get all \" +\n+            s\"barrier sync requests for barrier epoch $barrierEpoch from $barrierId within \" +\n+            s\"$timeoutInSecs second(s).\")))\n+          cleanupBarrierStage(barrierId)\n+        }\n+      }\n+    }\n+\n+    // Cancel the current active TimerTask and release resources.\n+    private def cancelTimerTask(): Unit = {\n+      if (timerTask != null) {\n+        timerTask.cancel()\n+        timerTask = null\n+      }\n+    }\n+\n+    // Process the global sync request. The barrier() call succeed if collected enough requests\n+    // within a configured time, otherwise fail all the pending requests.\n+    def handleRequest(requester: RpcCallContext, request: RequestToSync): Unit = synchronized {\n+      val taskId = request.taskAttemptId\n+      val epoch = request.barrierEpoch\n+\n+      // Require the number of tasks is correctly set from the BarrierTaskContext.\n+      require(request.numTasks == numTasks, s\"Number of tasks of $barrierId is \" +\n+        s\"${request.numTasks} from Task $taskId, previously it was $numTasks.\")\n+\n+      // Check whether the epoch from the barrier tasks matches current barrierEpoch.\n+      logInfo(s\"Current barrier epoch for $barrierId is $barrierEpoch.\")\n+      if (epoch != barrierEpoch) {\n+        requester.sendFailure(new SparkException(s\"The request to sync of $barrierId with \" +\n+          s\"barrier epoch $barrierEpoch has already finished. Maybe task $taskId is not \" +\n+          \"properly killed.\"))\n+      } else {\n+        // If this is the first sync message received for a barrier() call, start timer to ensure\n+        // we may timeout for the sync.\n+        if (requesters.isEmpty) {\n+          initTimerTask()\n+          timer.schedule(timerTask, timeoutInSecs * 1000)\n+        }\n+        // Add the requester to array of RPCCallContexts pending for reply.\n+        requesters += requester\n+        logInfo(s\"Barrier sync epoch $barrierEpoch from $barrierId received update from Task \" +\n+          s\"$taskId, current progress: ${requesters.size}/$numTasks.\")\n+        if (maybeFinishAllRequesters(requesters, numTasks)) {\n+          // Finished current barrier() call successfully, clean up ContextBarrierState and\n+          // increase the barrier epoch.\n+          logInfo(s\"Barrier sync epoch $barrierEpoch from $barrierId received all updates from \" +\n+            s\"tasks, finished successfully.\")\n+          barrierEpoch += 1\n+          requesters.clear()\n+          cancelTimerTask()\n+        }\n+      }\n+    }\n+\n+    // Finish all the blocking barrier sync requests from a stage attempt successfully if we\n+    // have received all the sync requests.\n+    private def maybeFinishAllRequesters(\n+        requesters: ArrayBuffer[RpcCallContext],\n+        numTasks: Int): Boolean = {\n+      if (requesters.size == numTasks) {\n+        requesters.foreach(_.reply(()))\n+        true\n+      } else {\n+        false\n+      }\n+    }\n+\n+    // Cleanup the internal state of a barrier stage attempt.\n+    def clear(): Unit = synchronized {\n+      // The global sync fails so the stage is expected to retry another attempt, all sync\n+      // messages come from current stage attempt shall fail.\n+      barrierEpoch = -1\n+      requesters.clear()\n+      cancelTimerTask()\n+    }\n+  }\n+\n+  // Clean up the [[ContextBarrierState]] that correspond to a specific stage attempt.\n+  private def cleanupBarrierStage(barrierId: ContextBarrierId): Unit = {\n+    val barrierState = states.remove(barrierId)\n+    if (barrierState != null) {\n+      barrierState.clear()\n+    }\n+  }\n+\n+  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {\n+    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _) =>\n+      // Get or init the ContextBarrierState correspond to the stage attempt.\n+      val barrierId = ContextBarrierId(stageId, stageAttemptId)\n+      states.putIfAbsent(barrierId, new ContextBarrierState(barrierId, numTasks))\n+      val barrierState = states.get(barrierId)\n+\n+      barrierState.handleRequest(context, request)\n+  }\n+\n+  private val clearStateConsumer = new Consumer[ContextBarrierState] {\n+    override def accept(state: ContextBarrierState) = state.clear()\n+  }\n+}\n+\n+private[spark] sealed trait BarrierCoordinatorMessage extends Serializable\n+\n+/**\n+ * A global sync request message from BarrierTaskContext, by `barrier()` call. Each request is\n+ * identified by stageId + stageAttemptId + barrierEpoch.\n+ *\n+ * @param numTasks The number of global sync requests the BarrierCoordinator shall receive\n+ * @param stageId ID of current stage\n+ * @param stageAttemptId ID of current stage attempt\n+ * @param taskAttemptId Unique ID of current task\n+ * @param barrierEpoch ID of the `barrier()` call, a task may consists multiple `barrier()` calls.","path":"core/src/main/scala/org/apache/spark/BarrierCoordinator.scala","position":223,"original_position":223,"commit_id":"a8fa8db6a478009c9be6b9063fbddf30dcdaa316","original_commit_id":"a8fa8db6a478009c9be6b9063fbddf30dcdaa316","user":{"login":"kiszk","id":1315079,"node_id":"MDQ6VXNlcjEzMTUwNzk=","avatar_url":"https://avatars2.githubusercontent.com/u/1315079?v=4","gravatar_id":"","url":"https://api.github.com/users/kiszk","html_url":"https://github.com/kiszk","followers_url":"https://api.github.com/users/kiszk/followers","following_url":"https://api.github.com/users/kiszk/following{/other_user}","gists_url":"https://api.github.com/users/kiszk/gists{/gist_id}","starred_url":"https://api.github.com/users/kiszk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kiszk/subscriptions","organizations_url":"https://api.github.com/users/kiszk/orgs","repos_url":"https://api.github.com/users/kiszk/repos","events_url":"https://api.github.com/users/kiszk/events{/privacy}","received_events_url":"https://api.github.com/users/kiszk/received_events","type":"User","site_admin":false},"body":"nit: `consists` -> `consist`","created_at":"2018-08-05T17:41:42Z","updated_at":"2018-08-05T17:41:42Z","html_url":"https://github.com/apache/spark/pull/21898#discussion_r207743563","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/21898","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/207743563"},"html":{"href":"https://github.com/apache/spark/pull/21898#discussion_r207743563"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/21898"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/21898","id":204504131,"node_id":"MDExOlB1bGxSZXF1ZXN0MjA0NTA0MTMx","html_url":"https://github.com/apache/spark/pull/21898","diff_url":"https://github.com/apache/spark/pull/21898.diff","patch_url":"https://github.com/apache/spark/pull/21898.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/21898","number":21898,"state":"open","locked":false,"title":"[SPARK-24817][Core] Implement BarrierTaskContext.barrier()","user":{"login":"jiangxb1987","id":4784782,"node_id":"MDQ6VXNlcjQ3ODQ3ODI=","avatar_url":"https://avatars1.githubusercontent.com/u/4784782?v=4","gravatar_id":"","url":"https://api.github.com/users/jiangxb1987","html_url":"https://github.com/jiangxb1987","followers_url":"https://api.github.com/users/jiangxb1987/followers","following_url":"https://api.github.com/users/jiangxb1987/following{/other_user}","gists_url":"https://api.github.com/users/jiangxb1987/gists{/gist_id}","starred_url":"https://api.github.com/users/jiangxb1987/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jiangxb1987/subscriptions","organizations_url":"https://api.github.com/users/jiangxb1987/orgs","repos_url":"https://api.github.com/users/jiangxb1987/repos","events_url":"https://api.github.com/users/jiangxb1987/events{/privacy}","received_events_url":"https://api.github.com/users/jiangxb1987/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nImplement BarrierTaskContext.barrier(), to support global sync between all the tasks in a barrier stage.\r\nThe function set a global barrier and waits until all tasks in this stage hit this barrier. Similar to MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same stage have reached this routine. The global sync shall finish immediately once all tasks in the same barrier stage reaches the same barrier.\r\n\r\nThis PR implements BarrierTaskContext.barrier() based on netty-based RPC client, introduces new `BarrierCoordinator` and new `BarrierCoordinatorMessage`, and new config to handle timeout issue.\r\n\r\n## How was this patch tested?\r\nAdd `BarrierTaskContextSuite` to test `BarrierTaskContext.barrier()`","created_at":"2018-07-27T18:16:39Z","updated_at":"2018-08-05T17:41:42Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d1dce98925b8768e6d292eea3e09900fffa8e051","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/21898/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/21898/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/21898/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/a8fa8db6a478009c9be6b9063fbddf30dcdaa316","head":{"label":"jiangxb1987:taskcontext.barrier","ref":"taskcontext.barrier","sha":"a8fa8db6a478009c9be6b9063fbddf30dcdaa316","user":{"login":"jiangxb1987","id":4784782,"node_id":"MDQ6VXNlcjQ3ODQ3ODI=","avatar_url":"https://avatars1.githubusercontent.com/u/4784782?v=4","gravatar_id":"","url":"https://api.github.com/users/jiangxb1987","html_url":"https://github.com/jiangxb1987","followers_url":"https://api.github.com/users/jiangxb1987/followers","following_url":"https://api.github.com/users/jiangxb1987/following{/other_user}","gists_url":"https://api.github.com/users/jiangxb1987/gists{/gist_id}","starred_url":"https://api.github.com/users/jiangxb1987/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jiangxb1987/subscriptions","organizations_url":"https://api.github.com/users/jiangxb1987/orgs","repos_url":"https://api.github.com/users/jiangxb1987/repos","events_url":"https://api.github.com/users/jiangxb1987/events{/privacy}","received_events_url":"https://api.github.com/users/jiangxb1987/received_events","type":"User","site_admin":false},"repo":{"id":35214686,"node_id":"MDEwOlJlcG9zaXRvcnkzNTIxNDY4Ng==","name":"spark","full_name":"jiangxb1987/spark","owner":{"login":"jiangxb1987","id":4784782,"node_id":"MDQ6VXNlcjQ3ODQ3ODI=","avatar_url":"https://avatars1.githubusercontent.com/u/4784782?v=4","gravatar_id":"","url":"https://api.github.com/users/jiangxb1987","html_url":"https://github.com/jiangxb1987","followers_url":"https://api.github.com/users/jiangxb1987/followers","following_url":"https://api.github.com/users/jiangxb1987/following{/other_user}","gists_url":"https://api.github.com/users/jiangxb1987/gists{/gist_id}","starred_url":"https://api.github.com/users/jiangxb1987/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jiangxb1987/subscriptions","organizations_url":"https://api.github.com/users/jiangxb1987/orgs","repos_url":"https://api.github.com/users/jiangxb1987/repos","events_url":"https://api.github.com/users/jiangxb1987/events{/privacy}","received_events_url":"https://api.github.com/users/jiangxb1987/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jiangxb1987/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/jiangxb1987/spark","forks_url":"https://api.github.com/repos/jiangxb1987/spark/forks","keys_url":"https://api.github.com/repos/jiangxb1987/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jiangxb1987/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jiangxb1987/spark/teams","hooks_url":"https://api.github.com/repos/jiangxb1987/spark/hooks","issue_events_url":"https://api.github.com/repos/jiangxb1987/spark/issues/events{/number}","events_url":"https://api.github.com/repos/jiangxb1987/spark/events","assignees_url":"https://api.github.com/repos/jiangxb1987/spark/assignees{/user}","branches_url":"https://api.github.com/repos/jiangxb1987/spark/branches{/branch}","tags_url":"https://api.github.com/repos/jiangxb1987/spark/tags","blobs_url":"https://api.github.com/repos/jiangxb1987/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jiangxb1987/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jiangxb1987/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/jiangxb1987/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jiangxb1987/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/jiangxb1987/spark/languages","stargazers_url":"https://api.github.com/repos/jiangxb1987/spark/stargazers","contributors_url":"https://api.github.com/repos/jiangxb1987/spark/contributors","subscribers_url":"https://api.github.com/repos/jiangxb1987/spark/subscribers","subscription_url":"https://api.github.com/repos/jiangxb1987/spark/subscription","commits_url":"https://api.github.com/repos/jiangxb1987/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/jiangxb1987/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/jiangxb1987/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/jiangxb1987/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/jiangxb1987/spark/contents/{+path}","compare_url":"https://api.github.com/repos/jiangxb1987/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jiangxb1987/spark/merges","archive_url":"https://api.github.com/repos/jiangxb1987/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jiangxb1987/spark/downloads","issues_url":"https://api.github.com/repos/jiangxb1987/spark/issues{/number}","pulls_url":"https://api.github.com/repos/jiangxb1987/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/jiangxb1987/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/jiangxb1987/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jiangxb1987/spark/labels{/name}","releases_url":"https://api.github.com/repos/jiangxb1987/spark/releases{/id}","deployments_url":"https://api.github.com/repos/jiangxb1987/spark/deployments","created_at":"2015-05-07T10:48:35Z","updated_at":"2018-07-15T14:39:33Z","pushed_at":"2018-08-05T15:47:27Z","git_url":"git://github.com/jiangxb1987/spark.git","ssh_url":"git@github.com:jiangxb1987/spark.git","clone_url":"https://github.com/jiangxb1987/spark.git","svn_url":"https://github.com/jiangxb1987/spark","homepage":null,"size":267806,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"b3f2911eebeb418631ce296f68a7cc68083659cd","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"node_id":"MDEwOlJlcG9zaXRvcnkxNzE2NTY1OA==","name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2018-08-05T17:39:15Z","pushed_at":"2018-08-05T15:57:54Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":278564,"stargazers_count":18310,"watchers_count":18310,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":16586,"mirror_url":null,"archived":false,"open_issues_count":489,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":16586,"open_issues":489,"watchers":18310,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/21898"},"html":{"href":"https://github.com/apache/spark/pull/21898"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/21898"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/21898/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/21898/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/21898/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/a8fa8db6a478009c9be6b9063fbddf30dcdaa316"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"kiszk"}}