{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/190364971","pull_request_review_id":122702552,"id":190364971,"diff_hunk":"@@ -251,6 +261,233 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit\n     }\n   }\n \n+  /**\n+   * Test executor metrics update logging functionality. This checks that a\n+   * SparkListenerExecutorMetricsUpdate event is added to the Spark history\n+   * log if one of the executor metrics is larger than any previously\n+   * recorded value for the metric, per executor per stage. The task metrics\n+   * should not be added.\n+   */\n+  private def testExecutorMetricsUpdateEventLogging() {\n+    val conf = getLoggingConf(testDirPath, None)\n+    val logName = \"executorMetricsUpdated-test\"\n+    val eventLogger = new EventLoggingListener(logName, None, testDirPath.toUri(), conf)\n+    val listenerBus = new LiveListenerBus(conf)\n+\n+    // expected ExecutorMetricsUpdate, for the given stage id and executor id\n+    val expectedMetricsEvents: Map[(Int, String), SparkListenerExecutorMetricsUpdate] =\n+      Map(\n+        ((0, \"1\"),\n+          createExecutorMetricsUpdateEvent(1,\n+            new ExecutorMetrics(-1L, 5000L, 50L, 50L, 20L, 50L, 10L, 100L, 30L, 70L, 20L))),\n+        ((0, \"2\"),\n+          createExecutorMetricsUpdateEvent(2,\n+            new ExecutorMetrics(-1L, 7000L, 70L, 50L, 20L, 10L, 10L, 50L, 30L, 80L, 40L))),\n+        ((1, \"1\"),\n+          createExecutorMetricsUpdateEvent(1,\n+            new ExecutorMetrics(-1L, 7000L, 70L, 50L, 30L, 60L, 30L, 80L, 55L, 50L, 0L))),\n+        ((1, \"2\"),\n+          createExecutorMetricsUpdateEvent(2,\n+            new ExecutorMetrics(-1L, 7000L, 70L, 50L, 40L, 10L, 30L, 50L, 60L, 40L, 40L))))\n+\n+    // Events to post.\n+    val events = Array(\n+      SparkListenerApplicationStart(\"executionMetrics\", None,\n+        1L, \"update\", None),\n+      createExecutorAddedEvent(1),\n+      createExecutorAddedEvent(2),\n+      createStageSubmittedEvent(0),\n+      createExecutorMetricsUpdateEvent(1,\n+        new ExecutorMetrics(10L, 4000L, 50L, 20L, 0L, 40L, 0L, 60L, 0L, 70L, 20L)),\n+      createExecutorMetricsUpdateEvent(2,\n+        new ExecutorMetrics(10L, 1500L, 50L, 20L, 0L, 0L, 0L, 20L, 0L, 70L, 0L)),\n+      createExecutorMetricsUpdateEvent(1,\n+        new ExecutorMetrics(15L, 4000L, 50L, 50L, 0L, 50L, 0L, 100L, 0L, 70L, 20L)),\n+      createExecutorMetricsUpdateEvent(2,\n+        new ExecutorMetrics(15L, 2000L, 50L, 10L, 0L, 10L, 0L, 30L, 0L, 70L, 0L)),\n+      createExecutorMetricsUpdateEvent(1,\n+        new ExecutorMetrics(20L, 2000L, 40L, 50L, 0L, 40L, 10L, 90L, 10L, 50L, 0L)),\n+      createExecutorMetricsUpdateEvent(2,\n+        new ExecutorMetrics(20L, 3500L, 50L, 15L, 0L, 10L, 10L, 35L, 10L, 80L, 0L)),\n+      createStageSubmittedEvent(1),\n+      createExecutorMetricsUpdateEvent(1,\n+        new ExecutorMetrics(25L, 5000L, 30L, 50L, 20L, 30L, 10L, 80L, 30L, 50L, 0L)),\n+      createExecutorMetricsUpdateEvent(2,\n+        new ExecutorMetrics(25L, 7000L, 70L, 50L, 20L, 0L, 10L, 50L, 30L, 10L, 40L)),\n+      createStageCompletedEvent(0),\n+      createExecutorMetricsUpdateEvent(1,\n+        new ExecutorMetrics(30L, 6000L, 70L, 20L, 30L, 10L, 0L, 30L, 30L, 30L, 0L)),\n+      createExecutorMetricsUpdateEvent(2,\n+        new ExecutorMetrics(30L, 5500L, 30L, 20L, 40L, 10L, 0L, 30L, 40L, 40L, 20L)),\n+      createExecutorMetricsUpdateEvent(1,\n+        new ExecutorMetrics(35L, 7000L, 70L, 5L, 25L, 60L, 30L, 65L, 55L, 30L, 0L)),\n+      createExecutorMetricsUpdateEvent(2,\n+        new ExecutorMetrics(35L, 5500L, 40L, 25L, 30L, 10L, 30L, 35L, 60L, 0L, 20L)),\n+      createExecutorMetricsUpdateEvent(1,\n+        new ExecutorMetrics(40L, 5500L, 70L, 15L, 20L, 55L, 20L, 70L, 40L, 20L, 0L)),\n+      createExecutorRemovedEvent(1),\n+      createExecutorMetricsUpdateEvent(2,\n+        new ExecutorMetrics(40L, 4000L, 20L, 25L, 30L, 10L, 30L, 35L, 60L, 0L, 0L)),\n+      createStageCompletedEvent(1),\n+      SparkListenerApplicationEnd(1000L))\n+\n+    // play the events for the event logger\n+    eventLogger.start()\n+    listenerBus.start(Mockito.mock(classOf[SparkContext]), Mockito.mock(classOf[MetricsSystem]))\n+    listenerBus.addToEventLogQueue(eventLogger)\n+    events.foreach(event => listenerBus.post(event))\n+    listenerBus.stop()\n+    eventLogger.stop()\n+\n+    // Verify the log file contains the expected events.\n+    // Posted events should be logged, except for ExecutorMetricsUpdate events -- these\n+    // are consolidated, and the peak values for each stage are logged at stage end.\n+    val logData = EventLoggingListener.openEventLog(new Path(eventLogger.logPath), fileSystem)\n+    try {\n+      val lines = readLines(logData)\n+      val logStart = SparkListenerLogStart(SPARK_VERSION)\n+      assert(lines.size === 14)\n+      assert(lines(0).contains(\"SparkListenerLogStart\"))\n+      assert(lines(1).contains(\"SparkListenerApplicationStart\"))\n+      assert(JsonProtocol.sparkEventFromJson(parse(lines(0))) === logStart)\n+      var i = 1\n+      events.foreach {event =>\n+        event match {\n+          case metricsUpdate: SparkListenerExecutorMetricsUpdate =>\n+          case stageCompleted: SparkListenerStageCompleted =>\n+            for (j <- 1 to 2) {\n+              checkExecutorMetricsUpdate(lines(i), stageCompleted.stageInfo.stageId,\n+                expectedMetricsEvents)\n+                i += 1\n+             }\n+            checkEvent(lines(i), event)\n+            i += 1\n+        case _ =>\n+          checkEvent(lines(i), event)\n+          i += 1\n+        }\n+      }\n+    } finally {\n+      logData.close()\n+    }\n+  }\n+\n+  /** Create a stage submitted event for the specified stage Id. */\n+  private def createStageSubmittedEvent(stageId: Int) = {\n+    SparkListenerStageSubmitted(new StageInfo(stageId, 0, stageId.toString, 0,\n+      Seq.empty, Seq.empty, \"details\"))\n+  }\n+\n+  /** Create a stage completed event for the specified stage Id. */\n+  private def createStageCompletedEvent(stageId: Int) = {\n+    SparkListenerStageCompleted(new StageInfo(stageId, 0, stageId.toString, 0,\n+      Seq.empty, Seq.empty, \"details\"))\n+  }\n+\n+  /** Create an executor added event for the specified executor Id. */\n+  private def createExecutorAddedEvent(executorId: Int) = {\n+    SparkListenerExecutorAdded(0L, executorId.toString, new ExecutorInfo(\"host1\", 1, Map.empty))\n+  }\n+\n+  /** Create an executor added event for the specified executor Id. */\n+  private def createExecutorRemovedEvent(executorId: Int) = {\n+    SparkListenerExecutorRemoved(0L, executorId.toString, \"test\")\n+  }\n+\n+  /** Create an executor metrics update event, with the specified executor metrics values. */\n+  private def createExecutorMetricsUpdateEvent(\n+      executorId: Int,\n+      executorMetrics: ExecutorMetrics): SparkListenerExecutorMetricsUpdate = {\n+    val taskMetrics = TaskMetrics.empty\n+    taskMetrics.incDiskBytesSpilled(111)\n+    taskMetrics.incMemoryBytesSpilled(222)\n+    val accum = Array((333L, 1, 1, taskMetrics.accumulators().map(AccumulatorSuite.makeInfo)))\n+    SparkListenerExecutorMetricsUpdate(executorId.toString, accum, Some(executorMetrics))\n+  }\n+\n+  /** Check that the two ExecutorMetrics match */\n+  private def checkExecutorMetrics(\n+      executorMetrics1: Option[ExecutorMetrics],\n+      executorMetrics2: Option[ExecutorMetrics]) = {\n+    (executorMetrics1, executorMetrics2) match {\n+      case (Some(e1), Some(e2)) =>\n+        assert(e1.timestamp === e2.timestamp)\n+        assert(e1.jvmUsedHeapMemory === e2.jvmUsedHeapMemory)\n+        assert(e1.jvmUsedNonHeapMemory === e2.jvmUsedNonHeapMemory)\n+        assert(e1.onHeapExecutionMemory === e2.onHeapExecutionMemory)\n+        assert(e1.offHeapExecutionMemory === e2.offHeapExecutionMemory)\n+        assert(e1.onHeapStorageMemory === e2.onHeapStorageMemory)\n+        assert(e1.offHeapStorageMemory === e2.offHeapStorageMemory)\n+        assert(e1.onHeapUnifiedMemory === e2.onHeapUnifiedMemory)\n+        assert(e1.offHeapUnifiedMemory === e2.offHeapUnifiedMemory)\n+        assert(e1.directMemory === e2.directMemory)\n+        assert(e1.mappedMemory === e2.mappedMemory)\n+      case (None, None) =>\n+      case _ =>\n+        assert(false)\n+    }\n+  }\n+\n+  /** Check that the Spark history log line matches the expected event. */\n+  private def checkEvent(line: String, event: SparkListenerEvent): Unit = {\n+    assert(line.contains(event.getClass.toString.split(\"\\\\.\").last))\n+    event match {\n+      case executorMetrics: SparkListenerExecutorMetricsUpdate =>","path":"core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala","position":219,"original_position":219,"commit_id":"10ed328bfcf160711e7619aac23472f97bf1c976","original_commit_id":"10ed328bfcf160711e7619aac23472f97bf1c976","user":{"login":"squito","id":71240,"avatar_url":"https://avatars2.githubusercontent.com/u/71240?v=4","gravatar_id":"","url":"https://api.github.com/users/squito","html_url":"https://github.com/squito","followers_url":"https://api.github.com/users/squito/followers","following_url":"https://api.github.com/users/squito/following{/other_user}","gists_url":"https://api.github.com/users/squito/gists{/gist_id}","starred_url":"https://api.github.com/users/squito/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/squito/subscriptions","organizations_url":"https://api.github.com/users/squito/orgs","repos_url":"https://api.github.com/users/squito/repos","events_url":"https://api.github.com/users/squito/events{/privacy}","received_events_url":"https://api.github.com/users/squito/received_events","type":"User","site_admin":false},"body":"you're never using this w/ `SparkListenerExecutorMetricsUpdate`, right?","created_at":"2018-05-23T19:04:12Z","updated_at":"2018-05-23T19:10:33Z","html_url":"https://github.com/apache/spark/pull/21221#discussion_r190364971","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/21221","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/190364971"},"html":{"href":"https://github.com/apache/spark/pull/21221#discussion_r190364971"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/21221"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/21221","id":185590243,"html_url":"https://github.com/apache/spark/pull/21221","diff_url":"https://github.com/apache/spark/pull/21221.diff","patch_url":"https://github.com/apache/spark/pull/21221.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/21221","number":21221,"state":"open","locked":false,"title":"[SPARK-23429][CORE] Add executor memory metrics to heartbeat and expose in executors REST API","user":{"login":"edwinalu","id":26153035,"avatar_url":"https://avatars0.githubusercontent.com/u/26153035?v=4","gravatar_id":"","url":"https://api.github.com/users/edwinalu","html_url":"https://github.com/edwinalu","followers_url":"https://api.github.com/users/edwinalu/followers","following_url":"https://api.github.com/users/edwinalu/following{/other_user}","gists_url":"https://api.github.com/users/edwinalu/gists{/gist_id}","starred_url":"https://api.github.com/users/edwinalu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edwinalu/subscriptions","organizations_url":"https://api.github.com/users/edwinalu/orgs","repos_url":"https://api.github.com/users/edwinalu/repos","events_url":"https://api.github.com/users/edwinalu/events{/privacy}","received_events_url":"https://api.github.com/users/edwinalu/received_events","type":"User","site_admin":false},"body":"The original PR #20940 is messed up, and the dif shows changes not related to SPARK-23429. This is a cleaned up version of that pull request.\r\n\r\nAdd new executor level memory metrics (JVM used memory, on/off heap execution memory, on/off heap storage memory, on/off heap unified memory, direct memory, and mapped memory), and expose via the executors REST API. This information will help provide insight into how executor and driver JVM memory is used, and for the different memory regions. It can be used to help determine good values for spark.executor.memory, spark.driver.memory, spark.memory.fraction, and spark.memory.storageFraction.\r\n\r\n## What changes were proposed in this pull request?\r\n\r\nAn ExecutorMetrics class is added, with jvmUsedHeapMemory, jvmUsedNonHeapMemory, onHeapExecutionMemory, offHeapExecutionMemory, onHeapStorageMemory, and offHeapStorageMemory, onHeapUnifiedMemory, offHeapUnifiedMemory, directMemory and mappedMemory. The new ExecutorMetrics is sent by executors to the driver as part of the Heartbeat. A heartbeat is added for the driver as well, to collect these metrics for the driver.\r\n\r\nThe EventLoggingListener store information about the peak values for each metric, per active stage and executor. When a StageCompleted event is seen, and ExecutorMetricsUpdate event will be logged for each executor, with peal values for the stage. Only the ExecutorMetrics will be logged, and not the TaskMetrics, to minimize additional logging.\r\n\r\nThe AppStatusListener records the peak values for each memory metric.\r\n\r\nThe new memory metrics are added to the executors REST API.\r\n\r\n## How was this patch tested?\r\n\r\nNew unit tests have been added. This was also tested on our cluster.","created_at":"2018-05-02T23:57:26Z","updated_at":"2018-05-23T19:10:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8176f46e544f299a3495436b7ea7a8b0d33063ae","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/21221/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/21221/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/21221/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/10ed328bfcf160711e7619aac23472f97bf1c976","head":{"label":"edwinalu:SPARK-23429.2","ref":"SPARK-23429.2","sha":"10ed328bfcf160711e7619aac23472f97bf1c976","user":{"login":"edwinalu","id":26153035,"avatar_url":"https://avatars0.githubusercontent.com/u/26153035?v=4","gravatar_id":"","url":"https://api.github.com/users/edwinalu","html_url":"https://github.com/edwinalu","followers_url":"https://api.github.com/users/edwinalu/followers","following_url":"https://api.github.com/users/edwinalu/following{/other_user}","gists_url":"https://api.github.com/users/edwinalu/gists{/gist_id}","starred_url":"https://api.github.com/users/edwinalu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edwinalu/subscriptions","organizations_url":"https://api.github.com/users/edwinalu/orgs","repos_url":"https://api.github.com/users/edwinalu/repos","events_url":"https://api.github.com/users/edwinalu/events{/privacy}","received_events_url":"https://api.github.com/users/edwinalu/received_events","type":"User","site_admin":false},"repo":{"id":98354731,"name":"spark","full_name":"edwinalu/spark","owner":{"login":"edwinalu","id":26153035,"avatar_url":"https://avatars0.githubusercontent.com/u/26153035?v=4","gravatar_id":"","url":"https://api.github.com/users/edwinalu","html_url":"https://github.com/edwinalu","followers_url":"https://api.github.com/users/edwinalu/followers","following_url":"https://api.github.com/users/edwinalu/following{/other_user}","gists_url":"https://api.github.com/users/edwinalu/gists{/gist_id}","starred_url":"https://api.github.com/users/edwinalu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edwinalu/subscriptions","organizations_url":"https://api.github.com/users/edwinalu/orgs","repos_url":"https://api.github.com/users/edwinalu/repos","events_url":"https://api.github.com/users/edwinalu/events{/privacy}","received_events_url":"https://api.github.com/users/edwinalu/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/edwinalu/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/edwinalu/spark","forks_url":"https://api.github.com/repos/edwinalu/spark/forks","keys_url":"https://api.github.com/repos/edwinalu/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edwinalu/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edwinalu/spark/teams","hooks_url":"https://api.github.com/repos/edwinalu/spark/hooks","issue_events_url":"https://api.github.com/repos/edwinalu/spark/issues/events{/number}","events_url":"https://api.github.com/repos/edwinalu/spark/events","assignees_url":"https://api.github.com/repos/edwinalu/spark/assignees{/user}","branches_url":"https://api.github.com/repos/edwinalu/spark/branches{/branch}","tags_url":"https://api.github.com/repos/edwinalu/spark/tags","blobs_url":"https://api.github.com/repos/edwinalu/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edwinalu/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edwinalu/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/edwinalu/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edwinalu/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/edwinalu/spark/languages","stargazers_url":"https://api.github.com/repos/edwinalu/spark/stargazers","contributors_url":"https://api.github.com/repos/edwinalu/spark/contributors","subscribers_url":"https://api.github.com/repos/edwinalu/spark/subscribers","subscription_url":"https://api.github.com/repos/edwinalu/spark/subscription","commits_url":"https://api.github.com/repos/edwinalu/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/edwinalu/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/edwinalu/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/edwinalu/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/edwinalu/spark/contents/{+path}","compare_url":"https://api.github.com/repos/edwinalu/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edwinalu/spark/merges","archive_url":"https://api.github.com/repos/edwinalu/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edwinalu/spark/downloads","issues_url":"https://api.github.com/repos/edwinalu/spark/issues{/number}","pulls_url":"https://api.github.com/repos/edwinalu/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/edwinalu/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/edwinalu/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edwinalu/spark/labels{/name}","releases_url":"https://api.github.com/repos/edwinalu/spark/releases{/id}","deployments_url":"https://api.github.com/repos/edwinalu/spark/deployments","created_at":"2017-07-25T22:19:57Z","updated_at":"2018-05-02T21:56:59Z","pushed_at":"2018-05-15T00:57:46Z","git_url":"git://github.com/edwinalu/spark.git","ssh_url":"git@github.com:edwinalu/spark.git","clone_url":"https://github.com/edwinalu/spark.git","svn_url":"https://github.com/edwinalu/spark","homepage":null,"size":267300,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"e4c91c089a701117af82f585d14d8afc5245fc64","user":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2018-05-23T18:58:53Z","pushed_at":"2018-05-23T18:52:31Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":271018,"stargazers_count":17369,"watchers_count":17369,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":15886,"mirror_url":null,"archived":false,"open_issues_count":548,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":15886,"open_issues":548,"watchers":17369,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/21221"},"html":{"href":"https://github.com/apache/spark/pull/21221"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/21221"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/21221/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/21221/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/21221/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/10ed328bfcf160711e7619aac23472f97bf1c976"}},"author_association":"NONE"}}
{"actor":{"display_login":"squito"}}