{"action":"created","comment":{"url":"https://api.github.com/repos/foxish/spark/pulls/comments/87814404","pull_request_review_id":8410227,"id":87814404,"diff_hunk":"@@ -40,41 +41,78 @@ private[spark] class KubernetesClusterSchedulerBackend(\n \n   val DEFAULT_NUMBER_EXECUTORS = 2\n   val sparkExecutorName = s\"spark-executor-${Random.alphanumeric take 5 mkString(\"\")}\".toLowerCase()\n-  var executorPods = mutable.ArrayBuffer[String]()\n+\n+  // key is executor id, value is pod name\n+  var executorToPod = mutable.Map.empty[String, String]\n+  var executorID = 0\n \n   val sparkDriverImage = sc.getConf.get(\"spark.kubernetes.driver.image\")\n   val clientJarUri = sc.getConf.get(\"spark.executor.jar\")\n   val ns = sc.getConf.get(\"spark.kubernetes.namespace\")\n+  val dynamicExecutors = Utils.isDynamicAllocationEnabled(sc.getConf)\n+\n+  // executor back-ends take their configuration this way\n+  if (dynamicExecutors) {\n+    sc.getConf.setExecutorEnv(\"spark.dynamicAllocation.enabled\", \"true\")\n+    sc.getConf.setExecutorEnv(\"spark.shuffle.service.enabled\", \"true\")\n+  }\n \n-  override def start() {\n+  override def start(): Unit = {\n     super.start()\n-    var i = 0\n-    for(i <- 1 to getInitialTargetExecutorNumber(sc.conf)){\n-      executorPods += createExecutorPod(i)\n-    }\n-    None\n+    createExecutorPods(getInitialTargetExecutorNumber(sc.getConf))\n   }\n \n   override def stop(): Unit = {\n-    for (i <- 0 to executorPods.length) {\n-      client.pods().inNamespace(ns).withName(executorPods(i)).delete()\n-    }\n+    killExecutorPods(executorToPod.toVector)\n     super.stop()\n   }\n \n   // Dynamic allocation interfaces\n-  override def doRequestTotalExecutors(requestedTotal: Int): scala.concurrent.Future[Boolean] = {\n-    return super.doRequestTotalExecutors(requestedTotal)\n+  override def doRequestTotalExecutors(requestedTotal: Int): Future[Boolean] = {\n+    logInfo(s\"Received doRequestTotalExecutors: $requestedTotal\")\n+    val n = executorToPod.size\n+    val delta = requestedTotal - n\n+    if (delta > 0) {\n+      logInfo(s\"Adding $delta new executor Pods\")\n+      createExecutorPods(delta)\n+    } else if (delta < 0) {\n+      logInfo(s\"Deleting ${-delta} new executor Pods\")","path":"kubernetes/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterSchedulerBackend.scala","position":59,"original_position":59,"commit_id":"7ec754680020269b0af7605ddf804e33a8edad55","original_commit_id":"7ec754680020269b0af7605ddf804e33a8edad55","user":{"login":"erikerlandson","id":259898,"avatar_url":"https://avatars.githubusercontent.com/u/259898?v=3","gravatar_id":"","url":"https://api.github.com/users/erikerlandson","html_url":"https://github.com/erikerlandson","followers_url":"https://api.github.com/users/erikerlandson/followers","following_url":"https://api.github.com/users/erikerlandson/following{/other_user}","gists_url":"https://api.github.com/users/erikerlandson/gists{/gist_id}","starred_url":"https://api.github.com/users/erikerlandson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erikerlandson/subscriptions","organizations_url":"https://api.github.com/users/erikerlandson/orgs","repos_url":"https://api.github.com/users/erikerlandson/repos","events_url":"https://api.github.com/users/erikerlandson/events{/privacy}","received_events_url":"https://api.github.com/users/erikerlandson/received_events","type":"User","site_admin":false},"body":"In the dynamic-executor case, Spark will scale back what it's requesting.   For example, if a job starts to complete and there are fewer partitions left to compute, it will start requesting numbers of executors fewer than what it is running, and eventually drops to zero.","created_at":"2016-11-14T14:50:51Z","updated_at":"2016-11-14T14:50:51Z","html_url":"https://github.com/foxish/spark/pull/4#discussion_r87814404","pull_request_url":"https://api.github.com/repos/foxish/spark/pulls/4","_links":{"self":{"href":"https://api.github.com/repos/foxish/spark/pulls/comments/87814404"},"html":{"href":"https://github.com/foxish/spark/pull/4#discussion_r87814404"},"pull_request":{"href":"https://api.github.com/repos/foxish/spark/pulls/4"}}},"pull_request":{"url":"https://api.github.com/repos/foxish/spark/pulls/4","id":93418648,"html_url":"https://github.com/foxish/spark/pull/4","diff_url":"https://github.com/foxish/spark/pull/4.diff","patch_url":"https://github.com/foxish/spark/pull/4.patch","issue_url":"https://api.github.com/repos/foxish/spark/issues/4","number":4,"state":"open","locked":false,"title":"Add support for dynamic executors","user":{"login":"erikerlandson","id":259898,"avatar_url":"https://avatars.githubusercontent.com/u/259898?v=3","gravatar_id":"","url":"https://api.github.com/users/erikerlandson","html_url":"https://github.com/erikerlandson","followers_url":"https://api.github.com/users/erikerlandson/followers","following_url":"https://api.github.com/users/erikerlandson/following{/other_user}","gists_url":"https://api.github.com/users/erikerlandson/gists{/gist_id}","starred_url":"https://api.github.com/users/erikerlandson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erikerlandson/subscriptions","organizations_url":"https://api.github.com/users/erikerlandson/orgs","repos_url":"https://api.github.com/users/erikerlandson/repos","events_url":"https://api.github.com/users/erikerlandson/events{/privacy}","received_events_url":"https://api.github.com/users/erikerlandson/received_events","type":"User","site_admin":false},"body":"This PR adds support for dynamic executors.  It works by running the shuffle server in the same container as the executor.\r\n\r\nTo use it, you will want to run with my latest image:\r\n`manyangled/kube-spark:dynamic`\r\n\r\nYou must configure dynamic executors, for example submit using `--conf spark.dynamicAllocation.enabled=true`","created_at":"2016-11-12T00:21:31Z","updated_at":"2016-11-14T14:50:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7c3fd4c5a80f5fff57ce4076109408dfccda3b43","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/foxish/spark/pulls/4/commits","review_comments_url":"https://api.github.com/repos/foxish/spark/pulls/4/comments","review_comment_url":"https://api.github.com/repos/foxish/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/foxish/spark/issues/4/comments","statuses_url":"https://api.github.com/repos/foxish/spark/statuses/7ec754680020269b0af7605ddf804e33a8edad55","head":{"label":"erikerlandson:k8s-eje-dynamic","ref":"k8s-eje-dynamic","sha":"7ec754680020269b0af7605ddf804e33a8edad55","user":{"login":"erikerlandson","id":259898,"avatar_url":"https://avatars.githubusercontent.com/u/259898?v=3","gravatar_id":"","url":"https://api.github.com/users/erikerlandson","html_url":"https://github.com/erikerlandson","followers_url":"https://api.github.com/users/erikerlandson/followers","following_url":"https://api.github.com/users/erikerlandson/following{/other_user}","gists_url":"https://api.github.com/users/erikerlandson/gists{/gist_id}","starred_url":"https://api.github.com/users/erikerlandson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erikerlandson/subscriptions","organizations_url":"https://api.github.com/users/erikerlandson/orgs","repos_url":"https://api.github.com/users/erikerlandson/repos","events_url":"https://api.github.com/users/erikerlandson/events{/privacy}","received_events_url":"https://api.github.com/users/erikerlandson/received_events","type":"User","site_admin":false},"repo":{"id":19623230,"name":"spark","full_name":"erikerlandson/spark","owner":{"login":"erikerlandson","id":259898,"avatar_url":"https://avatars.githubusercontent.com/u/259898?v=3","gravatar_id":"","url":"https://api.github.com/users/erikerlandson","html_url":"https://github.com/erikerlandson","followers_url":"https://api.github.com/users/erikerlandson/followers","following_url":"https://api.github.com/users/erikerlandson/following{/other_user}","gists_url":"https://api.github.com/users/erikerlandson/gists{/gist_id}","starred_url":"https://api.github.com/users/erikerlandson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erikerlandson/subscriptions","organizations_url":"https://api.github.com/users/erikerlandson/orgs","repos_url":"https://api.github.com/users/erikerlandson/repos","events_url":"https://api.github.com/users/erikerlandson/events{/privacy}","received_events_url":"https://api.github.com/users/erikerlandson/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/erikerlandson/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/erikerlandson/spark","forks_url":"https://api.github.com/repos/erikerlandson/spark/forks","keys_url":"https://api.github.com/repos/erikerlandson/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/erikerlandson/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/erikerlandson/spark/teams","hooks_url":"https://api.github.com/repos/erikerlandson/spark/hooks","issue_events_url":"https://api.github.com/repos/erikerlandson/spark/issues/events{/number}","events_url":"https://api.github.com/repos/erikerlandson/spark/events","assignees_url":"https://api.github.com/repos/erikerlandson/spark/assignees{/user}","branches_url":"https://api.github.com/repos/erikerlandson/spark/branches{/branch}","tags_url":"https://api.github.com/repos/erikerlandson/spark/tags","blobs_url":"https://api.github.com/repos/erikerlandson/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/erikerlandson/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/erikerlandson/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/erikerlandson/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/erikerlandson/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/erikerlandson/spark/languages","stargazers_url":"https://api.github.com/repos/erikerlandson/spark/stargazers","contributors_url":"https://api.github.com/repos/erikerlandson/spark/contributors","subscribers_url":"https://api.github.com/repos/erikerlandson/spark/subscribers","subscription_url":"https://api.github.com/repos/erikerlandson/spark/subscription","commits_url":"https://api.github.com/repos/erikerlandson/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/erikerlandson/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/erikerlandson/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/erikerlandson/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/erikerlandson/spark/contents/{+path}","compare_url":"https://api.github.com/repos/erikerlandson/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/erikerlandson/spark/merges","archive_url":"https://api.github.com/repos/erikerlandson/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/erikerlandson/spark/downloads","issues_url":"https://api.github.com/repos/erikerlandson/spark/issues{/number}","pulls_url":"https://api.github.com/repos/erikerlandson/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/erikerlandson/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/erikerlandson/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/erikerlandson/spark/labels{/name}","releases_url":"https://api.github.com/repos/erikerlandson/spark/releases{/id}","deployments_url":"https://api.github.com/repos/erikerlandson/spark/deployments","created_at":"2014-05-09T19:42:39Z","updated_at":"2015-03-17T02:52:11Z","pushed_at":"2016-11-12T17:39:41Z","git_url":"git://github.com/erikerlandson/spark.git","ssh_url":"git@github.com:erikerlandson/spark.git","clone_url":"https://github.com/erikerlandson/spark.git","svn_url":"https://github.com/erikerlandson/spark","homepage":null,"size":238436,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":0,"forks":2,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"foxish:k8s-support","ref":"k8s-support","sha":"4e88119c18ab76d732857a36f767c46535de378d","user":{"login":"foxish","id":906471,"avatar_url":"https://avatars.githubusercontent.com/u/906471?v=3","gravatar_id":"","url":"https://api.github.com/users/foxish","html_url":"https://github.com/foxish","followers_url":"https://api.github.com/users/foxish/followers","following_url":"https://api.github.com/users/foxish/following{/other_user}","gists_url":"https://api.github.com/users/foxish/gists{/gist_id}","starred_url":"https://api.github.com/users/foxish/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foxish/subscriptions","organizations_url":"https://api.github.com/users/foxish/orgs","repos_url":"https://api.github.com/users/foxish/repos","events_url":"https://api.github.com/users/foxish/events{/privacy}","received_events_url":"https://api.github.com/users/foxish/received_events","type":"User","site_admin":false},"repo":{"id":67230308,"name":"spark","full_name":"foxish/spark","owner":{"login":"foxish","id":906471,"avatar_url":"https://avatars.githubusercontent.com/u/906471?v=3","gravatar_id":"","url":"https://api.github.com/users/foxish","html_url":"https://github.com/foxish","followers_url":"https://api.github.com/users/foxish/followers","following_url":"https://api.github.com/users/foxish/following{/other_user}","gists_url":"https://api.github.com/users/foxish/gists{/gist_id}","starred_url":"https://api.github.com/users/foxish/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foxish/subscriptions","organizations_url":"https://api.github.com/users/foxish/orgs","repos_url":"https://api.github.com/users/foxish/repos","events_url":"https://api.github.com/users/foxish/events{/privacy}","received_events_url":"https://api.github.com/users/foxish/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/foxish/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/foxish/spark","forks_url":"https://api.github.com/repos/foxish/spark/forks","keys_url":"https://api.github.com/repos/foxish/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/foxish/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/foxish/spark/teams","hooks_url":"https://api.github.com/repos/foxish/spark/hooks","issue_events_url":"https://api.github.com/repos/foxish/spark/issues/events{/number}","events_url":"https://api.github.com/repos/foxish/spark/events","assignees_url":"https://api.github.com/repos/foxish/spark/assignees{/user}","branches_url":"https://api.github.com/repos/foxish/spark/branches{/branch}","tags_url":"https://api.github.com/repos/foxish/spark/tags","blobs_url":"https://api.github.com/repos/foxish/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/foxish/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/foxish/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/foxish/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/foxish/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/foxish/spark/languages","stargazers_url":"https://api.github.com/repos/foxish/spark/stargazers","contributors_url":"https://api.github.com/repos/foxish/spark/contributors","subscribers_url":"https://api.github.com/repos/foxish/spark/subscribers","subscription_url":"https://api.github.com/repos/foxish/spark/subscription","commits_url":"https://api.github.com/repos/foxish/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/foxish/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/foxish/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/foxish/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/foxish/spark/contents/{+path}","compare_url":"https://api.github.com/repos/foxish/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/foxish/spark/merges","archive_url":"https://api.github.com/repos/foxish/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/foxish/spark/downloads","issues_url":"https://api.github.com/repos/foxish/spark/issues{/number}","pulls_url":"https://api.github.com/repos/foxish/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/foxish/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/foxish/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/foxish/spark/labels{/name}","releases_url":"https://api.github.com/repos/foxish/spark/releases{/id}","deployments_url":"https://api.github.com/repos/foxish/spark/deployments","created_at":"2016-09-02T14:40:35Z","updated_at":"2016-11-04T18:59:34Z","pushed_at":"2016-11-12T17:39:43Z","git_url":"git://github.com/foxish/spark.git","ssh_url":"git@github.com:foxish/spark.git","clone_url":"https://github.com/foxish/spark.git","svn_url":"https://github.com/foxish/spark","homepage":null,"size":248746,"stargazers_count":2,"watchers_count":2,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":3,"forks":1,"open_issues":3,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/foxish/spark/pulls/4"},"html":{"href":"https://github.com/foxish/spark/pull/4"},"issue":{"href":"https://api.github.com/repos/foxish/spark/issues/4"},"comments":{"href":"https://api.github.com/repos/foxish/spark/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/foxish/spark/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/foxish/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/foxish/spark/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/foxish/spark/statuses/7ec754680020269b0af7605ddf804e33a8edad55"}}}}
{"actor":{"display_login":"erikerlandson"}}