{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/73413028","id":73413028,"diff_hunk":"@@ -357,4 +375,194 @@ private[mesos] trait MesosSchedulerUtils extends Logging {\n     sc.conf.getTimeAsSeconds(\"spark.mesos.rejectOfferDurationForReachedMaxCores\", \"120s\")\n   }\n \n+  /**\n+   * Checks executor ports if they are within some range of the offered list of ports ranges,\n+   *\n+   * @param conf the Spark Config\n+   * @param ports the list of ports to check\n+   * @return true if ports are within range false otherwise\n+   */\n+  protected def checkPorts(conf: SparkConf, ports: List[(Long, Long)]): Boolean = {\n+\n+    def checkIfInRange(port: Long, ps: List[(Long, Long)]): Boolean = {\n+      ps.exists(r => r._1 <= port & r._2 >= port)\n+    }\n+\n+    val portsToCheck = nonZeroPortValuesFromConfig(conf)\n+    val withinRange = portsToCheck.forall(p => checkIfInRange(p, ports))\n+    // make sure we have enough ports to allocate per offer\n+    ports.map(r => r._2 - r._1 + 1).sum >= portsToCheck.size && withinRange\n+  }\n+\n+  /**\n+   * Partitions port resources.\n+   *\n+   * @param portsToAssign non-zero ports to assign\n+   * @param offeredResources the resources offered\n+   * @return resources left, port resources to be used.\n+   */\n+  def partitionPortResources(portsToAssign: List[Long], offeredResources: List[Resource])\n+    : (List[Resource], List[Resource]) = {\n+    if (portsToAssign.isEmpty) {\n+     return (offeredResources, List[Resource]())\n+    }\n+    // partition port offers\n+    val (resourcesWithoutPorts, portResources) = filterPortResources(offeredResources)\n+    val offeredPortRanges = getRangeResourceWithRoleInfo(portResources.asJava, \"ports\")\n+    // reserve non-zero ports\n+    val nonZeroResources = reservePorts(offeredPortRanges, portsToAssign)\n+\n+    val (leftPortResources, assignedPortResources) =\n+      createResourcesFromAssignedPorts(nonZeroResources)\n+\n+    (resourcesWithoutPorts ++ leftPortResources, assignedPortResources)\n+  }\n+\n+  /**\n+   * Returns known port name used by the executor process.\n+   * @return the port name\n+   */\n+  def managedPortNames() : List[String] = List(\"spark.executor.port\", \"spark.blockManager.port\")\n+\n+  /**\n+   * The values of the non-zero ports to be used by the executor process.\n+   * @param conf the spark config to use\n+   * @return the ono-zero values of the ports\n+   */\n+  def nonZeroPortValuesFromConfig(conf: SparkConf): List[Long] = {\n+    managedPortNames().map(conf.getLong(_, 0)).filter( _ != 0)\n+  }\n+\n+  /**\n+   * It gets a tuple for the non-zero port assigned resources.\n+   * First member of the tuple represents resources left while the second\n+   * resources used. A tuple is returned with final resources left (fist member)\n+   * and the resources used (second member).\n+   */\n+  private def createResourcesFromAssignedPorts(\n+      nonZero: (List[PortRangeResourceInfo], List[PortRangeResourceInfo]))\n+    : (List[Resource], List[Resource]) = {\n+        (nonZero._1.flatMap{port => createMesosPortResource(port.range, Some(port.role))},\n+          nonZero._2.flatMap{port => createMesosPortResource(port.range, Some(port.role))})\n+  }\n+\n+  private case class PortRangeResourceInfo(role: String, range: List[(Long, Long)])","path":"core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala","position":195,"original_position":195,"commit_id":"bfa7b0fbf4229a72921ec2e538e635b0cc92ebce","original_commit_id":"bfa7b0fbf4229a72921ec2e538e635b0cc92ebce","user":{"login":"mgummelt","id":153734,"avatar_url":"https://avatars.githubusercontent.com/u/153734?v=3","gravatar_id":"","url":"https://api.github.com/users/mgummelt","html_url":"https://github.com/mgummelt","followers_url":"https://api.github.com/users/mgummelt/followers","following_url":"https://api.github.com/users/mgummelt/following{/other_user}","gists_url":"https://api.github.com/users/mgummelt/gists{/gist_id}","starred_url":"https://api.github.com/users/mgummelt/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mgummelt/subscriptions","organizations_url":"https://api.github.com/users/mgummelt/orgs","repos_url":"https://api.github.com/users/mgummelt/repos","events_url":"https://api.github.com/users/mgummelt/events{/privacy}","received_events_url":"https://api.github.com/users/mgummelt/received_events","type":"User","site_admin":false},"body":"s/range/ranges","created_at":"2016-08-03T20:21:31Z","updated_at":"2016-08-03T20:21:31Z","html_url":"https://github.com/apache/spark/pull/11157#discussion_r73413028","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/11157","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/73413028"},"html":{"href":"https://github.com/apache/spark/pull/11157#discussion_r73413028"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/11157"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/11157","id":58952129,"html_url":"https://github.com/apache/spark/pull/11157","diff_url":"https://github.com/apache/spark/pull/11157.diff","patch_url":"https://github.com/apache/spark/pull/11157.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/11157","number":11157,"state":"open","locked":false,"title":"[SPARK-11714][Mesos] Make Spark on Mesos honor port restrictions on coarse grain mode","user":{"login":"skonto","id":7945591,"avatar_url":"https://avatars.githubusercontent.com/u/7945591?v=3","gravatar_id":"","url":"https://api.github.com/users/skonto","html_url":"https://github.com/skonto","followers_url":"https://api.github.com/users/skonto/followers","following_url":"https://api.github.com/users/skonto/following{/other_user}","gists_url":"https://api.github.com/users/skonto/gists{/gist_id}","starred_url":"https://api.github.com/users/skonto/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/skonto/subscriptions","organizations_url":"https://api.github.com/users/skonto/orgs","repos_url":"https://api.github.com/users/skonto/repos","events_url":"https://api.github.com/users/skonto/events{/privacy}","received_events_url":"https://api.github.com/users/skonto/received_events","type":"User","site_admin":false},"body":"- Make mesos coarse grained scheduler accept port offers and pre-assign ports\r\n\r\nPrevious attempt was for fine grained: https://github.com/apache/spark/pull/10808","created_at":"2016-02-10T20:56:12Z","updated_at":"2016-08-03T20:21:32Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2a58470f773a3efc9b3c9a55b2e00fdb707e110c","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/11157/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/11157/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/11157/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/bfa7b0fbf4229a72921ec2e538e635b0cc92ebce","head":{"label":"skonto:honour_ports_coarse","ref":"honour_ports_coarse","sha":"bfa7b0fbf4229a72921ec2e538e635b0cc92ebce","user":{"login":"skonto","id":7945591,"avatar_url":"https://avatars.githubusercontent.com/u/7945591?v=3","gravatar_id":"","url":"https://api.github.com/users/skonto","html_url":"https://github.com/skonto","followers_url":"https://api.github.com/users/skonto/followers","following_url":"https://api.github.com/users/skonto/following{/other_user}","gists_url":"https://api.github.com/users/skonto/gists{/gist_id}","starred_url":"https://api.github.com/users/skonto/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/skonto/subscriptions","organizations_url":"https://api.github.com/users/skonto/orgs","repos_url":"https://api.github.com/users/skonto/repos","events_url":"https://api.github.com/users/skonto/events{/privacy}","received_events_url":"https://api.github.com/users/skonto/received_events","type":"User","site_admin":false},"repo":{"id":48892104,"name":"spark","full_name":"skonto/spark","owner":{"login":"skonto","id":7945591,"avatar_url":"https://avatars.githubusercontent.com/u/7945591?v=3","gravatar_id":"","url":"https://api.github.com/users/skonto","html_url":"https://github.com/skonto","followers_url":"https://api.github.com/users/skonto/followers","following_url":"https://api.github.com/users/skonto/following{/other_user}","gists_url":"https://api.github.com/users/skonto/gists{/gist_id}","starred_url":"https://api.github.com/users/skonto/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/skonto/subscriptions","organizations_url":"https://api.github.com/users/skonto/orgs","repos_url":"https://api.github.com/users/skonto/repos","events_url":"https://api.github.com/users/skonto/events{/privacy}","received_events_url":"https://api.github.com/users/skonto/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/skonto/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/skonto/spark","forks_url":"https://api.github.com/repos/skonto/spark/forks","keys_url":"https://api.github.com/repos/skonto/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/skonto/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/skonto/spark/teams","hooks_url":"https://api.github.com/repos/skonto/spark/hooks","issue_events_url":"https://api.github.com/repos/skonto/spark/issues/events{/number}","events_url":"https://api.github.com/repos/skonto/spark/events","assignees_url":"https://api.github.com/repos/skonto/spark/assignees{/user}","branches_url":"https://api.github.com/repos/skonto/spark/branches{/branch}","tags_url":"https://api.github.com/repos/skonto/spark/tags","blobs_url":"https://api.github.com/repos/skonto/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/skonto/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/skonto/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/skonto/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/skonto/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/skonto/spark/languages","stargazers_url":"https://api.github.com/repos/skonto/spark/stargazers","contributors_url":"https://api.github.com/repos/skonto/spark/contributors","subscribers_url":"https://api.github.com/repos/skonto/spark/subscribers","subscription_url":"https://api.github.com/repos/skonto/spark/subscription","commits_url":"https://api.github.com/repos/skonto/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/skonto/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/skonto/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/skonto/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/skonto/spark/contents/{+path}","compare_url":"https://api.github.com/repos/skonto/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/skonto/spark/merges","archive_url":"https://api.github.com/repos/skonto/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/skonto/spark/downloads","issues_url":"https://api.github.com/repos/skonto/spark/issues{/number}","pulls_url":"https://api.github.com/repos/skonto/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/skonto/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/skonto/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/skonto/spark/labels{/name}","releases_url":"https://api.github.com/repos/skonto/spark/releases{/id}","deployments_url":"https://api.github.com/repos/skonto/spark/deployments","created_at":"2016-01-01T23:02:52Z","updated_at":"2016-01-18T15:47:02Z","pushed_at":"2016-08-03T10:09:36Z","git_url":"git://github.com/skonto/spark.git","ssh_url":"git@github.com:skonto/spark.git","clone_url":"https://github.com/skonto/spark.git","svn_url":"https://github.com/skonto/spark","homepage":null,"size":173383,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"ae226283e19ce396216c73b0ae2470efa122b65b","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2016-08-03T17:23:34Z","pushed_at":"2016-08-03T20:16:43Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":507137,"stargazers_count":9534,"watchers_count":9534,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":8850,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":471,"forks":8850,"open_issues":471,"watchers":9534,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/11157"},"html":{"href":"https://github.com/apache/spark/pull/11157"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/11157"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/11157/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/11157/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/11157/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/bfa7b0fbf4229a72921ec2e538e635b0cc92ebce"}}}}
{"actor":{"display_login":"mgummelt"}}