{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/224830769","pull_request_review_id":164300031,"id":224830769,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyNDgzMDc2OQ==","diff_hunk":"@@ -0,0 +1,140 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.deploy.yarn\n+\n+import java.lang.{Long => JLong}\n+import java.lang.reflect.InvocationTargetException\n+\n+import scala.collection.mutable\n+import scala.util.Try\n+\n+import org.apache.hadoop.yarn.api.records.Resource\n+\n+import org.apache.spark.{SparkConf, SparkException}\n+import org.apache.spark.deploy.yarn.config._\n+import org.apache.spark.internal.Logging\n+import org.apache.spark.internal.config._\n+import org.apache.spark.util.Utils\n+\n+/**\n+ * This helper class uses some of Hadoop 3 methods from the YARN API,\n+ * so we need to use reflection to avoid compile error when building against Hadoop 2.x\n+ */\n+private object ResourceRequestHelper extends Logging {\n+  private val AMOUNT_AND_UNIT_REGEX = \"([0-9]+)([A-Za-z]*)\".r\n+  private val RESOURCE_INFO_CLASS = \"org.apache.hadoop.yarn.api.records.ResourceInformation\"\n+\n+  /**\n+   * Validates sparkConf and throws a SparkException if any of standard resources (memory or cores)\n+   * is defined with the property spark.yarn.x.resource.y\n+   */\n+  def validateResources(sparkConf: SparkConf): Unit = {\n+    val resourceDefinitions = Seq[(String, String)](\n+      (AM_MEMORY.key, YARN_AM_RESOURCE_TYPES_PREFIX + \"memory\"),","path":"resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala","position":48,"original_position":48,"commit_id":"f3955f8f7716ef0369ed3f2289eda2cc921c8937","original_commit_id":"f509153abe2857d086e85ff557d095863fbb9ecb","user":{"login":"vanzin","id":1694083,"node_id":"MDQ6VXNlcjE2OTQwODM=","avatar_url":"https://avatars0.githubusercontent.com/u/1694083?v=4","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"body":"Still waiting for a word on this.","created_at":"2018-10-12T15:41:48Z","updated_at":"2018-10-12T15:41:48Z","html_url":"https://github.com/apache/spark/pull/20761#discussion_r224830769","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/20761","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/224830769"},"html":{"href":"https://github.com/apache/spark/pull/20761#discussion_r224830769"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/20761"}},"in_reply_to_id":224270816},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/20761","id":173527330,"node_id":"MDExOlB1bGxSZXF1ZXN0MTczNTI3MzMw","html_url":"https://github.com/apache/spark/pull/20761","diff_url":"https://github.com/apache/spark/pull/20761.diff","patch_url":"https://github.com/apache/spark/pull/20761.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/20761","number":20761,"state":"open","locked":false,"title":"[SPARK-20327][CORE][YARN] Add CLI support for YARN custom resources, like GPUs","user":{"login":"szyszy","id":954799,"node_id":"MDQ6VXNlcjk1NDc5OQ==","avatar_url":"https://avatars3.githubusercontent.com/u/954799?v=4","gravatar_id":"","url":"https://api.github.com/users/szyszy","html_url":"https://github.com/szyszy","followers_url":"https://api.github.com/users/szyszy/followers","following_url":"https://api.github.com/users/szyszy/following{/other_user}","gists_url":"https://api.github.com/users/szyszy/gists{/gist_id}","starred_url":"https://api.github.com/users/szyszy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/szyszy/subscriptions","organizations_url":"https://api.github.com/users/szyszy/orgs","repos_url":"https://api.github.com/users/szyszy/repos","events_url":"https://api.github.com/users/szyszy/events{/privacy}","received_events_url":"https://api.github.com/users/szyszy/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nThis PR adds CLI support for YARN custom resources, e.g. GPUs and any other resources YARN defines.\r\nThe custom resources are defined with Spark properties, no additional CLI arguments were introduced.\r\n\r\n\r\nThe properties can be defined in the following form:\r\n\r\n**AM resources, client mode:** \r\nFormat: `spark.yarn.am.resource.<resource-name>`\r\nThe property name follows the naming convention of YARN AM cores / memory properties: `spark.yarn.am.memory and spark.yarn.am.cores\r\n`\r\n\r\n**Driver resources, cluster mode:** \r\nFormat: `spark.yarn.driver.resource.<resource-name>`\r\nThe property name follows the naming convention of driver cores / memory properties: `spark.driver.memory and spark.driver.cores.`\r\n\r\n**Executor resources:** \r\nFormat: `spark.yarn.executor.resource.<resource-name>`\r\nThe property name follows the naming convention of executor cores / memory properties: `spark.executor.memory / spark.executor.cores`.\r\n\r\n\r\nFor the driver resources (cluster mode) and executor resources properties, we use the `yarn` prefix here as custom resource types are specific to YARN, currently.\r\n\r\n\r\n**Validation:** \r\nPlease note that a validation logic is added to avoid having requested resources defined in 2 ways, for example defining the following configs:\r\n```\r\n\"--conf\", \"spark.driver.memory=2G\",\r\n\"--conf\", \"spark.yarn.driver.resource.memory=1G\"\r\n```\r\n\r\nwill not start execution and will print an error message.\r\n\r\n## How was this patch tested?\r\nUnit tests + manual execution with Hadoop2 and Hadoop 3 builds.\r\n\r\nTesting have been performed on a real cluster with Spark and YARN configured: \r\nCluster and client mode\r\nRequest Resource Types with lowercase and uppercase units\r\nStart Spark job with only requesting standard resources (mem / cpu)\r\nError handling cases: \r\n- Request unknown resource type\r\n- Request Resource type (either memory / cpu) with duplicate configs at the same time (e.g. with this config: \r\n```\r\n--conf spark.yarn.am.resource.memory=1G \\\r\n  --conf spark.yarn.driver.resource.memory=2G \\\r\n  --conf spark.yarn.executor.resource.memory=3G \\\r\n```\r\n), ResourceTypeValidator handles these cases well, so it is not permitted\r\n- Request standard resource (memory / cpu) with the new style configs, e.g. --conf spark.yarn.am.resource.memory=1G,  this is not permitted and handled well.\r\n\r\n\r\nAn example about how I ran the testcases: \r\n```\r\ncd ~;export HADOOP_CONF_DIR=/opt/hadoop/etc/hadoop/;\r\n./spark-2.4.0-SNAPSHOT-bin-custom-spark/bin/spark-submit \\\r\n  --class org.apache.spark.examples.SparkPi \\\r\n  --master yarn \\\r\n  --deploy-mode cluster \\\r\n  --driver-memory 1G \\\r\n  --driver-cores 1 \\\r\n  --executor-memory 1G \\\r\n  --executor-cores 1 \\\r\n  --conf spark.logConf=true \\\r\n  --conf spark.yarn.executor.resource.gpu=3G \\\r\n  --verbose \\\r\n  ./spark-2.4.0-SNAPSHOT-bin-custom-spark/examples/jars/spark-examples_2.11-2.4.0-SNAPSHOT.jar \\\r\n  10;\r\n```\r\n","created_at":"2018-03-07T17:52:05Z","updated_at":"2018-10-12T15:41:48Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8efd67b9ff8f504ee427fab9552a65e4d451ad69","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/20761/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/20761/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/20761/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/f3955f8f7716ef0369ed3f2289eda2cc921c8937","head":{"label":"szyszy:SPARK-20327","ref":"SPARK-20327","sha":"f3955f8f7716ef0369ed3f2289eda2cc921c8937","user":{"login":"szyszy","id":954799,"node_id":"MDQ6VXNlcjk1NDc5OQ==","avatar_url":"https://avatars3.githubusercontent.com/u/954799?v=4","gravatar_id":"","url":"https://api.github.com/users/szyszy","html_url":"https://github.com/szyszy","followers_url":"https://api.github.com/users/szyszy/followers","following_url":"https://api.github.com/users/szyszy/following{/other_user}","gists_url":"https://api.github.com/users/szyszy/gists{/gist_id}","starred_url":"https://api.github.com/users/szyszy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/szyszy/subscriptions","organizations_url":"https://api.github.com/users/szyszy/orgs","repos_url":"https://api.github.com/users/szyszy/repos","events_url":"https://api.github.com/users/szyszy/events{/privacy}","received_events_url":"https://api.github.com/users/szyszy/received_events","type":"User","site_admin":false},"repo":{"id":121649508,"node_id":"MDEwOlJlcG9zaXRvcnkxMjE2NDk1MDg=","name":"spark","full_name":"szyszy/spark","private":false,"owner":{"login":"szyszy","id":954799,"node_id":"MDQ6VXNlcjk1NDc5OQ==","avatar_url":"https://avatars3.githubusercontent.com/u/954799?v=4","gravatar_id":"","url":"https://api.github.com/users/szyszy","html_url":"https://github.com/szyszy","followers_url":"https://api.github.com/users/szyszy/followers","following_url":"https://api.github.com/users/szyszy/following{/other_user}","gists_url":"https://api.github.com/users/szyszy/gists{/gist_id}","starred_url":"https://api.github.com/users/szyszy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/szyszy/subscriptions","organizations_url":"https://api.github.com/users/szyszy/orgs","repos_url":"https://api.github.com/users/szyszy/repos","events_url":"https://api.github.com/users/szyszy/events{/privacy}","received_events_url":"https://api.github.com/users/szyszy/received_events","type":"User","site_admin":false},"html_url":"https://github.com/szyszy/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/szyszy/spark","forks_url":"https://api.github.com/repos/szyszy/spark/forks","keys_url":"https://api.github.com/repos/szyszy/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/szyszy/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/szyszy/spark/teams","hooks_url":"https://api.github.com/repos/szyszy/spark/hooks","issue_events_url":"https://api.github.com/repos/szyszy/spark/issues/events{/number}","events_url":"https://api.github.com/repos/szyszy/spark/events","assignees_url":"https://api.github.com/repos/szyszy/spark/assignees{/user}","branches_url":"https://api.github.com/repos/szyszy/spark/branches{/branch}","tags_url":"https://api.github.com/repos/szyszy/spark/tags","blobs_url":"https://api.github.com/repos/szyszy/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/szyszy/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/szyszy/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/szyszy/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/szyszy/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/szyszy/spark/languages","stargazers_url":"https://api.github.com/repos/szyszy/spark/stargazers","contributors_url":"https://api.github.com/repos/szyszy/spark/contributors","subscribers_url":"https://api.github.com/repos/szyszy/spark/subscribers","subscription_url":"https://api.github.com/repos/szyszy/spark/subscription","commits_url":"https://api.github.com/repos/szyszy/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/szyszy/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/szyszy/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/szyszy/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/szyszy/spark/contents/{+path}","compare_url":"https://api.github.com/repos/szyszy/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/szyszy/spark/merges","archive_url":"https://api.github.com/repos/szyszy/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/szyszy/spark/downloads","issues_url":"https://api.github.com/repos/szyszy/spark/issues{/number}","pulls_url":"https://api.github.com/repos/szyszy/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/szyszy/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/szyszy/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/szyszy/spark/labels{/name}","releases_url":"https://api.github.com/repos/szyszy/spark/releases{/id}","deployments_url":"https://api.github.com/repos/szyszy/spark/deployments","created_at":"2018-02-15T16:02:30Z","updated_at":"2018-02-15T16:02:57Z","pushed_at":"2018-10-11T20:42:30Z","git_url":"git://github.com/szyszy/spark.git","ssh_url":"git@github.com:szyszy/spark.git","clone_url":"https://github.com/szyszy/spark.git","svn_url":"https://github.com/szyszy/spark","homepage":null,"size":281427,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"80813e198033cd63cc6100ee6ffe7d1eb1dff27b","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"node_id":"MDEwOlJlcG9zaXRvcnkxNzE2NTY1OA==","name":"spark","full_name":"apache/spark","private":false,"owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2018-10-12T13:02:17Z","pushed_at":"2018-10-12T15:30:22Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":284161,"stargazers_count":19065,"watchers_count":19065,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":17123,"mirror_url":null,"archived":false,"open_issues_count":522,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":17123,"open_issues":522,"watchers":19065,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/20761"},"html":{"href":"https://github.com/apache/spark/pull/20761"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/20761"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/20761/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/20761/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/20761/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/f3955f8f7716ef0369ed3f2289eda2cc921c8937"}},"author_association":"NONE"}}
{"actor":{"display_login":"vanzin"}}