{"action":"created","comment":{"url":"https://api.github.com/repos/cloudera/livy/pulls/comments/73948352","id":73948352,"diff_hunk":"@@ -0,0 +1,230 @@\n+/*\n+ * Licensed to Cloudera, Inc. under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  Cloudera, Inc. licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package com.cloudera.livy.utils\n+\n+import java.util.concurrent.Executors\n+import java.util.concurrent.TimeoutException\n+\n+import scala.annotation.tailrec\n+import scala.collection.JavaConverters._\n+import scala.collection.mutable.ArrayBuffer\n+import scala.concurrent._\n+import scala.concurrent.duration._\n+import scala.language.postfixOps\n+\n+import org.apache.hadoop.yarn.api.records.{ApplicationId, ApplicationReport, FinalApplicationStatus, YarnApplicationState}\n+import org.apache.hadoop.yarn.client.api.YarnClient\n+import org.apache.hadoop.yarn.conf.YarnConfiguration\n+\n+import com.cloudera.livy.{LivyConf, Logging, Utils}\n+import com.cloudera.livy.rsc.{Utils => RSCUtils}\n+import com.cloudera.livy.util.LineBufferedProcess\n+\n+object SparkYarnApp extends Logging {\n+  // Create a non daemon thread pool to run getAppIdFromTag().\n+  // getAppIdFromTag() might take a while and should not use the global default thread pool.\n+  private implicit val ec = ExecutionContext.fromExecutor(\n+    Executors.newCachedThreadPool(RSCUtils.newDaemonThreadFactory(\"getAppIdFromTag-%d\")))\n+\n+  // YarnClient is thread safe. Create once, share it across threads.\n+  lazy val yarnClient = {\n+    val c = YarnClient.createYarnClient()\n+    c.init(new YarnConfiguration())\n+    c.start()\n+    c\n+  }\n+\n+  def fromAppTag(\n+      appTag: String,\n+      process: Option[LineBufferedProcess],\n+      listener: Option[SparkAppListener],\n+      livyConf: LivyConf): SparkYarnApp = {\n+    val deadline = getYarnTagToAppIdTimeout(livyConf).fromNow\n+    val pollInterval = getYarnPollInterval(livyConf)\n+    new SparkYarnApp(getAppIdFromTag(appTag, pollInterval, deadline), process, listener, livyConf)\n+  }\n+\n+  /**\n+   * Find the corresponding YARN application id from an application tag.\n+   *\n+   * @param appTag The application tag tagged on the target application.\n+   *               If the tag is not unique, it returns the first application it found.\n+   *               It will be converted to lower case to match YARN's behaviour.\n+   * @return ApplicationId or the failure.\n+   */\n+  @tailrec\n+  private def getAppIdFromTag(\n+      appTag: String,\n+      pollInterval: Duration,\n+      deadline: Deadline): ApplicationId = {\n+    val appTagLowerCase = appTag.toLowerCase()\n+\n+    // FIXME Should not loop thru all YARN applications but YarnClient doesn't offer an API.\n+    // Consider calling rmClient in YarnClient directly.\n+    val appType = Set(\"SPARK\").asJava\n+    yarnClient.getApplications(appType).asScala.find(_.getApplicationTags.contains(appTagLowerCase))\n+    match {\n+      case Some(app) => app.getApplicationId\n+      case None =>\n+        if (deadline.isOverdue) {\n+          throw new Exception(s\"No YARN application is tagged with $appTagLowerCase.\")\n+        } else {\n+          Clock.sleep(pollInterval.toMillis)","path":"server/src/main/scala/com/cloudera/livy/utils/SparkYarnApp.scala","position":87,"original_position":87,"commit_id":"305e9b1db052e740169ec101b851d221c43ca245","original_commit_id":"305e9b1db052e740169ec101b851d221c43ca245","user":{"login":"vanzin","id":1694083,"avatar_url":"https://avatars.githubusercontent.com/u/1694083?v=3","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"body":"BTW you could avoid this altogether if you try to merge this logic with the logic in `yarnAppMonitorThread` (`waitForAppId`). That way, you can use a blocking call as much as you want, since in that code you have one thread per session.","created_at":"2016-08-08T20:31:40Z","updated_at":"2016-08-08T20:31:40Z","html_url":"https://github.com/cloudera/livy/pull/163#discussion_r73948352","pull_request_url":"https://api.github.com/repos/cloudera/livy/pulls/163","_links":{"self":{"href":"https://api.github.com/repos/cloudera/livy/pulls/comments/73948352"},"html":{"href":"https://github.com/cloudera/livy/pull/163#discussion_r73948352"},"pull_request":{"href":"https://api.github.com/repos/cloudera/livy/pulls/163"}}},"pull_request":{"url":"https://api.github.com/repos/cloudera/livy/pulls/163","id":76833284,"html_url":"https://github.com/cloudera/livy/pull/163","diff_url":"https://github.com/cloudera/livy/pull/163.diff","patch_url":"https://github.com/cloudera/livy/pull/163.patch","issue_url":"https://api.github.com/repos/cloudera/livy/issues/163","number":163,"state":"open","locked":false,"title":"LIVY-184. Better YARN integration for Batch Session.","user":{"login":"tc0312","id":8400790,"avatar_url":"https://avatars.githubusercontent.com/u/8400790?v=3","gravatar_id":"","url":"https://api.github.com/users/tc0312","html_url":"https://github.com/tc0312","followers_url":"https://api.github.com/users/tc0312/followers","following_url":"https://api.github.com/users/tc0312/following{/other_user}","gists_url":"https://api.github.com/users/tc0312/gists{/gist_id}","starred_url":"https://api.github.com/users/tc0312/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tc0312/subscriptions","organizations_url":"https://api.github.com/users/tc0312/orgs","repos_url":"https://api.github.com/users/tc0312/repos","events_url":"https://api.github.com/users/tc0312/events{/privacy}","received_events_url":"https://api.github.com/users/tc0312/received_events","type":"User","site_admin":false},"body":"When livy is running with YARN master:\r\n- Stop and log session using YARN API.\r\n- Exposed app id.\r\n- Livy now requires Spark 1.6+. Added a check so Livy won't start if it's running with older Spark.\r\n\r\nTest changes:\r\n- Added a test to verify killing the batch session will kill its YARN app.\r\n- Added a test to verify killing the YARN app will change the batch session state.\r\n- Added YarnClient integration in test framework.\r\n- Added halt mode in SimpleSparkApp in test lib for kill session test.\r\n- Added a method to dump logs when Batch test fails.\r\n- Added a Clock interface to allow unit testing to mock out time.","created_at":"2016-07-09T01:18:24Z","updated_at":"2016-08-08T20:31:40Z","closed_at":null,"merged_at":null,"merge_commit_sha":"53523f282dea17298e216c49a92eedc88756985d","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/cloudera/livy/pulls/163/commits","review_comments_url":"https://api.github.com/repos/cloudera/livy/pulls/163/comments","review_comment_url":"https://api.github.com/repos/cloudera/livy/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cloudera/livy/issues/163/comments","statuses_url":"https://api.github.com/repos/cloudera/livy/statuses/305e9b1db052e740169ec101b851d221c43ca245","head":{"label":"hdinsight:batch-yarn","ref":"batch-yarn","sha":"305e9b1db052e740169ec101b851d221c43ca245","user":{"login":"hdinsight","id":5924018,"avatar_url":"https://avatars.githubusercontent.com/u/5924018?v=3","gravatar_id":"","url":"https://api.github.com/users/hdinsight","html_url":"https://github.com/hdinsight","followers_url":"https://api.github.com/users/hdinsight/followers","following_url":"https://api.github.com/users/hdinsight/following{/other_user}","gists_url":"https://api.github.com/users/hdinsight/gists{/gist_id}","starred_url":"https://api.github.com/users/hdinsight/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hdinsight/subscriptions","organizations_url":"https://api.github.com/users/hdinsight/orgs","repos_url":"https://api.github.com/users/hdinsight/repos","events_url":"https://api.github.com/users/hdinsight/events{/privacy}","received_events_url":"https://api.github.com/users/hdinsight/received_events","type":"Organization","site_admin":false},"repo":{"id":47885827,"name":"livy","full_name":"hdinsight/livy","owner":{"login":"hdinsight","id":5924018,"avatar_url":"https://avatars.githubusercontent.com/u/5924018?v=3","gravatar_id":"","url":"https://api.github.com/users/hdinsight","html_url":"https://github.com/hdinsight","followers_url":"https://api.github.com/users/hdinsight/followers","following_url":"https://api.github.com/users/hdinsight/following{/other_user}","gists_url":"https://api.github.com/users/hdinsight/gists{/gist_id}","starred_url":"https://api.github.com/users/hdinsight/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hdinsight/subscriptions","organizations_url":"https://api.github.com/users/hdinsight/orgs","repos_url":"https://api.github.com/users/hdinsight/repos","events_url":"https://api.github.com/users/hdinsight/events{/privacy}","received_events_url":"https://api.github.com/users/hdinsight/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hdinsight/livy","description":"Livy is an open source REST interface for interacting with Apache Spark from anywhere","fork":true,"url":"https://api.github.com/repos/hdinsight/livy","forks_url":"https://api.github.com/repos/hdinsight/livy/forks","keys_url":"https://api.github.com/repos/hdinsight/livy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hdinsight/livy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hdinsight/livy/teams","hooks_url":"https://api.github.com/repos/hdinsight/livy/hooks","issue_events_url":"https://api.github.com/repos/hdinsight/livy/issues/events{/number}","events_url":"https://api.github.com/repos/hdinsight/livy/events","assignees_url":"https://api.github.com/repos/hdinsight/livy/assignees{/user}","branches_url":"https://api.github.com/repos/hdinsight/livy/branches{/branch}","tags_url":"https://api.github.com/repos/hdinsight/livy/tags","blobs_url":"https://api.github.com/repos/hdinsight/livy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hdinsight/livy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hdinsight/livy/git/refs{/sha}","trees_url":"https://api.github.com/repos/hdinsight/livy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hdinsight/livy/statuses/{sha}","languages_url":"https://api.github.com/repos/hdinsight/livy/languages","stargazers_url":"https://api.github.com/repos/hdinsight/livy/stargazers","contributors_url":"https://api.github.com/repos/hdinsight/livy/contributors","subscribers_url":"https://api.github.com/repos/hdinsight/livy/subscribers","subscription_url":"https://api.github.com/repos/hdinsight/livy/subscription","commits_url":"https://api.github.com/repos/hdinsight/livy/commits{/sha}","git_commits_url":"https://api.github.com/repos/hdinsight/livy/git/commits{/sha}","comments_url":"https://api.github.com/repos/hdinsight/livy/comments{/number}","issue_comment_url":"https://api.github.com/repos/hdinsight/livy/issues/comments{/number}","contents_url":"https://api.github.com/repos/hdinsight/livy/contents/{+path}","compare_url":"https://api.github.com/repos/hdinsight/livy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hdinsight/livy/merges","archive_url":"https://api.github.com/repos/hdinsight/livy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hdinsight/livy/downloads","issues_url":"https://api.github.com/repos/hdinsight/livy/issues{/number}","pulls_url":"https://api.github.com/repos/hdinsight/livy/pulls{/number}","milestones_url":"https://api.github.com/repos/hdinsight/livy/milestones{/number}","notifications_url":"https://api.github.com/repos/hdinsight/livy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hdinsight/livy/labels{/name}","releases_url":"https://api.github.com/repos/hdinsight/livy/releases{/id}","deployments_url":"https://api.github.com/repos/hdinsight/livy/deployments","created_at":"2015-12-12T16:39:58Z","updated_at":"2016-08-01T06:20:59Z","pushed_at":"2016-08-05T00:57:58Z","git_url":"git://github.com/hdinsight/livy.git","ssh_url":"git@github.com:hdinsight/livy.git","clone_url":"https://github.com/hdinsight/livy.git","svn_url":"https://github.com/hdinsight/livy","homepage":"","size":1404,"stargazers_count":6,"watchers_count":6,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":0,"forks":2,"open_issues":0,"watchers":6,"default_branch":"master"}},"base":{"label":"cloudera:master","ref":"master","sha":"d1386405f1ada8be5e899594cecbd34fffdba244","user":{"login":"cloudera","id":87383,"avatar_url":"https://avatars.githubusercontent.com/u/87383?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudera","html_url":"https://github.com/cloudera","followers_url":"https://api.github.com/users/cloudera/followers","following_url":"https://api.github.com/users/cloudera/following{/other_user}","gists_url":"https://api.github.com/users/cloudera/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudera/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudera/subscriptions","organizations_url":"https://api.github.com/users/cloudera/orgs","repos_url":"https://api.github.com/users/cloudera/repos","events_url":"https://api.github.com/users/cloudera/events{/privacy}","received_events_url":"https://api.github.com/users/cloudera/received_events","type":"Organization","site_admin":false},"repo":{"id":46328956,"name":"livy","full_name":"cloudera/livy","owner":{"login":"cloudera","id":87383,"avatar_url":"https://avatars.githubusercontent.com/u/87383?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudera","html_url":"https://github.com/cloudera","followers_url":"https://api.github.com/users/cloudera/followers","following_url":"https://api.github.com/users/cloudera/following{/other_user}","gists_url":"https://api.github.com/users/cloudera/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudera/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudera/subscriptions","organizations_url":"https://api.github.com/users/cloudera/orgs","repos_url":"https://api.github.com/users/cloudera/repos","events_url":"https://api.github.com/users/cloudera/events{/privacy}","received_events_url":"https://api.github.com/users/cloudera/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cloudera/livy","description":"Livy is an open source REST interface for interacting with Apache Spark from anywhere","fork":false,"url":"https://api.github.com/repos/cloudera/livy","forks_url":"https://api.github.com/repos/cloudera/livy/forks","keys_url":"https://api.github.com/repos/cloudera/livy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cloudera/livy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cloudera/livy/teams","hooks_url":"https://api.github.com/repos/cloudera/livy/hooks","issue_events_url":"https://api.github.com/repos/cloudera/livy/issues/events{/number}","events_url":"https://api.github.com/repos/cloudera/livy/events","assignees_url":"https://api.github.com/repos/cloudera/livy/assignees{/user}","branches_url":"https://api.github.com/repos/cloudera/livy/branches{/branch}","tags_url":"https://api.github.com/repos/cloudera/livy/tags","blobs_url":"https://api.github.com/repos/cloudera/livy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cloudera/livy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cloudera/livy/git/refs{/sha}","trees_url":"https://api.github.com/repos/cloudera/livy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cloudera/livy/statuses/{sha}","languages_url":"https://api.github.com/repos/cloudera/livy/languages","stargazers_url":"https://api.github.com/repos/cloudera/livy/stargazers","contributors_url":"https://api.github.com/repos/cloudera/livy/contributors","subscribers_url":"https://api.github.com/repos/cloudera/livy/subscribers","subscription_url":"https://api.github.com/repos/cloudera/livy/subscription","commits_url":"https://api.github.com/repos/cloudera/livy/commits{/sha}","git_commits_url":"https://api.github.com/repos/cloudera/livy/git/commits{/sha}","comments_url":"https://api.github.com/repos/cloudera/livy/comments{/number}","issue_comment_url":"https://api.github.com/repos/cloudera/livy/issues/comments{/number}","contents_url":"https://api.github.com/repos/cloudera/livy/contents/{+path}","compare_url":"https://api.github.com/repos/cloudera/livy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cloudera/livy/merges","archive_url":"https://api.github.com/repos/cloudera/livy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cloudera/livy/downloads","issues_url":"https://api.github.com/repos/cloudera/livy/issues{/number}","pulls_url":"https://api.github.com/repos/cloudera/livy/pulls{/number}","milestones_url":"https://api.github.com/repos/cloudera/livy/milestones{/number}","notifications_url":"https://api.github.com/repos/cloudera/livy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cloudera/livy/labels{/name}","releases_url":"https://api.github.com/repos/cloudera/livy/releases{/id}","deployments_url":"https://api.github.com/repos/cloudera/livy/deployments","created_at":"2015-11-17T06:55:45Z","updated_at":"2016-08-08T08:01:33Z","pushed_at":"2016-08-05T18:01:19Z","git_url":"git://github.com/cloudera/livy.git","ssh_url":"git@github.com:cloudera/livy.git","clone_url":"https://github.com/cloudera/livy.git","svn_url":"https://github.com/cloudera/livy","homepage":"http://livy.io/","size":2041,"stargazers_count":209,"watchers_count":209,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":85,"mirror_url":null,"open_issues_count":4,"forks":85,"open_issues":4,"watchers":209,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cloudera/livy/pulls/163"},"html":{"href":"https://github.com/cloudera/livy/pull/163"},"issue":{"href":"https://api.github.com/repos/cloudera/livy/issues/163"},"comments":{"href":"https://api.github.com/repos/cloudera/livy/issues/163/comments"},"review_comments":{"href":"https://api.github.com/repos/cloudera/livy/pulls/163/comments"},"review_comment":{"href":"https://api.github.com/repos/cloudera/livy/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cloudera/livy/pulls/163/commits"},"statuses":{"href":"https://api.github.com/repos/cloudera/livy/statuses/305e9b1db052e740169ec101b851d221c43ca245"}}}}
{"actor":{"display_login":"vanzin"}}