{"action":"created","comment":{"url":"https://api.github.com/repos/cloudera/livy/pulls/comments/81262296","pull_request_review_id":2255291,"id":81262296,"diff_hunk":"@@ -26,17 +26,79 @@ import scala.concurrent.{Await, ExecutionContext, Future}\n import scala.concurrent.duration.Duration\n \n import com.cloudera.livy.{LivyConf, Logging}\n+import com.cloudera.livy.server.batch.{BatchRecoveryMetadata, BatchSession}\n+import com.cloudera.livy.server.recovery.SessionStore\n \n object SessionManager {\n+  val SESSION_RECOVERY_MODE_OFF = \"off\"\n+  val SESSION_RECOVERY_MODE_RECOVERY = \"recovery\"\n   val SESSION_TIMEOUT = LivyConf.Entry(\"livy.server.session.timeout\", \"1h\")\n }\n \n-class SessionManager[S <: Session](val livyConf: LivyConf) extends Logging {\n+// TODO Replace SessionManager with this class when interactive sessions support recovery.\n+class BatchSessionManager(\n+    livyConf: LivyConf,\n+    sessionStore: SessionStore,\n+    mockSessions: Option[Seq[BatchSession]] = None)\n+  extends SessionManager[BatchSession](livyConf) {\n \n-  private implicit def executor: ExecutionContext = ExecutionContext.global\n+  import SessionManager._\n \n-  private[this] final val idCounter = new AtomicInteger()\n-  private[this] final val sessions = mutable.LinkedHashMap[Int, S]()\n+  private val sessionType: String = \"batch\"\n+\n+  mockSessions.getOrElse(recover()).foreach(register)\n+\n+  override def nextId(): Int = synchronized {\n+    val id = idCounter.getAndIncrement()\n+    sessionStore.saveNextSessionId(sessionType, idCounter.get())\n+    id\n+  }\n+\n+  override def delete(session: BatchSession): Future[Unit] = {\n+    session.stop().map { case _ =>\n+      sessionStore.remove(sessionType, session.id)\n+      synchronized {\n+        sessions.remove(session.id)\n+      }\n+    }\n+  }\n+\n+  override def shutdown(): Unit = {\n+    val recoveryEnabled = livyConf.get(LivyConf.RECOVERY_MODE) != SESSION_RECOVERY_MODE_OFF\n+    if (!recoveryEnabled) {\n+      super.shutdown()\n+    }\n+  }\n+\n+  private def recover(): Seq[BatchSession] = {\n+    // Recover next session id from state store and create SessionManager.\n+    idCounter.set(sessionStore.getNextSessionId(sessionType))\n+\n+    // Retrieve session recovery metadata from state store.\n+    val sessionMetadata = sessionStore.getAllSessions[BatchRecoveryMetadata](sessionType)\n+\n+    // Call recoveryFun to recover session from session recovery metadata.","path":"server/src/main/scala/com/cloudera/livy/sessions/SessionManager.scala","position":59,"original_position":59,"commit_id":"51bd97d513b778fbfec51005e3bf7c89c8111893","original_commit_id":"51bd97d513b778fbfec51005e3bf7c89c8111893","user":{"login":"tc0312","id":8400790,"avatar_url":"https://avatars.githubusercontent.com/u/8400790?v=3","gravatar_id":"","url":"https://api.github.com/users/tc0312","html_url":"https://github.com/tc0312","followers_url":"https://api.github.com/users/tc0312/followers","following_url":"https://api.github.com/users/tc0312/following{/other_user}","gists_url":"https://api.github.com/users/tc0312/gists{/gist_id}","starred_url":"https://api.github.com/users/tc0312/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tc0312/subscriptions","organizations_url":"https://api.github.com/users/tc0312/orgs","repos_url":"https://api.github.com/users/tc0312/repos","events_url":"https://api.github.com/users/tc0312/events{/privacy}","received_events_url":"https://api.github.com/users/tc0312/received_events","type":"User","site_admin":false},"body":"Fixed.","created_at":"2016-09-30T00:34:33Z","updated_at":"2016-09-30T00:34:33Z","html_url":"https://github.com/cloudera/livy/pull/195#discussion_r81262296","pull_request_url":"https://api.github.com/repos/cloudera/livy/pulls/195","_links":{"self":{"href":"https://api.github.com/repos/cloudera/livy/pulls/comments/81262296"},"html":{"href":"https://github.com/cloudera/livy/pull/195#discussion_r81262296"},"pull_request":{"href":"https://api.github.com/repos/cloudera/livy/pulls/195"}}},"pull_request":{"url":"https://api.github.com/repos/cloudera/livy/pulls/195","id":85996823,"html_url":"https://github.com/cloudera/livy/pull/195","diff_url":"https://github.com/cloudera/livy/pull/195.diff","patch_url":"https://github.com/cloudera/livy/pull/195.patch","issue_url":"https://api.github.com/repos/cloudera/livy/issues/195","number":195,"state":"open","locked":false,"title":"LIVY-211. Implemented session recovery for batch session. Only work with YARN.","user":{"login":"tc0312","id":8400790,"avatar_url":"https://avatars.githubusercontent.com/u/8400790?v=3","gravatar_id":"","url":"https://api.github.com/users/tc0312","html_url":"https://github.com/tc0312","followers_url":"https://api.github.com/users/tc0312/followers","following_url":"https://api.github.com/users/tc0312/following{/other_user}","gists_url":"https://api.github.com/users/tc0312/gists{/gist_id}","starred_url":"https://api.github.com/users/tc0312/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tc0312/subscriptions","organizations_url":"https://api.github.com/users/tc0312/orgs","repos_url":"https://api.github.com/users/tc0312/repos","events_url":"https://api.github.com/users/tc0312/events{/privacy}","received_events_url":"https://api.github.com/users/tc0312/received_events","type":"User","site_admin":false},"body":"livy-server can now recover batch sessions after restart.\r\n\r\nAdded a few new LivyConf:\r\n- livy.server.recovery.mode: Possible values: [\"off\", \"recovery\"]. Default is \"off\".\r\n - off: Sessions are not persisted. Every time Livy server restarts, it loses all previous sessions. It also stops all batch sessions before shutting down.\r\n - recovery: Sessions are persisted to the state store. Requires YARN.\r\n\r\n- livy.server.recovery.state-store: Possible values: [\"filesystem\", \"zookeeper\"].\r\n - filesystem: Store state on a file system.\r\n - zookeeper: Store state in a Zookeeper instance.\r\n\r\n- livy.server.recovery.state-store.url\r\n - For filesystem state store, the path of the state store directory. e.g. file:///tmp/livy or hdfs:///\r\n - For zookeeper, the address to the Zookeeper servers. e.g. host1:port1,host2:port2","created_at":"2016-09-20T15:17:16Z","updated_at":"2016-09-30T00:34:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7ee7a4994459698c75172ddb609601bf44cb7a74","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/cloudera/livy/pulls/195/commits","review_comments_url":"https://api.github.com/repos/cloudera/livy/pulls/195/comments","review_comment_url":"https://api.github.com/repos/cloudera/livy/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cloudera/livy/issues/195/comments","statuses_url":"https://api.github.com/repos/cloudera/livy/statuses/51bd97d513b778fbfec51005e3bf7c89c8111893","head":{"label":"hdinsight:batch-recovery","ref":"batch-recovery","sha":"51bd97d513b778fbfec51005e3bf7c89c8111893","user":{"login":"hdinsight","id":5924018,"avatar_url":"https://avatars.githubusercontent.com/u/5924018?v=3","gravatar_id":"","url":"https://api.github.com/users/hdinsight","html_url":"https://github.com/hdinsight","followers_url":"https://api.github.com/users/hdinsight/followers","following_url":"https://api.github.com/users/hdinsight/following{/other_user}","gists_url":"https://api.github.com/users/hdinsight/gists{/gist_id}","starred_url":"https://api.github.com/users/hdinsight/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hdinsight/subscriptions","organizations_url":"https://api.github.com/users/hdinsight/orgs","repos_url":"https://api.github.com/users/hdinsight/repos","events_url":"https://api.github.com/users/hdinsight/events{/privacy}","received_events_url":"https://api.github.com/users/hdinsight/received_events","type":"Organization","site_admin":false},"repo":{"id":47885827,"name":"livy","full_name":"hdinsight/livy","owner":{"login":"hdinsight","id":5924018,"avatar_url":"https://avatars.githubusercontent.com/u/5924018?v=3","gravatar_id":"","url":"https://api.github.com/users/hdinsight","html_url":"https://github.com/hdinsight","followers_url":"https://api.github.com/users/hdinsight/followers","following_url":"https://api.github.com/users/hdinsight/following{/other_user}","gists_url":"https://api.github.com/users/hdinsight/gists{/gist_id}","starred_url":"https://api.github.com/users/hdinsight/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hdinsight/subscriptions","organizations_url":"https://api.github.com/users/hdinsight/orgs","repos_url":"https://api.github.com/users/hdinsight/repos","events_url":"https://api.github.com/users/hdinsight/events{/privacy}","received_events_url":"https://api.github.com/users/hdinsight/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hdinsight/livy","description":"Livy is an open source REST interface for interacting with Apache Spark from anywhere","fork":true,"url":"https://api.github.com/repos/hdinsight/livy","forks_url":"https://api.github.com/repos/hdinsight/livy/forks","keys_url":"https://api.github.com/repos/hdinsight/livy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hdinsight/livy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hdinsight/livy/teams","hooks_url":"https://api.github.com/repos/hdinsight/livy/hooks","issue_events_url":"https://api.github.com/repos/hdinsight/livy/issues/events{/number}","events_url":"https://api.github.com/repos/hdinsight/livy/events","assignees_url":"https://api.github.com/repos/hdinsight/livy/assignees{/user}","branches_url":"https://api.github.com/repos/hdinsight/livy/branches{/branch}","tags_url":"https://api.github.com/repos/hdinsight/livy/tags","blobs_url":"https://api.github.com/repos/hdinsight/livy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hdinsight/livy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hdinsight/livy/git/refs{/sha}","trees_url":"https://api.github.com/repos/hdinsight/livy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hdinsight/livy/statuses/{sha}","languages_url":"https://api.github.com/repos/hdinsight/livy/languages","stargazers_url":"https://api.github.com/repos/hdinsight/livy/stargazers","contributors_url":"https://api.github.com/repos/hdinsight/livy/contributors","subscribers_url":"https://api.github.com/repos/hdinsight/livy/subscribers","subscription_url":"https://api.github.com/repos/hdinsight/livy/subscription","commits_url":"https://api.github.com/repos/hdinsight/livy/commits{/sha}","git_commits_url":"https://api.github.com/repos/hdinsight/livy/git/commits{/sha}","comments_url":"https://api.github.com/repos/hdinsight/livy/comments{/number}","issue_comment_url":"https://api.github.com/repos/hdinsight/livy/issues/comments{/number}","contents_url":"https://api.github.com/repos/hdinsight/livy/contents/{+path}","compare_url":"https://api.github.com/repos/hdinsight/livy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hdinsight/livy/merges","archive_url":"https://api.github.com/repos/hdinsight/livy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hdinsight/livy/downloads","issues_url":"https://api.github.com/repos/hdinsight/livy/issues{/number}","pulls_url":"https://api.github.com/repos/hdinsight/livy/pulls{/number}","milestones_url":"https://api.github.com/repos/hdinsight/livy/milestones{/number}","notifications_url":"https://api.github.com/repos/hdinsight/livy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hdinsight/livy/labels{/name}","releases_url":"https://api.github.com/repos/hdinsight/livy/releases{/id}","deployments_url":"https://api.github.com/repos/hdinsight/livy/deployments","created_at":"2015-12-12T16:39:58Z","updated_at":"2016-09-08T03:26:58Z","pushed_at":"2016-09-29T23:54:40Z","git_url":"git://github.com/hdinsight/livy.git","ssh_url":"git@github.com:hdinsight/livy.git","clone_url":"https://github.com/hdinsight/livy.git","svn_url":"https://github.com/hdinsight/livy","homepage":"","size":1605,"stargazers_count":7,"watchers_count":7,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":0,"forks":3,"open_issues":0,"watchers":7,"default_branch":"master"}},"base":{"label":"cloudera:master","ref":"master","sha":"c9a7a898f9aa0dff7039141f634ebe7db6756a69","user":{"login":"cloudera","id":87383,"avatar_url":"https://avatars.githubusercontent.com/u/87383?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudera","html_url":"https://github.com/cloudera","followers_url":"https://api.github.com/users/cloudera/followers","following_url":"https://api.github.com/users/cloudera/following{/other_user}","gists_url":"https://api.github.com/users/cloudera/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudera/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudera/subscriptions","organizations_url":"https://api.github.com/users/cloudera/orgs","repos_url":"https://api.github.com/users/cloudera/repos","events_url":"https://api.github.com/users/cloudera/events{/privacy}","received_events_url":"https://api.github.com/users/cloudera/received_events","type":"Organization","site_admin":false},"repo":{"id":46328956,"name":"livy","full_name":"cloudera/livy","owner":{"login":"cloudera","id":87383,"avatar_url":"https://avatars.githubusercontent.com/u/87383?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudera","html_url":"https://github.com/cloudera","followers_url":"https://api.github.com/users/cloudera/followers","following_url":"https://api.github.com/users/cloudera/following{/other_user}","gists_url":"https://api.github.com/users/cloudera/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudera/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudera/subscriptions","organizations_url":"https://api.github.com/users/cloudera/orgs","repos_url":"https://api.github.com/users/cloudera/repos","events_url":"https://api.github.com/users/cloudera/events{/privacy}","received_events_url":"https://api.github.com/users/cloudera/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cloudera/livy","description":"Livy is an open source REST interface for interacting with Apache Spark from anywhere","fork":false,"url":"https://api.github.com/repos/cloudera/livy","forks_url":"https://api.github.com/repos/cloudera/livy/forks","keys_url":"https://api.github.com/repos/cloudera/livy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cloudera/livy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cloudera/livy/teams","hooks_url":"https://api.github.com/repos/cloudera/livy/hooks","issue_events_url":"https://api.github.com/repos/cloudera/livy/issues/events{/number}","events_url":"https://api.github.com/repos/cloudera/livy/events","assignees_url":"https://api.github.com/repos/cloudera/livy/assignees{/user}","branches_url":"https://api.github.com/repos/cloudera/livy/branches{/branch}","tags_url":"https://api.github.com/repos/cloudera/livy/tags","blobs_url":"https://api.github.com/repos/cloudera/livy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cloudera/livy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cloudera/livy/git/refs{/sha}","trees_url":"https://api.github.com/repos/cloudera/livy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cloudera/livy/statuses/{sha}","languages_url":"https://api.github.com/repos/cloudera/livy/languages","stargazers_url":"https://api.github.com/repos/cloudera/livy/stargazers","contributors_url":"https://api.github.com/repos/cloudera/livy/contributors","subscribers_url":"https://api.github.com/repos/cloudera/livy/subscribers","subscription_url":"https://api.github.com/repos/cloudera/livy/subscription","commits_url":"https://api.github.com/repos/cloudera/livy/commits{/sha}","git_commits_url":"https://api.github.com/repos/cloudera/livy/git/commits{/sha}","comments_url":"https://api.github.com/repos/cloudera/livy/comments{/number}","issue_comment_url":"https://api.github.com/repos/cloudera/livy/issues/comments{/number}","contents_url":"https://api.github.com/repos/cloudera/livy/contents/{+path}","compare_url":"https://api.github.com/repos/cloudera/livy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cloudera/livy/merges","archive_url":"https://api.github.com/repos/cloudera/livy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cloudera/livy/downloads","issues_url":"https://api.github.com/repos/cloudera/livy/issues{/number}","pulls_url":"https://api.github.com/repos/cloudera/livy/pulls{/number}","milestones_url":"https://api.github.com/repos/cloudera/livy/milestones{/number}","notifications_url":"https://api.github.com/repos/cloudera/livy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cloudera/livy/labels{/name}","releases_url":"https://api.github.com/repos/cloudera/livy/releases{/id}","deployments_url":"https://api.github.com/repos/cloudera/livy/deployments","created_at":"2015-11-17T06:55:45Z","updated_at":"2016-09-29T12:21:26Z","pushed_at":"2016-09-29T23:54:41Z","git_url":"git://github.com/cloudera/livy.git","ssh_url":"git@github.com:cloudera/livy.git","clone_url":"https://github.com/cloudera/livy.git","svn_url":"https://github.com/cloudera/livy","homepage":"http://livy.io/","size":6849,"stargazers_count":256,"watchers_count":256,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":101,"mirror_url":null,"open_issues_count":8,"forks":101,"open_issues":8,"watchers":256,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cloudera/livy/pulls/195"},"html":{"href":"https://github.com/cloudera/livy/pull/195"},"issue":{"href":"https://api.github.com/repos/cloudera/livy/issues/195"},"comments":{"href":"https://api.github.com/repos/cloudera/livy/issues/195/comments"},"review_comments":{"href":"https://api.github.com/repos/cloudera/livy/pulls/195/comments"},"review_comment":{"href":"https://api.github.com/repos/cloudera/livy/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cloudera/livy/pulls/195/commits"},"statuses":{"href":"https://api.github.com/repos/cloudera/livy/statuses/51bd97d513b778fbfec51005e3bf7c89c8111893"}}}}
{"actor":{"display_login":"tc0312"}}