{"action":"created","comment":{"url":"https://api.github.com/repos/mesosphere/marathon/pulls/comments/62008373","id":62008373,"diff_hunk":"@@ -0,0 +1,195 @@\n+package mesosphere.marathon.core.election.impl\n+\n+import akka.actor.ActorSystem\n+import akka.event.EventStream\n+import com.codahale.metrics.MetricRegistry\n+import mesosphere.chaos.http.HttpConf\n+import mesosphere.marathon.MarathonConf\n+import mesosphere.marathon.core.base.ShutdownHooks\n+import mesosphere.marathon.metrics.Metrics\n+import org.apache.curator.{ RetrySleeper, RetryPolicy }\n+import org.apache.curator.framework.{ CuratorFramework, CuratorFrameworkFactory }\n+import org.apache.curator.framework.recipes.leader.{ LeaderLatch, LeaderLatchListener }\n+import org.apache.zookeeper.{ ZooDefs, KeeperException, CreateMode }\n+import org.slf4j.LoggerFactory\n+\n+import scala.concurrent.{ Await, Future }\n+import scala.util.control.NonFatal\n+import scala.concurrent.duration._\n+\n+class CuratorElectionService(\n+  config: MarathonConf,\n+  system: ActorSystem,\n+  eventStream: EventStream,\n+  http: HttpConf,\n+  metrics: Metrics = new Metrics(new MetricRegistry),\n+  hostPort: String,\n+  backoff: ExponentialBackoff,\n+  shutdownHooks: ShutdownHooks) extends ElectionServiceBase(\n+  config, system, eventStream, metrics, backoff, shutdownHooks\n+) {\n+  private lazy val log = LoggerFactory.getLogger(getClass.getName)\n+\n+  private lazy val client = provideCuratorClient()\n+  private var latch: Option[LeaderLatch] = None\n+\n+  override def leaderHostPortImpl: Option[String] = synchronized {\n+    try {\n+      latch match {\n+        case None => None\n+        case Some(l) =>\n+          val participant = l.getLeader\n+          if (participant.isLeader) Some(participant.getId) else None\n+      }\n+    }\n+    catch {\n+      case NonFatal(e) =>\n+        log.error(\"error while getting current leader\", e)\n+        None\n+    }\n+  }\n+\n+  override def offerLeadershipImpl(): Unit = synchronized {\n+    log.info(\"Using HA and therefore offering leadership\")\n+    latch match {\n+      case Some(l) =>\n+        log.error(\"Offering leadership while being candidate\")\n+        l.close()\n+      case _ =>\n+    }\n+    latch = Some(new LeaderLatch(\n+      client, config.zooKeeperLeaderPath + \"-curator\", hostPort, LeaderLatch.CloseMode.NOTIFY_LEADER\n+    ))\n+    latch.get.addListener(Listener) // idem-potent","path":"src/main/scala/mesosphere/marathon/core/election/impl/CuratorElectionService.scala","position":63,"original_position":63,"commit_id":"bc69e4a7ca5ad3c09d85dc3c4f31be0174d55763","original_commit_id":"bc69e4a7ca5ad3c09d85dc3c4f31be0174d55763","user":{"login":"sttts","id":730123,"avatar_url":"https://avatars.githubusercontent.com/u/730123?v=3","gravatar_id":"","url":"https://api.github.com/users/sttts","html_url":"https://github.com/sttts","followers_url":"https://api.github.com/users/sttts/followers","following_url":"https://api.github.com/users/sttts/following{/other_user}","gists_url":"https://api.github.com/users/sttts/gists{/gist_id}","starred_url":"https://api.github.com/users/sttts/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sttts/subscriptions","organizations_url":"https://api.github.com/users/sttts/orgs","repos_url":"https://api.github.com/users/sttts/repos","events_url":"https://api.github.com/users/sttts/events{/privacy}","received_events_url":"https://api.github.com/users/sttts/received_events","type":"User","site_admin":false},"body":"I prefer `get` because \"fail early\" is good.","created_at":"2016-05-04T09:01:51Z","updated_at":"2016-05-04T09:01:51Z","html_url":"https://github.com/mesosphere/marathon/pull/3777#discussion_r62008373","pull_request_url":"https://api.github.com/repos/mesosphere/marathon/pulls/3777","_links":{"self":{"href":"https://api.github.com/repos/mesosphere/marathon/pulls/comments/62008373"},"html":{"href":"https://github.com/mesosphere/marathon/pull/3777#discussion_r62008373"},"pull_request":{"href":"https://api.github.com/repos/mesosphere/marathon/pulls/3777"}}},"pull_request":{"url":"https://api.github.com/repos/mesosphere/marathon/pulls/3777","id":66494612,"html_url":"https://github.com/mesosphere/marathon/pull/3777","diff_url":"https://github.com/mesosphere/marathon/pull/3777.diff","patch_url":"https://github.com/mesosphere/marathon/pull/3777.patch","issue_url":"https://api.github.com/repos/mesosphere/marathon/issues/3777","number":3777,"state":"open","locked":false,"title":"Factor out leader election code into core module and add curator support","user":{"login":"sttts","id":730123,"avatar_url":"https://avatars.githubusercontent.com/u/730123?v=3","gravatar_id":"","url":"https://api.github.com/users/sttts","html_url":"https://github.com/sttts","followers_url":"https://api.github.com/users/sttts/followers","following_url":"https://api.github.com/users/sttts/following{/other_user}","gists_url":"https://api.github.com/users/sttts/gists{/gist_id}","starred_url":"https://api.github.com/users/sttts/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sttts/subscriptions","organizations_url":"https://api.github.com/users/sttts/orgs","repos_url":"https://api.github.com/users/sttts/repos","events_url":"https://api.github.com/users/sttts/events{/privacy}","received_events_url":"https://api.github.com/users/sttts/received_events","type":"User","site_admin":false},"body":"- create core module \"election\"\r\n- move all generic leadership logic from `MarathonSchedulerService` into `ElectionService` implementation\r\n- add curator support which is active by default. A \"always wins the election\" tombstone is created in the twitter common `/leader` group such that old Marathon seamlessly agree on the new master of the new election code\r\n- add a burn-in integration test which re-elects 10 times with ~~half a seconds~~ a random value between 0 to 1000ms in between\r\n\r\nTODO:\r\n- [x] ~~deduplicate/reuse ExponentialBackoff logic~~ – haven't found anything in the code base\r\n- [x] document traits\r\n- [x] ~~analyze and possibly fix burn-in tests with `Thread.sleep(500L)` (probably they fail without the sleep because the `DELETE` call on the leader endpoints in the tests fails)~~ randomized the sleep now, with a 10% chance that the wait is actually zero. This should make us aware of any flakyness\r\n- [ ] decide whether we switch on the curator leader election immediately or avoid two big changes at once: the refactoring + the curator lib switch.","created_at":"2016-04-14T14:33:48Z","updated_at":"2016-05-04T09:01:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7f7850e774d6e3a09aef52beb69b4c2c003c2bb3","assignee":{"login":"sttts","id":730123,"avatar_url":"https://avatars.githubusercontent.com/u/730123?v=3","gravatar_id":"","url":"https://api.github.com/users/sttts","html_url":"https://github.com/sttts","followers_url":"https://api.github.com/users/sttts/followers","following_url":"https://api.github.com/users/sttts/following{/other_user}","gists_url":"https://api.github.com/users/sttts/gists{/gist_id}","starred_url":"https://api.github.com/users/sttts/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sttts/subscriptions","organizations_url":"https://api.github.com/users/sttts/orgs","repos_url":"https://api.github.com/users/sttts/repos","events_url":"https://api.github.com/users/sttts/events{/privacy}","received_events_url":"https://api.github.com/users/sttts/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/mesosphere/marathon/pulls/3777/commits","review_comments_url":"https://api.github.com/repos/mesosphere/marathon/pulls/3777/comments","review_comment_url":"https://api.github.com/repos/mesosphere/marathon/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mesosphere/marathon/issues/3777/comments","statuses_url":"https://api.github.com/repos/mesosphere/marathon/statuses/bc69e4a7ca5ad3c09d85dc3c4f31be0174d55763","head":{"label":"mesosphere:sttts-election-refactoring","ref":"sttts-election-refactoring","sha":"bc69e4a7ca5ad3c09d85dc3c4f31be0174d55763","user":{"login":"mesosphere","id":3458585,"avatar_url":"https://avatars.githubusercontent.com/u/3458585?v=3","gravatar_id":"","url":"https://api.github.com/users/mesosphere","html_url":"https://github.com/mesosphere","followers_url":"https://api.github.com/users/mesosphere/followers","following_url":"https://api.github.com/users/mesosphere/following{/other_user}","gists_url":"https://api.github.com/users/mesosphere/gists{/gist_id}","starred_url":"https://api.github.com/users/mesosphere/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mesosphere/subscriptions","organizations_url":"https://api.github.com/users/mesosphere/orgs","repos_url":"https://api.github.com/users/mesosphere/repos","events_url":"https://api.github.com/users/mesosphere/events{/privacy}","received_events_url":"https://api.github.com/users/mesosphere/received_events","type":"Organization","site_admin":false},"repo":{"id":10941409,"name":"marathon","full_name":"mesosphere/marathon","owner":{"login":"mesosphere","id":3458585,"avatar_url":"https://avatars.githubusercontent.com/u/3458585?v=3","gravatar_id":"","url":"https://api.github.com/users/mesosphere","html_url":"https://github.com/mesosphere","followers_url":"https://api.github.com/users/mesosphere/followers","following_url":"https://api.github.com/users/mesosphere/following{/other_user}","gists_url":"https://api.github.com/users/mesosphere/gists{/gist_id}","starred_url":"https://api.github.com/users/mesosphere/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mesosphere/subscriptions","organizations_url":"https://api.github.com/users/mesosphere/orgs","repos_url":"https://api.github.com/users/mesosphere/repos","events_url":"https://api.github.com/users/mesosphere/events{/privacy}","received_events_url":"https://api.github.com/users/mesosphere/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mesosphere/marathon","description":"Deploy and manage containers (including Docker) on top of Apache Mesos at scale.","fork":false,"url":"https://api.github.com/repos/mesosphere/marathon","forks_url":"https://api.github.com/repos/mesosphere/marathon/forks","keys_url":"https://api.github.com/repos/mesosphere/marathon/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mesosphere/marathon/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mesosphere/marathon/teams","hooks_url":"https://api.github.com/repos/mesosphere/marathon/hooks","issue_events_url":"https://api.github.com/repos/mesosphere/marathon/issues/events{/number}","events_url":"https://api.github.com/repos/mesosphere/marathon/events","assignees_url":"https://api.github.com/repos/mesosphere/marathon/assignees{/user}","branches_url":"https://api.github.com/repos/mesosphere/marathon/branches{/branch}","tags_url":"https://api.github.com/repos/mesosphere/marathon/tags","blobs_url":"https://api.github.com/repos/mesosphere/marathon/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mesosphere/marathon/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mesosphere/marathon/git/refs{/sha}","trees_url":"https://api.github.com/repos/mesosphere/marathon/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mesosphere/marathon/statuses/{sha}","languages_url":"https://api.github.com/repos/mesosphere/marathon/languages","stargazers_url":"https://api.github.com/repos/mesosphere/marathon/stargazers","contributors_url":"https://api.github.com/repos/mesosphere/marathon/contributors","subscribers_url":"https://api.github.com/repos/mesosphere/marathon/subscribers","subscription_url":"https://api.github.com/repos/mesosphere/marathon/subscription","commits_url":"https://api.github.com/repos/mesosphere/marathon/commits{/sha}","git_commits_url":"https://api.github.com/repos/mesosphere/marathon/git/commits{/sha}","comments_url":"https://api.github.com/repos/mesosphere/marathon/comments{/number}","issue_comment_url":"https://api.github.com/repos/mesosphere/marathon/issues/comments{/number}","contents_url":"https://api.github.com/repos/mesosphere/marathon/contents/{+path}","compare_url":"https://api.github.com/repos/mesosphere/marathon/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mesosphere/marathon/merges","archive_url":"https://api.github.com/repos/mesosphere/marathon/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mesosphere/marathon/downloads","issues_url":"https://api.github.com/repos/mesosphere/marathon/issues{/number}","pulls_url":"https://api.github.com/repos/mesosphere/marathon/pulls{/number}","milestones_url":"https://api.github.com/repos/mesosphere/marathon/milestones{/number}","notifications_url":"https://api.github.com/repos/mesosphere/marathon/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mesosphere/marathon/labels{/name}","releases_url":"https://api.github.com/repos/mesosphere/marathon/releases{/id}","deployments_url":"https://api.github.com/repos/mesosphere/marathon/deployments","created_at":"2013-06-25T14:56:34Z","updated_at":"2016-05-04T08:37:52Z","pushed_at":"2016-05-03T23:00:56Z","git_url":"git://github.com/mesosphere/marathon.git","ssh_url":"git@github.com:mesosphere/marathon.git","clone_url":"https://github.com/mesosphere/marathon.git","svn_url":"https://github.com/mesosphere/marathon","homepage":"https://mesosphere.github.io/marathon/","size":23040,"stargazers_count":2467,"watchers_count":2467,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":536,"mirror_url":null,"open_issues_count":553,"forks":536,"open_issues":553,"watchers":2467,"default_branch":"master"}},"base":{"label":"mesosphere:master","ref":"master","sha":"3587f8a92f11a8ca5bf09d3c942bf75d5c6d215e","user":{"login":"mesosphere","id":3458585,"avatar_url":"https://avatars.githubusercontent.com/u/3458585?v=3","gravatar_id":"","url":"https://api.github.com/users/mesosphere","html_url":"https://github.com/mesosphere","followers_url":"https://api.github.com/users/mesosphere/followers","following_url":"https://api.github.com/users/mesosphere/following{/other_user}","gists_url":"https://api.github.com/users/mesosphere/gists{/gist_id}","starred_url":"https://api.github.com/users/mesosphere/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mesosphere/subscriptions","organizations_url":"https://api.github.com/users/mesosphere/orgs","repos_url":"https://api.github.com/users/mesosphere/repos","events_url":"https://api.github.com/users/mesosphere/events{/privacy}","received_events_url":"https://api.github.com/users/mesosphere/received_events","type":"Organization","site_admin":false},"repo":{"id":10941409,"name":"marathon","full_name":"mesosphere/marathon","owner":{"login":"mesosphere","id":3458585,"avatar_url":"https://avatars.githubusercontent.com/u/3458585?v=3","gravatar_id":"","url":"https://api.github.com/users/mesosphere","html_url":"https://github.com/mesosphere","followers_url":"https://api.github.com/users/mesosphere/followers","following_url":"https://api.github.com/users/mesosphere/following{/other_user}","gists_url":"https://api.github.com/users/mesosphere/gists{/gist_id}","starred_url":"https://api.github.com/users/mesosphere/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mesosphere/subscriptions","organizations_url":"https://api.github.com/users/mesosphere/orgs","repos_url":"https://api.github.com/users/mesosphere/repos","events_url":"https://api.github.com/users/mesosphere/events{/privacy}","received_events_url":"https://api.github.com/users/mesosphere/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mesosphere/marathon","description":"Deploy and manage containers (including Docker) on top of Apache Mesos at scale.","fork":false,"url":"https://api.github.com/repos/mesosphere/marathon","forks_url":"https://api.github.com/repos/mesosphere/marathon/forks","keys_url":"https://api.github.com/repos/mesosphere/marathon/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mesosphere/marathon/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mesosphere/marathon/teams","hooks_url":"https://api.github.com/repos/mesosphere/marathon/hooks","issue_events_url":"https://api.github.com/repos/mesosphere/marathon/issues/events{/number}","events_url":"https://api.github.com/repos/mesosphere/marathon/events","assignees_url":"https://api.github.com/repos/mesosphere/marathon/assignees{/user}","branches_url":"https://api.github.com/repos/mesosphere/marathon/branches{/branch}","tags_url":"https://api.github.com/repos/mesosphere/marathon/tags","blobs_url":"https://api.github.com/repos/mesosphere/marathon/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mesosphere/marathon/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mesosphere/marathon/git/refs{/sha}","trees_url":"https://api.github.com/repos/mesosphere/marathon/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mesosphere/marathon/statuses/{sha}","languages_url":"https://api.github.com/repos/mesosphere/marathon/languages","stargazers_url":"https://api.github.com/repos/mesosphere/marathon/stargazers","contributors_url":"https://api.github.com/repos/mesosphere/marathon/contributors","subscribers_url":"https://api.github.com/repos/mesosphere/marathon/subscribers","subscription_url":"https://api.github.com/repos/mesosphere/marathon/subscription","commits_url":"https://api.github.com/repos/mesosphere/marathon/commits{/sha}","git_commits_url":"https://api.github.com/repos/mesosphere/marathon/git/commits{/sha}","comments_url":"https://api.github.com/repos/mesosphere/marathon/comments{/number}","issue_comment_url":"https://api.github.com/repos/mesosphere/marathon/issues/comments{/number}","contents_url":"https://api.github.com/repos/mesosphere/marathon/contents/{+path}","compare_url":"https://api.github.com/repos/mesosphere/marathon/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mesosphere/marathon/merges","archive_url":"https://api.github.com/repos/mesosphere/marathon/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mesosphere/marathon/downloads","issues_url":"https://api.github.com/repos/mesosphere/marathon/issues{/number}","pulls_url":"https://api.github.com/repos/mesosphere/marathon/pulls{/number}","milestones_url":"https://api.github.com/repos/mesosphere/marathon/milestones{/number}","notifications_url":"https://api.github.com/repos/mesosphere/marathon/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mesosphere/marathon/labels{/name}","releases_url":"https://api.github.com/repos/mesosphere/marathon/releases{/id}","deployments_url":"https://api.github.com/repos/mesosphere/marathon/deployments","created_at":"2013-06-25T14:56:34Z","updated_at":"2016-05-04T08:37:52Z","pushed_at":"2016-05-03T23:00:56Z","git_url":"git://github.com/mesosphere/marathon.git","ssh_url":"git@github.com:mesosphere/marathon.git","clone_url":"https://github.com/mesosphere/marathon.git","svn_url":"https://github.com/mesosphere/marathon","homepage":"https://mesosphere.github.io/marathon/","size":23040,"stargazers_count":2467,"watchers_count":2467,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":536,"mirror_url":null,"open_issues_count":553,"forks":536,"open_issues":553,"watchers":2467,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mesosphere/marathon/pulls/3777"},"html":{"href":"https://github.com/mesosphere/marathon/pull/3777"},"issue":{"href":"https://api.github.com/repos/mesosphere/marathon/issues/3777"},"comments":{"href":"https://api.github.com/repos/mesosphere/marathon/issues/3777/comments"},"review_comments":{"href":"https://api.github.com/repos/mesosphere/marathon/pulls/3777/comments"},"review_comment":{"href":"https://api.github.com/repos/mesosphere/marathon/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mesosphere/marathon/pulls/3777/commits"},"statuses":{"href":"https://api.github.com/repos/mesosphere/marathon/statuses/bc69e4a7ca5ad3c09d85dc3c4f31be0174d55763"}}}}