{"action":"created","comment":{"url":"https://api.github.com/repos/twosigma/waiter/pulls/comments/186462765","pull_request_review_id":118044894,"id":186462765,"diff_hunk":"@@ -0,0 +1,483 @@\n+;;\n+;;       Copyright (c) 2017 Two Sigma Investments, LP.\n+;;       All Rights Reserved\n+;;\n+;;       THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE OF\n+;;       Two Sigma Investments, LP.\n+;;\n+;;       The copyright notice above does not evidence any\n+;;       actual or intended publication of such source code.\n+;;\n+(ns waiter.scheduler.kubernetes\n+  (:require [clj-time.core :as t]\n+            [clojure.core.async :as async]\n+            [clojure.data.json :as json]\n+            [clojure.edn :as edn]\n+            [clojure.java.shell :as shell]\n+            [clojure.string :as string]\n+            [clojure.tools.logging :as log]\n+            [plumbing.core :as pc]\n+            [slingshot.slingshot :as ss]\n+            [waiter.mesos.utils :as mesos-utils]\n+            [waiter.scheduler :as scheduler]\n+            [waiter.service-description :as sd]\n+            [waiter.util.utils :as utils])\n+  (:import (org.joda.time.format DateTimeFormat)))\n+\n+(def service-id->failed-instances-transient-store (atom {}))\n+(def instance-x->failed-instances-transient-store (atom {}))\n+(def k8s-api-auth-str (atom nil))\n+\n+;; XXX - this shouldn't be a global constant (it should be read from the config file)\n+(def k8s-name-length-limit 63)\n+\n+(def k8s-timestamp-format (DateTimeFormat/forPattern \"yyyy-MM-dd'T'HH:mm:ss'Z'\"))\n+\n+(defn- timestamp-str->datetime [k8s-timestamp-str]\n+  (utils/str-to-date k8s-timestamp-str k8s-timestamp-format))\n+\n+(defn- service-id->app-name [service-id]\n+  (let [[_ app-prefix x y z] (re-find #\"([^-]+)-(\\w{8})(\\w+)(\\w{8})$\" service-id)\n+        app-name-max-length (- k8s-name-length-limit 6)\n+        suffix (if (< app-name-max-length 48)\n+                 (str \\- x z)\n+                 (str \\- x y z))\n+        prefix-max-length (- app-name-max-length (count suffix))\n+        app-prefix' (cond-> app-prefix\n+                      (< prefix-max-length (count app-prefix))\n+                      (subs 0 prefix-max-length))]\n+      (str app-prefix' suffix)))\n+\n+(defn- kw->str\n+  \"Convert a keyword to a string, preserving the keyword's namespace (if any).\"\n+  [^clojure.lang.Keyword kw]\n+  (str (.-sym kw)))\n+\n+(defn- as-json\n+  \"Convert Clojure data structures to JSON, preserving namespaces on keys.\"\n+  [data]\n+  (json/write-str data :key-fn #(if (keyword? %) (kw->str %) %)))\n+\n+(defn- replicaset->Service\n+  ;; FIXME - catch exceptions and return nil\n+  [replicaset]","path":"waiter/src/waiter/scheduler/kubernetes.clj","position":75,"original_position":63,"commit_id":"f67759da9e634ffb36f620f11cfddd67bdf051bc","original_commit_id":"d671e5efc3af161e774f494aa3c33d5c6b22607b","user":{"login":"DaoWen","id":1783674,"avatar_url":"https://avatars3.githubusercontent.com/u/1783674?v=4","gravatar_id":"","url":"https://api.github.com/users/DaoWen","html_url":"https://github.com/DaoWen","followers_url":"https://api.github.com/users/DaoWen/followers","following_url":"https://api.github.com/users/DaoWen/following{/other_user}","gists_url":"https://api.github.com/users/DaoWen/gists{/gist_id}","starred_url":"https://api.github.com/users/DaoWen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DaoWen/subscriptions","organizations_url":"https://api.github.com/users/DaoWen/orgs","repos_url":"https://api.github.com/users/DaoWen/repos","events_url":"https://api.github.com/users/DaoWen/events{/privacy}","received_events_url":"https://api.github.com/users/DaoWen/received_events","type":"User","site_admin":false},"body":"❌ ","created_at":"2018-05-07T15:42:55Z","updated_at":"2018-05-07T15:42:55Z","html_url":"https://github.com/twosigma/waiter/pull/285#discussion_r186462765","pull_request_url":"https://api.github.com/repos/twosigma/waiter/pulls/285","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/twosigma/waiter/pulls/comments/186462765"},"html":{"href":"https://github.com/twosigma/waiter/pull/285#discussion_r186462765"},"pull_request":{"href":"https://api.github.com/repos/twosigma/waiter/pulls/285"}},"in_reply_to_id":182126688},"pull_request":{"url":"https://api.github.com/repos/twosigma/waiter/pulls/285","id":181990019,"html_url":"https://github.com/twosigma/waiter/pull/285","diff_url":"https://github.com/twosigma/waiter/pull/285.diff","patch_url":"https://github.com/twosigma/waiter/pull/285.patch","issue_url":"https://api.github.com/repos/twosigma/waiter/issues/285","number":285,"state":"open","locked":false,"title":"Add Kubernetes scheduler implementation","user":{"login":"DaoWen","id":1783674,"avatar_url":"https://avatars3.githubusercontent.com/u/1783674?v=4","gravatar_id":"","url":"https://api.github.com/users/DaoWen","html_url":"https://github.com/DaoWen","followers_url":"https://api.github.com/users/DaoWen/followers","following_url":"https://api.github.com/users/DaoWen/following{/other_user}","gists_url":"https://api.github.com/users/DaoWen/gists{/gist_id}","starred_url":"https://api.github.com/users/DaoWen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DaoWen/subscriptions","organizations_url":"https://api.github.com/users/DaoWen/orgs","repos_url":"https://api.github.com/users/DaoWen/repos","events_url":"https://api.github.com/users/DaoWen/events{/privacy}","received_events_url":"https://api.github.com/users/DaoWen/received_events","type":"User","site_admin":false},"body":"## Changes proposed in this PR\r\n\r\nAdd Kubernetes scheduler implementation.\r\n\r\n## Why are we making these changes?\r\n\r\nWe want the option to migrate Waiter apps from existing Marathon clusters to Kubernetes clusters.\r\n\r\n## :construction: Work in Progress :construction:\r\n\r\nI'm still working on some things in this branch; however, most of the functionality is done and I think this code is ready for an initial review. Some of the outstanding work items are listed below.\r\n\r\n- [ ] Exception handling that allows retries on API request failures.\r\n- [ ] Allow the user to define the ReplicaSet specification template's path in the config file.\r\n- [ ] Docstrings on functions in `waiter.scheduler.kubernetes`.\r\n- [ ] Unit tests patterned after the Marathon scheduler tests.","created_at":"2018-04-16T21:13:21Z","updated_at":"2018-05-07T15:42:55Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f194fc9a6c6a2c018ee4fe1248a3e1bc08ed7b01","assignee":null,"assignees":[],"requested_reviewers":[{"login":"sradack","id":6118583,"avatar_url":"https://avatars2.githubusercontent.com/u/6118583?v=4","gravatar_id":"","url":"https://api.github.com/users/sradack","html_url":"https://github.com/sradack","followers_url":"https://api.github.com/users/sradack/followers","following_url":"https://api.github.com/users/sradack/following{/other_user}","gists_url":"https://api.github.com/users/sradack/gists{/gist_id}","starred_url":"https://api.github.com/users/sradack/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sradack/subscriptions","organizations_url":"https://api.github.com/users/sradack/orgs","repos_url":"https://api.github.com/users/sradack/repos","events_url":"https://api.github.com/users/sradack/events{/privacy}","received_events_url":"https://api.github.com/users/sradack/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[{"id":758303536,"url":"https://api.github.com/repos/twosigma/waiter/labels/wip","name":"wip","color":"000000","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/twosigma/waiter/pulls/285/commits","review_comments_url":"https://api.github.com/repos/twosigma/waiter/pulls/285/comments","review_comment_url":"https://api.github.com/repos/twosigma/waiter/pulls/comments{/number}","comments_url":"https://api.github.com/repos/twosigma/waiter/issues/285/comments","statuses_url":"https://api.github.com/repos/twosigma/waiter/statuses/f67759da9e634ffb36f620f11cfddd67bdf051bc","head":{"label":"DaoWen:feature/kubernetes-scheduler","ref":"feature/kubernetes-scheduler","sha":"f67759da9e634ffb36f620f11cfddd67bdf051bc","user":{"login":"DaoWen","id":1783674,"avatar_url":"https://avatars3.githubusercontent.com/u/1783674?v=4","gravatar_id":"","url":"https://api.github.com/users/DaoWen","html_url":"https://github.com/DaoWen","followers_url":"https://api.github.com/users/DaoWen/followers","following_url":"https://api.github.com/users/DaoWen/following{/other_user}","gists_url":"https://api.github.com/users/DaoWen/gists{/gist_id}","starred_url":"https://api.github.com/users/DaoWen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DaoWen/subscriptions","organizations_url":"https://api.github.com/users/DaoWen/orgs","repos_url":"https://api.github.com/users/DaoWen/repos","events_url":"https://api.github.com/users/DaoWen/events{/privacy}","received_events_url":"https://api.github.com/users/DaoWen/received_events","type":"User","site_admin":false},"repo":{"id":105570659,"name":"waiter","full_name":"DaoWen/waiter","owner":{"login":"DaoWen","id":1783674,"avatar_url":"https://avatars3.githubusercontent.com/u/1783674?v=4","gravatar_id":"","url":"https://api.github.com/users/DaoWen","html_url":"https://github.com/DaoWen","followers_url":"https://api.github.com/users/DaoWen/followers","following_url":"https://api.github.com/users/DaoWen/following{/other_user}","gists_url":"https://api.github.com/users/DaoWen/gists{/gist_id}","starred_url":"https://api.github.com/users/DaoWen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DaoWen/subscriptions","organizations_url":"https://api.github.com/users/DaoWen/orgs","repos_url":"https://api.github.com/users/DaoWen/repos","events_url":"https://api.github.com/users/DaoWen/events{/privacy}","received_events_url":"https://api.github.com/users/DaoWen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/DaoWen/waiter","description":"Runs, manages, and autoscales web services on Mesos","fork":true,"url":"https://api.github.com/repos/DaoWen/waiter","forks_url":"https://api.github.com/repos/DaoWen/waiter/forks","keys_url":"https://api.github.com/repos/DaoWen/waiter/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DaoWen/waiter/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DaoWen/waiter/teams","hooks_url":"https://api.github.com/repos/DaoWen/waiter/hooks","issue_events_url":"https://api.github.com/repos/DaoWen/waiter/issues/events{/number}","events_url":"https://api.github.com/repos/DaoWen/waiter/events","assignees_url":"https://api.github.com/repos/DaoWen/waiter/assignees{/user}","branches_url":"https://api.github.com/repos/DaoWen/waiter/branches{/branch}","tags_url":"https://api.github.com/repos/DaoWen/waiter/tags","blobs_url":"https://api.github.com/repos/DaoWen/waiter/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DaoWen/waiter/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DaoWen/waiter/git/refs{/sha}","trees_url":"https://api.github.com/repos/DaoWen/waiter/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DaoWen/waiter/statuses/{sha}","languages_url":"https://api.github.com/repos/DaoWen/waiter/languages","stargazers_url":"https://api.github.com/repos/DaoWen/waiter/stargazers","contributors_url":"https://api.github.com/repos/DaoWen/waiter/contributors","subscribers_url":"https://api.github.com/repos/DaoWen/waiter/subscribers","subscription_url":"https://api.github.com/repos/DaoWen/waiter/subscription","commits_url":"https://api.github.com/repos/DaoWen/waiter/commits{/sha}","git_commits_url":"https://api.github.com/repos/DaoWen/waiter/git/commits{/sha}","comments_url":"https://api.github.com/repos/DaoWen/waiter/comments{/number}","issue_comment_url":"https://api.github.com/repos/DaoWen/waiter/issues/comments{/number}","contents_url":"https://api.github.com/repos/DaoWen/waiter/contents/{+path}","compare_url":"https://api.github.com/repos/DaoWen/waiter/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DaoWen/waiter/merges","archive_url":"https://api.github.com/repos/DaoWen/waiter/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DaoWen/waiter/downloads","issues_url":"https://api.github.com/repos/DaoWen/waiter/issues{/number}","pulls_url":"https://api.github.com/repos/DaoWen/waiter/pulls{/number}","milestones_url":"https://api.github.com/repos/DaoWen/waiter/milestones{/number}","notifications_url":"https://api.github.com/repos/DaoWen/waiter/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DaoWen/waiter/labels{/name}","releases_url":"https://api.github.com/repos/DaoWen/waiter/releases{/id}","deployments_url":"https://api.github.com/repos/DaoWen/waiter/deployments","created_at":"2017-10-02T18:39:29Z","updated_at":"2018-05-04T19:00:21Z","pushed_at":"2018-05-07T15:14:56Z","git_url":"git://github.com/DaoWen/waiter.git","ssh_url":"git@github.com:DaoWen/waiter.git","clone_url":"https://github.com/DaoWen/waiter.git","svn_url":"https://github.com/DaoWen/waiter","homepage":"","size":1864,"stargazers_count":0,"watchers_count":0,"language":"Clojure","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":3,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":3,"watchers":0,"default_branch":"master"}},"base":{"label":"twosigma:master","ref":"master","sha":"b4f027f36f33eb4f47f546c89f5430c316446762","user":{"login":"twosigma","id":4049181,"avatar_url":"https://avatars3.githubusercontent.com/u/4049181?v=4","gravatar_id":"","url":"https://api.github.com/users/twosigma","html_url":"https://github.com/twosigma","followers_url":"https://api.github.com/users/twosigma/followers","following_url":"https://api.github.com/users/twosigma/following{/other_user}","gists_url":"https://api.github.com/users/twosigma/gists{/gist_id}","starred_url":"https://api.github.com/users/twosigma/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twosigma/subscriptions","organizations_url":"https://api.github.com/users/twosigma/orgs","repos_url":"https://api.github.com/users/twosigma/repos","events_url":"https://api.github.com/users/twosigma/events{/privacy}","received_events_url":"https://api.github.com/users/twosigma/received_events","type":"Organization","site_admin":false},"repo":{"id":81378873,"name":"waiter","full_name":"twosigma/waiter","owner":{"login":"twosigma","id":4049181,"avatar_url":"https://avatars3.githubusercontent.com/u/4049181?v=4","gravatar_id":"","url":"https://api.github.com/users/twosigma","html_url":"https://github.com/twosigma","followers_url":"https://api.github.com/users/twosigma/followers","following_url":"https://api.github.com/users/twosigma/following{/other_user}","gists_url":"https://api.github.com/users/twosigma/gists{/gist_id}","starred_url":"https://api.github.com/users/twosigma/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twosigma/subscriptions","organizations_url":"https://api.github.com/users/twosigma/orgs","repos_url":"https://api.github.com/users/twosigma/repos","events_url":"https://api.github.com/users/twosigma/events{/privacy}","received_events_url":"https://api.github.com/users/twosigma/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twosigma/waiter","description":"Runs, manages, and autoscales web services on Mesos","fork":false,"url":"https://api.github.com/repos/twosigma/waiter","forks_url":"https://api.github.com/repos/twosigma/waiter/forks","keys_url":"https://api.github.com/repos/twosigma/waiter/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twosigma/waiter/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twosigma/waiter/teams","hooks_url":"https://api.github.com/repos/twosigma/waiter/hooks","issue_events_url":"https://api.github.com/repos/twosigma/waiter/issues/events{/number}","events_url":"https://api.github.com/repos/twosigma/waiter/events","assignees_url":"https://api.github.com/repos/twosigma/waiter/assignees{/user}","branches_url":"https://api.github.com/repos/twosigma/waiter/branches{/branch}","tags_url":"https://api.github.com/repos/twosigma/waiter/tags","blobs_url":"https://api.github.com/repos/twosigma/waiter/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twosigma/waiter/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twosigma/waiter/git/refs{/sha}","trees_url":"https://api.github.com/repos/twosigma/waiter/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twosigma/waiter/statuses/{sha}","languages_url":"https://api.github.com/repos/twosigma/waiter/languages","stargazers_url":"https://api.github.com/repos/twosigma/waiter/stargazers","contributors_url":"https://api.github.com/repos/twosigma/waiter/contributors","subscribers_url":"https://api.github.com/repos/twosigma/waiter/subscribers","subscription_url":"https://api.github.com/repos/twosigma/waiter/subscription","commits_url":"https://api.github.com/repos/twosigma/waiter/commits{/sha}","git_commits_url":"https://api.github.com/repos/twosigma/waiter/git/commits{/sha}","comments_url":"https://api.github.com/repos/twosigma/waiter/comments{/number}","issue_comment_url":"https://api.github.com/repos/twosigma/waiter/issues/comments{/number}","contents_url":"https://api.github.com/repos/twosigma/waiter/contents/{+path}","compare_url":"https://api.github.com/repos/twosigma/waiter/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twosigma/waiter/merges","archive_url":"https://api.github.com/repos/twosigma/waiter/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twosigma/waiter/downloads","issues_url":"https://api.github.com/repos/twosigma/waiter/issues{/number}","pulls_url":"https://api.github.com/repos/twosigma/waiter/pulls{/number}","milestones_url":"https://api.github.com/repos/twosigma/waiter/milestones{/number}","notifications_url":"https://api.github.com/repos/twosigma/waiter/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twosigma/waiter/labels{/name}","releases_url":"https://api.github.com/repos/twosigma/waiter/releases{/id}","deployments_url":"https://api.github.com/repos/twosigma/waiter/deployments","created_at":"2017-02-08T21:40:37Z","updated_at":"2018-05-05T20:11:33Z","pushed_at":"2018-05-07T15:14:58Z","git_url":"git://github.com/twosigma/waiter.git","ssh_url":"git@github.com:twosigma/waiter.git","clone_url":"https://github.com/twosigma/waiter.git","svn_url":"https://github.com/twosigma/waiter","homepage":"","size":2378,"stargazers_count":22,"watchers_count":22,"language":"Clojure","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":7,"mirror_url":null,"archived":false,"open_issues_count":20,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":7,"open_issues":20,"watchers":22,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/twosigma/waiter/pulls/285"},"html":{"href":"https://github.com/twosigma/waiter/pull/285"},"issue":{"href":"https://api.github.com/repos/twosigma/waiter/issues/285"},"comments":{"href":"https://api.github.com/repos/twosigma/waiter/issues/285/comments"},"review_comments":{"href":"https://api.github.com/repos/twosigma/waiter/pulls/285/comments"},"review_comment":{"href":"https://api.github.com/repos/twosigma/waiter/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/twosigma/waiter/pulls/285/commits"},"statuses":{"href":"https://api.github.com/repos/twosigma/waiter/statuses/f67759da9e634ffb36f620f11cfddd67bdf051bc"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"DaoWen"}}