{"action":"created","comment":{"url":"https://api.github.com/repos/square/p2/pulls/comments/69040904","id":69040904,"diff_hunk":"@@ -273,66 +275,48 @@ func (r replication) updateOne(node types.NodeName, done chan<- types.NodeName,\n \t\t)\n \t}\n \n-\trealityResults := make(chan kp.ManifestResult)\n-\trealityErr := make(chan error)\n-\trealityQuit := make(chan struct{})\n-\tdefer close(realityQuit)\n-\tgo r.store.WatchPod(\n-\t\tkp.REALITY_TREE,\n-\t\tnode,\n-\t\tr.manifest.ID(),\n-\t\trealityQuit,\n-\t\trealityErr,\n-\t\trealityResults,\n-\t)\n-REALITY_LOOP:\n+\tr.ensureInReality(node, quitCh, nodeLogger, targetSHA)\n+\tr.ensureHealthy(node, done, quitCh, nodeLogger, aggregateHealth)\n+}\n+\n+func (r *replication) ensureInReality(node types.NodeName, quitCh <-chan struct{}, nodeLogger logging.Logger, targetSHA string) {\n \tfor {\n \t\tselect {\n \t\tcase <-quitCh:\n \t\t\treturn\n-\t\tcase err := <-realityErr:\n-\t\t\tnodeLogger.WithError(err).Errorln(\"Could not read reality store\")\n-\t\t\tselect {\n-\t\t\tcase r.errCh <- err:\n-\t\t\tcase <-quitCh:\n-\t\t\t}\n-\t\tcase mResult := <-realityResults:\n-\t\t\t// if the pod key doesn't exist yet, that's okay just wait longer\n-\t\t\tif mResult.Manifest != nil {\n-\t\t\t\treceivedSHA, _ := mResult.Manifest.SHA()\n+\t\tcase <-time.After(time.Duration(20+rand.Intn(10)) * time.Second): // random value for staggering\n+\t\t\tman, _, err := r.store.Pod(kp.REALITY_TREE, node, r.manifest.ID())\n+\t\t\tif err == pods.NoCurrentManifest {\n+\t\t\t\t// if the pod key doesn't exist yet, that's okay just wait longer\n+\t\t\t} else if err != nil {\n+\t\t\t\tnodeLogger.WithErrorAndFields(err, logrus.Fields{\n+\t\t\t\t\t\"node\": node,\n+\t\t\t\t}).Errorln(\"Could not read reality for pod manifest\")\n+\t\t\t} else {\n+\t\t\t\treceivedSHA, _ := man.SHA()\n \t\t\t\tif receivedSHA == targetSHA {\n-\t\t\t\t\tbreak REALITY_LOOP\n+\t\t\t\t\tnodeLogger.NoFields().Infoln(\"Node is current\")\n+\t\t\t\t\treturn\n \t\t\t\t} else {\n \t\t\t\t\tnodeLogger.WithFields(logrus.Fields{\"current\": receivedSHA, \"target\": targetSHA}).Infoln(\"Waiting for current\")\n \t\t\t\t}\n \t\t\t}\n \t\t}\n \t}\n-\tnodeLogger.NoFields().Infoln(\"Node is current\")\n+}\n \n-\thealthResults := make(chan health.Result)\n-\thealthErr := make(chan error)\n-\thealthQuit := make(chan struct{})\n-\tdefer close(healthQuit)\n-\tgo r.health.WatchNodeService(\n-\t\tnode,\n-\t\tstring(r.manifest.ID()),\n-\t\thealthResults,\n-\t\thealthErr,\n-\t\thealthQuit,\n-\t)\n-HEALTH_LOOP:\n+func (r *replication) ensureHealthy(node types.NodeName, done chan<- types.NodeName, quitCh <-chan struct{}, nodeLogger logging.Logger, aggregateHealth podHealth) {\n \tfor {\n \t\tselect {\n \t\tcase <-quitCh:\n \t\t\treturn\n-\t\tcase err := <-healthErr:\n-\t\t\tnodeLogger.WithError(err).Errorln(\"Could not read health check\")\n-\t\t\tselect {\n-\t\t\tcase r.errCh <- err:\n-\t\t\tcase <-quitCh:\n+\t\tcase <-time.After(1 * time.Second):\n+\t\t\tres, err := aggregateHealth.GetHealth(node)\n+\t\t\tif err != nil {","path":"pkg/replication/replication.go","position":114,"original_position":114,"commit_id":"8f22f261a072bfcc890f7b6f96746b2f9580fc37","original_commit_id":"8f22f261a072bfcc890f7b6f96746b2f9580fc37","user":{"login":"mpuncel","id":1444855,"avatar_url":"https://avatars.githubusercontent.com/u/1444855?v=3","gravatar_id":"","url":"https://api.github.com/users/mpuncel","html_url":"https://github.com/mpuncel","followers_url":"https://api.github.com/users/mpuncel/followers","following_url":"https://api.github.com/users/mpuncel/following{/other_user}","gists_url":"https://api.github.com/users/mpuncel/gists{/gist_id}","starred_url":"https://api.github.com/users/mpuncel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mpuncel/subscriptions","organizations_url":"https://api.github.com/users/mpuncel/orgs","repos_url":"https://api.github.com/users/mpuncel/repos","events_url":"https://api.github.com/users/mpuncel/events{/privacy}","received_events_url":"https://api.github.com/users/mpuncel/received_events","type":"User","site_admin":false},"body":"i mean it's been deployed but maybe no health value yet","created_at":"2016-06-29T22:35:02Z","updated_at":"2016-06-29T22:35:02Z","html_url":"https://github.com/square/p2/pull/535#discussion_r69040904","pull_request_url":"https://api.github.com/repos/square/p2/pulls/535","_links":{"self":{"href":"https://api.github.com/repos/square/p2/pulls/comments/69040904"},"html":{"href":"https://github.com/square/p2/pull/535#discussion_r69040904"},"pull_request":{"href":"https://api.github.com/repos/square/p2/pulls/535"}}},"pull_request":{"url":"https://api.github.com/repos/square/p2/pulls/535","id":75563358,"html_url":"https://github.com/square/p2/pull/535","diff_url":"https://github.com/square/p2/pull/535.diff","patch_url":"https://github.com/square/p2/pull/535.patch","issue_url":"https://api.github.com/repos/square/p2/issues/535","number":535,"state":"open","locked":false,"title":"Optimizations to outstanding connection counts in pkg/replication.","user":{"login":"anthonybishopric","id":540783,"avatar_url":"https://avatars.githubusercontent.com/u/540783?v=3","gravatar_id":"","url":"https://api.github.com/users/anthonybishopric","html_url":"https://github.com/anthonybishopric","followers_url":"https://api.github.com/users/anthonybishopric/followers","following_url":"https://api.github.com/users/anthonybishopric/following{/other_user}","gists_url":"https://api.github.com/users/anthonybishopric/gists{/gist_id}","starred_url":"https://api.github.com/users/anthonybishopric/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anthonybishopric/subscriptions","organizations_url":"https://api.github.com/users/anthonybishopric/orgs","repos_url":"https://api.github.com/users/anthonybishopric/repos","events_url":"https://api.github.com/users/anthonybishopric/events{/privacy}","received_events_url":"https://api.github.com/users/anthonybishopric/received_events","type":"User","site_admin":false},"body":"`pkg/replication` is primarily used by human operators attempting to\r\ndeploy many pods simultaneously (in the absence of daemon sets, which\r\nare being actively worked on). For this reason, we want to be considerate\r\nof the number of open file descriptors used by the replication package\r\nat any given time.\r\n\r\nThis PR switches reality observations into periodic polls (20 + rand(10) seconds)\r\nand changes health to use a single service-wide aggregation. Previously on my\r\nMacbook I was unable to execute any deploy with a parallelism factor over 100. With\r\nthis change I was able to perform a deployment at a 200 node parallelism factor without\r\nissue.","created_at":"2016-06-28T22:48:33Z","updated_at":"2016-06-29T22:35:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"11e7b61d63fcece8553320c451f31412bf676f37","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/square/p2/pulls/535/commits","review_comments_url":"https://api.github.com/repos/square/p2/pulls/535/comments","review_comment_url":"https://api.github.com/repos/square/p2/pulls/comments{/number}","comments_url":"https://api.github.com/repos/square/p2/issues/535/comments","statuses_url":"https://api.github.com/repos/square/p2/statuses/8f22f261a072bfcc890f7b6f96746b2f9580fc37","head":{"label":"anthonybishopric:abishopric/replication_connections","ref":"abishopric/replication_connections","sha":"8f22f261a072bfcc890f7b6f96746b2f9580fc37","user":{"login":"anthonybishopric","id":540783,"avatar_url":"https://avatars.githubusercontent.com/u/540783?v=3","gravatar_id":"","url":"https://api.github.com/users/anthonybishopric","html_url":"https://github.com/anthonybishopric","followers_url":"https://api.github.com/users/anthonybishopric/followers","following_url":"https://api.github.com/users/anthonybishopric/following{/other_user}","gists_url":"https://api.github.com/users/anthonybishopric/gists{/gist_id}","starred_url":"https://api.github.com/users/anthonybishopric/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anthonybishopric/subscriptions","organizations_url":"https://api.github.com/users/anthonybishopric/orgs","repos_url":"https://api.github.com/users/anthonybishopric/repos","events_url":"https://api.github.com/users/anthonybishopric/events{/privacy}","received_events_url":"https://api.github.com/users/anthonybishopric/received_events","type":"User","site_admin":false},"repo":{"id":25321022,"name":"p2","full_name":"anthonybishopric/p2","owner":{"login":"anthonybishopric","id":540783,"avatar_url":"https://avatars.githubusercontent.com/u/540783?v=3","gravatar_id":"","url":"https://api.github.com/users/anthonybishopric","html_url":"https://github.com/anthonybishopric","followers_url":"https://api.github.com/users/anthonybishopric/followers","following_url":"https://api.github.com/users/anthonybishopric/following{/other_user}","gists_url":"https://api.github.com/users/anthonybishopric/gists{/gist_id}","starred_url":"https://api.github.com/users/anthonybishopric/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anthonybishopric/subscriptions","organizations_url":"https://api.github.com/users/anthonybishopric/orgs","repos_url":"https://api.github.com/users/anthonybishopric/repos","events_url":"https://api.github.com/users/anthonybishopric/events{/privacy}","received_events_url":"https://api.github.com/users/anthonybishopric/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/anthonybishopric/p2","description":"Platypus Platform: Tools for Scalable Deployment","fork":true,"url":"https://api.github.com/repos/anthonybishopric/p2","forks_url":"https://api.github.com/repos/anthonybishopric/p2/forks","keys_url":"https://api.github.com/repos/anthonybishopric/p2/keys{/key_id}","collaborators_url":"https://api.github.com/repos/anthonybishopric/p2/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/anthonybishopric/p2/teams","hooks_url":"https://api.github.com/repos/anthonybishopric/p2/hooks","issue_events_url":"https://api.github.com/repos/anthonybishopric/p2/issues/events{/number}","events_url":"https://api.github.com/repos/anthonybishopric/p2/events","assignees_url":"https://api.github.com/repos/anthonybishopric/p2/assignees{/user}","branches_url":"https://api.github.com/repos/anthonybishopric/p2/branches{/branch}","tags_url":"https://api.github.com/repos/anthonybishopric/p2/tags","blobs_url":"https://api.github.com/repos/anthonybishopric/p2/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/anthonybishopric/p2/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/anthonybishopric/p2/git/refs{/sha}","trees_url":"https://api.github.com/repos/anthonybishopric/p2/git/trees{/sha}","statuses_url":"https://api.github.com/repos/anthonybishopric/p2/statuses/{sha}","languages_url":"https://api.github.com/repos/anthonybishopric/p2/languages","stargazers_url":"https://api.github.com/repos/anthonybishopric/p2/stargazers","contributors_url":"https://api.github.com/repos/anthonybishopric/p2/contributors","subscribers_url":"https://api.github.com/repos/anthonybishopric/p2/subscribers","subscription_url":"https://api.github.com/repos/anthonybishopric/p2/subscription","commits_url":"https://api.github.com/repos/anthonybishopric/p2/commits{/sha}","git_commits_url":"https://api.github.com/repos/anthonybishopric/p2/git/commits{/sha}","comments_url":"https://api.github.com/repos/anthonybishopric/p2/comments{/number}","issue_comment_url":"https://api.github.com/repos/anthonybishopric/p2/issues/comments{/number}","contents_url":"https://api.github.com/repos/anthonybishopric/p2/contents/{+path}","compare_url":"https://api.github.com/repos/anthonybishopric/p2/compare/{base}...{head}","merges_url":"https://api.github.com/repos/anthonybishopric/p2/merges","archive_url":"https://api.github.com/repos/anthonybishopric/p2/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/anthonybishopric/p2/downloads","issues_url":"https://api.github.com/repos/anthonybishopric/p2/issues{/number}","pulls_url":"https://api.github.com/repos/anthonybishopric/p2/pulls{/number}","milestones_url":"https://api.github.com/repos/anthonybishopric/p2/milestones{/number}","notifications_url":"https://api.github.com/repos/anthonybishopric/p2/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/anthonybishopric/p2/labels{/name}","releases_url":"https://api.github.com/repos/anthonybishopric/p2/releases{/id}","deployments_url":"https://api.github.com/repos/anthonybishopric/p2/deployments","created_at":"2014-10-16T20:45:38Z","updated_at":"2016-02-10T18:14:28Z","pushed_at":"2016-06-28T22:48:26Z","git_url":"git://github.com/anthonybishopric/p2.git","ssh_url":"git@github.com:anthonybishopric/p2.git","clone_url":"https://github.com/anthonybishopric/p2.git","svn_url":"https://github.com/anthonybishopric/p2","homepage":"","size":40445,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"square:master","ref":"master","sha":"8d4cfaa6c6f045f84a6d26cb11c36e769a2aa130","user":{"login":"square","id":82592,"avatar_url":"https://avatars.githubusercontent.com/u/82592?v=3","gravatar_id":"","url":"https://api.github.com/users/square","html_url":"https://github.com/square","followers_url":"https://api.github.com/users/square/followers","following_url":"https://api.github.com/users/square/following{/other_user}","gists_url":"https://api.github.com/users/square/gists{/gist_id}","starred_url":"https://api.github.com/users/square/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/square/subscriptions","organizations_url":"https://api.github.com/users/square/orgs","repos_url":"https://api.github.com/users/square/repos","events_url":"https://api.github.com/users/square/events{/privacy}","received_events_url":"https://api.github.com/users/square/received_events","type":"Organization","site_admin":false},"repo":{"id":25320424,"name":"p2","full_name":"square/p2","owner":{"login":"square","id":82592,"avatar_url":"https://avatars.githubusercontent.com/u/82592?v=3","gravatar_id":"","url":"https://api.github.com/users/square","html_url":"https://github.com/square","followers_url":"https://api.github.com/users/square/followers","following_url":"https://api.github.com/users/square/following{/other_user}","gists_url":"https://api.github.com/users/square/gists{/gist_id}","starred_url":"https://api.github.com/users/square/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/square/subscriptions","organizations_url":"https://api.github.com/users/square/orgs","repos_url":"https://api.github.com/users/square/repos","events_url":"https://api.github.com/users/square/events{/privacy}","received_events_url":"https://api.github.com/users/square/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/square/p2","description":"Platypus Platform: Tools for Scalable Deployment","fork":false,"url":"https://api.github.com/repos/square/p2","forks_url":"https://api.github.com/repos/square/p2/forks","keys_url":"https://api.github.com/repos/square/p2/keys{/key_id}","collaborators_url":"https://api.github.com/repos/square/p2/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/square/p2/teams","hooks_url":"https://api.github.com/repos/square/p2/hooks","issue_events_url":"https://api.github.com/repos/square/p2/issues/events{/number}","events_url":"https://api.github.com/repos/square/p2/events","assignees_url":"https://api.github.com/repos/square/p2/assignees{/user}","branches_url":"https://api.github.com/repos/square/p2/branches{/branch}","tags_url":"https://api.github.com/repos/square/p2/tags","blobs_url":"https://api.github.com/repos/square/p2/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/square/p2/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/square/p2/git/refs{/sha}","trees_url":"https://api.github.com/repos/square/p2/git/trees{/sha}","statuses_url":"https://api.github.com/repos/square/p2/statuses/{sha}","languages_url":"https://api.github.com/repos/square/p2/languages","stargazers_url":"https://api.github.com/repos/square/p2/stargazers","contributors_url":"https://api.github.com/repos/square/p2/contributors","subscribers_url":"https://api.github.com/repos/square/p2/subscribers","subscription_url":"https://api.github.com/repos/square/p2/subscription","commits_url":"https://api.github.com/repos/square/p2/commits{/sha}","git_commits_url":"https://api.github.com/repos/square/p2/git/commits{/sha}","comments_url":"https://api.github.com/repos/square/p2/comments{/number}","issue_comment_url":"https://api.github.com/repos/square/p2/issues/comments{/number}","contents_url":"https://api.github.com/repos/square/p2/contents/{+path}","compare_url":"https://api.github.com/repos/square/p2/compare/{base}...{head}","merges_url":"https://api.github.com/repos/square/p2/merges","archive_url":"https://api.github.com/repos/square/p2/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/square/p2/downloads","issues_url":"https://api.github.com/repos/square/p2/issues{/number}","pulls_url":"https://api.github.com/repos/square/p2/pulls{/number}","milestones_url":"https://api.github.com/repos/square/p2/milestones{/number}","notifications_url":"https://api.github.com/repos/square/p2/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/square/p2/labels{/name}","releases_url":"https://api.github.com/repos/square/p2/releases{/id}","deployments_url":"https://api.github.com/repos/square/p2/deployments","created_at":"2014-10-16T20:31:12Z","updated_at":"2016-06-29T18:39:33Z","pushed_at":"2016-06-29T07:15:15Z","git_url":"git://github.com/square/p2.git","ssh_url":"git@github.com:square/p2.git","clone_url":"https://github.com/square/p2.git","svn_url":"https://github.com/square/p2","homepage":"","size":40788,"stargazers_count":113,"watchers_count":113,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":20,"mirror_url":null,"open_issues_count":7,"forks":20,"open_issues":7,"watchers":113,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/square/p2/pulls/535"},"html":{"href":"https://github.com/square/p2/pull/535"},"issue":{"href":"https://api.github.com/repos/square/p2/issues/535"},"comments":{"href":"https://api.github.com/repos/square/p2/issues/535/comments"},"review_comments":{"href":"https://api.github.com/repos/square/p2/pulls/535/comments"},"review_comment":{"href":"https://api.github.com/repos/square/p2/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/square/p2/pulls/535/commits"},"statuses":{"href":"https://api.github.com/repos/square/p2/statuses/8f22f261a072bfcc890f7b6f96746b2f9580fc37"}}}}
{"actor":{"display_login":"mpuncel"}}