{"action":"created","comment":{"url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/comments/70463157","id":70463157,"diff_hunk":"@@ -0,0 +1,623 @@\n+package azure\n+\n+import (\n+\t\"fmt\"\n+\t\"strconv\"\n+\t\"strings\"\n+\n+\t\"k8s.io/kubernetes/pkg/api\"\n+\tutilerrors \"k8s.io/kubernetes/pkg/util/errors\"\n+\n+\t\"github.com/Azure/azure-sdk-for-go/arm/network\"\n+\t\"github.com/Azure/go-autorest/autorest/to\"\n+\t\"github.com/golang/glog\"\n+)\n+\n+// GetLoadBalancer returns whether the specified load balancer exists, and\n+// if so, what its status is.\n+func (az *AzureCloud) GetLoadBalancer(clusterName string, service *api.Service) (status *api.LoadBalancerStatus, exists bool, err error) {\n+\tlbName := getLoadBalancerName(clusterName)\n+\tpipName := getPublicIPName(clusterName, service)\n+\tserviceName := getServiceName(service)\n+\tglog.Infof(\"get(%s): START clusterName=%q lbName=%q\", serviceName, clusterName, lbName)\n+\n+\tglog.Infof(\"get(%s): lb(%s) - retrieving\", serviceName, lbName)\n+\t_, err = az.LoadBalancerClient.Get(az.ResourceGroup, lbName, \"\")\n+\tif existsLb, err := checkResourceExistsFromError(err); err != nil {\n+\t\tglog.Errorf(\"get(%s): lb(%s) - retrieving failed: %q\", serviceName, lbName, err)\n+\t\treturn nil, false, err\n+\t} else if !existsLb {\n+\t\tglog.Infof(\"get(%s): lb(%s) - doesn't exist\", serviceName, lbName)\n+\t\treturn nil, false, nil\n+\t}\n+\n+\tglog.Infof(\"get(%s): pip(%s) - retrieving\", serviceName, pipName)\n+\tpip, err := az.PublicIPAddressesClient.Get(az.ResourceGroup, pipName, \"\")\n+\tif existsLbPip, err := checkResourceExistsFromError(err); err != nil {\n+\t\tglog.Errorf(\"get(%s): pip(%s) - retrieving failed: %q\", serviceName, pipName, err)\n+\t\treturn nil, false, err\n+\t} else if !existsLbPip {\n+\t\tglog.Infof(\"get(%s): pip(%s) - doesn't exist\", serviceName, pipName)\n+\t\treturn nil, false, nil\n+\t}\n+\n+\tglog.Infof(\"get(%s): FINISH\")\n+\treturn &api.LoadBalancerStatus{\n+\t\tIngress: []api.LoadBalancerIngress{{IP: *pip.Properties.IPAddress}},\n+\t}, true, nil\n+}\n+\n+// EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer\n+func (az *AzureCloud) EnsureLoadBalancer(clusterName string, service *api.Service, hosts []string) (*api.LoadBalancerStatus, error) {\n+\tlbName := getLoadBalancerName(clusterName)\n+\tpipName := getPublicIPName(clusterName, service)\n+\tserviceName := getServiceName(service)\n+\tglog.Infof(\"ensure(%s): START clusterName=%q lbName=%q\", serviceName, clusterName, lbName)\n+\n+\tpip, err := az.ensurePublicIPExists(serviceName, pipName)\n+\tif err != nil {\n+\t\treturn nil, err\n+\t}\n+\n+\tglog.Infof(\"ensure(%s): sg(%s) - retrieving\", serviceName, az.SecurityGroupName)\n+\tsg, err := az.SecurityGroupsClient.Get(az.ResourceGroup, az.SecurityGroupName, \"\")\n+\tif err != nil {\n+\t\tglog.Errorf(\"ensure(%s): sg(%s) - retrieving failed: %q\", serviceName, *sg.Name, err)\n+\t\treturn nil, err\n+\t}\n+\tsg, sgNeedsUpdate, err := az.reconcileSecurityGroup(sg, clusterName, service)\n+\tif err != nil {\n+\t\treturn nil, err\n+\t}\n+\tif sgNeedsUpdate {\n+\t\tglog.Infof(\"ensure(%s): sg(%s) - updating\", serviceName, *sg.Name)\n+\t\t_, err := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *sg.Name, sg, nil)\n+\t\tif err != nil {\n+\t\t\tglog.Errorf(\"ensure(%s): sg(%s) - updating failed: %q\", serviceName, *sg.Name, err)\n+\t\t\treturn nil, fmt.Errorf(\"failed to update security group. err=%q\", err)\n+\t\t}\n+\t}\n+\n+\tlbNeedsCreate := false\n+\tglog.Infof(\"ensure(%s): lb(%s) - retrieving\", serviceName, lbName)\n+\tlb, err := az.LoadBalancerClient.Get(az.ResourceGroup, lbName, \"\")\n+\tif existsLb, err := checkResourceExistsFromError(err); err != nil {\n+\t\tglog.Infof(\"ensure(%s): lb(%s) - retrieving failed: %q\", serviceName, lbName, err)\n+\t\treturn nil, err\n+\t} else if !existsLb {\n+\t\tlb = network.LoadBalancer{\n+\t\t\tName:       &lbName,\n+\t\t\tLocation:   &az.Location,\n+\t\t\tProperties: &network.LoadBalancerPropertiesFormat{},\n+\t\t}\n+\t\tlbNeedsCreate = true\n+\t\tglog.Infof(\"ensure(%s): lb(%s) - needs creation\")\n+\t}\n+\n+\tlb, lbNeedsUpdate, err := az.reconcileLoadBalancer(lb, pip, clusterName, service, hosts)\n+\tif err != nil {\n+\t\treturn nil, err\n+\t}\n+\tif lbNeedsCreate || lbNeedsUpdate {\n+\t\tglog.Infof(\"ensure(%s): lb(%s) - updating\", serviceName, lbName)\n+\t\t_, err = az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil)\n+\t\tif err != nil {\n+\t\t\tglog.Errorf(\"ensure(%s): lb(%s) - updating failed: %q\", serviceName, lbName, err)\n+\t\t\treturn nil, err\n+\t\t}\n+\t}\n+\n+\t// Add the machines to the backend pool if they're not already\n+\tlbBackendName := getBackendPoolName(clusterName)\n+\tlbBackendPoolID := az.getBackendPoolID(lbName, lbBackendName)\n+\thostUpdates := make([]func() error, len(hosts))\n+\tfor i, host := range hosts {\n+\t\tf := func(serviceName, host, lbBackendPoolID string) func() error {\n+\t\t\treturn func() error {\n+\t\t\t\tglog.Infof(\"ensureHostInPool(%s): host(%s) - calling\", serviceName, host)\n+\t\t\t\treturn az.ensureHostInPool(serviceName, host, lbBackendPoolID)\n+\t\t\t}\n+\t\t}(serviceName, host, lbBackendPoolID)\n+\t\thostUpdates[i] = f\n+\t}\n+\n+\terrs := utilerrors.AggregateGoroutines(hostUpdates...)\n+\tif errs != nil {\n+\t\treturn nil, utilerrors.Flatten(errs)\n+\t}\n+\n+\tglog.Infof(\"ensure(%s): FINISH - %s\", service.Name, *pip.Properties.IPAddress)\n+\treturn &api.LoadBalancerStatus{\n+\t\tIngress: []api.LoadBalancerIngress{{IP: *pip.Properties.IPAddress}},\n+\t}, nil\n+}\n+\n+// UpdateLoadBalancer updates hosts under the specified load balancer.\n+func (az *AzureCloud) UpdateLoadBalancer(clusterName string, service *api.Service, hosts []string) error {\n+\tserviceName := getServiceName(service)\n+\tglog.Infof(\"update(%s): START\", serviceName)\n+\t_, err := az.EnsureLoadBalancer(clusterName, service, hosts)\n+\tglog.Infof(\"update(%s): FINISH\", serviceName)\n+\treturn err\n+}\n+\n+// EnsureLoadBalancerDeleted deletes the specified load balancer if it\n+// exists, returning nil if the load balancer specified either didn't exist or\n+// was successfully deleted.\n+// This construction is useful because many cloud providers' load balancers\n+// have multiple underlying components, meaning a Get could say that the LB\n+// doesn't exist even if some part of it is still laying around.\n+func (az *AzureCloud) EnsureLoadBalancerDeleted(clusterName string, service *api.Service) error {\n+\tlbName := getLoadBalancerName(clusterName)\n+\tpipName := getPublicIPName(clusterName, service)\n+\tserviceName := getServiceName(service)\n+\tglog.Infof(\"delete(%s): START clusterName=%q lbName=%q\", clusterName, lbName)\n+\n+\t// reconcile logic is capable of fully reconcile, so we can use this to delete\n+\tservice.Spec.Ports = []api.ServicePort{}\n+\n+\tglog.Infof(\"delete(%s): lb(%s) - retrieving\", serviceName, lbName)\n+\tlb, err := az.LoadBalancerClient.Get(az.ResourceGroup, lbName, \"\")\n+\tif existsLb, err := checkResourceExistsFromError(err); err != nil {\n+\t\treturn err\n+\t} else if existsLb {\n+\t\tlb, lbNeedsUpdate, err := az.reconcileLoadBalancer(lb, nil, clusterName, service, []string{})\n+\t\tif err != nil {\n+\t\t\treturn err\n+\t\t}\n+\t\tif lbNeedsUpdate {\n+\t\t\tif len(*lb.Properties.FrontendIPConfigurations) > 0 {\n+\t\t\t\t// if we have no more frontend ip configs, we need to remove the whole load balancer\n+\t\t\t\tglog.Infof(\"delete(%s): lb(%s) - updating\", serviceName, lbName)\n+\t\t\t\t_, err = az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil)\n+\t\t\t\tif err != nil {\n+\t\t\t\t\tglog.Errorf(\"delete(%s): lb(%s) - updating failed: %q\", serviceName, az.SecurityGroupName, err)\n+\t\t\t\t\treturn err\n+\t\t\t\t}\n+\t\t\t} else {\n+\t\t\t\tglog.Infof(\"delete(%s): lb(%s) - deleting due to no remaining frontendipconfigs\", serviceName, lbName)\n+\t\t\t\t_, err = az.LoadBalancerClient.Delete(az.ResourceGroup, lbName, nil)\n+\t\t\t\tif err != nil {\n+\t\t\t\t\tglog.Errorf(\"delete(%s): lb(%s) - deleting failed: %q\", serviceName, az.SecurityGroupName, err)\n+\t\t\t\t\treturn err\n+\t\t\t\t}\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\tglog.Infof(\"delete(%s): sg(%s) - retrieving\", serviceName, az.SecurityGroupName)\n+\tsg, err := az.SecurityGroupsClient.Get(az.ResourceGroup, az.SecurityGroupName, \"\")\n+\tif existsSg, err := checkResourceExistsFromError(err); err != nil {\n+\t\tglog.Infof(\"delete(%s): sg(%s) - retrieving failed: %q\", serviceName, az.SecurityGroupName, err)\n+\t\treturn err\n+\t} else if existsSg {\n+\t\tsg, sgNeedsUpdate, err := az.reconcileSecurityGroup(sg, clusterName, service)\n+\t\tif err != nil {\n+\t\t\treturn err\n+\t\t}\n+\t\tif sgNeedsUpdate {\n+\t\t\tglog.Infof(\"delete(%s): sg(%s) - updating\", serviceName, az.SecurityGroupName)\n+\t\t\t_, err := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *sg.Name, sg, nil)\n+\t\t\tif err != nil {\n+\t\t\t\tglog.Errorf(\"delete(%s): sg(%s) - updating failed: %q\", serviceName, az.SecurityGroupName, err)\n+\t\t\t\treturn fmt.Errorf(\"failed to update security group. err=%q\", err)\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\terr = az.ensurePublicIPDeleted(serviceName, pipName)\n+\tif err != nil {\n+\t\treturn err\n+\t}\n+\n+\tglog.Infof(\"delete(%s): FINISH\", serviceName)\n+\treturn nil\n+}\n+\n+func (az *AzureCloud) ensurePublicIPExists(serviceName, pipName string) (*network.PublicIPAddress, error) {\n+\tpip, err := az.PublicIPAddressesClient.Get(az.ResourceGroup, pipName, \"\")\n+\tif existsPip, err := checkResourceExistsFromError(err); err != nil {\n+\t\treturn nil, err\n+\t} else if existsPip {\n+\t\tglog.Infof(\"ensure(%s): pip(%s) - already exists\", serviceName, *pip.Name)\n+\t\treturn &pip, nil\n+\t} else {\n+\t\tpip.Name = to.StringPtr(pipName)\n+\t\tpip.Location = to.StringPtr(az.Location)\n+\t\tpip.Properties = &network.PublicIPAddressPropertiesFormat{\n+\t\t\tPublicIPAllocationMethod: network.Static,\n+\t\t}\n+\t\tpip.Tags = &map[string]*string{\"service\": &serviceName}\n+\n+\t\tglog.Infof(\"ensure(%s): pip(%s) - creating\", serviceName, *pip.Name)\n+\t\t_, err = az.PublicIPAddressesClient.CreateOrUpdate(az.ResourceGroup, *pip.Name, pip, nil)\n+\t\tif err != nil {\n+\t\t\tglog.Errorf(\"ensure(%s): pip(%s) - creating failed: %q\", serviceName, *pip.Name, err)\n+\t\t\treturn nil, err\n+\t\t}\n+\n+\t\tglog.Infof(\"ensure(%s): pip(%s) - retrieving\", serviceName, *pip.Name)\n+\t\tpip, err = az.PublicIPAddressesClient.Get(az.ResourceGroup, *pip.Name, \"\")\n+\t\tif err != nil {\n+\t\t\tglog.Errorf(\"ensure(%s): pip(%s) - retrieving failed: %q\", serviceName, *pip.Name, err)\n+\t\t\treturn nil, err\n+\t\t}\n+\n+\t\treturn &pip, nil\n+\t}\n+}\n+\n+func (az *AzureCloud) ensurePublicIPDeleted(serviceName, pipName string) error {\n+\tglog.Infof(\"delete(%s): pip(%s) - deleting pip\", serviceName, pipName)\n+\t_, err := az.PublicIPAddressesClient.Delete(az.ResourceGroup, pipName, nil)\n+\tif _, err := checkResourceExistsFromError(err); err != nil {\n+\t\tglog.Errorf(\"delete(%s): pip(%s) - deleting failed: %q\", serviceName, pipName, err)\n+\t\treturn fmt.Errorf(\"failed to delete public ip: %q\", err)\n+\t}\n+\treturn nil\n+}\n+\n+// This ensures load balancer exists and the frontend ip config is setup.\n+// This also reconciles the Service's Ports  with the LoadBalancer config.\n+// This entails adding rules/probes for expected Ports and removing stale rules/ports.\n+func (az *AzureCloud) reconcileLoadBalancer(lb network.LoadBalancer, pip *network.PublicIPAddress, clusterName string, service *api.Service, hosts []string) (network.LoadBalancer, bool, error) {\n+\tlbName := getLoadBalancerName(clusterName)\n+\tserviceName := getServiceName(service)\n+\tlbFrontendIPConfigName := getFrontendIPConfigName(service)\n+\tlbFrontendIPConfigID := az.getFrontendIPConfigID(lbName, lbFrontendIPConfigName)\n+\tlbBackendPoolName := getBackendPoolName(clusterName)\n+\tlbBackendPoolID := az.getBackendPoolID(lbName, lbBackendPoolName)\n+\n+\twantLb := (len(service.Spec.Ports) > 0)\n+\tdirtyLb := false\n+\n+\t// Ensure LoadBalancer's Backend Pool Configuration\n+\tif wantLb {\n+\t\tif lb.Properties.BackendAddressPools == nil ||\n+\t\t\tlen(*lb.Properties.BackendAddressPools) == 0 {\n+\t\t\tlb.Properties.BackendAddressPools = &[]network.BackendAddressPool{\n+\t\t\t\tnetwork.BackendAddressPool{\n+\t\t\t\t\tName: to.StringPtr(lbBackendPoolName),\n+\t\t\t\t},\n+\t\t\t}\n+\t\t\tglog.Infof(\"reconcile(%s)(%t): lb backendpool - adding\", serviceName, wantLb)\n+\t\t\tdirtyLb = true\n+\t\t} else if len(*lb.Properties.BackendAddressPools) != 1 ||\n+\t\t\t!strings.EqualFold(*(*lb.Properties.BackendAddressPools)[0].Name, lbBackendPoolName) {\n+\t\t\tglog.Errorf(\"reconcile(%s)(%t): lb backendpool - misconfigured\", serviceName, wantLb)\n+\t\t\treturn lb, false, fmt.Errorf(\"loadbalancer is misconfigured with a different backend pool\")\n+\t\t}\n+\t}\n+\n+\t// Ensure LoadBalancer's Frontend IP Configurations\n+\tdirtyConfigs := false\n+\tnewConfigs := []network.FrontendIPConfiguration{}\n+\tif lb.Properties.FrontendIPConfigurations != nil {\n+\t\tnewConfigs = *lb.Properties.FrontendIPConfigurations\n+\t}\n+\tif !wantLb {\n+\t\tfor i := len(newConfigs) - 1; i >= 0; i-- {\n+\t\t\tconfig := newConfigs[i]\n+\t\t\tif strings.EqualFold(*config.Name, lbFrontendIPConfigName) {\n+\t\t\t\tglog.Infof(\"reconcile(%s)(%t): lb frontendconfig(%s) - dropping\", serviceName, wantLb, lbFrontendIPConfigName)\n+\t\t\t\tnewConfigs = append(newConfigs[:i],\n+\t\t\t\t\tnewConfigs[i+1:]...)","path":"pkg/cloudprovider/providers/azure/azure_loadbalancer.go","position":304,"original_position":304,"commit_id":"8a288e1e54df4b1710f17593eb42f4ec020b5a71","original_commit_id":"8a288e1e54df4b1710f17593eb42f4ec020b5a71","user":{"login":"brendandburns","id":5751682,"avatar_url":"https://avatars.githubusercontent.com/u/5751682?v=3","gravatar_id":"","url":"https://api.github.com/users/brendandburns","html_url":"https://github.com/brendandburns","followers_url":"https://api.github.com/users/brendandburns/followers","following_url":"https://api.github.com/users/brendandburns/following{/other_user}","gists_url":"https://api.github.com/users/brendandburns/gists{/gist_id}","starred_url":"https://api.github.com/users/brendandburns/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brendandburns/subscriptions","organizations_url":"https://api.github.com/users/brendandburns/orgs","repos_url":"https://api.github.com/users/brendandburns/repos","events_url":"https://api.github.com/users/brendandburns/events{/privacy}","received_events_url":"https://api.github.com/users/brendandburns/received_events","type":"User","site_admin":false},"body":"nit: remove line break.","created_at":"2016-07-12T15:32:40Z","updated_at":"2016-07-12T15:32:40Z","html_url":"https://github.com/kubernetes/kubernetes/pull/28821#discussion_r70463157","pull_request_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/28821","_links":{"self":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/comments/70463157"},"html":{"href":"https://github.com/kubernetes/kubernetes/pull/28821#discussion_r70463157"},"pull_request":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/28821"}}},"pull_request":{"url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/28821","id":77065790,"html_url":"https://github.com/kubernetes/kubernetes/pull/28821","diff_url":"https://github.com/kubernetes/kubernetes/pull/28821.diff","patch_url":"https://github.com/kubernetes/kubernetes/pull/28821.patch","issue_url":"https://api.github.com/repos/kubernetes/kubernetes/issues/28821","number":28821,"state":"open","locked":false,"title":"Azure CloudProvider Implementation","user":{"login":"colemickens","id":327028,"avatar_url":"https://avatars.githubusercontent.com/u/327028?v=3","gravatar_id":"","url":"https://api.github.com/users/colemickens","html_url":"https://github.com/colemickens","followers_url":"https://api.github.com/users/colemickens/followers","following_url":"https://api.github.com/users/colemickens/following{/other_user}","gists_url":"https://api.github.com/users/colemickens/gists{/gist_id}","starred_url":"https://api.github.com/users/colemickens/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/colemickens/subscriptions","organizations_url":"https://api.github.com/users/colemickens/orgs","repos_url":"https://api.github.com/users/colemickens/repos","events_url":"https://api.github.com/users/colemickens/events{/privacy}","received_events_url":"https://api.github.com/users/colemickens/received_events","type":"User","site_admin":false},"body":"This PR adds `Azure` as a cloudprovider provider for Kubernetes. It specifically adds support for native pod networking (via Azure User Defined Routes) and L4 Load Balancing (via Azure Load Balancers).\r\n\r\nI did have to add `clusterName` as a parameter to the `LoadBalancers` methods. This is because Azure only allows one \"LoadBalancer\" object per set of backend machines. This means a single \"LoadBalancer\" object must be shared across the cluster. The \"LoadBalancer\" is named via the `cluster-name` parameter passed to `kube-controller-manager` so as to enable multiple clusters per resource group if the user desires such a configuration.\r\n\r\nThere are few things that I'm a bit unsure about:\r\n\r\n1. The implementation of the `Instances` interface. It's not extensively documented, it's not really clear what the different functions are used for, and my questions on the ML didn't get an answer.\r\n\r\n2. Counter to the comments on the `LoadBalancers` Interface, I modify the `api.Service` object in `EnsureLoadBalancerDeleted`, but not with the intention of affecting Kube's view of the Service. I simply do it so that I can remove the `Port`s on the `Service` object and then re-use my reconciliation logic that can handle removing stale/deleted Ports. \r\n\r\n3. The logging is a bit verbose. I'm looking for guidance on the appropriate log level to use for the chattier bits.\r\n\r\nDue to the (current) lack of Instance Metadata Service and lack of Virtual Machine Identity in Azure, the user is required to do a few things to opt-in to this provider. These things are called-out as they are in contrast to AWS/GCE:\r\n\r\n1. The user must provision an Azure Active Directory ServicePrincipal with `Contributor` level access to the resource group that the cluster is deployed in. This creation process is documented [by Hashicorp](https://www.packer.io/docs/builders/azure-setup.html) or [on the MSDN Blog](https://blogs.msdn.microsoft.com/arsen/2016/05/11/how-to-create-and-test-azure-service-principal-using-azure-cli/).\r\n\r\n2. The user must place a JSON file somewhere on each Node that conforms to the `AzureConfig` struct defined in `azure.go`. (This is automatically done in the Azure flavor of [Kubernetes-Anywhere](https://github.com/kubernetes/kubernetes-anywhere).)\r\n\r\n3. The user must specify `--cloud-config=/path/to/azure.json` as an option to `kube-apiserver` and `kube-controller-manager` similarly to how the user would need to pass `--cloud-provider=azure`.\r\n\r\nI've been running approximately this code for a month and a half. I only encountered one bug which has since been fixed and covered by a unit test. I've just deployed a new cluster (and a Type=LoadBalancer nginx Service) using this code (via `kubernetes-anywhere`) and have posted [the `kube-controller-manager` logs](https://gist.github.com/colemickens/1bf6a26e7ef9484a72a30b1fcf9fc3cb) for anyone who is interested in seeing the logs of the logic.\r\n\r\nIf you're interested in this PR, you can use the instructions in my [`azure-kubernetes-demo` repository](https://github.com/colemickens/azure-kubernetes-demo) to deploy a cluster with minimal effort via [`kubernetes-anywhere`](https://github.com/kubernetes/kubernetes-anywhere). (There is currently [a pending PR in `kubernetes-anywhere` that is needed](https://github.com/kubernetes/kubernetes-anywhere/pull/172) in conjuncture with this PR). I also have a pre-built `hyperkube` image: `docker.io/colemickens/hyperkube-amd64:v1.4.0-alpha.0-azure`, which will be kept in sync with the branch this PR stems from.\r\n\r\nI'm hoping this can land in the Kubernetes 1.4 timeframe.\r\n\r\nCC (potential code reviewers from Azure): @ahmetalpbalkan @brendandixon @paulmey\r\n\r\nCC (other interested Azure folk): @brendandburns @johngossman @anandramakrishna @jmspring @jimzim\r\n\r\nCC (others who've expressed interest): @codefx9 @edevil @thockin @rootfs","created_at":"2016-07-12T06:06:15Z","updated_at":"2016-07-12T15:32:40Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4a93a2ce99ad51ae87dca7802f72d4ab82e12bf1","assignee":{"login":"mikedanese","id":1787169,"avatar_url":"https://avatars.githubusercontent.com/u/1787169?v=3","gravatar_id":"","url":"https://api.github.com/users/mikedanese","html_url":"https://github.com/mikedanese","followers_url":"https://api.github.com/users/mikedanese/followers","following_url":"https://api.github.com/users/mikedanese/following{/other_user}","gists_url":"https://api.github.com/users/mikedanese/gists{/gist_id}","starred_url":"https://api.github.com/users/mikedanese/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikedanese/subscriptions","organizations_url":"https://api.github.com/users/mikedanese/orgs","repos_url":"https://api.github.com/users/mikedanese/repos","events_url":"https://api.github.com/users/mikedanese/events{/privacy}","received_events_url":"https://api.github.com/users/mikedanese/received_events","type":"User","site_admin":false},"assignees":[{"login":"mikedanese","id":1787169,"avatar_url":"https://avatars.githubusercontent.com/u/1787169?v=3","gravatar_id":"","url":"https://api.github.com/users/mikedanese","html_url":"https://github.com/mikedanese","followers_url":"https://api.github.com/users/mikedanese/followers","following_url":"https://api.github.com/users/mikedanese/following{/other_user}","gists_url":"https://api.github.com/users/mikedanese/gists{/gist_id}","starred_url":"https://api.github.com/users/mikedanese/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikedanese/subscriptions","organizations_url":"https://api.github.com/users/mikedanese/orgs","repos_url":"https://api.github.com/users/mikedanese/repos","events_url":"https://api.github.com/users/mikedanese/events{/privacy}","received_events_url":"https://api.github.com/users/mikedanese/received_events","type":"User","site_admin":false},{"login":"brendandburns","id":5751682,"avatar_url":"https://avatars.githubusercontent.com/u/5751682?v=3","gravatar_id":"","url":"https://api.github.com/users/brendandburns","html_url":"https://github.com/brendandburns","followers_url":"https://api.github.com/users/brendandburns/followers","following_url":"https://api.github.com/users/brendandburns/following{/other_user}","gists_url":"https://api.github.com/users/brendandburns/gists{/gist_id}","starred_url":"https://api.github.com/users/brendandburns/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brendandburns/subscriptions","organizations_url":"https://api.github.com/users/brendandburns/orgs","repos_url":"https://api.github.com/users/brendandburns/repos","events_url":"https://api.github.com/users/brendandburns/events{/privacy}","received_events_url":"https://api.github.com/users/brendandburns/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/28821/commits","review_comments_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/28821/comments","review_comment_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/comments{/number}","comments_url":"https://api.github.com/repos/kubernetes/kubernetes/issues/28821/comments","statuses_url":"https://api.github.com/repos/kubernetes/kubernetes/statuses/8a288e1e54df4b1710f17593eb42f4ec020b5a71","head":{"label":"colemickens:azure-cloudprovider-pr","ref":"azure-cloudprovider-pr","sha":"8a288e1e54df4b1710f17593eb42f4ec020b5a71","user":{"login":"colemickens","id":327028,"avatar_url":"https://avatars.githubusercontent.com/u/327028?v=3","gravatar_id":"","url":"https://api.github.com/users/colemickens","html_url":"https://github.com/colemickens","followers_url":"https://api.github.com/users/colemickens/followers","following_url":"https://api.github.com/users/colemickens/following{/other_user}","gists_url":"https://api.github.com/users/colemickens/gists{/gist_id}","starred_url":"https://api.github.com/users/colemickens/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/colemickens/subscriptions","organizations_url":"https://api.github.com/users/colemickens/orgs","repos_url":"https://api.github.com/users/colemickens/repos","events_url":"https://api.github.com/users/colemickens/events{/privacy}","received_events_url":"https://api.github.com/users/colemickens/received_events","type":"User","site_admin":false},"repo":{"id":45073694,"name":"kubernetes","full_name":"colemickens/kubernetes","owner":{"login":"colemickens","id":327028,"avatar_url":"https://avatars.githubusercontent.com/u/327028?v=3","gravatar_id":"","url":"https://api.github.com/users/colemickens","html_url":"https://github.com/colemickens","followers_url":"https://api.github.com/users/colemickens/followers","following_url":"https://api.github.com/users/colemickens/following{/other_user}","gists_url":"https://api.github.com/users/colemickens/gists{/gist_id}","starred_url":"https://api.github.com/users/colemickens/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/colemickens/subscriptions","organizations_url":"https://api.github.com/users/colemickens/orgs","repos_url":"https://api.github.com/users/colemickens/repos","events_url":"https://api.github.com/users/colemickens/events{/privacy}","received_events_url":"https://api.github.com/users/colemickens/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/colemickens/kubernetes","description":"Container Cluster Manager from Google","fork":true,"url":"https://api.github.com/repos/colemickens/kubernetes","forks_url":"https://api.github.com/repos/colemickens/kubernetes/forks","keys_url":"https://api.github.com/repos/colemickens/kubernetes/keys{/key_id}","collaborators_url":"https://api.github.com/repos/colemickens/kubernetes/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/colemickens/kubernetes/teams","hooks_url":"https://api.github.com/repos/colemickens/kubernetes/hooks","issue_events_url":"https://api.github.com/repos/colemickens/kubernetes/issues/events{/number}","events_url":"https://api.github.com/repos/colemickens/kubernetes/events","assignees_url":"https://api.github.com/repos/colemickens/kubernetes/assignees{/user}","branches_url":"https://api.github.com/repos/colemickens/kubernetes/branches{/branch}","tags_url":"https://api.github.com/repos/colemickens/kubernetes/tags","blobs_url":"https://api.github.com/repos/colemickens/kubernetes/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/colemickens/kubernetes/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/colemickens/kubernetes/git/refs{/sha}","trees_url":"https://api.github.com/repos/colemickens/kubernetes/git/trees{/sha}","statuses_url":"https://api.github.com/repos/colemickens/kubernetes/statuses/{sha}","languages_url":"https://api.github.com/repos/colemickens/kubernetes/languages","stargazers_url":"https://api.github.com/repos/colemickens/kubernetes/stargazers","contributors_url":"https://api.github.com/repos/colemickens/kubernetes/contributors","subscribers_url":"https://api.github.com/repos/colemickens/kubernetes/subscribers","subscription_url":"https://api.github.com/repos/colemickens/kubernetes/subscription","commits_url":"https://api.github.com/repos/colemickens/kubernetes/commits{/sha}","git_commits_url":"https://api.github.com/repos/colemickens/kubernetes/git/commits{/sha}","comments_url":"https://api.github.com/repos/colemickens/kubernetes/comments{/number}","issue_comment_url":"https://api.github.com/repos/colemickens/kubernetes/issues/comments{/number}","contents_url":"https://api.github.com/repos/colemickens/kubernetes/contents/{+path}","compare_url":"https://api.github.com/repos/colemickens/kubernetes/compare/{base}...{head}","merges_url":"https://api.github.com/repos/colemickens/kubernetes/merges","archive_url":"https://api.github.com/repos/colemickens/kubernetes/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/colemickens/kubernetes/downloads","issues_url":"https://api.github.com/repos/colemickens/kubernetes/issues{/number}","pulls_url":"https://api.github.com/repos/colemickens/kubernetes/pulls{/number}","milestones_url":"https://api.github.com/repos/colemickens/kubernetes/milestones{/number}","notifications_url":"https://api.github.com/repos/colemickens/kubernetes/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/colemickens/kubernetes/labels{/name}","releases_url":"https://api.github.com/repos/colemickens/kubernetes/releases{/id}","deployments_url":"https://api.github.com/repos/colemickens/kubernetes/deployments","created_at":"2015-10-27T22:21:42Z","updated_at":"2016-03-29T23:49:45Z","pushed_at":"2016-07-12T06:06:01Z","git_url":"git://github.com/colemickens/kubernetes.git","ssh_url":"git@github.com:colemickens/kubernetes.git","clone_url":"https://github.com/colemickens/kubernetes.git","svn_url":"https://github.com/colemickens/kubernetes","homepage":"http://kubernetes.io","size":309094,"stargazers_count":1,"watchers_count":1,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":1,"default_branch":"master"}},"base":{"label":"kubernetes:master","ref":"master","sha":"9b5827691a8c12c6a8e2222e3f1e0b4bd1692f87","user":{"login":"kubernetes","id":13629408,"avatar_url":"https://avatars.githubusercontent.com/u/13629408?v=3","gravatar_id":"","url":"https://api.github.com/users/kubernetes","html_url":"https://github.com/kubernetes","followers_url":"https://api.github.com/users/kubernetes/followers","following_url":"https://api.github.com/users/kubernetes/following{/other_user}","gists_url":"https://api.github.com/users/kubernetes/gists{/gist_id}","starred_url":"https://api.github.com/users/kubernetes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kubernetes/subscriptions","organizations_url":"https://api.github.com/users/kubernetes/orgs","repos_url":"https://api.github.com/users/kubernetes/repos","events_url":"https://api.github.com/users/kubernetes/events{/privacy}","received_events_url":"https://api.github.com/users/kubernetes/received_events","type":"Organization","site_admin":false},"repo":{"id":20580498,"name":"kubernetes","full_name":"kubernetes/kubernetes","owner":{"login":"kubernetes","id":13629408,"avatar_url":"https://avatars.githubusercontent.com/u/13629408?v=3","gravatar_id":"","url":"https://api.github.com/users/kubernetes","html_url":"https://github.com/kubernetes","followers_url":"https://api.github.com/users/kubernetes/followers","following_url":"https://api.github.com/users/kubernetes/following{/other_user}","gists_url":"https://api.github.com/users/kubernetes/gists{/gist_id}","starred_url":"https://api.github.com/users/kubernetes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kubernetes/subscriptions","organizations_url":"https://api.github.com/users/kubernetes/orgs","repos_url":"https://api.github.com/users/kubernetes/repos","events_url":"https://api.github.com/users/kubernetes/events{/privacy}","received_events_url":"https://api.github.com/users/kubernetes/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/kubernetes/kubernetes","description":"Production-Grade Container Scheduling and Management","fork":false,"url":"https://api.github.com/repos/kubernetes/kubernetes","forks_url":"https://api.github.com/repos/kubernetes/kubernetes/forks","keys_url":"https://api.github.com/repos/kubernetes/kubernetes/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kubernetes/kubernetes/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kubernetes/kubernetes/teams","hooks_url":"https://api.github.com/repos/kubernetes/kubernetes/hooks","issue_events_url":"https://api.github.com/repos/kubernetes/kubernetes/issues/events{/number}","events_url":"https://api.github.com/repos/kubernetes/kubernetes/events","assignees_url":"https://api.github.com/repos/kubernetes/kubernetes/assignees{/user}","branches_url":"https://api.github.com/repos/kubernetes/kubernetes/branches{/branch}","tags_url":"https://api.github.com/repos/kubernetes/kubernetes/tags","blobs_url":"https://api.github.com/repos/kubernetes/kubernetes/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kubernetes/kubernetes/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kubernetes/kubernetes/git/refs{/sha}","trees_url":"https://api.github.com/repos/kubernetes/kubernetes/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kubernetes/kubernetes/statuses/{sha}","languages_url":"https://api.github.com/repos/kubernetes/kubernetes/languages","stargazers_url":"https://api.github.com/repos/kubernetes/kubernetes/stargazers","contributors_url":"https://api.github.com/repos/kubernetes/kubernetes/contributors","subscribers_url":"https://api.github.com/repos/kubernetes/kubernetes/subscribers","subscription_url":"https://api.github.com/repos/kubernetes/kubernetes/subscription","commits_url":"https://api.github.com/repos/kubernetes/kubernetes/commits{/sha}","git_commits_url":"https://api.github.com/repos/kubernetes/kubernetes/git/commits{/sha}","comments_url":"https://api.github.com/repos/kubernetes/kubernetes/comments{/number}","issue_comment_url":"https://api.github.com/repos/kubernetes/kubernetes/issues/comments{/number}","contents_url":"https://api.github.com/repos/kubernetes/kubernetes/contents/{+path}","compare_url":"https://api.github.com/repos/kubernetes/kubernetes/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kubernetes/kubernetes/merges","archive_url":"https://api.github.com/repos/kubernetes/kubernetes/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kubernetes/kubernetes/downloads","issues_url":"https://api.github.com/repos/kubernetes/kubernetes/issues{/number}","pulls_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls{/number}","milestones_url":"https://api.github.com/repos/kubernetes/kubernetes/milestones{/number}","notifications_url":"https://api.github.com/repos/kubernetes/kubernetes/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kubernetes/kubernetes/labels{/name}","releases_url":"https://api.github.com/repos/kubernetes/kubernetes/releases{/id}","deployments_url":"https://api.github.com/repos/kubernetes/kubernetes/deployments","created_at":"2014-06-06T22:56:04Z","updated_at":"2016-07-12T14:28:47Z","pushed_at":"2016-07-12T15:26:31Z","git_url":"git://github.com/kubernetes/kubernetes.git","ssh_url":"git@github.com:kubernetes/kubernetes.git","clone_url":"https://github.com/kubernetes/kubernetes.git","svn_url":"https://github.com/kubernetes/kubernetes","homepage":"http://kubernetes.io","size":490517,"stargazers_count":15471,"watchers_count":15471,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":4854,"mirror_url":null,"open_issues_count":3926,"forks":4854,"open_issues":3926,"watchers":15471,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/28821"},"html":{"href":"https://github.com/kubernetes/kubernetes/pull/28821"},"issue":{"href":"https://api.github.com/repos/kubernetes/kubernetes/issues/28821"},"comments":{"href":"https://api.github.com/repos/kubernetes/kubernetes/issues/28821/comments"},"review_comments":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/28821/comments"},"review_comment":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/28821/commits"},"statuses":{"href":"https://api.github.com/repos/kubernetes/kubernetes/statuses/8a288e1e54df4b1710f17593eb42f4ec020b5a71"}}}}
{"actor":{"display_login":"brendandburns"}}