{"action":"created","comment":{"url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/comments/59942755","id":59942755,"diff_hunk":"@@ -0,0 +1,995 @@\n+#!/bin/sh\n+\n+# Copyright 2016 The Kubernetes Authors All rights reserved.\n+#\n+# Licensed under the Apache License, Version 2.0 (the \"License\");\n+# you may not use this file except in compliance with the License.\n+# You may obtain a copy of the License at\n+#\n+#     http://www.apache.org/licenses/LICENSE-2.0\n+#\n+# Unless required by applicable law or agreed to in writing, software\n+# distributed under the License is distributed on an \"AS IS\" BASIS,\n+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+# See the License for the specific language governing permissions and\n+# limitations under the License.\n+\n+# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts.\n+\n+KUBE_ROOT=$(dirname \"${BASH_SOURCE}\")/../..\n+source \"${KUBE_ROOT}/cluster/photon-controller/config-common.sh\"\n+source \"${KUBE_ROOT}/cluster/photon-controller/${KUBE_CONFIG_FILE-\"config-default.sh\"}\"\n+source \"${KUBE_ROOT}/cluster/common.sh\"\n+\n+PHOTON=\"photon -n\"\n+\n+#####################################################################\n+#\n+# Public API\n+#\n+#####################################################################\n+\n+#\n+# detect-master will query Photon Controller for the Kubernetes master.\n+# It assumes that the VM name for the master is unique.\n+# It will set KUBE_MASTER_ID to be the VM ID of the master\n+# It will set KUBE_MASTER_IP to be the IP address of the master\n+# If the silent parameter is passed, it will not print when the master\n+# is found: this is used internally just to find the MASTER\n+#\n+function detect-master {\n+    local silent=${1:-\"\"}\n+    local tenant_args=\"--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}\"\n+\n+    KUBE_MASTER=${MASTER_NAME}\n+    KUBE_MASTER_ID=${KUBE_MASTER_ID:-\"\"}\n+    KUBE_MASTER_IP=${KUBE_MASTER_IP:-\"\"}\n+\n+    # We don't want silent failure: we check for failure\n+    set +o pipefail\n+    if [ -z ${KUBE_MASTER_ID} ]; then\n+        KUBE_MASTER_ID=$($PHOTON vm list ${tenant_args} | grep $'\\t'\"kubernetes-master\"$'\\t' | awk '{print $1}')\n+    fi\n+    if [ -z ${KUBE_MASTER_ID} ]; then\n+        kube::log::error \"Could not find Kubernetes master node ID. Make sure you've launched a cluster with kube-up.sh\"\n+        exit 1\n+    fi\n+\n+    if [ -z \"${KUBE_MASTER_IP-}\" ]; then\n+        # Make sure to ignore lines where it's not attached to a portgroup\n+        # Make sure to ignore lines that have a network interface but no address\n+        KUBE_MASTER_IP=$($PHOTON vm networks ${KUBE_MASTER_ID} | grep -v \"^-\" | grep -E '[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+' | head -1 | awk -F'\\t' '{print $3}')\n+    fi\n+    if [ -z \"${KUBE_MASTER_IP-}\" ]; then\n+        kube::log::error \"Could not find Kubernetes master node IP. Make sure you've launched a cluster with 'kube-up.sh'\" >&2\n+        exit 1\n+    fi\n+    if [ -z $silent ]; then\n+        kube::log::status \"Master: $KUBE_MASTER ($KUBE_MASTER_IP)\"\n+    fi\n+    # Reset default set in common.sh\n+    set -o pipefail\n+}\n+\n+#\n+# detect-nodes will query Photon Controller for the Kubernetes nodes\n+# It assumes that the VM name for the nodes are unique.\n+# It assumes that NODE_NAMES has been set\n+# It will set KUBE_NODE_IP_ADDRESSES to be the VM IPs of the nodes\n+# It will set the KUBE_NODE_IDS to be the VM IDs of the nodes\n+# If the silent parameter is passed, it will not print when the nodes\n+# are found: this is used internally just to find the MASTER\n+#\n+function detect-nodes {\n+    local silent=${1:-\"\"}\n+    local failure=0\n+    local tenant_args=\"--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}\"\n+\n+    KUBE_NODE_IP_ADDRESSES=()\n+    KUBE_NODE_IDS=()\n+    # We don't want silent failure: we check for failure\n+    set +o pipefail\n+    for (( i=0; i<${#NODE_NAMES[@]}; i++)); do\n+\n+        local node_id=$($PHOTON vm list ${tenant_args} | grep $'\\t'${NODE_NAMES[$i]}$'\\t' | awk '{print $1}')\n+        if [ -z ${node_id} ]; then\n+            kube::log::error \"Could not find ${NODE_NAMES[$i]}\"\n+            failure=1\n+        fi\n+        KUBE_NODE_IDS+=(\"${node_id}\")\n+\n+        # Make sure to ignore lines where it's not attached to a portgroup\n+        # Make sure to ignore lines that have a network interface but no address\n+        local node_ip=$($PHOTON vm networks ${node_id} | grep -v \"^-\" | grep -E '[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+' | head -1 | awk -F'\\t' '{print $3}')\n+        KUBE_NODE_IP_ADDRESSES+=(\"${node_ip}\")\n+\n+        if [ -z $silent ]; then\n+            kube::log::status \"Node: ${NODE_NAMES[$i]} (${KUBE_NODE_IP_ADDRESSES[$i]})\"\n+        fi\n+    done\n+\n+    if [ $failure -ne 0 ]; then\n+        exit 1\n+    fi\n+    # Reset default set in common.sh\n+    set -o pipefail\n+}\n+\n+# Get node names if they are not static.\n+function detect-node-names {\n+\techo \"TODO: detect-node-names\" 1>&2\n+}\n+\n+#\n+# Verifies that this computer has sufficient software installed\n+# so that it can run the rest of the script.\n+#\n+function verify-prereqs {\n+    verify-cmd-in-path photon\n+    verify-cmd-in-path ssh\n+    verify-cmd-in-path scp\n+    verify-cmd-in-path ssh-add\n+    verify-cmd-in-path openssl\n+    verify-cmd-in-path mkisofs\n+}\n+\n+#\n+# The entry point for bringing up a Kubernetes cluster\n+#\n+function kube-up {\n+    verify-prereqs\n+    verify-ssh-prereqs\n+    verify-photon-config\n+    ensure-temp-dir\n+    find-release-tars\n+    find-image-id\n+\n+    load-or-gen-kube-basicauth\n+    gen-cloud-init-iso\n+    gen-master-start\n+    create-master-vm\n+    install-salt-on-master\n+\n+    gen-node-start\n+    install-nodes\n+\n+    detect-nodes -s\n+\n+    install-kubernetes-on-master\n+    install-kubernetes-on-nodes\n+\n+    wait-master-api\n+    wait-node-apis\n+\n+    setup-pod-routes\n+\n+    copy-kube-certs\n+    kube::log::status \"Creating kubeconfig...\"\n+    create-kubeconfig\n+}\n+\n+# Delete a kubernetes cluster\n+function kube-down {\n+    detect-master\n+    detect-nodes\n+\n+    pc-delete-vm $KUBE_MASTER $KUBE_MASTER_ID\n+    for (( node=0; node<${#KUBE_NODE_IDS[@]}; node++)); do\n+        pc-delete-vm ${NODE_NAMES[$node]} ${KUBE_NODE_IDS[$node]}\n+    done\n+}\n+\n+# Update a kubernetes cluster\n+function kube-push {\n+\techo \"TODO: kube-push\" 1>&2\n+}\n+\n+# Prepare update a kubernetes component\n+function prepare-push {\n+\techo \"TODO: prepare-push\" 1>&2\n+}\n+\n+# Update a kubernetes master\n+function push-master {\n+\techo \"TODO: push-master\" 1>&2\n+}\n+\n+# Update a kubernetes node\n+function push-node {\n+\techo \"TODO: push-node\" 1>&2\n+}\n+\n+# Execute prior to running tests to build a release if required for env\n+function test-build-release {\n+\techo \"TODO: test-build-release\" 1>&2\n+}\n+\n+# Execute prior to running tests to initialize required structure\n+function test-setup {\n+\techo \"TODO: test-setup\" 1>&2\n+}\n+\n+# Execute after running tests to perform any required clean-up\n+function test-teardown {\n+\techo \"TODO: test-teardown\" 1>&2\n+}\n+\n+#####################################################################\n+#\n+# Internal functions\n+#\n+#####################################################################\n+\n+#\n+# Uses Photon Controller to make a VM\n+# Takes two parameters:\n+#   - The name of the VM (Assumed to be unique)\n+#   - The name of the flavor to create the VM (Assumed to be unique)\n+#\n+# It assumes that the variables in config-common.sh (PHOTON_TENANT, etc)\n+# are set correctly.\n+#\n+# It also assumes the cloud-init ISO has been generated\n+#\n+# When it completes, it sets two environment variables for use by the\n+# caller: _VM_ID (the ID of the created VM) and _VM_IP (the IP address\n+# of the created VM)\n+#\n+function pc-create-vm {\n+    local vm_name=\"$1\"\n+    local vm_flavor=\"$2\"\n+    local rc=0\n+    local i=0\n+\n+    # Create the VM\n+    local tenant_args=\"--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}\"\n+    local vm_args=\"--name ${vm_name} --image ${PHOTON_IMAGE_ID} --flavor ${vm_flavor}\"\n+    local disk_args=\"disk-1 ${PHOTON_DISK_FLAVOR} boot=true\"\n+\n+    rc=0\n+    _VM_ID=$($PHOTON vm create ${tenant_args} ${vm_args} --disks \"${disk_args}\" 2>&1) || rc=$?\n+    if [ $rc -ne 0 ]; then\n+        kube::log::error \"Failed to create VM. Error output:\"\n+        echo ${_VM_ID}\n+        exit 1\n+    fi\n+    kube::log::status \"Created VM ${vm_name}: ${_VM_ID}\"\n+\n+    # Start the VM\n+    # Note that the VM has cloud-init in it, and we attach an ISO that\n+    # contains a user-data.txt file for cloud-init. When the VM starts,\n+    # cloud-init will temporarily mount the ISO and configure the VM\n+    # Our user-data will configure the 'kube' user and set up the ssh\n+    # authorized keys to allow us to ssh to the VM and do further work.\n+    run-cmd \"$PHOTON vm attach-iso -p ${KUBE_TEMP}/cloud-init.iso ${_VM_ID}\"\n+    run-cmd \"$PHOTON vm start ${_VM_ID}\"\n+    kube::log::status \"Started VM ${vm_name}, waiting for network address...\"\n+\n+    # Wait for the VM to be started and connected to the network\n+    have_network=0\n+    for i in $(seq 120); do\n+        # photon -n vm networks print several fields:\n+        # NETWORK MAC IP GATEWAY CONNECTED?\n+        # We wait until CONNECTED is True\n+        rc=0\n+        networks=$($PHOTON vm networks ${_VM_ID}) || rc=$?\n+        if [ $rc -ne 0 ]; then\n+            kube::log::error \"'$PHOTON vm networks ${_VM_ID}' failed. Error output: \"\n+            echo $networks\n+        fi\n+        networks=$(echo \"$networks\" | grep True) || rc=$?\n+        if [ $rc -eq 0 ]; then\n+            have_network=1\n+            break;\n+        fi\n+        sleep 1\n+    done\n+\n+    # Fail if the VM didn't come up\n+    if [ $have_network -eq 0 ]; then\n+        kube::log::error \"VM ${vm_name} failed to start up: no IP was found\"\n+        exit 1\n+    fi\n+\n+    # Find the IP address of the VM\n+    _VM_IP=$($PHOTON vm networks ${_VM_ID} | head -1 | awk -F'\\t' '{print $3}')\n+    kube::log::status \"VM ${vm_name} has IP: ${_VM_IP}\"\n+}\n+\n+#\n+# Delete one of our VMs\n+# If it is STARTED, it will be stopped first.\n+#\n+function pc-delete-vm {\n+    local vm_name=\"$1\"\n+    local vm_id=\"$2\"\n+    local rc=0\n+\n+    kube::log::status \"Deleting VM ${vm_name}\"\n+    $PHOTON vm show $vm_id | head -1 | grep STARTED > /dev/null 2>&1 || rc=$?\n+    if [ $rc -eq 0 ]; then\n+        $PHOTON vm stop $vm_id > /dev/null 2>&1 || rc=$?\n+        if [ $rc -ne 0 ]; then\n+            kube::log::error \"Error: could not stop ${vm_name} ($vm_id)\"\n+            kube::log::error \"Please investigate and stop manually\"\n+            return\n+        fi\n+    fi\n+\n+    rc=0\n+    $PHOTON vm delete $vm_id > /dev/null 2>&1 || rc=$?\n+    if [ $rc -ne 0 ]; then\n+        kube::log::error \"Error: could not delete ${vm_name} ($vm_id)\"\n+        kube::log::error \"Please investigate and delete manually\"\n+    fi\n+}\n+\n+#\n+# Looks for the image named PHOTON_IMAGE\n+# Sets PHOTON_IMAGE_ID to be the id of that image.\n+# We currently assume there is exactly one image with name\n+#\n+function find-image-id {\n+    local rc=0\n+    PHOTON_IMAGE_ID=$($PHOTON image list | head -1 | grep $'\\t'${PHOTON_IMAGE}$'\\t' | grep READY | awk -F'\\t' '{print $1}')\n+    if [ $rc -ne 0 ]; then\n+        kube::log::error \"Cannot find image \\\"${PHOTON_IMAGE}\\\"\"\n+        fail=1\n+    fi\n+}\n+\n+#\n+# Generate an ISO with a single file called user-data.txt\n+# This ISO will be used to configure cloud-init (which is already\n+# on the VM). We will tell cloud-init to create the kube user/group\n+# and give ourselves the ability to ssh to the VM with ssh. We also\n+# allow people to ssh with the same password that was randomly\n+# generated for access to Kubernetes as a backup method.\n+#\n+# Assumes environment variables:\n+#   - VM_USER\n+#   - KUBE_PASSWORD (randomly generated password)\n+#\n+function gen-cloud-init-iso {\n+    local password_hash\n+    password_hash=$(openssl passwd -1 ${KUBE_PASSWORD})\n+\n+    local ssh_key\n+    ssh_key=$(ssh-add -L | head -1)\n+\n+    # Make the user-data file that will be used by cloud-init\n+    (\n+        echo \"#cloud-config\"\n+        echo \"\"\n+        echo \"groups:\"\n+        echo \"  - ${VM_USER}\"\n+        echo \"\"\n+        echo \"users:\"\n+        echo \"  - name: ${VM_USER}\"\n+        echo \"    gecos: Kubernetes\"\n+        echo \"    primary-group: ${VM_USER}\"\n+        echo \"    lock-passwd: false\"\n+        echo \"    passwd: ${password_hash}\"\n+        echo \"    ssh-authorized-keys: \"\n+        echo \"      - ${ssh_key}\"\n+        echo \"    sudo: ALL=(ALL) NOPASSWD:ALL\"\n+        echo \"    shell: /bin/bash\"\n+        echo \"\"\n+        echo \"hostname:\"\n+        echo \"  - hostname: kube\"\n+    ) > ${KUBE_TEMP}/user-data.txt\n+\n+    # Make the ISO that will contain the user-data\n+    # The -rock option means that we'll generate real filenames (long and with case)\n+    run-cmd \"mkisofs -rock -o ${KUBE_TEMP}/cloud-init.iso ${KUBE_TEMP}/user-data.txt\"\n+}\n+\n+#\n+# Generate a script used to install salt on the master\n+# It is placed into $KUBE_TEMP/master-start.sh\n+#\n+function gen-master-start {\n+    python \"${KUBE_ROOT}/third_party/htpasswd/htpasswd.py\" \\\n+        -b -c \"${KUBE_TEMP}/htpasswd\" \"$KUBE_USER\" \"$KUBE_PASSWORD\"\n+    local htpasswd\n+    htpasswd=$(cat \"${KUBE_TEMP}/htpasswd\")\n+\n+    (\n+        echo \"#! /bin/bash\"\n+        echo \"readonly MY_NAME=${MASTER_NAME}\"\n+        grep -v \"^#\" \"${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh\"\n+        echo \"cd /home/kube/cache/kubernetes-install\"\n+        echo \"readonly MASTER_NAME='${MASTER_NAME}'\"\n+        echo \"readonly MASTER_IP_RANGE='${MASTER_IP_RANGE}'\"\n+        echo \"readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'\"\n+        echo \"readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-node'\"\n+        echo \"readonly NODE_IP_RANGES='${NODE_IP_RANGES}'\"\n+        echo \"readonly SERVICE_CLUSTER_IP_RANGE='${SERVICE_CLUSTER_IP_RANGE}'\"\n+        echo \"readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'\"\n+        echo \"readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'\"\n+        echo \"readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'\"\n+        echo \"readonly ENABLE_CLUSTER_UI='${ENABLE_CLUSTER_UI:-false}'\"\n+        echo \"readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'\"\n+        echo \"readonly DNS_DOMAIN='${DNS_DOMAIN:-}'\"\n+        echo \"readonly KUBE_USER='${KUBE_USER:-}'\"\n+        echo \"readonly KUBE_PASSWORD='${KUBE_PASSWORD:-}'\"\n+        echo \"readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'\"\n+        echo \"readonly SALT_TAR='${SALT_TAR##*/}'\"\n+        echo \"readonly MASTER_HTPASSWD='${htpasswd}'\"\n+        echo \"readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'\"\n+        echo \"readonly MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'\"\n+        grep -v \"^#\" \"${KUBE_ROOT}/cluster/photon-controller/templates/create-dynamic-salt-files.sh\"\n+        grep -v \"^#\" \"${KUBE_ROOT}/cluster/photon-controller/templates/install-release.sh\"\n+        grep -v \"^#\" \"${KUBE_ROOT}/cluster/photon-controller/templates/salt-master.sh\"\n+    ) > \"${KUBE_TEMP}/master-start.sh\"\n+}\n+\n+#\n+# Generate the scripts for each node to install salt\n+#\n+function gen-node-start {\n+    local i\n+    for (( i=0; i<${#NODE_NAMES[@]}; i++)); do\n+        (\n+            echo \"#! /bin/bash\"\n+            echo \"readonly MY_NAME=${NODE_NAMES[$i]}\"\n+            grep -v \"^#\" \"${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh\"\n+            echo \"KUBE_MASTER=${KUBE_MASTER}\"\n+            echo \"KUBE_MASTER_IP=${KUBE_MASTER_IP}\"\n+            echo \"NODE_IP_RANGE=$NODE_IP_RANGES\"\n+            grep -v \"^#\" \"${KUBE_ROOT}/cluster/photon-controller/templates/salt-minion.sh\"\n+        ) > \"${KUBE_TEMP}/node-start-${i}.sh\"\n+  done\n+}\n+\n+#\n+# Create a script that will run on the Kubernetes master and will run salt\n+# to configure the master. We make it a script instead of just running a\n+# single ssh command so that we can get logging.\n+#\n+function gen-master-salt {\n+     (\n+        echo '#!/bin/bash'\n+        echo 'echo $(date) >> /tmp/master-salt.log'\n+        echo \"salt kubernetes-master state.highstate -t 30 --no-color > /tmp/master-salt.out\"\n+        echo 'grep -E \"Failed:[[:space:]]+0\" /tmp/master-salt.out'\n+        echo 'success=$?'\n+        echo 'cat /tmp/master-salt.out >> /tmp/master-salt.log'\n+        echo 'exit $success'\n+    ) > ${KUBE_TEMP}/salt-master.sh\n+}\n+\n+#\n+# Create scripts that will be run on the Kubernets master. Each of these\n+# will invoke salt to configure one of the nodes\n+#\n+function gen-node-salt {\n+    local i\n+    for (( i=0; i<${#NODE_NAMES[@]}; i++)); do\n+        (\n+            echo '#!/bin/bash'\n+            echo \"echo \\$(date) >> /tmp/${NODE_NAMES[$i]}-salt.log\"\n+            echo \"salt ${NODE_NAMES[$i]} state.highstate -t 30 --no-color > /tmp/${NODE_NAMES[$i]}-salt.out\"\n+            echo \"grep -E \\\"Failed:[[:space:]]+0\\\" /tmp/${NODE_NAMES[$i]}-salt.out\"\n+            echo 'success=$?'\n+            echo \"cat /tmp/${NODE_NAMES[$i]}-salt.out >> /tmp/${NODE_NAMES[$i]}-salt.log\"\n+            echo 'exit $success'\n+        ) > \"${KUBE_TEMP}/${NODE_NAMES[$i]}-salt.sh\"\n+    done\n+}\n+\n+function create-master-vm {\n+    kube::log::status \"Starting master VM...\"\n+    pc-create-vm ${MASTER_NAME} ${PHOTON_MASTER_FLAVOR}\n+    KUBE_MASTER=${MASTER_NAME}\n+    KUBE_MASTER_ID=$_VM_ID\n+    KUBE_MASTER_IP=$_VM_IP\n+}\n+\n+function install-salt-on-master {\n+    kube::log::status \"Installing salt on master...\"\n+    upload-server-tars ${MASTER_NAME} ${KUBE_MASTER_IP}\n+    run-script-remotely ${KUBE_MASTER_IP} ${KUBE_TEMP}/master-start.sh\n+}\n+\n+function install-nodes {\n+    kube::log::status \"Creating nodes and installing salt on them...\"\n+\n+    # Start each of the VMs in parallel\n+    local node\n+    for (( node=0; node<${#NODE_NAMES[@]}; node++)); do\n+    (\n+        pc-create-vm ${NODE_NAMES[$node]} ${PHOTON_NODE_FLAVOR}\n+        run-script-remotely ${_VM_IP} \"${KUBE_TEMP}/node-start-${node}.sh\"\n+    ) &\n+    done\n+\n+    # Wait for the node VM startups to complete\n+    local fail=0\n+    local job\n+    for job in $(jobs -p); do\n+        wait \"${job}\" || fail=$((fail + 1))\n+    done\n+    if (( $fail != 0 )); then\n+        kube::log::error \"Failed to start ${fail}/${NUM_NODES} nodes\"\n+        exit 1\n+    fi\n+}\n+\n+function install-kubernetes-on-master {\n+    # Wait until salt-master is running: it may take a bit\n+    try-until-success-ssh ${KUBE_MASTER_IP} \\\n+        \"Waiting for salt-master to start on ${KUBE_MASTER}\" \\\n+        \"pgrep salt-master\"\n+    gen-master-salt\n+    copy-file-to-vm ${_VM_IP} ${KUBE_TEMP}/salt-master.sh \"/tmp/master-salt.sh\"\n+    try-until-success-ssh ${KUBE_MASTER_IP} \\\n+        \"Installing Kubernetes on ${KUBE_MASTER} via salt\" \\\n+        \"sudo /bin/bash /tmp/master-salt.sh\"\n+}\n+\n+function install-kubernetes-on-nodes {\n+    gen-node-salt\n+\n+    # Run in parallel to bring up the cluster faster\n+    # TODO: Batch this so that we run up to N in parallel, so\n+    # we don't overload this machine\n+    local node\n+    for (( node=0; node<${#NODE_NAMES[@]}; node++)); do\n+    (\n+        copy-file-to-vm ${_VM_IP} ${KUBE_TEMP}/${NODE_NAMES[$node]}-salt.sh \"/tmp/${NODE_NAMES[$node]}-salt.sh\"\n+        try-until-success-ssh ${KUBE_NODE_IP_ADDRESSES[$node]} \\\n+            \"Waiting for salt-master to start on ${NODE_NAMES[$node]}\" \\\n+            \"pgrep salt-minion\"\n+        try-until-success-ssh ${KUBE_MASTER_IP} \\\n+            \"Installing Kubernetes on ${NODE_NAMES[$node]} via salt\" \\\n+            \"sudo /bin/bash /tmp/${NODE_NAMES[$node]}-salt.sh\"\n+    ) &\n+    done\n+\n+    # Wait for the Kubernetes installations to complete\n+    local fail=0\n+    local job\n+    for job in $(jobs -p); do\n+        wait \"${job}\" || fail=$((fail + 1))\n+    done\n+    if (( $fail != 0 )); then\n+        kube::log::error \"Failed to start install Kubernetes on ${fail} out of ${NUM_NODES} nodess\"\n+        exit 1\n+    fi\n+}\n+\n+#\n+# Upload the Kubernetes tarballs to the master\n+#\n+function upload-server-tars {\n+    vm_name=$1\n+    vm_ip=$2\n+\n+    run-ssh-cmd ${vm_ip} \"mkdir -p /home/kube/cache/kubernetes-install\"\n+\n+    local tar\n+    for tar in \"${SERVER_BINARY_TAR}\" \"${SALT_TAR}\"; do\n+        local base_tar\n+        base_tar=$(basename $tar)\n+        kube::log::status \"Uploading ${base_tar} to ${vm_name}...\"\n+        copy-file-to-vm ${vm_ip} \"${tar}\" \"/home/kube/cache/kubernetes-install/${tar##*/}\"\n+    done\n+}\n+\n+#\n+# Wait for the Kubernets healthz API to be responsive on the master\n+#\n+function wait-master-api {\n+    local curl_creds=\"--insecure --user ${KUBE_USER}:${KUBE_PASSWORD}\"\n+    local curl_output=\"--fail --output /dev/null --silent\"\n+    local curl_net=\"--max-time 1\"\n+\n+    try-until-success \"Waiting for Kubernetes API on ${KUBE_MASTER}\" \\\n+        \"curl ${curl_creds} ${curl_output} ${curl_net} https://${KUBE_MASTER_IP}/healthz\"\n+}\n+\n+#\n+# Wait for the Kubernetes healthz API to be responsive on each node\n+#\n+function wait-node-apis {\n+    local curl_output=\"--fail --output /dev/null --silent\"\n+    local curl_net=\"--max-time 1\"\n+\n+    for (( i=0; i<${#NODE_NAMES[@]}; i++)); do\n+        try-until-success \"Waiting for Kubernetes API on ${NODE_NAMES[$i]}...\" \\\n+            \"curl ${curl_output} ${curl_net} http://${KUBE_NODE_IP_ADDRESSES[$i]}:10250/healthz\"\n+    done\n+}\n+\n+#\n+# Configure the nodes so the pods can communicate\n+# Each node will have a bridge named cbr0 for the NODE_IP_RANGES\n+# defined in config-default.sh. This finds the IP subnet (assigned\n+# by Kubernetes) to nodes and configures routes so they can communicate\n+#\n+# Also configure the master to be able to talk to the nodes. This is\n+# useful so that you can get to the UI from the master.\n+#\n+function setup-pod-routes {\n+    local node\n+\n+    KUBE_NODE_BRIDGE_NETWORK=()\n+    for (( node=0; node<${#NODE_NAMES[@]}; node++)); do\n+\n+        # This happens in two steps (wait for an address, wait for a non 172.x.x.x address)\n+        # because it's both simpler and more clear what's happening.\n+        try-until-success-ssh ${KUBE_NODE_IP_ADDRESSES[$node]} \\\n+            \"Waiting for cbr0 bridge on ${NODE_NAMES[$node]} to have an address\"  \\\n+            'sudo ifconfig cbr0  | grep -oP \"inet addr:\\K\\S+\"'\n+\n+        try-until-success-ssh ${KUBE_NODE_IP_ADDRESSES[$node]} \\\n+            \"Waiting for cbr0 bridge on ${NODE_NAMES[$node]} to have correct address\"  \\\n+            'sudo ifconfig cbr0  | grep -oP \"inet addr:\\K\\S+\" | grep -v  \"^172.\"'\n+\n+        run-ssh-cmd ${KUBE_NODE_IP_ADDRESSES[$node]} 'sudo ip route show | grep -E \"dev cbr0\" | cut -d \" \" -f1'\n+        KUBE_NODE_BRIDGE_NETWORK+=($_OUTPUT)\n+        kube::log::status \"cbr0 on ${NODE_NAMES[$node]} is ${_OUTPUT}\"\n+    done\n+\n+    local i\n+    local j\n+    for (( i=0; i<${#NODE_NAMES[@]}; i++)); do\n+        kube::log::status \"Configuring pod routes on ${NODE_NAMES[$i]}...\"\n+        run-ssh-cmd ${KUBE_MASTER_IP} \"sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$i]} gw ${KUBE_NODE_IP_ADDRESSES[$i]}\"\n+        for (( j=0; j<${#NODE_NAMES[@]}; j++)); do\n+            if [[ $i != $j ]]; then\n+                run-ssh-cmd ${KUBE_NODE_IP_ADDRESSES[$i]} \"sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$j]} gw ${KUBE_NODE_IP_ADDRESSES[$j]}\"\n+            fi\n+        done\n+    done\n+}\n+\n+#\n+# Copy the certificate/key from the Kubernetes master\n+# These are used to create the kubeconfig file, which allows\n+# users to use kubectl easily\n+#\n+function copy-kube-certs {\n+    local cert=\"kubecfg.crt\"\n+    local key=\"kubecfg.key\"\n+    local ca=\"ca.crt\"\n+    local cert_dir=\"/srv/kubernetes\"\n+\n+    kube::log::status \"Copying credentials from ${KUBE_MASTER}\"\n+\n+    KUBE_CERT=\"${KUBE_TEMP}/${cert}\"\n+    KUBE_KEY=\"${KUBE_TEMP}/${key}\"\n+    CA_CERT=\"${KUBE_TEMP}/${ca}\"","path":"cluster/photon-controller/util.sh","position":663,"original_position":663,"commit_id":"5699d51efadb41b6c010f45672c4844d84de0b84","original_commit_id":"5699d51efadb41b6c010f45672c4844d84de0b84","user":{"login":"mikedanese","id":1787169,"avatar_url":"https://avatars.githubusercontent.com/u/1787169?v=3","gravatar_id":"","url":"https://api.github.com/users/mikedanese","html_url":"https://github.com/mikedanese","followers_url":"https://api.github.com/users/mikedanese/followers","following_url":"https://api.github.com/users/mikedanese/following{/other_user}","gists_url":"https://api.github.com/users/mikedanese/gists{/gist_id}","starred_url":"https://api.github.com/users/mikedanese/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikedanese/subscriptions","organizations_url":"https://api.github.com/users/mikedanese/orgs","repos_url":"https://api.github.com/users/mikedanese/repos","events_url":"https://api.github.com/users/mikedanese/events{/privacy}","received_events_url":"https://api.github.com/users/mikedanese/received_events","type":"User","site_admin":false},"body":"These are very global :(","created_at":"2016-04-15T21:20:45Z","updated_at":"2016-04-15T21:20:45Z","html_url":"https://github.com/kubernetes/kubernetes/pull/24124#discussion_r59942755","pull_request_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/24124","_links":{"self":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/comments/59942755"},"html":{"href":"https://github.com/kubernetes/kubernetes/pull/24124#discussion_r59942755"},"pull_request":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/24124"}}},"pull_request":{"url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/24124","id":66081887,"html_url":"https://github.com/kubernetes/kubernetes/pull/24124","diff_url":"https://github.com/kubernetes/kubernetes/pull/24124.diff","patch_url":"https://github.com/kubernetes/kubernetes/pull/24124.patch","issue_url":"https://api.github.com/repos/kubernetes/kubernetes/issues/24124","number":24124,"state":"open","locked":false,"title":"Initial kube-up support for VMware's Photon Controller","user":{"login":"AlainRoy","id":887364,"avatar_url":"https://avatars.githubusercontent.com/u/887364?v=3","gravatar_id":"","url":"https://api.github.com/users/AlainRoy","html_url":"https://github.com/AlainRoy","followers_url":"https://api.github.com/users/AlainRoy/followers","following_url":"https://api.github.com/users/AlainRoy/following{/other_user}","gists_url":"https://api.github.com/users/AlainRoy/gists{/gist_id}","starred_url":"https://api.github.com/users/AlainRoy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AlainRoy/subscriptions","organizations_url":"https://api.github.com/users/AlainRoy/orgs","repos_url":"https://api.github.com/users/AlainRoy/repos","events_url":"https://api.github.com/users/AlainRoy/events{/privacy}","received_events_url":"https://api.github.com/users/AlainRoy/received_events","type":"User","site_admin":false},"body":"This is for: https://github.com/kubernetes/kubernetes/issues/24121\r\n\r\nPhoton Controller is an open-source cloud management platform. More\r\ninformation is available at:\r\nhttp://vmware.github.io/photon-controller/\r\n\r\nThis commit provides initial support for Photon Controller. The\r\nfollowing features are tested and working:\r\n- kube-up and kube-down\r\n- Basic pod and service management\r\n- Networking within the Kubernetes cluster\r\n- UI and DNS addons\r\n\r\nIt has been tested with a Kubernetes cluster of up to 10\r\nnodes. Further work on scaling is planned for the near future.\r\n\r\nInternally we have implemented continuous integration testing and will\r\nrun it multiple times per day against the Kubernetes master branch\r\nonce this is integrated so we can quickly react to problems.\r\n\r\nA few things have not yet been implemented, but are planned:\r\n- Support for kube-push\r\n- Support for test-build-release, test-setup, test-teardown\r\n\r\nAssuming this is accepted for inclusion, we will write documentation\r\nfor the kubernetes.io site.\r\n\r\nWe have included a script to help users configure Photon Controller\r\nfor use with Kubernetes. While not required, it will help some\r\nusers get started more quickly. It will be documented.\r\n\r\nWe are aware of the kube-deploy efforts and will track them and\r\nsupport them as appropriate.","created_at":"2016-04-12T00:00:04Z","updated_at":"2016-04-15T21:20:45Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3845a25aa9bb85e702954cea2dafb0eb5d4eec8e","assignee":{"login":"mikedanese","id":1787169,"avatar_url":"https://avatars.githubusercontent.com/u/1787169?v=3","gravatar_id":"","url":"https://api.github.com/users/mikedanese","html_url":"https://github.com/mikedanese","followers_url":"https://api.github.com/users/mikedanese/followers","following_url":"https://api.github.com/users/mikedanese/following{/other_user}","gists_url":"https://api.github.com/users/mikedanese/gists{/gist_id}","starred_url":"https://api.github.com/users/mikedanese/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikedanese/subscriptions","organizations_url":"https://api.github.com/users/mikedanese/orgs","repos_url":"https://api.github.com/users/mikedanese/repos","events_url":"https://api.github.com/users/mikedanese/events{/privacy}","received_events_url":"https://api.github.com/users/mikedanese/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/24124/commits","review_comments_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/24124/comments","review_comment_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls/comments{/number}","comments_url":"https://api.github.com/repos/kubernetes/kubernetes/issues/24124/comments","statuses_url":"https://api.github.com/repos/kubernetes/kubernetes/statuses/5699d51efadb41b6c010f45672c4844d84de0b84","head":{"label":"vmware:photon-controller","ref":"photon-controller","sha":"5699d51efadb41b6c010f45672c4844d84de0b84","user":{"login":"vmware","id":473334,"avatar_url":"https://avatars.githubusercontent.com/u/473334?v=3","gravatar_id":"","url":"https://api.github.com/users/vmware","html_url":"https://github.com/vmware","followers_url":"https://api.github.com/users/vmware/followers","following_url":"https://api.github.com/users/vmware/following{/other_user}","gists_url":"https://api.github.com/users/vmware/gists{/gist_id}","starred_url":"https://api.github.com/users/vmware/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vmware/subscriptions","organizations_url":"https://api.github.com/users/vmware/orgs","repos_url":"https://api.github.com/users/vmware/repos","events_url":"https://api.github.com/users/vmware/events{/privacy}","received_events_url":"https://api.github.com/users/vmware/received_events","type":"Organization","site_admin":false},"repo":{"id":49536898,"name":"kubernetes","full_name":"vmware/kubernetes","owner":{"login":"vmware","id":473334,"avatar_url":"https://avatars.githubusercontent.com/u/473334?v=3","gravatar_id":"","url":"https://api.github.com/users/vmware","html_url":"https://github.com/vmware","followers_url":"https://api.github.com/users/vmware/followers","following_url":"https://api.github.com/users/vmware/following{/other_user}","gists_url":"https://api.github.com/users/vmware/gists{/gist_id}","starred_url":"https://api.github.com/users/vmware/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vmware/subscriptions","organizations_url":"https://api.github.com/users/vmware/orgs","repos_url":"https://api.github.com/users/vmware/repos","events_url":"https://api.github.com/users/vmware/events{/privacy}","received_events_url":"https://api.github.com/users/vmware/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/vmware/kubernetes","description":"Container Cluster Manager from Google","fork":true,"url":"https://api.github.com/repos/vmware/kubernetes","forks_url":"https://api.github.com/repos/vmware/kubernetes/forks","keys_url":"https://api.github.com/repos/vmware/kubernetes/keys{/key_id}","collaborators_url":"https://api.github.com/repos/vmware/kubernetes/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/vmware/kubernetes/teams","hooks_url":"https://api.github.com/repos/vmware/kubernetes/hooks","issue_events_url":"https://api.github.com/repos/vmware/kubernetes/issues/events{/number}","events_url":"https://api.github.com/repos/vmware/kubernetes/events","assignees_url":"https://api.github.com/repos/vmware/kubernetes/assignees{/user}","branches_url":"https://api.github.com/repos/vmware/kubernetes/branches{/branch}","tags_url":"https://api.github.com/repos/vmware/kubernetes/tags","blobs_url":"https://api.github.com/repos/vmware/kubernetes/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/vmware/kubernetes/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/vmware/kubernetes/git/refs{/sha}","trees_url":"https://api.github.com/repos/vmware/kubernetes/git/trees{/sha}","statuses_url":"https://api.github.com/repos/vmware/kubernetes/statuses/{sha}","languages_url":"https://api.github.com/repos/vmware/kubernetes/languages","stargazers_url":"https://api.github.com/repos/vmware/kubernetes/stargazers","contributors_url":"https://api.github.com/repos/vmware/kubernetes/contributors","subscribers_url":"https://api.github.com/repos/vmware/kubernetes/subscribers","subscription_url":"https://api.github.com/repos/vmware/kubernetes/subscription","commits_url":"https://api.github.com/repos/vmware/kubernetes/commits{/sha}","git_commits_url":"https://api.github.com/repos/vmware/kubernetes/git/commits{/sha}","comments_url":"https://api.github.com/repos/vmware/kubernetes/comments{/number}","issue_comment_url":"https://api.github.com/repos/vmware/kubernetes/issues/comments{/number}","contents_url":"https://api.github.com/repos/vmware/kubernetes/contents/{+path}","compare_url":"https://api.github.com/repos/vmware/kubernetes/compare/{base}...{head}","merges_url":"https://api.github.com/repos/vmware/kubernetes/merges","archive_url":"https://api.github.com/repos/vmware/kubernetes/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/vmware/kubernetes/downloads","issues_url":"https://api.github.com/repos/vmware/kubernetes/issues{/number}","pulls_url":"https://api.github.com/repos/vmware/kubernetes/pulls{/number}","milestones_url":"https://api.github.com/repos/vmware/kubernetes/milestones{/number}","notifications_url":"https://api.github.com/repos/vmware/kubernetes/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/vmware/kubernetes/labels{/name}","releases_url":"https://api.github.com/repos/vmware/kubernetes/releases{/id}","deployments_url":"https://api.github.com/repos/vmware/kubernetes/deployments","created_at":"2016-01-12T23:58:29Z","updated_at":"2016-04-01T20:42:32Z","pushed_at":"2016-04-14T16:20:52Z","git_url":"git://github.com/vmware/kubernetes.git","ssh_url":"git@github.com:vmware/kubernetes.git","clone_url":"https://github.com/vmware/kubernetes.git","svn_url":"https://github.com/vmware/kubernetes","homepage":"http://kubernetes.io","size":188212,"stargazers_count":1,"watchers_count":1,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":1,"forks":4,"open_issues":1,"watchers":1,"default_branch":"master"}},"base":{"label":"kubernetes:master","ref":"master","sha":"d800dca7f8d4a484a12f1d94b16e902e95f86b23","user":{"login":"kubernetes","id":13629408,"avatar_url":"https://avatars.githubusercontent.com/u/13629408?v=3","gravatar_id":"","url":"https://api.github.com/users/kubernetes","html_url":"https://github.com/kubernetes","followers_url":"https://api.github.com/users/kubernetes/followers","following_url":"https://api.github.com/users/kubernetes/following{/other_user}","gists_url":"https://api.github.com/users/kubernetes/gists{/gist_id}","starred_url":"https://api.github.com/users/kubernetes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kubernetes/subscriptions","organizations_url":"https://api.github.com/users/kubernetes/orgs","repos_url":"https://api.github.com/users/kubernetes/repos","events_url":"https://api.github.com/users/kubernetes/events{/privacy}","received_events_url":"https://api.github.com/users/kubernetes/received_events","type":"Organization","site_admin":false},"repo":{"id":20580498,"name":"kubernetes","full_name":"kubernetes/kubernetes","owner":{"login":"kubernetes","id":13629408,"avatar_url":"https://avatars.githubusercontent.com/u/13629408?v=3","gravatar_id":"","url":"https://api.github.com/users/kubernetes","html_url":"https://github.com/kubernetes","followers_url":"https://api.github.com/users/kubernetes/followers","following_url":"https://api.github.com/users/kubernetes/following{/other_user}","gists_url":"https://api.github.com/users/kubernetes/gists{/gist_id}","starred_url":"https://api.github.com/users/kubernetes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kubernetes/subscriptions","organizations_url":"https://api.github.com/users/kubernetes/orgs","repos_url":"https://api.github.com/users/kubernetes/repos","events_url":"https://api.github.com/users/kubernetes/events{/privacy}","received_events_url":"https://api.github.com/users/kubernetes/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/kubernetes/kubernetes","description":"Container Cluster Manager from Google","fork":false,"url":"https://api.github.com/repos/kubernetes/kubernetes","forks_url":"https://api.github.com/repos/kubernetes/kubernetes/forks","keys_url":"https://api.github.com/repos/kubernetes/kubernetes/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kubernetes/kubernetes/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kubernetes/kubernetes/teams","hooks_url":"https://api.github.com/repos/kubernetes/kubernetes/hooks","issue_events_url":"https://api.github.com/repos/kubernetes/kubernetes/issues/events{/number}","events_url":"https://api.github.com/repos/kubernetes/kubernetes/events","assignees_url":"https://api.github.com/repos/kubernetes/kubernetes/assignees{/user}","branches_url":"https://api.github.com/repos/kubernetes/kubernetes/branches{/branch}","tags_url":"https://api.github.com/repos/kubernetes/kubernetes/tags","blobs_url":"https://api.github.com/repos/kubernetes/kubernetes/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kubernetes/kubernetes/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kubernetes/kubernetes/git/refs{/sha}","trees_url":"https://api.github.com/repos/kubernetes/kubernetes/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kubernetes/kubernetes/statuses/{sha}","languages_url":"https://api.github.com/repos/kubernetes/kubernetes/languages","stargazers_url":"https://api.github.com/repos/kubernetes/kubernetes/stargazers","contributors_url":"https://api.github.com/repos/kubernetes/kubernetes/contributors","subscribers_url":"https://api.github.com/repos/kubernetes/kubernetes/subscribers","subscription_url":"https://api.github.com/repos/kubernetes/kubernetes/subscription","commits_url":"https://api.github.com/repos/kubernetes/kubernetes/commits{/sha}","git_commits_url":"https://api.github.com/repos/kubernetes/kubernetes/git/commits{/sha}","comments_url":"https://api.github.com/repos/kubernetes/kubernetes/comments{/number}","issue_comment_url":"https://api.github.com/repos/kubernetes/kubernetes/issues/comments{/number}","contents_url":"https://api.github.com/repos/kubernetes/kubernetes/contents/{+path}","compare_url":"https://api.github.com/repos/kubernetes/kubernetes/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kubernetes/kubernetes/merges","archive_url":"https://api.github.com/repos/kubernetes/kubernetes/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kubernetes/kubernetes/downloads","issues_url":"https://api.github.com/repos/kubernetes/kubernetes/issues{/number}","pulls_url":"https://api.github.com/repos/kubernetes/kubernetes/pulls{/number}","milestones_url":"https://api.github.com/repos/kubernetes/kubernetes/milestones{/number}","notifications_url":"https://api.github.com/repos/kubernetes/kubernetes/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kubernetes/kubernetes/labels{/name}","releases_url":"https://api.github.com/repos/kubernetes/kubernetes/releases{/id}","deployments_url":"https://api.github.com/repos/kubernetes/kubernetes/deployments","created_at":"2014-06-06T22:56:04Z","updated_at":"2016-04-15T20:52:52Z","pushed_at":"2016-04-15T21:20:00Z","git_url":"git://github.com/kubernetes/kubernetes.git","ssh_url":"git@github.com:kubernetes/kubernetes.git","clone_url":"https://github.com/kubernetes/kubernetes.git","svn_url":"https://github.com/kubernetes/kubernetes","homepage":"http://kubernetes.io","size":375917,"stargazers_count":13880,"watchers_count":13880,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":4109,"mirror_url":null,"open_issues_count":3427,"forks":4109,"open_issues":3427,"watchers":13880,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/24124"},"html":{"href":"https://github.com/kubernetes/kubernetes/pull/24124"},"issue":{"href":"https://api.github.com/repos/kubernetes/kubernetes/issues/24124"},"comments":{"href":"https://api.github.com/repos/kubernetes/kubernetes/issues/24124/comments"},"review_comments":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/24124/comments"},"review_comment":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/kubernetes/kubernetes/pulls/24124/commits"},"statuses":{"href":"https://api.github.com/repos/kubernetes/kubernetes/statuses/5699d51efadb41b6c010f45672c4844d84de0b84"}}}}