204 lines
6.8 KiB
Go
204 lines
6.8 KiB
Go
/*
|
|
Copyright 2017 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package apiclient
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
|
|
"k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
clientset "k8s.io/client-go/kubernetes"
|
|
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
|
)
|
|
|
|
// Waiter is an interface for waiting for criterias in Kubernetes to happen
|
|
type Waiter interface {
|
|
// WaitForAPI waits for the API Server's /healthz endpoint to become "ok"
|
|
WaitForAPI() error
|
|
// WaitForPodsWithLabel waits for Pods in the kube-system namespace to become Ready
|
|
WaitForPodsWithLabel(kvLabel string) error
|
|
// WaitForPodToDisappear waits for the given Pod in the kube-system namespace to be deleted
|
|
WaitForPodToDisappear(staticPodName string) error
|
|
// WaitForStaticPodControlPlaneHashes
|
|
WaitForStaticPodControlPlaneHashes(nodeName string) (map[string]string, error)
|
|
// WaitForStaticPodControlPlaneHashChange
|
|
WaitForStaticPodControlPlaneHashChange(nodeName, component, previousHash string) error
|
|
// SetTimeout adjusts the timeout to the specified duration
|
|
SetTimeout(timeout time.Duration)
|
|
}
|
|
|
|
// KubeWaiter is an implementation of Waiter that is backed by a Kubernetes client
|
|
type KubeWaiter struct {
|
|
client clientset.Interface
|
|
timeout time.Duration
|
|
writer io.Writer
|
|
}
|
|
|
|
// NewKubeWaiter returns a new Waiter object that talks to the given Kubernetes cluster
|
|
func NewKubeWaiter(client clientset.Interface, timeout time.Duration, writer io.Writer) Waiter {
|
|
return &KubeWaiter{
|
|
client: client,
|
|
timeout: timeout,
|
|
writer: writer,
|
|
}
|
|
}
|
|
|
|
// WaitForAPI waits for the API Server's /healthz endpoint to report "ok"
|
|
func (w *KubeWaiter) WaitForAPI() error {
|
|
start := time.Now()
|
|
return wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) {
|
|
healthStatus := 0
|
|
w.client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
|
|
if healthStatus != http.StatusOK {
|
|
return false, nil
|
|
}
|
|
|
|
fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds())
|
|
return true, nil
|
|
})
|
|
}
|
|
|
|
// WaitForPodsWithLabel will lookup pods with the given label and wait until they are all
|
|
// reporting status as running.
|
|
func (w *KubeWaiter) WaitForPodsWithLabel(kvLabel string) error {
|
|
|
|
lastKnownPodNumber := -1
|
|
return wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) {
|
|
listOpts := metav1.ListOptions{LabelSelector: kvLabel}
|
|
pods, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts)
|
|
if err != nil {
|
|
fmt.Fprintf(w.writer, "[apiclient] Error getting Pods with label selector %q [%v]\n", kvLabel, err)
|
|
return false, nil
|
|
}
|
|
|
|
if lastKnownPodNumber != len(pods.Items) {
|
|
fmt.Fprintf(w.writer, "[apiclient] Found %d Pods for label selector %s\n", len(pods.Items), kvLabel)
|
|
lastKnownPodNumber = len(pods.Items)
|
|
}
|
|
|
|
if len(pods.Items) == 0 {
|
|
return false, nil
|
|
}
|
|
|
|
for _, pod := range pods.Items {
|
|
if pod.Status.Phase != v1.PodRunning {
|
|
return false, nil
|
|
}
|
|
}
|
|
|
|
return true, nil
|
|
})
|
|
}
|
|
|
|
// WaitForPodToDisappear blocks until it timeouts or gets a "NotFound" response from the API Server when getting the Static Pod in question
|
|
func (w *KubeWaiter) WaitForPodToDisappear(podName string) error {
|
|
return wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) {
|
|
_, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).Get(podName, metav1.GetOptions{})
|
|
if apierrors.IsNotFound(err) {
|
|
fmt.Printf("[apiclient] The old Pod %q is now removed (which is desired)\n", podName)
|
|
return true, nil
|
|
}
|
|
return false, nil
|
|
})
|
|
}
|
|
|
|
// SetTimeout adjusts the timeout to the specified duration
|
|
func (w *KubeWaiter) SetTimeout(timeout time.Duration) {
|
|
w.timeout = timeout
|
|
}
|
|
|
|
// WaitForStaticPodControlPlaneHashes blocks until it timeouts or gets a hash map for all components and their Static Pods
|
|
func (w *KubeWaiter) WaitForStaticPodControlPlaneHashes(nodeName string) (map[string]string, error) {
|
|
|
|
var mirrorPodHashes map[string]string
|
|
err := wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) {
|
|
|
|
hashes, err := getStaticPodControlPlaneHashes(w.client, nodeName)
|
|
if err != nil {
|
|
return false, nil
|
|
}
|
|
mirrorPodHashes = hashes
|
|
return true, nil
|
|
})
|
|
return mirrorPodHashes, err
|
|
}
|
|
|
|
// WaitForStaticPodControlPlaneHashChange blocks until it timeouts or notices that the Mirror Pod (for the Static Pod, respectively) has changed
|
|
// This implicitely means this function blocks until the kubelet has restarted the Static Pod in question
|
|
func (w *KubeWaiter) WaitForStaticPodControlPlaneHashChange(nodeName, component, previousHash string) error {
|
|
return wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) {
|
|
|
|
hashes, err := getStaticPodControlPlaneHashes(w.client, nodeName)
|
|
if err != nil {
|
|
return false, nil
|
|
}
|
|
// We should continue polling until the UID changes
|
|
if hashes[component] == previousHash {
|
|
return false, nil
|
|
}
|
|
|
|
return true, nil
|
|
})
|
|
}
|
|
|
|
// getStaticPodControlPlaneHashes computes hashes for all the control plane's Static Pod resources
|
|
func getStaticPodControlPlaneHashes(client clientset.Interface, nodeName string) (map[string]string, error) {
|
|
|
|
mirrorPodHashes := map[string]string{}
|
|
for _, component := range constants.MasterComponents {
|
|
staticPodName := fmt.Sprintf("%s-%s", component, nodeName)
|
|
staticPod, err := client.CoreV1().Pods(metav1.NamespaceSystem).Get(staticPodName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
podBytes, err := json.Marshal(staticPod)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
mirrorPodHashes[component] = fmt.Sprintf("%x", sha256.Sum256(podBytes))
|
|
}
|
|
return mirrorPodHashes, nil
|
|
}
|
|
|
|
// TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned
|
|
func TryRunCommand(f func() error, failureThreshold uint8) error {
|
|
var numFailures uint8
|
|
return wait.PollImmediate(5*time.Second, 20*time.Minute, func() (bool, error) {
|
|
err := f()
|
|
if err != nil {
|
|
numFailures++
|
|
// If we've reached the maximum amount of failures, error out
|
|
if numFailures == failureThreshold {
|
|
return false, err
|
|
}
|
|
// Retry
|
|
return false, nil
|
|
}
|
|
// The last f() call was a success!
|
|
return true, nil
|
|
})
|
|
}
|