github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update DeploymentReaper.Stop to use ObservedGeneration to remove race condition

Browse Source
This commit is contained in:
nikhiljindal
2016-02-23 20:27:24 -08:00
parent 9c1d8bf99d
commit 7e50fa6df0
5 changed files with 39 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
pkg/kubectl/stop.go
View File

@@ -30,6 +30,7 @@ import (
client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/util"
deploymentutil "k8s.io/kubernetes/pkg/util/deployment"
utilerrors "k8s.io/kubernetes/pkg/util/errors"
"k8s.io/kubernetes/pkg/util/wait"
)
@@ -368,52 +369,30 @@ func (reaper *DeploymentReaper) Stop(namespace, name string, timeout time.Durati
zero := 0
d.Spec.RevisionHistoryLimit = &zero
d.Spec.Replicas = 0
// TODO: un-pausing should not be necessary, remove when this is fixed:
// https://github.com/kubernetes/kubernetes/issues/20966
// Instead deployment should be Paused at this point and not at next TODO.
d.Spec.Paused = false
})
if err != nil {
return err
}
// wait for total no of pods drop to 0
if err := wait.Poll(reaper.pollInterval, reaper.timeout, func() (bool, error) {
curr, err := deployments.Get(name)
// if deployment was not found it must have been deleted, error out
if err != nil && errors.IsNotFound(err) {
return false, err
}
// if other errors happen, retry
if err != nil {
return false, nil
}
// check if deployment wasn't recreated with the same name
// TODO use generations when deployment will have them
if curr.UID != deployment.UID {
return false, errors.NewNotFound(extensions.Resource("Deployment"), name)
}
return curr.Status.Replicas == 0, nil
}); err != nil {
return err
}
// TODO: When deployments will allow running cleanup policy while being
// paused, move pausing to above update operation. Without it, we need to
// pause deployment before stopping RSs, to prevent creating new RSs.
// See https://github.com/kubernetes/kubernetes/issues/20966
deployment, err = reaper.updateDeploymentWithRetries(namespace, name, func(d *extensions.Deployment) {
d.Spec.Paused = true
})
if err != nil {
return err
}
// remove remaining RSs
// Use observedGeneration to determine if the deployment controller noticed the pause.
if err := deploymentutil.WaitForObservedDeployment(func() (*extensions.Deployment, error) {
return deployments.Get(name)
}, deployment.Generation, 10*time.Millisecond, 1*time.Minute); err != nil {
return err
}
// Delete deployment.
if err := deployments.Delete(name, gracePeriod); err != nil {
return err
}
// Stop all replica sets.
selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
if err != nil {
return err
}
options := api.ListOptions{LabelSelector: selector}
rsList, err := replicaSets.List(options)
if err != nil {
@@ -430,9 +409,7 @@ func (reaper *DeploymentReaper) Stop(namespace, name string, timeout time.Durati
if len(errList) > 0 {
return utilerrors.NewAggregate(errList)
}
// and finally deployment
return deployments.Delete(name, gracePeriod)
return nil
}
type updateDeploymentFunc func(d *extensions.Deployment)