3da15535b6
When etcd is down today we don't specifically handle the error involved, which means clients get a generic 500 error. This commit adds a formal error type internally for both WatchExpired and EtcdUnreachable, and then converts them to api/errors before returning to the client. It also upgrades the client to retry on any 429 or 5xx error that has a Retry-After header, instead of just 429. In combination, this allows the apiserver to exert backpressure on controllers that are hotlooping. Picked 2 seconds by default, but we could potentially ramp that up even further in a future iteration.
88 lines
2.9 KiB
Go
88 lines
2.9 KiB
Go
/*
|
|
Copyright 2014 The Kubernetes Authors All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package etcd
|
|
|
|
import (
|
|
"k8s.io/kubernetes/pkg/api/errors"
|
|
etcdstorage "k8s.io/kubernetes/pkg/storage/etcd"
|
|
)
|
|
|
|
// InterpretListError converts a generic etcd error on a retrieval
|
|
// operation into the appropriate API error.
|
|
func InterpretListError(err error, kind string) error {
|
|
switch {
|
|
case etcdstorage.IsEtcdNotFound(err):
|
|
return errors.NewNotFound(kind, "")
|
|
case etcdstorage.IsEtcdUnreachable(err):
|
|
return errors.NewServerTimeout(kind, "list", 2) // TODO: make configurable or handled at a higher level
|
|
default:
|
|
return err
|
|
}
|
|
}
|
|
|
|
// InterpretGetError converts a generic etcd error on a retrieval
|
|
// operation into the appropriate API error.
|
|
func InterpretGetError(err error, kind, name string) error {
|
|
switch {
|
|
case etcdstorage.IsEtcdNotFound(err):
|
|
return errors.NewNotFound(kind, name)
|
|
case etcdstorage.IsEtcdUnreachable(err):
|
|
return errors.NewServerTimeout(kind, "get", 2) // TODO: make configurable or handled at a higher level
|
|
default:
|
|
return err
|
|
}
|
|
}
|
|
|
|
// InterpretCreateError converts a generic etcd error on a create
|
|
// operation into the appropriate API error.
|
|
func InterpretCreateError(err error, kind, name string) error {
|
|
switch {
|
|
case etcdstorage.IsEtcdNodeExist(err):
|
|
return errors.NewAlreadyExists(kind, name)
|
|
case etcdstorage.IsEtcdUnreachable(err):
|
|
return errors.NewServerTimeout(kind, "create", 2) // TODO: make configurable or handled at a higher level
|
|
default:
|
|
return err
|
|
}
|
|
}
|
|
|
|
// InterpretUpdateError converts a generic etcd error on a update
|
|
// operation into the appropriate API error.
|
|
func InterpretUpdateError(err error, kind, name string) error {
|
|
switch {
|
|
case etcdstorage.IsEtcdTestFailed(err), etcdstorage.IsEtcdNodeExist(err):
|
|
return errors.NewConflict(kind, name, err)
|
|
case etcdstorage.IsEtcdUnreachable(err):
|
|
return errors.NewServerTimeout(kind, "update", 2) // TODO: make configurable or handled at a higher level
|
|
default:
|
|
return err
|
|
}
|
|
}
|
|
|
|
// InterpretDeleteError converts a generic etcd error on a delete
|
|
// operation into the appropriate API error.
|
|
func InterpretDeleteError(err error, kind, name string) error {
|
|
switch {
|
|
case etcdstorage.IsEtcdNotFound(err):
|
|
return errors.NewNotFound(kind, name)
|
|
case etcdstorage.IsEtcdUnreachable(err):
|
|
return errors.NewServerTimeout(kind, "delete", 2) // TODO: make configurable or handled at a higher level
|
|
default:
|
|
return err
|
|
}
|
|
}
|