github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rate limiting everywhere

Browse Source 
not waiting to rate limit until we get an error response from the API, doing so on initial request for all API requests
This commit is contained in:
Jack Francis
2017-06-06 22:09:57 -07:00
parent 148e923f65
commit 6d73a09dcc
4 changed files with 14 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/cloudprovider/providers/azure/azure_instances.go
View File

@@ -60,6 +60,7 @@ func (az *Cloud) InstanceID(name types.NodeName) (string, error) {
var machine compute.VirtualMachine var machine compute.VirtualMachine
var exists bool var exists bool
var err error var err error
az.operationPollRateLimiter.Accept()
machine, exists, err = az.getVirtualMachine(name) machine, exists, err = az.getVirtualMachine(name)
if err != nil { if err != nil {
if az.CloudProviderBackoff { if az.CloudProviderBackoff {

8
pkg/cloudprovider/providers/azure/azure_loadbalancer.go
View File

@@ -151,6 +151,7 @@ func (az *Cloud) EnsureLoadBalancer(clusterName string, service *v1.Service, nod
// to nil. This is a workaround until https://github.com/Azure/go-autorest/issues/112 is fixed // to nil. This is a workaround until https://github.com/Azure/go-autorest/issues/112 is fixed
sg.SecurityGroupPropertiesFormat.NetworkInterfaces = nil sg.SecurityGroupPropertiesFormat.NetworkInterfaces = nil
sg.SecurityGroupPropertiesFormat.Subnets = nil sg.SecurityGroupPropertiesFormat.Subnets = nil
az.operationPollRateLimiter.Accept()
resp, err := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *sg.Name, sg, nil) resp, err := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *sg.Name, sg, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("ensure(%s) backing off: sg(%s) - updating", serviceName, *sg.Name) glog.V(2).Infof("ensure(%s) backing off: sg(%s) - updating", serviceName, *sg.Name)
@@ -229,6 +230,7 @@ func (az *Cloud) EnsureLoadBalancer(clusterName string, service *v1.Service, nod
} }
if !existsLb || lbNeedsUpdate { if !existsLb || lbNeedsUpdate {
glog.V(3).Infof("ensure(%s): lb(%s) - updating", serviceName, lbName) glog.V(3).Infof("ensure(%s): lb(%s) - updating", serviceName, lbName)
az.operationPollRateLimiter.Accept()
resp, err := az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil) resp, err := az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("ensure(%s) backing off: lb(%s) - updating", serviceName, lbName) glog.V(2).Infof("ensure(%s) backing off: lb(%s) - updating", serviceName, lbName)
@@ -328,6 +330,7 @@ func (az *Cloud) EnsureLoadBalancerDeleted(clusterName string, service *v1.Servi
// to nil. This is a workaround until https://github.com/Azure/go-autorest/issues/112 is fixed // to nil. This is a workaround until https://github.com/Azure/go-autorest/issues/112 is fixed
sg.SecurityGroupPropertiesFormat.NetworkInterfaces = nil sg.SecurityGroupPropertiesFormat.NetworkInterfaces = nil
sg.SecurityGroupPropertiesFormat.Subnets = nil sg.SecurityGroupPropertiesFormat.Subnets = nil
az.operationPollRateLimiter.Accept()
resp, err := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *reconciledSg.Name, reconciledSg, nil) resp, err := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *reconciledSg.Name, reconciledSg, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("delete(%s) backing off: sg(%s) - updating", serviceName, az.SecurityGroupName) glog.V(2).Infof("delete(%s) backing off: sg(%s) - updating", serviceName, az.SecurityGroupName)
@@ -365,6 +368,7 @@ func (az *Cloud) cleanupLoadBalancer(clusterName string, service *v1.Service, is
if lbNeedsUpdate { if lbNeedsUpdate {
if len(*lb.FrontendIPConfigurations) > 0 { if len(*lb.FrontendIPConfigurations) > 0 {
glog.V(3).Infof("delete(%s): lb(%s) - updating", serviceName, lbName) glog.V(3).Infof("delete(%s): lb(%s) - updating", serviceName, lbName)
az.operationPollRateLimiter.Accept()
resp, err := az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil) resp, err := az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("delete(%s) backing off: sg(%s) - updating", serviceName, az.SecurityGroupName) glog.V(2).Infof("delete(%s) backing off: sg(%s) - updating", serviceName, az.SecurityGroupName)
@@ -380,6 +384,7 @@ func (az *Cloud) cleanupLoadBalancer(clusterName string, service *v1.Service, is
} else { } else {
glog.V(3).Infof("delete(%s): lb(%s) - deleting; no remaining frontendipconfigs", serviceName, lbName) glog.V(3).Infof("delete(%s): lb(%s) - deleting; no remaining frontendipconfigs", serviceName, lbName)
az.operationPollRateLimiter.Accept()
resp, err := az.LoadBalancerClient.Delete(az.ResourceGroup, lbName, nil) resp, err := az.LoadBalancerClient.Delete(az.ResourceGroup, lbName, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("delete(%s) backing off: lb(%s) - deleting; no remaining frontendipconfigs", serviceName, lbName) glog.V(2).Infof("delete(%s) backing off: lb(%s) - deleting; no remaining frontendipconfigs", serviceName, lbName)
@@ -434,6 +439,7 @@ func (az *Cloud) ensurePublicIPExists(serviceName, pipName string) (*network.Pub
pip.Tags = &map[string]*string{"service": &serviceName} pip.Tags = &map[string]*string{"service": &serviceName}
glog.V(3).Infof("ensure(%s): pip(%s) - creating", serviceName, *pip.Name) glog.V(3).Infof("ensure(%s): pip(%s) - creating", serviceName, *pip.Name)
az.operationPollRateLimiter.Accept()
resp, err := az.PublicIPAddressesClient.CreateOrUpdate(az.ResourceGroup, *pip.Name, pip, nil) resp, err := az.PublicIPAddressesClient.CreateOrUpdate(az.ResourceGroup, *pip.Name, pip, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("ensure(%s) backing off: pip(%s) - creating", serviceName, *pip.Name) glog.V(2).Infof("ensure(%s) backing off: pip(%s) - creating", serviceName, *pip.Name)
@@ -459,6 +465,7 @@ func (az *Cloud) ensurePublicIPExists(serviceName, pipName string) (*network.Pub
func (az *Cloud) ensurePublicIPDeleted(serviceName, pipName string) error { func (az *Cloud) ensurePublicIPDeleted(serviceName, pipName string) error {
glog.V(2).Infof("ensure(%s): pip(%s) - deleting", serviceName, pipName) glog.V(2).Infof("ensure(%s): pip(%s) - deleting", serviceName, pipName)
az.operationPollRateLimiter.Accept()
resp, deleteErr := az.PublicIPAddressesClient.Delete(az.ResourceGroup, pipName, nil) resp, deleteErr := az.PublicIPAddressesClient.Delete(az.ResourceGroup, pipName, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, deleteErr) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, deleteErr) {
glog.V(2).Infof("ensure(%s) backing off: pip(%s) - deleting", serviceName, pipName) glog.V(2).Infof("ensure(%s) backing off: pip(%s) - deleting", serviceName, pipName)
@@ -910,6 +917,7 @@ func (az *Cloud) ensureHostInPool(serviceName string, nodeName types.NodeName, b
primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
glog.V(3).Infof("nicupdate(%s): nic(%s) - updating", serviceName, nicName) glog.V(3).Infof("nicupdate(%s): nic(%s) - updating", serviceName, nicName)
az.operationPollRateLimiter.Accept()
resp, err := az.InterfacesClient.CreateOrUpdate(az.ResourceGroup, *nic.Name, nic, nil) resp, err := az.InterfacesClient.CreateOrUpdate(az.ResourceGroup, *nic.Name, nic, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("nicupdate(%s) backing off: nic(%s) - updating, err=%v", serviceName, nicName, err) glog.V(2).Infof("nicupdate(%s) backing off: nic(%s) - updating, err=%v", serviceName, nicName, err)

3
pkg/cloudprovider/providers/azure/azure_routes.go
View File

@@ -77,6 +77,7 @@ func (az *Cloud) CreateRoute(clusterName string, nameHint string, kubeRoute *clo
} }
glog.V(3).Infof("create: creating routetable. routeTableName=%q", az.RouteTableName) glog.V(3).Infof("create: creating routetable. routeTableName=%q", az.RouteTableName)
az.operationPollRateLimiter.Accept()
resp, err := az.RouteTablesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, routeTable, nil) resp, err := az.RouteTablesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, routeTable, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("create backing off: creating routetable. routeTableName=%q", az.RouteTableName) glog.V(2).Infof("create backing off: creating routetable. routeTableName=%q", az.RouteTableName)
@@ -112,6 +113,7 @@ func (az *Cloud) CreateRoute(clusterName string, nameHint string, kubeRoute *clo
} }
glog.V(3).Infof("create: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) glog.V(3).Infof("create: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
az.operationPollRateLimiter.Accept()
resp, err := az.RoutesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, *route.Name, route, nil) resp, err := az.RoutesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, *route.Name, route, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("create backing off: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) glog.V(2).Infof("create backing off: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
@@ -135,6 +137,7 @@ func (az *Cloud) DeleteRoute(clusterName string, kubeRoute *cloudprovider.Route)
glog.V(2).Infof("delete: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) glog.V(2).Infof("delete: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
routeName := mapNodeNameToRouteName(kubeRoute.TargetNode) routeName := mapNodeNameToRouteName(kubeRoute.TargetNode)
az.operationPollRateLimiter.Accept()
resp, err := az.RoutesClient.Delete(az.ResourceGroup, az.RouteTableName, routeName, nil) resp, err := az.RoutesClient.Delete(az.ResourceGroup, az.RouteTableName, routeName, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("delete backing off: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) glog.V(2).Infof("delete backing off: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)

2
pkg/cloudprovider/providers/azure/azure_storage.go
View File

@@ -65,6 +65,7 @@ func (az *Cloud) AttachDisk(diskName, diskURI string, nodeName types.NodeName, l
} }
vmName := mapNodeNameToVMName(nodeName) vmName := mapNodeNameToVMName(nodeName)
glog.V(2).Infof("create(%s): vm(%s)", az.ResourceGroup, vmName) glog.V(2).Infof("create(%s): vm(%s)", az.ResourceGroup, vmName)
az.operationPollRateLimiter.Accept()
resp, err := az.VirtualMachinesClient.CreateOrUpdate(az.ResourceGroup, vmName, newVM, nil) resp, err := az.VirtualMachinesClient.CreateOrUpdate(az.ResourceGroup, vmName, newVM, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("create(%s) backing off: vm(%s)", az.ResourceGroup, vmName) glog.V(2).Infof("create(%s) backing off: vm(%s)", az.ResourceGroup, vmName)
@@ -145,6 +146,7 @@ func (az *Cloud) DetachDiskByName(diskName, diskURI string, nodeName types.NodeN
} }
vmName := mapNodeNameToVMName(nodeName) vmName := mapNodeNameToVMName(nodeName)
glog.V(2).Infof("create(%s): vm(%s)", az.ResourceGroup, vmName) glog.V(2).Infof("create(%s): vm(%s)", az.ResourceGroup, vmName)
az.operationPollRateLimiter.Accept()
resp, err := az.VirtualMachinesClient.CreateOrUpdate(az.ResourceGroup, vmName, newVM, nil) resp, err := az.VirtualMachinesClient.CreateOrUpdate(az.ResourceGroup, vmName, newVM, nil)
if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
glog.V(2).Infof("create(%s) backing off: vm(%s)", az.ResourceGroup, vmName) glog.V(2).Infof("create(%s) backing off: vm(%s)", az.ResourceGroup, vmName)