github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #115838 from torredil/remove-aws

Browse Source 
Remove AWS legacy cloud provider + EBS in-tree storage plugin
This commit is contained in:
Kubernetes Prow Robot
2023-03-06 08:18:29 -08:00
committed by GitHub
parent 778b24c97e 6aebda9b1e
commit d6e9cff212
100 changed files with 31 additions and 82645 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
staging/publishing/rules.yaml
View File

@@ -1673,8 +1673,6 @@ rules:
branch: master
- repository: cloud-provider
branch: master
- repository: csi-translation-lib
branch: master
- repository: apiserver
branch: master
- repository: component-base

1
staging/src/k8s.io/cloud-provider/plugins.go
View File

@@ -40,7 +40,6 @@ var (
external bool
detail string
}{
{"aws", false, "The AWS provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-aws"},
{"azure", false, "The Azure provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes-sigs/cloud-provider-azure"},
{"gce", false, "The GCE provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-gcp"},
{"vsphere", false, "The vSphere provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-vsphere"},

18
staging/src/k8s.io/legacy-cloud-providers/aws/OWNERS
View File

@@ -1,18 +0,0 @@
# See the OWNERS docs at https://go.k8s.io/owners
# We are no longer accepting features into k8s.io/legacy-cloud-providers.
# Any kind/feature PRs must be approved by SIG Cloud Provider going forward.
emeritus_approvers:
- justinsb
- gnufied
- jsafrane
- micahhausler
- m00nf1sh
- zmerlynn
- mcrute
reviewers:
- gnufied
- jsafrane
- justinsb
- nckturner
- m00nf1sh

5099
staging/src/k8s.io/legacy-cloud-providers/aws/aws.go
View File

File diff suppressed because it is too large Load Diff

62
staging/src/k8s.io/legacy-cloud-providers/aws/aws_assumerole_provider.go
View File

@@ -1,62 +0,0 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"sync"
"time"
"github.com/aws/aws-sdk-go/aws/credentials"
)
const (
invalidateCredsCacheAfter = 1 * time.Second
)
// assumeRoleProviderWithRateLimiting makes sure we call the underlying provider only
// once after `invalidateCredsCacheAfter` period
type assumeRoleProviderWithRateLimiting struct {
provider credentials.Provider
invalidateCredsCacheAfter time.Duration
sync.RWMutex
lastError error
lastValue credentials.Value
lastRetrieveTime time.Time
}
func assumeRoleProvider(provider credentials.Provider) credentials.Provider {
return &assumeRoleProviderWithRateLimiting{provider: provider,
invalidateCredsCacheAfter: invalidateCredsCacheAfter}
}
func (l *assumeRoleProviderWithRateLimiting) Retrieve() (credentials.Value, error) {
l.Lock()
defer l.Unlock()
if time.Since(l.lastRetrieveTime) < l.invalidateCredsCacheAfter {
if l.lastError != nil {
return credentials.Value{}, l.lastError
}
return l.lastValue, nil
}
l.lastValue, l.lastError = l.provider.Retrieve()
l.lastRetrieveTime = time.Now()
return l.lastValue, l.lastError
}
func (l *assumeRoleProviderWithRateLimiting) IsExpired() bool {
return l.provider.IsExpired()
}

133
staging/src/k8s.io/legacy-cloud-providers/aws/aws_assumerole_provider_test.go
View File

@@ -1,133 +0,0 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"reflect"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws/credentials"
)
func Test_assumeRoleProviderWithRateLimiting_Retrieve(t *testing.T) {
type fields struct {
provider credentials.Provider
invalidateCredsCacheAfter time.Duration
RWMutex sync.RWMutex
lastError error
lastValue credentials.Value
lastRetrieveTime time.Time
}
tests := []struct {
name string
fields fields
want credentials.Value
wantProviderCalled bool
sleepBeforeCallingProvider time.Duration
wantErr bool
wantErrString string
}{{
name: "Call assume role provider and verify access ID returned",
fields: fields{provider: &fakeAssumeRoleProvider{accesskeyID: "fakeID"}},
want: credentials.Value{AccessKeyID: "fakeID"},
wantProviderCalled: true,
}, {
name: "Immediate call to assume role API, shouldn't call the underlying provider and return the last value",
fields: fields{
provider: &fakeAssumeRoleProvider{accesskeyID: "fakeID"},
invalidateCredsCacheAfter: 100 * time.Millisecond,
lastValue: credentials.Value{AccessKeyID: "fakeID1"},
lastRetrieveTime: time.Now(),
},
want: credentials.Value{AccessKeyID: "fakeID1"},
wantProviderCalled: false,
sleepBeforeCallingProvider: 10 * time.Millisecond,
}, {
name: "Assume role provider returns an error when trying to assume a role",
fields: fields{
provider: &fakeAssumeRoleProvider{err: fmt.Errorf("can't assume fake role")},
invalidateCredsCacheAfter: 10 * time.Millisecond,
lastRetrieveTime: time.Now(),
},
wantProviderCalled: true,
wantErr: true,
wantErrString: "can't assume fake role",
sleepBeforeCallingProvider: 15 * time.Millisecond,
}, {
name: "Immediate call to assume role API, shouldn't call the underlying provider and return the last error value",
fields: fields{
provider: &fakeAssumeRoleProvider{},
invalidateCredsCacheAfter: 100 * time.Millisecond,
lastRetrieveTime: time.Now(),
},
want: credentials.Value{},
wantProviderCalled: false,
wantErr: true,
wantErrString: "can't assume fake role",
}, {
name: "Delayed call to assume role API, should call the underlying provider",
fields: fields{
provider: &fakeAssumeRoleProvider{accesskeyID: "fakeID2"},
invalidateCredsCacheAfter: 20 * time.Millisecond,
lastRetrieveTime: time.Now(),
},
want: credentials.Value{AccessKeyID: "fakeID2"},
wantProviderCalled: true,
sleepBeforeCallingProvider: 25 * time.Millisecond,
}}
//nolint:govet // ignore copying of sync.RWMutex, it is empty
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
l := &assumeRoleProviderWithRateLimiting{
provider: tt.fields.provider,
invalidateCredsCacheAfter: tt.fields.invalidateCredsCacheAfter,
lastError: tt.fields.lastError,
lastValue: tt.fields.lastValue,
lastRetrieveTime: tt.fields.lastRetrieveTime,
}
time.Sleep(tt.sleepBeforeCallingProvider)
got, err := l.Retrieve()
if (err != nil) != tt.wantErr && (tt.wantErr && reflect.DeepEqual(err, tt.wantErrString)) {
t.Errorf("assumeRoleProviderWithRateLimiting.Retrieve() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("assumeRoleProviderWithRateLimiting.Retrieve() got = %v, want %v", got, tt.want)
return
}
if tt.wantProviderCalled != tt.fields.provider.(*fakeAssumeRoleProvider).providerCalled {
t.Errorf("provider called %v, want %v", tt.fields.provider.(*fakeAssumeRoleProvider).providerCalled, tt.wantProviderCalled)
}
})
}
}
type fakeAssumeRoleProvider struct {
accesskeyID string
err error
providerCalled bool
}
func (f *fakeAssumeRoleProvider) Retrieve() (credentials.Value, error) {
f.providerCalled = true
return credentials.Value{AccessKeyID: f.accesskeyID}, f.err
}
func (f *fakeAssumeRoleProvider) IsExpired() bool { return true }

688
staging/src/k8s.io/legacy-cloud-providers/aws/aws_fakes.go
View File

@@ -1,688 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"sort"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/elb"
"github.com/aws/aws-sdk-go/service/elbv2"
"github.com/aws/aws-sdk-go/service/kms"
_ "github.com/stretchr/testify/mock"
"k8s.io/klog/v2"
)
// FakeAWSServices is an fake AWS session used for testing
type FakeAWSServices struct {
region string
instances []*ec2.Instance
selfInstance *ec2.Instance
networkInterfacesMacs []string
networkInterfacesPrivateIPs [][]string
networkInterfacesVpcIDs []string
ec2 FakeEC2
elb ELB
elbv2 ELBV2
asg *FakeASG
metadata *FakeMetadata
kms *FakeKMS
}
// NewFakeAWSServices creates a new FakeAWSServices
func NewFakeAWSServices(clusterID string) *FakeAWSServices {
s := &FakeAWSServices{}
s.region = "us-east-1"
s.ec2 = &FakeEC2Impl{aws: s}
s.elb = &FakeELB{aws: s}
s.elbv2 = &FakeELBV2{aws: s}
s.asg = &FakeASG{aws: s}
s.metadata = &FakeMetadata{aws: s}
s.kms = &FakeKMS{aws: s}
s.networkInterfacesMacs = []string{"aa:bb:cc:dd:ee:00", "aa:bb:cc:dd:ee:01"}
s.networkInterfacesVpcIDs = []string{"vpc-mac0", "vpc-mac1"}
selfInstance := &ec2.Instance{}
selfInstance.InstanceId = aws.String("i-self")
selfInstance.Placement = &ec2.Placement{
AvailabilityZone: aws.String("us-east-1a"),
}
selfInstance.PrivateDnsName = aws.String("ip-172-20-0-100.ec2.internal")
selfInstance.PrivateIpAddress = aws.String("192.168.0.1")
selfInstance.PublicIpAddress = aws.String("1.2.3.4")
s.selfInstance = selfInstance
s.instances = []*ec2.Instance{selfInstance}
var tag ec2.Tag
tag.Key = aws.String(TagNameKubernetesClusterLegacy)
tag.Value = aws.String(clusterID)
selfInstance.Tags = []*ec2.Tag{&tag}
return s
}
// WithAz sets the ec2 placement availability zone
func (s *FakeAWSServices) WithAz(az string) *FakeAWSServices {
if s.selfInstance.Placement == nil {
s.selfInstance.Placement = &ec2.Placement{}
}
s.selfInstance.Placement.AvailabilityZone = aws.String(az)
return s
}
// Compute returns a fake EC2 client
func (s *FakeAWSServices) Compute(region string) (EC2, error) {
return s.ec2, nil
}
// LoadBalancing returns a fake ELB client
func (s *FakeAWSServices) LoadBalancing(region string) (ELB, error) {
return s.elb, nil
}
// LoadBalancingV2 returns a fake ELBV2 client
func (s *FakeAWSServices) LoadBalancingV2(region string) (ELBV2, error) {
return s.elbv2, nil
}
// Autoscaling returns a fake ASG client
func (s *FakeAWSServices) Autoscaling(region string) (ASG, error) {
return s.asg, nil
}
// Metadata returns a fake EC2Metadata client
func (s *FakeAWSServices) Metadata() (EC2Metadata, error) {
return s.metadata, nil
}
// KeyManagement returns a fake KMS client
func (s *FakeAWSServices) KeyManagement(region string) (KMS, error) {
return s.kms, nil
}
// FakeEC2 is a fake EC2 client used for testing
type FakeEC2 interface {
EC2
CreateSubnet(*ec2.Subnet) (*ec2.CreateSubnetOutput, error)
RemoveSubnets()
CreateRouteTable(*ec2.RouteTable) (*ec2.CreateRouteTableOutput, error)
RemoveRouteTables()
}
// FakeEC2Impl is an implementation of the FakeEC2 interface used for testing
type FakeEC2Impl struct {
aws *FakeAWSServices
Subnets []*ec2.Subnet
DescribeSubnetsInput *ec2.DescribeSubnetsInput
RouteTables []*ec2.RouteTable
DescribeRouteTablesInput *ec2.DescribeRouteTablesInput
}
// DescribeInstances returns fake instance descriptions
func (ec2i *FakeEC2Impl) DescribeInstances(request *ec2.DescribeInstancesInput) ([]*ec2.Instance, error) {
matches := []*ec2.Instance{}
for _, instance := range ec2i.aws.instances {
if request.InstanceIds != nil {
if instance.InstanceId == nil {
klog.Warning("Instance with no instance id: ", instance)
continue
}
found := false
for _, instanceID := range request.InstanceIds {
if *instanceID == *instance.InstanceId {
found = true
break
}
}
if !found {
continue
}
}
if request.Filters != nil {
allMatch := true
for _, filter := range request.Filters {
if !instanceMatchesFilter(instance, filter) {
allMatch = false
break
}
}
if !allMatch {
continue
}
}
matches = append(matches, instance)
}
return matches, nil
}
// AttachVolume is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) AttachVolume(request *ec2.AttachVolumeInput) (resp *ec2.VolumeAttachment, err error) {
panic("Not implemented")
}
// DetachVolume is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) DetachVolume(request *ec2.DetachVolumeInput) (resp *ec2.VolumeAttachment, err error) {
panic("Not implemented")
}
// DescribeVolumes is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) DescribeVolumes(request *ec2.DescribeVolumesInput) ([]*ec2.Volume, error) {
panic("Not implemented")
}
// CreateVolume is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) CreateVolume(request *ec2.CreateVolumeInput) (resp *ec2.Volume, err error) {
panic("Not implemented")
}
// DeleteVolume is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) DeleteVolume(request *ec2.DeleteVolumeInput) (resp *ec2.DeleteVolumeOutput, err error) {
panic("Not implemented")
}
// DescribeSecurityGroups is not implemented but is required for interface
// conformance
func (ec2i *FakeEC2Impl) DescribeSecurityGroups(request *ec2.DescribeSecurityGroupsInput) ([]*ec2.SecurityGroup, error) {
panic("Not implemented")
}
// CreateSecurityGroup is not implemented but is required for interface
// conformance
func (ec2i *FakeEC2Impl) CreateSecurityGroup(*ec2.CreateSecurityGroupInput) (*ec2.CreateSecurityGroupOutput, error) {
panic("Not implemented")
}
// DeleteSecurityGroup is not implemented but is required for interface
// conformance
func (ec2i *FakeEC2Impl) DeleteSecurityGroup(*ec2.DeleteSecurityGroupInput) (*ec2.DeleteSecurityGroupOutput, error) {
panic("Not implemented")
}
// AuthorizeSecurityGroupIngress is not implemented but is required for
// interface conformance
func (ec2i *FakeEC2Impl) AuthorizeSecurityGroupIngress(*ec2.AuthorizeSecurityGroupIngressInput) (*ec2.AuthorizeSecurityGroupIngressOutput, error) {
panic("Not implemented")
}
// RevokeSecurityGroupIngress is not implemented but is required for interface
// conformance
func (ec2i *FakeEC2Impl) RevokeSecurityGroupIngress(*ec2.RevokeSecurityGroupIngressInput) (*ec2.RevokeSecurityGroupIngressOutput, error) {
panic("Not implemented")
}
// DescribeVolumeModifications is not implemented but is required for interface
// conformance
func (ec2i *FakeEC2Impl) DescribeVolumeModifications(*ec2.DescribeVolumesModificationsInput) ([]*ec2.VolumeModification, error) {
panic("Not implemented")
}
// ModifyVolume is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) ModifyVolume(*ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error) {
panic("Not implemented")
}
// CreateSubnet creates fake subnets
func (ec2i *FakeEC2Impl) CreateSubnet(request *ec2.Subnet) (*ec2.CreateSubnetOutput, error) {
ec2i.Subnets = append(ec2i.Subnets, request)
response := &ec2.CreateSubnetOutput{
Subnet: request,
}
return response, nil
}
// DescribeSubnets returns fake subnet descriptions
func (ec2i *FakeEC2Impl) DescribeSubnets(request *ec2.DescribeSubnetsInput) ([]*ec2.Subnet, error) {
ec2i.DescribeSubnetsInput = request
return ec2i.Subnets, nil
}
// RemoveSubnets clears subnets on client
func (ec2i *FakeEC2Impl) RemoveSubnets() {
ec2i.Subnets = ec2i.Subnets[:0]
}
// CreateTags is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) CreateTags(*ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) {
panic("Not implemented")
}
// DescribeRouteTables returns fake route table descriptions
func (ec2i *FakeEC2Impl) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) {
ec2i.DescribeRouteTablesInput = request
return ec2i.RouteTables, nil
}
// CreateRouteTable creates fake route tables
func (ec2i *FakeEC2Impl) CreateRouteTable(request *ec2.RouteTable) (*ec2.CreateRouteTableOutput, error) {
ec2i.RouteTables = append(ec2i.RouteTables, request)
response := &ec2.CreateRouteTableOutput{
RouteTable: request,
}
return response, nil
}
// RemoveRouteTables clears route tables on client
func (ec2i *FakeEC2Impl) RemoveRouteTables() {
ec2i.RouteTables = ec2i.RouteTables[:0]
}
// CreateRoute is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) CreateRoute(request *ec2.CreateRouteInput) (*ec2.CreateRouteOutput, error) {
panic("Not implemented")
}
// DeleteRoute is not implemented but is required for interface conformance
func (ec2i *FakeEC2Impl) DeleteRoute(request *ec2.DeleteRouteInput) (*ec2.DeleteRouteOutput, error) {
panic("Not implemented")
}
// ModifyInstanceAttribute is not implemented but is required for interface
// conformance
func (ec2i *FakeEC2Impl) ModifyInstanceAttribute(request *ec2.ModifyInstanceAttributeInput) (*ec2.ModifyInstanceAttributeOutput, error) {
panic("Not implemented")
}
// DescribeVpcs returns fake VPC descriptions
func (ec2i *FakeEC2Impl) DescribeVpcs(request *ec2.DescribeVpcsInput) (*ec2.DescribeVpcsOutput, error) {
return &ec2.DescribeVpcsOutput{Vpcs: []*ec2.Vpc{{CidrBlock: aws.String("172.20.0.0/16")}}}, nil
}
// FakeMetadata is a fake EC2 metadata service client used for testing
type FakeMetadata struct {
aws *FakeAWSServices
}
// GetMetadata returns fake EC2 metadata for testing
func (m *FakeMetadata) GetMetadata(key string) (string, error) {
networkInterfacesPrefix := "network/interfaces/macs/"
i := m.aws.selfInstance
if key == "placement/availability-zone" {
az := ""
if i.Placement != nil {
az = aws.StringValue(i.Placement.AvailabilityZone)
}
return az, nil
} else if key == "instance-id" {
return aws.StringValue(i.InstanceId), nil
} else if key == "local-hostname" {
return aws.StringValue(i.PrivateDnsName), nil
} else if key == "public-hostname" {
return aws.StringValue(i.PublicDnsName), nil
} else if key == "local-ipv4" {
return aws.StringValue(i.PrivateIpAddress), nil
} else if key == "public-ipv4" {
return aws.StringValue(i.PublicIpAddress), nil
} else if strings.HasPrefix(key, networkInterfacesPrefix) {
if key == networkInterfacesPrefix {
// Return the MACs sorted lexically rather than in device-number
// order; this matches AWS's observed behavior and lets us test
// that we fix up the ordering correctly in NodeAddresses().
macs := make([]string, len(m.aws.networkInterfacesMacs))
copy(macs, m.aws.networkInterfacesMacs)
sort.Strings(macs)
return strings.Join(macs, "/\n") + "/\n", nil
}
keySplit := strings.Split(key, "/")
macParam := keySplit[3]
if len(keySplit) == 5 && keySplit[4] == "vpc-id" {
for i, macElem := range m.aws.networkInterfacesMacs {
if macParam == macElem {
return m.aws.networkInterfacesVpcIDs[i], nil
}
}
}
if len(keySplit) == 5 && keySplit[4] == "device-number" {
for i, macElem := range m.aws.networkInterfacesMacs {
if macParam == macElem {
n := i
if n > 0 {
// Introduce an artificial gap, just to test eg: [eth0, eth2]
n++
}
return fmt.Sprintf("%d\n", n), nil
}
}
}
if len(keySplit) == 5 && keySplit[4] == "local-ipv4s" {
for i, macElem := range m.aws.networkInterfacesMacs {
if macParam == macElem {
return strings.Join(m.aws.networkInterfacesPrivateIPs[i], "/\n"), nil
}
}
}
return "", nil
}
return "", nil
}
// FakeELB is a fake ELB client used for testing
type FakeELB struct {
aws *FakeAWSServices
}
// CreateLoadBalancer is not implemented but is required for interface
// conformance
func (elb *FakeELB) CreateLoadBalancer(*elb.CreateLoadBalancerInput) (*elb.CreateLoadBalancerOutput, error) {
panic("Not implemented")
}
// DeleteLoadBalancer is not implemented but is required for interface
// conformance
func (elb *FakeELB) DeleteLoadBalancer(input *elb.DeleteLoadBalancerInput) (*elb.DeleteLoadBalancerOutput, error) {
panic("Not implemented")
}
// DescribeLoadBalancers is not implemented but is required for interface
// conformance
func (elb *FakeELB) DescribeLoadBalancers(input *elb.DescribeLoadBalancersInput) (*elb.DescribeLoadBalancersOutput, error) {
panic("Not implemented")
}
// AddTags is not implemented but is required for interface conformance
func (elb *FakeELB) AddTags(input *elb.AddTagsInput) (*elb.AddTagsOutput, error) {
panic("Not implemented")
}
// RegisterInstancesWithLoadBalancer is not implemented but is required for
// interface conformance
func (elb *FakeELB) RegisterInstancesWithLoadBalancer(*elb.RegisterInstancesWithLoadBalancerInput) (*elb.RegisterInstancesWithLoadBalancerOutput, error) {
panic("Not implemented")
}
// DeregisterInstancesFromLoadBalancer is not implemented but is required for
// interface conformance
func (elb *FakeELB) DeregisterInstancesFromLoadBalancer(*elb.DeregisterInstancesFromLoadBalancerInput) (*elb.DeregisterInstancesFromLoadBalancerOutput, error) {
panic("Not implemented")
}
// DetachLoadBalancerFromSubnets is not implemented but is required for
// interface conformance
func (elb *FakeELB) DetachLoadBalancerFromSubnets(*elb.DetachLoadBalancerFromSubnetsInput) (*elb.DetachLoadBalancerFromSubnetsOutput, error) {
panic("Not implemented")
}
// AttachLoadBalancerToSubnets is not implemented but is required for interface
// conformance
func (elb *FakeELB) AttachLoadBalancerToSubnets(*elb.AttachLoadBalancerToSubnetsInput) (*elb.AttachLoadBalancerToSubnetsOutput, error) {
panic("Not implemented")
}
// CreateLoadBalancerListeners is not implemented but is required for interface
// conformance
func (elb *FakeELB) CreateLoadBalancerListeners(*elb.CreateLoadBalancerListenersInput) (*elb.CreateLoadBalancerListenersOutput, error) {
panic("Not implemented")
}
// DeleteLoadBalancerListeners is not implemented but is required for interface
// conformance
func (elb *FakeELB) DeleteLoadBalancerListeners(*elb.DeleteLoadBalancerListenersInput) (*elb.DeleteLoadBalancerListenersOutput, error) {
panic("Not implemented")
}
// ApplySecurityGroupsToLoadBalancer is not implemented but is required for
// interface conformance
func (elb *FakeELB) ApplySecurityGroupsToLoadBalancer(*elb.ApplySecurityGroupsToLoadBalancerInput) (*elb.ApplySecurityGroupsToLoadBalancerOutput, error) {
panic("Not implemented")
}
// ConfigureHealthCheck is not implemented but is required for interface
// conformance
func (elb *FakeELB) ConfigureHealthCheck(*elb.ConfigureHealthCheckInput) (*elb.ConfigureHealthCheckOutput, error) {
panic("Not implemented")
}
// CreateLoadBalancerPolicy is not implemented but is required for interface
// conformance
func (elb *FakeELB) CreateLoadBalancerPolicy(*elb.CreateLoadBalancerPolicyInput) (*elb.CreateLoadBalancerPolicyOutput, error) {
panic("Not implemented")
}
// SetLoadBalancerPoliciesForBackendServer is not implemented but is required
// for interface conformance
func (elb *FakeELB) SetLoadBalancerPoliciesForBackendServer(*elb.SetLoadBalancerPoliciesForBackendServerInput) (*elb.SetLoadBalancerPoliciesForBackendServerOutput, error) {
panic("Not implemented")
}
// SetLoadBalancerPoliciesOfListener is not implemented but is required for
// interface conformance
func (elb *FakeELB) SetLoadBalancerPoliciesOfListener(input *elb.SetLoadBalancerPoliciesOfListenerInput) (*elb.SetLoadBalancerPoliciesOfListenerOutput, error) {
panic("Not implemented")
}
// DescribeLoadBalancerPolicies is not implemented but is required for
// interface conformance
func (elb *FakeELB) DescribeLoadBalancerPolicies(input *elb.DescribeLoadBalancerPoliciesInput) (*elb.DescribeLoadBalancerPoliciesOutput, error) {
panic("Not implemented")
}
// DescribeLoadBalancerAttributes is not implemented but is required for
// interface conformance
func (elb *FakeELB) DescribeLoadBalancerAttributes(*elb.DescribeLoadBalancerAttributesInput) (*elb.DescribeLoadBalancerAttributesOutput, error) {
panic("Not implemented")
}
// ModifyLoadBalancerAttributes is not implemented but is required for
// interface conformance
func (elb *FakeELB) ModifyLoadBalancerAttributes(*elb.ModifyLoadBalancerAttributesInput) (*elb.ModifyLoadBalancerAttributesOutput, error) {
panic("Not implemented")
}
// FakeELBV2 is a fake ELBV2 client used for testing
type FakeELBV2 struct {
aws *FakeAWSServices
}
// AddTags is not implemented but is required for interface conformance
func (elb *FakeELBV2) AddTags(input *elbv2.AddTagsInput) (*elbv2.AddTagsOutput, error) {
panic("Not implemented")
}
// CreateLoadBalancer is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) CreateLoadBalancer(*elbv2.CreateLoadBalancerInput) (*elbv2.CreateLoadBalancerOutput, error) {
panic("Not implemented")
}
// DescribeLoadBalancers is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) DescribeLoadBalancers(*elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) {
panic("Not implemented")
}
// DeleteLoadBalancer is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) DeleteLoadBalancer(*elbv2.DeleteLoadBalancerInput) (*elbv2.DeleteLoadBalancerOutput, error) {
panic("Not implemented")
}
// ModifyLoadBalancerAttributes is not implemented but is required for
// interface conformance
func (elb *FakeELBV2) ModifyLoadBalancerAttributes(*elbv2.ModifyLoadBalancerAttributesInput) (*elbv2.ModifyLoadBalancerAttributesOutput, error) {
panic("Not implemented")
}
// DescribeLoadBalancerAttributes is not implemented but is required for
// interface conformance
func (elb *FakeELBV2) DescribeLoadBalancerAttributes(*elbv2.DescribeLoadBalancerAttributesInput) (*elbv2.DescribeLoadBalancerAttributesOutput, error) {
panic("Not implemented")
}
// CreateTargetGroup is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) CreateTargetGroup(*elbv2.CreateTargetGroupInput) (*elbv2.CreateTargetGroupOutput, error) {
panic("Not implemented")
}
// DescribeTargetGroups is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) DescribeTargetGroups(*elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) {
panic("Not implemented")
}
// ModifyTargetGroup is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) ModifyTargetGroup(*elbv2.ModifyTargetGroupInput) (*elbv2.ModifyTargetGroupOutput, error) {
panic("Not implemented")
}
// DeleteTargetGroup is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) DeleteTargetGroup(*elbv2.DeleteTargetGroupInput) (*elbv2.DeleteTargetGroupOutput, error) {
panic("Not implemented")
}
// DescribeTargetHealth is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) DescribeTargetHealth(input *elbv2.DescribeTargetHealthInput) (*elbv2.DescribeTargetHealthOutput, error) {
panic("Not implemented")
}
// DescribeTargetGroupAttributes is not implemented but is required for
// interface conformance
func (elb *FakeELBV2) DescribeTargetGroupAttributes(*elbv2.DescribeTargetGroupAttributesInput) (*elbv2.DescribeTargetGroupAttributesOutput, error) {
panic("Not implemented")
}
// ModifyTargetGroupAttributes is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) ModifyTargetGroupAttributes(*elbv2.ModifyTargetGroupAttributesInput) (*elbv2.ModifyTargetGroupAttributesOutput, error) {
panic("Not implemented")
}
// RegisterTargets is not implemented but is required for interface conformance
func (elb *FakeELBV2) RegisterTargets(*elbv2.RegisterTargetsInput) (*elbv2.RegisterTargetsOutput, error) {
panic("Not implemented")
}
// DeregisterTargets is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) DeregisterTargets(*elbv2.DeregisterTargetsInput) (*elbv2.DeregisterTargetsOutput, error) {
panic("Not implemented")
}
// CreateListener is not implemented but is required for interface conformance
func (elb *FakeELBV2) CreateListener(*elbv2.CreateListenerInput) (*elbv2.CreateListenerOutput, error) {
panic("Not implemented")
}
// DescribeListeners is not implemented but is required for interface
// conformance
func (elb *FakeELBV2) DescribeListeners(*elbv2.DescribeListenersInput) (*elbv2.DescribeListenersOutput, error) {
panic("Not implemented")
}
// DeleteListener is not implemented but is required for interface conformance
func (elb *FakeELBV2) DeleteListener(*elbv2.DeleteListenerInput) (*elbv2.DeleteListenerOutput, error) {
panic("Not implemented")
}
// ModifyListener is not implemented but is required for interface conformance
func (elb *FakeELBV2) ModifyListener(*elbv2.ModifyListenerInput) (*elbv2.ModifyListenerOutput, error) {
panic("Not implemented")
}
// WaitUntilLoadBalancersDeleted is not implemented but is required for
// interface conformance
func (elb *FakeELBV2) WaitUntilLoadBalancersDeleted(*elbv2.DescribeLoadBalancersInput) error {
panic("Not implemented")
}
// FakeASG is a fake Autoscaling client used for testing
type FakeASG struct {
aws *FakeAWSServices
}
// UpdateAutoScalingGroup is not implemented but is required for interface
// conformance
func (a *FakeASG) UpdateAutoScalingGroup(*autoscaling.UpdateAutoScalingGroupInput) (*autoscaling.UpdateAutoScalingGroupOutput, error) {
panic("Not implemented")
}
// DescribeAutoScalingGroups is not implemented but is required for interface
// conformance
func (a *FakeASG) DescribeAutoScalingGroups(*autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) {
panic("Not implemented")
}
// FakeKMS is a fake KMS client used for testing
type FakeKMS struct {
aws *FakeAWSServices
}
// DescribeKey is not implemented but is required for interface conformance
func (kms *FakeKMS) DescribeKey(*kms.DescribeKeyInput) (*kms.DescribeKeyOutput, error) {
panic("Not implemented")
}
func instanceMatchesFilter(instance *ec2.Instance, filter *ec2.Filter) bool {
name := *filter.Name
if name == "private-dns-name" {
if instance.PrivateDnsName == nil {
return false
}
return contains(filter.Values, *instance.PrivateDnsName)
}
if name == "instance-state-name" {
return contains(filter.Values, *instance.State.Name)
}
if name == "tag-key" {
for _, instanceTag := range instance.Tags {
if contains(filter.Values, aws.StringValue(instanceTag.Key)) {
return true
}
}
return false
}
if strings.HasPrefix(name, "tag:") {
tagName := name[4:]
for _, instanceTag := range instance.Tags {
if aws.StringValue(instanceTag.Key) == tagName && contains(filter.Values, aws.StringValue(instanceTag.Value)) {
return true
}
}
return false
}
panic("Unknown filter name: " + name)
}
func contains(haystack []*string, needle string) bool {
for _, s := range haystack {
// (deliberately panic if s == nil)
if needle == *s {
return true
}
}
return false
}

92
staging/src/k8s.io/legacy-cloud-providers/aws/aws_instancegroups.go
View File

@@ -1,92 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/autoscaling"
"k8s.io/klog/v2"
)
// AWSCloud implements InstanceGroups
var _ InstanceGroups = &Cloud{}
// ResizeInstanceGroup sets the size of the specificed instancegroup Exported
// so it can be used by the e2e tests, which don't want to instantiate a full
// cloudprovider.
func ResizeInstanceGroup(asg ASG, instanceGroupName string, size int) error {
request := &autoscaling.UpdateAutoScalingGroupInput{
AutoScalingGroupName: aws.String(instanceGroupName),
DesiredCapacity: aws.Int64(int64(size)),
}
if _, err := asg.UpdateAutoScalingGroup(request); err != nil {
return fmt.Errorf("error resizing AWS autoscaling group: %q", err)
}
return nil
}
// ResizeInstanceGroup implements InstanceGroups.ResizeInstanceGroup
// Set the size to the fixed size
func (c *Cloud) ResizeInstanceGroup(instanceGroupName string, size int) error {
return ResizeInstanceGroup(c.asg, instanceGroupName, size)
}
// DescribeInstanceGroup gets info about the specified instancegroup
// Exported so it can be used by the e2e tests,
// which don't want to instantiate a full cloudprovider.
func DescribeInstanceGroup(asg ASG, instanceGroupName string) (InstanceGroupInfo, error) {
request := &autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{aws.String(instanceGroupName)},
}
response, err := asg.DescribeAutoScalingGroups(request)
if err != nil {
return nil, fmt.Errorf("error listing AWS autoscaling group (%s): %q", instanceGroupName, err)
}
if len(response.AutoScalingGroups) == 0 {
return nil, nil
}
if len(response.AutoScalingGroups) > 1 {
klog.Warning("AWS returned multiple autoscaling groups with name ", instanceGroupName)
}
group := response.AutoScalingGroups[0]
return &awsInstanceGroup{group: group}, nil
}
// DescribeInstanceGroup implements InstanceGroups.DescribeInstanceGroup
// Queries the cloud provider for information about the specified instance group
func (c *Cloud) DescribeInstanceGroup(instanceGroupName string) (InstanceGroupInfo, error) {
return DescribeInstanceGroup(c.asg, instanceGroupName)
}
// awsInstanceGroup implements InstanceGroupInfo
var _ InstanceGroupInfo = &awsInstanceGroup{}
type awsInstanceGroup struct {
group *autoscaling.Group
}
// Implement InstanceGroupInfo.CurrentSize
// The number of instances currently running under control of this group
func (g *awsInstanceGroup) CurrentSize() (int, error) {
return len(g.group.Instances), nil
}

1662
staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go
View File

File diff suppressed because it is too large Load Diff

1081
staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer_test.go
View File

File diff suppressed because it is too large Load Diff

75
staging/src/k8s.io/legacy-cloud-providers/aws/aws_metrics.go
View File

@@ -1,75 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"sync"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
var (
awsAPIMetric = metrics.NewHistogramVec(
&metrics.HistogramOpts{
Name: "cloudprovider_aws_api_request_duration_seconds",
Help: "Latency of AWS API calls",
StabilityLevel: metrics.ALPHA,
},
[]string{"request"})
awsAPIErrorMetric = metrics.NewCounterVec(
&metrics.CounterOpts{
Name: "cloudprovider_aws_api_request_errors",
Help: "AWS API errors",
StabilityLevel: metrics.ALPHA,
},
[]string{"request"})
awsAPIThrottlesMetric = metrics.NewCounterVec(
&metrics.CounterOpts{
Name: "cloudprovider_aws_api_throttled_requests_total",
Help: "AWS API throttled requests",
StabilityLevel: metrics.ALPHA,
},
[]string{"operation_name"})
)
func recordAWSMetric(actionName string, timeTaken float64, err error) {
if err != nil {
awsAPIErrorMetric.With(metrics.Labels{"request": actionName}).Inc()
} else {
awsAPIMetric.With(metrics.Labels{"request": actionName}).Observe(timeTaken)
}
}
func recordAWSThrottlesMetric(operation string) {
awsAPIThrottlesMetric.With(metrics.Labels{"operation_name": operation}).Inc()
}
var registerOnce sync.Once
func registerMetrics() {
registerOnce.Do(func() {
legacyregistry.MustRegister(awsAPIMetric)
legacyregistry.MustRegister(awsAPIErrorMetric)
legacyregistry.MustRegister(awsAPIThrottlesMetric)
})
}

222
staging/src/k8s.io/legacy-cloud-providers/aws/aws_routes.go
View File

@@ -1,222 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/klog/v2"
cloudprovider "k8s.io/cloud-provider"
)
func (c *Cloud) findRouteTable(clusterName string) (*ec2.RouteTable, error) {
// This should be unnecessary (we already filter on TagNameKubernetesCluster,
// and something is broken if cluster name doesn't match, but anyway...
// TODO: All clouds should be cluster-aware by default
var tables []*ec2.RouteTable
if c.cfg.Global.RouteTableID != "" {
request := &ec2.DescribeRouteTablesInput{Filters: []*ec2.Filter{newEc2Filter("route-table-id", c.cfg.Global.RouteTableID)}}
response, err := c.ec2.DescribeRouteTables(request)
if err != nil {
return nil, err
}
tables = response
} else {
request := &ec2.DescribeRouteTablesInput{}
response, err := c.ec2.DescribeRouteTables(request)
if err != nil {
return nil, err
}
for _, table := range response {
if c.tagging.hasClusterTag(table.Tags) {
tables = append(tables, table)
}
}
}
if len(tables) == 0 {
return nil, fmt.Errorf("unable to find route table for AWS cluster: %s", clusterName)
}
if len(tables) != 1 {
return nil, fmt.Errorf("found multiple matching AWS route tables for AWS cluster: %s", clusterName)
}
return tables[0], nil
}
// ListRoutes implements Routes.ListRoutes
// List all routes that match the filter
func (c *Cloud) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) {
table, err := c.findRouteTable(clusterName)
if err != nil {
return nil, err
}
var routes []*cloudprovider.Route
var instanceIDs []*string
for _, r := range table.Routes {
instanceID := aws.StringValue(r.InstanceId)
if instanceID == "" {
continue
}
instanceIDs = append(instanceIDs, &instanceID)
}
instances, err := c.getInstancesByIDs(instanceIDs)
if err != nil {
return nil, err
}
for _, r := range table.Routes {
destinationCIDR := aws.StringValue(r.DestinationCidrBlock)
if destinationCIDR == "" {
continue
}
route := &cloudprovider.Route{
Name: clusterName + "-" + destinationCIDR,
DestinationCIDR: destinationCIDR,
}
// Capture blackhole routes
if aws.StringValue(r.State) == ec2.RouteStateBlackhole {
route.Blackhole = true
routes = append(routes, route)
continue
}
// Capture instance routes
instanceID := aws.StringValue(r.InstanceId)
if instanceID != "" {
instance, found := instances[instanceID]
if found {
route.TargetNode = mapInstanceToNodeName(instance)
routes = append(routes, route)
} else {
klog.Warningf("unable to find instance ID %s in the list of instances being routed to", instanceID)
}
}
}
return routes, nil
}
// Sets the instance attribute "source-dest-check" to the specified value
func (c *Cloud) configureInstanceSourceDestCheck(instanceID string, sourceDestCheck bool) error {
request := &ec2.ModifyInstanceAttributeInput{}
request.InstanceId = aws.String(instanceID)
request.SourceDestCheck = &ec2.AttributeBooleanValue{Value: aws.Bool(sourceDestCheck)}
_, err := c.ec2.ModifyInstanceAttribute(request)
if err != nil {
return fmt.Errorf("error configuring source-dest-check on instance %s: %q", instanceID, err)
}
return nil
}
// CreateRoute implements Routes.CreateRoute
// Create the described route
func (c *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint string, route *cloudprovider.Route) error {
instance, err := c.getInstanceByNodeName(route.TargetNode)
if err != nil {
return err
}
// In addition to configuring the route itself, we also need to configure the instance to accept that traffic
// On AWS, this requires turning source-dest checks off
err = c.configureInstanceSourceDestCheck(aws.StringValue(instance.InstanceId), false)
if err != nil {
return err
}
table, err := c.findRouteTable(clusterName)
if err != nil {
return err
}
var deleteRoute *ec2.Route
for _, r := range table.Routes {
destinationCIDR := aws.StringValue(r.DestinationCidrBlock)
if destinationCIDR != route.DestinationCIDR {
continue
}
if aws.StringValue(r.State) == ec2.RouteStateBlackhole {
deleteRoute = r
}
}
if deleteRoute != nil {
klog.Infof("deleting blackholed route: %s", aws.StringValue(deleteRoute.DestinationCidrBlock))
request := &ec2.DeleteRouteInput{}
request.DestinationCidrBlock = deleteRoute.DestinationCidrBlock
request.RouteTableId = table.RouteTableId
_, err = c.ec2.DeleteRoute(request)
if err != nil {
return fmt.Errorf("error deleting blackholed AWS route (%s): %q", aws.StringValue(deleteRoute.DestinationCidrBlock), err)
}
}
request := &ec2.CreateRouteInput{}
// TODO: use ClientToken for idempotency?
request.DestinationCidrBlock = aws.String(route.DestinationCIDR)
request.InstanceId = instance.InstanceId
request.RouteTableId = table.RouteTableId
_, err = c.ec2.CreateRoute(request)
if err != nil {
return fmt.Errorf("error creating AWS route (%s): %q", route.DestinationCIDR, err)
}
return nil
}
// DeleteRoute implements Routes.DeleteRoute
// Delete the specified route
func (c *Cloud) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error {
table, err := c.findRouteTable(clusterName)
if err != nil {
return err
}
request := &ec2.DeleteRouteInput{}
request.DestinationCidrBlock = aws.String(route.DestinationCIDR)
request.RouteTableId = table.RouteTableId
_, err = c.ec2.DeleteRoute(request)
if err != nil {
return fmt.Errorf("error deleting AWS route (%s): %q", route.DestinationCIDR, err)
}
return nil
}

3578
staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go
View File

File diff suppressed because it is too large Load Diff

48
staging/src/k8s.io/legacy-cloud-providers/aws/aws_utils.go
View File

@@ -1,48 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"github.com/aws/aws-sdk-go/aws"
"k8s.io/apimachinery/pkg/util/sets"
)
func stringSetToPointers(in sets.String) []*string {
if in == nil {
return nil
}
out := make([]*string, 0, len(in))
for k := range in {
out = append(out, aws.String(k))
}
return out
}
func stringSetFromPointers(in []*string) sets.String {
if in == nil {
return nil
}
out := sets.NewString()
for i := range in {
out.Insert(aws.StringValue(in[i]))
}
return out
}

135
staging/src/k8s.io/legacy-cloud-providers/aws/device_allocator.go
View File

@@ -1,135 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"sort"
"sync"
)
// ExistingDevices is a map of assigned devices. Presence of a key with a device
// name in the map means that the device is allocated. Value is irrelevant and
// can be used for anything that DeviceAllocator user wants.
// Only the relevant part of device name should be in the map, e.g. "ba" for
// "/dev/xvdba".
type ExistingDevices map[mountDevice]EBSVolumeID
// DeviceAllocator finds available device name, taking into account already
// assigned device names from ExistingDevices map. It tries to find the next
// device name to the previously assigned one (from previous DeviceAllocator
// call), so all available device names are used eventually and it minimizes
// device name reuse.
//
// All these allocations are in-memory, nothing is written to / read from
// /dev directory.
//
// On AWS, we should assign new (not yet used) device names to attached volumes.
// If we reuse a previously used name, we may get the volume "attaching" forever,
// see https://aws.amazon.com/premiumsupport/knowledge-center/ebs-stuck-attaching/.
type DeviceAllocator interface {
// GetNext returns a free device name or error when there is no free device
// name. Only the device suffix is returned, e.g. "ba" for "/dev/xvdba".
// It's up to the called to add appropriate "/dev/sd" or "/dev/xvd" prefix.
GetNext(existingDevices ExistingDevices) (mountDevice, error)
// Deprioritize the device so as it can't be used immediately again
Deprioritize(mountDevice)
// Lock the deviceAllocator
Lock()
// Unlock the deviceAllocator
Unlock()
}
type deviceAllocator struct {
possibleDevices map[mountDevice]int
counter int
deviceLock sync.Mutex
}
var _ DeviceAllocator = &deviceAllocator{}
type devicePair struct {
deviceName mountDevice
deviceIndex int
}
type devicePairList []devicePair
func (p devicePairList) Len() int { return len(p) }
func (p devicePairList) Less(i, j int) bool { return p[i].deviceIndex < p[j].deviceIndex }
func (p devicePairList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// NewDeviceAllocator allocates device names according to scheme ba..bz, ca..cz
// it moves along the ring and always picks next device until device list is
// exhausted.
func NewDeviceAllocator() DeviceAllocator {
possibleDevices := make(map[mountDevice]int)
for _, firstChar := range []rune{'b', 'c'} {
for i := 'a'; i <= 'z'; i++ {
dev := mountDevice([]rune{firstChar, i})
possibleDevices[dev] = 0
}
}
return &deviceAllocator{
possibleDevices: possibleDevices,
counter: 0,
}
}
// GetNext gets next available device from the pool, this function assumes that caller
// holds the necessary lock on deviceAllocator
func (d *deviceAllocator) GetNext(existingDevices ExistingDevices) (mountDevice, error) {
for _, devicePair := range d.sortByCount() {
if _, found := existingDevices[devicePair.deviceName]; !found {
return devicePair.deviceName, nil
}
}
return "", fmt.Errorf("no devices are available")
}
func (d *deviceAllocator) sortByCount() devicePairList {
dpl := make(devicePairList, 0)
for deviceName, deviceIndex := range d.possibleDevices {
dpl = append(dpl, devicePair{deviceName, deviceIndex})
}
sort.Sort(dpl)
return dpl
}
func (d *deviceAllocator) Lock() {
d.deviceLock.Lock()
}
func (d *deviceAllocator) Unlock() {
d.deviceLock.Unlock()
}
// Deprioritize the device so as it can't be used immediately again
func (d *deviceAllocator) Deprioritize(chosen mountDevice) {
d.deviceLock.Lock()
defer d.deviceLock.Unlock()
if _, ok := d.possibleDevices[chosen]; ok {
d.counter++
d.possibleDevices[chosen] = d.counter
}
}

84
staging/src/k8s.io/legacy-cloud-providers/aws/device_allocator_test.go
View File

@@ -1,84 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import "testing"
func TestDeviceAllocator(t *testing.T) {
tests := []struct {
name string
existingDevices ExistingDevices
deviceMap map[mountDevice]int
expectedOutput mountDevice
}{
{
"empty device list with wrap",
ExistingDevices{},
generateUnsortedDeviceList(),
"bd", // next to 'zz' is the first one, 'ba'
},
}
for _, test := range tests {
allocator := NewDeviceAllocator().(*deviceAllocator)
for k, v := range test.deviceMap {
allocator.possibleDevices[k] = v
}
got, err := allocator.GetNext(test.existingDevices)
if err != nil {
t.Errorf("text %q: unexpected error: %v", test.name, err)
}
if got != test.expectedOutput {
t.Errorf("text %q: expected %q, got %q", test.name, test.expectedOutput, got)
}
}
}
func generateUnsortedDeviceList() map[mountDevice]int {
possibleDevices := make(map[mountDevice]int)
for _, firstChar := range []rune{'b', 'c'} {
for i := 'a'; i <= 'z'; i++ {
dev := mountDevice([]rune{firstChar, i})
possibleDevices[dev] = 3
}
}
possibleDevices["bd"] = 0
return possibleDevices
}
func TestDeviceAllocatorError(t *testing.T) {
allocator := NewDeviceAllocator().(*deviceAllocator)
existingDevices := ExistingDevices{}
// make all devices used
var first, second byte
for first = 'b'; first <= 'c'; first++ {
for second = 'a'; second <= 'z'; second++ {
device := [2]byte{first, second}
existingDevices[mountDevice(device[:])] = "used"
}
}
device, err := allocator.GetNext(existingDevices)
if err == nil {
t.Errorf("expected error, got device %q", device)
}
}

17
staging/src/k8s.io/legacy-cloud-providers/aws/doc.go
View File

@@ -1,17 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws

276
staging/src/k8s.io/legacy-cloud-providers/aws/instances.go
View File

@@ -1,276 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"net/url"
"regexp"
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/klog/v2"
"k8s.io/api/core/v1"
)
// awsInstanceRegMatch represents Regex Match for AWS instance.
var awsInstanceRegMatch = regexp.MustCompile("^i-[^/]*$")
// InstanceID represents the ID of the instance in the AWS API, e.g. i-12345678
// The "traditional" format is "i-12345678"
// A new longer format is also being introduced: "i-12345678abcdef01"
// We should not assume anything about the length or format, though it seems
// reasonable to assume that instances will continue to start with "i-".
type InstanceID string
func (i InstanceID) awsString() *string {
return aws.String(string(i))
}
// KubernetesInstanceID represents the id for an instance in the kubernetes API;
// the following form
// - aws:///<zone>/<awsInstanceId>
// - aws:////<awsInstanceId>
// - <awsInstanceId>
type KubernetesInstanceID string
// MapToAWSInstanceID extracts the InstanceID from the KubernetesInstanceID
func (name KubernetesInstanceID) MapToAWSInstanceID() (InstanceID, error) {
s := string(name)
if !strings.HasPrefix(s, "aws://") {
// Assume a bare aws volume id (vol-1234...)
// Build a URL with an empty host (AZ)
s = "aws://" + "/" + "/" + s
}
url, err := url.Parse(s)
if err != nil {
return "", fmt.Errorf("Invalid instance name (%s): %v", name, err)
}
if url.Scheme != "aws" {
return "", fmt.Errorf("Invalid scheme for AWS instance (%s)", name)
}
awsID := ""
tokens := strings.Split(strings.Trim(url.Path, "/"), "/")
if len(tokens) == 1 {
// instanceId
awsID = tokens[0]
} else if len(tokens) == 2 {
// az/instanceId
awsID = tokens[1]
}
// We sanity check the resulting volume; the two known formats are
// i-12345678 and i-12345678abcdef01
if awsID == "" || !awsInstanceRegMatch.MatchString(awsID) {
return "", fmt.Errorf("Invalid format for AWS instance (%s)", name)
}
return InstanceID(awsID), nil
}
// mapToAWSInstanceID extracts the InstanceIDs from the Nodes, returning an error if a Node cannot be mapped
func mapToAWSInstanceIDs(nodes []*v1.Node) ([]InstanceID, error) {
var instanceIDs []InstanceID
for _, node := range nodes {
if node.Spec.ProviderID == "" {
return nil, fmt.Errorf("node %q did not have ProviderID set", node.Name)
}
instanceID, err := KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID()
if err != nil {
return nil, fmt.Errorf("unable to parse ProviderID %q for node %q", node.Spec.ProviderID, node.Name)
}
instanceIDs = append(instanceIDs, instanceID)
}
return instanceIDs, nil
}
// mapToAWSInstanceIDsTolerant extracts the InstanceIDs from the Nodes, skipping Nodes that cannot be mapped
func mapToAWSInstanceIDsTolerant(nodes []*v1.Node) []InstanceID {
var instanceIDs []InstanceID
for _, node := range nodes {
if node.Spec.ProviderID == "" {
klog.Warningf("node %q did not have ProviderID set", node.Name)
continue
}
instanceID, err := KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID()
if err != nil {
klog.Warningf("unable to parse ProviderID %q for node %q", node.Spec.ProviderID, node.Name)
continue
}
instanceIDs = append(instanceIDs, instanceID)
}
return instanceIDs
}
// Gets the full information about this instance from the EC2 API
func describeInstance(ec2Client EC2, instanceID InstanceID) (*ec2.Instance, error) {
request := &ec2.DescribeInstancesInput{
InstanceIds: []*string{instanceID.awsString()},
}
instances, err := ec2Client.DescribeInstances(request)
if err != nil {
return nil, err
}
if len(instances) == 0 {
return nil, fmt.Errorf("no instances found for instance: %s", instanceID)
}
if len(instances) > 1 {
return nil, fmt.Errorf("multiple instances found for instance: %s", instanceID)
}
return instances[0], nil
}
// instanceCache manages the cache of DescribeInstances
type instanceCache struct {
// TODO: Get rid of this field, send all calls through the instanceCache
cloud *Cloud
mutex sync.Mutex
snapshot *allInstancesSnapshot
}
// Gets the full information about these instance from the EC2 API
func (c *instanceCache) describeAllInstancesUncached() (*allInstancesSnapshot, error) {
now := time.Now()
klog.V(4).Infof("EC2 DescribeInstances - fetching all instances")
var filters []*ec2.Filter
instances, err := c.cloud.describeInstances(filters)
if err != nil {
return nil, err
}
m := make(map[InstanceID]*ec2.Instance)
for _, i := range instances {
id := InstanceID(aws.StringValue(i.InstanceId))
m[id] = i
}
snapshot := &allInstancesSnapshot{now, m}
c.mutex.Lock()
defer c.mutex.Unlock()
if c.snapshot != nil && snapshot.olderThan(c.snapshot) {
// If this happens a lot, we could run this function in a mutex and only return one result
klog.Infof("Not caching concurrent AWS DescribeInstances results")
} else {
c.snapshot = snapshot
}
return snapshot, nil
}
// cacheCriteria holds criteria that must hold to use a cached snapshot
type cacheCriteria struct {
// MaxAge indicates the maximum age of a cached snapshot we can accept.
// If set to 0 (i.e. unset), cached values will not time out because of age.
MaxAge time.Duration
// HasInstances is a list of InstanceIDs that must be in a cached snapshot for it to be considered valid.
// If an instance is not found in the cached snapshot, the snapshot be ignored and we will re-fetch.
HasInstances []InstanceID
}
// describeAllInstancesCached returns all instances, using cached results if applicable
func (c *instanceCache) describeAllInstancesCached(criteria cacheCriteria) (*allInstancesSnapshot, error) {
var err error
snapshot := c.getSnapshot()
if snapshot != nil && !snapshot.MeetsCriteria(criteria) {
snapshot = nil
}
if snapshot == nil {
snapshot, err = c.describeAllInstancesUncached()
if err != nil {
return nil, err
}
} else {
klog.V(6).Infof("EC2 DescribeInstances - using cached results")
}
return snapshot, nil
}
// getSnapshot returns a snapshot if one exists
func (c *instanceCache) getSnapshot() *allInstancesSnapshot {
c.mutex.Lock()
defer c.mutex.Unlock()
return c.snapshot
}
// olderThan is a simple helper to encapsulate timestamp comparison
func (s *allInstancesSnapshot) olderThan(other *allInstancesSnapshot) bool {
// After() is technically broken by time changes until we have monotonic time
return other.timestamp.After(s.timestamp)
}
// MeetsCriteria returns true if the snapshot meets the criteria in cacheCriteria
func (s *allInstancesSnapshot) MeetsCriteria(criteria cacheCriteria) bool {
if criteria.MaxAge > 0 {
// Sub() is technically broken by time changes until we have monotonic time
now := time.Now()
if now.Sub(s.timestamp) > criteria.MaxAge {
klog.V(6).Infof("instanceCache snapshot cannot be used as is older than MaxAge=%s", criteria.MaxAge)
return false
}
}
if len(criteria.HasInstances) != 0 {
for _, id := range criteria.HasInstances {
if nil == s.instances[id] {
klog.V(6).Infof("instanceCache snapshot cannot be used as does not contain instance %s", id)
return false
}
}
}
return true
}
// allInstancesSnapshot holds the results from querying for all instances,
// along with the timestamp for cache-invalidation purposes
type allInstancesSnapshot struct {
timestamp time.Time
instances map[InstanceID]*ec2.Instance
}
// FindInstances returns the instances corresponding to the specified ids. If an id is not found, it is ignored.
func (s *allInstancesSnapshot) FindInstances(ids []InstanceID) map[InstanceID]*ec2.Instance {
m := make(map[InstanceID]*ec2.Instance)
for _, id := range ids {
instance := s.instances[id]
if instance != nil {
m[id] = instance
}
}
return m
}

204
staging/src/k8s.io/legacy-cloud-providers/aws/instances_test.go
View File

@@ -1,204 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
)
func TestMapToAWSInstanceIDs(t *testing.T) {
tests := []struct {
Kubernetes KubernetesInstanceID
Aws InstanceID
ExpectError bool
}{
{
Kubernetes: "aws:///us-east-1a/i-12345678",
Aws: "i-12345678",
},
{
Kubernetes: "aws:////i-12345678",
Aws: "i-12345678",
},
{
Kubernetes: "i-12345678",
Aws: "i-12345678",
},
{
Kubernetes: "aws:///us-east-1a/i-12345678abcdef01",
Aws: "i-12345678abcdef01",
},
{
Kubernetes: "aws:////i-12345678abcdef01",
Aws: "i-12345678abcdef01",
},
{
Kubernetes: "i-12345678abcdef01",
Aws: "i-12345678abcdef01",
},
{
Kubernetes: "vol-123456789",
ExpectError: true,
},
{
Kubernetes: "aws:///us-east-1a/vol-12345678abcdef01",
ExpectError: true,
},
{
Kubernetes: "aws://accountid/us-east-1a/vol-12345678abcdef01",
ExpectError: true,
},
{
Kubernetes: "aws:///us-east-1a/vol-12345678abcdef01/suffix",
ExpectError: true,
},
{
Kubernetes: "",
ExpectError: true,
},
}
for _, test := range tests {
awsID, err := test.Kubernetes.MapToAWSInstanceID()
if err != nil {
if !test.ExpectError {
t.Errorf("unexpected error parsing %s: %v", test.Kubernetes, err)
}
} else {
if test.ExpectError {
t.Errorf("expected error parsing %s", test.Kubernetes)
} else if test.Aws != awsID {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsID)
}
}
}
for _, test := range tests {
node := &v1.Node{}
node.Spec.ProviderID = string(test.Kubernetes)
awsInstanceIds, err := mapToAWSInstanceIDs([]*v1.Node{node})
if err != nil {
if !test.ExpectError {
t.Errorf("unexpected error parsing %s: %v", test.Kubernetes, err)
}
} else {
if test.ExpectError {
t.Errorf("expected error parsing %s", test.Kubernetes)
} else if len(awsInstanceIds) != 1 {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
} else if awsInstanceIds[0] != test.Aws {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
}
}
awsInstanceIds = mapToAWSInstanceIDsTolerant([]*v1.Node{node})
if test.ExpectError {
if len(awsInstanceIds) != 0 {
t.Errorf("unexpected results parsing %s: %s", test.Kubernetes, awsInstanceIds)
}
} else {
if len(awsInstanceIds) != 1 {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
} else if awsInstanceIds[0] != test.Aws {
t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
}
}
}
}
func TestSnapshotMeetsCriteria(t *testing.T) {
snapshot := &allInstancesSnapshot{timestamp: time.Now().Add(-3601 * time.Second)}
if !snapshot.MeetsCriteria(cacheCriteria{}) {
t.Errorf("Snapshot should always meet empty criteria")
}
if snapshot.MeetsCriteria(cacheCriteria{MaxAge: time.Hour}) {
t.Errorf("Snapshot did not honor MaxAge")
}
if snapshot.MeetsCriteria(cacheCriteria{HasInstances: []InstanceID{InstanceID("i-12345678")}}) {
t.Errorf("Snapshot did not honor HasInstances with missing instances")
}
snapshot.instances = make(map[InstanceID]*ec2.Instance)
snapshot.instances[InstanceID("i-12345678")] = &ec2.Instance{}
if !snapshot.MeetsCriteria(cacheCriteria{HasInstances: []InstanceID{InstanceID("i-12345678")}}) {
t.Errorf("Snapshot did not honor HasInstances with matching instances")
}
if snapshot.MeetsCriteria(cacheCriteria{HasInstances: []InstanceID{InstanceID("i-12345678"), InstanceID("i-00000000")}}) {
t.Errorf("Snapshot did not honor HasInstances with partially matching instances")
}
}
func TestOlderThan(t *testing.T) {
t1 := time.Now()
t2 := t1.Add(time.Second)
s1 := &allInstancesSnapshot{timestamp: t1}
s2 := &allInstancesSnapshot{timestamp: t2}
assert.True(t, s1.olderThan(s2), "s1 should be olderThan s2")
assert.False(t, s2.olderThan(s1), "s2 not should be olderThan s1")
assert.False(t, s1.olderThan(s1), "s1 not should be olderThan itself")
}
func TestSnapshotFindInstances(t *testing.T) {
snapshot := &allInstancesSnapshot{}
snapshot.instances = make(map[InstanceID]*ec2.Instance)
{
id := InstanceID("i-12345678")
snapshot.instances[id] = &ec2.Instance{InstanceId: id.awsString()}
}
{
id := InstanceID("i-23456789")
snapshot.instances[id] = &ec2.Instance{InstanceId: id.awsString()}
}
instances := snapshot.FindInstances([]InstanceID{InstanceID("i-12345678"), InstanceID("i-23456789"), InstanceID("i-00000000")})
if len(instances) != 2 {
t.Errorf("findInstances returned %d results, expected 2", len(instances))
}
for _, id := range []InstanceID{InstanceID("i-12345678"), InstanceID("i-23456789")} {
i := instances[id]
if i == nil {
t.Errorf("findInstances did not return %s", id)
continue
}
if aws.StringValue(i.InstanceId) != string(id) {
t.Errorf("findInstances did not return expected instanceId for %s", id)
}
if i != snapshot.instances[id] {
t.Errorf("findInstances did not return expected instance (reference equality) for %s", id)
}
}
}

51
staging/src/k8s.io/legacy-cloud-providers/aws/log_handler.go
View File

@@ -1,51 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"github.com/aws/aws-sdk-go/aws/request"
"k8s.io/klog/v2"
)
// Handler for aws-sdk-go that logs all requests
func awsHandlerLogger(req *request.Request) {
service, name := awsServiceAndName(req)
klog.V(4).Infof("AWS request: %s %s", service, name)
}
func awsSendHandlerLogger(req *request.Request) {
service, name := awsServiceAndName(req)
klog.V(4).Infof("AWS API Send: %s %s %v %v", service, name, req.Operation, req.Params)
}
func awsValidateResponseHandlerLogger(req *request.Request) {
service, name := awsServiceAndName(req)
klog.V(4).Infof("AWS API ValidateResponse: %s %s %v %v %s", service, name, req.Operation, req.Params, req.HTTPResponse.Status)
}
func awsServiceAndName(req *request.Request) (string, string) {
service := req.ClientInfo.ServiceName
name := "?"
if req.Operation != nil {
name = req.Operation.Name
}
return service, name
}

178
staging/src/k8s.io/legacy-cloud-providers/aws/retry_handler.go
View File

@@ -1,178 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"math"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"k8s.io/klog/v2"
)
const (
decayIntervalSeconds = 20
decayFraction = 0.8
maxDelay = 60 * time.Second
)
// CrossRequestRetryDelay inserts delays before AWS calls, when we are observing RequestLimitExceeded errors
// Note that we share a CrossRequestRetryDelay across multiple AWS requests; this is a process-wide back-off,
// whereas the aws-sdk-go implements a per-request exponential backoff/retry
type CrossRequestRetryDelay struct {
backoff Backoff
}
// NewCrossRequestRetryDelay creates a new CrossRequestRetryDelay
func NewCrossRequestRetryDelay() *CrossRequestRetryDelay {
c := &CrossRequestRetryDelay{}
c.backoff.init(decayIntervalSeconds, decayFraction, maxDelay)
return c
}
// BeforeSign is added to the Sign chain; called before each request
func (c *CrossRequestRetryDelay) BeforeSign(r *request.Request) {
now := time.Now()
delay := c.backoff.ComputeDelayForRequest(now)
if delay > 0 {
klog.Warningf("Inserting delay before AWS request (%s) to avoid RequestLimitExceeded: %s",
describeRequest(r), delay.String())
if sleepFn := r.Config.SleepDelay; sleepFn != nil {
// Support SleepDelay for backwards compatibility
sleepFn(delay)
} else if err := aws.SleepWithContext(r.Context(), delay); err != nil {
r.Error = awserr.New(request.CanceledErrorCode, "request context canceled", err)
r.Retryable = aws.Bool(false)
return
}
// Avoid clock skew problems
r.Time = now
}
}
// Return the operation name, for use in log messages and metrics
func operationName(r *request.Request) string {
name := "?"
if r.Operation != nil {
name = r.Operation.Name
}
return name
}
// Return a user-friendly string describing the request, for use in log messages
func describeRequest(r *request.Request) string {
service := r.ClientInfo.ServiceName
return service + "::" + operationName(r)
}
// AfterRetry is added to the AfterRetry chain; called after any error
func (c *CrossRequestRetryDelay) AfterRetry(r *request.Request) {
if r.Error == nil {
return
}
awsError, ok := r.Error.(awserr.Error)
if !ok {
return
}
if awsError.Code() == "RequestLimitExceeded" {
c.backoff.ReportError()
recordAWSThrottlesMetric(operationName(r))
klog.Warningf("Got RequestLimitExceeded error on AWS request (%s)",
describeRequest(r))
}
}
// Backoff manages a backoff that varies based on the recently observed failures
type Backoff struct {
decayIntervalSeconds int64
decayFraction float64
maxDelay time.Duration
mutex sync.Mutex
// We count all requests & the number of requests which hit a
// RequestLimit. We only really care about 'recent' requests, so we
// decay the counts exponentially to bias towards recent values.
countErrorsRequestLimit float32
countRequests float32
lastDecay int64
}
func (b *Backoff) init(decayIntervalSeconds int, decayFraction float64, maxDelay time.Duration) {
b.lastDecay = time.Now().Unix()
// Bias so that if the first request hits the limit we don't immediately apply the full delay
b.countRequests = 4
b.decayIntervalSeconds = int64(decayIntervalSeconds)
b.decayFraction = decayFraction
b.maxDelay = maxDelay
}
// ComputeDelayForRequest computes the delay required for a request, also
// updates internal state to count this request
func (b *Backoff) ComputeDelayForRequest(now time.Time) time.Duration {
b.mutex.Lock()
defer b.mutex.Unlock()
// Apply exponential decay to the counters
timeDeltaSeconds := now.Unix() - b.lastDecay
if timeDeltaSeconds > b.decayIntervalSeconds {
intervals := float64(timeDeltaSeconds) / float64(b.decayIntervalSeconds)
decay := float32(math.Pow(b.decayFraction, intervals))
b.countErrorsRequestLimit *= decay
b.countRequests *= decay
b.lastDecay = now.Unix()
}
// Count this request
b.countRequests += 1.0
// Compute the failure rate
errorFraction := float32(0.0)
if b.countRequests > 0.5 {
// Avoid tiny residuals & rounding errors
errorFraction = b.countErrorsRequestLimit / b.countRequests
}
// Ignore a low fraction of errors
// This also allows them to time-out
if errorFraction < 0.1 {
return time.Duration(0)
}
// Delay by the max delay multiplied by the recent error rate
// (i.e. we apply a linear delay function)
// TODO: This is pretty arbitrary
delay := time.Nanosecond * time.Duration(float32(b.maxDelay.Nanoseconds())*errorFraction)
// Round down to the nearest second for sanity
return time.Second * time.Duration(int(delay.Seconds()))
}
// ReportError is called when we observe a throttling error
func (b *Backoff) ReportError() {
b.mutex.Lock()
defer b.mutex.Unlock()
b.countErrorsRequestLimit += 1.0
}

138
staging/src/k8s.io/legacy-cloud-providers/aws/retry_handler_test.go
View File

@@ -1,138 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"testing"
"time"
)
// There follows a group of tests for the backoff logic. There's nothing
// particularly special about the values chosen: if we tweak the values in the
// backoff logic then we might well have to update the tests. However the key
// behavioural elements should remain (e.g. no errors => no backoff), and these
// are each tested by one of the tests below.
// Test that we don't apply any delays when there are no errors
func TestBackoffNoErrors(t *testing.T) {
b := &Backoff{}
b.init(decayIntervalSeconds, decayFraction, maxDelay)
now := time.Now()
for i := 0; i < 100; i++ {
d := b.ComputeDelayForRequest(now)
if d.Nanoseconds() != 0 {
t.Fatalf("unexpected delay during no-error case")
}
now = now.Add(time.Second)
}
}
// Test that we always apply a delay when there are errors, and also that we
// don't "flap" - that our own delay doesn't cause us to oscillate between
// delay and no-delay.
func TestBackoffAllErrors(t *testing.T) {
b := &Backoff{}
b.init(decayIntervalSeconds, decayFraction, maxDelay)
now := time.Now()
// Warm up
for i := 0; i < 10; i++ {
_ = b.ComputeDelayForRequest(now)
b.ReportError()
now = now.Add(time.Second)
}
for i := 0; i < 100; i++ {
d := b.ComputeDelayForRequest(now)
b.ReportError()
if d.Seconds() < 5 {
t.Fatalf("unexpected short-delay during all-error case: %v", d)
}
t.Logf("delay @%d %v", i, d)
now = now.Add(d)
}
}
// Test that we do come close to our max delay, when we see all errors at 1
// second intervals (this simulates multiple concurrent requests, because we
// don't wait for delay in between requests)
func TestBackoffHitsMax(t *testing.T) {
b := &Backoff{}
b.init(decayIntervalSeconds, decayFraction, maxDelay)
now := time.Now()
for i := 0; i < 100; i++ {
_ = b.ComputeDelayForRequest(now)
b.ReportError()
now = now.Add(time.Second)
}
for i := 0; i < 10; i++ {
d := b.ComputeDelayForRequest(now)
b.ReportError()
if float32(d.Nanoseconds()) < (float32(maxDelay.Nanoseconds()) * 0.95) {
t.Fatalf("expected delay to be >= 95 percent of max delay, was %v", d)
}
t.Logf("delay @%d %v", i, d)
now = now.Add(time.Second)
}
}
// Test that after a phase of errors, we eventually stop applying a delay once there are
// no more errors.
func TestBackoffRecovers(t *testing.T) {
b := &Backoff{}
b.init(decayIntervalSeconds, decayFraction, maxDelay)
now := time.Now()
// Phase of all-errors
for i := 0; i < 100; i++ {
_ = b.ComputeDelayForRequest(now)
b.ReportError()
now = now.Add(time.Second)
}
for i := 0; i < 10; i++ {
d := b.ComputeDelayForRequest(now)
b.ReportError()
if d.Seconds() < 5 {
t.Fatalf("unexpected short-delay during all-error phase: %v", d)
}
t.Logf("error phase delay @%d %v", i, d)
now = now.Add(time.Second)
}
// Phase of no errors
for i := 0; i < 100; i++ {
_ = b.ComputeDelayForRequest(now)
now = now.Add(3 * time.Second)
}
for i := 0; i < 10; i++ {
d := b.ComputeDelayForRequest(now)
if d.Seconds() != 0 {
t.Fatalf("unexpected delay during error recovery phase: %v", d)
}
t.Logf("no-error phase delay @%d %v", i, d)
now = now.Add(time.Second)
}
}

219
staging/src/k8s.io/legacy-cloud-providers/aws/sets_ippermissions.go
View File

@@ -1,219 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
)
// IPPermissionSet maps IP strings of strings to EC2 IpPermissions
type IPPermissionSet map[string]*ec2.IpPermission
// IPPermissionPredicate is an predicate to test whether IPPermission matches some condition.
type IPPermissionPredicate interface {
// Test checks whether specified IPPermission matches condition.
Test(perm *ec2.IpPermission) bool
}
// NewIPPermissionSet creates a new IPPermissionSet
func NewIPPermissionSet(items ...*ec2.IpPermission) IPPermissionSet {
s := make(IPPermissionSet)
s.Insert(items...)
return s
}
// Ungroup splits permissions out into individual permissions
// EC2 will combine permissions with the same port but different SourceRanges together, for example
// We ungroup them so we can process them
func (s IPPermissionSet) Ungroup() IPPermissionSet {
l := []*ec2.IpPermission{}
for _, p := range s.List() {
if len(p.IpRanges) <= 1 {
l = append(l, p)
continue
}
for _, ipRange := range p.IpRanges {
c := &ec2.IpPermission{}
*c = *p
c.IpRanges = []*ec2.IpRange{ipRange}
l = append(l, c)
}
}
l2 := []*ec2.IpPermission{}
for _, p := range l {
if len(p.UserIdGroupPairs) <= 1 {
l2 = append(l2, p)
continue
}
for _, u := range p.UserIdGroupPairs {
c := &ec2.IpPermission{}
*c = *p
c.UserIdGroupPairs = []*ec2.UserIdGroupPair{u}
l2 = append(l, c)
}
}
l3 := []*ec2.IpPermission{}
for _, p := range l2 {
if len(p.PrefixListIds) <= 1 {
l3 = append(l3, p)
continue
}
for _, v := range p.PrefixListIds {
c := &ec2.IpPermission{}
*c = *p
c.PrefixListIds = []*ec2.PrefixListId{v}
l3 = append(l3, c)
}
}
return NewIPPermissionSet(l3...)
}
// Insert adds items to the set.
func (s IPPermissionSet) Insert(items ...*ec2.IpPermission) {
for _, p := range items {
k := keyForIPPermission(p)
s[k] = p
}
}
// Delete delete permission from the set.
func (s IPPermissionSet) Delete(items ...*ec2.IpPermission) {
for _, p := range items {
k := keyForIPPermission(p)
delete(s, k)
}
}
// DeleteIf delete permission from the set if permission matches predicate.
func (s IPPermissionSet) DeleteIf(predicate IPPermissionPredicate) {
for k, p := range s {
if predicate.Test(p) {
delete(s, k)
}
}
}
// List returns the contents as a slice. Order is not defined.
func (s IPPermissionSet) List() []*ec2.IpPermission {
res := make([]*ec2.IpPermission, 0, len(s))
for _, v := range s {
res = append(res, v)
}
return res
}
// IsSuperset returns true if and only if s is a superset of s2.
func (s IPPermissionSet) IsSuperset(s2 IPPermissionSet) bool {
for k := range s2 {
_, found := s[k]
if !found {
return false
}
}
return true
}
// Equal returns true if and only if s is equal (as a set) to s2.
// Two sets are equal if their membership is identical.
// (In practice, this means same elements, order doesn't matter)
func (s IPPermissionSet) Equal(s2 IPPermissionSet) bool {
return len(s) == len(s2) && s.IsSuperset(s2)
}
// Difference returns a set of objects that are not in s2
// For example:
// s1 = {a1, a2, a3}
// s2 = {a1, a2, a4, a5}
// s1.Difference(s2) = {a3}
// s2.Difference(s1) = {a4, a5}
func (s IPPermissionSet) Difference(s2 IPPermissionSet) IPPermissionSet {
result := NewIPPermissionSet()
for k, v := range s {
_, found := s2[k]
if !found {
result[k] = v
}
}
return result
}
// Len returns the size of the set.
func (s IPPermissionSet) Len() int {
return len(s)
}
func keyForIPPermission(p *ec2.IpPermission) string {
v, err := json.Marshal(p)
if err != nil {
panic(fmt.Sprintf("error building JSON representation of ec2.IpPermission: %v", err))
}
return string(v)
}
var _ IPPermissionPredicate = IPPermissionMatchDesc{}
// IPPermissionMatchDesc checks whether specific IPPermission contains description.
type IPPermissionMatchDesc struct {
Description string
}
// Test whether specific IPPermission contains description.
func (p IPPermissionMatchDesc) Test(perm *ec2.IpPermission) bool {
for _, v4Range := range perm.IpRanges {
if aws.StringValue(v4Range.Description) == p.Description {
return true
}
}
for _, v6Range := range perm.Ipv6Ranges {
if aws.StringValue(v6Range.Description) == p.Description {
return true
}
}
for _, prefixListID := range perm.PrefixListIds {
if aws.StringValue(prefixListID.Description) == p.Description {
return true
}
}
for _, group := range perm.UserIdGroupPairs {
if aws.StringValue(group.Description) == p.Description {
return true
}
}
return false
}
var _ IPPermissionPredicate = IPPermissionNotMatch{}
// IPPermissionNotMatch is the *not* operator for Predicate
type IPPermissionNotMatch struct {
Predicate IPPermissionPredicate
}
// Test whether specific IPPermission not match the embed predicate.
func (p IPPermissionNotMatch) Test(perm *ec2.IpPermission) bool {
return !p.Predicate.Test(perm)
}

315
staging/src/k8s.io/legacy-cloud-providers/aws/tags.go
View File

@@ -1,315 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/klog/v2"
"k8s.io/apimachinery/pkg/util/wait"
)
// TagNameKubernetesClusterPrefix is the tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.
// The tag key = TagNameKubernetesClusterPrefix + clusterID
// The tag value is an ownership value
const TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/"
// TagNameKubernetesClusterLegacy is the legacy tag name we use to differentiate multiple
// logically independent clusters running in the same AZ. The problem with it was that it
// did not allow shared resources.
const TagNameKubernetesClusterLegacy = "KubernetesCluster"
// ResourceLifecycle is the cluster lifecycle state used in tagging
type ResourceLifecycle string
const (
// ResourceLifecycleOwned is the value we use when tagging resources to indicate
// that the resource is considered owned and managed by the cluster,
// and in particular that the lifecycle is tied to the lifecycle of the cluster.
ResourceLifecycleOwned = "owned"
// ResourceLifecycleShared is the value we use when tagging resources to indicate
// that the resource is shared between multiple clusters, and should not be destroyed
// if the cluster is destroyed.
ResourceLifecycleShared = "shared"
)
type awsTagging struct {
// ClusterID is our cluster identifier: we tag AWS resources with this value,
// and thus we can run two independent clusters in the same VPC or subnets.
// This gives us similar functionality to GCE projects.
ClusterID string
// usesLegacyTags is true if we are using the legacy TagNameKubernetesClusterLegacy tags
usesLegacyTags bool
}
func (t *awsTagging) init(legacyClusterID string, clusterID string) error {
if legacyClusterID != "" {
if clusterID != "" && legacyClusterID != clusterID {
return fmt.Errorf("clusterID tags did not match: %q vs %q", clusterID, legacyClusterID)
}
t.usesLegacyTags = true
clusterID = legacyClusterID
}
t.ClusterID = clusterID
if clusterID != "" {
klog.Infof("AWS cloud filtering on ClusterID: %v", clusterID)
} else {
return fmt.Errorf("AWS cloud failed to find ClusterID")
}
return nil
}
// Extracts a clusterID from the given tags, if one is present
// If no clusterID is found, returns "", nil
// If multiple (different) clusterIDs are found, returns an error
func (t *awsTagging) initFromTags(tags []*ec2.Tag) error {
legacyClusterID, newClusterID, err := findClusterIDs(tags)
if err != nil {
return err
}
if legacyClusterID == "" && newClusterID == "" {
klog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...")
}
return t.init(legacyClusterID, newClusterID)
}
// Extracts the legacy & new cluster ids from the given tags, if they are present
// If duplicate tags are found, returns an error
func findClusterIDs(tags []*ec2.Tag) (string, string, error) {
legacyClusterID := ""
newClusterID := ""
for _, tag := range tags {
tagKey := aws.StringValue(tag.Key)
if strings.HasPrefix(tagKey, TagNameKubernetesClusterPrefix) {
id := strings.TrimPrefix(tagKey, TagNameKubernetesClusterPrefix)
if newClusterID != "" {
return "", "", fmt.Errorf("Found multiple cluster tags with prefix %s (%q and %q)", TagNameKubernetesClusterPrefix, newClusterID, id)
}
newClusterID = id
}
if tagKey == TagNameKubernetesClusterLegacy {
id := aws.StringValue(tag.Value)
if legacyClusterID != "" {
return "", "", fmt.Errorf("Found multiple %s tags (%q and %q)", TagNameKubernetesClusterLegacy, legacyClusterID, id)
}
legacyClusterID = id
}
}
return legacyClusterID, newClusterID, nil
}
func (t *awsTagging) clusterTagKey() string {
return TagNameKubernetesClusterPrefix + t.ClusterID
}
func (t *awsTagging) hasClusterTag(tags []*ec2.Tag) bool {
// if the clusterID is not configured -- we consider all instances.
if len(t.ClusterID) == 0 {
return true
}
clusterTagKey := t.clusterTagKey()
for _, tag := range tags {
tagKey := aws.StringValue(tag.Key)
// For 1.6, we continue to recognize the legacy tags, for the 1.5 -> 1.6 upgrade
// Note that we want to continue traversing tag list if we see a legacy tag with value != ClusterID
if (tagKey == TagNameKubernetesClusterLegacy) && (aws.StringValue(tag.Value) == t.ClusterID) {
return true
}
if tagKey == clusterTagKey {
return true
}
}
return false
}
func (t *awsTagging) hasNoClusterPrefixTag(tags []*ec2.Tag) bool {
for _, tag := range tags {
if strings.HasPrefix(aws.StringValue(tag.Key), TagNameKubernetesClusterPrefix) {
return false
}
}
return true
}
// Ensure that a resource has the correct tags
// If it has no tags, we assume that this was a problem caused by an error in between creation and tagging,
// and we add the tags. If it has a different cluster's tags, that is an error.
func (t *awsTagging) readRepairClusterTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string, observedTags []*ec2.Tag) error {
actualTagMap := make(map[string]string)
for _, tag := range observedTags {
actualTagMap[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
}
expectedTags := t.buildTags(lifecycle, additionalTags)
addTags := make(map[string]string)
for k, expected := range expectedTags {
actual := actualTagMap[k]
if actual == expected {
continue
}
if actual == "" {
klog.Warningf("Resource %q was missing expected cluster tag %q. Will add (with value %q)", resourceID, k, expected)
addTags[k] = expected
} else {
return fmt.Errorf("resource %q has tag belonging to another cluster: %q=%q (expected %q)", resourceID, k, actual, expected)
}
}
if len(addTags) == 0 {
return nil
}
if err := t.createTags(client, resourceID, lifecycle, addTags); err != nil {
return fmt.Errorf("error adding missing tags to resource %q: %q", resourceID, err)
}
return nil
}
// createTags calls EC2 CreateTags, but adds retry-on-failure logic
// We retry mainly because if we create an object, we cannot tag it until it is "fully created" (eventual consistency)
// The error code varies though (depending on what we are tagging), so we simply retry on all errors
func (t *awsTagging) createTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string) error {
tags := t.buildTags(lifecycle, additionalTags)
if tags == nil || len(tags) == 0 {
return nil
}
var awsTags []*ec2.Tag
for k, v := range tags {
tag := &ec2.Tag{
Key: aws.String(k),
Value: aws.String(v),
}
awsTags = append(awsTags, tag)
}
backoff := wait.Backoff{
Duration: createTagInitialDelay,
Factor: createTagFactor,
Steps: createTagSteps,
}
request := &ec2.CreateTagsInput{}
request.Resources = []*string{&resourceID}
request.Tags = awsTags
var lastErr error
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
_, err := client.CreateTags(request)
if err == nil {
return true, nil
}
// We could check that the error is retryable, but the error code changes based on what we are tagging
// SecurityGroup: InvalidGroup.NotFound
klog.V(2).Infof("Failed to create tags; will retry. Error was %q", err)
lastErr = err
return false, nil
})
if err == wait.ErrWaitTimeout {
// return real CreateTags error instead of timeout
err = lastErr
}
return err
}
// Add additional filters, to match on our tags
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addFilters(filters []*ec2.Filter) []*ec2.Filter {
// if there are no clusterID configured - no filtering by special tag names
// should be applied to revert to legacy behaviour.
if len(t.ClusterID) == 0 {
if len(filters) == 0 {
// We can't pass a zero-length Filters to AWS (it's an error)
// So if we end up with no filters; just return nil
return nil
}
return filters
}
f := newEc2Filter("tag-key", t.clusterTagKey())
filters = append(filters, f)
return filters
}
// Add additional filters, to match on our tags. This uses the tag for legacy
// 1.5 -> 1.6 clusters and exists for backwards compatibility
//
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addLegacyFilters(filters []*ec2.Filter) []*ec2.Filter {
// if there are no clusterID configured - no filtering by special tag names
// should be applied to revert to legacy behaviour.
if len(t.ClusterID) == 0 {
if len(filters) == 0 {
// We can't pass a zero-length Filters to AWS (it's an error)
// So if we end up with no filters; just return nil
return nil
}
return filters
}
f := newEc2Filter(fmt.Sprintf("tag:%s", TagNameKubernetesClusterLegacy), t.ClusterID)
// We can't pass a zero-length Filters to AWS (it's an error)
// So if we end up with no filters; we need to return nil
filters = append(filters, f)
return filters
}
func (t *awsTagging) buildTags(lifecycle ResourceLifecycle, additionalTags map[string]string) map[string]string {
tags := make(map[string]string)
for k, v := range additionalTags {
tags[k] = v
}
// no clusterID is a sign of misconfigured cluster, but we can't be tagging the resources with empty
// strings
if len(t.ClusterID) == 0 {
return tags
}
// We only create legacy tags if we are using legacy tags, i.e. if we have seen a legacy tag on our instance
if t.usesLegacyTags {
tags[TagNameKubernetesClusterLegacy] = t.ClusterID
}
tags[t.clusterTagKey()] = string(lifecycle)
return tags
}
func (t *awsTagging) clusterID() string {
return t.ClusterID
}

235
staging/src/k8s.io/legacy-cloud-providers/aws/tags_test.go
View File

@@ -1,235 +0,0 @@
//go:build !nolegacyroviders
// +build !nolegacyroviders
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/stretchr/testify/assert"
)
func TestFilterTags(t *testing.T) {
awsServices := NewFakeAWSServices(TestClusterID)
c, err := newAWSCloud(CloudConfig{}, awsServices)
if err != nil {
t.Errorf("Error building aws cloud: %v", err)
return
}
if c.tagging.ClusterID != TestClusterID {
t.Errorf("unexpected ClusterID: %v", c.tagging.ClusterID)
}
}
func TestFindClusterID(t *testing.T) {
grid := []struct {
Tags map[string]string
ExpectedNew string
ExpectedLegacy string
ExpectError bool
}{
{
Tags: map[string]string{},
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: "a",
},
ExpectedLegacy: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + "a": "owned",
},
ExpectedNew: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + "a": "shared",
},
ExpectedNew: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + "a": "",
},
ExpectedNew: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: "a",
TagNameKubernetesClusterPrefix + "a": "",
},
ExpectedLegacy: "a",
ExpectedNew: "a",
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + "a": "",
TagNameKubernetesClusterPrefix + "b": "",
},
ExpectError: true,
},
}
for _, g := range grid {
var ec2Tags []*ec2.Tag
for k, v := range g.Tags {
ec2Tags = append(ec2Tags, &ec2.Tag{Key: aws.String(k), Value: aws.String(v)})
}
actualLegacy, actualNew, err := findClusterIDs(ec2Tags)
if g.ExpectError {
if err == nil {
t.Errorf("expected error for tags %v", g.Tags)
continue
}
} else {
if err != nil {
t.Errorf("unexpected error for tags %v: %v", g.Tags, err)
continue
}
if g.ExpectedNew != actualNew {
t.Errorf("unexpected new clusterid for tags %v: %s vs %s", g.Tags, g.ExpectedNew, actualNew)
continue
}
if g.ExpectedLegacy != actualLegacy {
t.Errorf("unexpected new clusterid for tags %v: %s vs %s", g.Tags, g.ExpectedLegacy, actualLegacy)
continue
}
}
}
}
func TestHasClusterTag(t *testing.T) {
awsServices := NewFakeAWSServices(TestClusterID)
c, err := newAWSCloud(CloudConfig{}, awsServices)
if err != nil {
t.Errorf("Error building aws cloud: %v", err)
return
}
grid := []struct {
Tags map[string]string
Expected bool
}{
{
Tags: map[string]string{},
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: TestClusterID,
},
Expected: true,
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: "a",
},
Expected: false,
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + TestClusterID: "owned",
},
Expected: true,
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + TestClusterID: "",
},
Expected: true,
},
{
Tags: map[string]string{
TagNameKubernetesClusterLegacy: "a",
TagNameKubernetesClusterPrefix + TestClusterID: "shared",
},
Expected: true,
},
{
Tags: map[string]string{
TagNameKubernetesClusterPrefix + TestClusterID: "shared",
TagNameKubernetesClusterPrefix + "b": "shared",
},
Expected: true,
},
}
for _, g := range grid {
var ec2Tags []*ec2.Tag
for k, v := range g.Tags {
ec2Tags = append(ec2Tags, &ec2.Tag{Key: aws.String(k), Value: aws.String(v)})
}
result := c.tagging.hasClusterTag(ec2Tags)
if result != g.Expected {
t.Errorf("Unexpected result for tags %v: %t", g.Tags, result)
}
}
}
func TestHasNoClusterPrefixTag(t *testing.T) {
awsServices := NewFakeAWSServices(TestClusterID)
c, err := newAWSCloud(CloudConfig{}, awsServices)
if err != nil {
t.Errorf("Error building aws cloud: %v", err)
return
}
tests := []struct {
name string
tags []*ec2.Tag
want bool
}{
{
name: "no tags",
want: true,
},
{
name: "no cluster tags",
tags: []*ec2.Tag{
{
Key: aws.String("not a cluster tag"),
Value: aws.String("true"),
},
},
want: true,
},
{
name: "contains cluster tags",
tags: []*ec2.Tag{
{
Key: aws.String("tag1"),
Value: aws.String("value1"),
},
{
Key: aws.String("kubernetes.io/cluster/test.cluster"),
Value: aws.String("owned"),
},
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, c.tagging.hasNoClusterPrefixTag(tt.tags))
})
}
}

120
staging/src/k8s.io/legacy-cloud-providers/aws/volumes.go
View File

@@ -1,120 +0,0 @@
//go:build !providerless
// +build !providerless
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
csimigration "k8s.io/csi-translation-lib/plugins"
"k8s.io/klog/v2"
"k8s.io/apimachinery/pkg/types"
)
// EBSVolumeID represents the ID of the volume in the AWS API, e.g.
// vol-12345678 The "traditional" format is "vol-12345678" A new longer format
// is also being introduced: "vol-12345678abcdef01" We should not assume
// anything about the length or format, though it seems reasonable to assume
// that volumes will continue to start with "vol-".
type EBSVolumeID string
func (i EBSVolumeID) awsString() *string {
return aws.String(string(i))
}
// KubernetesVolumeID represents the id for a volume in the kubernetes API;
// a few forms are recognized:
// - aws://<zone>/<awsVolumeId>
// - aws:///<awsVolumeId>
// - <awsVolumeId>
type KubernetesVolumeID string
// DiskInfo returns aws disk information in easy to use manner
type diskInfo struct {
ec2Instance *ec2.Instance
nodeName types.NodeName
volumeState string
attachmentState string
hasAttachment bool
disk *awsDisk
}
// MapToAWSVolumeID extracts the EBSVolumeID from the KubernetesVolumeID
func (name KubernetesVolumeID) MapToAWSVolumeID() (EBSVolumeID, error) {
awsID, err := csimigration.KubernetesVolumeIDToEBSVolumeID(string(name))
if err != nil {
return "", err
}
return EBSVolumeID(awsID), nil
}
// GetAWSVolumeID converts a Kubernetes volume ID to an AWS volume ID
func GetAWSVolumeID(kubeVolumeID string) (string, error) {
kid := KubernetesVolumeID(kubeVolumeID)
awsID, err := kid.MapToAWSVolumeID()
return string(awsID), err
}
func (c *Cloud) checkIfAttachedToNode(diskName KubernetesVolumeID, nodeName types.NodeName) (*diskInfo, bool, error) {
disk, err := newAWSDisk(c, diskName)
if err != nil {
return nil, true, err
}
awsDiskInfo := &diskInfo{
disk: disk,
}
info, err := disk.describeVolume()
if err != nil {
klog.Warningf("Error describing volume %s with %v", diskName, err)
awsDiskInfo.volumeState = "unknown"
return awsDiskInfo, false, err
}
awsDiskInfo.volumeState = aws.StringValue(info.State)
if len(info.Attachments) > 0 {
attachment := info.Attachments[0]
awsDiskInfo.attachmentState = aws.StringValue(attachment.State)
instanceID := aws.StringValue(attachment.InstanceId)
instanceInfo, err := c.getInstanceByID(instanceID)
// This should never happen but if it does it could mean there was a race and instance
// has been deleted
if err != nil {
fetchErr := fmt.Errorf("error fetching instance %s for volume %s", instanceID, diskName)
klog.Warning(fetchErr)
return awsDiskInfo, false, fetchErr
}
awsDiskInfo.ec2Instance = instanceInfo
awsDiskInfo.nodeName = mapInstanceToNodeName(instanceInfo)
awsDiskInfo.hasAttachment = true
if awsDiskInfo.nodeName == nodeName {
return awsDiskInfo, true, nil
}
}
return awsDiskInfo, false, nil
}

5
staging/src/k8s.io/legacy-cloud-providers/go.mod
View File

@@ -11,7 +11,6 @@ require (
github.com/Azure/go-autorest/autorest/adal v0.9.20
github.com/Azure/go-autorest/autorest/mocks v0.4.2
github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b
github.com/aws/aws-sdk-go v1.44.147
github.com/golang/mock v1.6.0
github.com/google/go-cmp v0.5.9
github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021
@@ -26,7 +25,6 @@ require (
k8s.io/client-go v0.0.0
k8s.io/cloud-provider v0.0.0
k8s.io/component-base v0.0.0
k8s.io/csi-translation-lib v0.0.0
k8s.io/klog/v2 v2.90.1
k8s.io/utils v0.0.0-20230209194617-a36077c30491
sigs.k8s.io/yaml v1.3.0
@@ -59,7 +57,6 @@ require (
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
@@ -74,7 +71,6 @@ require (
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sys v0.5.0 // indirect
@@ -103,7 +99,6 @@ replace (
k8s.io/component-base => ../component-base
k8s.io/component-helpers => ../component-helpers
k8s.io/controller-manager => ../controller-manager
k8s.io/csi-translation-lib => ../csi-translation-lib
k8s.io/kms => ../kms
k8s.io/legacy-cloud-providers => ../legacy-cloud-providers
)

18
staging/src/k8s.io/legacy-cloud-providers/go.sum generated
View File

@@ -77,8 +77,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/aws/aws-sdk-go v1.44.147 h1:C/YQv0QAvRHio4cESBTFGh8aI/JM9VdRislDIOz/Dx4=
github.com/aws/aws-sdk-go v1.44.147/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -242,10 +240,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -337,7 +331,6 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -356,7 +349,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -411,7 +403,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -453,8 +444,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -486,7 +475,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -544,14 +532,10 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -563,7 +547,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -624,7 +607,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=