github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add InstanceExists* methods to cloud provider interface for CCM

Browse Source
This commit is contained in:
Josh Horwitz
2017-08-21 14:55:43 -04:00
committed by Josh Horwitz
parent 73a6ee1dcc
commit 2f1ea47c83
13 changed files with 235 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
pkg/controller/cloud/node_controller.go
View File

@@ -237,26 +237,36 @@ func (cnc *CloudNodeController) MonitorNode() {
if currentReadyCondition.Status != v1.ConditionTrue {
// Check with the cloud provider to see if the node still exists. If it
// doesn't, delete the node immediately.
if _, err := instances.ExternalID(types.NodeName(node.Name)); err != nil {
if err == cloudprovider.InstanceNotFound {
glog.V(2).Infof("Deleting node no longer present in cloud provider: %s", node.Name)
ref := &v1.ObjectReference{
Kind: "Node",
Name: node.Name,
UID: types.UID(node.UID),
Namespace: "",
}
glog.V(2).Infof("Recording %s event message for node %s", "DeletingNode", node.Name)
cnc.recorder.Eventf(ref, v1.EventTypeNormal, fmt.Sprintf("Deleting Node %v because it's not present according to cloud provider", node.Name), "Node %s event: %s", node.Name, "DeletingNode")
go func(nodeName string) {
defer utilruntime.HandleCrash()
if err := cnc.kubeClient.CoreV1().Nodes().Delete(node.Name, nil); err != nil {
glog.Errorf("unable to delete node %q: %v", node.Name, err)
}
}(node.Name)
}
glog.Errorf("Error getting node data from cloud: %v", err)
exists, err := ensureNodeExistsByProviderIDOrName(instances, node)
if err != nil {
glog.Errorf("Error getting data for node %s from cloud: %v", node.Name, err)
continue
}
if exists {
// Continue checking the remaining nodes since the current one is fine.
continue
}
glog.V(2).Infof("Deleting node since it is no longer present in cloud provider: %s", node.Name)
ref := &v1.ObjectReference{
Kind: "Node",
Name: node.Name,
UID: types.UID(node.UID),
Namespace: "",
}
glog.V(2).Infof("Recording %s event message for node %s", "DeletingNode", node.Name)
cnc.recorder.Eventf(ref, v1.EventTypeNormal, fmt.Sprintf("Deleting Node %v because it's not present according to cloud provider", node.Name), "Node %s event: %s", node.Name, "DeletingNode")
go func(nodeName string) {
defer utilruntime.HandleCrash()
if err := cnc.kubeClient.CoreV1().Nodes().Delete(nodeName, nil); err != nil {
glog.Errorf("unable to delete node %q: %v", nodeName, err)
}
}(node.Name)
}
}
}
@@ -372,6 +382,25 @@ func excludeTaintFromList(taints []v1.Taint, toExclude v1.Taint) []v1.Taint {
return newTaints
}
// ensureNodeExistsByProviderIDOrName first checks if the instance exists by the provider id and then by node name
func ensureNodeExistsByProviderIDOrName(instances cloudprovider.Instances, node *v1.Node) (bool, error) {
exists, err := instances.InstanceExistsByProviderID(node.Spec.ProviderID)
if err != nil {
providerIDErr := err
_, err = instances.ExternalID(types.NodeName(node.Name))
if err == nil {
return true, nil
}
if err == cloudprovider.InstanceNotFound {
return false, nil
}
return false, fmt.Errorf("InstanceExistsByProviderID: Error fetching by providerID: %v Error fetching by NodeName: %v", providerIDErr, err)
}
return exists, nil
}
func getNodeAddressesByProviderIDOrName(instances cloudprovider.Instances, node *v1.Node) ([]v1.NodeAddress, error) {
nodeAddresses, err := instances.NodeAddressesByProviderID(node.Spec.ProviderID)
if err != nil {

130
pkg/controller/cloud/node_controller_test.go
View File

@@ -17,6 +17,8 @@ limitations under the License.
package cloud
import (
"errors"
"reflect"
"testing"
"time"
@@ -39,6 +41,119 @@ import (
"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
)
func TestEnsureNodeExistsByProviderIDOrNodeName(t *testing.T) {
testCases := []struct {
testName string
node *v1.Node
expectedCalls []string
existsByNodeName bool
existsByProviderID bool
nodeNameErr error
providerIDErr error
}{
{
testName: "node exists by provider id",
existsByProviderID: true,
providerIDErr: nil,
existsByNodeName: false,
nodeNameErr: errors.New("unimplemented"),
expectedCalls: []string{"instance-exists-by-provider-id"},
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node0",
},
Spec: v1.NodeSpec{
ProviderID: "node0",
},
},
},
{
testName: "does not exist by provider id",
existsByProviderID: false,
providerIDErr: nil,
existsByNodeName: false,
nodeNameErr: errors.New("unimplemented"),
expectedCalls: []string{"instance-exists-by-provider-id"},
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node0",
},
Spec: v1.NodeSpec{
ProviderID: "node0",
},
},
},
{
testName: "node exists by node name",
existsByProviderID: false,
providerIDErr: errors.New("unimplemented"),
existsByNodeName: true,
nodeNameErr: nil,
expectedCalls: []string{"instance-exists-by-provider-id", "external-id"},
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node0",
},
Spec: v1.NodeSpec{
ProviderID: "node0",
},
},
},
{
testName: "does not exist by node name",
existsByProviderID: false,
providerIDErr: errors.New("unimplemented"),
existsByNodeName: false,
nodeNameErr: cloudprovider.InstanceNotFound,
expectedCalls: []string{"instance-exists-by-provider-id", "external-id"},
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node0",
},
Spec: v1.NodeSpec{
ProviderID: "node0",
},
},
},
}
for _, tc := range testCases {
t.Run(tc.testName, func(t *testing.T) {
fc := &fakecloud.FakeCloud{
Exists: tc.existsByNodeName,
ExistsByProviderID: tc.existsByProviderID,
Err: tc.nodeNameErr,
ErrByProviderID: tc.providerIDErr,
}
instances, _ := fc.Instances()
exists, err := ensureNodeExistsByProviderIDOrName(instances, tc.node)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(fc.Calls, tc.expectedCalls) {
t.Fatalf("expected cloud provider methods `%v` to be called but `%v` was called ", tc.expectedCalls, fc.Calls)
}
if tc.existsByProviderID && tc.existsByProviderID != exists {
t.Fatalf("expected exist by provider id to be `%t` but got `%t`", tc.existsByProviderID, exists)
}
if tc.existsByNodeName && tc.existsByNodeName != exists {
t.Fatalf("expected exist by node name to be `%t` but got `%t`", tc.existsByNodeName, exists)
}
if !tc.existsByNodeName && !tc.existsByProviderID && exists {
t.Fatal("node is not supposed to exist")
}
})
}
}
// This test checks that the node is deleted when kubelet stops reporting
// and cloud provider says node is gone
func TestNodeDeleted(t *testing.T) {
@@ -105,9 +220,12 @@ func TestNodeDeleted(t *testing.T) {
eventBroadcaster := record.NewBroadcaster()
cloudNodeController := &CloudNodeController{
kubeClient: fnh,
nodeInformer: factory.Core().V1().Nodes(),
cloud: &fakecloud.FakeCloud{Err: cloudprovider.InstanceNotFound},
kubeClient: fnh,
nodeInformer: factory.Core().V1().Nodes(),
cloud: &fakecloud.FakeCloud{
ExistsByProviderID: false,
Err: nil,
},
nodeMonitorPeriod: 1 * time.Second,
recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-controller-manager"}),
nodeStatusUpdateFrequency: 1 * time.Second,
@@ -520,7 +638,8 @@ func TestNodeAddresses(t *testing.T) {
FailureDomain: "us-west-1a",
Region: "us-west",
},
Err: nil,
ExistsByProviderID: true,
Err: nil,
}
eventBroadcaster := record.NewBroadcaster()
@@ -639,7 +758,8 @@ func TestNodeProvidedIPAddresses(t *testing.T) {
FailureDomain: "us-west-1a",
Region: "us-west",
},
Err: nil,
ExistsByProviderID: true,
Err: nil,
}
eventBroadcaster := record.NewBroadcaster()