github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #83098 from ddebroy/disable-intree

Browse Source 
CSI Migration phase 2: disable probing of in-tree plugins
This commit is contained in:
Kubernetes Prow Robot
2019-11-14 20:51:42 -08:00
committed by GitHub
parent cb2684c416 129f15328b
commit 372ebd24f5
75 changed files with 1220 additions and 851 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/kube-controller-manager/app/BUILD
View File

@@ -92,6 +92,7 @@ go_library(
"//pkg/volume/azure_file:go_default_library",
"//pkg/volume/cinder:go_default_library",
"//pkg/volume/csi:go_default_library",
"//pkg/volume/csimigration:go_default_library",
"//pkg/volume/fc:go_default_library",
"//pkg/volume/flexvolume:go_default_library",
"//pkg/volume/flocker:go_default_library",
@@ -140,10 +141,12 @@ go_library(
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
"//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
"//staging/src/k8s.io/component-base/metrics/prometheus/ratelimiter:go_default_library",
"//staging/src/k8s.io/component-base/version:go_default_library",
"//staging/src/k8s.io/component-base/version/verflag:go_default_library",
"//staging/src/k8s.io/csi-translation-lib:go_default_library",
"//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library",
"//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1:go_default_library",
"//staging/src/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
"//staging/src/k8s.io/metrics/pkg/client/external_metrics:go_default_library",

26
cmd/kube-controller-manager/app/core.go
View File

@@ -65,6 +65,7 @@ import (
kubefeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/quota/v1/generic"
quotainstall "k8s.io/kubernetes/pkg/quota/v1/install"
"k8s.io/kubernetes/pkg/volume/csimigration"
netutils "k8s.io/utils/net"
)
@@ -254,10 +255,14 @@ func startRouteController(ctx ControllerContext) (http.Handler, bool, error) {
}
func startPersistentVolumeBinderController(ctx ControllerContext) (http.Handler, bool, error) {
plugins, err := ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration)
if err != nil {
return nil, true, fmt.Errorf("failed to probe volume plugins when starting persistentvolume controller: %v", err)
}
params := persistentvolumecontroller.ControllerParameters{
KubeClient: ctx.ClientBuilder.ClientOrDie("persistent-volume-binder"),
SyncPeriod: ctx.ComponentConfig.PersistentVolumeBinderController.PVClaimBinderSyncPeriod.Duration,
VolumePlugins: ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
VolumePlugins: plugins,
Cloud: ctx.Cloud,
ClusterName: ctx.ComponentConfig.KubeCloudShared.ClusterName,
VolumeInformer: ctx.InformerFactory.Core().V1().PersistentVolumes(),
@@ -291,6 +296,11 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
csiDriverInformer = ctx.InformerFactory.Storage().V1beta1().CSIDrivers()
}
plugins, err := ProbeAttachableVolumePlugins()
if err != nil {
return nil, true, fmt.Errorf("failed to probe volume plugins when starting attach/detach controller: %v", err)
}
attachDetachController, attachDetachControllerErr :=
attachdetach.NewAttachDetachController(
ctx.ClientBuilder.ClientOrDie("attachdetach-controller"),
@@ -301,7 +311,7 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
csiNodeInformer,
csiDriverInformer,
ctx.Cloud,
ProbeAttachableVolumePlugins(),
plugins,
GetDynamicPluginProber(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
ctx.ComponentConfig.AttachDetachController.DisableAttachDetachReconcilerSync,
ctx.ComponentConfig.AttachDetachController.ReconcilerSyncLoopPeriod.Duration,
@@ -316,17 +326,23 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, error) {
if utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
plugins, err := ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration)
if err != nil {
return nil, true, fmt.Errorf("failed to probe volume plugins when starting volume expand controller: %v", err)
}
csiTranslator := csitrans.New()
expandController, expandControllerErr := expand.NewExpandController(
ctx.ClientBuilder.ClientOrDie("expand-controller"),
ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
ctx.InformerFactory.Core().V1().PersistentVolumes(),
ctx.InformerFactory.Storage().V1().StorageClasses(),
ctx.Cloud,
ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
csitrans.New())
plugins,
csiTranslator,
csimigration.NewPluginManager(csiTranslator))
if expandControllerErr != nil {
return nil, true, fmt.Errorf("failed to start volume expand controller : %v", expandControllerErr)
return nil, true, fmt.Errorf("failed to start volume expand controller: %v", expandControllerErr)
}
go expandController.Run(ctx.Stop)
return nil, true, nil

32
cmd/kube-controller-manager/app/plugins.go
View File

@@ -57,10 +57,13 @@ import (
// detach controller.
// The list of plugins is manually compiled. This code and the plugin
// initialization code for kubelet really, really need a through refactor.
func ProbeAttachableVolumePlugins() []volume.VolumePlugin {
func ProbeAttachableVolumePlugins() ([]volume.VolumePlugin, error) {
var err error
allPlugins := []volume.VolumePlugin{}
allPlugins = appendAttachableLegacyProviderVolumes(allPlugins)
allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
if err != nil {
return allPlugins, err
}
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
@@ -68,7 +71,7 @@ func ProbeAttachableVolumePlugins() []volume.VolumePlugin {
allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
return allPlugins
return allPlugins, nil
}
// GetDynamicPluginProber gets the probers of dynamically discoverable plugins
@@ -79,23 +82,26 @@ func GetDynamicPluginProber(config persistentvolumeconfig.VolumeConfiguration) v
}
// ProbeExpandableVolumePlugins returns volume plugins which are expandable
func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) []volume.VolumePlugin {
func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) {
var err error
allPlugins := []volume.VolumePlugin{}
allPlugins = appendExpandableLegacyProviderVolumes(allPlugins)
allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
if err != nil {
return allPlugins, err
}
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
return allPlugins
return allPlugins, nil
}
// ProbeControllerVolumePlugins collects all persistent volume plugins into an
// easy to use list. Only volume plugins that implement any of
// provisioner/recycler/deleter interface should be returned.
func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persistentvolumeconfig.VolumeConfiguration) []volume.VolumePlugin {
func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) {
allPlugins := []volume.VolumePlugin{}
// The list of plugins to probe is decided by this binary, not
@@ -131,7 +137,11 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persiste
// add rbd provisioner
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
allPlugins = appendLegacyProviderVolumes(allPlugins)
var err error
allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
if err != nil {
return allPlugins, err
}
allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
@@ -143,7 +153,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persiste
allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
}
return allPlugins
return allPlugins, nil
}
// AttemptToLoadRecycler tries decoding a pod from a filepath for use as a recycler for a volume.

82
cmd/kube-controller-manager/app/plugins_providers.go
View File

@@ -19,34 +19,84 @@ limitations under the License.
package app
import (
"k8s.io/component-base/featuregate"
"k8s.io/csi-translation-lib/plugins"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/awsebs"
"k8s.io/kubernetes/pkg/volume/azure_dd"
"k8s.io/kubernetes/pkg/volume/azure_file"
"k8s.io/kubernetes/pkg/volume/cinder"
"k8s.io/kubernetes/pkg/volume/csimigration"
"k8s.io/kubernetes/pkg/volume/gcepd"
"k8s.io/kubernetes/pkg/volume/vsphere_volume"
)
func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin {
allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
return allPlugins
type probeFn func() []volume.VolumePlugin
func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) {
// Skip appending the in-tree plugin to the list of plugins to be probed/initialized
// if the CSIMigration feature flag and plugin specific feature flag indicating
// CSI migration is complete
err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete)
if err != nil {
klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err)
// TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly
}
if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) {
klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete)
return plugins, nil
}
plugins = append(plugins, fn()...)
return plugins, nil
}
func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin {
return appendLegacyProviderVolumes(allPlugins)
type pluginInfo struct {
pluginMigrationFeature featuregate.Feature
pluginMigrationCompleteFeature featuregate.Feature
pluginProbeFunction probeFn
}
func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin {
allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...)
func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
pluginMigrationStatus := make(map[string]pluginInfo)
pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins}
pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azure_dd.ProbeVolumePlugins}
var err error
for pluginName, pluginInfo := range pluginMigrationStatus {
allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction)
if err != nil {
return allPlugins, err
}
}
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
return allPlugins
return allPlugins, nil
}
func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
return appendLegacyProviderVolumes(allPlugins, featureGate)
}
func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
pluginMigrationStatus := make(map[string]pluginInfo)
pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins}
pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azure_dd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins}
var err error
for pluginName, pluginInfo := range pluginMigrationStatus {
allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction)
if err != nil {
return allPlugins, err
}
}
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
return allPlugins, nil
}