From 5b97b2860d9b490eba3325bb1e3ae0a748222d9d Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Mon, 19 Nov 2018 17:17:45 -0500 Subject: [PATCH 1/6] Change fsGroup to mounterArgs in volume.SetUp() to allow for future extension. --- pkg/kubelet/kubelet_volumes_test.go | 4 +-- pkg/volume/awsebs/aws_ebs.go | 8 ++--- pkg/volume/awsebs/aws_ebs_test.go | 4 +-- pkg/volume/azure_dd/azure_mounter.go | 8 ++--- pkg/volume/azure_file/azure_file.go | 8 ++--- pkg/volume/azure_file/azure_file_test.go | 2 +- pkg/volume/cephfs/cephfs.go | 6 ++-- pkg/volume/cephfs/cephfs_test.go | 2 +- pkg/volume/cinder/cinder.go | 8 ++--- pkg/volume/cinder/cinder_test.go | 2 +- pkg/volume/configmap/configmap.go | 12 ++++---- pkg/volume/configmap/configmap_test.go | 30 ++++++++++++------- pkg/volume/csi/csi_mounter.go | 8 ++--- pkg/volume/csi/csi_mounter_test.go | 12 +++++--- pkg/volume/csi/csi_test.go | 4 ++- pkg/volume/downwardapi/downwardapi.go | 12 ++++---- pkg/volume/downwardapi/downwardapi_test.go | 4 +-- pkg/volume/emptydir/empty_dir.go | 8 ++--- pkg/volume/emptydir/empty_dir_test.go | 2 +- pkg/volume/fc/fc.go | 8 ++--- pkg/volume/fc/fc_test.go | 2 +- pkg/volume/flexvolume/driver-call.go | 2 +- pkg/volume/flexvolume/mounter-defaults.go | 2 +- pkg/volume/flexvolume/mounter.go | 14 ++++----- pkg/volume/flexvolume/mounter_test.go | 13 ++++---- pkg/volume/flocker/flocker.go | 8 ++--- pkg/volume/gcepd/gce_pd.go | 8 ++--- pkg/volume/gcepd/gce_pd_test.go | 4 +-- pkg/volume/git_repo/git_repo.go | 10 +++---- pkg/volume/git_repo/git_repo_test.go | 2 +- pkg/volume/glusterfs/glusterfs.go | 6 ++-- pkg/volume/glusterfs/glusterfs_test.go | 2 +- pkg/volume/hostpath/host_path.go | 4 +-- pkg/volume/hostpath/host_path_test.go | 4 +-- pkg/volume/iscsi/iscsi.go | 8 ++--- pkg/volume/iscsi/iscsi_test.go | 2 +- pkg/volume/local/local.go | 12 ++++---- pkg/volume/local/local_test.go | 10 ++++--- pkg/volume/nfs/nfs.go | 6 ++-- pkg/volume/nfs/nfs_test.go | 2 +- pkg/volume/photon_pd/photon_pd.go | 6 ++-- pkg/volume/photon_pd/photon_pd_test.go | 2 +- pkg/volume/portworx/portworx.go | 8 ++--- pkg/volume/portworx/portworx_test.go | 2 +- pkg/volume/projected/projected.go | 12 ++++---- pkg/volume/projected/projected_test.go | 11 +++---- pkg/volume/quobyte/quobyte.go | 6 ++-- pkg/volume/quobyte/quobyte_test.go | 2 +- pkg/volume/rbd/rbd.go | 8 ++--- pkg/volume/rbd/rbd_test.go | 2 +- pkg/volume/scaleio/sio_volume.go | 10 +++---- pkg/volume/scaleio/sio_volume_test.go | 4 +-- pkg/volume/secret/secret.go | 12 ++++---- pkg/volume/secret/secret_test.go | 11 +++---- pkg/volume/storageos/storageos.go | 8 ++--- pkg/volume/storageos/storageos_test.go | 2 +- pkg/volume/testing/testing.go | 6 ++-- .../operationexecutor/operation_generator.go | 5 +++- pkg/volume/volume.go | 10 +++++-- pkg/volume/vsphere_volume/vsphere_volume.go | 8 ++--- .../vsphere_volume/vsphere_volume_test.go | 2 +- 61 files changed, 221 insertions(+), 189 deletions(-) diff --git a/pkg/kubelet/kubelet_volumes_test.go b/pkg/kubelet/kubelet_volumes_test.go index a82aa061632..6678118d3fb 100644 --- a/pkg/kubelet/kubelet_volumes_test.go +++ b/pkg/kubelet/kubelet_volumes_test.go @@ -439,11 +439,11 @@ func (f *stubVolume) CanMount() error { return nil } -func (f *stubVolume) SetUp(fsGroup *int64) error { +func (f *stubVolume) SetUp(mounterArgs volume.MounterArgs) error { return nil } -func (f *stubVolume) SetUpAt(dir string, fsGroup *int64) error { +func (f *stubVolume) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { return nil } diff --git a/pkg/volume/awsebs/aws_ebs.go b/pkg/volume/awsebs/aws_ebs.go index 9532d96ed70..fb0b3ebf6c7 100644 --- a/pkg/volume/awsebs/aws_ebs.go +++ b/pkg/volume/awsebs/aws_ebs.go @@ -382,12 +382,12 @@ func (b *awsElasticBlockStoreMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (b *awsElasticBlockStoreMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *awsElasticBlockStoreMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } // SetUpAt attaches the disk and bind mounts to the volume path. -func (b *awsElasticBlockStoreMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *awsElasticBlockStoreMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { // TODO: handle failed mounts here. notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) klog.V(4).Infof("PersistentDisk set up: %s %v %v", dir, !notMnt, err) @@ -440,7 +440,7 @@ func (b *awsElasticBlockStoreMounter) SetUpAt(dir string, fsGroup *int64) error } if !b.readOnly { - volume.SetVolumeOwnership(b, fsGroup) + volume.SetVolumeOwnership(b, mounterArgs.FsGroup) } klog.V(4).Infof("Successfully mounted %s", dir) diff --git a/pkg/volume/awsebs/aws_ebs_test.go b/pkg/volume/awsebs/aws_ebs_test.go index 58716757042..b418479cf1b 100644 --- a/pkg/volume/awsebs/aws_ebs_test.go +++ b/pkg/volume/awsebs/aws_ebs_test.go @@ -136,7 +136,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { @@ -367,7 +367,7 @@ func TestMountOptions(t *testing.T) { t.Errorf("Got a nil Mounter") } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } mountOptions := fakeMounter.MountPoints[0].Opts diff --git a/pkg/volume/azure_dd/azure_mounter.go b/pkg/volume/azure_dd/azure_mounter.go index a47d709b0ca..dd4ff36486b 100644 --- a/pkg/volume/azure_dd/azure_mounter.go +++ b/pkg/volume/azure_dd/azure_mounter.go @@ -62,15 +62,15 @@ func (m *azureDiskMounter) CanMount() error { return nil } -func (m *azureDiskMounter) SetUp(fsGroup *int64) error { - return m.SetUpAt(m.GetPath(), fsGroup) +func (m *azureDiskMounter) SetUp(mounterArgs volume.MounterArgs) error { + return m.SetUpAt(m.GetPath(), mounterArgs) } func (m *azureDiskMounter) GetPath() string { return getPath(m.dataDisk.podUID, m.dataDisk.volumeName, m.plugin.host) } -func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { +func (m *azureDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { mounter := m.plugin.host.GetMounter(m.plugin.GetPluginName()) volumeSource, _, err := getVolumeSource(m.spec) @@ -161,7 +161,7 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { } if volumeSource.ReadOnly == nil || !*volumeSource.ReadOnly { - volume.SetVolumeOwnership(m, fsGroup) + volume.SetVolumeOwnership(m, mounterArgs.FsGroup) } klog.V(2).Infof("azureDisk - successfully mounted disk %s on %s", diskName, dir) diff --git a/pkg/volume/azure_file/azure_file.go b/pkg/volume/azure_file/azure_file.go index 5db49ba2e82..68755ac37bd 100644 --- a/pkg/volume/azure_file/azure_file.go +++ b/pkg/volume/azure_file/azure_file.go @@ -236,11 +236,11 @@ func (b *azureFileMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (b *azureFileMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *azureFileMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } -func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *azureFileMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) klog.V(4).Infof("AzureFile mount set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { @@ -283,7 +283,7 @@ func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error { options = append(options, "ro") } mountOptions = volutil.JoinMountOptions(b.mountOptions, options) - mountOptions = appendDefaultMountOptions(mountOptions, fsGroup) + mountOptions = appendDefaultMountOptions(mountOptions, mounterArgs.FsGroup) } err = b.mounter.Mount(source, dir, "cifs", mountOptions) diff --git a/pkg/volume/azure_file/azure_file_test.go b/pkg/volume/azure_file/azure_file_test.go index 1c6a35cdbad..7efa11c4884 100644 --- a/pkg/volume/azure_file/azure_file_test.go +++ b/pkg/volume/azure_file/azure_file_test.go @@ -155,7 +155,7 @@ func testPlugin(t *testing.T, tmpDir string, volumeHost volume.VolumeHost) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { diff --git a/pkg/volume/cephfs/cephfs.go b/pkg/volume/cephfs/cephfs.go index c45fc1ab149..59f394606ae 100644 --- a/pkg/volume/cephfs/cephfs.go +++ b/pkg/volume/cephfs/cephfs.go @@ -222,12 +222,12 @@ func (cephfsVolume *cephfsMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (cephfsVolume *cephfsMounter) SetUp(fsGroup *int64) error { - return cephfsVolume.SetUpAt(cephfsVolume.GetPath(), fsGroup) +func (cephfsVolume *cephfsMounter) SetUp(mounterArgs volume.MounterArgs) error { + return cephfsVolume.SetUpAt(cephfsVolume.GetPath(), mounterArgs) } // SetUpAt attaches the disk and bind mounts to the volume path. -func (cephfsVolume *cephfsMounter) SetUpAt(dir string, fsGroup *int64) error { +func (cephfsVolume *cephfsMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { notMnt, err := cephfsVolume.mounter.IsLikelyNotMountPoint(dir) klog.V(4).Infof("CephFS mount set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { diff --git a/pkg/volume/cephfs/cephfs_test.go b/pkg/volume/cephfs/cephfs_test.go index 45fb4cf74de..3f9058442a0 100644 --- a/pkg/volume/cephfs/cephfs_test.go +++ b/pkg/volume/cephfs/cephfs_test.go @@ -87,7 +87,7 @@ func TestPlugin(t *testing.T) { if volumePath != volpath { t.Errorf("Got unexpected path: %s", volumePath) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(volumePath); err != nil { diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index e456476ec40..9f1d8492be8 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -377,12 +377,12 @@ func (b *cinderVolumeMounter) CanMount() error { return nil } -func (b *cinderVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *cinderVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } // SetUp bind mounts to the volume path. -func (b *cinderVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *cinderVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(5).Infof("Cinder SetUp %s to %s", b.pdName, dir) b.plugin.volumeLocks.LockKey(b.pdName) @@ -442,7 +442,7 @@ func (b *cinderVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } if !b.readOnly { - volume.SetVolumeOwnership(b, fsGroup) + volume.SetVolumeOwnership(b, mounterArgs.FsGroup) } klog.V(3).Infof("Cinder volume %s mounted to %s", b.pdName, dir) diff --git a/pkg/volume/cinder/cinder_test.go b/pkg/volume/cinder/cinder_test.go index cf0efd7e58e..9dc710e4076 100644 --- a/pkg/volume/cinder/cinder_test.go +++ b/pkg/volume/cinder/cinder_test.go @@ -166,7 +166,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { diff --git a/pkg/volume/configmap/configmap.go b/pkg/volume/configmap/configmap.go index 60196dfee51..cdaa54ac014 100644 --- a/pkg/volume/configmap/configmap.go +++ b/pkg/volume/configmap/configmap.go @@ -183,11 +183,11 @@ func (b *configMapVolumeMounter) CanMount() error { return nil } -func (b *configMapVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *configMapVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } -func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *configMapVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir) // Wrap EmptyDir, let it do the setup. @@ -224,7 +224,7 @@ func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } setupSuccess := false - if err := wrapped.SetUpAt(dir, fsGroup); err != nil { + if err := wrapped.SetUpAt(dir, mounterArgs); err != nil { return err } if err := volumeutil.MakeNestedMountpoints(b.volName, dir, b.pod); err != nil { @@ -259,9 +259,9 @@ func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return err } - err = volume.SetVolumeOwnership(b, fsGroup) + err = volume.SetVolumeOwnership(b, mounterArgs.FsGroup) if err != nil { - klog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) + klog.Errorf("Error applying volume ownership settings for group: %v", mounterArgs.FsGroup) return err } setupSuccess = true diff --git a/pkg/volume/configmap/configmap_test.go b/pkg/volume/configmap/configmap_test.go index f1945a26873..596916d9f07 100644 --- a/pkg/volume/configmap/configmap_test.go +++ b/pkg/volume/configmap/configmap_test.go @@ -365,8 +365,10 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - fsGroup := int64(1001) - err = mounter.SetUp(&fsGroup) + var mounterArgs volume.MounterArgs + group := int64(1001) + mounterArgs.FsGroup = &group + err = mounter.SetUp(mounterArgs) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -423,8 +425,10 @@ func TestPluginReboot(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - fsGroup := int64(1001) - err = mounter.SetUp(&fsGroup) + var mounterArgs volume.MounterArgs + group := int64(1001) + mounterArgs.FsGroup = &group + err = mounter.SetUp(mounterArgs) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -485,8 +489,10 @@ func TestPluginOptional(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - fsGroup := int64(1001) - err = mounter.SetUp(&fsGroup) + var mounterArgs volume.MounterArgs + group := int64(1001) + mounterArgs.FsGroup = &group + err = mounter.SetUp(mounterArgs) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -582,8 +588,10 @@ func TestPluginKeysOptional(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - fsGroup := int64(1001) - err = mounter.SetUp(&fsGroup) + var mounterArgs volume.MounterArgs + group := int64(1001) + mounterArgs.FsGroup = &group + err = mounter.SetUp(mounterArgs) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -660,8 +668,10 @@ func TestInvalidConfigMapSetup(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - fsGroup := int64(1001) - err = mounter.SetUp(&fsGroup) + var mounterArgs volume.MounterArgs + group := int64(1001) + mounterArgs.FsGroup = &group + err = mounter.SetUp(mounterArgs) if err == nil { t.Errorf("Expected setup to fail") } diff --git a/pkg/volume/csi/csi_mounter.go b/pkg/volume/csi/csi_mounter.go index 20793fa82f5..1af3d0af5a3 100644 --- a/pkg/volume/csi/csi_mounter.go +++ b/pkg/volume/csi/csi_mounter.go @@ -93,11 +93,11 @@ func (c *csiMountMgr) CanMount() error { return nil } -func (c *csiMountMgr) SetUp(fsGroup *int64) error { - return c.SetUpAt(c.GetPath(), fsGroup) +func (c *csiMountMgr) SetUp(mounterArgs volume.MounterArgs) error { + return c.SetUpAt(c.GetPath(), mounterArgs) } -func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { +func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(4).Infof(log("Mounter.SetUpAt(%s)", dir)) mounted, err := isDirMounted(c.plugin, dir) @@ -268,7 +268,7 @@ func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { // if fstype is "", then skip fsgroup (could be indication of non-block filesystem) // if fstype is provided and pv.AccessMode == ReadWriteOnly, then apply fsgroup - err = c.applyFSGroup(fsType, fsGroup) + err = c.applyFSGroup(fsType, mounterArgs.FsGroup) if err != nil { // attempt to rollback mount. fsGrpErr := fmt.Errorf("applyFSGroup failed for vol %s: %v", c.volumeID, err) diff --git a/pkg/volume/csi/csi_mounter_test.go b/pkg/volume/csi/csi_mounter_test.go index 500b0be530b..82f75a8a679 100644 --- a/pkg/volume/csi/csi_mounter_test.go +++ b/pkg/volume/csi/csi_mounter_test.go @@ -203,8 +203,10 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) { } // Mounter.SetUp() + var mounterArgs volume.MounterArgs fsGroup := int64(2000) - if err := csiMounter.SetUp(&fsGroup); err != nil { + mounterArgs.FsGroup = &fsGroup + if err := csiMounter.SetUp(mounterArgs); err != nil { t.Fatalf("mounter.Setup failed: %v", err) } @@ -344,7 +346,7 @@ func TestMounterSetUpSimple(t *testing.T) { } // Mounter.SetUp() - if err := csiMounter.SetUp(nil); err != nil { + if err := csiMounter.SetUp(volume.MounterArgs{}); err != nil { t.Fatalf("mounter.Setup failed: %v", err) } @@ -475,7 +477,7 @@ func TestMounterSetUpWithInline(t *testing.T) { } // Mounter.SetUp() - if err := csiMounter.SetUp(nil); err != nil { + if err := csiMounter.SetUp(volume.MounterArgs{}); err != nil { t.Fatalf("mounter.Setup failed: %v", err) } @@ -621,12 +623,14 @@ func TestMounterSetUpWithFSGroup(t *testing.T) { } // Mounter.SetUp() + var mounterArgs volume.MounterArgs var fsGroupPtr *int64 if tc.setFsGroup { fsGroup := tc.fsGroup fsGroupPtr = &fsGroup } - if err := csiMounter.SetUp(fsGroupPtr); err != nil { + mounterArgs.FsGroup = fsGroupPtr + if err := csiMounter.SetUp(mounterArgs); err != nil { t.Fatalf("mounter.Setup failed: %v", err) } diff --git a/pkg/volume/csi/csi_test.go b/pkg/volume/csi/csi_test.go index b15ed703513..8dd147fa846 100644 --- a/pkg/volume/csi/csi_test.go +++ b/pkg/volume/csi/csi_test.go @@ -268,7 +268,9 @@ func TestCSI_VolumeAll(t *testing.T) { csiMounter := mounter.(*csiMountMgr) csiMounter.csiClient = csiClient - if err := csiMounter.SetUp(fsGroup); err != nil { + var mounterArgs volume.MounterArgs + mounterArgs.FsGroup = fsGroup + if err := csiMounter.SetUp(mounterArgs); err != nil { t.Fatalf("csiTest.VolumeAll mounter.Setup(fsGroup) failed: %s", err) } t.Log("csiTest.VolumeAll mounter.Setup(fsGroup) done OK") diff --git a/pkg/volume/downwardapi/downwardapi.go b/pkg/volume/downwardapi/downwardapi.go index 0a888914458..8d2aabb019a 100644 --- a/pkg/volume/downwardapi/downwardapi.go +++ b/pkg/volume/downwardapi/downwardapi.go @@ -174,11 +174,11 @@ func (b *downwardAPIVolumeMounter) CanMount() error { // This function is not idempotent by design. We want the data to be refreshed periodically. // The internal sync interval of kubelet will drive the refresh of data. // TODO: Add volume specific ticker and refresh loop -func (b *downwardAPIVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *downwardAPIVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } -func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *downwardAPIVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(3).Infof("Setting up a downwardAPI volume %v for pod %v/%v at %v", b.volName, b.pod.Namespace, b.pod.Name, dir) // Wrap EmptyDir. Here we rely on the idempotency of the wrapped plugin to avoid repeatedly mounting wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), b.pod, *b.opts) @@ -194,7 +194,7 @@ func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } setupSuccess := false - if err := wrapped.SetUpAt(dir, fsGroup); err != nil { + if err := wrapped.SetUpAt(dir, mounterArgs); err != nil { klog.Errorf("Unable to setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) return err } @@ -231,9 +231,9 @@ func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return err } - err = volume.SetVolumeOwnership(b, fsGroup) + err = volume.SetVolumeOwnership(b, mounterArgs.FsGroup) if err != nil { - klog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) + klog.Errorf("Error applying volume ownership settings for group: %v", mounterArgs.FsGroup) return err } diff --git a/pkg/volume/downwardapi/downwardapi_test.go b/pkg/volume/downwardapi/downwardapi_test.go index def65d12190..b0291570cc0 100644 --- a/pkg/volume/downwardapi/downwardapi_test.go +++ b/pkg/volume/downwardapi/downwardapi_test.go @@ -253,7 +253,7 @@ func newDownwardAPITest(t *testing.T, name string, volumeFiles, podLabels, podAn volumePath := mounter.GetPath() - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -380,7 +380,7 @@ func (step reSetUp) run(test *downwardAPITest) { } // now re-run Setup - if err = test.mounter.SetUp(nil); err != nil { + if err = test.mounter.SetUp(volume.MounterArgs{}); err != nil { test.t.Errorf("Failed to re-setup volume: %v", err) } diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go index fad947f128c..1c1f3792fc3 100644 --- a/pkg/volume/emptydir/empty_dir.go +++ b/pkg/volume/emptydir/empty_dir.go @@ -193,12 +193,12 @@ func (ed *emptyDir) CanMount() error { } // SetUp creates new directory. -func (ed *emptyDir) SetUp(fsGroup *int64) error { - return ed.SetUpAt(ed.GetPath(), fsGroup) +func (ed *emptyDir) SetUp(mounterArgs volume.MounterArgs) error { + return ed.SetUpAt(ed.GetPath(), mounterArgs) } // SetUpAt creates new directory. -func (ed *emptyDir) SetUpAt(dir string, fsGroup *int64) error { +func (ed *emptyDir) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { notMnt, err := ed.mounter.IsLikelyNotMountPoint(dir) // Getting an os.IsNotExist err from is a contingency; the directory // may not exist yet, in which case, setup should run. @@ -229,7 +229,7 @@ func (ed *emptyDir) SetUpAt(dir string, fsGroup *int64) error { err = fmt.Errorf("unknown storage medium %q", ed.medium) } - volume.SetVolumeOwnership(ed, fsGroup) + volume.SetVolumeOwnership(ed, mounterArgs.FsGroup) if err == nil { volumeutil.SetReady(ed.getMetaDir()) diff --git a/pkg/volume/emptydir/empty_dir_test.go b/pkg/volume/emptydir/empty_dir_test.go index 075a0f26ac4..b069c6d7c19 100644 --- a/pkg/volume/emptydir/empty_dir_test.go +++ b/pkg/volume/emptydir/empty_dir_test.go @@ -163,7 +163,7 @@ func doTestPlugin(t *testing.T, config pluginTestConfig) { t.Errorf("Got unexpected path: %s", volPath) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } diff --git a/pkg/volume/fc/fc.go b/pkg/volume/fc/fc.go index 3086743d7a5..ee8a09226e4 100644 --- a/pkg/volume/fc/fc.go +++ b/pkg/volume/fc/fc.go @@ -371,13 +371,13 @@ func (b *fcDiskMounter) CanMount() error { return nil } -func (b *fcDiskMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *fcDiskMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } -func (b *fcDiskMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *fcDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { // diskSetUp checks mountpoints and prevent repeated calls - err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup) + err := diskSetUp(b.manager, *b, dir, b.mounter, mounterArgs.FsGroup) if err != nil { klog.Errorf("fc: failed to setup") } diff --git a/pkg/volume/fc/fc_test.go b/pkg/volume/fc/fc_test.go index 632b05c2f1e..b0ce92e7f36 100644 --- a/pkg/volume/fc/fc_test.go +++ b/pkg/volume/fc/fc_test.go @@ -179,7 +179,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { t.Errorf("Unexpected path, expected %q, got: %q", expectedPath, path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { diff --git a/pkg/volume/flexvolume/driver-call.go b/pkg/volume/flexvolume/driver-call.go index c25b5999191..6eac275c9dc 100644 --- a/pkg/volume/flexvolume/driver-call.go +++ b/pkg/volume/flexvolume/driver-call.go @@ -51,7 +51,7 @@ const ( optionFSType = "kubernetes.io/fsType" optionReadWrite = "kubernetes.io/readwrite" optionKeySecret = "kubernetes.io/secret" - optionFSGroup = "kubernetes.io/fsGroup" + optionFSGroup = "kubernetes.io/mounterArgs.FsGroup" optionMountsDir = "kubernetes.io/mountsDir" optionPVorVolumeName = "kubernetes.io/pvOrVolumeName" diff --git a/pkg/volume/flexvolume/mounter-defaults.go b/pkg/volume/flexvolume/mounter-defaults.go index 93080611897..d535fac6a7a 100644 --- a/pkg/volume/flexvolume/mounter-defaults.go +++ b/pkg/volume/flexvolume/mounter-defaults.go @@ -26,7 +26,7 @@ type mounterDefaults flexVolumeMounter // SetUpAt is part of the volume.Mounter interface. // This implementation relies on the attacher's device mount path and does a bind mount to dir. -func (f *mounterDefaults) SetUpAt(dir string, fsGroup *int64) error { +func (f *mounterDefaults) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.Warning(logPrefix(f.plugin), "using default SetUpAt to ", dir) src, err := f.plugin.getDeviceMountPath(f.spec) diff --git a/pkg/volume/flexvolume/mounter.go b/pkg/volume/flexvolume/mounter.go index 8c246043cb7..94229d0d833 100644 --- a/pkg/volume/flexvolume/mounter.go +++ b/pkg/volume/flexvolume/mounter.go @@ -39,12 +39,12 @@ var _ volume.Mounter = &flexVolumeMounter{} // Mounter interface // SetUp creates new directory. -func (f *flexVolumeMounter) SetUp(fsGroup *int64) error { - return f.SetUpAt(f.GetPath(), fsGroup) +func (f *flexVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return f.SetUpAt(f.GetPath(), mounterArgs) } // SetUpAt creates new directory. -func (f *flexVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (f *flexVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { // Mount only once. alreadyMounted, err := prepareForMount(f.mounter, dir) if err != nil { @@ -75,15 +75,15 @@ func (f *flexVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } // Implicit parameters - if fsGroup != nil { - extraOptions[optionFSGroup] = strconv.FormatInt(int64(*fsGroup), 10) + if mounterArgs.FsGroup != nil { + extraOptions[optionFSGroup] = strconv.FormatInt(int64(*mounterArgs.FsGroup), 10) } call.AppendSpec(f.spec, f.plugin.host, extraOptions) _, err = call.Run() if isCmdNotSupportedErr(err) { - err = (*mounterDefaults)(f).SetUpAt(dir, fsGroup) + err = (*mounterDefaults)(f).SetUpAt(dir, mounterArgs) } if err != nil { @@ -93,7 +93,7 @@ func (f *flexVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if !f.readOnly { if f.plugin.capabilities.FSGroup { - volume.SetVolumeOwnership(f, fsGroup) + volume.SetVolumeOwnership(f, mounterArgs.FsGroup) } } diff --git a/pkg/volume/flexvolume/mounter_test.go b/pkg/volume/flexvolume/mounter_test.go index bf10fd74817..7e39d5c8879 100644 --- a/pkg/volume/flexvolume/mounter_test.go +++ b/pkg/volume/flexvolume/mounter_test.go @@ -23,6 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/test/utils/harness" ) @@ -46,7 +47,7 @@ func TestSetUpAt(tt *testing.T) { plugin, rootDir := testPlugin(t) plugin.unsupportedCommands = []string{"unsupportedCmd"} plugin.runner = fakeRunner( - // first call without fsGroup + // first call without mounterArgs.FsGroup assertDriverCall(t, successOutput(), mountCmd, rootDir+"/mount-dir", specJSON(plugin, spec, map[string]string{ optionKeyPodName: "my-pod", @@ -55,7 +56,7 @@ func TestSetUpAt(tt *testing.T) { optionKeyServiceAccountName: "my-sa", })), - // second test has fsGroup + // second test has mounterArgs.FsGroup assertDriverCall(t, notSupportedOutput(), mountCmd, rootDir+"/mount-dir", specJSON(plugin, spec, map[string]string{ optionFSGroup: "42", @@ -69,8 +70,10 @@ func TestSetUpAt(tt *testing.T) { ) m, _ := plugin.newMounterInternal(spec, pod, mounter, plugin.runner) - m.SetUpAt(rootDir+"/mount-dir", nil) + var mounterArgs volume.MounterArgs + m.SetUpAt(rootDir+"/mount-dir", mounterArgs) - fsGroup := int64(42) - m.SetUpAt(rootDir+"/mount-dir", &fsGroup) + group := int64(42) + mounterArgs.FsGroup = &group + m.SetUpAt(rootDir+"/mount-dir", mounterArgs) } diff --git a/pkg/volume/flocker/flocker.go b/pkg/volume/flocker/flocker.go index 41a30fa2af4..c69a96776d0 100644 --- a/pkg/volume/flocker/flocker.go +++ b/pkg/volume/flocker/flocker.go @@ -235,8 +235,8 @@ func (b *flockerVolumeMounter) GetPath() string { } // SetUp bind mounts the disk global mount to the volume path. -func (b *flockerVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *flockerVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } // newFlockerClient uses environment variables and pod attributes to return a @@ -277,7 +277,7 @@ control service: need to update the Primary UUID for this volume. 5. Wait until the Primary UUID was updated or timeout. */ -func (b *flockerVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *flockerVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { var err error if b.flockerClient == nil { b.flockerClient, err = b.newFlockerClient() @@ -365,7 +365,7 @@ func (b *flockerVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } if !b.readOnly { - volume.SetVolumeOwnership(b, fsGroup) + volume.SetVolumeOwnership(b, mounterArgs.FsGroup) } klog.V(4).Infof("successfully mounted %s", dir) diff --git a/pkg/volume/gcepd/gce_pd.go b/pkg/volume/gcepd/gce_pd.go index f4f29a9c63d..837266e8533 100644 --- a/pkg/volume/gcepd/gce_pd.go +++ b/pkg/volume/gcepd/gce_pd.go @@ -354,12 +354,12 @@ func (b *gcePersistentDiskMounter) CanMount() error { } // SetUp bind mounts the disk global mount to the volume path. -func (b *gcePersistentDiskMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *gcePersistentDiskMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } // SetUp bind mounts the disk global mount to the give volume path. -func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *gcePersistentDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { // TODO: handle failed mounts here. notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) klog.V(4).Infof("GCE PersistentDisk set up: Dir (%s) PD name (%q) Mounted (%t) Error (%v), ReadOnly (%t)", dir, b.pdName, !notMnt, err, b.readOnly) @@ -419,7 +419,7 @@ func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { } if !b.readOnly { - volume.SetVolumeOwnership(b, fsGroup) + volume.SetVolumeOwnership(b, mounterArgs.FsGroup) } klog.V(4).Infof("Successfully mounted %s", dir) diff --git a/pkg/volume/gcepd/gce_pd_test.go b/pkg/volume/gcepd/gce_pd_test.go index 36fdef08138..479a5a7b96e 100644 --- a/pkg/volume/gcepd/gce_pd_test.go +++ b/pkg/volume/gcepd/gce_pd_test.go @@ -141,7 +141,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { @@ -279,7 +279,7 @@ func TestMountOptions(t *testing.T) { t.Errorf("Got a nil Mounter") } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } mountOptions := fakeMounter.MountPoints[0].Opts diff --git a/pkg/volume/git_repo/git_repo.go b/pkg/volume/git_repo/git_repo.go index 46d6c5298aa..932296c2845 100644 --- a/pkg/volume/git_repo/git_repo.go +++ b/pkg/volume/git_repo/git_repo.go @@ -179,12 +179,12 @@ func (b *gitRepoVolumeMounter) CanMount() error { } // SetUp creates new directory and clones a git repo. -func (b *gitRepoVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *gitRepoVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } // SetUpAt creates new directory and clones a git repo. -func (b *gitRepoVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *gitRepoVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { if volumeutil.IsReady(b.getMetaDir()) { return nil } @@ -194,7 +194,7 @@ func (b *gitRepoVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if err != nil { return err } - if err := wrapped.SetUpAt(dir, fsGroup); err != nil { + if err := wrapped.SetUpAt(dir, mounterArgs); err != nil { return err } @@ -240,7 +240,7 @@ func (b *gitRepoVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return fmt.Errorf("failed to exec 'git reset --hard': %s: %v", output, err) } - volume.SetVolumeOwnership(b, fsGroup) + volume.SetVolumeOwnership(b, mounterArgs.FsGroup) volumeutil.SetReady(b.getMetaDir()) return nil diff --git a/pkg/volume/git_repo/git_repo_test.go b/pkg/volume/git_repo/git_repo_test.go index 6bb9e6eb734..64ef2eccf7a 100644 --- a/pkg/volume/git_repo/git_repo_test.go +++ b/pkg/volume/git_repo/git_repo_test.go @@ -422,7 +422,7 @@ func doTestSetUp(scenario struct { g := mounter.(*gitRepoVolumeMounter) g.exec = &fake - g.SetUp(nil) + g.SetUp(volume.MounterArgs{}) if fake.CommandCalls != len(expecteds) { allErrs = append(allErrs, diff --git a/pkg/volume/glusterfs/glusterfs.go b/pkg/volume/glusterfs/glusterfs.go index 51bb55f3bb5..63cb44498f5 100644 --- a/pkg/volume/glusterfs/glusterfs.go +++ b/pkg/volume/glusterfs/glusterfs.go @@ -275,11 +275,11 @@ func (b *glusterfsMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (b *glusterfsMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *glusterfsMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } -func (b *glusterfsMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *glusterfsMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) klog.V(4).Infof("mount setup: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { diff --git a/pkg/volume/glusterfs/glusterfs_test.go b/pkg/volume/glusterfs/glusterfs_test.go index 0a8a505a0aa..68428ff7ff0 100644 --- a/pkg/volume/glusterfs/glusterfs_test.go +++ b/pkg/volume/glusterfs/glusterfs_test.go @@ -118,7 +118,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { if volumePath != expectedPath { t.Errorf("Unexpected path, expected %q, got: %q", expectedPath, volumePath) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(volumePath); err != nil { diff --git a/pkg/volume/hostpath/host_path.go b/pkg/volume/hostpath/host_path.go index 070b12d0d82..0f4c46ff4bf 100644 --- a/pkg/volume/hostpath/host_path.go +++ b/pkg/volume/hostpath/host_path.go @@ -222,7 +222,7 @@ func (b *hostPathMounter) CanMount() error { } // SetUp does nothing. -func (b *hostPathMounter) SetUp(fsGroup *int64) error { +func (b *hostPathMounter) SetUp(mounterArgs volume.MounterArgs) error { err := validation.ValidatePathNoBacksteps(b.GetPath()) if err != nil { return fmt.Errorf("invalid HostPath `%s`: %v", b.GetPath(), err) @@ -235,7 +235,7 @@ func (b *hostPathMounter) SetUp(fsGroup *int64) error { } // SetUpAt does not make sense for host paths - probably programmer error. -func (b *hostPathMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *hostPathMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { return fmt.Errorf("SetUpAt() does not make sense for host paths") } diff --git a/pkg/volume/hostpath/host_path_test.go b/pkg/volume/hostpath/host_path_test.go index ac2cd64eb32..23da5d3d6af 100644 --- a/pkg/volume/hostpath/host_path_test.go +++ b/pkg/volume/hostpath/host_path_test.go @@ -217,7 +217,7 @@ func TestInvalidHostPath(t *testing.T) { t.Fatal(err) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) expectedMsg := "invalid HostPath `/no/backsteps/allowed/..`: must not contain '..'" if err.Error() != expectedMsg { t.Fatalf("expected error `%s` but got `%s`", expectedMsg, err) @@ -253,7 +253,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index 3dd5ddf3ca2..1500a6e4b0e 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -330,13 +330,13 @@ func (b *iscsiDiskMounter) CanMount() error { return nil } -func (b *iscsiDiskMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *iscsiDiskMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } -func (b *iscsiDiskMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *iscsiDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { // diskSetUp checks mountpoints and prevent repeated calls - err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup) + err := diskSetUp(b.manager, *b, dir, b.mounter, mounterArgs.FsGroup) if err != nil { klog.Errorf("iscsi: failed to setup") } diff --git a/pkg/volume/iscsi/iscsi_test.go b/pkg/volume/iscsi/iscsi_test.go index 4c2f3f960f5..c6554562745 100644 --- a/pkg/volume/iscsi/iscsi_test.go +++ b/pkg/volume/iscsi/iscsi_test.go @@ -175,7 +175,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { t.Errorf("Unexpected path, expected %q, got: %q", expectedPath, path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { diff --git a/pkg/volume/local/local.go b/pkg/volume/local/local.go index 5b3c30fab15..4457f9aadb9 100644 --- a/pkg/volume/local/local.go +++ b/pkg/volume/local/local.go @@ -423,12 +423,12 @@ func (m *localVolumeMounter) CanMount() error { } // SetUp bind mounts the directory to the volume path -func (m *localVolumeMounter) SetUp(fsGroup *int64) error { - return m.SetUpAt(m.GetPath(), fsGroup) +func (m *localVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return m.SetUpAt(m.GetPath(), mounterArgs) } // SetUpAt bind mounts the directory to the volume path and sets up volume ownership -func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (m *localVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { m.plugin.volumeLocks.LockKey(m.globalPath) defer m.plugin.volumeLocks.UnlockKey(m.globalPath) @@ -452,7 +452,7 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return nil } refs, err := m.mounter.GetMountRefs(m.globalPath) - if fsGroup != nil { + if mounterArgs.FsGroup != nil { if err != nil { klog.Errorf("cannot collect mounting information: %s %v", m.globalPath, err) return err @@ -461,7 +461,7 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { // Only count mounts from other pods refs = m.filterPodMounts(refs) if len(refs) > 0 { - fsGroupNew := int64(*fsGroup) + fsGroupNew := int64(*mounterArgs.FsGroup) fsGroupOld, err := m.mounter.GetFSGroup(m.globalPath) if err != nil { return fmt.Errorf("failed to check fsGroup for %s (%v)", m.globalPath, err) @@ -519,7 +519,7 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if !m.readOnly { // Volume owner will be written only once on the first volume mount if len(refs) == 0 { - return volume.SetVolumeOwnership(m, fsGroup) + return volume.SetVolumeOwnership(m, mounterArgs.FsGroup) } } return nil diff --git a/pkg/volume/local/local_test.go b/pkg/volume/local/local_test.go index c6a3161556c..dcad1a056c9 100644 --- a/pkg/volume/local/local_test.go +++ b/pkg/volume/local/local_test.go @@ -200,7 +200,7 @@ func TestInvalidLocalPath(t *testing.T) { t.Fatal(err) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) expectedMsg := "invalid path: /no/backsteps/allowed/.. must not contain '..'" if err.Error() != expectedMsg { t.Fatalf("expected error `%s` but got `%s`", expectedMsg, err) @@ -307,7 +307,7 @@ func TestMountUnmount(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } @@ -412,7 +412,9 @@ func testFSGroupMount(plug volume.VolumePlugin, pod *v1.Pod, tmpDir string, fsGr return fmt.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(&fsGroup); err != nil { + var mounterArgs volume.MounterArgs + mounterArgs.FsGroup = &fsGroup + if err := mounter.SetUp(mounterArgs); err != nil { return err } return nil @@ -517,7 +519,7 @@ func TestMountOptions(t *testing.T) { fakeMounter := &mount.FakeMounter{} mounter.(*localVolumeMounter).mounter = fakeMounter - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } mountOptions := fakeMounter.MountPoints[0].Opts diff --git a/pkg/volume/nfs/nfs.go b/pkg/volume/nfs/nfs.go index a94959fe1c8..93c12375b64 100644 --- a/pkg/volume/nfs/nfs.go +++ b/pkg/volume/nfs/nfs.go @@ -233,11 +233,11 @@ func (nfsMounter *nfsMounter) GetAttributes() volume.Attributes { } // SetUp attaches the disk and bind mounts to the volume path. -func (nfsMounter *nfsMounter) SetUp(fsGroup *int64) error { - return nfsMounter.SetUpAt(nfsMounter.GetPath(), fsGroup) +func (nfsMounter *nfsMounter) SetUp(mounterArgs volume.MounterArgs) error { + return nfsMounter.SetUpAt(nfsMounter.GetPath(), mounterArgs) } -func (nfsMounter *nfsMounter) SetUpAt(dir string, fsGroup *int64) error { +func (nfsMounter *nfsMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { notMnt, err := mount.IsNotMountPoint(nfsMounter.mounter, dir) klog.V(4).Infof("NFS mount set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { diff --git a/pkg/volume/nfs/nfs_test.go b/pkg/volume/nfs/nfs_test.go index 2c6b7667c76..3fa03e8a298 100644 --- a/pkg/volume/nfs/nfs_test.go +++ b/pkg/volume/nfs/nfs_test.go @@ -122,7 +122,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { if volumePath != expectedPath { t.Errorf("Unexpected path, expected %q, got: %q", expectedPath, volumePath) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(volumePath); err != nil { diff --git a/pkg/volume/photon_pd/photon_pd.go b/pkg/volume/photon_pd/photon_pd.go index f89364658e2..54fe82fa02b 100644 --- a/pkg/volume/photon_pd/photon_pd.go +++ b/pkg/volume/photon_pd/photon_pd.go @@ -200,12 +200,12 @@ func (b *photonPersistentDiskMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (b *photonPersistentDiskMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *photonPersistentDiskMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } // SetUp attaches the disk and bind mounts to the volume path. -func (b *photonPersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *photonPersistentDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(4).Infof("Photon Persistent Disk setup %s to %s", b.pdID, dir) // TODO: handle failed mounts here. diff --git a/pkg/volume/photon_pd/photon_pd_test.go b/pkg/volume/photon_pd/photon_pd_test.go index 34a5576c1e7..8b2beaeb619 100644 --- a/pkg/volume/photon_pd/photon_pd_test.go +++ b/pkg/volume/photon_pd/photon_pd_test.go @@ -128,7 +128,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { diff --git a/pkg/volume/portworx/portworx.go b/pkg/volume/portworx/portworx.go index b1371e1c4c1..c9c30f4bd49 100644 --- a/pkg/volume/portworx/portworx.go +++ b/pkg/volume/portworx/portworx.go @@ -298,12 +298,12 @@ func (b *portworxVolumeMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (b *portworxVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *portworxVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } // SetUpAt attaches the disk and bind mounts to the volume path. -func (b *portworxVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *portworxVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) klog.Infof("Portworx Volume set up. Dir: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { @@ -331,7 +331,7 @@ func (b *portworxVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return err } if !b.readOnly { - volume.SetVolumeOwnership(b, fsGroup) + volume.SetVolumeOwnership(b, mounterArgs.FsGroup) } klog.Infof("Portworx Volume %s setup at %s", b.volumeID, dir) return nil diff --git a/pkg/volume/portworx/portworx_test.go b/pkg/volume/portworx/portworx_test.go index 2fa15360e28..7f261c9258e 100644 --- a/pkg/volume/portworx/portworx_test.go +++ b/pkg/volume/portworx/portworx_test.go @@ -163,7 +163,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { diff --git a/pkg/volume/projected/projected.go b/pkg/volume/projected/projected.go index e2da1bec276..7aaee7138ac 100644 --- a/pkg/volume/projected/projected.go +++ b/pkg/volume/projected/projected.go @@ -188,11 +188,11 @@ func (s *projectedVolumeMounter) CanMount() error { return nil } -func (s *projectedVolumeMounter) SetUp(fsGroup *int64) error { - return s.SetUpAt(s.GetPath(), fsGroup) +func (s *projectedVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return s.SetUpAt(s.GetPath(), mounterArgs) } -func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (s *projectedVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(3).Infof("Setting up volume %v for pod %v at %v", s.volName, s.pod.UID, dir) wrapped, err := s.plugin.host.NewWrapperMounter(s.volName, wrappedVolumeSpec(), s.pod, *s.opts) @@ -207,7 +207,7 @@ func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } setupSuccess := false - if err := wrapped.SetUpAt(dir, fsGroup); err != nil { + if err := wrapped.SetUpAt(dir, mounterArgs); err != nil { return err } @@ -243,9 +243,9 @@ func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return err } - err = volume.SetVolumeOwnership(s, fsGroup) + err = volume.SetVolumeOwnership(s, mounterArgs.FsGroup) if err != nil { - klog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) + klog.Errorf("Error applying volume ownership settings for group: %v", mounterArgs.FsGroup) return err } setupSuccess = true diff --git a/pkg/volume/projected/projected_test.go b/pkg/volume/projected/projected_test.go index b4d73fc68a0..841582114b4 100644 --- a/pkg/volume/projected/projected_test.go +++ b/pkg/volume/projected/projected_test.go @@ -878,7 +878,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -943,7 +943,8 @@ func TestInvalidPathProjected(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + var mounterArgs volume.MounterArgs + err = mounter.SetUp(mounterArgs) if err == nil { t.Errorf("Expected error while setting up secret") } @@ -994,7 +995,7 @@ func TestPluginReboot(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -1046,7 +1047,7 @@ func TestPluginOptional(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -1144,7 +1145,7 @@ func TestPluginOptionalKeys(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } diff --git a/pkg/volume/quobyte/quobyte.go b/pkg/volume/quobyte/quobyte.go index ec14a5c6c58..3cfc7a800ec 100644 --- a/pkg/volume/quobyte/quobyte.go +++ b/pkg/volume/quobyte/quobyte.go @@ -237,12 +237,12 @@ func (mounter *quobyteMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (mounter *quobyteMounter) SetUp(fsGroup *int64) error { +func (mounter *quobyteMounter) SetUp(mounterArgs volume.MounterArgs) error { pluginDir := mounter.plugin.host.GetPluginDir(utilstrings.EscapeQualifiedName(quobytePluginName)) - return mounter.SetUpAt(pluginDir, fsGroup) + return mounter.SetUpAt(pluginDir, mounterArgs) } -func (mounter *quobyteMounter) SetUpAt(dir string, fsGroup *int64) error { +func (mounter *quobyteMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { // Check if Quobyte is already mounted on the host in the Plugin Dir // if so we can use this mountpoint instead of creating a new one // IsLikelyNotMountPoint wouldn't check the mount type diff --git a/pkg/volume/quobyte/quobyte_test.go b/pkg/volume/quobyte/quobyte_test.go index 61c061ad0f2..71e031d3c84 100644 --- a/pkg/volume/quobyte/quobyte_test.go +++ b/pkg/volume/quobyte/quobyte_test.go @@ -101,7 +101,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { if volumePath != fmt.Sprintf("%s/plugins/kubernetes.io~quobyte/root#root@vol", tmpDir) { t.Errorf("Got unexpected path: %s expected: %s", volumePath, fmt.Sprintf("%s/plugins/kubernetes.io~quobyte/root#root@vol", tmpDir)) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } unmounter, err := plug.(*quobytePlugin).newUnmounterInternal("vol", types.UID("poduid"), &mount.FakeMounter{}) diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index bae23bc7334..da07ba64765 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -825,14 +825,14 @@ func (b *rbdMounter) CanMount() error { return nil } -func (b *rbdMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *rbdMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } -func (b *rbdMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *rbdMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { // diskSetUp checks mountpoints and prevent repeated calls klog.V(4).Infof("rbd: attempting to setup at %s", dir) - err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup) + err := diskSetUp(b.manager, *b, dir, b.mounter, mounterArgs.FsGroup) if err != nil { klog.Errorf("rbd: failed to setup at %s %v", dir, err) } diff --git a/pkg/volume/rbd/rbd_test.go b/pkg/volume/rbd/rbd_test.go index 799194c0760..4e133d1d551 100644 --- a/pkg/volume/rbd/rbd_test.go +++ b/pkg/volume/rbd/rbd_test.go @@ -305,7 +305,7 @@ func doTestPlugin(t *testing.T, c *testcase) { t.Errorf("Unexpected path, expected %q, got: %q", c.expectedPodMountPath, path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { diff --git a/pkg/volume/scaleio/sio_volume.go b/pkg/volume/scaleio/sio_volume.go index 3c84ddecd3d..4b7fd8ebb23 100644 --- a/pkg/volume/scaleio/sio_volume.go +++ b/pkg/volume/scaleio/sio_volume.go @@ -77,12 +77,12 @@ func (v *sioVolume) CanMount() error { return nil } -func (v *sioVolume) SetUp(fsGroup *int64) error { - return v.SetUpAt(v.GetPath(), fsGroup) +func (v *sioVolume) SetUp(mounterArgs volume.MounterArgs) error { + return v.SetUpAt(v.GetPath(), mounterArgs) } // SetUp bind mounts the disk global mount to the volume path. -func (v *sioVolume) SetUpAt(dir string, fsGroup *int64) error { +func (v *sioVolume) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { v.plugin.volumeMtx.LockKey(v.volSpecName) defer v.plugin.volumeMtx.UnlockKey(v.volSpecName) @@ -154,9 +154,9 @@ func (v *sioVolume) SetUpAt(dir string, fsGroup *int64) error { return err } - if !v.readOnly && fsGroup != nil { + if !v.readOnly && mounterArgs.FsGroup != nil { klog.V(4).Info(log("applying value FSGroup ownership")) - volume.SetVolumeOwnership(v, fsGroup) + volume.SetVolumeOwnership(v, mounterArgs.FsGroup) } klog.V(4).Info(log("successfully setup PV %s: volume %s mapped as %s mounted at %s", v.volSpecName, v.volName, devicePath, dir)) diff --git a/pkg/volume/scaleio/sio_volume_test.go b/pkg/volume/scaleio/sio_volume_test.go index f7cf263ad7a..196eb1aee44 100644 --- a/pkg/volume/scaleio/sio_volume_test.go +++ b/pkg/volume/scaleio/sio_volume_test.go @@ -191,7 +191,7 @@ func TestVolumeMounterUnmounter(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := sioMounter.SetUp(nil); err != nil { + if err := sioMounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(path); err != nil { @@ -345,7 +345,7 @@ func TestVolumeProvisioner(t *testing.T) { t.Fatalf("failed to create sio mgr: %v", err) } sioVol.sioMgr.client = sio - if err := sioMounter.SetUp(nil); err != nil { + if err := sioMounter.SetUp(volume.MounterArgs{}); err != nil { t.Fatalf("Expected success, got: %v", err) } diff --git a/pkg/volume/secret/secret.go b/pkg/volume/secret/secret.go index b548acfb3c0..90345d3f96a 100644 --- a/pkg/volume/secret/secret.go +++ b/pkg/volume/secret/secret.go @@ -178,11 +178,11 @@ func (b *secretVolumeMounter) CanMount() error { return nil } -func (b *secretVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *secretVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } -func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *secretVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir) // Wrap EmptyDir, let it do the setup. @@ -219,7 +219,7 @@ func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } setupSuccess := false - if err := wrapped.SetUpAt(dir, fsGroup); err != nil { + if err := wrapped.SetUpAt(dir, mounterArgs); err != nil { return err } if err := volumeutil.MakeNestedMountpoints(b.volName, dir, b.pod); err != nil { @@ -254,9 +254,9 @@ func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return err } - err = volume.SetVolumeOwnership(b, fsGroup) + err = volume.SetVolumeOwnership(b, mounterArgs.FsGroup) if err != nil { - klog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) + klog.Errorf("Error applying volume ownership settings for group: %v", mounterArgs.FsGroup) return err } setupSuccess = true diff --git a/pkg/volume/secret/secret_test.go b/pkg/volume/secret/secret_test.go index c83bc99bc0e..0f4ada8f50f 100644 --- a/pkg/volume/secret/secret_test.go +++ b/pkg/volume/secret/secret_test.go @@ -327,7 +327,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -401,7 +401,8 @@ func TestInvalidPathSecret(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + var mounterArgs volume.MounterArgs + err = mounter.SetUp(mounterArgs) if err == nil { t.Errorf("Expected error while setting up secret") } @@ -452,7 +453,7 @@ func TestPluginReboot(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -504,7 +505,7 @@ func TestPluginOptional(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } @@ -602,7 +603,7 @@ func TestPluginOptionalKeys(t *testing.T) { t.Errorf("Got unexpected path: %s", volumePath) } - err = mounter.SetUp(nil) + err = mounter.SetUp(volume.MounterArgs{}) if err != nil { t.Errorf("Failed to setup volume: %v", err) } diff --git a/pkg/volume/storageos/storageos.go b/pkg/volume/storageos/storageos.go index dd5201081cb..a126d50799a 100644 --- a/pkg/volume/storageos/storageos.go +++ b/pkg/volume/storageos/storageos.go @@ -345,7 +345,7 @@ func (b *storageosMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (b *storageosMounter) SetUp(fsGroup *int64) error { +func (b *storageosMounter) SetUp(mounterArgs volume.MounterArgs) error { // Need a namespace to find the volume, try pod's namespace if not set. if b.volNamespace == "" { klog.V(2).Infof("Setting StorageOS volume namespace to pod namespace: %s", b.podNamespace) @@ -375,11 +375,11 @@ func (b *storageosMounter) SetUp(fsGroup *int64) error { klog.V(4).Infof("Successfully mounted StorageOS volume %s into global mount directory", b.volName) // Bind mount the volume into the pod - return b.SetUpAt(b.GetPath(), fsGroup) + return b.SetUpAt(b.GetPath(), mounterArgs) } // SetUp bind mounts the disk global mount to the give volume path. -func (b *storageosMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *storageosMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) klog.V(4).Infof("StorageOS volume set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { @@ -433,7 +433,7 @@ func (b *storageosMounter) SetUpAt(dir string, fsGroup *int64) error { } if !b.readOnly { - volume.SetVolumeOwnership(b, fsGroup) + volume.SetVolumeOwnership(b, mounterArgs.FsGroup) } klog.V(4).Infof("StorageOS volume setup complete on %s", dir) return nil diff --git a/pkg/volume/storageos/storageos_test.go b/pkg/volume/storageos/storageos_test.go index e3d544567b2..c51dfe25732 100644 --- a/pkg/volume/storageos/storageos_test.go +++ b/pkg/volume/storageos/storageos_test.go @@ -208,7 +208,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Expected path: '%s' got: '%s'", expectedPath, volPath) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } if _, err := os.Stat(volPath); err != nil { diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 2c8e34e4d66..df8f6d192cb 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -765,11 +765,11 @@ func (fv *FakeVolume) CanMount() error { return nil } -func (fv *FakeVolume) SetUp(fsGroup *int64) error { +func (fv *FakeVolume) SetUp(mounterArgs MounterArgs) error { fv.Lock() defer fv.Unlock() fv.SetUpCallCount++ - return fv.SetUpAt(fv.getPath(), fsGroup) + return fv.SetUpAt(fv.getPath(), mounterArgs) } func (fv *FakeVolume) GetSetUpCallCount() int { @@ -778,7 +778,7 @@ func (fv *FakeVolume) GetSetUpCallCount() int { return fv.SetUpCallCount } -func (fv *FakeVolume) SetUpAt(dir string, fsGroup *int64) error { +func (fv *FakeVolume) SetUpAt(dir string, mounterArgs MounterArgs) error { return os.MkdirAll(dir, 0750) } diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 7300d843992..6acb6443d70 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -701,7 +701,10 @@ func (og *operationGenerator) GenerateMountVolumeFunc( } // Execute mount - mountErr := volumeMounter.SetUp(fsGroup) + mountErr := volumeMounter.SetUp(volume.MounterArgs{ + FsGroup: fsGroup, + PodUID: string(volumeToMount.Pod.UID), + }) if mountErr != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MountVolume.SetUp failed", mountErr) diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index f597ba1662d..a5abb923b24 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -101,6 +101,12 @@ type Attributes struct { SupportsSELinux bool } +// MounterArgs provides more easily extensible arguments to Mounter +type MounterArgs struct { + FsGroup *int64 + PodUID string +} + // Mounter interface provides methods to set up/mount the volume. type Mounter interface { // Uses Interface to provide the path for Docker binds. @@ -122,14 +128,14 @@ type Mounter interface { // content should be owned by 'fsGroup' so that it can be // accessed by the pod. This may be called more than once, so // implementations must be idempotent. - SetUp(fsGroup *int64) error + SetUp(mounterArgs MounterArgs) error // SetUpAt prepares and mounts/unpacks the volume to the // specified directory path, which may or may not exist yet. // The mount point and its content should be owned by // 'fsGroup' so that it can be accessed by the pod. This may // be called more than once, so implementations must be // idempotent. - SetUpAt(dir string, fsGroup *int64) error + SetUpAt(dir string, mounterArgs MounterArgs) error // GetAttributes returns the attributes of the mounter. // This function is called after SetUp()/SetUpAt(). GetAttributes() Attributes diff --git a/pkg/volume/vsphere_volume/vsphere_volume.go b/pkg/volume/vsphere_volume/vsphere_volume.go index db2596b99de..30c8a8d09d9 100644 --- a/pkg/volume/vsphere_volume/vsphere_volume.go +++ b/pkg/volume/vsphere_volume/vsphere_volume.go @@ -208,8 +208,8 @@ func (b *vsphereVolumeMounter) GetAttributes() volume.Attributes { } // SetUp attaches the disk and bind mounts to the volume path. -func (b *vsphereVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) +func (b *vsphereVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error { + return b.SetUpAt(b.GetPath(), mounterArgs) } // Checks prior to mount operations to verify that the required components (binaries, etc.) @@ -220,7 +220,7 @@ func (b *vsphereVolumeMounter) CanMount() error { } // SetUp attaches the disk and bind mounts to the volume path. -func (b *vsphereVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { +func (b *vsphereVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { klog.V(5).Infof("vSphere volume setup %s to %s", b.volPath, dir) // TODO: handle failed mounts here. @@ -269,7 +269,7 @@ func (b *vsphereVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { os.Remove(dir) return err } - volume.SetVolumeOwnership(b, fsGroup) + volume.SetVolumeOwnership(b, mounterArgs.FsGroup) klog.V(3).Infof("vSphere volume %s mounted to %s", b.volPath, dir) return nil diff --git a/pkg/volume/vsphere_volume/vsphere_volume_test.go b/pkg/volume/vsphere_volume/vsphere_volume_test.go index 0bae75180fc..a25540e3b2c 100644 --- a/pkg/volume/vsphere_volume/vsphere_volume_test.go +++ b/pkg/volume/vsphere_volume/vsphere_volume_test.go @@ -123,7 +123,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Got unexpected path: %s", path) } - if err := mounter.SetUp(nil); err != nil { + if err := mounter.SetUp(volume.MounterArgs{}); err != nil { t.Errorf("Expected success, got: %v", err) } From 448e0c44c6919b8e874f042cb934609c0bf5a790 Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Mon, 19 Nov 2018 17:43:41 -0500 Subject: [PATCH 2/6] Apply quotas via syscalls using cgo. --- pkg/features/kube_features.go | 8 + pkg/kubelet/eviction/helpers.go | 20 +- pkg/kubelet/volumemanager/cache/BUILD | 2 + .../cache/desired_state_of_world.go | 21 +- pkg/volume/emptydir/BUILD | 1 + pkg/volume/emptydir/empty_dir.go | 18 +- pkg/volume/util/BUILD | 1 + pkg/volume/util/fs/BUILD | 2 + pkg/volume/util/fs/fs.go | 13 + .../operationexecutor/operation_executor.go | 4 + .../operationexecutor/operation_generator.go | 5 +- pkg/volume/util/quota/BUILD | 62 ++ pkg/volume/util/quota/common/BUILD | 33 + .../util/quota/common/quota_linux_common.go | 77 ++ .../quota/common/quota_linux_common_impl.go | 152 ++++ pkg/volume/util/quota/extfs/BUILD | 31 + pkg/volume/util/quota/extfs/quota_extfs.go | 153 ++++ pkg/volume/util/quota/project.go | 353 ++++++++ pkg/volume/util/quota/quota.go | 47 ++ pkg/volume/util/quota/quota_linux.go | 432 ++++++++++ pkg/volume/util/quota/quota_linux_test.go | 762 ++++++++++++++++++ pkg/volume/util/quota/quota_unsupported.go | 54 ++ pkg/volume/util/quota/xfs/BUILD | 31 + pkg/volume/util/quota/xfs/quota_xfs.go | 153 ++++ pkg/volume/volume.go | 5 +- 25 files changed, 2431 insertions(+), 9 deletions(-) create mode 100644 pkg/volume/util/quota/BUILD create mode 100644 pkg/volume/util/quota/common/BUILD create mode 100644 pkg/volume/util/quota/common/quota_linux_common.go create mode 100644 pkg/volume/util/quota/common/quota_linux_common_impl.go create mode 100644 pkg/volume/util/quota/extfs/BUILD create mode 100644 pkg/volume/util/quota/extfs/quota_extfs.go create mode 100644 pkg/volume/util/quota/project.go create mode 100644 pkg/volume/util/quota/quota.go create mode 100644 pkg/volume/util/quota/quota_linux.go create mode 100644 pkg/volume/util/quota/quota_linux_test.go create mode 100644 pkg/volume/util/quota/quota_unsupported.go create mode 100644 pkg/volume/util/quota/xfs/BUILD create mode 100644 pkg/volume/util/quota/xfs/quota_xfs.go diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 911f8c5a4c1..86d164b895a 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -440,6 +440,13 @@ const ( // // Enables the regional PD feature on GCE. deprecatedGCERegionalPersistentDisk featuregate.Feature = "GCERegionalPersistentDisk" + + // owner: @RobertKrawitz + // alpha: v1.15 + // + // Allow use of filesystems for ephemeral storage monitoring. + // Only applies if LocalStorageCapacityIsolation is set. + FSQuotaForLSCIMonitoring = "FSQuotaForLSCIMonitoring" ) func init() { @@ -514,6 +521,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, KubeletPodResources: {Default: false, PreRelease: featuregate.Alpha}, WindowsGMSA: {Default: false, PreRelease: featuregate.Alpha}, + FSQuotaForLSCIMonitoring: {Default: false, PreRelease: utilfeature.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 74229b75905..0e8f843539a 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -395,13 +395,27 @@ func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsSt }, nil } +func internalIsLocalEphemeralVolume(pod *v1.Pod, volume v1.Volume) bool { + return volume.GitRepo != nil || + (volume.EmptyDir != nil && volume.EmptyDir.Medium != v1.StorageMediumMemory) || + volume.ConfigMap != nil || volume.DownwardAPI != nil +} + +// IsLocalEphemeralVolume determines whether a given volume name is ephemeral +func IsLocalEphemeralVolume(pod *v1.Pod, volumeName string) (bool, error) { + for _, volume := range pod.Spec.Volumes { + if volume.Name == volumeName { + return internalIsLocalEphemeralVolume(pod, volume), nil + } + } + return false, fmt.Errorf("Volume %s not found in pod %v", volumeName, pod) +} + // localEphemeralVolumeNames returns the set of ephemeral volumes for the pod that are local func localEphemeralVolumeNames(pod *v1.Pod) []string { result := []string{} for _, volume := range pod.Spec.Volumes { - if volume.GitRepo != nil || - (volume.EmptyDir != nil && volume.EmptyDir.Medium != v1.StorageMediumMemory) || - volume.ConfigMap != nil || volume.DownwardAPI != nil { + if internalIsLocalEphemeralVolume(pod, volume) { result = append(result, volume.Name) } } diff --git a/pkg/kubelet/volumemanager/cache/BUILD b/pkg/kubelet/volumemanager/cache/BUILD index 3bc1a3886bb..35028f5846e 100644 --- a/pkg/kubelet/volumemanager/cache/BUILD +++ b/pkg/kubelet/volumemanager/cache/BUILD @@ -14,7 +14,9 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache", deps = [ + "//pkg/api/v1/resource:go_default_library", "//pkg/features:go_default_library", + "//pkg/kubelet/eviction:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index 1c748505a26..c1d9fea084b 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -25,6 +25,8 @@ import ( "sync" "k8s.io/api/core/v1" + apiv1resource "k8s.io/kubernetes/pkg/api/v1/resource" + limits "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" @@ -160,6 +162,10 @@ type volumeToMount struct { // reportedInUse indicates that the volume was successfully added to the // VolumesInUse field in the node's status. reportedInUse bool + + // desiredSizeLimit indicates the desired upper bound on the size of the volume + // (if so implemented) + desiredSizeLimit int64 } // The pod object represents a pod that references the underlying volume and @@ -226,6 +232,17 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( } if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists { + var sizeLimit int64 + sizeLimit = 0 + isLocal, _ := limits.IsLocalEphemeralVolume(pod, volumeSpec.Name()) + if isLocal { + _, podLimits := apiv1resource.PodRequestsAndLimits(pod) + ephemeralStorageLimit := podLimits[v1.ResourceEphemeralStorage] + sizeLimit = ephemeralStorageLimit.Value() + if sizeLimit == 0 { + sizeLimit = -1 + } + } dsw.volumesToMount[volumeName] = volumeToMount{ volumeName: volumeName, podsToMount: make(map[types.UniquePodName]podToMount), @@ -233,6 +250,7 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( pluginIsDeviceMountable: deviceMountable, volumeGidValue: volumeGidValue, reportedInUse: false, + desiredSizeLimit: sizeLimit, } } @@ -360,7 +378,8 @@ func (dsw *desiredStateOfWorld) GetVolumesToMount() []VolumeToMount { PluginIsDeviceMountable: volumeObj.pluginIsDeviceMountable, OuterVolumeSpecName: podObj.outerVolumeSpecName, VolumeGidValue: volumeObj.volumeGidValue, - ReportedInUse: volumeObj.reportedInUse}}) + ReportedInUse: volumeObj.reportedInUse, + DesiredSizeLimit: volumeObj.desiredSizeLimit}}) } } return volumesToMount diff --git a/pkg/volume/emptydir/BUILD b/pkg/volume/emptydir/BUILD index 08c2bc334fc..4833bc31f51 100644 --- a/pkg/volume/emptydir/BUILD +++ b/pkg/volume/emptydir/BUILD @@ -20,6 +20,7 @@ go_library( "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", + "//pkg/volume/util/quota:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go index 1c1f3792fc3..a2604b7c465 100644 --- a/pkg/volume/emptydir/empty_dir.go +++ b/pkg/volume/emptydir/empty_dir.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/kubernetes/pkg/volume/util/quota" utilstrings "k8s.io/utils/strings" ) @@ -174,6 +175,7 @@ type emptyDir struct { mounter mount.Interface mountDetector mountDetector plugin *emptyDirPlugin + desiredSize int64 volume.MetricsProvider } @@ -234,6 +236,15 @@ func (ed *emptyDir) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { if err == nil { volumeutil.SetReady(ed.getMetaDir()) } + if mounterArgs.DesiredSize != 0 { + if hasQuotas, _ := quota.SupportsQuotas(ed.mounter, dir); hasQuotas { + klog.V(3).Infof("emptydir trying to assign quota") + err := quota.AssignQuota(ed.mounter, dir, mounterArgs.PodUID, mounterArgs.DesiredSize) + if err != nil { + klog.V(3).Infof("Set quota failed %v", err) + } + } + } return err } @@ -397,9 +408,14 @@ func (ed *emptyDir) TearDownAt(dir string) error { } func (ed *emptyDir) teardownDefault(dir string) error { + // Remove any quota + err := quota.ClearQuota(ed.mounter, dir) + if err != nil { + klog.V(3).Infof("Failed to clear quota on %s: %v", dir, err) + } // Renaming the directory is not required anymore because the operation executor // now handles duplicate operations on the same volume - err := os.RemoveAll(dir) + err = os.RemoveAll(dir) if err != nil { return err } diff --git a/pkg/volume/util/BUILD b/pkg/volume/util/BUILD index 950fb621661..a94e304a482 100644 --- a/pkg/volume/util/BUILD +++ b/pkg/volume/util/BUILD @@ -91,6 +91,7 @@ filegroup( "//pkg/volume/util/nestedpendingoperations:all-srcs", "//pkg/volume/util/nsenter:all-srcs", "//pkg/volume/util/operationexecutor:all-srcs", + "//pkg/volume/util/quota:all-srcs", "//pkg/volume/util/recyclerclient:all-srcs", "//pkg/volume/util/subpath:all-srcs", "//pkg/volume/util/types:all-srcs", diff --git a/pkg/volume/util/fs/BUILD b/pkg/volume/util/fs/BUILD index 38602bac3e1..504175781f2 100644 --- a/pkg/volume/util/fs/BUILD +++ b/pkg/volume/util/fs/BUILD @@ -14,6 +14,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ + "//pkg/volume/util/quota:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", ], @@ -24,6 +25,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/volume/util/quota:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", ], diff --git a/pkg/volume/util/fs/fs.go b/pkg/volume/util/fs/fs.go index a80a167eea7..dd88367f020 100644 --- a/pkg/volume/util/fs/fs.go +++ b/pkg/volume/util/fs/fs.go @@ -27,6 +27,7 @@ import ( "golang.org/x/sys/unix" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/kubernetes/pkg/volume/util/quota" ) // FSInfo linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error) @@ -56,6 +57,13 @@ func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { // DiskUsage gets disk usage of specified path. func DiskUsage(path string) (*resource.Quantity, error) { + // First check whether the quota system knows about this directory + data, err := quota.GetConsumption(path) + if err == nil { + var q resource.Quantity + q.Set(data) + return &q, nil + } // Uses the same niceness level as cadvisor.fs does when running du // Uses -B 1 to always scale to a blocksize of 1 byte out, err := exec.Command("nice", "-n", "19", "du", "-s", "-B", "1", path).CombinedOutput() @@ -76,6 +84,11 @@ func Find(path string) (int64, error) { if path == "" { return 0, fmt.Errorf("invalid directory") } + // First check whether the quota system knows about this directory + inodes, err := quota.GetInodes(path) + if err == nil { + return inodes, nil + } var counter byteCounter var stderr bytes.Buffer findCmd := exec.Command("find", path, "-xdev", "-printf", ".") diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 18668f8d293..379829ad50c 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -346,6 +346,10 @@ type VolumeToMount struct { // ReportedInUse indicates that the volume was successfully added to the // VolumesInUse field in the node's status. ReportedInUse bool + + // DesiredSizeLimit indicates the desired upper bound on the size of the volume + // (if so implemented) + DesiredSizeLimit int64 } // GenerateMsgDetailed returns detailed msgs for volumes to mount diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 6acb6443d70..d3195413742 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -702,8 +702,9 @@ func (og *operationGenerator) GenerateMountVolumeFunc( // Execute mount mountErr := volumeMounter.SetUp(volume.MounterArgs{ - FsGroup: fsGroup, - PodUID: string(volumeToMount.Pod.UID), + FsGroup: fsGroup, + DesiredSize: volumeToMount.DesiredSizeLimit, + PodUID: string(volumeToMount.Pod.UID), }) if mountErr != nil { // On failure, return error. Caller will log and retry. diff --git a/pkg/volume/util/quota/BUILD b/pkg/volume/util/quota/BUILD new file mode 100644 index 00000000000..2d1df604472 --- /dev/null +++ b/pkg/volume/util/quota/BUILD @@ -0,0 +1,62 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "project.go", + "quota.go", + "quota_linux.go", + "quota_unsupported.go", + ], + importpath = "k8s.io/kubernetes/pkg/volume/util/quota", + visibility = ["//visibility:public"], + deps = [ + "//pkg/features:go_default_library", + "//pkg/util/mount:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + ] + select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/volume/util/quota/common:go_default_library", + "//pkg/volume/util/quota/extfs:go_default_library", + "//pkg/volume/util/quota/xfs:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "go_default_test", + srcs = ["quota_linux_test.go"], + embed = [":go_default_library"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/features:go_default_library", + "//pkg/util/mount:go_default_library", + "//pkg/volume/util/quota/common:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//pkg/volume/util/quota/common:all-srcs", + "//pkg/volume/util/quota/extfs:all-srcs", + "//pkg/volume/util/quota/xfs:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/volume/util/quota/common/BUILD b/pkg/volume/util/quota/common/BUILD new file mode 100644 index 00000000000..536a67450b4 --- /dev/null +++ b/pkg/volume/util/quota/common/BUILD @@ -0,0 +1,33 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "quota_linux_common.go", + "quota_linux_common_impl.go", + ], + cgo = True, + importpath = "k8s.io/kubernetes/pkg/volume/util/quota/common", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/volume/util/quota/common/quota_linux_common.go b/pkg/volume/util/quota/common/quota_linux_common.go new file mode 100644 index 00000000000..a5774686acf --- /dev/null +++ b/pkg/volume/util/quota/common/quota_linux_common.go @@ -0,0 +1,77 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +// QuotaID -- generic quota identifier +type QuotaID int32 + +const ( + // BadQuotaID -- Invalid quota + BadQuotaID QuotaID = 0 +) + +// FirstQuota is the quota ID we start with. +// XXXXXXX Need a better way of doing this... +var FirstQuota QuotaID = 1048577 + +// LinuxVolumeQuotaProvider returns an appropriate quota applier +// object if we can support quotas on this device +type LinuxVolumeQuotaProvider interface { + // GetQuotaApplier retrieves an object that can apply + // quotas (or nil if this provider cannot support quotas + // on the device) + GetQuotaApplier(mountpoint string, backingDev string) LinuxVolumeQuotaApplier +} + +// LinuxVolumeQuotaApplier is a generic interface to any quota +// mechanism supported by Linux +type LinuxVolumeQuotaApplier interface { + // GetQuotaOnDir gets the quota ID (if any) that applies to + // this directory + GetQuotaOnDir(path string) (QuotaID, error) + + // SetQuotaOnDir applies the specified quota ID to a directory. + // Negative value for bytes means that a non-enforcing quota + // should be applied (perhaps by setting a quota too large to + // be hit) + SetQuotaOnDir(path string, id QuotaID, bytes int64) error + + // QuotaIDIsInUse determines whether the quota ID is in use. + // Implementations should not check /etc/project or /etc/projid, + // only whether their underlying mechanism already has the ID in + // use. + // Return value of false with no error means that the ID is not + // in use; true means that it is already in use. An error + // return means that any quota ID will fail. + QuotaIDIsInUse(path string, id QuotaID) (bool, error) + + // GetConsumption returns the consumption (in bytes) of the + // directory, determined by the implementation's quota-based + // mechanism. If it is unable to do so using that mechanism, + // it should return an error and allow higher layers to + // enumerate the directory. + GetConsumption(path string, id QuotaID) (int64, error) + + // GetInodes returns the number of inodes used by the + // directory, determined by the implementation's quota-based + // mechanism. If it is unable to do so using that mechanism, + // it should return an error and allow higher layers to + // enumerate the directory. + GetInodes(path string, id QuotaID) (int64, error) +} diff --git a/pkg/volume/util/quota/common/quota_linux_common_impl.go b/pkg/volume/util/quota/common/quota_linux_common_impl.go new file mode 100644 index 00000000000..5df76fd532f --- /dev/null +++ b/pkg/volume/util/quota/common/quota_linux_common_impl.go @@ -0,0 +1,152 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +/* +#include +#include +#include +#include +#include +#include + +#ifndef FS_XFLAG_PROJINHERIT +struct fsxattr { + __u32 fsx_xflags; + __u32 fsx_extsize; + __u32 fsx_nextents; + __u32 fsx_projid; + unsigned char fsx_pad[12]; +}; +#define FS_XFLAG_PROJINHERIT 0x00000200 +#endif +#ifndef FS_IOC_FSGETXATTR +#define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) +#endif +#ifndef FS_IOC_FSSETXATTR +#define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr) +#endif + +#ifndef PRJQUOTA +#define PRJQUOTA 2 +#endif +#ifndef Q_XGETQSTAT_PRJQUOTA +#define Q_XGETQSTAT_PRJQUOTA QCMD(Q_XGETQSTAT, PRJQUOTA) +#endif +*/ +import "C" + +import ( + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" + "k8s.io/klog" +) + +// IsFilesystemOfType determines whether the filesystem specified is of the type +// specified by the magic number +func IsFilesystemOfType(mountpoint string, backingDev string, magic int64) bool { + var buf syscall.Statfs_t + err := syscall.Statfs(mountpoint, &buf) + if err != nil { + klog.V(3).Infof("Extfs Unable to statfs %s: %v", mountpoint, err) + return false + } + if buf.Type != magic { + return false + } + + var qstat C.fs_quota_stat_t + CPath := C.CString(backingDev) + defer free(CPath) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(CPath)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0) + return errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 +} + +func free(p *C.char) { + C.free(unsafe.Pointer(p)) +} + +func openDir(path string) (*C.DIR, error) { + Cpath := C.CString(path) + defer free(Cpath) + + dir := C.opendir(Cpath) + if dir == nil { + return nil, fmt.Errorf("Can't open dir") + } + return dir, nil +} + +func closeDir(dir *C.DIR) { + if dir != nil { + C.closedir(dir) + } +} + +func getDirFd(dir *C.DIR) uintptr { + return uintptr(C.dirfd(dir)) +} + +// GetQuotaOnDir retrieves the quota ID (if any) associated with the specified directory +func GetQuotaOnDir(path string) (QuotaID, error) { + dir, err := openDir(path) + if err != nil { + klog.V(3).Infof("Can't open directory %s: %#+v", path, err) + return BadQuotaID, err + } + defer closeDir(dir) + var fsx C.struct_fsxattr + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return BadQuotaID, fmt.Errorf("Failed to get quota ID for %s: %v", path, errno.Error()) + } + if fsx.fsx_projid == 0 { + return BadQuotaID, fmt.Errorf("Failed to get quota ID for %s: %s", path, "no applicable quota") + } + return QuotaID(fsx.fsx_projid), nil +} + +// ApplyProjectToDir applies the specified quota ID to the specified directory +func ApplyProjectToDir(path string, id QuotaID) error { + dir, err := openDir(path) + if err != nil { + return err + } + defer closeDir(dir) + + var fsx C.struct_fsxattr + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return fmt.Errorf("Failed to get quota ID for %s: %v", path, errno.Error()) + } + fsx.fsx_projid = C.__u32(id) + fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT + _, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return fmt.Errorf("Failed to set quota ID for %s: %v", path, errno.Error()) + } + return nil +} diff --git a/pkg/volume/util/quota/extfs/BUILD b/pkg/volume/util/quota/extfs/BUILD new file mode 100644 index 00000000000..431386beed0 --- /dev/null +++ b/pkg/volume/util/quota/extfs/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["quota_extfs.go"], + cgo = True, + importpath = "k8s.io/kubernetes/pkg/volume/util/quota/extfs", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/volume/util/quota/common:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/volume/util/quota/extfs/quota_extfs.go b/pkg/volume/util/quota/extfs/quota_extfs.go new file mode 100644 index 00000000000..397e3ba79e9 --- /dev/null +++ b/pkg/volume/util/quota/extfs/quota_extfs.go @@ -0,0 +1,153 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extfs + +/* +#include +#include +#include +#include +#include + +#ifndef PRJQUOTA +#define PRJQUOTA 2 +#endif +#ifndef Q_SETPQUOTA +#define Q_SETPQUOTA (unsigned) QCMD(Q_SETQUOTA, PRJQUOTA) +#endif +#ifndef Q_GETPQUOTA +#define Q_GETPQUOTA (unsigned) QCMD(Q_GETQUOTA, PRJQUOTA) +#endif +*/ +import "C" + +import ( + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/volume/util/quota/common" +) + +// ext4fs empirically has a maximum quota size of 2^48 - 1 1KiB blocks (256 petabytes) +const ( + linuxExtfsMagic = 0xef53 + quotaBsize = 1024 // extfs specific + bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 + maxQuota int64 = (1<<(bitsPerWord-1) - 1) & (1<<58 - 1) // either 1<<31 - 1 or 1<<58 - 1 +) + +// VolumeProvider supplies a quota applier to the generic code. +type VolumeProvider struct { +} + +// GetQuotaApplier -- does this backing device support quotas that +// can be applied to directories? +func (*VolumeProvider) GetQuotaApplier(mountpoint string, backingDev string) common.LinuxVolumeQuotaApplier { + if common.IsFilesystemOfType(mountpoint, backingDev, linuxExtfsMagic) { + return extfsVolumeQuota{backingDev} + } + return nil +} + +type extfsVolumeQuota struct { + backingDev string +} + +// GetQuotaOnDir -- get the quota ID that applies to this directory. + +func (v extfsVolumeQuota) GetQuotaOnDir(path string) (common.QuotaID, error) { + return common.GetQuotaOnDir(path) +} + +// SetQuotaOnDir -- apply the specified quota to the directory. If +// bytes is not greater than zero, the quota should be applied in a +// way that is non-enforcing (either explicitly so or by setting a +// quota larger than anything the user may possibly create) +func (v extfsVolumeQuota) SetQuotaOnDir(path string, id common.QuotaID, bytes int64) error { + klog.V(3).Infof("extfsSetQuotaOn %s ID %v bytes %v", path, id, bytes) + if bytes < 0 || bytes > maxQuota { + bytes = maxQuota + } + + var d C.struct_if_dqblk + + d.dqb_bhardlimit = C.__u64(bytes / quotaBsize) + d.dqb_bsoftlimit = d.dqb_bhardlimit + d.dqb_ihardlimit = 0 + d.dqb_isoftlimit = 0 + d.dqb_valid = C.QIF_LIMITS + + var cs = C.CString(v.backingDev) + defer C.free(unsafe.Pointer(cs)) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_SETPQUOTA, + uintptr(unsafe.Pointer(cs)), uintptr(id), + uintptr(unsafe.Pointer(&d)), 0, 0) + if errno != 0 { + return fmt.Errorf("Failed to set quota limit for ID %d on %s: %v", + id, path, errno.Error()) + } + return common.ApplyProjectToDir(path, id) +} + +func (v extfsVolumeQuota) getQuotaInfo(path string, id common.QuotaID) (C.struct_if_dqblk, syscall.Errno) { + var d C.struct_if_dqblk + + var cs = C.CString(v.backingDev) + defer C.free(unsafe.Pointer(cs)) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_GETPQUOTA, + uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(id)), + uintptr(unsafe.Pointer(&d)), 0, 0) + return d, errno +} + +// QuotaIDIsInUse -- determine whether the quota ID is already in use. +func (v extfsVolumeQuota) QuotaIDIsInUse(path string, id common.QuotaID) (bool, error) { + d, errno := v.getQuotaInfo(path, id) + isInUse := !(d.dqb_bhardlimit == 0 && d.dqb_bsoftlimit == 0 && d.dqb_curspace == 0 && + d.dqb_ihardlimit == 0 && d.dqb_isoftlimit == 0 && d.dqb_curinodes == 0 && + d.dqb_btime == 0 && d.dqb_itime == 0) + return errno == 0 && isInUse, nil +} + +// GetConsumption -- retrieve the consumption (in bytes) of the directory +// Note that with ext[[:digit:]]fs the quota consumption is in bytes +// per man quotactl +func (v extfsVolumeQuota) GetConsumption(path string, id common.QuotaID) (int64, error) { + d, errno := v.getQuotaInfo(path, id) + if errno != 0 { + return 0, fmt.Errorf("Failed to get quota for %s: %s", path, errno.Error()) + } + klog.V(3).Infof("Consumption for %s is %v", path, d.dqb_curspace) + return int64(d.dqb_curspace), nil +} + +// GetInodes -- retrieve the number of inodes in use under the directory +func (v extfsVolumeQuota) GetInodes(path string, id common.QuotaID) (int64, error) { + d, errno := v.getQuotaInfo(path, id) + if errno != 0 { + return 0, fmt.Errorf("Failed to get quota for %s: %s", path, errno.Error()) + } + klog.V(3).Infof("Inode consumption for %s is %v", path, d.dqb_curinodes) + return int64(d.dqb_curinodes), nil +} diff --git a/pkg/volume/util/quota/project.go b/pkg/volume/util/quota/project.go new file mode 100644 index 00000000000..4caa7ecd671 --- /dev/null +++ b/pkg/volume/util/quota/project.go @@ -0,0 +1,353 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strconv" + "sync" + + "golang.org/x/sys/unix" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/volume/util/quota/common" +) + +var projectsFile = "/etc/projects" +var projidFile = "/etc/projid" + +var projectsParseRegexp *regexp.Regexp = regexp.MustCompilePOSIX("^([[:digit:]]+):(.*)$") +var projidParseRegexp *regexp.Regexp = regexp.MustCompilePOSIX("^([^#][^:]*):([[:digit:]]+)$") + +var quotaIDLock sync.RWMutex + +const maxUnusedQuotasToSearch = 128 // Don't go into an infinite loop searching for an unused quota + +type projectType struct { + isValid bool // False if we need to remove this line + id common.QuotaID + data string // Project name (projid) or directory (projects) + line string +} + +type projectsList struct { + projects []projectType + projid []projectType +} + +func projFilesAreOK() error { + if sf, err := os.Lstat(projectsFile); err != nil || sf.Mode().IsRegular() { + if sf, err := os.Lstat(projidFile); err != nil || sf.Mode().IsRegular() { + return nil + } + return fmt.Errorf("%s exists but is not a plain file, cannot continue", projidFile) + } + return fmt.Errorf("%s exists but is not a plain file, cannot continue", projectsFile) +} + +func lockFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_EX) +} + +func unlockFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_UN) +} + +// openAndLockProjectFiles opens /etc/projects and /etc/projid locked. +// Creates them if they don't exist +func openAndLockProjectFiles() (*os.File, *os.File, error) { + // Make sure neither project-related file is a symlink! + if err := projFilesAreOK(); err != nil { + return nil, nil, err + } + // We don't actually modify the original files; we create temporaries and + // move them over the originals + fProjects, err := os.OpenFile(projectsFile, os.O_RDONLY|os.O_CREATE, 0644) + if err != nil { + return nil, nil, err + } + fProjid, err := os.OpenFile(projidFile, os.O_RDONLY|os.O_CREATE, 0644) + if err == nil { + // Check once more, to ensure nothing got changed out from under us + if err := projFilesAreOK(); err == nil { + err = lockFile(fProjects) + if err == nil { + err = lockFile(fProjid) + if err == nil { + return fProjects, fProjid, nil + } + // Nothing useful we can do if we get an error here + unlockFile(fProjects) + } + } + fProjid.Close() + } + fProjects.Close() + return nil, nil, err +} + +func closeProjectFiles(fProjects *os.File, fProjid *os.File) error { + // Nothing useful we can do if either of these fail, + // but we have to close (and thereby unlock) the files anyway. + var err error + var err1 error + if fProjid != nil { + err = fProjid.Close() + } + if fProjects != nil { + err1 = fProjects.Close() + } + if err == nil { + return err1 + } + return err +} + +func parseProject(l string) projectType { + if match := projectsParseRegexp.FindStringSubmatch(l); match != nil { + i, err := strconv.Atoi(match[1]) + if err == nil { + return projectType{true, common.QuotaID(i), match[2], l} + } + } + return projectType{true, common.BadQuotaID, "", l} +} + +func parseProjid(l string) projectType { + if match := projidParseRegexp.FindStringSubmatch(l); match != nil { + i, err := strconv.Atoi(match[2]) + if err == nil { + return projectType{true, common.QuotaID(i), match[1], l} + } + } + return projectType{true, common.BadQuotaID, "", l} +} + +func parseProjFile(f *os.File, parser func(l string) projectType) []projectType { + var answer []projectType + scanner := bufio.NewScanner(f) + for scanner.Scan() { + answer = append(answer, parser(scanner.Text())) + } + return answer +} + +func readProjectFiles(projects *os.File, projid *os.File) projectsList { + return projectsList{parseProjFile(projects, parseProject), parseProjFile(projid, parseProjid)} +} + +func findAvailableQuota(path string, idMap map[common.QuotaID]bool) (common.QuotaID, error) { + unusedQuotasSearched := 0 + for id := common.FirstQuota; id == id; id++ { + if _, ok := idMap[id]; !ok { + isInUse, err := getApplier(path).QuotaIDIsInUse(path, id) + if err != nil { + return common.BadQuotaID, err + } else if !isInUse { + return id, nil + } + unusedQuotasSearched++ + if unusedQuotasSearched > maxUnusedQuotasToSearch { + break + } + } + } + return common.BadQuotaID, fmt.Errorf("Cannot find available quota ID") +} + +func addDirToProject(path string, id common.QuotaID, list *projectsList) (common.QuotaID, bool, error) { + idMap := make(map[common.QuotaID]bool) + for _, project := range list.projects { + if project.data == path { + if id != project.id { + return common.BadQuotaID, false, fmt.Errorf("Attempt to reassign project ID for %s", path) + } + // Trying to reassign a directory to the project it's + // already in. Maybe this should be an error, but for + // now treat it as an idempotent operation + return id, false, nil + } + idMap[project.id] = true + } + var needToAddProjid = true + for _, projid := range list.projid { + idMap[projid.id] = true + if projid.id == id && id != common.BadQuotaID { + needToAddProjid = false + } + } + var err error + if id == common.BadQuotaID { + id, err = findAvailableQuota(path, idMap) + if err != nil { + return common.BadQuotaID, false, err + } + needToAddProjid = true + } + if needToAddProjid { + name := fmt.Sprintf("volume%v", id) + line := fmt.Sprintf("%s:%v", name, id) + list.projid = append(list.projid, projectType{true, id, name, line}) + } + line := fmt.Sprintf("%v:%s", id, path) + list.projects = append(list.projects, projectType{true, id, path, line}) + return id, needToAddProjid, nil +} + +func removeDirFromProject(path string, id common.QuotaID, list *projectsList) (bool, error) { + if id == common.BadQuotaID { + return false, fmt.Errorf("Attempt to remove invalid quota ID from %s", path) + } + foundAt := -1 + countByID := make(map[common.QuotaID]int) + for i, project := range list.projects { + if project.data == path { + if id != project.id { + return false, fmt.Errorf("Attempting to remove quota ID %v from path %s, but expecting ID %v", id, path, project.id) + } else if foundAt != -1 { + return false, fmt.Errorf("Found multiple quota IDs for path %s", path) + } + // Faster and easier than deleting an element + list.projects[i].isValid = false + foundAt = i + } + countByID[project.id]++ + } + if foundAt == -1 { + return false, fmt.Errorf("Cannot find quota associated with path %s", path) + } + if countByID[id] <= 1 { + // Removing the last entry means that we're no longer using + // the quota ID, so remove that as well + for i, projid := range list.projid { + if projid.id == id { + list.projid[i].isValid = false + } + } + return true, nil + } + return false, nil +} + +func writeProjectFile(base *os.File, projects []projectType) (string, error) { + oname := base.Name() + stat, err := base.Stat() + if err != nil { + return "", err + } + mode := stat.Mode() & os.ModePerm + f, err := ioutil.TempFile(filepath.Dir(oname), filepath.Base(oname)) + if err != nil { + return "", err + } + filename := f.Name() + if err := os.Chmod(filename, mode); err != nil { + return "", err + } + for _, proj := range projects { + if proj.isValid { + if _, err := f.WriteString(fmt.Sprintf("%s\n", proj.line)); err != nil { + f.Close() + os.Remove(filename) + return "", err + } + } + } + if err := f.Close(); err != nil { + os.Remove(filename) + return "", err + } + return filename, nil +} + +func writeProjectFiles(fProjects *os.File, fProjid *os.File, writeProjid bool, list projectsList) error { + tmpProjects, err := writeProjectFile(fProjects, list.projects) + if err == nil { + // Ensure that both files are written before we try to rename either. + if writeProjid { + tmpProjid, err := writeProjectFile(fProjid, list.projid) + if err == nil { + err = os.Rename(tmpProjid, fProjid.Name()) + if err != nil { + os.Remove(tmpProjid) + } + } + } + if err == nil { + err = os.Rename(tmpProjects, fProjects.Name()) + if err == nil { + return nil + } + // We're in a bit of trouble here; at this + // point we've successfully renamed tmpProjid + // to the real thing, but renaming tmpProject + // to the real file failed. There's not much we + // can do in this position. Anything we could do + // to try to undo it would itself be likely to fail. + } + os.Remove(tmpProjects) + } + klog.V(3).Infof("Unable to write project files: %v", err) + return err +} + +func createProjectID(path string, ID common.QuotaID) (common.QuotaID, error) { + quotaIDLock.Lock() + defer quotaIDLock.Unlock() + fProjects, fProjid, err := openAndLockProjectFiles() + if err == nil { + defer closeProjectFiles(fProjects, fProjid) + list := readProjectFiles(fProjects, fProjid) + writeProjid := true + ID, writeProjid, err = addDirToProject(path, ID, &list) + if err == nil && ID != common.BadQuotaID { + if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil { + return ID, nil + } + } + } + klog.V(3).Infof("addQuotaID %s %v failed %v", path, ID, err) + return common.BadQuotaID, err +} + +func removeProjectID(path string, ID common.QuotaID) error { + if ID == common.BadQuotaID { + return fmt.Errorf("attempting to remove invalid quota ID %v", ID) + } + quotaIDLock.Lock() + defer quotaIDLock.Unlock() + fProjects, fProjid, err := openAndLockProjectFiles() + if err == nil { + defer closeProjectFiles(fProjects, fProjid) + list := readProjectFiles(fProjects, fProjid) + writeProjid := true + writeProjid, err = removeDirFromProject(path, ID, &list) + if err == nil { + if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil { + return nil + } + } + } + klog.V(3).Infof("removeQuotaID %s %v failed %v", path, ID, err) + return err +} diff --git a/pkg/volume/util/quota/quota.go b/pkg/volume/util/quota/quota.go new file mode 100644 index 00000000000..1bfa7d7c69f --- /dev/null +++ b/pkg/volume/util/quota/quota.go @@ -0,0 +1,47 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/util/mount" +) + +// Interface -- quota interface +type Interface interface { + // Does the path provided support quotas, and if so, what types + SupportsQuotas(m mount.Interface, path string) (bool, error) + // Assign a quota (picked by the quota mechanism) to a path, + // and return it. + AssignQuota(m mount.Interface, path string, poduid string, bytes int64) error + + // Get the quota-based storage consumption for the path + GetConsumption(path string) (int64, error) + + // Get the quota-based inode consumption for the path + GetInodes(path string) (int64, error) + + // Remove the quota from a path + // Implementations may assume that any data covered by the + // quota has already been removed. + ClearQuota(m mount.Interface, path string, poduid string) error +} + +func enabledQuotasForMonitoring() bool { + return utilfeature.DefaultFeatureGate.Enabled(features.FSQuotaForLSCIMonitoring) +} diff --git a/pkg/volume/util/quota/quota_linux.go b/pkg/volume/util/quota/quota_linux.go new file mode 100644 index 00000000000..664d2bc38d3 --- /dev/null +++ b/pkg/volume/util/quota/quota_linux.go @@ -0,0 +1,432 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "sync" + + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume/util/quota/common" + "k8s.io/kubernetes/pkg/volume/util/quota/extfs" + "k8s.io/kubernetes/pkg/volume/util/quota/xfs" +) + +// Pod -> ID +var podQuotaMap = make(map[string]common.QuotaID) + +// Dir -> ID (for convenience) +var dirQuotaMap = make(map[string]common.QuotaID) + +// ID -> pod +var quotaPodMap = make(map[common.QuotaID]string) + +// Directory -> pod +var dirPodMap = make(map[string]string) + +// Backing device -> applier +// This is *not* cleaned up; its size will be bounded. +var devApplierMap = make(map[string]common.LinuxVolumeQuotaApplier) + +// Directory -> applier +var dirApplierMap = make(map[string]common.LinuxVolumeQuotaApplier) +var dirApplierLock sync.RWMutex + +// Pod -> refcount +var podDirCountMap = make(map[string]int) + +// ID -> size +var quotaSizeMap = make(map[common.QuotaID]int64) +var quotaLock sync.RWMutex + +var supportsQuotasMap = make(map[string]bool) +var supportsQuotasLock sync.RWMutex + +var mountParseRegexp *regexp.Regexp = regexp.MustCompilePOSIX("^([^ ]*)[ \t]*([^ ]*)[ \t]*([^ ]*)") // Ignore options etc. + +// Directory -> backingDev +var backingDevMap = make(map[string]string) +var backingDevLock sync.RWMutex + +var mountpointMap = make(map[string]string) +var mountpointLock sync.RWMutex + +var mountsFile = "/proc/self/mounts" + +var providers = []common.LinuxVolumeQuotaProvider{ + &extfs.VolumeProvider{}, + &xfs.VolumeProvider{}, +} + +// Separate the innards for ease of testing +func detectBackingDevInternal(mountpoint string, mounts string) (string, error) { + file, err := os.Open(mounts) + if err != nil { + return "", err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + match := mountParseRegexp.FindStringSubmatch(scanner.Text()) + if match != nil { + device := match[1] + mount := match[2] + if mount == mountpoint { + return device, nil + } + } + } + return "", fmt.Errorf("couldn't find backing device for %s", mountpoint) +} + +// detectBackingDev assumes that the mount point provided is valid +func detectBackingDev(_ mount.Interface, mountpoint string) (string, error) { + return detectBackingDevInternal(mountpoint, mountsFile) +} + +func clearBackingDev(path string) { + backingDevLock.Lock() + defer backingDevLock.Unlock() + delete(backingDevMap, path) +} + +// Assumes that the path has been fully canonicalized +// Breaking this up helps with testing +func detectMountpointInternal(m mount.Interface, path string) (string, error) { + for path != "" && path != "/" { + // per pkg/util/mount/mount_linux this detects all but + // a bind mount from one part of a mount to another. + // For our purposes that's fine; we simply want the "true" + // mount point + // + // IsNotMountPoint proved much more troublesome; it actually + // scans the mounts, and when a lot of mount/unmount + // activity takes place, it is not able to get a consistent + // view of /proc/self/mounts, causing it to time out and + // report incorrectly. + isNotMount, err := m.IsLikelyNotMountPoint(path) + if err != nil { + return "/", err + } + if !isNotMount { + return path, nil + } + path = filepath.Dir(path) + } + return "/", nil +} + +func detectMountpoint(m mount.Interface, path string) (string, error) { + xpath, err := filepath.Abs(path) + if err == nil { + if xpath, err = filepath.EvalSymlinks(xpath); err == nil { + if xpath, err = detectMountpointInternal(m, xpath); err == nil { + return xpath, nil + } + } + } + return "/", err +} + +func clearMountpoint(path string) { + mountpointLock.Lock() + defer mountpointLock.Unlock() + delete(mountpointMap, path) +} + +// getFSInfo Returns mountpoint and backing device +// getFSInfo should cache the mountpoint and backing device for the +// path. +func getFSInfo(m mount.Interface, path string) (string, string, error) { + mountpointLock.Lock() + defer mountpointLock.Unlock() + + backingDevLock.Lock() + defer backingDevLock.Unlock() + + var err error + + mountpoint, okMountpoint := mountpointMap[path] + if !okMountpoint { + mountpoint, err = detectMountpoint(m, path) + klog.V(3).Infof("Mountpoint %s -> %s (%v)", path, mountpoint, err) + if err != nil { + return "", "", err + } + } + + backingDev, okBackingDev := backingDevMap[path] + if !okBackingDev { + backingDev, err = detectBackingDev(m, mountpoint) + klog.V(3).Infof("Backing dev %s -> %s (%v)", path, backingDev, err) + if err != nil { + return "", "", err + } + } + mountpointMap[path] = mountpoint + backingDevMap[path] = backingDev + return mountpoint, backingDev, nil +} + +func clearFSInfo(path string) { + clearMountpoint(path) + clearBackingDev(path) +} + +func getApplier(path string) common.LinuxVolumeQuotaApplier { + dirApplierLock.Lock() + defer dirApplierLock.Unlock() + return dirApplierMap[path] +} + +func setApplier(path string, applier common.LinuxVolumeQuotaApplier) { + dirApplierLock.Lock() + defer dirApplierLock.Unlock() + dirApplierMap[path] = applier +} + +func clearApplier(path string) { + dirApplierLock.Lock() + defer dirApplierLock.Unlock() + delete(dirApplierMap, path) +} + +func setQuotaOnDir(path string, id common.QuotaID, bytes int64) error { + return getApplier(path).SetQuotaOnDir(path, id, bytes) +} + +func getQuotaOnDir(m mount.Interface, path string) (common.QuotaID, error) { + _, _, err := getFSInfo(m, path) + if err != nil { + return common.BadQuotaID, err + } + return getApplier(path).GetQuotaOnDir(path) +} + +func clearQuotaOnDir(m mount.Interface, path string) error { + // Since we may be called without path being in the map, + // we explicitly have to check in this case. + klog.V(3).Infof("clearQuotaOnDir %s", path) + supportsQuotas, err := SupportsQuotas(m, path) + if !supportsQuotas { + return nil + } + projid, err := getQuotaOnDir(m, path) + if err == nil && projid != common.BadQuotaID { + klog.V(3).Infof("clearQuotaOnDir clearing quota") + // This means that we have a quota on the directory but + // we can't clear it. That's not good. + err = setQuotaOnDir(path, projid, 0) + if err != nil { + klog.V(3).Infof("Attempt to clear quota failed: %v", err) + } + err1 := removeProjectID(path, projid) + if err1 != nil { + klog.V(3).Infof("Attempt to remove quota ID from system files failed: %v", err1) + } + clearFSInfo(path) + if err != nil { + return err + } + return err1 + } + klog.V(3).Infof("clearQuotaOnDir fails %v", err) + // If we couldn't get a quota, that's fine -- there may + // never have been one, and we have no way to know otherwise + return nil +} + +// SupportsQuotas -- Does the path support quotas +// Cache the applier for paths that support quotas. For paths that don't, +// don't cache the result because nothing will clean it up. +// However, do cache the device->applier map; the number of devices +// is bounded. +func SupportsQuotas(m mount.Interface, path string) (bool, error) { + if !enabledQuotasForMonitoring() { + klog.V(3).Info("SupportsQuotas called, but quotas disabled") + return false, nil + } + supportsQuotasLock.Lock() + defer supportsQuotasLock.Unlock() + if supportsQuotas, ok := supportsQuotasMap[path]; ok { + return supportsQuotas, nil + } + mount, dev, err := getFSInfo(m, path) + klog.V(3).Infof("SupportsQuotas %s -> mount %s dev %s %v", path, mount, dev, err) + if err != nil { + return false, err + } + // Do we know about this device? + applier, ok := devApplierMap[mount] + if !ok { + for _, provider := range providers { + if applier = provider.GetQuotaApplier(mount, dev); applier != nil { + devApplierMap[mount] = applier + break + } + } + } + if applier != nil { + klog.V(3).Infof("SupportsQuotas got applier %v", applier) + supportsQuotasMap[path] = true + setApplier(path, applier) + return true, nil + } + klog.V(3).Infof("SupportsQuotas got no applier") + delete(backingDevMap, path) + delete(mountpointMap, path) + return false, nil +} + +// AssignQuota -- assign a quota to the specified directory. +// AssignQuota chooses the quota ID based on the pod UID and path. +// If the pod UID is identical to another one known, it may (but presently +// doesn't) choose the same quota ID as other volumes in the pod. +func AssignQuota(m mount.Interface, path string, poduid string, bytes int64) error { + if ok, err := SupportsQuotas(m, path); !ok { + return fmt.Errorf("Quotas not supported on %s: %v", path, err) + } + quotaLock.Lock() + defer quotaLock.Unlock() + // Current policy is to set individual quotas on each volumes. + // If we decide later that we want to assign one quota for all + // volumes in a pod, we can simply remove this line of code. + // If and when we decide permanently that we're going to adop + // one quota per volume, we can rip all of the pod code out. + poduid = string(uuid.NewUUID()) + klog.V(3).Infof("Synthesizing pod ID %s for directory %s in AssignQuota", poduid, path) + if pod, ok := dirPodMap[path]; ok && pod != poduid { + return fmt.Errorf("Requesting quota on existing directory %s but different pod %s %s", path, pod, poduid) + } + oid, ok := podQuotaMap[poduid] + if ok { + if quotaSizeMap[oid] != bytes { + return fmt.Errorf("Requesting quota of different size: old %v new %v", quotaSizeMap[oid], bytes) + } + } else { + oid = common.BadQuotaID + } + id, err := createProjectID(path, oid) + if err == nil { + if oid != common.BadQuotaID && oid != id { + klog.V(3).Infof("Attempt to reassign quota %v to %v", oid, id) + return fmt.Errorf("Attempt to reassign quota %v to %v", oid, id) + } + // When enforcing quotas are enabled, we'll condition this + // on their being disabled also. + if bytes > 0 { + bytes = -1 + } + if err = setQuotaOnDir(path, id, bytes); err == nil { + quotaPodMap[id] = poduid + quotaSizeMap[id] = bytes + podQuotaMap[poduid] = id + dirQuotaMap[path] = id + dirPodMap[path] = poduid + podDirCountMap[poduid]++ + return nil + } + removeProjectID(path, id) + } + klog.V(3).Infof("Assign quota FAILED %v", err) + return err +} + +// GetConsumption -- retrieve the consumption (in bytes) of the directory +func GetConsumption(path string) (int64, error) { + // Note that we actually need to hold the lock at least through + // running the quota command, so it can't get recycled behind our back + quotaLock.Lock() + defer quotaLock.Unlock() + applier := getApplier(path) + if applier == nil { + return 0, fmt.Errorf("No quota available for %s", path) + } + return applier.GetConsumption(path, dirQuotaMap[path]) +} + +// GetInodes -- retrieve the number of inodes in use under the directory +func GetInodes(path string) (int64, error) { + // Note that we actually need to hold the lock at least through + // running the quota command, so it can't get recycled behind our back + quotaLock.Lock() + defer quotaLock.Unlock() + applier := getApplier(path) + if applier == nil { + return 0, fmt.Errorf("No quota available for %s", path) + } + return applier.GetInodes(path, dirQuotaMap[path]) +} + +// ClearQuota -- remove the quota assigned to a directory +func ClearQuota(m mount.Interface, path string) error { + klog.V(3).Infof("ClearQuota %s", path) + if !enabledQuotasForMonitoring() { + klog.V(3).Info("ClearQuota called, but quotas disabled") + return fmt.Errorf("ClearQuota called, but quotas disabled") + } + quotaLock.Lock() + defer quotaLock.Unlock() + poduid, ok := dirPodMap[path] + if !ok { + // Nothing in the map either means that there was no + // quota to begin with or that we're clearing a + // stale directory, so if we find a quota, just remove it. + // The process of clearing the quota requires that an applier + // be found, which needs to be cleaned up. + defer delete(supportsQuotasMap, path) + defer clearApplier(path) + return clearQuotaOnDir(m, path) + } + _, ok = podQuotaMap[poduid] + if !ok { + return fmt.Errorf("ClearQuota: No quota available for %s", path) + } + var err error + projid, err := getQuotaOnDir(m, path) + if projid != dirQuotaMap[path] { + return fmt.Errorf("Expected quota ID %v on dir %s does not match actual %v", dirQuotaMap[path], path, projid) + } + count, ok := podDirCountMap[poduid] + if count <= 1 || !ok { + err = clearQuotaOnDir(m, path) + if err != nil { + klog.V(3).Infof("Unable to clear quota %v %s: %v", dirQuotaMap[path], path, err) + } + delete(quotaSizeMap, podQuotaMap[poduid]) + delete(quotaPodMap, podQuotaMap[poduid]) + delete(podDirCountMap, poduid) + delete(podQuotaMap, poduid) + } else { + err = removeProjectID(path, projid) + podDirCountMap[poduid]-- + klog.V(3).Infof("Not clearing quota for pod %s; still %v dirs outstanding", poduid, podDirCountMap[poduid]) + } + delete(dirPodMap, path) + delete(dirQuotaMap, path) + delete(supportsQuotasMap, path) + clearApplier(path) + return err +} diff --git a/pkg/volume/util/quota/quota_linux_test.go b/pkg/volume/util/quota/quota_linux_test.go new file mode 100644 index 00000000000..9366821c75c --- /dev/null +++ b/pkg/volume/util/quota/quota_linux_test.go @@ -0,0 +1,762 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "fmt" + "io/ioutil" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume/util/quota/common" + "os" + "strings" + "testing" +) + +const dummyMountData = `sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 +proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 +devtmpfs /dev devtmpfs rw,nosuid,size=6133536k,nr_inodes=1533384,mode=755 0 0 +tmpfs /tmp tmpfs rw,nosuid,nodev 0 0 +/dev/sda1 /boot ext4 rw,relatime 0 0 +/dev/mapper/fedora-root / ext4 rw,noatime 0 0 +/dev/mapper/fedora-home /home ext4 rw,noatime 0 0 +/dev/sdb1 /virt xfs rw,noatime,attr2,inode64,usrquota,prjquota 0 0 +` + +const dummyMountDataPquota = `tmpfs /tmp tmpfs rw,nosuid,nodev 0 0 +/dev/sda1 /boot ext4 rw,relatime 0 0 +/dev/mapper/fedora-root / ext4 rw,noatime 0 0 +/dev/mapper/fedora-home /home ext4 rw,noatime 0 0 +/dev/sdb1 /mnt/virt xfs rw,noatime,attr2,inode64,usrquota,prjquota 0 0 +` +const dummyMountDataNoPquota = `tmpfs /tmp tmpfs rw,nosuid,nodev 0 0 +/dev/sda1 /boot ext4 rw,relatime 0 0 +/dev/mapper/fedora-root / ext4 rw,noatime 0 0 +/dev/mapper/fedora-home /home ext4 rw,noatime 0 0 +/dev/sdb1 /mnt/virt xfs rw,noatime,attr2,inode64,usrquota 0 0 +` + +const dummyMountTest = `/dev/sda1 / ext4 rw,noatime 0 0 +/dev/sda2 /quota ext4 rw,prjquota 0 0 +/dev/sda3 /noquota ext4 rw 0 0 +` + +func dummyFakeMount1() mount.Interface { + return &mount.FakeMounter{ + MountPoints: []mount.MountPoint{ + { + Device: "tmpfs", + Path: "/tmp", + Type: "tmpfs", + Opts: []string{"rw", "nosuid", "nodev"}, + }, + { + Device: "/dev/sda1", + Path: "/boot", + Type: "ext4", + Opts: []string{"rw", "relatime"}, + }, + { + Device: "dev/mapper/fedora-root", + Path: "/", + Type: "ext4", + Opts: []string{"rw", "relatime"}, + }, + { + Device: "dev/mapper/fedora-home", + Path: "/home", + Type: "ext4", + Opts: []string{"rw", "relatime"}, + }, + { + Device: "/dev/sdb1", + Path: "/mnt/virt", + Type: "xfs", + Opts: []string{"rw", "relatime", "attr2", "inode64", "usrquota", "prjquota"}, + }, + }, + } +} + +type backingDevTest struct { + path string + mountdata string + expectedResult string + expectFailure bool +} + +type mountpointTest struct { + path string + mounter mount.Interface + expectedResult string + expectFailure bool +} + +func testBackingDev1(testcase backingDevTest) error { + tmpfile, err := ioutil.TempFile("", "backingdev") + if err != nil { + return err + } + defer os.Remove(tmpfile.Name()) + if _, err = tmpfile.WriteString(testcase.mountdata); err != nil { + return err + } + + backingDev, err := detectBackingDevInternal(testcase.path, tmpfile.Name()) + if err != nil { + if testcase.expectFailure { + return nil + } + return err + } + if testcase.expectFailure { + return fmt.Errorf("Path %s expected to fail; succeeded and got %s", testcase.path, backingDev) + } + if backingDev == testcase.expectedResult { + return nil + } + return fmt.Errorf("Mismatch: path %s expects mountpoint %s got %s", testcase.path, testcase.expectedResult, backingDev) +} + +func TestBackingDev(t *testing.T) { + testcasesBackingDev := map[string]backingDevTest{ + "Root": { + "/", + dummyMountData, + "/dev/mapper/fedora-root", + false, + }, + "tmpfs": { + "/tmp", + dummyMountData, + "tmpfs", + false, + }, + "user filesystem": { + "/virt", + dummyMountData, + "/dev/sdb1", + false, + }, + "empty mountpoint": { + "", + dummyMountData, + "", + true, + }, + "bad mountpoint": { + "/kiusf", + dummyMountData, + "", + true, + }, + } + for name, testcase := range testcasesBackingDev { + err := testBackingDev1(testcase) + if err != nil { + t.Errorf("%s failed: %s", name, err.Error()) + } + } +} + +func TestDetectMountPoint(t *testing.T) { + testcasesMount := map[string]mountpointTest{ + "Root": { + "/", + dummyFakeMount1(), + "/", + false, + }, + "(empty)": { + "", + dummyFakeMount1(), + "/", + false, + }, + "(invalid)": { + "", + dummyFakeMount1(), + "/", + false, + }, + "/usr": { + "/usr", + dummyFakeMount1(), + "/", + false, + }, + "/var/tmp": { + "/var/tmp", + dummyFakeMount1(), + "/", + false, + }, + } + for name, testcase := range testcasesMount { + mountpoint, err := detectMountpointInternal(testcase.mounter, testcase.path) + if err == nil && testcase.expectFailure { + t.Errorf("Case %s expected failure, but succeeded, returning mountpoint %s", name, mountpoint) + } else if err != nil { + t.Errorf("Case %s failed: %s", name, err.Error()) + } else if mountpoint != testcase.expectedResult { + t.Errorf("Case %s got mountpoint %s, expected %s", name, mountpoint, testcase.expectedResult) + } + } +} + +var dummyMountPoints = []mount.MountPoint{ + { + Device: "/dev/sda2", + Path: "/quota1", + Type: "ext4", + Opts: []string{"rw", "relatime", "prjquota"}, + }, + { + Device: "/dev/sda3", + Path: "/quota2", + Type: "ext4", + Opts: []string{"rw", "relatime", "prjquota"}, + }, + { + Device: "/dev/sda3", + Path: "/noquota", + Type: "ext4", + Opts: []string{"rw", "relatime"}, + }, + { + Device: "/dev/sda1", + Path: "/", + Type: "ext4", + Opts: []string{"rw", "relatime"}, + }, +} + +func dummyQuotaTest() mount.Interface { + return &mount.FakeMounter{ + MountPoints: dummyMountPoints, + } +} + +func dummySetFSInfo(path string) { + if enabledQuotasForMonitoring() { + for _, mount := range dummyMountPoints { + if strings.HasPrefix(path, mount.Path) { + mountpointMap[path] = mount.Path + backingDevMap[path] = mount.Device + return + } + } + } +} + +type VolumeProvider1 struct { +} + +type VolumeProvider2 struct { +} + +type testVolumeQuota struct { +} + +func logAllMaps(where string) { + fmt.Printf("Maps at %s\n", where) + fmt.Printf(" Map podQuotaMap contents:\n") + for key, val := range podQuotaMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map dirQuotaMap contents:\n") + for key, val := range dirQuotaMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map quotaPodMap contents:\n") + for key, val := range quotaPodMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map dirPodMap contents:\n") + for key, val := range dirPodMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map devApplierMap contents:\n") + for key, val := range devApplierMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map dirApplierMap contents:\n") + for key, val := range dirApplierMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map podDirCountMap contents:\n") + for key, val := range podDirCountMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map quotaSizeMap contents:\n") + for key, val := range quotaSizeMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map supportsQuotasMap contents:\n") + for key, val := range supportsQuotasMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map backingDevMap contents:\n") + for key, val := range backingDevMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf(" Map mountpointMap contents:\n") + for key, val := range mountpointMap { + fmt.Printf(" %v -> %v\n", key, val) + } + fmt.Printf("End maps %s\n", where) +} + +var testIDQuotaMap = make(map[common.QuotaID]string) +var testQuotaIDMap = make(map[string]common.QuotaID) + +func (*VolumeProvider1) GetQuotaApplier(mountpoint string, backingDev string) common.LinuxVolumeQuotaApplier { + if strings.HasPrefix(mountpoint, "/quota1") { + return testVolumeQuota{} + } + return nil +} + +func (*VolumeProvider2) GetQuotaApplier(mountpoint string, backingDev string) common.LinuxVolumeQuotaApplier { + if strings.HasPrefix(mountpoint, "/quota2") { + return testVolumeQuota{} + } + return nil +} + +func (v testVolumeQuota) SetQuotaOnDir(dir string, id common.QuotaID, _ int64) error { + odir, ok := testIDQuotaMap[id] + if ok && dir != odir { + return fmt.Errorf("ID %v is already in use", id) + } + oid, ok := testQuotaIDMap[dir] + if ok && id != oid { + return fmt.Errorf("Directory %s already has a quota applied", dir) + } + testQuotaIDMap[dir] = id + testIDQuotaMap[id] = dir + return nil +} + +func (v testVolumeQuota) GetQuotaOnDir(path string) (common.QuotaID, error) { + id, ok := testQuotaIDMap[path] + if ok { + return id, nil + } + return common.BadQuotaID, fmt.Errorf("No quota available for %s", path) +} + +func (v testVolumeQuota) QuotaIDIsInUse(_ string, id common.QuotaID) (bool, error) { + if _, ok := testIDQuotaMap[id]; ok { + return true, nil + } + // So that we reject some + if id%3 == 0 { + return false, nil + } + return false, nil +} + +func (v testVolumeQuota) GetConsumption(_ string, _ common.QuotaID) (int64, error) { + return 4096, nil +} + +func (v testVolumeQuota) GetInodes(_ string, _ common.QuotaID) (int64, error) { + return 1, nil +} + +func fakeSupportsQuotas(path string) (bool, error) { + dummySetFSInfo(path) + return SupportsQuotas(dummyQuotaTest(), path) +} + +func fakeAssignQuota(path string, poduid string, bytes int64) error { + dummySetFSInfo(path) + return AssignQuota(dummyQuotaTest(), path, poduid, bytes) +} + +func fakeClearQuota(path string) error { + dummySetFSInfo(path) + return ClearQuota(dummyQuotaTest(), path) +} + +type quotaTestCase struct { + path string + poduid string + bytes int64 + op string + expectedProjects string + expectedProjid string + supportsQuota bool + expectsSetQuota bool + deltaExpectedPodQuotaCount int + deltaExpectedDirQuotaCount int + deltaExpectedQuotaPodCount int + deltaExpectedDirPodCount int + deltaExpectedDevApplierCount int + deltaExpectedDirApplierCount int + deltaExpectedPodDirCountCount int + deltaExpectedQuotaSizeCount int + deltaExpectedSupportsQuotasCount int + deltaExpectedBackingDevCount int + deltaExpectedMountpointCount int +} + +const ( + projectsHeader = `# This is a /etc/projects header +1048578:/quota/d +` + projects1 = `1048577:/quota1/a +` + projects2 = `1048577:/quota1/a +1048580:/quota1/b +` + projects3 = `1048577:/quota1/a +1048580:/quota1/b +1048581:/quota2/b +` + projects4 = `1048577:/quota1/a +1048581:/quota2/b +` + projects5 = `1048581:/quota2/b +` + + projidHeader = `# This is a /etc/projid header +xxxxxx:1048579 +` + projid1 = `volume1048577:1048577 +` + projid2 = `volume1048577:1048577 +volume1048580:1048580 +` + projid3 = `volume1048577:1048577 +volume1048580:1048580 +volume1048581:1048581 +` + projid4 = `volume1048577:1048577 +volume1048581:1048581 +` + projid5 = `volume1048581:1048581 +` +) + +var quotaTestCases = []quotaTestCase{ + { + "/quota1/a", "", 1024, "Supports", "", "", + true, true, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, + }, + { + "/quota1/a", "", 1024, "Set", projects1, projid1, + true, true, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, + }, + { + "/quota1/b", "x", 1024, "Set", projects2, projid2, + true, true, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, + }, + { + "/quota2/b", "x", 1024, "Set", projects3, projid3, + true, true, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + }, + { + "/quota1/b", "x", 1024, "Set", projects3, projid3, + true, false, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }, + { + "/quota1/b", "", 1024, "Clear", projects4, projid4, + true, true, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, + }, + { + "/noquota/a", "", 1024, "Supports", projects4, projid4, + false, false, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }, + { + "/quota1/a", "", 1024, "Clear", projects5, projid5, + true, true, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, + }, + { + "/quota1/a", "", 1024, "Clear", projects5, projid5, + true, false, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }, + { + "/quota2/b", "", 1024, "Clear", "", "", + true, true, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, + }, +} + +func compareProjectsFiles(t *testing.T, testcase quotaTestCase, projectsFile string, projidFile string, enabled bool) { + bytes, err := ioutil.ReadFile(projectsFile) + if err != nil { + t.Error(err.Error()) + } else { + s := string(bytes) + p := projectsHeader + if enabled { + p += testcase.expectedProjects + } + if s != p { + t.Errorf("Case %v /etc/projects miscompare: expected\n`%s`\ngot\n`%s`\n", testcase.path, p, s) + } + } + bytes, err = ioutil.ReadFile(projidFile) + if err != nil { + t.Error(err.Error()) + } else { + s := string(bytes) + p := projidHeader + if enabled { + p += testcase.expectedProjid + } + if s != p { + t.Errorf("Case %v /etc/projid miscompare: expected\n`%s`\ngot\n`%s`\n", testcase.path, p, s) + } + } +} + +func setFeature(feature utilfeature.Feature, value bool) error { + v := "true" + if !value { + v = "false" + } + return utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=%s", string(feature), v)) +} + +func runCaseEnabled(t *testing.T, testcase quotaTestCase, seq int) bool { + fail := false + var err error + switch testcase.op { + case "Supports": + supports, err := fakeSupportsQuotas(testcase.path) + if err != nil { + fail = true + t.Errorf("Case %v (%s, %v) Got error in fakeSupportsQuotas: %v", seq, testcase.path, true, err) + } + if supports != testcase.supportsQuota { + fail = true + t.Errorf("Case %v (%s, %v) fakeSupportsQuotas got %v, expect %v", seq, testcase.path, true, supports, testcase.supportsQuota) + } + return fail + case "Set": + err = fakeAssignQuota(testcase.path, testcase.poduid, testcase.bytes) + case "Clear": + err = fakeClearQuota(testcase.path) + case "GetConsumption": + _, err = GetConsumption(testcase.path) + case "GetInodes": + _, err = GetInodes(testcase.path) + default: + t.Errorf("Case %v (%s, %v) unknown operation %s", seq, testcase.path, true, testcase.op) + return true + } + if err != nil && testcase.expectsSetQuota { + fail = true + t.Errorf("Case %v (%s, %v) %s expected to clear quota but failed %v", seq, testcase.path, true, testcase.op, err) + } else if err == nil && !testcase.expectsSetQuota { + fail = true + t.Errorf("Case %v (%s, %v) %s expected not to clear quota but succeeded", seq, testcase.path, true, testcase.op) + } + return fail +} + +func runCaseDisabled(t *testing.T, testcase quotaTestCase, seq int) bool { + var err error + var supports bool + switch testcase.op { + case "Supports": + if supports, err = fakeSupportsQuotas(testcase.path); supports { + t.Errorf("Case %v (%s, %v) supports quotas but shouldn't", seq, testcase.path, false) + return true + } + return false + case "Set": + err = fakeAssignQuota(testcase.path, testcase.poduid, testcase.bytes) + case "Clear": + err = fakeClearQuota(testcase.path) + case "GetConsumption": + _, err = GetConsumption(testcase.path) + case "GetInodes": + _, err = GetInodes(testcase.path) + default: + t.Errorf("Case %v (%s, %v) unknown operation %s", seq, testcase.path, false, testcase.op) + return true + } + if err == nil { + t.Errorf("Case %v (%s, %v) %s: supports quotas but shouldn't", seq, testcase.path, false, testcase.op) + return true + } + return false +} + +func testAddRemoveQuotas(t *testing.T, enabled bool) { + if err := setFeature(features.FSQuotaForLSCIMonitoring, enabled); err != nil { + t.Errorf("Unable to enable LSCI monitoring: %v", err) + } + tmpProjectsFile, err := ioutil.TempFile("", "projects") + if err == nil { + _, err = tmpProjectsFile.WriteString(projectsHeader) + } + if err != nil { + t.Errorf("Unable to create fake projects file") + } + projectsFile = tmpProjectsFile.Name() + tmpProjectsFile.Close() + tmpProjidFile, err := ioutil.TempFile("", "projid") + if err == nil { + _, err = tmpProjidFile.WriteString(projidHeader) + } + if err != nil { + t.Errorf("Unable to create fake projid file") + } + projidFile = tmpProjidFile.Name() + tmpProjidFile.Close() + providers = []common.LinuxVolumeQuotaProvider{ + &VolumeProvider1{}, + &VolumeProvider2{}, + } + for k := range podQuotaMap { + delete(podQuotaMap, k) + } + for k := range dirQuotaMap { + delete(dirQuotaMap, k) + } + for k := range quotaPodMap { + delete(quotaPodMap, k) + } + for k := range dirPodMap { + delete(dirPodMap, k) + } + for k := range devApplierMap { + delete(devApplierMap, k) + } + for k := range dirApplierMap { + delete(dirApplierMap, k) + } + for k := range podDirCountMap { + delete(podDirCountMap, k) + } + for k := range quotaSizeMap { + delete(quotaSizeMap, k) + } + for k := range supportsQuotasMap { + delete(supportsQuotasMap, k) + } + for k := range backingDevMap { + delete(backingDevMap, k) + } + for k := range mountpointMap { + delete(mountpointMap, k) + } + for k := range testIDQuotaMap { + delete(testIDQuotaMap, k) + } + for k := range testQuotaIDMap { + delete(testQuotaIDMap, k) + } + expectedPodQuotaCount := 0 + expectedDirQuotaCount := 0 + expectedQuotaPodCount := 0 + expectedDirPodCount := 0 + expectedDevApplierCount := 0 + expectedDirApplierCount := 0 + expectedPodDirCountCount := 0 + expectedQuotaSizeCount := 0 + expectedSupportsQuotasCount := 0 + expectedBackingDevCount := 0 + expectedMountpointCount := 0 + for seq, testcase := range quotaTestCases { + if enabled { + expectedPodQuotaCount += testcase.deltaExpectedPodQuotaCount + expectedDirQuotaCount += testcase.deltaExpectedDirQuotaCount + expectedQuotaPodCount += testcase.deltaExpectedQuotaPodCount + expectedDirPodCount += testcase.deltaExpectedDirPodCount + expectedDevApplierCount += testcase.deltaExpectedDevApplierCount + expectedDirApplierCount += testcase.deltaExpectedDirApplierCount + expectedPodDirCountCount += testcase.deltaExpectedPodDirCountCount + expectedQuotaSizeCount += testcase.deltaExpectedQuotaSizeCount + expectedSupportsQuotasCount += testcase.deltaExpectedSupportsQuotasCount + expectedBackingDevCount += testcase.deltaExpectedBackingDevCount + expectedMountpointCount += testcase.deltaExpectedMountpointCount + } + fail := false + if enabled { + fail = runCaseEnabled(t, testcase, seq) + } else { + fail = runCaseDisabled(t, testcase, seq) + } + + compareProjectsFiles(t, testcase, projectsFile, projidFile, enabled) + if len(podQuotaMap) != expectedPodQuotaCount { + fail = true + t.Errorf("Case %v (%s, %v) podQuotaCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(podQuotaMap), expectedPodQuotaCount) + } + if len(dirQuotaMap) != expectedDirQuotaCount { + fail = true + t.Errorf("Case %v (%s, %v) dirQuotaCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(dirQuotaMap), expectedDirQuotaCount) + } + if len(quotaPodMap) != expectedQuotaPodCount { + fail = true + t.Errorf("Case %v (%s, %v) quotaPodCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(quotaPodMap), expectedQuotaPodCount) + } + if len(dirPodMap) != expectedDirPodCount { + fail = true + t.Errorf("Case %v (%s, %v) dirPodCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(dirPodMap), expectedDirPodCount) + } + if len(devApplierMap) != expectedDevApplierCount { + fail = true + t.Errorf("Case %v (%s, %v) devApplierCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(devApplierMap), expectedDevApplierCount) + } + if len(dirApplierMap) != expectedDirApplierCount { + fail = true + t.Errorf("Case %v (%s, %v) dirApplierCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(dirApplierMap), expectedDirApplierCount) + } + if len(podDirCountMap) != expectedPodDirCountCount { + fail = true + t.Errorf("Case %v (%s, %v) podDirCountCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(podDirCountMap), expectedPodDirCountCount) + } + if len(quotaSizeMap) != expectedQuotaSizeCount { + fail = true + t.Errorf("Case %v (%s, %v) quotaSizeCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(quotaSizeMap), expectedQuotaSizeCount) + } + if len(supportsQuotasMap) != expectedSupportsQuotasCount { + fail = true + t.Errorf("Case %v (%s, %v) supportsQuotasCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(supportsQuotasMap), expectedSupportsQuotasCount) + } + if len(backingDevMap) != expectedBackingDevCount { + fail = true + t.Errorf("Case %v (%s, %v) BackingDevCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(backingDevMap), expectedBackingDevCount) + } + if len(mountpointMap) != expectedMountpointCount { + fail = true + t.Errorf("Case %v (%s, %v) MountpointCount mismatch: got %v, expect %v", seq, testcase.path, enabled, len(mountpointMap), expectedMountpointCount) + } + if fail { + logAllMaps(fmt.Sprintf("%v %s", seq, testcase.path)) + } + } + os.Remove(projectsFile) + os.Remove(projidFile) +} + +func TestAddRemoveQuotasEnabled(t *testing.T) { + testAddRemoveQuotas(t, true) +} + +func TestAddRemoveQuotasDisabled(t *testing.T) { + testAddRemoveQuotas(t, false) +} diff --git a/pkg/volume/util/quota/quota_unsupported.go b/pkg/volume/util/quota/quota_unsupported.go new file mode 100644 index 00000000000..a7d1b5ecf34 --- /dev/null +++ b/pkg/volume/util/quota/quota_unsupported.go @@ -0,0 +1,54 @@ +// +build !linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "errors" + "k8s.io/kubernetes/pkg/util/mount" +) + +// Dummy quota implementation for systems that do not implement support +// for volume quotas + +var errNotImplemented = errors.New("not implemented") + +// SupportsQuotas -- dummy implementation +func SupportsQuotas(_ mount.Interface, _ string) (bool, error) { + return false, errNotImplemented +} + +// AssignQuota -- dummy implementation +func AssignQuota(_ mount.Interface, _ string, _ string, _ int64) error { + return errNotImplemented +} + +// GetConsumption -- dummy implementation +func GetConsumption(_ string) (int64, error) { + return 0, errNotImplemented +} + +// GetInodes -- dummy implementation +func GetInodes(_ string) (int64, error) { + return 0, errNotImplemented +} + +// ClearQuota -- dummy implementation +func ClearQuota(_ mount.Interface, _ string) error { + return errNotImplemented +} diff --git a/pkg/volume/util/quota/xfs/BUILD b/pkg/volume/util/quota/xfs/BUILD new file mode 100644 index 00000000000..431587cf09e --- /dev/null +++ b/pkg/volume/util/quota/xfs/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["quota_xfs.go"], + cgo = True, + importpath = "k8s.io/kubernetes/pkg/volume/util/quota/xfs", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/volume/util/quota/common:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/volume/util/quota/xfs/quota_xfs.go b/pkg/volume/util/quota/xfs/quota_xfs.go new file mode 100644 index 00000000000..c6669323c3a --- /dev/null +++ b/pkg/volume/util/quota/xfs/quota_xfs.go @@ -0,0 +1,153 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package xfs + +/* +#include +#include +#include +#include +#include +#include + +#ifndef PRJQUOTA +#define PRJQUOTA 2 +#endif +#ifndef XFS_PROJ_QUOTA +#define XFS_PROJ_QUOTA 2 +#endif +#ifndef Q_XSETPQLIM +#define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA) +#endif +#ifndef Q_XGETPQUOTA +#define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) +#endif +*/ +import "C" + +import ( + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/volume/util/quota/common" +) + +const ( + linuxXfsMagic = 0x58465342 + // Documented in man xfs_quota(8); not necessarily the same + // as the filesystem blocksize + quotaBsize = 512 + bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 + maxQuota int64 = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 +) + +// VolumeProvider supplies a quota applier to the generic code. +type VolumeProvider struct { +} + +// GetQuotaApplier -- does this backing device support quotas that +// can be applied to directories? +func (*VolumeProvider) GetQuotaApplier(mountpoint string, backingDev string) common.LinuxVolumeQuotaApplier { + if common.IsFilesystemOfType(mountpoint, backingDev, linuxXfsMagic) { + return xfsVolumeQuota{backingDev} + } + return nil +} + +type xfsVolumeQuota struct { + backingDev string +} + +// GetQuotaOnDir -- get the quota ID that applies to this directory. +func (v xfsVolumeQuota) GetQuotaOnDir(path string) (common.QuotaID, error) { + return common.GetQuotaOnDir(path) +} + +// SetQuotaOnDir -- apply the specified quota to the directory. If +// bytes is not greater than zero, the quota should be applied in a +// way that is non-enforcing (either explicitly so or by setting a +// quota larger than anything the user may possibly create) +func (v xfsVolumeQuota) SetQuotaOnDir(path string, id common.QuotaID, bytes int64) error { + klog.V(3).Infof("xfsSetQuotaOn %s ID %v bytes %v", path, id, bytes) + if bytes < 0 || bytes > maxQuota { + bytes = maxQuota + } + + var d C.fs_disk_quota_t + d.d_version = C.FS_DQUOT_VERSION + d.d_id = C.__u32(id) + d.d_flags = C.XFS_PROJ_QUOTA + + d.d_fieldmask = C.FS_DQ_BHARD + d.d_blk_hardlimit = C.__u64(bytes / quotaBsize) + d.d_blk_softlimit = d.d_blk_hardlimit + + var cs = C.CString(v.backingDev) + defer C.free(unsafe.Pointer(cs)) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, + uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), + uintptr(unsafe.Pointer(&d)), 0, 0) + if errno != 0 { + return fmt.Errorf("Failed to set quota limit for ID %d on %s: %v", + id, path, errno.Error()) + } + return common.ApplyProjectToDir(path, id) +} + +func (v xfsVolumeQuota) getQuotaInfo(path string, id common.QuotaID) (C.fs_disk_quota_t, syscall.Errno) { + var d C.fs_disk_quota_t + + var cs = C.CString(v.backingDev) + defer C.free(unsafe.Pointer(cs)) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA, + uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(id)), + uintptr(unsafe.Pointer(&d)), 0, 0) + return d, errno +} + +// QuotaIDIsInUse -- determine whether the quota ID is already in use. +func (v xfsVolumeQuota) QuotaIDIsInUse(path string, id common.QuotaID) (bool, error) { + _, errno := v.getQuotaInfo(path, id) + return errno == 0, nil +} + +// GetConsumption -- retrieve the consumption (in bytes) of the directory +func (v xfsVolumeQuota) GetConsumption(path string, id common.QuotaID) (int64, error) { + d, errno := v.getQuotaInfo(path, id) + if errno != 0 { + return 0, fmt.Errorf("Failed to get quota for %s: %s", path, errno.Error()) + } + klog.V(3).Infof("Consumption for %s is %v", path, d.d_bcount*quotaBsize) + return int64(d.d_bcount) * quotaBsize, nil +} + +// GetInodes -- retrieve the number of inodes in use under the directory +func (v xfsVolumeQuota) GetInodes(path string, id common.QuotaID) (int64, error) { + d, errno := v.getQuotaInfo(path, id) + if errno != 0 { + return 0, fmt.Errorf("Failed to get quota for %s: %s", path, errno.Error()) + } + klog.V(3).Infof("Inode consumption for %s is %v", path, d.d_icount) + return int64(d.d_icount), nil +} diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index a5abb923b24..fdcc30647da 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -103,8 +103,9 @@ type Attributes struct { // MounterArgs provides more easily extensible arguments to Mounter type MounterArgs struct { - FsGroup *int64 - PodUID string + FsGroup *int64 + DesiredSize int64 + PodUID string } // Mounter interface provides methods to set up/mount the volume. From f8661d62406f98e422366f95cb5c5ef4f1c86a0f Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Mon, 19 Nov 2018 18:01:37 -0500 Subject: [PATCH 3/6] Use xfs_quota command to apply quotas --- pkg/features/kube_features.go | 4 +- pkg/kubelet/eviction/BUILD | 1 + pkg/kubelet/eviction/helpers.go | 19 +- pkg/kubelet/volumemanager/cache/BUILD | 2 +- .../cache/desired_state_of_world.go | 19 +- pkg/volume/emptydir/empty_dir.go | 17 +- pkg/volume/util/fs/fs.go | 18 +- pkg/volume/util/operationexecutor/BUILD | 1 + .../operationexecutor/operation_executor.go | 3 +- pkg/volume/util/quota/BUILD | 7 +- pkg/volume/util/quota/common/BUILD | 2 - .../util/quota/common/quota_linux_common.go | 32 +- .../quota/common/quota_linux_common_impl.go | 362 ++++++++++++------ pkg/volume/util/quota/extfs/BUILD | 31 -- pkg/volume/util/quota/extfs/quota_extfs.go | 153 -------- pkg/volume/util/quota/project.go | 26 +- pkg/volume/util/quota/quota.go | 9 +- pkg/volume/util/quota/quota_linux.go | 100 ++--- pkg/volume/util/quota/quota_linux_test.go | 22 +- pkg/volume/util/quota/quota_unsupported.go | 11 +- pkg/volume/util/quota/xfs/BUILD | 31 -- pkg/volume/util/quota/xfs/quota_xfs.go | 153 -------- pkg/volume/util/util.go | 8 + pkg/volume/volume.go | 2 +- 24 files changed, 416 insertions(+), 617 deletions(-) delete mode 100644 pkg/volume/util/quota/extfs/BUILD delete mode 100644 pkg/volume/util/quota/extfs/quota_extfs.go delete mode 100644 pkg/volume/util/quota/xfs/BUILD delete mode 100644 pkg/volume/util/quota/xfs/quota_xfs.go diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 86d164b895a..0429c092989 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -446,7 +446,7 @@ const ( // // Allow use of filesystems for ephemeral storage monitoring. // Only applies if LocalStorageCapacityIsolation is set. - FSQuotaForLSCIMonitoring = "FSQuotaForLSCIMonitoring" + LocalStorageCapacityIsolationFSQuotaMonitoring featuregate.Feature = "LocalStorageCapacityIsolationFSQuotaMonitoring" ) func init() { @@ -521,7 +521,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, KubeletPodResources: {Default: false, PreRelease: featuregate.Alpha}, WindowsGMSA: {Default: false, PreRelease: featuregate.Alpha}, - FSQuotaForLSCIMonitoring: {Default: false, PreRelease: utilfeature.Alpha}, + LocalStorageCapacityIsolationFSQuotaMonitoring: {Default: false, PreRelease: featuregate.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: diff --git a/pkg/kubelet/eviction/BUILD b/pkg/kubelet/eviction/BUILD index 68d188aafa7..2e6a49bd6f0 100644 --- a/pkg/kubelet/eviction/BUILD +++ b/pkg/kubelet/eviction/BUILD @@ -62,6 +62,7 @@ go_library( "//pkg/kubelet/util/format:go_default_library", "//pkg/scheduler/api:go_default_library", "//pkg/scheduler/util:go_default_library", + "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 0e8f843539a..50279fd1977 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -32,6 +32,7 @@ import ( evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" schedulerutils "k8s.io/kubernetes/pkg/scheduler/util" + volumeutils "k8s.io/kubernetes/pkg/volume/util" ) const ( @@ -395,27 +396,11 @@ func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsSt }, nil } -func internalIsLocalEphemeralVolume(pod *v1.Pod, volume v1.Volume) bool { - return volume.GitRepo != nil || - (volume.EmptyDir != nil && volume.EmptyDir.Medium != v1.StorageMediumMemory) || - volume.ConfigMap != nil || volume.DownwardAPI != nil -} - -// IsLocalEphemeralVolume determines whether a given volume name is ephemeral -func IsLocalEphemeralVolume(pod *v1.Pod, volumeName string) (bool, error) { - for _, volume := range pod.Spec.Volumes { - if volume.Name == volumeName { - return internalIsLocalEphemeralVolume(pod, volume), nil - } - } - return false, fmt.Errorf("Volume %s not found in pod %v", volumeName, pod) -} - // localEphemeralVolumeNames returns the set of ephemeral volumes for the pod that are local func localEphemeralVolumeNames(pod *v1.Pod) []string { result := []string{} for _, volume := range pod.Spec.Volumes { - if internalIsLocalEphemeralVolume(pod, volume) { + if volumeutils.IsLocalEphemeralVolume(volume) { result = append(result, volume.Name) } } diff --git a/pkg/kubelet/volumemanager/cache/BUILD b/pkg/kubelet/volumemanager/cache/BUILD index 35028f5846e..9d6a95049d6 100644 --- a/pkg/kubelet/volumemanager/cache/BUILD +++ b/pkg/kubelet/volumemanager/cache/BUILD @@ -16,12 +16,12 @@ go_library( deps = [ "//pkg/api/v1/resource:go_default_library", "//pkg/features:go_default_library", - "//pkg/kubelet/eviction:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", "//pkg/volume/util/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index c1d9fea084b..c80b03e7062 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -25,8 +25,8 @@ import ( "sync" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" apiv1resource "k8s.io/kubernetes/pkg/api/v1/resource" - limits "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" @@ -165,7 +165,7 @@ type volumeToMount struct { // desiredSizeLimit indicates the desired upper bound on the size of the volume // (if so implemented) - desiredSizeLimit int64 + desiredSizeLimit *resource.Quantity } // The pod object represents a pod that references the underlying volume and @@ -232,15 +232,12 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( } if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists { - var sizeLimit int64 - sizeLimit = 0 - isLocal, _ := limits.IsLocalEphemeralVolume(pod, volumeSpec.Name()) - if isLocal { - _, podLimits := apiv1resource.PodRequestsAndLimits(pod) - ephemeralStorageLimit := podLimits[v1.ResourceEphemeralStorage] - sizeLimit = ephemeralStorageLimit.Value() - if sizeLimit == 0 { - sizeLimit = -1 + var sizeLimit *resource.Quantity + if volumeSpec.Volume != nil { + if util.IsLocalEphemeralVolume(*volumeSpec.Volume) { + _, podLimits := apiv1resource.PodRequestsAndLimits(pod) + ephemeralStorageLimit := podLimits[v1.ResourceEphemeralStorage] + sizeLimit = resource.NewQuantity(ephemeralStorageLimit.Value(), resource.BinarySI) } } dsw.volumesToMount[volumeName] = volumeToMount{ diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go index a2604b7c465..e6e05b31a9b 100644 --- a/pkg/volume/emptydir/empty_dir.go +++ b/pkg/volume/emptydir/empty_dir.go @@ -236,17 +236,20 @@ func (ed *emptyDir) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { if err == nil { volumeutil.SetReady(ed.getMetaDir()) } - if mounterArgs.DesiredSize != 0 { - if hasQuotas, _ := quota.SupportsQuotas(ed.mounter, dir); hasQuotas { - klog.V(3).Infof("emptydir trying to assign quota") + if mounterArgs.DesiredSize != nil { + hasQuotas, err := quota.SupportsQuotas(ed.mounter, dir) + if err != nil { + return fmt.Errorf("Unable to check for quota support on %s: %s", dir, err.Error()) + } + if hasQuotas { + klog.V(4).Infof("emptydir trying to assign quota %v on %s", mounterArgs.DesiredSize, dir) err := quota.AssignQuota(ed.mounter, dir, mounterArgs.PodUID, mounterArgs.DesiredSize) if err != nil { - klog.V(3).Infof("Set quota failed %v", err) + return fmt.Errorf("Set quota on %s failed %s", dir, err.Error()) } } } - - return err + return nil } // setupTmpfs creates a tmpfs mount at the specified directory. @@ -411,7 +414,7 @@ func (ed *emptyDir) teardownDefault(dir string) error { // Remove any quota err := quota.ClearQuota(ed.mounter, dir) if err != nil { - klog.V(3).Infof("Failed to clear quota on %s: %v", dir, err) + klog.Warningf("Warning: Failed to clear quota on %s: %v", dir, err) } // Renaming the directory is not required anymore because the operation executor // now handles duplicate operations on the same volume diff --git a/pkg/volume/util/fs/fs.go b/pkg/volume/util/fs/fs.go index dd88367f020..6118a0bac7c 100644 --- a/pkg/volume/util/fs/fs.go +++ b/pkg/volume/util/fs/fs.go @@ -58,11 +58,13 @@ func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { // DiskUsage gets disk usage of specified path. func DiskUsage(path string) (*resource.Quantity, error) { // First check whether the quota system knows about this directory + // A nil quantity with no error means that the path does not support quotas + // and we should use other mechanisms. data, err := quota.GetConsumption(path) - if err == nil { - var q resource.Quantity - q.Set(data) - return &q, nil + if data != nil { + return data, nil + } else if err != nil { + return nil, fmt.Errorf("unable to retrieve disk consumption via quota for %s: %v", path, err) } // Uses the same niceness level as cadvisor.fs does when running du // Uses -B 1 to always scale to a blocksize of 1 byte @@ -85,9 +87,13 @@ func Find(path string) (int64, error) { return 0, fmt.Errorf("invalid directory") } // First check whether the quota system knows about this directory + // A nil quantity with no error means that the path does not support quotas + // and we should use other mechanisms. inodes, err := quota.GetInodes(path) - if err == nil { - return inodes, nil + if inodes != nil { + return inodes.Value(), nil + } else if err != nil { + return 0, fmt.Errorf("unable to retrieve inode consumption via quota for %s: %v", path, err) } var counter byteCounter var stderr bytes.Buffer diff --git a/pkg/volume/util/operationexecutor/BUILD b/pkg/volume/util/operationexecutor/BUILD index 8ae0ce393bc..ca290f29d7a 100644 --- a/pkg/volume/util/operationexecutor/BUILD +++ b/pkg/volume/util/operationexecutor/BUILD @@ -25,6 +25,7 @@ go_library( "//pkg/volume/util/volumepathhandler:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 379829ad50c..9206d5cb9e6 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -27,6 +27,7 @@ import ( "k8s.io/klog" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" @@ -349,7 +350,7 @@ type VolumeToMount struct { // DesiredSizeLimit indicates the desired upper bound on the size of the volume // (if so implemented) - DesiredSizeLimit int64 + DesiredSizeLimit *resource.Quantity } // GenerateMsgDetailed returns detailed msgs for volumes to mount diff --git a/pkg/volume/util/quota/BUILD b/pkg/volume/util/quota/BUILD index 2d1df604472..278469809e5 100644 --- a/pkg/volume/util/quota/BUILD +++ b/pkg/volume/util/quota/BUILD @@ -13,12 +13,11 @@ go_library( deps = [ "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//pkg/volume/util/quota/common:go_default_library", - "//pkg/volume/util/quota/extfs:go_default_library", - "//pkg/volume/util/quota/xfs:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/klog:go_default_library", @@ -36,7 +35,9 @@ go_test( "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume/util/quota/common:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", ], "//conditions:default": [], }), @@ -54,8 +55,6 @@ filegroup( srcs = [ ":package-srcs", "//pkg/volume/util/quota/common:all-srcs", - "//pkg/volume/util/quota/extfs:all-srcs", - "//pkg/volume/util/quota/xfs:all-srcs", ], tags = ["automanaged"], visibility = ["//visibility:public"], diff --git a/pkg/volume/util/quota/common/BUILD b/pkg/volume/util/quota/common/BUILD index 536a67450b4..9261c0cf01d 100644 --- a/pkg/volume/util/quota/common/BUILD +++ b/pkg/volume/util/quota/common/BUILD @@ -6,12 +6,10 @@ go_library( "quota_linux_common.go", "quota_linux_common_impl.go", ], - cgo = True, importpath = "k8s.io/kubernetes/pkg/volume/util/quota/common", visibility = ["//visibility:public"], deps = select({ "@io_bazel_rules_go//go/platform:linux": [ - "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], diff --git a/pkg/volume/util/quota/common/quota_linux_common.go b/pkg/volume/util/quota/common/quota_linux_common.go index a5774686acf..f73e7c7b8b1 100644 --- a/pkg/volume/util/quota/common/quota_linux_common.go +++ b/pkg/volume/util/quota/common/quota_linux_common.go @@ -18,18 +18,46 @@ limitations under the License. package common -// QuotaID -- generic quota identifier +import ( + "regexp" +) + +// QuotaID is generic quota identifier. +// Data type based on quotactl(2). type QuotaID int32 const ( + // UnknownQuotaID -- cannot determine whether a quota is in force + UnknownQuotaID QuotaID = -1 // BadQuotaID -- Invalid quota BadQuotaID QuotaID = 0 ) +const ( + acct = iota + enforcing = iota +) + +// QuotaType -- type of quota to be applied +type QuotaType int + +const ( + // FSQuotaAccounting for quotas for accounting only + FSQuotaAccounting QuotaType = 1 << iota + // FSQuotaEnforcing for quotas for enforcement + FSQuotaEnforcing QuotaType = 1 << iota +) + // FirstQuota is the quota ID we start with. // XXXXXXX Need a better way of doing this... var FirstQuota QuotaID = 1048577 +// MountsFile is the location of the system mount data +var MountsFile = "/proc/self/mounts" + +// MountParseRegexp parses out /proc/sys/self/mounts +var MountParseRegexp = regexp.MustCompilePOSIX("^([^ ]*)[ \t]*([^ ]*)[ \t]*([^ ]*)") // Ignore options etc. + // LinuxVolumeQuotaProvider returns an appropriate quota applier // object if we can support quotas on this device type LinuxVolumeQuotaProvider interface { @@ -59,7 +87,7 @@ type LinuxVolumeQuotaApplier interface { // Return value of false with no error means that the ID is not // in use; true means that it is already in use. An error // return means that any quota ID will fail. - QuotaIDIsInUse(path string, id QuotaID) (bool, error) + QuotaIDIsInUse(id QuotaID) (bool, error) // GetConsumption returns the consumption (in bytes) of the // directory, determined by the implementation's quota-based diff --git a/pkg/volume/util/quota/common/quota_linux_common_impl.go b/pkg/volume/util/quota/common/quota_linux_common_impl.go index 5df76fd532f..7f17b10baa4 100644 --- a/pkg/volume/util/quota/common/quota_linux_common_impl.go +++ b/pkg/volume/util/quota/common/quota_linux_common_impl.go @@ -18,135 +18,269 @@ limitations under the License. package common -/* -#include -#include -#include -#include -#include -#include - -#ifndef FS_XFLAG_PROJINHERIT -struct fsxattr { - __u32 fsx_xflags; - __u32 fsx_extsize; - __u32 fsx_nextents; - __u32 fsx_projid; - unsigned char fsx_pad[12]; -}; -#define FS_XFLAG_PROJINHERIT 0x00000200 -#endif -#ifndef FS_IOC_FSGETXATTR -#define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) -#endif -#ifndef FS_IOC_FSSETXATTR -#define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr) -#endif - -#ifndef PRJQUOTA -#define PRJQUOTA 2 -#endif -#ifndef Q_XGETQSTAT_PRJQUOTA -#define Q_XGETQSTAT_PRJQUOTA QCMD(Q_XGETQSTAT, PRJQUOTA) -#endif -*/ -import "C" - import ( + "bufio" "fmt" + "io/ioutil" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "sync" "syscall" - "unsafe" - "golang.org/x/sys/unix" "k8s.io/klog" ) -// IsFilesystemOfType determines whether the filesystem specified is of the type -// specified by the magic number -func IsFilesystemOfType(mountpoint string, backingDev string, magic int64) bool { - var buf syscall.Statfs_t - err := syscall.Statfs(mountpoint, &buf) - if err != nil { - klog.V(3).Infof("Extfs Unable to statfs %s: %v", mountpoint, err) - return false - } - if buf.Type != magic { - return false - } +var quotaCmd string +var quotaCmdInitialized bool +var quotaCmdLock sync.RWMutex - var qstat C.fs_quota_stat_t - CPath := C.CString(backingDev) - defer free(CPath) - - _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(CPath)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0) - return errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 +// If we later get a filesystem that uses project quota semantics other than +// XFS, we'll need to change this. +// Higher levels don't need to know what's inside +type linuxFilesystemType struct { + name string + typeMagic int64 // Filesystem magic number, per statfs(2) + maxQuota int64 + allowEmptyOutput bool // Accept empty output from "quota" command } -func free(p *C.char) { - C.free(unsafe.Pointer(p)) +const ( + bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 +) + +var ( + linuxSupportedFilesystems = []linuxFilesystemType{ + { + name: "XFS", + typeMagic: 0x58465342, + maxQuota: 1<<(bitsPerWord-1) - 1, + allowEmptyOutput: true, // XFS filesystems report nothing if a quota is not present + }, { + name: "ext4fs", + typeMagic: 0xef53, + maxQuota: (1<<(bitsPerWord-1) - 1) & (1<<58 - 1), + allowEmptyOutput: false, // ext4 filesystems always report something even if a quota is not present + }, + } +) + +// VolumeProvider supplies a quota applier to the generic code. +type VolumeProvider struct { } -func openDir(path string) (*C.DIR, error) { - Cpath := C.CString(path) - defer free(Cpath) +var quotaCmds = []string{"/sbin/xfs_quota", + "/usr/sbin/xfs_quota", + "/bin/xfs_quota"} - dir := C.opendir(Cpath) - if dir == nil { - return nil, fmt.Errorf("Can't open dir") - } - return dir, nil -} +var quotaParseRegexp = regexp.MustCompilePOSIX("^[^ \t]*[ \t]*([0-9]+)") -func closeDir(dir *C.DIR) { - if dir != nil { - C.closedir(dir) - } -} +var lsattrCmd = "/usr/bin/lsattr" +var lsattrParseRegexp = regexp.MustCompilePOSIX("^ *([0-9]+) [^ ]+ (.*)$") -func getDirFd(dir *C.DIR) uintptr { - return uintptr(C.dirfd(dir)) -} - -// GetQuotaOnDir retrieves the quota ID (if any) associated with the specified directory -func GetQuotaOnDir(path string) (QuotaID, error) { - dir, err := openDir(path) - if err != nil { - klog.V(3).Infof("Can't open directory %s: %#+v", path, err) - return BadQuotaID, err - } - defer closeDir(dir) - var fsx C.struct_fsxattr - _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, - uintptr(unsafe.Pointer(&fsx))) - if errno != 0 { - return BadQuotaID, fmt.Errorf("Failed to get quota ID for %s: %v", path, errno.Error()) - } - if fsx.fsx_projid == 0 { - return BadQuotaID, fmt.Errorf("Failed to get quota ID for %s: %s", path, "no applicable quota") - } - return QuotaID(fsx.fsx_projid), nil -} - -// ApplyProjectToDir applies the specified quota ID to the specified directory -func ApplyProjectToDir(path string, id QuotaID) error { - dir, err := openDir(path) - if err != nil { - return err - } - defer closeDir(dir) - - var fsx C.struct_fsxattr - _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, - uintptr(unsafe.Pointer(&fsx))) - if errno != 0 { - return fmt.Errorf("Failed to get quota ID for %s: %v", path, errno.Error()) - } - fsx.fsx_projid = C.__u32(id) - fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT - _, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR, - uintptr(unsafe.Pointer(&fsx))) - if errno != 0 { - return fmt.Errorf("Failed to set quota ID for %s: %v", path, errno.Error()) +// GetQuotaApplier -- does this backing device support quotas that +// can be applied to directories? +func (*VolumeProvider) GetQuotaApplier(mountpoint string, backingDev string) LinuxVolumeQuotaApplier { + for _, fsType := range linuxSupportedFilesystems { + if isFilesystemOfType(mountpoint, backingDev, fsType.typeMagic) { + return linuxVolumeQuotaApplier{mountpoint: mountpoint, + maxQuota: fsType.maxQuota, + allowEmptyOutput: fsType.allowEmptyOutput, + } + } } return nil } + +type linuxVolumeQuotaApplier struct { + mountpoint string + maxQuota int64 + allowEmptyOutput bool +} + +func getXFSQuotaCmd() (string, error) { + quotaCmdLock.Lock() + defer quotaCmdLock.Unlock() + if quotaCmdInitialized { + return quotaCmd, nil + } + for _, program := range quotaCmds { + fileinfo, err := os.Stat(program) + if err == nil && ((fileinfo.Mode().Perm() & (1 << 6)) != 0) { + klog.V(3).Infof("Found xfs_quota program %s", program) + quotaCmd = program + quotaCmdInitialized = true + return quotaCmd, nil + } + } + quotaCmdInitialized = true + return "", fmt.Errorf("No xfs_quota program found") +} + +func doRunXFSQuotaCommand(mountpoint string, mountsFile, command string) (string, error) { + quotaCmd, err := getXFSQuotaCmd() + if err != nil { + return "", err + } + // We're using numeric project IDs directly; no need to scan + // /etc/projects or /etc/projid + klog.V(4).Infof("runXFSQuotaCommand %s -t %s -P/dev/null -D/dev/null -x -f %s -c %s", quotaCmd, mountsFile, mountpoint, command) + cmd := exec.Command(quotaCmd, "-t", mountsFile, "-P/dev/null", "-D/dev/null", "-x", "-f", mountpoint, "-c", command) + + data, err := cmd.Output() + if err != nil { + return "", err + } + klog.V(4).Infof("runXFSQuotaCommand output %q", string(data)) + return string(data), nil +} + +// Extract the mountpoint we care about into a temporary mounts file so that xfs_quota does +// not attempt to scan every mount on the filesystem, which could hang if e. g. +// a stuck NFS mount is present. +// See https://bugzilla.redhat.com/show_bug.cgi?id=237120 for an example +// of the problem that could be caused if this were to happen. +func runXFSQuotaCommand(mountpoint string, command string) (string, error) { + tmpMounts, err := ioutil.TempFile("", "mounts") + if err != nil { + return "", fmt.Errorf("Cannot create temporary mount file: %v", err) + } + tmpMountsFileName := tmpMounts.Name() + defer tmpMounts.Close() + defer os.Remove(tmpMountsFileName) + + mounts, err := os.Open(MountsFile) + if err != nil { + return "", fmt.Errorf("Cannot open mounts file %s: %v", MountsFile, err) + } + defer mounts.Close() + + scanner := bufio.NewScanner(mounts) + for scanner.Scan() { + match := MountParseRegexp.FindStringSubmatch(scanner.Text()) + if match != nil { + mount := match[2] + if mount == mountpoint { + if _, err := tmpMounts.WriteString(fmt.Sprintf("%s\n", scanner.Text())); err != nil { + return "", fmt.Errorf("Cannot write temporary mounts file: %v", err) + } + if err := tmpMounts.Sync(); err != nil { + return "", fmt.Errorf("Cannot sync temporary mounts file: %v", err) + } + return doRunXFSQuotaCommand(mountpoint, tmpMountsFileName, command) + } + } + } + return "", fmt.Errorf("Cannot run xfs_quota: cannot find mount point %s in %s", mountpoint, MountsFile) +} + +// SupportsQuotas determines whether the filesystem supports quotas. +func SupportsQuotas(mountpoint string, qType QuotaType) (bool, error) { + data, err := runXFSQuotaCommand(mountpoint, "state -p") + if err != nil { + return false, err + } + if qType == FSQuotaEnforcing { + return strings.Contains(data, "Enforcement: ON"), nil + } + return strings.Contains(data, "Accounting: ON"), nil +} + +func isFilesystemOfType(mountpoint string, backingDev string, typeMagic int64) bool { + var buf syscall.Statfs_t + err := syscall.Statfs(mountpoint, &buf) + if err != nil { + klog.Warningf("Warning: Unable to statfs %s: %v", mountpoint, err) + return false + } + if int64(buf.Type) != typeMagic { + return false + } + if answer, _ := SupportsQuotas(mountpoint, FSQuotaAccounting); answer { + return true + } + return false +} + +// GetQuotaOnDir retrieves the quota ID (if any) associated with the specified directory +// If we can't make system calls, all we can say is that we don't know whether +// it has a quota, and higher levels have to make the call. +func (v linuxVolumeQuotaApplier) GetQuotaOnDir(path string) (QuotaID, error) { + cmd := exec.Command(lsattrCmd, "-pd", path) + data, err := cmd.Output() + if err != nil { + return BadQuotaID, fmt.Errorf("cannot run lsattr: %v", err) + } + match := lsattrParseRegexp.FindStringSubmatch(string(data)) + if match == nil { + return BadQuotaID, fmt.Errorf("Unable to parse lsattr -pd %s output %s", path, string(data)) + } + if match[2] != path { + return BadQuotaID, fmt.Errorf("Mismatch between supplied and returned path (%s != %s)", path, match[2]) + } + projid, err := strconv.ParseInt(match[1], 10, 32) + if err != nil { + return BadQuotaID, fmt.Errorf("Unable to parse project ID from %s (%v)", match[1], err) + } + return QuotaID(projid), nil +} + +// SetQuotaOnDir applies a quota to the specified directory under the specified mountpoint. +func (v linuxVolumeQuotaApplier) SetQuotaOnDir(path string, id QuotaID, bytes int64) error { + if bytes < 0 || bytes > v.maxQuota { + bytes = v.maxQuota + } + _, err := runXFSQuotaCommand(v.mountpoint, fmt.Sprintf("limit -p bhard=%v bsoft=%v %v", bytes, bytes, id)) + if err != nil { + return err + } + + _, err = runXFSQuotaCommand(v.mountpoint, fmt.Sprintf("project -s -p %s %v", path, id)) + return err +} + +func getQuantity(mountpoint string, id QuotaID, xfsQuotaArg string, multiplier int64, allowEmptyOutput bool) (int64, error) { + data, err := runXFSQuotaCommand(mountpoint, fmt.Sprintf("quota -p -N -n -v %s %v", xfsQuotaArg, id)) + if err != nil { + return 0, fmt.Errorf("Unable to run xfs_quota: %v", err) + } + if data == "" && allowEmptyOutput { + return 0, nil + } + match := quotaParseRegexp.FindStringSubmatch(data) + if match == nil { + return 0, fmt.Errorf("Unable to parse quota output '%s'", data) + } + size, err := strconv.ParseInt(match[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("Unable to parse data size '%s' from '%s': %v", match[1], data, err) + } + klog.V(4).Infof("getQuantity %s %d %s %d => %d %v", mountpoint, id, xfsQuotaArg, multiplier, size, err) + return size * multiplier, nil +} + +// GetConsumption returns the consumption in bytes if available via quotas +func (v linuxVolumeQuotaApplier) GetConsumption(_ string, id QuotaID) (int64, error) { + return getQuantity(v.mountpoint, id, "-b", 1024, v.allowEmptyOutput) +} + +// GetInodes returns the inodes in use if available via quotas +func (v linuxVolumeQuotaApplier) GetInodes(_ string, id QuotaID) (int64, error) { + return getQuantity(v.mountpoint, id, "-i", 1, v.allowEmptyOutput) +} + +// QuotaIDIsInUse checks whether the specified quota ID is in use on the specified +// filesystem +func (v linuxVolumeQuotaApplier) QuotaIDIsInUse(id QuotaID) (bool, error) { + bytes, err := v.GetConsumption(v.mountpoint, id) + if err != nil { + return false, err + } + if bytes > 0 { + return true, nil + } + inodes, err := v.GetInodes(v.mountpoint, id) + return inodes > 0, err +} diff --git a/pkg/volume/util/quota/extfs/BUILD b/pkg/volume/util/quota/extfs/BUILD deleted file mode 100644 index 431386beed0..00000000000 --- a/pkg/volume/util/quota/extfs/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["quota_extfs.go"], - cgo = True, - importpath = "k8s.io/kubernetes/pkg/volume/util/quota/extfs", - visibility = ["//visibility:public"], - deps = select({ - "@io_bazel_rules_go//go/platform:linux": [ - "//pkg/volume/util/quota/common:go_default_library", - "//vendor/golang.org/x/sys/unix:go_default_library", - "//vendor/k8s.io/klog:go_default_library", - ], - "//conditions:default": [], - }), -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/pkg/volume/util/quota/extfs/quota_extfs.go b/pkg/volume/util/quota/extfs/quota_extfs.go deleted file mode 100644 index 397e3ba79e9..00000000000 --- a/pkg/volume/util/quota/extfs/quota_extfs.go +++ /dev/null @@ -1,153 +0,0 @@ -// +build linux - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package extfs - -/* -#include -#include -#include -#include -#include - -#ifndef PRJQUOTA -#define PRJQUOTA 2 -#endif -#ifndef Q_SETPQUOTA -#define Q_SETPQUOTA (unsigned) QCMD(Q_SETQUOTA, PRJQUOTA) -#endif -#ifndef Q_GETPQUOTA -#define Q_GETPQUOTA (unsigned) QCMD(Q_GETQUOTA, PRJQUOTA) -#endif -*/ -import "C" - -import ( - "fmt" - "syscall" - "unsafe" - - "golang.org/x/sys/unix" - "k8s.io/klog" - "k8s.io/kubernetes/pkg/volume/util/quota/common" -) - -// ext4fs empirically has a maximum quota size of 2^48 - 1 1KiB blocks (256 petabytes) -const ( - linuxExtfsMagic = 0xef53 - quotaBsize = 1024 // extfs specific - bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 - maxQuota int64 = (1<<(bitsPerWord-1) - 1) & (1<<58 - 1) // either 1<<31 - 1 or 1<<58 - 1 -) - -// VolumeProvider supplies a quota applier to the generic code. -type VolumeProvider struct { -} - -// GetQuotaApplier -- does this backing device support quotas that -// can be applied to directories? -func (*VolumeProvider) GetQuotaApplier(mountpoint string, backingDev string) common.LinuxVolumeQuotaApplier { - if common.IsFilesystemOfType(mountpoint, backingDev, linuxExtfsMagic) { - return extfsVolumeQuota{backingDev} - } - return nil -} - -type extfsVolumeQuota struct { - backingDev string -} - -// GetQuotaOnDir -- get the quota ID that applies to this directory. - -func (v extfsVolumeQuota) GetQuotaOnDir(path string) (common.QuotaID, error) { - return common.GetQuotaOnDir(path) -} - -// SetQuotaOnDir -- apply the specified quota to the directory. If -// bytes is not greater than zero, the quota should be applied in a -// way that is non-enforcing (either explicitly so or by setting a -// quota larger than anything the user may possibly create) -func (v extfsVolumeQuota) SetQuotaOnDir(path string, id common.QuotaID, bytes int64) error { - klog.V(3).Infof("extfsSetQuotaOn %s ID %v bytes %v", path, id, bytes) - if bytes < 0 || bytes > maxQuota { - bytes = maxQuota - } - - var d C.struct_if_dqblk - - d.dqb_bhardlimit = C.__u64(bytes / quotaBsize) - d.dqb_bsoftlimit = d.dqb_bhardlimit - d.dqb_ihardlimit = 0 - d.dqb_isoftlimit = 0 - d.dqb_valid = C.QIF_LIMITS - - var cs = C.CString(v.backingDev) - defer C.free(unsafe.Pointer(cs)) - - _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_SETPQUOTA, - uintptr(unsafe.Pointer(cs)), uintptr(id), - uintptr(unsafe.Pointer(&d)), 0, 0) - if errno != 0 { - return fmt.Errorf("Failed to set quota limit for ID %d on %s: %v", - id, path, errno.Error()) - } - return common.ApplyProjectToDir(path, id) -} - -func (v extfsVolumeQuota) getQuotaInfo(path string, id common.QuotaID) (C.struct_if_dqblk, syscall.Errno) { - var d C.struct_if_dqblk - - var cs = C.CString(v.backingDev) - defer C.free(unsafe.Pointer(cs)) - - _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_GETPQUOTA, - uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(id)), - uintptr(unsafe.Pointer(&d)), 0, 0) - return d, errno -} - -// QuotaIDIsInUse -- determine whether the quota ID is already in use. -func (v extfsVolumeQuota) QuotaIDIsInUse(path string, id common.QuotaID) (bool, error) { - d, errno := v.getQuotaInfo(path, id) - isInUse := !(d.dqb_bhardlimit == 0 && d.dqb_bsoftlimit == 0 && d.dqb_curspace == 0 && - d.dqb_ihardlimit == 0 && d.dqb_isoftlimit == 0 && d.dqb_curinodes == 0 && - d.dqb_btime == 0 && d.dqb_itime == 0) - return errno == 0 && isInUse, nil -} - -// GetConsumption -- retrieve the consumption (in bytes) of the directory -// Note that with ext[[:digit:]]fs the quota consumption is in bytes -// per man quotactl -func (v extfsVolumeQuota) GetConsumption(path string, id common.QuotaID) (int64, error) { - d, errno := v.getQuotaInfo(path, id) - if errno != 0 { - return 0, fmt.Errorf("Failed to get quota for %s: %s", path, errno.Error()) - } - klog.V(3).Infof("Consumption for %s is %v", path, d.dqb_curspace) - return int64(d.dqb_curspace), nil -} - -// GetInodes -- retrieve the number of inodes in use under the directory -func (v extfsVolumeQuota) GetInodes(path string, id common.QuotaID) (int64, error) { - d, errno := v.getQuotaInfo(path, id) - if errno != 0 { - return 0, fmt.Errorf("Failed to get quota for %s: %s", path, errno.Error()) - } - klog.V(3).Infof("Inode consumption for %s is %v", path, d.dqb_curinodes) - return int64(d.dqb_curinodes), nil -} diff --git a/pkg/volume/util/quota/project.go b/pkg/volume/util/quota/project.go index 4caa7ecd671..b3aa1b244f0 100644 --- a/pkg/volume/util/quota/project.go +++ b/pkg/volume/util/quota/project.go @@ -29,15 +29,14 @@ import ( "sync" "golang.org/x/sys/unix" - "k8s.io/klog" "k8s.io/kubernetes/pkg/volume/util/quota/common" ) var projectsFile = "/etc/projects" var projidFile = "/etc/projid" -var projectsParseRegexp *regexp.Regexp = regexp.MustCompilePOSIX("^([[:digit:]]+):(.*)$") -var projidParseRegexp *regexp.Regexp = regexp.MustCompilePOSIX("^([^#][^:]*):([[:digit:]]+)$") +var projectsParseRegexp = regexp.MustCompilePOSIX("^([[:digit:]]+):(.*)$") +var projidParseRegexp = regexp.MustCompilePOSIX("^([^#][^:]*):([[:digit:]]+)$") var quotaIDLock sync.RWMutex @@ -78,12 +77,13 @@ func unlockFile(file *os.File) error { func openAndLockProjectFiles() (*os.File, *os.File, error) { // Make sure neither project-related file is a symlink! if err := projFilesAreOK(); err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("system project files failed verification: %v", err) } // We don't actually modify the original files; we create temporaries and // move them over the originals fProjects, err := os.OpenFile(projectsFile, os.O_RDONLY|os.O_CREATE, 0644) if err != nil { + err = fmt.Errorf("unable to open %s: %v", projectsFile, err) return nil, nil, err } fProjid, err := os.OpenFile(projidFile, os.O_RDONLY|os.O_CREATE, 0644) @@ -97,10 +97,17 @@ func openAndLockProjectFiles() (*os.File, *os.File, error) { return fProjects, fProjid, nil } // Nothing useful we can do if we get an error here + err = fmt.Errorf("unable to lock %s: %v", projidFile, err) unlockFile(fProjects) + } else { + err = fmt.Errorf("unable to lock %s: %v", projectsFile, err) } + } else { + err = fmt.Errorf("system project files failed re-verification: %v", err) } fProjid.Close() + } else { + err = fmt.Errorf("unable to open %s: %v", projidFile, err) } fProjects.Close() return nil, nil, err @@ -160,7 +167,7 @@ func findAvailableQuota(path string, idMap map[common.QuotaID]bool) (common.Quot unusedQuotasSearched := 0 for id := common.FirstQuota; id == id; id++ { if _, ok := idMap[id]; !ok { - isInUse, err := getApplier(path).QuotaIDIsInUse(path, id) + isInUse, err := getApplier(path).QuotaIDIsInUse(id) if err != nil { return common.BadQuotaID, err } else if !isInUse { @@ -307,8 +314,7 @@ func writeProjectFiles(fProjects *os.File, fProjid *os.File, writeProjid bool, l } os.Remove(tmpProjects) } - klog.V(3).Infof("Unable to write project files: %v", err) - return err + return fmt.Errorf("Unable to write project files: %v", err) } func createProjectID(path string, ID common.QuotaID) (common.QuotaID, error) { @@ -326,8 +332,7 @@ func createProjectID(path string, ID common.QuotaID) (common.QuotaID, error) { } } } - klog.V(3).Infof("addQuotaID %s %v failed %v", path, ID, err) - return common.BadQuotaID, err + return common.BadQuotaID, fmt.Errorf("createProjectID %s %v failed %v", path, ID, err) } func removeProjectID(path string, ID common.QuotaID) error { @@ -348,6 +353,5 @@ func removeProjectID(path string, ID common.QuotaID) error { } } } - klog.V(3).Infof("removeQuotaID %s %v failed %v", path, ID, err) - return err + return fmt.Errorf("removeProjectID %s %v failed %v", path, ID, err) } diff --git a/pkg/volume/util/quota/quota.go b/pkg/volume/util/quota/quota.go index 1bfa7d7c69f..1e578ff5ba5 100644 --- a/pkg/volume/util/quota/quota.go +++ b/pkg/volume/util/quota/quota.go @@ -17,6 +17,7 @@ limitations under the License. package quota import ( + "k8s.io/apimachinery/pkg/api/resource" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" @@ -28,13 +29,13 @@ type Interface interface { SupportsQuotas(m mount.Interface, path string) (bool, error) // Assign a quota (picked by the quota mechanism) to a path, // and return it. - AssignQuota(m mount.Interface, path string, poduid string, bytes int64) error + AssignQuota(m mount.Interface, path string, poduid string, bytes *resource.Quantity) error // Get the quota-based storage consumption for the path - GetConsumption(path string) (int64, error) + GetConsumption(path string) (*resource.Quantity, error) // Get the quota-based inode consumption for the path - GetInodes(path string) (int64, error) + GetInodes(path string) (*resource.Quantity, error) // Remove the quota from a path // Implementations may assume that any data covered by the @@ -43,5 +44,5 @@ type Interface interface { } func enabledQuotasForMonitoring() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.FSQuotaForLSCIMonitoring) + return utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolationFSQuotaMonitoring) } diff --git a/pkg/volume/util/quota/quota_linux.go b/pkg/volume/util/quota/quota_linux.go index 664d2bc38d3..fdf2f00a2cb 100644 --- a/pkg/volume/util/quota/quota_linux.go +++ b/pkg/volume/util/quota/quota_linux.go @@ -23,15 +23,13 @@ import ( "fmt" "os" "path/filepath" - "regexp" "sync" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume/util/quota/common" - "k8s.io/kubernetes/pkg/volume/util/quota/extfs" - "k8s.io/kubernetes/pkg/volume/util/quota/xfs" ) // Pod -> ID @@ -64,8 +62,6 @@ var quotaLock sync.RWMutex var supportsQuotasMap = make(map[string]bool) var supportsQuotasLock sync.RWMutex -var mountParseRegexp *regexp.Regexp = regexp.MustCompilePOSIX("^([^ ]*)[ \t]*([^ ]*)[ \t]*([^ ]*)") // Ignore options etc. - // Directory -> backingDev var backingDevMap = make(map[string]string) var backingDevLock sync.RWMutex @@ -73,11 +69,8 @@ var backingDevLock sync.RWMutex var mountpointMap = make(map[string]string) var mountpointLock sync.RWMutex -var mountsFile = "/proc/self/mounts" - var providers = []common.LinuxVolumeQuotaProvider{ - &extfs.VolumeProvider{}, - &xfs.VolumeProvider{}, + &common.VolumeProvider{}, } // Separate the innards for ease of testing @@ -89,7 +82,7 @@ func detectBackingDevInternal(mountpoint string, mounts string) (string, error) defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { - match := mountParseRegexp.FindStringSubmatch(scanner.Text()) + match := common.MountParseRegexp.FindStringSubmatch(scanner.Text()) if match != nil { device := match[1] mount := match[2] @@ -103,7 +96,7 @@ func detectBackingDevInternal(mountpoint string, mounts string) (string, error) // detectBackingDev assumes that the mount point provided is valid func detectBackingDev(_ mount.Interface, mountpoint string) (string, error) { - return detectBackingDevInternal(mountpoint, mountsFile) + return detectBackingDevInternal(mountpoint, common.MountsFile) } func clearBackingDev(path string) { @@ -140,12 +133,15 @@ func detectMountpointInternal(m mount.Interface, path string) (string, error) { func detectMountpoint(m mount.Interface, path string) (string, error) { xpath, err := filepath.Abs(path) - if err == nil { - if xpath, err = filepath.EvalSymlinks(xpath); err == nil { - if xpath, err = detectMountpointInternal(m, xpath); err == nil { - return xpath, nil - } - } + if err != nil { + return "/", err + } + xpath, err = filepath.EvalSymlinks(xpath) + if err != nil { + return "/", err + } + if xpath, err = detectMountpointInternal(m, xpath); err == nil { + return xpath, nil } return "/", err } @@ -171,18 +167,16 @@ func getFSInfo(m mount.Interface, path string) (string, string, error) { mountpoint, okMountpoint := mountpointMap[path] if !okMountpoint { mountpoint, err = detectMountpoint(m, path) - klog.V(3).Infof("Mountpoint %s -> %s (%v)", path, mountpoint, err) if err != nil { - return "", "", err + return "", "", fmt.Errorf("Cannot determine mountpoint for %s: %v", path, err) } } backingDev, okBackingDev := backingDevMap[path] if !okBackingDev { backingDev, err = detectBackingDev(m, mountpoint) - klog.V(3).Infof("Backing dev %s -> %s (%v)", path, backingDev, err) if err != nil { - return "", "", err + return "", "", fmt.Errorf("Cannot determine backing device for %s: %v", path, err) } } mountpointMap[path] = mountpoint @@ -228,20 +222,21 @@ func getQuotaOnDir(m mount.Interface, path string) (common.QuotaID, error) { func clearQuotaOnDir(m mount.Interface, path string) error { // Since we may be called without path being in the map, // we explicitly have to check in this case. - klog.V(3).Infof("clearQuotaOnDir %s", path) + klog.V(4).Infof("clearQuotaOnDir %s", path) supportsQuotas, err := SupportsQuotas(m, path) if !supportsQuotas { return nil } projid, err := getQuotaOnDir(m, path) if err == nil && projid != common.BadQuotaID { - klog.V(3).Infof("clearQuotaOnDir clearing quota") // This means that we have a quota on the directory but // we can't clear it. That's not good. err = setQuotaOnDir(path, projid, 0) if err != nil { klog.V(3).Infof("Attempt to clear quota failed: %v", err) } + // Even if clearing the quota failed, we still need to + // try to remove the project ID, or that may be left dangling. err1 := removeProjectID(path, projid) if err1 != nil { klog.V(3).Infof("Attempt to remove quota ID from system files failed: %v", err1) @@ -252,9 +247,9 @@ func clearQuotaOnDir(m mount.Interface, path string) error { } return err1 } - klog.V(3).Infof("clearQuotaOnDir fails %v", err) // If we couldn't get a quota, that's fine -- there may // never have been one, and we have no way to know otherwise + klog.V(3).Infof("clearQuotaOnDir fails %v", err) return nil } @@ -274,7 +269,6 @@ func SupportsQuotas(m mount.Interface, path string) (bool, error) { return supportsQuotas, nil } mount, dev, err := getFSInfo(m, path) - klog.V(3).Infof("SupportsQuotas %s -> mount %s dev %s %v", path, mount, dev, err) if err != nil { return false, err } @@ -289,12 +283,10 @@ func SupportsQuotas(m mount.Interface, path string) (bool, error) { } } if applier != nil { - klog.V(3).Infof("SupportsQuotas got applier %v", applier) supportsQuotasMap[path] = true setApplier(path, applier) return true, nil } - klog.V(3).Infof("SupportsQuotas got no applier") delete(backingDevMap, path) delete(mountpointMap, path) return false, nil @@ -304,7 +296,11 @@ func SupportsQuotas(m mount.Interface, path string) (bool, error) { // AssignQuota chooses the quota ID based on the pod UID and path. // If the pod UID is identical to another one known, it may (but presently // doesn't) choose the same quota ID as other volumes in the pod. -func AssignQuota(m mount.Interface, path string, poduid string, bytes int64) error { +func AssignQuota(m mount.Interface, path string, poduid string, bytes *resource.Quantity) error { + if bytes == nil { + return fmt.Errorf("Attempting to assign null quota to %s", path) + } + ibytes := bytes.Value() if ok, err := SupportsQuotas(m, path); !ok { return fmt.Errorf("Quotas not supported on %s: %v", path, err) } @@ -316,13 +312,12 @@ func AssignQuota(m mount.Interface, path string, poduid string, bytes int64) err // If and when we decide permanently that we're going to adop // one quota per volume, we can rip all of the pod code out. poduid = string(uuid.NewUUID()) - klog.V(3).Infof("Synthesizing pod ID %s for directory %s in AssignQuota", poduid, path) if pod, ok := dirPodMap[path]; ok && pod != poduid { return fmt.Errorf("Requesting quota on existing directory %s but different pod %s %s", path, pod, poduid) } oid, ok := podQuotaMap[poduid] if ok { - if quotaSizeMap[oid] != bytes { + if quotaSizeMap[oid] != ibytes { return fmt.Errorf("Requesting quota of different size: old %v new %v", quotaSizeMap[oid], bytes) } } else { @@ -331,60 +326,68 @@ func AssignQuota(m mount.Interface, path string, poduid string, bytes int64) err id, err := createProjectID(path, oid) if err == nil { if oid != common.BadQuotaID && oid != id { - klog.V(3).Infof("Attempt to reassign quota %v to %v", oid, id) return fmt.Errorf("Attempt to reassign quota %v to %v", oid, id) } // When enforcing quotas are enabled, we'll condition this // on their being disabled also. - if bytes > 0 { - bytes = -1 + if ibytes > 0 { + ibytes = -1 } - if err = setQuotaOnDir(path, id, bytes); err == nil { + if err = setQuotaOnDir(path, id, ibytes); err == nil { quotaPodMap[id] = poduid - quotaSizeMap[id] = bytes + quotaSizeMap[id] = ibytes podQuotaMap[poduid] = id dirQuotaMap[path] = id dirPodMap[path] = poduid podDirCountMap[poduid]++ + klog.V(4).Infof("Assigning quota ID %d (%d) to %s", id, ibytes, path) return nil } removeProjectID(path, id) } - klog.V(3).Infof("Assign quota FAILED %v", err) - return err + return fmt.Errorf("Assign quota FAILED %v", err) } // GetConsumption -- retrieve the consumption (in bytes) of the directory -func GetConsumption(path string) (int64, error) { +func GetConsumption(path string) (*resource.Quantity, error) { // Note that we actually need to hold the lock at least through // running the quota command, so it can't get recycled behind our back quotaLock.Lock() defer quotaLock.Unlock() applier := getApplier(path) + // No applier means directory is not under quota management if applier == nil { - return 0, fmt.Errorf("No quota available for %s", path) + return nil, nil } - return applier.GetConsumption(path, dirQuotaMap[path]) + ibytes, err := applier.GetConsumption(path, dirQuotaMap[path]) + if err != nil { + return nil, err + } + return resource.NewQuantity(ibytes, resource.DecimalSI), nil } // GetInodes -- retrieve the number of inodes in use under the directory -func GetInodes(path string) (int64, error) { +func GetInodes(path string) (*resource.Quantity, error) { // Note that we actually need to hold the lock at least through // running the quota command, so it can't get recycled behind our back quotaLock.Lock() defer quotaLock.Unlock() applier := getApplier(path) + // No applier means directory is not under quota management if applier == nil { - return 0, fmt.Errorf("No quota available for %s", path) + return nil, nil } - return applier.GetInodes(path, dirQuotaMap[path]) + inodes, err := applier.GetInodes(path, dirQuotaMap[path]) + if err != nil { + return nil, err + } + return resource.NewQuantity(inodes, resource.DecimalSI), nil } // ClearQuota -- remove the quota assigned to a directory func ClearQuota(m mount.Interface, path string) error { klog.V(3).Infof("ClearQuota %s", path) if !enabledQuotasForMonitoring() { - klog.V(3).Info("ClearQuota called, but quotas disabled") return fmt.Errorf("ClearQuota called, but quotas disabled") } quotaLock.Lock() @@ -412,6 +415,8 @@ func ClearQuota(m mount.Interface, path string) error { count, ok := podDirCountMap[poduid] if count <= 1 || !ok { err = clearQuotaOnDir(m, path) + // This error should be noted; we still need to clean up + // and otherwise handle in the same way. if err != nil { klog.V(3).Infof("Unable to clear quota %v %s: %v", dirQuotaMap[path], path, err) } @@ -422,11 +427,14 @@ func ClearQuota(m mount.Interface, path string) error { } else { err = removeProjectID(path, projid) podDirCountMap[poduid]-- - klog.V(3).Infof("Not clearing quota for pod %s; still %v dirs outstanding", poduid, podDirCountMap[poduid]) + klog.V(4).Infof("Not clearing quota for pod %s; still %v dirs outstanding", poduid, podDirCountMap[poduid]) } delete(dirPodMap, path) delete(dirQuotaMap, path) delete(supportsQuotasMap, path) clearApplier(path) - return err + if err != nil { + return fmt.Errorf("Unable to clear quota for %s: %v", path, err) + } + return nil } diff --git a/pkg/volume/util/quota/quota_linux_test.go b/pkg/volume/util/quota/quota_linux_test.go index 9366821c75c..a7acfcd2022 100644 --- a/pkg/volume/util/quota/quota_linux_test.go +++ b/pkg/volume/util/quota/quota_linux_test.go @@ -21,7 +21,9 @@ package quota import ( "fmt" "io/ioutil" + "k8s.io/apimachinery/pkg/api/resource" utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume/util/quota/common" @@ -74,13 +76,13 @@ func dummyFakeMount1() mount.Interface { Opts: []string{"rw", "relatime"}, }, { - Device: "dev/mapper/fedora-root", + Device: "/dev/mapper/fedora-root", Path: "/", Type: "ext4", Opts: []string{"rw", "relatime"}, }, { - Device: "dev/mapper/fedora-home", + Device: "/dev/mapper/fedora-home", Path: "/home", Type: "ext4", Opts: []string{"rw", "relatime"}, @@ -363,7 +365,7 @@ func (v testVolumeQuota) GetQuotaOnDir(path string) (common.QuotaID, error) { return common.BadQuotaID, fmt.Errorf("No quota available for %s", path) } -func (v testVolumeQuota) QuotaIDIsInUse(_ string, id common.QuotaID) (bool, error) { +func (v testVolumeQuota) QuotaIDIsInUse(id common.QuotaID) (bool, error) { if _, ok := testIDQuotaMap[id]; ok { return true, nil } @@ -389,7 +391,7 @@ func fakeSupportsQuotas(path string) (bool, error) { func fakeAssignQuota(path string, poduid string, bytes int64) error { dummySetFSInfo(path) - return AssignQuota(dummyQuotaTest(), path, poduid, bytes) + return AssignQuota(dummyQuotaTest(), path, poduid, resource.NewQuantity(bytes, resource.DecimalSI)) } func fakeClearQuota(path string) error { @@ -529,14 +531,6 @@ func compareProjectsFiles(t *testing.T, testcase quotaTestCase, projectsFile str } } -func setFeature(feature utilfeature.Feature, value bool) error { - v := "true" - if !value { - v = "false" - } - return utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=%s", string(feature), v)) -} - func runCaseEnabled(t *testing.T, testcase quotaTestCase, seq int) bool { fail := false var err error @@ -604,9 +598,7 @@ func runCaseDisabled(t *testing.T, testcase quotaTestCase, seq int) bool { } func testAddRemoveQuotas(t *testing.T, enabled bool) { - if err := setFeature(features.FSQuotaForLSCIMonitoring, enabled); err != nil { - t.Errorf("Unable to enable LSCI monitoring: %v", err) - } + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolationFSQuotaMonitoring, enabled)() tmpProjectsFile, err := ioutil.TempFile("", "projects") if err == nil { _, err = tmpProjectsFile.WriteString(projectsHeader) diff --git a/pkg/volume/util/quota/quota_unsupported.go b/pkg/volume/util/quota/quota_unsupported.go index a7d1b5ecf34..16cbfaffd1a 100644 --- a/pkg/volume/util/quota/quota_unsupported.go +++ b/pkg/volume/util/quota/quota_unsupported.go @@ -20,6 +20,7 @@ package quota import ( "errors" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/util/mount" ) @@ -34,18 +35,18 @@ func SupportsQuotas(_ mount.Interface, _ string) (bool, error) { } // AssignQuota -- dummy implementation -func AssignQuota(_ mount.Interface, _ string, _ string, _ int64) error { +func AssignQuota(_ mount.Interface, _ string, _ string, _ *resource.Quantity) error { return errNotImplemented } // GetConsumption -- dummy implementation -func GetConsumption(_ string) (int64, error) { - return 0, errNotImplemented +func GetConsumption(_ string) (*resource.Quantity, error) { + return nil, errNotImplemented } // GetInodes -- dummy implementation -func GetInodes(_ string) (int64, error) { - return 0, errNotImplemented +func GetInodes(_ string) (*resource.Quantity, error) { + return nil, errNotImplemented } // ClearQuota -- dummy implementation diff --git a/pkg/volume/util/quota/xfs/BUILD b/pkg/volume/util/quota/xfs/BUILD deleted file mode 100644 index 431587cf09e..00000000000 --- a/pkg/volume/util/quota/xfs/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["quota_xfs.go"], - cgo = True, - importpath = "k8s.io/kubernetes/pkg/volume/util/quota/xfs", - visibility = ["//visibility:public"], - deps = select({ - "@io_bazel_rules_go//go/platform:linux": [ - "//pkg/volume/util/quota/common:go_default_library", - "//vendor/golang.org/x/sys/unix:go_default_library", - "//vendor/k8s.io/klog:go_default_library", - ], - "//conditions:default": [], - }), -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/pkg/volume/util/quota/xfs/quota_xfs.go b/pkg/volume/util/quota/xfs/quota_xfs.go deleted file mode 100644 index c6669323c3a..00000000000 --- a/pkg/volume/util/quota/xfs/quota_xfs.go +++ /dev/null @@ -1,153 +0,0 @@ -// +build linux - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package xfs - -/* -#include -#include -#include -#include -#include -#include - -#ifndef PRJQUOTA -#define PRJQUOTA 2 -#endif -#ifndef XFS_PROJ_QUOTA -#define XFS_PROJ_QUOTA 2 -#endif -#ifndef Q_XSETPQLIM -#define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA) -#endif -#ifndef Q_XGETPQUOTA -#define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) -#endif -*/ -import "C" - -import ( - "fmt" - "syscall" - "unsafe" - - "golang.org/x/sys/unix" - "k8s.io/klog" - "k8s.io/kubernetes/pkg/volume/util/quota/common" -) - -const ( - linuxXfsMagic = 0x58465342 - // Documented in man xfs_quota(8); not necessarily the same - // as the filesystem blocksize - quotaBsize = 512 - bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 - maxQuota int64 = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 -) - -// VolumeProvider supplies a quota applier to the generic code. -type VolumeProvider struct { -} - -// GetQuotaApplier -- does this backing device support quotas that -// can be applied to directories? -func (*VolumeProvider) GetQuotaApplier(mountpoint string, backingDev string) common.LinuxVolumeQuotaApplier { - if common.IsFilesystemOfType(mountpoint, backingDev, linuxXfsMagic) { - return xfsVolumeQuota{backingDev} - } - return nil -} - -type xfsVolumeQuota struct { - backingDev string -} - -// GetQuotaOnDir -- get the quota ID that applies to this directory. -func (v xfsVolumeQuota) GetQuotaOnDir(path string) (common.QuotaID, error) { - return common.GetQuotaOnDir(path) -} - -// SetQuotaOnDir -- apply the specified quota to the directory. If -// bytes is not greater than zero, the quota should be applied in a -// way that is non-enforcing (either explicitly so or by setting a -// quota larger than anything the user may possibly create) -func (v xfsVolumeQuota) SetQuotaOnDir(path string, id common.QuotaID, bytes int64) error { - klog.V(3).Infof("xfsSetQuotaOn %s ID %v bytes %v", path, id, bytes) - if bytes < 0 || bytes > maxQuota { - bytes = maxQuota - } - - var d C.fs_disk_quota_t - d.d_version = C.FS_DQUOT_VERSION - d.d_id = C.__u32(id) - d.d_flags = C.XFS_PROJ_QUOTA - - d.d_fieldmask = C.FS_DQ_BHARD - d.d_blk_hardlimit = C.__u64(bytes / quotaBsize) - d.d_blk_softlimit = d.d_blk_hardlimit - - var cs = C.CString(v.backingDev) - defer C.free(unsafe.Pointer(cs)) - - _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, - uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), - uintptr(unsafe.Pointer(&d)), 0, 0) - if errno != 0 { - return fmt.Errorf("Failed to set quota limit for ID %d on %s: %v", - id, path, errno.Error()) - } - return common.ApplyProjectToDir(path, id) -} - -func (v xfsVolumeQuota) getQuotaInfo(path string, id common.QuotaID) (C.fs_disk_quota_t, syscall.Errno) { - var d C.fs_disk_quota_t - - var cs = C.CString(v.backingDev) - defer C.free(unsafe.Pointer(cs)) - - _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA, - uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(id)), - uintptr(unsafe.Pointer(&d)), 0, 0) - return d, errno -} - -// QuotaIDIsInUse -- determine whether the quota ID is already in use. -func (v xfsVolumeQuota) QuotaIDIsInUse(path string, id common.QuotaID) (bool, error) { - _, errno := v.getQuotaInfo(path, id) - return errno == 0, nil -} - -// GetConsumption -- retrieve the consumption (in bytes) of the directory -func (v xfsVolumeQuota) GetConsumption(path string, id common.QuotaID) (int64, error) { - d, errno := v.getQuotaInfo(path, id) - if errno != 0 { - return 0, fmt.Errorf("Failed to get quota for %s: %s", path, errno.Error()) - } - klog.V(3).Infof("Consumption for %s is %v", path, d.d_bcount*quotaBsize) - return int64(d.d_bcount) * quotaBsize, nil -} - -// GetInodes -- retrieve the number of inodes in use under the directory -func (v xfsVolumeQuota) GetInodes(path string, id common.QuotaID) (int64, error) { - d, errno := v.getQuotaInfo(path, id) - if errno != 0 { - return 0, fmt.Errorf("Failed to get quota for %s: %s", path, errno.Error()) - } - klog.V(3).Infof("Inode consumption for %s is %v", path, d.d_icount) - return int64(d.d_icount), nil -} diff --git a/pkg/volume/util/util.go b/pkg/volume/util/util.go index cb8ee4882df..f1a8affb9fb 100644 --- a/pkg/volume/util/util.go +++ b/pkg/volume/util/util.go @@ -540,3 +540,11 @@ func GetPluginMountDir(host volume.VolumeHost, name string) string { mntDir := filepath.Join(host.GetPluginDir(name), MountsInGlobalPDPath) return mntDir } + +// IsLocalEphemeralVolume determines whether the argument is a local ephemeral +// volume vs. some other type +func IsLocalEphemeralVolume(volume v1.Volume) bool { + return volume.GitRepo != nil || + (volume.EmptyDir != nil && volume.EmptyDir.Medium != v1.StorageMediumMemory) || + volume.ConfigMap != nil || volume.DownwardAPI != nil +} diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index fdcc30647da..bfc4b31d656 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -104,7 +104,7 @@ type Attributes struct { // MounterArgs provides more easily extensible arguments to Mounter type MounterArgs struct { FsGroup *int64 - DesiredSize int64 + DesiredSize *resource.Quantity PodUID string } From 46b3c7511320dbd6f12d2cc4370842f1fa7acc94 Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Fri, 11 Jan 2019 17:26:42 -0500 Subject: [PATCH 4/6] Add e2e test for quota-based eviction. Positive test is skipped if quotas not available. --- test/e2e_node/BUILD | 8 ++ test/e2e_node/image_list.go | 1 + test/e2e_node/quota_lsci_test.go | 147 ++++++++++++++++++++++++++ test/e2e_node/util.go | 15 +++ test/e2e_node/util_xfs_linux.go | 74 +++++++++++++ test/e2e_node/util_xfs_unsupported.go | 23 ++++ test/utils/image/manifest.go | 3 + 7 files changed, 271 insertions(+) create mode 100644 test/e2e_node/quota_lsci_test.go create mode 100644 test/e2e_node/util_xfs_linux.go create mode 100644 test/e2e_node/util_xfs_unsupported.go diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 230d8dbebc9..384714fc4e6 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -19,6 +19,8 @@ go_library( "node_problem_detector_linux.go", "resource_collector.go", "util.go", + "util_xfs_linux.go", + "util_xfs_unsupported.go", ], importpath = "k8s.io/kubernetes/test/e2e_node", visibility = ["//visibility:public"], @@ -41,7 +43,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//staging/src/k8s.io/component-base/featuregate:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library", "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", @@ -62,6 +66,7 @@ go_library( "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/util/mount:go_default_library", "//pkg/util/procfs:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", @@ -105,6 +110,7 @@ go_test( "node_perf_test.go", "pids_test.go", "pods_container_manager_test.go", + "quota_lsci_test.go", "resource_metrics_test.go", "resource_usage_test.go", "restart_test.go", @@ -138,6 +144,8 @@ go_test( "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/security/apparmor:go_default_library", + "//pkg/util/mount:go_default_library", + "//pkg/volume/util/quota:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/scheduling/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index 34c548029b1..416bf509ea4 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -49,6 +49,7 @@ var NodeImageWhiteList = sets.NewString( busyboxImage, "k8s.gcr.io/busybox@sha256:4bdd623e848417d96127e16037743f0cd8b528c026e9175e22a84f639eca58ff", imageutils.GetE2EImage(imageutils.Nginx), + imageutils.GetE2EImage(imageutils.Perl), imageutils.GetE2EImage(imageutils.ServeHostname), imageutils.GetE2EImage(imageutils.Netexec), imageutils.GetE2EImage(imageutils.Nonewprivs), diff --git a/test/e2e_node/quota_lsci_test.go b/test/e2e_node/quota_lsci_test.go new file mode 100644 index 00000000000..2fc09ff4053 --- /dev/null +++ b/test/e2e_node/quota_lsci_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "fmt" + "path/filepath" + "time" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/features" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume/util/quota" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + + . "github.com/onsi/ginkgo" +) + +const ( + LSCIQuotaFeature = features.LocalStorageCapacityIsolationFSQuotaMonitoring +) + +func runOneQuotaTest(f *framework.Framework, quotasRequested bool) { + evictionTestTimeout := 10 * time.Minute + sizeLimit := resource.MustParse("100Mi") + useOverLimit := 101 /* Mb */ + useUnderLimit := 99 /* Mb */ + // TODO: remove hardcoded kubelet volume directory path + // framework.TestContext.KubeVolumeDir is currently not populated for node e2e + // As for why we do this: see comment below at isXfs. + if isXfs("/var/lib/kubelet") { + useUnderLimit = 50 /* Mb */ + } + priority := 0 + if quotasRequested { + priority = 1 + } + Context(fmt.Sprintf(testContextFmt, fmt.Sprintf("use quotas for LSCI monitoring (quotas enabled: %v)", quotasRequested)), func() { + tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { + defer withFeatureGate(LSCIQuotaFeature, quotasRequested)() + // TODO: remove hardcoded kubelet volume directory path + // framework.TestContext.KubeVolumeDir is currently not populated for node e2e + if quotasRequested && !supportsQuotas("/var/lib/kubelet") { + // No point in running this as a positive test if quotas are not + // enabled on the underlying filesystem. + framework.Skipf("Cannot run LocalStorageCapacityIsolationQuotaMonitoring on filesystem without project quota enabled") + } + // setting a threshold to 0% disables; non-empty map overrides default value (necessary due to omitempty) + initialConfig.EvictionHard = map[string]string{"memory.available": "0%"} + initialConfig.FeatureGates[string(LSCIQuotaFeature)] = quotasRequested + }) + runEvictionTest(f, evictionTestTimeout, noPressure, noStarvedResource, logDiskMetrics, []podEvictSpec{ + { + evictionPriority: priority, // This pod should be evicted because of emptyDir violation only if quotas are enabled + pod: diskConcealingPod(fmt.Sprintf("emptydir-concealed-disk-over-sizelimit-quotas-%v", quotasRequested), useOverLimit, &v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit}, + }, v1.ResourceRequirements{}), + }, + { + evictionPriority: 0, // This pod should not be evicted because it uses less than its limit (test for quotas) + pod: diskConcealingPod(fmt.Sprintf("emptydir-concealed-disk-under-sizelimit-quotas-%v", quotasRequested), useUnderLimit, &v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit}, + }, v1.ResourceRequirements{}), + }, + }) + }) +} + +// LocalStorageCapacityIsolationQuotaMonitoring tests that quotas are +// used for monitoring rather than du. The mechanism is to create a +// pod that creates a file, deletes it, and writes data to it. If +// quotas are used to monitor, it will detect this deleted-but-in-use +// file; if du is used to monitor, it will not detect this. +var _ = framework.KubeDescribe("LocalStorageCapacityIsolationQuotaMonitoring [Slow] [Serial] [Disruptive] [Feature:LocalStorageCapacityIsolationQuota][NodeFeature:LSCIQuotaMonitoring]", func() { + f := framework.NewDefaultFramework("localstorage-quota-monitoring-test") + runOneQuotaTest(f, true) + runOneQuotaTest(f, false) +}) + +const ( + writeConcealedPodCommand = ` +my $file = "%s.bin"; +open OUT, ">$file" || die "Cannot open $file: $!\n"; +unlink "$file" || die "Cannot unlink $file: $!\n"; +my $a = "a"; +foreach (1..20) { $a = "$a$a"; } +foreach (1..%d) { syswrite(OUT, $a); } +sleep 999999;` +) + +// This is needed for testing eviction of pods using disk space in concealed files; the shell has no convenient +// way of performing I/O to a concealed file, and the busybox image doesn't contain Perl. +func diskConcealingPod(name string, diskConsumedMB int, volumeSource *v1.VolumeSource, resources v1.ResourceRequirements) *v1.Pod { + path := "" + volumeMounts := []v1.VolumeMount{} + volumes := []v1.Volume{} + if volumeSource != nil { + path = volumeMountPath + volumeMounts = []v1.VolumeMount{{MountPath: volumeMountPath, Name: volumeName}} + volumes = []v1.Volume{{Name: volumeName, VolumeSource: *volumeSource}} + } + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-pod", name)}, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Image: imageutils.GetE2EImage(imageutils.Perl), + Name: fmt.Sprintf("%s-container", name), + Command: []string{ + "perl", + "-e", + fmt.Sprintf(writeConcealedPodCommand, filepath.Join(path, "file"), diskConsumedMB), + }, + Resources: resources, + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + } +} + +// Don't bother returning an error; if something goes wrong, +// simply treat it as "no". +func supportsQuotas(dir string) bool { + supportsQuota, err := quota.SupportsQuotas(mount.New(""), dir) + return supportsQuota && err == nil +} diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index cb78d4289a4..9db6272f016 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -34,7 +34,9 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/component-base/featuregate" internalapi "k8s.io/cri-api/pkg/apis" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/pkg/features" @@ -62,6 +64,7 @@ var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "H var startServices = flag.Bool("start-services", true, "If true, start local node services") var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests") var busyboxImage = imageutils.GetE2EImage(imageutils.BusyBox) +var perlImage = imageutils.GetE2EImage(imageutils.Perl) const ( // Kubelet internal cgroup name for node allocatable cgroup. @@ -440,3 +443,15 @@ func reduceAllocatableMemoryUsage() { _, err := exec.Command("sudo", "sh", "-c", cmd).CombinedOutput() framework.ExpectNoError(err) } + +// Equivalent of featuregatetesting.SetFeatureGateDuringTest +// which can't be used here because we're not in a Testing context. +// This must be in a non-"_test" file to pass +// make verify WHAT=test-featuregates +func withFeatureGate(feature featuregate.Feature, desired bool) func() { + current := utilfeature.DefaultFeatureGate.Enabled(feature) + utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%v", string(feature), desired)) + return func() { + utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%v", string(feature), current)) + } +} diff --git a/test/e2e_node/util_xfs_linux.go b/test/e2e_node/util_xfs_linux.go new file mode 100644 index 00000000000..5f758134a0b --- /dev/null +++ b/test/e2e_node/util_xfs_linux.go @@ -0,0 +1,74 @@ +// +build linux + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "path/filepath" + "syscall" + + "k8s.io/kubernetes/pkg/util/mount" +) + +func detectMountpoint(m mount.Interface, path string) string { + path, err := filepath.Abs(path) + if err == nil { + path, err = filepath.EvalSymlinks(path) + } + if err != nil { + return "" + } + for path != "" && path != "/" { + isNotMount, err := m.IsLikelyNotMountPoint(path) + if err != nil { + return "" + } + if !isNotMount { + return path + } + path = filepath.Dir(path) + } + return "/" +} + +const ( + xfsMagic = 0x58465342 +) + +// XFS over-allocates and then eventually removes that excess allocation. +// That can lead to a file growing beyond its eventual size, causing +// an unnecessary eviction: +// +// % ls -ls +// total 32704 +// 32704 -rw-r--r-- 1 rkrawitz rkrawitz 20971520 Jan 15 13:16 foo.bin +// +// This issue can be hit regardless of the means used to count storage. +// It is not present in ext4fs. +func isXfs(dir string) bool { + mountpoint := detectMountpoint(mount.New(""), dir) + if mountpoint == "" { + return false + } + var buf syscall.Statfs_t + err := syscall.Statfs(mountpoint, &buf) + if err != nil { + return false + } + return buf.Type == xfsMagic +} diff --git a/test/e2e_node/util_xfs_unsupported.go b/test/e2e_node/util_xfs_unsupported.go new file mode 100644 index 00000000000..01fdb802b30 --- /dev/null +++ b/test/e2e_node/util_xfs_unsupported.go @@ -0,0 +1,23 @@ +// +build !linux + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +func isXfs(dir string) bool { + return false +} diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index dc6b2e1ad26..6fb3db8ea85 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -171,6 +171,8 @@ const ( // Pause - when these values are updated, also update cmd/kubelet/app/options/container_runtime.go // Pause image Pause + // Perl image + Perl // Porter image Porter // PortForwardTester image @@ -236,6 +238,7 @@ func initImageConfigs() map[int]Config { configs[NoSnatTestProxy] = Config{e2eRegistry, "no-snat-test-proxy", "1.0"} // Pause - when these values are updated, also update cmd/kubelet/app/options/container_runtime.go configs[Pause] = Config{gcRegistry, "pause", "3.1"} + configs[Perl] = Config{dockerLibraryRegistry, "perl", "5.26"} configs[Porter] = Config{e2eRegistry, "porter", "1.0"} configs[PortForwardTester] = Config{e2eRegistry, "port-forward-tester", "1.0"} configs[Redis] = Config{e2eRegistry, "redis", "1.0"} From 9f4a4d1f4952480e25f2477994f9ed2605ff3ae5 Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Tue, 21 May 2019 15:47:28 -0400 Subject: [PATCH 5/6] Log message rather than error if setting quota fails --- pkg/volume/emptydir/empty_dir.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go index e6e05b31a9b..4655f076f3f 100644 --- a/pkg/volume/emptydir/empty_dir.go +++ b/pkg/volume/emptydir/empty_dir.go @@ -233,23 +233,27 @@ func (ed *emptyDir) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { volume.SetVolumeOwnership(ed, mounterArgs.FsGroup) + // If setting up the quota fails, just log a message but don't actually error out. + // We'll use the old du mechanism in this case, at least until we support + // enforcement. if err == nil { volumeutil.SetReady(ed.getMetaDir()) - } - if mounterArgs.DesiredSize != nil { - hasQuotas, err := quota.SupportsQuotas(ed.mounter, dir) - if err != nil { - return fmt.Errorf("Unable to check for quota support on %s: %s", dir, err.Error()) - } - if hasQuotas { - klog.V(4).Infof("emptydir trying to assign quota %v on %s", mounterArgs.DesiredSize, dir) - err := quota.AssignQuota(ed.mounter, dir, mounterArgs.PodUID, mounterArgs.DesiredSize) + if mounterArgs.DesiredSize != nil { + // Deliberately shadow the outer use of err as noted + // above. + hasQuotas, err := quota.SupportsQuotas(ed.mounter, dir) if err != nil { - return fmt.Errorf("Set quota on %s failed %s", dir, err.Error()) + klog.V(3).Infof("Unable to check for quota support on %s: %s", dir, err.Error()) + } else if hasQuotas { + klog.V(4).Infof("emptydir trying to assign quota %v on %s", mounterArgs.DesiredSize, dir) + err := quota.AssignQuota(ed.mounter, dir, mounterArgs.PodUID, mounterArgs.DesiredSize) + if err != nil { + klog.V(3).Infof("Set quota on %s failed %s", dir, err.Error()) + } } } } - return nil + return err } // setupTmpfs creates a tmpfs mount at the specified directory. From 38753efcc758e16cb501906b9a1aabb7308a0246 Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Tue, 21 May 2019 18:30:37 -0400 Subject: [PATCH 6/6] Set sizeLimit to the smaller of the ephemeral storage limit and the emptydir limit (if set) --- pkg/kubelet/volumemanager/cache/desired_state_of_world.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index c80b03e7062..28b15b5f161 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -238,6 +238,12 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( _, podLimits := apiv1resource.PodRequestsAndLimits(pod) ephemeralStorageLimit := podLimits[v1.ResourceEphemeralStorage] sizeLimit = resource.NewQuantity(ephemeralStorageLimit.Value(), resource.BinarySI) + if volumeSpec.Volume.EmptyDir != nil && + volumeSpec.Volume.EmptyDir.SizeLimit != nil && + volumeSpec.Volume.EmptyDir.SizeLimit.Value() > 0 && + volumeSpec.Volume.EmptyDir.SizeLimit.Value() < sizeLimit.Value() { + sizeLimit = resource.NewQuantity(volumeSpec.Volume.EmptyDir.SizeLimit.Value(), resource.BinarySI) + } } } dsw.volumesToMount[volumeName] = volumeToMount{