Add e2e test for wrapper volumes
Use git server in e2e and refactor wrapper builder Update e2e test to use a fake git server
This commit is contained in:
parent
c0d49450e4
commit
0fa5b6c4f7
@ -55,10 +55,7 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/util"
|
"k8s.io/kubernetes/pkg/util"
|
||||||
"k8s.io/kubernetes/pkg/util/sets"
|
"k8s.io/kubernetes/pkg/util/sets"
|
||||||
"k8s.io/kubernetes/pkg/util/wait"
|
"k8s.io/kubernetes/pkg/util/wait"
|
||||||
"k8s.io/kubernetes/pkg/volume"
|
|
||||||
"k8s.io/kubernetes/pkg/volume/empty_dir"
|
"k8s.io/kubernetes/pkg/volume/empty_dir"
|
||||||
"k8s.io/kubernetes/pkg/volume/git_repo"
|
|
||||||
"k8s.io/kubernetes/pkg/volume/secret"
|
|
||||||
"k8s.io/kubernetes/plugin/pkg/scheduler"
|
"k8s.io/kubernetes/plugin/pkg/scheduler"
|
||||||
_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"
|
_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"
|
||||||
"k8s.io/kubernetes/plugin/pkg/scheduler/factory"
|
"k8s.io/kubernetes/plugin/pkg/scheduler/factory"
|
||||||
@ -214,7 +211,7 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string
|
|||||||
10250, /* KubeletPort */
|
10250, /* KubeletPort */
|
||||||
0, /* ReadOnlyPort */
|
0, /* ReadOnlyPort */
|
||||||
api.NamespaceDefault,
|
api.NamespaceDefault,
|
||||||
probeVolumePlugins(),
|
empty_dir.ProbeVolumePlugins(),
|
||||||
nil,
|
nil,
|
||||||
cadvisorInterface,
|
cadvisorInterface,
|
||||||
configFilePath,
|
configFilePath,
|
||||||
@ -247,7 +244,7 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string
|
|||||||
10251, /* KubeletPort */
|
10251, /* KubeletPort */
|
||||||
0, /* ReadOnlyPort */
|
0, /* ReadOnlyPort */
|
||||||
api.NamespaceDefault,
|
api.NamespaceDefault,
|
||||||
probeVolumePlugins(),
|
empty_dir.ProbeVolumePlugins(),
|
||||||
nil,
|
nil,
|
||||||
cadvisorInterface,
|
cadvisorInterface,
|
||||||
"",
|
"",
|
||||||
@ -268,16 +265,6 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string
|
|||||||
return apiServer.URL, configFilePath
|
return apiServer.URL, configFilePath
|
||||||
}
|
}
|
||||||
|
|
||||||
// The list of plugins to probe in this test
|
|
||||||
func probeVolumePlugins() []volume.VolumePlugin {
|
|
||||||
allPlugins := []volume.VolumePlugin{}
|
|
||||||
|
|
||||||
allPlugins = append(allPlugins, empty_dir.ProbeVolumePlugins()...)
|
|
||||||
allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
|
|
||||||
allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...)
|
|
||||||
return allPlugins
|
|
||||||
}
|
|
||||||
|
|
||||||
func makeTempDirOrDie(prefix string, baseDir string) string {
|
func makeTempDirOrDie(prefix string, baseDir string) string {
|
||||||
if baseDir == "" {
|
if baseDir == "" {
|
||||||
baseDir = "/tmp"
|
baseDir = "/tmp"
|
||||||
@ -1002,7 +989,6 @@ func main() {
|
|||||||
runServiceTest,
|
runServiceTest,
|
||||||
runAPIVersionsTest,
|
runAPIVersionsTest,
|
||||||
runMasterServiceTest,
|
runMasterServiceTest,
|
||||||
runWrapperVolumesPluginsTest,
|
|
||||||
func(c *client.Client) {
|
func(c *client.Client) {
|
||||||
runSelfLinkTestOnNamespace(c, api.NamespaceDefault)
|
runSelfLinkTestOnNamespace(c, api.NamespaceDefault)
|
||||||
runSelfLinkTestOnNamespace(c, "other")
|
runSelfLinkTestOnNamespace(c, "other")
|
||||||
@ -1052,8 +1038,8 @@ func main() {
|
|||||||
// 1 pod infra container + 1 container from the service test.
|
// 1 pod infra container + 1 container from the service test.
|
||||||
// The total number of container created is 9
|
// The total number of container created is 9
|
||||||
|
|
||||||
if len(createdConts) != 14 {
|
if len(createdConts) != 12 {
|
||||||
glog.Fatalf("Expected 14 containers; got %v\n\nlist of created containers:\n\n%#v\n\nDocker 1 Created:\n\n%#v\n\nDocker 2 Created:\n\n%#v\n\n", len(createdConts), createdConts.List(), fakeDocker1.Created, fakeDocker2.Created)
|
glog.Fatalf("Expected 12 containers; got %v\n\nlist of created containers:\n\n%#v\n\nDocker 1 Created:\n\n%#v\n\nDocker 2 Created:\n\n%#v\n\n", len(createdConts), createdConts.List(), fakeDocker1.Created, fakeDocker2.Created)
|
||||||
}
|
}
|
||||||
glog.Infof("OK - found created containers: %#v", createdConts.List())
|
glog.Infof("OK - found created containers: %#v", createdConts.List())
|
||||||
|
|
||||||
@ -1111,72 +1097,3 @@ const (
|
|||||||
}
|
}
|
||||||
}`
|
}`
|
||||||
)
|
)
|
||||||
|
|
||||||
// This is a test to make sure multiple empty_dir wrapper based volumes should not affect each other
|
|
||||||
func runWrapperVolumesPluginsTest(client *client.Client) {
|
|
||||||
// Make a secret object.
|
|
||||||
ns := api.NamespaceDefault
|
|
||||||
s := api.Secret{
|
|
||||||
ObjectMeta: api.ObjectMeta{
|
|
||||||
Name: "secret-name",
|
|
||||||
Namespace: ns,
|
|
||||||
},
|
|
||||||
Data: map[string][]byte{
|
|
||||||
"data": []byte("value1\n"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := client.Secrets(s.Namespace).Create(&s); err != nil {
|
|
||||||
glog.Fatalf("unable to create test secret: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
pod := &api.Pod{
|
|
||||||
Spec: api.PodSpec{
|
|
||||||
Containers: []api.Container{
|
|
||||||
{
|
|
||||||
Name: "vol-container",
|
|
||||||
Image: "kubernetes/pause",
|
|
||||||
VolumeMounts: []api.VolumeMount{
|
|
||||||
{
|
|
||||||
Name: "vol-secret",
|
|
||||||
MountPath: "/fake/path-1",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "vol-git",
|
|
||||||
MountPath: "/fake/path-2",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Volumes: []api.Volume{
|
|
||||||
{
|
|
||||||
Name: "vol-secret",
|
|
||||||
VolumeSource: api.VolumeSource{
|
|
||||||
Secret: &api.SecretVolumeSource{
|
|
||||||
SecretName: "secret-name",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "vol-git",
|
|
||||||
VolumeSource: api.VolumeSource{
|
|
||||||
GitRepo: &api.GitRepoVolumeSource{
|
|
||||||
Repository: "https://github.com/kubernetes/kubedash",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
pod.ObjectMeta.Name = "wrappedvolumes.foo"
|
|
||||||
foo, err := client.Pods(api.NamespaceDefault).Create(pod)
|
|
||||||
if err != nil {
|
|
||||||
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
|
|
||||||
}
|
|
||||||
if err := wait.Poll(time.Second, longTestTimeout, podRunning(client, foo.Namespace, foo.Name)); err != nil {
|
|
||||||
glog.Fatalf("FAILED: pod never started running %v", err)
|
|
||||||
}
|
|
||||||
glog.Info("empty_dir wrapper based volumes doesn't affect each other: test passed.")
|
|
||||||
}
|
|
||||||
|
@ -467,11 +467,11 @@ func (c *PersistentVolumeProvisionerController) GetKubeClient() client.Interface
|
|||||||
return c.client.GetKubeClient()
|
return c.client.GetKubeClient()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *PersistentVolumeProvisionerController) NewWrapperBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
func (c *PersistentVolumeProvisionerController) NewWrapperBuilder(volName string, spec volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
||||||
return nil, fmt.Errorf("NewWrapperBuilder not supported by PVClaimBinder's VolumeHost implementation")
|
return nil, fmt.Errorf("NewWrapperBuilder not supported by PVClaimBinder's VolumeHost implementation")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *PersistentVolumeProvisionerController) NewWrapperCleaner(spec *volume.Spec, podUID types.UID) (volume.Cleaner, error) {
|
func (c *PersistentVolumeProvisionerController) NewWrapperCleaner(volName string, spec volume.Spec, podUID types.UID) (volume.Cleaner, error) {
|
||||||
return nil, fmt.Errorf("NewWrapperCleaner not supported by PVClaimBinder's VolumeHost implementation")
|
return nil, fmt.Errorf("NewWrapperCleaner not supported by PVClaimBinder's VolumeHost implementation")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -289,11 +289,11 @@ func (f *PersistentVolumeRecycler) GetKubeClient() client.Interface {
|
|||||||
return f.kubeClient
|
return f.kubeClient
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *PersistentVolumeRecycler) NewWrapperBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
func (f *PersistentVolumeRecycler) NewWrapperBuilder(volName string, spec volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
||||||
return nil, fmt.Errorf("NewWrapperBuilder not supported by PVClaimBinder's VolumeHost implementation")
|
return nil, fmt.Errorf("NewWrapperBuilder not supported by PVClaimBinder's VolumeHost implementation")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *PersistentVolumeRecycler) NewWrapperCleaner(spec *volume.Spec, podUID types.UID) (volume.Cleaner, error) {
|
func (f *PersistentVolumeRecycler) NewWrapperCleaner(volName string, spec volume.Spec, podUID types.UID) (volume.Cleaner, error) {
|
||||||
return nil, fmt.Errorf("NewWrapperCleaner not supported by PVClaimBinder's VolumeHost implementation")
|
return nil, fmt.Errorf("NewWrapperCleaner not supported by PVClaimBinder's VolumeHost implementation")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -57,16 +57,28 @@ func (vh *volumeHost) GetKubeClient() client.Interface {
|
|||||||
return vh.kubelet.kubeClient
|
return vh.kubelet.kubeClient
|
||||||
}
|
}
|
||||||
|
|
||||||
func (vh *volumeHost) NewWrapperBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
func (vh *volumeHost) NewWrapperBuilder(volName string, spec volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
||||||
b, err := vh.kubelet.newVolumeBuilderFromPlugins(spec, pod, opts)
|
// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
|
||||||
|
wrapperVolumeName := "wrapped_" + volName
|
||||||
|
if spec.Volume != nil {
|
||||||
|
spec.Volume.Name = wrapperVolumeName
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := vh.kubelet.newVolumeBuilderFromPlugins(&spec, pod, opts)
|
||||||
if err == nil && b == nil {
|
if err == nil && b == nil {
|
||||||
return nil, errUnsupportedVolumeType
|
return nil, errUnsupportedVolumeType
|
||||||
}
|
}
|
||||||
return b, nil
|
return b, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (vh *volumeHost) NewWrapperCleaner(spec *volume.Spec, podUID types.UID) (volume.Cleaner, error) {
|
func (vh *volumeHost) NewWrapperCleaner(volName string, spec volume.Spec, podUID types.UID) (volume.Cleaner, error) {
|
||||||
plugin, err := vh.kubelet.volumePluginMgr.FindPluginBySpec(spec)
|
// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
|
||||||
|
wrapperVolumeName := "wrapped_" + volName
|
||||||
|
if spec.Volume != nil {
|
||||||
|
spec.Volume.Name = wrapperVolumeName
|
||||||
|
}
|
||||||
|
|
||||||
|
plugin, err := vh.kubelet.volumePluginMgr.FindPluginBySpec(&spec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -51,6 +51,10 @@ type downwardAPIPlugin struct {
|
|||||||
|
|
||||||
var _ volume.VolumePlugin = &downwardAPIPlugin{}
|
var _ volume.VolumePlugin = &downwardAPIPlugin{}
|
||||||
|
|
||||||
|
var wrappedVolumeSpec = volume.Spec{
|
||||||
|
Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: api.StorageMediumMemory}}},
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *downwardAPIPlugin) Init(host volume.VolumeHost) error {
|
func (plugin *downwardAPIPlugin) Init(host volume.VolumeHost) error {
|
||||||
plugin.host = host
|
plugin.host = host
|
||||||
return nil
|
return nil
|
||||||
@ -66,11 +70,10 @@ func (plugin *downwardAPIPlugin) CanSupport(spec *volume.Spec) bool {
|
|||||||
|
|
||||||
func (plugin *downwardAPIPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
func (plugin *downwardAPIPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
||||||
v := &downwardAPIVolume{
|
v := &downwardAPIVolume{
|
||||||
volName: spec.Name(),
|
volName: spec.Name(),
|
||||||
pod: pod,
|
pod: pod,
|
||||||
podUID: pod.UID,
|
podUID: pod.UID,
|
||||||
plugin: plugin,
|
plugin: plugin,
|
||||||
wrappedVolumeSpec: volume.GetWrappedVolumeSpec(spec.Name(), downwardAPIPluginName),
|
|
||||||
}
|
}
|
||||||
v.fieldReferenceFileNames = make(map[string]string)
|
v.fieldReferenceFileNames = make(map[string]string)
|
||||||
for _, fileInfo := range spec.Volume.DownwardAPI.Items {
|
for _, fileInfo := range spec.Volume.DownwardAPI.Items {
|
||||||
@ -78,15 +81,17 @@ func (plugin *downwardAPIPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opt
|
|||||||
}
|
}
|
||||||
return &downwardAPIVolumeBuilder{
|
return &downwardAPIVolumeBuilder{
|
||||||
downwardAPIVolume: v,
|
downwardAPIVolume: v,
|
||||||
opts: &opts}, nil
|
opts: &opts,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (plugin *downwardAPIPlugin) NewCleaner(volName string, podUID types.UID) (volume.Cleaner, error) {
|
func (plugin *downwardAPIPlugin) NewCleaner(volName string, podUID types.UID) (volume.Cleaner, error) {
|
||||||
return &downwardAPIVolumeCleaner{&downwardAPIVolume{
|
return &downwardAPIVolumeCleaner{
|
||||||
volName: volName,
|
&downwardAPIVolume{
|
||||||
podUID: podUID,
|
volName: volName,
|
||||||
plugin: plugin,
|
podUID: podUID,
|
||||||
wrappedVolumeSpec: volume.GetWrappedVolumeSpec(volName, downwardAPIPluginName)},
|
plugin: plugin,
|
||||||
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -98,7 +103,6 @@ type downwardAPIVolume struct {
|
|||||||
podUID types.UID // TODO: remove this redundancy as soon NewCleaner func will have *api.POD and not only types.UID
|
podUID types.UID // TODO: remove this redundancy as soon NewCleaner func will have *api.POD and not only types.UID
|
||||||
plugin *downwardAPIPlugin
|
plugin *downwardAPIPlugin
|
||||||
volume.MetricsNil
|
volume.MetricsNil
|
||||||
wrappedVolumeSpec *volume.Spec
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// downwardAPIVolumeBuilder fetches info from downward API from the pod
|
// downwardAPIVolumeBuilder fetches info from downward API from the pod
|
||||||
@ -132,7 +136,7 @@ func (b *downwardAPIVolumeBuilder) SetUp() error {
|
|||||||
func (b *downwardAPIVolumeBuilder) SetUpAt(dir string) error {
|
func (b *downwardAPIVolumeBuilder) SetUpAt(dir string) error {
|
||||||
glog.V(3).Infof("Setting up a downwardAPI volume %v for pod %v/%v at %v", b.volName, b.pod.Namespace, b.pod.Name, dir)
|
glog.V(3).Infof("Setting up a downwardAPI volume %v for pod %v/%v at %v", b.volName, b.pod.Namespace, b.pod.Name, dir)
|
||||||
// Wrap EmptyDir. Here we rely on the idempotency of the wrapped plugin to avoid repeatedly mounting
|
// Wrap EmptyDir. Here we rely on the idempotency of the wrapped plugin to avoid repeatedly mounting
|
||||||
wrapped, err := b.plugin.host.NewWrapperBuilder(b.wrappedVolumeSpec, b.pod, *b.opts)
|
wrapped, err := b.plugin.host.NewWrapperBuilder(b.volName, wrappedVolumeSpec, b.pod, *b.opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("Couldn't setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error())
|
glog.Errorf("Couldn't setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error())
|
||||||
return err
|
return err
|
||||||
@ -366,7 +370,7 @@ func (c *downwardAPIVolumeCleaner) TearDownAt(dir string) error {
|
|||||||
glog.V(3).Infof("Tearing down volume %v for pod %v at %v", c.volName, c.podUID, dir)
|
glog.V(3).Infof("Tearing down volume %v for pod %v at %v", c.volName, c.podUID, dir)
|
||||||
|
|
||||||
// Wrap EmptyDir, let it do the teardown.
|
// Wrap EmptyDir, let it do the teardown.
|
||||||
wrapped, err := c.plugin.host.NewWrapperCleaner(c.wrappedVolumeSpec, c.podUID)
|
wrapped, err := c.plugin.host.NewWrapperCleaner(c.volName, wrappedVolumeSpec, c.podUID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -143,7 +143,7 @@ func TestLabels(t *testing.T) {
|
|||||||
t.Errorf("Failed to setup volume: %v", err)
|
t.Errorf("Failed to setup volume: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// gitRepo volume should create it's own empty wrapper path
|
// downwardAPI volume should create its own empty wrapper path
|
||||||
podWrapperMetadataDir := fmt.Sprintf("%v/pods/%v/plugins/kubernetes.io~empty-dir/wrapped_%v", rootDir, testPodUID, testVolumeName)
|
podWrapperMetadataDir := fmt.Sprintf("%v/pods/%v/plugins/kubernetes.io~empty-dir/wrapped_%v", rootDir, testPodUID, testVolumeName)
|
||||||
|
|
||||||
if _, err := os.Stat(podWrapperMetadataDir); err != nil {
|
if _, err := os.Stat(podWrapperMetadataDir); err != nil {
|
||||||
|
@ -41,6 +41,10 @@ type gitRepoPlugin struct {
|
|||||||
|
|
||||||
var _ volume.VolumePlugin = &gitRepoPlugin{}
|
var _ volume.VolumePlugin = &gitRepoPlugin{}
|
||||||
|
|
||||||
|
var wrappedVolumeSpec = volume.Spec{
|
||||||
|
Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
|
||||||
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
gitRepoPluginName = "kubernetes.io/git-repo"
|
gitRepoPluginName = "kubernetes.io/git-repo"
|
||||||
)
|
)
|
||||||
@ -61,10 +65,9 @@ func (plugin *gitRepoPlugin) CanSupport(spec *volume.Spec) bool {
|
|||||||
func (plugin *gitRepoPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
func (plugin *gitRepoPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Builder, error) {
|
||||||
return &gitRepoVolumeBuilder{
|
return &gitRepoVolumeBuilder{
|
||||||
gitRepoVolume: &gitRepoVolume{
|
gitRepoVolume: &gitRepoVolume{
|
||||||
volName: spec.Name(),
|
volName: spec.Name(),
|
||||||
podUID: pod.UID,
|
podUID: pod.UID,
|
||||||
plugin: plugin,
|
plugin: plugin,
|
||||||
wrappedVolumeSpec: volume.GetWrappedVolumeSpec(spec.Name(), gitRepoPluginName),
|
|
||||||
},
|
},
|
||||||
pod: *pod,
|
pod: *pod,
|
||||||
source: spec.Volume.GitRepo.Repository,
|
source: spec.Volume.GitRepo.Repository,
|
||||||
@ -78,10 +81,9 @@ func (plugin *gitRepoPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opts vo
|
|||||||
func (plugin *gitRepoPlugin) NewCleaner(volName string, podUID types.UID) (volume.Cleaner, error) {
|
func (plugin *gitRepoPlugin) NewCleaner(volName string, podUID types.UID) (volume.Cleaner, error) {
|
||||||
return &gitRepoVolumeCleaner{
|
return &gitRepoVolumeCleaner{
|
||||||
&gitRepoVolume{
|
&gitRepoVolume{
|
||||||
volName: volName,
|
volName: volName,
|
||||||
podUID: podUID,
|
podUID: podUID,
|
||||||
plugin: plugin,
|
plugin: plugin,
|
||||||
wrappedVolumeSpec: volume.GetWrappedVolumeSpec(volName, gitRepoPluginName),
|
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@ -89,10 +91,9 @@ func (plugin *gitRepoPlugin) NewCleaner(volName string, podUID types.UID) (volum
|
|||||||
// gitRepo volumes are directories which are pre-filled from a git repository.
|
// gitRepo volumes are directories which are pre-filled from a git repository.
|
||||||
// These do not persist beyond the lifetime of a pod.
|
// These do not persist beyond the lifetime of a pod.
|
||||||
type gitRepoVolume struct {
|
type gitRepoVolume struct {
|
||||||
volName string
|
volName string
|
||||||
podUID types.UID
|
podUID types.UID
|
||||||
plugin *gitRepoPlugin
|
plugin *gitRepoPlugin
|
||||||
wrappedVolumeSpec *volume.Spec
|
|
||||||
volume.MetricsNil
|
volume.MetricsNil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -138,7 +139,7 @@ func (b *gitRepoVolumeBuilder) SetUpAt(dir string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Wrap EmptyDir, let it do the setup.
|
// Wrap EmptyDir, let it do the setup.
|
||||||
wrapped, err := b.plugin.host.NewWrapperBuilder(b.wrappedVolumeSpec, &b.pod, b.opts)
|
wrapped, err := b.plugin.host.NewWrapperBuilder(b.volName, wrappedVolumeSpec, &b.pod, b.opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -218,7 +219,7 @@ func (c *gitRepoVolumeCleaner) TearDown() error {
|
|||||||
func (c *gitRepoVolumeCleaner) TearDownAt(dir string) error {
|
func (c *gitRepoVolumeCleaner) TearDownAt(dir string) error {
|
||||||
|
|
||||||
// Wrap EmptyDir, let it do the teardown.
|
// Wrap EmptyDir, let it do the teardown.
|
||||||
wrapped, err := c.plugin.host.NewWrapperCleaner(c.wrappedVolumeSpec, c.podUID)
|
wrapped, err := c.plugin.host.NewWrapperCleaner(c.volName, wrappedVolumeSpec, c.podUID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -266,7 +266,7 @@ func doTestPlugin(scenario struct {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// gitRepo volume should create it's own empty wrapper path
|
// gitRepo volume should create its own empty wrapper path
|
||||||
podWrapperMetadataDir := fmt.Sprintf("%v/pods/poduid/plugins/kubernetes.io~empty-dir/wrapped_%v", rootDir, scenario.vol.Name)
|
podWrapperMetadataDir := fmt.Sprintf("%v/pods/poduid/plugins/kubernetes.io~empty-dir/wrapped_%v", rootDir, scenario.vol.Name)
|
||||||
|
|
||||||
if _, err := os.Stat(podWrapperMetadataDir); err != nil {
|
if _, err := os.Stat(podWrapperMetadataDir); err != nil {
|
||||||
|
@ -80,7 +80,7 @@ func (plugin *persistentClaimPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod,
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
builder, err := plugin.host.NewWrapperBuilder(volume.NewSpecFromPersistentVolume(pv, spec.ReadOnly), pod, opts)
|
builder, err := plugin.host.NewWrapperBuilder(claim.Spec.VolumeName, *volume.NewSpecFromPersistentVolume(pv, spec.ReadOnly), pod, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("Error creating builder for claim: %+v\n", claim.Name)
|
glog.Errorf("Error creating builder for claim: %+v\n", claim.Name)
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -144,12 +144,12 @@ type VolumeHost interface {
|
|||||||
// the provided spec. This is used to implement volume plugins which
|
// the provided spec. This is used to implement volume plugins which
|
||||||
// "wrap" other plugins. For example, the "secret" volume is
|
// "wrap" other plugins. For example, the "secret" volume is
|
||||||
// implemented in terms of the "emptyDir" volume.
|
// implemented in terms of the "emptyDir" volume.
|
||||||
NewWrapperBuilder(spec *Spec, pod *api.Pod, opts VolumeOptions) (Builder, error)
|
NewWrapperBuilder(volName string, spec Spec, pod *api.Pod, opts VolumeOptions) (Builder, error)
|
||||||
|
|
||||||
// NewWrapperCleaner finds an appropriate plugin with which to handle
|
// NewWrapperCleaner finds an appropriate plugin with which to handle
|
||||||
// the provided spec. See comments on NewWrapperBuilder for more
|
// the provided spec. See comments on NewWrapperBuilder for more
|
||||||
// context.
|
// context.
|
||||||
NewWrapperCleaner(spec *Spec, podUID types.UID) (Cleaner, error)
|
NewWrapperCleaner(volName string, spec Spec, podUID types.UID) (Cleaner, error)
|
||||||
|
|
||||||
// Get cloud provider from kubelet.
|
// Get cloud provider from kubelet.
|
||||||
GetCloudProvider() cloudprovider.Interface
|
GetCloudProvider() cloudprovider.Interface
|
||||||
|
@ -47,6 +47,10 @@ type secretPlugin struct {
|
|||||||
|
|
||||||
var _ volume.VolumePlugin = &secretPlugin{}
|
var _ volume.VolumePlugin = &secretPlugin{}
|
||||||
|
|
||||||
|
var wrappedVolumeSpec = volume.Spec{
|
||||||
|
Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: api.StorageMediumMemory}}},
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *secretPlugin) Init(host volume.VolumeHost) error {
|
func (plugin *secretPlugin) Init(host volume.VolumeHost) error {
|
||||||
plugin.host = host
|
plugin.host = host
|
||||||
return nil
|
return nil
|
||||||
@ -69,7 +73,6 @@ func (plugin *secretPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opts vol
|
|||||||
plugin.host.GetMounter(),
|
plugin.host.GetMounter(),
|
||||||
plugin.host.GetWriter(),
|
plugin.host.GetWriter(),
|
||||||
volume.MetricsNil{},
|
volume.MetricsNil{},
|
||||||
volume.GetWrappedVolumeSpec(spec.Name(), secretPluginName),
|
|
||||||
},
|
},
|
||||||
secretName: spec.Volume.Secret.SecretName,
|
secretName: spec.Volume.Secret.SecretName,
|
||||||
pod: *pod,
|
pod: *pod,
|
||||||
@ -86,7 +89,6 @@ func (plugin *secretPlugin) NewCleaner(volName string, podUID types.UID) (volume
|
|||||||
plugin.host.GetMounter(),
|
plugin.host.GetMounter(),
|
||||||
plugin.host.GetWriter(),
|
plugin.host.GetWriter(),
|
||||||
volume.MetricsNil{},
|
volume.MetricsNil{},
|
||||||
volume.GetWrappedVolumeSpec(volName, secretPluginName),
|
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@ -98,7 +100,6 @@ type secretVolume struct {
|
|||||||
mounter mount.Interface
|
mounter mount.Interface
|
||||||
writer ioutil.Writer
|
writer ioutil.Writer
|
||||||
volume.MetricsNil
|
volume.MetricsNil
|
||||||
wrappedVolumeSpec *volume.Spec
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ volume.Volume = &secretVolume{}
|
var _ volume.Volume = &secretVolume{}
|
||||||
@ -152,7 +153,7 @@ func (b *secretVolumeBuilder) SetUpAt(dir string) error {
|
|||||||
glog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir)
|
glog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir)
|
||||||
|
|
||||||
// Wrap EmptyDir, let it do the setup.
|
// Wrap EmptyDir, let it do the setup.
|
||||||
wrapped, err := b.plugin.host.NewWrapperBuilder(b.wrappedVolumeSpec, &b.pod, *b.opts)
|
wrapped, err := b.plugin.host.NewWrapperBuilder(b.volName, wrappedVolumeSpec, &b.pod, *b.opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -217,7 +218,7 @@ func (c *secretVolumeCleaner) TearDownAt(dir string) error {
|
|||||||
glog.V(3).Infof("Tearing down volume %v for pod %v at %v", c.volName, c.podUID, dir)
|
glog.V(3).Infof("Tearing down volume %v for pod %v at %v", c.volName, c.podUID, dir)
|
||||||
|
|
||||||
// Wrap EmptyDir, let it do the teardown.
|
// Wrap EmptyDir, let it do the teardown.
|
||||||
wrapped, err := c.plugin.host.NewWrapperCleaner(c.wrappedVolumeSpec, c.podUID)
|
wrapped, err := c.plugin.host.NewWrapperCleaner(c.volName, wrappedVolumeSpec, c.podUID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -110,7 +110,7 @@ func TestPlugin(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// secret volume should create it's own empty wrapper path
|
// secret volume should create its own empty wrapper path
|
||||||
podWrapperMetadataDir := fmt.Sprintf("%v/pods/test_pod_uid/plugins/kubernetes.io~empty-dir/wrapped_test_volume_name", rootDir)
|
podWrapperMetadataDir := fmt.Sprintf("%v/pods/test_pod_uid/plugins/kubernetes.io~empty-dir/wrapped_test_volume_name", rootDir)
|
||||||
|
|
||||||
if _, err := os.Stat(podWrapperMetadataDir); err != nil {
|
if _, err := os.Stat(podWrapperMetadataDir); err != nil {
|
||||||
|
@ -80,16 +80,26 @@ func (f *fakeVolumeHost) GetWriter() io.Writer {
|
|||||||
return f.writer
|
return f.writer
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeVolumeHost) NewWrapperBuilder(spec *Spec, pod *api.Pod, opts VolumeOptions) (Builder, error) {
|
func (f *fakeVolumeHost) NewWrapperBuilder(volName string, spec Spec, pod *api.Pod, opts VolumeOptions) (Builder, error) {
|
||||||
plug, err := f.pluginMgr.FindPluginBySpec(spec)
|
// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
|
||||||
|
wrapperVolumeName := "wrapped_" + volName
|
||||||
|
if spec.Volume != nil {
|
||||||
|
spec.Volume.Name = wrapperVolumeName
|
||||||
|
}
|
||||||
|
plug, err := f.pluginMgr.FindPluginBySpec(&spec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return plug.NewBuilder(spec, pod, opts)
|
return plug.NewBuilder(&spec, pod, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeVolumeHost) NewWrapperCleaner(spec *Spec, podUID types.UID) (Cleaner, error) {
|
func (f *fakeVolumeHost) NewWrapperCleaner(volName string, spec Spec, podUID types.UID) (Cleaner, error) {
|
||||||
plug, err := f.pluginMgr.FindPluginBySpec(spec)
|
// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
|
||||||
|
wrapperVolumeName := "wrapped_" + volName
|
||||||
|
if spec.Volume != nil {
|
||||||
|
spec.Volume.Name = wrapperVolumeName
|
||||||
|
}
|
||||||
|
plug, err := f.pluginMgr.FindPluginBySpec(&spec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -17,7 +17,6 @@ limitations under the License.
|
|||||||
package volume
|
package volume
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"k8s.io/kubernetes/pkg/api"
|
"k8s.io/kubernetes/pkg/api"
|
||||||
"k8s.io/kubernetes/pkg/api/resource"
|
"k8s.io/kubernetes/pkg/api/resource"
|
||||||
@ -25,10 +24,6 @@ import (
|
|||||||
"path"
|
"path"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
randomLength = 5
|
|
||||||
)
|
|
||||||
|
|
||||||
// Volume represents a directory used by pods or hosts on a node.
|
// Volume represents a directory used by pods or hosts on a node.
|
||||||
// All method implementations of methods in the volume interface must be idempotent.
|
// All method implementations of methods in the volume interface must be idempotent.
|
||||||
type Volume interface {
|
type Volume interface {
|
||||||
@ -137,22 +132,3 @@ func RenameDirectory(oldPath, newName string) (string, error) {
|
|||||||
}
|
}
|
||||||
return newPath, nil
|
return newPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return the wrapper volume spec of specific volume plugin
|
|
||||||
func GetWrappedVolumeSpec(volName string, pluginName string) *Spec {
|
|
||||||
// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
|
|
||||||
wrapperVolumeName := fmt.Sprintf("wrapped_%v", volName)
|
|
||||||
|
|
||||||
var volumeSpec *Spec
|
|
||||||
switch pluginName {
|
|
||||||
case "kubernetes.io/downward-api", "kubernetes.io/secret":
|
|
||||||
volumeSpec = &Spec{
|
|
||||||
Volume: &api.Volume{Name: wrapperVolumeName, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: api.StorageMediumMemory}}},
|
|
||||||
}
|
|
||||||
case "kubernetes.io/git-repo":
|
|
||||||
volumeSpec = &Spec{
|
|
||||||
Volume: &api.Volume{Name: wrapperVolumeName, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return volumeSpec
|
|
||||||
}
|
|
||||||
|
179
test/e2e/empty_dir_wrapper.go
Normal file
179
test/e2e/empty_dir_wrapper.go
Normal file
@ -0,0 +1,179 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package e2e
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/kubernetes/pkg/api"
|
||||||
|
"k8s.io/kubernetes/pkg/util"
|
||||||
|
"k8s.io/kubernetes/pkg/util/intstr"
|
||||||
|
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
. "github.com/onsi/ginkgo"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This test will create a pod with a secret volume and gitRepo volume
|
||||||
|
// Thus requests a secret, a git server pod, and a git server service
|
||||||
|
var _ = Describe("EmptyDir wrapper volumes", func() {
|
||||||
|
f := NewFramework("secrets")
|
||||||
|
|
||||||
|
It("should becomes running [Conformance]", func() {
|
||||||
|
name := "secret-test-" + string(util.NewUUID())
|
||||||
|
volumeName := "secret-volume"
|
||||||
|
volumeMountPath := "/etc/secret-volume"
|
||||||
|
|
||||||
|
secret := &api.Secret{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Namespace: f.Namespace.Name,
|
||||||
|
Name: name,
|
||||||
|
},
|
||||||
|
Data: map[string][]byte{
|
||||||
|
"data-1": []byte("value-1\n"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
|
||||||
|
Failf("unable to create test secret %s: %v", secret.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
gitServerPodName := "git-server-" + string(util.NewUUID())
|
||||||
|
containerPort := 8000
|
||||||
|
|
||||||
|
labels := map[string]string{"name": gitServerPodName}
|
||||||
|
|
||||||
|
gitServerPod := &api.Pod{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Name: gitServerPodName,
|
||||||
|
Labels: labels,
|
||||||
|
},
|
||||||
|
Spec: api.PodSpec{
|
||||||
|
Containers: []api.Container{
|
||||||
|
{
|
||||||
|
Name: "git-repo",
|
||||||
|
Image: "gcr.io/google_containers/fakegitserver:0.1",
|
||||||
|
ImagePullPolicy: "IfNotPresent",
|
||||||
|
Ports: []api.ContainerPort{
|
||||||
|
{ContainerPort: containerPort},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if gitServerPod, err = f.Client.Pods(f.Namespace.Name).Create(gitServerPod); err != nil {
|
||||||
|
Failf("unable to create test git server pod %s: %v", gitServerPod.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Portal IP and port
|
||||||
|
httpPort := 2345
|
||||||
|
|
||||||
|
gitServerSvc := &api.Service{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Name: "git-server-svc",
|
||||||
|
},
|
||||||
|
Spec: api.ServiceSpec{
|
||||||
|
Selector: labels,
|
||||||
|
Ports: []api.ServicePort{
|
||||||
|
{
|
||||||
|
Name: "http-portal",
|
||||||
|
Port: httpPort,
|
||||||
|
TargetPort: intstr.FromInt(containerPort),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if gitServerSvc, err = f.Client.Services(f.Namespace.Name).Create(gitServerSvc); err != nil {
|
||||||
|
Failf("unable to create test git server service %s: %v", gitServerSvc.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
gitVolumeName := "git-volume"
|
||||||
|
gitVolumeMountPath := "/etc/git-volume"
|
||||||
|
gitURL := "http://" + gitServerSvc.Spec.ClusterIP + ":" + strconv.Itoa(httpPort)
|
||||||
|
gitRepo := "test"
|
||||||
|
|
||||||
|
pod := &api.Pod{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Name: "pod-secrets-" + string(util.NewUUID()),
|
||||||
|
},
|
||||||
|
Spec: api.PodSpec{
|
||||||
|
Volumes: []api.Volume{
|
||||||
|
{
|
||||||
|
Name: volumeName,
|
||||||
|
VolumeSource: api.VolumeSource{
|
||||||
|
Secret: &api.SecretVolumeSource{
|
||||||
|
SecretName: name,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: gitVolumeName,
|
||||||
|
VolumeSource: api.VolumeSource{
|
||||||
|
GitRepo: &api.GitRepoVolumeSource{
|
||||||
|
Repository: gitURL,
|
||||||
|
Directory: gitRepo,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Containers: []api.Container{
|
||||||
|
{
|
||||||
|
Name: "secret-test",
|
||||||
|
Image: "gcr.io/google_containers/test-webserver",
|
||||||
|
VolumeMounts: []api.VolumeMount{
|
||||||
|
{
|
||||||
|
Name: volumeName,
|
||||||
|
MountPath: volumeMountPath,
|
||||||
|
ReadOnly: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: gitVolumeName,
|
||||||
|
MountPath: gitVolumeMountPath,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if pod, err = f.Client.Pods(f.Namespace.Name).Create(pod); err != nil {
|
||||||
|
Failf("unable to create pod %v: %v", pod.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
By("Cleaning up the secret")
|
||||||
|
if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
|
||||||
|
Failf("unable to delete secret %v: %v", secret.Name, err)
|
||||||
|
}
|
||||||
|
By("Cleaning up the git server pod")
|
||||||
|
if err = f.Client.Pods(f.Namespace.Name).Delete(gitServerPod.Name, api.NewDeleteOptions(0)); err != nil {
|
||||||
|
Failf("unable to delete git server pod %v: %v", gitServerPod.Name, err)
|
||||||
|
}
|
||||||
|
By("Cleaning up the git server svc")
|
||||||
|
if err = f.Client.Services(f.Namespace.Name).Delete(gitServerSvc.Name); err != nil {
|
||||||
|
Failf("unable to delete git server svc %v: %v", gitServerSvc.Name, err)
|
||||||
|
}
|
||||||
|
By("Cleaning up the git vol pod")
|
||||||
|
if err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil {
|
||||||
|
Failf("unable to delete git vol pod %v: %v", pod.Name, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
expectNoError(waitForPodRunningInNamespace(f.Client, pod.Name, f.Namespace.Name))
|
||||||
|
})
|
||||||
|
})
|
Loading…
Reference in New Issue
Block a user