Merge pull request #49840 from andrewrynhard/variable_certs_dir
Automatic merge from submit-queue (batch tested with PRs 49840, 54937, 54543). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. kubeadm: Make it possible to configure volume mounts via the config file **What this PR does / why we need it**: Kubeadm mounts host CA certs into api server and controller manager. It uses `/etc/pki` and does not allow for the path to be configurable. This PR adds a default to `/etc/pki` but also allows a user to configure the path in the config file. In the case of using Container Linux, the CAs are located at `/usr/share/ca-certificates`, so without this PR the hardcoded `/etc/pki` path is used and will break, for example, the `--cloud-provider` flag because of missing CAs. Fixes https://github.com/kubernetes/kubeadm/issues/484 Fixes https://github.com/kubernetes/kubeadm/issues/476 Fixes https://github.com/kubernetes/kubeadm/issues/441 /cc @luxas
This commit is contained in:
@@ -42,6 +42,10 @@ type MasterConfiguration struct {
|
|||||||
ControllerManagerExtraArgs map[string]string
|
ControllerManagerExtraArgs map[string]string
|
||||||
SchedulerExtraArgs map[string]string
|
SchedulerExtraArgs map[string]string
|
||||||
|
|
||||||
|
APIServerExtraVolumes []HostPathMount
|
||||||
|
ControllerManagerExtraVolumes []HostPathMount
|
||||||
|
SchedulerExtraVolumes []HostPathMount
|
||||||
|
|
||||||
// APIServerCertSANs sets extra Subject Alternative Names for the API Server signing cert
|
// APIServerCertSANs sets extra Subject Alternative Names for the API Server signing cert
|
||||||
APIServerCertSANs []string
|
APIServerCertSANs []string
|
||||||
// CertificatesDir specifies where to store or look for all required certificates
|
// CertificatesDir specifies where to store or look for all required certificates
|
||||||
@@ -137,3 +141,11 @@ func (cfg *MasterConfiguration) GetControlPlaneImageRepository() string {
|
|||||||
}
|
}
|
||||||
return cfg.ImageRepository
|
return cfg.ImageRepository
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HostPathMount contains elements describing volumes that are mounted from the
|
||||||
|
// host
|
||||||
|
type HostPathMount struct {
|
||||||
|
Name string
|
||||||
|
HostPath string
|
||||||
|
MountPath string
|
||||||
|
}
|
||||||
|
@@ -42,6 +42,10 @@ type MasterConfiguration struct {
|
|||||||
ControllerManagerExtraArgs map[string]string `json:"controllerManagerExtraArgs,omitempty"`
|
ControllerManagerExtraArgs map[string]string `json:"controllerManagerExtraArgs,omitempty"`
|
||||||
SchedulerExtraArgs map[string]string `json:"schedulerExtraArgs,omitempty"`
|
SchedulerExtraArgs map[string]string `json:"schedulerExtraArgs,omitempty"`
|
||||||
|
|
||||||
|
APIServerExtraVolumes []HostPathMount `json:"apiServerExtraVolumes,omitempty"`
|
||||||
|
ControllerManagerExtraVolumes []HostPathMount `json:"controllerManagerExtraVolumes,omitempty"`
|
||||||
|
SchedulerExtraVolumes []HostPathMount `json:"schedulerExtraVolumes,omitempty"`
|
||||||
|
|
||||||
// APIServerCertSANs sets extra Subject Alternative Names for the API Server signing cert
|
// APIServerCertSANs sets extra Subject Alternative Names for the API Server signing cert
|
||||||
APIServerCertSANs []string `json:"apiServerCertSANs,omitempty"`
|
APIServerCertSANs []string `json:"apiServerCertSANs,omitempty"`
|
||||||
// CertificatesDir specifies where to store or look for all required certificates
|
// CertificatesDir specifies where to store or look for all required certificates
|
||||||
@@ -119,3 +123,11 @@ type NodeConfiguration struct {
|
|||||||
// the security of kubeadm since other nodes can impersonate the master.
|
// the security of kubeadm since other nodes can impersonate the master.
|
||||||
DiscoveryTokenUnsafeSkipCAVerification bool `json:"discoveryTokenUnsafeSkipCAVerification"`
|
DiscoveryTokenUnsafeSkipCAVerification bool `json:"discoveryTokenUnsafeSkipCAVerification"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HostPathMount contains elements describing volumes that are mounted from the
|
||||||
|
// host
|
||||||
|
type HostPathMount struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
HostPath string `json:"hostPath"`
|
||||||
|
MountPath string `json:"mountPath"`
|
||||||
|
}
|
||||||
|
@@ -39,6 +39,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
|
|||||||
Convert_kubeadm_API_To_v1alpha1_API,
|
Convert_kubeadm_API_To_v1alpha1_API,
|
||||||
Convert_v1alpha1_Etcd_To_kubeadm_Etcd,
|
Convert_v1alpha1_Etcd_To_kubeadm_Etcd,
|
||||||
Convert_kubeadm_Etcd_To_v1alpha1_Etcd,
|
Convert_kubeadm_Etcd_To_v1alpha1_Etcd,
|
||||||
|
Convert_v1alpha1_HostPathMount_To_kubeadm_HostPathMount,
|
||||||
|
Convert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount,
|
||||||
Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration,
|
Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration,
|
||||||
Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration,
|
Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration,
|
||||||
Convert_v1alpha1_Networking_To_kubeadm_Networking,
|
Convert_v1alpha1_Networking_To_kubeadm_Networking,
|
||||||
@@ -104,6 +106,30 @@ func Convert_kubeadm_Etcd_To_v1alpha1_Etcd(in *kubeadm.Etcd, out *Etcd, s conver
|
|||||||
return autoConvert_kubeadm_Etcd_To_v1alpha1_Etcd(in, out, s)
|
return autoConvert_kubeadm_Etcd_To_v1alpha1_Etcd(in, out, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func autoConvert_v1alpha1_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMount, out *kubeadm.HostPathMount, s conversion.Scope) error {
|
||||||
|
out.Name = in.Name
|
||||||
|
out.HostPath = in.HostPath
|
||||||
|
out.MountPath = in.MountPath
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert_v1alpha1_HostPathMount_To_kubeadm_HostPathMount is an autogenerated conversion function.
|
||||||
|
func Convert_v1alpha1_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMount, out *kubeadm.HostPathMount, s conversion.Scope) error {
|
||||||
|
return autoConvert_v1alpha1_HostPathMount_To_kubeadm_HostPathMount(in, out, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func autoConvert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error {
|
||||||
|
out.Name = in.Name
|
||||||
|
out.HostPath = in.HostPath
|
||||||
|
out.MountPath = in.MountPath
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount is an autogenerated conversion function.
|
||||||
|
func Convert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error {
|
||||||
|
return autoConvert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount(in, out, s)
|
||||||
|
}
|
||||||
|
|
||||||
func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in *MasterConfiguration, out *kubeadm.MasterConfiguration, s conversion.Scope) error {
|
func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in *MasterConfiguration, out *kubeadm.MasterConfiguration, s conversion.Scope) error {
|
||||||
if err := Convert_v1alpha1_API_To_kubeadm_API(&in.API, &out.API, s); err != nil {
|
if err := Convert_v1alpha1_API_To_kubeadm_API(&in.API, &out.API, s); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -123,6 +149,9 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in
|
|||||||
out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
|
out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
|
||||||
out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs))
|
out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs))
|
||||||
out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs))
|
out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs))
|
||||||
|
out.APIServerExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.APIServerExtraVolumes))
|
||||||
|
out.ControllerManagerExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.ControllerManagerExtraVolumes))
|
||||||
|
out.SchedulerExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.SchedulerExtraVolumes))
|
||||||
out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs))
|
out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs))
|
||||||
out.CertificatesDir = in.CertificatesDir
|
out.CertificatesDir = in.CertificatesDir
|
||||||
out.ImageRepository = in.ImageRepository
|
out.ImageRepository = in.ImageRepository
|
||||||
@@ -155,6 +184,9 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in
|
|||||||
out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
|
out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
|
||||||
out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs))
|
out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs))
|
||||||
out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs))
|
out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs))
|
||||||
|
out.APIServerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.APIServerExtraVolumes))
|
||||||
|
out.ControllerManagerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ControllerManagerExtraVolumes))
|
||||||
|
out.SchedulerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.SchedulerExtraVolumes))
|
||||||
out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs))
|
out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs))
|
||||||
out.CertificatesDir = in.CertificatesDir
|
out.CertificatesDir = in.CertificatesDir
|
||||||
out.ImageRepository = in.ImageRepository
|
out.ImageRepository = in.ImageRepository
|
||||||
|
@@ -39,6 +39,10 @@ func GetGeneratedDeepCopyFuncs() []conversion.GeneratedDeepCopyFunc {
|
|||||||
in.(*Etcd).DeepCopyInto(out.(*Etcd))
|
in.(*Etcd).DeepCopyInto(out.(*Etcd))
|
||||||
return nil
|
return nil
|
||||||
}, InType: reflect.TypeOf(&Etcd{})},
|
}, InType: reflect.TypeOf(&Etcd{})},
|
||||||
|
{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
|
||||||
|
in.(*HostPathMount).DeepCopyInto(out.(*HostPathMount))
|
||||||
|
return nil
|
||||||
|
}, InType: reflect.TypeOf(&HostPathMount{})},
|
||||||
{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
|
{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
|
||||||
in.(*MasterConfiguration).DeepCopyInto(out.(*MasterConfiguration))
|
in.(*MasterConfiguration).DeepCopyInto(out.(*MasterConfiguration))
|
||||||
return nil
|
return nil
|
||||||
@@ -102,6 +106,22 @@ func (in *Etcd) DeepCopy() *Etcd {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
func (in *HostPathMount) DeepCopyInto(out *HostPathMount) {
|
||||||
|
*out = *in
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathMount.
|
||||||
|
func (in *HostPathMount) DeepCopy() *HostPathMount {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := new(HostPathMount)
|
||||||
|
in.DeepCopyInto(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
|
func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
|
||||||
*out = *in
|
*out = *in
|
||||||
@@ -136,6 +156,21 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
|
|||||||
(*out)[key] = val
|
(*out)[key] = val
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if in.APIServerExtraVolumes != nil {
|
||||||
|
in, out := &in.APIServerExtraVolumes, &out.APIServerExtraVolumes
|
||||||
|
*out = make([]HostPathMount, len(*in))
|
||||||
|
copy(*out, *in)
|
||||||
|
}
|
||||||
|
if in.ControllerManagerExtraVolumes != nil {
|
||||||
|
in, out := &in.ControllerManagerExtraVolumes, &out.ControllerManagerExtraVolumes
|
||||||
|
*out = make([]HostPathMount, len(*in))
|
||||||
|
copy(*out, *in)
|
||||||
|
}
|
||||||
|
if in.SchedulerExtraVolumes != nil {
|
||||||
|
in, out := &in.SchedulerExtraVolumes, &out.SchedulerExtraVolumes
|
||||||
|
*out = make([]HostPathMount, len(*in))
|
||||||
|
copy(*out, *in)
|
||||||
|
}
|
||||||
if in.APIServerCertSANs != nil {
|
if in.APIServerCertSANs != nil {
|
||||||
in, out := &in.APIServerCertSANs, &out.APIServerCertSANs
|
in, out := &in.APIServerCertSANs, &out.APIServerCertSANs
|
||||||
*out = make([]string, len(*in))
|
*out = make([]string, len(*in))
|
||||||
|
@@ -44,6 +44,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
|
|||||||
in.(*Etcd).DeepCopyInto(out.(*Etcd))
|
in.(*Etcd).DeepCopyInto(out.(*Etcd))
|
||||||
return nil
|
return nil
|
||||||
}, InType: reflect.TypeOf(&Etcd{})},
|
}, InType: reflect.TypeOf(&Etcd{})},
|
||||||
|
conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
|
||||||
|
in.(*HostPathMount).DeepCopyInto(out.(*HostPathMount))
|
||||||
|
return nil
|
||||||
|
}, InType: reflect.TypeOf(&HostPathMount{})},
|
||||||
conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
|
conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
|
||||||
in.(*MasterConfiguration).DeepCopyInto(out.(*MasterConfiguration))
|
in.(*MasterConfiguration).DeepCopyInto(out.(*MasterConfiguration))
|
||||||
return nil
|
return nil
|
||||||
@@ -107,6 +111,22 @@ func (in *Etcd) DeepCopy() *Etcd {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
func (in *HostPathMount) DeepCopyInto(out *HostPathMount) {
|
||||||
|
*out = *in
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathMount.
|
||||||
|
func (in *HostPathMount) DeepCopy() *HostPathMount {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := new(HostPathMount)
|
||||||
|
in.DeepCopyInto(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
|
func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
|
||||||
*out = *in
|
*out = *in
|
||||||
@@ -141,6 +161,21 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
|
|||||||
(*out)[key] = val
|
(*out)[key] = val
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if in.APIServerExtraVolumes != nil {
|
||||||
|
in, out := &in.APIServerExtraVolumes, &out.APIServerExtraVolumes
|
||||||
|
*out = make([]HostPathMount, len(*in))
|
||||||
|
copy(*out, *in)
|
||||||
|
}
|
||||||
|
if in.ControllerManagerExtraVolumes != nil {
|
||||||
|
in, out := &in.ControllerManagerExtraVolumes, &out.ControllerManagerExtraVolumes
|
||||||
|
*out = make([]HostPathMount, len(*in))
|
||||||
|
copy(*out, *in)
|
||||||
|
}
|
||||||
|
if in.SchedulerExtraVolumes != nil {
|
||||||
|
in, out := &in.SchedulerExtraVolumes, &out.SchedulerExtraVolumes
|
||||||
|
*out = make([]HostPathMount, len(*in))
|
||||||
|
copy(*out, *in)
|
||||||
|
}
|
||||||
if in.APIServerCertSANs != nil {
|
if in.APIServerCertSANs != nil {
|
||||||
in, out := &in.APIServerCertSANs, &out.APIServerCertSANs
|
in, out := &in.APIServerCertSANs, &out.APIServerCertSANs
|
||||||
*out = make([]string, len(*in))
|
*out = make([]string, len(*in))
|
||||||
|
@@ -77,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
|
|||||||
Name: kubeadmconstants.KubeAPIServer,
|
Name: kubeadmconstants.KubeAPIServer,
|
||||||
Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
|
Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
|
||||||
Command: getAPIServerCommand(cfg, k8sVersion),
|
Command: getAPIServerCommand(cfg, k8sVersion),
|
||||||
VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer),
|
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)),
|
||||||
LivenessProbe: staticpodutil.ComponentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
|
LivenessProbe: staticpodutil.ComponentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
|
||||||
Resources: staticpodutil.ComponentResources("250m"),
|
Resources: staticpodutil.ComponentResources("250m"),
|
||||||
Env: getProxyEnvVars(),
|
Env: getProxyEnvVars(),
|
||||||
@@ -86,7 +86,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
|
|||||||
Name: kubeadmconstants.KubeControllerManager,
|
Name: kubeadmconstants.KubeControllerManager,
|
||||||
Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
|
Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
|
||||||
Command: getControllerManagerCommand(cfg, k8sVersion),
|
Command: getControllerManagerCommand(cfg, k8sVersion),
|
||||||
VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager),
|
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
|
||||||
LivenessProbe: staticpodutil.ComponentProbe(10252, "/healthz", v1.URISchemeHTTP),
|
LivenessProbe: staticpodutil.ComponentProbe(10252, "/healthz", v1.URISchemeHTTP),
|
||||||
Resources: staticpodutil.ComponentResources("200m"),
|
Resources: staticpodutil.ComponentResources("200m"),
|
||||||
Env: getProxyEnvVars(),
|
Env: getProxyEnvVars(),
|
||||||
@@ -95,7 +95,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
|
|||||||
Name: kubeadmconstants.KubeScheduler,
|
Name: kubeadmconstants.KubeScheduler,
|
||||||
Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
|
Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
|
||||||
Command: getSchedulerCommand(cfg),
|
Command: getSchedulerCommand(cfg),
|
||||||
VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler),
|
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
|
||||||
LivenessProbe: staticpodutil.ComponentProbe(10251, "/healthz", v1.URISchemeHTTP),
|
LivenessProbe: staticpodutil.ComponentProbe(10251, "/healthz", v1.URISchemeHTTP),
|
||||||
Resources: staticpodutil.ComponentResources("100m"),
|
Resources: staticpodutil.ComponentResources("100m"),
|
||||||
Env: getProxyEnvVars(),
|
Env: getProxyEnvVars(),
|
||||||
|
@@ -86,40 +86,85 @@ func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) c
|
|||||||
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true, &hostPathDirectoryOrCreate)
|
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true, &hostPathDirectoryOrCreate)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Merge user defined mounts and ensure unique volume and volume mount
|
||||||
|
// names
|
||||||
|
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeAPIServer, cfg.APIServerExtraVolumes, true, &hostPathDirectoryOrCreate)
|
||||||
|
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeControllerManager, cfg.ControllerManagerExtraVolumes, true, &hostPathDirectoryOrCreate)
|
||||||
|
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeScheduler, cfg.SchedulerExtraVolumes, true, &hostPathDirectoryOrCreate)
|
||||||
|
|
||||||
return mounts
|
return mounts
|
||||||
}
|
}
|
||||||
|
|
||||||
// controlPlaneHostPathMounts is a helper struct for handling all the control plane's hostPath mounts in an easy way
|
// controlPlaneHostPathMounts is a helper struct for handling all the control plane's hostPath mounts in an easy way
|
||||||
type controlPlaneHostPathMounts struct {
|
type controlPlaneHostPathMounts struct {
|
||||||
volumes map[string][]v1.Volume
|
// volumes is a nested map that forces a unique volumes. The outer map's
|
||||||
volumeMounts map[string][]v1.VolumeMount
|
// keys are a string that should specify the target component to add the
|
||||||
|
// volume to. The values (inner map) of the outer map are maps with string
|
||||||
|
// keys and v1.Volume values. The inner map's key should specify the volume
|
||||||
|
// name.
|
||||||
|
volumes map[string]map[string]v1.Volume
|
||||||
|
// volumeMounts is a nested map that forces a unique volume mounts. The
|
||||||
|
// outer map's keys are a string that should specify the target component
|
||||||
|
// to add the volume mount to. The values (inner map) of the outer map are
|
||||||
|
// maps with string keys and v1.VolumeMount values. The inner map's key
|
||||||
|
// should specify the volume mount name.
|
||||||
|
volumeMounts map[string]map[string]v1.VolumeMount
|
||||||
}
|
}
|
||||||
|
|
||||||
func newControlPlaneHostPathMounts() controlPlaneHostPathMounts {
|
func newControlPlaneHostPathMounts() controlPlaneHostPathMounts {
|
||||||
return controlPlaneHostPathMounts{
|
return controlPlaneHostPathMounts{
|
||||||
volumes: map[string][]v1.Volume{},
|
volumes: map[string]map[string]v1.Volume{},
|
||||||
volumeMounts: map[string][]v1.VolumeMount{},
|
volumeMounts: map[string]map[string]v1.VolumeMount{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool, hostPathType *v1.HostPathType) {
|
func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool, hostPathType *v1.HostPathType) {
|
||||||
c.volumes[component] = append(c.volumes[component], staticpodutil.NewVolume(mountName, hostPath, hostPathType))
|
vol := staticpodutil.NewVolume(mountName, hostPath, hostPathType)
|
||||||
c.volumeMounts[component] = append(c.volumeMounts[component], staticpodutil.NewVolumeMount(mountName, containerPath, readOnly))
|
c.addComponentVolume(component, vol)
|
||||||
|
volMount := staticpodutil.NewVolumeMount(mountName, containerPath, readOnly)
|
||||||
|
c.addComponentVolumeMount(component, volMount)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols []v1.Volume, volMounts []v1.VolumeMount) {
|
func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols []v1.Volume, volMounts []v1.VolumeMount) {
|
||||||
c.volumes[component] = append(c.volumes[component], vols...)
|
for _, v := range vols {
|
||||||
c.volumeMounts[component] = append(c.volumeMounts[component], volMounts...)
|
c.addComponentVolume(component, v)
|
||||||
|
}
|
||||||
|
for _, v := range volMounts {
|
||||||
|
c.addComponentVolumeMount(component, v)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *controlPlaneHostPathMounts) GetVolumes(component string) []v1.Volume {
|
// AddExtraHostPathMounts adds host path mounts and overwrites the default
|
||||||
|
// paths in the case that a user specifies the same volume/volume mount name.
|
||||||
|
func (c *controlPlaneHostPathMounts) AddExtraHostPathMounts(component string, extraVols []kubeadmapi.HostPathMount, readOnly bool, hostPathType *v1.HostPathType) {
|
||||||
|
for _, extraVol := range extraVols {
|
||||||
|
fmt.Printf("[controlplane] Adding extra host path mount %q to %q\n", extraVol.Name, component)
|
||||||
|
c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, readOnly, hostPathType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *controlPlaneHostPathMounts) GetVolumes(component string) map[string]v1.Volume {
|
||||||
return c.volumes[component]
|
return c.volumes[component]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *controlPlaneHostPathMounts) GetVolumeMounts(component string) []v1.VolumeMount {
|
func (c *controlPlaneHostPathMounts) GetVolumeMounts(component string) map[string]v1.VolumeMount {
|
||||||
return c.volumeMounts[component]
|
return c.volumeMounts[component]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *controlPlaneHostPathMounts) addComponentVolume(component string, vol v1.Volume) {
|
||||||
|
if _, ok := c.volumes[component]; !ok {
|
||||||
|
c.volumes[component] = map[string]v1.Volume{}
|
||||||
|
}
|
||||||
|
c.volumes[component][vol.Name] = vol
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *controlPlaneHostPathMounts) addComponentVolumeMount(component string, volMount v1.VolumeMount) {
|
||||||
|
if _, ok := c.volumeMounts[component]; !ok {
|
||||||
|
c.volumeMounts[component] = map[string]v1.VolumeMount{}
|
||||||
|
}
|
||||||
|
c.volumeMounts[component][volMount.Name] = volMount
|
||||||
|
}
|
||||||
|
|
||||||
// getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster
|
// getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster
|
||||||
func getEtcdCertVolumes(etcdCfg kubeadmapi.Etcd) ([]v1.Volume, []v1.VolumeMount) {
|
func getEtcdCertVolumes(etcdCfg kubeadmapi.Etcd) ([]v1.Volume, []v1.VolumeMount) {
|
||||||
certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile}
|
certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile}
|
||||||
|
@@ -249,10 +249,251 @@ func TestGetEtcdCertVolumes(t *testing.T) {
|
|||||||
func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
|
func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
|
||||||
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
|
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
|
||||||
hostPathFileOrCreate := v1.HostPathFileOrCreate
|
hostPathFileOrCreate := v1.HostPathFileOrCreate
|
||||||
|
volMap := make(map[string]map[string]v1.Volume)
|
||||||
|
volMap[kubeadmconstants.KubeAPIServer] = map[string]v1.Volume{}
|
||||||
|
volMap[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.Volume{
|
||||||
|
Name: "k8s-certs",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: testCertsDir,
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.Volume{
|
||||||
|
Name: "ca-certs",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/ssl/certs",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap[kubeadmconstants.KubeControllerManager] = map[string]v1.Volume{}
|
||||||
|
volMap[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.Volume{
|
||||||
|
Name: "k8s-certs",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: testCertsDir,
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.Volume{
|
||||||
|
Name: "ca-certs",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/ssl/certs",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.Volume{
|
||||||
|
Name: "kubeconfig",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/kubernetes/controller-manager.conf",
|
||||||
|
Type: &hostPathFileOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap[kubeadmconstants.KubeControllerManager]["flexvolume-dir"] = v1.Volume{
|
||||||
|
Name: "flexvolume-dir",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap[kubeadmconstants.KubeScheduler] = map[string]v1.Volume{}
|
||||||
|
volMap[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.Volume{
|
||||||
|
Name: "kubeconfig",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/kubernetes/scheduler.conf",
|
||||||
|
Type: &hostPathFileOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMountMap := make(map[string]map[string]v1.VolumeMount)
|
||||||
|
volMountMap[kubeadmconstants.KubeAPIServer] = map[string]v1.VolumeMount{}
|
||||||
|
volMountMap[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.VolumeMount{
|
||||||
|
Name: "k8s-certs",
|
||||||
|
MountPath: testCertsDir,
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.VolumeMount{
|
||||||
|
Name: "ca-certs",
|
||||||
|
MountPath: "/etc/ssl/certs",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap[kubeadmconstants.KubeControllerManager] = map[string]v1.VolumeMount{}
|
||||||
|
volMountMap[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.VolumeMount{
|
||||||
|
Name: "k8s-certs",
|
||||||
|
MountPath: testCertsDir,
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.VolumeMount{
|
||||||
|
Name: "ca-certs",
|
||||||
|
MountPath: "/etc/ssl/certs",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.VolumeMount{
|
||||||
|
Name: "kubeconfig",
|
||||||
|
MountPath: "/etc/kubernetes/controller-manager.conf",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap[kubeadmconstants.KubeControllerManager]["flexvolume-dir"] = v1.VolumeMount{
|
||||||
|
Name: "flexvolume-dir",
|
||||||
|
MountPath: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec",
|
||||||
|
ReadOnly: false,
|
||||||
|
}
|
||||||
|
volMountMap[kubeadmconstants.KubeScheduler] = map[string]v1.VolumeMount{}
|
||||||
|
volMountMap[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.VolumeMount{
|
||||||
|
Name: "kubeconfig",
|
||||||
|
MountPath: "/etc/kubernetes/scheduler.conf",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
volMap2 := make(map[string]map[string]v1.Volume)
|
||||||
|
volMap2[kubeadmconstants.KubeAPIServer] = map[string]v1.Volume{}
|
||||||
|
volMap2[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.Volume{
|
||||||
|
Name: "k8s-certs",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: testCertsDir,
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap2[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.Volume{
|
||||||
|
Name: "ca-certs",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/ssl/certs",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-0"] = v1.Volume{
|
||||||
|
Name: "etcd-certs-0",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/certs/etcd",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-1"] = v1.Volume{
|
||||||
|
Name: "etcd-certs-1",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/var/lib/certs/etcd",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap2[kubeadmconstants.KubeControllerManager] = map[string]v1.Volume{}
|
||||||
|
volMap2[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.Volume{
|
||||||
|
Name: "k8s-certs",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: testCertsDir,
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap2[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.Volume{
|
||||||
|
Name: "ca-certs",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/ssl/certs",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap2[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.Volume{
|
||||||
|
Name: "kubeconfig",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/kubernetes/controller-manager.conf",
|
||||||
|
Type: &hostPathFileOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap2[kubeadmconstants.KubeControllerManager]["flexvolume-dir"] = v1.Volume{
|
||||||
|
Name: "flexvolume-dir",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMap2[kubeadmconstants.KubeScheduler] = map[string]v1.Volume{}
|
||||||
|
volMap2[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.Volume{
|
||||||
|
Name: "kubeconfig",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/etc/kubernetes/scheduler.conf",
|
||||||
|
Type: &hostPathFileOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMountMap2 := make(map[string]map[string]v1.VolumeMount)
|
||||||
|
volMountMap2[kubeadmconstants.KubeAPIServer] = map[string]v1.VolumeMount{}
|
||||||
|
volMountMap2[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.VolumeMount{
|
||||||
|
Name: "k8s-certs",
|
||||||
|
MountPath: testCertsDir,
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap2[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.VolumeMount{
|
||||||
|
Name: "ca-certs",
|
||||||
|
MountPath: "/etc/ssl/certs",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-0"] = v1.VolumeMount{
|
||||||
|
Name: "etcd-certs-0",
|
||||||
|
MountPath: "/etc/certs/etcd",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-1"] = v1.VolumeMount{
|
||||||
|
Name: "etcd-certs-1",
|
||||||
|
MountPath: "/var/lib/certs/etcd",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap2[kubeadmconstants.KubeControllerManager] = map[string]v1.VolumeMount{}
|
||||||
|
volMountMap2[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.VolumeMount{
|
||||||
|
Name: "k8s-certs",
|
||||||
|
MountPath: testCertsDir,
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap2[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.VolumeMount{
|
||||||
|
Name: "ca-certs",
|
||||||
|
MountPath: "/etc/ssl/certs",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap2[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.VolumeMount{
|
||||||
|
Name: "kubeconfig",
|
||||||
|
MountPath: "/etc/kubernetes/controller-manager.conf",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
|
volMountMap2[kubeadmconstants.KubeControllerManager]["flexvolume-dir"] = v1.VolumeMount{
|
||||||
|
Name: "flexvolume-dir",
|
||||||
|
MountPath: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec",
|
||||||
|
ReadOnly: false,
|
||||||
|
}
|
||||||
|
volMountMap2[kubeadmconstants.KubeScheduler] = map[string]v1.VolumeMount{}
|
||||||
|
volMountMap2[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.VolumeMount{
|
||||||
|
Name: "kubeconfig",
|
||||||
|
MountPath: "/etc/kubernetes/scheduler.conf",
|
||||||
|
ReadOnly: true,
|
||||||
|
}
|
||||||
var tests = []struct {
|
var tests = []struct {
|
||||||
cfg *kubeadmapi.MasterConfiguration
|
cfg *kubeadmapi.MasterConfiguration
|
||||||
vol map[string][]v1.Volume
|
vol map[string]map[string]v1.Volume
|
||||||
volMount map[string][]v1.VolumeMount
|
volMount map[string]map[string]v1.VolumeMount
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
// Should ignore files in /etc/ssl/certs
|
// Should ignore files in /etc/ssl/certs
|
||||||
@@ -260,120 +501,8 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
|
|||||||
CertificatesDir: testCertsDir,
|
CertificatesDir: testCertsDir,
|
||||||
Etcd: kubeadmapi.Etcd{},
|
Etcd: kubeadmapi.Etcd{},
|
||||||
},
|
},
|
||||||
vol: map[string][]v1.Volume{
|
vol: volMap,
|
||||||
kubeadmconstants.KubeAPIServer: {
|
volMount: volMountMap,
|
||||||
{
|
|
||||||
Name: "k8s-certs",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: testCertsDir,
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "ca-certs",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/ssl/certs",
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
kubeadmconstants.KubeControllerManager: {
|
|
||||||
{
|
|
||||||
Name: "k8s-certs",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: testCertsDir,
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "ca-certs",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/ssl/certs",
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "kubeconfig",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/kubernetes/controller-manager.conf",
|
|
||||||
Type: &hostPathFileOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "flexvolume-dir",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec",
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
kubeadmconstants.KubeScheduler: {
|
|
||||||
{
|
|
||||||
Name: "kubeconfig",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/kubernetes/scheduler.conf",
|
|
||||||
Type: &hostPathFileOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
volMount: map[string][]v1.VolumeMount{
|
|
||||||
kubeadmconstants.KubeAPIServer: {
|
|
||||||
{
|
|
||||||
Name: "k8s-certs",
|
|
||||||
MountPath: testCertsDir,
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "ca-certs",
|
|
||||||
MountPath: "/etc/ssl/certs",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
kubeadmconstants.KubeControllerManager: {
|
|
||||||
{
|
|
||||||
Name: "k8s-certs",
|
|
||||||
MountPath: testCertsDir,
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "ca-certs",
|
|
||||||
MountPath: "/etc/ssl/certs",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "kubeconfig",
|
|
||||||
MountPath: "/etc/kubernetes/controller-manager.conf",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "flexvolume-dir",
|
|
||||||
MountPath: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec",
|
|
||||||
ReadOnly: false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
kubeadmconstants.KubeScheduler: {
|
|
||||||
{
|
|
||||||
Name: "kubeconfig",
|
|
||||||
MountPath: "/etc/kubernetes/scheduler.conf",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// Should ignore files in /etc/ssl/certs
|
// Should ignore files in /etc/ssl/certs
|
||||||
@@ -386,148 +515,8 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
|
|||||||
KeyFile: "/var/lib/certs/etcd/my-etcd.key",
|
KeyFile: "/var/lib/certs/etcd/my-etcd.key",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
vol: map[string][]v1.Volume{
|
vol: volMap2,
|
||||||
kubeadmconstants.KubeAPIServer: {
|
volMount: volMountMap2,
|
||||||
{
|
|
||||||
Name: "k8s-certs",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: testCertsDir,
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "ca-certs",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/ssl/certs",
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "etcd-certs-0",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/certs/etcd",
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "etcd-certs-1",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/var/lib/certs/etcd",
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
kubeadmconstants.KubeControllerManager: {
|
|
||||||
{
|
|
||||||
Name: "k8s-certs",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: testCertsDir,
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "ca-certs",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/ssl/certs",
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "kubeconfig",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/kubernetes/controller-manager.conf",
|
|
||||||
Type: &hostPathFileOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "flexvolume-dir",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec",
|
|
||||||
Type: &hostPathDirectoryOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
kubeadmconstants.KubeScheduler: {
|
|
||||||
{
|
|
||||||
Name: "kubeconfig",
|
|
||||||
VolumeSource: v1.VolumeSource{
|
|
||||||
HostPath: &v1.HostPathVolumeSource{
|
|
||||||
Path: "/etc/kubernetes/scheduler.conf",
|
|
||||||
Type: &hostPathFileOrCreate,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
volMount: map[string][]v1.VolumeMount{
|
|
||||||
kubeadmconstants.KubeAPIServer: {
|
|
||||||
{
|
|
||||||
Name: "k8s-certs",
|
|
||||||
MountPath: testCertsDir,
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "ca-certs",
|
|
||||||
MountPath: "/etc/ssl/certs",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "etcd-certs-0",
|
|
||||||
MountPath: "/etc/certs/etcd",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "etcd-certs-1",
|
|
||||||
MountPath: "/var/lib/certs/etcd",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
kubeadmconstants.KubeControllerManager: {
|
|
||||||
{
|
|
||||||
Name: "k8s-certs",
|
|
||||||
MountPath: testCertsDir,
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "ca-certs",
|
|
||||||
MountPath: "/etc/ssl/certs",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "kubeconfig",
|
|
||||||
MountPath: "/etc/kubernetes/controller-manager.conf",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "flexvolume-dir",
|
|
||||||
MountPath: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec",
|
|
||||||
ReadOnly: false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
kubeadmconstants.KubeScheduler: {
|
|
||||||
{
|
|
||||||
Name: "kubeconfig",
|
|
||||||
MountPath: "/etc/kubernetes/scheduler.conf",
|
|
||||||
ReadOnly: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -559,3 +548,70 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAddExtraHostPathMounts(t *testing.T) {
|
||||||
|
mounts := newControlPlaneHostPathMounts()
|
||||||
|
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
|
||||||
|
hostPathFileOrCreate := v1.HostPathFileOrCreate
|
||||||
|
vols := []v1.Volume{
|
||||||
|
{
|
||||||
|
Name: "foo",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/tmp/foo",
|
||||||
|
Type: &hostPathDirectoryOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "bar",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: "/tmp/bar",
|
||||||
|
Type: &hostPathFileOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volMounts := []v1.VolumeMount{
|
||||||
|
{
|
||||||
|
Name: "foo",
|
||||||
|
MountPath: "/tmp/foo",
|
||||||
|
ReadOnly: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "bar",
|
||||||
|
MountPath: "/tmp/bar",
|
||||||
|
ReadOnly: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
mounts.AddHostPathMounts("component", vols, volMounts)
|
||||||
|
hostPathMounts := []kubeadmapi.HostPathMount{
|
||||||
|
{
|
||||||
|
Name: "foo",
|
||||||
|
HostPath: "/tmp/qux",
|
||||||
|
MountPath: "/tmp/qux",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
mounts.AddExtraHostPathMounts("component", hostPathMounts, true, &hostPathDirectoryOrCreate)
|
||||||
|
if _, ok := mounts.volumes["component"]["foo"]; !ok {
|
||||||
|
t.Errorf("Expected to find volume %q", "foo")
|
||||||
|
}
|
||||||
|
vol, _ := mounts.volumes["component"]["foo"]
|
||||||
|
if vol.Name != "foo" {
|
||||||
|
t.Errorf("Expected volume name %q", "foo")
|
||||||
|
}
|
||||||
|
if vol.HostPath.Path != "/tmp/qux" {
|
||||||
|
t.Errorf("Expected host path %q", "/tmp/qux")
|
||||||
|
}
|
||||||
|
if _, ok := mounts.volumeMounts["component"]["foo"]; !ok {
|
||||||
|
t.Errorf("Expected to find volume mount %q", "foo")
|
||||||
|
}
|
||||||
|
volMount, _ := mounts.volumeMounts["component"]["foo"]
|
||||||
|
if volMount.Name != "foo" {
|
||||||
|
t.Errorf("Expected volume mount name %q", "foo")
|
||||||
|
}
|
||||||
|
if volMount.MountPath != "/tmp/qux" {
|
||||||
|
t.Errorf("Expected container path %q", "/tmp/qux")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -50,6 +50,9 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma
|
|||||||
// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod mainfests.
|
// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod mainfests.
|
||||||
func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
|
func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
|
||||||
pathType := v1.HostPathDirectoryOrCreate
|
pathType := v1.HostPathDirectoryOrCreate
|
||||||
|
etcdMounts := map[string]v1.Volume{
|
||||||
|
etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType),
|
||||||
|
}
|
||||||
return staticpodutil.ComponentPod(v1.Container{
|
return staticpodutil.ComponentPod(v1.Container{
|
||||||
Name: kubeadmconstants.Etcd,
|
Name: kubeadmconstants.Etcd,
|
||||||
Command: getEtcdCommand(cfg),
|
Command: getEtcdCommand(cfg),
|
||||||
@@ -57,7 +60,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
|
|||||||
// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
|
// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
|
||||||
VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)},
|
VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)},
|
||||||
LivenessProbe: staticpodutil.ComponentProbe(2379, "/health", v1.URISchemeHTTP),
|
LivenessProbe: staticpodutil.ComponentProbe(2379, "/health", v1.URISchemeHTTP),
|
||||||
}, []v1.Volume{staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType)})
|
}, etcdMounts)
|
||||||
}
|
}
|
||||||
|
|
||||||
// getEtcdCommand builds the right etcd command from the given config object
|
// getEtcdCommand builds the right etcd command from the given config object
|
||||||
|
@@ -31,7 +31,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// ComponentPod returns a Pod object from the container and volume specifications
|
// ComponentPod returns a Pod object from the container and volume specifications
|
||||||
func ComponentPod(container v1.Container, volumes []v1.Volume) v1.Pod {
|
func ComponentPod(container v1.Container, volumes map[string]v1.Volume) v1.Pod {
|
||||||
return v1.Pod{
|
return v1.Pod{
|
||||||
TypeMeta: metav1.TypeMeta{
|
TypeMeta: metav1.TypeMeta{
|
||||||
APIVersion: "v1",
|
APIVersion: "v1",
|
||||||
@@ -48,7 +48,7 @@ func ComponentPod(container v1.Container, volumes []v1.Volume) v1.Pod {
|
|||||||
Spec: v1.PodSpec{
|
Spec: v1.PodSpec{
|
||||||
Containers: []v1.Container{container},
|
Containers: []v1.Container{container},
|
||||||
HostNetwork: true,
|
HostNetwork: true,
|
||||||
Volumes: volumes,
|
Volumes: VolumeMapToSlice(volumes),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -102,6 +102,28 @@ func NewVolumeMount(name, path string, readOnly bool) v1.VolumeMount {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// VolumeMapToSlice returns a slice of volumes from a map's values
|
||||||
|
func VolumeMapToSlice(volumes map[string]v1.Volume) []v1.Volume {
|
||||||
|
v := make([]v1.Volume, 0, len(volumes))
|
||||||
|
|
||||||
|
for _, vol := range volumes {
|
||||||
|
v = append(v, vol)
|
||||||
|
}
|
||||||
|
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// VolumeMountMapToSlice returns a slice of volumes from a map's values
|
||||||
|
func VolumeMountMapToSlice(volumeMounts map[string]v1.VolumeMount) []v1.VolumeMount {
|
||||||
|
v := make([]v1.VolumeMount, 0, len(volumeMounts))
|
||||||
|
|
||||||
|
for _, volMount := range volumeMounts {
|
||||||
|
v = append(v, volMount)
|
||||||
|
}
|
||||||
|
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
// GetExtraParameters builds a list of flag arguments two string-string maps, one with default, base commands and one with overrides
|
// GetExtraParameters builds a list of flag arguments two string-string maps, one with default, base commands and one with overrides
|
||||||
func GetExtraParameters(overrides map[string]string, defaults map[string]string) []string {
|
func GetExtraParameters(overrides map[string]string, defaults map[string]string) []string {
|
||||||
var command []string
|
var command []string
|
||||||
|
@@ -111,7 +111,7 @@ func TestComponentPod(t *testing.T) {
|
|||||||
|
|
||||||
for _, rt := range tests {
|
for _, rt := range tests {
|
||||||
c := v1.Container{Name: rt.name}
|
c := v1.Container{Name: rt.name}
|
||||||
actual := ComponentPod(c, []v1.Volume{})
|
actual := ComponentPod(c, map[string]v1.Volume{})
|
||||||
if !reflect.DeepEqual(rt.expected, actual) {
|
if !reflect.DeepEqual(rt.expected, actual) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"failed componentPod:\n\texpected: %v\n\t actual: %v",
|
"failed componentPod:\n\texpected: %v\n\t actual: %v",
|
||||||
@@ -198,6 +198,35 @@ func TestNewVolumeMount(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
func TestVolumeMapToSlice(t *testing.T) {
|
||||||
|
testVolumes := map[string]v1.Volume{
|
||||||
|
"foo": {
|
||||||
|
Name: "foo",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volumeSlice := VolumeMapToSlice(testVolumes)
|
||||||
|
if len(volumeSlice) != 1 {
|
||||||
|
t.Errorf("Expected slice length of 1, got %d", len(volumeSlice))
|
||||||
|
}
|
||||||
|
if volumeSlice[0].Name != "foo" {
|
||||||
|
t.Errorf("Expected volume name \"foo\", got %s", volumeSlice[0].Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVolumeMountMapToSlice(t *testing.T) {
|
||||||
|
testVolumeMounts := map[string]v1.VolumeMount{
|
||||||
|
"foo": {
|
||||||
|
Name: "foo",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
volumeMountSlice := VolumeMountMapToSlice(testVolumeMounts)
|
||||||
|
if len(volumeMountSlice) != 1 {
|
||||||
|
t.Errorf("Expected slice length of 1, got %d", len(volumeMountSlice))
|
||||||
|
}
|
||||||
|
if volumeMountSlice[0].Name != "foo" {
|
||||||
|
t.Errorf("Expected volume mount name \"foo\", got %s", volumeMountSlice[0].Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGetExtraParameters(t *testing.T) {
|
func TestGetExtraParameters(t *testing.T) {
|
||||||
var tests = []struct {
|
var tests = []struct {
|
||||||
|
Reference in New Issue
Block a user