diff --git a/docs/config.md b/docs/config.md index 402967001..4d2a3df8f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -135,6 +135,13 @@ version = 2 # i.e pass host devices through to privileged containers. privileged_without_host_devices = false + # base_runtime_spec is a file path to a JSON file with the OCI spec that will be used as the base spec that all + # container's are created from. + # Use containerd's `ctr oci spec > /etc/containerd/cri-base.json` to output initial spec file. + # Spec files are loaded at launch, so containerd daemon must be restared on any changes to refresh default specs. + # Still running containers and restarted containers will still be using the original spec from which that container was created. + base_runtime_spec = "" + # 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options' is options specific to # "io.containerd.runc.v1" and "io.containerd.runc.v2". Its corresponding options type is: # https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26 . diff --git a/pkg/config/config.go b/pkg/config/config.go index d2e10f192..3e3e59105 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -52,6 +52,8 @@ type Runtime struct { // PrivilegedWithoutHostDevices overloads the default behaviour for adding host devices to the // runtime spec when the container is privileged. Defaults to false. PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"` + // BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from. + BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"` } // ContainerdConfig contains toml config related to containerd diff --git a/pkg/server/container_create.go b/pkg/server/container_create.go index 833501de0..cbed9e236 100644 --- a/pkg/server/container_create.go +++ b/pkg/server/container_create.go @@ -297,12 +297,36 @@ func (c *criService) volumeMounts(containerRootDir string, criMounts []*runtime. } // runtimeSpec returns a default runtime spec used in cri-containerd. -func runtimeSpec(id string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) { +func (c *criService) runtimeSpec(id string, baseSpecFile string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) { // GenerateSpec needs namespace. ctx := ctrdutil.NamespacedContext() - spec, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: id}, opts...) - if err != nil { - return nil, err + container := &containers.Container{ID: id} + + if baseSpecFile != "" { + baseSpec, ok := c.baseOCISpecs[baseSpecFile] + if !ok { + return nil, errors.Errorf("can't find base OCI spec %q", baseSpecFile) + } + + spec := oci.Spec{} + if err := util.DeepCopy(&spec, &baseSpec); err != nil { + return nil, errors.Wrap(err, "failed to clone OCI spec") + } + + // Fix up cgroups path + applyOpts := append([]oci.SpecOpts{oci.WithNamespacedCgroup()}, opts...) + + if err := oci.ApplyOpts(ctx, nil, container, &spec, applyOpts...); err != nil { + return nil, errors.Wrap(err, "failed to apply OCI options") + } + + return &spec, nil } + + spec, err := oci.GenerateSpec(ctx, nil, container, opts...) + if err != nil { + return nil, errors.Wrap(err, "failed to generate spec") + } + return spec, nil } diff --git a/pkg/server/container_create_test.go b/pkg/server/container_create_test.go index 6c7219e66..b1c6cc8b3 100644 --- a/pkg/server/container_create_test.go +++ b/pkg/server/container_create_test.go @@ -21,6 +21,7 @@ import ( "path/filepath" "testing" + "github.com/containerd/containerd/oci" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtimespec "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" @@ -28,6 +29,7 @@ import ( runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "github.com/containerd/cri/pkg/config" + "github.com/containerd/cri/pkg/constants" "github.com/containerd/cri/pkg/containerd/opts" ) @@ -381,3 +383,25 @@ func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) { }) } } + +func TestBaseRuntimeSpec(t *testing.T) { + c := newTestCRIService() + c.baseOCISpecs = map[string]*oci.Spec{ + "/etc/containerd/cri-base.json": { + Version: "1.0.2", + Hostname: "old", + }, + } + + out, err := c.runtimeSpec("id1", "/etc/containerd/cri-base.json", oci.WithHostname("new")) + assert.NoError(t, err) + + assert.Equal(t, "1.0.2", out.Version) + assert.Equal(t, "new", out.Hostname) + + // Make sure original base spec not changed + assert.NotEqual(t, out, c.baseOCISpecs["/etc/containerd/cri-base.json"]) + assert.Equal(t, c.baseOCISpecs["/etc/containerd/cri-base.json"].Hostname, "old") + + assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath) +} diff --git a/pkg/server/container_create_unix.go b/pkg/server/container_create_unix.go index d5f0bc955..d0ff20e3f 100644 --- a/pkg/server/container_create_unix.go +++ b/pkg/server/container_create_unix.go @@ -262,7 +262,7 @@ func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint3 Type: runtimespec.CgroupNamespace, })) } - return runtimeSpec(id, specOpts...) + return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...) } func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) { diff --git a/pkg/server/container_create_unix_test.go b/pkg/server/container_create_unix_test.go index 0967b81b1..479c0849b 100644 --- a/pkg/server/container_create_unix_test.go +++ b/pkg/server/container_create_unix_test.go @@ -1203,3 +1203,45 @@ func TestPrivilegedDevices(t *testing.T) { } } } + +func TestBaseOCISpec(t *testing.T) { + c := newTestCRIService() + baseLimit := int64(100) + c.baseOCISpecs = map[string]*oci.Spec{ + "/etc/containerd/cri-base.json": { + Process: &runtimespec.Process{ + User: runtimespec.User{AdditionalGids: []uint32{9999}}, + Capabilities: &runtimespec.LinuxCapabilities{ + Permitted: []string{"CAP_SETUID"}, + }, + }, + Linux: &runtimespec.Linux{ + Resources: &runtimespec.LinuxResources{ + Memory: &runtimespec.LinuxMemory{Limit: &baseLimit}, // Will be overwritten by `getCreateContainerTestData` + }, + }, + }, + } + + ociRuntime := config.Runtime{} + ociRuntime.BaseRuntimeSpec = "/etc/containerd/cri-base.json" + + testID := "test-id" + testSandboxID := "sandbox-id" + testContainerName := "container-name" + testPid := uint32(1234) + containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() + + spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime) + assert.NoError(t, err) + + specCheck(t, testID, testSandboxID, testPid, spec) + + assert.Contains(t, spec.Process.User.AdditionalGids, uint32(9999)) + assert.Len(t, spec.Process.User.AdditionalGids, 3) + + assert.Contains(t, spec.Process.Capabilities.Permitted, "CAP_SETUID") + assert.Len(t, spec.Process.Capabilities.Permitted, 1) + + assert.Equal(t, *spec.Linux.Resources.Memory.Limit, containerConfig.Linux.Resources.MemoryLimitInBytes) +} diff --git a/pkg/server/container_create_windows.go b/pkg/server/container_create_windows.go index 2211008eb..e8c81808d 100644 --- a/pkg/server/container_create_windows.go +++ b/pkg/server/container_create_windows.go @@ -91,8 +91,7 @@ func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint3 customopts.WithAnnotation(annotations.SandboxID, sandboxID), customopts.WithAnnotation(annotations.ContainerName, containerName), ) - - return runtimeSpec(id, specOpts...) + return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...) } // No extra spec options needed for windows. diff --git a/pkg/server/sandbox_run_unix.go b/pkg/server/sandbox_run_unix.go index 8391872c4..ad0b85254 100644 --- a/pkg/server/sandbox_run_unix.go +++ b/pkg/server/sandbox_run_unix.go @@ -156,7 +156,7 @@ func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxC customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()), ) - return runtimeSpec(id, specOpts...) + return c.runtimeSpec(id, "", specOpts...) } // sandboxContainerSpecOpts generates OCI spec options for diff --git a/pkg/server/sandbox_run_windows.go b/pkg/server/sandbox_run_windows.go index 4f8f054a9..85105c299 100644 --- a/pkg/server/sandbox_run_windows.go +++ b/pkg/server/sandbox_run_windows.go @@ -67,7 +67,7 @@ func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxC customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()), ) - return runtimeSpec(id, specOpts...) + return c.runtimeSpec(id, "", specOpts...) } // No sandbox container spec options for windows yet. diff --git a/pkg/server/service.go b/pkg/server/service.go index d4a0d2817..43512ff97 100644 --- a/pkg/server/service.go +++ b/pkg/server/service.go @@ -17,15 +17,17 @@ package server import ( + "encoding/json" "fmt" "io" "net/http" + "os" "path/filepath" "time" "github.com/containerd/containerd" + "github.com/containerd/containerd/oci" "github.com/containerd/containerd/plugin" - "github.com/containerd/cri/pkg/store/label" cni "github.com/containerd/go-cni" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -33,6 +35,8 @@ import ( runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/server/streaming" + "github.com/containerd/cri/pkg/store/label" + "github.com/containerd/cri/pkg/atomic" criconfig "github.com/containerd/cri/pkg/config" ctrdutil "github.com/containerd/cri/pkg/containerd/util" @@ -95,6 +99,8 @@ type criService struct { // cniNetConfMonitor is used to reload cni network conf if there is // any valid fs change events from cni network conf dir. cniNetConfMonitor *cniNetConfSyncer + // baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec` + baseOCISpecs map[string]*oci.Spec } // NewCRIService returns a new instance of CRIService @@ -138,6 +144,12 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi return nil, errors.Wrap(err, "failed to create cni conf monitor") } + // Preload base OCI specs + c.baseOCISpecs, err = loadBaseOCISpecs(&config) + if err != nil { + return nil, err + } + return c, nil } @@ -273,3 +285,41 @@ func (c *criService) register(s *grpc.Server) error { func imageFSPath(rootDir, snapshotter string) string { return filepath.Join(rootDir, fmt.Sprintf("%s.%s", plugin.SnapshotPlugin, snapshotter)) } + +func loadOCISpec(filename string) (*oci.Spec, error) { + file, err := os.Open(filename) + if err != nil { + return nil, errors.Wrapf(err, "failed to open base OCI spec: %s", filename) + } + defer file.Close() + + spec := oci.Spec{} + if err := json.NewDecoder(file).Decode(&spec); err != nil { + return nil, errors.Wrap(err, "failed to parse base OCI spec file") + } + + return &spec, nil +} + +func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) { + specs := map[string]*oci.Spec{} + for _, cfg := range config.Runtimes { + if cfg.BaseRuntimeSpec == "" { + continue + } + + // Don't load same file twice + if _, ok := specs[cfg.BaseRuntimeSpec]; ok { + continue + } + + spec, err := loadOCISpec(cfg.BaseRuntimeSpec) + if err != nil { + return nil, errors.Wrapf(err, "failed to load base OCI spec from file: %s", cfg.BaseRuntimeSpec) + } + + specs[cfg.BaseRuntimeSpec] = spec + } + + return specs, nil +} diff --git a/pkg/server/service_test.go b/pkg/server/service_test.go index 0bef174b7..a39f0b9ce 100644 --- a/pkg/server/service_test.go +++ b/pkg/server/service_test.go @@ -17,6 +17,15 @@ package server import ( + "encoding/json" + "io/ioutil" + "os" + "testing" + + "github.com/containerd/containerd/oci" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + criconfig "github.com/containerd/cri/pkg/config" ostesting "github.com/containerd/cri/pkg/os/testing" "github.com/containerd/cri/pkg/registrar" @@ -60,3 +69,34 @@ func newTestCRIService() *criService { netPlugin: servertesting.NewFakeCNIPlugin(), } } + +func TestLoadBaseOCISpec(t *testing.T) { + spec := oci.Spec{Version: "1.0.2", Hostname: "default"} + + file, err := ioutil.TempFile("", "spec-test-") + require.NoError(t, err) + + defer func() { + assert.NoError(t, file.Close()) + assert.NoError(t, os.RemoveAll(file.Name())) + }() + + err = json.NewEncoder(file).Encode(&spec) + assert.NoError(t, err) + + config := criconfig.Config{} + config.Runtimes = map[string]criconfig.Runtime{ + "runc": {BaseRuntimeSpec: file.Name()}, + } + + specs, err := loadBaseOCISpecs(&config) + assert.NoError(t, err) + + assert.Len(t, specs, 1) + + out, ok := specs[file.Name()] + assert.True(t, ok, "expected spec with file name %q", file.Name()) + + assert.Equal(t, "1.0.2", out.Version) + assert.Equal(t, "default", out.Hostname) +}