Merge pull request #1498 from mxpv/base

Specify base OCI runtime spec

docs/config.md

@@ -135,6 +135,13 @@ version = 2
       # i.e pass host devices through to privileged containers.
       privileged_without_host_devices = false
 
+      # base_runtime_spec is a file path to a JSON file with the OCI spec that will be used as the base spec that all
+      # container's are created from.
+      # Use containerd's `ctr oci spec > /etc/containerd/cri-base.json` to output initial spec file.
+      # Spec files are loaded at launch, so containerd daemon must be restared on any changes to refresh default specs.
+      # Still running containers and restarted containers will still be using the original spec from which that container was created.
+      base_runtime_spec = ""
+
       # 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options' is options specific to
       # "io.containerd.runc.v1" and "io.containerd.runc.v2". Its corresponding options type is:
       #   https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26 .

pkg/config/config.go

@@ -52,6 +52,8 @@ type Runtime struct {
 	// PrivilegedWithoutHostDevices overloads the default behaviour for adding host devices to the
 	// runtime spec when the container is privileged. Defaults to false.
 	PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"`
+	// BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from.
+	BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"`
 }
 
 // ContainerdConfig contains toml config related to containerd

pkg/server/container_create.go

@@ -297,12 +297,36 @@ func (c *criService) volumeMounts(containerRootDir string, criMounts []*runtime.
 }
 
 // runtimeSpec returns a default runtime spec used in cri-containerd.
-func runtimeSpec(id string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) {
+func (c *criService) runtimeSpec(id string, baseSpecFile string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) {
 	// GenerateSpec needs namespace.
 	ctx := ctrdutil.NamespacedContext()
-	spec, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: id}, opts...)
+	container := &containers.Container{ID: id}
-	if err != nil {
+
-		return nil, err
+	if baseSpecFile != "" {
+		baseSpec, ok := c.baseOCISpecs[baseSpecFile]
+		if !ok {
+			return nil, errors.Errorf("can't find base OCI spec %q", baseSpecFile)
 		}
+
+		spec := oci.Spec{}
+		if err := util.DeepCopy(&spec, &baseSpec); err != nil {
+			return nil, errors.Wrap(err, "failed to clone OCI spec")
+		}
+
+		// Fix up cgroups path
+		applyOpts := append([]oci.SpecOpts{oci.WithNamespacedCgroup()}, opts...)
+
+		if err := oci.ApplyOpts(ctx, nil, container, &spec, applyOpts...); err != nil {
+			return nil, errors.Wrap(err, "failed to apply OCI options")
+		}
+
+		return &spec, nil
+	}
+
+	spec, err := oci.GenerateSpec(ctx, nil, container, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate spec")
+	}
+
 	return spec, nil
 }

pkg/server/container_create_test.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/containerd/containerd/oci"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
@@ -28,6 +29,7 @@ import (
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 
 	"github.com/containerd/cri/pkg/config"
+	"github.com/containerd/cri/pkg/constants"
 	"github.com/containerd/cri/pkg/containerd/opts"
 )
 
@@ -381,3 +383,25 @@ func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) {
 		})
 	}
 }
+
+func TestBaseRuntimeSpec(t *testing.T) {
+	c := newTestCRIService()
+	c.baseOCISpecs = map[string]*oci.Spec{
+		"/etc/containerd/cri-base.json": {
+			Version:  "1.0.2",
+			Hostname: "old",
+		},
+	}
+
+	out, err := c.runtimeSpec("id1", "/etc/containerd/cri-base.json", oci.WithHostname("new"))
+	assert.NoError(t, err)
+
+	assert.Equal(t, "1.0.2", out.Version)
+	assert.Equal(t, "new", out.Hostname)
+
+	// Make sure original base spec not changed
+	assert.NotEqual(t, out, c.baseOCISpecs["/etc/containerd/cri-base.json"])
+	assert.Equal(t, c.baseOCISpecs["/etc/containerd/cri-base.json"].Hostname, "old")
+
+	assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath)
+}

pkg/server/container_create_unix.go

@@ -262,7 +262,7 @@ func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint3
 				Type: runtimespec.CgroupNamespace,
 			}))
 	}
-	return runtimeSpec(id, specOpts...)
+	return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...)
 }
 
 func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {

pkg/server/container_create_unix_test.go

@@ -1203,3 +1203,45 @@ func TestPrivilegedDevices(t *testing.T) {
 		}
 	}
 }
+
+func TestBaseOCISpec(t *testing.T) {
+	c := newTestCRIService()
+	baseLimit := int64(100)
+	c.baseOCISpecs = map[string]*oci.Spec{
+		"/etc/containerd/cri-base.json": {
+			Process: &runtimespec.Process{
+				User: runtimespec.User{AdditionalGids: []uint32{9999}},
+				Capabilities: &runtimespec.LinuxCapabilities{
+					Permitted: []string{"CAP_SETUID"},
+				},
+			},
+			Linux: &runtimespec.Linux{
+				Resources: &runtimespec.LinuxResources{
+					Memory: &runtimespec.LinuxMemory{Limit: &baseLimit}, // Will be overwritten by `getCreateContainerTestData`
+				},
+			},
+		},
+	}
+
+	ociRuntime := config.Runtime{}
+	ociRuntime.BaseRuntimeSpec = "/etc/containerd/cri-base.json"
+
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
+	assert.NoError(t, err)
+
+	specCheck(t, testID, testSandboxID, testPid, spec)
+
+	assert.Contains(t, spec.Process.User.AdditionalGids, uint32(9999))
+	assert.Len(t, spec.Process.User.AdditionalGids, 3)
+
+	assert.Contains(t, spec.Process.Capabilities.Permitted, "CAP_SETUID")
+	assert.Len(t, spec.Process.Capabilities.Permitted, 1)
+
+	assert.Equal(t, *spec.Linux.Resources.Memory.Limit, containerConfig.Linux.Resources.MemoryLimitInBytes)
+}

pkg/server/container_create_windows.go

@@ -91,8 +91,7 @@ func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint3
 		customopts.WithAnnotation(annotations.SandboxID, sandboxID),
 		customopts.WithAnnotation(annotations.ContainerName, containerName),
 	)
-
+	return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...)
-	return runtimeSpec(id, specOpts...)
 }
 
 // No extra spec options needed for windows.

pkg/server/sandbox_run_unix.go

@@ -156,7 +156,7 @@ func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxC
 		customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
 	)
 
-	return runtimeSpec(id, specOpts...)
+	return c.runtimeSpec(id, "", specOpts...)
 }
 
 // sandboxContainerSpecOpts generates OCI spec options for

pkg/server/sandbox_run_windows.go

@@ -67,7 +67,7 @@ func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxC
 		customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
 	)
 
-	return runtimeSpec(id, specOpts...)
+	return c.runtimeSpec(id, "", specOpts...)
 }
 
 // No sandbox container spec options for windows yet.

pkg/server/service.go

@@ -17,15 +17,17 @@
 package server
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/plugin"
-	"github.com/containerd/cri/pkg/store/label"
 	cni "github.com/containerd/go-cni"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -33,6 +35,8 @@ import (
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 
+	"github.com/containerd/cri/pkg/store/label"
+
 	"github.com/containerd/cri/pkg/atomic"
 	criconfig "github.com/containerd/cri/pkg/config"
 	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
@@ -95,6 +99,8 @@ type criService struct {
 	// cniNetConfMonitor is used to reload cni network conf if there is
 	// any valid fs change events from cni network conf dir.
 	cniNetConfMonitor *cniNetConfSyncer
+	// baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
+	baseOCISpecs map[string]*oci.Spec
 }
 
 // NewCRIService returns a new instance of CRIService
@@ -138,6 +144,12 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
 		return nil, errors.Wrap(err, "failed to create cni conf monitor")
 	}
 
+	// Preload base OCI specs
+	c.baseOCISpecs, err = loadBaseOCISpecs(&config)
+	if err != nil {
+		return nil, err
+	}
+
 	return c, nil
 }
 
@@ -273,3 +285,41 @@ func (c *criService) register(s *grpc.Server) error {
 func imageFSPath(rootDir, snapshotter string) string {
 	return filepath.Join(rootDir, fmt.Sprintf("%s.%s", plugin.SnapshotPlugin, snapshotter))
 }
+
+func loadOCISpec(filename string) (*oci.Spec, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to open base OCI spec: %s", filename)
+	}
+	defer file.Close()
+
+	spec := oci.Spec{}
+	if err := json.NewDecoder(file).Decode(&spec); err != nil {
+		return nil, errors.Wrap(err, "failed to parse base OCI spec file")
+	}
+
+	return &spec, nil
+}
+
+func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) {
+	specs := map[string]*oci.Spec{}
+	for _, cfg := range config.Runtimes {
+		if cfg.BaseRuntimeSpec == "" {
+			continue
+		}
+
+		// Don't load same file twice
+		if _, ok := specs[cfg.BaseRuntimeSpec]; ok {
+			continue
+		}
+
+		spec, err := loadOCISpec(cfg.BaseRuntimeSpec)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to load base OCI spec from file: %s", cfg.BaseRuntimeSpec)
+		}
+
+		specs[cfg.BaseRuntimeSpec] = spec
+	}
+
+	return specs, nil
+}

pkg/server/service_test.go

@@ -17,6 +17,15 @@
 package server
 
 import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/containerd/containerd/oci"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	criconfig "github.com/containerd/cri/pkg/config"
 	ostesting "github.com/containerd/cri/pkg/os/testing"
 	"github.com/containerd/cri/pkg/registrar"
@@ -60,3 +69,34 @@ func newTestCRIService() *criService {
 		netPlugin:          servertesting.NewFakeCNIPlugin(),
 	}
 }
+
+func TestLoadBaseOCISpec(t *testing.T) {
+	spec := oci.Spec{Version: "1.0.2", Hostname: "default"}
+
+	file, err := ioutil.TempFile("", "spec-test-")
+	require.NoError(t, err)
+
+	defer func() {
+		assert.NoError(t, file.Close())
+		assert.NoError(t, os.RemoveAll(file.Name()))
+	}()
+
+	err = json.NewEncoder(file).Encode(&spec)
+	assert.NoError(t, err)
+
+	config := criconfig.Config{}
+	config.Runtimes = map[string]criconfig.Runtime{
+		"runc": {BaseRuntimeSpec: file.Name()},
+	}
+
+	specs, err := loadBaseOCISpecs(&config)
+	assert.NoError(t, err)
+
+	assert.Len(t, specs, 1)
+
+	out, ok := specs[file.Name()]
+	assert.True(t, ok, "expected spec with file name %q", file.Name())
+
+	assert.Equal(t, "1.0.2", out.Version)
+	assert.Equal(t, "default", out.Hostname)
+}