diff --git a/pkg/config/config_windows.go b/pkg/config/config_windows.go index 3489bc5df..f56514c0b 100644 --- a/pkg/config/config_windows.go +++ b/pkg/config/config_windows.go @@ -18,7 +18,53 @@ limitations under the License. package config +import ( + "os" + "path/filepath" + + "github.com/containerd/containerd" + "k8s.io/kubernetes/pkg/kubelet/server/streaming" +) + // DefaultConfig returns default configurations of cri plugin. func DefaultConfig() PluginConfig { - return PluginConfig{} + return PluginConfig{ + CniConfig: CniConfig{ + NetworkPluginBinDir: filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "bin"), + NetworkPluginConfDir: filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "conf"), + NetworkPluginMaxConfNum: 1, + NetworkPluginConfTemplate: "", + }, + ContainerdConfig: ContainerdConfig{ + Snapshotter: containerd.DefaultSnapshotter, + DefaultRuntimeName: "runhcs-wcow-process", + NoPivot: false, + Runtimes: map[string]Runtime{ + "runhcs-wcow-process": { + Type: "io.containerd.runhcs.v1", + }, + }, + }, + DisableTCPService: true, + StreamServerAddress: "127.0.0.1", + StreamServerPort: "0", + StreamIdleTimeout: streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour + EnableTLSStreaming: false, + X509KeyPairStreaming: X509KeyPairStreaming{ + TLSKeyFile: "", + TLSCertFile: "", + }, + SandboxImage: "mcr.microsoft.com/k8s/core/pause:1.0.0", + StatsCollectPeriod: 10, + MaxContainerLogLineSize: 16 * 1024, + Registry: Registry{ + Mirrors: map[string]Mirror{ + "docker.io": { + Endpoints: []string{"https://registry-1.docker.io"}, + }, + }, + }, + MaxConcurrentDownloads: 3, + // TODO(windows): Add platform specific config, so that most common defaults can be shared. + } } diff --git a/pkg/containerd/opts/spec.go b/pkg/containerd/opts/spec.go index da0dec584..bc34fc78e 100644 --- a/pkg/containerd/opts/spec.go +++ b/pkg/containerd/opts/spec.go @@ -42,6 +42,12 @@ func WithRelativeRoot(root string) oci.SpecOpts { } } +// WithoutRoot sets the root to nil for the container. +func WithoutRoot(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error { + s.Root = nil + return nil +} + // WithProcessArgs sets the process args on the spec based on the image and runtime config func WithProcessArgs(config *runtime.ContainerConfig, image *imagespec.ImageConfig) oci.SpecOpts { return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) { diff --git a/pkg/containerd/opts/spec_unix.go b/pkg/containerd/opts/spec_unix.go index 94a072afd..ef0b20269 100644 --- a/pkg/containerd/opts/spec_unix.go +++ b/pkg/containerd/opts/spec_unix.go @@ -137,7 +137,6 @@ func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*ru mounts = append(mounts, e) } } - // --- // Sort mounts in number of parts. This ensures that high level mounts don't // shadow other mounts. diff --git a/pkg/containerd/opts/spec_windows.go b/pkg/containerd/opts/spec_windows.go new file mode 100644 index 000000000..cbee12a1b --- /dev/null +++ b/pkg/containerd/opts/spec_windows.go @@ -0,0 +1,174 @@ +// +build windows + +/* +Copyright The containerd Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opts + +import ( + "context" + "path/filepath" + "sort" + + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/oci" + runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + + osinterface "github.com/containerd/cri/pkg/os" +) + +// WithWindowsNetworkNamespace sets windows network namespace for container. +// TODO(windows): Move this into container/containerd. +func WithWindowsNetworkNamespace(path string) oci.SpecOpts { + return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error { + if s.Windows == nil { + s.Windows = &runtimespec.Windows{} + } + if s.Windows.Network == nil { + s.Windows.Network = &runtimespec.WindowsNetwork{} + } + s.Windows.Network.NetworkNamespace = path + return nil + } +} + +// WithWindowsMounts sorts and adds runtime and CRI mounts to the spec for +// windows container. +func WithWindowsMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount) oci.SpecOpts { + return func(ctx context.Context, client oci.Client, _ *containers.Container, s *runtimespec.Spec) error { + // mergeMounts merge CRI mounts with extra mounts. If a mount destination + // is mounted by both a CRI mount and an extra mount, the CRI mount will + // be kept. + var ( + criMounts = config.GetMounts() + mounts = append([]*runtime.Mount{}, criMounts...) + ) + // Copy all mounts from extra mounts, except for mounts overriden by CRI. + for _, e := range extra { + found := false + for _, c := range criMounts { + if filepath.Clean(e.ContainerPath) == filepath.Clean(c.ContainerPath) { + found = true + break + } + } + if !found { + mounts = append(mounts, e) + } + } + + // Sort mounts in number of parts. This ensures that high level mounts don't + // shadow other mounts. + sort.Sort(orderedMounts(mounts)) + + // Copy all mounts from default mounts, except for + // - mounts overriden by supplied mount; + // - all mounts under /dev if a supplied /dev is present. + mountSet := make(map[string]struct{}) + for _, m := range mounts { + mountSet[filepath.Clean(m.ContainerPath)] = struct{}{} + } + + defaultMounts := s.Mounts + s.Mounts = nil + + for _, m := range defaultMounts { + dst := filepath.Clean(m.Destination) + if _, ok := mountSet[dst]; ok { + // filter out mount overridden by a supplied mount + continue + } + s.Mounts = append(s.Mounts, m) + } + + for _, mount := range mounts { + var ( + dst = mount.GetContainerPath() + src = mount.GetHostPath() + ) + // TODO(windows): Support special mount sources, e.g. named pipe. + // Create the host path if it doesn't exist. + if _, err := osi.Stat(src); err != nil { + // If the source doesn't exist, return an error instead + // of creating the source. This aligns with Docker's + // behavior on windows. + return errors.Wrapf(err, "failed to stat %q", src) + } + src, err := osi.ResolveSymbolicLink(src) + if err != nil { + return errors.Wrapf(err, "failed to resolve symlink %q", src) + } + + var options []string + // NOTE(random-liu): we don't change all mounts to `ro` when root filesystem + // is readonly. This is different from docker's behavior, but make more sense. + if mount.GetReadonly() { + options = append(options, "ro") + } else { + options = append(options, "rw") + } + s.Mounts = append(s.Mounts, runtimespec.Mount{ + Source: src, + Destination: dst, + Options: options, + }) + } + return nil + } +} + +// WithWindowsResources sets the provided resource restrictions for windows. +func WithWindowsResources(resources *runtime.WindowsContainerResources) oci.SpecOpts { + return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error { + if resources == nil { + return nil + } + if s.Windows == nil { + s.Windows = &runtimespec.Windows{} + } + if s.Windows.Resources == nil { + s.Windows.Resources = &runtimespec.WindowsResources{} + } + if s.Windows.Resources.CPU == nil { + s.Windows.Resources.CPU = &runtimespec.WindowsCPUResources{} + } + if s.Windows.Resources.Memory == nil { + s.Windows.Resources.Memory = &runtimespec.WindowsMemoryResources{} + } + + var ( + count = uint64(resources.GetCpuCount()) + shares = uint16(resources.GetCpuShares()) + max = uint16(resources.GetCpuMaximum()) + limit = uint64(resources.GetMemoryLimitInBytes()) + ) + if count != 0 { + s.Windows.Resources.CPU.Count = &count + } + if shares != 0 { + s.Windows.Resources.CPU.Shares = &shares + } + if max != 0 { + s.Windows.Resources.CPU.Maximum = &max + } + if limit != 0 { + s.Windows.Resources.Memory.Limit = &limit + } + return nil + } +} diff --git a/pkg/netns/netns_windows.go b/pkg/netns/netns_windows.go index 6cbc512c0..994cd1157 100644 --- a/pkg/netns/netns_windows.go +++ b/pkg/netns/netns_windows.go @@ -18,33 +18,61 @@ limitations under the License. package netns -// TODO(windows): Implement netns for windows. -// NetNS holds network namespace. +import "github.com/Microsoft/hcsshim/hcn" + +// NetNS holds network namespace for sandbox type NetNS struct { + path string } -// NewNetNS creates a network namespace. +// NewNetNS creates a network namespace for the sandbox func NewNetNS() (*NetNS, error) { - return nil, nil + temp := hcn.HostComputeNamespace{} + hcnNamespace, err := temp.Create() + if err != nil { + return nil, err + } + + return &NetNS{path: string(hcnNamespace.Id)}, nil } // LoadNetNS loads existing network namespace. func LoadNetNS(path string) *NetNS { - return nil + return &NetNS{path: path} } -// Remove removes network namepace. Remove is idempotent, meaning it might -// be invoked multiple times and provides consistent result. +// Remove removes network namepace if it exists and not closed. Remove is idempotent, +// meaning it might be invoked multiple times and provides consistent result. func (n *NetNS) Remove() error { - return nil + hcnNamespace, err := hcn.GetNamespaceByID(n.path) + if err != nil { + if hcn.IsNotFoundError(err) { + return nil + } + return err + } + err = hcnNamespace.Delete() + if err == nil || hcn.IsNotFoundError(err) { + return nil + } + return err } // Closed checks whether the network namespace has been closed. func (n *NetNS) Closed() (bool, error) { - return false, nil + _, err := hcn.GetNamespaceByID(n.path) + if err == nil { + return false, nil + } + if hcn.IsNotFoundError(err) { + return true, nil + } + return false, err } // GetPath returns network namespace path for sandbox container func (n *NetNS) GetPath() string { - return "" + return n.path } + +// NOTE: Do function is not supported. diff --git a/pkg/server/container_create.go b/pkg/server/container_create.go index 41785c373..cfb5d6aad 100644 --- a/pkg/server/container_create.go +++ b/pkg/server/container_create.go @@ -284,8 +284,7 @@ func (c *criService) volumeMounts(containerRootDir string, criMounts []*runtime. } // runtimeSpec returns a default runtime spec used in cri-containerd. -// TODO(windows): Remove nolint after windows starts using this helper. -func runtimeSpec(id string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) { // nolint: deadcode, unused +func runtimeSpec(id string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) { // GenerateSpec needs namespace. ctx := ctrdutil.NamespacedContext() spec, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: id}, opts...) diff --git a/pkg/server/container_create_test.go b/pkg/server/container_create_test.go index 76fe93344..3eb18f80f 100644 --- a/pkg/server/container_create_test.go +++ b/pkg/server/container_create_test.go @@ -17,14 +17,183 @@ limitations under the License. package server import ( + "context" "path/filepath" "testing" imagespec "github.com/opencontainers/image-spec/specs-go/v1" + runtimespec "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + + "github.com/containerd/cri/pkg/config" + "github.com/containerd/cri/pkg/containerd/opts" ) +func checkMount(t *testing.T, mounts []runtimespec.Mount, src, dest, typ string, + contains, notcontains []string) { + found := false + for _, m := range mounts { + if m.Source == src && m.Destination == dest { + assert.Equal(t, m.Type, typ) + for _, c := range contains { + assert.Contains(t, m.Options, c) + } + for _, n := range notcontains { + assert.NotContains(t, m.Options, n) + } + found = true + break + } + } + assert.True(t, found, "mount from %q to %q not found", src, dest) +} + +func TestGeneralContainerSpec(t *testing.T) { + testID := "test-id" + testPid := uint32(1234) + containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() + ociRuntime := config.Runtime{} + c := newTestCRIService() + testSandboxID := "sandbox-id" + spec, err := c.containerSpec(testID, testSandboxID, testPid, "", containerConfig, sandboxConfig, imageConfig, nil, ociRuntime) + require.NoError(t, err) + specCheck(t, testID, testSandboxID, testPid, spec) +} + +func TestPodAnnotationPassthroughContainerSpec(t *testing.T) { + testID := "test-id" + testSandboxID := "sandbox-id" + testPid := uint32(1234) + + for desc, test := range map[string]struct { + podAnnotations []string + configChange func(*runtime.PodSandboxConfig) + specCheck func(*testing.T, *runtimespec.Spec) + }{ + "a passthrough annotation should be passed as an OCI annotation": { + podAnnotations: []string{"c"}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + assert.Equal(t, spec.Annotations["c"], "d") + }, + }, + "a non-passthrough annotation should not be passed as an OCI annotation": { + configChange: func(c *runtime.PodSandboxConfig) { + c.Annotations["d"] = "e" + }, + podAnnotations: []string{"c"}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + assert.Equal(t, spec.Annotations["c"], "d") + _, ok := spec.Annotations["d"] + assert.False(t, ok) + }, + }, + "passthrough annotations should support wildcard match": { + configChange: func(c *runtime.PodSandboxConfig) { + c.Annotations["t.f"] = "j" + c.Annotations["z.g"] = "o" + c.Annotations["z"] = "o" + c.Annotations["y.ca"] = "b" + c.Annotations["y"] = "b" + }, + podAnnotations: []string{"t*", "z.*", "y.c*"}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + t.Logf("%+v", spec.Annotations) + assert.Equal(t, spec.Annotations["t.f"], "j") + assert.Equal(t, spec.Annotations["z.g"], "o") + assert.Equal(t, spec.Annotations["y.ca"], "b") + _, ok := spec.Annotations["y"] + assert.False(t, ok) + _, ok = spec.Annotations["z"] + assert.False(t, ok) + }, + }, + } { + t.Run(desc, func(t *testing.T) { + c := newTestCRIService() + containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() + if test.configChange != nil { + test.configChange(sandboxConfig) + } + + ociRuntime := config.Runtime{ + PodAnnotations: test.podAnnotations, + } + spec, err := c.containerSpec(testID, testSandboxID, testPid, "", + containerConfig, sandboxConfig, imageConfig, nil, ociRuntime) + assert.NoError(t, err) + assert.NotNil(t, spec) + specCheck(t, testID, testSandboxID, testPid, spec) + if test.specCheck != nil { + test.specCheck(t, spec) + } + }) + } +} + +func TestContainerSpecCommand(t *testing.T) { + for desc, test := range map[string]struct { + criEntrypoint []string + criArgs []string + imageEntrypoint []string + imageArgs []string + expected []string + expectErr bool + }{ + "should use cri entrypoint if it's specified": { + criEntrypoint: []string{"a", "b"}, + imageEntrypoint: []string{"c", "d"}, + imageArgs: []string{"e", "f"}, + expected: []string{"a", "b"}, + }, + "should use cri entrypoint if it's specified even if it's empty": { + criEntrypoint: []string{}, + criArgs: []string{"a", "b"}, + imageEntrypoint: []string{"c", "d"}, + imageArgs: []string{"e", "f"}, + expected: []string{"a", "b"}, + }, + "should use cri entrypoint and args if they are specified": { + criEntrypoint: []string{"a", "b"}, + criArgs: []string{"c", "d"}, + imageEntrypoint: []string{"e", "f"}, + imageArgs: []string{"g", "h"}, + expected: []string{"a", "b", "c", "d"}, + }, + "should use image entrypoint if cri entrypoint is not specified": { + criArgs: []string{"a", "b"}, + imageEntrypoint: []string{"c", "d"}, + imageArgs: []string{"e", "f"}, + expected: []string{"c", "d", "a", "b"}, + }, + "should use image args if both cri entrypoint and args are not specified": { + imageEntrypoint: []string{"c", "d"}, + imageArgs: []string{"e", "f"}, + expected: []string{"c", "d", "e", "f"}, + }, + "should return error if both entrypoint and args are empty": { + expectErr: true, + }, + } { + + config, _, imageConfig, _ := getCreateContainerTestData() + config.Command = test.criEntrypoint + config.Args = test.criArgs + imageConfig.Entrypoint = test.imageEntrypoint + imageConfig.Cmd = test.imageArgs + + var spec runtimespec.Spec + err := opts.WithProcessArgs(config, imageConfig)(context.Background(), nil, nil, &spec) + if test.expectErr { + assert.Error(t, err) + continue + } + assert.NoError(t, err) + assert.Equal(t, test.expected, spec.Process.Args, desc) + } +} + func TestVolumeMounts(t *testing.T) { testContainerRootDir := "test-container-root" for desc, test := range map[string]struct { @@ -95,3 +264,117 @@ func TestVolumeMounts(t *testing.T) { } } } + +func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) { + testID := "test-id" + testSandboxID := "sandbox-id" + testPid := uint32(1234) + + for desc, test := range map[string]struct { + podAnnotations []string + containerAnnotations []string + podConfigChange func(*runtime.PodSandboxConfig) + configChange func(*runtime.ContainerConfig) + specCheck func(*testing.T, *runtimespec.Spec) + }{ + "passthrough annotations from pod and container should be passed as an OCI annotation": { + podConfigChange: func(p *runtime.PodSandboxConfig) { + p.Annotations["pod.annotation.1"] = "1" + p.Annotations["pod.annotation.2"] = "2" + p.Annotations["pod.annotation.3"] = "3" + }, + configChange: func(c *runtime.ContainerConfig) { + c.Annotations["container.annotation.1"] = "1" + c.Annotations["container.annotation.2"] = "2" + c.Annotations["container.annotation.3"] = "3" + }, + podAnnotations: []string{"pod.annotation.1"}, + containerAnnotations: []string{"container.annotation.1"}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + assert.Equal(t, "1", spec.Annotations["container.annotation.1"]) + _, ok := spec.Annotations["container.annotation.2"] + assert.False(t, ok) + _, ok = spec.Annotations["container.annotation.3"] + assert.False(t, ok) + assert.Equal(t, "1", spec.Annotations["pod.annotation.1"]) + _, ok = spec.Annotations["pod.annotation.2"] + assert.False(t, ok) + _, ok = spec.Annotations["pod.annotation.3"] + assert.False(t, ok) + }, + }, + "passthrough annotations from pod and container should support wildcard": { + podConfigChange: func(p *runtime.PodSandboxConfig) { + p.Annotations["pod.annotation.1"] = "1" + p.Annotations["pod.annotation.2"] = "2" + p.Annotations["pod.annotation.3"] = "3" + }, + configChange: func(c *runtime.ContainerConfig) { + c.Annotations["container.annotation.1"] = "1" + c.Annotations["container.annotation.2"] = "2" + c.Annotations["container.annotation.3"] = "3" + }, + podAnnotations: []string{"pod.annotation.*"}, + containerAnnotations: []string{"container.annotation.*"}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + assert.Equal(t, "1", spec.Annotations["container.annotation.1"]) + assert.Equal(t, "2", spec.Annotations["container.annotation.2"]) + assert.Equal(t, "3", spec.Annotations["container.annotation.3"]) + assert.Equal(t, "1", spec.Annotations["pod.annotation.1"]) + assert.Equal(t, "2", spec.Annotations["pod.annotation.2"]) + assert.Equal(t, "3", spec.Annotations["pod.annotation.3"]) + }, + }, + "annotations should not pass through if no passthrough annotations are configured": { + podConfigChange: func(p *runtime.PodSandboxConfig) { + p.Annotations["pod.annotation.1"] = "1" + p.Annotations["pod.annotation.2"] = "2" + p.Annotations["pod.annotation.3"] = "3" + }, + configChange: func(c *runtime.ContainerConfig) { + c.Annotations["container.annotation.1"] = "1" + c.Annotations["container.annotation.2"] = "2" + c.Annotations["container.annotation.3"] = "3" + }, + podAnnotations: []string{}, + containerAnnotations: []string{}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + _, ok := spec.Annotations["container.annotation.1"] + assert.False(t, ok) + _, ok = spec.Annotations["container.annotation.2"] + assert.False(t, ok) + _, ok = spec.Annotations["container.annotation.3"] + assert.False(t, ok) + _, ok = spec.Annotations["pod.annotation.1"] + assert.False(t, ok) + _, ok = spec.Annotations["pod.annotation.2"] + assert.False(t, ok) + _, ok = spec.Annotations["pod.annotation.3"] + assert.False(t, ok) + }, + }, + } { + t.Run(desc, func(t *testing.T) { + c := newTestCRIService() + containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() + if test.configChange != nil { + test.configChange(containerConfig) + } + if test.podConfigChange != nil { + test.podConfigChange(sandboxConfig) + } + ociRuntime := config.Runtime{ + PodAnnotations: test.podAnnotations, + ContainerAnnotations: test.containerAnnotations, + } + spec, err := c.containerSpec(testID, testSandboxID, testPid, "", + containerConfig, sandboxConfig, imageConfig, nil, ociRuntime) + assert.NoError(t, err) + assert.NotNil(t, spec) + specCheck(t, testID, testSandboxID, testPid, spec) + if test.specCheck != nil { + test.specCheck(t, spec) + } + }) + } +} diff --git a/pkg/server/container_create_unix_test.go b/pkg/server/container_create_unix_test.go index 08b37dd81..15073a680 100644 --- a/pkg/server/container_create_unix_test.go +++ b/pkg/server/container_create_unix_test.go @@ -31,12 +31,6 @@ import ( "github.com/containerd/containerd/contrib/seccomp" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/oci" - "github.com/containerd/cri/pkg/annotations" - "github.com/containerd/cri/pkg/config" - "github.com/containerd/cri/pkg/containerd/opts" - ctrdutil "github.com/containerd/cri/pkg/containerd/util" - ostesting "github.com/containerd/cri/pkg/os/testing" - "github.com/containerd/cri/pkg/util" imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runc/libcontainer/devices" runtimespec "github.com/opencontainers/runtime-spec/specs-go" @@ -44,26 +38,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" -) -func checkMount(t *testing.T, mounts []runtimespec.Mount, src, dest, typ string, - contains, notcontains []string) { - found := false - for _, m := range mounts { - if m.Source == src && m.Destination == dest { - assert.Equal(t, m.Type, typ) - for _, c := range contains { - assert.Contains(t, m.Options, c) - } - for _, n := range notcontains { - assert.NotContains(t, m.Options, n) - } - found = true - break - } - } - assert.True(t, found, "mount from %q to %q not found", src, dest) -} + "github.com/containerd/cri/pkg/annotations" + "github.com/containerd/cri/pkg/config" + "github.com/containerd/cri/pkg/containerd/opts" + ctrdutil "github.com/containerd/cri/pkg/containerd/util" + ostesting "github.com/containerd/cri/pkg/os/testing" + "github.com/containerd/cri/pkg/util" +) func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, string, uint32, *runtimespec.Spec)) { @@ -195,18 +177,6 @@ func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandbox return config, sandboxConfig, imageConfig, specCheck } -func TestGeneralContainerSpec(t *testing.T) { - testID := "test-id" - testPid := uint32(1234) - containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - ociRuntime := config.Runtime{} - c := newTestCRIService() - testSandboxID := "sandbox-id" - spec, err := c.containerSpec(testID, testSandboxID, testPid, "", containerConfig, sandboxConfig, imageConfig, nil, ociRuntime) - require.NoError(t, err) - specCheck(t, testID, testSandboxID, testPid, spec) -} - func TestContainerCapabilities(t *testing.T) { testID := "test-id" testSandboxID := "sandbox-id" @@ -299,134 +269,6 @@ func TestContainerSpecTty(t *testing.T) { } } -func TestPodAnnotationPassthroughContainerSpec(t *testing.T) { - testID := "test-id" - testSandboxID := "sandbox-id" - testPid := uint32(1234) - - for desc, test := range map[string]struct { - podAnnotations []string - configChange func(*runtime.PodSandboxConfig) - specCheck func(*testing.T, *runtimespec.Spec) - }{ - "a passthrough annotation should be passed as an OCI annotation": { - podAnnotations: []string{"c"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, spec.Annotations["c"], "d") - }, - }, - "a non-passthrough annotation should not be passed as an OCI annotation": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Annotations["d"] = "e" - }, - podAnnotations: []string{"c"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, spec.Annotations["c"], "d") - _, ok := spec.Annotations["d"] - assert.False(t, ok) - }, - }, - "passthrough annotations should support wildcard match": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Annotations["t.f"] = "j" - c.Annotations["z.g"] = "o" - c.Annotations["z"] = "o" - c.Annotations["y.ca"] = "b" - c.Annotations["y"] = "b" - }, - podAnnotations: []string{"t*", "z.*", "y.c*"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - t.Logf("%+v", spec.Annotations) - assert.Equal(t, spec.Annotations["t.f"], "j") - assert.Equal(t, spec.Annotations["z.g"], "o") - assert.Equal(t, spec.Annotations["y.ca"], "b") - _, ok := spec.Annotations["y"] - assert.False(t, ok) - _, ok = spec.Annotations["z"] - assert.False(t, ok) - }, - }, - } { - t.Run(desc, func(t *testing.T) { - c := newTestCRIService() - containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - if test.configChange != nil { - test.configChange(sandboxConfig) - } - - ociRuntime := config.Runtime{ - PodAnnotations: test.podAnnotations, - } - spec, err := c.containerSpec(testID, testSandboxID, testPid, "", - containerConfig, sandboxConfig, imageConfig, nil, ociRuntime) - assert.NoError(t, err) - assert.NotNil(t, spec) - specCheck(t, testID, testSandboxID, testPid, spec) - if test.specCheck != nil { - test.specCheck(t, spec) - } - }) - } -} - -func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) { - testID := "test-id" - testSandboxID := "sandbox-id" - testPid := uint32(1234) - - for desc, test := range map[string]struct { - podAnnotations []string - containerAnnotations []string - configChange func(*runtime.PodSandboxConfig) - specCheck func(*testing.T, *runtimespec.Spec) - }{ - "passthrough annotations from pod and container should be passed as an OCI annotation": { - podAnnotations: []string{"c"}, - containerAnnotations: []string{"c*"}, // wildcard should pick up ca-c->ca-d pair in container - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, "d", spec.Annotations["c"]) - assert.Equal(t, "ca-d", spec.Annotations["ca-c"]) - }, - }, - "annotations should not pass through if no passthrough annotations are configured": { - podAnnotations: []string{}, - containerAnnotations: []string{}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, "", spec.Annotations["c"]) - assert.Equal(t, "", spec.Annotations["ca-c"]) - }, - }, - "unmatched annotations should not pass through even if passthrough annotations are configured": { - podAnnotations: []string{"x"}, - containerAnnotations: []string{"x*"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, "", spec.Annotations["c"]) - assert.Equal(t, "", spec.Annotations["ca-c"]) - }, - }, - } { - t.Run(desc, func(t *testing.T) { - c := newTestCRIService() - containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() - if test.configChange != nil { - test.configChange(sandboxConfig) - } - ociRuntime := config.Runtime{ - PodAnnotations: test.podAnnotations, - ContainerAnnotations: test.containerAnnotations, - } - spec, err := c.containerSpec(testID, testSandboxID, testPid, "", - containerConfig, sandboxConfig, imageConfig, nil, ociRuntime) - assert.NoError(t, err) - assert.NotNil(t, spec) - specCheck(t, testID, testSandboxID, testPid, spec) - if test.specCheck != nil { - test.specCheck(t, spec) - } - }) - } -} - func TestContainerSpecReadonlyRootfs(t *testing.T) { testID := "test-id" testSandboxID := "sandbox-id" @@ -550,68 +392,6 @@ func TestContainerAndSandboxPrivileged(t *testing.T) { } } -func TestContainerSpecCommand(t *testing.T) { - for desc, test := range map[string]struct { - criEntrypoint []string - criArgs []string - imageEntrypoint []string - imageArgs []string - expected []string - expectErr bool - }{ - "should use cri entrypoint if it's specified": { - criEntrypoint: []string{"a", "b"}, - imageEntrypoint: []string{"c", "d"}, - imageArgs: []string{"e", "f"}, - expected: []string{"a", "b"}, - }, - "should use cri entrypoint if it's specified even if it's empty": { - criEntrypoint: []string{}, - criArgs: []string{"a", "b"}, - imageEntrypoint: []string{"c", "d"}, - imageArgs: []string{"e", "f"}, - expected: []string{"a", "b"}, - }, - "should use cri entrypoint and args if they are specified": { - criEntrypoint: []string{"a", "b"}, - criArgs: []string{"c", "d"}, - imageEntrypoint: []string{"e", "f"}, - imageArgs: []string{"g", "h"}, - expected: []string{"a", "b", "c", "d"}, - }, - "should use image entrypoint if cri entrypoint is not specified": { - criArgs: []string{"a", "b"}, - imageEntrypoint: []string{"c", "d"}, - imageArgs: []string{"e", "f"}, - expected: []string{"c", "d", "a", "b"}, - }, - "should use image args if both cri entrypoint and args are not specified": { - imageEntrypoint: []string{"c", "d"}, - imageArgs: []string{"e", "f"}, - expected: []string{"c", "d", "e", "f"}, - }, - "should return error if both entrypoint and args are empty": { - expectErr: true, - }, - } { - - config, _, imageConfig, _ := getCreateContainerTestData() - config.Command = test.criEntrypoint - config.Args = test.criArgs - imageConfig.Entrypoint = test.imageEntrypoint - imageConfig.Cmd = test.imageArgs - - var spec runtimespec.Spec - err := opts.WithProcessArgs(config, imageConfig)(context.Background(), nil, nil, &spec) - if test.expectErr { - assert.Error(t, err) - continue - } - assert.NoError(t, err) - assert.Equal(t, test.expected, spec.Process.Args, desc) - } -} - func TestContainerMounts(t *testing.T) { const testSandboxID = "test-id" for desc, test := range map[string]struct { diff --git a/pkg/server/container_create_windows.go b/pkg/server/container_create_windows.go index af597f506..f14a2cf6f 100644 --- a/pkg/server/container_create_windows.go +++ b/pkg/server/container_create_windows.go @@ -19,13 +19,14 @@ limitations under the License. package server import ( - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/oci" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtimespec "github.com/opencontainers/runtime-spec/specs-go" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + "github.com/containerd/cri/pkg/annotations" "github.com/containerd/cri/pkg/config" + customopts "github.com/containerd/cri/pkg/containerd/opts" ) // No container mounts for windows. @@ -33,11 +34,63 @@ func (c *criService) containerMounts(sandboxID string, config *runtime.Container return nil } -// TODO(windows): Add windows container spec. func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint32, netNSPath string, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig, extraMounts []*runtime.Mount, ociRuntime config.Runtime) (*runtimespec.Spec, error) { - return nil, errdefs.ErrNotImplemented + specOpts := []oci.SpecOpts{ + customopts.WithProcessArgs(config, imageConfig), + } + if config.GetWorkingDir() != "" { + specOpts = append(specOpts, oci.WithProcessCwd(config.GetWorkingDir())) + } else if imageConfig.WorkingDir != "" { + specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir)) + } + + if config.GetTty() { + specOpts = append(specOpts, oci.WithTTY) + } + + // Apply envs from image config first, so that envs from container config + // can override them. + env := imageConfig.Env + for _, e := range config.GetEnvs() { + env = append(env, e.GetKey()+"="+e.GetValue()) + } + specOpts = append(specOpts, oci.WithEnv(env)) + + specOpts = append(specOpts, + // Clear the root location since hcsshim expects it. + // NOTE: readonly rootfs doesn't work on windows. + customopts.WithoutRoot, + customopts.WithWindowsNetworkNamespace(netNSPath), + ) + + specOpts = append(specOpts, customopts.WithWindowsMounts(c.os, config, extraMounts)) + + specOpts = append(specOpts, customopts.WithWindowsResources(config.GetWindows().GetResources())) + + username := config.GetWindows().GetSecurityContext().GetRunAsUsername() + if username != "" { + specOpts = append(specOpts, oci.WithUser(username)) + } + // TODO(windows): Add CredentialSpec support. + + for pKey, pValue := range getPassthroughAnnotations(sandboxConfig.Annotations, + ociRuntime.PodAnnotations) { + specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue)) + } + + for pKey, pValue := range getPassthroughAnnotations(config.Annotations, + ociRuntime.ContainerAnnotations) { + specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue)) + } + + specOpts = append(specOpts, + customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer), + customopts.WithAnnotation(annotations.SandboxID, sandboxID), + ) + + return runtimeSpec(id, specOpts...) } // No extra spec options needed for windows. diff --git a/pkg/server/container_create_windows_test.go b/pkg/server/container_create_windows_test.go new file mode 100644 index 000000000..db2fa4c51 --- /dev/null +++ b/pkg/server/container_create_windows_test.go @@ -0,0 +1,140 @@ +// +build windows + +/* +Copyright The containerd Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "testing" + + imagespec "github.com/opencontainers/image-spec/specs-go/v1" + runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + + "github.com/containerd/cri/pkg/annotations" + "github.com/containerd/cri/pkg/config" +) + +func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandboxConfig, + *imagespec.ImageConfig, func(*testing.T, string, string, uint32, *runtimespec.Spec)) { + config := &runtime.ContainerConfig{ + Metadata: &runtime.ContainerMetadata{ + Name: "test-name", + Attempt: 1, + }, + Image: &runtime.ImageSpec{ + Image: "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113799", + }, + Command: []string{"test", "command"}, + Args: []string{"test", "args"}, + WorkingDir: "test-cwd", + Envs: []*runtime.KeyValue{ + {Key: "k1", Value: "v1"}, + {Key: "k2", Value: "v2"}, + {Key: "k3", Value: "v3=v3bis"}, + {Key: "k4", Value: "v4=v4bis=foop"}, + }, + Mounts: []*runtime.Mount{ + // everything default + { + ContainerPath: "container-path-1", + HostPath: "host-path-1", + }, + // readOnly + { + ContainerPath: "container-path-2", + HostPath: "host-path-2", + Readonly: true, + }, + }, + Labels: map[string]string{"a": "b"}, + Annotations: map[string]string{"c": "d"}, + Windows: &runtime.WindowsContainerConfig{ + Resources: &runtime.WindowsContainerResources{ + CpuShares: 100, + CpuCount: 200, + CpuMaximum: 300, + MemoryLimitInBytes: 400, + }, + SecurityContext: &runtime.WindowsContainerSecurityContext{ + RunAsUsername: "test-user", + }, + }, + } + sandboxConfig := &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: "test-sandbox-name", + Uid: "test-sandbox-uid", + Namespace: "test-sandbox-ns", + Attempt: 2, + }, + Annotations: map[string]string{"c": "d"}, + } + imageConfig := &imagespec.ImageConfig{ + Env: []string{"ik1=iv1", "ik2=iv2", "ik3=iv3=iv3bis", "ik4=iv4=iv4bis=boop"}, + Entrypoint: []string{"/entrypoint"}, + Cmd: []string{"cmd"}, + WorkingDir: "/workspace", + } + specCheck := func(t *testing.T, id string, sandboxID string, sandboxPid uint32, spec *runtimespec.Spec) { + assert.Nil(t, spec.Root) + assert.Equal(t, []string{"test", "command", "test", "args"}, spec.Process.Args) + assert.Equal(t, "test-cwd", spec.Process.Cwd) + assert.Contains(t, spec.Process.Env, "k1=v1", "k2=v2", "k3=v3=v3bis", "ik4=iv4=iv4bis=boop") + assert.Contains(t, spec.Process.Env, "ik1=iv1", "ik2=iv2", "ik3=iv3=iv3bis", "k4=v4=v4bis=foop") + + t.Logf("Check bind mount") + checkMount(t, spec.Mounts, "host-path-1", "container-path-1", "", []string{"rw"}, nil) + checkMount(t, spec.Mounts, "host-path-2", "container-path-2", "", []string{"ro"}, nil) + + t.Logf("Check resource limits") + assert.EqualValues(t, *spec.Windows.Resources.CPU.Shares, 100) + assert.EqualValues(t, *spec.Windows.Resources.CPU.Count, 200) + assert.EqualValues(t, *spec.Windows.Resources.CPU.Maximum, 300) + assert.EqualValues(t, *spec.Windows.Resources.CPU.Maximum, 300) + assert.EqualValues(t, *spec.Windows.Resources.Memory.Limit, 400) + + t.Logf("Check username") + assert.Contains(t, spec.Process.User.Username, "test-user") + + t.Logf("Check PodSandbox annotations") + assert.Contains(t, spec.Annotations, annotations.SandboxID) + assert.EqualValues(t, spec.Annotations[annotations.SandboxID], sandboxID) + + assert.Contains(t, spec.Annotations, annotations.ContainerType) + assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeContainer) + } + return config, sandboxConfig, imageConfig, specCheck +} + +func TestContainerWindowsNetworkNamespace(t *testing.T) { + testID := "test-id" + testSandboxID := "sandbox-id" + testPid := uint32(1234) + nsPath := "test-cni" + c := newTestCRIService() + + containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData() + spec, err := c.containerSpec(testID, testSandboxID, testPid, nsPath, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}) + assert.NoError(t, err) + assert.NotNil(t, spec) + specCheck(t, testID, testSandboxID, testPid, spec) + assert.NotNil(t, spec.Windows) + assert.NotNil(t, spec.Windows.Network) + assert.Equal(t, nsPath, spec.Windows.Network.NetworkNamespace) +} diff --git a/pkg/server/container_stats.go b/pkg/server/container_stats.go index 802134d3a..076ac6b35 100644 --- a/pkg/server/container_stats.go +++ b/pkg/server/container_stats.go @@ -25,8 +25,6 @@ import ( // ContainerStats returns stats of the container. If the container does not // exist, the call returns an error. -// TODO(windows): hcsshim Stats is not implemented, add windows support after -// that is implemented. func (c *criService) ContainerStats(ctx context.Context, in *runtime.ContainerStatsRequest) (*runtime.ContainerStatsResponse, error) { cntr, err := c.containerStore.Get(in.GetContainerId()) if err != nil { diff --git a/pkg/server/container_stats_list_windows.go b/pkg/server/container_stats_list_windows.go index 05ed85355..d82e5201a 100644 --- a/pkg/server/container_stats_list_windows.go +++ b/pkg/server/container_stats_list_windows.go @@ -25,11 +25,36 @@ import ( containerstore "github.com/containerd/cri/pkg/store/container" ) -// TODO(windows): Implement a dummy version of this, and actually support this -// when stats is supported by the hcs containerd shim. func (c *criService) containerMetrics( meta containerstore.Metadata, stats *types.Metric, ) (*runtime.ContainerStats, error) { - return nil, nil + var cs runtime.ContainerStats + var usedBytes, inodesUsed uint64 + sn, err := c.snapshotStore.Get(meta.ID) + // If snapshotstore doesn't have cached snapshot information + // set WritableLayer usage to zero + if err == nil { + usedBytes = sn.Size + inodesUsed = sn.Inodes + } + cs.WritableLayer = &runtime.FilesystemUsage{ + Timestamp: sn.Timestamp, + FsId: &runtime.FilesystemIdentifier{ + Mountpoint: c.imageFSPath, + }, + UsedBytes: &runtime.UInt64Value{Value: usedBytes}, + InodesUsed: &runtime.UInt64Value{Value: inodesUsed}, + } + cs.Attributes = &runtime.ContainerAttributes{ + Id: meta.ID, + Metadata: meta.Config.GetMetadata(), + Labels: meta.Config.GetLabels(), + Annotations: meta.Config.GetAnnotations(), + } + + // TODO(windows): hcsshim Stats is not implemented, add windows support after + // that is implemented. + + return &cs, nil } diff --git a/pkg/server/helpers.go b/pkg/server/helpers.go index 8563a8921..98fe09d49 100644 --- a/pkg/server/helpers.go +++ b/pkg/server/helpers.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/BurntSushi/toml" + runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" "github.com/containerd/containerd" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/plugin" @@ -86,6 +87,9 @@ const ( // defaultIfName is the default network interface for the pods defaultIfName = "eth0" + + // runtimeRunhcsV1 is the runtime type for runhcs. + runtimeRunhcsV1 = "io.containerd.runhcs.v1" ) // makeSandboxName generates sandbox name from sandbox metadata. The name @@ -321,6 +325,8 @@ func getRuntimeOptionsType(t string) interface{} { return &runcoptions.Options{} case plugin.RuntimeLinuxV1: return &runctypes.RuncOptions{} + case runtimeRunhcsV1: + return &runhcsoptions.Options{} default: return &runtimeoptions.Options{} } diff --git a/pkg/server/io/helpers.go b/pkg/server/io/helpers.go index 4e797c9ef..a62b884a0 100644 --- a/pkg/server/io/helpers.go +++ b/pkg/server/io/helpers.go @@ -112,7 +112,7 @@ func newStdioPipes(fifos *cio.FIFOSet) (_ *stdioPipes, _ *wgCloser, err error) { }() if fifos.Stdin != "" { - if f, err = openFifo(ctx, fifos.Stdin, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { + if f, err = openPipe(ctx, fifos.Stdin, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { return nil, nil, err } p.stdin = f @@ -120,7 +120,7 @@ func newStdioPipes(fifos *cio.FIFOSet) (_ *stdioPipes, _ *wgCloser, err error) { } if fifos.Stdout != "" { - if f, err = openFifo(ctx, fifos.Stdout, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { + if f, err = openPipe(ctx, fifos.Stdout, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { return nil, nil, err } p.stdout = f @@ -128,7 +128,7 @@ func newStdioPipes(fifos *cio.FIFOSet) (_ *stdioPipes, _ *wgCloser, err error) { } if fifos.Stderr != "" { - if f, err = openFifo(ctx, fifos.Stderr, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { + if f, err = openPipe(ctx, fifos.Stderr, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { return nil, nil, err } p.stderr = f diff --git a/pkg/server/io/helpers_unix.go b/pkg/server/io/helpers_unix.go index ee5dc252c..c365a4373 100644 --- a/pkg/server/io/helpers_unix.go +++ b/pkg/server/io/helpers_unix.go @@ -26,6 +26,6 @@ import ( "golang.org/x/net/context" ) -func openFifo(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) { +func openPipe(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) { return fifo.OpenFifo(ctx, fn, flag, perm) } diff --git a/pkg/server/io/helpers_windows.go b/pkg/server/io/helpers_windows.go index 9ea6a4f2b..3a364050d 100644 --- a/pkg/server/io/helpers_windows.go +++ b/pkg/server/io/helpers_windows.go @@ -20,12 +20,62 @@ package io import ( "io" + "net" "os" + "sync" + winio "github.com/Microsoft/go-winio" + "github.com/pkg/errors" "golang.org/x/net/context" ) -// TODO(windows): Add windows FIFO support. -func openFifo(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) { - return nil, nil +type pipe struct { + l net.Listener + con net.Conn + conErr error + conWg sync.WaitGroup +} + +func openPipe(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) { + l, err := winio.ListenPipe(fn, nil) + if err != nil { + return nil, err + } + p := &pipe{l: l} + p.conWg.Add(1) + go func() { + defer p.conWg.Done() + c, err := l.Accept() + if err != nil { + p.conErr = err + return + } + p.con = c + }() + return p, nil +} + +func (p *pipe) Write(b []byte) (int, error) { + p.conWg.Wait() + if p.conErr != nil { + return 0, errors.Wrap(p.conErr, "connection error") + } + return p.con.Write(b) +} + +func (p *pipe) Read(b []byte) (int, error) { + p.conWg.Wait() + if p.conErr != nil { + return 0, errors.Wrap(p.conErr, "connection error") + } + return p.con.Read(b) +} + +func (p *pipe) Close() error { + p.l.Close() + p.conWg.Wait() + if p.con != nil { + return p.con.Close() + } + return p.conErr } diff --git a/pkg/server/sandbox_run_test.go b/pkg/server/sandbox_run_test.go index c40ca3ed4..c937d8e06 100644 --- a/pkg/server/sandbox_run_test.go +++ b/pkg/server/sandbox_run_test.go @@ -21,13 +21,140 @@ import ( "testing" cni "github.com/containerd/go-cni" + "github.com/containerd/typeurl" + imagespec "github.com/opencontainers/image-spec/specs-go/v1" + runtimespec "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "github.com/containerd/cri/pkg/annotations" criconfig "github.com/containerd/cri/pkg/config" + sandboxstore "github.com/containerd/cri/pkg/store/sandbox" ) +func TestSandboxContainerSpec(t *testing.T) { + testID := "test-id" + nsPath := "test-cni" + for desc, test := range map[string]struct { + configChange func(*runtime.PodSandboxConfig) + podAnnotations []string + imageConfigChange func(*imagespec.ImageConfig) + specCheck func(*testing.T, *runtimespec.Spec) + expectErr bool + }{ + "should return error when entrypoint and cmd are empty": { + imageConfigChange: func(c *imagespec.ImageConfig) { + c.Entrypoint = nil + c.Cmd = nil + }, + expectErr: true, + }, + "a passthrough annotation should be passed as an OCI annotation": { + podAnnotations: []string{"c"}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + assert.Equal(t, spec.Annotations["c"], "d") + }, + }, + "a non-passthrough annotation should not be passed as an OCI annotation": { + configChange: func(c *runtime.PodSandboxConfig) { + c.Annotations["d"] = "e" + }, + podAnnotations: []string{"c"}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + assert.Equal(t, spec.Annotations["c"], "d") + _, ok := spec.Annotations["d"] + assert.False(t, ok) + }, + }, + "passthrough annotations should support wildcard match": { + configChange: func(c *runtime.PodSandboxConfig) { + c.Annotations["t.f"] = "j" + c.Annotations["z.g"] = "o" + c.Annotations["z"] = "o" + c.Annotations["y.ca"] = "b" + c.Annotations["y"] = "b" + }, + podAnnotations: []string{"t*", "z.*", "y.c*"}, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + assert.Equal(t, spec.Annotations["t.f"], "j") + assert.Equal(t, spec.Annotations["z.g"], "o") + assert.Equal(t, spec.Annotations["y.ca"], "b") + _, ok := spec.Annotations["y"] + assert.False(t, ok) + _, ok = spec.Annotations["z"] + assert.False(t, ok) + }, + }, + } { + t.Logf("TestCase %q", desc) + c := newTestCRIService() + config, imageConfig, specCheck := getRunPodSandboxTestData() + if test.configChange != nil { + test.configChange(config) + } + + if test.imageConfigChange != nil { + test.imageConfigChange(imageConfig) + } + spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, + test.podAnnotations) + if test.expectErr { + assert.Error(t, err) + assert.Nil(t, spec) + continue + } + assert.NoError(t, err) + assert.NotNil(t, spec) + specCheck(t, testID, spec) + if test.specCheck != nil { + test.specCheck(t, spec) + } + } +} + +func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) { + for desc, test := range map[string]struct { + configChange func(*runtime.PodSandboxConfig) + }{ + "should marshal original config": {}, + "should marshal Linux": { + configChange: func(c *runtime.PodSandboxConfig) { + if c.Linux == nil { + c.Linux = &runtime.LinuxPodSandboxConfig{} + } + c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ + NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_NODE, + Pid: runtime.NamespaceMode_NODE, + Ipc: runtime.NamespaceMode_NODE, + }, + SupplementalGroups: []int64{1111, 2222}, + } + }, + }, + } { + t.Logf("TestCase %q", desc) + meta := &sandboxstore.Metadata{ + ID: "1", + Name: "sandbox_1", + NetNSPath: "/home/cloud", + } + meta.Config, _, _ = getRunPodSandboxTestData() + if test.configChange != nil { + test.configChange(meta.Config) + } + + any, err := typeurl.MarshalAny(meta) + assert.NoError(t, err) + data, err := typeurl.UnmarshalAny(any) + assert.NoError(t, err) + assert.IsType(t, &sandboxstore.Metadata{}, data) + curMeta, ok := data.(*sandboxstore.Metadata) + assert.True(t, ok) + assert.Equal(t, meta, curMeta) + } +} + func TestToCNIPortMappings(t *testing.T) { for desc, test := range map[string]struct { criPortMappings []*runtime.PortMapping diff --git a/pkg/server/sandbox_run_unix_test.go b/pkg/server/sandbox_run_unix_test.go index 151850564..f3866aec7 100644 --- a/pkg/server/sandbox_run_unix_test.go +++ b/pkg/server/sandbox_run_unix_test.go @@ -23,7 +23,6 @@ import ( "path/filepath" "testing" - "github.com/containerd/typeurl" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtimespec "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" @@ -33,7 +32,6 @@ import ( "github.com/containerd/cri/pkg/annotations" "github.com/containerd/cri/pkg/containerd/opts" ostesting "github.com/containerd/cri/pkg/os/testing" - sandboxstore "github.com/containerd/cri/pkg/store/sandbox" ) func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) { @@ -82,15 +80,13 @@ func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConf return config, imageConfig, specCheck } -func TestSandboxContainerSpec(t *testing.T) { +func TestLinuxSandboxContainerSpec(t *testing.T) { testID := "test-id" nsPath := "test-cni" for desc, test := range map[string]struct { - configChange func(*runtime.PodSandboxConfig) - podAnnotations []string - imageConfigChange func(*imagespec.ImageConfig) - specCheck func(*testing.T, *runtimespec.Spec) - expectErr bool + configChange func(*runtime.PodSandboxConfig) + specCheck func(*testing.T, *runtimespec.Spec) + expectErr bool }{ "spec should reflect original config": { specCheck: func(t *testing.T, spec *runtimespec.Spec) { @@ -138,13 +134,6 @@ func TestSandboxContainerSpec(t *testing.T) { }) }, }, - "should return error when entrypoint and cmd are empty": { - imageConfigChange: func(c *imagespec.ImageConfig) { - c.Entrypoint = nil - c.Cmd = nil - }, - expectErr: true, - }, "should set supplemental groups correctly": { configChange: func(c *runtime.PodSandboxConfig) { c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ @@ -157,42 +146,6 @@ func TestSandboxContainerSpec(t *testing.T) { assert.Contains(t, spec.Process.User.AdditionalGids, uint32(2222)) }, }, - "a passthrough annotation should be passed as an OCI annotation": { - podAnnotations: []string{"c"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, spec.Annotations["c"], "d") - }, - }, - "a non-passthrough annotation should not be passed as an OCI annotation": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Annotations["d"] = "e" - }, - podAnnotations: []string{"c"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, spec.Annotations["c"], "d") - _, ok := spec.Annotations["d"] - assert.False(t, ok) - }, - }, - "passthrough annotations should support wildcard match": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Annotations["t.f"] = "j" - c.Annotations["z.g"] = "o" - c.Annotations["z"] = "o" - c.Annotations["y.ca"] = "b" - c.Annotations["y"] = "b" - }, - podAnnotations: []string{"t*", "z.*", "y.c*"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, spec.Annotations["t.f"], "j") - assert.Equal(t, spec.Annotations["z.g"], "o") - assert.Equal(t, spec.Annotations["y.ca"], "b") - _, ok := spec.Annotations["y"] - assert.False(t, ok) - _, ok = spec.Annotations["z"] - assert.False(t, ok) - }, - }, } { t.Logf("TestCase %q", desc) c := newTestCRIService() @@ -200,12 +153,7 @@ func TestSandboxContainerSpec(t *testing.T) { if test.configChange != nil { test.configChange(config) } - - if test.imageConfigChange != nil { - test.imageConfigChange(imageConfig) - } - spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, - test.podAnnotations) + spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil) if test.expectErr { assert.Error(t, err) assert.Nil(t, spec) @@ -460,47 +408,6 @@ options timeout:1 } } -// TODO(windows): Move this to sandbox_run_test.go -func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) { - for desc, test := range map[string]struct { - configChange func(*runtime.PodSandboxConfig) - }{ - "should marshal original config": {}, - "should marshal Linux": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ - NamespaceOptions: &runtime.NamespaceOption{ - Network: runtime.NamespaceMode_NODE, - Pid: runtime.NamespaceMode_NODE, - Ipc: runtime.NamespaceMode_NODE, - }, - SupplementalGroups: []int64{1111, 2222}, - } - }, - }, - } { - t.Logf("TestCase %q", desc) - meta := &sandboxstore.Metadata{ - ID: "1", - Name: "sandbox_1", - NetNSPath: "/home/cloud", - } - meta.Config, _, _ = getRunPodSandboxTestData() - if test.configChange != nil { - test.configChange(meta.Config) - } - - any, err := typeurl.MarshalAny(meta) - assert.NoError(t, err) - data, err := typeurl.UnmarshalAny(any) - assert.NoError(t, err) - assert.IsType(t, &sandboxstore.Metadata{}, data) - curMeta, ok := data.(*sandboxstore.Metadata) - assert.True(t, ok) - assert.Equal(t, meta, curMeta) - } -} - func TestSandboxDisableCgroup(t *testing.T) { config, imageConfig, _ := getRunPodSandboxTestData() c := newTestCRIService() diff --git a/pkg/server/sandbox_run_windows.go b/pkg/server/sandbox_run_windows.go index c265e1941..216e75dee 100644 --- a/pkg/server/sandbox_run_windows.go +++ b/pkg/server/sandbox_run_windows.go @@ -20,18 +20,53 @@ package server import ( "github.com/containerd/containerd" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/oci" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + + "github.com/containerd/cri/pkg/annotations" + customopts "github.com/containerd/cri/pkg/containerd/opts" ) -// TODO(windows): Add windows support. // TODO(windows): Configure windows sandbox shares func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (*runtimespec.Spec, error) { - return nil, errdefs.ErrNotImplemented + // Creates a spec Generator with the default spec. + specOpts := []oci.SpecOpts{ + oci.WithEnv(imageConfig.Env), + oci.WithHostname(config.GetHostname()), + } + if imageConfig.WorkingDir != "" { + specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir)) + } + + if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 { + // Pause image must have entrypoint or cmd. + return nil, errors.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig) + } + specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...)) + + specOpts = append(specOpts, + // Clear the root location since hcsshim expects it. + // NOTE: readonly rootfs doesn't work on windows. + customopts.WithoutRoot, + customopts.WithWindowsNetworkNamespace(nsPath), + ) + + for pKey, pValue := range getPassthroughAnnotations(config.Annotations, + runtimePodAnnotations) { + specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue)) + } + + specOpts = append(specOpts, + customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox), + customopts.WithAnnotation(annotations.SandboxID, id), + customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()), + ) + + return runtimeSpec(id, specOpts...) } // No sandbox container spec options for windows yet. diff --git a/pkg/server/sandbox_run_windows_test.go b/pkg/server/sandbox_run_windows_test.go new file mode 100644 index 000000000..92f7f2d2a --- /dev/null +++ b/pkg/server/sandbox_run_windows_test.go @@ -0,0 +1,84 @@ +// +build windows + +/* +Copyright The containerd Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "testing" + + imagespec "github.com/opencontainers/image-spec/specs-go/v1" + runtimespec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + + "github.com/containerd/cri/pkg/annotations" +) + +func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) { + config := &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: "test-name", + Uid: "test-uid", + Namespace: "test-ns", + Attempt: 1, + }, + Hostname: "test-hostname", + LogDirectory: "test-log-directory", + Labels: map[string]string{"a": "b"}, + Annotations: map[string]string{"c": "d"}, + } + imageConfig := &imagespec.ImageConfig{ + Env: []string{"a=b", "c=d"}, + Entrypoint: []string{"/pause"}, + Cmd: []string{"forever"}, + WorkingDir: "/workspace", + } + specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) { + assert.Equal(t, "test-hostname", spec.Hostname) + assert.Nil(t, spec.Root) + assert.Contains(t, spec.Process.Env, "a=b", "c=d") + assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args) + assert.Equal(t, "/workspace", spec.Process.Cwd) + + t.Logf("Check PodSandbox annotations") + assert.Contains(t, spec.Annotations, annotations.SandboxID) + assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id) + + assert.Contains(t, spec.Annotations, annotations.ContainerType) + assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox) + + assert.Contains(t, spec.Annotations, annotations.SandboxLogDir) + assert.EqualValues(t, spec.Annotations[annotations.SandboxLogDir], "test-log-directory") + } + return config, imageConfig, specCheck +} + +func TestSandboxWindowsNetworkNamespace(t *testing.T) { + testID := "test-id" + nsPath := "test-cni" + c := newTestCRIService() + + config, imageConfig, specCheck := getRunPodSandboxTestData() + spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil) + assert.NoError(t, err) + assert.NotNil(t, spec) + specCheck(t, testID, spec) + assert.NotNil(t, spec.Windows) + assert.NotNil(t, spec.Windows.Network) + assert.Equal(t, nsPath, spec.Windows.Network.NetworkNamespace) +} diff --git a/pkg/server/service_windows.go b/pkg/server/service_windows.go index edeb91054..7b1c35f2f 100644 --- a/pkg/server/service_windows.go +++ b/pkg/server/service_windows.go @@ -20,16 +20,39 @@ package server import ( cni "github.com/containerd/go-cni" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) +// windowsNetworkAttachCount is the minimum number of networks the PodSandbox +// attaches to +const windowsNetworkAttachCount = 1 + // initPlatform handles linux specific initialization for the CRI service. -// TODO(windows): Initialize CRI plugin for windows func (c *criService) initPlatform() error { + var err error + // For windows, the loopback network is added as default. + // There is no need to explicitly add one hence networkAttachCount is 1. + // If there are more network configs the pod will be attached to all the + // networks but we will only use the ip of the default network interface + // as the pod IP. + c.netPlugin, err = cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount), + cni.WithPluginConfDir(c.config.NetworkPluginConfDir), + cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum), + cni.WithPluginDir([]string{c.config.NetworkPluginBinDir})) + if err != nil { + return errors.Wrap(err, "failed to initialize cni") + } + + // Try to load the config if it exists. Just log the error if load fails + // This is not disruptive for containerd to panic + if err := c.netPlugin.Load(c.cniLoadOptions()...); err != nil { + logrus.WithError(err).Error("Failed to load cni during init, please check CRI plugin status before setting up network for pods") + } return nil } // cniLoadOptions returns cni load options for the windows. -// TODO(windows): Implement CNI options for windows. func (c *criService) cniLoadOptions() []cni.CNIOpt { - return nil + return []cni.CNIOpt{cni.WithDefaultConf} }