Merge pull request #5163 from claudiubelu/test-windows

tests: Adds support for Windows cri-integration tests
This commit is contained in:
Phil Estes 2021-05-17 09:24:31 -04:00 committed by GitHub
commit a152049d7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 153 additions and 81 deletions

View File

@ -194,7 +194,7 @@ bin/cri-integration.test:
cri-integration: binaries bin/cri-integration.test ## run cri integration tests
@echo "$(WHALE) $@"
@./script/test/cri-integration.sh
@bash -x ./script/test/cri-integration.sh
@rm -rf bin/cri-integration.test
benchmark: ## run benchmarks tests

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -35,6 +33,9 @@ func TestContainerRestart(t *testing.T) {
assert.NoError(t, runtimeService.StopPodSandbox(sb))
assert.NoError(t, runtimeService.RemovePodSandbox(sb))
}()
EnsureImageExists(t, pauseImage)
t.Logf("Create a container config and run container in a pod")
containerConfig := ContainerConfig(
"container1",

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -40,6 +38,9 @@ func TestContainerStats(t *testing.T) {
assert.NoError(t, runtimeService.StopPodSandbox(sb))
assert.NoError(t, runtimeService.RemovePodSandbox(sb))
}()
EnsureImageExists(t, pauseImage)
t.Logf("Create a container config and run container in a pod")
containerConfig := ContainerConfig(
"container1",
@ -64,8 +65,7 @@ func TestContainerStats(t *testing.T) {
if err != nil {
return false, err
}
if s.GetWritableLayer().GetUsedBytes().GetValue() != 0 &&
s.GetWritableLayer().GetInodesUsed().GetValue() != 0 {
if s.GetWritableLayer().GetUsedBytes().GetValue() != 0 {
return true, nil
}
return false, nil
@ -162,6 +162,9 @@ func TestContainerListStats(t *testing.T) {
assert.NoError(t, runtimeService.StopPodSandbox(sb))
assert.NoError(t, runtimeService.RemovePodSandbox(sb))
}()
EnsureImageExists(t, pauseImage)
t.Logf("Create a container config and run containers in a pod")
containerConfigMap := make(map[string]*runtime.ContainerConfig)
for i := 0; i < 3; i++ {
@ -192,8 +195,7 @@ func TestContainerListStats(t *testing.T) {
return false, err
}
for _, s := range stats {
if s.GetWritableLayer().GetUsedBytes().GetValue() == 0 &&
s.GetWritableLayer().GetInodesUsed().GetValue() == 0 {
if s.GetWritableLayer().GetUsedBytes().GetValue() == 0 {
return false, nil
}
}
@ -217,6 +219,9 @@ func TestContainerListStatsWithIdFilter(t *testing.T) {
assert.NoError(t, runtimeService.StopPodSandbox(sb))
assert.NoError(t, runtimeService.RemovePodSandbox(sb))
}()
EnsureImageExists(t, pauseImage)
t.Logf("Create a container config and run containers in a pod")
containerConfigMap := make(map[string]*runtime.ContainerConfig)
for i := 0; i < 3; i++ {
@ -251,8 +256,7 @@ func TestContainerListStatsWithIdFilter(t *testing.T) {
if len(stats) != 1 {
return false, errors.New("unexpected stats length")
}
if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 &&
stats[0].GetWritableLayer().GetInodesUsed().GetValue() != 0 {
if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 {
return true, nil
}
return false, nil
@ -277,6 +281,9 @@ func TestContainerListStatsWithSandboxIdFilter(t *testing.T) {
assert.NoError(t, runtimeService.StopPodSandbox(sb))
assert.NoError(t, runtimeService.RemovePodSandbox(sb))
}()
EnsureImageExists(t, pauseImage)
t.Logf("Create a container config and run containers in a pod")
containerConfigMap := make(map[string]*runtime.ContainerConfig)
for i := 0; i < 3; i++ {
@ -310,12 +317,12 @@ func TestContainerListStatsWithSandboxIdFilter(t *testing.T) {
if len(stats) != 3 {
return false, errors.New("unexpected stats length")
}
if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 &&
stats[0].GetWritableLayer().GetInodesUsed().GetValue() != 0 {
if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 {
return true, nil
}
return false, nil
}, time.Second, 30*time.Second))
}, time.Second, 45*time.Second))
// TODO(claudiub): Reduce the timer above to 30 seconds once Windows flakiness has been addressed.
t.Logf("Verify container stats for sandbox %q", sb)
for _, s := range stats {
testStats(t, s, containerConfigMap[s.GetAttributes().GetId()])
@ -333,6 +340,9 @@ func TestContainerListStatsWithIdSandboxIdFilter(t *testing.T) {
assert.NoError(t, runtimeService.StopPodSandbox(sb))
assert.NoError(t, runtimeService.RemovePodSandbox(sb))
}()
EnsureImageExists(t, pauseImage)
t.Logf("Create container config and run containers in a pod")
containerConfigMap := make(map[string]*runtime.ContainerConfig)
for i := 0; i < 3; i++ {
@ -366,8 +376,7 @@ func TestContainerListStatsWithIdSandboxIdFilter(t *testing.T) {
if len(stats) != 1 {
return false, errors.New("unexpected stats length")
}
if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 &&
stats[0].GetWritableLayer().GetInodesUsed().GetValue() != 0 {
if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 {
return true, nil
}
return false, nil
@ -389,8 +398,7 @@ func TestContainerListStatsWithIdSandboxIdFilter(t *testing.T) {
if len(stats) != 1 {
return false, errors.New("unexpected stats length")
}
if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 &&
stats[0].GetWritableLayer().GetInodesUsed().GetValue() != 0 {
if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 {
return true, nil
}
return false, nil
@ -421,5 +429,9 @@ func testStats(t *testing.T,
require.NotEmpty(t, s.GetWritableLayer().GetTimestamp())
require.NotEmpty(t, s.GetWritableLayer().GetFsId().GetMountpoint())
require.NotEmpty(t, s.GetWritableLayer().GetUsedBytes().GetValue())
// Windows does not collect inodes stats.
if goruntime.GOOS != "windows" {
require.NotEmpty(t, s.GetWritableLayer().GetInodesUsed().GetValue())
}
}

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -20,6 +18,7 @@ package integration
import (
"context"
goruntime "runtime"
"testing"
"time"
@ -73,6 +72,9 @@ func TestSharedPidMultiProcessContainerStop(t *testing.T) {
}
func TestContainerStopCancellation(t *testing.T) {
if goruntime.GOOS == "windows" {
t.Skip("Skipped on Windows.")
}
t.Log("Create a pod sandbox")
sbConfig := PodSandboxConfig("sandbox", "cancel-container-stop")
sb, err := runtimeService.RunPodSandbox(sbConfig, *runtimeHandler)

View File

@ -39,6 +39,7 @@ func checkMemoryLimit(t *testing.T, spec *runtimespec.Spec, memLimit int64) {
}
func TestUpdateContainerResources(t *testing.T) {
// TODO(claudiub): Make this test work once https://github.com/microsoft/hcsshim/pull/931 merges.
t.Log("Create a sandbox")
sbConfig := PodSandboxConfig("sandbox", "update-container-resources")
sb, err := runtimeService.RunPodSandbox(sbConfig, *runtimeHandler)
@ -48,6 +49,8 @@ func TestUpdateContainerResources(t *testing.T) {
assert.NoError(t, runtimeService.RemovePodSandbox(sb))
}()
EnsureImageExists(t, pauseImage)
t.Log("Create a container with memory limit")
cnConfig := ContainerConfig(
"container",

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -135,7 +133,7 @@ func TestContainerdImage(t *testing.T) {
cnConfig := ContainerConfig(
"test-container",
id,
WithCommand("top"),
WithCommand("sleep", "300"),
)
cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
require.NoError(t, err)

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -39,6 +37,8 @@ func TestDuplicateName(t *testing.T) {
_, err = runtimeService.RunPodSandbox(sbConfig, *runtimeHandler)
require.Error(t, err)
EnsureImageExists(t, pauseImage)
t.Logf("Create a container")
cnConfig := ContainerConfig(
"container",

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -22,6 +20,7 @@ import (
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
@ -41,11 +40,12 @@ func TestImageLoad(t *testing.T) {
t.Logf("docker save image into tarball")
output, err := exec.Command("docker", "pull", testImage).CombinedOutput()
require.NoError(t, err, "output: %q", output)
tarF, err := ioutil.TempFile("", "image-load")
tar := tarF.Name()
// ioutil.TempFile also opens a file, which might prevent us from overwriting that file with docker save.
tarDir, err := ioutil.TempDir("", "image-load")
tar := filepath.Join(tarDir, "image.tar")
require.NoError(t, err)
defer func() {
assert.NoError(t, os.RemoveAll(tar))
assert.NoError(t, os.RemoveAll(tarDir))
}()
output, err = exec.Command("docker", "save", testImage, "-o", tar).CombinedOutput()
require.NoError(t, err, "output: %q", output)

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -61,7 +59,6 @@ func TestImageFSInfo(t *testing.T) {
info = stats[0]
if info.GetTimestamp() != 0 &&
info.GetUsedBytes().GetValue() != 0 &&
info.GetInodesUsed().GetValue() != 0 &&
info.GetFsId().GetMountpoint() != "" {
return true, nil
}

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -97,6 +95,7 @@ func ConnectDaemons() error {
}
// containerdEndpoint is the same with criEndpoint now
containerdEndpoint = strings.TrimPrefix(*criEndpoint, "unix://")
containerdEndpoint = strings.TrimPrefix(containerdEndpoint, "npipe:")
containerdClient, err = containerd.New(containerdEndpoint, containerd.WithDefaultNamespace(k8sNamespace))
if err != nil {
return errors.Wrap(err, "failed to connect containerd")
@ -198,7 +197,7 @@ func WithTestAnnotations() ContainerOpts {
}
// Add container resource limits.
func WithResources(r *runtime.LinuxContainerResources) ContainerOpts {
func WithResources(r *runtime.LinuxContainerResources) ContainerOpts { //nolint:unused
return func(c *runtime.ContainerConfig) {
if c.Linux == nil {
c.Linux = &runtime.LinuxContainerConfig{}
@ -240,7 +239,7 @@ func WithLogPath(path string) ContainerOpts {
}
// WithSupplementalGroups adds supplemental groups.
func WithSupplementalGroups(gids []int64) ContainerOpts {
func WithSupplementalGroups(gids []int64) ContainerOpts { //nolint:unused
return func(c *runtime.ContainerConfig) {
if c.Linux == nil {
c.Linux = &runtime.LinuxContainerConfig{}
@ -325,7 +324,7 @@ func KillProcess(name string) error {
}
// KillPid kills the process by pid. kill is used.
func KillPid(pid int) error {
func KillPid(pid int) error { //nolint:unused
output, err := exec.Command("kill", strconv.Itoa(pid)).CombinedOutput()
if err != nil {
return errors.Errorf("failed to kill %d - error: %v, output: %q", pid, err, output)
@ -379,7 +378,7 @@ func CRIConfig() (*criconfig.Config, error) {
}
// SandboxInfo gets sandbox info.
func SandboxInfo(id string) (*runtime.PodSandboxStatus, *server.SandboxInfo, error) {
func SandboxInfo(id string) (*runtime.PodSandboxStatus, *server.SandboxInfo, error) { //nolint:unused
client, err := RawRuntimeClient()
if err != nil {
return nil, nil, errors.Wrap(err, "failed to get raw runtime client")

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -24,6 +22,7 @@ import (
"os"
"path/filepath"
"regexp"
goruntime "runtime"
"testing"
"time"
@ -54,10 +53,16 @@ func TestPodDualStack(t *testing.T) {
EnsureImageExists(t, testImage)
t.Log("Create a container to print env")
var command ContainerOpts
if goruntime.GOOS == "windows" {
command = WithCommand("ipconfig")
} else {
command = WithCommand("ip", "address", "show", "dev", "eth0")
}
cnConfig := ContainerConfig(
containerName,
testImage,
WithCommand("ip", "address", "show", "dev", "eth0"),
command,
WithLogPath(containerName),
)
cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
@ -85,9 +90,18 @@ func TestPodDualStack(t *testing.T) {
ip := status.GetNetwork().GetIp()
additionalIps := status.GetNetwork().GetAdditionalIps()
ipv4Enabled, err := regexp.MatchString("inet .* scope global", string(content))
var ipv4Regex, ipv6Regex string
if goruntime.GOOS == "windows" {
ipv4Regex = "^\\s*IPv4 Address"
ipv6Regex = "^\\s*IPv6 Address"
} else {
ipv4Regex = "inet .* scope global"
ipv6Regex = "inet6 .* scope global"
}
ipv4Enabled, err := regexp.MatchString(ipv4Regex, string(content))
assert.NoError(t, err)
ipv6Enabled, err := regexp.MatchString("inet6 .* scope global", string(content))
ipv6Enabled, err := regexp.MatchString(ipv6Regex, string(content))
assert.NoError(t, err)
if ipv4Enabled && ipv6Enabled {

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -22,6 +20,8 @@ import (
"io/ioutil"
"os"
"path/filepath"
goruntime "runtime"
"strings"
"testing"
"time"
@ -37,6 +37,7 @@ func TestPodHostname(t *testing.T) {
opts []PodSandboxOpts
expectedHostname string
expectErr bool
needsHostNetwork bool
}{
"regular pod with custom hostname": {
opts: []PodSandboxOpts{
@ -49,6 +50,7 @@ func TestPodHostname(t *testing.T) {
WithHostNetwork,
},
expectedHostname: hostname,
needsHostNetwork: true,
},
"host network pod with custom hostname should fail": {
opts: []PodSandboxOpts{
@ -56,9 +58,13 @@ func TestPodHostname(t *testing.T) {
WithPodHostname("test-hostname"),
},
expectErr: true,
needsHostNetwork: true,
},
} {
t.Run(name, func(t *testing.T) {
if test.needsHostNetwork && goruntime.GOOS == "windows" {
t.Skip("Skipped on Windows.")
}
testPodLogDir, err := ioutil.TempDir("/tmp", "hostname")
require.NoError(t, err)
defer os.RemoveAll(testPodLogDir)
@ -94,7 +100,7 @@ func TestPodHostname(t *testing.T) {
containerName,
testImage,
WithCommand("sh", "-c",
"echo -n /etc/hostname= && cat /etc/hostname && env"),
"echo -n /etc/hostname= && hostname && env"),
WithLogPath(containerName),
)
cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
@ -119,7 +125,11 @@ func TestPodHostname(t *testing.T) {
assert.NoError(t, err)
t.Log("Search hostname env in container log")
if goruntime.GOOS == "windows" {
assert.Contains(t, string(content), "COMPUTERNAME="+strings.ToUpper(test.expectedHostname))
} else {
assert.Contains(t, string(content), "HOSTNAME="+test.expectedHostname)
}
t.Log("Search /etc/hostname content in container log")
assert.Contains(t, string(content), "/etc/hostname="+test.expectedHostname)

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -19,6 +17,7 @@
package integration
import (
goruntime "runtime"
"sort"
"testing"
@ -33,6 +32,9 @@ import (
// Restart test must run sequentially.
func TestContainerdRestart(t *testing.T) {
if goruntime.GOOS == "windows" {
t.Skip("Skipped on Windows.")
}
type container struct {
name string
id string
@ -100,6 +102,9 @@ func TestContainerdRestart(t *testing.T) {
runtimeService.StopPodSandbox(sid)
runtimeService.RemovePodSandbox(sid)
}()
EnsureImageExists(t, pauseImage)
s.id = sid
for j := range s.containers {
c := &s.containers[j]

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -19,6 +17,7 @@
package integration
import (
goruntime "runtime"
"testing"
"github.com/stretchr/testify/assert"
@ -82,7 +81,7 @@ func TestTruncIndex(t *testing.T) {
cnConfig := ContainerConfig(
"containerTruncIndex",
appImage,
WithCommand("top"),
WithCommand("sleep", "300"),
)
cn, err := runtimeService.CreateContainer(sbTruncIndex, cnConfig, sbConfig)
require.NoError(t, err)
@ -112,11 +111,15 @@ func TestTruncIndex(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, cn, cStats.Attributes.Id)
if goruntime.GOOS != "windows" {
// TODO(claudiub): remove this when UpdateContainerResources works on running Windows Containers.
// https://github.com/containerd/containerd/issues/5187
t.Logf("Update container memory limit after started")
err = runtimeService.UpdateContainerResources(cnTruncIndex, &runtimeapi.LinuxContainerResources{
MemoryLimitInBytes: 50 * 1024 * 1024,
})
assert.NoError(t, err)
}
t.Logf("Execute cmd in container")
execReq := &runtimeapi.ExecRequest{

View File

@ -1,5 +1,3 @@
// +build linux
/*
Copyright The containerd Authors.
@ -21,6 +19,7 @@ package integration
import (
"fmt"
"os/exec"
goruntime "runtime"
"testing"
"time"
@ -29,6 +28,11 @@ import (
)
func TestVolumeCopyUp(t *testing.T) {
if goruntime.GOOS == "windows" {
// TODO(claudiub): Remove this when the volume-copy-up image has Windows support.
// https://github.com/containerd/containerd/pull/5162
t.Skip("Skipped on Windows.")
}
var (
testImage = GetImage(VolumeCopyUp)
execTimeout = time.Minute
@ -89,6 +93,9 @@ func TestVolumeCopyUp(t *testing.T) {
}
func TestVolumeOwnership(t *testing.T) {
if goruntime.GOOS == "windows" {
t.Skip("Skipped on Windows.")
}
var (
testImage = GetImage(VolumeOwnership)
execTimeout = time.Minute

View File

@ -33,7 +33,7 @@ mkdir -p ${REPORT_DIR}
test_setup ${REPORT_DIR}
# Run integration test.
sudo PATH=${PATH} bin/cri-integration.test --test.run="${FOCUS}" --test.v \
${sudo} bin/cri-integration.test --test.run="${FOCUS}" --test.v \
--cri-endpoint=${CONTAINERD_SOCK} \
--cri-root=${CRI_ROOT} \
--runtime-handler=${RUNTIME} \

View File

@ -16,6 +16,11 @@
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/../..
IS_WINDOWS=0
if [ -v "OS" ] && [ "${OS}" == "Windows_NT" ]; then
IS_WINDOWS=1
fi
# RESTART_WAIT_PERIOD is the period to wait before restarting containerd.
RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10}
# CONTAINERD_FLAGS contains all containerd flags.
@ -52,17 +57,35 @@ CONTAINERD_ROOT=${CONTAINERD_ROOT:-"/var/lib/containerd${CONTAINERD_TEST_SUFFIX}
# The containerd state directory.
CONTAINERD_STATE=${CONTAINERD_STATE:-"/run/containerd${CONTAINERD_TEST_SUFFIX}"}
# The containerd socket address.
if [ $IS_WINDOWS -eq 0 ]; then
CONTAINERD_SOCK=${CONTAINERD_SOCK:-unix://${CONTAINERD_STATE}/containerd.sock}
TRIMMED_CONTAINERD_SOCK="${CONTAINERD_SOCK#unix://}"
else
CONTAINERD_SOCK=${CONTAINERD_SOCK:-npipe://./pipe/${CONTAINERD_STATE}/containerd}
TRIMMED_CONTAINERD_SOCK="${CONTAINERD_SOCK#npipe:}"
fi
# The containerd binary name.
CONTAINERD_BIN=${CONTAINERD_BIN:-"containerd"} # don't need a suffix now
EXE_SUFFIX=""
if [ $IS_WINDOWS -eq 1 ]; then
EXE_SUFFIX=".exe"
fi
CONTAINERD_BIN=${CONTAINERD_BIN:-"containerd"}${EXE_SUFFIX}
if [ -f "${CONTAINERD_CONFIG_FILE}" ]; then
CONTAINERD_FLAGS+="--config ${CONTAINERD_CONFIG_FILE} "
fi
CONTAINERD_FLAGS+="--address ${CONTAINERD_SOCK#"unix://"} \
CONTAINERD_FLAGS+="--address ${TRIMMED_CONTAINERD_SOCK} \
--state ${CONTAINERD_STATE} \
--root ${CONTAINERD_ROOT}"
containerd_groupid=
kill_containerd_cmd=
# NOTE: We don't have the sudo command on Windows.
sudo=""
if [ $(id -u) -ne 0 ] && command -v sudo &> /dev/null; then
sudo="sudo PATH=${PATH}"
fi
# test_setup starts containerd.
test_setup() {
@ -75,11 +98,17 @@ test_setup() {
set -m
# Create containerd in a different process group
# so that we can easily clean them up.
keepalive "sudo PATH=${PATH} bin/containerd ${CONTAINERD_FLAGS}" \
keepalive "${sudo} bin/containerd ${CONTAINERD_FLAGS}" \
${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log &
pid=$!
set +m
containerd_groupid=$(ps -o pgid= -p ${pid})
if [ $IS_WINDOWS -eq 1 ]; then
kill_containerd_cmd="${sudo} kill ${pid}"
else
kill_containerd_cmd="${sudo} pkill -g $(ps -o pgid= -p ${pid})"
fi
# Wait for containerd to be running by using the containerd client ctr to check the version
# of the containerd server. Wait an increasing amount of time after each of five attempts
local -r crictl_path=$(which crictl)
@ -87,14 +116,14 @@ test_setup() {
echo "crictl is not in PATH"
exit 1
fi
readiness_check "sudo bin/ctr --address ${CONTAINERD_SOCK#"unix://"} version"
readiness_check "sudo ${crictl_path} --runtime-endpoint=${CONTAINERD_SOCK} info"
readiness_check "${sudo} bin/ctr --address ${TRIMMED_CONTAINERD_SOCK} version"
readiness_check "${sudo} ${crictl_path} --runtime-endpoint=${CONTAINERD_SOCK} info"
}
# test_teardown kills containerd.
test_teardown() {
if [ -n "${containerd_groupid}" ]; then
sudo pkill -g ${containerd_groupid}
if [ -n "${kill_containerd_cmd}" ]; then
${kill_containerd_cmd}
fi
}