github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add integration test framework

Browse Source 
Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu
2017-09-18 07:58:17 +00:00
parent 491400c892
commit 8fb57da18e
8 changed files with 268 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@@ -61,9 +61,12 @@ jobs:
script:
- make install.deps
- make test
- make test-integration
- make test-cri
after_script:
# Abuse travis to preserve the log.
- cat /tmp/test-integration/cri-containerd.log
- cat /tmp/test-integration/containerd.log
- cat /tmp/test-cri/cri-containerd.log
- cat /tmp/test-cri/containerd.log
go: 1.8.x

39
Makefile
View File

@@ -26,7 +26,8 @@ VERSION := $(VERSION:v%=%)
TARBALL := cri-containerd-$(VERSION).tar.gz
BUILD_TAGS := apparmor
GO_LDFLAGS := -X $(PROJECT)/pkg/version.criContainerdVersion=$(VERSION)
SOURCES := $(shell find . -name '*.go')
SOURCES := $(shell find cmd/ pkg/ vendor/ -name '*.go')
INTEGRATION_SOURCES := $(shell find integration/ -name '*.go')
all: binaries
@@ -35,20 +36,21 @@ default: help
help:
@echo "Usage: make <target>"
@echo
@echo " * 'install' - Install binaries to system locations"
@echo " * 'binaries' - Build cri-containerd"
@echo " * 'static-binaries - Build static cri-containerd"
@echo " * 'release' - Build release tarball"
@echo " * 'push' - Push release tarball to GCS"
@echo " * 'test' - Test cri-containerd"
@echo " * 'test-cri' - Test cri-containerd with cri validation test"
@echo " * 'test-e2e-node' - Test cri-containerd with Kubernetes node e2e test"
@echo " * 'clean' - Clean artifacts"
@echo " * 'verify' - Execute the source code verification tools"
@echo " * 'install.tools' - Install tools used by verify"
@echo " * 'install.deps' - Install dependencies of cri-containerd (containerd, runc, cni) Note: BUILDTAGS defaults to 'seccomp apparmor' for runc build"
@echo " * 'uninstall' - Remove installed binaries from system locations"
@echo " * 'version' - Print current cri-containerd release version"
@echo " * 'install' - Install binaries to system locations"
@echo " * 'binaries' - Build cri-containerd"
@echo " * 'static-binaries - Build static cri-containerd"
@echo " * 'release' - Build release tarball"
@echo " * 'push' - Push release tarball to GCS"
@echo " * 'test' - Test cri-containerd with unit test"
@echo " * 'test-integration' - Test cri-containerd with integration test"
@echo " * 'test-cri' - Test cri-containerd with cri validation test"
@echo " * 'test-e2e-node' - Test cri-containerd with Kubernetes node e2e test"
@echo " * 'clean' - Clean artifacts"
@echo " * 'verify' - Execute the source code verification tools"
@echo " * 'install.tools' - Install tools used by verify"
@echo " * 'install.deps' - Install dependencies of cri-containerd (containerd, runc, cni) Note: BUILDTAGS defaults to 'seccomp apparmor' for runc build"
@echo " * 'uninstall' - Remove installed binaries from system locations"
@echo " * 'version' - Print current cri-containerd release version"
verify: lint gofmt boiler
@@ -80,6 +82,12 @@ test:
-ldflags '$(GO_LDFLAGS)' \
-gcflags '$(GO_GCFLAGS)'
$(BUILD_DIR)/integration.test: $(INTEGRATION_SOURCES)
go test -c $(PROJECT)/integration -o $(BUILD_DIR)/integration.test
test-integration: $(BUILD_DIR)/integration.test binaries
@./hack/test-integration.sh
test-cri: binaries
@./hack/test-cri.sh
@@ -147,6 +155,7 @@ install.tools: .install.gitvalidation .install.gometalinter
install \
lint \
test \
test-integration \
test-cri \
test-e2e-node \
uninstall \

6
hack/test-cri.sh
View File

@@ -22,6 +22,7 @@ source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh
FOCUS=${FOCUS:-}
# SKIP skips the test to skip.
SKIP=${SKIP:-""}
# REPORT_DIR is the the directory to store test logs.
REPORT_DIR=${REPORT_DIR:-"/tmp/test-cri"}
# Check GOPATH
@@ -35,7 +36,6 @@ GOPATH=${GOPATH%%:*}
CRITEST=${GOPATH}/bin/critest
CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools
CRICONTAINERD_SOCK=/var/run/cri-containerd.sock
# Install critest
if [ ! -x "$(command -v ${CRITEST})" ]; then
@@ -48,12 +48,12 @@ fi
which ${CRITEST}
mkdir -p ${REPORT_DIR}
start_cri_containerd ${REPORT_DIR}
test_setup ${REPORT_DIR}
# Run cri validation test
sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CRICONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\"" validation
test_exit_code=$?
kill_cri_containerd
test_teardown
exit ${test_exit_code}

8
hack/test-e2e-node.sh
View File

@@ -26,7 +26,9 @@ DEFAULT_SKIP+="|ImageID"
export FOCUS=${FOCUS:-""}
# SKIP skips the test to skip.
export SKIP=${SKIP:-${DEFAULT_SKIP}}
# REPORT_DIR is the the directory to store test logs.
REPORT_DIR=${REPORT_DIR:-"/tmp/test-e2e-node"}
# UPLOAD_LOG indicates whether to upload test log to gcs.
UPLOAD_LOG=${UPLOAD_LOG:-false}
# Check GOPATH
@@ -67,18 +69,18 @@ git fetch --all
git checkout ${KUBERNETES_VERSION}
mkdir -p ${REPORT_DIR}
start_cri_containerd ${REPORT_DIR}
test_setup ${REPORT_DIR}
make test-e2e-node \
RUNTIME=remote \
CONTAINER_RUNTIME_ENDPOINT=unix:///var/run/cri-containerd.sock \
CONTAINER_RUNTIME_ENDPOINT=unix://${CRICONTAINERD_SOCK} \
ARTIFACTS=${REPORT_DIR} \
TEST_ARGS='--kubelet-flags=--cgroups-per-qos=true \
--kubelet-flags=--cgroup-root=/ \
--prepull-images=false'
test_exit_code=$?
kill_cri_containerd
test_teardown
sudo iptables-restore < ${ORIGINAL_RULES}
rm ${ORIGINAL_RULES}

36
hack/test-integration.sh Executable file
View File

@@ -0,0 +1,36 @@
#!/bin/bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o nounset
set -o pipefail
source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh
cd ${ROOT}
# FOCUS focuses the test to run.
FOCUS=${FOCUS:-""}
# REPORT_DIR is the the directory to store test logs.
REPORT_DIR=${REPORT_DIR:-"/tmp/test-integration"}
mkdir -p ${REPORT_DIR}
test_setup ${REPORT_DIR}
# Run integration test.
sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v
test_exit_code=$?
test_teardown
exit ${test_exit_code}

36
hack/test-utils.sh
View File

@@ -19,8 +19,10 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
# CRI_CONTAINERD_FLAGS are the extra flags to use when start cri-containerd.
CRI_CONTAINERD_FLAGS=${CRI_CONTAINERD_FLAGS:-""}
# start_cri_containerd starts containerd and cri-containerd.
start_cri_containerd() {
CRICONTAINERD_SOCK=/var/run/cri-containerd.sock
# test_setup starts containerd and cri-containerd.
test_setup() {
local report_dir=$1
if [ ! -x ${ROOT}/_output/cri-containerd ]; then
echo "cri-containerd is not built"
@@ -32,29 +34,35 @@ start_cri_containerd() {
echo "containerd is not installed, please run hack/install-deps.sh"
exit 1
fi
kill_cri_containerd
sudo containerd -l debug &> ${report_dir}/containerd.log &
sudo pkill containerd
sudo containerd &> ${report_dir}/containerd.log &
# Wait for containerd to be running by using the containerd client ctr to check the version
# of the containerd server. Wait an increasing amount of time after each of five attempts
local MAX_ATTEMPTS=5
local attempt_num=1
until sudo ctr version &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
do
echo "attempt $attempt_num to connect to containerd failed! Trying again in $attempt_num seconds..."
sleep $(( attempt_num++ ))
done
readiness_check "sudo ctr version"
# Start cri-containerd
sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 ${CRI_CONTAINERD_FLAGS} \
&> ${report_dir}/cri-containerd.log &
readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CRICONTAINERD_SOCK} info"
}
# kill_cri_containerd kills containerd and cri-containerd.
kill_cri_containerd() {
# test_teardown kills containerd and cri-containerd.
test_teardown() {
sudo pkill containerd
}
# readiness_check checks readiness of a daemon with specified command.
readiness_check() {
local command=$1
local MAX_ATTEMPTS=5
local attempt_num=1
until ${command} &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
do
echo "$attempt_num attempt \"$command\"! Trying again in $attempt_num seconds..."
sleep $(( attempt_num++ ))
done
}
# upload_logs_to_gcs uploads test logs to gcs.
# Var set:
# 1. Bucket: gcs bucket to upload logs.

80
integration/imagefs_info_test.go Normal file
View File

@@ -0,0 +1,80 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"fmt"
"io/ioutil"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
)
func TestImageFSInfo(t *testing.T) {
t.Logf("Pull an image to make sure image fs is not empty")
img, err := imageService.PullImage(&runtime.ImageSpec{Image: "busybox"}, nil)
require.NoError(t, err)
defer func() {
err := imageService.RemoveImage(&runtime.ImageSpec{Image: img})
assert.NoError(t, err)
}()
t.Logf("Create a sandbox to make sure there is an active snapshot")
config := PodSandboxConfig("running-pod", "imagefs")
sb, err := runtimeService.RunPodSandbox(config)
require.NoError(t, err)
defer func() {
assert.NoError(t, runtimeService.StopPodSandbox(sb))
assert.NoError(t, runtimeService.RemovePodSandbox(sb))
}()
// It takes time to populate imagefs stats. Use eventually
// to check for a period of time.
t.Logf("Check imagefs info")
var info *runtime.FilesystemUsage
require.NoError(t, Eventually(func() (bool, error) {
stats, err := imageService.ImageFsInfo()
if err != nil {
return false, err
}
if len(stats) == 0 {
return false, nil
}
if len(stats) >= 2 {
return false, fmt.Errorf("unexpected stats length: %d", len(stats))
}
info = stats[0]
if info.GetTimestamp() != 0 &&
info.GetUsedBytes().GetValue() != 0 &&
info.GetInodesUsed().GetValue() != 0 &&
info.GetStorageId().GetUuid() != "" {
return true, nil
}
return false, nil
}, time.Second, 30*time.Second))
t.Logf("Device uuid should exist")
files, err := ioutil.ReadDir("/dev/disk/by-uuid")
require.NoError(t, err)
var names []string
for _, f := range files {
names = append(names, f.Name())
}
assert.Contains(t, names, info.GetStorageId().GetUuid())
}

95
integration/test_utils.go Normal file
View File

@@ -0,0 +1,95 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"errors"
"time"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/kubelet/apis/cri"
"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/remote"
"github.com/kubernetes-incubator/cri-containerd/pkg/util"
)
const (
sock = "/var/run/cri-containerd.sock"
timeout = 1 * time.Minute
)
var (
runtimeService cri.RuntimeService
imageService cri.ImageManagerService
)
func init() {
var err error
runtimeService, err = remote.NewRemoteRuntimeService(sock, timeout)
if err != nil {
glog.Exitf("Failed to create runtime service: %v", err)
}
imageService, err = remote.NewRemoteImageService(sock, timeout)
if err != nil {
glog.Exitf("Failed to create image service: %v", err)
}
}
// Opts sets specific information in pod sandbox config.
type PodSandboxOpts func(*runtime.PodSandboxConfig)
// PodSandboxConfig generates a pod sandbox config for test.
func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandboxConfig {
config := &runtime.PodSandboxConfig{
Metadata: &runtime.PodSandboxMetadata{
Name: name,
// Using random id as uuid is good enough for local
// integration test.
Uid: util.GenerateID(),
Namespace: ns,
},
Linux: &runtime.LinuxPodSandboxConfig{},
}
for _, opt := range opts {
opt(config)
}
return config
}
// CheckFunc is the function used to check a condition is true/false.
type CheckFunc func() (bool, error)
// Eventually waits for f to return true, it checks every period, and
// returns error if timeout exceeds. If f returns error, Eventually
// will return the same error immediately.
func Eventually(f CheckFunc, period, timeout time.Duration) error {
start := time.Now()
for {
done, err := f()
if done {
return nil
}
if err != nil {
return err
}
if time.Since(start) >= timeout {
return errors.New("timeout exceeded")
}
time.Sleep(period)
}
}