diff --git a/.travis.yml b/.travis.yml index aea15291c..6b9332096 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,9 +61,12 @@ jobs: script: - make install.deps - make test + - make test-integration - make test-cri after_script: # Abuse travis to preserve the log. + - cat /tmp/test-integration/cri-containerd.log + - cat /tmp/test-integration/containerd.log - cat /tmp/test-cri/cri-containerd.log - cat /tmp/test-cri/containerd.log go: 1.8.x diff --git a/Makefile b/Makefile index c724ac997..176578d4e 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,8 @@ VERSION := $(VERSION:v%=%) TARBALL := cri-containerd-$(VERSION).tar.gz BUILD_TAGS := apparmor GO_LDFLAGS := -X $(PROJECT)/pkg/version.criContainerdVersion=$(VERSION) -SOURCES := $(shell find . -name '*.go') +SOURCES := $(shell find cmd/ pkg/ vendor/ -name '*.go') +INTEGRATION_SOURCES := $(shell find integration/ -name '*.go') all: binaries @@ -35,20 +36,21 @@ default: help help: @echo "Usage: make " @echo - @echo " * 'install' - Install binaries to system locations" - @echo " * 'binaries' - Build cri-containerd" - @echo " * 'static-binaries - Build static cri-containerd" - @echo " * 'release' - Build release tarball" - @echo " * 'push' - Push release tarball to GCS" - @echo " * 'test' - Test cri-containerd" - @echo " * 'test-cri' - Test cri-containerd with cri validation test" - @echo " * 'test-e2e-node' - Test cri-containerd with Kubernetes node e2e test" - @echo " * 'clean' - Clean artifacts" - @echo " * 'verify' - Execute the source code verification tools" - @echo " * 'install.tools' - Install tools used by verify" - @echo " * 'install.deps' - Install dependencies of cri-containerd (containerd, runc, cni) Note: BUILDTAGS defaults to 'seccomp apparmor' for runc build" - @echo " * 'uninstall' - Remove installed binaries from system locations" - @echo " * 'version' - Print current cri-containerd release version" + @echo " * 'install' - Install binaries to system locations" + @echo " * 'binaries' - Build cri-containerd" + @echo " * 'static-binaries - Build static cri-containerd" + @echo " * 'release' - Build release tarball" + @echo " * 'push' - Push release tarball to GCS" + @echo " * 'test' - Test cri-containerd with unit test" + @echo " * 'test-integration' - Test cri-containerd with integration test" + @echo " * 'test-cri' - Test cri-containerd with cri validation test" + @echo " * 'test-e2e-node' - Test cri-containerd with Kubernetes node e2e test" + @echo " * 'clean' - Clean artifacts" + @echo " * 'verify' - Execute the source code verification tools" + @echo " * 'install.tools' - Install tools used by verify" + @echo " * 'install.deps' - Install dependencies of cri-containerd (containerd, runc, cni) Note: BUILDTAGS defaults to 'seccomp apparmor' for runc build" + @echo " * 'uninstall' - Remove installed binaries from system locations" + @echo " * 'version' - Print current cri-containerd release version" verify: lint gofmt boiler @@ -80,6 +82,12 @@ test: -ldflags '$(GO_LDFLAGS)' \ -gcflags '$(GO_GCFLAGS)' +$(BUILD_DIR)/integration.test: $(INTEGRATION_SOURCES) + go test -c $(PROJECT)/integration -o $(BUILD_DIR)/integration.test + +test-integration: $(BUILD_DIR)/integration.test binaries + @./hack/test-integration.sh + test-cri: binaries @./hack/test-cri.sh @@ -147,6 +155,7 @@ install.tools: .install.gitvalidation .install.gometalinter install \ lint \ test \ + test-integration \ test-cri \ test-e2e-node \ uninstall \ diff --git a/hack/test-cri.sh b/hack/test-cri.sh index a1083a55d..cd6ba4882 100755 --- a/hack/test-cri.sh +++ b/hack/test-cri.sh @@ -22,6 +22,7 @@ source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh FOCUS=${FOCUS:-} # SKIP skips the test to skip. SKIP=${SKIP:-""} +# REPORT_DIR is the the directory to store test logs. REPORT_DIR=${REPORT_DIR:-"/tmp/test-cri"} # Check GOPATH @@ -35,7 +36,6 @@ GOPATH=${GOPATH%%:*} CRITEST=${GOPATH}/bin/critest CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools -CRICONTAINERD_SOCK=/var/run/cri-containerd.sock # Install critest if [ ! -x "$(command -v ${CRITEST})" ]; then @@ -48,12 +48,12 @@ fi which ${CRITEST} mkdir -p ${REPORT_DIR} -start_cri_containerd ${REPORT_DIR} +test_setup ${REPORT_DIR} # Run cri validation test sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CRICONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\"" validation test_exit_code=$? -kill_cri_containerd +test_teardown exit ${test_exit_code} diff --git a/hack/test-e2e-node.sh b/hack/test-e2e-node.sh index ed2969503..fe02927dc 100755 --- a/hack/test-e2e-node.sh +++ b/hack/test-e2e-node.sh @@ -26,7 +26,9 @@ DEFAULT_SKIP+="|ImageID" export FOCUS=${FOCUS:-""} # SKIP skips the test to skip. export SKIP=${SKIP:-${DEFAULT_SKIP}} +# REPORT_DIR is the the directory to store test logs. REPORT_DIR=${REPORT_DIR:-"/tmp/test-e2e-node"} +# UPLOAD_LOG indicates whether to upload test log to gcs. UPLOAD_LOG=${UPLOAD_LOG:-false} # Check GOPATH @@ -67,18 +69,18 @@ git fetch --all git checkout ${KUBERNETES_VERSION} mkdir -p ${REPORT_DIR} -start_cri_containerd ${REPORT_DIR} +test_setup ${REPORT_DIR} make test-e2e-node \ RUNTIME=remote \ - CONTAINER_RUNTIME_ENDPOINT=unix:///var/run/cri-containerd.sock \ + CONTAINER_RUNTIME_ENDPOINT=unix://${CRICONTAINERD_SOCK} \ ARTIFACTS=${REPORT_DIR} \ TEST_ARGS='--kubelet-flags=--cgroups-per-qos=true \ --kubelet-flags=--cgroup-root=/ \ --prepull-images=false' test_exit_code=$? -kill_cri_containerd +test_teardown sudo iptables-restore < ${ORIGINAL_RULES} rm ${ORIGINAL_RULES} diff --git a/hack/test-integration.sh b/hack/test-integration.sh new file mode 100755 index 000000000..62a54aaf3 --- /dev/null +++ b/hack/test-integration.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -o nounset +set -o pipefail + +source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh +cd ${ROOT} + +# FOCUS focuses the test to run. +FOCUS=${FOCUS:-""} +# REPORT_DIR is the the directory to store test logs. +REPORT_DIR=${REPORT_DIR:-"/tmp/test-integration"} + +mkdir -p ${REPORT_DIR} +test_setup ${REPORT_DIR} + +# Run integration test. +sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v +test_exit_code=$? + +test_teardown + +exit ${test_exit_code} diff --git a/hack/test-utils.sh b/hack/test-utils.sh index 231699273..7bdbed267 100644 --- a/hack/test-utils.sh +++ b/hack/test-utils.sh @@ -19,8 +19,10 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/.. # CRI_CONTAINERD_FLAGS are the extra flags to use when start cri-containerd. CRI_CONTAINERD_FLAGS=${CRI_CONTAINERD_FLAGS:-""} -# start_cri_containerd starts containerd and cri-containerd. -start_cri_containerd() { +CRICONTAINERD_SOCK=/var/run/cri-containerd.sock + +# test_setup starts containerd and cri-containerd. +test_setup() { local report_dir=$1 if [ ! -x ${ROOT}/_output/cri-containerd ]; then echo "cri-containerd is not built" @@ -32,29 +34,35 @@ start_cri_containerd() { echo "containerd is not installed, please run hack/install-deps.sh" exit 1 fi - kill_cri_containerd - sudo containerd -l debug &> ${report_dir}/containerd.log & - + sudo pkill containerd + sudo containerd &> ${report_dir}/containerd.log & # Wait for containerd to be running by using the containerd client ctr to check the version # of the containerd server. Wait an increasing amount of time after each of five attempts - local MAX_ATTEMPTS=5 - local attempt_num=1 - until sudo ctr version &> /dev/null || (( attempt_num == MAX_ATTEMPTS )) - do - echo "attempt $attempt_num to connect to containerd failed! Trying again in $attempt_num seconds..." - sleep $(( attempt_num++ )) - done + readiness_check "sudo ctr version" # Start cri-containerd sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 ${CRI_CONTAINERD_FLAGS} \ &> ${report_dir}/cri-containerd.log & + readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CRICONTAINERD_SOCK} info" } -# kill_cri_containerd kills containerd and cri-containerd. -kill_cri_containerd() { +# test_teardown kills containerd and cri-containerd. +test_teardown() { sudo pkill containerd } +# readiness_check checks readiness of a daemon with specified command. +readiness_check() { + local command=$1 + local MAX_ATTEMPTS=5 + local attempt_num=1 + until ${command} &> /dev/null || (( attempt_num == MAX_ATTEMPTS )) + do + echo "$attempt_num attempt \"$command\"! Trying again in $attempt_num seconds..." + sleep $(( attempt_num++ )) + done +} + # upload_logs_to_gcs uploads test logs to gcs. # Var set: # 1. Bucket: gcs bucket to upload logs. diff --git a/integration/imagefs_info_test.go b/integration/imagefs_info_test.go new file mode 100644 index 000000000..8ec94ffe6 --- /dev/null +++ b/integration/imagefs_info_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "fmt" + "io/ioutil" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" +) + +func TestImageFSInfo(t *testing.T) { + t.Logf("Pull an image to make sure image fs is not empty") + img, err := imageService.PullImage(&runtime.ImageSpec{Image: "busybox"}, nil) + require.NoError(t, err) + defer func() { + err := imageService.RemoveImage(&runtime.ImageSpec{Image: img}) + assert.NoError(t, err) + }() + t.Logf("Create a sandbox to make sure there is an active snapshot") + config := PodSandboxConfig("running-pod", "imagefs") + sb, err := runtimeService.RunPodSandbox(config) + require.NoError(t, err) + defer func() { + assert.NoError(t, runtimeService.StopPodSandbox(sb)) + assert.NoError(t, runtimeService.RemovePodSandbox(sb)) + }() + + // It takes time to populate imagefs stats. Use eventually + // to check for a period of time. + t.Logf("Check imagefs info") + var info *runtime.FilesystemUsage + require.NoError(t, Eventually(func() (bool, error) { + stats, err := imageService.ImageFsInfo() + if err != nil { + return false, err + } + if len(stats) == 0 { + return false, nil + } + if len(stats) >= 2 { + return false, fmt.Errorf("unexpected stats length: %d", len(stats)) + } + info = stats[0] + if info.GetTimestamp() != 0 && + info.GetUsedBytes().GetValue() != 0 && + info.GetInodesUsed().GetValue() != 0 && + info.GetStorageId().GetUuid() != "" { + return true, nil + } + return false, nil + }, time.Second, 30*time.Second)) + + t.Logf("Device uuid should exist") + files, err := ioutil.ReadDir("/dev/disk/by-uuid") + require.NoError(t, err) + var names []string + for _, f := range files { + names = append(names, f.Name()) + } + assert.Contains(t, names, info.GetStorageId().GetUuid()) +} diff --git a/integration/test_utils.go b/integration/test_utils.go new file mode 100644 index 000000000..91cdc27fe --- /dev/null +++ b/integration/test_utils.go @@ -0,0 +1,95 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "errors" + "time" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/kubelet/apis/cri" + "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" + "k8s.io/kubernetes/pkg/kubelet/remote" + + "github.com/kubernetes-incubator/cri-containerd/pkg/util" +) + +const ( + sock = "/var/run/cri-containerd.sock" + timeout = 1 * time.Minute +) + +var ( + runtimeService cri.RuntimeService + imageService cri.ImageManagerService +) + +func init() { + var err error + runtimeService, err = remote.NewRemoteRuntimeService(sock, timeout) + if err != nil { + glog.Exitf("Failed to create runtime service: %v", err) + } + imageService, err = remote.NewRemoteImageService(sock, timeout) + if err != nil { + glog.Exitf("Failed to create image service: %v", err) + } +} + +// Opts sets specific information in pod sandbox config. +type PodSandboxOpts func(*runtime.PodSandboxConfig) + +// PodSandboxConfig generates a pod sandbox config for test. +func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandboxConfig { + config := &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: name, + // Using random id as uuid is good enough for local + // integration test. + Uid: util.GenerateID(), + Namespace: ns, + }, + Linux: &runtime.LinuxPodSandboxConfig{}, + } + for _, opt := range opts { + opt(config) + } + return config +} + +// CheckFunc is the function used to check a condition is true/false. +type CheckFunc func() (bool, error) + +// Eventually waits for f to return true, it checks every period, and +// returns error if timeout exceeds. If f returns error, Eventually +// will return the same error immediately. +func Eventually(f CheckFunc, period, timeout time.Duration) error { + start := time.Now() + for { + done, err := f() + if done { + return nil + } + if err != nil { + return err + } + if time.Since(start) >= timeout { + return errors.New("timeout exceeded") + } + time.Sleep(period) + } +}