Add integration test framework

Signed-off-by: Lantao Liu <lantaol@google.com>

.travis.yml

@@ -61,9 +61,12 @@ jobs:
       script:
         - make install.deps
         - make test
+        - make test-integration
         - make test-cri
       after_script:
         # Abuse travis to preserve the log.
+        - cat /tmp/test-integration/cri-containerd.log
+        - cat /tmp/test-integration/containerd.log
         - cat /tmp/test-cri/cri-containerd.log
         - cat /tmp/test-cri/containerd.log
       go: 1.8.x

Makefile

@@ -26,7 +26,8 @@ VERSION := $(VERSION:v%=%)
 TARBALL := cri-containerd-$(VERSION).tar.gz
 BUILD_TAGS := apparmor
 GO_LDFLAGS := -X $(PROJECT)/pkg/version.criContainerdVersion=$(VERSION)
-SOURCES := $(shell find . -name '*.go')
+SOURCES := $(shell find cmd/ pkg/ vendor/ -name '*.go')
+INTEGRATION_SOURCES := $(shell find integration/ -name '*.go')
 
 all: binaries
 
@@ -35,20 +36,21 @@ default: help
 help:
 	@echo "Usage: make <target>"
 	@echo
-	@echo " * 'install'        - Install binaries to system locations"
+	@echo " * 'install'          - Install binaries to system locations"
-	@echo " * 'binaries'       - Build cri-containerd"
+	@echo " * 'binaries'         - Build cri-containerd"
-	@echo " * 'static-binaries - Build static cri-containerd"
+	@echo " * 'static-binaries   - Build static cri-containerd"
-	@echo " * 'release'        - Build release tarball"
+	@echo " * 'release'          - Build release tarball"
-	@echo " * 'push'           - Push release tarball to GCS"
+	@echo " * 'push'             - Push release tarball to GCS"
-	@echo " * 'test'           - Test cri-containerd"
+	@echo " * 'test'             - Test cri-containerd with unit test"
-	@echo " * 'test-cri'       - Test cri-containerd with cri validation test"
+	@echo " * 'test-integration' - Test cri-containerd with integration test"
-	@echo " * 'test-e2e-node'  - Test cri-containerd with Kubernetes node e2e test"
+	@echo " * 'test-cri'         - Test cri-containerd with cri validation test"
-	@echo " * 'clean'          - Clean artifacts"
+	@echo " * 'test-e2e-node'    - Test cri-containerd with Kubernetes node e2e test"
-	@echo " * 'verify'         - Execute the source code verification tools"
+	@echo " * 'clean'            - Clean artifacts"
-	@echo " * 'install.tools'  - Install tools used by verify"
+	@echo " * 'verify'           - Execute the source code verification tools"
-	@echo " * 'install.deps'   - Install dependencies of cri-containerd (containerd, runc, cni) Note: BUILDTAGS defaults to 'seccomp apparmor' for runc build"
+	@echo " * 'install.tools'    - Install tools used by verify"
-	@echo " * 'uninstall'      - Remove installed binaries from system locations"
+	@echo " * 'install.deps'     - Install dependencies of cri-containerd (containerd, runc, cni) Note: BUILDTAGS defaults to 'seccomp apparmor' for runc build"
-	@echo " * 'version'        - Print current cri-containerd release version"
+	@echo " * 'uninstall'        - Remove installed binaries from system locations"
+	@echo " * 'version'          - Print current cri-containerd release version"
 
 verify: lint gofmt boiler
 
@@ -80,6 +82,12 @@ test:
 	        -ldflags '$(GO_LDFLAGS)' \
 		-gcflags '$(GO_GCFLAGS)'
 
+$(BUILD_DIR)/integration.test: $(INTEGRATION_SOURCES)
+	go test -c $(PROJECT)/integration -o $(BUILD_DIR)/integration.test
+
+test-integration: $(BUILD_DIR)/integration.test binaries
+	@./hack/test-integration.sh
+
 test-cri: binaries
 	@./hack/test-cri.sh
 
@@ -147,6 +155,7 @@ install.tools: .install.gitvalidation .install.gometalinter
 	install \
 	lint \
 	test \
+	test-integration \
 	test-cri \
 	test-e2e-node \
 	uninstall \

hack/test-cri.sh

@@ -22,6 +22,7 @@ source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh
 FOCUS=${FOCUS:-}
 # SKIP skips the test to skip.
 SKIP=${SKIP:-""}
+# REPORT_DIR is the the directory to store test logs.
 REPORT_DIR=${REPORT_DIR:-"/tmp/test-cri"}
 
 # Check GOPATH
@@ -35,7 +36,6 @@ GOPATH=${GOPATH%%:*}
 
 CRITEST=${GOPATH}/bin/critest
 CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools
-CRICONTAINERD_SOCK=/var/run/cri-containerd.sock
 
 # Install critest
 if [ ! -x "$(command -v ${CRITEST})" ]; then
@@ -48,12 +48,12 @@ fi
 which ${CRITEST}
 
 mkdir -p ${REPORT_DIR}
-start_cri_containerd ${REPORT_DIR}
+test_setup ${REPORT_DIR}
 
 # Run cri validation test
 sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CRICONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\"" validation
 test_exit_code=$?
 
-kill_cri_containerd
+test_teardown
 
 exit ${test_exit_code}

hack/test-e2e-node.sh

@@ -26,7 +26,9 @@ DEFAULT_SKIP+="|ImageID"
 export FOCUS=${FOCUS:-""}
 # SKIP skips the test to skip.
 export SKIP=${SKIP:-${DEFAULT_SKIP}}
+# REPORT_DIR is the the directory to store test logs.
 REPORT_DIR=${REPORT_DIR:-"/tmp/test-e2e-node"}
+# UPLOAD_LOG indicates whether to upload test log to gcs.
 UPLOAD_LOG=${UPLOAD_LOG:-false}
 
 # Check GOPATH
@@ -67,18 +69,18 @@ git fetch --all
 git checkout ${KUBERNETES_VERSION}
 
 mkdir -p ${REPORT_DIR}
-start_cri_containerd ${REPORT_DIR}
+test_setup ${REPORT_DIR}
 
 make test-e2e-node \
 	RUNTIME=remote \
-	CONTAINER_RUNTIME_ENDPOINT=unix:///var/run/cri-containerd.sock \
+	CONTAINER_RUNTIME_ENDPOINT=unix://${CRICONTAINERD_SOCK} \
 	ARTIFACTS=${REPORT_DIR} \
 	TEST_ARGS='--kubelet-flags=--cgroups-per-qos=true \
 	--kubelet-flags=--cgroup-root=/ \
 	--prepull-images=false'
 test_exit_code=$?
 
-kill_cri_containerd
+test_teardown
 
 sudo iptables-restore < ${ORIGINAL_RULES}
 rm ${ORIGINAL_RULES}

hack/test-integration.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -o nounset
+set -o pipefail
+
+source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh
+cd ${ROOT}
+
+# FOCUS focuses the test to run.
+FOCUS=${FOCUS:-""}
+# REPORT_DIR is the the directory to store test logs.
+REPORT_DIR=${REPORT_DIR:-"/tmp/test-integration"}
+
+mkdir -p ${REPORT_DIR}
+test_setup ${REPORT_DIR}
+
+# Run integration test.
+sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v
+test_exit_code=$?
+
+test_teardown
+
+exit ${test_exit_code}

hack/test-utils.sh

@@ -19,8 +19,10 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
 # CRI_CONTAINERD_FLAGS are the extra flags to use when start cri-containerd.
 CRI_CONTAINERD_FLAGS=${CRI_CONTAINERD_FLAGS:-""}
 
-# start_cri_containerd starts containerd and cri-containerd.
+CRICONTAINERD_SOCK=/var/run/cri-containerd.sock
-start_cri_containerd() {
+
+# test_setup starts containerd and cri-containerd.
+test_setup() {
   local report_dir=$1 
   if [ ! -x ${ROOT}/_output/cri-containerd ]; then
     echo "cri-containerd is not built"
@@ -32,29 +34,35 @@ start_cri_containerd() {
     echo "containerd is not installed, please run hack/install-deps.sh"
     exit 1
   fi
-  kill_cri_containerd
+  sudo pkill containerd
-  sudo containerd -l debug &> ${report_dir}/containerd.log &
+  sudo containerd &> ${report_dir}/containerd.log &
-
   # Wait for containerd to be running by using the containerd client ctr to check the version
   # of the containerd server. Wait an increasing amount of time after each of five attempts
-  local MAX_ATTEMPTS=5
+  readiness_check "sudo ctr version"
-  local attempt_num=1
-  until sudo ctr version &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
-  do
-      echo "attempt $attempt_num to connect to containerd failed! Trying again in $attempt_num seconds..."
-      sleep $(( attempt_num++ ))
-  done
 
   # Start cri-containerd
   sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 ${CRI_CONTAINERD_FLAGS} \
 	  &> ${report_dir}/cri-containerd.log &
+  readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CRICONTAINERD_SOCK} info"
 }
 
-# kill_cri_containerd kills containerd and cri-containerd.
+# test_teardown kills containerd and cri-containerd.
-kill_cri_containerd() {
+test_teardown() {
   sudo pkill containerd
 }
 
+# readiness_check checks readiness of a daemon with specified command.
+readiness_check() {
+  local command=$1
+  local MAX_ATTEMPTS=5
+  local attempt_num=1
+  until ${command} &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
+  do
+      echo "$attempt_num attempt \"$command\"! Trying again in $attempt_num seconds..."
+      sleep $(( attempt_num++ ))
+  done
+}
+
 # upload_logs_to_gcs uploads test logs to gcs.
 # Var set:
 # 1. Bucket: gcs bucket to upload logs.

integration/imagefs_info_test.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+func TestImageFSInfo(t *testing.T) {
+	t.Logf("Pull an image to make sure image fs is not empty")
+	img, err := imageService.PullImage(&runtime.ImageSpec{Image: "busybox"}, nil)
+	require.NoError(t, err)
+	defer func() {
+		err := imageService.RemoveImage(&runtime.ImageSpec{Image: img})
+		assert.NoError(t, err)
+	}()
+	t.Logf("Create a sandbox to make sure there is an active snapshot")
+	config := PodSandboxConfig("running-pod", "imagefs")
+	sb, err := runtimeService.RunPodSandbox(config)
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, runtimeService.StopPodSandbox(sb))
+		assert.NoError(t, runtimeService.RemovePodSandbox(sb))
+	}()
+
+	// It takes time to populate imagefs stats. Use eventually
+	// to check for a period of time.
+	t.Logf("Check imagefs info")
+	var info *runtime.FilesystemUsage
+	require.NoError(t, Eventually(func() (bool, error) {
+		stats, err := imageService.ImageFsInfo()
+		if err != nil {
+			return false, err
+		}
+		if len(stats) == 0 {
+			return false, nil
+		}
+		if len(stats) >= 2 {
+			return false, fmt.Errorf("unexpected stats length: %d", len(stats))
+		}
+		info = stats[0]
+		if info.GetTimestamp() != 0 &&
+			info.GetUsedBytes().GetValue() != 0 &&
+			info.GetInodesUsed().GetValue() != 0 &&
+			info.GetStorageId().GetUuid() != "" {
+			return true, nil
+		}
+		return false, nil
+	}, time.Second, 30*time.Second))
+
+	t.Logf("Device uuid should exist")
+	files, err := ioutil.ReadDir("/dev/disk/by-uuid")
+	require.NoError(t, err)
+	var names []string
+	for _, f := range files {
+		names = append(names, f.Name())
+	}
+	assert.Contains(t, names, info.GetStorageId().GetUuid())
+}

integration/test_utils.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"errors"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/kubelet/apis/cri"
+	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	"k8s.io/kubernetes/pkg/kubelet/remote"
+
+	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
+)
+
+const (
+	sock    = "/var/run/cri-containerd.sock"
+	timeout = 1 * time.Minute
+)
+
+var (
+	runtimeService cri.RuntimeService
+	imageService   cri.ImageManagerService
+)
+
+func init() {
+	var err error
+	runtimeService, err = remote.NewRemoteRuntimeService(sock, timeout)
+	if err != nil {
+		glog.Exitf("Failed to create runtime service: %v", err)
+	}
+	imageService, err = remote.NewRemoteImageService(sock, timeout)
+	if err != nil {
+		glog.Exitf("Failed to create image service: %v", err)
+	}
+}
+
+// Opts sets specific information in pod sandbox config.
+type PodSandboxOpts func(*runtime.PodSandboxConfig)
+
+// PodSandboxConfig generates a pod sandbox config for test.
+func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandboxConfig {
+	config := &runtime.PodSandboxConfig{
+		Metadata: &runtime.PodSandboxMetadata{
+			Name: name,
+			// Using random id as uuid is good enough for local
+			// integration test.
+			Uid:       util.GenerateID(),
+			Namespace: ns,
+		},
+		Linux: &runtime.LinuxPodSandboxConfig{},
+	}
+	for _, opt := range opts {
+		opt(config)
+	}
+	return config
+}
+
+// CheckFunc is the function used to check a condition is true/false.
+type CheckFunc func() (bool, error)
+
+// Eventually waits for f to return true, it checks every period, and
+// returns error if timeout exceeds. If f returns error, Eventually
+// will return the same error immediately.
+func Eventually(f CheckFunc, period, timeout time.Duration) error {
+	start := time.Now()
+	for {
+		done, err := f()
+		if done {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		if time.Since(start) >= timeout {
+			return errors.New("timeout exceeded")
+		}
+		time.Sleep(period)
+	}
+}