Add CRI Validation test.

Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu 2019-09-09 16:17:04 -07:00
parent dc964de85f
commit 7e96aaab09
16 changed files with 440 additions and 26 deletions

View File

@ -37,7 +37,6 @@ before_script:
script:
- make .install.gitvalidation
- make .gitvalidation
- make binaries
- make install.deps
- make containerd
- sudo PATH=$PATH GOPATH=$GOPATH make install-containerd

View File

@ -129,14 +129,14 @@ containerd: $(BUILD_DIR)/containerd ## build a customized containerd with CRI pl
install-containerd: containerd ## installs customized containerd to system location
@echo "$(WHALE) $@"
@install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd
@install -D -m 755 $(BUILD_DIR)/containerd "$(BINDIR)/containerd"
install: install-containerd ## installs customized containerd to system location
@echo "$(WHALE) $@"
uninstall: ## remove containerd from system location
@echo "$(WHALE) $@"
@rm -f $(BINDIR)/containerd
@rm -f "$(BINDIR)/containerd"
$(BUILD_DIR)/$(TARBALL): static-binaries vendor.conf
@BUILD_DIR=$(BUILD_DIR) TARBALL=$(TARBALL) VERSION=$(VERSION) ./hack/release.sh
@ -152,12 +152,22 @@ proto: ## update protobuf of the cri plugin api
@API_PATH=pkg/api/v1 hack/update-proto.sh
@API_PATH=pkg/api/runtimeoptions/v1 hack/update-proto.sh
.PHONY: install.deps
.PHONY: install.deps .install.deps.linux .install.deps.windows
install.deps: ## install dependencies of cri (default 'seccomp apparmor' BUILDTAGS for runc build)
ifeq ($(GOOS),windows)
install.deps: .install.deps.windows ## install windows deps on windows
else
install.deps: .install.deps.linux ## install windows deps on linux
endif
.install.deps.linux: ## install dependencies of cri (default 'seccomp apparmor' BUILDTAGS for runc build)
@echo "$(WHALE) $@"
@./hack/install/install-deps.sh
.install.deps.windows: ## install dependencies of cri on windows
@echo "$(WHALE) $@"
@./hack/install/windows/install-deps.sh
.PHONY: .gitvalidation
# When this is running in travis, it will only check the travis commit range.
# When running outside travis, it will check from $(EPOCH_TEST_COMMIT)..HEAD.

View File

@ -23,8 +23,7 @@ CNI_DIR=${DESTDIR}/opt/cni
CNI_PKG=github.com/containernetworking/plugins
# Create a temporary GOPATH for cni installation.
TMPGOPATH=$(mktemp -d /tmp/cri-install-cni.XXXX)
GOPATH=${TMPGOPATH}
GOPATH=$(mktemp -d /tmp/cri-install-cni.XXXX)
# Install cni
from-vendor CNI github.com/containernetworking/plugins
@ -35,4 +34,4 @@ ${SUDO} mkdir -p ${CNI_DIR}
${SUDO} cp -r ./bin ${CNI_DIR}
# Clean the tmp GOPATH dir.
rm -rf ${TMPGOPATH}
rm -rf ${GOPATH}

View File

@ -19,7 +19,7 @@ set -o nounset
set -o pipefail
source $(dirname "${BASH_SOURCE[0]}")/utils.sh
CONTAINERD_DIR=${DESTDIR}/usr/local
CONTAINERD_DIR=${CONTAINERD_DIR:-"${DESTDIR}/usr/local"}
CONTAINERD_PKG=github.com/containerd/containerd
# CHECKOUT_CONTAINERD indicates whether to checkout containerd repo.
@ -29,8 +29,7 @@ CHECKOUT_CONTAINERD=${CHECKOUT_CONTAINERD:-true}
if ${CHECKOUT_CONTAINERD}; then
# Create a temporary GOPATH for containerd installation.
TMPGOPATH=$(mktemp -d /tmp/cri-install-containerd.XXXX)
GOPATH=${TMPGOPATH}
GOPATH=$(mktemp -d /tmp/cri-install-containerd.XXXX)
from-vendor CONTAINERD github.com/containerd/containerd
checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO}
fi
@ -40,9 +39,11 @@ cd ${GOPATH}/src/${CONTAINERD_PKG}
make BUILDTAGS="${BUILDTAGS}"
# containerd make install requires `go` to work. Explicitly
# set PATH to make sure it can find `go` even with `sudo`.
${SUDO} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}"
# The single quote is required because containerd Makefile
# can't handle spaces in the path.
${SUDO} make install -e DESTDIR="'${CONTAINERD_DIR}'"
# Clean the tmp GOPATH dir.
if ${CHECKOUT_CONTAINERD}; then
rm -rf ${TMPGOPATH}
rm -rf ${GOPATH}
fi

View File

@ -19,22 +19,22 @@ set -o nounset
set -o pipefail
source $(dirname "${BASH_SOURCE[0]}")/utils.sh
CRITOOL_DIR=${DESTDIR}/usr/local/bin
CRICTL_CONFIG_DIR=${DESTDIR}/etc
CRITOOL_DIR="${CRITOOL_DIR:-${DESTDIR}/usr/local/bin}"
CRICTL_CONFIG_DIR="${CRICTL_CONFIG_DIR:-"${DESTDIR}/etc"}"
CRICTL_RUNTIME_ENDPOINT=${CRICTL_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
# Create a temporary GOPATH for crictl installation.
TMPGOPATH=$(mktemp -d /tmp/cri-install-crictl.XXXX)
GOPATH=${TMPGOPATH}
GOPATH=$(mktemp -d /tmp/cri-install-crictl.XXXX)
#Install crictl
checkout_repo ${CRITOOL_PKG} ${CRITOOL_VERSION} ${CRITOOL_REPO}
cd ${GOPATH}/src/${CRITOOL_PKG}
make VERSION=${CRITOOL_VERSION}
${SUDO} make install -e BINDIR=${CRITOOL_DIR} GOPATH=${GOPATH}
${SUDO} make install -e BINDIR="\"${CRITOOL_DIR}\"" GOPATH=${GOPATH}
${SUDO} mkdir -p ${CRICTL_CONFIG_DIR}
${SUDO} bash -c 'cat >'${CRICTL_CONFIG_DIR}'/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
${SUDO} bash -c 'cat >"'"${CRICTL_CONFIG_DIR}"'"/crictl.yaml <<EOF
runtime-endpoint: '${CRICTL_RUNTIME_ENDPOINT}'
EOF'
# Clean the tmp GOPATH dir.
rm -rf ${TMPGOPATH}
rm -rf ${GOPATH}

View File

@ -23,8 +23,7 @@ RUNC_DIR=${DESTDIR}
RUNC_PKG=github.com/opencontainers/runc
# Create a temporary GOPATH for runc installation.
TMPGOPATH=$(mktemp -d /tmp/cri-install-runc.XXXX)
GOPATH=${TMPGOPATH}
GOPATH=$(mktemp -d /tmp/cri-install-runc.XXXX)
# Install runc
from-vendor RUNC github.com/opencontainers/runc
@ -35,4 +34,4 @@ ${SUDO} make install -e DESTDIR=${RUNC_DIR}
# Clean the tmp GOPATH dir. Use sudo because runc build generates
# some privileged files.
${SUDO} rm -rf ${TMPGOPATH}
${SUDO} rm -rf ${GOPATH}

View File

@ -0,0 +1,86 @@
#!/bin/bash
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
CNI_CONFIG_DIR="${CNI_CONFIG_DIR:-"C:\\Program Files\\containerd\\cni\\conf"}"
mkdir -p "${CNI_CONFIG_DIR}"
# split_ip splits ip into a 4-element array.
split_ip() {
local -r varname="$1"
local -r ip="$2"
for i in {0..3}; do
eval "$varname"[$i]=$( echo "$ip" | cut -d '.' -f $((i + 1)) )
done
}
# subnet gets subnet for a gateway, e.g. 192.168.100.0/24.
calculate_subnet() {
local -r gateway="$1"
local -r prefix_len="$2"
split_ip gateway_array "$gateway"
local len=$prefix_len
for i in {0..3}; do
if (( len >= 8 )); then
mask=255
elif (( len > 0 )); then
mask=$(( 256 - 2 ** ( 8 - len ) ))
else
mask=0
fi
(( len -= 8 ))
result_array[i]=$(( gateway_array[i] & mask ))
done
result="$(printf ".%s" "${result_array[@]}")"
result="${result:1}"
echo "$result/$((32 - prefix_len))"
}
# nat already exists on the Windows VM, the subnet and gateway
# we specify should match that.
gateway="$(powershell -c "(Get-NetIPAddress -InterfaceAlias 'vEthernet (nat)' -AddressFamily IPv4).IPAddress")"
prefix_len="$(powershell -c "(Get-NetIPAddress -InterfaceAlias 'vEthernet (nat)' -AddressFamily IPv4).PrefixLength")"
subnet="$(calculate_subnet "$gateway" "$prefix_len")"
# The "name" field in the config is used as the underlying
# network type right now (see
# https://github.com/microsoft/windows-container-networking/pull/45),
# so it must match a network type in:
# https://docs.microsoft.com/en-us/windows-server/networking/technologies/hcn/hcn-json-document-schemas
bash -c 'cat >"'"${CNI_CONFIG_DIR}"'"/0-containerd-nat.conf <<EOF
{
"cniVersion": "0.2.0",
"name": "nat",
"type": "nat",
"master": "Ethernet",
"ipam": {
"subnet": "'$subnet'",
"routes": [
{
"gateway": "'$gateway'"
}
]
},
"capabilities": {
"portMappings": true,
"dns": true
}
}
EOF'

View File

@ -0,0 +1,37 @@
#!/bin/bash
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
source $(dirname "${BASH_SOURCE[0]}")/../utils.sh
# WINCNI_BIN_DIR is the cni plugin directory
WINCNI_BIN_DIR="${WINCNI_BIN_DIR:-"C:\\Program Files\\containerd\\cni\\bin"}"
WINCNI_PKG=github.com/Microsoft/windows-container-networking
WINCNI_VERSION=33bc4764ea3ad7c6ec58c5716370d329f5eb1266
# Create a temporary GOPATH for cni installation.
GOPATH="$(mktemp -d /tmp/cri-install-cni.XXXX)"
# Install cni
checkout_repo "${WINCNI_PKG}" "${WINCNI_VERSION}" "${WINCNI_PKG}"
cd "${GOPATH}/src/${WINCNI_PKG}"
go build "${WINCNI_PKG}/plugins/nat"
install -D -m 755 "nat.exe" "${WINCNI_BIN_DIR}/nat.exe"
# Clean the tmp GOPATH dir.
rm -rf "${GOPATH}"

View File

@ -0,0 +1,47 @@
#!/bin/bash
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
cd $(dirname "${BASH_SOURCE[0]}")
# Install hcsshim
./install-hcsshim.sh
# Install cni
./install-cni.sh
# Install cni config
./install-cni-config.sh
# Install containerd
NOSUDO=true \
BUILDTAGS="" \
CONTAINERD_DIR='C:\Program Files\Containerd' \
../install-containerd.sh
# Containerd makefile always installs into a "bin" directory.
# Use slash instead of bach slash so that `*` can work.
mv C:/'Program Files'/Containerd/bin/* 'C:\Program Files\Containerd\'
rm -rf 'C:\Program Files\Containerd\bin'
#Install critools
NOSUDO=true \
CRITOOL_DIR='C:\Program Files\Containerd' \
CRICTL_RUNTIME_ENDPOINT="npipe:////./pipe/containerd-containerd" \
CRICTL_CONFIG_DIR="C:\\Users\\$(id -u -n)\\.crictl" \
../install-critools.sh

View File

@ -0,0 +1,37 @@
#!/bin/bash
# Copyright 2018 The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
source $(dirname "${BASH_SOURCE[0]}")/../utils.sh
HCSSHIM_DIR="${HCSSHIM_DIR:-"C:\\Program Files\\Containerd"}"
HCSSHIM_PKG=github.com/Microsoft/hcsshim
# Create a temporary GOPATH for hcsshim installation.
GOPATH="$(mktemp -d /tmp/cri-install-hcsshim.XXXX)"
# Install hcsshim
from-vendor HCSSHIM "${HCSSHIM_PKG}"
checkout_repo "${HCSSHIM_PKG}" "${HCSSHIM_VERSION}" "${HCSSHIM_REPO}"
cd "${GOPATH}/src/${HCSSHIM_PKG}"
go build "${HCSSHIM_PKG}/cmd/containerd-shim-runhcs-v1"
install -D -m 755 containerd-shim-runhcs-v1 "${HCSSHIM_DIR}"/containerd-shim-runhcs-v1
# Clean the tmp GOPATH dir. Use sudo because runc build generates
# some privileged files.
rm -rf ${GOPATH}

View File

@ -52,7 +52,8 @@ if [ ! -x "$(command -v ${CRITEST})" ]; then
cd ${GOPATH}/src/${CRITOOL_PKG}
git fetch --all
git checkout ${CRITOOL_VERSION}
make
make critest
make install-critest -e BINDIR="${GOPATH}/bin"
fi
which ${CRITEST}

View File

@ -17,7 +17,7 @@
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
# Not from vendor.conf.
CRITOOL_VERSION=427262054f59f3b849391310856a19474acb7e83
CRITOOL_VERSION=84b8540c69e82671bfc244a0b0fe28a6a98381ce
CRITOOL_PKG=github.com/kubernetes-sigs/cri-tools
CRITOOL_REPO=github.com/kubernetes-sigs/cri-tools

98
test/windows/runner.sh Executable file
View File

@ -0,0 +1,98 @@
#!/bin/bash
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
set -o xtrace
GCE_PROJECT="${GCE_PROJECT:-"cri-containerd-node-e2e"}"
GCE_IMAGE="${GCE_IMAGE:-"windows-server-1809-dc-core-for-containers-v20190827"}"
GCE_IMAGE_PROJECT="${GCE_IMAGE_PROJECT:-"windows-cloud"}"
ZONE="${ZONE:-"us-west1-b"}"
ARTIFACTS="${ARTIFACTS:-"/tmp/test-cri-windows/_artifacts"}"
CLEANUP="${CLEANUP:-"true"}"
root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/../..
script_path="${root}/test/windows"
node_name="windows-cri-$(cat /proc/sys/kernel/random/uuid)"
# log logs the test logs.
function log() {
echo "$(date) $1"
}
function cleanup() {
if [[ "$CLEANUP" == "true" ]]; then
log "Delete the test instance"
gcloud compute instances delete -q "${node_name}"
fi
}
function retry() {
local -r MAX_ATTEMPTS=$1
local attempts=1
shift
until "$@" || (( attempts == MAX_ATTEMPTS ))
do
log "$* failed, retry in 1 second..."
(( attempts++ ))
sleep 1
done
}
gcloud config set compute/zone "${ZONE}"
gcloud config set project "${GCE_PROJECT}"
log "Create the test instance"
gcloud compute instances create "${node_name}" --machine-type=n1-standard-2 \
--image="${GCE_IMAGE}" --image-project="${GCE_IMAGE_PROJECT}" \
--metadata-from-file=windows-startup-script-ps1="${script_path}/setup-ssh.ps1"
trap cleanup EXIT
log "Wait for ssh to be ready"
retry 180 gcloud compute ssh "${node_name}" --command="echo ssh ready"
log "Setup test environment in the test instance"
retry 5 gcloud compute scp "${script_path}/setup-vm.ps1" "${node_name}":"C:/setup-vm.ps1"
gcloud compute ssh "${node_name}" --command="powershell /c C:/setup-vm.ps1"
log "Reboot the test instance to refresh environment variables"
gcloud compute ssh "${node_name}" --command="powershell /c Restart-Computer"
log "Wait for ssh to be ready"
retry 180 gcloud compute ssh "${node_name}" --command="echo ssh ready"
log "Run test on the test instance"
cri_tar="/tmp/cri.tar.gz"
tar -zcf "${cri_tar}" -C "${root}" . --owner=0 --group=0
retry 5 gcloud compute scp "${script_path}/test.sh" "${node_name}":"C:/test.sh"
retry 5 gcloud compute scp "${cri_tar}" "${node_name}":"C:/cri.tar.gz"
rm "${cri_tar}"
# git-bash doesn't return test exit code, the command should
# succeed. We'll collect test exit code from _artifacts/.
gcloud compute ssh "${node_name}" --command='powershell /c "Start-Process -FilePath \"C:\Program Files\Git\git-bash.exe\" -ArgumentList \"-elc\",\"`\"/c/test.sh &> /c/test.log`\"\" -Wait"'
log "Collect test logs"
mkdir -p "${ARTIFACTS}"
retry 5 gcloud compute scp "${node_name}":"C:/test.log" "${ARTIFACTS}"
retry 5 gcloud compute scp --recurse "${node_name}":"C:/_artifacts/*" "${ARTIFACTS}"
log "Test output:"
cat "${ARTIFACTS}/test.log"
exit_code="$(cat "${ARTIFACTS}/exitcode")"
exit "${exit_code}"

View File

@ -0,0 +1,32 @@
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$ProgressPreference = 'SilentlyContinue'
$k8sversion = ("v1.16.0-beta.2")
$url = ("https://raw.githubusercontent.com/kubernetes/kubernetes/$k8sversion/cluster/gce/windows/testonly/user-profile.psm1")
Invoke-WebRequest $url -OutFile C:\user-profile.psm1
$url = ("https://raw.githubusercontent.com/kubernetes/kubernetes/$k8sversion/cluster/gce/windows/common.psm1")
Invoke-WebRequest $url -OutFile C:\common.psm1
$url = ("https://raw.githubusercontent.com/kubernetes/kubernetes/$k8sversion/cluster/gce/windows/testonly/install-ssh.psm1")
Invoke-WebRequest $url -OutFile C:\install-ssh.psm1
Import-Module -Force C:\install-ssh.psm1
InstallAndStart-OpenSsh
StartProcess-WriteSshKeys

23
test/windows/setup-vm.ps1 Normal file
View File

@ -0,0 +1,23 @@
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
choco install -y --no-progress git
choco install -y --no-progress golang
choco install -y --no-progress make
choco install -y --no-progress mingw

45
test/windows/test.sh Executable file
View File

@ -0,0 +1,45 @@
#!/bin/bash
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
export PATH="/c/Program Files/Containerd:$PATH"
REPO_TAR="${REPO_TAR:-"/c/cri.tar.gz"}"
FOCUS="${FOCUS:-"Conformance"}"
SKIP="${SKIP:-"portforward"}"
REPORT_DIR="${REPORT_DIR:-"/c/_artifacts"}"
repo="$GOPATH/src/github.com/containerd/cri"
mkdir -p "${repo}"
cd "${repo}"
tar -xzf "${REPO_TAR}"
make install.deps
make install -e BINDIR="/c/Program Files/Containerd"
mkdir -p "${REPORT_DIR}"
containerd -log-level debug &> "${REPORT_DIR}/containerd.log" &
pid=$!
ctr version
set +o errexit
critest --runtime-endpoint=npipe:////./pipe/containerd-containerd --ginkgo.focus="${FOCUS}" --ginkgo.skip="${SKIP}" --report-dir="${REPORT_DIR}" --report-prefix="windows"
TEST_RC=$?
set -o errexit
kill -9 $pid
echo -n "${TEST_RC}" > "${REPORT_DIR}/exitcode"