contrib/Dockerfile.test: add "integration", "cri-integration", "critest" stages
For ease of running the entire tests locally ``` cd contrib docker build -t containerd-test -f Dockerfile.test --target integration .. docker run --privileged containerd-test docker build -t containerd-test -f Dockerfile.test --target cri-integration .. docker run --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 containerd-test docker build -t containerd-test -f Dockerfile.test --target critest .. docker run --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 containerd-test ``` Tested on Ubuntu 22.10 (amd64, cgroup v2). Known issues: - cri-integration and critest: require `--sysctl net.ipv6.conf.all.disable_ipv6=0` to avoid errors like `failed to set bridge addr: could not add IP address to "cni0": permission denied` - critest: Often fails due to Docker Hub rate limits. Fix is coming in kubernetes-sigs/cri-tools PR 1053 Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
This commit is contained in:
		| @@ -10,6 +10,25 @@ | |||||||
| # | # | ||||||
| # docker build -t containerd-test --build-arg RUNC_VERSION=v1.0.0-rc94 -f Dockerfile.test ../ | # docker build -t containerd-test --build-arg RUNC_VERSION=v1.0.0-rc94 -f Dockerfile.test ../ | ||||||
|  |  | ||||||
|  | # ------------------------------------------------------------------------------ | ||||||
|  | # Public stages: | ||||||
|  | # "integration": for running integration tests: | ||||||
|  | #   docker build -t containerd-test -f Dockerfile.test --target integration ../ | ||||||
|  | #   docker run --privileged containerd-test | ||||||
|  | # | ||||||
|  | # "cri-integration": for running cri-integration tests: | ||||||
|  | #   docker build -t containerd-test -f Dockerfile.test --target cri-integration ../ | ||||||
|  | #   docker run --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 containerd-test | ||||||
|  | # | ||||||
|  | # "critest: for running critest: | ||||||
|  | #   docker build -t containerd-test -f Dockerfile.test --target critest ../ | ||||||
|  | #   docker run --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 containerd-test | ||||||
|  | # | ||||||
|  | # "cri-in-userns": for running critest with "CRI-in-UserNS" mode; needs Rootless Docker/Podman/nerdctl: | ||||||
|  | #   docker build -t containerd-test -f Dockerfile.test --target cri-in-userns ../ | ||||||
|  | #   docker run --privileged containerd-test | ||||||
|  | # ------------------------------------------------------------------------------ | ||||||
|  |  | ||||||
| ARG GOLANG_VERSION=1.19.4 | ARG GOLANG_VERSION=1.19.4 | ||||||
| ARG GOLANG_IMAGE=golang | ARG GOLANG_IMAGE=golang | ||||||
|  |  | ||||||
| @@ -48,10 +67,44 @@ ARG DESTDIR=/build | |||||||
| COPY script/setup/install-critools script/setup/critools-version  ./ | COPY script/setup/install-critools script/setup/critools-version  ./ | ||||||
| RUN GOBIN=$DESTDIR/usr/local/bin ./install-critools | RUN GOBIN=$DESTDIR/usr/local/bin ./install-critools | ||||||
|  |  | ||||||
| FROM build-env AS containerd | # integration stage is for running integration tests. | ||||||
| ARG DESTDIR=/build | FROM build-env AS integration | ||||||
|  | RUN apt-get update && apt-get install -y --no-install-recommends \ | ||||||
|  |     lsof \ | ||||||
|  |     && rm -rf /var/lib/apt/lists/* | ||||||
|  | COPY --from=runc       /build/ / | ||||||
|  | COPY contrib/Dockerfile.test.d/docker-entrypoint.sh /docker-entrypoint.sh | ||||||
| COPY . . | COPY . . | ||||||
| RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install | RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install | ||||||
|  | VOLUME /tmp | ||||||
|  | # TestMain wants to unlink /var/lib/containerd-test, so the entire /var/lib has to be volumified. | ||||||
|  | VOLUME /var/lib | ||||||
|  | # The entrypoint script is needed for nesting cgroup v2. | ||||||
|  | ENTRYPOINT ["/docker-entrypoint.sh"] | ||||||
|  | CMD ["make", "integration"] | ||||||
|  |  | ||||||
|  | # cri-integration stage is for running cri-integration tests. | ||||||
|  | FROM integration AS cri-integration | ||||||
|  | RUN apt-get update && apt-get install -y --no-install-recommends \ | ||||||
|  |     sudo iptables \ | ||||||
|  |     && rm -rf /var/lib/apt/lists/* | ||||||
|  | COPY --from=cni                /build/ / | ||||||
|  | COPY --from=critools           /build/ / | ||||||
|  | RUN make BUILDTAGS="no_btrfs no_devmapper" bin/cri-integration.test | ||||||
|  | # install-failpoint-binaries cannot be easily executed in a substage as it does not support custom DESTDIR. | ||||||
|  | RUN ./script/setup/install-failpoint-binaries | ||||||
|  | # The test scripts need these env vars to be explicitly set | ||||||
|  | ENV GITHUB_WORKSPACE="" | ||||||
|  | ENV ENABLE_CRI_SANDBOXES="" | ||||||
|  | ENV CONTAINERD_RUNTIME="io.containerd.runc.v2" | ||||||
|  | CMD ["make", "cri-integration"] | ||||||
|  |  | ||||||
|  | # critest stage is for running critest. | ||||||
|  | FROM cri-integration AS critest | ||||||
|  | # critest wants to create mounts under this directory, so it has to be volumified. | ||||||
|  | VOLUME /go/src/github.com/containerd/containerd | ||||||
|  | ENV TEST_RUNTIME="io.containerd.runc.v2" | ||||||
|  | CMD ["script/critest.sh", "/tmp"] | ||||||
|  |  | ||||||
| # cri-in-userns stage is for testing "CRI-in-UserNS", which should be used in conjunction with | # cri-in-userns stage is for testing "CRI-in-UserNS", which should be used in conjunction with | ||||||
| # "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless | # "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless | ||||||
| @@ -59,17 +112,9 @@ RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install | |||||||
| # | # | ||||||
| # Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ | # Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ | ||||||
| # (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves) | # (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves) | ||||||
| FROM build-env AS cri-in-userns | FROM critest AS cri-in-userns | ||||||
| RUN apt-get update && apt-get install -y --no-install-recommends \ |  | ||||||
|     iptables \ |  | ||||||
|     && rm -rf /var/lib/apt/lists/* |  | ||||||
| COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml | COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml | ||||||
| COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh       /docker-entrypoint.sh | COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh       /docker-entrypoint.sh | ||||||
| COPY --from=runc       /build/ / |  | ||||||
| COPY --from=cni        /build/ / |  | ||||||
| COPY --from=critools   /build/ / |  | ||||||
| COPY --from=containerd /build/ / |  | ||||||
| VOLUME /var/lib/containerd |  | ||||||
| ENTRYPOINT ["/docker-entrypoint.sh"] | ENTRYPOINT ["/docker-entrypoint.sh"] | ||||||
| # Skip "runtime should support unsafe sysctls": `container init caused:  write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied` | # Skip "runtime should support unsafe sysctls": `container init caused:  write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied` | ||||||
| # Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied` | # Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied` | ||||||
|   | |||||||
							
								
								
									
										28
									
								
								contrib/Dockerfile.test.d/docker-entrypoint.sh
									
									
									
									
									
										Executable file
									
								
							
							
						
						
									
										28
									
								
								contrib/Dockerfile.test.d/docker-entrypoint.sh
									
									
									
									
									
										Executable file
									
								
							| @@ -0,0 +1,28 @@ | |||||||
|  | #!/bin/bash | ||||||
|  |  | ||||||
|  | #   Copyright The containerd Authors. | ||||||
|  |  | ||||||
|  | #   Licensed under the Apache License, Version 2.0 (the "License"); | ||||||
|  | #   you may not use this file except in compliance with the License. | ||||||
|  | #   You may obtain a copy of the License at | ||||||
|  |  | ||||||
|  | #       http://www.apache.org/licenses/LICENSE-2.0 | ||||||
|  |  | ||||||
|  | #   Unless required by applicable law or agreed to in writing, software | ||||||
|  | #   distributed under the License is distributed on an "AS IS" BASIS, | ||||||
|  | #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||||||
|  | #   See the License for the specific language governing permissions and | ||||||
|  | #   limitations under the License. | ||||||
|  |  | ||||||
|  | set -eu -o pipefail | ||||||
|  |  | ||||||
|  | if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then | ||||||
|  | 	echo >&2 "Enabling cgroup v2 nesting" | ||||||
|  | 	# https://github.com/moby/moby/blob/v20.10.7/hack/dind#L28-L38 | ||||||
|  | 	mkdir -p /sys/fs/cgroup/init | ||||||
|  | 	xargs -rn1 </sys/fs/cgroup/cgroup.procs >/sys/fs/cgroup/init/cgroup.procs || : | ||||||
|  | 	sed -e 's/ / +/g' -e 's/^/+/' </sys/fs/cgroup/cgroup.controllers \ | ||||||
|  | 		>/sys/fs/cgroup/cgroup.subtree_control | ||||||
|  | fi | ||||||
|  |  | ||||||
|  | exec "$@" | ||||||
		Reference in New Issue
	
	Block a user
	 Akihiro Suda
					Akihiro Suda