From 546538971cbde5c9fd95da32d0d77daba63c72ad Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 14:00:58 +0200 Subject: [PATCH 1/9] install-critools: make sure DESTDIR exists Signed-off-by: Sebastiaan van Stijn --- script/setup/install-critools | 1 + 1 file changed, 1 insertion(+) diff --git a/script/setup/install-critools b/script/setup/install-critools index ddbb48c48..aa03029dd 100755 --- a/script/setup/install-critools +++ b/script/setup/install-critools @@ -34,6 +34,7 @@ git checkout "$CRITEST_COMMIT" make make install -e DESTDIR=${DESTDIR:=''} BINDIR=/usr/local/bin +mkdir -p ${DESTDIR:=''}/etc/ cat << EOF | tee ${DESTDIR:=''}/etc/crictl.yaml runtime-endpoint: unix:///run/containerd/containerd.sock EOF From 25fada0cc7082d3f6c2e54e58e1794234a605daf Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 14:06:11 +0200 Subject: [PATCH 2/9] Dockerfile.test: skip curl, gcc, git and make install These are already installed by default in the golang image. Signed-off-by: Sebastiaan van Stijn --- contrib/Dockerfile.test | 4 ---- 1 file changed, 4 deletions(-) diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index da23f982d..e6016fcc7 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -32,7 +32,6 @@ RUN ./install-protobuf # Install runc FROM golang-base AS runc RUN apt-get update && apt-get install -y \ - curl \ libseccomp-dev \ --no-install-recommends @@ -46,10 +45,7 @@ FROM golang-base AS dev RUN apt-get update && apt-get install -y \ libbtrfs-dev \ btrfs-progs \ - gcc \ - git \ libseccomp-dev \ - make \ xfsprogs \ --no-install-recommends From e9f26eb877faf37dd0b6f33fe1292970b6183fd1 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 14:26:00 +0200 Subject: [PATCH 3/9] Dockerfile.test: split dev stage, and optimize order This makes the following changes: - The containerd/config.toml, and docker-entrypoint.sh only occasionally change, so copy them before copying the source code to allow them to be cached. - The cri-in-userns stage does not need files from proto3, so do not copy them - The dev environment does need the file from the proto3 stage, so copy them there. - Change the order of stages. Our CI uses `podman build` which (I think) does not skips stages that are not used for the specified target (like BuildKit does). So I moved stages that are not used for the `cri-in-userns` after that stage. Signed-off-by: Sebastiaan van Stijn --- contrib/Dockerfile.test | 90 ++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index e6016fcc7..3da24cf32 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -12,12 +12,54 @@ ARG GOLANG_VERSION=1.16.6 -FROM golang:${GOLANG_VERSION} AS golang-base +FROM golang:${GOLANG_VERSION} AS golang + +# Install runc +FROM golang AS runc +RUN apt-get update && apt-get install -y \ + libseccomp-dev \ + --no-install-recommends + +COPY script/setup/runc-version script/setup/install-runc ./ +# Allow overriding the version of runc to install through build-args +ARG RUNC_VERSION +ARG GOPROXY=direct +RUN ./install-runc + +FROM golang AS build-env +RUN apt-get update && apt-get install -y \ + libbtrfs-dev \ + btrfs-progs \ + libseccomp-dev \ + xfsprogs \ + --no-install-recommends RUN mkdir -p /go/src/github.com/containerd/containerd WORKDIR /go/src/github.com/containerd/containerd +# cri-in-userns stage is for testing "CRI-in-UserNS", which should be used in conjunction with +# "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless +# This feature is mostly expected to be used for `kind` and `minikube`. +# +# Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ +# (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves) +FROM build-env AS cri-in-userns +RUN apt-get update && apt-get install -y iptables +COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml +COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh +COPY --from=runc /usr/local/sbin/runc /usr/local/go/bin/runc +COPY . . +RUN ./script/setup/install-cni +RUN ./script/setup/install-critools +RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install +VOLUME /var/lib/containerd +ENTRYPOINT ["/docker-entrypoint.sh"] +# Skip "runtime should support unsafe sysctls": `container init caused: write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied` +# Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied` +# Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:} stream:stdout log:Effective uid: 1000) }] +CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"] + # Install proto3 -FROM golang-base AS proto3 +FROM golang AS proto3 RUN apt-get update && apt-get install -y \ autoconf \ automake \ @@ -29,50 +71,8 @@ RUN apt-get update && apt-get install -y \ COPY script/setup/install-protobuf install-protobuf RUN ./install-protobuf -# Install runc -FROM golang-base AS runc -RUN apt-get update && apt-get install -y \ - libseccomp-dev \ - --no-install-recommends - -COPY script/setup/runc-version script/setup/install-runc ./ -# Allow overriding the version of runc to install through build-args -ARG RUNC_VERSION -ARG GOPROXY=direct -RUN ./install-runc - -FROM golang-base AS dev -RUN apt-get update && apt-get install -y \ - libbtrfs-dev \ - btrfs-progs \ - libseccomp-dev \ - xfsprogs \ - --no-install-recommends - +FROM build-env AS dev COPY --from=proto3 /usr/local/bin/protoc /usr/local/bin/protoc COPY --from=proto3 /usr/local/include/google /usr/local/include/google COPY --from=runc /usr/local/sbin/runc /usr/local/go/bin/runc - COPY . . - -# cri-in-userns stage is for testing "CRI-in-UserNS", which should be used in conjunction with -# "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless -# This feature is mostly expected to be used for `kind` and `minikube`. -# -# Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ -# (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves) -FROM dev AS cri-in-userns -RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install -RUN apt-get update && apt-get install -y iptables -RUN ./script/setup/install-cni -RUN ./script/setup/install-critools -COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml -COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh -VOLUME /var/lib/containerd -ENTRYPOINT ["/docker-entrypoint.sh"] -# Skip "runtime should support unsafe sysctls": `container init caused: write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied` -# Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied` -# Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:} stream:stdout log:Effective uid: 1000) }] -CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"] - -FROM dev AS default From f9f423c078f3b106fe1a87fd60c86eb2d606e2bf Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 14:33:17 +0200 Subject: [PATCH 4/9] Dockerfile.test: standard directory to collect build aftifacts This allows for easier copying artifacts from stages, by just copying the directory content to the stage where it's used. These stages are not used to be run individually so do not have to be "runnable". Each stage is "responsible" for colllecting all aftifacts in the directory, so that "consumer" stages do not have to be aware of what needs to be copied. Signed-off-by: Sebastiaan van Stijn --- contrib/Dockerfile.test | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index 3da24cf32..c823daf4f 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -24,6 +24,7 @@ COPY script/setup/runc-version script/setup/install-runc ./ # Allow overriding the version of runc to install through build-args ARG RUNC_VERSION ARG GOPROXY=direct +ARG DESTDIR=/build RUN ./install-runc FROM golang AS build-env @@ -46,7 +47,7 @@ FROM build-env AS cri-in-userns RUN apt-get update && apt-get install -y iptables COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh -COPY --from=runc /usr/local/sbin/runc /usr/local/go/bin/runc +COPY --from=runc /build/ / COPY . . RUN ./script/setup/install-cni RUN ./script/setup/install-critools @@ -60,19 +61,20 @@ CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe # Install proto3 FROM golang AS proto3 -RUN apt-get update && apt-get install -y \ - autoconf \ - automake \ - g++ \ - libtool \ - unzip \ - --no-install-recommends +ARG DESTDIR=/build +RUN apt-get update && apt-get install -y --no-install-recommends \ + autoconf \ + automake \ + g++ \ + libtool \ + unzip COPY script/setup/install-protobuf install-protobuf -RUN ./install-protobuf +RUN ./install-protobuf \ + && mv /usr/local/bin/protoc $DESTDIR/usr/local/bin/protoc \ + && mv /usr/local/include/google $DESTDIR/usr/local/include/google FROM build-env AS dev -COPY --from=proto3 /usr/local/bin/protoc /usr/local/bin/protoc -COPY --from=proto3 /usr/local/include/google /usr/local/include/google -COPY --from=runc /usr/local/sbin/runc /usr/local/go/bin/runc +COPY --from=proto3 /build/ / +COPY --from=runc /build/ / COPY . . From 7ec8e2d369606e870875f52a116ca2bf32e7317d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 14:37:17 +0200 Subject: [PATCH 5/9] Dockerfile.test: build cni in a separate stage Building cni only requires the install script, and the go.mod (to determin the version to install). Moving it to a separate stage prevents it from being rebuilt if unrelated changes were made in the codebase. Signed-off-by: Sebastiaan van Stijn --- contrib/Dockerfile.test | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index c823daf4f..c1cf55247 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -37,6 +37,12 @@ RUN apt-get update && apt-get install -y \ RUN mkdir -p /go/src/github.com/containerd/containerd WORKDIR /go/src/github.com/containerd/containerd +FROM golang AS cni +ENV DESTDIR=/build +COPY script/setup/install-cni ./ +COPY go.mod /go/src/github.com/containerd/containerd/go.mod +RUN ./install-cni + # cri-in-userns stage is for testing "CRI-in-UserNS", which should be used in conjunction with # "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless # This feature is mostly expected to be used for `kind` and `minikube`. @@ -48,8 +54,8 @@ RUN apt-get update && apt-get install -y iptables COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh COPY --from=runc /build/ / +COPY --from=cni /build/ / COPY . . -RUN ./script/setup/install-cni RUN ./script/setup/install-critools RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install VOLUME /var/lib/containerd From 9f7e6335c46f9a46cfe2f0955759842b095addc4 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 14:39:36 +0200 Subject: [PATCH 6/9] Dockerfile.test: build critools in a separate stage Building critools only requires the install script and the critools-version file (to determin the version to build). Moving it to a separate stage prevents rebuilding it if unrelated changes are made in the code. Signed-off-by: Sebastiaan van Stijn --- contrib/Dockerfile.test | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index c1cf55247..04409af8d 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -43,6 +43,11 @@ COPY script/setup/install-cni ./ COPY go.mod /go/src/github.com/containerd/containerd/go.mod RUN ./install-cni +FROM golang AS critools +ARG DESTDIR=/build +COPY script/setup/install-critools script/setup/critools-version ./ +RUN GOBIN=$DESTDIR/usr/local/bin ./install-critools + # cri-in-userns stage is for testing "CRI-in-UserNS", which should be used in conjunction with # "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless # This feature is mostly expected to be used for `kind` and `minikube`. @@ -55,8 +60,8 @@ COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/con COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh COPY --from=runc /build/ / COPY --from=cni /build/ / +COPY --from=critools /build/ / COPY . . -RUN ./script/setup/install-critools RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install VOLUME /var/lib/containerd ENTRYPOINT ["/docker-entrypoint.sh"] From 8faacfca1647e6b8d2d37e35f11d0cf9bd5d42f6 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 14:46:07 +0200 Subject: [PATCH 7/9] Dockerfile.test: clean up apt indexes after installing Not critical for intermediate stages, but a minor optimization to reduce the image cache. Ideally, this would use cache-mounts for this, but those may not be supported by podman, so taking the traditional approach. Signed-off-by: Sebastiaan van Stijn --- contrib/Dockerfile.test | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index 04409af8d..ed8ae1162 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -16,9 +16,9 @@ FROM golang:${GOLANG_VERSION} AS golang # Install runc FROM golang AS runc -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ libseccomp-dev \ - --no-install-recommends + && rm -rf /var/lib/apt/lists/* COPY script/setup/runc-version script/setup/install-runc ./ # Allow overriding the version of runc to install through build-args @@ -28,12 +28,12 @@ ARG DESTDIR=/build RUN ./install-runc FROM golang AS build-env -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ libbtrfs-dev \ btrfs-progs \ libseccomp-dev \ xfsprogs \ - --no-install-recommends + && rm -rf /var/lib/apt/lists/* RUN mkdir -p /go/src/github.com/containerd/containerd WORKDIR /go/src/github.com/containerd/containerd @@ -55,7 +55,9 @@ RUN GOBIN=$DESTDIR/usr/local/bin ./install-critools # Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ # (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves) FROM build-env AS cri-in-userns -RUN apt-get update && apt-get install -y iptables +RUN apt-get update && apt-get install -y --no-install-recommends \ + iptables \ + && rm -rf /var/lib/apt/lists/* COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh COPY --from=runc /build/ / @@ -78,7 +80,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ automake \ g++ \ libtool \ - unzip + unzip \ + && rm -rf /var/lib/apt/lists/* COPY script/setup/install-protobuf install-protobuf RUN ./install-protobuf \ From 36be5ef3a26d5db8f487514e06179a17d7771b49 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 14:51:07 +0200 Subject: [PATCH 8/9] Dockerfile.test: add GOLANG_IMAGE build arg to allow overriding This allows the base image itself to be overridden with an alternative image. Signed-off-by: Sebastiaan van Stijn --- contrib/Dockerfile.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index ed8ae1162..9cb9f7408 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -11,8 +11,9 @@ # docker build -t containerd-test --build-arg RUNC_VERSION=v1.0.0-rc94 -f Dockerfile.test ../ ARG GOLANG_VERSION=1.16.6 +ARG GOLANG_IMAGE=golang -FROM golang:${GOLANG_VERSION} AS golang +FROM ${GOLANG_IMAGE}:${GOLANG_VERSION} AS golang # Install runc FROM golang AS runc From 9537bc2654869bbc6fc4b740a674acfbcec9ab00 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 17 Jul 2021 17:13:47 +0200 Subject: [PATCH 9/9] Dockerfile.test: build containerd in separate stage Looking at how this image is used, I think we don't even need the source in the final image, so we can build containerd in a separate stage, and copy the binaries. Signed-off-by: Sebastiaan van Stijn --- contrib/Dockerfile.test | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index 9cb9f7408..2bb7ad603 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -49,6 +49,11 @@ ARG DESTDIR=/build COPY script/setup/install-critools script/setup/critools-version ./ RUN GOBIN=$DESTDIR/usr/local/bin ./install-critools +FROM build-env AS containerd +ARG DESTDIR=/build +COPY . . +RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install + # cri-in-userns stage is for testing "CRI-in-UserNS", which should be used in conjunction with # "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless # This feature is mostly expected to be used for `kind` and `minikube`. @@ -64,8 +69,7 @@ COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker- COPY --from=runc /build/ / COPY --from=cni /build/ / COPY --from=critools /build/ / -COPY . . -RUN make BUILDTAGS="no_btrfs no_devmapper" binaries install +COPY --from=containerd /build/ / VOLUME /var/lib/containerd ENTRYPOINT ["/docker-entrypoint.sh"] # Skip "runtime should support unsafe sysctls": `container init caused: write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied`