From b4376e9865848e6985c4d3377a57947ce3bcce63 Mon Sep 17 00:00:00 2001 From: Jacob Blain Christen Date: Mon, 3 Aug 2020 16:01:05 -0700 Subject: [PATCH] Update Vagrantfile for testing SELinux `vagrant up` will build and install containerd and all dependencies, setting up proper SELinux contexts on the runc and containerd binaries. The VM is configured to be SELinux Enforcing by default but this gets changed during various CI passes via a matrix param to Disabled and Permissive before running tests. I have an open PR to fix the container-selinux policy for containerd at https://github.com/containers/container-selinux/pull/98 which once accepted we will want to update the CI matrix to use Enforcing mode instead of Permissive. All tests currently pass in SELinux permissive mode with containerd configured with `enable_selinux=true`. To see which tests are failing with SELinux enforcing and an already spun up VM: `SELINUX=Enforcing vagrant up --provision-with=selinux,test-cri` To test SELinux enforcing in a new VM: `vagrant destroy -force; SELINUX=Enforcing vagrant up --provision-with=shell,selinux,test-cri` The `selinux` shell provisioner, parameterized by the SELINUX envvar, will configure the system as you would expect, with the side effect that containerd is configured with `enable_selinux=true` via `/etc/containerd/config.toml` for Permissive or Enforcing modes and `enable_selinux=false` when SELINUX=Disabled. Provided that virtualization is suported, this Vagrantfile and provisioners make it easy to test containerd/cri for conformance under SELinux on non-SELinux systems. Signed-off-by: Jacob Blain Christen --- .github/workflows/ci.yml | 18 ++- Vagrantfile | 255 +++++++++++++++++++++++++-------- script/setup/config-containerd | 36 +++++ script/setup/config-selinux | 48 +++++++ script/setup/install-critools | 3 + script/setup/install-runc | 2 +- 6 files changed, 299 insertions(+), 63 deletions(-) create mode 100755 script/setup/config-containerd create mode 100755 script/setup/config-selinux diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a3e067a..79b834b87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,7 +423,7 @@ jobs: test $TEST_RC -eq 0 || /bin/false cgroup2: - name: CGroups v2 Integration Test + name: CGroupsV2 and SELinux Integration # nested virtualization is only available on macOS hosts runs-on: macos-10.15 timeout-minutes: 40 @@ -436,12 +436,20 @@ jobs: uses: actions/checkout@v2 - name: Start vagrant - env: - RUNC_FLAVOR: ${{ matrix.runc }} run: vagrant up - name: Integration - run: vagrant ssh default -- sudo -i /integration.sh + env: + RUNC_FLAVOR: ${{ matrix.runc }} + # SELinux: replace Permissive with Enforcing after https://github.com/containers/container-selinux/pull/98 + # is merged and the package becomes generally available. + SELINUX: Permissive + run: vagrant up --provision-with=selinux,install-runc,test-integration - name: CRI test - run: vagrant ssh default -- sudo -i /critest.sh + env: + RUNC_FLAVOR: ${{ matrix.runc }} + # SELinux: replace Permissive with Enforcing after https://github.com/containers/container-selinux/pull/98 + # is merged and the package becomes generally available. + SELINUX: Permissive + run: vagrant up --provision-with=selinux,install-runc,test-cri diff --git a/Vagrantfile b/Vagrantfile index 4464b94a7..991e5d88c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Vagrantfile for cgroup2 +# Vagrantfile for cgroup2 and SELinux Vagrant.configure("2") do |config| config.vm.box = "fedora/32-cloud-base" config.vm.provider :virtualbox do |v| @@ -26,78 +26,219 @@ Vagrant.configure("2") do |config| v.memory = 2048 v.cpus = 2 end - config.vm.provision "shell", env: {"RUNC_FLAVOR"=>ENV["RUNC_FLAVOR"]}, inline: <<-SHELL - set -eux -o pipefail - # configuration - GO_VERSION="1.13.15" - # install dnf deps - dnf install -y container-selinux gcc git iptables libseccomp-devel lsof make + # Disabled by default. To run: + # vagrant up --provision-with=upgrade-packages + # To upgrade only specific packages: + # UPGRADE_PACKAGES=selinux vagrant up --provision-with=upgrade-packages + # + config.vm.provision "upgrade-packages", type: "shell", run: "never" do |sh| + sh.upload_path = "/tmp/vagrant-upgrade-packages" + sh.env = { + 'UPGRADE_PACKAGES': ENV['UPGRADE_PACKAGES'], + } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + dnf -y upgrade ${UPGRADE_PACKAGES} + SHELL + end - # install Go - curl -fsSL "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar Cxz /usr/local + # To re-run, installing CNI from RPM: + # INSTALL_PACKAGES="containernetworking-plugins" vagrant up --provision-with=install-packages + # + config.vm.provision "install-packages", type: "shell", run: "once" do |sh| + sh.upload_path = "/tmp/vagrant-install-packages" + sh.env = { + 'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'], + } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + dnf -y install \ + container-selinux \ + curl \ + gcc \ + git \ + iptables \ + libseccomp-devel \ + libselinux-devel \ + lsof \ + make \ + ${INSTALL_PACKAGES} + SHELL + end - # setup env vars - cat >> /etc/environment <> /etc/environment <> /etc/profile.d/sh.local <> /etc/profile.d/sh.local < /etc/containerd/config.toml < /integration.sh < /tmp/containerd.log + systemctl stop containerd + } + selinux=$(getenforce) + if [[ $selinux == Enforcing ]]; then + setenforce 0 + fi + systemctl enable --now ${GOPATH}/src/github.com/containerd/containerd/containerd.service + if [[ $selinux == Enforcing ]]; then + setenforce 1 + fi + trap cleanup EXIT + ctr version + critest --parallel=$(nproc) ${CRITEST_ARGS} + SHELL + end - # create /critest.sh - cat > /critest.sh < /tmp/containerd-cri.log & -critest --runtime-endpoint=unix:///var/run/containerd/containerd.sock --parallel=2 -TEST_RC=\\$? -test \\$TEST_RC -ne 0 && cat /tmp/containerd-cri.log -pkill containerd -rm -rf /etc/containerd -exit \\$TEST_RC -EOF - chmod +x /critest.sh - SHELL end diff --git a/script/setup/config-containerd b/script/setup/config-containerd new file mode 100755 index 000000000..945172a5a --- /dev/null +++ b/script/setup/config-containerd @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# establishes /etc/containerd/config.toml +# parameterized by the current SELinux mode +# +set -eux -o pipefail + +enable_selinux=false + +if type -p getenforce &>/dev/null && [[ $(getenforce) != Disabled ]]; then + enable_selinux=true +fi + +mkdir -p /etc/containerd + +cat << EOF | sudo tee /etc/containerd/config.toml +version = 2 +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + enable_selinux = ${enable_selinux} +EOF diff --git a/script/setup/config-selinux b/script/setup/config-selinux new file mode 100755 index 000000000..16faf0662 --- /dev/null +++ b/script/setup/config-selinux @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# set the desired SELinux mode via envvar +# +set -eux -o pipefail + +if ! type -p getenforce setenforce &>/dev/null; then + echo SELinux is Disabled + exit 0 +fi + +case "${SELINUX}" in + Disabled) + if mountpoint -q /sys/fs/selinux; then + setenforce 0 + umount -v /sys/fs/selinux + fi + ;; + Enforcing) + mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux + setenforce 1 + ;; + Permissive) + mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux + setenforce 0 + ;; + *) + echo "SELinux mode not supported: ${SELINUX}" >&2 + exit 1 + ;; +esac + +echo SELinux is $(getenforce) diff --git a/script/setup/install-critools b/script/setup/install-critools index 2ec2e3a04..b9b5891d3 100755 --- a/script/setup/install-critools +++ b/script/setup/install-critools @@ -27,3 +27,6 @@ cd "$GOPATH"/src/github.com/kubernetes-sigs/cri-tools git checkout $CRITEST_COMMIT make sudo make install +cat << EOF | sudo tee /etc/crictl.yaml +runtime-endpoint: unix:///run/containerd/containerd.sock +EOF diff --git a/script/setup/install-runc b/script/setup/install-runc index c8468ad74..0f142e828 100755 --- a/script/setup/install-runc +++ b/script/setup/install-runc @@ -36,7 +36,7 @@ function install_crun() { chmod +x /usr/local/sbin/runc } -: ${RUNC_FLAVOR=runc} +: ${RUNC_FLAVOR:=runc} case ${RUNC_FLAVOR} in runc) install_runc ;; crun) install_crun ;;