Update Vagrantfile for testing SELinux

`vagrant up` will build and install containerd and all dependencies,
setting up proper SELinux contexts on the runc and containerd binaries.
The VM is configured to be SELinux Enforcing by default but this gets
changed during various CI passes via a matrix param to Disabled and
Permissive before running tests. I have an open PR to fix the
container-selinux policy for containerd at
https://github.com/containers/container-selinux/pull/98 which once
accepted we will want to update the CI matrix to use Enforcing mode
instead of Permissive.

All tests currently pass in SELinux permissive mode with containerd
configured with `enable_selinux=true`. To see which tests are failing
with SELinux enforcing and an already spun up VM:
`SELINUX=Enforcing vagrant up --provision-with=selinux,test-cri`
To test SELinux enforcing in a new VM:
`vagrant destroy -force; SELINUX=Enforcing vagrant up --provision-with=shell,selinux,test-cri`

The `selinux` shell provisioner, parameterized by the SELINUX envvar,
will configure the system as you would expect, with the side effect that
containerd is configured with `enable_selinux=true` via
`/etc/containerd/config.toml` for Permissive or Enforcing modes and
`enable_selinux=false` when SELINUX=Disabled.

Provided that virtualization is suported, this Vagrantfile and provisioners
make it easy to test containerd/cri for conformance under SELinux on
non-SELinux systems.

Signed-off-by: Jacob Blain Christen <jacob@rancher.com>

.github/workflows/ci.yml

@@ -423,7 +423,7 @@ jobs:
           test $TEST_RC -eq 0 || /bin/false
 
   cgroup2:
-    name: CGroups v2 Integration Test
+    name: CGroupsV2 and SELinux Integration
     # nested virtualization is only available on macOS hosts
     runs-on: macos-10.15
     timeout-minutes: 40
@@ -436,12 +436,20 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Start vagrant
-        env:
-          RUNC_FLAVOR: ${{ matrix.runc }}
         run: vagrant up
 
       - name: Integration
-        run: vagrant ssh default -- sudo -i /integration.sh
+        env:
+          RUNC_FLAVOR: ${{ matrix.runc }}
+          # SELinux: replace Permissive with Enforcing after https://github.com/containers/container-selinux/pull/98
+          # is merged and the package becomes generally available.
+          SELINUX: Permissive
+        run: vagrant up --provision-with=selinux,install-runc,test-integration
 
       - name: CRI test
-        run: vagrant ssh default -- sudo -i /critest.sh
+        env:
+          RUNC_FLAVOR: ${{ matrix.runc }}
+          # SELinux: replace Permissive with Enforcing after https://github.com/containers/container-selinux/pull/98
+          # is merged and the package becomes generally available.
+          SELINUX: Permissive
+        run: vagrant up --provision-with=selinux,install-runc,test-cri

Vagrantfile

@@ -15,7 +15,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-# Vagrantfile for cgroup2
+# Vagrantfile for cgroup2 and SELinux
 Vagrant.configure("2") do |config|
   config.vm.box = "fedora/32-cloud-base"
   config.vm.provider :virtualbox do |v|
@@ -26,18 +26,61 @@ Vagrant.configure("2") do |config|
     v.memory = 2048
     v.cpus = 2
   end
-  config.vm.provision "shell", env: {"RUNC_FLAVOR"=>ENV["RUNC_FLAVOR"]}, inline: <<-SHELL
+
+  # Disabled by default. To run:
+  #   vagrant up --provision-with=upgrade-packages
+  # To upgrade only specific packages:
+  #   UPGRADE_PACKAGES=selinux vagrant up --provision-with=upgrade-packages
+  #
+  config.vm.provision "upgrade-packages", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/vagrant-upgrade-packages"
+    sh.env = {
+        'UPGRADE_PACKAGES': ENV['UPGRADE_PACKAGES'],
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
         set -eux -o pipefail
-    # configuration
+        dnf -y upgrade ${UPGRADE_PACKAGES}
-    GO_VERSION="1.13.15"
+    SHELL
+  end
 
-    # install dnf deps
+  # To re-run, installing CNI from RPM:
-    dnf install -y container-selinux gcc git iptables libseccomp-devel lsof make
+  #   INSTALL_PACKAGES="containernetworking-plugins" vagrant up --provision-with=install-packages
+  #
+  config.vm.provision "install-packages", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-packages"
+    sh.env = {
+        'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'],
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        set -eux -o pipefail
+        dnf -y install \
+            container-selinux \
+            curl \
+            gcc \
+            git \
+            iptables \
+            libseccomp-devel \
+            libselinux-devel \
+            lsof \
+            make \
+            ${INSTALL_PACKAGES}
+    SHELL
+  end
 
-    # install Go
+  # To re-run this provisioner, installing a different version of go:
+  #   GO_VERSION="1.14.6" vagrant up --provision-with=install-golang
+  #
+  config.vm.provision "install-golang", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-golang"
+    sh.env = {
+        'GO_VERSION': ENV['GO_VERSION'] || "1.13.15",
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        set -eux -o pipefail
         curl -fsSL "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar Cxz /usr/local
-
-    # setup env vars
         cat >> /etc/environment <<EOF
 PATH=/usr/local/go/bin:$PATH
 GO111MODULE=off
@@ -49,55 +92,153 @@ PATH=\\$GOPATH/bin:\\$PATH
 export GOPATH PATH
 EOF
     source /etc/profile.d/sh.local
-
-    # enter /root/go/src/github.com/containerd/containerd
-    mkdir -p /root/go/src/github.com/containerd
-    ln -s /vagrant /root/go/src/github.com/containerd/containerd
-    cd /root/go/src/github.com/containerd/containerd
-
-    # install runc (or crun) and other components
-    ./script/setup/install-runc
-    ./script/setup/install-cni
-    ./script/setup/install-critools
-
-    # install containerd
-    make BUILDTAGS="seccomp selinux no_aufs no_btrfs no_devmapper no_zfs" binaries install
-
-    # FIXME: enable SELinux
-    setenforce 0
-    umount /sys/fs/selinux
-
-    # create the daemon config
-    mkdir -p /etc/containerd
-    cat > /etc/containerd/config.toml <<EOF
-version = 2
-[plugins]
-  [plugins."io.containerd.grpc.v1.cri"]
-# FIXME: enable SELinux
-    enable_selinux = false
-EOF
-
-    # create /integration.sh
-    cat > /integration.sh <<EOF
-#!/bin/bash
-set -eux -o pipefail
-cd /root/go/src/github.com/containerd/containerd
-make integration EXTRA_TESTFLAGS=-no-criu TEST_RUNTIME=io.containerd.runc.v2 RUNC_FLAVOR=$RUNC_FLAVOR
-EOF
-    chmod +x /integration.sh
-
-    # create /critest.sh
-    cat > /critest.sh <<EOF
-#!/bin/bash
-set -eux -o pipefail
-containerd -log-level debug &> /tmp/containerd-cri.log &
-critest --runtime-endpoint=unix:///var/run/containerd/containerd.sock --parallel=2
-TEST_RC=\\$?
-test \\$TEST_RC -ne 0 && cat /tmp/containerd-cri.log
-pkill containerd
-rm -rf /etc/containerd
-exit \\$TEST_RC
-EOF
-    chmod +x /critest.sh
     SHELL
+  end
+
+  config.vm.provision "setup-gopath", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-setup-gopath"
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        mkdir -p ${GOPATH}/src/github.com/containerd
+        ln -fnsv /vagrant ${GOPATH}/src/github.com/containerd/containerd
+    SHELL
+  end
+
+  config.vm.provision "install-runc", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-runc"
+    sh.env = {
+        'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-runc
+        type runc
+        runc --version
+        chcon -v -t container_runtime_exec_t $(type -ap runc)
+    SHELL
+  end
+
+  config.vm.provision "install-cni", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-cni"
+    sh.env = {
+        'CNI_BINARIES': 'bridge dhcp flannel host-device host-local ipvlan loopback macvlan portmap ptp tuning vlan',
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-cni
+        PATH=/opt/cni/bin:$PATH type ${CNI_BINARIES} || true
+    SHELL
+  end
+
+  config.vm.provision "install-cri-tools", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-cri-tools"
+    sh.env = {
+        'CRI_TOOLS_VERSION': ENV['CRI_TOOLS_VERSION'] || '16911795a3c33833fa0ec83dac1ade3172f6989e',
+        'GOBIN': '/usr/local/bin',
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-critools
+        type crictl critest
+        critest --version
+    SHELL
+  end
+
+  config.vm.provision "install-containerd", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-containerd"
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        cd ${GOPATH}/src/github.com/containerd/containerd
+        make BUILDTAGS="seccomp selinux no_aufs no_btrfs no_devmapper no_zfs" binaries install
+        type containerd
+        containerd --version
+        chcon -v -t container_runtime_exec_t /usr/local/bin/{containerd,containerd-shim*}
+        ./script/setup/config-containerd
+    SHELL
+  end
+
+  # SELinux is Enforcing by default.
+  # To set SELinux as Disabled on a VM that has already been provisioned:
+  #   SELINUX=Disabled vagrant up --provision-with=selinux
+  # To set SELinux as Permissive on a VM that has already been provsioned
+  #   SELINUX=Permissive vagrant up --provision-with=selinux
+  config.vm.provision "selinux", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/vagrant-selinux"
+    sh.env = {
+        'SELINUX': ENV['SELINUX'] || "Enforcing"
+    }
+    sh.inline = <<~SHELL
+        /vagrant/script/setup/config-selinux
+        /vagrant/script/setup/config-containerd
+    SHELL
+  end
+
+  # SELinux is permissive by default (via provisioning) in this VM. To re-run with SELinux enforcing:
+  #   vagrant up --provision-with=selinux-enforcing,test-integration
+  #
+  config.vm.provision "test-integration", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/test-integration"
+    sh.env = {
+        'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        rm -rf /var/lib/containerd-test /run/containerd-test
+        cd ${GOPATH}/src/github.com/containerd/containerd
+        make integration EXTRA_TESTFLAGS="-no-criu -test.v" TEST_RUNTIME=io.containerd.runc.v2 RUNC_FLAVOR=$RUNC_FLAVOR
+    SHELL
+  end
+
+  # SELinux is permissive by default (via provisioning) in this VM. To re-run with SELinux enforcing:
+  #   vagrant up --provision-with=selinux-enforcing,test-cri
+  #
+  config.vm.provision "test-cri", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/test-cri"
+    sh.env = {
+        'CRITEST_ARGS': ENV['CRITEST_ARGS'],
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        systemctl disable --now containerd || true
+        rm -rf /var/lib/containerd /run/containerd
+        function cleanup()
+        {
+            journalctl -u containerd > /tmp/containerd.log
+            systemctl stop containerd
+        }
+        selinux=$(getenforce)
+        if [[ $selinux == Enforcing ]]; then
+            setenforce 0
+        fi
+        systemctl enable --now ${GOPATH}/src/github.com/containerd/containerd/containerd.service
+        if [[ $selinux == Enforcing ]]; then
+            setenforce 1
+        fi
+        trap cleanup EXIT
+        ctr version
+        critest --parallel=$(nproc) ${CRITEST_ARGS}
+    SHELL
+  end
+
 end

script/setup/config-containerd

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+#
+# establishes /etc/containerd/config.toml
+# parameterized by the current SELinux mode
+#
+set -eux -o pipefail
+
+enable_selinux=false
+
+if type -p getenforce &>/dev/null && [[ $(getenforce) != Disabled ]]; then
+  enable_selinux=true
+fi
+
+mkdir -p /etc/containerd
+
+cat << EOF | sudo tee /etc/containerd/config.toml
+version = 2
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    enable_selinux = ${enable_selinux}
+EOF

script/setup/config-selinux

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+#
+# set the desired SELinux mode via envvar
+#
+set -eux -o pipefail
+
+if ! type -p getenforce setenforce &>/dev/null; then
+  echo SELinux is Disabled
+  exit 0
+fi
+
+case "${SELINUX}" in
+  Disabled)
+    if mountpoint -q /sys/fs/selinux; then
+      setenforce 0
+      umount -v /sys/fs/selinux
+    fi
+    ;;
+  Enforcing)
+    mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux
+    setenforce 1
+    ;;
+  Permissive)
+    mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux
+    setenforce 0
+    ;;
+  *)
+    echo "SELinux mode not supported: ${SELINUX}" >&2
+    exit 1
+    ;;
+esac
+
+echo SELinux is $(getenforce)

script/setup/install-critools

@@ -27,3 +27,6 @@ cd "$GOPATH"/src/github.com/kubernetes-sigs/cri-tools
 git checkout $CRITEST_COMMIT
 make
 sudo make install
+cat << EOF | sudo tee /etc/crictl.yaml
+runtime-endpoint: unix:///run/containerd/containerd.sock
+EOF

script/setup/install-runc

@@ -36,7 +36,7 @@ function install_crun() {
 	chmod +x /usr/local/sbin/runc
 }
 
-: ${RUNC_FLAVOR=runc}
+: ${RUNC_FLAVOR:=runc}
 case ${RUNC_FLAVOR} in
 runc) install_runc ;;
 crun) install_crun ;;