Merge pull request #4451 from dweomer/test-selinux

Update Vagrantfile for testing SELinux

.github/workflows/ci.yml

@@ -423,7 +423,7 @@ jobs:
           test $TEST_RC -eq 0 || /bin/false
 
   cgroup2:
-    name: CGroups v2 Integration Test
+    name: CGroupsV2 and SELinux Integration
     # nested virtualization is only available on macOS hosts
     runs-on: macos-10.15
     timeout-minutes: 40
@@ -436,12 +436,20 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Start vagrant
-        env:
-          RUNC_FLAVOR: ${{ matrix.runc }}
         run: vagrant up
 
       - name: Integration
-        run: vagrant ssh default -- sudo -i /integration.sh
+        env:
+          RUNC_FLAVOR: ${{ matrix.runc }}
+          # SELinux: replace Permissive with Enforcing after https://github.com/containers/container-selinux/pull/98
+          # is merged and the package becomes generally available.
+          SELINUX: Permissive
+        run: vagrant up --provision-with=selinux,install-runc,test-integration
 
       - name: CRI test
-        run: vagrant ssh default -- sudo -i /critest.sh
+        env:
+          RUNC_FLAVOR: ${{ matrix.runc }}
+          # SELinux: replace Permissive with Enforcing after https://github.com/containers/container-selinux/pull/98
+          # is merged and the package becomes generally available.
+          SELINUX: Permissive
+        run: vagrant up --provision-with=selinux,install-runc,test-cri

Vagrantfile

@@ -15,7 +15,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-# Vagrantfile for cgroup2
+# Vagrantfile for cgroup2 and SELinux
 Vagrant.configure("2") do |config|
   config.vm.box = "fedora/32-cloud-base"
   config.vm.provider :virtualbox do |v|
@@ -26,18 +26,61 @@ Vagrant.configure("2") do |config|
     v.memory = 2048
     v.cpus = 2
   end
-  config.vm.provision "shell", env: {"RUNC_FLAVOR"=>ENV["RUNC_FLAVOR"]}, inline: <<-SHELL
+
+  # Disabled by default. To run:
+  #   vagrant up --provision-with=upgrade-packages
+  # To upgrade only specific packages:
+  #   UPGRADE_PACKAGES=selinux vagrant up --provision-with=upgrade-packages
+  #
+  config.vm.provision "upgrade-packages", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/vagrant-upgrade-packages"
+    sh.env = {
+        'UPGRADE_PACKAGES': ENV['UPGRADE_PACKAGES'],
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
         set -eux -o pipefail
-    # configuration
+        dnf -y upgrade ${UPGRADE_PACKAGES}
-    GO_VERSION="1.13.15"
+    SHELL
+  end
 
-    # install dnf deps
+  # To re-run, installing CNI from RPM:
-    dnf install -y container-selinux gcc git iptables libseccomp-devel lsof make
+  #   INSTALL_PACKAGES="containernetworking-plugins" vagrant up --provision-with=install-packages
+  #
+  config.vm.provision "install-packages", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-packages"
+    sh.env = {
+        'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'],
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        set -eux -o pipefail
+        dnf -y install \
+            container-selinux \
+            curl \
+            gcc \
+            git \
+            iptables \
+            libseccomp-devel \
+            libselinux-devel \
+            lsof \
+            make \
+            ${INSTALL_PACKAGES}
+    SHELL
+  end
 
-    # install Go
+  # To re-run this provisioner, installing a different version of go:
+  #   GO_VERSION="1.14.6" vagrant up --provision-with=install-golang
+  #
+  config.vm.provision "install-golang", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-golang"
+    sh.env = {
+        'GO_VERSION': ENV['GO_VERSION'] || "1.13.15",
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        set -eux -o pipefail
         curl -fsSL "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar Cxz /usr/local
-
-    # setup env vars
         cat >> /etc/environment <<EOF
 PATH=/usr/local/go/bin:$PATH
 GO111MODULE=off
@@ -49,55 +92,153 @@ PATH=\\$GOPATH/bin:\\$PATH
 export GOPATH PATH
 EOF
     source /etc/profile.d/sh.local
-
-    # enter /root/go/src/github.com/containerd/containerd
-    mkdir -p /root/go/src/github.com/containerd
-    ln -s /vagrant /root/go/src/github.com/containerd/containerd
-    cd /root/go/src/github.com/containerd/containerd
-
-    # install runc (or crun) and other components
-    ./script/setup/install-runc
-    ./script/setup/install-cni
-    ./script/setup/install-critools
-
-    # install containerd
-    make BUILDTAGS="seccomp selinux no_aufs no_btrfs no_devmapper no_zfs" binaries install
-
-    # FIXME: enable SELinux
-    setenforce 0
-    umount /sys/fs/selinux
-
-    # create the daemon config
-    mkdir -p /etc/containerd
-    cat > /etc/containerd/config.toml <<EOF
-version = 2
-[plugins]
-  [plugins."io.containerd.grpc.v1.cri"]
-# FIXME: enable SELinux
-    enable_selinux = false
-EOF
-
-    # create /integration.sh
-    cat > /integration.sh <<EOF
-#!/bin/bash
-set -eux -o pipefail
-cd /root/go/src/github.com/containerd/containerd
-make integration EXTRA_TESTFLAGS=-no-criu TEST_RUNTIME=io.containerd.runc.v2 RUNC_FLAVOR=$RUNC_FLAVOR
-EOF
-    chmod +x /integration.sh
-
-    # create /critest.sh
-    cat > /critest.sh <<EOF
-#!/bin/bash
-set -eux -o pipefail
-containerd -log-level debug &> /tmp/containerd-cri.log &
-critest --runtime-endpoint=unix:///var/run/containerd/containerd.sock --parallel=2
-TEST_RC=\\$?
-test \\$TEST_RC -ne 0 && cat /tmp/containerd-cri.log
-pkill containerd
-rm -rf /etc/containerd
-exit \\$TEST_RC
-EOF
-    chmod +x /critest.sh
     SHELL
   end
+
+  config.vm.provision "setup-gopath", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-setup-gopath"
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        mkdir -p ${GOPATH}/src/github.com/containerd
+        ln -fnsv /vagrant ${GOPATH}/src/github.com/containerd/containerd
+    SHELL
+  end
+
+  config.vm.provision "install-runc", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-runc"
+    sh.env = {
+        'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-runc
+        type runc
+        runc --version
+        chcon -v -t container_runtime_exec_t $(type -ap runc)
+    SHELL
+  end
+
+  config.vm.provision "install-cni", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-cni"
+    sh.env = {
+        'CNI_BINARIES': 'bridge dhcp flannel host-device host-local ipvlan loopback macvlan portmap ptp tuning vlan',
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-cni
+        PATH=/opt/cni/bin:$PATH type ${CNI_BINARIES} || true
+    SHELL
+  end
+
+  config.vm.provision "install-cri-tools", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-cri-tools"
+    sh.env = {
+        'CRI_TOOLS_VERSION': ENV['CRI_TOOLS_VERSION'] || '16911795a3c33833fa0ec83dac1ade3172f6989e',
+        'GOBIN': '/usr/local/bin',
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-critools
+        type crictl critest
+        critest --version
+    SHELL
+  end
+
+  config.vm.provision "install-containerd", type: "shell", run: "once" do |sh|
+    sh.upload_path = "/tmp/vagrant-install-containerd"
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        cd ${GOPATH}/src/github.com/containerd/containerd
+        make BUILDTAGS="seccomp selinux no_aufs no_btrfs no_devmapper no_zfs" binaries install
+        type containerd
+        containerd --version
+        chcon -v -t container_runtime_exec_t /usr/local/bin/{containerd,containerd-shim*}
+        ./script/setup/config-containerd
+    SHELL
+  end
+
+  # SELinux is Enforcing by default.
+  # To set SELinux as Disabled on a VM that has already been provisioned:
+  #   SELINUX=Disabled vagrant up --provision-with=selinux
+  # To set SELinux as Permissive on a VM that has already been provsioned
+  #   SELINUX=Permissive vagrant up --provision-with=selinux
+  config.vm.provision "selinux", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/vagrant-selinux"
+    sh.env = {
+        'SELINUX': ENV['SELINUX'] || "Enforcing"
+    }
+    sh.inline = <<~SHELL
+        /vagrant/script/setup/config-selinux
+        /vagrant/script/setup/config-containerd
+    SHELL
+  end
+
+  # SELinux is permissive by default (via provisioning) in this VM. To re-run with SELinux enforcing:
+  #   vagrant up --provision-with=selinux-enforcing,test-integration
+  #
+  config.vm.provision "test-integration", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/test-integration"
+    sh.env = {
+        'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        rm -rf /var/lib/containerd-test /run/containerd-test
+        cd ${GOPATH}/src/github.com/containerd/containerd
+        make integration EXTRA_TESTFLAGS="-no-criu -test.v" TEST_RUNTIME=io.containerd.runc.v2 RUNC_FLAVOR=$RUNC_FLAVOR
+    SHELL
+  end
+
+  # SELinux is permissive by default (via provisioning) in this VM. To re-run with SELinux enforcing:
+  #   vagrant up --provision-with=selinux-enforcing,test-cri
+  #
+  config.vm.provision "test-cri", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/test-cri"
+    sh.env = {
+        'CRITEST_ARGS': ENV['CRITEST_ARGS'],
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        systemctl disable --now containerd || true
+        rm -rf /var/lib/containerd /run/containerd
+        function cleanup()
+        {
+            journalctl -u containerd > /tmp/containerd.log
+            systemctl stop containerd
+        }
+        selinux=$(getenforce)
+        if [[ $selinux == Enforcing ]]; then
+            setenforce 0
+        fi
+        systemctl enable --now ${GOPATH}/src/github.com/containerd/containerd/containerd.service
+        if [[ $selinux == Enforcing ]]; then
+            setenforce 1
+        fi
+        trap cleanup EXIT
+        ctr version
+        critest --parallel=$(nproc) ${CRITEST_ARGS}
+    SHELL
+  end
+
+end

script/setup/config-containerd

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+#
+# establishes /etc/containerd/config.toml
+# parameterized by the current SELinux mode
+#
+set -eux -o pipefail
+
+enable_selinux=false
+
+if type -p getenforce &>/dev/null && [[ $(getenforce) != Disabled ]]; then
+  enable_selinux=true
+fi
+
+mkdir -p /etc/containerd
+
+cat << EOF | sudo tee /etc/containerd/config.toml
+version = 2
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    enable_selinux = ${enable_selinux}
+EOF

script/setup/config-selinux

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+#
+# set the desired SELinux mode via envvar
+#
+set -eux -o pipefail
+
+if ! type -p getenforce setenforce &>/dev/null; then
+  echo SELinux is Disabled
+  exit 0
+fi
+
+case "${SELINUX}" in
+  Disabled)
+    if mountpoint -q /sys/fs/selinux; then
+      setenforce 0
+      umount -v /sys/fs/selinux
+    fi
+    ;;
+  Enforcing)
+    mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux
+    setenforce 1
+    ;;
+  Permissive)
+    mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux
+    setenforce 0
+    ;;
+  *)
+    echo "SELinux mode not supported: ${SELINUX}" >&2
+    exit 1
+    ;;
+esac
+
+echo SELinux is $(getenforce)

script/setup/install-critools

@@ -27,3 +27,6 @@ cd "$GOPATH"/src/github.com/kubernetes-sigs/cri-tools
 git checkout $CRITEST_COMMIT
 make
 sudo make install
+cat << EOF | sudo tee /etc/crictl.yaml
+runtime-endpoint: unix:///run/containerd/containerd.sock
+EOF

script/setup/install-runc

@@ -36,7 +36,7 @@ function install_crun() {
 	chmod +x /usr/local/sbin/runc
 }
 
-: ${RUNC_FLAVOR=runc}
+: ${RUNC_FLAVOR:=runc}
 case ${RUNC_FLAVOR} in
 runc) install_runc ;;
 crun) install_crun ;;