Update Vagrantfile for testing SELinux
`vagrant up` will build and install containerd and all dependencies, setting up proper SELinux contexts on the runc and containerd binaries. The VM is configured to be SELinux Enforcing by default but this gets changed during various CI passes via a matrix param to Disabled and Permissive before running tests. I have an open PR to fix the container-selinux policy for containerd at https://github.com/containers/container-selinux/pull/98 which once accepted we will want to update the CI matrix to use Enforcing mode instead of Permissive. All tests currently pass in SELinux permissive mode with containerd configured with `enable_selinux=true`. To see which tests are failing with SELinux enforcing and an already spun up VM: `SELINUX=Enforcing vagrant up --provision-with=selinux,test-cri` To test SELinux enforcing in a new VM: `vagrant destroy -force; SELINUX=Enforcing vagrant up --provision-with=shell,selinux,test-cri` The `selinux` shell provisioner, parameterized by the SELINUX envvar, will configure the system as you would expect, with the side effect that containerd is configured with `enable_selinux=true` via `/etc/containerd/config.toml` for Permissive or Enforcing modes and `enable_selinux=false` when SELINUX=Disabled. Provided that virtualization is suported, this Vagrantfile and provisioners make it easy to test containerd/cri for conformance under SELinux on non-SELinux systems. Signed-off-by: Jacob Blain Christen <jacob@rancher.com>
This commit is contained in:
Jacob Blain Christen
Jacob Blain Christen
parent
23934e8686
commit
b4376e9865
18
.github/workflows/ci.yml
vendored
18
.github/workflows/ci.yml
vendored
@ -423,7 +423,7 @@ jobs:
|
||||
test $TEST_RC -eq 0 || /bin/false
|
||||
|
||||
cgroup2:
|
||||
name: CGroups v2 Integration Test
|
||||
name: CGroupsV2 and SELinux Integration
|
||||
# nested virtualization is only available on macOS hosts
|
||||
runs-on: macos-10.15
|
||||
timeout-minutes: 40
|
||||
@ -436,12 +436,20 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Start vagrant
|
||||
env:
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
run: vagrant up
|
||||
|
||||
- name: Integration
|
||||
run: vagrant ssh default -- sudo -i /integration.sh
|
||||
env:
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
# SELinux: replace Permissive with Enforcing after https://github.com/containers/container-selinux/pull/98
|
||||
# is merged and the package becomes generally available.
|
||||
SELINUX: Permissive
|
||||
run: vagrant up --provision-with=selinux,install-runc,test-integration
|
||||
|
||||
- name: CRI test
|
||||
run: vagrant ssh default -- sudo -i /critest.sh
|
||||
env:
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
# SELinux: replace Permissive with Enforcing after https://github.com/containers/container-selinux/pull/98
|
||||
# is merged and the package becomes generally available.
|
||||
SELINUX: Permissive
|
||||
run: vagrant up --provision-with=selinux,install-runc,test-cri
|
||||
|
||||
255
Vagrantfile
vendored
255
Vagrantfile
vendored
@ -15,7 +15,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Vagrantfile for cgroup2
|
||||
# Vagrantfile for cgroup2 and SELinux
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vm.box = "fedora/32-cloud-base"
|
||||
config.vm.provider :virtualbox do |v|
|
||||
@ -26,78 +26,219 @@ Vagrant.configure("2") do |config|
|
||||
v.memory = 2048
|
||||
v.cpus = 2
|
||||
end
|
||||
config.vm.provision "shell", env: {"RUNC_FLAVOR"=>ENV["RUNC_FLAVOR"]}, inline: <<-SHELL
|
||||
set -eux -o pipefail
|
||||
# configuration
|
||||
GO_VERSION="1.13.15"
|
||||
|
||||
# install dnf deps
|
||||
dnf install -y container-selinux gcc git iptables libseccomp-devel lsof make
|
||||
# Disabled by default. To run:
|
||||
# vagrant up --provision-with=upgrade-packages
|
||||
# To upgrade only specific packages:
|
||||
# UPGRADE_PACKAGES=selinux vagrant up --provision-with=upgrade-packages
|
||||
#
|
||||
config.vm.provision "upgrade-packages", type: "shell", run: "never" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-upgrade-packages"
|
||||
sh.env = {
|
||||
'UPGRADE_PACKAGES': ENV['UPGRADE_PACKAGES'],
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
set -eux -o pipefail
|
||||
dnf -y upgrade ${UPGRADE_PACKAGES}
|
||||
SHELL
|
||||
end
|
||||
|
||||
# install Go
|
||||
curl -fsSL "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar Cxz /usr/local
|
||||
# To re-run, installing CNI from RPM:
|
||||
# INSTALL_PACKAGES="containernetworking-plugins" vagrant up --provision-with=install-packages
|
||||
#
|
||||
config.vm.provision "install-packages", type: "shell", run: "once" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-install-packages"
|
||||
sh.env = {
|
||||
'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'],
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
set -eux -o pipefail
|
||||
dnf -y install \
|
||||
container-selinux \
|
||||
curl \
|
||||
gcc \
|
||||
git \
|
||||
iptables \
|
||||
libseccomp-devel \
|
||||
libselinux-devel \
|
||||
lsof \
|
||||
make \
|
||||
${INSTALL_PACKAGES}
|
||||
SHELL
|
||||
end
|
||||
|
||||
# setup env vars
|
||||
cat >> /etc/environment <<EOF
|
||||
# To re-run this provisioner, installing a different version of go:
|
||||
# GO_VERSION="1.14.6" vagrant up --provision-with=install-golang
|
||||
#
|
||||
config.vm.provision "install-golang", type: "shell", run: "once" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-install-golang"
|
||||
sh.env = {
|
||||
'GO_VERSION': ENV['GO_VERSION'] || "1.13.15",
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
set -eux -o pipefail
|
||||
curl -fsSL "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar Cxz /usr/local
|
||||
cat >> /etc/environment <<EOF
|
||||
PATH=/usr/local/go/bin:$PATH
|
||||
GO111MODULE=off
|
||||
EOF
|
||||
source /etc/environment
|
||||
cat >> /etc/profile.d/sh.local <<EOF
|
||||
source /etc/environment
|
||||
cat >> /etc/profile.d/sh.local <<EOF
|
||||
GOPATH=\\$HOME/go
|
||||
PATH=\\$GOPATH/bin:\\$PATH
|
||||
export GOPATH PATH
|
||||
EOF
|
||||
source /etc/profile.d/sh.local
|
||||
SHELL
|
||||
end
|
||||
|
||||
# enter /root/go/src/github.com/containerd/containerd
|
||||
mkdir -p /root/go/src/github.com/containerd
|
||||
ln -s /vagrant /root/go/src/github.com/containerd/containerd
|
||||
cd /root/go/src/github.com/containerd/containerd
|
||||
config.vm.provision "setup-gopath", type: "shell", run: "once" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-setup-gopath"
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
source /etc/profile.d/sh.local
|
||||
set -eux -o pipefail
|
||||
mkdir -p ${GOPATH}/src/github.com/containerd
|
||||
ln -fnsv /vagrant ${GOPATH}/src/github.com/containerd/containerd
|
||||
SHELL
|
||||
end
|
||||
|
||||
# install runc (or crun) and other components
|
||||
./script/setup/install-runc
|
||||
./script/setup/install-cni
|
||||
./script/setup/install-critools
|
||||
config.vm.provision "install-runc", type: "shell", run: "once" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-install-runc"
|
||||
sh.env = {
|
||||
'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
source /etc/profile.d/sh.local
|
||||
set -eux -o pipefail
|
||||
${GOPATH}/src/github.com/containerd/containerd/script/setup/install-runc
|
||||
type runc
|
||||
runc --version
|
||||
chcon -v -t container_runtime_exec_t $(type -ap runc)
|
||||
SHELL
|
||||
end
|
||||
|
||||
# install containerd
|
||||
make BUILDTAGS="seccomp selinux no_aufs no_btrfs no_devmapper no_zfs" binaries install
|
||||
config.vm.provision "install-cni", type: "shell", run: "once" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-install-cni"
|
||||
sh.env = {
|
||||
'CNI_BINARIES': 'bridge dhcp flannel host-device host-local ipvlan loopback macvlan portmap ptp tuning vlan',
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
source /etc/profile.d/sh.local
|
||||
set -eux -o pipefail
|
||||
${GOPATH}/src/github.com/containerd/containerd/script/setup/install-cni
|
||||
PATH=/opt/cni/bin:$PATH type ${CNI_BINARIES} || true
|
||||
SHELL
|
||||
end
|
||||
|
||||
# FIXME: enable SELinux
|
||||
setenforce 0
|
||||
umount /sys/fs/selinux
|
||||
config.vm.provision "install-cri-tools", type: "shell", run: "once" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-install-cri-tools"
|
||||
sh.env = {
|
||||
'CRI_TOOLS_VERSION': ENV['CRI_TOOLS_VERSION'] || '16911795a3c33833fa0ec83dac1ade3172f6989e',
|
||||
'GOBIN': '/usr/local/bin',
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
source /etc/profile.d/sh.local
|
||||
set -eux -o pipefail
|
||||
${GOPATH}/src/github.com/containerd/containerd/script/setup/install-critools
|
||||
type crictl critest
|
||||
critest --version
|
||||
SHELL
|
||||
end
|
||||
|
||||
# create the daemon config
|
||||
mkdir -p /etc/containerd
|
||||
cat > /etc/containerd/config.toml <<EOF
|
||||
version = 2
|
||||
[plugins]
|
||||
[plugins."io.containerd.grpc.v1.cri"]
|
||||
# FIXME: enable SELinux
|
||||
enable_selinux = false
|
||||
EOF
|
||||
config.vm.provision "install-containerd", type: "shell", run: "once" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-install-containerd"
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
source /etc/profile.d/sh.local
|
||||
set -eux -o pipefail
|
||||
cd ${GOPATH}/src/github.com/containerd/containerd
|
||||
make BUILDTAGS="seccomp selinux no_aufs no_btrfs no_devmapper no_zfs" binaries install
|
||||
type containerd
|
||||
containerd --version
|
||||
chcon -v -t container_runtime_exec_t /usr/local/bin/{containerd,containerd-shim*}
|
||||
./script/setup/config-containerd
|
||||
SHELL
|
||||
end
|
||||
|
||||
# create /integration.sh
|
||||
cat > /integration.sh <<EOF
|
||||
#!/bin/bash
|
||||
set -eux -o pipefail
|
||||
cd /root/go/src/github.com/containerd/containerd
|
||||
make integration EXTRA_TESTFLAGS=-no-criu TEST_RUNTIME=io.containerd.runc.v2 RUNC_FLAVOR=$RUNC_FLAVOR
|
||||
EOF
|
||||
chmod +x /integration.sh
|
||||
# SELinux is Enforcing by default.
|
||||
# To set SELinux as Disabled on a VM that has already been provisioned:
|
||||
# SELINUX=Disabled vagrant up --provision-with=selinux
|
||||
# To set SELinux as Permissive on a VM that has already been provsioned
|
||||
# SELINUX=Permissive vagrant up --provision-with=selinux
|
||||
config.vm.provision "selinux", type: "shell", run: "never" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-selinux"
|
||||
sh.env = {
|
||||
'SELINUX': ENV['SELINUX'] || "Enforcing"
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
/vagrant/script/setup/config-selinux
|
||||
/vagrant/script/setup/config-containerd
|
||||
SHELL
|
||||
end
|
||||
|
||||
# SELinux is permissive by default (via provisioning) in this VM. To re-run with SELinux enforcing:
|
||||
# vagrant up --provision-with=selinux-enforcing,test-integration
|
||||
#
|
||||
config.vm.provision "test-integration", type: "shell", run: "never" do |sh|
|
||||
sh.upload_path = "/tmp/test-integration"
|
||||
sh.env = {
|
||||
'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
source /etc/profile.d/sh.local
|
||||
set -eux -o pipefail
|
||||
rm -rf /var/lib/containerd-test /run/containerd-test
|
||||
cd ${GOPATH}/src/github.com/containerd/containerd
|
||||
make integration EXTRA_TESTFLAGS="-no-criu -test.v" TEST_RUNTIME=io.containerd.runc.v2 RUNC_FLAVOR=$RUNC_FLAVOR
|
||||
SHELL
|
||||
end
|
||||
|
||||
# SELinux is permissive by default (via provisioning) in this VM. To re-run with SELinux enforcing:
|
||||
# vagrant up --provision-with=selinux-enforcing,test-cri
|
||||
#
|
||||
config.vm.provision "test-cri", type: "shell", run: "never" do |sh|
|
||||
sh.upload_path = "/tmp/test-cri"
|
||||
sh.env = {
|
||||
'CRITEST_ARGS': ENV['CRITEST_ARGS'],
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
source /etc/profile.d/sh.local
|
||||
set -eux -o pipefail
|
||||
systemctl disable --now containerd || true
|
||||
rm -rf /var/lib/containerd /run/containerd
|
||||
function cleanup()
|
||||
{
|
||||
journalctl -u containerd > /tmp/containerd.log
|
||||
systemctl stop containerd
|
||||
}
|
||||
selinux=$(getenforce)
|
||||
if [[ $selinux == Enforcing ]]; then
|
||||
setenforce 0
|
||||
fi
|
||||
systemctl enable --now ${GOPATH}/src/github.com/containerd/containerd/containerd.service
|
||||
if [[ $selinux == Enforcing ]]; then
|
||||
setenforce 1
|
||||
fi
|
||||
trap cleanup EXIT
|
||||
ctr version
|
||||
critest --parallel=$(nproc) ${CRITEST_ARGS}
|
||||
SHELL
|
||||
end
|
||||
|
||||
# create /critest.sh
|
||||
cat > /critest.sh <<EOF
|
||||
#!/bin/bash
|
||||
set -eux -o pipefail
|
||||
containerd -log-level debug &> /tmp/containerd-cri.log &
|
||||
critest --runtime-endpoint=unix:///var/run/containerd/containerd.sock --parallel=2
|
||||
TEST_RC=\\$?
|
||||
test \\$TEST_RC -ne 0 && cat /tmp/containerd-cri.log
|
||||
pkill containerd
|
||||
rm -rf /etc/containerd
|
||||
exit \\$TEST_RC
|
||||
EOF
|
||||
chmod +x /critest.sh
|
||||
SHELL
|
||||
end
|
||||
|
||||
36
script/setup/config-containerd
Executable file
36
script/setup/config-containerd
Executable file
@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright The containerd Authors.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#
|
||||
# establishes /etc/containerd/config.toml
|
||||
# parameterized by the current SELinux mode
|
||||
#
|
||||
set -eux -o pipefail
|
||||
|
||||
enable_selinux=false
|
||||
|
||||
if type -p getenforce &>/dev/null && [[ $(getenforce) != Disabled ]]; then
|
||||
enable_selinux=true
|
||||
fi
|
||||
|
||||
mkdir -p /etc/containerd
|
||||
|
||||
cat << EOF | sudo tee /etc/containerd/config.toml
|
||||
version = 2
|
||||
[plugins]
|
||||
[plugins."io.containerd.grpc.v1.cri"]
|
||||
enable_selinux = ${enable_selinux}
|
||||
EOF
|
||||
48
script/setup/config-selinux
Executable file
48
script/setup/config-selinux
Executable file
@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright The containerd Authors.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#
|
||||
# set the desired SELinux mode via envvar
|
||||
#
|
||||
set -eux -o pipefail
|
||||
|
||||
if ! type -p getenforce setenforce &>/dev/null; then
|
||||
echo SELinux is Disabled
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "${SELINUX}" in
|
||||
Disabled)
|
||||
if mountpoint -q /sys/fs/selinux; then
|
||||
setenforce 0
|
||||
umount -v /sys/fs/selinux
|
||||
fi
|
||||
;;
|
||||
Enforcing)
|
||||
mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux
|
||||
setenforce 1
|
||||
;;
|
||||
Permissive)
|
||||
mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux
|
||||
setenforce 0
|
||||
;;
|
||||
*)
|
||||
echo "SELinux mode not supported: ${SELINUX}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo SELinux is $(getenforce)
|
||||
@ -27,3 +27,6 @@ cd "$GOPATH"/src/github.com/kubernetes-sigs/cri-tools
|
||||
git checkout $CRITEST_COMMIT
|
||||
make
|
||||
sudo make install
|
||||
cat << EOF | sudo tee /etc/crictl.yaml
|
||||
runtime-endpoint: unix:///run/containerd/containerd.sock
|
||||
EOF
|
||||
|
||||
@ -36,7 +36,7 @@ function install_crun() {
|
||||
chmod +x /usr/local/sbin/runc
|
||||
}
|
||||
|
||||
: ${RUNC_FLAVOR=runc}
|
||||
: ${RUNC_FLAVOR:=runc}
|
||||
case ${RUNC_FLAVOR} in
|
||||
runc) install_runc ;;
|
||||
crun) install_crun ;;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user