github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Use sandboxed CRI by default

Browse Source 
Signed-off-by: Maksym Pavlenko <pavlenko.maksym@gmail.com>
This commit is contained in:
Maksym Pavlenko
2023-08-21 10:52:20 -07:00
parent 55c877297a
commit c3f3cad287
9 changed files with 25 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/ci.yml vendored
View File

@@ -244,7 +244,7 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
os: [windows-2019, windows-2022] os: [windows-2019, windows-2022]
enable_cri_sandboxes: ["", "sandboxed"] disable_cri_sandboxes: ["", "legacyCRI"]
defaults: defaults:
run: run:
@@ -335,7 +335,7 @@ jobs:
- name: Integration 1 - name: Integration 1
env: env:
CGO_ENABLED: 1 CGO_ENABLED: 1
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} DISABLE_CRI_SANDBOXES: ${{ matrix.disable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
EXTRA_TESTFLAGS: "-timeout=20m" EXTRA_TESTFLAGS: "-timeout=20m"
@@ -353,7 +353,7 @@ jobs:
TESTFLAGS_PARALLEL: 1 TESTFLAGS_PARALLEL: 1
EXTRA_TESTFLAGS: "-short" EXTRA_TESTFLAGS: "-short"
CGO_ENABLED: 1 CGO_ENABLED: 1
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} DISABLE_CRI_SANDBOXES: ${{ matrix.disable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
run: mingw32-make.exe integration run: mingw32-make.exe integration
@@ -366,14 +366,14 @@ jobs:
- name: CRI Integration Test - name: CRI Integration Test
env: env:
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} DISABLE_CRI_SANDBOXES: ${{ matrix.disable_cri_sandboxes }}
TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
run: | run: |
make cri-integration make cri-integration
- name: cri-tools critest - name: cri-tools critest
env: env:
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} DISABLE_CRI_SANDBOXES: ${{ matrix.disable_cri_sandboxes }}
CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
shell: powershell shell: powershell
run: | run: |
@@ -409,7 +409,7 @@ jobs:
runtime: runtime:
- io.containerd.runc.v2 - io.containerd.runc.v2
runc: [runc, crun] runc: [runc, crun]
enable_cri_sandboxes: ["", "sandboxed"] DISABLE_CRI_SANDBOXES: ["", "legacyCRI"]
env: env:
GOTEST: gotestsum -- GOTEST: gotestsum --
@@ -464,7 +464,7 @@ jobs:
env: env:
TEST_RUNTIME: ${{ matrix.runtime }} TEST_RUNTIME: ${{ matrix.runtime }}
RUNC_FLAVOR: ${{ matrix.runc }} RUNC_FLAVOR: ${{ matrix.runc }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} DISABLE_CRI_SANDBOXES: ${{ matrix.disable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
run: | run: |
@@ -483,7 +483,7 @@ jobs:
env: env:
TEST_RUNTIME: ${{ matrix.runtime }} TEST_RUNTIME: ${{ matrix.runtime }}
RUNC_FLAVOR: ${{ matrix.runc }} RUNC_FLAVOR: ${{ matrix.runc }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} DISABLE_CRI_SANDBOXES: ${{ matrix.disable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
run: | run: |
@@ -500,14 +500,14 @@ jobs:
- name: CRI Integration Test - name: CRI Integration Test
env: env:
TEST_RUNTIME: ${{ matrix.runtime }} TEST_RUNTIME: ${{ matrix.runtime }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} DISABLE_CRI_SANDBOXES: ${{ matrix.disable_cri_sandboxes }}
run: | run: |
CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration
- name: cri-tools critest - name: cri-tools critest
env: env:
TEST_RUNTIME: ${{ matrix.runtime }} TEST_RUNTIME: ${{ matrix.runtime }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} DISABLE_CRI_SANDBOXES: ${{ matrix.disable_cri_sandboxes }}
run: | run: |
sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report" sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report"

2
Vagrantfile vendored
View File

@@ -272,7 +272,7 @@ EOF
'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'], 'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'], 'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'],
'GITHUB_WORKSPACE': '', 'GITHUB_WORKSPACE': '',
'ENABLE_CRI_SANDBOXES': ENV['ENABLE_CRI_SANDBOXES'], 'DISABLE_CRI_SANDBOXES': ENV['DISABLE_CRI_SANDBOXES'],
} }
sh.inline = <<~SHELL sh.inline = <<~SHELL
#!/usr/bin/env bash #!/usr/bin/env bash

4
containerd.service
View File

@@ -18,8 +18,8 @@ Documentation=https://containerd.io
After=network.target local-fs.target After=network.target local-fs.target
[Service] [Service]
#uncomment to enable the experimental sbservice (sandboxed) version of containerd/cri integration #uncomment to fallback to legacy CRI plugin implementation with podsandbox support.
#Environment="ENABLE_CRI_SANDBOXES=sandboxed" #Environment="DISABLE_CRI_SANDBOXES=1"
ExecStartPre=-/sbin/modprobe overlay ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd ExecStart=/usr/local/bin/containerd

2
contrib/Dockerfile.test
View File

@@ -94,7 +94,7 @@ RUN make BUILDTAGS="no_btrfs no_devmapper" bin/cri-integration.test
RUN ./script/setup/install-failpoint-binaries RUN ./script/setup/install-failpoint-binaries
# The test scripts need these env vars to be explicitly set # The test scripts need these env vars to be explicitly set
ENV GITHUB_WORKSPACE="" ENV GITHUB_WORKSPACE=""
ENV ENABLE_CRI_SANDBOXES="" ENV DISABLE_CRI_SANDBOXES=""
ENV CONTAINERD_RUNTIME="io.containerd.runc.v2" ENV CONTAINERD_RUNTIME="io.containerd.runc.v2"
CMD ["make", "cri-integration"] CMD ["make", "cri-integration"]

6
integration/sandbox_run_rollback_test.go
View File

@@ -293,8 +293,7 @@ func TestRunPodSandboxAndTeardownCNISlow(t *testing.T) {
assert.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid) assert.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid)
assert.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt) assert.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt)
switch os.Getenv("ENABLE_CRI_SANDBOXES") { if os.Getenv("DISABLE_CRI_SANDBOXES") != "" {
case "":
// non-sbserver // non-sbserver
t.Log("Get sandbox info (non-sbserver)") t.Log("Get sandbox info (non-sbserver)")
_, info, err := SandboxInfo(sb.Id) _, info, err := SandboxInfo(sb.Id)
@@ -319,7 +318,7 @@ func TestRunPodSandboxAndTeardownCNISlow(t *testing.T) {
metadata, ok := i.(*sandbox.Metadata) metadata, ok := i.(*sandbox.Metadata)
require.True(t, ok) require.True(t, ok)
assert.Equal(t, netNS, metadata.NetNSPath, "network namespace path should be the same in runtime spec and sandbox metadata") assert.Equal(t, netNS, metadata.NetNSPath, "network namespace path should be the same in runtime spec and sandbox metadata")
default: } else {
// sbserver // sbserver
t.Log("Get sandbox info (sbserver)") t.Log("Get sandbox info (sbserver)")
_, info, err := sbserverSandboxInfo(sb.Id) _, info, err := sbserverSandboxInfo(sb.Id)
@@ -328,7 +327,6 @@ func TestRunPodSandboxAndTeardownCNISlow(t *testing.T) {
assert.NotEmpty(t, info.Metadata.NetNSPath, "network namespace should be set") assert.NotEmpty(t, info.Metadata.NetNSPath, "network namespace should be set")
} }
} }
// sbserverSandboxInfo gets sandbox info. // sbserverSandboxInfo gets sandbox info.

2
pkg/cri/config/config.go
View File

@@ -78,7 +78,7 @@ type Runtime struct {
// See https://github.com/containerd/containerd/issues/6657 for details. // See https://github.com/containerd/containerd/issues/6657 for details.
Snapshotter string `toml:"snapshotter" json:"snapshotter"` Snapshotter string `toml:"snapshotter" json:"snapshotter"`
// SandboxMode defines which sandbox runtime to use when scheduling pods // SandboxMode defines which sandbox runtime to use when scheduling pods
// This features requires experimental CRI server to be enabled (use ENABLE_CRI_SANDBOXES=1) // This features requires the new CRI server implementation (enabled by default in 2.0)
// shim - means use whatever Controller implementation provided by shim (e.g. use RemoteController). // shim - means use whatever Controller implementation provided by shim (e.g. use RemoteController).
// podsandbox - means use Controller implementation from sbserver podsandbox package. // podsandbox - means use Controller implementation from sbserver podsandbox package.
SandboxMode string `toml:"sandbox_mode" json:"sandboxMode"` SandboxMode string `toml:"sandbox_mode" json:"sandboxMode"`

4
pkg/cri/cri.go
View File

@@ -88,8 +88,8 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
} }
var s server.CRIService var s server.CRIService
if os.Getenv("ENABLE_CRI_SANDBOXES") != "" { if os.Getenv("DISABLE_CRI_SANDBOXES") == "" {
log.G(ctx).Info("using experimental CRI Sandbox server - unset ENABLE_CRI_SANDBOXES to disable") log.G(ctx).Info("using CRI Sandbox server - use DISABLE_CRI_SANDBOXES=1 to fallback to legacy CRI")
s, err = sbserver.NewCRIService(c, client, getNRIAPI(ic)) s, err = sbserver.NewCRIService(c, client, getNRIAPI(ic))
} else { } else {
log.G(ctx).Info("using legacy CRI server") log.G(ctx).Info("using legacy CRI server")

6
script/test/cri-integration.sh
View File

@@ -44,9 +44,9 @@ test_setup "${REPORT_DIR}"
CMD="" CMD=""
if [ -n "${sudo}" ]; then if [ -n "${sudo}" ]; then
CMD+="${sudo} " CMD+="${sudo} "
# sudo strips environment variables, so add ENABLE_CRI_SANDBOXES back if present # sudo strips environment variables, so add DISABLE_CRI_SANDBOXES back if present
if [ -n "${ENABLE_CRI_SANDBOXES}" ]; then if [ -n "${DISABLE_CRI_SANDBOXES}" ]; then
CMD+="ENABLE_CRI_SANDBOXES='${ENABLE_CRI_SANDBOXES}' " CMD+="DISABLE_CRI_SANDBOXES='${DISABLE_CRI_SANDBOXES}' "
fi fi
fi fi
CMD+="${PWD}/bin/cri-integration.test" CMD+="${PWD}/bin/cri-integration.test"

6
script/test/utils.sh
View File

@@ -215,9 +215,9 @@ run_containerd() {
CMD="" CMD=""
if [ -n "${sudo}" ]; then if [ -n "${sudo}" ]; then
CMD+="${sudo} " CMD+="${sudo} "
# sudo strips environment variables, so add ENABLE_CRI_SANDBOXES back if present # sudo strips environment variables, so add DISABLE_CRI_SANDBOXES back if present
if [ -n "${ENABLE_CRI_SANDBOXES}" ]; then if [ -n "${DISABLE_CRI_SANDBOXES}" ]; then
CMD+="ENABLE_CRI_SANDBOXES='${ENABLE_CRI_SANDBOXES}' " CMD+="DISABLE_CRI_SANDBOXES='${DISABLE_CRI_SANDBOXES}' "
fi fi
fi fi
CMD+="${PWD}/bin/containerd" CMD+="${PWD}/bin/containerd"