From bd9e9ce56eee03900b0b7f1fa29c380008a17688 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 4 Jan 2018 02:43:00 +0000 Subject: [PATCH 1/3] Add containerd plugin mode. Signed-off-by: Lantao Liu --- cmd/cri-containerd/options/options.go | 10 ++-- cri.go | 70 +++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 cri.go diff --git a/cmd/cri-containerd/options/options.go b/cmd/cri-containerd/options/options.go index f9df285e3..c7d328e8a 100644 --- a/cmd/cri-containerd/options/options.go +++ b/cmd/cri-containerd/options/options.go @@ -116,7 +116,7 @@ func NewCRIContainerdOptions() *CRIContainerdOptions { // AddFlags adds cri-containerd command line options to pflag. func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) { - defaults := defaultConfig() + defaults := DefaultConfig() fs.StringVar(&c.ConfigFilePath, configFilePathArgName, defaultConfigFilePath, "Path to the config file.") fs.StringVar(&c.SocketPath, "socket-path", @@ -192,7 +192,7 @@ func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error { // PrintDefaultTomlConfig print default toml config of cri-containerd. func PrintDefaultTomlConfig() { - if err := toml.NewEncoder(os.Stdout).Encode(defaultConfig()); err != nil { + if err := toml.NewEncoder(os.Stdout).Encode(DefaultConfig()); err != nil { fmt.Println(err) return } @@ -200,13 +200,13 @@ func PrintDefaultTomlConfig() { // AddGRPCFlags add flags for grpc connection. func AddGRPCFlags(fs *pflag.FlagSet) (*string, *time.Duration) { - endpoint := fs.String("endpoint", defaultConfig().SocketPath, "cri-containerd endpoint.") + endpoint := fs.String("endpoint", DefaultConfig().SocketPath, "cri-containerd endpoint.") timeout := fs.Duration("timeout", connectionTimeout, "cri-containerd connection timeout.") return endpoint, timeout } -// defaultConfig returns default configurations of cri-containerd. -func defaultConfig() Config { +// DefaultConfig returns default configurations of cri-containerd. +func DefaultConfig() Config { return Config{ ContainerdConfig: ContainerdConfig{ RootDir: "/var/lib/containerd", diff --git a/cri.go b/cri.go new file mode 100644 index 000000000..dbe853f55 --- /dev/null +++ b/cri.go @@ -0,0 +1,70 @@ +/* +Copyright 2018 The Containerd Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cri + +import ( + "github.com/containerd/containerd/plugin" + "github.com/golang/glog" + + "github.com/containerd/cri-containerd/cmd/cri-containerd/options" + "github.com/containerd/cri-containerd/pkg/server" +) + +// Register CRI service plugin +func init() { + plugin.Register(&plugin.Registration{ + // In fact, cri is not strictly a GRPC plugin now. + Type: plugin.GRPCPlugin, + ID: "cri", + Requires: []plugin.Type{ + plugin.RuntimePlugin, + plugin.SnapshotPlugin, + plugin.TaskMonitorPlugin, + plugin.DiffPlugin, + plugin.MetadataPlugin, + plugin.ContentPlugin, + plugin.GCPlugin, + }, + InitFn: initCRIService, + }) +} + +func initCRIService(_ *plugin.InitContext) (interface{}, error) { + // TODO(random-liu): Support Config through Registration.Config. + // TODO(random-liu): Validate the configuration. + // TODO(random-liu): Leverage other fields in InitContext, such as Root. + // TODO(random-liu): Register GRPC service onto containerd GRPC server. + // TODO(random-liu): Separate cri plugin config from cri-containerd server config, + // because many options only make sense to cri-containerd server. + // TODO(random-liu): Change all glog to logrus. + // TODO(random-liu): Handle graceful stop. + c := options.DefaultConfig() + glog.V(0).Infof("Start cri plugin with config %+v", c) + // Use a goroutine to start cri service. The reason is that currently + // cri service requires containerd to be running. + // TODO(random-liu): Resolve the circular dependency. + go func() { + s, err := server.NewCRIContainerdService(c) + if err != nil { + glog.Exitf("Failed to create CRI service: %v", err) + } + if err := s.Run(); err != nil { + glog.Exitf("Failed to run CRI grpc server: %v", err) + } + }() + return nil, nil +} From 794543065a238b1a6ce1c826b1fdf23db91c9651 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 5 Jan 2018 06:38:25 +0000 Subject: [PATCH 2/3] Add script to automatically update containerd with newest cri plugin. Signed-off-by: Lantao Liu --- hack/install-deps.sh | 43 +++++++++++++++++++++++++++++++++---- hack/test-integration.sh | 4 +++- hack/test-utils.sh | 10 ++++++--- integration/restart_test.go | 7 ++++++ integration/test_utils.go | 1 + 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/hack/install-deps.sh b/hack/install-deps.sh index 7fb1ba9ee..74c3b75ad 100755 --- a/hack/install-deps.sh +++ b/hack/install-deps.sh @@ -50,6 +50,11 @@ fi # and configurations in cluster. INSTALL_CNI=${INSTALL_CNI:-true} +# COOK_CONTAINERD indicates whether to update containerd with newest +# cri plugin before install. This is mainly used for testing new cri +# plugin change. +COOK_CONTAINERD=${COOK_CONTAINERD:-false} + CONTAINERD_DIR=${DESTDIR}/usr/local RUNC_DIR=${DESTDIR} CNI_DIR=${DESTDIR}/opt/cni @@ -61,6 +66,7 @@ RUNC_PKG=github.com/opencontainers/runc CNI_PKG=github.com/containernetworking/plugins CONTAINERD_PKG=github.com/containerd/containerd CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools +CRI_CONTAINERD_PKG=github.com/containerd/cri-containerd # Create a temporary GOPATH for make install.deps. TMPGOPATH=$(mktemp -d /tmp/cri-containerd-install-deps.XXXX) @@ -73,9 +79,9 @@ GOPATH=${TMPGOPATH} # 2) Version. # 3) Repo name (optional); checkout_repo() { - pkg=$1 - version=$2 - repo=${3:-""} + local -r pkg=$1 + local -r version=$2 + local repo=${3:-""} if [ -z "${repo}" ]; then repo=${pkg} fi @@ -134,7 +140,36 @@ fi # Install containerd checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO} cd ${GOPATH}/src/${CONTAINERD_PKG} -make +if ${COOK_CONTAINERD}; then + # Verify that vendor.conf is in sync with containerd before cook containerd, + # this is a hard requirement. + if ! ${ROOT}/hack/update-vendor.sh -only-verify; then + echo "Please run hack/update-vendor.sh before cook containerd." + exit 1 + fi + # Import cri plugin into containerd. + # TODO(random-liu): Remove this after containerd starts to vendor cri plugin. + echo "import _ \"${CRI_CONTAINERD_PKG}\"" >> cmd/containerd/builtins_linux.go + # 1. Copy all cri-containerd vendors into containerd vendor. This makes sure + # all dependencies introduced by cri-containerd will be updated. There might + # be unnecessary vendors introduced, but it still builds. + cp -rT ${ROOT}/vendor/ vendor/ + # 2. Remove containerd repo itself from vendor. + rm -rf vendor/${CONTAINERD_PKG} + # 3. Create cri-containerd vendor in containerd vendor, and copy the newest + # cri-containerd there. + if [ -d "vendor/${CRI_CONTAINERD_PKG}" ]; then + rm -rf vendor/${CRI_CONTAINERD_PKG}/* + else + mkdir -p vendor/${CRI_CONTAINERD_PKG} + fi + cp -rT ${ROOT} vendor/${CRI_CONTAINERD_PKG} + # 4. Remove the extra vendor in cri-containerd. + rm -rf vendor/${CRI_CONTAINERD_PKG}/vendor + # After the 4 steps above done, we have a containerd with newest cri-containerd + # plugin. +fi +make BUILDTAGS="${BUILDTAGS}" # containerd make install requires `go` to work. Explicitly # set PATH to make sure it can find `go` even with `sudo`. ${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}" diff --git a/hack/test-integration.sh b/hack/test-integration.sh index f2c99ff9f..64cbd97c6 100755 --- a/hack/test-integration.sh +++ b/hack/test-integration.sh @@ -29,7 +29,9 @@ mkdir -p ${REPORT_DIR} test_setup ${REPORT_DIR} # Run integration test. -sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v +# Set STANDALONE_CRI_CONTAINERD so that integration test can see it. +# Some integration test needs the env to skip itself. +sudo STANDALONE_CRI_CONTAINERD=${STANDALONE_CRI_CONTAINERD} ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v test_exit_code=$? test_teardown diff --git a/hack/test-utils.sh b/hack/test-utils.sh index adf4b975d..dea502161 100644 --- a/hack/test-utils.sh +++ b/hack/test-utils.sh @@ -21,6 +21,8 @@ source ${ROOT}/hack/versions CRI_CONTAINERD_FLAGS=${CRI_CONTAINERD_FLAGS:-""} # RESTART_WAIT_PERIOD is the period to wait before restarting cri-containerd/containerd. RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10} +# STANDALONE_CRI_CONTAINERD indicates whether to run standalone cri-containerd. +STANDALONE_CRI_CONTAINERD=${STANDALONE_CRI_CONTAINERD:-true} CRICONTAINERD_SOCK=/var/run/cri-containerd.sock @@ -48,9 +50,11 @@ test_setup() { readiness_check "sudo ctr version" # Start cri-containerd - keepalive "sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 ${CRI_CONTAINERD_FLAGS}" \ - ${RESTART_WAIT_PERIOD} &> ${report_dir}/cri-containerd.log & - cri_containerd_pid=$! + if ${STANDALONE_CRI_CONTAINERD}; then + keepalive "sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 ${CRI_CONTAINERD_FLAGS}" \ + ${RESTART_WAIT_PERIOD} &> ${report_dir}/cri-containerd.log & + cri_containerd_pid=$! + fi readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CRICONTAINERD_SOCK} info" } diff --git a/integration/restart_test.go b/integration/restart_test.go index 768b45fc9..494f92b8e 100644 --- a/integration/restart_test.go +++ b/integration/restart_test.go @@ -31,8 +31,12 @@ import ( ) // Restart test must run sequentially. +// NOTE(random-liu): Current restart test only support standalone cri-containerd mode. func TestSandboxAcrossCRIContainerdRestart(t *testing.T) { + if os.Getenv(standaloneEnvKey) == "false" { + t.Skip("Skip because cri-containerd does not run in standalone mode") + } ctx := context.Background() sandboxNS := "sandbox-restart-cri-containerd" sandboxes := []struct { @@ -149,6 +153,9 @@ func TestSandboxAcrossCRIContainerdRestart(t *testing.T) { // teardown the network properly. // This test uses host network sandbox to avoid resource leakage. func TestSandboxDeletionAcrossCRIContainerdRestart(t *testing.T) { + if os.Getenv(standaloneEnvKey) == "false" { + t.Skip("Skip because cri-containerd does not run in standalone mode") + } ctx := context.Background() sandboxNS := "sandbox-delete-restart-cri-containerd" t.Logf("Make sure no sandbox is running before test") diff --git a/integration/test_utils.go b/integration/test_utils.go index 58080a9fc..06f755033 100644 --- a/integration/test_utils.go +++ b/integration/test_utils.go @@ -41,6 +41,7 @@ const ( containerdEndpoint = "/run/containerd/containerd.sock" criContainerdEndpoint = "/var/run/cri-containerd.sock" criContainerdRoot = "/var/lib/cri-containerd" + standaloneEnvKey = "STANDALONE_CRI_CONTAINERD" ) var ( From 15940c93199dc8bfaf87cf3ea44a09c18abe64be Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 5 Jan 2018 07:07:55 +0000 Subject: [PATCH 3/3] Add travis integration test for plugin mode. Signed-off-by: Lantao Liu --- .travis.yml | 11 +++++++++++ Makefile | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/.travis.yml b/.travis.yml index bcdd794d2..70cadd641 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,11 +32,13 @@ jobs: - make install.tools - make .gitvalidation - make binaries + - make plugin go: 1.9.x - script: - make install.tools - make .gitvalidation - make binaries + - make plugin go: tip - stage: Test script: @@ -51,3 +53,12 @@ jobs: - cat /tmp/test-cri/cri-containerd.log - cat /tmp/test-cri/containerd.log go: 1.9.x + - script: + - make install.deps COOK_CONTAINERD=true + - make test-integration STANDALONE_CRI_CONTAINERD=false + - make test-cri STANDALONE_CRI_CONTAINERD=false + after_script: + # Abuse travis to preserve the log. + - cat /tmp/test-integration/containerd.log + - cat /tmp/test-cri/containerd.log + go: 1.9.x diff --git a/Makefile b/Makefile index 2ee79d3e9..a7cd6d277 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ TARBALL := $(TARBALL_PREFIX)-$(VERSION).$(GOOS)-$(GOARCH).tar.gz BUILD_TAGS := seccomp apparmor GO_LDFLAGS := -X $(PROJECT)/pkg/version.CRIContainerdVersion=$(VERSION) SOURCES := $(shell find cmd/ pkg/ vendor/ -name '*.go') +PLUGIN_SOURCES := $(shell ls *.go) INTEGRATION_SOURCES := $(shell find integration/ -name '*.go') all: binaries @@ -41,6 +42,7 @@ help: @echo @echo " * 'install' - Install binaries to system locations" @echo " * 'binaries' - Build cri-containerd" + @echo " * 'plugin' - Build cri-containerd as a plugin package" @echo " * 'static-binaries - Build static cri-containerd" @echo " * 'release' - Build release tarball" @echo " * 'push' - Push release tarball to GCS" @@ -107,6 +109,13 @@ clean: binaries: $(BUILD_DIR)/cri-containerd +# TODO(random-liu): Make this only build when source files change and +# add this to target all. +plugin: $(PLUGIN_SOURCES) $(SOURCES) + $(GO) build -tags '$(BUILD_TAGS)' \ + -ldflags '$(GO_LDFLAGS)' \ + -gcflags '$(GO_GCFLAGS)' \ + static-binaries: GO_LDFLAGS += -extldflags "-fno-PIC -static" static-binaries: $(BUILD_DIR)/cri-containerd @@ -156,6 +165,7 @@ install.tools: .install.gitvalidation .install.gometalinter .PHONY: \ binaries \ static-binaries \ + plugin \ release \ push \ boiler \