Merge pull request #527 from Random-Liu/add-plugin-mode

Add initial version of plugin mode

.travis.yml

@@ -32,11 +32,13 @@ jobs:
         - make install.tools
         - make .gitvalidation
         - make binaries
+        - make plugin
       go: 1.9.x
     - script:
         - make install.tools
         - make .gitvalidation
         - make binaries
+        - make plugin
       go: tip
     - stage: Test
       script:
@@ -51,3 +53,12 @@ jobs:
         - cat /tmp/test-cri/cri-containerd.log
         - cat /tmp/test-cri/containerd.log
       go: 1.9.x
+    - script:
+        - make install.deps COOK_CONTAINERD=true
+        - make test-integration STANDALONE_CRI_CONTAINERD=false
+        - make test-cri STANDALONE_CRI_CONTAINERD=false
+      after_script:
+        # Abuse travis to preserve the log.
+        - cat /tmp/test-integration/containerd.log
+        - cat /tmp/test-cri/containerd.log
+      go: 1.9.x

Makefile

@@ -30,6 +30,7 @@ TARBALL := $(TARBALL_PREFIX)-$(VERSION).$(GOOS)-$(GOARCH).tar.gz
 BUILD_TAGS := seccomp apparmor
 GO_LDFLAGS := -X $(PROJECT)/pkg/version.CRIContainerdVersion=$(VERSION)
 SOURCES := $(shell find cmd/ pkg/ vendor/ -name '*.go')
+PLUGIN_SOURCES := $(shell ls *.go)
 INTEGRATION_SOURCES := $(shell find integration/ -name '*.go')
 
 all: binaries
@@ -41,6 +42,7 @@ help:
 	@echo
 	@echo " * 'install'          - Install binaries to system locations"
 	@echo " * 'binaries'         - Build cri-containerd"
+	@echo " * 'plugin'           - Build cri-containerd as a plugin package"
 	@echo " * 'static-binaries   - Build static cri-containerd"
 	@echo " * 'release'          - Build release tarball"
 	@echo " * 'push'             - Push release tarball to GCS"
@@ -107,6 +109,13 @@ clean:
 
 binaries: $(BUILD_DIR)/cri-containerd
 
+# TODO(random-liu): Make this only build when source files change and
+# add this to target all.
+plugin: $(PLUGIN_SOURCES) $(SOURCES)
+	$(GO) build -tags '$(BUILD_TAGS)' \
+		-ldflags '$(GO_LDFLAGS)' \
+		-gcflags '$(GO_GCFLAGS)' \
+
 static-binaries: GO_LDFLAGS += -extldflags "-fno-PIC -static"
 static-binaries: $(BUILD_DIR)/cri-containerd
 
@@ -156,6 +165,7 @@ install.tools: .install.gitvalidation .install.gometalinter
 .PHONY: \
 	binaries \
 	static-binaries \
+	plugin \
 	release \
 	push \
 	boiler \

cmd/cri-containerd/options/options.go

@@ -116,7 +116,7 @@ func NewCRIContainerdOptions() *CRIContainerdOptions {
 
 // AddFlags adds cri-containerd command line options to pflag.
 func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) {
-	defaults := defaultConfig()
+	defaults := DefaultConfig()
 	fs.StringVar(&c.ConfigFilePath, configFilePathArgName,
 		defaultConfigFilePath, "Path to the config file.")
 	fs.StringVar(&c.SocketPath, "socket-path",
@@ -192,7 +192,7 @@ func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error {
 
 // PrintDefaultTomlConfig print default toml config of cri-containerd.
 func PrintDefaultTomlConfig() {
-	if err := toml.NewEncoder(os.Stdout).Encode(defaultConfig()); err != nil {
+	if err := toml.NewEncoder(os.Stdout).Encode(DefaultConfig()); err != nil {
 		fmt.Println(err)
 		return
 	}
@@ -200,13 +200,13 @@ func PrintDefaultTomlConfig() {
 
 // AddGRPCFlags add flags for grpc connection.
 func AddGRPCFlags(fs *pflag.FlagSet) (*string, *time.Duration) {
-	endpoint := fs.String("endpoint", defaultConfig().SocketPath, "cri-containerd endpoint.")
+	endpoint := fs.String("endpoint", DefaultConfig().SocketPath, "cri-containerd endpoint.")
 	timeout := fs.Duration("timeout", connectionTimeout, "cri-containerd connection timeout.")
 	return endpoint, timeout
 }
 
-// defaultConfig returns default configurations of cri-containerd.
-func defaultConfig() Config {
+// DefaultConfig returns default configurations of cri-containerd.
+func DefaultConfig() Config {
 	return Config{
 		ContainerdConfig: ContainerdConfig{
 			RootDir:       "/var/lib/containerd",

cri.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2018 The Containerd Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cri
+
+import (
+	"github.com/containerd/containerd/plugin"
+	"github.com/golang/glog"
+
+	"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
+	"github.com/containerd/cri-containerd/pkg/server"
+)
+
+// Register CRI service plugin
+func init() {
+	plugin.Register(&plugin.Registration{
+		// In fact, cri is not strictly a GRPC plugin now.
+		Type: plugin.GRPCPlugin,
+		ID:   "cri",
+		Requires: []plugin.Type{
+			plugin.RuntimePlugin,
+			plugin.SnapshotPlugin,
+			plugin.TaskMonitorPlugin,
+			plugin.DiffPlugin,
+			plugin.MetadataPlugin,
+			plugin.ContentPlugin,
+			plugin.GCPlugin,
+		},
+		InitFn: initCRIService,
+	})
+}
+
+func initCRIService(_ *plugin.InitContext) (interface{}, error) {
+	// TODO(random-liu): Support Config through Registration.Config.
+	// TODO(random-liu): Validate the configuration.
+	// TODO(random-liu): Leverage other fields in InitContext, such as Root.
+	// TODO(random-liu): Register GRPC service onto containerd GRPC server.
+	// TODO(random-liu): Separate cri plugin config from cri-containerd server config,
+	// because many options only make sense to cri-containerd server.
+	// TODO(random-liu): Change all glog to logrus.
+	// TODO(random-liu): Handle graceful stop.
+	c := options.DefaultConfig()
+	glog.V(0).Infof("Start cri plugin with config %+v", c)
+	// Use a goroutine to start cri service. The reason is that currently
+	// cri service requires containerd to be running.
+	// TODO(random-liu): Resolve the circular dependency.
+	go func() {
+		s, err := server.NewCRIContainerdService(c)
+		if err != nil {
+			glog.Exitf("Failed to create CRI service: %v", err)
+		}
+		if err := s.Run(); err != nil {
+			glog.Exitf("Failed to run CRI grpc server: %v", err)
+		}
+	}()
+	return nil, nil
+}

hack/install-deps.sh

@@ -50,6 +50,11 @@ fi
 # and configurations in cluster.
 INSTALL_CNI=${INSTALL_CNI:-true}
 
+# COOK_CONTAINERD indicates whether to update containerd with newest
+# cri plugin before install. This is mainly used for testing new cri
+# plugin change.
+COOK_CONTAINERD=${COOK_CONTAINERD:-false}
+
 CONTAINERD_DIR=${DESTDIR}/usr/local
 RUNC_DIR=${DESTDIR}
 CNI_DIR=${DESTDIR}/opt/cni
@@ -61,6 +66,7 @@ RUNC_PKG=github.com/opencontainers/runc
 CNI_PKG=github.com/containernetworking/plugins
 CONTAINERD_PKG=github.com/containerd/containerd
 CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools
+CRI_CONTAINERD_PKG=github.com/containerd/cri-containerd
 
 # Create a temporary GOPATH for make install.deps.
 TMPGOPATH=$(mktemp -d /tmp/cri-containerd-install-deps.XXXX)
@@ -73,9 +79,9 @@ GOPATH=${TMPGOPATH}
 # 2) Version.
 # 3) Repo name (optional);
 checkout_repo() {
-  pkg=$1
-  version=$2
-  repo=${3:-""}
+  local -r pkg=$1
+  local -r version=$2
+  local repo=${3:-""}
   if [ -z "${repo}" ]; then
     repo=${pkg}
   fi
@@ -134,7 +140,36 @@ fi
 # Install containerd
 checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO}
 cd ${GOPATH}/src/${CONTAINERD_PKG}
-make
+if ${COOK_CONTAINERD}; then
+  # Verify that vendor.conf is in sync with containerd before cook containerd,
+  # this is a hard requirement.
+  if ! ${ROOT}/hack/update-vendor.sh -only-verify; then
+    echo "Please run hack/update-vendor.sh before cook containerd."
+    exit 1
+  fi
+  # Import cri plugin into containerd.
+  # TODO(random-liu): Remove this after containerd starts to vendor cri plugin.
+  echo "import _ \"${CRI_CONTAINERD_PKG}\"" >> cmd/containerd/builtins_linux.go
+  # 1. Copy all cri-containerd vendors into containerd vendor. This makes sure
+  # all dependencies introduced by cri-containerd will be updated. There might
+  # be unnecessary vendors introduced, but it still builds.
+  cp -rT ${ROOT}/vendor/ vendor/
+  # 2. Remove containerd repo itself from vendor.
+  rm -rf vendor/${CONTAINERD_PKG}
+  # 3. Create cri-containerd vendor in containerd vendor, and copy the newest
+  # cri-containerd there.
+  if [ -d "vendor/${CRI_CONTAINERD_PKG}" ]; then
+    rm -rf vendor/${CRI_CONTAINERD_PKG}/*
+  else
+    mkdir -p vendor/${CRI_CONTAINERD_PKG}
+  fi
+  cp -rT ${ROOT} vendor/${CRI_CONTAINERD_PKG}
+  # 4. Remove the extra vendor in cri-containerd.
+  rm -rf vendor/${CRI_CONTAINERD_PKG}/vendor
+  # After the 4 steps above done, we have a containerd with newest cri-containerd
+  # plugin.
+fi
+make BUILDTAGS="${BUILDTAGS}"
 # containerd make install requires `go` to work. Explicitly
 # set PATH to make sure it can find `go` even with `sudo`.
 ${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}"

hack/test-integration.sh

@@ -29,7 +29,9 @@ mkdir -p ${REPORT_DIR}
 test_setup ${REPORT_DIR}
 
 # Run integration test.
-sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v
+# Set STANDALONE_CRI_CONTAINERD so that integration test can see it.
+# Some integration test needs the env to skip itself.
+sudo STANDALONE_CRI_CONTAINERD=${STANDALONE_CRI_CONTAINERD} ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v
 test_exit_code=$?
 
 test_teardown

hack/test-utils.sh

@@ -21,6 +21,8 @@ source ${ROOT}/hack/versions
 CRI_CONTAINERD_FLAGS=${CRI_CONTAINERD_FLAGS:-""}
 # RESTART_WAIT_PERIOD is the period to wait before restarting cri-containerd/containerd.
 RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10}
+# STANDALONE_CRI_CONTAINERD indicates whether to run standalone cri-containerd.
+STANDALONE_CRI_CONTAINERD=${STANDALONE_CRI_CONTAINERD:-true}
 
 CRICONTAINERD_SOCK=/var/run/cri-containerd.sock
 
@@ -48,9 +50,11 @@ test_setup() {
   readiness_check "sudo ctr version"
 
   # Start cri-containerd
+  if ${STANDALONE_CRI_CONTAINERD}; then
     keepalive "sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 ${CRI_CONTAINERD_FLAGS}" \
       ${RESTART_WAIT_PERIOD} &> ${report_dir}/cri-containerd.log &
     cri_containerd_pid=$!
+  fi
   readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CRICONTAINERD_SOCK} info"
 }
 

integration/restart_test.go

@@ -31,8 +31,12 @@ import (
 )
 
 // Restart test must run sequentially.
+// NOTE(random-liu): Current restart test only support standalone cri-containerd mode.
 
 func TestSandboxAcrossCRIContainerdRestart(t *testing.T) {
+	if os.Getenv(standaloneEnvKey) == "false" {
+		t.Skip("Skip because cri-containerd does not run in standalone mode")
+	}
 	ctx := context.Background()
 	sandboxNS := "sandbox-restart-cri-containerd"
 	sandboxes := []struct {
@@ -149,6 +153,9 @@ func TestSandboxAcrossCRIContainerdRestart(t *testing.T) {
 // teardown the network properly.
 // This test uses host network sandbox to avoid resource leakage.
 func TestSandboxDeletionAcrossCRIContainerdRestart(t *testing.T) {
+	if os.Getenv(standaloneEnvKey) == "false" {
+		t.Skip("Skip because cri-containerd does not run in standalone mode")
+	}
 	ctx := context.Background()
 	sandboxNS := "sandbox-delete-restart-cri-containerd"
 	t.Logf("Make sure no sandbox is running before test")

integration/test_utils.go

@@ -41,6 +41,7 @@ const (
 	containerdEndpoint    = "/run/containerd/containerd.sock"
 	criContainerdEndpoint = "/var/run/cri-containerd.sock"
 	criContainerdRoot     = "/var/lib/cri-containerd"
+	standaloneEnvKey      = "STANDALONE_CRI_CONTAINERD"
 )
 
 var (