From d1e9960180bc49fa5aee12f84d1fcd8730ab3b93 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 5 Mar 2018 19:13:29 +0000 Subject: [PATCH 1/4] Remove standalone mode Signed-off-by: Lantao Liu --- cmd/cri-containerd/cri_containerd.go | 229 --------------------- cmd/cri-containerd/options/options.go | 273 ------------------------- cmd/cri-containerd/options/registry.go | 88 -------- cmd/ctrcri/ctrcri.go | 6 +- cri.go | 38 +++- pkg/config/config.go | 127 ++++++++++++ pkg/server/service.go | 65 +----- pkg/server/service_test.go | 6 +- 8 files changed, 173 insertions(+), 659 deletions(-) delete mode 100644 cmd/cri-containerd/cri_containerd.go delete mode 100644 cmd/cri-containerd/options/options.go delete mode 100644 cmd/cri-containerd/options/registry.go create mode 100644 pkg/config/config.go diff --git a/cmd/cri-containerd/cri_containerd.go b/cmd/cri-containerd/cri_containerd.go deleted file mode 100644 index 9b40cda19..000000000 --- a/cmd/cri-containerd/cri_containerd.go +++ /dev/null @@ -1,229 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "net" - "net/http" - "net/http/pprof" - "os" - "os/signal" - "runtime" - "syscall" - - "github.com/containerd/cgroups" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/sys" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "k8s.io/kubernetes/pkg/util/interrupt" - - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" - "github.com/containerd/cri-containerd/pkg/server" - "github.com/containerd/cri-containerd/pkg/version" -) - -// Add \u200B to avoid the space trimming. -const desc = "\u200B" + ` _ __ _ __ - __________(_) _________ ____ / /_____ _(_)____ ___ _________/ / - / ___/ ___/ /______/ ___/ __ \/ __ \/ __/ __ ` + "`" + `/ // __ \/ _ \/ ___/ __ / -/ /__/ / / //_____/ /__/ /_/ / / / / /_/ /_/ / // / / / __/ / / /_/ / -\___/_/ /_/ \___/\____/_/ /_/\__/\__,_/_//_/ /_/\___/_/ \__,_/ - -A containerd based Kubernetes CRI implementation. -` - -var cmd = &cobra.Command{ - Use: "cri-containerd", - Short: "A containerd based Kubernetes CRI implementation.", - Long: desc, -} - -func defaultConfigCommand() *cobra.Command { - return &cobra.Command{ - Use: "default-config", - Short: "Print default toml config of cri-containerd.", - Run: func(cmd *cobra.Command, args []string) { - options.PrintDefaultTomlConfig() - }, - } -} - -func versionCommand() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Print cri-containerd version information.", - Run: func(cmd *cobra.Command, args []string) { - version.PrintVersion() - }, - } -} - -func main() { - o := options.NewCRIContainerdOptions() - - o.AddFlags(cmd.Flags()) - cmd.AddCommand(defaultConfigCommand()) - cmd.AddCommand(versionCommand()) - - cmd.RunE = func(cmd *cobra.Command, args []string) error { - setupDumpStacksTrap() - if err := o.InitFlags(cmd.Flags()); err != nil { - return fmt.Errorf("failed to init CRI containerd flags: %v", err) - } - - if err := setLogLevel(o.LogLevel); err != nil { - return fmt.Errorf("failed to set log level: %v", err) - } - - logrus.Infof("Run cri-containerd %+v", o) - - if o.CgroupPath != "" { - _, err := loadCgroup(o.CgroupPath) - if err != nil { - return fmt.Errorf("failed to load cgroup for cgroup path %v: %v", o.CgroupPath, err) - } - } - - if o.OOMScore != 0 { - if err := sys.SetOOMScore(os.Getpid(), o.OOMScore); err != nil { - return fmt.Errorf("failed to set OOMScore to %v: %v", o.OOMScore, err) - } - } - - // Start profiling server if enable. - if o.EnableProfiling { - logrus.Info("Start profiling server") - go startProfilingServer(o.ProfilingAddress, o.ProfilingPort) - } - - logrus.Infof("Run cri-containerd grpc server on socket %q", o.SocketPath) - s, err := server.NewCRIContainerdService(o.Config) - if err != nil { - return fmt.Errorf("failed to create CRI containerd service: %v", err) - } - // Use interrupt handler to make sure the server is stopped properly. - // Pass in non-empty final function to avoid os.Exit(1). We expect `Run` - // to return itself. - h := interrupt.New(func(os.Signal) {}, func() { - if err := s.Close(); err != nil { - logrus.WithError(err).Error("Failed to stop cri service") - } - }) - if err := h.Run(func() error { return s.Run(true) }); err != nil { - return fmt.Errorf("failed to run cri-containerd with grpc server: %v", err) - } - return nil - } - - if err := cmd.Execute(); err != nil { - // Error should have been reported. - os.Exit(1) - } -} - -func setupDumpStacksTrap() { - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGUSR1) - go func() { - for range c { - dumpStacks() - } - }() -} - -func dumpStacks() { - buf := make([]byte, 1024) - for { - n := runtime.Stack(buf, true) - if n < len(buf) { - buf = buf[:n] - break - } - buf = make([]byte, 2*len(buf)) - } - logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) -} - -// startProfilingServer start http server to profiling via web interface -func startProfilingServer(host string, port string) { - endpoint := net.JoinHostPort(host, port) - mux := http.NewServeMux() - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - if err := http.ListenAndServe(endpoint, mux); err != nil { - logrus.WithError(err).Error("Failed to start profiling server") - } -} - -func setLogLevel(l string) error { - lvl, err := log.ParseLevel(l) - if err != nil { - return err - } - if err := setGLogLevel(lvl); err != nil { - return err - } - logrus.SetLevel(lvl) - return nil -} - -// TODO(random-liu): Set glog level in plugin mode. -func setGLogLevel(l logrus.Level) error { - if err := flag.Set("logtostderr", "true"); err != nil { - return err - } - switch l { - case log.TraceLevel: - return flag.Set("v", "5") - case logrus.DebugLevel: - return flag.Set("v", "4") - case logrus.InfoLevel: - return flag.Set("v", "2") - // glog doesn't support following filters. Defaults to v=0. - case logrus.WarnLevel: - case logrus.ErrorLevel: - case logrus.FatalLevel: - case logrus.PanicLevel: - } - return nil -} - -// loadCgroup loads the cgroup associated with path if it exists and moves the current process into the cgroup. If the cgroup -// is not created it is created and returned. -func loadCgroup(cgroupPath string) (cgroups.Cgroup, error) { - cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(cgroupPath)) - if err != nil { - if err != cgroups.ErrCgroupDeleted { - return nil, err - } - if cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(cgroupPath), &runtimespec.LinuxResources{}); err != nil { - return nil, err - } - } - if err := cg.Add(cgroups.Process{ - Pid: os.Getpid(), - }); err != nil { - return nil, err - } - return cg, nil -} diff --git a/cmd/cri-containerd/options/options.go b/cmd/cri-containerd/options/options.go deleted file mode 100644 index 3f1edfecb..000000000 --- a/cmd/cri-containerd/options/options.go +++ /dev/null @@ -1,273 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "os" - - "github.com/BurntSushi/toml" - "github.com/containerd/containerd" - "github.com/spf13/pflag" -) - -const ( - // configFilePathArgName is the path to the config file. - configFilePathArgName = "config" - // defaultConfigFilePath is the default config file path. - defaultConfigFilePath = "/etc/cri-containerd/config.toml" -) - -// ContainerdConfig contains toml config related to containerd -type ContainerdConfig struct { - // RootDir is the root directory path for containerd. - // TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode. - RootDir string `toml:"root_dir" json:"rootDir,omitempty"` - // Snapshotter is the snapshotter used by containerd. - Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"` - // Endpoint is the containerd endpoint path. - // TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode. - Endpoint string `toml:"endpoint" json:"endpoint,omitempty"` - // Runtime is the runtime to use in containerd. We may support - // other runtimes in the future. - Runtime string `toml:"runtime" json:"runtime,omitempty"` - // RuntimeEngine is the name of the runtime engine used by containerd. - // Containerd default should be "runc" - // We may support other runtime engines in the future. - RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"` - // RuntimeRoot is the directory used by containerd for runtime state. - // Containerd default should be "/run/containerd/runc" - RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"` -} - -// CniConfig contains toml config related to cni -type CniConfig struct { - // NetworkPluginBinDir is the directory in which the binaries for the plugin is kept. - NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"` - // NetworkPluginConfDir is the directory in which the admin places a CNI conf. - NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"` -} - -// PluginConfig contains toml config related to CRI plugin, -// it is a subset of Config. -type PluginConfig struct { - // ContainerdConfig contains config related to containerd - ContainerdConfig `toml:"containerd" json:"containerd,omitempty"` - // CniConfig contains config related to cni - CniConfig `toml:"cni" json:"cni,omitempty"` - // Registry contains config related to the registry - Registry `toml:"registry" json:"registry,omitempty"` - // StreamServerAddress is the ip address streaming server is listening on. - StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"` - // StreamServerPort is the port streaming server is listening on. - StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"` - // EnableSelinux indicates to enable the selinux support. - EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"` - // SandboxImage is the image used by sandbox container. - SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"` - // StatsCollectPeriod is the period (in seconds) of snapshots stats collection. - StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"` - // SystemdCgroup enables systemd cgroup support. - SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"` - // EnableIPv6DAD enables IPv6 DAD. - // TODO(random-liu): Use optimistic_dad when it's GA. - EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"` -} - -// Config contains toml config related cri-containerd daemon. -// TODO(random-liu): Make this an internal config object when we no longer support cri-containerd -// standalone mode. At that time, we can clean this up. -type Config struct { - // PluginConfig is the config for CRI plugin. - PluginConfig - // ContainerdRootDir is the root directory path for containerd. - ContainerdRootDir string `toml:"-" json:"containerdRootDir,omitempty"` - // ContainerdEndpoint is the containerd endpoint path. - ContainerdEndpoint string `toml:"-" json:"containerdEndpoint,omitempty"` - // SocketPath is the path to the socket which cri-containerd serves on. - // TODO(random-liu): Remove SocketPath when no longer support cri-containerd - // standalone mode. - SocketPath string `toml:"socket_path" json:"socketPath,omitempty"` - // RootDir is the root directory path for managing cri-containerd files - // (metadata checkpoint etc.) - RootDir string `toml:"root_dir" json:"rootDir,omitempty"` - // TODO(random-liu): Remove following fields when we no longer support cri-containerd - // standalone mode. - // CgroupPath is the path for the cgroup that cri-containerd is placed in. - CgroupPath string `toml:"cgroup_path" json:"cgroupPath,omitempty"` - // OOMScore adjust the cri-containerd's oom score - OOMScore int `toml:"oom_score" json:"oomScore,omitempty"` - // EnableProfiling is used for enable profiling via host:port/debug/pprof/ - EnableProfiling bool `toml:"profiling" json:"enableProfiling,omitempty"` - // ProfilingPort is the port for profiling via host:port/debug/pprof/ - ProfilingPort string `toml:"profiling_port" json:"profilingPort,omitempty"` - // ProfilingAddress is address for profiling via host:port/debug/pprof/ - ProfilingAddress string `toml:"profiling_addr" json:"profilingAddress,omitempty"` - // LogLevel is the logrus log level. - LogLevel string `toml:"log_level" json:"logLevel,omitempty"` -} - -// CRIContainerdOptions contains cri-containerd command line and toml options. -type CRIContainerdOptions struct { - // Config contains cri-containerd toml config - Config - // ConfigFilePath is the path to the TOML config file. - ConfigFilePath string `toml:"-"` -} - -// NewCRIContainerdOptions returns a reference to CRIContainerdOptions -func NewCRIContainerdOptions() *CRIContainerdOptions { - return &CRIContainerdOptions{} -} - -// AddFlags adds cri-containerd command line options to pflag. -func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) { - defaults := DefaultConfig() - fs.StringVar(&c.ConfigFilePath, configFilePathArgName, - defaultConfigFilePath, "Path to the config file.") - fs.StringVar(&c.LogLevel, "log-level", - defaults.LogLevel, "Set the logging level [trace, debug, info, warn, error, fatal, panic].") - fs.StringVar(&c.SocketPath, "socket-path", - defaults.SocketPath, "Path to the socket which cri-containerd serves on.") - fs.StringVar(&c.RootDir, "root-dir", - defaults.RootDir, "Root directory path for cri-containerd managed files (metadata checkpoint etc).") - fs.StringVar(&c.ContainerdRootDir, "containerd-root-dir", - defaults.ContainerdRootDir, "Root directory path where containerd stores persistent data.") - fs.StringVar(&c.ContainerdEndpoint, "containerd-endpoint", - defaults.ContainerdEndpoint, "Path to the containerd endpoint.") - fs.StringVar(&c.ContainerdConfig.Snapshotter, "containerd-snapshotter", - defaults.ContainerdConfig.Snapshotter, "The snapshotter used by containerd.") - fs.StringVar(&c.ContainerdConfig.Runtime, "containerd-runtime", - defaults.ContainerdConfig.Runtime, "The runtime used by containerd.") - fs.StringVar(&c.ContainerdConfig.RuntimeEngine, "containerd-runtime-engine", - defaults.ContainerdConfig.RuntimeEngine, "Runtime engine used by containerd. Defaults to containerd's default if not specified.") - fs.StringVar(&c.ContainerdConfig.RuntimeRoot, "containerd-runtime-root", - defaults.ContainerdConfig.RuntimeRoot, "The directory used by containerd for runtime state. Defaults to containerd's default if not specified.") - fs.StringVar(&c.NetworkPluginBinDir, "network-bin-dir", - defaults.NetworkPluginBinDir, "The directory for putting network binaries.") - fs.StringVar(&c.NetworkPluginConfDir, "network-conf-dir", - defaults.NetworkPluginConfDir, "The directory for putting network plugin configuration files.") - fs.StringVar(&c.StreamServerAddress, "stream-addr", - defaults.StreamServerAddress, "The ip address streaming server is listening on. The default host interface is used if not specified.") - fs.StringVar(&c.StreamServerPort, "stream-port", - defaults.StreamServerPort, "The port streaming server is listening on.") - fs.StringVar(&c.CgroupPath, "cgroup-path", - defaults.CgroupPath, "The cgroup that cri-containerd is part of. Cri-containerd is not placed in a cgroup if none is specified.") - fs.BoolVar(&c.EnableSelinux, "enable-selinux", - defaults.EnableSelinux, "Enable selinux support. By default not enabled.") - fs.StringVar(&c.SandboxImage, "sandbox-image", - defaults.SandboxImage, "The image used by sandbox container.") - fs.IntVar(&c.StatsCollectPeriod, "stats-collect-period", - defaults.StatsCollectPeriod, "The period (in seconds) of snapshots stats collection.") - fs.BoolVar(&c.SystemdCgroup, "systemd-cgroup", - defaults.SystemdCgroup, "Enables systemd cgroup support. By default not enabled.") - fs.IntVar(&c.OOMScore, "oom-score", - defaults.OOMScore, "Adjust the cri-containerd's oom score.") - fs.BoolVar(&c.EnableProfiling, "profiling", - defaults.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/.") - fs.StringVar(&c.ProfilingPort, "profiling-port", - defaults.ProfilingPort, "Profiling port for web interface host:port/debug/pprof/.") - fs.StringVar(&c.ProfilingAddress, "profiling-addr", - defaults.ProfilingAddress, "Profiling address for web interface host:port/debug/pprof/.") - fs.BoolVar(&c.EnableIPv6DAD, "enable-ipv6-dad", - defaults.EnableIPv6DAD, "Enable IPv6 DAD (duplicate address detection) for pod sandbox network. Enabling this will increase pod sandbox start latency by several seconds.") - fs.Var(&c.Registry, "registry", - "Registry config for image pull eg --registry=myregistry.io=https://mymirror.io/ --registry=myregistry2.io=https://mymirror2.io/") -} - -// InitFlags load configurations from config file, and then overwrite with flags. -// This function must be called inside `Run`, at that time flags should have been -// parsed once. -// precedence: commandline > configfile > default -func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error { - // Load default config file if none provided - if _, err := toml.DecodeFile(c.ConfigFilePath, &c.Config); err != nil { - // the absence of default config file is normal case. - if !fs.Changed(configFilePathArgName) && os.IsNotExist(err) { - return nil - } - return err - } - // Add this for backward compatibility. - // TODO(random-liu): Remove this when we no longer support cri-containerd standalone mode. - if c.ContainerdConfig.RootDir != "" { - c.ContainerdRootDir = c.ContainerdConfig.RootDir - } - if c.ContainerdConfig.Endpoint != "" { - c.ContainerdEndpoint = c.ContainerdConfig.Endpoint - } - - // What is the reason for applying the command line twice? - // Because the values from command line have the highest priority. - // The path of toml configuration file if from the command line, - // and triggers the first parse. - // The first parse generates the default value and the value from command line at the same time. - // But the priority of the toml config value is higher than the default value, - // Without a way to insert the toml config value between the default value and the command line value. - // We parse twice one for default value, one for commandline value. - return fs.Parse(os.Args[1:]) -} - -// PrintDefaultTomlConfig print default toml config of cri-containerd. -func PrintDefaultTomlConfig() { - if err := toml.NewEncoder(os.Stdout).Encode(DefaultConfig()); err != nil { - fmt.Println(err) - return - } -} - -// DefaultConfig returns default configurations of cri-containerd. -func DefaultConfig() Config { - return Config{ - PluginConfig: PluginConfig{ - CniConfig: CniConfig{ - NetworkPluginBinDir: "/opt/cni/bin", - NetworkPluginConfDir: "/etc/cni/net.d", - }, - ContainerdConfig: ContainerdConfig{ - Snapshotter: containerd.DefaultSnapshotter, - Runtime: "io.containerd.runtime.v1.linux", - RuntimeEngine: "", - RuntimeRoot: "", - }, - StreamServerAddress: "", - StreamServerPort: "10010", - EnableSelinux: false, - SandboxImage: "gcr.io/google_containers/pause:3.0", - StatsCollectPeriod: 10, - SystemdCgroup: false, - EnableIPv6DAD: false, - Registry: Registry{ - Mirrors: map[string]Mirror{ - "docker.io": { - Endpoints: []string{"https://registry-1.docker.io"}, - }, - }, - }, - }, - ContainerdRootDir: "/var/lib/containerd", - ContainerdEndpoint: "/run/containerd/containerd.sock", - SocketPath: "/var/run/cri-containerd.sock", - RootDir: "/var/lib/cri-containerd", - CgroupPath: "", - OOMScore: -999, - EnableProfiling: true, - ProfilingPort: "10011", - ProfilingAddress: "127.0.0.1", - LogLevel: "info", - } -} diff --git a/cmd/cri-containerd/options/registry.go b/cmd/cri-containerd/options/registry.go deleted file mode 100644 index 7f057ee4c..000000000 --- a/cmd/cri-containerd/options/registry.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2018 The Containerd Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "net/url" - "strings" -) - -// Mirror contains the config related to the registry mirror -type Mirror struct { - Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"` - // TODO (Abhi) We might need to add auth per namespace. Looks like - // image auth information is passed by kube itself. -} - -// Registry is registry settings configured -type Registry struct { - Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"` -} - -// String returns the string format of registry type -func (r *Registry) String() string { - // Its not used hence return empty string - return "" -} - -// Set validates and converts into the internal registry struct -func (r *Registry) Set(s string) error { - // --registry docker.io=https://mymirror.io,http://mymirror2.io - // If no option is set then return format error - if len(s) == 0 { - return fmt.Errorf("incomplete registry mirror option") - } - var mirrors []string - host := "docker.io" - opt := strings.Split(s, "=") - if len(opt) > 1 { - // If option is set in the format "mynamespace.io=https://mymirror.io,https://mymirror2.io" - // Then associate the mirror urls for the namespace only" - host = opt[0] - mirrors = strings.Split(opt[1], ",") - } else { - // If option is set in the format "https://mymirror.io,https://mymirror.io" - // Then associate mirror against default docker.io namespace - mirrors = strings.Split(opt[0], ",") - } - - // Validate the format of the urls passed - for _, u := range mirrors { - _, err := url.Parse(u) - if err != nil { - return fmt.Errorf("invalid registry mirror url format %v: %v", u, err) - } - } - - if r.Mirrors == nil { - r.Mirrors = make(map[string]Mirror) - } - if _, ok := r.Mirrors[host]; !ok { - r.Mirrors[host] = Mirror{} - } - m := r.Mirrors[host] - m.Endpoints = append(m.Endpoints, mirrors...) - r.Mirrors[host] = m - - return nil -} - -// Type returns a string name for the option type -func (r *Registry) Type() string { - return "list" -} diff --git a/cmd/ctrcri/ctrcri.go b/cmd/ctrcri/ctrcri.go index 63276cf75..67811d4c3 100644 --- a/cmd/ctrcri/ctrcri.go +++ b/cmd/ctrcri/ctrcri.go @@ -20,10 +20,10 @@ import ( "os" "time" + "github.com/containerd/containerd/defaults" "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" "github.com/containerd/cri-containerd/pkg/version" ) @@ -56,9 +56,7 @@ var ( ) func addGlobalFlags(fs *pflag.FlagSet) { - // TODO(random-liu): Change default to containerd/defaults.DefaultAddress after cri plugin - // become default. - fs.StringVar(&address, "address", options.DefaultConfig().SocketPath, "address for containerd's GRPC server.") + fs.StringVar(&address, "address", defaults.DefaultAddress, "address for containerd's GRPC server.") fs.DurationVar(&timeout, "timeout", defaultTimeout, "timeout for containerd grpc connection.") } diff --git a/cri.go b/cri.go index fdcdf81cb..4da21e910 100644 --- a/cri.go +++ b/cri.go @@ -17,6 +17,7 @@ limitations under the License. package cri import ( + "flag" "path/filepath" "github.com/containerd/containerd/log" @@ -24,8 +25,9 @@ import ( "github.com/containerd/containerd/plugin" imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" + criconfig "github.com/containerd/cri-containerd/pkg/config" "github.com/containerd/cri-containerd/pkg/server" ) @@ -35,7 +37,7 @@ const criVersion = "v1alpha2" // TODO(random-liu): Use github.com/pkg/errors for our errors. // Register CRI service plugin func init() { - config := options.DefaultConfig().PluginConfig + config := criconfig.DefaultConfig() plugin.Register(&plugin.Registration{ Type: plugin.GRPCPlugin, ID: "cri", @@ -57,8 +59,8 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()} ic.Meta.Exports = map[string]string{"CRIVersion": criVersion} ctx := ic.Context - pluginConfig := ic.Config.(*options.PluginConfig) - c := options.Config{ + pluginConfig := ic.Config.(*criconfig.PluginConfig) + c := criconfig.Config{ PluginConfig: *pluginConfig, // This is a hack. We assume that containerd root directory // is one level above plugin directory. @@ -69,6 +71,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { } log.G(ctx).Infof("Start cri plugin with config %+v", c) + if err := setGLogLevel(); err != nil { + return nil, errors.Wrap(err, "failed to set glog level") + } + s, err := server.NewCRIContainerdService(c) if err != nil { return nil, errors.Wrap(err, "failed to create CRI service") @@ -77,10 +83,32 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { // Use a goroutine to initialize cri service. The reason is that currently // cri service requires containerd to be initialize. go func() { - if err := s.Run(false); err != nil { + if err := s.Run(); err != nil { log.G(ctx).WithError(err).Fatal("Failed to run CRI service") } // TODO(random-liu): Whether and how we can stop containerd. }() return s, nil } + +// Set glog level. +func setGLogLevel() error { + l := logrus.GetLevel() + if err := flag.Set("logtostderr", "true"); err != nil { + return err + } + switch l { + case log.TraceLevel: + return flag.Set("v", "5") + case logrus.DebugLevel: + return flag.Set("v", "4") + case logrus.InfoLevel: + return flag.Set("v", "2") + // glog doesn't support following filters. Defaults to v=0. + case logrus.WarnLevel: + case logrus.ErrorLevel: + case logrus.FatalLevel: + case logrus.PanicLevel: + } + return nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 000000000..102be9f14 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,127 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import "github.com/containerd/containerd" + +// ContainerdConfig contains toml config related to containerd +type ContainerdConfig struct { + // Snapshotter is the snapshotter used by containerd. + Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"` + // Runtime is the runtime to use in containerd. We may support + // other runtimes in the future. + Runtime string `toml:"runtime" json:"runtime,omitempty"` + // RuntimeEngine is the name of the runtime engine used by containerd. + // Containerd default should be "runc" + // We may support other runtime engines in the future. + RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"` + // RuntimeRoot is the directory used by containerd for runtime state. + // Containerd default should be "/run/containerd/runc" + RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"` +} + +// CniConfig contains toml config related to cni +type CniConfig struct { + // NetworkPluginBinDir is the directory in which the binaries for the plugin is kept. + NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"` + // NetworkPluginConfDir is the directory in which the admin places a CNI conf. + NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"` +} + +// Mirror contains the config related to the registry mirror +type Mirror struct { + // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints + // one by one until a working one is found. + Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"` + // TODO (Abhi) We might need to add auth per namespace. Looks like + // image auth information is passed by kube itself. +} + +// Registry is registry settings configured +type Registry struct { + // Mirrors are namespace to mirror mapping for all namespaces. + Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"` +} + +// PluginConfig contains toml config related to CRI plugin, +// it is a subset of Config. +type PluginConfig struct { + // ContainerdConfig contains config related to containerd + ContainerdConfig `toml:"containerd" json:"containerd,omitempty"` + // CniConfig contains config related to cni + CniConfig `toml:"cni" json:"cni,omitempty"` + // Registry contains config related to the registry + Registry `toml:"registry" json:"registry,omitempty"` + // StreamServerAddress is the ip address streaming server is listening on. + StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"` + // StreamServerPort is the port streaming server is listening on. + StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"` + // EnableSelinux indicates to enable the selinux support. + EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"` + // SandboxImage is the image used by sandbox container. + SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"` + // StatsCollectPeriod is the period (in seconds) of snapshots stats collection. + StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"` + // SystemdCgroup enables systemd cgroup support. + SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"` + // EnableIPv6DAD enables IPv6 DAD. + // TODO(random-liu): Use optimistic_dad when it's GA. + EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"` +} + +// Config contains all configurations for cri server. +type Config struct { + // PluginConfig is the config for CRI plugin. + PluginConfig + // ContainerdRootDir is the root directory path for containerd. + ContainerdRootDir string `json:"containerdRootDir,omitempty"` + // ContainerdEndpoint is the containerd endpoint path. + ContainerdEndpoint string `json:"containerdEndpoint,omitempty"` + // RootDir is the root directory path for managing cri-containerd files + // (metadata checkpoint etc.) + RootDir string `json:"rootDir,omitempty"` +} + +// DefaultConfig returns default configurations of cri plugin. +func DefaultConfig() PluginConfig { + return PluginConfig{ + CniConfig: CniConfig{ + NetworkPluginBinDir: "/opt/cni/bin", + NetworkPluginConfDir: "/etc/cni/net.d", + }, + ContainerdConfig: ContainerdConfig{ + Snapshotter: containerd.DefaultSnapshotter, + Runtime: "io.containerd.runtime.v1.linux", + RuntimeEngine: "", + RuntimeRoot: "", + }, + StreamServerAddress: "", + StreamServerPort: "10010", + EnableSelinux: false, + SandboxImage: "gcr.io/google_containers/pause:3.0", + StatsCollectPeriod: 10, + SystemdCgroup: false, + EnableIPv6DAD: false, + Registry: Registry{ + Mirrors: map[string]Mirror{ + "docker.io": { + Endpoints: []string{"https://registry-1.docker.io"}, + }, + }, + }, + } +} diff --git a/pkg/server/service.go b/pkg/server/service.go index f502709db..f5930800e 100644 --- a/pkg/server/service.go +++ b/pkg/server/service.go @@ -19,10 +19,7 @@ package server import ( "fmt" "io" - "net" - "os" "path/filepath" - "syscall" "time" "github.com/containerd/containerd" @@ -37,9 +34,9 @@ import ( runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/server/streaming" - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" api "github.com/containerd/cri-containerd/pkg/api/v1" "github.com/containerd/cri-containerd/pkg/atomic" + criconfig "github.com/containerd/cri-containerd/pkg/config" osinterface "github.com/containerd/cri-containerd/pkg/os" "github.com/containerd/cri-containerd/pkg/registrar" containerstore "github.com/containerd/cri-containerd/pkg/store/container" @@ -48,12 +45,8 @@ import ( snapshotstore "github.com/containerd/cri-containerd/pkg/store/snapshot" ) -const ( - // k8sContainerdNamespace is the namespace we use to connect containerd. - k8sContainerdNamespace = "k8s.io" - // unixProtocol is the network protocol of unix socket. - unixProtocol = "unix" -) +// k8sContainerdNamespace is the namespace we use to connect containerd. +const k8sContainerdNamespace = "k8s.io" // grpcServices are all the grpc services provided by cri containerd. type grpcServices interface { @@ -64,7 +57,7 @@ type grpcServices interface { // CRIContainerdService is the interface implement CRI remote service server. type CRIContainerdService interface { - Run(bool) error + Run() error // io.Closer is used by containerd to gracefully stop cri service. io.Closer plugin.Service @@ -74,15 +67,13 @@ type CRIContainerdService interface { // criContainerdService implements CRIContainerdService. type criContainerdService struct { // config contains all configurations. - config options.Config + config criconfig.Config // imageFSPath is the path to image filesystem. imageFSPath string // apparmorEnabled indicates whether apparmor is enabled. apparmorEnabled bool // seccompEnabled indicates whether seccomp is enabled. seccompEnabled bool - // server is the grpc server. - server *grpc.Server // os is an interface for all required os operations. os osinterface.OS // sandboxStore stores all resources associated with sandboxes. @@ -113,7 +104,7 @@ type criContainerdService struct { } // NewCRIContainerdService returns a new instance of CRIContainerdService -func NewCRIContainerdService(config options.Config) (CRIContainerdService, error) { +func NewCRIContainerdService(config criconfig.Config) (CRIContainerdService, error) { var err error c := &criContainerdService{ config: config, @@ -153,12 +144,6 @@ func NewCRIContainerdService(config options.Config) (CRIContainerdService, error c.eventMonitor = newEventMonitor(c.containerStore, c.sandboxStore) - // To avoid race condition between `Run` and `Stop`, still create grpc server - // although we may not use it. It's just a small in-memory data structure. - // TODO(random-liu): Get rid of the grpc server when completely switch - // to plugin mode. - c.server = grpc.NewServer() - return c, nil } @@ -172,11 +157,8 @@ func (c *criContainerdService) Register(s *grpc.Server) error { return nil } -// Run starts the cri-containerd service. startGRPC specifies -// whether to start grpc server in this function. -// TODO(random-liu): Remove `startRPC=true` case when we no longer support cri-containerd -// standalone mode. -func (c *criContainerdService) Run(startGRPC bool) error { +// Run starts the cri-containerd service. +func (c *criContainerdService) Run() error { logrus.Info("Start cri-containerd service") // Connect containerd service here, to get rid of the containerd dependency @@ -226,35 +208,10 @@ func (c *criContainerdService) Run(startGRPC bool) error { // Set the server as initialized. GRPC services could start serving traffic. c.initialized.Set() - grpcServerCloseCh := make(chan struct{}) - if startGRPC { - // Create the grpc server and register runtime and image services. - c.Register(c.server) // nolint: errcheck - // Start grpc server. - // Unlink to cleanup the previous socket file. - logrus.Info("Start grpc server") - err := syscall.Unlink(c.config.SocketPath) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to unlink socket file %q: %v", c.config.SocketPath, err) - } - l, err := net.Listen(unixProtocol, c.config.SocketPath) - if err != nil { - return fmt.Errorf("failed to listen on %q: %v", c.config.SocketPath, err) - } - go func() { - if err := c.server.Serve(l); err != nil { - logrus.WithError(err).Error("Failed to serve grpc request") - } - close(grpcServerCloseCh) - }() - } - // Keep grpcServerCloseCh open if grpc server is not started. - // Stop the whole cri-containerd service if any of the critical service exits. select { case <-eventMonitorCloseCh: case <-streamServerCloseCh: - case <-grpcServerCloseCh: } if err := c.Close(); err != nil { return fmt.Errorf("failed to stop cri service: %v", err) @@ -277,11 +234,6 @@ func (c *criContainerdService) Run(startGRPC bool) error { case <-time.After(streamServerStopTimeout): logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout) } - if startGRPC { - // Only wait for grpc server close channel when grpc server is started. - <-grpcServerCloseCh - logrus.Info("GRPC server stopped") - } return nil } @@ -293,7 +245,6 @@ func (c *criContainerdService) Close() error { if err := c.streamServer.Stop(); err != nil { return fmt.Errorf("failed to stop stream server: %v", err) } - c.server.Stop() return nil } diff --git a/pkg/server/service_test.go b/pkg/server/service_test.go index 0743ee4d4..51bdf34d8 100644 --- a/pkg/server/service_test.go +++ b/pkg/server/service_test.go @@ -17,7 +17,7 @@ limitations under the License. package server import ( - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" + criconfig "github.com/containerd/cri-containerd/pkg/config" ostesting "github.com/containerd/cri-containerd/pkg/os/testing" "github.com/containerd/cri-containerd/pkg/registrar" servertesting "github.com/containerd/cri-containerd/pkg/server/testing" @@ -39,9 +39,9 @@ const ( // newTestCRIContainerdService creates a fake criContainerdService for test. func newTestCRIContainerdService() *criContainerdService { return &criContainerdService{ - config: options.Config{ + config: criconfig.Config{ RootDir: testRootDir, - PluginConfig: options.PluginConfig{ + PluginConfig: criconfig.PluginConfig{ SandboxImage: testSandboxImage, }, }, From 05bcc3e88c33ca7b4f60c2e916d88b1e854d6f4e Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 5 Mar 2018 19:14:14 +0000 Subject: [PATCH 2/4] Update makefile and all integration test Signed-off-by: Lantao Liu --- .travis.yml | 16 ++-------- Makefile | 46 ++++++++++++++--------------- hack/install-deps.sh | 5 ++-- hack/release.sh | 18 ++++++++--- hack/test-cri.sh | 2 +- hack/test-e2e-node.sh | 2 +- hack/test-integration.sh | 12 ++------ hack/test-utils.sh | 45 ++++++---------------------- integration/container_stats_test.go | 1 - integration/test_utils.go | 9 +++--- 10 files changed, 59 insertions(+), 97 deletions(-) diff --git a/.travis.yml b/.travis.yml index 15e1479e6..9acfce780 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,27 +35,15 @@ jobs: - make install.tools - make .gitvalidation - make binaries - - make containerd go: "1.10" - stage: Test script: - make install.deps + - make containerd + - sudo make install-containerd - make test - make test-integration - make test-cri - after_script: - # Abuse travis to preserve the log. - - cat /tmp/test-integration/cri-containerd.log - - cat /tmp/test-integration/containerd.log - - cat /tmp/test-cri/cri-containerd.log - - cat /tmp/test-cri/containerd.log - go: "1.10" - - script: - - make install.deps - - make containerd - - sudo make install-containerd - - make test-integration STANDALONE_CRI_CONTAINERD=false - - make test-cri STANDALONE_CRI_CONTAINERD=false after_script: # Abuse travis to preserve the log. - cat /tmp/test-integration/containerd.log diff --git a/Makefile b/Makefile index 8b1f75eb1..2da939e8f 100644 --- a/Makefile +++ b/Makefile @@ -41,12 +41,14 @@ help: @echo "Usage: make " @echo @echo " * 'install' - Install binaries to system locations" - @echo " * 'binaries' - Build cri-containerd and ctrcri" - @echo " * 'static-binaries - Build static cri-containerd and ctrcri" + @echo " * 'binaries' - Build containerd and ctrcri" + @echo " * 'static-binaries - Build static containerd and ctrcri" + @echo " * 'ctrcri' - Build ctrcri" + @echo " * 'install-ctrcri' - Install ctrcri" + @echo " * 'containerd' - Build a customized containerd with CRI plugin for testing" + @echo " * 'install-containerd' - Install customized containerd to system location" @echo " * 'release' - Build release tarball" @echo " * 'push' - Push release tarball to GCS" - @echo " * 'containerd' - Build a customized containerd with CRI plugin for testing" - @echo " * 'install-containerd' - Install customized containerd to system location" @echo " * 'test' - Test cri-containerd with unit test" @echo " * 'test-integration' - Test cri-containerd with integration test" @echo " * 'test-cri' - Test cri-containerd with cri validation test" @@ -89,13 +91,6 @@ sync-vendor: update-vendor: sync-vendor sort-vendor -$(BUILD_DIR)/cri-containerd: $(SOURCES) - $(GO) build -o $@ \ - -tags '$(BUILD_TAGS)' \ - -ldflags '$(GO_LDFLAGS)' \ - -gcflags '$(GO_GCFLAGS)' \ - $(PROJECT)/cmd/cri-containerd - $(BUILD_DIR)/ctrcri: $(SOURCES) $(GO) build -o $@ \ -tags '$(BUILD_TAGS)' \ @@ -131,17 +126,25 @@ test-e2e-node: binaries clean: rm -rf $(BUILD_DIR)/* -binaries: $(BUILD_DIR)/cri-containerd $(BUILD_DIR)/ctrcri +binaries: $(BUILD_DIR)/containerd $(BUILD_DIR)/ctrcri static-binaries: GO_LDFLAGS += -extldflags "-fno-PIC -static" -static-binaries: $(BUILD_DIR)/cri-containerd $(BUILD_DIR)/ctrcri +static-binaries: $(BUILD_DIR)/containerd $(BUILD_DIR)/ctrcri -install: binaries - install -D -m 755 $(BUILD_DIR)/cri-containerd $(BINDIR)/cri-containerd +ctrcri: $(BUILD_DIR)/ctrcri + +install-ctrcri: ctrcri install -D -m 755 $(BUILD_DIR)/ctrcri $(BINDIR)/ctrcri +containerd: $(BUILD_DIR)/containerd + +install-containerd: containerd + install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd + +install: install-ctrcri install-containerd + uninstall: - rm -f $(BINDIR)/cri-containerd + rm -f $(BINDIR)/containerd rm -f $(BINDIR)/ctrcri $(BUILD_DIR)/$(TARBALL): static-binaries vendor.conf @@ -152,11 +155,6 @@ release: $(BUILD_DIR)/$(TARBALL) push: $(BUILD_DIR)/$(TARBALL) @BUILD_DIR=$(BUILD_DIR) TARBALL=$(TARBALL) VERSION=$(VERSION) ./hack/push.sh -containerd: $(BUILD_DIR)/containerd - -install-containerd: containerd - install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd - proto: @hack/update-proto.sh @@ -189,6 +187,10 @@ install.tools: .install.gitvalidation .install.gometalinter .PHONY: \ binaries \ static-binaries \ + ctrcri \ + install-ctrcri \ + containerd \ + install-containerd \ release \ push \ boiler \ @@ -198,8 +200,6 @@ install.tools: .install.gitvalidation .install.gometalinter help \ install \ lint \ - containerd \ - install-containerd \ test \ test-integration \ test-cri \ diff --git a/hack/install-deps.sh b/hack/install-deps.sh index 803d95701..76a73665f 100755 --- a/hack/install-deps.sh +++ b/hack/install-deps.sh @@ -133,8 +133,7 @@ fi # Install containerd checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO} cd ${GOPATH}/src/${CONTAINERD_PKG} -# Build no_cri version and run standalone cri-containerd. -make BUILDTAGS="${BUILDTAGS} no_cri" +make BUILDTAGS="${BUILDTAGS}" # containerd make install requires `go` to work. Explicitly # set PATH to make sure it can find `go` even with `sudo`. ${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}" @@ -146,7 +145,7 @@ make crictl ${sudo} make install-crictl -e BINDIR=${CRICTL_DIR} GOPATH=${GOPATH} ${sudo} mkdir -p ${CRICTL_CONFIG_DIR} ${sudo} bash -c 'cat >'${CRICTL_CONFIG_DIR}'/crictl.yaml < \ - /etc/profile.d/cri-containerd_env.sh +echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/containerd_env.sh diff --git a/cluster/gce/env b/cluster/gce/env index 184d86042..231d1c914 100644 --- a/cluster/gce/env +++ b/cluster/gce/env @@ -9,11 +9,11 @@ if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then fi version_file=$(mktemp /tmp/version.XXXX) echo "${CRI_CONTAINERD_VERSION}" > "$version_file" -export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" -export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" -export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" -export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" +export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" +export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" diff --git a/cluster/health-monitor.sh b/cluster/health-monitor.sh index 0320a534d..800f4df07 100755 --- a/cluster/health-monitor.sh +++ b/cluster/health-monitor.sh @@ -20,16 +20,15 @@ set -o pipefail # CRICTL is the path of crictl CRICTL=${CRICTL:-"crictl"} # INITIAL_WAIT_ATTEMPTS is the number to attempt, before start -# performing health check. The problem is that cri-containerd -# and containerd are started around the same time with health -# monitor, they may not be ready yet when health-monitor is started. +# performing health check. The problem is that containerd is +# started around the same time with health monitor, it may +# not be ready yet when health-monitor is started. INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5} # COMMAND_TIMEOUT is the timeout for the health check command. COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} # CHECK_PERIOD is the health check period. CHECK_PERIOD=${CHECK_PERIOD:-10} -# SLEEP_SECONDS is the time to sleep after killing cri-containerd -# and containerd. +# SLEEP_SECONDS is the time to sleep after killing containerd. SLEEP_SECONDS=${SLEEP_SECONDS:-120} attempt=1 @@ -41,11 +40,8 @@ done echo "Start performing health check." while true; do - # Use crictl pods because it requires both containerd and - # cri-containerd to be working. if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then echo "\"$CRICTL pods\" failed!" - pkill -x cri-containerd pkill -x containerd # Wait for a while, as we don't want to kill it again before it is really up. sleep ${SLEEP_SECONDS} diff --git a/test/build.sh b/test/build.sh index a5a73c536..d246d1663 100755 --- a/test/build.sh +++ b/test/build.sh @@ -55,4 +55,4 @@ make clean DEPLOY_DIR=${DEPLOY_DIR:-""} make push # Build and push node e2e tarball. PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \ - make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true + make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true CUSTOM_CONTAINERD=true diff --git a/test/configure.sh b/test/configure.sh index 2961c5472..c33d8b71e 100755 --- a/test/configure.sh +++ b/test/configure.sh @@ -19,9 +19,9 @@ set -o errexit set -o nounset set -o pipefail -# CRI_CONTAINERD_HOME is the directory for cri-containerd. -CRI_CONTAINERD_HOME="/home/cri-containerd" -cd "${CRI_CONTAINERD_HOME}" +# CONTAINERD_HOME is the directory for containerd. +CONTAINERD_HOME="/home/containerd" +cd "${CONTAINERD_HOME}" # fetch_metadata fetches metadata from GCE metadata server. # Var set: @@ -72,11 +72,11 @@ curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 -- tar xvf "${TARBALL}" # Copy crictl config. -cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc +cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc # TODO(random-liu): Stop docker on the node, this may break docker. -echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ - /etc/profile.d/cri-containerd_env.sh +echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/containerd_env.sh # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded. EXTRA_INIT_SCRIPT="extra-init.sh" diff --git a/test/e2e_node/benchmark-config.yaml b/test/e2e_node/benchmark-config.yaml index 8c7e2d2b8..7541bd7ae 100644 --- a/test/e2e_node/benchmark-config.yaml +++ b/test/e2e_node/benchmark-config.yaml @@ -4,21 +4,21 @@ images: image: cos-stable-60-9592-90-0 project: cos-cloud machine: n1-standard-1 - metadata: "user-data Date: Mon, 5 Mar 2018 19:15:04 +0000 Subject: [PATCH 4/4] Update ansible setup. Signed-off-by: Lantao Liu --- contrib/ansible/README.md | 2 +- contrib/ansible/cri-containerd.yaml | 19 +++---------------- contrib/ansible/tasks/binaries.yaml | 2 +- contrib/ansible/vars/vars.yaml | 3 --- contrib/systemd-units/cri-containerd.service | 17 ----------------- 5 files changed, 5 insertions(+), 38 deletions(-) delete mode 100644 contrib/systemd-units/cri-containerd.service diff --git a/contrib/ansible/README.md b/contrib/ansible/README.md index 6c049d578..a75a8759d 100644 --- a/contrib/ansible/README.md +++ b/contrib/ansible/README.md @@ -1,4 +1,4 @@ -# Kubernetes Cluster with Containerd and CRI-Containerd +# Kubernetes Cluster with Containerd

diff --git a/contrib/ansible/cri-containerd.yaml b/contrib/ansible/cri-containerd.yaml index 52aa1a8b5..3eb70c6f6 100644 --- a/contrib/ansible/cri-containerd.yaml +++ b/contrib/ansible/cri-containerd.yaml @@ -8,7 +8,7 @@ - include_tasks: tasks/bootstrap_centos.yaml # Contains tasks bootstrap components for centos systems when: ansible_distribution == "CentOS" - include_tasks: tasks/k8s.yaml # Contains tasks kubernetes component installation - - include_tasks: tasks/binaries.yaml # Contains tasks for pulling containerd and cri-containerd components + - include_tasks: tasks/binaries.yaml # Contains tasks for pulling containerd components - name: "Create a directory for containerd config" file: path=/etc/containerd state=directory @@ -21,22 +21,9 @@ [cgroup] path = "/runtime" - - name: "Create a directory for cri-containerd config" - file: path=/etc/cri-containerd state=directory - - - name: "Add cri-containerd config file" - blockinfile: - path: /etc/cri-containerd/config.toml - create: yes - block: | - cgroup_path = "/runtime" - - name: "Start Containerd" systemd: name=containerd daemon_reload=yes state=started enabled=yes - - name: "Start CRI-Containerd" - systemd: name=cri-containerd daemon_reload=yes state=started enabled=yes - - name: "Load br_netfilter kernel module" modprobe: name: br_netfilter @@ -59,7 +46,7 @@ - name: "Add runtime args in kubelet conf" lineinfile: dest: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/runtime --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=/var/run/cri-containerd.sock\"" + line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/runtime --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=/run/containerd/containerd.sock\"" insertafter: '\[Service\]' when: check_args.stdout == "" @@ -70,5 +57,5 @@ - name: "Pre-pull pause container image" shell: | /usr/local/bin/ctr pull gcr.io/google_containers/pause:3.0 - /usr/local/bin/crictl --runtime-endpoint /var/run/cri-containerd.sock \ + /usr/local/bin/crictl --runtime-endpoint /run/containerd/containerd.sock \ pull gcr.io/google_containers/pause:3.0 diff --git a/contrib/ansible/tasks/binaries.yaml b/contrib/ansible/tasks/binaries.yaml index 2bcc44495..808c7a72a 100644 --- a/contrib/ansible/tasks/binaries.yaml +++ b/contrib/ansible/tasks/binaries.yaml @@ -1,5 +1,5 @@ --- -- name: "Get Containerd and CRI-Containerd" +- name: "Get Containerd" unarchive: src: "https://storage.googleapis.com/cri-containerd-release/cri-containerd-{{ cri_containerd_release_version }}.linux-amd64.tar.gz" dest: "/" diff --git a/contrib/ansible/vars/vars.yaml b/contrib/ansible/vars/vars.yaml index 8cbe743bd..d9c828a4c 100644 --- a/contrib/ansible/vars/vars.yaml +++ b/contrib/ansible/vars/vars.yaml @@ -1,7 +1,4 @@ --- cri_containerd_release_version: 1.0.0-beta.0 -cri_release_directory: /opt/cri-containerd/ -local_bin_dir: /usr/local/bin/ -local_sbin_dir: /usr/local/sbin/ cni_bin_dir: /opt/cni/bin/ cni_conf_dir: /etc/cni/net.d/ diff --git a/contrib/systemd-units/cri-containerd.service b/contrib/systemd-units/cri-containerd.service deleted file mode 100644 index b82e85e27..000000000 --- a/contrib/systemd-units/cri-containerd.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Kubernetes containerd CRI shim -Requires=network-online.target -After=containerd.service - -[Service] -Restart=always -RestartSec=5 -LimitNOFILE=1048576 -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNPROC=infinity -LimitCORE=infinity -ExecStart=/usr/local/bin/cri-containerd - -[Install] -WantedBy=multi-user.target