diff --git a/cmd/cri-containerd/cri_containerd.go b/cmd/cri-containerd/cri_containerd.go deleted file mode 100644 index 9b40cda19..000000000 --- a/cmd/cri-containerd/cri_containerd.go +++ /dev/null @@ -1,229 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "net" - "net/http" - "net/http/pprof" - "os" - "os/signal" - "runtime" - "syscall" - - "github.com/containerd/cgroups" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/sys" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "k8s.io/kubernetes/pkg/util/interrupt" - - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" - "github.com/containerd/cri-containerd/pkg/server" - "github.com/containerd/cri-containerd/pkg/version" -) - -// Add \u200B to avoid the space trimming. -const desc = "\u200B" + ` _ __ _ __ - __________(_) _________ ____ / /_____ _(_)____ ___ _________/ / - / ___/ ___/ /______/ ___/ __ \/ __ \/ __/ __ ` + "`" + `/ // __ \/ _ \/ ___/ __ / -/ /__/ / / //_____/ /__/ /_/ / / / / /_/ /_/ / // / / / __/ / / /_/ / -\___/_/ /_/ \___/\____/_/ /_/\__/\__,_/_//_/ /_/\___/_/ \__,_/ - -A containerd based Kubernetes CRI implementation. -` - -var cmd = &cobra.Command{ - Use: "cri-containerd", - Short: "A containerd based Kubernetes CRI implementation.", - Long: desc, -} - -func defaultConfigCommand() *cobra.Command { - return &cobra.Command{ - Use: "default-config", - Short: "Print default toml config of cri-containerd.", - Run: func(cmd *cobra.Command, args []string) { - options.PrintDefaultTomlConfig() - }, - } -} - -func versionCommand() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Print cri-containerd version information.", - Run: func(cmd *cobra.Command, args []string) { - version.PrintVersion() - }, - } -} - -func main() { - o := options.NewCRIContainerdOptions() - - o.AddFlags(cmd.Flags()) - cmd.AddCommand(defaultConfigCommand()) - cmd.AddCommand(versionCommand()) - - cmd.RunE = func(cmd *cobra.Command, args []string) error { - setupDumpStacksTrap() - if err := o.InitFlags(cmd.Flags()); err != nil { - return fmt.Errorf("failed to init CRI containerd flags: %v", err) - } - - if err := setLogLevel(o.LogLevel); err != nil { - return fmt.Errorf("failed to set log level: %v", err) - } - - logrus.Infof("Run cri-containerd %+v", o) - - if o.CgroupPath != "" { - _, err := loadCgroup(o.CgroupPath) - if err != nil { - return fmt.Errorf("failed to load cgroup for cgroup path %v: %v", o.CgroupPath, err) - } - } - - if o.OOMScore != 0 { - if err := sys.SetOOMScore(os.Getpid(), o.OOMScore); err != nil { - return fmt.Errorf("failed to set OOMScore to %v: %v", o.OOMScore, err) - } - } - - // Start profiling server if enable. - if o.EnableProfiling { - logrus.Info("Start profiling server") - go startProfilingServer(o.ProfilingAddress, o.ProfilingPort) - } - - logrus.Infof("Run cri-containerd grpc server on socket %q", o.SocketPath) - s, err := server.NewCRIContainerdService(o.Config) - if err != nil { - return fmt.Errorf("failed to create CRI containerd service: %v", err) - } - // Use interrupt handler to make sure the server is stopped properly. - // Pass in non-empty final function to avoid os.Exit(1). We expect `Run` - // to return itself. - h := interrupt.New(func(os.Signal) {}, func() { - if err := s.Close(); err != nil { - logrus.WithError(err).Error("Failed to stop cri service") - } - }) - if err := h.Run(func() error { return s.Run(true) }); err != nil { - return fmt.Errorf("failed to run cri-containerd with grpc server: %v", err) - } - return nil - } - - if err := cmd.Execute(); err != nil { - // Error should have been reported. - os.Exit(1) - } -} - -func setupDumpStacksTrap() { - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGUSR1) - go func() { - for range c { - dumpStacks() - } - }() -} - -func dumpStacks() { - buf := make([]byte, 1024) - for { - n := runtime.Stack(buf, true) - if n < len(buf) { - buf = buf[:n] - break - } - buf = make([]byte, 2*len(buf)) - } - logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) -} - -// startProfilingServer start http server to profiling via web interface -func startProfilingServer(host string, port string) { - endpoint := net.JoinHostPort(host, port) - mux := http.NewServeMux() - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - if err := http.ListenAndServe(endpoint, mux); err != nil { - logrus.WithError(err).Error("Failed to start profiling server") - } -} - -func setLogLevel(l string) error { - lvl, err := log.ParseLevel(l) - if err != nil { - return err - } - if err := setGLogLevel(lvl); err != nil { - return err - } - logrus.SetLevel(lvl) - return nil -} - -// TODO(random-liu): Set glog level in plugin mode. -func setGLogLevel(l logrus.Level) error { - if err := flag.Set("logtostderr", "true"); err != nil { - return err - } - switch l { - case log.TraceLevel: - return flag.Set("v", "5") - case logrus.DebugLevel: - return flag.Set("v", "4") - case logrus.InfoLevel: - return flag.Set("v", "2") - // glog doesn't support following filters. Defaults to v=0. - case logrus.WarnLevel: - case logrus.ErrorLevel: - case logrus.FatalLevel: - case logrus.PanicLevel: - } - return nil -} - -// loadCgroup loads the cgroup associated with path if it exists and moves the current process into the cgroup. If the cgroup -// is not created it is created and returned. -func loadCgroup(cgroupPath string) (cgroups.Cgroup, error) { - cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(cgroupPath)) - if err != nil { - if err != cgroups.ErrCgroupDeleted { - return nil, err - } - if cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(cgroupPath), &runtimespec.LinuxResources{}); err != nil { - return nil, err - } - } - if err := cg.Add(cgroups.Process{ - Pid: os.Getpid(), - }); err != nil { - return nil, err - } - return cg, nil -} diff --git a/cmd/cri-containerd/options/options.go b/cmd/cri-containerd/options/options.go deleted file mode 100644 index 3f1edfecb..000000000 --- a/cmd/cri-containerd/options/options.go +++ /dev/null @@ -1,273 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "os" - - "github.com/BurntSushi/toml" - "github.com/containerd/containerd" - "github.com/spf13/pflag" -) - -const ( - // configFilePathArgName is the path to the config file. - configFilePathArgName = "config" - // defaultConfigFilePath is the default config file path. - defaultConfigFilePath = "/etc/cri-containerd/config.toml" -) - -// ContainerdConfig contains toml config related to containerd -type ContainerdConfig struct { - // RootDir is the root directory path for containerd. - // TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode. - RootDir string `toml:"root_dir" json:"rootDir,omitempty"` - // Snapshotter is the snapshotter used by containerd. - Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"` - // Endpoint is the containerd endpoint path. - // TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode. - Endpoint string `toml:"endpoint" json:"endpoint,omitempty"` - // Runtime is the runtime to use in containerd. We may support - // other runtimes in the future. - Runtime string `toml:"runtime" json:"runtime,omitempty"` - // RuntimeEngine is the name of the runtime engine used by containerd. - // Containerd default should be "runc" - // We may support other runtime engines in the future. - RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"` - // RuntimeRoot is the directory used by containerd for runtime state. - // Containerd default should be "/run/containerd/runc" - RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"` -} - -// CniConfig contains toml config related to cni -type CniConfig struct { - // NetworkPluginBinDir is the directory in which the binaries for the plugin is kept. - NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"` - // NetworkPluginConfDir is the directory in which the admin places a CNI conf. - NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"` -} - -// PluginConfig contains toml config related to CRI plugin, -// it is a subset of Config. -type PluginConfig struct { - // ContainerdConfig contains config related to containerd - ContainerdConfig `toml:"containerd" json:"containerd,omitempty"` - // CniConfig contains config related to cni - CniConfig `toml:"cni" json:"cni,omitempty"` - // Registry contains config related to the registry - Registry `toml:"registry" json:"registry,omitempty"` - // StreamServerAddress is the ip address streaming server is listening on. - StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"` - // StreamServerPort is the port streaming server is listening on. - StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"` - // EnableSelinux indicates to enable the selinux support. - EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"` - // SandboxImage is the image used by sandbox container. - SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"` - // StatsCollectPeriod is the period (in seconds) of snapshots stats collection. - StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"` - // SystemdCgroup enables systemd cgroup support. - SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"` - // EnableIPv6DAD enables IPv6 DAD. - // TODO(random-liu): Use optimistic_dad when it's GA. - EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"` -} - -// Config contains toml config related cri-containerd daemon. -// TODO(random-liu): Make this an internal config object when we no longer support cri-containerd -// standalone mode. At that time, we can clean this up. -type Config struct { - // PluginConfig is the config for CRI plugin. - PluginConfig - // ContainerdRootDir is the root directory path for containerd. - ContainerdRootDir string `toml:"-" json:"containerdRootDir,omitempty"` - // ContainerdEndpoint is the containerd endpoint path. - ContainerdEndpoint string `toml:"-" json:"containerdEndpoint,omitempty"` - // SocketPath is the path to the socket which cri-containerd serves on. - // TODO(random-liu): Remove SocketPath when no longer support cri-containerd - // standalone mode. - SocketPath string `toml:"socket_path" json:"socketPath,omitempty"` - // RootDir is the root directory path for managing cri-containerd files - // (metadata checkpoint etc.) - RootDir string `toml:"root_dir" json:"rootDir,omitempty"` - // TODO(random-liu): Remove following fields when we no longer support cri-containerd - // standalone mode. - // CgroupPath is the path for the cgroup that cri-containerd is placed in. - CgroupPath string `toml:"cgroup_path" json:"cgroupPath,omitempty"` - // OOMScore adjust the cri-containerd's oom score - OOMScore int `toml:"oom_score" json:"oomScore,omitempty"` - // EnableProfiling is used for enable profiling via host:port/debug/pprof/ - EnableProfiling bool `toml:"profiling" json:"enableProfiling,omitempty"` - // ProfilingPort is the port for profiling via host:port/debug/pprof/ - ProfilingPort string `toml:"profiling_port" json:"profilingPort,omitempty"` - // ProfilingAddress is address for profiling via host:port/debug/pprof/ - ProfilingAddress string `toml:"profiling_addr" json:"profilingAddress,omitempty"` - // LogLevel is the logrus log level. - LogLevel string `toml:"log_level" json:"logLevel,omitempty"` -} - -// CRIContainerdOptions contains cri-containerd command line and toml options. -type CRIContainerdOptions struct { - // Config contains cri-containerd toml config - Config - // ConfigFilePath is the path to the TOML config file. - ConfigFilePath string `toml:"-"` -} - -// NewCRIContainerdOptions returns a reference to CRIContainerdOptions -func NewCRIContainerdOptions() *CRIContainerdOptions { - return &CRIContainerdOptions{} -} - -// AddFlags adds cri-containerd command line options to pflag. -func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) { - defaults := DefaultConfig() - fs.StringVar(&c.ConfigFilePath, configFilePathArgName, - defaultConfigFilePath, "Path to the config file.") - fs.StringVar(&c.LogLevel, "log-level", - defaults.LogLevel, "Set the logging level [trace, debug, info, warn, error, fatal, panic].") - fs.StringVar(&c.SocketPath, "socket-path", - defaults.SocketPath, "Path to the socket which cri-containerd serves on.") - fs.StringVar(&c.RootDir, "root-dir", - defaults.RootDir, "Root directory path for cri-containerd managed files (metadata checkpoint etc).") - fs.StringVar(&c.ContainerdRootDir, "containerd-root-dir", - defaults.ContainerdRootDir, "Root directory path where containerd stores persistent data.") - fs.StringVar(&c.ContainerdEndpoint, "containerd-endpoint", - defaults.ContainerdEndpoint, "Path to the containerd endpoint.") - fs.StringVar(&c.ContainerdConfig.Snapshotter, "containerd-snapshotter", - defaults.ContainerdConfig.Snapshotter, "The snapshotter used by containerd.") - fs.StringVar(&c.ContainerdConfig.Runtime, "containerd-runtime", - defaults.ContainerdConfig.Runtime, "The runtime used by containerd.") - fs.StringVar(&c.ContainerdConfig.RuntimeEngine, "containerd-runtime-engine", - defaults.ContainerdConfig.RuntimeEngine, "Runtime engine used by containerd. Defaults to containerd's default if not specified.") - fs.StringVar(&c.ContainerdConfig.RuntimeRoot, "containerd-runtime-root", - defaults.ContainerdConfig.RuntimeRoot, "The directory used by containerd for runtime state. Defaults to containerd's default if not specified.") - fs.StringVar(&c.NetworkPluginBinDir, "network-bin-dir", - defaults.NetworkPluginBinDir, "The directory for putting network binaries.") - fs.StringVar(&c.NetworkPluginConfDir, "network-conf-dir", - defaults.NetworkPluginConfDir, "The directory for putting network plugin configuration files.") - fs.StringVar(&c.StreamServerAddress, "stream-addr", - defaults.StreamServerAddress, "The ip address streaming server is listening on. The default host interface is used if not specified.") - fs.StringVar(&c.StreamServerPort, "stream-port", - defaults.StreamServerPort, "The port streaming server is listening on.") - fs.StringVar(&c.CgroupPath, "cgroup-path", - defaults.CgroupPath, "The cgroup that cri-containerd is part of. Cri-containerd is not placed in a cgroup if none is specified.") - fs.BoolVar(&c.EnableSelinux, "enable-selinux", - defaults.EnableSelinux, "Enable selinux support. By default not enabled.") - fs.StringVar(&c.SandboxImage, "sandbox-image", - defaults.SandboxImage, "The image used by sandbox container.") - fs.IntVar(&c.StatsCollectPeriod, "stats-collect-period", - defaults.StatsCollectPeriod, "The period (in seconds) of snapshots stats collection.") - fs.BoolVar(&c.SystemdCgroup, "systemd-cgroup", - defaults.SystemdCgroup, "Enables systemd cgroup support. By default not enabled.") - fs.IntVar(&c.OOMScore, "oom-score", - defaults.OOMScore, "Adjust the cri-containerd's oom score.") - fs.BoolVar(&c.EnableProfiling, "profiling", - defaults.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/.") - fs.StringVar(&c.ProfilingPort, "profiling-port", - defaults.ProfilingPort, "Profiling port for web interface host:port/debug/pprof/.") - fs.StringVar(&c.ProfilingAddress, "profiling-addr", - defaults.ProfilingAddress, "Profiling address for web interface host:port/debug/pprof/.") - fs.BoolVar(&c.EnableIPv6DAD, "enable-ipv6-dad", - defaults.EnableIPv6DAD, "Enable IPv6 DAD (duplicate address detection) for pod sandbox network. Enabling this will increase pod sandbox start latency by several seconds.") - fs.Var(&c.Registry, "registry", - "Registry config for image pull eg --registry=myregistry.io=https://mymirror.io/ --registry=myregistry2.io=https://mymirror2.io/") -} - -// InitFlags load configurations from config file, and then overwrite with flags. -// This function must be called inside `Run`, at that time flags should have been -// parsed once. -// precedence: commandline > configfile > default -func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error { - // Load default config file if none provided - if _, err := toml.DecodeFile(c.ConfigFilePath, &c.Config); err != nil { - // the absence of default config file is normal case. - if !fs.Changed(configFilePathArgName) && os.IsNotExist(err) { - return nil - } - return err - } - // Add this for backward compatibility. - // TODO(random-liu): Remove this when we no longer support cri-containerd standalone mode. - if c.ContainerdConfig.RootDir != "" { - c.ContainerdRootDir = c.ContainerdConfig.RootDir - } - if c.ContainerdConfig.Endpoint != "" { - c.ContainerdEndpoint = c.ContainerdConfig.Endpoint - } - - // What is the reason for applying the command line twice? - // Because the values from command line have the highest priority. - // The path of toml configuration file if from the command line, - // and triggers the first parse. - // The first parse generates the default value and the value from command line at the same time. - // But the priority of the toml config value is higher than the default value, - // Without a way to insert the toml config value between the default value and the command line value. - // We parse twice one for default value, one for commandline value. - return fs.Parse(os.Args[1:]) -} - -// PrintDefaultTomlConfig print default toml config of cri-containerd. -func PrintDefaultTomlConfig() { - if err := toml.NewEncoder(os.Stdout).Encode(DefaultConfig()); err != nil { - fmt.Println(err) - return - } -} - -// DefaultConfig returns default configurations of cri-containerd. -func DefaultConfig() Config { - return Config{ - PluginConfig: PluginConfig{ - CniConfig: CniConfig{ - NetworkPluginBinDir: "/opt/cni/bin", - NetworkPluginConfDir: "/etc/cni/net.d", - }, - ContainerdConfig: ContainerdConfig{ - Snapshotter: containerd.DefaultSnapshotter, - Runtime: "io.containerd.runtime.v1.linux", - RuntimeEngine: "", - RuntimeRoot: "", - }, - StreamServerAddress: "", - StreamServerPort: "10010", - EnableSelinux: false, - SandboxImage: "gcr.io/google_containers/pause:3.0", - StatsCollectPeriod: 10, - SystemdCgroup: false, - EnableIPv6DAD: false, - Registry: Registry{ - Mirrors: map[string]Mirror{ - "docker.io": { - Endpoints: []string{"https://registry-1.docker.io"}, - }, - }, - }, - }, - ContainerdRootDir: "/var/lib/containerd", - ContainerdEndpoint: "/run/containerd/containerd.sock", - SocketPath: "/var/run/cri-containerd.sock", - RootDir: "/var/lib/cri-containerd", - CgroupPath: "", - OOMScore: -999, - EnableProfiling: true, - ProfilingPort: "10011", - ProfilingAddress: "127.0.0.1", - LogLevel: "info", - } -} diff --git a/cmd/cri-containerd/options/registry.go b/cmd/cri-containerd/options/registry.go deleted file mode 100644 index 7f057ee4c..000000000 --- a/cmd/cri-containerd/options/registry.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2018 The Containerd Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "net/url" - "strings" -) - -// Mirror contains the config related to the registry mirror -type Mirror struct { - Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"` - // TODO (Abhi) We might need to add auth per namespace. Looks like - // image auth information is passed by kube itself. -} - -// Registry is registry settings configured -type Registry struct { - Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"` -} - -// String returns the string format of registry type -func (r *Registry) String() string { - // Its not used hence return empty string - return "" -} - -// Set validates and converts into the internal registry struct -func (r *Registry) Set(s string) error { - // --registry docker.io=https://mymirror.io,http://mymirror2.io - // If no option is set then return format error - if len(s) == 0 { - return fmt.Errorf("incomplete registry mirror option") - } - var mirrors []string - host := "docker.io" - opt := strings.Split(s, "=") - if len(opt) > 1 { - // If option is set in the format "mynamespace.io=https://mymirror.io,https://mymirror2.io" - // Then associate the mirror urls for the namespace only" - host = opt[0] - mirrors = strings.Split(opt[1], ",") - } else { - // If option is set in the format "https://mymirror.io,https://mymirror.io" - // Then associate mirror against default docker.io namespace - mirrors = strings.Split(opt[0], ",") - } - - // Validate the format of the urls passed - for _, u := range mirrors { - _, err := url.Parse(u) - if err != nil { - return fmt.Errorf("invalid registry mirror url format %v: %v", u, err) - } - } - - if r.Mirrors == nil { - r.Mirrors = make(map[string]Mirror) - } - if _, ok := r.Mirrors[host]; !ok { - r.Mirrors[host] = Mirror{} - } - m := r.Mirrors[host] - m.Endpoints = append(m.Endpoints, mirrors...) - r.Mirrors[host] = m - - return nil -} - -// Type returns a string name for the option type -func (r *Registry) Type() string { - return "list" -} diff --git a/cmd/ctrcri/ctrcri.go b/cmd/ctrcri/ctrcri.go index 63276cf75..67811d4c3 100644 --- a/cmd/ctrcri/ctrcri.go +++ b/cmd/ctrcri/ctrcri.go @@ -20,10 +20,10 @@ import ( "os" "time" + "github.com/containerd/containerd/defaults" "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" "github.com/containerd/cri-containerd/pkg/version" ) @@ -56,9 +56,7 @@ var ( ) func addGlobalFlags(fs *pflag.FlagSet) { - // TODO(random-liu): Change default to containerd/defaults.DefaultAddress after cri plugin - // become default. - fs.StringVar(&address, "address", options.DefaultConfig().SocketPath, "address for containerd's GRPC server.") + fs.StringVar(&address, "address", defaults.DefaultAddress, "address for containerd's GRPC server.") fs.DurationVar(&timeout, "timeout", defaultTimeout, "timeout for containerd grpc connection.") } diff --git a/cri.go b/cri.go index fdcdf81cb..4da21e910 100644 --- a/cri.go +++ b/cri.go @@ -17,6 +17,7 @@ limitations under the License. package cri import ( + "flag" "path/filepath" "github.com/containerd/containerd/log" @@ -24,8 +25,9 @@ import ( "github.com/containerd/containerd/plugin" imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" + criconfig "github.com/containerd/cri-containerd/pkg/config" "github.com/containerd/cri-containerd/pkg/server" ) @@ -35,7 +37,7 @@ const criVersion = "v1alpha2" // TODO(random-liu): Use github.com/pkg/errors for our errors. // Register CRI service plugin func init() { - config := options.DefaultConfig().PluginConfig + config := criconfig.DefaultConfig() plugin.Register(&plugin.Registration{ Type: plugin.GRPCPlugin, ID: "cri", @@ -57,8 +59,8 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()} ic.Meta.Exports = map[string]string{"CRIVersion": criVersion} ctx := ic.Context - pluginConfig := ic.Config.(*options.PluginConfig) - c := options.Config{ + pluginConfig := ic.Config.(*criconfig.PluginConfig) + c := criconfig.Config{ PluginConfig: *pluginConfig, // This is a hack. We assume that containerd root directory // is one level above plugin directory. @@ -69,6 +71,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { } log.G(ctx).Infof("Start cri plugin with config %+v", c) + if err := setGLogLevel(); err != nil { + return nil, errors.Wrap(err, "failed to set glog level") + } + s, err := server.NewCRIContainerdService(c) if err != nil { return nil, errors.Wrap(err, "failed to create CRI service") @@ -77,10 +83,32 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { // Use a goroutine to initialize cri service. The reason is that currently // cri service requires containerd to be initialize. go func() { - if err := s.Run(false); err != nil { + if err := s.Run(); err != nil { log.G(ctx).WithError(err).Fatal("Failed to run CRI service") } // TODO(random-liu): Whether and how we can stop containerd. }() return s, nil } + +// Set glog level. +func setGLogLevel() error { + l := logrus.GetLevel() + if err := flag.Set("logtostderr", "true"); err != nil { + return err + } + switch l { + case log.TraceLevel: + return flag.Set("v", "5") + case logrus.DebugLevel: + return flag.Set("v", "4") + case logrus.InfoLevel: + return flag.Set("v", "2") + // glog doesn't support following filters. Defaults to v=0. + case logrus.WarnLevel: + case logrus.ErrorLevel: + case logrus.FatalLevel: + case logrus.PanicLevel: + } + return nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 000000000..102be9f14 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,127 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import "github.com/containerd/containerd" + +// ContainerdConfig contains toml config related to containerd +type ContainerdConfig struct { + // Snapshotter is the snapshotter used by containerd. + Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"` + // Runtime is the runtime to use in containerd. We may support + // other runtimes in the future. + Runtime string `toml:"runtime" json:"runtime,omitempty"` + // RuntimeEngine is the name of the runtime engine used by containerd. + // Containerd default should be "runc" + // We may support other runtime engines in the future. + RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"` + // RuntimeRoot is the directory used by containerd for runtime state. + // Containerd default should be "/run/containerd/runc" + RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"` +} + +// CniConfig contains toml config related to cni +type CniConfig struct { + // NetworkPluginBinDir is the directory in which the binaries for the plugin is kept. + NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"` + // NetworkPluginConfDir is the directory in which the admin places a CNI conf. + NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"` +} + +// Mirror contains the config related to the registry mirror +type Mirror struct { + // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints + // one by one until a working one is found. + Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"` + // TODO (Abhi) We might need to add auth per namespace. Looks like + // image auth information is passed by kube itself. +} + +// Registry is registry settings configured +type Registry struct { + // Mirrors are namespace to mirror mapping for all namespaces. + Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"` +} + +// PluginConfig contains toml config related to CRI plugin, +// it is a subset of Config. +type PluginConfig struct { + // ContainerdConfig contains config related to containerd + ContainerdConfig `toml:"containerd" json:"containerd,omitempty"` + // CniConfig contains config related to cni + CniConfig `toml:"cni" json:"cni,omitempty"` + // Registry contains config related to the registry + Registry `toml:"registry" json:"registry,omitempty"` + // StreamServerAddress is the ip address streaming server is listening on. + StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"` + // StreamServerPort is the port streaming server is listening on. + StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"` + // EnableSelinux indicates to enable the selinux support. + EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"` + // SandboxImage is the image used by sandbox container. + SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"` + // StatsCollectPeriod is the period (in seconds) of snapshots stats collection. + StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"` + // SystemdCgroup enables systemd cgroup support. + SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"` + // EnableIPv6DAD enables IPv6 DAD. + // TODO(random-liu): Use optimistic_dad when it's GA. + EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"` +} + +// Config contains all configurations for cri server. +type Config struct { + // PluginConfig is the config for CRI plugin. + PluginConfig + // ContainerdRootDir is the root directory path for containerd. + ContainerdRootDir string `json:"containerdRootDir,omitempty"` + // ContainerdEndpoint is the containerd endpoint path. + ContainerdEndpoint string `json:"containerdEndpoint,omitempty"` + // RootDir is the root directory path for managing cri-containerd files + // (metadata checkpoint etc.) + RootDir string `json:"rootDir,omitempty"` +} + +// DefaultConfig returns default configurations of cri plugin. +func DefaultConfig() PluginConfig { + return PluginConfig{ + CniConfig: CniConfig{ + NetworkPluginBinDir: "/opt/cni/bin", + NetworkPluginConfDir: "/etc/cni/net.d", + }, + ContainerdConfig: ContainerdConfig{ + Snapshotter: containerd.DefaultSnapshotter, + Runtime: "io.containerd.runtime.v1.linux", + RuntimeEngine: "", + RuntimeRoot: "", + }, + StreamServerAddress: "", + StreamServerPort: "10010", + EnableSelinux: false, + SandboxImage: "gcr.io/google_containers/pause:3.0", + StatsCollectPeriod: 10, + SystemdCgroup: false, + EnableIPv6DAD: false, + Registry: Registry{ + Mirrors: map[string]Mirror{ + "docker.io": { + Endpoints: []string{"https://registry-1.docker.io"}, + }, + }, + }, + } +} diff --git a/pkg/server/service.go b/pkg/server/service.go index f502709db..f5930800e 100644 --- a/pkg/server/service.go +++ b/pkg/server/service.go @@ -19,10 +19,7 @@ package server import ( "fmt" "io" - "net" - "os" "path/filepath" - "syscall" "time" "github.com/containerd/containerd" @@ -37,9 +34,9 @@ import ( runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/server/streaming" - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" api "github.com/containerd/cri-containerd/pkg/api/v1" "github.com/containerd/cri-containerd/pkg/atomic" + criconfig "github.com/containerd/cri-containerd/pkg/config" osinterface "github.com/containerd/cri-containerd/pkg/os" "github.com/containerd/cri-containerd/pkg/registrar" containerstore "github.com/containerd/cri-containerd/pkg/store/container" @@ -48,12 +45,8 @@ import ( snapshotstore "github.com/containerd/cri-containerd/pkg/store/snapshot" ) -const ( - // k8sContainerdNamespace is the namespace we use to connect containerd. - k8sContainerdNamespace = "k8s.io" - // unixProtocol is the network protocol of unix socket. - unixProtocol = "unix" -) +// k8sContainerdNamespace is the namespace we use to connect containerd. +const k8sContainerdNamespace = "k8s.io" // grpcServices are all the grpc services provided by cri containerd. type grpcServices interface { @@ -64,7 +57,7 @@ type grpcServices interface { // CRIContainerdService is the interface implement CRI remote service server. type CRIContainerdService interface { - Run(bool) error + Run() error // io.Closer is used by containerd to gracefully stop cri service. io.Closer plugin.Service @@ -74,15 +67,13 @@ type CRIContainerdService interface { // criContainerdService implements CRIContainerdService. type criContainerdService struct { // config contains all configurations. - config options.Config + config criconfig.Config // imageFSPath is the path to image filesystem. imageFSPath string // apparmorEnabled indicates whether apparmor is enabled. apparmorEnabled bool // seccompEnabled indicates whether seccomp is enabled. seccompEnabled bool - // server is the grpc server. - server *grpc.Server // os is an interface for all required os operations. os osinterface.OS // sandboxStore stores all resources associated with sandboxes. @@ -113,7 +104,7 @@ type criContainerdService struct { } // NewCRIContainerdService returns a new instance of CRIContainerdService -func NewCRIContainerdService(config options.Config) (CRIContainerdService, error) { +func NewCRIContainerdService(config criconfig.Config) (CRIContainerdService, error) { var err error c := &criContainerdService{ config: config, @@ -153,12 +144,6 @@ func NewCRIContainerdService(config options.Config) (CRIContainerdService, error c.eventMonitor = newEventMonitor(c.containerStore, c.sandboxStore) - // To avoid race condition between `Run` and `Stop`, still create grpc server - // although we may not use it. It's just a small in-memory data structure. - // TODO(random-liu): Get rid of the grpc server when completely switch - // to plugin mode. - c.server = grpc.NewServer() - return c, nil } @@ -172,11 +157,8 @@ func (c *criContainerdService) Register(s *grpc.Server) error { return nil } -// Run starts the cri-containerd service. startGRPC specifies -// whether to start grpc server in this function. -// TODO(random-liu): Remove `startRPC=true` case when we no longer support cri-containerd -// standalone mode. -func (c *criContainerdService) Run(startGRPC bool) error { +// Run starts the cri-containerd service. +func (c *criContainerdService) Run() error { logrus.Info("Start cri-containerd service") // Connect containerd service here, to get rid of the containerd dependency @@ -226,35 +208,10 @@ func (c *criContainerdService) Run(startGRPC bool) error { // Set the server as initialized. GRPC services could start serving traffic. c.initialized.Set() - grpcServerCloseCh := make(chan struct{}) - if startGRPC { - // Create the grpc server and register runtime and image services. - c.Register(c.server) // nolint: errcheck - // Start grpc server. - // Unlink to cleanup the previous socket file. - logrus.Info("Start grpc server") - err := syscall.Unlink(c.config.SocketPath) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to unlink socket file %q: %v", c.config.SocketPath, err) - } - l, err := net.Listen(unixProtocol, c.config.SocketPath) - if err != nil { - return fmt.Errorf("failed to listen on %q: %v", c.config.SocketPath, err) - } - go func() { - if err := c.server.Serve(l); err != nil { - logrus.WithError(err).Error("Failed to serve grpc request") - } - close(grpcServerCloseCh) - }() - } - // Keep grpcServerCloseCh open if grpc server is not started. - // Stop the whole cri-containerd service if any of the critical service exits. select { case <-eventMonitorCloseCh: case <-streamServerCloseCh: - case <-grpcServerCloseCh: } if err := c.Close(); err != nil { return fmt.Errorf("failed to stop cri service: %v", err) @@ -277,11 +234,6 @@ func (c *criContainerdService) Run(startGRPC bool) error { case <-time.After(streamServerStopTimeout): logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout) } - if startGRPC { - // Only wait for grpc server close channel when grpc server is started. - <-grpcServerCloseCh - logrus.Info("GRPC server stopped") - } return nil } @@ -293,7 +245,6 @@ func (c *criContainerdService) Close() error { if err := c.streamServer.Stop(); err != nil { return fmt.Errorf("failed to stop stream server: %v", err) } - c.server.Stop() return nil } diff --git a/pkg/server/service_test.go b/pkg/server/service_test.go index 0743ee4d4..51bdf34d8 100644 --- a/pkg/server/service_test.go +++ b/pkg/server/service_test.go @@ -17,7 +17,7 @@ limitations under the License. package server import ( - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" + criconfig "github.com/containerd/cri-containerd/pkg/config" ostesting "github.com/containerd/cri-containerd/pkg/os/testing" "github.com/containerd/cri-containerd/pkg/registrar" servertesting "github.com/containerd/cri-containerd/pkg/server/testing" @@ -39,9 +39,9 @@ const ( // newTestCRIContainerdService creates a fake criContainerdService for test. func newTestCRIContainerdService() *criContainerdService { return &criContainerdService{ - config: options.Config{ + config: criconfig.Config{ RootDir: testRootDir, - PluginConfig: options.PluginConfig{ + PluginConfig: criconfig.PluginConfig{ SandboxImage: testSandboxImage, }, },