github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Remove standalone mode

Browse Source 
Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu 2018-03-05 19:13:29 +00:00
parent 64b098a293
commit d1e9960180
8 changed files with 173 additions and 659 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

229
cmd/cri-containerd/cri_containerd.go
View File

@ -1,229 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"flag"
"fmt"
"net"
"net/http"
"net/http/pprof"
"os"
"os/signal"
"runtime"
"syscall"
"github.com/containerd/cgroups"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/sys"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"k8s.io/kubernetes/pkg/util/interrupt"
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
"github.com/containerd/cri-containerd/pkg/server"
"github.com/containerd/cri-containerd/pkg/version"
)
// Add \u200B to avoid the space trimming.
const desc = "\u200B" + ` _ __ _ __
__________(_) _________ ____ / /_____ _(_)____ ___ _________/ /
/ ___/ ___/ /______/ ___/ __ \/ __ \/ __/ __ ` + "`" + `/ // __ \/ _ \/ ___/ __ /
/ /__/ / / //_____/ /__/ /_/ / / / / /_/ /_/ / // / / / __/ / / /_/ /
\___/_/ /_/ \___/\____/_/ /_/\__/\__,_/_//_/ /_/\___/_/ \__,_/
A containerd based Kubernetes CRI implementation.
`
var cmd = &cobra.Command{
Use: "cri-containerd",
Short: "A containerd based Kubernetes CRI implementation.",
Long: desc,
}
func defaultConfigCommand() *cobra.Command {
return &cobra.Command{
Use: "default-config",
Short: "Print default toml config of cri-containerd.",
Run: func(cmd *cobra.Command, args []string) {
options.PrintDefaultTomlConfig()
},
}
}
func versionCommand() *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "Print cri-containerd version information.",
Run: func(cmd *cobra.Command, args []string) {
version.PrintVersion()
},
}
}
func main() {
o := options.NewCRIContainerdOptions()
o.AddFlags(cmd.Flags())
cmd.AddCommand(defaultConfigCommand())
cmd.AddCommand(versionCommand())
cmd.RunE = func(cmd *cobra.Command, args []string) error {
setupDumpStacksTrap()
if err := o.InitFlags(cmd.Flags()); err != nil {
return fmt.Errorf("failed to init CRI containerd flags: %v", err)
}
if err := setLogLevel(o.LogLevel); err != nil {
return fmt.Errorf("failed to set log level: %v", err)
}
logrus.Infof("Run cri-containerd %+v", o)
if o.CgroupPath != "" {
_, err := loadCgroup(o.CgroupPath)
if err != nil {
return fmt.Errorf("failed to load cgroup for cgroup path %v: %v", o.CgroupPath, err)
}
}
if o.OOMScore != 0 {
if err := sys.SetOOMScore(os.Getpid(), o.OOMScore); err != nil {
return fmt.Errorf("failed to set OOMScore to %v: %v", o.OOMScore, err)
}
}
// Start profiling server if enable.
if o.EnableProfiling {
logrus.Info("Start profiling server")
go startProfilingServer(o.ProfilingAddress, o.ProfilingPort)
}
logrus.Infof("Run cri-containerd grpc server on socket %q", o.SocketPath)
s, err := server.NewCRIContainerdService(o.Config)
if err != nil {
return fmt.Errorf("failed to create CRI containerd service: %v", err)
}
// Use interrupt handler to make sure the server is stopped properly.
// Pass in non-empty final function to avoid os.Exit(1). We expect `Run`
// to return itself.
h := interrupt.New(func(os.Signal) {}, func() {
if err := s.Close(); err != nil {
logrus.WithError(err).Error("Failed to stop cri service")
}
})
if err := h.Run(func() error { return s.Run(true) }); err != nil {
return fmt.Errorf("failed to run cri-containerd with grpc server: %v", err)
}
return nil
}
if err := cmd.Execute(); err != nil {
// Error should have been reported.
os.Exit(1)
}
}
func setupDumpStacksTrap() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGUSR1)
go func() {
for range c {
dumpStacks()
}
}()
}
func dumpStacks() {
buf := make([]byte, 1024)
for {
n := runtime.Stack(buf, true)
if n < len(buf) {
buf = buf[:n]
break
}
buf = make([]byte, 2*len(buf))
}
logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
}
// startProfilingServer start http server to profiling via web interface
func startProfilingServer(host string, port string) {
endpoint := net.JoinHostPort(host, port)
mux := http.NewServeMux()
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
if err := http.ListenAndServe(endpoint, mux); err != nil {
logrus.WithError(err).Error("Failed to start profiling server")
}
}
func setLogLevel(l string) error {
lvl, err := log.ParseLevel(l)
if err != nil {
return err
}
if err := setGLogLevel(lvl); err != nil {
return err
}
logrus.SetLevel(lvl)
return nil
}
// TODO(random-liu): Set glog level in plugin mode.
func setGLogLevel(l logrus.Level) error {
if err := flag.Set("logtostderr", "true"); err != nil {
return err
}
switch l {
case log.TraceLevel:
return flag.Set("v", "5")
case logrus.DebugLevel:
return flag.Set("v", "4")
case logrus.InfoLevel:
return flag.Set("v", "2")
// glog doesn't support following filters. Defaults to v=0.
case logrus.WarnLevel:
case logrus.ErrorLevel:
case logrus.FatalLevel:
case logrus.PanicLevel:
}
return nil
}
// loadCgroup loads the cgroup associated with path if it exists and moves the current process into the cgroup. If the cgroup
// is not created it is created and returned.
func loadCgroup(cgroupPath string) (cgroups.Cgroup, error) {
cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(cgroupPath))
if err != nil {
if err != cgroups.ErrCgroupDeleted {
return nil, err
}
if cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(cgroupPath), &runtimespec.LinuxResources{}); err != nil {
return nil, err
}
}
if err := cg.Add(cgroups.Process{
Pid: os.Getpid(),
}); err != nil {
return nil, err
}
return cg, nil
}

273
cmd/cri-containerd/options/options.go
View File

@ -1,273 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"os"
"github.com/BurntSushi/toml"
"github.com/containerd/containerd"
"github.com/spf13/pflag"
)
const (
// configFilePathArgName is the path to the config file.
configFilePathArgName = "config"
// defaultConfigFilePath is the default config file path.
defaultConfigFilePath = "/etc/cri-containerd/config.toml"
)
// ContainerdConfig contains toml config related to containerd
type ContainerdConfig struct {
// RootDir is the root directory path for containerd.
// TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode.
RootDir string `toml:"root_dir" json:"rootDir,omitempty"`
// Snapshotter is the snapshotter used by containerd.
Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"`
// Endpoint is the containerd endpoint path.
// TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode.
Endpoint string `toml:"endpoint" json:"endpoint,omitempty"`
// Runtime is the runtime to use in containerd. We may support
// other runtimes in the future.
Runtime string `toml:"runtime" json:"runtime,omitempty"`
// RuntimeEngine is the name of the runtime engine used by containerd.
// Containerd default should be "runc"
// We may support other runtime engines in the future.
RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
// RuntimeRoot is the directory used by containerd for runtime state.
// Containerd default should be "/run/containerd/runc"
RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
}
// CniConfig contains toml config related to cni
type CniConfig struct {
// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"`
// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"`
}
// PluginConfig contains toml config related to CRI plugin,
// it is a subset of Config.
type PluginConfig struct {
// ContainerdConfig contains config related to containerd
ContainerdConfig `toml:"containerd" json:"containerd,omitempty"`
// CniConfig contains config related to cni
CniConfig `toml:"cni" json:"cni,omitempty"`
// Registry contains config related to the registry
Registry `toml:"registry" json:"registry,omitempty"`
// StreamServerAddress is the ip address streaming server is listening on.
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"`
// StreamServerPort is the port streaming server is listening on.
StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"`
// EnableSelinux indicates to enable the selinux support.
EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"`
// SandboxImage is the image used by sandbox container.
SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"`
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"`
// SystemdCgroup enables systemd cgroup support.
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"`
// EnableIPv6DAD enables IPv6 DAD.
// TODO(random-liu): Use optimistic_dad when it's GA.
EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"`
}
// Config contains toml config related cri-containerd daemon.
// TODO(random-liu): Make this an internal config object when we no longer support cri-containerd
// standalone mode. At that time, we can clean this up.
type Config struct {
// PluginConfig is the config for CRI plugin.
PluginConfig
// ContainerdRootDir is the root directory path for containerd.
ContainerdRootDir string `toml:"-" json:"containerdRootDir,omitempty"`
// ContainerdEndpoint is the containerd endpoint path.
ContainerdEndpoint string `toml:"-" json:"containerdEndpoint,omitempty"`
// SocketPath is the path to the socket which cri-containerd serves on.
// TODO(random-liu): Remove SocketPath when no longer support cri-containerd
// standalone mode.
SocketPath string `toml:"socket_path" json:"socketPath,omitempty"`
// RootDir is the root directory path for managing cri-containerd files
// (metadata checkpoint etc.)
RootDir string `toml:"root_dir" json:"rootDir,omitempty"`
// TODO(random-liu): Remove following fields when we no longer support cri-containerd
// standalone mode.
// CgroupPath is the path for the cgroup that cri-containerd is placed in.
CgroupPath string `toml:"cgroup_path" json:"cgroupPath,omitempty"`
// OOMScore adjust the cri-containerd's oom score
OOMScore int `toml:"oom_score" json:"oomScore,omitempty"`
// EnableProfiling is used for enable profiling via host:port/debug/pprof/
EnableProfiling bool `toml:"profiling" json:"enableProfiling,omitempty"`
// ProfilingPort is the port for profiling via host:port/debug/pprof/
ProfilingPort string `toml:"profiling_port" json:"profilingPort,omitempty"`
// ProfilingAddress is address for profiling via host:port/debug/pprof/
ProfilingAddress string `toml:"profiling_addr" json:"profilingAddress,omitempty"`
// LogLevel is the logrus log level.
LogLevel string `toml:"log_level" json:"logLevel,omitempty"`
}
// CRIContainerdOptions contains cri-containerd command line and toml options.
type CRIContainerdOptions struct {
// Config contains cri-containerd toml config
Config
// ConfigFilePath is the path to the TOML config file.
ConfigFilePath string `toml:"-"`
}
// NewCRIContainerdOptions returns a reference to CRIContainerdOptions
func NewCRIContainerdOptions() *CRIContainerdOptions {
return &CRIContainerdOptions{}
}
// AddFlags adds cri-containerd command line options to pflag.
func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) {
defaults := DefaultConfig()
fs.StringVar(&c.ConfigFilePath, configFilePathArgName,
defaultConfigFilePath, "Path to the config file.")
fs.StringVar(&c.LogLevel, "log-level",
defaults.LogLevel, "Set the logging level [trace, debug, info, warn, error, fatal, panic].")
fs.StringVar(&c.SocketPath, "socket-path",
defaults.SocketPath, "Path to the socket which cri-containerd serves on.")
fs.StringVar(&c.RootDir, "root-dir",
defaults.RootDir, "Root directory path for cri-containerd managed files (metadata checkpoint etc).")
fs.StringVar(&c.ContainerdRootDir, "containerd-root-dir",
defaults.ContainerdRootDir, "Root directory path where containerd stores persistent data.")
fs.StringVar(&c.ContainerdEndpoint, "containerd-endpoint",
defaults.ContainerdEndpoint, "Path to the containerd endpoint.")
fs.StringVar(&c.ContainerdConfig.Snapshotter, "containerd-snapshotter",
defaults.ContainerdConfig.Snapshotter, "The snapshotter used by containerd.")
fs.StringVar(&c.ContainerdConfig.Runtime, "containerd-runtime",
defaults.ContainerdConfig.Runtime, "The runtime used by containerd.")
fs.StringVar(&c.ContainerdConfig.RuntimeEngine, "containerd-runtime-engine",
defaults.ContainerdConfig.RuntimeEngine, "Runtime engine used by containerd. Defaults to containerd's default if not specified.")
fs.StringVar(&c.ContainerdConfig.RuntimeRoot, "containerd-runtime-root",
defaults.ContainerdConfig.RuntimeRoot, "The directory used by containerd for runtime state. Defaults to containerd's default if not specified.")
fs.StringVar(&c.NetworkPluginBinDir, "network-bin-dir",
defaults.NetworkPluginBinDir, "The directory for putting network binaries.")
fs.StringVar(&c.NetworkPluginConfDir, "network-conf-dir",
defaults.NetworkPluginConfDir, "The directory for putting network plugin configuration files.")
fs.StringVar(&c.StreamServerAddress, "stream-addr",
defaults.StreamServerAddress, "The ip address streaming server is listening on. The default host interface is used if not specified.")
fs.StringVar(&c.StreamServerPort, "stream-port",
defaults.StreamServerPort, "The port streaming server is listening on.")
fs.StringVar(&c.CgroupPath, "cgroup-path",
defaults.CgroupPath, "The cgroup that cri-containerd is part of. Cri-containerd is not placed in a cgroup if none is specified.")
fs.BoolVar(&c.EnableSelinux, "enable-selinux",
defaults.EnableSelinux, "Enable selinux support. By default not enabled.")
fs.StringVar(&c.SandboxImage, "sandbox-image",
defaults.SandboxImage, "The image used by sandbox container.")
fs.IntVar(&c.StatsCollectPeriod, "stats-collect-period",
defaults.StatsCollectPeriod, "The period (in seconds) of snapshots stats collection.")
fs.BoolVar(&c.SystemdCgroup, "systemd-cgroup",
defaults.SystemdCgroup, "Enables systemd cgroup support. By default not enabled.")
fs.IntVar(&c.OOMScore, "oom-score",
defaults.OOMScore, "Adjust the cri-containerd's oom score.")
fs.BoolVar(&c.EnableProfiling, "profiling",
defaults.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/.")
fs.StringVar(&c.ProfilingPort, "profiling-port",
defaults.ProfilingPort, "Profiling port for web interface host:port/debug/pprof/.")
fs.StringVar(&c.ProfilingAddress, "profiling-addr",
defaults.ProfilingAddress, "Profiling address for web interface host:port/debug/pprof/.")
fs.BoolVar(&c.EnableIPv6DAD, "enable-ipv6-dad",
defaults.EnableIPv6DAD, "Enable IPv6 DAD (duplicate address detection) for pod sandbox network. Enabling this will increase pod sandbox start latency by several seconds.")
fs.Var(&c.Registry, "registry",
"Registry config for image pull eg --registry=myregistry.io=https://mymirror.io/ --registry=myregistry2.io=https://mymirror2.io/")
}
// InitFlags load configurations from config file, and then overwrite with flags.
// This function must be called inside `Run`, at that time flags should have been
// parsed once.
// precedence: commandline > configfile > default
func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error {
// Load default config file if none provided
if _, err := toml.DecodeFile(c.ConfigFilePath, &c.Config); err != nil {
// the absence of default config file is normal case.
if !fs.Changed(configFilePathArgName) && os.IsNotExist(err) {
return nil
}
return err
}
// Add this for backward compatibility.
// TODO(random-liu): Remove this when we no longer support cri-containerd standalone mode.
if c.ContainerdConfig.RootDir != "" {
c.ContainerdRootDir = c.ContainerdConfig.RootDir
}
if c.ContainerdConfig.Endpoint != "" {
c.ContainerdEndpoint = c.ContainerdConfig.Endpoint
}
// What is the reason for applying the command line twice?
// Because the values from command line have the highest priority.
// The path of toml configuration file if from the command line,
// and triggers the first parse.
// The first parse generates the default value and the value from command line at the same time.
// But the priority of the toml config value is higher than the default value,
// Without a way to insert the toml config value between the default value and the command line value.
// We parse twice one for default value, one for commandline value.
return fs.Parse(os.Args[1:])
}
// PrintDefaultTomlConfig print default toml config of cri-containerd.
func PrintDefaultTomlConfig() {
if err := toml.NewEncoder(os.Stdout).Encode(DefaultConfig()); err != nil {
fmt.Println(err)
return
}
}
// DefaultConfig returns default configurations of cri-containerd.
func DefaultConfig() Config {
return Config{
PluginConfig: PluginConfig{
CniConfig: CniConfig{
NetworkPluginBinDir: "/opt/cni/bin",
NetworkPluginConfDir: "/etc/cni/net.d",
},
ContainerdConfig: ContainerdConfig{
Snapshotter: containerd.DefaultSnapshotter,
Runtime: "io.containerd.runtime.v1.linux",
RuntimeEngine: "",
RuntimeRoot: "",
},
StreamServerAddress: "",
StreamServerPort: "10010",
EnableSelinux: false,
SandboxImage: "gcr.io/google_containers/pause:3.0",
StatsCollectPeriod: 10,
SystemdCgroup: false,
EnableIPv6DAD: false,
Registry: Registry{
Mirrors: map[string]Mirror{
"docker.io": {
Endpoints: []string{"https://registry-1.docker.io"},
},
},
},
},
ContainerdRootDir: "/var/lib/containerd",
ContainerdEndpoint: "/run/containerd/containerd.sock",
SocketPath: "/var/run/cri-containerd.sock",
RootDir: "/var/lib/cri-containerd",
CgroupPath: "",
OOMScore: -999,
EnableProfiling: true,
ProfilingPort: "10011",
ProfilingAddress: "127.0.0.1",
LogLevel: "info",
}
}

88
cmd/cri-containerd/options/registry.go
View File

@ -1,88 +0,0 @@
/*
Copyright 2018 The Containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"net/url"
"strings"
)
// Mirror contains the config related to the registry mirror
type Mirror struct {
Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"`
// TODO (Abhi) We might need to add auth per namespace. Looks like
// image auth information is passed by kube itself.
}
// Registry is registry settings configured
type Registry struct {
Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"`
}
// String returns the string format of registry type
func (r *Registry) String() string {
// Its not used hence return empty string
return ""
}
// Set validates and converts into the internal registry struct
func (r *Registry) Set(s string) error {
// --registry docker.io=https://mymirror.io,http://mymirror2.io
// If no option is set then return format error
if len(s) == 0 {
return fmt.Errorf("incomplete registry mirror option")
}
var mirrors []string
host := "docker.io"
opt := strings.Split(s, "=")
if len(opt) > 1 {
// If option is set in the format "mynamespace.io=https://mymirror.io,https://mymirror2.io"
// Then associate the mirror urls for the namespace only"
host = opt[0]
mirrors = strings.Split(opt[1], ",")
} else {
// If option is set in the format "https://mymirror.io,https://mymirror.io"
// Then associate mirror against default docker.io namespace
mirrors = strings.Split(opt[0], ",")
}
// Validate the format of the urls passed
for _, u := range mirrors {
_, err := url.Parse(u)
if err != nil {
return fmt.Errorf("invalid registry mirror url format %v: %v", u, err)
}
}
if r.Mirrors == nil {
r.Mirrors = make(map[string]Mirror)
}
if _, ok := r.Mirrors[host]; !ok {
r.Mirrors[host] = Mirror{}
}
m := r.Mirrors[host]
m.Endpoints = append(m.Endpoints, mirrors...)
r.Mirrors[host] = m
return nil
}
// Type returns a string name for the option type
func (r *Registry) Type() string {
return "list"
}

6
cmd/ctrcri/ctrcri.go
View File

@ -20,10 +20,10 @@ import (
"os" "os"
"time" "time"
"github.com/containerd/containerd/defaults"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
"github.com/containerd/cri-containerd/pkg/version" "github.com/containerd/cri-containerd/pkg/version"
) )
@ -56,9 +56,7 @@ var (
) )
func addGlobalFlags(fs *pflag.FlagSet) { func addGlobalFlags(fs *pflag.FlagSet) {
// TODO(random-liu): Change default to containerd/defaults.DefaultAddress after cri plugin fs.StringVar(&address, "address", defaults.DefaultAddress, "address for containerd's GRPC server.")
// become default.
fs.StringVar(&address, "address", options.DefaultConfig().SocketPath, "address for containerd's GRPC server.")
fs.DurationVar(&timeout, "timeout", defaultTimeout, "timeout for containerd grpc connection.") fs.DurationVar(&timeout, "timeout", defaultTimeout, "timeout for containerd grpc connection.")
} }

38
cri.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package cri package cri
import ( import (
"flag"
"path/filepath" "path/filepath"
"github.com/containerd/containerd/log" "github.com/containerd/containerd/log"
@ -24,8 +25,9 @@ import (
"github.com/containerd/containerd/plugin" "github.com/containerd/containerd/plugin"
imagespec "github.com/opencontainers/image-spec/specs-go/v1" imagespec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/containerd/cri-containerd/cmd/cri-containerd/options" criconfig "github.com/containerd/cri-containerd/pkg/config"
"github.com/containerd/cri-containerd/pkg/server" "github.com/containerd/cri-containerd/pkg/server"
) )
@ -35,7 +37,7 @@ const criVersion = "v1alpha2"
// TODO(random-liu): Use github.com/pkg/errors for our errors. // TODO(random-liu): Use github.com/pkg/errors for our errors.
// Register CRI service plugin // Register CRI service plugin
func init() { func init() {
config := options.DefaultConfig().PluginConfig config := criconfig.DefaultConfig()
plugin.Register(&plugin.Registration{ plugin.Register(&plugin.Registration{
Type: plugin.GRPCPlugin, Type: plugin.GRPCPlugin,
ID: "cri", ID: "cri",
@ -57,8 +59,8 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()} ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()}
ic.Meta.Exports = map[string]string{"CRIVersion": criVersion} ic.Meta.Exports = map[string]string{"CRIVersion": criVersion}
ctx := ic.Context ctx := ic.Context
pluginConfig := ic.Config.(*options.PluginConfig) pluginConfig := ic.Config.(*criconfig.PluginConfig)
c := options.Config{ c := criconfig.Config{
PluginConfig: *pluginConfig, PluginConfig: *pluginConfig,
// This is a hack. We assume that containerd root directory // This is a hack. We assume that containerd root directory
// is one level above plugin directory. // is one level above plugin directory.
@ -69,6 +71,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
} }
log.G(ctx).Infof("Start cri plugin with config %+v", c) log.G(ctx).Infof("Start cri plugin with config %+v", c)
if err := setGLogLevel(); err != nil {
return nil, errors.Wrap(err, "failed to set glog level")
}
s, err := server.NewCRIContainerdService(c) s, err := server.NewCRIContainerdService(c)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to create CRI service") return nil, errors.Wrap(err, "failed to create CRI service")
@ -77,10 +83,32 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
// Use a goroutine to initialize cri service. The reason is that currently // Use a goroutine to initialize cri service. The reason is that currently
// cri service requires containerd to be initialize. // cri service requires containerd to be initialize.
go func() { go func() {
if err := s.Run(false); err != nil { if err := s.Run(); err != nil {
log.G(ctx).WithError(err).Fatal("Failed to run CRI service") log.G(ctx).WithError(err).Fatal("Failed to run CRI service")
} }
// TODO(random-liu): Whether and how we can stop containerd. // TODO(random-liu): Whether and how we can stop containerd.
}() }()
return s, nil return s, nil
} }
// Set glog level.
func setGLogLevel() error {
l := logrus.GetLevel()
if err := flag.Set("logtostderr", "true"); err != nil {
return err
}
switch l {
case log.TraceLevel:
return flag.Set("v", "5")
case logrus.DebugLevel:
return flag.Set("v", "4")
case logrus.InfoLevel:
return flag.Set("v", "2")
// glog doesn't support following filters. Defaults to v=0.
case logrus.WarnLevel:
case logrus.ErrorLevel:
case logrus.FatalLevel:
case logrus.PanicLevel:
}
return nil
}

127
pkg/config/config.go Normal file
View File

@ -0,0 +1,127 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import "github.com/containerd/containerd"
// ContainerdConfig contains toml config related to containerd
type ContainerdConfig struct {
// Snapshotter is the snapshotter used by containerd.
Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"`
// Runtime is the runtime to use in containerd. We may support
// other runtimes in the future.
Runtime string `toml:"runtime" json:"runtime,omitempty"`
// RuntimeEngine is the name of the runtime engine used by containerd.
// Containerd default should be "runc"
// We may support other runtime engines in the future.
RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
// RuntimeRoot is the directory used by containerd for runtime state.
// Containerd default should be "/run/containerd/runc"
RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
}
// CniConfig contains toml config related to cni
type CniConfig struct {
// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"`
// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"`
}
// Mirror contains the config related to the registry mirror
type Mirror struct {
// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
// one by one until a working one is found.
Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"`
// TODO (Abhi) We might need to add auth per namespace. Looks like
// image auth information is passed by kube itself.
}
// Registry is registry settings configured
type Registry struct {
// Mirrors are namespace to mirror mapping for all namespaces.
Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"`
}
// PluginConfig contains toml config related to CRI plugin,
// it is a subset of Config.
type PluginConfig struct {
// ContainerdConfig contains config related to containerd
ContainerdConfig `toml:"containerd" json:"containerd,omitempty"`
// CniConfig contains config related to cni
CniConfig `toml:"cni" json:"cni,omitempty"`
// Registry contains config related to the registry
Registry `toml:"registry" json:"registry,omitempty"`
// StreamServerAddress is the ip address streaming server is listening on.
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"`
// StreamServerPort is the port streaming server is listening on.
StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"`
// EnableSelinux indicates to enable the selinux support.
EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"`
// SandboxImage is the image used by sandbox container.
SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"`
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"`
// SystemdCgroup enables systemd cgroup support.
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"`
// EnableIPv6DAD enables IPv6 DAD.
// TODO(random-liu): Use optimistic_dad when it's GA.
EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"`
}
// Config contains all configurations for cri server.
type Config struct {
// PluginConfig is the config for CRI plugin.
PluginConfig
// ContainerdRootDir is the root directory path for containerd.
ContainerdRootDir string `json:"containerdRootDir,omitempty"`
// ContainerdEndpoint is the containerd endpoint path.
ContainerdEndpoint string `json:"containerdEndpoint,omitempty"`
// RootDir is the root directory path for managing cri-containerd files
// (metadata checkpoint etc.)
RootDir string `json:"rootDir,omitempty"`
}
// DefaultConfig returns default configurations of cri plugin.
func DefaultConfig() PluginConfig {
return PluginConfig{
CniConfig: CniConfig{
NetworkPluginBinDir: "/opt/cni/bin",
NetworkPluginConfDir: "/etc/cni/net.d",
},
ContainerdConfig: ContainerdConfig{
Snapshotter: containerd.DefaultSnapshotter,
Runtime: "io.containerd.runtime.v1.linux",
RuntimeEngine: "",
RuntimeRoot: "",
},
StreamServerAddress: "",
StreamServerPort: "10010",
EnableSelinux: false,
SandboxImage: "gcr.io/google_containers/pause:3.0",
StatsCollectPeriod: 10,
SystemdCgroup: false,
EnableIPv6DAD: false,
Registry: Registry{
Mirrors: map[string]Mirror{
"docker.io": {
Endpoints: []string{"https://registry-1.docker.io"},
},
},
},
}
}

65
pkg/server/service.go
View File

@ -19,10 +19,7 @@ package server
import ( import (
"fmt" "fmt"
"io" "io"
"net"
"os"
"path/filepath" "path/filepath"
"syscall"
"time" "time"
"github.com/containerd/containerd" "github.com/containerd/containerd"
@ -37,9 +34,9 @@ import (
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/kubelet/server/streaming" "k8s.io/kubernetes/pkg/kubelet/server/streaming"
"github.com/containerd/cri-containerd/cmd/cri-containerd/options"
api "github.com/containerd/cri-containerd/pkg/api/v1" api "github.com/containerd/cri-containerd/pkg/api/v1"
"github.com/containerd/cri-containerd/pkg/atomic" "github.com/containerd/cri-containerd/pkg/atomic"
criconfig "github.com/containerd/cri-containerd/pkg/config"
osinterface "github.com/containerd/cri-containerd/pkg/os" osinterface "github.com/containerd/cri-containerd/pkg/os"
"github.com/containerd/cri-containerd/pkg/registrar" "github.com/containerd/cri-containerd/pkg/registrar"
containerstore "github.com/containerd/cri-containerd/pkg/store/container" containerstore "github.com/containerd/cri-containerd/pkg/store/container"
@ -48,12 +45,8 @@ import (
snapshotstore "github.com/containerd/cri-containerd/pkg/store/snapshot" snapshotstore "github.com/containerd/cri-containerd/pkg/store/snapshot"
) )
const ( // k8sContainerdNamespace is the namespace we use to connect containerd.
// k8sContainerdNamespace is the namespace we use to connect containerd. const k8sContainerdNamespace = "k8s.io"
k8sContainerdNamespace = "k8s.io"
// unixProtocol is the network protocol of unix socket.
unixProtocol = "unix"
)
// grpcServices are all the grpc services provided by cri containerd. // grpcServices are all the grpc services provided by cri containerd.
type grpcServices interface { type grpcServices interface {
@ -64,7 +57,7 @@ type grpcServices interface {
// CRIContainerdService is the interface implement CRI remote service server. // CRIContainerdService is the interface implement CRI remote service server.
type CRIContainerdService interface { type CRIContainerdService interface {
Run(bool) error Run() error
// io.Closer is used by containerd to gracefully stop cri service. // io.Closer is used by containerd to gracefully stop cri service.
io.Closer io.Closer
plugin.Service plugin.Service
@ -74,15 +67,13 @@ type CRIContainerdService interface {
// criContainerdService implements CRIContainerdService. // criContainerdService implements CRIContainerdService.
type criContainerdService struct { type criContainerdService struct {
// config contains all configurations. // config contains all configurations.
config options.Config config criconfig.Config
// imageFSPath is the path to image filesystem. // imageFSPath is the path to image filesystem.
imageFSPath string imageFSPath string
// apparmorEnabled indicates whether apparmor is enabled. // apparmorEnabled indicates whether apparmor is enabled.
apparmorEnabled bool apparmorEnabled bool
// seccompEnabled indicates whether seccomp is enabled. // seccompEnabled indicates whether seccomp is enabled.
seccompEnabled bool seccompEnabled bool
// server is the grpc server.
server *grpc.Server
// os is an interface for all required os operations. // os is an interface for all required os operations.
os osinterface.OS os osinterface.OS
// sandboxStore stores all resources associated with sandboxes. // sandboxStore stores all resources associated with sandboxes.
@ -113,7 +104,7 @@ type criContainerdService struct {
} }
// NewCRIContainerdService returns a new instance of CRIContainerdService // NewCRIContainerdService returns a new instance of CRIContainerdService
func NewCRIContainerdService(config options.Config) (CRIContainerdService, error) { func NewCRIContainerdService(config criconfig.Config) (CRIContainerdService, error) {
var err error var err error
c := &criContainerdService{ c := &criContainerdService{
config: config, config: config,
@ -153,12 +144,6 @@ func NewCRIContainerdService(config options.Config) (CRIContainerdService, error
c.eventMonitor = newEventMonitor(c.containerStore, c.sandboxStore) c.eventMonitor = newEventMonitor(c.containerStore, c.sandboxStore)
// To avoid race condition between `Run` and `Stop`, still create grpc server
// although we may not use it. It's just a small in-memory data structure.
// TODO(random-liu): Get rid of the grpc server when completely switch
// to plugin mode.
c.server = grpc.NewServer()
return c, nil return c, nil
} }
@ -172,11 +157,8 @@ func (c *criContainerdService) Register(s *grpc.Server) error {
return nil return nil
} }
// Run starts the cri-containerd service. startGRPC specifies // Run starts the cri-containerd service.
// whether to start grpc server in this function. func (c *criContainerdService) Run() error {
// TODO(random-liu): Remove `startRPC=true` case when we no longer support cri-containerd
// standalone mode.
func (c *criContainerdService) Run(startGRPC bool) error {
logrus.Info("Start cri-containerd service") logrus.Info("Start cri-containerd service")
// Connect containerd service here, to get rid of the containerd dependency // Connect containerd service here, to get rid of the containerd dependency
@ -226,35 +208,10 @@ func (c *criContainerdService) Run(startGRPC bool) error {
// Set the server as initialized. GRPC services could start serving traffic. // Set the server as initialized. GRPC services could start serving traffic.
c.initialized.Set() c.initialized.Set()
grpcServerCloseCh := make(chan struct{})
if startGRPC {
// Create the grpc server and register runtime and image services.
c.Register(c.server) // nolint: errcheck
// Start grpc server.
// Unlink to cleanup the previous socket file.
logrus.Info("Start grpc server")
err := syscall.Unlink(c.config.SocketPath)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to unlink socket file %q: %v", c.config.SocketPath, err)
}
l, err := net.Listen(unixProtocol, c.config.SocketPath)
if err != nil {
return fmt.Errorf("failed to listen on %q: %v", c.config.SocketPath, err)
}
go func() {
if err := c.server.Serve(l); err != nil {
logrus.WithError(err).Error("Failed to serve grpc request")
}
close(grpcServerCloseCh)
}()
}
// Keep grpcServerCloseCh open if grpc server is not started.
// Stop the whole cri-containerd service if any of the critical service exits. // Stop the whole cri-containerd service if any of the critical service exits.
select { select {
case <-eventMonitorCloseCh: case <-eventMonitorCloseCh:
case <-streamServerCloseCh: case <-streamServerCloseCh:
case <-grpcServerCloseCh:
} }
if err := c.Close(); err != nil { if err := c.Close(); err != nil {
return fmt.Errorf("failed to stop cri service: %v", err) return fmt.Errorf("failed to stop cri service: %v", err)
@ -277,11 +234,6 @@ func (c *criContainerdService) Run(startGRPC bool) error {
case <-time.After(streamServerStopTimeout): case <-time.After(streamServerStopTimeout):
logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout) logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout)
} }
if startGRPC {
// Only wait for grpc server close channel when grpc server is started.
<-grpcServerCloseCh
logrus.Info("GRPC server stopped")
}
return nil return nil
} }
@ -293,7 +245,6 @@ func (c *criContainerdService) Close() error {
if err := c.streamServer.Stop(); err != nil { if err := c.streamServer.Stop(); err != nil {
return fmt.Errorf("failed to stop stream server: %v", err) return fmt.Errorf("failed to stop stream server: %v", err)
} }
c.server.Stop()
return nil return nil
} }

6
pkg/server/service_test.go
View File

@ -17,7 +17,7 @@ limitations under the License.
package server package server
import ( import (
"github.com/containerd/cri-containerd/cmd/cri-containerd/options" criconfig "github.com/containerd/cri-containerd/pkg/config"
ostesting "github.com/containerd/cri-containerd/pkg/os/testing" ostesting "github.com/containerd/cri-containerd/pkg/os/testing"
"github.com/containerd/cri-containerd/pkg/registrar" "github.com/containerd/cri-containerd/pkg/registrar"
servertesting "github.com/containerd/cri-containerd/pkg/server/testing" servertesting "github.com/containerd/cri-containerd/pkg/server/testing"
@ -39,9 +39,9 @@ const (
// newTestCRIContainerdService creates a fake criContainerdService for test. // newTestCRIContainerdService creates a fake criContainerdService for test.
func newTestCRIContainerdService() *criContainerdService { func newTestCRIContainerdService() *criContainerdService {
return &criContainerdService{ return &criContainerdService{
config: options.Config{ config: criconfig.Config{
RootDir: testRootDir, RootDir: testRootDir,
PluginConfig: options.PluginConfig{ PluginConfig: criconfig.PluginConfig{
SandboxImage: testSandboxImage, SandboxImage: testSandboxImage,
}, },
}, },