Move cri base plugin to CRI runtime service
Create new plugin type for CRI runtime and image services. Signed-off-by: Derek McGowan <derek@mcg.dev>
This commit is contained in:
parent
22e1a2e65c
commit
9795677fe9
@ -21,4 +21,5 @@ package builtins
|
|||||||
import (
|
import (
|
||||||
_ "github.com/containerd/containerd/v2/pkg/cri"
|
_ "github.com/containerd/containerd/v2/pkg/cri"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/cri/images"
|
_ "github.com/containerd/containerd/v2/plugins/cri/images"
|
||||||
|
_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
|
||||||
)
|
)
|
||||||
|
@ -23,6 +23,7 @@ import (
|
|||||||
_ "github.com/containerd/containerd/v2/pkg/events/plugin"
|
_ "github.com/containerd/containerd/v2/pkg/events/plugin"
|
||||||
_ "github.com/containerd/containerd/v2/pkg/nri/plugin"
|
_ "github.com/containerd/containerd/v2/pkg/nri/plugin"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/cri/images"
|
_ "github.com/containerd/containerd/v2/plugins/cri/images"
|
||||||
|
_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
|
_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/gc"
|
_ "github.com/containerd/containerd/v2/plugins/gc"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/imageverifier"
|
_ "github.com/containerd/containerd/v2/plugins/imageverifier"
|
||||||
|
@ -29,6 +29,7 @@ import (
|
|||||||
"github.com/containerd/containerd/v2/pkg/cri/server"
|
"github.com/containerd/containerd/v2/pkg/cri/server"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/server/images"
|
"github.com/containerd/containerd/v2/pkg/cri/server/images"
|
||||||
"github.com/containerd/containerd/v2/pkg/oci"
|
"github.com/containerd/containerd/v2/pkg/oci"
|
||||||
|
"github.com/containerd/errdefs"
|
||||||
)
|
)
|
||||||
|
|
||||||
func FuzzCRIServer(data []byte) int {
|
func FuzzCRIServer(data []byte) int {
|
||||||
@ -42,7 +43,6 @@ func FuzzCRIServer(data []byte) int {
|
|||||||
}
|
}
|
||||||
defer client.Close()
|
defer client.Close()
|
||||||
|
|
||||||
config := criconfig.Config{}
|
|
||||||
imageConfig := criconfig.ImageConfig{}
|
imageConfig := criconfig.ImageConfig{}
|
||||||
|
|
||||||
imageService, err := images.NewService(imageConfig, &images.CRIImageServiceOptions{
|
imageService, err := images.NewService(imageConfig, &images.CRIImageServiceOptions{
|
||||||
@ -52,10 +52,10 @@ func FuzzCRIServer(data []byte) int {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
c, rs, err := server.NewCRIService(config, &server.CRIServiceOptions{
|
c, rs, err := server.NewCRIService(&server.CRIServiceOptions{
|
||||||
ImageService: imageService,
|
RuntimeService: &fakeRuntimeService{},
|
||||||
Client: client,
|
ImageService: imageService,
|
||||||
BaseOCISpecs: map[string]*oci.Spec{},
|
Client: client,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
@ -68,6 +68,16 @@ func FuzzCRIServer(data []byte) int {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeRuntimeService struct{}
|
||||||
|
|
||||||
|
func (fakeRuntimeService) Config() criconfig.Config {
|
||||||
|
return criconfig.Config{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fakeRuntimeService) LoadOCISpec(string) (*oci.Spec, error) {
|
||||||
|
return nil, errdefs.ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
type service struct {
|
type service struct {
|
||||||
server.CRIService
|
server.CRIService
|
||||||
runtime.RuntimeServiceServer
|
runtime.RuntimeServiceServer
|
||||||
|
@ -38,6 +38,7 @@ import (
|
|||||||
_ "github.com/containerd/containerd/v2/core/runtime/v2/runc/options"
|
_ "github.com/containerd/containerd/v2/core/runtime/v2/runc/options"
|
||||||
_ "github.com/containerd/containerd/v2/pkg/events/plugin"
|
_ "github.com/containerd/containerd/v2/pkg/events/plugin"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/cri/images"
|
_ "github.com/containerd/containerd/v2/plugins/cri/images"
|
||||||
|
_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
|
_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/gc"
|
_ "github.com/containerd/containerd/v2/plugins/gc"
|
||||||
_ "github.com/containerd/containerd/v2/plugins/leases"
|
_ "github.com/containerd/containerd/v2/plugins/leases"
|
||||||
|
@ -319,8 +319,6 @@ type PluginConfig struct {
|
|||||||
ContainerdConfig `toml:"containerd" json:"containerd"`
|
ContainerdConfig `toml:"containerd" json:"containerd"`
|
||||||
// CniConfig contains config related to cni
|
// CniConfig contains config related to cni
|
||||||
CniConfig `toml:"cni" json:"cni"`
|
CniConfig `toml:"cni" json:"cni"`
|
||||||
// DisableTCPService disables serving CRI on the TCP server.
|
|
||||||
DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"`
|
|
||||||
// StreamServerAddress is the ip address streaming server is listening on.
|
// StreamServerAddress is the ip address streaming server is listening on.
|
||||||
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
|
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
|
||||||
// StreamServerPort is the port streaming server is listening on.
|
// StreamServerPort is the port streaming server is listening on.
|
||||||
@ -433,6 +431,12 @@ type Config struct {
|
|||||||
StateDir string `json:"stateDir"`
|
StateDir string `json:"stateDir"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ServiceConfig contains all the configuration for the CRI API server.
|
||||||
|
type ServiceConfig struct {
|
||||||
|
// DisableTCPService disables serving CRI on the TCP server.
|
||||||
|
DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"`
|
||||||
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime
|
// RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime
|
||||||
RuntimeUntrusted = "untrusted"
|
RuntimeUntrusted = "untrusted"
|
||||||
|
@ -89,7 +89,6 @@ func DefaultConfig() PluginConfig {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
DisableTCPService: true,
|
|
||||||
StreamServerAddress: "127.0.0.1",
|
StreamServerAddress: "127.0.0.1",
|
||||||
StreamServerPort: "0",
|
StreamServerPort: "0",
|
||||||
StreamIdleTimeout: streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour
|
StreamIdleTimeout: streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour
|
||||||
|
@ -78,7 +78,6 @@ func DefaultConfig() PluginConfig {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
DisableTCPService: true,
|
|
||||||
StreamServerAddress: "127.0.0.1",
|
StreamServerAddress: "127.0.0.1",
|
||||||
StreamServerPort: "0",
|
StreamServerPort: "0",
|
||||||
StreamIdleTimeout: streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour
|
StreamIdleTimeout: streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour
|
||||||
|
@ -17,6 +17,7 @@
|
|||||||
package cri
|
package cri
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
@ -25,13 +26,13 @@ import (
|
|||||||
"github.com/containerd/plugin/registry"
|
"github.com/containerd/plugin/registry"
|
||||||
|
|
||||||
containerd "github.com/containerd/containerd/v2/client"
|
containerd "github.com/containerd/containerd/v2/client"
|
||||||
|
srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config"
|
||||||
"github.com/containerd/containerd/v2/core/sandbox"
|
"github.com/containerd/containerd/v2/core/sandbox"
|
||||||
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
|
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/constants"
|
"github.com/containerd/containerd/v2/pkg/cri/constants"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/instrument"
|
"github.com/containerd/containerd/v2/pkg/cri/instrument"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/nri"
|
"github.com/containerd/containerd/v2/pkg/cri/nri"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/server"
|
"github.com/containerd/containerd/v2/pkg/cri/server"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/server/base"
|
|
||||||
nriservice "github.com/containerd/containerd/v2/pkg/nri"
|
nriservice "github.com/containerd/containerd/v2/pkg/nri"
|
||||||
"github.com/containerd/containerd/v2/plugins"
|
"github.com/containerd/containerd/v2/plugins"
|
||||||
"github.com/containerd/platforms"
|
"github.com/containerd/platforms"
|
||||||
@ -43,13 +44,11 @@ import (
|
|||||||
|
|
||||||
// Register CRI service plugin
|
// Register CRI service plugin
|
||||||
func init() {
|
func init() {
|
||||||
|
|
||||||
registry.Register(&plugin.Registration{
|
registry.Register(&plugin.Registration{
|
||||||
Type: plugins.GRPCPlugin,
|
Type: plugins.GRPCPlugin,
|
||||||
ID: "cri",
|
ID: "cri",
|
||||||
Requires: []plugin.Type{
|
Requires: []plugin.Type{
|
||||||
plugins.CRIImagePlugin,
|
plugins.CRIServicePlugin,
|
||||||
plugins.InternalPlugin,
|
|
||||||
plugins.SandboxControllerPlugin,
|
plugins.SandboxControllerPlugin,
|
||||||
plugins.NRIApiPlugin,
|
plugins.NRIApiPlugin,
|
||||||
plugins.EventPlugin,
|
plugins.EventPlugin,
|
||||||
@ -58,23 +57,46 @@ func init() {
|
|||||||
plugins.SandboxStorePlugin,
|
plugins.SandboxStorePlugin,
|
||||||
plugins.TransferPlugin,
|
plugins.TransferPlugin,
|
||||||
},
|
},
|
||||||
|
Config: &criconfig.ServiceConfig{
|
||||||
|
DisableTCPService: true,
|
||||||
|
},
|
||||||
|
ConfigMigration: func(ctx context.Context, version int, pluginConfigs map[string]interface{}) error {
|
||||||
|
if version >= srvconfig.CurrentConfigVersion {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
const pluginName = string(plugins.GRPCPlugin) + ".cri"
|
||||||
|
original, ok := pluginConfigs[pluginName]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
src := original.(map[string]interface{})
|
||||||
|
|
||||||
|
// Currently only a single key migrated
|
||||||
|
if val, ok := src["disable_tcp_service"]; ok {
|
||||||
|
pluginConfigs[pluginName] = map[string]interface{}{
|
||||||
|
"disable_tcp_service": val,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
delete(pluginConfigs, pluginName)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
InitFn: initCRIService,
|
InitFn: initCRIService,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func initCRIService(ic *plugin.InitContext) (interface{}, error) {
|
func initCRIService(ic *plugin.InitContext) (interface{}, error) {
|
||||||
ctx := ic.Context
|
ctx := ic.Context
|
||||||
|
config := ic.Config.(*criconfig.ServiceConfig)
|
||||||
|
|
||||||
// Get base CRI dependencies.
|
// Get runtime service.
|
||||||
criBasePlugin, err := ic.GetByID(plugins.InternalPlugin, "cri")
|
criRuntimePlugin, err := ic.GetByID(plugins.CRIServicePlugin, "runtime")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err)
|
return nil, fmt.Errorf("unable to load CRI runtime service plugin dependency: %w", err)
|
||||||
}
|
}
|
||||||
criBase := criBasePlugin.(*base.CRIBase)
|
|
||||||
c := criBase.Config
|
|
||||||
|
|
||||||
// Get image service.
|
// Get image service.
|
||||||
criImagePlugin, err := ic.GetSingle(plugins.CRIImagePlugin)
|
criImagePlugin, err := ic.GetByID(plugins.CRIServicePlugin, "images")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err)
|
return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err)
|
||||||
}
|
}
|
||||||
@ -98,15 +120,16 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
options := &server.CRIServiceOptions{
|
options := &server.CRIServiceOptions{
|
||||||
|
RuntimeService: criRuntimePlugin.(server.RuntimeService),
|
||||||
ImageService: criImagePlugin.(server.ImageService),
|
ImageService: criImagePlugin.(server.ImageService),
|
||||||
NRI: getNRIAPI(ic),
|
NRI: getNRIAPI(ic),
|
||||||
Client: client,
|
Client: client,
|
||||||
SandboxControllers: sbControllers,
|
SandboxControllers: sbControllers,
|
||||||
BaseOCISpecs: criBase.BaseOCISpecs,
|
|
||||||
}
|
}
|
||||||
is := criImagePlugin.(imageService).GRPCService()
|
is := criImagePlugin.(imageService).GRPCService()
|
||||||
|
|
||||||
s, rs, err := server.NewCRIService(criBase.Config, options)
|
// TODO: More options specifically for grpc service?
|
||||||
|
s, rs, err := server.NewCRIService(options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create CRI service: %w", err)
|
return nil, fmt.Errorf("failed to create CRI service: %w", err)
|
||||||
}
|
}
|
||||||
@ -127,7 +150,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
|
|||||||
initializer: s,
|
initializer: s,
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.DisableTCPService {
|
if config.DisableTCPService {
|
||||||
return service, nil
|
return service, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -394,9 +394,9 @@ func (c *criService) runtimeSpec(id string, platform platforms.Platform, baseSpe
|
|||||||
container := &containers.Container{ID: id}
|
container := &containers.Container{ID: id}
|
||||||
|
|
||||||
if baseSpecFile != "" {
|
if baseSpecFile != "" {
|
||||||
baseSpec, ok := c.baseOCISpecs[baseSpecFile]
|
baseSpec, err := c.LoadOCISpec(baseSpecFile)
|
||||||
if !ok {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("can't find base OCI spec %q", baseSpecFile)
|
return nil, fmt.Errorf("can't load base OCI spec %q: %w", baseSpecFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
spec := oci.Spec{}
|
spec := oci.Spec{}
|
||||||
|
@ -1680,23 +1680,24 @@ func TestPrivilegedDevices(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBaseOCISpec(t *testing.T) {
|
func TestBaseOCISpec(t *testing.T) {
|
||||||
c := newTestCRIService()
|
|
||||||
baseLimit := int64(100)
|
baseLimit := int64(100)
|
||||||
c.baseOCISpecs = map[string]*oci.Spec{
|
c := newTestCRIService(withRuntimeService(&fakeRuntimeService{
|
||||||
"/etc/containerd/cri-base.json": {
|
ocispecs: map[string]*oci.Spec{
|
||||||
Process: &runtimespec.Process{
|
"/etc/containerd/cri-base.json": {
|
||||||
User: runtimespec.User{AdditionalGids: []uint32{9999}},
|
Process: &runtimespec.Process{
|
||||||
Capabilities: &runtimespec.LinuxCapabilities{
|
User: runtimespec.User{AdditionalGids: []uint32{9999}},
|
||||||
Permitted: []string{"CAP_SETUID"},
|
Capabilities: &runtimespec.LinuxCapabilities{
|
||||||
|
Permitted: []string{"CAP_SETUID"},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
Linux: &runtimespec.Linux{
|
||||||
Linux: &runtimespec.Linux{
|
Resources: &runtimespec.LinuxResources{
|
||||||
Resources: &runtimespec.LinuxResources{
|
Memory: &runtimespec.LinuxMemory{Limit: &baseLimit}, // Will be overwritten by `getCreateContainerTestData`
|
||||||
Memory: &runtimespec.LinuxMemory{Limit: &baseLimit}, // Will be overwritten by `getCreateContainerTestData`
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}))
|
||||||
|
|
||||||
ociRuntime := config.Runtime{}
|
ociRuntime := config.Runtime{}
|
||||||
ociRuntime.BaseRuntimeSpec = "/etc/containerd/cri-base.json"
|
ociRuntime.BaseRuntimeSpec = "/etc/containerd/cri-base.json"
|
||||||
|
@ -524,13 +524,14 @@ func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBaseRuntimeSpec(t *testing.T) {
|
func TestBaseRuntimeSpec(t *testing.T) {
|
||||||
c := newTestCRIService()
|
c := newTestCRIService(withRuntimeService(&fakeRuntimeService{
|
||||||
c.baseOCISpecs = map[string]*oci.Spec{
|
ocispecs: map[string]*oci.Spec{
|
||||||
"/etc/containerd/cri-base.json": {
|
"/etc/containerd/cri-base.json": {
|
||||||
Version: "1.0.2",
|
Version: "1.0.2",
|
||||||
Hostname: "old",
|
Hostname: "old",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}))
|
||||||
|
|
||||||
out, err := c.runtimeSpec(
|
out, err := c.runtimeSpec(
|
||||||
"id1",
|
"id1",
|
||||||
@ -546,8 +547,10 @@ func TestBaseRuntimeSpec(t *testing.T) {
|
|||||||
assert.Equal(t, "new-domain", out.Domainname)
|
assert.Equal(t, "new-domain", out.Domainname)
|
||||||
|
|
||||||
// Make sure original base spec not changed
|
// Make sure original base spec not changed
|
||||||
assert.NotEqual(t, out, c.baseOCISpecs["/etc/containerd/cri-base.json"])
|
spec, err := c.LoadOCISpec("/etc/containerd/cri-base.json")
|
||||||
assert.Equal(t, c.baseOCISpecs["/etc/containerd/cri-base.json"].Hostname, "old")
|
assert.NoError(t, err)
|
||||||
|
assert.NotEqual(t, out, spec)
|
||||||
|
assert.Equal(t, spec.Hostname, "old")
|
||||||
|
|
||||||
assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath)
|
assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath)
|
||||||
}
|
}
|
||||||
|
@ -31,7 +31,6 @@ import (
|
|||||||
"github.com/containerd/containerd/v2/core/sandbox"
|
"github.com/containerd/containerd/v2/core/sandbox"
|
||||||
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
|
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/constants"
|
"github.com/containerd/containerd/v2/pkg/cri/constants"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/server/base"
|
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/server/podsandbox/types"
|
"github.com/containerd/containerd/v2/pkg/cri/server/podsandbox/types"
|
||||||
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
|
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
|
||||||
ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util"
|
ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util"
|
||||||
@ -51,8 +50,7 @@ func init() {
|
|||||||
plugins.EventPlugin,
|
plugins.EventPlugin,
|
||||||
plugins.LeasePlugin,
|
plugins.LeasePlugin,
|
||||||
plugins.SandboxStorePlugin,
|
plugins.SandboxStorePlugin,
|
||||||
plugins.InternalPlugin,
|
plugins.CRIServicePlugin,
|
||||||
plugins.CRIImagePlugin,
|
|
||||||
plugins.ServicePlugin,
|
plugins.ServicePlugin,
|
||||||
},
|
},
|
||||||
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
|
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
|
||||||
@ -66,26 +64,26 @@ func init() {
|
|||||||
return nil, fmt.Errorf("unable to init client for podsandbox: %w", err)
|
return nil, fmt.Errorf("unable to init client for podsandbox: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get base CRI dependencies.
|
// Get runtime service.
|
||||||
criBasePlugin, err := ic.GetByID(plugins.InternalPlugin, "cri")
|
criRuntimePlugin, err := ic.GetByID(plugins.CRIServicePlugin, "runtime")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err)
|
return nil, fmt.Errorf("unable to load CRI runtime service plugin dependency: %w", err)
|
||||||
}
|
}
|
||||||
criBase := criBasePlugin.(*base.CRIBase)
|
runtimeService := criRuntimePlugin.(RuntimeService)
|
||||||
|
|
||||||
// Get image service.
|
// Get image service.
|
||||||
criImagePlugin, err := ic.GetSingle(plugins.CRIImagePlugin)
|
criImagePlugin, err := ic.GetByID(plugins.CRIServicePlugin, "images")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err)
|
return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
c := Controller{
|
c := Controller{
|
||||||
client: client,
|
client: client,
|
||||||
config: criBase.Config,
|
config: runtimeService.Config(),
|
||||||
os: osinterface.RealOS{},
|
os: osinterface.RealOS{},
|
||||||
baseOCISpecs: criBase.BaseOCISpecs,
|
runtimeService: runtimeService,
|
||||||
imageService: criImagePlugin.(ImageService),
|
imageService: criImagePlugin.(ImageService),
|
||||||
store: NewStore(),
|
store: NewStore(),
|
||||||
}
|
}
|
||||||
return &c, nil
|
return &c, nil
|
||||||
},
|
},
|
||||||
@ -99,6 +97,12 @@ type CRIService interface {
|
|||||||
BackOffEvent(id string, event interface{})
|
BackOffEvent(id string, event interface{})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RuntimeService specifies dependencies to CRI runtime service.
|
||||||
|
type RuntimeService interface {
|
||||||
|
Config() criconfig.Config
|
||||||
|
LoadOCISpec(string) (*oci.Spec, error)
|
||||||
|
}
|
||||||
|
|
||||||
// ImageService specifies dependencies to CRI image service.
|
// ImageService specifies dependencies to CRI image service.
|
||||||
type ImageService interface {
|
type ImageService interface {
|
||||||
LocalResolve(refOrID string) (imagestore.Image, error)
|
LocalResolve(refOrID string) (imagestore.Image, error)
|
||||||
@ -113,14 +117,14 @@ type Controller struct {
|
|||||||
config criconfig.Config
|
config criconfig.Config
|
||||||
// client is an instance of the containerd client
|
// client is an instance of the containerd client
|
||||||
client *containerd.Client
|
client *containerd.Client
|
||||||
|
// runtimeService is a dependency to CRI runtime service.
|
||||||
|
runtimeService RuntimeService
|
||||||
// imageService is a dependency to CRI image service.
|
// imageService is a dependency to CRI image service.
|
||||||
imageService ImageService
|
imageService ImageService
|
||||||
// os is an interface for all required os operations.
|
// os is an interface for all required os operations.
|
||||||
os osinterface.OS
|
os osinterface.OS
|
||||||
// cri is CRI service that provides missing gaps needed by controller.
|
// cri is CRI service that provides missing gaps needed by controller.
|
||||||
cri CRIService
|
cri CRIService
|
||||||
// baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
|
|
||||||
baseOCISpecs map[string]*oci.Spec
|
|
||||||
|
|
||||||
store *Store
|
store *Store
|
||||||
}
|
}
|
||||||
|
@ -159,9 +159,9 @@ func (c *Controller) runtimeSpec(id string, baseSpecFile string, opts ...oci.Spe
|
|||||||
container := &containers.Container{ID: id}
|
container := &containers.Container{ID: id}
|
||||||
|
|
||||||
if baseSpecFile != "" {
|
if baseSpecFile != "" {
|
||||||
baseSpec, ok := c.baseOCISpecs[baseSpecFile]
|
baseSpec, err := c.runtimeService.LoadOCISpec(baseSpecFile)
|
||||||
if !ok {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("can't find base OCI spec %q", baseSpecFile)
|
return nil, fmt.Errorf("can't load base OCI spec %q: %w", baseSpecFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
spec := oci.Spec{}
|
spec := oci.Spec{}
|
||||||
|
@ -65,6 +65,15 @@ type sandboxService interface {
|
|||||||
SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error)
|
SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RuntimeService specifies dependencies to runtime service which provides
|
||||||
|
// the runtime configuration and OCI spec loading.
|
||||||
|
type RuntimeService interface {
|
||||||
|
Config() criconfig.Config
|
||||||
|
|
||||||
|
// LoadCISpec loads cached OCI specs via `Runtime.BaseRuntimeSpec`
|
||||||
|
LoadOCISpec(string) (*oci.Spec, error)
|
||||||
|
}
|
||||||
|
|
||||||
// ImageService specifies dependencies to image service.
|
// ImageService specifies dependencies to image service.
|
||||||
type ImageService interface {
|
type ImageService interface {
|
||||||
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string
|
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string
|
||||||
@ -84,6 +93,7 @@ type ImageService interface {
|
|||||||
|
|
||||||
// criService implements CRIService.
|
// criService implements CRIService.
|
||||||
type criService struct {
|
type criService struct {
|
||||||
|
RuntimeService
|
||||||
ImageService
|
ImageService
|
||||||
// config contains all configurations.
|
// config contains all configurations.
|
||||||
config criconfig.Config
|
config criconfig.Config
|
||||||
@ -115,8 +125,6 @@ type criService struct {
|
|||||||
// cniNetConfMonitor is used to reload cni network conf if there is
|
// cniNetConfMonitor is used to reload cni network conf if there is
|
||||||
// any valid fs change events from cni network conf dir.
|
// any valid fs change events from cni network conf dir.
|
||||||
cniNetConfMonitor map[string]*cniNetConfSyncer
|
cniNetConfMonitor map[string]*cniNetConfSyncer
|
||||||
// baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
|
|
||||||
baseOCISpecs map[string]*oci.Spec
|
|
||||||
// allCaps is the list of the capabilities.
|
// allCaps is the list of the capabilities.
|
||||||
// When nil, parsed from CapEff of /proc/self/status.
|
// When nil, parsed from CapEff of /proc/self/status.
|
||||||
allCaps []string //nolint:nolintlint,unused // Ignore on non-Linux
|
allCaps []string //nolint:nolintlint,unused // Ignore on non-Linux
|
||||||
@ -130,6 +138,8 @@ type criService struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type CRIServiceOptions struct {
|
type CRIServiceOptions struct {
|
||||||
|
RuntimeService RuntimeService
|
||||||
|
|
||||||
ImageService ImageService
|
ImageService ImageService
|
||||||
|
|
||||||
NRI *nri.API
|
NRI *nri.API
|
||||||
@ -137,9 +147,6 @@ type CRIServiceOptions struct {
|
|||||||
// SandboxControllers is a map of all the loaded sandbox controllers
|
// SandboxControllers is a map of all the loaded sandbox controllers
|
||||||
SandboxControllers map[string]sandbox.Controller
|
SandboxControllers map[string]sandbox.Controller
|
||||||
|
|
||||||
// BaseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
|
|
||||||
BaseOCISpecs map[string]*oci.Spec
|
|
||||||
|
|
||||||
// Client is the base containerd client used for accessing services,
|
// Client is the base containerd client used for accessing services,
|
||||||
//
|
//
|
||||||
// TODO: Replace this gradually with directly configured instances
|
// TODO: Replace this gradually with directly configured instances
|
||||||
@ -147,18 +154,18 @@ type CRIServiceOptions struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewCRIService returns a new instance of CRIService
|
// NewCRIService returns a new instance of CRIService
|
||||||
// TODO: Add criBase.BaseOCISpecs to options
|
func NewCRIService(options *CRIServiceOptions) (CRIService, runtime.RuntimeServiceServer, error) {
|
||||||
func NewCRIService(config criconfig.Config, options *CRIServiceOptions) (CRIService, runtime.RuntimeServiceServer, error) {
|
|
||||||
var err error
|
var err error
|
||||||
labels := label.NewStore()
|
labels := label.NewStore()
|
||||||
|
config := options.RuntimeService.Config()
|
||||||
|
|
||||||
c := &criService{
|
c := &criService{
|
||||||
|
RuntimeService: options.RuntimeService,
|
||||||
ImageService: options.ImageService,
|
ImageService: options.ImageService,
|
||||||
config: config,
|
config: config,
|
||||||
client: options.Client,
|
client: options.Client,
|
||||||
imageFSPaths: options.ImageService.ImageFSPaths(),
|
imageFSPaths: options.ImageService.ImageFSPaths(),
|
||||||
os: osinterface.RealOS{},
|
os: osinterface.RealOS{},
|
||||||
baseOCISpecs: options.BaseOCISpecs,
|
|
||||||
sandboxStore: sandboxstore.NewStore(labels),
|
sandboxStore: sandboxstore.NewStore(labels),
|
||||||
containerStore: containerstore.NewStore(labels),
|
containerStore: containerstore.NewStore(labels),
|
||||||
sandboxNameIndex: registrar.NewRegistrar(),
|
sandboxNameIndex: registrar.NewRegistrar(),
|
||||||
|
@ -25,10 +25,12 @@ import (
|
|||||||
"github.com/containerd/containerd/v2/api/types"
|
"github.com/containerd/containerd/v2/api/types"
|
||||||
"github.com/containerd/containerd/v2/core/sandbox"
|
"github.com/containerd/containerd/v2/core/sandbox"
|
||||||
"github.com/containerd/containerd/v2/internal/registrar"
|
"github.com/containerd/containerd/v2/internal/registrar"
|
||||||
|
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
|
||||||
containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container"
|
containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container"
|
||||||
"github.com/containerd/containerd/v2/pkg/cri/store/label"
|
"github.com/containerd/containerd/v2/pkg/cri/store/label"
|
||||||
sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox"
|
sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox"
|
||||||
servertesting "github.com/containerd/containerd/v2/pkg/cri/testing"
|
servertesting "github.com/containerd/containerd/v2/pkg/cri/testing"
|
||||||
|
"github.com/containerd/containerd/v2/pkg/oci"
|
||||||
ostesting "github.com/containerd/containerd/v2/pkg/os/testing"
|
ostesting "github.com/containerd/containerd/v2/pkg/os/testing"
|
||||||
"github.com/containerd/errdefs"
|
"github.com/containerd/errdefs"
|
||||||
"github.com/containerd/platforms"
|
"github.com/containerd/platforms"
|
||||||
@ -74,11 +76,34 @@ func (f fakeSandboxController) Metrics(ctx context.Context, sandboxID string) (*
|
|||||||
return &types.Metric{}, errdefs.ErrNotImplemented
|
return &types.Metric{}, errdefs.ErrNotImplemented
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeRuntimeService struct {
|
||||||
|
ocispecs map[string]*oci.Spec
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeRuntimeService) Config() criconfig.Config {
|
||||||
|
return testConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeRuntimeService) LoadOCISpec(filename string) (*oci.Spec, error) {
|
||||||
|
spec, ok := f.ocispecs[filename]
|
||||||
|
if !ok {
|
||||||
|
return nil, errdefs.ErrNotFound
|
||||||
|
}
|
||||||
|
return spec, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type testOpt func(*criService)
|
||||||
|
|
||||||
|
func withRuntimeService(rs RuntimeService) testOpt {
|
||||||
|
return func(service *criService) {
|
||||||
|
service.RuntimeService = rs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// newTestCRIService creates a fake criService for test.
|
// newTestCRIService creates a fake criService for test.
|
||||||
func newTestCRIService() *criService {
|
func newTestCRIService(opts ...testOpt) *criService {
|
||||||
labels := label.NewStore()
|
labels := label.NewStore()
|
||||||
return &criService{
|
service := &criService{
|
||||||
ImageService: &fakeImageService{},
|
|
||||||
config: testConfig,
|
config: testConfig,
|
||||||
os: ostesting.NewFakeOS(),
|
os: ostesting.NewFakeOS(),
|
||||||
sandboxStore: sandboxstore.NewStore(labels),
|
sandboxStore: sandboxstore.NewStore(labels),
|
||||||
@ -90,4 +115,15 @@ func newTestCRIService() *criService {
|
|||||||
},
|
},
|
||||||
sandboxService: &fakeSandboxService{},
|
sandboxService: &fakeSandboxService{},
|
||||||
}
|
}
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(service)
|
||||||
|
}
|
||||||
|
if service.RuntimeService == nil {
|
||||||
|
service.RuntimeService = &fakeRuntimeService{}
|
||||||
|
}
|
||||||
|
if service.ImageService == nil {
|
||||||
|
service.ImageService = &fakeImageService{}
|
||||||
|
}
|
||||||
|
|
||||||
|
return service
|
||||||
}
|
}
|
||||||
|
@ -40,15 +40,14 @@ func init() {
|
|||||||
config := criconfig.DefaultImageConfig()
|
config := criconfig.DefaultImageConfig()
|
||||||
|
|
||||||
registry.Register(&plugin.Registration{
|
registry.Register(&plugin.Registration{
|
||||||
Type: plugins.CRIImagePlugin,
|
Type: plugins.CRIServicePlugin,
|
||||||
ID: "local",
|
ID: "images",
|
||||||
Config: &config,
|
Config: &config,
|
||||||
Requires: []plugin.Type{
|
Requires: []plugin.Type{
|
||||||
plugins.LeasePlugin,
|
plugins.LeasePlugin,
|
||||||
plugins.EventPlugin,
|
plugins.EventPlugin,
|
||||||
plugins.MetadataPlugin,
|
plugins.MetadataPlugin,
|
||||||
plugins.SandboxStorePlugin,
|
plugins.SandboxStorePlugin,
|
||||||
plugins.InternalPlugin, // For config migration ordering
|
|
||||||
plugins.ServicePlugin, // For client
|
plugins.ServicePlugin, // For client
|
||||||
plugins.SnapshotPlugin, // For root directory properties
|
plugins.SnapshotPlugin, // For root directory properties
|
||||||
},
|
},
|
||||||
@ -152,12 +151,12 @@ func configMigration(ctx context.Context, version int, pluginConfigs map[string]
|
|||||||
if version >= srvconfig.CurrentConfigVersion {
|
if version >= srvconfig.CurrentConfigVersion {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
original, ok := pluginConfigs[string(plugins.InternalPlugin)+".cri"]
|
original, ok := pluginConfigs[string(plugins.GRPCPlugin)+".cri"]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
src := original.(map[string]interface{})
|
src := original.(map[string]interface{})
|
||||||
updated, ok := pluginConfigs[string(plugins.CRIImagePlugin)+".local"]
|
updated, ok := pluginConfigs[string(plugins.CRIServicePlugin)+".images"]
|
||||||
var dst map[string]interface{}
|
var dst map[string]interface{}
|
||||||
if ok {
|
if ok {
|
||||||
dst = updated.(map[string]interface{})
|
dst = updated.(map[string]interface{})
|
||||||
@ -166,7 +165,7 @@ func configMigration(ctx context.Context, version int, pluginConfigs map[string]
|
|||||||
}
|
}
|
||||||
|
|
||||||
migrateConfig(dst, src)
|
migrateConfig(dst, src)
|
||||||
pluginConfigs[string(plugins.CRIImagePlugin)+".local"] = dst
|
pluginConfigs[string(plugins.CRIServicePlugin)+".images"] = dst
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func migrateConfig(dst, src map[string]interface{}) {
|
func migrateConfig(dst, src map[string]interface{}) {
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package base
|
package runtime
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
@ -14,7 +14,7 @@
|
|||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package base
|
package runtime
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@ -36,47 +36,39 @@ import (
|
|||||||
"github.com/containerd/containerd/v2/pkg/oci"
|
"github.com/containerd/containerd/v2/pkg/oci"
|
||||||
"github.com/containerd/containerd/v2/plugins"
|
"github.com/containerd/containerd/v2/plugins"
|
||||||
"github.com/containerd/containerd/v2/plugins/services/warning"
|
"github.com/containerd/containerd/v2/plugins/services/warning"
|
||||||
|
"github.com/containerd/errdefs"
|
||||||
"github.com/containerd/platforms"
|
"github.com/containerd/platforms"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CRIBase contains common dependencies for CRI's runtime, image, and podsandbox services.
|
|
||||||
type CRIBase struct {
|
|
||||||
// Config contains all configurations.
|
|
||||||
Config criconfig.Config
|
|
||||||
// BaseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
|
|
||||||
BaseOCISpecs map[string]*oci.Spec
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
config := criconfig.DefaultConfig()
|
config := criconfig.DefaultConfig()
|
||||||
|
|
||||||
// Base plugin that other CRI services depend on.
|
// Base plugin that other CRI services depend on.
|
||||||
registry.Register(&plugin.Registration{
|
registry.Register(&plugin.Registration{
|
||||||
Type: plugins.InternalPlugin,
|
Type: plugins.CRIServicePlugin,
|
||||||
ID: "cri",
|
ID: "runtime",
|
||||||
Config: &config,
|
Config: &config,
|
||||||
Requires: []plugin.Type{
|
Requires: []plugin.Type{
|
||||||
plugins.WarningPlugin,
|
plugins.WarningPlugin,
|
||||||
},
|
},
|
||||||
ConfigMigration: func(ctx context.Context, version int, plugins map[string]interface{}) error {
|
ConfigMigration: func(ctx context.Context, version int, pluginConfigs map[string]interface{}) error {
|
||||||
if version >= srvconfig.CurrentConfigVersion {
|
if version >= srvconfig.CurrentConfigVersion {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
c, ok := plugins["io.containerd.grpc.v1.cri"]
|
c, ok := pluginConfigs[string(plugins.GRPCPlugin)+".cri"]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
conf := c.(map[string]interface{})
|
conf := c.(map[string]interface{})
|
||||||
migrateConfig(conf)
|
migrateConfig(conf)
|
||||||
plugins["io.containerd.internal.v1.cri"] = conf
|
pluginConfigs[string(plugins.CRIServicePlugin)+".runtime"] = conf
|
||||||
delete(plugins, "io.containerd.grpc.v1.cri")
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
InitFn: initCRIBase,
|
InitFn: initCRIRuntime,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func initCRIBase(ic *plugin.InitContext) (interface{}, error) {
|
func initCRIRuntime(ic *plugin.InitContext) (interface{}, error) {
|
||||||
ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()}
|
ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()}
|
||||||
ic.Meta.Exports = map[string]string{"CRIVersion": constants.CRIVersion}
|
ic.Meta.Exports = map[string]string{"CRIVersion": constants.CRIVersion}
|
||||||
ctx := ic.Context
|
ctx := ic.Context
|
||||||
@ -118,12 +110,33 @@ func initCRIBase(ic *plugin.InitContext) (interface{}, error) {
|
|||||||
return nil, fmt.Errorf("failed to create load basic oci spec: %w", err)
|
return nil, fmt.Errorf("failed to create load basic oci spec: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &CRIBase{
|
return &runtime{
|
||||||
Config: c,
|
config: c,
|
||||||
BaseOCISpecs: ociSpec,
|
baseOCISpecs: ociSpec,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runtime contains common dependencies for CRI's runtime, image, and podsandbox services.
|
||||||
|
type runtime struct {
|
||||||
|
// Config contains all configurations.
|
||||||
|
config criconfig.Config
|
||||||
|
// BaseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
|
||||||
|
baseOCISpecs map[string]*oci.Spec
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *runtime) Config() criconfig.Config {
|
||||||
|
return r.config
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *runtime) LoadOCISpec(filename string) (*oci.Spec, error) {
|
||||||
|
spec, ok := r.baseOCISpecs[filename]
|
||||||
|
if !ok {
|
||||||
|
// TODO: Load here or only allow preloading...
|
||||||
|
return nil, errdefs.ErrNotFound
|
||||||
|
}
|
||||||
|
return spec, nil
|
||||||
|
}
|
||||||
|
|
||||||
func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) {
|
func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) {
|
||||||
specs := map[string]*oci.Spec{}
|
specs := map[string]*oci.Spec{}
|
||||||
for _, cfg := range config.Runtimes {
|
for _, cfg := range config.Runtimes {
|
@ -67,8 +67,8 @@ const (
|
|||||||
ImageVerifierPlugin plugin.Type = "io.containerd.image-verifier.v1"
|
ImageVerifierPlugin plugin.Type = "io.containerd.image-verifier.v1"
|
||||||
// WarningPlugin implements a warning service
|
// WarningPlugin implements a warning service
|
||||||
WarningPlugin plugin.Type = "io.containerd.warning.v1"
|
WarningPlugin plugin.Type = "io.containerd.warning.v1"
|
||||||
// CRIImagePlugin implements a cri image service
|
// CRIServicePlugin implements a cri service
|
||||||
CRIImagePlugin plugin.Type = "io.containerd.cri.image.v1"
|
CRIServicePlugin plugin.Type = "io.containerd.cri.v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
Loading…
Reference in New Issue
Block a user