Move cri server packages under pkg/cri

Organizes the cri related server packages under pkg/cri

Signed-off-by: Derek McGowan <derek@mcg.dev>

pkg/cri/store/sandbox/metadata.go

@@ -0,0 +1,89 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sandbox
+
+import (
+	"encoding/json"
+
+	cni "github.com/containerd/go-cni"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// NOTE(random-liu):
+// 1) Metadata is immutable after created.
+// 2) Metadata is checkpointed as containerd container label.
+
+// metadataVersion is current version of sandbox metadata.
+const metadataVersion = "v1" // nolint
+
+// versionedMetadata is the internal versioned sandbox metadata.
+// nolint
+type versionedMetadata struct {
+	// Version indicates the version of the versioned sandbox metadata.
+	Version string
+	// Metadata's type is metadataInternal. If not there will be a recursive call in MarshalJSON.
+	Metadata metadataInternal
+}
+
+// metadataInternal is for internal use.
+type metadataInternal Metadata
+
+// Metadata is the unversioned sandbox metadata.
+type Metadata struct {
+	// ID is the sandbox id.
+	ID string
+	// Name is the sandbox name.
+	Name string
+	// Config is the CRI sandbox config.
+	Config *runtime.PodSandboxConfig
+	// NetNSPath is the network namespace used by the sandbox.
+	NetNSPath string
+	// IP of Pod if it is attached to non host network
+	IP string
+	// AdditionalIPs of the Pod if it is attached to non host network
+	AdditionalIPs []string
+	// RuntimeHandler is the runtime handler name of the pod.
+	RuntimeHandler string
+	// CNIresult resulting configuration for attached network namespace interfaces
+	CNIResult *cni.CNIResult
+	// ProcessLabel is the SELinux process label for the container
+	ProcessLabel string
+}
+
+// MarshalJSON encodes Metadata into bytes in json format.
+func (c *Metadata) MarshalJSON() ([]byte, error) {
+	return json.Marshal(&versionedMetadata{
+		Version:  metadataVersion,
+		Metadata: metadataInternal(*c),
+	})
+}
+
+// UnmarshalJSON decodes Metadata from bytes.
+func (c *Metadata) UnmarshalJSON(data []byte) error {
+	versioned := &versionedMetadata{}
+	if err := json.Unmarshal(data, versioned); err != nil {
+		return err
+	}
+	// Handle old version after upgrade.
+	switch versioned.Version {
+	case metadataVersion:
+		*c = Metadata(versioned.Metadata)
+		return nil
+	}
+	return errors.Errorf("unsupported version: %q", versioned.Version)
+}

pkg/cri/store/sandbox/metadata_test.go

@@ -0,0 +1,79 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sandbox
+
+import (
+	"encoding/json"
+	"testing"
+
+	assertlib "github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+func TestMetadataMarshalUnmarshal(t *testing.T) {
+	meta := &Metadata{
+		ID:   "test-id",
+		Name: "test-name",
+		Config: &runtime.PodSandboxConfig{
+			Metadata: &runtime.PodSandboxMetadata{
+				Name:      "test-name",
+				Uid:       "test-uid",
+				Namespace: "test-namespace",
+				Attempt:   1,
+			},
+		},
+	}
+	assert := assertlib.New(t)
+	newMeta := &Metadata{}
+	newVerMeta := &versionedMetadata{}
+
+	t.Logf("should be able to do json.marshal")
+	data, err := json.Marshal(meta)
+	assert.NoError(err)
+	data1, err := json.Marshal(&versionedMetadata{
+		Version:  metadataVersion,
+		Metadata: metadataInternal(*meta),
+	})
+	assert.NoError(err)
+	assert.Equal(data, data1)
+
+	t.Logf("should be able to do MarshalJSON")
+	data, err = meta.MarshalJSON()
+	assert.NoError(err)
+	assert.NoError(newMeta.UnmarshalJSON(data))
+	assert.Equal(meta, newMeta)
+
+	t.Logf("should be able to do MarshalJSON and json.Unmarshal")
+	data, err = meta.MarshalJSON()
+	assert.NoError(err)
+	assert.NoError(json.Unmarshal(data, newVerMeta))
+	assert.Equal(meta, (*Metadata)(&newVerMeta.Metadata))
+
+	t.Logf("should be able to do json.Marshal and UnmarshalJSON")
+	data, err = json.Marshal(meta)
+	assert.NoError(err)
+	assert.NoError(newMeta.UnmarshalJSON(data))
+	assert.Equal(meta, newMeta)
+
+	t.Logf("should json.Unmarshal fail for unsupported version")
+	unsupported, err := json.Marshal(&versionedMetadata{
+		Version:  "random-test-version",
+		Metadata: metadataInternal(*meta),
+	})
+	assert.NoError(err)
+	assert.Error(json.Unmarshal(unsupported, &newMeta))
+}

pkg/cri/store/sandbox/sandbox.go

@@ -0,0 +1,137 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sandbox
+
+import (
+	"sync"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/pkg/cri/store/label"
+	"github.com/docker/docker/pkg/truncindex"
+
+	"github.com/containerd/containerd/pkg/cri/store"
+	"github.com/containerd/containerd/pkg/netns"
+)
+
+// Sandbox contains all resources associated with the sandbox. All methods to
+// mutate the internal state are thread safe.
+type Sandbox struct {
+	// Metadata is the metadata of the sandbox, it is immutable after created.
+	Metadata
+	// Status stores the status of the sandbox.
+	Status StatusStorage
+	// Container is the containerd sandbox container client.
+	Container containerd.Container
+	// CNI network namespace client.
+	// For hostnetwork pod, this is always nil;
+	// For non hostnetwork pod, this should never be nil.
+	NetNS *netns.NetNS
+	// StopCh is used to propagate the stop information of the sandbox.
+	*store.StopCh
+}
+
+// NewSandbox creates an internally used sandbox type. This functions reminds
+// the caller that a sandbox must have a status.
+func NewSandbox(metadata Metadata, status Status) Sandbox {
+	s := Sandbox{
+		Metadata: metadata,
+		Status:   StoreStatus(status),
+		StopCh:   store.NewStopCh(),
+	}
+	if status.State == StateNotReady {
+		s.Stop()
+	}
+	return s
+}
+
+// Store stores all sandboxes.
+type Store struct {
+	lock      sync.RWMutex
+	sandboxes map[string]Sandbox
+	idIndex   *truncindex.TruncIndex
+	labels    *label.Store
+}
+
+// NewStore creates a sandbox store.
+func NewStore(labels *label.Store) *Store {
+	return &Store{
+		sandboxes: make(map[string]Sandbox),
+		idIndex:   truncindex.NewTruncIndex([]string{}),
+		labels:    labels,
+	}
+}
+
+// Add a sandbox into the store.
+func (s *Store) Add(sb Sandbox) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if _, ok := s.sandboxes[sb.ID]; ok {
+		return store.ErrAlreadyExist
+	}
+	if err := s.labels.Reserve(sb.ProcessLabel); err != nil {
+		return err
+	}
+	if err := s.idIndex.Add(sb.ID); err != nil {
+		return err
+	}
+	s.sandboxes[sb.ID] = sb
+	return nil
+}
+
+// Get returns the sandbox with specified id.
+// Returns store.ErrNotExist if the sandbox doesn't exist.
+func (s *Store) Get(id string) (Sandbox, error) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	id, err := s.idIndex.Get(id)
+	if err != nil {
+		if err == truncindex.ErrNotExist {
+			err = store.ErrNotExist
+		}
+		return Sandbox{}, err
+	}
+	if sb, ok := s.sandboxes[id]; ok {
+		return sb, nil
+	}
+	return Sandbox{}, store.ErrNotExist
+}
+
+// List lists all sandboxes.
+func (s *Store) List() []Sandbox {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	var sandboxes []Sandbox
+	for _, sb := range s.sandboxes {
+		sandboxes = append(sandboxes, sb)
+	}
+	return sandboxes
+}
+
+// Delete deletes the sandbox with specified id.
+func (s *Store) Delete(id string) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	id, err := s.idIndex.Get(id)
+	if err != nil {
+		// Note: The idIndex.Delete and delete doesn't handle truncated index.
+		// So we need to return if there are error.
+		return
+	}
+	s.labels.Release(s.sandboxes[id].ProcessLabel)
+	s.idIndex.Delete(id) // nolint: errcheck
+	delete(s.sandboxes, id)
+}

pkg/cri/store/sandbox/sandbox_test.go

@@ -0,0 +1,156 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sandbox
+
+import (
+	"testing"
+
+	"github.com/containerd/containerd/pkg/cri/store/label"
+	assertlib "github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/cri/store"
+)
+
+func TestSandboxStore(t *testing.T) {
+	sandboxes := map[string]Sandbox{
+		"1": NewSandbox(
+			Metadata{
+				ID:   "1",
+				Name: "Sandbox-1",
+				Config: &runtime.PodSandboxConfig{
+					Metadata: &runtime.PodSandboxMetadata{
+						Name:      "TestPod-1",
+						Uid:       "TestUid-1",
+						Namespace: "TestNamespace-1",
+						Attempt:   1,
+					},
+				},
+				NetNSPath: "TestNetNS-1",
+			},
+			Status{State: StateReady},
+		),
+		"2abcd": NewSandbox(
+			Metadata{
+				ID:   "2abcd",
+				Name: "Sandbox-2abcd",
+				Config: &runtime.PodSandboxConfig{
+					Metadata: &runtime.PodSandboxMetadata{
+						Name:      "TestPod-2abcd",
+						Uid:       "TestUid-2abcd",
+						Namespace: "TestNamespace-2abcd",
+						Attempt:   2,
+					},
+				},
+				NetNSPath: "TestNetNS-2",
+			},
+			Status{State: StateNotReady},
+		),
+		"4a333": NewSandbox(
+			Metadata{
+				ID:   "4a333",
+				Name: "Sandbox-4a333",
+				Config: &runtime.PodSandboxConfig{
+					Metadata: &runtime.PodSandboxMetadata{
+						Name:      "TestPod-4a333",
+						Uid:       "TestUid-4a333",
+						Namespace: "TestNamespace-4a333",
+						Attempt:   3,
+					},
+				},
+				NetNSPath: "TestNetNS-3",
+			},
+			Status{State: StateNotReady},
+		),
+		"4abcd": NewSandbox(
+			Metadata{
+				ID:   "4abcd",
+				Name: "Sandbox-4abcd",
+				Config: &runtime.PodSandboxConfig{
+					Metadata: &runtime.PodSandboxMetadata{
+						Name:      "TestPod-4abcd",
+						Uid:       "TestUid-4abcd",
+						Namespace: "TestNamespace-4abcd",
+						Attempt:   1,
+					},
+				},
+				NetNSPath: "TestNetNS-4abcd",
+			},
+			Status{State: StateReady},
+		),
+	}
+	unknown := NewSandbox(
+		Metadata{
+			ID:   "3defg",
+			Name: "Sandbox-3defg",
+			Config: &runtime.PodSandboxConfig{
+				Metadata: &runtime.PodSandboxMetadata{
+					Name:      "TestPod-3defg",
+					Uid:       "TestUid-3defg",
+					Namespace: "TestNamespace-3defg",
+					Attempt:   1,
+				},
+			},
+			NetNSPath: "TestNetNS-3defg",
+		},
+		Status{State: StateUnknown},
+	)
+	assert := assertlib.New(t)
+	s := NewStore(label.NewStore())
+
+	t.Logf("should be able to add sandbox")
+	for _, sb := range sandboxes {
+		assert.NoError(s.Add(sb))
+	}
+	assert.NoError(s.Add(unknown))
+
+	t.Logf("should be able to get sandbox")
+	genTruncIndex := func(normalName string) string { return normalName[:(len(normalName)+1)/2] }
+	for id, sb := range sandboxes {
+		got, err := s.Get(genTruncIndex(id))
+		assert.NoError(err)
+		assert.Equal(sb, got)
+	}
+
+	t.Logf("should be able to get sandbox in unknown state with Get")
+	got, err := s.Get(unknown.ID)
+	assert.NoError(err)
+	assert.Equal(unknown, got)
+
+	t.Logf("should be able to list sandboxes")
+	sbNum := len(sandboxes) + 1
+	sbs := s.List()
+	assert.Len(sbs, sbNum)
+
+	for testID, v := range sandboxes {
+		truncID := genTruncIndex(testID)
+
+		t.Logf("add should return already exists error for duplicated sandbox")
+		assert.Equal(store.ErrAlreadyExist, s.Add(v))
+
+		t.Logf("should be able to delete sandbox")
+		s.Delete(truncID)
+		sbNum--
+		sbs = s.List()
+		assert.Len(sbs, sbNum)
+
+		t.Logf("get should return not exist error after deletion")
+		sb, err := s.Get(truncID)
+		assert.Equal(Sandbox{}, sb)
+		assert.Equal(store.ErrNotExist, err)
+	}
+}

pkg/cri/store/sandbox/status.go

@@ -0,0 +1,151 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sandbox
+
+import (
+	"strconv"
+	"sync"
+	"time"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// The sandbox state machine in the CRI plugin:
+//                    +              +
+//                    |              |
+//                    | Create(Run)  | Load
+//                    |              |
+//      Start         |              |
+//     (failed)       |              |
+// +------------------+              +-----------+
+// |                  |              |           |
+// |                  |              |           |
+// |                  |              |           |
+// |                  | Start(Run)   |           |
+// |                  |              |           |
+// | PortForward +----v----+         |           |
+// |      +------+         |         |           |
+// |      |      |  READY  <---------+           |
+// |      +------>         |         |           |
+// |             +----+----+         |           |
+// |                  |              |           |
+// |                  | Stop/Exit    |           |
+// |                  |              |           |
+// |             +----v----+         |           |
+// |             |         <---------+      +----v----+
+// |             | NOTREADY|                |         |
+// |             |         <----------------+ UNKNOWN |
+// |             +----+----+       Stop     |         |
+// |                  |                     +---------+
+// |                  | Remove
+// |                  v
+// +-------------> DELETED
+
+// State is the sandbox state we use in containerd/cri.
+// It includes unknown, which is internal states not defined in CRI.
+// The state mapping from internal states to CRI states:
+// * ready -> ready
+// * not ready -> not ready
+// * unknown -> not ready
+type State uint32
+
+const (
+	// StateReady is ready state, it means sandbox container
+	// is running.
+	StateReady State = iota
+	// StateNotReady is notready state, it ONLY means sandbox
+	// container is not running.
+	// StopPodSandbox should still be called for NOTREADY sandbox to
+	// cleanup resources other than sandbox container, e.g. network namespace.
+	// This is an assumption made in CRI.
+	StateNotReady
+	// StateUnknown is unknown state. Sandbox only goes
+	// into unknown state when its status fails to be loaded.
+	StateUnknown
+)
+
+// String returns the string representation of the state
+func (s State) String() string {
+	switch s {
+	case StateReady:
+		return runtime.PodSandboxState_SANDBOX_READY.String()
+	case StateNotReady:
+		return runtime.PodSandboxState_SANDBOX_NOTREADY.String()
+	case StateUnknown:
+		// PodSandboxState doesn't have an unknown state, but State does, so return a string using the same convention
+		return "SANDBOX_UNKNOWN"
+	default:
+		return "invalid sandbox state value: " + strconv.Itoa(int(s))
+	}
+}
+
+// Status is the status of a sandbox.
+type Status struct {
+	// Pid is the init process id of the sandbox container.
+	Pid uint32
+	// CreatedAt is the created timestamp.
+	CreatedAt time.Time
+	// State is the state of the sandbox.
+	State State
+}
+
+// UpdateFunc is function used to update the sandbox status. If there
+// is an error, the update will be rolled back.
+type UpdateFunc func(Status) (Status, error)
+
+// StatusStorage manages the sandbox status.
+// The status storage for sandbox is different from container status storage,
+// because we don't checkpoint sandbox status. If we need checkpoint in the
+// future, we should combine this with container status storage.
+type StatusStorage interface {
+	// Get a sandbox status.
+	Get() Status
+	// Update the sandbox status. Note that the update MUST be applied
+	// in one transaction.
+	Update(UpdateFunc) error
+}
+
+// StoreStatus creates the storage containing the passed in sandbox status with the
+// specified id.
+// The status MUST be created in one transaction.
+func StoreStatus(status Status) StatusStorage {
+	return &statusStorage{status: status}
+}
+
+type statusStorage struct {
+	sync.RWMutex
+	status Status
+}
+
+// Get a copy of sandbox status.
+func (s *statusStorage) Get() Status {
+	s.RLock()
+	defer s.RUnlock()
+	return s.status
+}
+
+// Update the sandbox status.
+func (s *statusStorage) Update(u UpdateFunc) error {
+	s.Lock()
+	defer s.Unlock()
+	newStatus, err := u(s.status)
+	if err != nil {
+		return err
+	}
+	s.status = newStatus
+	return nil
+}

pkg/cri/store/sandbox/status_test.go

@@ -0,0 +1,69 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sandbox
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	assertlib "github.com/stretchr/testify/assert"
+)
+
+func TestStatus(t *testing.T) {
+	testStatus := Status{
+		Pid:       123,
+		CreatedAt: time.Now(),
+		State:     StateUnknown,
+	}
+	updateStatus := Status{
+		Pid:       456,
+		CreatedAt: time.Now(),
+		State:     StateReady,
+	}
+	updateErr := errors.New("update error")
+	assert := assertlib.New(t)
+
+	t.Logf("simple store and get")
+	s := StoreStatus(testStatus)
+	old := s.Get()
+	assert.Equal(testStatus, old)
+
+	t.Logf("failed update should not take effect")
+	err := s.Update(func(o Status) (Status, error) {
+		o = updateStatus
+		return o, updateErr
+	})
+	assert.Equal(updateErr, err)
+	assert.Equal(testStatus, s.Get())
+
+	t.Logf("successful update should take effect but not checkpoint")
+	err = s.Update(func(o Status) (Status, error) {
+		o = updateStatus
+		return o, nil
+	})
+	assert.NoError(err)
+	assert.Equal(updateStatus, s.Get())
+}
+
+func TestStateStringConversion(t *testing.T) {
+	assert := assertlib.New(t)
+	assert.Equal("SANDBOX_READY", StateReady.String())
+	assert.Equal("SANDBOX_NOTREADY", StateNotReady.String())
+	assert.Equal("SANDBOX_UNKNOWN", StateUnknown.String())
+	assert.Equal("invalid sandbox state value: 123", State(123).String())
+}