containerd/core/snapshots/snapshotter.go

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package snapshots

import (
	"context"
	"encoding/json"
	"strings"
	"time"

	snapshotsapi "github.com/containerd/containerd/v2/api/services/snapshots/v1"
	"github.com/containerd/containerd/v2/core/mount"
	"github.com/containerd/containerd/v2/protobuf"
)

const (
	// UnpackKeyPrefix is the beginning of the key format used for snapshots that will have
	// image content unpacked into them.
	UnpackKeyPrefix = "extract"
	// UnpackKeyFormat is the format for the snapshotter keys used for extraction
	UnpackKeyFormat       = UnpackKeyPrefix + "-%s %s"
	inheritedLabelsPrefix = "containerd.io/snapshot/"
	labelSnapshotRef      = "containerd.io/snapshot.ref"

	// LabelSnapshotUIDMapping is the label used for UID mappings
	LabelSnapshotUIDMapping = "containerd.io/snapshot/uidmapping"
	// LabelSnapshotGIDMapping is the label used for GID mappings
	LabelSnapshotGIDMapping = "containerd.io/snapshot/gidmapping"
)

// Kind identifies the kind of snapshot.
type Kind uint8

// definitions of snapshot kinds
const (
	KindUnknown Kind = iota
	KindView
	KindActive
	KindCommitted
)

// ParseKind parses the provided string into a Kind
//
// If the string cannot be parsed KindUnknown is returned
func ParseKind(s string) Kind {
	s = strings.ToLower(s)
	switch s {
	case "view":
		return KindView
	case "active":
		return KindActive
	case "committed":
		return KindCommitted
	}

	return KindUnknown
}

// String returns the string representation of the Kind
func (k Kind) String() string {
	switch k {
	case KindView:
		return "View"
	case KindActive:
		return "Active"
	case KindCommitted:
		return "Committed"
	}

	return "Unknown"
}

// MarshalJSON the Kind to JSON
func (k Kind) MarshalJSON() ([]byte, error) {
	return json.Marshal(k.String())
}

// UnmarshalJSON the Kind from JSON
func (k *Kind) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err != nil {
		return err
	}

	*k = ParseKind(s)
	return nil
}

// KindToProto converts from [Kind] to the protobuf definition [snapshots.Kind].
func KindToProto(kind Kind) snapshotsapi.Kind {
	if kind == KindActive {
		return snapshotsapi.Kind_ACTIVE
	}
	if kind == KindView {
		return snapshotsapi.Kind_VIEW
	}
	return snapshotsapi.Kind_COMMITTED
}

// KindFromProto converts from the protobuf definition [snapshots.Kind] to
// [Kind].
func KindFromProto(kind snapshotsapi.Kind) Kind {
	if kind == snapshotsapi.Kind_ACTIVE {
		return KindActive
	}
	if kind == snapshotsapi.Kind_VIEW {
		return KindView
	}
	return KindCommitted
}

// Info provides information about a particular snapshot.
// JSON marshalling is supported for interacting with tools like ctr,
type Info struct {
	Kind   Kind   // active or committed snapshot
	Name   string // name or key of snapshot
	Parent string `json:",omitempty"` // name of parent snapshot

	// Labels for a snapshot.
	//
	// Note: only labels prefixed with `containerd.io/snapshot/` will be inherited
	// by the snapshotter's `Prepare`, `View`, or `Commit` calls.
	Labels  map[string]string `json:",omitempty"`
	Created time.Time         `json:",omitempty"` // Created time
	Updated time.Time         `json:",omitempty"` // Last update time
}

// InfoToProto converts from [Info] to the protobuf definition [snapshots.Info].
func InfoToProto(info Info) *snapshotsapi.Info {
	return &snapshotsapi.Info{
		Name:      info.Name,
		Parent:    info.Parent,
		Kind:      KindToProto(info.Kind),
		CreatedAt: protobuf.ToTimestamp(info.Created),
		UpdatedAt: protobuf.ToTimestamp(info.Updated),
		Labels:    info.Labels,
	}
}

// InfoFromProto converts from the protobuf definition [snapshots.Info] to
// [Info].
func InfoFromProto(info *snapshotsapi.Info) Info {
	return Info{
		Name:    info.Name,
		Parent:  info.Parent,
		Kind:    KindFromProto(info.Kind),
		Created: protobuf.FromTimestamp(info.CreatedAt),
		Updated: protobuf.FromTimestamp(info.UpdatedAt),
		Labels:  info.Labels,
	}
}

// Usage defines statistics for disk resources consumed by the snapshot.
//
// These resources only include the resources consumed by the snapshot itself
// and does not include resources usage by the parent.
type Usage struct {
	Inodes int64 // number of inodes in use.
	Size   int64 // provides usage, in bytes, of snapshot
}

// Add the provided usage to the current usage
func (u *Usage) Add(other Usage) {
	u.Size += other.Size

	// TODO(stevvooe): assumes independent inodes, but provides an upper
	// bound. This should be pretty close, assuming the inodes for a
	// snapshot are roughly unique to it. Don't trust this assumption.
	u.Inodes += other.Inodes
}

// UsageFromProto converts from the protobuf definition [snapshots.Usage] to
// [Usage].
func UsageFromProto(resp *snapshotsapi.UsageResponse) Usage {
	return Usage{
		Inodes: resp.Inodes,
		Size:   resp.Size,
	}
}

// UsageToProto converts from [Usage] to the protobuf definition [snapshots.Usage].
func UsageToProto(usage Usage) *snapshotsapi.UsageResponse {
	return &snapshotsapi.UsageResponse{
		Inodes: usage.Inodes,
		Size:   usage.Size,
	}
}

// WalkFunc defines the callback for a snapshot walk.
type WalkFunc func(context.Context, Info) error

// Snapshotter defines the methods required to implement a snapshot snapshotter for
// allocating, snapshotting and mounting filesystem changesets. The model works
// by building up sets of changes with parent-child relationships.
//
// A snapshot represents a filesystem state. Every snapshot has a parent, where
// the empty parent is represented by the empty string. A diff can be taken
// between a parent and its snapshot to generate a classic layer.
//
// An active snapshot is created by calling `Prepare`. After mounting, changes
// can be made to the snapshot. The act of committing creates a committed
// snapshot. The committed snapshot will get the parent of active snapshot. The
// committed snapshot can then be used as a parent. Active snapshots can never
// act as a parent.
//
// Snapshots are best understood by their lifecycle. Active snapshots are
// always created with Prepare or View. Committed snapshots are always created
// with Commit.  Active snapshots never become committed snapshots and vice
// versa. All snapshots may be removed.
//
// For consistency, we define the following terms to be used throughout this
// interface for snapshotter implementations:
//
//	`ctx` - refers to a context.Context
//	`key` - refers to an active snapshot
//	`name` - refers to a committed snapshot
//	`parent` - refers to the parent in relation
//
// Most methods take various combinations of these identifiers. Typically,
// `name` and `parent` will be used in cases where a method *only* takes
// committed snapshots. `key` will be used to refer to active snapshots in most
// cases, except where noted. All variables used to access snapshots use the
// same key space. For example, an active snapshot may not share the same key
// with a committed snapshot.
//
// We cover several examples below to demonstrate the utility of the snapshotter.
//
// # Importing a Layer
//
// To import a layer, we simply have the snapshotter provide a list of
// mounts to be applied such that our dst will capture a changeset. We start
// out by getting a path to the layer tar file and creating a temp location to
// unpack it to:
//
//	layerPath, tmpDir := getLayerPath(), mkTmpDir() // just a path to layer tar file.
//
// We start by using the snapshotter to Prepare a new snapshot transaction, using a
// key and descending from the empty parent "". To prevent our layer from being
// garbage collected during unpacking, we add the `containerd.io/gc.root` label:
//
//	noGcOpt := snapshots.WithLabels(map[string]string{
//		"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339),
//	})
//	mounts, err := snapshotter.Prepare(ctx, key, "", noGcOpt)
//	if err != nil { ... }
//
// We get back a list of mounts from snapshotter.Prepare(), with the key identifying
// the active snapshot. Mount this to the temporary location with the
// following:
//
//	if err := mount.All(mounts, tmpDir); err != nil { ... }
//
// Once the mounts are performed, our temporary location is ready to capture
// a diff. In practice, this works similar to a filesystem transaction. The
// next step is to unpack the layer. We have a special function unpackLayer
// that applies the contents of the layer to target location and calculates the
// DiffID of the unpacked layer (this is a requirement for docker
// implementation):
//
//	layer, err := os.Open(layerPath)
//	if err != nil { ... }
//	digest, err := unpackLayer(tmpLocation, layer) // unpack into layer location
//	if err != nil { ... }
//
// When the above completes, we should have a filesystem that represents the
// contents of the layer. Careful implementations should verify that digest
// matches the expected DiffID. When completed, we unmount the mounts:
//
//	unmount(mounts) // optional, for now
//
// Now that we've verified and unpacked our layer, we commit the active
// snapshot to a name. For this example, we are just going to use the layer
// digest, but in practice, this will probably be the ChainID. This also removes
// the active snapshot:
//
//	if err := snapshotter.Commit(ctx, digest.String(), key, noGcOpt); err != nil { ... }
//
// Now, we have a layer in the snapshotter that can be accessed with the digest
// provided during commit.
//
// # Importing the Next Layer
//
// Making a layer depend on the above is identical to the process described
// above except that the parent is provided as parent when calling
// snapshotter.Prepare(), assuming a clean, unique key identifier:
//
//	mounts, err := snapshotter.Prepare(ctx, key, parentDigest, noGcOpt)
//
// We then mount, apply and commit, as we did above. The new snapshot will be
// based on the content of the previous one.
//
// # Running a Container
//
// To run a container, we simply provide snapshotter.Prepare() the committed image
// snapshot as the parent. After mounting, the prepared path can
// be used directly as the container's filesystem:
//
//	mounts, err := snapshotter.Prepare(ctx, containerKey, imageRootFSChainID)
//
// The returned mounts can then be passed directly to the container runtime. If
// one would like to create a new image from the filesystem, snapshotter.Commit() is
// called:
//
//	if err := snapshotter.Commit(ctx, newImageSnapshot, containerKey); err != nil { ... }
//
// Alternatively, for most container runs, snapshotter.Remove() will be called to
// signal the snapshotter to abandon the changes.
type Snapshotter interface {
	// Stat returns the info for an active or committed snapshot by name or
	// key.
	//
	// Should be used for parent resolution, existence checks and to discern
	// the kind of snapshot.
	Stat(ctx context.Context, key string) (Info, error)

	// Update updates the info for a snapshot.
	//
	// Only mutable properties of a snapshot may be updated.
	Update(ctx context.Context, info Info, fieldpaths ...string) (Info, error)

	// Usage returns the resource usage of an active or committed snapshot
	// excluding the usage of parent snapshots.
	//
	// The running time of this call for active snapshots is dependent on
	// implementation, but may be proportional to the size of the resource.
	// Callers should take this into consideration. Implementations should
	// attempt to honor context cancellation and avoid taking locks when making
	// the calculation.
	Usage(ctx context.Context, key string) (Usage, error)

	// Mounts returns the mounts for the active snapshot transaction identified
	// by key. Can be called on a read-write or readonly transaction. This is
	// available only for active snapshots.
	//
	// This can be used to recover mounts after calling View or Prepare.
	Mounts(ctx context.Context, key string) ([]mount.Mount, error)

	// Prepare creates an active snapshot identified by key descending from the
	// provided parent.  The returned mounts can be used to mount the snapshot
	// to capture changes.
	//
	// If a parent is provided, after performing the mounts, the destination
	// will start with the content of the parent. The parent must be a
	// committed snapshot. Changes to the mounted destination will be captured
	// in relation to the parent. The default parent, "", is an empty
	// directory.
	//
	// The changes may be saved to a committed snapshot by calling Commit. When
	// one is done with the transaction, Remove should be called on the key.
	//
	// Multiple calls to Prepare or View with the same key should fail.
	Prepare(ctx context.Context, key, parent string, opts ...Opt) ([]mount.Mount, error)

	// View behaves identically to Prepare except the result may not be
	// committed back to the snapshot snapshotter. View returns a readonly view on
	// the parent, with the active snapshot being tracked by the given key.
	//
	// This method operates identically to Prepare, except the mounts returned
	// may have the readonly flag set. Any modifications to the underlying
	// filesystem will be ignored. Implementations may perform this in a more
	// efficient manner that differs from what would be attempted with
	// `Prepare`.
	//
	// Commit may not be called on the provided key and will return an error.
	// To collect the resources associated with key, Remove must be called with
	// key as the argument.
	View(ctx context.Context, key, parent string, opts ...Opt) ([]mount.Mount, error)

	// Commit captures the changes between key and its parent into a snapshot
	// identified by name.  The name can then be used with the snapshotter's other
	// methods to create subsequent snapshots.
	//
	// A committed snapshot will be created under name with the parent of the
	// active snapshot.
	//
	// After commit, the snapshot identified by key is removed.
	Commit(ctx context.Context, name, key string, opts ...Opt) error

	// Remove the committed or active snapshot by the provided key.
	//
	// All resources associated with the key will be removed.
	//
	// If the snapshot is a parent of another snapshot, its children must be
	// removed before proceeding.
	Remove(ctx context.Context, key string) error

	// Walk will call the provided function for each snapshot in the
	// snapshotter which match the provided filters. If no filters are
	// given all items will be walked.
	// Filters:
	//  name
	//  parent
	//  kind (active,view,committed)
	//  labels.(label)
	Walk(ctx context.Context, fn WalkFunc, filters ...string) error

	// Close releases the internal resources.
	//
	// Close is expected to be called on the end of the lifecycle of the snapshotter,
	// but not mandatory.
	//
	// Close returns nil when it is already closed.
	Close() error
}

// Cleaner defines a type capable of performing asynchronous resource cleanup.
// The Cleaner interface should be used by snapshotters which implement fast
// removal and deferred resource cleanup. This prevents snapshots from needing
// to perform lengthy resource cleanup before acknowledging a snapshot key
// has been removed and available for re-use. This is also useful when
// performing multi-key removal with the intent of cleaning up all the
// resources after each snapshot key has been removed.
type Cleaner interface {
	Cleanup(ctx context.Context) error
}

// Opt allows setting mutable snapshot properties on creation
type Opt func(info *Info) error

// WithLabels appends labels to a created snapshot
func WithLabels(labels map[string]string) Opt {
	return func(info *Info) error {
		if info.Labels == nil {
			info.Labels = make(map[string]string)
		}

		for k, v := range labels {
			info.Labels[k] = v
		}

		return nil
	}
}

// FilterInheritedLabels filters the provided labels by removing any key which
// isn't a snapshot label. Snapshot labels have a prefix of "containerd.io/snapshot/"
// or are the "containerd.io/snapshot.ref" label.
func FilterInheritedLabels(labels map[string]string) map[string]string {
	if labels == nil {
		return nil
	}

	filtered := make(map[string]string)
	for k, v := range labels {
		if k == labelSnapshotRef || strings.HasPrefix(k, inheritedLabelsPrefix) {
			filtered[k] = v
		}
	}
	return filtered
}