Merge pull request #4414 from dmcgowan/discard-content

Set content labels based on content type

client.go

@@ -351,6 +351,10 @@ type RemoteContext struct {
 
 	// AllMetadata downloads all manifests and known-configuration files
 	AllMetadata bool
+
+	// ChildLabelMap sets the labels used to reference child objects in the content
+	// store. By default, all GC reference labels will be set for all fetched content.
+	ChildLabelMap func(ocispec.Descriptor) []string
 }
 
 func defaultRemoteContext() *RemoteContext {

client_opts.go

@@ -23,6 +23,7 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/snapshots"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	"google.golang.org/grpc"
 )
@@ -175,6 +176,18 @@ func WithPullLabels(labels map[string]string) RemoteOpt {
 	}
 }
 
+// WithChildLabelMap sets the map function used to define the labels set
+// on referenced child content in the content store. This can be used
+// to overwrite the default GC labels or filter which labels get set
+// for content.
+// The default is `images.ChildGCLabels`.
+func WithChildLabelMap(fn func(ocispec.Descriptor) []string) RemoteOpt {
+	return func(_ *Client, c *RemoteContext) error {
+		c.ChildLabelMap = fn
+		return nil
+	}
+}
+
 // WithSchema1Conversion is used to convert Docker registry schema 1
 // manifests to oci manifests on pull. Without this option schema 1
 // manifests will return a not supported error.

client_test.go

@@ -29,13 +29,17 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/defaults"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/leases"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/log/logtest"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/pkg/testutil"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/sys"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/identity"
 	"github.com/sirupsen/logrus"
 )
 
@@ -215,6 +219,85 @@ func TestImagePull(t *testing.T) {
 	}
 }
 
+func TestImagePullWithDiscardContent(t *testing.T) {
+	client, err := newClient(t, address)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	ctx, cancel := testContext(t)
+	defer cancel()
+
+	err = client.ImageService().Delete(ctx, testImage, images.SynchronousDelete())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ls := client.LeasesService()
+	l, err := ls.Create(ctx, leases.WithRandomID(), leases.WithExpiration(24*time.Hour))
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx = leases.WithLease(ctx, l.ID)
+	img, err := client.Pull(ctx, testImage,
+		WithPlatformMatcher(platforms.Default()),
+		WithPullUnpack,
+		WithChildLabelMap(images.ChildGCLabelsFilterLayers),
+	)
+	// Synchronously garbage collect contents
+	if errL := ls.Delete(ctx, l, leases.SynchronousDelete); errL != nil {
+		t.Fatal(errL)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check if all layer contents have been unpacked and aren't preserved
+	var (
+		diffIDs []digest.Digest
+		layers  []digest.Digest
+	)
+	cs := client.ContentStore()
+	manifest, err := images.Manifest(ctx, cs, img.Target(), platforms.Default())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(manifest.Layers) == 0 {
+		t.Fatalf("failed to get children from %v", img.Target())
+	}
+	for _, l := range manifest.Layers {
+		layers = append(layers, l.Digest)
+	}
+	config, err := images.Config(ctx, cs, img.Target(), platforms.Default())
+	if err != nil {
+		t.Fatal(err)
+	}
+	diffIDs, err = images.RootFS(ctx, cs, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(layers) != len(diffIDs) {
+		t.Fatalf("number of layers and diffIDs don't match: %d != %d", len(layers), len(diffIDs))
+	} else if len(layers) == 0 {
+		t.Fatalf("there is no layers in the target image(parent: %v)", img.Target())
+	}
+	var (
+		sn    = client.SnapshotService("")
+		chain []digest.Digest
+	)
+	for i, dgst := range layers {
+		chain = append(chain, diffIDs[i])
+		chainID := identity.ChainID(chain).String()
+		if _, err := sn.Stat(ctx, chainID); err != nil {
+			t.Errorf("snapshot %v must exist: %v", chainID, err)
+		}
+		if _, err := cs.Info(ctx, dgst); err == nil || !errdefs.IsNotFound(err) {
+			t.Errorf("content %v must be garbage collected: %v", dgst, err)
+		}
+	}
+}
+
 func TestImagePullAllPlatforms(t *testing.T) {
 	client, err := newClient(t, address)
 	if err != nil {

images/handlers.go

@@ -170,6 +170,19 @@ func ChildrenHandler(provider content.Provider) HandlerFunc {
 // the children returned by the handler and passes through the children.
 // Must follow a handler that returns the children to be labeled.
 func SetChildrenLabels(manager content.Manager, f HandlerFunc) HandlerFunc {
+	return SetChildrenMappedLabels(manager, f, nil)
+}
+
+// SetChildrenMappedLabels is a handler wrapper which sets labels for the content on
+// the children returned by the handler and passes through the children.
+// Must follow a handler that returns the children to be labeled.
+// The label map allows the caller to control the labels per child descriptor.
+// For returned labels, the index of the child will be appended to the end
+// except for the first index when the returned label does not end with '.'.
+func SetChildrenMappedLabels(manager content.Manager, f HandlerFunc, labelMap func(ocispec.Descriptor) []string) HandlerFunc {
+	if labelMap == nil {
+		labelMap = ChildGCLabels
+	}
 	return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		children, err := f(ctx, desc)
 		if err != nil {
@@ -177,14 +190,26 @@ func SetChildrenLabels(manager content.Manager, f HandlerFunc) HandlerFunc {
 		}
 
 		if len(children) > 0 {
-			info := content.Info{
-				Digest: desc.Digest,
-				Labels: map[string]string{},
-			}
-			fields := []string{}
-			for i, ch := range children {
-				info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = ch.Digest.String()
-				fields = append(fields, fmt.Sprintf("labels.containerd.io/gc.ref.content.%d", i))
+			var (
+				info = content.Info{
+					Digest: desc.Digest,
+					Labels: map[string]string{},
+				}
+				fields = []string{}
+				keys   = map[string]uint{}
+			)
+			for _, ch := range children {
+				labelKeys := labelMap(ch)
+				for _, key := range labelKeys {
+					idx := keys[key]
+					keys[key] = idx + 1
+					if idx > 0 || key[len(key)-1] == '.' {
+						key = fmt.Sprintf("%s%d", key, idx)
+					}
+
+					info.Labels[key] = ch.Digest.String()
+					fields = append(fields, "labels."+key)
+				}
 			}
 
 			_, err := manager.Update(ctx, info, fields...)

images/mediatypes.go

@@ -125,3 +125,31 @@ func IsKnownConfig(mt string) bool {
 	}
 	return false
 }
+
+// ChildGCLabels returns the label for a given descriptor to reference it
+func ChildGCLabels(desc ocispec.Descriptor) []string {
+	mt := desc.MediaType
+	if IsKnownConfig(mt) {
+		return []string{"containerd.io/gc.ref.content.config"}
+	}
+
+	switch mt {
+	case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+		return []string{"containerd.io/gc.ref.content.m."}
+	}
+
+	if IsLayerType(mt) {
+		return []string{"containerd.io/gc.ref.content.l."}
+	}
+
+	return []string{"containerd.io/gc.ref.content."}
+}
+
+// ChildGCLabelsFilterLayers returns the labels for a given descriptor to
+// reference it, skipping layer media types
+func ChildGCLabelsFilterLayers(desc ocispec.Descriptor) []string {
+	if IsLayerType(desc.MediaType) {
+		return nil
+	}
+	return ChildGCLabels(desc)
+}

pull.go

@@ -159,7 +159,7 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim
 		// Get all the children for a descriptor
 		childrenHandler := images.ChildrenHandler(store)
 		// Set any children labels for that content
-		childrenHandler = images.SetChildrenLabels(store, childrenHandler)
+		childrenHandler = images.SetChildrenMappedLabels(store, childrenHandler, rCtx.ChildLabelMap)
 		if rCtx.AllMetadata {
 			// Filter manifests by platforms but allow to handle manifest
 			// and configuration for not-target platforms