diff --git a/client.go b/client.go index 98c2bdd04..e72433c9b 100644 --- a/client.go +++ b/client.go @@ -351,6 +351,10 @@ type RemoteContext struct { // AllMetadata downloads all manifests and known-configuration files AllMetadata bool + + // ChildLabelMap sets the labels used to reference child objects in the content + // store. By default, all GC reference labels will be set for all fetched content. + ChildLabelMap func(ocispec.Descriptor) []string } func defaultRemoteContext() *RemoteContext { diff --git a/client_opts.go b/client_opts.go index d0f884f83..5271b94ee 100644 --- a/client_opts.go +++ b/client_opts.go @@ -23,6 +23,7 @@ import ( "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/snapshots" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "google.golang.org/grpc" ) @@ -175,6 +176,18 @@ func WithPullLabels(labels map[string]string) RemoteOpt { } } +// WithChildLabelMap sets the map function used to define the labels set +// on referenced child content in the content store. This can be used +// to overwrite the default GC labels or filter which labels get set +// for content. +// The default is `images.ChildGCLabels`. +func WithChildLabelMap(fn func(ocispec.Descriptor) []string) RemoteOpt { + return func(_ *Client, c *RemoteContext) error { + c.ChildLabelMap = fn + return nil + } +} + // WithSchema1Conversion is used to convert Docker registry schema 1 // manifests to oci manifests on pull. Without this option schema 1 // manifests will return a not supported error. diff --git a/client_test.go b/client_test.go index b706167e3..be57edde8 100644 --- a/client_test.go +++ b/client_test.go @@ -29,13 +29,17 @@ import ( "time" "github.com/containerd/containerd/defaults" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" "github.com/containerd/containerd/log" "github.com/containerd/containerd/log/logtest" "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/pkg/testutil" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/sys" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" "github.com/sirupsen/logrus" ) @@ -215,6 +219,85 @@ func TestImagePull(t *testing.T) { } } +func TestImagePullWithDiscardContent(t *testing.T) { + client, err := newClient(t, address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx, cancel := testContext(t) + defer cancel() + + err = client.ImageService().Delete(ctx, testImage, images.SynchronousDelete()) + if err != nil { + t.Fatal(err) + } + + ls := client.LeasesService() + l, err := ls.Create(ctx, leases.WithRandomID(), leases.WithExpiration(24*time.Hour)) + if err != nil { + t.Fatal(err) + } + ctx = leases.WithLease(ctx, l.ID) + img, err := client.Pull(ctx, testImage, + WithPlatformMatcher(platforms.Default()), + WithPullUnpack, + WithChildLabelMap(images.ChildGCLabelsFilterLayers), + ) + // Synchronously garbage collect contents + if errL := ls.Delete(ctx, l, leases.SynchronousDelete); errL != nil { + t.Fatal(errL) + } + if err != nil { + t.Fatal(err) + } + + // Check if all layer contents have been unpacked and aren't preserved + var ( + diffIDs []digest.Digest + layers []digest.Digest + ) + cs := client.ContentStore() + manifest, err := images.Manifest(ctx, cs, img.Target(), platforms.Default()) + if err != nil { + t.Fatal(err) + } + if len(manifest.Layers) == 0 { + t.Fatalf("failed to get children from %v", img.Target()) + } + for _, l := range manifest.Layers { + layers = append(layers, l.Digest) + } + config, err := images.Config(ctx, cs, img.Target(), platforms.Default()) + if err != nil { + t.Fatal(err) + } + diffIDs, err = images.RootFS(ctx, cs, config) + if err != nil { + t.Fatal(err) + } + if len(layers) != len(diffIDs) { + t.Fatalf("number of layers and diffIDs don't match: %d != %d", len(layers), len(diffIDs)) + } else if len(layers) == 0 { + t.Fatalf("there is no layers in the target image(parent: %v)", img.Target()) + } + var ( + sn = client.SnapshotService("") + chain []digest.Digest + ) + for i, dgst := range layers { + chain = append(chain, diffIDs[i]) + chainID := identity.ChainID(chain).String() + if _, err := sn.Stat(ctx, chainID); err != nil { + t.Errorf("snapshot %v must exist: %v", chainID, err) + } + if _, err := cs.Info(ctx, dgst); err == nil || !errdefs.IsNotFound(err) { + t.Errorf("content %v must be garbage collected: %v", dgst, err) + } + } +} + func TestImagePullAllPlatforms(t *testing.T) { client, err := newClient(t, address) if err != nil { diff --git a/images/handlers.go b/images/handlers.go index f89085bcc..05a9017bc 100644 --- a/images/handlers.go +++ b/images/handlers.go @@ -170,6 +170,19 @@ func ChildrenHandler(provider content.Provider) HandlerFunc { // the children returned by the handler and passes through the children. // Must follow a handler that returns the children to be labeled. func SetChildrenLabels(manager content.Manager, f HandlerFunc) HandlerFunc { + return SetChildrenMappedLabels(manager, f, nil) +} + +// SetChildrenMappedLabels is a handler wrapper which sets labels for the content on +// the children returned by the handler and passes through the children. +// Must follow a handler that returns the children to be labeled. +// The label map allows the caller to control the labels per child descriptor. +// For returned labels, the index of the child will be appended to the end +// except for the first index when the returned label does not end with '.'. +func SetChildrenMappedLabels(manager content.Manager, f HandlerFunc, labelMap func(ocispec.Descriptor) []string) HandlerFunc { + if labelMap == nil { + labelMap = ChildGCLabels + } return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { children, err := f(ctx, desc) if err != nil { @@ -177,14 +190,26 @@ func SetChildrenLabels(manager content.Manager, f HandlerFunc) HandlerFunc { } if len(children) > 0 { - info := content.Info{ - Digest: desc.Digest, - Labels: map[string]string{}, - } - fields := []string{} - for i, ch := range children { - info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = ch.Digest.String() - fields = append(fields, fmt.Sprintf("labels.containerd.io/gc.ref.content.%d", i)) + var ( + info = content.Info{ + Digest: desc.Digest, + Labels: map[string]string{}, + } + fields = []string{} + keys = map[string]uint{} + ) + for _, ch := range children { + labelKeys := labelMap(ch) + for _, key := range labelKeys { + idx := keys[key] + keys[key] = idx + 1 + if idx > 0 || key[len(key)-1] == '.' { + key = fmt.Sprintf("%s%d", key, idx) + } + + info.Labels[key] = ch.Digest.String() + fields = append(fields, "labels."+key) + } } _, err := manager.Update(ctx, info, fields...) diff --git a/images/mediatypes.go b/images/mediatypes.go index a0005d4e2..c51897d25 100644 --- a/images/mediatypes.go +++ b/images/mediatypes.go @@ -125,3 +125,31 @@ func IsKnownConfig(mt string) bool { } return false } + +// ChildGCLabels returns the label for a given descriptor to reference it +func ChildGCLabels(desc ocispec.Descriptor) []string { + mt := desc.MediaType + if IsKnownConfig(mt) { + return []string{"containerd.io/gc.ref.content.config"} + } + + switch mt { + case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + return []string{"containerd.io/gc.ref.content.m."} + } + + if IsLayerType(mt) { + return []string{"containerd.io/gc.ref.content.l."} + } + + return []string{"containerd.io/gc.ref.content."} +} + +// ChildGCLabelsFilterLayers returns the labels for a given descriptor to +// reference it, skipping layer media types +func ChildGCLabelsFilterLayers(desc ocispec.Descriptor) []string { + if IsLayerType(desc.MediaType) { + return nil + } + return ChildGCLabels(desc) +} diff --git a/pull.go b/pull.go index f1ee7ff67..36365513f 100644 --- a/pull.go +++ b/pull.go @@ -159,7 +159,7 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim // Get all the children for a descriptor childrenHandler := images.ChildrenHandler(store) // Set any children labels for that content - childrenHandler = images.SetChildrenLabels(store, childrenHandler) + childrenHandler = images.SetChildrenMappedLabels(store, childrenHandler, rCtx.ChildLabelMap) if rCtx.AllMetadata { // Filter manifests by platforms but allow to handle manifest // and configuration for not-target platforms