Merge pull request #9029 from dmcgowan/push-inherit-distribution-sources

push: inherit distribution sources from parent

cmd/ctr/commands/content/fetch.go

@@ -68,8 +68,13 @@ Most of this is experimental and there are few leaps to make this work.`,
 			Usage: "Pull content from all platforms",
 		},
 		cli.BoolFlag{
-			Name:  "all-metadata",
-			Usage: "Pull metadata for all platforms",
+			Name:   "all-metadata",
+			Usage:  "(Deprecated: use skip-metadata) Pull metadata for all platforms",
+			Hidden: true,
+		},
+		cli.BoolFlag{
+			Name:  "skip-metadata",
+			Usage: "Skips metadata for unused platforms (Image may be unable to be pushed without metadata)",
 		},
 		cli.BoolFlag{
 			Name:  "metadata-only",
@@ -141,7 +146,7 @@ func NewFetchConfig(ctx context.Context, clicontext *cli.Context) (*FetchConfig,
 		config.AllMetadata = true
 		// Any with an empty set is None
 		config.PlatformMatcher = platforms.Any()
-	} else if clicontext.Bool("all-metadata") {
+	} else if !clicontext.Bool("skip-metadata") {
 		config.AllMetadata = true
 	}
 

cmd/ctr/commands/images/pull.go

@@ -63,8 +63,13 @@ command. As part of this process, we do the following:
 			Usage: "Pull content and metadata from all platforms",
 		},
 		cli.BoolFlag{
-			Name:  "all-metadata",
-			Usage: "Pull metadata for all platforms",
+			Name:   "all-metadata",
+			Usage:  "(Deprecated: use skip-metadata) Pull metadata for all platforms",
+			Hidden: true,
+		},
+		cli.BoolFlag{
+			Name:  "skip-metadata",
+			Usage: "Skips metadata for unused platforms (Image may be unable to be pushed without metadata)",
 		},
 		cli.BoolFlag{
 			Name:  "print-chainid",
@@ -123,7 +128,7 @@ command. As part of this process, we do the following:
 				// Any with an empty set is None
 				// TODO: Specify way to specify not default platform
 				// config.PlatformMatcher = platforms.Any()
-			} else if context.Bool("all-metadata") {
+			} else if !context.Bool("skip-metadata") {
 				sopts = append(sopts, image.WithAllMetadata)
 			}
 

content/content.go

@@ -87,9 +87,6 @@ type IngestManager interface {
 }
 
 // Info holds content specific information
-//
-// TODO(stevvooe): Consider a very different name for this struct. Info is way
-// to general. It also reads very weird in certain context, like pluralization.
 type Info struct {
 	Digest    digest.Digest
 	Size      int64
@@ -111,12 +108,17 @@ type Status struct {
 // WalkFunc defines the callback for a blob walk.
 type WalkFunc func(Info) error
 
-// Manager provides methods for inspecting, listing and removing content.
-type Manager interface {
+// InfoProvider provides info for content inspection.
+type InfoProvider interface {
 	// Info will return metadata about content available in the content store.
 	//
 	// If the content is not present, ErrNotFound will be returned.
 	Info(ctx context.Context, dgst digest.Digest) (Info, error)
+}
+
+// Manager provides methods for inspecting, listing and removing content.
+type Manager interface {
+	InfoProvider
 
 	// Update updates mutable information related to content.
 	// If one or more fieldpaths are provided, only those

pkg/display/manifest_printer.go

@@ -27,7 +27,6 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
-	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -61,8 +60,7 @@ var LineTreeFormat = TreeFormat{
 
 type ContentReader interface {
 	content.Provider
-
-	Info(ctx context.Context, dgst digest.Digest) (content.Info, error)
+	content.InfoProvider
 }
 
 type ImageTreePrinter struct {

remotes/handlers.go

@@ -204,8 +204,9 @@ func push(ctx context.Context, provider content.Provider, pusher Pusher, desc oc
 // Base handlers can be provided which will be called before any push specific
 // handlers.
 //
-// If the passed in content.Provider is also a content.Manager then this will
-// also annotate the distribution sources in the manager.
+// If the passed in content.Provider is also a content.InfoProvider (such as
+// content.Manager) then this will also annotate the distribution sources using
+// labels prefixed with "containerd.io/distribution.source".
 func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, store content.Provider, limiter *semaphore.Weighted, platform platforms.MatchComparer, wrapper func(h images.Handler) images.Handler) error {
 
 	var m sync.Mutex
@@ -234,7 +235,7 @@ func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, st
 	platformFilterhandler := images.FilterPlatforms(images.ChildrenHandler(store), platform)
 
 	var handler images.Handler
-	if m, ok := store.(content.Manager); ok {
+	if m, ok := store.(content.InfoProvider); ok {
 		annotateHandler := annotateDistributionSourceHandler(platformFilterhandler, m)
 		handler = images.Handlers(annotateHandler, filterHandler, pushHandler)
 	} else {
@@ -340,14 +341,15 @@ func FilterManifestByPlatformHandler(f images.HandlerFunc, m platforms.Matcher)
 
 // annotateDistributionSourceHandler add distribution source label into
 // annotation of config or blob descriptor.
-func annotateDistributionSourceHandler(f images.HandlerFunc, manager content.Manager) images.HandlerFunc {
+func annotateDistributionSourceHandler(f images.HandlerFunc, provider content.InfoProvider) images.HandlerFunc {
 	return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		children, err := f(ctx, desc)
 		if err != nil {
 			return nil, err
 		}
 
-		// only add distribution source for the config or blob data descriptor
+		// Distribution source is only used for config or blob but may be inherited from
+		// a manifest or manifest list
 		switch desc.MediaType {
 		case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest,
 			images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
@@ -355,12 +357,28 @@ func annotateDistributionSourceHandler(f images.HandlerFunc, manager content.Man
 			return children, nil
 		}
 
+		// parentInfo can be used to inherit info for non-existent blobs
+		var parentInfo *content.Info
+
 		for i := range children {
 			child := children[i]
 
-			info, err := manager.Info(ctx, child.Digest)
+			info, err := provider.Info(ctx, child.Digest)
 			if err != nil {
-				return nil, err
+				if !errdefs.IsNotFound(err) {
+					return nil, err
+				}
+				if parentInfo == nil {
+					pi, err := provider.Info(ctx, desc.Digest)
+					if err != nil {
+						return nil, err
+					}
+					parentInfo = &pi
+				}
+				// Blob may not exist locally, annotate with parent labels for cross repo
+				// mount or fetch. Parent sources may apply to all children since most
+				// registries enforce that children exist before the manifests.
+				info = *parentInfo
 			}
 
 			for k, v := range info.Labels {