Add configuration options to local transfer service
Signed-off-by: Tony Fang <nhfang@amazon.com>
This commit is contained in:
		| @@ -22,6 +22,9 @@ import ( | |||||||
| 	"os" | 	"os" | ||||||
| 	"time" | 	"time" | ||||||
|  |  | ||||||
|  | 	ocispec "github.com/opencontainers/image-spec/specs-go/v1" | ||||||
|  | 	"github.com/urfave/cli" | ||||||
|  |  | ||||||
| 	"github.com/containerd/containerd" | 	"github.com/containerd/containerd" | ||||||
| 	"github.com/containerd/containerd/cmd/ctr/commands" | 	"github.com/containerd/containerd/cmd/ctr/commands" | ||||||
| 	"github.com/containerd/containerd/images/archive" | 	"github.com/containerd/containerd/images/archive" | ||||||
| @@ -30,7 +33,6 @@ import ( | |||||||
| 	tarchive "github.com/containerd/containerd/pkg/transfer/archive" | 	tarchive "github.com/containerd/containerd/pkg/transfer/archive" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer/image" | 	"github.com/containerd/containerd/pkg/transfer/image" | ||||||
| 	"github.com/containerd/containerd/platforms" | 	"github.com/containerd/containerd/platforms" | ||||||
| 	"github.com/urfave/cli" |  | ||||||
| ) | ) | ||||||
|  |  | ||||||
| var importCommand = cli.Command{ | var importCommand = cli.Command{ | ||||||
| @@ -127,9 +129,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb | |||||||
| 				opts = append(opts, image.WithNamedPrefix(prefix, overwrite)) | 				opts = append(opts, image.WithNamedPrefix(prefix, overwrite)) | ||||||
| 			} | 			} | ||||||
|  |  | ||||||
| 			// TODO: Add platform options | 			var platSpec ocispec.Platform | ||||||
|  | 			//Only when all-platforms not specified, we will check platform value | ||||||
|  | 			//Implicitly if the platforms is empty, it means all-platforms | ||||||
|  | 			if !context.Bool("all-platforms") { | ||||||
|  | 				//If platform specified, use that one, if not use default | ||||||
|  | 				if platform := context.String("platform"); platform != "" { | ||||||
|  | 					platSpec, err = platforms.Parse(platform) | ||||||
|  | 					if err != nil { | ||||||
|  | 						return err | ||||||
|  | 					} | ||||||
|  | 				} else { | ||||||
|  | 					platSpec = platforms.DefaultSpec() | ||||||
|  | 				} | ||||||
|  | 				opts = append(opts, image.WithPlatforms(platSpec)) | ||||||
|  | 			} | ||||||
|  |  | ||||||
| 			// TODO: Add unpack options | 			if !context.Bool("no-unpack") { | ||||||
|  | 				snapshotter := context.String("snapshotter") | ||||||
|  | 				//If OS field is not empty, it means platSpec was updated in the above block | ||||||
|  | 				//i.e all-platforms was not specified | ||||||
|  | 				if platSpec.OS != "" { | ||||||
|  | 					opts = append(opts, image.WithUnpack(platSpec, snapshotter)) | ||||||
|  | 				} else { | ||||||
|  | 					//empty spec means all platforms | ||||||
|  | 					var emptySpec ocispec.Platform | ||||||
|  | 					opts = append(opts, image.WithUnpack(emptySpec, snapshotter)) | ||||||
|  | 				} | ||||||
|  | 			} | ||||||
|  |  | ||||||
| 			is := image.NewStore(context.String("index-name"), opts...) | 			is := image.NewStore(context.String("index-name"), opts...) | ||||||
|  |  | ||||||
|   | |||||||
| @@ -32,6 +32,7 @@ import ( | |||||||
| 	"github.com/containerd/containerd/pkg/progress" | 	"github.com/containerd/containerd/pkg/progress" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer" | 	"github.com/containerd/containerd/pkg/transfer" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer/image" | 	"github.com/containerd/containerd/pkg/transfer/image" | ||||||
|  | 	"github.com/containerd/containerd/pkg/transfer/registry" | ||||||
| 	"github.com/containerd/containerd/platforms" | 	"github.com/containerd/containerd/platforms" | ||||||
| 	"github.com/opencontainers/image-spec/identity" | 	"github.com/opencontainers/image-spec/identity" | ||||||
| 	ocispec "github.com/opencontainers/image-spec/specs-go/v1" | 	ocispec "github.com/opencontainers/image-spec/specs-go/v1" | ||||||
| @@ -112,6 +113,11 @@ command. As part of this process, we do the following: | |||||||
| 					p = append(p, platforms.DefaultSpec()) | 					p = append(p, platforms.DefaultSpec()) | ||||||
| 				} | 				} | ||||||
| 				sopts = append(sopts, image.WithPlatforms(p...)) | 				sopts = append(sopts, image.WithPlatforms(p...)) | ||||||
|  |  | ||||||
|  | 				//set unpack configuration | ||||||
|  | 				for _, platform := range p { | ||||||
|  | 					sopts = append(sopts, image.WithUnpack(platform, context.String("snapshotter"))) | ||||||
|  | 				} | ||||||
| 			} | 			} | ||||||
| 			// TODO: Support unpack for all platforms..? | 			// TODO: Support unpack for all platforms..? | ||||||
| 			// Pass in a *? | 			// Pass in a *? | ||||||
| @@ -125,7 +131,7 @@ command. As part of this process, we do the following: | |||||||
| 				sopts = append(sopts, image.WithAllMetadata) | 				sopts = append(sopts, image.WithAllMetadata) | ||||||
| 			} | 			} | ||||||
|  |  | ||||||
| 			reg := image.NewOCIRegistry(ref, nil, ch) | 			reg := registry.NewOCIRegistry(ref, nil, ch) | ||||||
| 			is := image.NewStore(ref, sopts...) | 			is := image.NewStore(ref, sopts...) | ||||||
|  |  | ||||||
| 			pf, done := ProgressHandler(ctx, os.Stdout) | 			pf, done := ProgressHandler(ctx, os.Stdout) | ||||||
|   | |||||||
| @@ -34,6 +34,7 @@ import ( | |||||||
| 	"github.com/containerd/containerd/pkg/progress" | 	"github.com/containerd/containerd/pkg/progress" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer" | 	"github.com/containerd/containerd/pkg/transfer" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer/image" | 	"github.com/containerd/containerd/pkg/transfer/image" | ||||||
|  | 	"github.com/containerd/containerd/pkg/transfer/registry" | ||||||
| 	"github.com/containerd/containerd/platforms" | 	"github.com/containerd/containerd/platforms" | ||||||
| 	"github.com/containerd/containerd/remotes" | 	"github.com/containerd/containerd/remotes" | ||||||
| 	"github.com/containerd/containerd/remotes/docker" | 	"github.com/containerd/containerd/remotes/docker" | ||||||
| @@ -103,7 +104,7 @@ var pushCommand = cli.Command{ | |||||||
| 			if local == "" { | 			if local == "" { | ||||||
| 				local = ref | 				local = ref | ||||||
| 			} | 			} | ||||||
| 			reg := image.NewOCIRegistry(ref, nil, ch) | 			reg := registry.NewOCIRegistry(ref, nil, ch) | ||||||
| 			is := image.NewStore(local) | 			is := image.NewStore(local) | ||||||
|  |  | ||||||
| 			pf, done := ProgressHandler(ctx, os.Stdout) | 			pf, done := ProgressHandler(ctx, os.Stdout) | ||||||
|   | |||||||
| @@ -32,7 +32,7 @@ import ( | |||||||
|  |  | ||||||
| 	"github.com/containerd/console" | 	"github.com/containerd/console" | ||||||
| 	"github.com/containerd/containerd/log" | 	"github.com/containerd/containerd/log" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer/image" | 	"github.com/containerd/containerd/pkg/transfer/registry" | ||||||
| 	"github.com/containerd/containerd/remotes" | 	"github.com/containerd/containerd/remotes" | ||||||
| 	"github.com/containerd/containerd/remotes/docker" | 	"github.com/containerd/containerd/remotes/docker" | ||||||
| 	"github.com/containerd/containerd/remotes/docker/config" | 	"github.com/containerd/containerd/remotes/docker/config" | ||||||
| @@ -218,7 +218,7 @@ type staticCredentials struct { | |||||||
| } | } | ||||||
|  |  | ||||||
| // NewStaticCredentials gets credentials from passing in cli context | // NewStaticCredentials gets credentials from passing in cli context | ||||||
| func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (image.CredentialHelper, error) { | func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (registry.CredentialHelper, error) { | ||||||
| 	username := clicontext.String("user") | 	username := clicontext.String("user") | ||||||
| 	var secret string | 	var secret string | ||||||
| 	if i := strings.IndexByte(username, ':'); i > 0 { | 	if i := strings.IndexByte(username, ':'); i > 0 { | ||||||
| @@ -248,12 +248,12 @@ func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref st | |||||||
| 	}, nil | 	}, nil | ||||||
| } | } | ||||||
|  |  | ||||||
| func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (image.Credentials, error) { | func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (registry.Credentials, error) { | ||||||
| 	if ref == sc.ref { | 	if ref == sc.ref { | ||||||
| 		return image.Credentials{ | 		return registry.Credentials{ | ||||||
| 			Username: sc.username, | 			Username: sc.username, | ||||||
| 			Secret:   sc.secret, | 			Secret:   sc.secret, | ||||||
| 		}, nil | 		}, nil | ||||||
| 	} | 	} | ||||||
| 	return image.Credentials{}, nil | 	return registry.Credentials{}, nil | ||||||
| } | } | ||||||
|   | |||||||
| @@ -27,8 +27,8 @@ import ( | |||||||
| 	"github.com/containerd/containerd/images" | 	"github.com/containerd/containerd/images" | ||||||
| 	"github.com/containerd/containerd/images/archive" | 	"github.com/containerd/containerd/images/archive" | ||||||
| 	"github.com/containerd/containerd/pkg/streaming" | 	"github.com/containerd/containerd/pkg/streaming" | ||||||
|  | 	"github.com/containerd/containerd/pkg/transfer" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer/plugins" | 	"github.com/containerd/containerd/pkg/transfer/plugins" | ||||||
| 	"github.com/containerd/containerd/pkg/unpack" |  | ||||||
| 	"github.com/containerd/containerd/platforms" | 	"github.com/containerd/containerd/platforms" | ||||||
| 	"github.com/containerd/containerd/remotes" | 	"github.com/containerd/containerd/remotes" | ||||||
| 	"github.com/containerd/typeurl/v2" | 	"github.com/containerd/typeurl/v2" | ||||||
| @@ -51,7 +51,7 @@ type Store struct { | |||||||
| 	// extraReferences are used to store or lookup multiple references | 	// extraReferences are used to store or lookup multiple references | ||||||
| 	extraReferences []Reference | 	extraReferences []Reference | ||||||
|  |  | ||||||
| 	unpacks []UnpackConfiguration | 	unpacks []transfer.UnpackConfiguration | ||||||
| } | } | ||||||
|  |  | ||||||
| // Reference is used to create or find a reference for an image | // Reference is used to create or find a reference for an image | ||||||
| @@ -84,14 +84,6 @@ type Reference struct { | |||||||
| 	SkipNamedDigest bool | 	SkipNamedDigest bool | ||||||
| } | } | ||||||
|  |  | ||||||
| // UnpackConfiguration specifies the platform and snapshotter to use for resolving |  | ||||||
| // the unpack Platform, if snapshotter is not specified the platform default will |  | ||||||
| // be used. |  | ||||||
| type UnpackConfiguration struct { |  | ||||||
| 	Platform    ocispec.Platform |  | ||||||
| 	Snapshotter string |  | ||||||
| } |  | ||||||
|  |  | ||||||
| // StoreOpt defines options when configuring an image store source or destination | // StoreOpt defines options when configuring an image store source or destination | ||||||
| type StoreOpt func(*Store) | type StoreOpt func(*Store) | ||||||
|  |  | ||||||
| @@ -171,7 +163,7 @@ func WithExtraReference(name string) StoreOpt { | |||||||
| // WithUnpack specifies a platform to unpack for and an optional snapshotter to use | // WithUnpack specifies a platform to unpack for and an optional snapshotter to use | ||||||
| func WithUnpack(p ocispec.Platform, snapshotter string) StoreOpt { | func WithUnpack(p ocispec.Platform, snapshotter string) StoreOpt { | ||||||
| 	return func(s *Store) { | 	return func(s *Store) { | ||||||
| 		s.unpacks = append(s.unpacks, UnpackConfiguration{ | 		s.unpacks = append(s.unpacks, transfer.UnpackConfiguration{ | ||||||
| 			Platform:    p, | 			Platform:    p, | ||||||
| 			Snapshotter: snapshotter, | 			Snapshotter: snapshotter, | ||||||
| 		}) | 		}) | ||||||
| @@ -333,11 +325,11 @@ func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, err | |||||||
| 	return store.Get(ctx, is.imageName) | 	return store.Get(ctx, is.imageName) | ||||||
| } | } | ||||||
|  |  | ||||||
| func (is *Store) UnpackPlatforms() []unpack.Platform { | func (is *Store) UnpackPlatforms() []transfer.UnpackConfiguration { | ||||||
| 	unpacks := make([]unpack.Platform, len(is.unpacks)) | 	unpacks := make([]transfer.UnpackConfiguration, len(is.unpacks)) | ||||||
| 	for i, uc := range is.unpacks { | 	for i, uc := range is.unpacks { | ||||||
| 		unpacks[i].SnapshotterKey = uc.Snapshotter | 		unpacks[i].Snapshotter = uc.Snapshotter | ||||||
| 		unpacks[i].Platform = platforms.Only(uc.Platform) | 		unpacks[i].Platform = uc.Platform | ||||||
| 	} | 	} | ||||||
| 	return unpacks | 	return unpacks | ||||||
| } | } | ||||||
| @@ -424,7 +416,7 @@ func referencesFromProto(references []*transfertypes.ImageReference) []Reference | |||||||
| 	} | 	} | ||||||
| 	return or | 	return or | ||||||
| } | } | ||||||
| func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguration { | func unpackToProto(uc []transfer.UnpackConfiguration) []*transfertypes.UnpackConfiguration { | ||||||
| 	auc := make([]*transfertypes.UnpackConfiguration, len(uc)) | 	auc := make([]*transfertypes.UnpackConfiguration, len(uc)) | ||||||
| 	for i := range uc { | 	for i := range uc { | ||||||
| 		p := types.Platform{ | 		p := types.Platform{ | ||||||
| @@ -440,8 +432,8 @@ func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguratio | |||||||
| 	return auc | 	return auc | ||||||
| } | } | ||||||
|  |  | ||||||
| func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []UnpackConfiguration { | func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []transfer.UnpackConfiguration { | ||||||
| 	uc := make([]UnpackConfiguration, len(auc)) | 	uc := make([]transfer.UnpackConfiguration, len(auc)) | ||||||
| 	for i := range auc { | 	for i := range auc { | ||||||
| 		uc[i].Snapshotter = auc[i].Snapshotter | 		uc[i].Snapshotter = auc[i].Snapshotter | ||||||
| 		if auc[i].Platform != nil { | 		if auc[i].Platform != nil { | ||||||
|   | |||||||
| @@ -19,13 +19,16 @@ package local | |||||||
| import ( | import ( | ||||||
| 	"context" | 	"context" | ||||||
| 	"encoding/json" | 	"encoding/json" | ||||||
|  | 	"fmt" | ||||||
|  |  | ||||||
|  | 	ocispec "github.com/opencontainers/image-spec/specs-go/v1" | ||||||
|  |  | ||||||
| 	"github.com/containerd/containerd/content" | 	"github.com/containerd/containerd/content" | ||||||
| 	"github.com/containerd/containerd/errdefs" | 	"github.com/containerd/containerd/errdefs" | ||||||
| 	"github.com/containerd/containerd/images" | 	"github.com/containerd/containerd/images" | ||||||
| 	"github.com/containerd/containerd/log" | 	"github.com/containerd/containerd/log" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer" | 	"github.com/containerd/containerd/pkg/transfer" | ||||||
| 	ocispec "github.com/opencontainers/image-spec/specs-go/v1" | 	"github.com/containerd/containerd/pkg/unpack" | ||||||
| ) | ) | ||||||
|  |  | ||||||
| func (ts *localTransferService) importStream(ctx context.Context, i transfer.ImageImporter, is transfer.ImageStorer, tops *transfer.Config) error { | func (ts *localTransferService) importStream(ctx context.Context, i transfer.ImageImporter, is transfer.ImageStorer, tops *transfer.Config) error { | ||||||
| @@ -46,12 +49,16 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima | |||||||
| 		return err | 		return err | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	var descriptors []ocispec.Descriptor | 	var ( | ||||||
|  | 		descriptors []ocispec.Descriptor | ||||||
|  | 		handler     images.Handler | ||||||
|  | 		unpacker    *unpack.Unpacker | ||||||
|  | 	) | ||||||
|  |  | ||||||
| 	// If save index, add index | 	// If save index, add index | ||||||
| 	descriptors = append(descriptors, index) | 	descriptors = append(descriptors, index) | ||||||
|  |  | ||||||
| 	var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { | 	var handlerFunc images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { | ||||||
| 		// Only save images at top level | 		// Only save images at top level | ||||||
| 		if desc.Digest != index.Digest { | 		if desc.Digest != index.Digest { | ||||||
| 			return images.Children(ctx, ts.content, desc) | 			return images.Children(ctx, ts.content, desc) | ||||||
| @@ -76,7 +83,33 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima | |||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	if f, ok := is.(transfer.ImageFilterer); ok { | 	if f, ok := is.(transfer.ImageFilterer); ok { | ||||||
| 		handler = f.ImageFilter(handler, ts.content) | 		handlerFunc = f.ImageFilter(handlerFunc, ts.content) | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	handler = images.Handlers(handlerFunc) | ||||||
|  |  | ||||||
|  | 	// First find suitable platforms to unpack into | ||||||
|  | 	//If image storer is also an unpacker type, i.e implemented UnpackPlatforms() func | ||||||
|  | 	if iu, ok := is.(transfer.ImageUnpacker); ok { | ||||||
|  | 		unpacks := iu.UnpackPlatforms() | ||||||
|  | 		if len(unpacks) > 0 { | ||||||
|  | 			uopts := []unpack.UnpackerOpt{} | ||||||
|  | 			for _, u := range unpacks { | ||||||
|  | 				matched, mu := getSupportedPlatform(u, ts.config.UnpackPlatforms) | ||||||
|  | 				if matched { | ||||||
|  | 					uopts = append(uopts, unpack.WithUnpackPlatform(mu)) | ||||||
|  | 				} | ||||||
|  | 			} | ||||||
|  |  | ||||||
|  | 			if ts.config.DuplicationSuppressor != nil { | ||||||
|  | 				uopts = append(uopts, unpack.WithDuplicationSuppressor(ts.config.DuplicationSuppressor)) | ||||||
|  | 			} | ||||||
|  | 			unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...) | ||||||
|  | 			if err != nil { | ||||||
|  | 				return fmt.Errorf("unable to initialize unpacker: %w", err) | ||||||
|  | 			} | ||||||
|  | 			handler = unpacker.Unpack(handler) | ||||||
|  | 		} | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	if err := images.WalkNotEmpty(ctx, handler, index); err != nil { | 	if err := images.WalkNotEmpty(ctx, handler, index); err != nil { | ||||||
|   | |||||||
| @@ -20,6 +20,7 @@ import ( | |||||||
| 	"context" | 	"context" | ||||||
| 	"fmt" | 	"fmt" | ||||||
|  |  | ||||||
|  | 	"github.com/containerd/containerd" | ||||||
| 	"github.com/containerd/containerd/content" | 	"github.com/containerd/containerd/content" | ||||||
| 	"github.com/containerd/containerd/errdefs" | 	"github.com/containerd/containerd/errdefs" | ||||||
| 	"github.com/containerd/containerd/images" | 	"github.com/containerd/containerd/images" | ||||||
| @@ -73,6 +74,8 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch | |||||||
| 	var ( | 	var ( | ||||||
| 		handler images.Handler | 		handler images.Handler | ||||||
|  |  | ||||||
|  | 		baseHandlers []images.Handler | ||||||
|  |  | ||||||
| 		unpacker *unpack.Unpacker | 		unpacker *unpack.Unpacker | ||||||
|  |  | ||||||
| 		// has a config media type bug (distribution#1622) | 		// has a config media type bug (distribution#1622) | ||||||
| @@ -97,12 +100,6 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch | |||||||
| 		childrenHandler = f.ImageFilter(childrenHandler, store) | 		childrenHandler = f.ImageFilter(childrenHandler, store) | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	// Sort and limit manifests if a finite number is needed |  | ||||||
| 	//if limit > 0 { |  | ||||||
| 	//	childrenHandler = images.LimitManifests(childrenHandler, rCtx.PlatformMatcher, limit) |  | ||||||
| 	//} |  | ||||||
| 	//SetChildrenMappedLabels(manager content.Manager, f HandlerFunc, labelMap func(ocispec.Descriptor) []string) HandlerFunc { |  | ||||||
|  |  | ||||||
| 	checkNeedsFix := images.HandlerFunc( | 	checkNeedsFix := images.HandlerFunc( | ||||||
| 		func(_ context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { | 		func(_ context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { | ||||||
| 			// set to true if there is application/octet-stream media type | 			// set to true if there is application/octet-stream media type | ||||||
| @@ -119,8 +116,12 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch | |||||||
| 		return err | 		return err | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	// TODO: Allow initialization from configuration | 	//Set up baseHandlers from service configuration if present or create a new one | ||||||
| 	baseHandlers := []images.Handler{} | 	if ts.config.BaseHandlers != nil { | ||||||
|  | 		baseHandlers = ts.config.BaseHandlers | ||||||
|  | 	} else { | ||||||
|  | 		baseHandlers = []images.Handler{} | ||||||
|  | 	} | ||||||
|  |  | ||||||
| 	if tops.Progress != nil { | 	if tops.Progress != nil { | ||||||
| 		baseHandlers = append(baseHandlers, images.HandlerFunc( | 		baseHandlers = append(baseHandlers, images.HandlerFunc( | ||||||
| @@ -149,22 +150,28 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch | |||||||
| 		appendDistSrcLabelHandler, | 		appendDistSrcLabelHandler, | ||||||
| 	)...) | 	)...) | ||||||
|  |  | ||||||
| 	// TODO: Should available platforms be a configuration of the service? |  | ||||||
| 	// First find suitable platforms to unpack into | 	// First find suitable platforms to unpack into | ||||||
| 	//if unpacker, ok := is. | 	//If image storer is also an unpacker type, i.e implemented UnpackPlatforms() func | ||||||
| 	if iu, ok := is.(transfer.ImageUnpacker); ok { | 	if iu, ok := is.(transfer.ImageUnpacker); ok { | ||||||
| 		unpacks := iu.UnpackPlatforms() | 		unpacks := iu.UnpackPlatforms() | ||||||
| 		if len(unpacks) > 0 { | 		if len(unpacks) > 0 { | ||||||
| 			uopts := []unpack.UnpackerOpt{} | 			uopts := []unpack.UnpackerOpt{} | ||||||
|  | 			//Only unpack if requested unpackconfig matches default/supported unpackconfigs | ||||||
| 			for _, u := range unpacks { | 			for _, u := range unpacks { | ||||||
| 				uopts = append(uopts, unpack.WithUnpackPlatform(u)) | 				matched, mu := getSupportedPlatform(u, ts.config.UnpackPlatforms) | ||||||
|  | 				if matched { | ||||||
|  | 					uopts = append(uopts, unpack.WithUnpackPlatform(mu)) | ||||||
| 				} | 				} | ||||||
| 			if ts.limiter != nil { |  | ||||||
| 				uopts = append(uopts, unpack.WithLimiter(ts.limiter)) |  | ||||||
| 			} | 			} | ||||||
| 			//if uconfig.DuplicationSuppressor != nil { |  | ||||||
| 			//	uopts = append(uopts, unpack.WithDuplicationSuppressor(uconfig.DuplicationSuppressor)) | 			if ts.limiterD != nil { | ||||||
| 			//} | 				uopts = append(uopts, unpack.WithLimiter(ts.limiterD)) | ||||||
|  | 			} | ||||||
|  |  | ||||||
|  | 			if ts.config.DuplicationSuppressor != nil { | ||||||
|  | 				uopts = append(uopts, unpack.WithDuplicationSuppressor(ts.config.DuplicationSuppressor)) | ||||||
|  | 			} | ||||||
|  |  | ||||||
| 			unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...) | 			unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...) | ||||||
| 			if err != nil { | 			if err != nil { | ||||||
| 				return fmt.Errorf("unable to initialize unpacker: %w", err) | 				return fmt.Errorf("unable to initialize unpacker: %w", err) | ||||||
| @@ -173,7 +180,7 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch | |||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	if err := images.Dispatch(ctx, handler, ts.limiter, desc); err != nil { | 	if err := images.Dispatch(ctx, handler, ts.limiterD, desc); err != nil { | ||||||
| 		if unpacker != nil { | 		if unpacker != nil { | ||||||
| 			// wait for unpacker to cleanup | 			// wait for unpacker to cleanup | ||||||
| 			unpacker.Wait() | 			unpacker.Wait() | ||||||
| @@ -241,3 +248,23 @@ func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher, pt *Progre | |||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
|  |  | ||||||
|  | // getSupportedPlatform returns a matched platform comparing input UnpackConfiguration to the supported platform/snapshotter combinations | ||||||
|  | // If input platform didn't specify snapshotter, default will be used if there is a match on platform. | ||||||
|  | func getSupportedPlatform(uc transfer.UnpackConfiguration, supportedPlatforms []unpack.Platform) (bool, unpack.Platform) { | ||||||
|  | 	var u unpack.Platform | ||||||
|  | 	for _, sp := range supportedPlatforms { | ||||||
|  | 		//If both platform and snapshotter match, return the supportPlatform | ||||||
|  | 		//If platform matched and SnapshotterKey is empty, we assume client didn't pass SnapshotterKey | ||||||
|  | 		//use default Snapshotter | ||||||
|  | 		if sp.Platform.Match(uc.Platform) { | ||||||
|  | 			//assuming sp.SnapshotterKey is not empty | ||||||
|  | 			if uc.Snapshotter == sp.SnapshotterKey { | ||||||
|  | 				return true, sp | ||||||
|  | 			} else if uc.Snapshotter == "" && sp.SnapshotterKey == containerd.DefaultSnapshotter { | ||||||
|  | 				return true, sp | ||||||
|  | 			} | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  | 	return false, u | ||||||
|  | } | ||||||
|   | |||||||
| @@ -105,8 +105,7 @@ func (ts *localTransferService) push(ctx context.Context, ig transfer.ImageGette | |||||||
| 			wrapper = pushCtx.HandlerWrapper | 			wrapper = pushCtx.HandlerWrapper | ||||||
| 		} | 		} | ||||||
| 	*/ | 	*/ | ||||||
|  | 	if err := remotes.PushContent(ctx, pusher, img.Target, ts.content, ts.limiterU, matcher, wrapper); err != nil { | ||||||
| 	if err := remotes.PushContent(ctx, pusher, img.Target, ts.content, ts.limiter, matcher, wrapper); err != nil { |  | ||||||
| 		return err | 		return err | ||||||
| 	} | 	} | ||||||
| 	if tops.Progress != nil { | 	if tops.Progress != nil { | ||||||
|   | |||||||
| @@ -22,11 +22,15 @@ import ( | |||||||
| 	"io" | 	"io" | ||||||
| 	"time" | 	"time" | ||||||
|  |  | ||||||
|  | 	"github.com/containerd/containerd" | ||||||
| 	"github.com/containerd/containerd/content" | 	"github.com/containerd/containerd/content" | ||||||
| 	"github.com/containerd/containerd/errdefs" | 	"github.com/containerd/containerd/errdefs" | ||||||
| 	"github.com/containerd/containerd/images" | 	"github.com/containerd/containerd/images" | ||||||
| 	"github.com/containerd/containerd/leases" | 	"github.com/containerd/containerd/leases" | ||||||
|  | 	"github.com/containerd/containerd/pkg/kmutex" | ||||||
| 	"github.com/containerd/containerd/pkg/transfer" | 	"github.com/containerd/containerd/pkg/transfer" | ||||||
|  | 	"github.com/containerd/containerd/pkg/unpack" | ||||||
|  | 	"github.com/containerd/containerd/platforms" | ||||||
| 	"github.com/containerd/typeurl/v2" | 	"github.com/containerd/typeurl/v2" | ||||||
| 	"golang.org/x/sync/semaphore" | 	"golang.org/x/sync/semaphore" | ||||||
| ) | ) | ||||||
| @@ -35,25 +39,21 @@ type localTransferService struct { | |||||||
| 	leases  leases.Manager | 	leases  leases.Manager | ||||||
| 	content content.Store | 	content content.Store | ||||||
| 	images  images.Store | 	images  images.Store | ||||||
|  | 	//limiter for upload | ||||||
| 	// semaphore.NewWeighted(int64(rCtx.MaxConcurrentDownloads)) | 	limiterU *semaphore.Weighted | ||||||
| 	limiter *semaphore.Weighted | 	//limiter for download operation | ||||||
|  | 	limiterD *semaphore.Weighted | ||||||
| 	// TODO: Duplication suppressor | 	config   TransferConfig | ||||||
|  |  | ||||||
| 	// Configuration |  | ||||||
| 	//  - Max downloads |  | ||||||
| 	//  - Max uploads |  | ||||||
|  |  | ||||||
| 	// Supported platforms |  | ||||||
| 	//  - Platform -> snapshotter defaults? |  | ||||||
| } | } | ||||||
|  |  | ||||||
| func NewTransferService(lm leases.Manager, cs content.Store, is images.Store) transfer.Transferrer { | func NewTransferService(lm leases.Manager, cs content.Store, is images.Store, tc *TransferConfig) transfer.Transferrer { | ||||||
| 	return &localTransferService{ | 	return &localTransferService{ | ||||||
| 		leases:   lm, | 		leases:   lm, | ||||||
| 		content:  cs, | 		content:  cs, | ||||||
| 		images:   is, | 		images:   is, | ||||||
|  | 		limiterU: semaphore.NewWeighted(int64(tc.MaxConcurrentUploadedLayers)), | ||||||
|  | 		limiterD: semaphore.NewWeighted(int64(tc.MaxConcurrentDownloads)), | ||||||
|  | 		config:   *tc, | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
|  |  | ||||||
| @@ -150,3 +150,40 @@ func (ts *localTransferService) withLease(ctx context.Context, opts ...leases.Op | |||||||
| 		return ls.Delete(ctx, l) | 		return ls.Delete(ctx, l) | ||||||
| 	}, nil | 	}, nil | ||||||
| } | } | ||||||
|  |  | ||||||
|  | type TransferConfig struct { | ||||||
|  | 	// MaxConcurrentDownloads is the max concurrent content downloads for pull. | ||||||
|  | 	MaxConcurrentDownloads int `toml:"max_concurrent_downloads"` | ||||||
|  |  | ||||||
|  | 	// MaxConcurrentUploadedLayers is the max concurrent uploads for push | ||||||
|  | 	MaxConcurrentUploadedLayers int `toml:"max_concurrent_uploaded_layers"` | ||||||
|  |  | ||||||
|  | 	// DuplicationSuppressor is used to make sure that there is only one | ||||||
|  | 	// in-flight fetch request or unpack handler for a given descriptor's | ||||||
|  | 	// digest or chain ID. | ||||||
|  | 	DuplicationSuppressor kmutex.KeyedLocker | ||||||
|  |  | ||||||
|  | 	// BaseHandlers are a set of handlers which get are called on dispatch. | ||||||
|  | 	// These handlers always get called before any operation specific | ||||||
|  | 	// handlers. | ||||||
|  | 	BaseHandlers []images.Handler | ||||||
|  |  | ||||||
|  | 	//UnpackPlatforms are used to specify supported combination of platforms and snapshotters | ||||||
|  | 	UnpackPlatforms []unpack.Platform `toml:"unpack_platforms"` | ||||||
|  |  | ||||||
|  | 	// RegistryConfigPath is a path to the root directory containing registry-specific configurations | ||||||
|  | 	RegistryConfigPath string `toml:"config_path"` | ||||||
|  | } | ||||||
|  |  | ||||||
|  | func DefaultConfig() *TransferConfig { | ||||||
|  | 	return &TransferConfig{ | ||||||
|  | 		MaxConcurrentDownloads:      3, | ||||||
|  | 		MaxConcurrentUploadedLayers: 3, | ||||||
|  | 		UnpackPlatforms: []unpack.Platform{ | ||||||
|  | 			{ | ||||||
|  | 				Platform:       platforms.Only(platforms.DefaultSpec()), | ||||||
|  | 				SnapshotterKey: containerd.DefaultSnapshotter, | ||||||
|  | 			}, | ||||||
|  | 		}, | ||||||
|  | 	} | ||||||
|  | } | ||||||
|   | |||||||
| @@ -21,6 +21,8 @@ import ( | |||||||
| 	"errors" | 	"errors" | ||||||
| 	"io" | 	"io" | ||||||
|  |  | ||||||
|  | 	"google.golang.org/protobuf/types/known/anypb" | ||||||
|  |  | ||||||
| 	transferapi "github.com/containerd/containerd/api/services/transfer/v1" | 	transferapi "github.com/containerd/containerd/api/services/transfer/v1" | ||||||
| 	transfertypes "github.com/containerd/containerd/api/types/transfer" | 	transfertypes "github.com/containerd/containerd/api/types/transfer" | ||||||
| 	"github.com/containerd/containerd/log" | 	"github.com/containerd/containerd/log" | ||||||
| @@ -28,7 +30,6 @@ import ( | |||||||
| 	"github.com/containerd/containerd/pkg/transfer" | 	"github.com/containerd/containerd/pkg/transfer" | ||||||
| 	tstreaming "github.com/containerd/containerd/pkg/transfer/streaming" | 	tstreaming "github.com/containerd/containerd/pkg/transfer/streaming" | ||||||
| 	"github.com/containerd/typeurl/v2" | 	"github.com/containerd/typeurl/v2" | ||||||
| 	"google.golang.org/protobuf/types/known/anypb" |  | ||||||
| ) | ) | ||||||
|  |  | ||||||
| type proxyTransferrer struct { | type proxyTransferrer struct { | ||||||
| @@ -36,7 +37,7 @@ type proxyTransferrer struct { | |||||||
| 	streamCreator streaming.StreamCreator | 	streamCreator streaming.StreamCreator | ||||||
| } | } | ||||||
|  |  | ||||||
| // NewTransferrer returns a new transferr which communicates over a GRPC | // NewTransferrer returns a new transferrer which communicates over a GRPC | ||||||
| // connection using the containerd transfer API | // connection using the containerd transfer API | ||||||
| func NewTransferrer(client transferapi.TransferClient, sc streaming.StreamCreator) transfer.Transferrer { | func NewTransferrer(client transferapi.TransferClient, sc streaming.StreamCreator) transfer.Transferrer { | ||||||
| 	return &proxyTransferrer{ | 	return &proxyTransferrer{ | ||||||
|   | |||||||
| @@ -14,7 +14,7 @@ | |||||||
|    limitations under the License. |    limitations under the License. | ||||||
| */ | */ | ||||||
| 
 | 
 | ||||||
| package image | package registry | ||||||
| 
 | 
 | ||||||
| import ( | import ( | ||||||
| 	"context" | 	"context" | ||||||
| @@ -20,10 +20,10 @@ import ( | |||||||
| 	"context" | 	"context" | ||||||
| 	"io" | 	"io" | ||||||
|  |  | ||||||
|  | 	ocispec "github.com/opencontainers/image-spec/specs-go/v1" | ||||||
|  |  | ||||||
| 	"github.com/containerd/containerd/content" | 	"github.com/containerd/containerd/content" | ||||||
| 	"github.com/containerd/containerd/images" | 	"github.com/containerd/containerd/images" | ||||||
| 	"github.com/containerd/containerd/pkg/unpack" |  | ||||||
| 	ocispec "github.com/opencontainers/image-spec/specs-go/v1" |  | ||||||
| ) | ) | ||||||
|  |  | ||||||
| type Transferrer interface { | type Transferrer interface { | ||||||
| @@ -86,8 +86,15 @@ type ImageExportStreamer interface { | |||||||
| } | } | ||||||
|  |  | ||||||
| type ImageUnpacker interface { | type ImageUnpacker interface { | ||||||
| 	// TODO: consider using unpack options | 	UnpackPlatforms() []UnpackConfiguration | ||||||
| 	UnpackPlatforms() []unpack.Platform | } | ||||||
|  |  | ||||||
|  | // UnpackConfiguration specifies the platform and snapshotter to use for resolving | ||||||
|  | // the unpack Platform, if snapshotter is not specified the platform default will | ||||||
|  | // be used. | ||||||
|  | type UnpackConfiguration struct { | ||||||
|  | 	Platform    ocispec.Platform | ||||||
|  | 	Snapshotter string | ||||||
| } | } | ||||||
|  |  | ||||||
| type ProgressFunc func(Progress) | type ProgressFunc func(Progress) | ||||||
|   | |||||||
| @@ -27,6 +27,7 @@ import ( | |||||||
| 	_ "github.com/containerd/containerd/pkg/transfer/image" | 	_ "github.com/containerd/containerd/pkg/transfer/image" | ||||||
| ) | ) | ||||||
|  |  | ||||||
|  | // Register local transfer service plugin | ||||||
| func init() { | func init() { | ||||||
| 	plugin.Register(&plugin.Registration{ | 	plugin.Register(&plugin.Registration{ | ||||||
| 		Type: plugin.TransferPlugin, | 		Type: plugin.TransferPlugin, | ||||||
| @@ -35,8 +36,9 @@ func init() { | |||||||
| 			plugin.LeasePlugin, | 			plugin.LeasePlugin, | ||||||
| 			plugin.MetadataPlugin, | 			plugin.MetadataPlugin, | ||||||
| 		}, | 		}, | ||||||
| 		Config: &transferConfig{}, | 		Config: local.DefaultConfig(), | ||||||
| 		InitFn: func(ic *plugin.InitContext) (interface{}, error) { | 		InitFn: func(ic *plugin.InitContext) (interface{}, error) { | ||||||
|  | 			config := ic.Config.(*local.TransferConfig) | ||||||
| 			m, err := ic.Get(plugin.MetadataPlugin) | 			m, err := ic.Get(plugin.MetadataPlugin) | ||||||
| 			if err != nil { | 			if err != nil { | ||||||
| 				return nil, err | 				return nil, err | ||||||
| @@ -47,12 +49,7 @@ func init() { | |||||||
| 				return nil, err | 				return nil, err | ||||||
| 			} | 			} | ||||||
|  |  | ||||||
| 			return local.NewTransferService(l.(leases.Manager), ms.ContentStore(), metadata.NewImageStore(ms)), nil | 			return local.NewTransferService(l.(leases.Manager), ms.ContentStore(), metadata.NewImageStore(ms), config), nil | ||||||
| 		}, | 		}, | ||||||
| 	}) | 	}) | ||||||
| } | } | ||||||
|  |  | ||||||
| type transferConfig struct { |  | ||||||
| 	// Max concurrent downloads |  | ||||||
| 	// Snapshotter platforms |  | ||||||
| } |  | ||||||
|   | |||||||
| @@ -34,7 +34,7 @@ type Resolver interface { | |||||||
| 	// reference a specific host or be matched against a specific handler. | 	// reference a specific host or be matched against a specific handler. | ||||||
| 	// | 	// | ||||||
| 	// The returned name should be used to identify the referenced entity. | 	// The returned name should be used to identify the referenced entity. | ||||||
| 	// Dependending on the remote namespace, this may be immutable or mutable. | 	// Depending on the remote namespace, this may be immutable or mutable. | ||||||
| 	// While the name may differ from ref, it should itself be a valid ref. | 	// While the name may differ from ref, it should itself be a valid ref. | ||||||
| 	// | 	// | ||||||
| 	// If the resolution fails, an error will be returned. | 	// If the resolution fails, an error will be returned. | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user
	 Tony Fang
					Tony Fang