github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add configuration options to local transfer service

Browse Source 
Signed-off-by: Tony Fang <nhfang@amazon.com>
This commit is contained in:
Tony Fang 2023-02-07 20:58:03 +00:00
parent e366facb87
commit 47305392c6
14 changed files with 209 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
cmd/ctr/commands/images/import.go
View File

@ -22,6 +22,9 @@ import (
"os"
"time"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/urfave/cli"
"github.com/containerd/containerd"
"github.com/containerd/containerd/cmd/ctr/commands"
"github.com/containerd/containerd/images/archive"
@ -30,7 +33,6 @@ import (
tarchive "github.com/containerd/containerd/pkg/transfer/archive"
"github.com/containerd/containerd/pkg/transfer/image"
"github.com/containerd/containerd/platforms"
"github.com/urfave/cli"
)
var importCommand = cli.Command{
@ -127,9 +129,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
opts = append(opts, image.WithNamedPrefix(prefix, overwrite))
}
// TODO: Add platform options
var platSpec ocispec.Platform
//Only when all-platforms not specified, we will check platform value
//Implicitly if the platforms is empty, it means all-platforms
if !context.Bool("all-platforms") {
//If platform specified, use that one, if not use default
if platform := context.String("platform"); platform != "" {
platSpec, err = platforms.Parse(platform)
if err != nil {
return err
}
} else {
platSpec = platforms.DefaultSpec()
}
opts = append(opts, image.WithPlatforms(platSpec))
}
// TODO: Add unpack options
if !context.Bool("no-unpack") {
snapshotter := context.String("snapshotter")
//If OS field is not empty, it means platSpec was updated in the above block
//i.e all-platforms was not specified
if platSpec.OS != "" {
opts = append(opts, image.WithUnpack(platSpec, snapshotter))
} else {
//empty spec means all platforms
var emptySpec ocispec.Platform
opts = append(opts, image.WithUnpack(emptySpec, snapshotter))
}
}
is := image.NewStore(context.String("index-name"), opts...)

8
cmd/ctr/commands/images/pull.go
View File

@ -32,6 +32,7 @@ import (
"github.com/containerd/containerd/pkg/progress"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/transfer/image"
"github.com/containerd/containerd/pkg/transfer/registry"
"github.com/containerd/containerd/platforms"
"github.com/opencontainers/image-spec/identity"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -112,6 +113,11 @@ command. As part of this process, we do the following:
p = append(p, platforms.DefaultSpec())
}
sopts = append(sopts, image.WithPlatforms(p...))
//set unpack configuration
for _, platform := range p {
sopts = append(sopts, image.WithUnpack(platform, context.String("snapshotter")))
}
}
// TODO: Support unpack for all platforms..?
// Pass in a *?
@ -125,7 +131,7 @@ command. As part of this process, we do the following:
sopts = append(sopts, image.WithAllMetadata)
}
reg := image.NewOCIRegistry(ref, nil, ch)
reg := registry.NewOCIRegistry(ref, nil, ch)
is := image.NewStore(ref, sopts...)
pf, done := ProgressHandler(ctx, os.Stdout)

3
cmd/ctr/commands/images/push.go
View File

@ -34,6 +34,7 @@ import (
"github.com/containerd/containerd/pkg/progress"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/transfer/image"
"github.com/containerd/containerd/pkg/transfer/registry"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
@ -103,7 +104,7 @@ var pushCommand = cli.Command{
if local == "" {
local = ref
}
reg := image.NewOCIRegistry(ref, nil, ch)
reg := registry.NewOCIRegistry(ref, nil, ch)
is := image.NewStore(local)
pf, done := ProgressHandler(ctx, os.Stdout)

10
cmd/ctr/commands/resolver.go
View File

@ -32,7 +32,7 @@ import (
"github.com/containerd/console"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/transfer/image"
"github.com/containerd/containerd/pkg/transfer/registry"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/remotes/docker/config"
@ -218,7 +218,7 @@ type staticCredentials struct {
}
// NewStaticCredentials gets credentials from passing in cli context
func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (image.CredentialHelper, error) {
func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (registry.CredentialHelper, error) {
username := clicontext.String("user")
var secret string
if i := strings.IndexByte(username, ':'); i > 0 {
@ -248,12 +248,12 @@ func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref st
}, nil
}
func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (image.Credentials, error) {
func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (registry.Credentials, error) {
if ref == sc.ref {
return image.Credentials{
return registry.Credentials{
Username: sc.username,
Secret: sc.secret,
}, nil
}
return image.Credentials{}, nil
return registry.Credentials{}, nil
}

28
pkg/transfer/image/imagestore.go
View File

@ -27,8 +27,8 @@ import (
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/images/archive"
"github.com/containerd/containerd/pkg/streaming"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/transfer/plugins"
"github.com/containerd/containerd/pkg/unpack"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/containerd/typeurl/v2"
@ -51,7 +51,7 @@ type Store struct {
// extraReferences are used to store or lookup multiple references
extraReferences []Reference
unpacks []UnpackConfiguration
unpacks []transfer.UnpackConfiguration
}
// Reference is used to create or find a reference for an image
@ -84,14 +84,6 @@ type Reference struct {
SkipNamedDigest bool
}
// UnpackConfiguration specifies the platform and snapshotter to use for resolving
// the unpack Platform, if snapshotter is not specified the platform default will
// be used.
type UnpackConfiguration struct {
Platform ocispec.Platform
Snapshotter string
}
// StoreOpt defines options when configuring an image store source or destination
type StoreOpt func(*Store)
@ -171,7 +163,7 @@ func WithExtraReference(name string) StoreOpt {
// WithUnpack specifies a platform to unpack for and an optional snapshotter to use
func WithUnpack(p ocispec.Platform, snapshotter string) StoreOpt {
return func(s *Store) {
s.unpacks = append(s.unpacks, UnpackConfiguration{
s.unpacks = append(s.unpacks, transfer.UnpackConfiguration{
Platform: p,
Snapshotter: snapshotter,
})
@ -333,11 +325,11 @@ func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, err
return store.Get(ctx, is.imageName)
}
func (is *Store) UnpackPlatforms() []unpack.Platform {
unpacks := make([]unpack.Platform, len(is.unpacks))
func (is *Store) UnpackPlatforms() []transfer.UnpackConfiguration {
unpacks := make([]transfer.UnpackConfiguration, len(is.unpacks))
for i, uc := range is.unpacks {
unpacks[i].SnapshotterKey = uc.Snapshotter
unpacks[i].Platform = platforms.Only(uc.Platform)
unpacks[i].Snapshotter = uc.Snapshotter
unpacks[i].Platform = uc.Platform
}
return unpacks
}
@ -424,7 +416,7 @@ func referencesFromProto(references []*transfertypes.ImageReference) []Reference
}
return or
}
func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguration {
func unpackToProto(uc []transfer.UnpackConfiguration) []*transfertypes.UnpackConfiguration {
auc := make([]*transfertypes.UnpackConfiguration, len(uc))
for i := range uc {
p := types.Platform{
@ -440,8 +432,8 @@ func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguratio
return auc
}
func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []UnpackConfiguration {
uc := make([]UnpackConfiguration, len(auc))
func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []transfer.UnpackConfiguration {
uc := make([]transfer.UnpackConfiguration, len(auc))
for i := range auc {
uc[i].Snapshotter = auc[i].Snapshotter
if auc[i].Platform != nil {

41
pkg/transfer/local/import.go
View File

@ -19,13 +19,16 @@ package local
import (
"context"
"encoding/json"
"fmt"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/transfer"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/containerd/containerd/pkg/unpack"
)
func (ts *localTransferService) importStream(ctx context.Context, i transfer.ImageImporter, is transfer.ImageStorer, tops *transfer.Config) error {
@ -46,12 +49,16 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
return err
}
var descriptors []ocispec.Descriptor
var (
descriptors []ocispec.Descriptor
handler images.Handler
unpacker *unpack.Unpacker
)
// If save index, add index
descriptors = append(descriptors, index)
var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
var handlerFunc images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
// Only save images at top level
if desc.Digest != index.Digest {
return images.Children(ctx, ts.content, desc)
@ -76,7 +83,33 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
}
if f, ok := is.(transfer.ImageFilterer); ok {
handler = f.ImageFilter(handler, ts.content)
handlerFunc = f.ImageFilter(handlerFunc, ts.content)
}
handler = images.Handlers(handlerFunc)
// First find suitable platforms to unpack into
//If image storer is also an unpacker type, i.e implemented UnpackPlatforms() func
if iu, ok := is.(transfer.ImageUnpacker); ok {
unpacks := iu.UnpackPlatforms()
if len(unpacks) > 0 {
uopts := []unpack.UnpackerOpt{}
for _, u := range unpacks {
matched, mu := getSupportedPlatform(u, ts.config.UnpackPlatforms)
if matched {
uopts = append(uopts, unpack.WithUnpackPlatform(mu))
}
}
if ts.config.DuplicationSuppressor != nil {
uopts = append(uopts, unpack.WithDuplicationSuppressor(ts.config.DuplicationSuppressor))
}
unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...)
if err != nil {
return fmt.Errorf("unable to initialize unpacker: %w", err)
}
handler = unpacker.Unpack(handler)
}
}
if err := images.WalkNotEmpty(ctx, handler, index); err != nil {

61
pkg/transfer/local/pull.go
View File

@ -20,6 +20,7 @@ import (
"context"
"fmt"
"github.com/containerd/containerd"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
@ -73,6 +74,8 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
var (
handler images.Handler
baseHandlers []images.Handler
unpacker *unpack.Unpacker
// has a config media type bug (distribution#1622)
@ -97,12 +100,6 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
childrenHandler = f.ImageFilter(childrenHandler, store)
}
// Sort and limit manifests if a finite number is needed
//if limit > 0 {
// childrenHandler = images.LimitManifests(childrenHandler, rCtx.PlatformMatcher, limit)
//}
//SetChildrenMappedLabels(manager content.Manager, f HandlerFunc, labelMap func(ocispec.Descriptor) []string) HandlerFunc {
checkNeedsFix := images.HandlerFunc(
func(_ context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
// set to true if there is application/octet-stream media type
@ -119,8 +116,12 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
return err
}
// TODO: Allow initialization from configuration
baseHandlers := []images.Handler{}
//Set up baseHandlers from service configuration if present or create a new one
if ts.config.BaseHandlers != nil {
baseHandlers = ts.config.BaseHandlers
} else {
baseHandlers = []images.Handler{}
}
if tops.Progress != nil {
baseHandlers = append(baseHandlers, images.HandlerFunc(
@ -149,22 +150,28 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
appendDistSrcLabelHandler,
)...)
// TODO: Should available platforms be a configuration of the service?
// First find suitable platforms to unpack into
//if unpacker, ok := is.
//If image storer is also an unpacker type, i.e implemented UnpackPlatforms() func
if iu, ok := is.(transfer.ImageUnpacker); ok {
unpacks := iu.UnpackPlatforms()
if len(unpacks) > 0 {
uopts := []unpack.UnpackerOpt{}
//Only unpack if requested unpackconfig matches default/supported unpackconfigs
for _, u := range unpacks {
uopts = append(uopts, unpack.WithUnpackPlatform(u))
matched, mu := getSupportedPlatform(u, ts.config.UnpackPlatforms)
if matched {
uopts = append(uopts, unpack.WithUnpackPlatform(mu))
}
if ts.limiter != nil {
uopts = append(uopts, unpack.WithLimiter(ts.limiter))
}
//if uconfig.DuplicationSuppressor != nil {
// uopts = append(uopts, unpack.WithDuplicationSuppressor(uconfig.DuplicationSuppressor))
//}
if ts.limiterD != nil {
uopts = append(uopts, unpack.WithLimiter(ts.limiterD))
}
if ts.config.DuplicationSuppressor != nil {
uopts = append(uopts, unpack.WithDuplicationSuppressor(ts.config.DuplicationSuppressor))
}
unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...)
if err != nil {
return fmt.Errorf("unable to initialize unpacker: %w", err)
@ -173,7 +180,7 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
}
}
if err := images.Dispatch(ctx, handler, ts.limiter, desc); err != nil {
if err := images.Dispatch(ctx, handler, ts.limiterD, desc); err != nil {
if unpacker != nil {
// wait for unpacker to cleanup
unpacker.Wait()
@ -241,3 +248,23 @@ func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher, pt *Progre
}
}
}
// getSupportedPlatform returns a matched platform comparing input UnpackConfiguration to the supported platform/snapshotter combinations
// If input platform didn't specify snapshotter, default will be used if there is a match on platform.
func getSupportedPlatform(uc transfer.UnpackConfiguration, supportedPlatforms []unpack.Platform) (bool, unpack.Platform) {
var u unpack.Platform
for _, sp := range supportedPlatforms {
//If both platform and snapshotter match, return the supportPlatform
//If platform matched and SnapshotterKey is empty, we assume client didn't pass SnapshotterKey
//use default Snapshotter
if sp.Platform.Match(uc.Platform) {
//assuming sp.SnapshotterKey is not empty
if uc.Snapshotter == sp.SnapshotterKey {
return true, sp
} else if uc.Snapshotter == "" && sp.SnapshotterKey == containerd.DefaultSnapshotter {
return true, sp
}
}
}
return false, u
}

3
pkg/transfer/local/push.go
View File

@ -105,8 +105,7 @@ func (ts *localTransferService) push(ctx context.Context, ig transfer.ImageGette
wrapper = pushCtx.HandlerWrapper
}
*/
if err := remotes.PushContent(ctx, pusher, img.Target, ts.content, ts.limiter, matcher, wrapper); err != nil {
if err := remotes.PushContent(ctx, pusher, img.Target, ts.content, ts.limiterU, matcher, wrapper); err != nil {
return err
}
if tops.Progress != nil {

63
pkg/transfer/local/transfer.go
View File

@ -22,11 +22,15 @@ import (
"io"
"time"
"github.com/containerd/containerd"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/leases"
"github.com/containerd/containerd/pkg/kmutex"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/unpack"
"github.com/containerd/containerd/platforms"
"github.com/containerd/typeurl/v2"
"golang.org/x/sync/semaphore"
)
@ -35,25 +39,21 @@ type localTransferService struct {
leases leases.Manager
content content.Store
images images.Store
// semaphore.NewWeighted(int64(rCtx.MaxConcurrentDownloads))
limiter *semaphore.Weighted
// TODO: Duplication suppressor
// Configuration
// - Max downloads
// - Max uploads
// Supported platforms
// - Platform -> snapshotter defaults?
//limiter for upload
limiterU *semaphore.Weighted
//limiter for download operation
limiterD *semaphore.Weighted
config TransferConfig
}
func NewTransferService(lm leases.Manager, cs content.Store, is images.Store) transfer.Transferrer {
func NewTransferService(lm leases.Manager, cs content.Store, is images.Store, tc *TransferConfig) transfer.Transferrer {
return &localTransferService{
leases: lm,
content: cs,
images: is,
limiterU: semaphore.NewWeighted(int64(tc.MaxConcurrentUploadedLayers)),
limiterD: semaphore.NewWeighted(int64(tc.MaxConcurrentDownloads)),
config: *tc,
}
}
@ -150,3 +150,40 @@ func (ts *localTransferService) withLease(ctx context.Context, opts ...leases.Op
return ls.Delete(ctx, l)
}, nil
}
type TransferConfig struct {
// MaxConcurrentDownloads is the max concurrent content downloads for pull.
MaxConcurrentDownloads int `toml:"max_concurrent_downloads"`
// MaxConcurrentUploadedLayers is the max concurrent uploads for push
MaxConcurrentUploadedLayers int `toml:"max_concurrent_uploaded_layers"`
// DuplicationSuppressor is used to make sure that there is only one
// in-flight fetch request or unpack handler for a given descriptor's
// digest or chain ID.
DuplicationSuppressor kmutex.KeyedLocker
// BaseHandlers are a set of handlers which get are called on dispatch.
// These handlers always get called before any operation specific
// handlers.
BaseHandlers []images.Handler
//UnpackPlatforms are used to specify supported combination of platforms and snapshotters
UnpackPlatforms []unpack.Platform `toml:"unpack_platforms"`
// RegistryConfigPath is a path to the root directory containing registry-specific configurations
RegistryConfigPath string `toml:"config_path"`
}
func DefaultConfig() *TransferConfig {
return &TransferConfig{
MaxConcurrentDownloads: 3,
MaxConcurrentUploadedLayers: 3,
UnpackPlatforms: []unpack.Platform{
{
Platform: platforms.Only(platforms.DefaultSpec()),
SnapshotterKey: containerd.DefaultSnapshotter,
},
},
}
}

5
pkg/transfer/proxy/transfer.go
View File

@ -21,6 +21,8 @@ import (
"errors"
"io"
"google.golang.org/protobuf/types/known/anypb"
transferapi "github.com/containerd/containerd/api/services/transfer/v1"
transfertypes "github.com/containerd/containerd/api/types/transfer"
"github.com/containerd/containerd/log"
@ -28,7 +30,6 @@ import (
"github.com/containerd/containerd/pkg/transfer"
tstreaming "github.com/containerd/containerd/pkg/transfer/streaming"
"github.com/containerd/typeurl/v2"
"google.golang.org/protobuf/types/known/anypb"
)
type proxyTransferrer struct {
@ -36,7 +37,7 @@ type proxyTransferrer struct {
streamCreator streaming.StreamCreator
}
// NewTransferrer returns a new transferr which communicates over a GRPC
// NewTransferrer returns a new transferrer which communicates over a GRPC
// connection using the containerd transfer API
func NewTransferrer(client transferapi.TransferClient, sc streaming.StreamCreator) transfer.Transferrer {
return &proxyTransferrer{

2
pkg/transfer/image/registry.go → pkg/transfer/registry/registry.go
View File

@ -14,7 +14,7 @@
limitations under the License.
*/
package image
package registry
import (
"context"

15
pkg/transfer/transfer.go
View File

@ -20,10 +20,10 @@ import (
"context"
"io"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/pkg/unpack"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type Transferrer interface {
@ -86,8 +86,15 @@ type ImageExportStreamer interface {
}
type ImageUnpacker interface {
// TODO: consider using unpack options
UnpackPlatforms() []unpack.Platform
UnpackPlatforms() []UnpackConfiguration
}
// UnpackConfiguration specifies the platform and snapshotter to use for resolving
// the unpack Platform, if snapshotter is not specified the platform default will
// be used.
type UnpackConfiguration struct {
Platform ocispec.Platform
Snapshotter string
}
type ProgressFunc func(Progress)

11
plugins/transfer/plugin.go
View File

@ -27,6 +27,7 @@ import (
_ "github.com/containerd/containerd/pkg/transfer/image"
)
// Register local transfer service plugin
func init() {
plugin.Register(&plugin.Registration{
Type: plugin.TransferPlugin,
@ -35,8 +36,9 @@ func init() {
plugin.LeasePlugin,
plugin.MetadataPlugin,
},
Config: &transferConfig{},
Config: local.DefaultConfig(),
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
config := ic.Config.(*local.TransferConfig)
m, err := ic.Get(plugin.MetadataPlugin)
if err != nil {
return nil, err
@ -47,12 +49,7 @@ func init() {
return nil, err
}
return local.NewTransferService(l.(leases.Manager), ms.ContentStore(), metadata.NewImageStore(ms)), nil
return local.NewTransferService(l.(leases.Manager), ms.ContentStore(), metadata.NewImageStore(ms), config), nil
},
})
}
type transferConfig struct {
// Max concurrent downloads
// Snapshotter platforms
}

2
remotes/resolver.go
View File

@ -34,7 +34,7 @@ type Resolver interface {
// reference a specific host or be matched against a specific handler.
//
// The returned name should be used to identify the referenced entity.
// Dependending on the remote namespace, this may be immutable or mutable.
// Depending on the remote namespace, this may be immutable or mutable.
// While the name may differ from ref, it should itself be a valid ref.
//
// If the resolution fails, an error will be returned.