Add configuration options to local transfer service
Signed-off-by: Tony Fang <nhfang@amazon.com>
This commit is contained in:
parent
e366facb87
commit
47305392c6
@ -22,6 +22,9 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
|
"github.com/urfave/cli"
|
||||||
|
|
||||||
"github.com/containerd/containerd"
|
"github.com/containerd/containerd"
|
||||||
"github.com/containerd/containerd/cmd/ctr/commands"
|
"github.com/containerd/containerd/cmd/ctr/commands"
|
||||||
"github.com/containerd/containerd/images/archive"
|
"github.com/containerd/containerd/images/archive"
|
||||||
@ -30,7 +33,6 @@ import (
|
|||||||
tarchive "github.com/containerd/containerd/pkg/transfer/archive"
|
tarchive "github.com/containerd/containerd/pkg/transfer/archive"
|
||||||
"github.com/containerd/containerd/pkg/transfer/image"
|
"github.com/containerd/containerd/pkg/transfer/image"
|
||||||
"github.com/containerd/containerd/platforms"
|
"github.com/containerd/containerd/platforms"
|
||||||
"github.com/urfave/cli"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var importCommand = cli.Command{
|
var importCommand = cli.Command{
|
||||||
@ -127,9 +129,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
|
|||||||
opts = append(opts, image.WithNamedPrefix(prefix, overwrite))
|
opts = append(opts, image.WithNamedPrefix(prefix, overwrite))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Add platform options
|
var platSpec ocispec.Platform
|
||||||
|
//Only when all-platforms not specified, we will check platform value
|
||||||
|
//Implicitly if the platforms is empty, it means all-platforms
|
||||||
|
if !context.Bool("all-platforms") {
|
||||||
|
//If platform specified, use that one, if not use default
|
||||||
|
if platform := context.String("platform"); platform != "" {
|
||||||
|
platSpec, err = platforms.Parse(platform)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
platSpec = platforms.DefaultSpec()
|
||||||
|
}
|
||||||
|
opts = append(opts, image.WithPlatforms(platSpec))
|
||||||
|
}
|
||||||
|
|
||||||
// TODO: Add unpack options
|
if !context.Bool("no-unpack") {
|
||||||
|
snapshotter := context.String("snapshotter")
|
||||||
|
//If OS field is not empty, it means platSpec was updated in the above block
|
||||||
|
//i.e all-platforms was not specified
|
||||||
|
if platSpec.OS != "" {
|
||||||
|
opts = append(opts, image.WithUnpack(platSpec, snapshotter))
|
||||||
|
} else {
|
||||||
|
//empty spec means all platforms
|
||||||
|
var emptySpec ocispec.Platform
|
||||||
|
opts = append(opts, image.WithUnpack(emptySpec, snapshotter))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
is := image.NewStore(context.String("index-name"), opts...)
|
is := image.NewStore(context.String("index-name"), opts...)
|
||||||
|
|
||||||
|
@ -32,6 +32,7 @@ import (
|
|||||||
"github.com/containerd/containerd/pkg/progress"
|
"github.com/containerd/containerd/pkg/progress"
|
||||||
"github.com/containerd/containerd/pkg/transfer"
|
"github.com/containerd/containerd/pkg/transfer"
|
||||||
"github.com/containerd/containerd/pkg/transfer/image"
|
"github.com/containerd/containerd/pkg/transfer/image"
|
||||||
|
"github.com/containerd/containerd/pkg/transfer/registry"
|
||||||
"github.com/containerd/containerd/platforms"
|
"github.com/containerd/containerd/platforms"
|
||||||
"github.com/opencontainers/image-spec/identity"
|
"github.com/opencontainers/image-spec/identity"
|
||||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
@ -112,6 +113,11 @@ command. As part of this process, we do the following:
|
|||||||
p = append(p, platforms.DefaultSpec())
|
p = append(p, platforms.DefaultSpec())
|
||||||
}
|
}
|
||||||
sopts = append(sopts, image.WithPlatforms(p...))
|
sopts = append(sopts, image.WithPlatforms(p...))
|
||||||
|
|
||||||
|
//set unpack configuration
|
||||||
|
for _, platform := range p {
|
||||||
|
sopts = append(sopts, image.WithUnpack(platform, context.String("snapshotter")))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// TODO: Support unpack for all platforms..?
|
// TODO: Support unpack for all platforms..?
|
||||||
// Pass in a *?
|
// Pass in a *?
|
||||||
@ -125,7 +131,7 @@ command. As part of this process, we do the following:
|
|||||||
sopts = append(sopts, image.WithAllMetadata)
|
sopts = append(sopts, image.WithAllMetadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
reg := image.NewOCIRegistry(ref, nil, ch)
|
reg := registry.NewOCIRegistry(ref, nil, ch)
|
||||||
is := image.NewStore(ref, sopts...)
|
is := image.NewStore(ref, sopts...)
|
||||||
|
|
||||||
pf, done := ProgressHandler(ctx, os.Stdout)
|
pf, done := ProgressHandler(ctx, os.Stdout)
|
||||||
|
@ -34,6 +34,7 @@ import (
|
|||||||
"github.com/containerd/containerd/pkg/progress"
|
"github.com/containerd/containerd/pkg/progress"
|
||||||
"github.com/containerd/containerd/pkg/transfer"
|
"github.com/containerd/containerd/pkg/transfer"
|
||||||
"github.com/containerd/containerd/pkg/transfer/image"
|
"github.com/containerd/containerd/pkg/transfer/image"
|
||||||
|
"github.com/containerd/containerd/pkg/transfer/registry"
|
||||||
"github.com/containerd/containerd/platforms"
|
"github.com/containerd/containerd/platforms"
|
||||||
"github.com/containerd/containerd/remotes"
|
"github.com/containerd/containerd/remotes"
|
||||||
"github.com/containerd/containerd/remotes/docker"
|
"github.com/containerd/containerd/remotes/docker"
|
||||||
@ -103,7 +104,7 @@ var pushCommand = cli.Command{
|
|||||||
if local == "" {
|
if local == "" {
|
||||||
local = ref
|
local = ref
|
||||||
}
|
}
|
||||||
reg := image.NewOCIRegistry(ref, nil, ch)
|
reg := registry.NewOCIRegistry(ref, nil, ch)
|
||||||
is := image.NewStore(local)
|
is := image.NewStore(local)
|
||||||
|
|
||||||
pf, done := ProgressHandler(ctx, os.Stdout)
|
pf, done := ProgressHandler(ctx, os.Stdout)
|
||||||
|
@ -32,7 +32,7 @@ import (
|
|||||||
|
|
||||||
"github.com/containerd/console"
|
"github.com/containerd/console"
|
||||||
"github.com/containerd/containerd/log"
|
"github.com/containerd/containerd/log"
|
||||||
"github.com/containerd/containerd/pkg/transfer/image"
|
"github.com/containerd/containerd/pkg/transfer/registry"
|
||||||
"github.com/containerd/containerd/remotes"
|
"github.com/containerd/containerd/remotes"
|
||||||
"github.com/containerd/containerd/remotes/docker"
|
"github.com/containerd/containerd/remotes/docker"
|
||||||
"github.com/containerd/containerd/remotes/docker/config"
|
"github.com/containerd/containerd/remotes/docker/config"
|
||||||
@ -218,7 +218,7 @@ type staticCredentials struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewStaticCredentials gets credentials from passing in cli context
|
// NewStaticCredentials gets credentials from passing in cli context
|
||||||
func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (image.CredentialHelper, error) {
|
func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (registry.CredentialHelper, error) {
|
||||||
username := clicontext.String("user")
|
username := clicontext.String("user")
|
||||||
var secret string
|
var secret string
|
||||||
if i := strings.IndexByte(username, ':'); i > 0 {
|
if i := strings.IndexByte(username, ':'); i > 0 {
|
||||||
@ -248,12 +248,12 @@ func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref st
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (image.Credentials, error) {
|
func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (registry.Credentials, error) {
|
||||||
if ref == sc.ref {
|
if ref == sc.ref {
|
||||||
return image.Credentials{
|
return registry.Credentials{
|
||||||
Username: sc.username,
|
Username: sc.username,
|
||||||
Secret: sc.secret,
|
Secret: sc.secret,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
return image.Credentials{}, nil
|
return registry.Credentials{}, nil
|
||||||
}
|
}
|
||||||
|
@ -27,8 +27,8 @@ import (
|
|||||||
"github.com/containerd/containerd/images"
|
"github.com/containerd/containerd/images"
|
||||||
"github.com/containerd/containerd/images/archive"
|
"github.com/containerd/containerd/images/archive"
|
||||||
"github.com/containerd/containerd/pkg/streaming"
|
"github.com/containerd/containerd/pkg/streaming"
|
||||||
|
"github.com/containerd/containerd/pkg/transfer"
|
||||||
"github.com/containerd/containerd/pkg/transfer/plugins"
|
"github.com/containerd/containerd/pkg/transfer/plugins"
|
||||||
"github.com/containerd/containerd/pkg/unpack"
|
|
||||||
"github.com/containerd/containerd/platforms"
|
"github.com/containerd/containerd/platforms"
|
||||||
"github.com/containerd/containerd/remotes"
|
"github.com/containerd/containerd/remotes"
|
||||||
"github.com/containerd/typeurl/v2"
|
"github.com/containerd/typeurl/v2"
|
||||||
@ -51,7 +51,7 @@ type Store struct {
|
|||||||
// extraReferences are used to store or lookup multiple references
|
// extraReferences are used to store or lookup multiple references
|
||||||
extraReferences []Reference
|
extraReferences []Reference
|
||||||
|
|
||||||
unpacks []UnpackConfiguration
|
unpacks []transfer.UnpackConfiguration
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reference is used to create or find a reference for an image
|
// Reference is used to create or find a reference for an image
|
||||||
@ -84,14 +84,6 @@ type Reference struct {
|
|||||||
SkipNamedDigest bool
|
SkipNamedDigest bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnpackConfiguration specifies the platform and snapshotter to use for resolving
|
|
||||||
// the unpack Platform, if snapshotter is not specified the platform default will
|
|
||||||
// be used.
|
|
||||||
type UnpackConfiguration struct {
|
|
||||||
Platform ocispec.Platform
|
|
||||||
Snapshotter string
|
|
||||||
}
|
|
||||||
|
|
||||||
// StoreOpt defines options when configuring an image store source or destination
|
// StoreOpt defines options when configuring an image store source or destination
|
||||||
type StoreOpt func(*Store)
|
type StoreOpt func(*Store)
|
||||||
|
|
||||||
@ -171,7 +163,7 @@ func WithExtraReference(name string) StoreOpt {
|
|||||||
// WithUnpack specifies a platform to unpack for and an optional snapshotter to use
|
// WithUnpack specifies a platform to unpack for and an optional snapshotter to use
|
||||||
func WithUnpack(p ocispec.Platform, snapshotter string) StoreOpt {
|
func WithUnpack(p ocispec.Platform, snapshotter string) StoreOpt {
|
||||||
return func(s *Store) {
|
return func(s *Store) {
|
||||||
s.unpacks = append(s.unpacks, UnpackConfiguration{
|
s.unpacks = append(s.unpacks, transfer.UnpackConfiguration{
|
||||||
Platform: p,
|
Platform: p,
|
||||||
Snapshotter: snapshotter,
|
Snapshotter: snapshotter,
|
||||||
})
|
})
|
||||||
@ -333,11 +325,11 @@ func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, err
|
|||||||
return store.Get(ctx, is.imageName)
|
return store.Get(ctx, is.imageName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (is *Store) UnpackPlatforms() []unpack.Platform {
|
func (is *Store) UnpackPlatforms() []transfer.UnpackConfiguration {
|
||||||
unpacks := make([]unpack.Platform, len(is.unpacks))
|
unpacks := make([]transfer.UnpackConfiguration, len(is.unpacks))
|
||||||
for i, uc := range is.unpacks {
|
for i, uc := range is.unpacks {
|
||||||
unpacks[i].SnapshotterKey = uc.Snapshotter
|
unpacks[i].Snapshotter = uc.Snapshotter
|
||||||
unpacks[i].Platform = platforms.Only(uc.Platform)
|
unpacks[i].Platform = uc.Platform
|
||||||
}
|
}
|
||||||
return unpacks
|
return unpacks
|
||||||
}
|
}
|
||||||
@ -424,7 +416,7 @@ func referencesFromProto(references []*transfertypes.ImageReference) []Reference
|
|||||||
}
|
}
|
||||||
return or
|
return or
|
||||||
}
|
}
|
||||||
func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguration {
|
func unpackToProto(uc []transfer.UnpackConfiguration) []*transfertypes.UnpackConfiguration {
|
||||||
auc := make([]*transfertypes.UnpackConfiguration, len(uc))
|
auc := make([]*transfertypes.UnpackConfiguration, len(uc))
|
||||||
for i := range uc {
|
for i := range uc {
|
||||||
p := types.Platform{
|
p := types.Platform{
|
||||||
@ -440,8 +432,8 @@ func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguratio
|
|||||||
return auc
|
return auc
|
||||||
}
|
}
|
||||||
|
|
||||||
func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []UnpackConfiguration {
|
func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []transfer.UnpackConfiguration {
|
||||||
uc := make([]UnpackConfiguration, len(auc))
|
uc := make([]transfer.UnpackConfiguration, len(auc))
|
||||||
for i := range auc {
|
for i := range auc {
|
||||||
uc[i].Snapshotter = auc[i].Snapshotter
|
uc[i].Snapshotter = auc[i].Snapshotter
|
||||||
if auc[i].Platform != nil {
|
if auc[i].Platform != nil {
|
||||||
|
@ -19,13 +19,16 @@ package local
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
|
|
||||||
"github.com/containerd/containerd/content"
|
"github.com/containerd/containerd/content"
|
||||||
"github.com/containerd/containerd/errdefs"
|
"github.com/containerd/containerd/errdefs"
|
||||||
"github.com/containerd/containerd/images"
|
"github.com/containerd/containerd/images"
|
||||||
"github.com/containerd/containerd/log"
|
"github.com/containerd/containerd/log"
|
||||||
"github.com/containerd/containerd/pkg/transfer"
|
"github.com/containerd/containerd/pkg/transfer"
|
||||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
"github.com/containerd/containerd/pkg/unpack"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (ts *localTransferService) importStream(ctx context.Context, i transfer.ImageImporter, is transfer.ImageStorer, tops *transfer.Config) error {
|
func (ts *localTransferService) importStream(ctx context.Context, i transfer.ImageImporter, is transfer.ImageStorer, tops *transfer.Config) error {
|
||||||
@ -46,12 +49,16 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var descriptors []ocispec.Descriptor
|
var (
|
||||||
|
descriptors []ocispec.Descriptor
|
||||||
|
handler images.Handler
|
||||||
|
unpacker *unpack.Unpacker
|
||||||
|
)
|
||||||
|
|
||||||
// If save index, add index
|
// If save index, add index
|
||||||
descriptors = append(descriptors, index)
|
descriptors = append(descriptors, index)
|
||||||
|
|
||||||
var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
var handlerFunc images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||||
// Only save images at top level
|
// Only save images at top level
|
||||||
if desc.Digest != index.Digest {
|
if desc.Digest != index.Digest {
|
||||||
return images.Children(ctx, ts.content, desc)
|
return images.Children(ctx, ts.content, desc)
|
||||||
@ -76,7 +83,33 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
|
|||||||
}
|
}
|
||||||
|
|
||||||
if f, ok := is.(transfer.ImageFilterer); ok {
|
if f, ok := is.(transfer.ImageFilterer); ok {
|
||||||
handler = f.ImageFilter(handler, ts.content)
|
handlerFunc = f.ImageFilter(handlerFunc, ts.content)
|
||||||
|
}
|
||||||
|
|
||||||
|
handler = images.Handlers(handlerFunc)
|
||||||
|
|
||||||
|
// First find suitable platforms to unpack into
|
||||||
|
//If image storer is also an unpacker type, i.e implemented UnpackPlatforms() func
|
||||||
|
if iu, ok := is.(transfer.ImageUnpacker); ok {
|
||||||
|
unpacks := iu.UnpackPlatforms()
|
||||||
|
if len(unpacks) > 0 {
|
||||||
|
uopts := []unpack.UnpackerOpt{}
|
||||||
|
for _, u := range unpacks {
|
||||||
|
matched, mu := getSupportedPlatform(u, ts.config.UnpackPlatforms)
|
||||||
|
if matched {
|
||||||
|
uopts = append(uopts, unpack.WithUnpackPlatform(mu))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ts.config.DuplicationSuppressor != nil {
|
||||||
|
uopts = append(uopts, unpack.WithDuplicationSuppressor(ts.config.DuplicationSuppressor))
|
||||||
|
}
|
||||||
|
unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to initialize unpacker: %w", err)
|
||||||
|
}
|
||||||
|
handler = unpacker.Unpack(handler)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := images.WalkNotEmpty(ctx, handler, index); err != nil {
|
if err := images.WalkNotEmpty(ctx, handler, index); err != nil {
|
||||||
|
@ -20,6 +20,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/containerd/containerd"
|
||||||
"github.com/containerd/containerd/content"
|
"github.com/containerd/containerd/content"
|
||||||
"github.com/containerd/containerd/errdefs"
|
"github.com/containerd/containerd/errdefs"
|
||||||
"github.com/containerd/containerd/images"
|
"github.com/containerd/containerd/images"
|
||||||
@ -73,6 +74,8 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
|
|||||||
var (
|
var (
|
||||||
handler images.Handler
|
handler images.Handler
|
||||||
|
|
||||||
|
baseHandlers []images.Handler
|
||||||
|
|
||||||
unpacker *unpack.Unpacker
|
unpacker *unpack.Unpacker
|
||||||
|
|
||||||
// has a config media type bug (distribution#1622)
|
// has a config media type bug (distribution#1622)
|
||||||
@ -97,12 +100,6 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
|
|||||||
childrenHandler = f.ImageFilter(childrenHandler, store)
|
childrenHandler = f.ImageFilter(childrenHandler, store)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sort and limit manifests if a finite number is needed
|
|
||||||
//if limit > 0 {
|
|
||||||
// childrenHandler = images.LimitManifests(childrenHandler, rCtx.PlatformMatcher, limit)
|
|
||||||
//}
|
|
||||||
//SetChildrenMappedLabels(manager content.Manager, f HandlerFunc, labelMap func(ocispec.Descriptor) []string) HandlerFunc {
|
|
||||||
|
|
||||||
checkNeedsFix := images.HandlerFunc(
|
checkNeedsFix := images.HandlerFunc(
|
||||||
func(_ context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
func(_ context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||||
// set to true if there is application/octet-stream media type
|
// set to true if there is application/octet-stream media type
|
||||||
@ -119,8 +116,12 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Allow initialization from configuration
|
//Set up baseHandlers from service configuration if present or create a new one
|
||||||
baseHandlers := []images.Handler{}
|
if ts.config.BaseHandlers != nil {
|
||||||
|
baseHandlers = ts.config.BaseHandlers
|
||||||
|
} else {
|
||||||
|
baseHandlers = []images.Handler{}
|
||||||
|
}
|
||||||
|
|
||||||
if tops.Progress != nil {
|
if tops.Progress != nil {
|
||||||
baseHandlers = append(baseHandlers, images.HandlerFunc(
|
baseHandlers = append(baseHandlers, images.HandlerFunc(
|
||||||
@ -149,22 +150,28 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
|
|||||||
appendDistSrcLabelHandler,
|
appendDistSrcLabelHandler,
|
||||||
)...)
|
)...)
|
||||||
|
|
||||||
// TODO: Should available platforms be a configuration of the service?
|
|
||||||
// First find suitable platforms to unpack into
|
// First find suitable platforms to unpack into
|
||||||
//if unpacker, ok := is.
|
//If image storer is also an unpacker type, i.e implemented UnpackPlatforms() func
|
||||||
if iu, ok := is.(transfer.ImageUnpacker); ok {
|
if iu, ok := is.(transfer.ImageUnpacker); ok {
|
||||||
unpacks := iu.UnpackPlatforms()
|
unpacks := iu.UnpackPlatforms()
|
||||||
if len(unpacks) > 0 {
|
if len(unpacks) > 0 {
|
||||||
uopts := []unpack.UnpackerOpt{}
|
uopts := []unpack.UnpackerOpt{}
|
||||||
|
//Only unpack if requested unpackconfig matches default/supported unpackconfigs
|
||||||
for _, u := range unpacks {
|
for _, u := range unpacks {
|
||||||
uopts = append(uopts, unpack.WithUnpackPlatform(u))
|
matched, mu := getSupportedPlatform(u, ts.config.UnpackPlatforms)
|
||||||
|
if matched {
|
||||||
|
uopts = append(uopts, unpack.WithUnpackPlatform(mu))
|
||||||
}
|
}
|
||||||
if ts.limiter != nil {
|
|
||||||
uopts = append(uopts, unpack.WithLimiter(ts.limiter))
|
|
||||||
}
|
}
|
||||||
//if uconfig.DuplicationSuppressor != nil {
|
|
||||||
// uopts = append(uopts, unpack.WithDuplicationSuppressor(uconfig.DuplicationSuppressor))
|
if ts.limiterD != nil {
|
||||||
//}
|
uopts = append(uopts, unpack.WithLimiter(ts.limiterD))
|
||||||
|
}
|
||||||
|
|
||||||
|
if ts.config.DuplicationSuppressor != nil {
|
||||||
|
uopts = append(uopts, unpack.WithDuplicationSuppressor(ts.config.DuplicationSuppressor))
|
||||||
|
}
|
||||||
|
|
||||||
unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...)
|
unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to initialize unpacker: %w", err)
|
return fmt.Errorf("unable to initialize unpacker: %w", err)
|
||||||
@ -173,7 +180,7 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := images.Dispatch(ctx, handler, ts.limiter, desc); err != nil {
|
if err := images.Dispatch(ctx, handler, ts.limiterD, desc); err != nil {
|
||||||
if unpacker != nil {
|
if unpacker != nil {
|
||||||
// wait for unpacker to cleanup
|
// wait for unpacker to cleanup
|
||||||
unpacker.Wait()
|
unpacker.Wait()
|
||||||
@ -241,3 +248,23 @@ func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher, pt *Progre
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getSupportedPlatform returns a matched platform comparing input UnpackConfiguration to the supported platform/snapshotter combinations
|
||||||
|
// If input platform didn't specify snapshotter, default will be used if there is a match on platform.
|
||||||
|
func getSupportedPlatform(uc transfer.UnpackConfiguration, supportedPlatforms []unpack.Platform) (bool, unpack.Platform) {
|
||||||
|
var u unpack.Platform
|
||||||
|
for _, sp := range supportedPlatforms {
|
||||||
|
//If both platform and snapshotter match, return the supportPlatform
|
||||||
|
//If platform matched and SnapshotterKey is empty, we assume client didn't pass SnapshotterKey
|
||||||
|
//use default Snapshotter
|
||||||
|
if sp.Platform.Match(uc.Platform) {
|
||||||
|
//assuming sp.SnapshotterKey is not empty
|
||||||
|
if uc.Snapshotter == sp.SnapshotterKey {
|
||||||
|
return true, sp
|
||||||
|
} else if uc.Snapshotter == "" && sp.SnapshotterKey == containerd.DefaultSnapshotter {
|
||||||
|
return true, sp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, u
|
||||||
|
}
|
||||||
|
@ -105,8 +105,7 @@ func (ts *localTransferService) push(ctx context.Context, ig transfer.ImageGette
|
|||||||
wrapper = pushCtx.HandlerWrapper
|
wrapper = pushCtx.HandlerWrapper
|
||||||
}
|
}
|
||||||
*/
|
*/
|
||||||
|
if err := remotes.PushContent(ctx, pusher, img.Target, ts.content, ts.limiterU, matcher, wrapper); err != nil {
|
||||||
if err := remotes.PushContent(ctx, pusher, img.Target, ts.content, ts.limiter, matcher, wrapper); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if tops.Progress != nil {
|
if tops.Progress != nil {
|
||||||
|
@ -22,11 +22,15 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/containerd/containerd"
|
||||||
"github.com/containerd/containerd/content"
|
"github.com/containerd/containerd/content"
|
||||||
"github.com/containerd/containerd/errdefs"
|
"github.com/containerd/containerd/errdefs"
|
||||||
"github.com/containerd/containerd/images"
|
"github.com/containerd/containerd/images"
|
||||||
"github.com/containerd/containerd/leases"
|
"github.com/containerd/containerd/leases"
|
||||||
|
"github.com/containerd/containerd/pkg/kmutex"
|
||||||
"github.com/containerd/containerd/pkg/transfer"
|
"github.com/containerd/containerd/pkg/transfer"
|
||||||
|
"github.com/containerd/containerd/pkg/unpack"
|
||||||
|
"github.com/containerd/containerd/platforms"
|
||||||
"github.com/containerd/typeurl/v2"
|
"github.com/containerd/typeurl/v2"
|
||||||
"golang.org/x/sync/semaphore"
|
"golang.org/x/sync/semaphore"
|
||||||
)
|
)
|
||||||
@ -35,25 +39,21 @@ type localTransferService struct {
|
|||||||
leases leases.Manager
|
leases leases.Manager
|
||||||
content content.Store
|
content content.Store
|
||||||
images images.Store
|
images images.Store
|
||||||
|
//limiter for upload
|
||||||
// semaphore.NewWeighted(int64(rCtx.MaxConcurrentDownloads))
|
limiterU *semaphore.Weighted
|
||||||
limiter *semaphore.Weighted
|
//limiter for download operation
|
||||||
|
limiterD *semaphore.Weighted
|
||||||
// TODO: Duplication suppressor
|
config TransferConfig
|
||||||
|
|
||||||
// Configuration
|
|
||||||
// - Max downloads
|
|
||||||
// - Max uploads
|
|
||||||
|
|
||||||
// Supported platforms
|
|
||||||
// - Platform -> snapshotter defaults?
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewTransferService(lm leases.Manager, cs content.Store, is images.Store) transfer.Transferrer {
|
func NewTransferService(lm leases.Manager, cs content.Store, is images.Store, tc *TransferConfig) transfer.Transferrer {
|
||||||
return &localTransferService{
|
return &localTransferService{
|
||||||
leases: lm,
|
leases: lm,
|
||||||
content: cs,
|
content: cs,
|
||||||
images: is,
|
images: is,
|
||||||
|
limiterU: semaphore.NewWeighted(int64(tc.MaxConcurrentUploadedLayers)),
|
||||||
|
limiterD: semaphore.NewWeighted(int64(tc.MaxConcurrentDownloads)),
|
||||||
|
config: *tc,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -150,3 +150,40 @@ func (ts *localTransferService) withLease(ctx context.Context, opts ...leases.Op
|
|||||||
return ls.Delete(ctx, l)
|
return ls.Delete(ctx, l)
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TransferConfig struct {
|
||||||
|
// MaxConcurrentDownloads is the max concurrent content downloads for pull.
|
||||||
|
MaxConcurrentDownloads int `toml:"max_concurrent_downloads"`
|
||||||
|
|
||||||
|
// MaxConcurrentUploadedLayers is the max concurrent uploads for push
|
||||||
|
MaxConcurrentUploadedLayers int `toml:"max_concurrent_uploaded_layers"`
|
||||||
|
|
||||||
|
// DuplicationSuppressor is used to make sure that there is only one
|
||||||
|
// in-flight fetch request or unpack handler for a given descriptor's
|
||||||
|
// digest or chain ID.
|
||||||
|
DuplicationSuppressor kmutex.KeyedLocker
|
||||||
|
|
||||||
|
// BaseHandlers are a set of handlers which get are called on dispatch.
|
||||||
|
// These handlers always get called before any operation specific
|
||||||
|
// handlers.
|
||||||
|
BaseHandlers []images.Handler
|
||||||
|
|
||||||
|
//UnpackPlatforms are used to specify supported combination of platforms and snapshotters
|
||||||
|
UnpackPlatforms []unpack.Platform `toml:"unpack_platforms"`
|
||||||
|
|
||||||
|
// RegistryConfigPath is a path to the root directory containing registry-specific configurations
|
||||||
|
RegistryConfigPath string `toml:"config_path"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func DefaultConfig() *TransferConfig {
|
||||||
|
return &TransferConfig{
|
||||||
|
MaxConcurrentDownloads: 3,
|
||||||
|
MaxConcurrentUploadedLayers: 3,
|
||||||
|
UnpackPlatforms: []unpack.Platform{
|
||||||
|
{
|
||||||
|
Platform: platforms.Only(platforms.DefaultSpec()),
|
||||||
|
SnapshotterKey: containerd.DefaultSnapshotter,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -21,6 +21,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
|
"google.golang.org/protobuf/types/known/anypb"
|
||||||
|
|
||||||
transferapi "github.com/containerd/containerd/api/services/transfer/v1"
|
transferapi "github.com/containerd/containerd/api/services/transfer/v1"
|
||||||
transfertypes "github.com/containerd/containerd/api/types/transfer"
|
transfertypes "github.com/containerd/containerd/api/types/transfer"
|
||||||
"github.com/containerd/containerd/log"
|
"github.com/containerd/containerd/log"
|
||||||
@ -28,7 +30,6 @@ import (
|
|||||||
"github.com/containerd/containerd/pkg/transfer"
|
"github.com/containerd/containerd/pkg/transfer"
|
||||||
tstreaming "github.com/containerd/containerd/pkg/transfer/streaming"
|
tstreaming "github.com/containerd/containerd/pkg/transfer/streaming"
|
||||||
"github.com/containerd/typeurl/v2"
|
"github.com/containerd/typeurl/v2"
|
||||||
"google.golang.org/protobuf/types/known/anypb"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type proxyTransferrer struct {
|
type proxyTransferrer struct {
|
||||||
@ -36,7 +37,7 @@ type proxyTransferrer struct {
|
|||||||
streamCreator streaming.StreamCreator
|
streamCreator streaming.StreamCreator
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTransferrer returns a new transferr which communicates over a GRPC
|
// NewTransferrer returns a new transferrer which communicates over a GRPC
|
||||||
// connection using the containerd transfer API
|
// connection using the containerd transfer API
|
||||||
func NewTransferrer(client transferapi.TransferClient, sc streaming.StreamCreator) transfer.Transferrer {
|
func NewTransferrer(client transferapi.TransferClient, sc streaming.StreamCreator) transfer.Transferrer {
|
||||||
return &proxyTransferrer{
|
return &proxyTransferrer{
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package image
|
package registry
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
@ -20,10 +20,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
|
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
|
|
||||||
"github.com/containerd/containerd/content"
|
"github.com/containerd/containerd/content"
|
||||||
"github.com/containerd/containerd/images"
|
"github.com/containerd/containerd/images"
|
||||||
"github.com/containerd/containerd/pkg/unpack"
|
|
||||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Transferrer interface {
|
type Transferrer interface {
|
||||||
@ -86,8 +86,15 @@ type ImageExportStreamer interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ImageUnpacker interface {
|
type ImageUnpacker interface {
|
||||||
// TODO: consider using unpack options
|
UnpackPlatforms() []UnpackConfiguration
|
||||||
UnpackPlatforms() []unpack.Platform
|
}
|
||||||
|
|
||||||
|
// UnpackConfiguration specifies the platform and snapshotter to use for resolving
|
||||||
|
// the unpack Platform, if snapshotter is not specified the platform default will
|
||||||
|
// be used.
|
||||||
|
type UnpackConfiguration struct {
|
||||||
|
Platform ocispec.Platform
|
||||||
|
Snapshotter string
|
||||||
}
|
}
|
||||||
|
|
||||||
type ProgressFunc func(Progress)
|
type ProgressFunc func(Progress)
|
||||||
|
@ -27,6 +27,7 @@ import (
|
|||||||
_ "github.com/containerd/containerd/pkg/transfer/image"
|
_ "github.com/containerd/containerd/pkg/transfer/image"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Register local transfer service plugin
|
||||||
func init() {
|
func init() {
|
||||||
plugin.Register(&plugin.Registration{
|
plugin.Register(&plugin.Registration{
|
||||||
Type: plugin.TransferPlugin,
|
Type: plugin.TransferPlugin,
|
||||||
@ -35,8 +36,9 @@ func init() {
|
|||||||
plugin.LeasePlugin,
|
plugin.LeasePlugin,
|
||||||
plugin.MetadataPlugin,
|
plugin.MetadataPlugin,
|
||||||
},
|
},
|
||||||
Config: &transferConfig{},
|
Config: local.DefaultConfig(),
|
||||||
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
|
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
|
||||||
|
config := ic.Config.(*local.TransferConfig)
|
||||||
m, err := ic.Get(plugin.MetadataPlugin)
|
m, err := ic.Get(plugin.MetadataPlugin)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -47,12 +49,7 @@ func init() {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return local.NewTransferService(l.(leases.Manager), ms.ContentStore(), metadata.NewImageStore(ms)), nil
|
return local.NewTransferService(l.(leases.Manager), ms.ContentStore(), metadata.NewImageStore(ms), config), nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
type transferConfig struct {
|
|
||||||
// Max concurrent downloads
|
|
||||||
// Snapshotter platforms
|
|
||||||
}
|
|
||||||
|
@ -34,7 +34,7 @@ type Resolver interface {
|
|||||||
// reference a specific host or be matched against a specific handler.
|
// reference a specific host or be matched against a specific handler.
|
||||||
//
|
//
|
||||||
// The returned name should be used to identify the referenced entity.
|
// The returned name should be used to identify the referenced entity.
|
||||||
// Dependending on the remote namespace, this may be immutable or mutable.
|
// Depending on the remote namespace, this may be immutable or mutable.
|
||||||
// While the name may differ from ref, it should itself be a valid ref.
|
// While the name may differ from ref, it should itself be a valid ref.
|
||||||
//
|
//
|
||||||
// If the resolution fails, an error will be returned.
|
// If the resolution fails, an error will be returned.
|
||||||
|
Loading…
Reference in New Issue
Block a user