diff --git a/client.go b/client.go index 55c1a36fd..370d4b89f 100644 --- a/client.go +++ b/client.go @@ -45,6 +45,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/pkg/dialer" + "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" @@ -284,7 +285,31 @@ func defaultRemoteContext() *RemoteContext { } } +// Fetch downloads the provided content into containerd's content store +// and returns a non-platform specific image reference +func (c *Client) Fetch(ctx context.Context, ref string, opts ...RemoteOpt) (images.Image, error) { + fetchCtx := defaultRemoteContext() + for _, o := range opts { + if err := o(c, fetchCtx); err != nil { + return images.Image{}, err + } + } + + if fetchCtx.Unpack { + return images.Image{}, errors.New("unpack on fetch not supported, try pull") + } + + ctx, done, err := c.WithLease(ctx) + if err != nil { + return images.Image{}, err + } + defer done(ctx) + + return c.fetch(ctx, fetchCtx, ref) +} + // Pull downloads the provided content into containerd's content store +// and returns a platform specific image object func (c *Client) Pull(ctx context.Context, ref string, opts ...RemoteOpt) (Image, error) { pullCtx := defaultRemoteContext() for _, o := range opts { @@ -292,7 +317,12 @@ func (c *Client) Pull(ctx context.Context, ref string, opts ...RemoteOpt) (Image return nil, err } } - store := c.ContentStore() + + if len(pullCtx.Platforms) > 1 { + return nil, errors.New("cannot pull multiplatform image locally, try Fetch") + } else if len(pullCtx.Platforms) == 0 { + pullCtx.Platforms = []string{platforms.Default()} + } ctx, done, err := c.WithLease(ctx) if err != nil { @@ -300,82 +330,92 @@ func (c *Client) Pull(ctx context.Context, ref string, opts ...RemoteOpt) (Image } defer done(ctx) - name, desc, err := pullCtx.Resolver.Resolve(ctx, ref) + img, err := c.fetch(ctx, pullCtx, ref) if err != nil { - return nil, errors.Wrapf(err, "failed to resolve reference %q", ref) + return nil, err } - fetcher, err := pullCtx.Resolver.Fetcher(ctx, name) + i := NewImageWithPlatform(c, img, pullCtx.Platforms[0]) + + if pullCtx.Unpack { + if err := i.Unpack(ctx, pullCtx.Snapshotter); err != nil { + return nil, errors.Wrapf(err, "failed to unpack image on snapshotter %s", pullCtx.Snapshotter) + } + } + + return i, nil +} + +func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string) (images.Image, error) { + store := c.ContentStore() + name, desc, err := rCtx.Resolver.Resolve(ctx, ref) if err != nil { - return nil, errors.Wrapf(err, "failed to get fetcher for %q", name) + return images.Image{}, errors.Wrapf(err, "failed to resolve reference %q", ref) + } + + fetcher, err := rCtx.Resolver.Fetcher(ctx, name) + if err != nil { + return images.Image{}, errors.Wrapf(err, "failed to get fetcher for %q", name) } var ( schema1Converter *schema1.Converter handler images.Handler ) - if desc.MediaType == images.MediaTypeDockerSchema1Manifest && pullCtx.ConvertSchema1 { + if desc.MediaType == images.MediaTypeDockerSchema1Manifest && rCtx.ConvertSchema1 { schema1Converter = schema1.NewConverter(store, fetcher) - handler = images.Handlers(append(pullCtx.BaseHandlers, schema1Converter)...) + handler = images.Handlers(append(rCtx.BaseHandlers, schema1Converter)...) } else { // Get all the children for a descriptor childrenHandler := images.ChildrenHandler(store) // Set any children labels for that content childrenHandler = images.SetChildrenLabels(store, childrenHandler) // Filter children by platforms - childrenHandler = images.FilterPlatforms(childrenHandler, pullCtx.Platforms...) + childrenHandler = images.FilterPlatforms(childrenHandler, rCtx.Platforms...) - handler = images.Handlers(append(pullCtx.BaseHandlers, + handler = images.Handlers(append(rCtx.BaseHandlers, remotes.FetchHandler(store, fetcher), childrenHandler, )...) } if err := images.Dispatch(ctx, handler, desc); err != nil { - return nil, err + return images.Image{}, err } if schema1Converter != nil { desc, err = schema1Converter.Convert(ctx) if err != nil { - return nil, err + return images.Image{}, err } } - img := &image{ - client: c, - i: images.Image{ - Name: name, - Target: desc, - Labels: pullCtx.Labels, - }, - } - - if pullCtx.Unpack { - if err := img.Unpack(ctx, pullCtx.Snapshotter); err != nil { - return nil, errors.Wrapf(err, "failed to unpack image on snapshotter %s", pullCtx.Snapshotter) - } + img := images.Image{ + Name: name, + Target: desc, + Labels: rCtx.Labels, } is := c.ImageService() for { - if created, err := is.Create(ctx, img.i); err != nil { + if created, err := is.Create(ctx, img); err != nil { if !errdefs.IsAlreadyExists(err) { - return nil, err + return images.Image{}, err } - updated, err := is.Update(ctx, img.i) + updated, err := is.Update(ctx, img) if err != nil { // if image was removed, try create again if errdefs.IsNotFound(err) { continue } - return nil, err + return images.Image{}, err } - img.i = updated + img = updated } else { - img.i = created + img = created } + return img, nil } } diff --git a/client_test.go b/client_test.go index 428e34086..eb5de7735 100644 --- a/client_test.go +++ b/client_test.go @@ -111,7 +111,7 @@ func TestMain(m *testing.M) { }).Info("running tests against containerd") // pull a seed image - if _, err = client.Pull(ctx, testImage, WithPullUnpack, WithPlatform(platforms.Default())); err != nil { + if _, err = client.Pull(ctx, testImage, WithPullUnpack); err != nil { ctrd.Stop() ctrd.Wait() fmt.Fprintf(os.Stderr, "%s: %s\n", err, buf.String()) @@ -198,11 +198,11 @@ func TestImagePullAllPlatforms(t *testing.T) { defer cancel() cs := client.ContentStore() - img, err := client.Pull(ctx, testImage) + img, err := client.Fetch(ctx, testImage) if err != nil { t.Fatal(err) } - index := img.Target() + index := img.Target manifests, err := images.Children(ctx, cs, index) if err != nil { t.Fatal(err) @@ -246,12 +246,12 @@ func TestImagePullSomePlatforms(t *testing.T) { opts = append(opts, WithPlatform(platform)) } - img, err := client.Pull(ctx, "docker.io/library/busybox:latest", opts...) + img, err := client.Fetch(ctx, "docker.io/library/busybox:latest", opts...) if err != nil { t.Fatal(err) } - index := img.Target() + index := img.Target manifests, err := images.Children(ctx, cs, index) if err != nil { t.Fatal(err) diff --git a/cmd/ctr/commands/content/fetch.go b/cmd/ctr/commands/content/fetch.go index 514010dd8..4169bd620 100644 --- a/cmd/ctr/commands/content/fetch.go +++ b/cmd/ctr/commands/content/fetch.go @@ -32,6 +32,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/log" "github.com/containerd/containerd/pkg/progress" + "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -56,7 +57,16 @@ not use this implementation as a guide. The end goal should be having metadata, content and snapshots ready for a direct use via the 'ctr run'. Most of this is experimental and there are few leaps to make this work.`, - Flags: append(commands.RegistryFlags, commands.LabelFlag), + Flags: append(commands.RegistryFlags, commands.LabelFlag, + cli.StringSliceFlag{ + Name: "platform", + Usage: "Pull content from a specific platform", + }, + cli.BoolFlag{ + Name: "all-platforms", + Usage: "pull content from all platforms", + }, + ), Action: func(clicontext *cli.Context) error { var ( ref = clicontext.Args().First() @@ -73,10 +83,10 @@ Most of this is experimental and there are few leaps to make this work.`, } // Fetch loads all resources into the content store and returns the image -func Fetch(ctx context.Context, client *containerd.Client, ref string, cliContext *cli.Context) (containerd.Image, error) { +func Fetch(ctx context.Context, client *containerd.Client, ref string, cliContext *cli.Context) (images.Image, error) { resolver, err := commands.GetResolver(ctx, cliContext) if err != nil { - return nil, err + return images.Image{}, err } ongoing := newJobs(ref) @@ -109,15 +119,19 @@ func Fetch(ctx context.Context, client *containerd.Client, ref string, cliContex } if !cliContext.Bool("all-platforms") { - for _, platform := range cliContext.StringSlice("platform") { + p := cliContext.StringSlice("platform") + if len(p) == 0 { + p = append(p, platforms.Default()) + } + for _, platform := range p { opts = append(opts, containerd.WithPlatform(platform)) } } - img, err := client.Pull(pctx, ref, opts...) + img, err := client.Fetch(pctx, ref, opts...) stopProgress() if err != nil { - return nil, err + return images.Image{}, err } <-progress diff --git a/cmd/ctr/commands/images/pull.go b/cmd/ctr/commands/images/pull.go index 10e4f1d52..de99ae4a6 100644 --- a/cmd/ctr/commands/images/pull.go +++ b/cmd/ctr/commands/images/pull.go @@ -19,10 +19,13 @@ package images import ( "fmt" + "github.com/containerd/containerd" "github.com/containerd/containerd/cmd/ctr/commands" "github.com/containerd/containerd/cmd/ctr/commands/content" + "github.com/containerd/containerd/images" "github.com/containerd/containerd/log" "github.com/containerd/containerd/platforms" + "github.com/pkg/errors" "github.com/urfave/cli" ) @@ -43,7 +46,7 @@ command. As part of this process, we do the following: cli.StringSliceFlag{ Name: "platform", Usage: "Pull content from a specific platform", - Value: &cli.StringSlice{platforms.Default()}, + Value: &cli.StringSlice{}, }, cli.BoolFlag{ Name: "all-platforms", @@ -78,11 +81,34 @@ command. As part of this process, we do the following: log.G(ctx).WithField("image", ref).Debug("unpacking") // TODO: Show unpack status - fmt.Printf("unpacking %s...\n", img.Target().Digest) - err = img.Unpack(ctx, context.String("snapshotter")) - if err == nil { - fmt.Println("done") + + var p []string + if context.Bool("all-platforms") { + all, err := images.Platforms(ctx, client.ContentStore(), img.Target) + if err != nil { + return errors.Wrap(err, "unable to resolve image platforms") + } + p = make([]string, len(all)) + for i := range all { + p[i] = platforms.Format(all[i]) + } + } else { + p = context.StringSlice("platform") } - return err + if len(p) == 0 { + p = append(p, platforms.Default()) + } + + for _, platform := range p { + fmt.Printf("unpacking %s %s...\n", platform, img.Target.Digest) + i := containerd.NewImageWithPlatform(client, img, platform) + err = i.Unpack(ctx, context.String("snapshotter")) + if err != nil { + return err + } + } + + fmt.Println("done") + return nil }, } diff --git a/export_test.go b/export_test.go index ad4d6ca96..d947c1830 100644 --- a/export_test.go +++ b/export_test.go @@ -40,11 +40,11 @@ func TestOCIExport(t *testing.T) { } defer client.Close() - pulled, err := client.Pull(ctx, testImage) + pulled, err := client.Fetch(ctx, testImage) if err != nil { t.Fatal(err) } - exportedStream, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target()) + exportedStream, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target) if err != nil { t.Fatal(err) } diff --git a/image.go b/image.go index 0f2bb0da9..6e286fcaf 100644 --- a/image.go +++ b/image.go @@ -56,15 +56,26 @@ var _ = (Image)(&image{}) // NewImage returns a client image object from the metadata image func NewImage(client *Client, i images.Image) Image { return &image{ - client: client, - i: i, + client: client, + i: i, + platform: platforms.Default(), + } +} + +// NewImageWithPlatform returns a client image object from the metadata image +func NewImageWithPlatform(client *Client, i images.Image, platform string) Image { + return &image{ + client: client, + i: i, + platform: platform, } } type image struct { client *Client - i images.Image + i images.Image + platform string } func (i *image) Name() string { @@ -77,24 +88,24 @@ func (i *image) Target() ocispec.Descriptor { func (i *image) RootFS(ctx context.Context) ([]digest.Digest, error) { provider := i.client.ContentStore() - return i.i.RootFS(ctx, provider, platforms.Default()) + return i.i.RootFS(ctx, provider, i.platform) } func (i *image) Size(ctx context.Context) (int64, error) { provider := i.client.ContentStore() - return i.i.Size(ctx, provider, platforms.Default()) + return i.i.Size(ctx, provider, i.platform) } func (i *image) Config(ctx context.Context) (ocispec.Descriptor, error) { provider := i.client.ContentStore() - return i.i.Config(ctx, provider, platforms.Default()) + return i.i.Config(ctx, provider, i.platform) } func (i *image) IsUnpacked(ctx context.Context, snapshotterName string) (bool, error) { sn := i.client.SnapshotService(snapshotterName) cs := i.client.ContentStore() - diffs, err := i.i.RootFS(ctx, cs, platforms.Default()) + diffs, err := i.i.RootFS(ctx, cs, i.platform) if err != nil { return false, err } @@ -117,7 +128,7 @@ func (i *image) Unpack(ctx context.Context, snapshotterName string) error { } defer done(ctx) - layers, err := i.getLayers(ctx, platforms.Default()) + layers, err := i.getLayers(ctx, i.platform) if err != nil { return err } @@ -154,7 +165,7 @@ func (i *image) Unpack(ctx context.Context, snapshotterName string) error { } if unpacked { - desc, err := i.i.Config(ctx, cs, platforms.Default()) + desc, err := i.i.Config(ctx, cs, i.platform) if err != nil { return err } diff --git a/import_test.go b/import_test.go index d04903f09..62813ac0c 100644 --- a/import_test.go +++ b/import_test.go @@ -39,12 +39,12 @@ func TestOCIExportAndImport(t *testing.T) { } defer client.Close() - pulled, err := client.Pull(ctx, testImage) + pulled, err := client.Fetch(ctx, testImage) if err != nil { t.Fatal(err) } - exported, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target()) + exported, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target) if err != nil { t.Fatal(err) }