From 61a7c4999c78e70f0be672c587feed501f9144f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 15 Dec 2023 16:31:42 +0100 Subject: [PATCH 1/3] import/export: Support references to missing content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow importing/exporting archives which doesn't have all the referenced blobs. This allows to export/import an image with only some of the platforms available locally while still persisting the full index. > The blobs directory MAY be missing referenced blobs, in which case the missing blobs SHOULD be fulfilled by an external blob store. https://github.com/opencontainers/image-spec/blob/v1.0/image-layout.md#blobs Signed-off-by: Paweł Gronowski --- client/import.go | 17 +++++++++++- content/helpers.go | 11 ++++++++ images/archive/exporter.go | 55 +++++++++++++++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/client/import.go b/client/import.go index 42aca54e7..79031f5c3 100644 --- a/client/import.go +++ b/client/import.go @@ -37,6 +37,7 @@ type importOpts struct { platformMatcher platforms.MatchComparer compress bool discardLayers bool + skipMissing bool } // ImportOpt allows the caller to specify import specific options @@ -113,6 +114,15 @@ func WithDiscardUnpackedLayers() ImportOpt { } } +// WithSkipMissing allows to import an archive which doesn't contain all the +// referenced blobs. +func WithSkipMissing() ImportOpt { + return func(c *importOpts) error { + c.skipMissing = true + return nil + } +} + // Import imports an image from a Tar stream using reader. // Caller needs to specify importer. Future version may use oci.v1 as the default. // Note that unreferenced blobs may be imported to the content store as well. @@ -162,7 +172,12 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { // Only save images at top level if desc.Digest != index.Digest { - return images.Children(ctx, cs, desc) + // Don't set labels on missing content. + children, err := images.Children(ctx, cs, desc) + if iopts.skipMissing && errdefs.IsNotFound(err) { + return nil, images.ErrSkipDesc + } + return children, err } idx, err := decodeIndex(ctx, cs, desc) diff --git a/content/helpers.go b/content/helpers.go index 180ff4848..1525c15e1 100644 --- a/content/helpers.go +++ b/content/helpers.go @@ -320,3 +320,14 @@ func copyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) { } return } + +// Exists returns whether an attempt to access the content would not error out +// with an ErrNotFound error. It will return an encountered error if it was +// different than ErrNotFound. +func Exists(ctx context.Context, provider InfoProvider, desc ocispec.Descriptor) (bool, error) { + _, err := provider.Info(ctx, desc.Digest) + if errdefs.IsNotFound(err) { + return false, nil + } + return err == nil, err +} diff --git a/images/archive/exporter.go b/images/archive/exporter.go index af1121772..87fac79e8 100644 --- a/images/archive/exporter.go +++ b/images/archive/exporter.go @@ -140,6 +140,45 @@ func WithSkipNonDistributableBlobs() ExportOpt { return WithBlobFilter(f) } +// WithSkipMissing excludes blobs referenced by manifests if not all blobs +// would be included in the archive. +// The manifest itself is excluded only if it's not present locally. +// This allows to export multi-platform images if not all platforms are present +// while still persisting the multi-platform index. +func WithSkipMissing(store ContentProvider) ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + o.blobRecordOptions.childrenHandler = images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) { + children, err := images.Children(ctx, store, desc) + if !images.IsManifestType(desc.MediaType) { + return children, err + } + + if err != nil { + // If manifest itself is missing, skip it from export. + if errdefs.IsNotFound(err) { + return nil, images.ErrSkipDesc + } + return nil, err + } + + // Don't export manifest descendants if any of them doesn't exist. + for _, child := range children { + exists, err := content.Exists(ctx, store, child) + if err != nil { + return nil, err + } + + // If any child is missing, only export the manifest, but don't export its descendants. + if !exists { + return nil, nil + } + } + return children, nil + }) + return nil + } +} + func addNameAnnotation(name string, base map[string]string) map[string]string { annotations := map[string]string{} for k, v := range base { @@ -152,8 +191,14 @@ func addNameAnnotation(name string, base map[string]string) map[string]string { return annotations } +// ContentProvider provides both content and info about content +type ContentProvider interface { + content.Provider + content.InfoProvider +} + // Export implements Exporter. -func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error { +func Export(ctx context.Context, store ContentProvider, writer io.Writer, opts ...ExportOpt) error { var eo exportOptions for _, opt := range opts { if err := opt(ctx, &eo); err != nil { @@ -291,7 +336,10 @@ func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descri return nil, nil } - childrenHandler := images.ChildrenHandler(store) + childrenHandler := brOpts.childrenHandler + if childrenHandler == nil { + childrenHandler = images.ChildrenHandler(store) + } handlers := images.Handlers( childrenHandler, @@ -313,7 +361,8 @@ type tarRecord struct { } type blobRecordOptions struct { - blobFilter BlobFilter + blobFilter BlobFilter + childrenHandler images.HandlerFunc } func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOptions) tarRecord { From b9af453f0ccc68d9ae37202712f1a9726d9cf5de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 15 Dec 2023 17:34:15 +0100 Subject: [PATCH 2/3] export: Copy distribution source labels to manifest annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persist manifest/indexes distribution source labels as annotations in the index.json. This could allow the importer to fetch the missing blobs from the external repository. These can't really be persisted directly in blob descriptors because that would alter the digests. Signed-off-by: Paweł Gronowski --- images/archive/exporter.go | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/images/archive/exporter.go b/images/archive/exporter.go index 87fac79e8..f343912a4 100644 --- a/images/archive/exporter.go +++ b/images/archive/exporter.go @@ -24,11 +24,14 @@ import ( "io" "path" "sort" + "strings" "github.com/containerd/containerd/v2/content" "github.com/containerd/containerd/v2/errdefs" "github.com/containerd/containerd/v2/images" + "github.com/containerd/containerd/v2/labels" "github.com/containerd/containerd/v2/platforms" + "github.com/containerd/log" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -191,6 +194,23 @@ func addNameAnnotation(name string, base map[string]string) map[string]string { return annotations } +func copySourceLabels(ctx context.Context, infoProvider content.InfoProvider, desc ocispec.Descriptor) (ocispec.Descriptor, error) { + info, err := infoProvider.Info(ctx, desc.Digest) + if err != nil { + return desc, err + } + for k, v := range info.Labels { + if strings.HasPrefix(k, labels.LabelDistributionSource) { + if desc.Annotations == nil { + desc.Annotations = map[string]string{k: v} + } else { + desc.Annotations[k] = v + } + } + } + return desc, nil +} + // ContentProvider provides both content and info about content type ContentProvider interface { content.Provider @@ -208,13 +228,22 @@ func Export(ctx context.Context, store ContentProvider, writer io.Writer, opts . records := []tarRecord{ ociLayoutFile(""), - ociIndexRecord(eo.manifests), + } + + manifests := make([]ocispec.Descriptor, 0, len(eo.manifests)) + for _, desc := range eo.manifests { + d, err := copySourceLabels(ctx, store, desc) + if err != nil { + log.G(ctx).WithError(err).WithField("desc", desc).Warn("failed to copy distribution.source labels") + continue + } + manifests = append(manifests, d) } algorithms := map[string]struct{}{} dManifests := map[digest.Digest]*exportManifest{} resolvedIndex := map[digest.Digest]digest.Digest{} - for _, desc := range eo.manifests { + for _, desc := range manifests { if images.IsManifestType(desc.MediaType) { mt, ok := dManifests[desc.Digest] if !ok { @@ -304,6 +333,8 @@ func Export(ctx context.Context, store ContentProvider, writer io.Writer, opts . } } + records = append(records, ociIndexRecord(manifests)) + if !eo.skipDockerManifest && len(dManifests) > 0 { tr, err := manifestsRecord(ctx, store, dManifests) if err != nil { From 0d3c3efe3b2bde4a0d38b3e22bc341ea7dcede99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 21 Dec 2023 14:41:55 +0100 Subject: [PATCH 3/3] integration/import-export: Add WithSkipMissing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also refactor tests to use the t.Run and run each test concurrently in a separate namespace. Signed-off-by: Paweł Gronowski --- integration/client/export_test.go | 388 +++++++++++++++++++++--------- integration/client/import_test.go | 171 +++++++++++-- 2 files changed, 426 insertions(+), 133 deletions(-) diff --git a/integration/client/export_test.go b/integration/client/export_test.go index 8a43c8e26..1abc3613e 100644 --- a/integration/client/export_test.go +++ b/integration/client/export_test.go @@ -19,144 +19,315 @@ package client import ( "archive/tar" "context" - "encoding/json" "io" "os" + "runtime" + "strings" "testing" . "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/content" + "github.com/containerd/containerd/v2/errdefs" "github.com/containerd/containerd/v2/images" "github.com/containerd/containerd/v2/images/archive" + "github.com/containerd/containerd/v2/namespaces" "github.com/containerd/containerd/v2/platforms" + "github.com/google/uuid" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// TestExport exports testImage as a tar stream -func TestExport(t *testing.T) { +func TestExportAllCases(t *testing.T) { if testing.Short() { t.Skip() } - ctx, cancel := testContext(t) - defer cancel() - client, err := New(address) - if err != nil { - t.Fatal(err) - } - defer client.Close() + for _, tc := range []struct { + name string + prepare func(context.Context, *testing.T, *Client) images.Image + check func(context.Context, *testing.T, *Client, *os.File, images.Image) + }{ + { + name: "export all platforms without SkipMissing", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + if runtime.GOOS == "windows" { + t.Skip("skipping test on windows - the testimage index has only one platform") + } + img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString()), WithAllMetadata()) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, _ images.Image) { + err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All)) + if !errdefs.IsNotFound(err) { + t.Fatal("should fail with not found error") + } + }, + }, + { + name: "export all platforms with SkipMissing", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + if runtime.GOOS == "windows" { + t.Skip("skipping test on windows - the testimage index has only one platform") + } + img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString()), WithAllMetadata()) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + defaultPlatformManifest, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default()) + if err != nil { + t.Fatal(err) + } + err = client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All), archive.WithSkipMissing(client.ContentStore())) + if err != nil { + t.Fatal(err) + } + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, true) - _, err = client.Fetch(ctx, testImage) - if err != nil { - t.Fatal(err) - } - dstFile, err := os.CreateTemp("", "export-import-test") - if err != nil { - t.Fatal(err) - } - defer func() { - dstFile.Close() - os.Remove(dstFile.Name()) - }() + // Check if archive contains only one manifest for the default platform + if !isImageInArchive(ctx, t, client, dstFile, defaultPlatformManifest) { + t.Fatal("archive does not contain manifest for the default platform") + } - err = client.Export(ctx, dstFile, archive.WithPlatform(platforms.Default()), archive.WithImage(client.ImageService(), testImage)) - if err != nil { - t.Fatal(err) - } + if isImageInArchive(ctx, t, client, dstFile, img.Target) { + t.Fatal("archive shouldn't contain all platforms") + } + }, + }, + { + name: "export full image", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + img, err := client.Fetch(ctx, testImage) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All), archive.WithImage(client.ImageService(), testImage)) + if err != nil { + t.Fatal(err) + } - // Seek to beginning of file before passing it to assertOCITar() - dstFile.Seek(0, 0) - assertOCITar(t, dstFile, true) + // Seek to beginning of file before passing it to assertOCITar() + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, true) + + // Archive should contain all platforms. + if !isImageInArchive(ctx, t, client, dstFile, img.Target) { + t.Fatalf("archive does not contain all platforms") + } + }, + }, + { + name: "export multi-platform with SkipDockerManifest", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + img, err := client.Fetch(ctx, testImage) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithManifest(img.Target), archive.WithSkipDockerManifest()) + if err != nil { + t.Fatal(err) + } + + // Seek to beginning of file before passing it to assertOCITar() + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, false) + + if !isImageInArchive(ctx, t, client, dstFile, img.Target) { + t.Fatalf("archive does not contain expected platform") + } + }, + }, + { + name: "export single-platform with SkipDockerManifest", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString())) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + result, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default()) + if err != nil { + t.Fatal(err) + } + + err = client.Export(ctx, dstFile, archive.WithManifest(result), archive.WithSkipDockerManifest()) + if err != nil { + t.Fatal(err) + } + + // Seek to beginning of file before passing it to assertOCITar() + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, false) + + if !isImageInArchive(ctx, t, client, dstFile, result) { + t.Fatalf("archive does not contain expected platform") + } + }, + }, + { + name: "export index only", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString())) + if err != nil { + t.Fatal(err) + } + + var all []ocispec.Descriptor + err = images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + ch, err := images.Children(ctx, client.ContentStore(), desc) + if err != nil { + if errdefs.IsNotFound(err) { + return nil, images.ErrSkipDesc + } + return nil, err + } + all = append(all, ch...) + + return ch, nil + }), img.Target) + if err != nil { + t.Fatal(err) + } + for _, d := range all { + if images.IsIndexType(d.MediaType) { + continue + } + if err := client.ContentStore().Delete(ctx, d.Digest); err != nil && !errdefs.IsNotFound(err) { + t.Fatalf("failed to delete %v: %v", d.Digest, err) + } + } + + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithSkipMissing(client.ContentStore())) + if err != nil { + t.Fatal(err) + } + + // Seek to beginning of file before passing it to assertOCITar() + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, false) + + defaultPlatformManifest, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default()) + if err != nil { + t.Fatal(err) + } + + // Check if archive contains only one manifest for the default platform + if isImageInArchive(ctx, t, client, dstFile, defaultPlatformManifest) { + t.Fatal("archive shouldn't contain manifest for the default platform") + } + }, + }, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx, cancel := testContext(t) + defer cancel() + + namespace := uuid.New().String() + client, err := newClient(t, address, WithDefaultNamespace(namespace)) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx = namespaces.WithNamespace(ctx, namespace) + + img := tc.prepare(ctx, t, client) + + dstFile, err := os.CreateTemp("", "export-test") + if err != nil { + t.Fatal(err) + } + defer func() { + dstFile.Close() + os.Remove(dstFile.Name()) + }() + + tc.check(ctx, t, client, dstFile, img) + }) + } } -// TestExportDockerManifest exports testImage as a tar stream, using the -// WithSkipDockerManifest option -func TestExportDockerManifest(t *testing.T) { - if testing.Short() { - t.Skip() - } - ctx, cancel := testContext(t) - defer cancel() - - client, err := New(address) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - _, err = client.Fetch(ctx, testImage) - if err != nil { - t.Fatal(err) - } - dstFile, err := os.CreateTemp("", "export-import-test") - if err != nil { - t.Fatal(err) - } - defer func() { - dstFile.Close() - os.Remove(dstFile.Name()) - }() - - img, err := client.ImageService().Get(ctx, testImage) - if err != nil { - t.Fatal(err) - } - - // test multi-platform export - err = client.Export(ctx, dstFile, archive.WithManifest(img.Target), archive.WithSkipDockerManifest()) - if err != nil { - t.Fatal(err) - } +func isImageInArchive(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, mfst ocispec.Descriptor) bool { dstFile.Seek(0, 0) - assertOCITar(t, dstFile, false) + tr := tar.NewReader(dstFile) - // reset to beginning - dstFile.Seek(0, 0) - - // test single-platform export - var result ocispec.Descriptor - err = images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - if images.IsManifestType(desc.MediaType) { - p, err := content.ReadBlob(ctx, client.ContentStore(), desc) - if err != nil { - return nil, err + var blobs []string + for { + h, err := tr.Next() + if err != nil { + if err == io.EOF { + break } - - var manifest ocispec.Manifest - if err := json.Unmarshal(p, &manifest); err != nil { - return nil, err - } - - if desc.Platform == nil || platforms.Default().Match(platforms.Normalize(*desc.Platform)) { - result = desc - } - return nil, nil - } else if images.IsIndexType(desc.MediaType) { - p, err := content.ReadBlob(ctx, client.ContentStore(), desc) - if err != nil { - return nil, err - } - - var idx ocispec.Index - if err := json.Unmarshal(p, &idx); err != nil { - return nil, err - } - return idx.Manifests, nil + t.Fatal(err) + } + + digest := strings.TrimPrefix(h.Name, "blobs/sha256/") + if digest != h.Name && digest != "" { + blobs = append(blobs, digest) } - return nil, nil - }), img.Target) - if err != nil { - t.Fatal(err) } - err = client.Export(ctx, dstFile, archive.WithManifest(result), archive.WithSkipDockerManifest()) + + allPresent := true + // Check if the archive contains all blobs referenced by the manifest. + images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + for _, b := range blobs { + if desc.Digest.Hex() == b { + return images.Children(ctx, client.ContentStore(), desc) + } + } + allPresent = false + return nil, images.ErrStopHandler + }), mfst) + + return allPresent +} + +func getPlatformManifest(ctx context.Context, cs content.Store, target ocispec.Descriptor, platform platforms.MatchComparer) (ocispec.Descriptor, error) { + mfst, err := images.LimitManifests(images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + children, err := images.Children(ctx, cs, desc) + if !images.IsManifestType(desc.MediaType) { + return children, err + } + + if err != nil { + if errdefs.IsNotFound(err) { + return nil, images.ErrSkipDesc + } + return nil, err + } + + return children, nil + }), platform, 1)(ctx, target) + if err != nil { - t.Fatal(err) + return ocispec.Descriptor{}, err } - dstFile.Seek(0, 0) - assertOCITar(t, dstFile, false) + if len(mfst) == 0 { + return ocispec.Descriptor{}, errdefs.ErrNotFound + } + return mfst[0], nil } func assertOCITar(t *testing.T, r io.Reader, docker bool) { + t.Helper() // TODO: add more assertion tr := tar.NewReader(r) foundOCILayout := false @@ -168,8 +339,7 @@ func assertOCITar(t *testing.T, r io.Reader, docker bool) { break } if err != nil { - t.Error(err) - continue + t.Fatal(err) } if h.Name == ocispec.ImageLayoutFile { foundOCILayout = true diff --git a/integration/client/import_test.go b/integration/client/import_test.go index dbabeb44a..e228d2039 100644 --- a/integration/client/import_test.go +++ b/integration/client/import_test.go @@ -35,14 +35,17 @@ import ( "github.com/containerd/containerd/v2/archive/compression" "github.com/containerd/containerd/v2/archive/tartest" . "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/content" "github.com/containerd/containerd/v2/images" "github.com/containerd/containerd/v2/images/archive" "github.com/containerd/containerd/v2/leases" + "github.com/containerd/containerd/v2/namespaces" "github.com/containerd/containerd/v2/oci" "github.com/containerd/containerd/v2/pkg/transfer" tarchive "github.com/containerd/containerd/v2/pkg/transfer/archive" "github.com/containerd/containerd/v2/pkg/transfer/image" "github.com/containerd/containerd/v2/platforms" + "github.com/google/uuid" digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go" @@ -159,12 +162,6 @@ func TestImport(t *testing.T) { ctx, cancel := testContext(t) defer cancel() - client, err := newClient(t, address) - if err != nil { - t.Fatal(err) - } - defer client.Close() - tc := tartest.TarContext{} b1, d1 := createContent(256, 1) @@ -176,9 +173,12 @@ func TestImport(t *testing.T) { m1, d3, expManifest := createManifest(c1, [][]byte{b1}) - provider := client.ContentStore() + c2, _ := createConfig(runtime.GOOS, runtime.GOARCH, "test2") + m2, d5, _ := createManifest(c2, [][]byte{{1, 2, 3, 4, 5}}) - checkManifest := func(ctx context.Context, t *testing.T, d ocispec.Descriptor, expManifest *ocispec.Manifest) { + ml1, d6 := createManifestList(m1, m2) + + checkManifest := func(ctx context.Context, t *testing.T, provider content.Provider, d ocispec.Descriptor, expManifest *ocispec.Manifest) { m, err := images.Manifest(ctx, provider, d, nil) if err != nil { t.Fatalf("unable to read target blob: %+v", err) @@ -209,9 +209,40 @@ func TestImport(t *testing.T) { for _, tc := range []struct { Name string Writer tartest.WriterToTar - Check func(*testing.T, []images.Image) + Check func(context.Context, *testing.T, *Client, []images.Image) Opts []ImportOpt }{ + { + Name: "OCI-IndexWithoutAnyManifest", + Writer: tartest.TarAll( + tc.Dir(ocispec.ImageBlobsDir, 0755), + tc.Dir(ocispec.ImageBlobsDir+"/sha256", 0755), + tc.File(ocispec.ImageIndexFile, createIndex(ml1, ocispec.MediaTypeImageIndex, "docker.io/library/sparse:ok"), 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d6.Encoded(), ml1, 0644), + tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), + ), + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { + checkImages(t, d6, imgs, "docker.io/library/sparse:ok") + mfsts, err := images.Children(ctx, client.ContentStore(), imgs[0].Target) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for _, m := range mfsts { + exists, err := content.Exists(ctx, client.ContentStore(), m) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if exists { + t.Fatal("no manifest should be imported") + } + } + }, + Opts: []ImportOpt{ + WithSkipMissing(), + }, + }, { Name: "DockerV2.0", Writer: tartest.TarAll( @@ -232,7 +263,7 @@ func TestImport(t *testing.T) { tc.File("e95212f7aa2cab51d0abd765cd43.json", c1, 0644), tc.File("manifest.json", []byte(`[{"Config":"e95212f7aa2cab51d0abd765cd43.json","RepoTags":["test-import:notlatest", "another/repo:tag"],"Layers":["bd765cd43e95212f7aa2cab51d0a/layer.tar"]}]`), 0644), ), - Check: func(t *testing.T, imgs []images.Image) { + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { if len(imgs) == 0 { t.Fatalf("no images") } @@ -243,7 +274,7 @@ func TestImport(t *testing.T) { } checkImages(t, imgs[0].Target.Digest, imgs, names...) - checkManifest(ctx, t, imgs[0].Target, nil) + checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, nil) }, }, { @@ -271,17 +302,17 @@ func TestImport(t *testing.T) { tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644), - tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "docker.io/lib/img:ok"), 0644), + tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "docker.io/lib/img:ok"), 0644), tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), ), - Check: func(t *testing.T, imgs []images.Image) { + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { names := []string{ "latest", "docker.io/lib/img:ok", } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target, expManifest) + checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest) }, }, { @@ -292,17 +323,17 @@ func TestImport(t *testing.T) { tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644), - tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "docker.io/lib/img:ok"), 0644), + tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "docker.io/lib/img:ok"), 0644), tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), ), - Check: func(t *testing.T, imgs []images.Image) { + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { names := []string{ "localhost:5000/myimage:latest", "docker.io/lib/img:ok", } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target, expManifest) + checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest) }, Opts: []ImportOpt{ WithImageRefTranslator(archive.AddRefPrefix("localhost:5000/myimage")), @@ -316,24 +347,94 @@ func TestImport(t *testing.T) { tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644), - tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "localhost:5000/myimage:old", "docker.io/lib/img:ok"), 0644), + tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "localhost:5000/myimage:old", "docker.io/lib/img:ok"), 0644), tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), ), - Check: func(t *testing.T, imgs []images.Image) { + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { names := []string{ "localhost:5000/myimage:latest", "localhost:5000/myimage:old", } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target, expManifest) + checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest) }, Opts: []ImportOpt{ WithImageRefTranslator(archive.FilterRefPrefix("localhost:5000/myimage")), }, }, + { + Name: "OCI-IndexWithMissingManifestDescendants", + Writer: tartest.TarAll( + tc.Dir(ocispec.ImageBlobsDir, 0755), + tc.Dir(ocispec.ImageBlobsDir+"/sha256", 0755), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d5.Encoded(), m2, 0644), + tc.File(ocispec.ImageIndexFile, createIndex(ml1, ocispec.MediaTypeImageIndex, "docker.io/library/sparse:ok"), 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d6.Encoded(), ml1, 0644), + tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), + ), + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { + checkImages(t, d6, imgs, "docker.io/library/sparse:ok") + mfsts, err := images.Children(ctx, client.ContentStore(), imgs[0].Target) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var importedManifest *ocispec.Descriptor + var secondManifest *ocispec.Descriptor + for _, m := range mfsts { + exists, err := content.Exists(ctx, client.ContentStore(), m) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if exists { + if m.Digest == d3 { + m := m + importedManifest = &m + } else if m.Digest == d5 { + m := m + secondManifest = &m + } else { + t.Fatalf("imported manifest with unexpected digest: %v", m.Digest) + } + } + } + if importedManifest == nil { + t.Fatal("the expected manifest was not loaded") + } + checkManifest(ctx, t, client.ContentStore(), *importedManifest, expManifest) + + if secondManifest == nil { + t.Fatal("the expected manifest was not loaded") + } + _, _, _, missing, err := images.Check(ctx, client.ContentStore(), *secondManifest, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(missing) != 2 { + t.Fatalf("expected 2 missing blobs, got %+v", missing) + } + + }, + Opts: []ImportOpt{ + WithSkipMissing(), + }, + }, } { t.Run(tc.Name, func(t *testing.T) { + client, err := newClient(t, address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx := namespaces.WithNamespace(ctx, uuid.New().String()) + images, err := client.Import(ctx, tartest.TarFromWriterTo(tc.Writer), tc.Opts...) if err != nil { if tc.Check != nil { @@ -344,7 +445,7 @@ func TestImport(t *testing.T) { t.Fatalf("expected error on import") } - tc.Check(t, images) + tc.Check(ctx, t, client, images) }) } } @@ -363,7 +464,8 @@ func checkImages(t *testing.T, target digest.Digest, actual []images.Image, name } if actual[i].Target.MediaType != ocispec.MediaTypeImageManifest && - actual[i].Target.MediaType != images.MediaTypeDockerSchema2Manifest { + actual[i].Target.MediaType != images.MediaTypeDockerSchema2Manifest && + actual[i].Target.MediaType != ocispec.MediaTypeImageIndex { t.Fatalf("image(%d) unexpected media type: %s", i, actual[i].Target.MediaType) } } @@ -430,14 +532,35 @@ func createManifest(config []byte, layers [][]byte) ([]byte, digest.Digest, *oci return b, digest.FromBytes(b), &manifest } -func createIndex(manifest []byte, tags ...string) []byte { +func createManifestList(manifests ...[]byte) ([]byte, digest.Digest) { + idx := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + } + + for _, manifest := range manifests { + d := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: digest.FromBytes(manifest), + Size: int64(len(manifest)), + } + idx.Manifests = append(idx.Manifests, d) + } + + b, _ := json.Marshal(idx) + + return b, digest.FromBytes(b) +} + +func createIndex(manifest []byte, mt string, tags ...string) []byte { idx := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, }, } d := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageManifest, + MediaType: mt, Digest: digest.FromBytes(manifest), Size: int64(len(manifest)), }