diff --git a/client/import.go b/client/import.go index 42aca54e7..79031f5c3 100644 --- a/client/import.go +++ b/client/import.go @@ -37,6 +37,7 @@ type importOpts struct { platformMatcher platforms.MatchComparer compress bool discardLayers bool + skipMissing bool } // ImportOpt allows the caller to specify import specific options @@ -113,6 +114,15 @@ func WithDiscardUnpackedLayers() ImportOpt { } } +// WithSkipMissing allows to import an archive which doesn't contain all the +// referenced blobs. +func WithSkipMissing() ImportOpt { + return func(c *importOpts) error { + c.skipMissing = true + return nil + } +} + // Import imports an image from a Tar stream using reader. // Caller needs to specify importer. Future version may use oci.v1 as the default. // Note that unreferenced blobs may be imported to the content store as well. @@ -162,7 +172,12 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { // Only save images at top level if desc.Digest != index.Digest { - return images.Children(ctx, cs, desc) + // Don't set labels on missing content. + children, err := images.Children(ctx, cs, desc) + if iopts.skipMissing && errdefs.IsNotFound(err) { + return nil, images.ErrSkipDesc + } + return children, err } idx, err := decodeIndex(ctx, cs, desc) diff --git a/content/helpers.go b/content/helpers.go index 180ff4848..1525c15e1 100644 --- a/content/helpers.go +++ b/content/helpers.go @@ -320,3 +320,14 @@ func copyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) { } return } + +// Exists returns whether an attempt to access the content would not error out +// with an ErrNotFound error. It will return an encountered error if it was +// different than ErrNotFound. +func Exists(ctx context.Context, provider InfoProvider, desc ocispec.Descriptor) (bool, error) { + _, err := provider.Info(ctx, desc.Digest) + if errdefs.IsNotFound(err) { + return false, nil + } + return err == nil, err +} diff --git a/images/archive/exporter.go b/images/archive/exporter.go index af1121772..f343912a4 100644 --- a/images/archive/exporter.go +++ b/images/archive/exporter.go @@ -24,11 +24,14 @@ import ( "io" "path" "sort" + "strings" "github.com/containerd/containerd/v2/content" "github.com/containerd/containerd/v2/errdefs" "github.com/containerd/containerd/v2/images" + "github.com/containerd/containerd/v2/labels" "github.com/containerd/containerd/v2/platforms" + "github.com/containerd/log" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -140,6 +143,45 @@ func WithSkipNonDistributableBlobs() ExportOpt { return WithBlobFilter(f) } +// WithSkipMissing excludes blobs referenced by manifests if not all blobs +// would be included in the archive. +// The manifest itself is excluded only if it's not present locally. +// This allows to export multi-platform images if not all platforms are present +// while still persisting the multi-platform index. +func WithSkipMissing(store ContentProvider) ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + o.blobRecordOptions.childrenHandler = images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) { + children, err := images.Children(ctx, store, desc) + if !images.IsManifestType(desc.MediaType) { + return children, err + } + + if err != nil { + // If manifest itself is missing, skip it from export. + if errdefs.IsNotFound(err) { + return nil, images.ErrSkipDesc + } + return nil, err + } + + // Don't export manifest descendants if any of them doesn't exist. + for _, child := range children { + exists, err := content.Exists(ctx, store, child) + if err != nil { + return nil, err + } + + // If any child is missing, only export the manifest, but don't export its descendants. + if !exists { + return nil, nil + } + } + return children, nil + }) + return nil + } +} + func addNameAnnotation(name string, base map[string]string) map[string]string { annotations := map[string]string{} for k, v := range base { @@ -152,8 +194,31 @@ func addNameAnnotation(name string, base map[string]string) map[string]string { return annotations } +func copySourceLabels(ctx context.Context, infoProvider content.InfoProvider, desc ocispec.Descriptor) (ocispec.Descriptor, error) { + info, err := infoProvider.Info(ctx, desc.Digest) + if err != nil { + return desc, err + } + for k, v := range info.Labels { + if strings.HasPrefix(k, labels.LabelDistributionSource) { + if desc.Annotations == nil { + desc.Annotations = map[string]string{k: v} + } else { + desc.Annotations[k] = v + } + } + } + return desc, nil +} + +// ContentProvider provides both content and info about content +type ContentProvider interface { + content.Provider + content.InfoProvider +} + // Export implements Exporter. -func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error { +func Export(ctx context.Context, store ContentProvider, writer io.Writer, opts ...ExportOpt) error { var eo exportOptions for _, opt := range opts { if err := opt(ctx, &eo); err != nil { @@ -163,13 +228,22 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts records := []tarRecord{ ociLayoutFile(""), - ociIndexRecord(eo.manifests), + } + + manifests := make([]ocispec.Descriptor, 0, len(eo.manifests)) + for _, desc := range eo.manifests { + d, err := copySourceLabels(ctx, store, desc) + if err != nil { + log.G(ctx).WithError(err).WithField("desc", desc).Warn("failed to copy distribution.source labels") + continue + } + manifests = append(manifests, d) } algorithms := map[string]struct{}{} dManifests := map[digest.Digest]*exportManifest{} resolvedIndex := map[digest.Digest]digest.Digest{} - for _, desc := range eo.manifests { + for _, desc := range manifests { if images.IsManifestType(desc.MediaType) { mt, ok := dManifests[desc.Digest] if !ok { @@ -259,6 +333,8 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts } } + records = append(records, ociIndexRecord(manifests)) + if !eo.skipDockerManifest && len(dManifests) > 0 { tr, err := manifestsRecord(ctx, store, dManifests) if err != nil { @@ -291,7 +367,10 @@ func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descri return nil, nil } - childrenHandler := images.ChildrenHandler(store) + childrenHandler := brOpts.childrenHandler + if childrenHandler == nil { + childrenHandler = images.ChildrenHandler(store) + } handlers := images.Handlers( childrenHandler, @@ -313,7 +392,8 @@ type tarRecord struct { } type blobRecordOptions struct { - blobFilter BlobFilter + blobFilter BlobFilter + childrenHandler images.HandlerFunc } func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOptions) tarRecord { diff --git a/integration/client/export_test.go b/integration/client/export_test.go index 8a43c8e26..1abc3613e 100644 --- a/integration/client/export_test.go +++ b/integration/client/export_test.go @@ -19,144 +19,315 @@ package client import ( "archive/tar" "context" - "encoding/json" "io" "os" + "runtime" + "strings" "testing" . "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/content" + "github.com/containerd/containerd/v2/errdefs" "github.com/containerd/containerd/v2/images" "github.com/containerd/containerd/v2/images/archive" + "github.com/containerd/containerd/v2/namespaces" "github.com/containerd/containerd/v2/platforms" + "github.com/google/uuid" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// TestExport exports testImage as a tar stream -func TestExport(t *testing.T) { +func TestExportAllCases(t *testing.T) { if testing.Short() { t.Skip() } - ctx, cancel := testContext(t) - defer cancel() - client, err := New(address) - if err != nil { - t.Fatal(err) - } - defer client.Close() + for _, tc := range []struct { + name string + prepare func(context.Context, *testing.T, *Client) images.Image + check func(context.Context, *testing.T, *Client, *os.File, images.Image) + }{ + { + name: "export all platforms without SkipMissing", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + if runtime.GOOS == "windows" { + t.Skip("skipping test on windows - the testimage index has only one platform") + } + img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString()), WithAllMetadata()) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, _ images.Image) { + err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All)) + if !errdefs.IsNotFound(err) { + t.Fatal("should fail with not found error") + } + }, + }, + { + name: "export all platforms with SkipMissing", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + if runtime.GOOS == "windows" { + t.Skip("skipping test on windows - the testimage index has only one platform") + } + img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString()), WithAllMetadata()) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + defaultPlatformManifest, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default()) + if err != nil { + t.Fatal(err) + } + err = client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All), archive.WithSkipMissing(client.ContentStore())) + if err != nil { + t.Fatal(err) + } + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, true) - _, err = client.Fetch(ctx, testImage) - if err != nil { - t.Fatal(err) - } - dstFile, err := os.CreateTemp("", "export-import-test") - if err != nil { - t.Fatal(err) - } - defer func() { - dstFile.Close() - os.Remove(dstFile.Name()) - }() + // Check if archive contains only one manifest for the default platform + if !isImageInArchive(ctx, t, client, dstFile, defaultPlatformManifest) { + t.Fatal("archive does not contain manifest for the default platform") + } - err = client.Export(ctx, dstFile, archive.WithPlatform(platforms.Default()), archive.WithImage(client.ImageService(), testImage)) - if err != nil { - t.Fatal(err) - } + if isImageInArchive(ctx, t, client, dstFile, img.Target) { + t.Fatal("archive shouldn't contain all platforms") + } + }, + }, + { + name: "export full image", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + img, err := client.Fetch(ctx, testImage) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All), archive.WithImage(client.ImageService(), testImage)) + if err != nil { + t.Fatal(err) + } - // Seek to beginning of file before passing it to assertOCITar() - dstFile.Seek(0, 0) - assertOCITar(t, dstFile, true) + // Seek to beginning of file before passing it to assertOCITar() + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, true) + + // Archive should contain all platforms. + if !isImageInArchive(ctx, t, client, dstFile, img.Target) { + t.Fatalf("archive does not contain all platforms") + } + }, + }, + { + name: "export multi-platform with SkipDockerManifest", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + img, err := client.Fetch(ctx, testImage) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithManifest(img.Target), archive.WithSkipDockerManifest()) + if err != nil { + t.Fatal(err) + } + + // Seek to beginning of file before passing it to assertOCITar() + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, false) + + if !isImageInArchive(ctx, t, client, dstFile, img.Target) { + t.Fatalf("archive does not contain expected platform") + } + }, + }, + { + name: "export single-platform with SkipDockerManifest", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString())) + if err != nil { + t.Fatal(err) + } + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + result, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default()) + if err != nil { + t.Fatal(err) + } + + err = client.Export(ctx, dstFile, archive.WithManifest(result), archive.WithSkipDockerManifest()) + if err != nil { + t.Fatal(err) + } + + // Seek to beginning of file before passing it to assertOCITar() + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, false) + + if !isImageInArchive(ctx, t, client, dstFile, result) { + t.Fatalf("archive does not contain expected platform") + } + }, + }, + { + name: "export index only", + prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image { + img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString())) + if err != nil { + t.Fatal(err) + } + + var all []ocispec.Descriptor + err = images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + ch, err := images.Children(ctx, client.ContentStore(), desc) + if err != nil { + if errdefs.IsNotFound(err) { + return nil, images.ErrSkipDesc + } + return nil, err + } + all = append(all, ch...) + + return ch, nil + }), img.Target) + if err != nil { + t.Fatal(err) + } + for _, d := range all { + if images.IsIndexType(d.MediaType) { + continue + } + if err := client.ContentStore().Delete(ctx, d.Digest); err != nil && !errdefs.IsNotFound(err) { + t.Fatalf("failed to delete %v: %v", d.Digest, err) + } + } + + return img + }, + check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) { + err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithSkipMissing(client.ContentStore())) + if err != nil { + t.Fatal(err) + } + + // Seek to beginning of file before passing it to assertOCITar() + dstFile.Seek(0, 0) + assertOCITar(t, dstFile, false) + + defaultPlatformManifest, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default()) + if err != nil { + t.Fatal(err) + } + + // Check if archive contains only one manifest for the default platform + if isImageInArchive(ctx, t, client, dstFile, defaultPlatformManifest) { + t.Fatal("archive shouldn't contain manifest for the default platform") + } + }, + }, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx, cancel := testContext(t) + defer cancel() + + namespace := uuid.New().String() + client, err := newClient(t, address, WithDefaultNamespace(namespace)) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx = namespaces.WithNamespace(ctx, namespace) + + img := tc.prepare(ctx, t, client) + + dstFile, err := os.CreateTemp("", "export-test") + if err != nil { + t.Fatal(err) + } + defer func() { + dstFile.Close() + os.Remove(dstFile.Name()) + }() + + tc.check(ctx, t, client, dstFile, img) + }) + } } -// TestExportDockerManifest exports testImage as a tar stream, using the -// WithSkipDockerManifest option -func TestExportDockerManifest(t *testing.T) { - if testing.Short() { - t.Skip() - } - ctx, cancel := testContext(t) - defer cancel() - - client, err := New(address) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - _, err = client.Fetch(ctx, testImage) - if err != nil { - t.Fatal(err) - } - dstFile, err := os.CreateTemp("", "export-import-test") - if err != nil { - t.Fatal(err) - } - defer func() { - dstFile.Close() - os.Remove(dstFile.Name()) - }() - - img, err := client.ImageService().Get(ctx, testImage) - if err != nil { - t.Fatal(err) - } - - // test multi-platform export - err = client.Export(ctx, dstFile, archive.WithManifest(img.Target), archive.WithSkipDockerManifest()) - if err != nil { - t.Fatal(err) - } +func isImageInArchive(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, mfst ocispec.Descriptor) bool { dstFile.Seek(0, 0) - assertOCITar(t, dstFile, false) + tr := tar.NewReader(dstFile) - // reset to beginning - dstFile.Seek(0, 0) - - // test single-platform export - var result ocispec.Descriptor - err = images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - if images.IsManifestType(desc.MediaType) { - p, err := content.ReadBlob(ctx, client.ContentStore(), desc) - if err != nil { - return nil, err + var blobs []string + for { + h, err := tr.Next() + if err != nil { + if err == io.EOF { + break } - - var manifest ocispec.Manifest - if err := json.Unmarshal(p, &manifest); err != nil { - return nil, err - } - - if desc.Platform == nil || platforms.Default().Match(platforms.Normalize(*desc.Platform)) { - result = desc - } - return nil, nil - } else if images.IsIndexType(desc.MediaType) { - p, err := content.ReadBlob(ctx, client.ContentStore(), desc) - if err != nil { - return nil, err - } - - var idx ocispec.Index - if err := json.Unmarshal(p, &idx); err != nil { - return nil, err - } - return idx.Manifests, nil + t.Fatal(err) + } + + digest := strings.TrimPrefix(h.Name, "blobs/sha256/") + if digest != h.Name && digest != "" { + blobs = append(blobs, digest) } - return nil, nil - }), img.Target) - if err != nil { - t.Fatal(err) } - err = client.Export(ctx, dstFile, archive.WithManifest(result), archive.WithSkipDockerManifest()) + + allPresent := true + // Check if the archive contains all blobs referenced by the manifest. + images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + for _, b := range blobs { + if desc.Digest.Hex() == b { + return images.Children(ctx, client.ContentStore(), desc) + } + } + allPresent = false + return nil, images.ErrStopHandler + }), mfst) + + return allPresent +} + +func getPlatformManifest(ctx context.Context, cs content.Store, target ocispec.Descriptor, platform platforms.MatchComparer) (ocispec.Descriptor, error) { + mfst, err := images.LimitManifests(images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + children, err := images.Children(ctx, cs, desc) + if !images.IsManifestType(desc.MediaType) { + return children, err + } + + if err != nil { + if errdefs.IsNotFound(err) { + return nil, images.ErrSkipDesc + } + return nil, err + } + + return children, nil + }), platform, 1)(ctx, target) + if err != nil { - t.Fatal(err) + return ocispec.Descriptor{}, err } - dstFile.Seek(0, 0) - assertOCITar(t, dstFile, false) + if len(mfst) == 0 { + return ocispec.Descriptor{}, errdefs.ErrNotFound + } + return mfst[0], nil } func assertOCITar(t *testing.T, r io.Reader, docker bool) { + t.Helper() // TODO: add more assertion tr := tar.NewReader(r) foundOCILayout := false @@ -168,8 +339,7 @@ func assertOCITar(t *testing.T, r io.Reader, docker bool) { break } if err != nil { - t.Error(err) - continue + t.Fatal(err) } if h.Name == ocispec.ImageLayoutFile { foundOCILayout = true diff --git a/integration/client/import_test.go b/integration/client/import_test.go index dbabeb44a..e228d2039 100644 --- a/integration/client/import_test.go +++ b/integration/client/import_test.go @@ -35,14 +35,17 @@ import ( "github.com/containerd/containerd/v2/archive/compression" "github.com/containerd/containerd/v2/archive/tartest" . "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/content" "github.com/containerd/containerd/v2/images" "github.com/containerd/containerd/v2/images/archive" "github.com/containerd/containerd/v2/leases" + "github.com/containerd/containerd/v2/namespaces" "github.com/containerd/containerd/v2/oci" "github.com/containerd/containerd/v2/pkg/transfer" tarchive "github.com/containerd/containerd/v2/pkg/transfer/archive" "github.com/containerd/containerd/v2/pkg/transfer/image" "github.com/containerd/containerd/v2/platforms" + "github.com/google/uuid" digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go" @@ -159,12 +162,6 @@ func TestImport(t *testing.T) { ctx, cancel := testContext(t) defer cancel() - client, err := newClient(t, address) - if err != nil { - t.Fatal(err) - } - defer client.Close() - tc := tartest.TarContext{} b1, d1 := createContent(256, 1) @@ -176,9 +173,12 @@ func TestImport(t *testing.T) { m1, d3, expManifest := createManifest(c1, [][]byte{b1}) - provider := client.ContentStore() + c2, _ := createConfig(runtime.GOOS, runtime.GOARCH, "test2") + m2, d5, _ := createManifest(c2, [][]byte{{1, 2, 3, 4, 5}}) - checkManifest := func(ctx context.Context, t *testing.T, d ocispec.Descriptor, expManifest *ocispec.Manifest) { + ml1, d6 := createManifestList(m1, m2) + + checkManifest := func(ctx context.Context, t *testing.T, provider content.Provider, d ocispec.Descriptor, expManifest *ocispec.Manifest) { m, err := images.Manifest(ctx, provider, d, nil) if err != nil { t.Fatalf("unable to read target blob: %+v", err) @@ -209,9 +209,40 @@ func TestImport(t *testing.T) { for _, tc := range []struct { Name string Writer tartest.WriterToTar - Check func(*testing.T, []images.Image) + Check func(context.Context, *testing.T, *Client, []images.Image) Opts []ImportOpt }{ + { + Name: "OCI-IndexWithoutAnyManifest", + Writer: tartest.TarAll( + tc.Dir(ocispec.ImageBlobsDir, 0755), + tc.Dir(ocispec.ImageBlobsDir+"/sha256", 0755), + tc.File(ocispec.ImageIndexFile, createIndex(ml1, ocispec.MediaTypeImageIndex, "docker.io/library/sparse:ok"), 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d6.Encoded(), ml1, 0644), + tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), + ), + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { + checkImages(t, d6, imgs, "docker.io/library/sparse:ok") + mfsts, err := images.Children(ctx, client.ContentStore(), imgs[0].Target) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for _, m := range mfsts { + exists, err := content.Exists(ctx, client.ContentStore(), m) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if exists { + t.Fatal("no manifest should be imported") + } + } + }, + Opts: []ImportOpt{ + WithSkipMissing(), + }, + }, { Name: "DockerV2.0", Writer: tartest.TarAll( @@ -232,7 +263,7 @@ func TestImport(t *testing.T) { tc.File("e95212f7aa2cab51d0abd765cd43.json", c1, 0644), tc.File("manifest.json", []byte(`[{"Config":"e95212f7aa2cab51d0abd765cd43.json","RepoTags":["test-import:notlatest", "another/repo:tag"],"Layers":["bd765cd43e95212f7aa2cab51d0a/layer.tar"]}]`), 0644), ), - Check: func(t *testing.T, imgs []images.Image) { + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { if len(imgs) == 0 { t.Fatalf("no images") } @@ -243,7 +274,7 @@ func TestImport(t *testing.T) { } checkImages(t, imgs[0].Target.Digest, imgs, names...) - checkManifest(ctx, t, imgs[0].Target, nil) + checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, nil) }, }, { @@ -271,17 +302,17 @@ func TestImport(t *testing.T) { tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644), - tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "docker.io/lib/img:ok"), 0644), + tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "docker.io/lib/img:ok"), 0644), tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), ), - Check: func(t *testing.T, imgs []images.Image) { + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { names := []string{ "latest", "docker.io/lib/img:ok", } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target, expManifest) + checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest) }, }, { @@ -292,17 +323,17 @@ func TestImport(t *testing.T) { tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644), - tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "docker.io/lib/img:ok"), 0644), + tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "docker.io/lib/img:ok"), 0644), tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), ), - Check: func(t *testing.T, imgs []images.Image) { + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { names := []string{ "localhost:5000/myimage:latest", "docker.io/lib/img:ok", } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target, expManifest) + checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest) }, Opts: []ImportOpt{ WithImageRefTranslator(archive.AddRefPrefix("localhost:5000/myimage")), @@ -316,24 +347,94 @@ func TestImport(t *testing.T) { tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644), tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644), - tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "localhost:5000/myimage:old", "docker.io/lib/img:ok"), 0644), + tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "localhost:5000/myimage:old", "docker.io/lib/img:ok"), 0644), tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), ), - Check: func(t *testing.T, imgs []images.Image) { + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { names := []string{ "localhost:5000/myimage:latest", "localhost:5000/myimage:old", } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target, expManifest) + checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest) }, Opts: []ImportOpt{ WithImageRefTranslator(archive.FilterRefPrefix("localhost:5000/myimage")), }, }, + { + Name: "OCI-IndexWithMissingManifestDescendants", + Writer: tartest.TarAll( + tc.Dir(ocispec.ImageBlobsDir, 0755), + tc.Dir(ocispec.ImageBlobsDir+"/sha256", 0755), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d5.Encoded(), m2, 0644), + tc.File(ocispec.ImageIndexFile, createIndex(ml1, ocispec.MediaTypeImageIndex, "docker.io/library/sparse:ok"), 0644), + tc.File(ocispec.ImageBlobsDir+"/sha256/"+d6.Encoded(), ml1, 0644), + tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644), + ), + Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) { + checkImages(t, d6, imgs, "docker.io/library/sparse:ok") + mfsts, err := images.Children(ctx, client.ContentStore(), imgs[0].Target) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var importedManifest *ocispec.Descriptor + var secondManifest *ocispec.Descriptor + for _, m := range mfsts { + exists, err := content.Exists(ctx, client.ContentStore(), m) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if exists { + if m.Digest == d3 { + m := m + importedManifest = &m + } else if m.Digest == d5 { + m := m + secondManifest = &m + } else { + t.Fatalf("imported manifest with unexpected digest: %v", m.Digest) + } + } + } + if importedManifest == nil { + t.Fatal("the expected manifest was not loaded") + } + checkManifest(ctx, t, client.ContentStore(), *importedManifest, expManifest) + + if secondManifest == nil { + t.Fatal("the expected manifest was not loaded") + } + _, _, _, missing, err := images.Check(ctx, client.ContentStore(), *secondManifest, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(missing) != 2 { + t.Fatalf("expected 2 missing blobs, got %+v", missing) + } + + }, + Opts: []ImportOpt{ + WithSkipMissing(), + }, + }, } { t.Run(tc.Name, func(t *testing.T) { + client, err := newClient(t, address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx := namespaces.WithNamespace(ctx, uuid.New().String()) + images, err := client.Import(ctx, tartest.TarFromWriterTo(tc.Writer), tc.Opts...) if err != nil { if tc.Check != nil { @@ -344,7 +445,7 @@ func TestImport(t *testing.T) { t.Fatalf("expected error on import") } - tc.Check(t, images) + tc.Check(ctx, t, client, images) }) } } @@ -363,7 +464,8 @@ func checkImages(t *testing.T, target digest.Digest, actual []images.Image, name } if actual[i].Target.MediaType != ocispec.MediaTypeImageManifest && - actual[i].Target.MediaType != images.MediaTypeDockerSchema2Manifest { + actual[i].Target.MediaType != images.MediaTypeDockerSchema2Manifest && + actual[i].Target.MediaType != ocispec.MediaTypeImageIndex { t.Fatalf("image(%d) unexpected media type: %s", i, actual[i].Target.MediaType) } } @@ -430,14 +532,35 @@ func createManifest(config []byte, layers [][]byte) ([]byte, digest.Digest, *oci return b, digest.FromBytes(b), &manifest } -func createIndex(manifest []byte, tags ...string) []byte { +func createManifestList(manifests ...[]byte) ([]byte, digest.Digest) { + idx := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + } + + for _, manifest := range manifests { + d := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: digest.FromBytes(manifest), + Size: int64(len(manifest)), + } + idx.Manifests = append(idx.Manifests, d) + } + + b, _ := json.Marshal(idx) + + return b, digest.FromBytes(b) +} + +func createIndex(manifest []byte, mt string, tags ...string) []byte { idx := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, }, } d := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageManifest, + MediaType: mt, Digest: digest.FromBytes(manifest), Size: int64(len(manifest)), }