github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #9554 from vvoland/archive-sparse

Browse Source 
import/export: Support references to missing content
This commit is contained in:
Maksym Pavlenko 2024-01-03 22:45:00 +00:00 committed by GitHub
commit d75bc05ef3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 538 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
client/import.go
View File

@ -37,6 +37,7 @@ type importOpts struct {
platformMatcher platforms.MatchComparer
compress bool
discardLayers bool
skipMissing bool
}
// ImportOpt allows the caller to specify import specific options
@ -113,6 +114,15 @@ func WithDiscardUnpackedLayers() ImportOpt {
}
}
// WithSkipMissing allows to import an archive which doesn't contain all the
// referenced blobs.
func WithSkipMissing() ImportOpt {
return func(c *importOpts) error {
c.skipMissing = true
return nil
}
}
// Import imports an image from a Tar stream using reader.
// Caller needs to specify importer. Future version may use oci.v1 as the default.
// Note that unreferenced blobs may be imported to the content store as well.
@ -162,7 +172,12 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt
var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
// Only save images at top level
if desc.Digest != index.Digest {
return images.Children(ctx, cs, desc)
// Don't set labels on missing content.
children, err := images.Children(ctx, cs, desc)
if iopts.skipMissing && errdefs.IsNotFound(err) {
return nil, images.ErrSkipDesc
}
return children, err
}
idx, err := decodeIndex(ctx, cs, desc)

11
content/helpers.go
View File

@ -320,3 +320,14 @@ func copyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) {
}
return
}
// Exists returns whether an attempt to access the content would not error out
// with an ErrNotFound error. It will return an encountered error if it was
// different than ErrNotFound.
func Exists(ctx context.Context, provider InfoProvider, desc ocispec.Descriptor) (bool, error) {
_, err := provider.Info(ctx, desc.Digest)
if errdefs.IsNotFound(err) {
return false, nil
}
return err == nil, err
}

90
images/archive/exporter.go
View File

@ -24,11 +24,14 @@ import (
"io"
"path"
"sort"
"strings"
"github.com/containerd/containerd/v2/content"
"github.com/containerd/containerd/v2/errdefs"
"github.com/containerd/containerd/v2/images"
"github.com/containerd/containerd/v2/labels"
"github.com/containerd/containerd/v2/platforms"
"github.com/containerd/log"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -140,6 +143,45 @@ func WithSkipNonDistributableBlobs() ExportOpt {
return WithBlobFilter(f)
}
// WithSkipMissing excludes blobs referenced by manifests if not all blobs
// would be included in the archive.
// The manifest itself is excluded only if it's not present locally.
// This allows to export multi-platform images if not all platforms are present
// while still persisting the multi-platform index.
func WithSkipMissing(store ContentProvider) ExportOpt {
return func(ctx context.Context, o *exportOptions) error {
o.blobRecordOptions.childrenHandler = images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
children, err := images.Children(ctx, store, desc)
if !images.IsManifestType(desc.MediaType) {
return children, err
}
if err != nil {
// If manifest itself is missing, skip it from export.
if errdefs.IsNotFound(err) {
return nil, images.ErrSkipDesc
}
return nil, err
}
// Don't export manifest descendants if any of them doesn't exist.
for _, child := range children {
exists, err := content.Exists(ctx, store, child)
if err != nil {
return nil, err
}
// If any child is missing, only export the manifest, but don't export its descendants.
if !exists {
return nil, nil
}
}
return children, nil
})
return nil
}
}
func addNameAnnotation(name string, base map[string]string) map[string]string {
annotations := map[string]string{}
for k, v := range base {
@ -152,8 +194,31 @@ func addNameAnnotation(name string, base map[string]string) map[string]string {
return annotations
}
func copySourceLabels(ctx context.Context, infoProvider content.InfoProvider, desc ocispec.Descriptor) (ocispec.Descriptor, error) {
info, err := infoProvider.Info(ctx, desc.Digest)
if err != nil {
return desc, err
}
for k, v := range info.Labels {
if strings.HasPrefix(k, labels.LabelDistributionSource) {
if desc.Annotations == nil {
desc.Annotations = map[string]string{k: v}
} else {
desc.Annotations[k] = v
}
}
}
return desc, nil
}
// ContentProvider provides both content and info about content
type ContentProvider interface {
content.Provider
content.InfoProvider
}
// Export implements Exporter.
func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error {
func Export(ctx context.Context, store ContentProvider, writer io.Writer, opts ...ExportOpt) error {
var eo exportOptions
for _, opt := range opts {
if err := opt(ctx, &eo); err != nil {
@ -163,13 +228,22 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts
records := []tarRecord{
ociLayoutFile(""),
ociIndexRecord(eo.manifests),
}
manifests := make([]ocispec.Descriptor, 0, len(eo.manifests))
for _, desc := range eo.manifests {
d, err := copySourceLabels(ctx, store, desc)
if err != nil {
log.G(ctx).WithError(err).WithField("desc", desc).Warn("failed to copy distribution.source labels")
continue
}
manifests = append(manifests, d)
}
algorithms := map[string]struct{}{}
dManifests := map[digest.Digest]*exportManifest{}
resolvedIndex := map[digest.Digest]digest.Digest{}
for _, desc := range eo.manifests {
for _, desc := range manifests {
if images.IsManifestType(desc.MediaType) {
mt, ok := dManifests[desc.Digest]
if !ok {
@ -259,6 +333,8 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts
}
}
records = append(records, ociIndexRecord(manifests))
if !eo.skipDockerManifest && len(dManifests) > 0 {
tr, err := manifestsRecord(ctx, store, dManifests)
if err != nil {
@ -291,7 +367,10 @@ func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descri
return nil, nil
}
childrenHandler := images.ChildrenHandler(store)
childrenHandler := brOpts.childrenHandler
if childrenHandler == nil {
childrenHandler = images.ChildrenHandler(store)
}
handlers := images.Handlers(
childrenHandler,
@ -313,7 +392,8 @@ type tarRecord struct {
}
type blobRecordOptions struct {
blobFilter BlobFilter
blobFilter BlobFilter
childrenHandler images.HandlerFunc
}
func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOptions) tarRecord {

388
integration/client/export_test.go
View File

@ -19,144 +19,315 @@ package client
import (
"archive/tar"
"context"
"encoding/json"
"io"
"os"
"runtime"
"strings"
"testing"
. "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/content"
"github.com/containerd/containerd/v2/errdefs"
"github.com/containerd/containerd/v2/images"
"github.com/containerd/containerd/v2/images/archive"
"github.com/containerd/containerd/v2/namespaces"
"github.com/containerd/containerd/v2/platforms"
"github.com/google/uuid"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
// TestExport exports testImage as a tar stream
func TestExport(t *testing.T) {
func TestExportAllCases(t *testing.T) {
if testing.Short() {
t.Skip()
}
ctx, cancel := testContext(t)
defer cancel()
client, err := New(address)
if err != nil {
t.Fatal(err)
}
defer client.Close()
for _, tc := range []struct {
name string
prepare func(context.Context, *testing.T, *Client) images.Image
check func(context.Context, *testing.T, *Client, *os.File, images.Image)
}{
{
name: "export all platforms without SkipMissing",
prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image {
if runtime.GOOS == "windows" {
t.Skip("skipping test on windows - the testimage index has only one platform")
}
img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString()), WithAllMetadata())
if err != nil {
t.Fatal(err)
}
return img
},
check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, _ images.Image) {
err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All))
if !errdefs.IsNotFound(err) {
t.Fatal("should fail with not found error")
}
},
},
{
name: "export all platforms with SkipMissing",
prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image {
if runtime.GOOS == "windows" {
t.Skip("skipping test on windows - the testimage index has only one platform")
}
img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString()), WithAllMetadata())
if err != nil {
t.Fatal(err)
}
return img
},
check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) {
defaultPlatformManifest, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default())
if err != nil {
t.Fatal(err)
}
err = client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All), archive.WithSkipMissing(client.ContentStore()))
if err != nil {
t.Fatal(err)
}
dstFile.Seek(0, 0)
assertOCITar(t, dstFile, true)
_, err = client.Fetch(ctx, testImage)
if err != nil {
t.Fatal(err)
}
dstFile, err := os.CreateTemp("", "export-import-test")
if err != nil {
t.Fatal(err)
}
defer func() {
dstFile.Close()
os.Remove(dstFile.Name())
}()
// Check if archive contains only one manifest for the default platform
if !isImageInArchive(ctx, t, client, dstFile, defaultPlatformManifest) {
t.Fatal("archive does not contain manifest for the default platform")
}
err = client.Export(ctx, dstFile, archive.WithPlatform(platforms.Default()), archive.WithImage(client.ImageService(), testImage))
if err != nil {
t.Fatal(err)
}
if isImageInArchive(ctx, t, client, dstFile, img.Target) {
t.Fatal("archive shouldn't contain all platforms")
}
},
},
{
name: "export full image",
prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image {
img, err := client.Fetch(ctx, testImage)
if err != nil {
t.Fatal(err)
}
return img
},
check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) {
err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithPlatform(platforms.All), archive.WithImage(client.ImageService(), testImage))
if err != nil {
t.Fatal(err)
}
// Seek to beginning of file before passing it to assertOCITar()
dstFile.Seek(0, 0)
assertOCITar(t, dstFile, true)
// Seek to beginning of file before passing it to assertOCITar()
dstFile.Seek(0, 0)
assertOCITar(t, dstFile, true)
// Archive should contain all platforms.
if !isImageInArchive(ctx, t, client, dstFile, img.Target) {
t.Fatalf("archive does not contain all platforms")
}
},
},
{
name: "export multi-platform with SkipDockerManifest",
prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image {
img, err := client.Fetch(ctx, testImage)
if err != nil {
t.Fatal(err)
}
return img
},
check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) {
err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithManifest(img.Target), archive.WithSkipDockerManifest())
if err != nil {
t.Fatal(err)
}
// Seek to beginning of file before passing it to assertOCITar()
dstFile.Seek(0, 0)
assertOCITar(t, dstFile, false)
if !isImageInArchive(ctx, t, client, dstFile, img.Target) {
t.Fatalf("archive does not contain expected platform")
}
},
},
{
name: "export single-platform with SkipDockerManifest",
prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image {
img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString()))
if err != nil {
t.Fatal(err)
}
return img
},
check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) {
result, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default())
if err != nil {
t.Fatal(err)
}
err = client.Export(ctx, dstFile, archive.WithManifest(result), archive.WithSkipDockerManifest())
if err != nil {
t.Fatal(err)
}
// Seek to beginning of file before passing it to assertOCITar()
dstFile.Seek(0, 0)
assertOCITar(t, dstFile, false)
if !isImageInArchive(ctx, t, client, dstFile, result) {
t.Fatalf("archive does not contain expected platform")
}
},
},
{
name: "export index only",
prepare: func(ctx context.Context, t *testing.T, client *Client) images.Image {
img, err := client.Fetch(ctx, testImage, WithPlatform(platforms.DefaultString()))
if err != nil {
t.Fatal(err)
}
var all []ocispec.Descriptor
err = images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
ch, err := images.Children(ctx, client.ContentStore(), desc)
if err != nil {
if errdefs.IsNotFound(err) {
return nil, images.ErrSkipDesc
}
return nil, err
}
all = append(all, ch...)
return ch, nil
}), img.Target)
if err != nil {
t.Fatal(err)
}
for _, d := range all {
if images.IsIndexType(d.MediaType) {
continue
}
if err := client.ContentStore().Delete(ctx, d.Digest); err != nil && !errdefs.IsNotFound(err) {
t.Fatalf("failed to delete %v: %v", d.Digest, err)
}
}
return img
},
check: func(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, img images.Image) {
err := client.Export(ctx, dstFile, archive.WithImage(client.ImageService(), testImage), archive.WithSkipMissing(client.ContentStore()))
if err != nil {
t.Fatal(err)
}
// Seek to beginning of file before passing it to assertOCITar()
dstFile.Seek(0, 0)
assertOCITar(t, dstFile, false)
defaultPlatformManifest, err := getPlatformManifest(ctx, client.ContentStore(), img.Target, platforms.Default())
if err != nil {
t.Fatal(err)
}
// Check if archive contains only one manifest for the default platform
if isImageInArchive(ctx, t, client, dstFile, defaultPlatformManifest) {
t.Fatal("archive shouldn't contain manifest for the default platform")
}
},
},
} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx, cancel := testContext(t)
defer cancel()
namespace := uuid.New().String()
client, err := newClient(t, address, WithDefaultNamespace(namespace))
if err != nil {
t.Fatal(err)
}
defer client.Close()
ctx = namespaces.WithNamespace(ctx, namespace)
img := tc.prepare(ctx, t, client)
dstFile, err := os.CreateTemp("", "export-test")
if err != nil {
t.Fatal(err)
}
defer func() {
dstFile.Close()
os.Remove(dstFile.Name())
}()
tc.check(ctx, t, client, dstFile, img)
})
}
}
// TestExportDockerManifest exports testImage as a tar stream, using the
// WithSkipDockerManifest option
func TestExportDockerManifest(t *testing.T) {
if testing.Short() {
t.Skip()
}
ctx, cancel := testContext(t)
defer cancel()
client, err := New(address)
if err != nil {
t.Fatal(err)
}
defer client.Close()
_, err = client.Fetch(ctx, testImage)
if err != nil {
t.Fatal(err)
}
dstFile, err := os.CreateTemp("", "export-import-test")
if err != nil {
t.Fatal(err)
}
defer func() {
dstFile.Close()
os.Remove(dstFile.Name())
}()
img, err := client.ImageService().Get(ctx, testImage)
if err != nil {
t.Fatal(err)
}
// test multi-platform export
err = client.Export(ctx, dstFile, archive.WithManifest(img.Target), archive.WithSkipDockerManifest())
if err != nil {
t.Fatal(err)
}
func isImageInArchive(ctx context.Context, t *testing.T, client *Client, dstFile *os.File, mfst ocispec.Descriptor) bool {
dstFile.Seek(0, 0)
assertOCITar(t, dstFile, false)
tr := tar.NewReader(dstFile)
// reset to beginning
dstFile.Seek(0, 0)
// test single-platform export
var result ocispec.Descriptor
err = images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
if images.IsManifestType(desc.MediaType) {
p, err := content.ReadBlob(ctx, client.ContentStore(), desc)
if err != nil {
return nil, err
var blobs []string
for {
h, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
var manifest ocispec.Manifest
if err := json.Unmarshal(p, &manifest); err != nil {
return nil, err
}
if desc.Platform == nil || platforms.Default().Match(platforms.Normalize(*desc.Platform)) {
result = desc
}
return nil, nil
} else if images.IsIndexType(desc.MediaType) {
p, err := content.ReadBlob(ctx, client.ContentStore(), desc)
if err != nil {
return nil, err
}
var idx ocispec.Index
if err := json.Unmarshal(p, &idx); err != nil {
return nil, err
}
return idx.Manifests, nil
t.Fatal(err)
}
digest := strings.TrimPrefix(h.Name, "blobs/sha256/")
if digest != h.Name && digest != "" {
blobs = append(blobs, digest)
}
return nil, nil
}), img.Target)
if err != nil {
t.Fatal(err)
}
err = client.Export(ctx, dstFile, archive.WithManifest(result), archive.WithSkipDockerManifest())
allPresent := true
// Check if the archive contains all blobs referenced by the manifest.
images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
for _, b := range blobs {
if desc.Digest.Hex() == b {
return images.Children(ctx, client.ContentStore(), desc)
}
}
allPresent = false
return nil, images.ErrStopHandler
}), mfst)
return allPresent
}
func getPlatformManifest(ctx context.Context, cs content.Store, target ocispec.Descriptor, platform platforms.MatchComparer) (ocispec.Descriptor, error) {
mfst, err := images.LimitManifests(images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
children, err := images.Children(ctx, cs, desc)
if !images.IsManifestType(desc.MediaType) {
return children, err
}
if err != nil {
if errdefs.IsNotFound(err) {
return nil, images.ErrSkipDesc
}
return nil, err
}
return children, nil
}), platform, 1)(ctx, target)
if err != nil {
t.Fatal(err)
return ocispec.Descriptor{}, err
}
dstFile.Seek(0, 0)
assertOCITar(t, dstFile, false)
if len(mfst) == 0 {
return ocispec.Descriptor{}, errdefs.ErrNotFound
}
return mfst[0], nil
}
func assertOCITar(t *testing.T, r io.Reader, docker bool) {
t.Helper()
// TODO: add more assertion
tr := tar.NewReader(r)
foundOCILayout := false
@ -168,8 +339,7 @@ func assertOCITar(t *testing.T, r io.Reader, docker bool) {
break
}
if err != nil {
t.Error(err)
continue
t.Fatal(err)
}
if h.Name == ocispec.ImageLayoutFile {
foundOCILayout = true

171
integration/client/import_test.go
View File

@ -35,14 +35,17 @@ import (
"github.com/containerd/containerd/v2/archive/compression"
"github.com/containerd/containerd/v2/archive/tartest"
. "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/content"
"github.com/containerd/containerd/v2/images"
"github.com/containerd/containerd/v2/images/archive"
"github.com/containerd/containerd/v2/leases"
"github.com/containerd/containerd/v2/namespaces"
"github.com/containerd/containerd/v2/oci"
"github.com/containerd/containerd/v2/pkg/transfer"
tarchive "github.com/containerd/containerd/v2/pkg/transfer/archive"
"github.com/containerd/containerd/v2/pkg/transfer/image"
"github.com/containerd/containerd/v2/platforms"
"github.com/google/uuid"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
@ -159,12 +162,6 @@ func TestImport(t *testing.T) {
ctx, cancel := testContext(t)
defer cancel()
client, err := newClient(t, address)
if err != nil {
t.Fatal(err)
}
defer client.Close()
tc := tartest.TarContext{}
b1, d1 := createContent(256, 1)
@ -176,9 +173,12 @@ func TestImport(t *testing.T) {
m1, d3, expManifest := createManifest(c1, [][]byte{b1})
provider := client.ContentStore()
c2, _ := createConfig(runtime.GOOS, runtime.GOARCH, "test2")
m2, d5, _ := createManifest(c2, [][]byte{{1, 2, 3, 4, 5}})
checkManifest := func(ctx context.Context, t *testing.T, d ocispec.Descriptor, expManifest *ocispec.Manifest) {
ml1, d6 := createManifestList(m1, m2)
checkManifest := func(ctx context.Context, t *testing.T, provider content.Provider, d ocispec.Descriptor, expManifest *ocispec.Manifest) {
m, err := images.Manifest(ctx, provider, d, nil)
if err != nil {
t.Fatalf("unable to read target blob: %+v", err)
@ -209,9 +209,40 @@ func TestImport(t *testing.T) {
for _, tc := range []struct {
Name string
Writer tartest.WriterToTar
Check func(*testing.T, []images.Image)
Check func(context.Context, *testing.T, *Client, []images.Image)
Opts []ImportOpt
}{
{
Name: "OCI-IndexWithoutAnyManifest",
Writer: tartest.TarAll(
tc.Dir(ocispec.ImageBlobsDir, 0755),
tc.Dir(ocispec.ImageBlobsDir+"/sha256", 0755),
tc.File(ocispec.ImageIndexFile, createIndex(ml1, ocispec.MediaTypeImageIndex, "docker.io/library/sparse:ok"), 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d6.Encoded(), ml1, 0644),
tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644),
),
Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) {
checkImages(t, d6, imgs, "docker.io/library/sparse:ok")
mfsts, err := images.Children(ctx, client.ContentStore(), imgs[0].Target)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, m := range mfsts {
exists, err := content.Exists(ctx, client.ContentStore(), m)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if exists {
t.Fatal("no manifest should be imported")
}
}
},
Opts: []ImportOpt{
WithSkipMissing(),
},
},
{
Name: "DockerV2.0",
Writer: tartest.TarAll(
@ -232,7 +263,7 @@ func TestImport(t *testing.T) {
tc.File("e95212f7aa2cab51d0abd765cd43.json", c1, 0644),
tc.File("manifest.json", []byte(`[{"Config":"e95212f7aa2cab51d0abd765cd43.json","RepoTags":["test-import:notlatest", "another/repo:tag"],"Layers":["bd765cd43e95212f7aa2cab51d0a/layer.tar"]}]`), 0644),
),
Check: func(t *testing.T, imgs []images.Image) {
Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) {
if len(imgs) == 0 {
t.Fatalf("no images")
}
@ -243,7 +274,7 @@ func TestImport(t *testing.T) {
}
checkImages(t, imgs[0].Target.Digest, imgs, names...)
checkManifest(ctx, t, imgs[0].Target, nil)
checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, nil)
},
},
{
@ -271,17 +302,17 @@ func TestImport(t *testing.T) {
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644),
tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "docker.io/lib/img:ok"), 0644),
tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "docker.io/lib/img:ok"), 0644),
tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644),
),
Check: func(t *testing.T, imgs []images.Image) {
Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) {
names := []string{
"latest",
"docker.io/lib/img:ok",
}
checkImages(t, d3, imgs, names...)
checkManifest(ctx, t, imgs[0].Target, expManifest)
checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest)
},
},
{
@ -292,17 +323,17 @@ func TestImport(t *testing.T) {
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644),
tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "docker.io/lib/img:ok"), 0644),
tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "docker.io/lib/img:ok"), 0644),
tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644),
),
Check: func(t *testing.T, imgs []images.Image) {
Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) {
names := []string{
"localhost:5000/myimage:latest",
"docker.io/lib/img:ok",
}
checkImages(t, d3, imgs, names...)
checkManifest(ctx, t, imgs[0].Target, expManifest)
checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest)
},
Opts: []ImportOpt{
WithImageRefTranslator(archive.AddRefPrefix("localhost:5000/myimage")),
@ -316,24 +347,94 @@ func TestImport(t *testing.T) {
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644),
tc.File(ocispec.ImageIndexFile, createIndex(m1, "latest", "localhost:5000/myimage:old", "docker.io/lib/img:ok"), 0644),
tc.File(ocispec.ImageIndexFile, createIndex(m1, ocispec.MediaTypeImageManifest, "latest", "localhost:5000/myimage:old", "docker.io/lib/img:ok"), 0644),
tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644),
),
Check: func(t *testing.T, imgs []images.Image) {
Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) {
names := []string{
"localhost:5000/myimage:latest",
"localhost:5000/myimage:old",
}
checkImages(t, d3, imgs, names...)
checkManifest(ctx, t, imgs[0].Target, expManifest)
checkManifest(ctx, t, client.ContentStore(), imgs[0].Target, expManifest)
},
Opts: []ImportOpt{
WithImageRefTranslator(archive.FilterRefPrefix("localhost:5000/myimage")),
},
},
{
Name: "OCI-IndexWithMissingManifestDescendants",
Writer: tartest.TarAll(
tc.Dir(ocispec.ImageBlobsDir, 0755),
tc.Dir(ocispec.ImageBlobsDir+"/sha256", 0755),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d1.Encoded(), b1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d2.Encoded(), c1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d3.Encoded(), m1, 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d5.Encoded(), m2, 0644),
tc.File(ocispec.ImageIndexFile, createIndex(ml1, ocispec.MediaTypeImageIndex, "docker.io/library/sparse:ok"), 0644),
tc.File(ocispec.ImageBlobsDir+"/sha256/"+d6.Encoded(), ml1, 0644),
tc.File(ocispec.ImageLayoutFile, []byte(`{"imageLayoutVersion":"`+ocispec.ImageLayoutVersion+`"}`), 0644),
),
Check: func(ctx context.Context, t *testing.T, client *Client, imgs []images.Image) {
checkImages(t, d6, imgs, "docker.io/library/sparse:ok")
mfsts, err := images.Children(ctx, client.ContentStore(), imgs[0].Target)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var importedManifest *ocispec.Descriptor
var secondManifest *ocispec.Descriptor
for _, m := range mfsts {
exists, err := content.Exists(ctx, client.ContentStore(), m)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if exists {
if m.Digest == d3 {
m := m
importedManifest = &m
} else if m.Digest == d5 {
m := m
secondManifest = &m
} else {
t.Fatalf("imported manifest with unexpected digest: %v", m.Digest)
}
}
}
if importedManifest == nil {
t.Fatal("the expected manifest was not loaded")
}
checkManifest(ctx, t, client.ContentStore(), *importedManifest, expManifest)
if secondManifest == nil {
t.Fatal("the expected manifest was not loaded")
}
_, _, _, missing, err := images.Check(ctx, client.ContentStore(), *secondManifest, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(missing) != 2 {
t.Fatalf("expected 2 missing blobs, got %+v", missing)
}
},
Opts: []ImportOpt{
WithSkipMissing(),
},
},
} {
t.Run(tc.Name, func(t *testing.T) {
client, err := newClient(t, address)
if err != nil {
t.Fatal(err)
}
defer client.Close()
ctx := namespaces.WithNamespace(ctx, uuid.New().String())
images, err := client.Import(ctx, tartest.TarFromWriterTo(tc.Writer), tc.Opts...)
if err != nil {
if tc.Check != nil {
@ -344,7 +445,7 @@ func TestImport(t *testing.T) {
t.Fatalf("expected error on import")
}
tc.Check(t, images)
tc.Check(ctx, t, client, images)
})
}
}
@ -363,7 +464,8 @@ func checkImages(t *testing.T, target digest.Digest, actual []images.Image, name
}
if actual[i].Target.MediaType != ocispec.MediaTypeImageManifest &&
actual[i].Target.MediaType != images.MediaTypeDockerSchema2Manifest {
actual[i].Target.MediaType != images.MediaTypeDockerSchema2Manifest &&
actual[i].Target.MediaType != ocispec.MediaTypeImageIndex {
t.Fatalf("image(%d) unexpected media type: %s", i, actual[i].Target.MediaType)
}
}
@ -430,14 +532,35 @@ func createManifest(config []byte, layers [][]byte) ([]byte, digest.Digest, *oci
return b, digest.FromBytes(b), &manifest
}
func createIndex(manifest []byte, tags ...string) []byte {
func createManifestList(manifests ...[]byte) ([]byte, digest.Digest) {
idx := ocispec.Index{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
}
for _, manifest := range manifests {
d := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
Digest: digest.FromBytes(manifest),
Size: int64(len(manifest)),
}
idx.Manifests = append(idx.Manifests, d)
}
b, _ := json.Marshal(idx)
return b, digest.FromBytes(b)
}
func createIndex(manifest []byte, mt string, tags ...string) []byte {
idx := ocispec.Index{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
}
d := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
MediaType: mt,
Digest: digest.FromBytes(manifest),
Size: int64(len(manifest)),
}