Move images to core/images
Signed-off-by: Derek McGowan <derek@mcg.dev>
This commit is contained in:
Derek McGowan
23
core/images/annotations.go
Normal file
23
core/images/annotations.go
Normal file
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package images
|
||||
|
||||
const (
|
||||
// AnnotationImageName is an annotation on a Descriptor in an index.json
|
||||
// containing the `Name` value as used by an `Image` struct
|
||||
AnnotationImageName = "io.containerd.image.name"
|
||||
)
|
||||
588
core/images/archive/exporter.go
Normal file
588
core/images/archive/exporter.go
Normal file
@@ -0,0 +1,588 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
"github.com/containerd/containerd/v2/labels"
|
||||
"github.com/containerd/containerd/v2/platforms"
|
||||
"github.com/containerd/log"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispecs "github.com/opencontainers/image-spec/specs-go"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
type exportOptions struct {
|
||||
manifests []ocispec.Descriptor
|
||||
platform platforms.MatchComparer
|
||||
allPlatforms bool
|
||||
skipDockerManifest bool
|
||||
blobRecordOptions blobRecordOptions
|
||||
}
|
||||
|
||||
// ExportOpt defines options for configuring exported descriptors
|
||||
type ExportOpt func(context.Context, *exportOptions) error
|
||||
|
||||
// WithPlatform defines the platform to require manifest lists have
|
||||
// not exporting all platforms.
|
||||
// Additionally, platform is used to resolve image configs for
|
||||
// Docker v1.1, v1.2 format compatibility.
|
||||
func WithPlatform(p platforms.MatchComparer) ExportOpt {
|
||||
return func(ctx context.Context, o *exportOptions) error {
|
||||
o.platform = p
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithAllPlatforms exports all manifests from a manifest list.
|
||||
// Missing content will fail the export.
|
||||
func WithAllPlatforms() ExportOpt {
|
||||
return func(ctx context.Context, o *exportOptions) error {
|
||||
o.allPlatforms = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithSkipDockerManifest skips creation of the Docker compatible
|
||||
// manifest.json file.
|
||||
func WithSkipDockerManifest() ExportOpt {
|
||||
return func(ctx context.Context, o *exportOptions) error {
|
||||
o.skipDockerManifest = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithImage adds the provided images to the exported archive.
|
||||
func WithImage(is images.Store, name string) ExportOpt {
|
||||
return func(ctx context.Context, o *exportOptions) error {
|
||||
img, err := is.Get(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
img.Target.Annotations = addNameAnnotation(name, img.Target.Annotations)
|
||||
o.manifests = append(o.manifests, img.Target)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithImages adds multiples images to the exported archive.
|
||||
func WithImages(imgs []images.Image) ExportOpt {
|
||||
return func(ctx context.Context, o *exportOptions) error {
|
||||
for _, img := range imgs {
|
||||
img.Target.Annotations = addNameAnnotation(img.Name, img.Target.Annotations)
|
||||
o.manifests = append(o.manifests, img.Target)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifest adds a manifest to the exported archive.
|
||||
// When names are given they will be set on the manifest in the
|
||||
// exported archive, creating an index record for each name.
|
||||
// When no names are provided, it is up to caller to put name annotation to
|
||||
// on the manifest descriptor if needed.
|
||||
func WithManifest(manifest ocispec.Descriptor, names ...string) ExportOpt {
|
||||
return func(ctx context.Context, o *exportOptions) error {
|
||||
if len(names) == 0 {
|
||||
o.manifests = append(o.manifests, manifest)
|
||||
}
|
||||
for _, name := range names {
|
||||
mc := manifest
|
||||
mc.Annotations = addNameAnnotation(name, manifest.Annotations)
|
||||
o.manifests = append(o.manifests, mc)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// BlobFilter returns false if the blob should not be included in the archive.
|
||||
type BlobFilter func(ocispec.Descriptor) bool
|
||||
|
||||
// WithBlobFilter specifies BlobFilter.
|
||||
func WithBlobFilter(f BlobFilter) ExportOpt {
|
||||
return func(ctx context.Context, o *exportOptions) error {
|
||||
o.blobRecordOptions.blobFilter = f
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithSkipNonDistributableBlobs excludes non-distributable blobs such as Windows base layers.
|
||||
func WithSkipNonDistributableBlobs() ExportOpt {
|
||||
f := func(desc ocispec.Descriptor) bool {
|
||||
return !images.IsNonDistributable(desc.MediaType)
|
||||
}
|
||||
return WithBlobFilter(f)
|
||||
}
|
||||
|
||||
// WithSkipMissing excludes blobs referenced by manifests if not all blobs
|
||||
// would be included in the archive.
|
||||
// The manifest itself is excluded only if it's not present locally.
|
||||
// This allows to export multi-platform images if not all platforms are present
|
||||
// while still persisting the multi-platform index.
|
||||
func WithSkipMissing(store content.InfoReaderProvider) ExportOpt {
|
||||
return func(ctx context.Context, o *exportOptions) error {
|
||||
o.blobRecordOptions.childrenHandler = images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
|
||||
children, err := images.Children(ctx, store, desc)
|
||||
if !images.IsManifestType(desc.MediaType) {
|
||||
return children, err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// If manifest itself is missing, skip it from export.
|
||||
if errdefs.IsNotFound(err) {
|
||||
return nil, images.ErrSkipDesc
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Don't export manifest descendants if any of them doesn't exist.
|
||||
for _, child := range children {
|
||||
exists, err := content.Exists(ctx, store, child)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If any child is missing, only export the manifest, but don't export its descendants.
|
||||
if !exists {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
return children, nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func addNameAnnotation(name string, base map[string]string) map[string]string {
|
||||
annotations := map[string]string{}
|
||||
for k, v := range base {
|
||||
annotations[k] = v
|
||||
}
|
||||
|
||||
annotations[images.AnnotationImageName] = name
|
||||
annotations[ocispec.AnnotationRefName] = ociReferenceName(name)
|
||||
|
||||
return annotations
|
||||
}
|
||||
|
||||
func copySourceLabels(ctx context.Context, infoProvider content.InfoProvider, desc ocispec.Descriptor) (ocispec.Descriptor, error) {
|
||||
info, err := infoProvider.Info(ctx, desc.Digest)
|
||||
if err != nil {
|
||||
return desc, err
|
||||
}
|
||||
for k, v := range info.Labels {
|
||||
if strings.HasPrefix(k, labels.LabelDistributionSource) {
|
||||
if desc.Annotations == nil {
|
||||
desc.Annotations = map[string]string{k: v}
|
||||
} else {
|
||||
desc.Annotations[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
return desc, nil
|
||||
}
|
||||
|
||||
// Export implements Exporter.
|
||||
func Export(ctx context.Context, store content.InfoReaderProvider, writer io.Writer, opts ...ExportOpt) error {
|
||||
var eo exportOptions
|
||||
for _, opt := range opts {
|
||||
if err := opt(ctx, &eo); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
records := []tarRecord{
|
||||
ociLayoutFile(""),
|
||||
}
|
||||
|
||||
manifests := make([]ocispec.Descriptor, 0, len(eo.manifests))
|
||||
for _, desc := range eo.manifests {
|
||||
d, err := copySourceLabels(ctx, store, desc)
|
||||
if err != nil {
|
||||
log.G(ctx).WithError(err).WithField("desc", desc).Warn("failed to copy distribution.source labels")
|
||||
continue
|
||||
}
|
||||
manifests = append(manifests, d)
|
||||
}
|
||||
|
||||
algorithms := map[string]struct{}{}
|
||||
dManifests := map[digest.Digest]*exportManifest{}
|
||||
resolvedIndex := map[digest.Digest]digest.Digest{}
|
||||
for _, desc := range manifests {
|
||||
if images.IsManifestType(desc.MediaType) {
|
||||
mt, ok := dManifests[desc.Digest]
|
||||
if !ok {
|
||||
// TODO(containerd): Skip if already added
|
||||
r, err := getRecords(ctx, store, desc, algorithms, &eo.blobRecordOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = append(records, r...)
|
||||
|
||||
mt = &exportManifest{
|
||||
manifest: desc,
|
||||
}
|
||||
dManifests[desc.Digest] = mt
|
||||
}
|
||||
|
||||
name := desc.Annotations[images.AnnotationImageName]
|
||||
if name != "" {
|
||||
mt.names = append(mt.names, name)
|
||||
}
|
||||
} else if images.IsIndexType(desc.MediaType) {
|
||||
d, ok := resolvedIndex[desc.Digest]
|
||||
if !ok {
|
||||
if err := desc.Digest.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
records = append(records, blobRecord(store, desc, &eo.blobRecordOptions))
|
||||
|
||||
p, err := content.ReadBlob(ctx, store, desc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var index ocispec.Index
|
||||
if err := json.Unmarshal(p, &index); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var manifests []ocispec.Descriptor
|
||||
for _, m := range index.Manifests {
|
||||
if eo.platform != nil {
|
||||
if m.Platform == nil || eo.platform.Match(*m.Platform) {
|
||||
manifests = append(manifests, m)
|
||||
} else if !eo.allPlatforms {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
r, err := getRecords(ctx, store, m, algorithms, &eo.blobRecordOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records = append(records, r...)
|
||||
}
|
||||
|
||||
if len(manifests) >= 1 {
|
||||
if len(manifests) > 1 {
|
||||
sort.SliceStable(manifests, func(i, j int) bool {
|
||||
if manifests[i].Platform == nil {
|
||||
return false
|
||||
}
|
||||
if manifests[j].Platform == nil {
|
||||
return true
|
||||
}
|
||||
return eo.platform.Less(*manifests[i].Platform, *manifests[j].Platform)
|
||||
})
|
||||
}
|
||||
d = manifests[0].Digest
|
||||
dManifests[d] = &exportManifest{
|
||||
manifest: manifests[0],
|
||||
}
|
||||
} else if eo.platform != nil {
|
||||
return fmt.Errorf("no manifest found for platform: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
resolvedIndex[desc.Digest] = d
|
||||
}
|
||||
if d != "" {
|
||||
if name := desc.Annotations[images.AnnotationImageName]; name != "" {
|
||||
mt := dManifests[d]
|
||||
mt.names = append(mt.names, name)
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("only manifests may be exported: %w", errdefs.ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
|
||||
records = append(records, ociIndexRecord(manifests))
|
||||
|
||||
if !eo.skipDockerManifest && len(dManifests) > 0 {
|
||||
tr, err := manifestsRecord(ctx, store, dManifests)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create manifests file: %w", err)
|
||||
}
|
||||
|
||||
records = append(records, tr)
|
||||
}
|
||||
|
||||
if len(algorithms) > 0 {
|
||||
records = append(records, directoryRecord("blobs/", 0755))
|
||||
for alg := range algorithms {
|
||||
records = append(records, directoryRecord("blobs/"+alg+"/", 0755))
|
||||
}
|
||||
}
|
||||
|
||||
tw := tar.NewWriter(writer)
|
||||
defer tw.Close()
|
||||
return writeTar(ctx, tw, records)
|
||||
}
|
||||
|
||||
func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descriptor, algorithms map[string]struct{}, brOpts *blobRecordOptions) ([]tarRecord, error) {
|
||||
var records []tarRecord
|
||||
exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
if err := desc.Digest.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, blobRecord(store, desc, brOpts))
|
||||
algorithms[desc.Digest.Algorithm().String()] = struct{}{}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
childrenHandler := brOpts.childrenHandler
|
||||
if childrenHandler == nil {
|
||||
childrenHandler = images.ChildrenHandler(store)
|
||||
}
|
||||
|
||||
handlers := images.Handlers(
|
||||
childrenHandler,
|
||||
images.HandlerFunc(exportHandler),
|
||||
)
|
||||
|
||||
// Walk sequentially since the number of fetches is likely one and doing in
|
||||
// parallel requires locking the export handler
|
||||
if err := images.Walk(ctx, handlers, desc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
type tarRecord struct {
|
||||
Header *tar.Header
|
||||
CopyTo func(context.Context, io.Writer) (int64, error)
|
||||
}
|
||||
|
||||
type blobRecordOptions struct {
|
||||
blobFilter BlobFilter
|
||||
childrenHandler images.HandlerFunc
|
||||
}
|
||||
|
||||
func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOptions) tarRecord {
|
||||
if opts != nil && opts.blobFilter != nil && !opts.blobFilter(desc) {
|
||||
return tarRecord{}
|
||||
}
|
||||
return tarRecord{
|
||||
Header: &tar.Header{
|
||||
Name: path.Join(ocispec.ImageBlobsDir, desc.Digest.Algorithm().String(), desc.Digest.Encoded()),
|
||||
Mode: 0444,
|
||||
Size: desc.Size,
|
||||
Typeflag: tar.TypeReg,
|
||||
},
|
||||
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
|
||||
r, err := cs.ReaderAt(ctx, desc)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get reader: %w", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
// Verify digest
|
||||
dgstr := desc.Digest.Algorithm().Digester()
|
||||
|
||||
n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to copy to tar: %w", err)
|
||||
}
|
||||
if dgstr.Digest() != desc.Digest {
|
||||
return 0, fmt.Errorf("unexpected digest %s copied", dgstr.Digest())
|
||||
}
|
||||
return n, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func directoryRecord(name string, mode int64) tarRecord {
|
||||
return tarRecord{
|
||||
Header: &tar.Header{
|
||||
Name: name,
|
||||
Mode: mode,
|
||||
Typeflag: tar.TypeDir,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ociLayoutFile(version string) tarRecord {
|
||||
if version == "" {
|
||||
version = ocispec.ImageLayoutVersion
|
||||
}
|
||||
layout := ocispec.ImageLayout{
|
||||
Version: version,
|
||||
}
|
||||
|
||||
b, err := json.Marshal(layout)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return tarRecord{
|
||||
Header: &tar.Header{
|
||||
Name: ocispec.ImageLayoutFile,
|
||||
Mode: 0444,
|
||||
Size: int64(len(b)),
|
||||
Typeflag: tar.TypeReg,
|
||||
},
|
||||
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
|
||||
n, err := w.Write(b)
|
||||
return int64(n), err
|
||||
},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func ociIndexRecord(manifests []ocispec.Descriptor) tarRecord {
|
||||
index := ocispec.Index{
|
||||
Versioned: ocispecs.Versioned{
|
||||
SchemaVersion: 2,
|
||||
},
|
||||
Manifests: manifests,
|
||||
}
|
||||
|
||||
b, err := json.Marshal(index)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return tarRecord{
|
||||
Header: &tar.Header{
|
||||
Name: ocispec.ImageIndexFile,
|
||||
Mode: 0644,
|
||||
Size: int64(len(b)),
|
||||
Typeflag: tar.TypeReg,
|
||||
},
|
||||
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
|
||||
n, err := w.Write(b)
|
||||
return int64(n), err
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type exportManifest struct {
|
||||
manifest ocispec.Descriptor
|
||||
names []string
|
||||
}
|
||||
|
||||
func manifestsRecord(ctx context.Context, store content.Provider, manifests map[digest.Digest]*exportManifest) (tarRecord, error) {
|
||||
mfsts := make([]struct {
|
||||
Config string
|
||||
RepoTags []string
|
||||
Layers []string
|
||||
}, len(manifests))
|
||||
|
||||
var i int
|
||||
for _, m := range manifests {
|
||||
p, err := content.ReadBlob(ctx, store, m.manifest)
|
||||
if err != nil {
|
||||
return tarRecord{}, err
|
||||
}
|
||||
|
||||
var manifest ocispec.Manifest
|
||||
if err := json.Unmarshal(p, &manifest); err != nil {
|
||||
return tarRecord{}, err
|
||||
}
|
||||
|
||||
dgst := manifest.Config.Digest
|
||||
if err := dgst.Validate(); err != nil {
|
||||
return tarRecord{}, err
|
||||
}
|
||||
mfsts[i].Config = path.Join(ocispec.ImageBlobsDir, dgst.Algorithm().String(), dgst.Encoded())
|
||||
for _, l := range manifest.Layers {
|
||||
mfsts[i].Layers = append(mfsts[i].Layers, path.Join(ocispec.ImageBlobsDir, l.Digest.Algorithm().String(), l.Digest.Encoded()))
|
||||
}
|
||||
|
||||
for _, name := range m.names {
|
||||
nname, err := familiarizeReference(name)
|
||||
if err != nil {
|
||||
return tarRecord{}, err
|
||||
}
|
||||
|
||||
mfsts[i].RepoTags = append(mfsts[i].RepoTags, nname)
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
b, err := json.Marshal(mfsts)
|
||||
if err != nil {
|
||||
return tarRecord{}, err
|
||||
}
|
||||
|
||||
return tarRecord{
|
||||
Header: &tar.Header{
|
||||
Name: "manifest.json",
|
||||
Mode: 0644,
|
||||
Size: int64(len(b)),
|
||||
Typeflag: tar.TypeReg,
|
||||
},
|
||||
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
|
||||
n, err := w.Write(b)
|
||||
return int64(n), err
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func writeTar(ctx context.Context, tw *tar.Writer, recordsWithEmpty []tarRecord) error {
|
||||
var records []tarRecord
|
||||
for _, r := range recordsWithEmpty {
|
||||
if r.Header != nil {
|
||||
records = append(records, r)
|
||||
}
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
return records[i].Header.Name < records[j].Header.Name
|
||||
})
|
||||
|
||||
var last string
|
||||
for _, record := range records {
|
||||
if record.Header.Name == last {
|
||||
continue
|
||||
}
|
||||
last = record.Header.Name
|
||||
if err := tw.WriteHeader(record.Header); err != nil {
|
||||
return err
|
||||
}
|
||||
if record.CopyTo != nil {
|
||||
n, err := record.CopyTo(ctx, tw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != record.Header.Size {
|
||||
return fmt.Errorf("unexpected copy size for %s", record.Header.Name)
|
||||
}
|
||||
} else if record.Header.Size > 0 {
|
||||
return fmt.Errorf("no content to write to record with non-zero size for %s", record.Header.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
420
core/images/archive/importer.go
Normal file
420
core/images/archive/importer.go
Normal file
@@ -0,0 +1,420 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package archive provides a Docker and OCI compatible importer
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
|
||||
"github.com/containerd/containerd/v2/archive/compression"
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
"github.com/containerd/containerd/v2/labels"
|
||||
"github.com/containerd/containerd/v2/platforms"
|
||||
"github.com/containerd/log"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
specs "github.com/opencontainers/image-spec/specs-go"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
type importOpts struct {
|
||||
compress bool
|
||||
}
|
||||
|
||||
// ImportOpt is an option for importing an OCI index
|
||||
type ImportOpt func(*importOpts) error
|
||||
|
||||
// WithImportCompression compresses uncompressed layers on import.
|
||||
// This is used for import formats which do not include the manifest.
|
||||
func WithImportCompression() ImportOpt {
|
||||
return func(io *importOpts) error {
|
||||
io.compress = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ImportIndex imports an index from a tar archive image bundle
|
||||
// - implements Docker v1.1, v1.2 and OCI v1.
|
||||
// - prefers OCI v1 when provided
|
||||
// - creates OCI index for Docker formats
|
||||
// - normalizes Docker references and adds as OCI ref name
|
||||
// e.g. alpine:latest -> docker.io/library/alpine:latest
|
||||
// - existing OCI reference names are untouched
|
||||
func ImportIndex(ctx context.Context, store content.Store, reader io.Reader, opts ...ImportOpt) (ocispec.Descriptor, error) {
|
||||
var (
|
||||
tr = tar.NewReader(reader)
|
||||
|
||||
ociLayout ocispec.ImageLayout
|
||||
mfsts []struct {
|
||||
Config string
|
||||
RepoTags []string
|
||||
Layers []string
|
||||
}
|
||||
symlinks = make(map[string]string)
|
||||
blobs = make(map[string]ocispec.Descriptor)
|
||||
iopts importOpts
|
||||
)
|
||||
|
||||
for _, o := range opts {
|
||||
if err := o(&iopts); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeSymlink {
|
||||
symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname)
|
||||
}
|
||||
|
||||
//nolint:staticcheck // TypeRegA is deprecated but we may still receive an external tar with TypeRegA
|
||||
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
|
||||
if hdr.Typeflag != tar.TypeDir {
|
||||
log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
hdrName := path.Clean(hdr.Name)
|
||||
if hdrName == ocispec.ImageLayoutFile {
|
||||
if err = onUntarJSON(tr, &ociLayout); err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("untar oci layout %q: %w", hdr.Name, err)
|
||||
}
|
||||
} else if hdrName == "manifest.json" {
|
||||
if err = onUntarJSON(tr, &mfsts); err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("untar manifest %q: %w", hdr.Name, err)
|
||||
}
|
||||
} else {
|
||||
dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("failed to ingest %q: %w", hdr.Name, err)
|
||||
}
|
||||
|
||||
blobs[hdrName] = ocispec.Descriptor{
|
||||
Digest: dgst,
|
||||
Size: hdr.Size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If OCI layout was given, interpret the tar as an OCI layout.
|
||||
// When not provided, the layout of the tar will be interpreted
|
||||
// as Docker v1.1 or v1.2.
|
||||
if ociLayout.Version != "" {
|
||||
if ociLayout.Version != ocispec.ImageLayoutVersion {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("unsupported OCI version %s", ociLayout.Version)
|
||||
}
|
||||
|
||||
idx, ok := blobs[ocispec.ImageIndexFile]
|
||||
if !ok {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion)
|
||||
}
|
||||
|
||||
idx.MediaType = ocispec.MediaTypeImageIndex
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
if mfsts == nil {
|
||||
return ocispec.Descriptor{}, errors.New("unrecognized image format")
|
||||
}
|
||||
|
||||
for name, linkname := range symlinks {
|
||||
desc, ok := blobs[linkname]
|
||||
if !ok {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("no target for symlink layer from %q to %q", name, linkname)
|
||||
}
|
||||
blobs[name] = desc
|
||||
}
|
||||
|
||||
idx := ocispec.Index{
|
||||
Versioned: specs.Versioned{
|
||||
SchemaVersion: 2,
|
||||
},
|
||||
}
|
||||
for _, mfst := range mfsts {
|
||||
config, ok := blobs[mfst.Config]
|
||||
if !ok {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("image config %q not found", mfst.Config)
|
||||
}
|
||||
config.MediaType = images.MediaTypeDockerSchema2Config
|
||||
|
||||
layers, err := resolveLayers(ctx, store, mfst.Layers, blobs, iopts.compress)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("failed to resolve layers: %w", err)
|
||||
}
|
||||
|
||||
manifest := struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Config ocispec.Descriptor `json:"config"`
|
||||
Layers []ocispec.Descriptor `json:"layers"`
|
||||
}{
|
||||
SchemaVersion: 2,
|
||||
MediaType: images.MediaTypeDockerSchema2Manifest,
|
||||
Config: config,
|
||||
Layers: layers,
|
||||
}
|
||||
|
||||
desc, err := writeManifest(ctx, store, manifest, manifest.MediaType)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("write docker manifest: %w", err)
|
||||
}
|
||||
|
||||
imgPlatforms, err := images.Platforms(ctx, store, desc)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("unable to resolve platform: %w", err)
|
||||
}
|
||||
if len(imgPlatforms) > 0 {
|
||||
// Only one platform can be resolved from non-index manifest,
|
||||
// The platform can only come from the config included above,
|
||||
// if the config has no platform it can be safely omitted.
|
||||
desc.Platform = &imgPlatforms[0]
|
||||
|
||||
// If the image we've just imported is a Windows image without the OSVersion set,
|
||||
// we could just assume it matches this host's OS Version. Without this, the
|
||||
// children labels might not be set on the image content, leading to it being
|
||||
// garbage collected, breaking the image.
|
||||
// See: https://github.com/containerd/containerd/issues/5690
|
||||
if desc.Platform.OS == "windows" && desc.Platform.OSVersion == "" {
|
||||
platform := platforms.DefaultSpec()
|
||||
desc.Platform.OSVersion = platform.OSVersion
|
||||
}
|
||||
}
|
||||
|
||||
if len(mfst.RepoTags) == 0 {
|
||||
idx.Manifests = append(idx.Manifests, desc)
|
||||
} else {
|
||||
// Add descriptor per tag
|
||||
for _, ref := range mfst.RepoTags {
|
||||
mfstdesc := desc
|
||||
|
||||
normalized, err := normalizeReference(ref)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
mfstdesc.Annotations = map[string]string{
|
||||
images.AnnotationImageName: normalized,
|
||||
ocispec.AnnotationRefName: ociReferenceName(normalized),
|
||||
}
|
||||
|
||||
idx.Manifests = append(idx.Manifests, mfstdesc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex)
|
||||
}
|
||||
|
||||
const (
|
||||
kib = 1024
|
||||
mib = 1024 * kib
|
||||
jsonLimit = 20 * mib
|
||||
)
|
||||
|
||||
func onUntarJSON(r io.Reader, j interface{}) error {
|
||||
return json.NewDecoder(io.LimitReader(r, jsonLimit)).Decode(j)
|
||||
}
|
||||
|
||||
func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) {
|
||||
dgstr := digest.Canonical.Digester()
|
||||
|
||||
if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return dgstr.Digest(), nil
|
||||
}
|
||||
|
||||
func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor, compress bool) ([]ocispec.Descriptor, error) {
|
||||
layers := make([]ocispec.Descriptor, len(layerFiles))
|
||||
descs := map[digest.Digest]*ocispec.Descriptor{}
|
||||
filters := []string{}
|
||||
for i, f := range layerFiles {
|
||||
desc, ok := blobs[f]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("layer %q not found", f)
|
||||
}
|
||||
layers[i] = desc
|
||||
descs[desc.Digest] = &layers[i]
|
||||
filters = append(filters, fmt.Sprintf("labels.\"%s\"==%s", labels.LabelUncompressed, desc.Digest.String()))
|
||||
}
|
||||
|
||||
err := store.Walk(ctx, func(info content.Info) error {
|
||||
dgst, ok := info.Labels[labels.LabelUncompressed]
|
||||
if ok {
|
||||
desc := descs[digest.Digest(dgst)]
|
||||
if desc != nil {
|
||||
desc.Digest = info.Digest
|
||||
desc.Size = info.Size
|
||||
mediaType, err := detectLayerMediaType(ctx, store, *desc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect media type of layer: %w", err)
|
||||
}
|
||||
desc.MediaType = mediaType
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, filters...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failure checking for compressed blobs: %w", err)
|
||||
}
|
||||
|
||||
for i, desc := range layers {
|
||||
if desc.MediaType != "" {
|
||||
continue
|
||||
}
|
||||
// Open blob, resolve media type
|
||||
ra, err := store.ReaderAt(ctx, desc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open %q (%s): %w", layerFiles[i], desc.Digest, err)
|
||||
}
|
||||
s, err := compression.DecompressStream(content.NewReader(ra))
|
||||
if err != nil {
|
||||
ra.Close()
|
||||
return nil, fmt.Errorf("failed to detect compression for %q: %w", layerFiles[i], err)
|
||||
}
|
||||
if s.GetCompression() == compression.Uncompressed {
|
||||
if compress {
|
||||
if err := desc.Digest.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
|
||||
labels := map[string]string{
|
||||
labels.LabelUncompressed: desc.Digest.String(),
|
||||
}
|
||||
layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
|
||||
if err != nil {
|
||||
s.Close()
|
||||
ra.Close()
|
||||
return nil, err
|
||||
}
|
||||
layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
|
||||
} else {
|
||||
layers[i].MediaType = images.MediaTypeDockerSchema2Layer
|
||||
}
|
||||
} else {
|
||||
layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
|
||||
}
|
||||
s.Close()
|
||||
ra.Close()
|
||||
}
|
||||
return layers, nil
|
||||
}
|
||||
|
||||
func compressBlob(ctx context.Context, cs content.Store, r io.Reader, ref string, opts ...content.Opt) (desc ocispec.Descriptor, err error) {
|
||||
w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("failed to open writer: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
w.Close()
|
||||
if err != nil {
|
||||
cs.Abort(ctx, ref)
|
||||
}
|
||||
}()
|
||||
if err := w.Truncate(0); err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("failed to truncate writer: %w", err)
|
||||
}
|
||||
|
||||
cw, err := compression.CompressStream(w, compression.Gzip)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(cw, r); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
if err := cw.Close(); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
cst, err := w.Status()
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("failed to get writer status: %w", err)
|
||||
}
|
||||
|
||||
desc.Digest = w.Digest()
|
||||
desc.Size = cst.Offset
|
||||
|
||||
if err := w.Commit(ctx, desc.Size, desc.Digest, opts...); err != nil {
|
||||
if !errdefs.IsAlreadyExists(err) {
|
||||
return ocispec.Descriptor{}, fmt.Errorf("failed to commit: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return desc, nil
|
||||
}
|
||||
|
||||
func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) {
|
||||
manifestBytes, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
desc := ocispec.Descriptor{
|
||||
MediaType: mediaType,
|
||||
Digest: digest.FromBytes(manifestBytes),
|
||||
Size: int64(len(manifestBytes)),
|
||||
}
|
||||
if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
return desc, nil
|
||||
}
|
||||
|
||||
func detectLayerMediaType(ctx context.Context, store content.Store, desc ocispec.Descriptor) (string, error) {
|
||||
var mediaType string
|
||||
// need to parse existing blob to use the proper media type
|
||||
bytes := make([]byte, 10)
|
||||
ra, err := store.ReaderAt(ctx, desc)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read content store to detect layer media type: %w", err)
|
||||
}
|
||||
defer ra.Close()
|
||||
_, err = ra.ReadAt(bytes, 0)
|
||||
if err != nil && err != io.EOF {
|
||||
return "", fmt.Errorf("failed to read header bytes from layer to detect media type: %w", err)
|
||||
}
|
||||
if err == io.EOF {
|
||||
// in the case of an empty layer then the media type should be uncompressed
|
||||
return images.MediaTypeDockerSchema2Layer, nil
|
||||
}
|
||||
switch c := compression.DetectCompression(bytes); c {
|
||||
case compression.Uncompressed:
|
||||
mediaType = images.MediaTypeDockerSchema2Layer
|
||||
default:
|
||||
mediaType = images.MediaTypeDockerSchema2LayerGzip
|
||||
}
|
||||
return mediaType, nil
|
||||
}
|
||||
115
core/images/archive/reference.go
Normal file
115
core/images/archive/reference.go
Normal file
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/containerd/containerd/v2/reference"
|
||||
distref "github.com/distribution/reference"
|
||||
"github.com/opencontainers/go-digest"
|
||||
)
|
||||
|
||||
// FilterRefPrefix restricts references to having the given image
|
||||
// prefix. Tag-only references will have the prefix prepended.
|
||||
func FilterRefPrefix(image string) func(string) string {
|
||||
return refTranslator(image, true)
|
||||
}
|
||||
|
||||
// AddRefPrefix prepends the given image prefix to tag-only references,
|
||||
// while leaving returning full references unmodified.
|
||||
func AddRefPrefix(image string) func(string) string {
|
||||
return refTranslator(image, false)
|
||||
}
|
||||
|
||||
// refTranslator creates a reference which only has a tag or verifies
|
||||
// a full reference.
|
||||
func refTranslator(image string, checkPrefix bool) func(string) string {
|
||||
return func(ref string) string {
|
||||
if image == "" {
|
||||
return ""
|
||||
}
|
||||
// Check if ref is full reference
|
||||
if strings.ContainsAny(ref, "/:@") {
|
||||
// If not prefixed, don't include image
|
||||
if checkPrefix && !isImagePrefix(ref, image) {
|
||||
return ""
|
||||
}
|
||||
return ref
|
||||
}
|
||||
return image + ":" + ref
|
||||
}
|
||||
}
|
||||
|
||||
func isImagePrefix(s, prefix string) bool {
|
||||
if !strings.HasPrefix(s, prefix) {
|
||||
return false
|
||||
}
|
||||
if len(s) > len(prefix) {
|
||||
switch s[len(prefix)] {
|
||||
case '/', ':', '@':
|
||||
// Prevent matching partial namespaces
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func normalizeReference(ref string) (string, error) {
|
||||
// TODO: Replace this function to not depend on reference package
|
||||
normalized, err := distref.ParseDockerRef(ref)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("normalize image ref %q: %w", ref, err)
|
||||
}
|
||||
|
||||
return normalized.String(), nil
|
||||
}
|
||||
|
||||
func familiarizeReference(ref string) (string, error) {
|
||||
named, err := distref.ParseNormalizedNamed(ref)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse %q: %w", ref, err)
|
||||
}
|
||||
named = distref.TagNameOnly(named)
|
||||
|
||||
return distref.FamiliarString(named), nil
|
||||
}
|
||||
|
||||
func ociReferenceName(name string) string {
|
||||
// OCI defines the reference name as only a tag excluding the
|
||||
// repository. The containerd annotation contains the full image name
|
||||
// since the tag is insufficient for correctly naming and referring to an
|
||||
// image
|
||||
var ociRef string
|
||||
if spec, err := reference.Parse(name); err == nil {
|
||||
ociRef = spec.Object
|
||||
} else {
|
||||
ociRef = name
|
||||
}
|
||||
|
||||
return ociRef
|
||||
}
|
||||
|
||||
// DigestTranslator creates a digest reference by adding the
|
||||
// digest to an image name
|
||||
func DigestTranslator(prefix string) func(digest.Digest) string {
|
||||
return func(dgst digest.Digest) string {
|
||||
return prefix + "@" + dgst.String()
|
||||
}
|
||||
}
|
||||
126
core/images/converter/converter.go
Normal file
126
core/images/converter/converter.go
Normal file
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package converter provides image converter
|
||||
package converter
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/leases"
|
||||
"github.com/containerd/containerd/v2/platforms"
|
||||
)
|
||||
|
||||
type convertOpts struct {
|
||||
layerConvertFunc ConvertFunc
|
||||
docker2oci bool
|
||||
indexConvertFunc ConvertFunc
|
||||
platformMC platforms.MatchComparer
|
||||
}
|
||||
|
||||
// Opt is an option for Convert()
|
||||
type Opt func(*convertOpts) error
|
||||
|
||||
// WithLayerConvertFunc specifies the function that converts layers.
|
||||
func WithLayerConvertFunc(fn ConvertFunc) Opt {
|
||||
return func(copts *convertOpts) error {
|
||||
copts.layerConvertFunc = fn
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithDockerToOCI converts Docker media types into OCI ones.
|
||||
func WithDockerToOCI(v bool) Opt {
|
||||
return func(copts *convertOpts) error {
|
||||
copts.docker2oci = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithPlatform specifies the platform.
|
||||
// Defaults to all platforms.
|
||||
func WithPlatform(p platforms.MatchComparer) Opt {
|
||||
return func(copts *convertOpts) error {
|
||||
copts.platformMC = p
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithIndexConvertFunc specifies the function that converts manifests and index (manifest lists).
|
||||
// Defaults to DefaultIndexConvertFunc.
|
||||
func WithIndexConvertFunc(fn ConvertFunc) Opt {
|
||||
return func(copts *convertOpts) error {
|
||||
copts.indexConvertFunc = fn
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Client is implemented by *containerd.Client .
|
||||
type Client interface {
|
||||
WithLease(ctx context.Context, opts ...leases.Opt) (context.Context, func(context.Context) error, error)
|
||||
ContentStore() content.Store
|
||||
ImageService() images.Store
|
||||
}
|
||||
|
||||
// Convert converts an image.
|
||||
func Convert(ctx context.Context, client Client, dstRef, srcRef string, opts ...Opt) (*images.Image, error) {
|
||||
var copts convertOpts
|
||||
for _, o := range opts {
|
||||
if err := o(&copts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if copts.platformMC == nil {
|
||||
copts.platformMC = platforms.All
|
||||
}
|
||||
if copts.indexConvertFunc == nil {
|
||||
copts.indexConvertFunc = DefaultIndexConvertFunc(copts.layerConvertFunc, copts.docker2oci, copts.platformMC)
|
||||
}
|
||||
|
||||
ctx, done, err := client.WithLease(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer done(ctx)
|
||||
|
||||
cs := client.ContentStore()
|
||||
is := client.ImageService()
|
||||
srcImg, err := is.Get(ctx, srcRef)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dstDesc, err := copts.indexConvertFunc(ctx, cs, srcImg.Target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dstImg := srcImg
|
||||
dstImg.Name = dstRef
|
||||
if dstDesc != nil {
|
||||
dstImg.Target = *dstDesc
|
||||
}
|
||||
var res images.Image
|
||||
if dstRef != srcRef {
|
||||
_ = is.Delete(ctx, dstRef)
|
||||
res, err = is.Create(ctx, dstImg)
|
||||
} else {
|
||||
res, err = is.Update(ctx, dstImg)
|
||||
}
|
||||
return &res, err
|
||||
}
|
||||
454
core/images/converter/default.go
Normal file
454
core/images/converter/default.go
Normal file
@@ -0,0 +1,454 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package converter
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/platforms"
|
||||
"github.com/containerd/log"
|
||||
"github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// ConvertFunc returns a converted content descriptor.
|
||||
// When the content was not converted, ConvertFunc returns nil.
|
||||
type ConvertFunc func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error)
|
||||
|
||||
// DefaultIndexConvertFunc is the default convert func used by Convert.
|
||||
func DefaultIndexConvertFunc(layerConvertFunc ConvertFunc, docker2oci bool, platformMC platforms.MatchComparer) ConvertFunc {
|
||||
c := &defaultConverter{
|
||||
layerConvertFunc: layerConvertFunc,
|
||||
docker2oci: docker2oci,
|
||||
platformMC: platformMC,
|
||||
diffIDMap: make(map[digest.Digest]digest.Digest),
|
||||
}
|
||||
return c.convert
|
||||
}
|
||||
|
||||
// ConvertHookFunc is a callback function called during conversion of a blob.
|
||||
// orgDesc is the target descriptor to convert. newDesc is passed if conversion happens.
|
||||
type ConvertHookFunc func(ctx context.Context, cs content.Store, orgDesc ocispec.Descriptor, newDesc *ocispec.Descriptor) (*ocispec.Descriptor, error)
|
||||
|
||||
// ConvertHooks is a configuration for hook callbacks called during blob conversion.
|
||||
type ConvertHooks struct {
|
||||
// PostConvertHook is a callback function called for each blob after conversion is done.
|
||||
PostConvertHook ConvertHookFunc
|
||||
}
|
||||
|
||||
// IndexConvertFuncWithHook is the convert func used by Convert with hook functions support.
|
||||
func IndexConvertFuncWithHook(layerConvertFunc ConvertFunc, docker2oci bool, platformMC platforms.MatchComparer, hooks ConvertHooks) ConvertFunc {
|
||||
c := &defaultConverter{
|
||||
layerConvertFunc: layerConvertFunc,
|
||||
docker2oci: docker2oci,
|
||||
platformMC: platformMC,
|
||||
diffIDMap: make(map[digest.Digest]digest.Digest),
|
||||
hooks: hooks,
|
||||
}
|
||||
return c.convert
|
||||
}
|
||||
|
||||
type defaultConverter struct {
|
||||
layerConvertFunc ConvertFunc
|
||||
docker2oci bool
|
||||
platformMC platforms.MatchComparer
|
||||
diffIDMap map[digest.Digest]digest.Digest // key: old diffID, value: new diffID
|
||||
diffIDMapMu sync.RWMutex
|
||||
hooks ConvertHooks
|
||||
}
|
||||
|
||||
// convert dispatches desc.MediaType and calls c.convert{Layer,Manifest,Index,Config}.
|
||||
//
|
||||
// Also converts media type if c.docker2oci is set.
|
||||
func (c *defaultConverter) convert(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
|
||||
var (
|
||||
newDesc *ocispec.Descriptor
|
||||
err error
|
||||
)
|
||||
if images.IsLayerType(desc.MediaType) {
|
||||
newDesc, err = c.convertLayer(ctx, cs, desc)
|
||||
} else if images.IsManifestType(desc.MediaType) {
|
||||
newDesc, err = c.convertManifest(ctx, cs, desc)
|
||||
} else if images.IsIndexType(desc.MediaType) {
|
||||
newDesc, err = c.convertIndex(ctx, cs, desc)
|
||||
} else if images.IsConfigType(desc.MediaType) {
|
||||
newDesc, err = c.convertConfig(ctx, cs, desc)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if c.hooks.PostConvertHook != nil {
|
||||
if newDescPost, err := c.hooks.PostConvertHook(ctx, cs, desc, newDesc); err != nil {
|
||||
return nil, err
|
||||
} else if newDescPost != nil {
|
||||
newDesc = newDescPost
|
||||
}
|
||||
}
|
||||
|
||||
if images.IsDockerType(desc.MediaType) {
|
||||
if c.docker2oci {
|
||||
if newDesc == nil {
|
||||
newDesc = copyDesc(desc)
|
||||
}
|
||||
newDesc.MediaType = ConvertDockerMediaTypeToOCI(newDesc.MediaType)
|
||||
} else if (newDesc == nil && len(desc.Annotations) != 0) || (newDesc != nil && len(newDesc.Annotations) != 0) {
|
||||
// Annotations is supported only on OCI manifest.
|
||||
// We need to remove annotations for Docker media types.
|
||||
if newDesc == nil {
|
||||
newDesc = copyDesc(desc)
|
||||
}
|
||||
newDesc.Annotations = nil
|
||||
}
|
||||
}
|
||||
log.G(ctx).WithField("old", desc).WithField("new", newDesc).Debugf("converted")
|
||||
return newDesc, nil
|
||||
}
|
||||
|
||||
func copyDesc(desc ocispec.Descriptor) *ocispec.Descriptor {
|
||||
descCopy := desc
|
||||
return &descCopy
|
||||
}
|
||||
|
||||
// convertLayer converts image layers if c.layerConvertFunc is set.
|
||||
//
|
||||
// c.layerConvertFunc can be nil, e.g., for converting Docker media types to OCI ones.
|
||||
func (c *defaultConverter) convertLayer(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
|
||||
if c.layerConvertFunc != nil {
|
||||
return c.layerConvertFunc(ctx, cs, desc)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// convertManifest converts image manifests.
|
||||
//
|
||||
// - converts `.mediaType` if the target format is OCI
|
||||
// - records diff ID changes in c.diffIDMap
|
||||
func (c *defaultConverter) convertManifest(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
|
||||
var (
|
||||
manifest ocispec.Manifest
|
||||
modified bool
|
||||
)
|
||||
labels, err := readJSON(ctx, cs, &manifest, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if labels == nil {
|
||||
labels = make(map[string]string)
|
||||
}
|
||||
if images.IsDockerType(manifest.MediaType) && c.docker2oci {
|
||||
manifest.MediaType = ConvertDockerMediaTypeToOCI(manifest.MediaType)
|
||||
modified = true
|
||||
}
|
||||
var mu sync.Mutex
|
||||
eg, ctx2 := errgroup.WithContext(ctx)
|
||||
for i, l := range manifest.Layers {
|
||||
i := i
|
||||
l := l
|
||||
oldDiffID, err := images.GetDiffID(ctx, cs, l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
eg.Go(func() error {
|
||||
newL, err := c.convert(ctx2, cs, l)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if newL != nil {
|
||||
mu.Lock()
|
||||
// update GC labels
|
||||
ClearGCLabels(labels, l.Digest)
|
||||
labelKey := fmt.Sprintf("containerd.io/gc.ref.content.l.%d", i)
|
||||
labels[labelKey] = newL.Digest.String()
|
||||
manifest.Layers[i] = *newL
|
||||
modified = true
|
||||
mu.Unlock()
|
||||
|
||||
// diffID changes if the tar entries were modified.
|
||||
// diffID stays same if only the compression type was changed.
|
||||
// When diffID changed, add a map entry so that we can update image config.
|
||||
newDiffID, err := images.GetDiffID(ctx, cs, *newL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if newDiffID != oldDiffID {
|
||||
c.diffIDMapMu.Lock()
|
||||
c.diffIDMap[oldDiffID] = newDiffID
|
||||
c.diffIDMapMu.Unlock()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := eg.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
newConfig, err := c.convert(ctx, cs, manifest.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if newConfig != nil {
|
||||
ClearGCLabels(labels, manifest.Config.Digest)
|
||||
labels["containerd.io/gc.ref.content.config"] = newConfig.Digest.String()
|
||||
manifest.Config = *newConfig
|
||||
modified = true
|
||||
}
|
||||
|
||||
if modified {
|
||||
return writeJSON(ctx, cs, &manifest, desc, labels)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// convertIndex converts image index.
|
||||
//
|
||||
// - converts `.mediaType` if the target format is OCI
|
||||
// - clears manifest entries that do not match c.platformMC
|
||||
func (c *defaultConverter) convertIndex(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
|
||||
var (
|
||||
index ocispec.Index
|
||||
modified bool
|
||||
)
|
||||
labels, err := readJSON(ctx, cs, &index, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if labels == nil {
|
||||
labels = make(map[string]string)
|
||||
}
|
||||
if images.IsDockerType(index.MediaType) && c.docker2oci {
|
||||
index.MediaType = ConvertDockerMediaTypeToOCI(index.MediaType)
|
||||
modified = true
|
||||
}
|
||||
|
||||
newManifests := make([]ocispec.Descriptor, len(index.Manifests))
|
||||
newManifestsToBeRemoved := make(map[int]struct{}) // slice index
|
||||
var mu sync.Mutex
|
||||
eg, ctx2 := errgroup.WithContext(ctx)
|
||||
for i, mani := range index.Manifests {
|
||||
i := i
|
||||
mani := mani
|
||||
labelKey := fmt.Sprintf("containerd.io/gc.ref.content.m.%d", i)
|
||||
eg.Go(func() error {
|
||||
if mani.Platform != nil && !c.platformMC.Match(*mani.Platform) {
|
||||
mu.Lock()
|
||||
ClearGCLabels(labels, mani.Digest)
|
||||
newManifestsToBeRemoved[i] = struct{}{}
|
||||
modified = true
|
||||
mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
newMani, err := c.convert(ctx2, cs, mani)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mu.Lock()
|
||||
if newMani != nil {
|
||||
ClearGCLabels(labels, mani.Digest)
|
||||
labels[labelKey] = newMani.Digest.String()
|
||||
// NOTE: for keeping manifest order, we specify `i` index explicitly
|
||||
newManifests[i] = *newMani
|
||||
modified = true
|
||||
} else {
|
||||
newManifests[i] = mani
|
||||
}
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := eg.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if modified {
|
||||
var newManifestsClean []ocispec.Descriptor
|
||||
for i, m := range newManifests {
|
||||
if _, ok := newManifestsToBeRemoved[i]; !ok {
|
||||
newManifestsClean = append(newManifestsClean, m)
|
||||
}
|
||||
}
|
||||
index.Manifests = newManifestsClean
|
||||
return writeJSON(ctx, cs, &index, desc, labels)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// convertConfig converts image config contents.
|
||||
//
|
||||
// - updates `.rootfs.diff_ids` using c.diffIDMap .
|
||||
//
|
||||
// - clears legacy `.config.Image` and `.container_config.Image` fields if `.rootfs.diff_ids` was updated.
|
||||
func (c *defaultConverter) convertConfig(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
|
||||
var (
|
||||
cfg DualConfig
|
||||
cfgAsOCI ocispec.Image // read only, used for parsing cfg
|
||||
modified bool
|
||||
)
|
||||
|
||||
labels, err := readJSON(ctx, cs, &cfg, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if labels == nil {
|
||||
labels = make(map[string]string)
|
||||
}
|
||||
if _, err := readJSON(ctx, cs, &cfgAsOCI, desc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if rootfs := cfgAsOCI.RootFS; rootfs.Type == "layers" {
|
||||
rootfsModified := false
|
||||
c.diffIDMapMu.RLock()
|
||||
for i, oldDiffID := range rootfs.DiffIDs {
|
||||
if newDiffID, ok := c.diffIDMap[oldDiffID]; ok && newDiffID != oldDiffID {
|
||||
rootfs.DiffIDs[i] = newDiffID
|
||||
rootfsModified = true
|
||||
}
|
||||
}
|
||||
c.diffIDMapMu.RUnlock()
|
||||
if rootfsModified {
|
||||
rootfsB, err := json.Marshal(rootfs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg["rootfs"] = (*json.RawMessage)(&rootfsB)
|
||||
modified = true
|
||||
}
|
||||
}
|
||||
|
||||
if modified {
|
||||
// cfg may have dummy value for legacy `.config.Image` and `.container_config.Image`
|
||||
// We should clear the ID if we changed the diff IDs.
|
||||
if _, err := clearDockerV1DummyID(cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return writeJSON(ctx, cs, &cfg, desc, labels)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// clearDockerV1DummyID clears the dummy values for legacy `.config.Image` and `.container_config.Image`.
|
||||
// Returns true if the cfg was modified.
|
||||
func clearDockerV1DummyID(cfg DualConfig) (bool, error) {
|
||||
var modified bool
|
||||
f := func(k string) error {
|
||||
if configX, ok := cfg[k]; ok && configX != nil {
|
||||
var configField map[string]*json.RawMessage
|
||||
if err := json.Unmarshal(*configX, &configField); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(configField, "Image")
|
||||
b, err := json.Marshal(configField)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg[k] = (*json.RawMessage)(&b)
|
||||
modified = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := f("config"); err != nil {
|
||||
return modified, err
|
||||
}
|
||||
if err := f("container_config"); err != nil {
|
||||
return modified, err
|
||||
}
|
||||
return modified, nil
|
||||
}
|
||||
|
||||
// DualConfig covers Docker config (v1.0, v1.1, v1.2) and OCI config.
|
||||
// Unmarshalled as map[string]*json.RawMessage to retain unknown fields on remarshalling.
|
||||
type DualConfig map[string]*json.RawMessage
|
||||
|
||||
func readJSON(ctx context.Context, cs content.Store, x interface{}, desc ocispec.Descriptor) (map[string]string, error) {
|
||||
info, err := cs.Info(ctx, desc.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labels := info.Labels
|
||||
b, err := content.ReadBlob(ctx, cs, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(b, x); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return labels, nil
|
||||
}
|
||||
|
||||
func writeJSON(ctx context.Context, cs content.Store, x interface{}, oldDesc ocispec.Descriptor, labels map[string]string) (*ocispec.Descriptor, error) {
|
||||
b, err := json.Marshal(x)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dgst := digest.SHA256.FromBytes(b)
|
||||
ref := fmt.Sprintf("converter-write-json-%s", dgst.String())
|
||||
w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := content.Copy(ctx, w, bytes.NewReader(b), int64(len(b)), dgst, content.WithLabels(labels)); err != nil {
|
||||
w.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newDesc := oldDesc
|
||||
newDesc.Size = int64(len(b))
|
||||
newDesc.Digest = dgst
|
||||
return &newDesc, nil
|
||||
}
|
||||
|
||||
// ConvertDockerMediaTypeToOCI converts a media type string
|
||||
func ConvertDockerMediaTypeToOCI(mt string) string {
|
||||
switch mt {
|
||||
case images.MediaTypeDockerSchema2ManifestList:
|
||||
return ocispec.MediaTypeImageIndex
|
||||
case images.MediaTypeDockerSchema2Manifest:
|
||||
return ocispec.MediaTypeImageManifest
|
||||
case images.MediaTypeDockerSchema2LayerGzip:
|
||||
return ocispec.MediaTypeImageLayerGzip
|
||||
case images.MediaTypeDockerSchema2LayerForeignGzip:
|
||||
return ocispec.MediaTypeImageLayerNonDistributableGzip //nolint:staticcheck // deprecated
|
||||
case images.MediaTypeDockerSchema2Layer:
|
||||
return ocispec.MediaTypeImageLayer
|
||||
case images.MediaTypeDockerSchema2LayerForeign:
|
||||
return ocispec.MediaTypeImageLayerNonDistributable //nolint:staticcheck // deprecated
|
||||
case images.MediaTypeDockerSchema2Config:
|
||||
return ocispec.MediaTypeImageConfig
|
||||
default:
|
||||
return mt
|
||||
}
|
||||
}
|
||||
|
||||
// ClearGCLabels clears GC labels for the given digest.
|
||||
func ClearGCLabels(labels map[string]string, dgst digest.Digest) {
|
||||
for k, v := range labels {
|
||||
if v == dgst.String() && strings.HasPrefix(k, "containerd.io/gc.ref.content") {
|
||||
delete(labels, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
122
core/images/converter/uncompress/uncompress.go
Normal file
122
core/images/converter/uncompress/uncompress.go
Normal file
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package uncompress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/containerd/containerd/v2/archive/compression"
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/core/images/converter"
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
"github.com/containerd/containerd/v2/labels"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
var _ converter.ConvertFunc = LayerConvertFunc
|
||||
|
||||
// LayerConvertFunc converts tar.gz layers into uncompressed tar layers.
|
||||
// Media type is changed, e.g., "application/vnd.oci.image.layer.v1.tar+gzip" -> "application/vnd.oci.image.layer.v1.tar"
|
||||
func LayerConvertFunc(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
|
||||
if !images.IsLayerType(desc.MediaType) || IsUncompressedType(desc.MediaType) {
|
||||
// No conversion. No need to return an error here.
|
||||
return nil, nil
|
||||
}
|
||||
info, err := cs.Info(ctx, desc.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
readerAt, err := cs.ReaderAt(ctx, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer readerAt.Close()
|
||||
sr := io.NewSectionReader(readerAt, 0, desc.Size)
|
||||
newR, err := compression.DecompressStream(sr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer newR.Close()
|
||||
ref := fmt.Sprintf("convert-uncompress-from-%s", desc.Digest)
|
||||
w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer w.Close()
|
||||
|
||||
// Reset the writing position
|
||||
// Old writer possibly remains without aborted
|
||||
// (e.g. conversion interrupted by a signal)
|
||||
if err := w.Truncate(0); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
n, err := io.Copy(w, newR)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := newR.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// no need to retain "containerd.io/uncompressed" label, but retain other labels ("containerd.io/distribution.source.*")
|
||||
labelsMap := info.Labels
|
||||
delete(labelsMap, labels.LabelUncompressed)
|
||||
if err = w.Commit(ctx, 0, "", content.WithLabels(labelsMap)); err != nil && !errdefs.IsAlreadyExists(err) {
|
||||
return nil, err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newDesc := desc
|
||||
newDesc.Digest = w.Digest()
|
||||
newDesc.Size = n
|
||||
newDesc.MediaType = convertMediaType(newDesc.MediaType)
|
||||
return &newDesc, nil
|
||||
}
|
||||
|
||||
// IsUncompressedType returns whether the provided media type is considered
|
||||
// an uncompressed layer type
|
||||
func IsUncompressedType(mt string) bool {
|
||||
switch mt {
|
||||
case
|
||||
images.MediaTypeDockerSchema2Layer,
|
||||
images.MediaTypeDockerSchema2LayerForeign,
|
||||
ocispec.MediaTypeImageLayer,
|
||||
ocispec.MediaTypeImageLayerNonDistributable: //nolint:staticcheck // deprecated
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func convertMediaType(mt string) string {
|
||||
switch mt {
|
||||
case images.MediaTypeDockerSchema2LayerGzip:
|
||||
return images.MediaTypeDockerSchema2Layer
|
||||
case images.MediaTypeDockerSchema2LayerForeignGzip:
|
||||
return images.MediaTypeDockerSchema2LayerForeign
|
||||
case ocispec.MediaTypeImageLayerGzip, ocispec.MediaTypeImageLayerZstd:
|
||||
return ocispec.MediaTypeImageLayer
|
||||
case ocispec.MediaTypeImageLayerNonDistributableGzip, ocispec.MediaTypeImageLayerNonDistributableZstd: //nolint:staticcheck // deprecated
|
||||
return ocispec.MediaTypeImageLayerNonDistributable //nolint:staticcheck // deprecated
|
||||
default:
|
||||
return mt
|
||||
}
|
||||
}
|
||||
82
core/images/diffid.go
Normal file
82
core/images/diffid.go
Normal file
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package images
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
||||
"github.com/containerd/containerd/v2/archive/compression"
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/labels"
|
||||
"github.com/containerd/log"
|
||||
)
|
||||
|
||||
// GetDiffID gets the diff ID of the layer blob descriptor.
|
||||
func GetDiffID(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (digest.Digest, error) {
|
||||
switch desc.MediaType {
|
||||
case
|
||||
// If the layer is already uncompressed, we can just return its digest
|
||||
MediaTypeDockerSchema2Layer,
|
||||
ocispec.MediaTypeImageLayer,
|
||||
MediaTypeDockerSchema2LayerForeign,
|
||||
ocispec.MediaTypeImageLayerNonDistributable: //nolint:staticcheck // deprecated
|
||||
return desc.Digest, nil
|
||||
}
|
||||
info, err := cs.Info(ctx, desc.Digest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
v, ok := info.Labels[labels.LabelUncompressed]
|
||||
if ok {
|
||||
// Fast path: if the image is already unpacked, we can use the label value
|
||||
return digest.Parse(v)
|
||||
}
|
||||
// if the image is not unpacked, we may not have the label
|
||||
ra, err := cs.ReaderAt(ctx, desc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer ra.Close()
|
||||
r := content.NewReader(ra)
|
||||
uR, err := compression.DecompressStream(r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer uR.Close()
|
||||
digester := digest.Canonical.Digester()
|
||||
hashW := digester.Hash()
|
||||
if _, err := io.Copy(hashW, uR); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := ra.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest := digester.Digest()
|
||||
// memorize the computed value
|
||||
if info.Labels == nil {
|
||||
info.Labels = make(map[string]string)
|
||||
}
|
||||
info.Labels[labels.LabelUncompressed] = digest.String()
|
||||
if _, err := cs.Update(ctx, info, "labels"); err != nil {
|
||||
log.G(ctx).WithError(err).Warnf("failed to set %s label for %s", labels.LabelUncompressed, desc.Digest)
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
320
core/images/handlers.go
Normal file
320
core/images/handlers.go
Normal file
@@ -0,0 +1,320 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package images
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
"github.com/containerd/containerd/v2/platforms"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrSkipDesc is used to skip processing of a descriptor and
|
||||
// its descendants.
|
||||
ErrSkipDesc = errors.New("skip descriptor")
|
||||
|
||||
// ErrStopHandler is used to signify that the descriptor
|
||||
// has been handled and should not be handled further.
|
||||
// This applies only to a single descriptor in a handler
|
||||
// chain and does not apply to descendant descriptors.
|
||||
ErrStopHandler = errors.New("stop handler")
|
||||
|
||||
// ErrEmptyWalk is used when the WalkNotEmpty handlers return no
|
||||
// children (e.g.: they were filtered out).
|
||||
ErrEmptyWalk = errors.New("image might be filtered out")
|
||||
)
|
||||
|
||||
// Handler handles image manifests
|
||||
type Handler interface {
|
||||
Handle(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error)
|
||||
}
|
||||
|
||||
// HandlerFunc function implementing the Handler interface
|
||||
type HandlerFunc func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error)
|
||||
|
||||
// Handle image manifests
|
||||
func (fn HandlerFunc) Handle(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
|
||||
return fn(ctx, desc)
|
||||
}
|
||||
|
||||
// Handlers returns a handler that will run the handlers in sequence.
|
||||
//
|
||||
// A handler may return `ErrStopHandler` to stop calling additional handlers
|
||||
func Handlers(handlers ...Handler) HandlerFunc {
|
||||
return func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
|
||||
var children []ocispec.Descriptor
|
||||
for _, handler := range handlers {
|
||||
ch, err := handler.Handle(ctx, desc)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrStopHandler) {
|
||||
break
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
children = append(children, ch...)
|
||||
}
|
||||
|
||||
return children, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Walk the resources of an image and call the handler for each. If the handler
|
||||
// decodes the sub-resources for each image,
|
||||
//
|
||||
// This differs from dispatch in that each sibling resource is considered
|
||||
// synchronously.
|
||||
func Walk(ctx context.Context, handler Handler, descs ...ocispec.Descriptor) error {
|
||||
for _, desc := range descs {
|
||||
|
||||
children, err := handler.Handle(ctx, desc)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrSkipDesc) {
|
||||
continue // don't traverse the children.
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if len(children) > 0 {
|
||||
if err := Walk(ctx, handler, children...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WalkNotEmpty works the same way Walk does, with the exception that it ensures that
|
||||
// some children are still found by Walking the descriptors (for example, not all of
|
||||
// them have been filtered out by one of the handlers). If there are no children,
|
||||
// then an ErrEmptyWalk error is returned.
|
||||
func WalkNotEmpty(ctx context.Context, handler Handler, descs ...ocispec.Descriptor) error {
|
||||
isEmpty := true
|
||||
var notEmptyHandler HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
children, err := handler.Handle(ctx, desc)
|
||||
if err != nil {
|
||||
return children, err
|
||||
}
|
||||
|
||||
if len(children) > 0 {
|
||||
isEmpty = false
|
||||
}
|
||||
|
||||
return children, nil
|
||||
}
|
||||
|
||||
err := Walk(ctx, notEmptyHandler, descs...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isEmpty {
|
||||
return ErrEmptyWalk
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Dispatch runs the provided handler for content specified by the descriptors.
|
||||
// If the handler decode subresources, they will be visited, as well.
|
||||
//
|
||||
// Handlers for siblings are run in parallel on the provided descriptors. A
|
||||
// handler may return `ErrSkipDesc` to signal to the dispatcher to not traverse
|
||||
// any children.
|
||||
//
|
||||
// A concurrency limiter can be passed in to limit the number of concurrent
|
||||
// handlers running. When limiter is nil, there is no limit.
|
||||
//
|
||||
// Typically, this function will be used with `FetchHandler`, often composed
|
||||
// with other handlers.
|
||||
//
|
||||
// If any handler returns an error, the dispatch session will be canceled.
|
||||
func Dispatch(ctx context.Context, handler Handler, limiter *semaphore.Weighted, descs ...ocispec.Descriptor) error {
|
||||
eg, ctx2 := errgroup.WithContext(ctx)
|
||||
for _, desc := range descs {
|
||||
desc := desc
|
||||
|
||||
if limiter != nil {
|
||||
if err := limiter.Acquire(ctx, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
eg.Go(func() error {
|
||||
desc := desc
|
||||
|
||||
children, err := handler.Handle(ctx2, desc)
|
||||
if limiter != nil {
|
||||
limiter.Release(1)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrSkipDesc) {
|
||||
return nil // don't traverse the children.
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if len(children) > 0 {
|
||||
return Dispatch(ctx2, handler, limiter, children...)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
return eg.Wait()
|
||||
}
|
||||
|
||||
// ChildrenHandler decodes well-known manifest types and returns their children.
|
||||
//
|
||||
// This is useful for supporting recursive fetch and other use cases where you
|
||||
// want to do a full walk of resources.
|
||||
//
|
||||
// One can also replace this with another implementation to allow descending of
|
||||
// arbitrary types.
|
||||
func ChildrenHandler(provider content.Provider) HandlerFunc {
|
||||
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
return Children(ctx, provider, desc)
|
||||
}
|
||||
}
|
||||
|
||||
// SetChildrenLabels is a handler wrapper which sets labels for the content on
|
||||
// the children returned by the handler and passes through the children.
|
||||
// Must follow a handler that returns the children to be labeled.
|
||||
func SetChildrenLabels(manager content.Manager, f HandlerFunc) HandlerFunc {
|
||||
return SetChildrenMappedLabels(manager, f, nil)
|
||||
}
|
||||
|
||||
// SetChildrenMappedLabels is a handler wrapper which sets labels for the content on
|
||||
// the children returned by the handler and passes through the children.
|
||||
// Must follow a handler that returns the children to be labeled.
|
||||
// The label map allows the caller to control the labels per child descriptor.
|
||||
// For returned labels, the index of the child will be appended to the end
|
||||
// except for the first index when the returned label does not end with '.'.
|
||||
func SetChildrenMappedLabels(manager content.Manager, f HandlerFunc, labelMap func(ocispec.Descriptor) []string) HandlerFunc {
|
||||
if labelMap == nil {
|
||||
labelMap = ChildGCLabels
|
||||
}
|
||||
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
children, err := f(ctx, desc)
|
||||
if err != nil {
|
||||
return children, err
|
||||
}
|
||||
|
||||
if len(children) > 0 {
|
||||
var (
|
||||
info = content.Info{
|
||||
Digest: desc.Digest,
|
||||
Labels: map[string]string{},
|
||||
}
|
||||
fields = []string{}
|
||||
keys = map[string]uint{}
|
||||
)
|
||||
for _, ch := range children {
|
||||
labelKeys := labelMap(ch)
|
||||
for _, key := range labelKeys {
|
||||
idx := keys[key]
|
||||
keys[key] = idx + 1
|
||||
if idx > 0 || key[len(key)-1] == '.' {
|
||||
key = fmt.Sprintf("%s%d", key, idx)
|
||||
}
|
||||
|
||||
info.Labels[key] = ch.Digest.String()
|
||||
fields = append(fields, "labels."+key)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := manager.Update(ctx, info, fields...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return children, err
|
||||
}
|
||||
}
|
||||
|
||||
// FilterPlatforms is a handler wrapper which limits the descriptors returned
|
||||
// based on matching the specified platform matcher.
|
||||
func FilterPlatforms(f HandlerFunc, m platforms.Matcher) HandlerFunc {
|
||||
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
children, err := f(ctx, desc)
|
||||
if err != nil {
|
||||
return children, err
|
||||
}
|
||||
|
||||
var descs []ocispec.Descriptor
|
||||
|
||||
if m == nil {
|
||||
descs = children
|
||||
} else {
|
||||
for _, d := range children {
|
||||
if d.Platform == nil || m.Match(*d.Platform) {
|
||||
descs = append(descs, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return descs, nil
|
||||
}
|
||||
}
|
||||
|
||||
// LimitManifests is a handler wrapper which filters the manifest descriptors
|
||||
// returned using the provided platform.
|
||||
// The results will be ordered according to the comparison operator and
|
||||
// use the ordering in the manifests for equal matches.
|
||||
// A limit of 0 or less is considered no limit.
|
||||
// A not found error is returned if no manifest is matched.
|
||||
func LimitManifests(f HandlerFunc, m platforms.MatchComparer, n int) HandlerFunc {
|
||||
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
children, err := f(ctx, desc)
|
||||
if err != nil {
|
||||
return children, err
|
||||
}
|
||||
|
||||
// only limit manifests from an index
|
||||
if IsIndexType(desc.MediaType) {
|
||||
sort.SliceStable(children, func(i, j int) bool {
|
||||
if children[i].Platform == nil {
|
||||
return false
|
||||
}
|
||||
if children[j].Platform == nil {
|
||||
return true
|
||||
}
|
||||
return m.Less(*children[i].Platform, *children[j].Platform)
|
||||
})
|
||||
|
||||
if n > 0 {
|
||||
if len(children) == 0 {
|
||||
return children, fmt.Errorf("no match for platform in manifest: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
if len(children) > n {
|
||||
children = children[:n]
|
||||
}
|
||||
}
|
||||
}
|
||||
return children, nil
|
||||
}
|
||||
}
|
||||
437
core/images/image.go
Normal file
437
core/images/image.go
Normal file
@@ -0,0 +1,437 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package images
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
"github.com/containerd/containerd/v2/platforms"
|
||||
"github.com/containerd/log"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
// Image provides the model for how containerd views container images.
|
||||
type Image struct {
|
||||
// Name of the image.
|
||||
//
|
||||
// To be pulled, it must be a reference compatible with resolvers.
|
||||
//
|
||||
// This field is required.
|
||||
Name string
|
||||
|
||||
// Labels provide runtime decoration for the image record.
|
||||
//
|
||||
// There is no default behavior for how these labels are propagated. They
|
||||
// only decorate the static metadata object.
|
||||
//
|
||||
// This field is optional.
|
||||
Labels map[string]string
|
||||
|
||||
// Target describes the root content for this image. Typically, this is
|
||||
// a manifest, index or manifest list.
|
||||
Target ocispec.Descriptor
|
||||
|
||||
CreatedAt, UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// DeleteOptions provide options on image delete
|
||||
type DeleteOptions struct {
|
||||
Synchronous bool
|
||||
Target *ocispec.Descriptor
|
||||
}
|
||||
|
||||
// DeleteOpt allows configuring a delete operation
|
||||
type DeleteOpt func(context.Context, *DeleteOptions) error
|
||||
|
||||
// SynchronousDelete is used to indicate that an image deletion and removal of
|
||||
// the image resources should occur synchronously before returning a result.
|
||||
func SynchronousDelete() DeleteOpt {
|
||||
return func(ctx context.Context, o *DeleteOptions) error {
|
||||
o.Synchronous = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteTarget is used to specify the target value an image is expected
|
||||
// to have when deleting. If the image has a different target, then
|
||||
// NotFound is returned.
|
||||
func DeleteTarget(target *ocispec.Descriptor) DeleteOpt {
|
||||
return func(ctx context.Context, o *DeleteOptions) error {
|
||||
o.Target = target
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Store and interact with images
|
||||
type Store interface {
|
||||
Get(ctx context.Context, name string) (Image, error)
|
||||
List(ctx context.Context, filters ...string) ([]Image, error)
|
||||
Create(ctx context.Context, image Image) (Image, error)
|
||||
|
||||
// Update will replace the data in the store with the provided image. If
|
||||
// one or more fieldpaths are provided, only those fields will be updated.
|
||||
Update(ctx context.Context, image Image, fieldpaths ...string) (Image, error)
|
||||
|
||||
Delete(ctx context.Context, name string, opts ...DeleteOpt) error
|
||||
}
|
||||
|
||||
// TODO(stevvooe): Many of these functions make strong platform assumptions,
|
||||
// which are untrue in a lot of cases. More refactoring must be done here to
|
||||
// make this work in all cases.
|
||||
|
||||
// Config resolves the image configuration descriptor.
|
||||
//
|
||||
// The caller can then use the descriptor to resolve and process the
|
||||
// configuration of the image.
|
||||
func (image *Image) Config(ctx context.Context, provider content.Provider, platform platforms.MatchComparer) (ocispec.Descriptor, error) {
|
||||
return Config(ctx, provider, image.Target, platform)
|
||||
}
|
||||
|
||||
// RootFS returns the unpacked diffids that make up and images rootfs.
|
||||
//
|
||||
// These are used to verify that a set of layers unpacked to the expected
|
||||
// values.
|
||||
func (image *Image) RootFS(ctx context.Context, provider content.Provider, platform platforms.MatchComparer) ([]digest.Digest, error) {
|
||||
desc, err := image.Config(ctx, provider, platform)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return RootFS(ctx, provider, desc)
|
||||
}
|
||||
|
||||
// Size returns the total size of an image's packed resources.
|
||||
func (image *Image) Size(ctx context.Context, provider content.Provider, platform platforms.MatchComparer) (int64, error) {
|
||||
var size int64
|
||||
return size, Walk(ctx, Handlers(HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
if desc.Size < 0 {
|
||||
return nil, fmt.Errorf("invalid size %v in %v (%v)", desc.Size, desc.Digest, desc.MediaType)
|
||||
}
|
||||
size += desc.Size
|
||||
return nil, nil
|
||||
}), LimitManifests(FilterPlatforms(ChildrenHandler(provider), platform), platform, 1)), image.Target)
|
||||
}
|
||||
|
||||
type platformManifest struct {
|
||||
p *ocispec.Platform
|
||||
m *ocispec.Manifest
|
||||
}
|
||||
|
||||
// Manifest resolves a manifest from the image for the given platform.
|
||||
//
|
||||
// When a manifest descriptor inside of a manifest index does not have
|
||||
// a platform defined, the platform from the image config is considered.
|
||||
//
|
||||
// If the descriptor points to a non-index manifest, then the manifest is
|
||||
// unmarshalled and returned without considering the platform inside of the
|
||||
// config.
|
||||
//
|
||||
// TODO(stevvooe): This violates the current platform agnostic approach to this
|
||||
// package by returning a specific manifest type. We'll need to refactor this
|
||||
// to return a manifest descriptor or decide that we want to bring the API in
|
||||
// this direction because this abstraction is not needed.
|
||||
func Manifest(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform platforms.MatchComparer) (ocispec.Manifest, error) {
|
||||
var (
|
||||
limit = 1
|
||||
m []platformManifest
|
||||
wasIndex bool
|
||||
)
|
||||
|
||||
if err := Walk(ctx, HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
if IsManifestType(desc.MediaType) {
|
||||
p, err := content.ReadBlob(ctx, provider, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateMediaType(p, desc.MediaType); err != nil {
|
||||
return nil, fmt.Errorf("manifest: invalid desc %s: %w", desc.Digest, err)
|
||||
}
|
||||
|
||||
var manifest ocispec.Manifest
|
||||
if err := json.Unmarshal(p, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if desc.Digest != image.Digest && platform != nil {
|
||||
if desc.Platform != nil && !platform.Match(*desc.Platform) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if desc.Platform == nil {
|
||||
imagePlatform, err := ConfigPlatform(ctx, provider, manifest.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !platform.Match(imagePlatform) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
m = append(m, platformManifest{
|
||||
p: desc.Platform,
|
||||
m: &manifest,
|
||||
})
|
||||
|
||||
return nil, nil
|
||||
} else if IsIndexType(desc.MediaType) {
|
||||
p, err := content.ReadBlob(ctx, provider, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateMediaType(p, desc.MediaType); err != nil {
|
||||
return nil, fmt.Errorf("manifest: invalid desc %s: %w", desc.Digest, err)
|
||||
}
|
||||
|
||||
var idx ocispec.Index
|
||||
if err := json.Unmarshal(p, &idx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if platform == nil {
|
||||
return idx.Manifests, nil
|
||||
}
|
||||
|
||||
var descs []ocispec.Descriptor
|
||||
for _, d := range idx.Manifests {
|
||||
if d.Platform == nil || platform.Match(*d.Platform) {
|
||||
descs = append(descs, d)
|
||||
}
|
||||
}
|
||||
|
||||
sort.SliceStable(descs, func(i, j int) bool {
|
||||
if descs[i].Platform == nil {
|
||||
return false
|
||||
}
|
||||
if descs[j].Platform == nil {
|
||||
return true
|
||||
}
|
||||
return platform.Less(*descs[i].Platform, *descs[j].Platform)
|
||||
})
|
||||
|
||||
wasIndex = true
|
||||
|
||||
if len(descs) > limit {
|
||||
return descs[:limit], nil
|
||||
}
|
||||
return descs, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected media type %v for %v: %w", desc.MediaType, desc.Digest, errdefs.ErrNotFound)
|
||||
}), image); err != nil {
|
||||
return ocispec.Manifest{}, err
|
||||
}
|
||||
|
||||
if len(m) == 0 {
|
||||
err := fmt.Errorf("manifest %v: %w", image.Digest, errdefs.ErrNotFound)
|
||||
if wasIndex {
|
||||
err = fmt.Errorf("no match for platform in manifest %v: %w", image.Digest, errdefs.ErrNotFound)
|
||||
}
|
||||
return ocispec.Manifest{}, err
|
||||
}
|
||||
return *m[0].m, nil
|
||||
}
|
||||
|
||||
// Config resolves the image configuration descriptor using a content provided
|
||||
// to resolve child resources on the image.
|
||||
//
|
||||
// The caller can then use the descriptor to resolve and process the
|
||||
// configuration of the image.
|
||||
func Config(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform platforms.MatchComparer) (ocispec.Descriptor, error) {
|
||||
manifest, err := Manifest(ctx, provider, image, platform)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
return manifest.Config, nil
|
||||
}
|
||||
|
||||
// Platforms returns one or more platforms supported by the image.
|
||||
func Platforms(ctx context.Context, provider content.Provider, image ocispec.Descriptor) ([]ocispec.Platform, error) {
|
||||
var platformSpecs []ocispec.Platform
|
||||
return platformSpecs, Walk(ctx, Handlers(HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
if desc.Platform != nil {
|
||||
platformSpecs = append(platformSpecs, *desc.Platform)
|
||||
return nil, ErrSkipDesc
|
||||
}
|
||||
|
||||
if IsConfigType(desc.MediaType) {
|
||||
imagePlatform, err := ConfigPlatform(ctx, provider, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
platformSpecs = append(platformSpecs, imagePlatform)
|
||||
}
|
||||
return nil, nil
|
||||
}), ChildrenHandler(provider)), image)
|
||||
}
|
||||
|
||||
// Check returns nil if the all components of an image are available in the
|
||||
// provider for the specified platform.
|
||||
//
|
||||
// If available is true, the caller can assume that required represents the
|
||||
// complete set of content required for the image.
|
||||
//
|
||||
// missing will have the components that are part of required but not available
|
||||
// in the provider.
|
||||
//
|
||||
// If there is a problem resolving content, an error will be returned.
|
||||
func Check(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform platforms.MatchComparer) (available bool, required, present, missing []ocispec.Descriptor, err error) {
|
||||
mfst, err := Manifest(ctx, provider, image, platform)
|
||||
if err != nil {
|
||||
if errdefs.IsNotFound(err) {
|
||||
return false, []ocispec.Descriptor{image}, nil, []ocispec.Descriptor{image}, nil
|
||||
}
|
||||
|
||||
return false, nil, nil, nil, fmt.Errorf("failed to check image %v: %w", image.Digest, err)
|
||||
}
|
||||
|
||||
// TODO(stevvooe): It is possible that referenced components could have
|
||||
// children, but this is rare. For now, we ignore this and only verify
|
||||
// that manifest components are present.
|
||||
required = append([]ocispec.Descriptor{mfst.Config}, mfst.Layers...)
|
||||
|
||||
for _, desc := range required {
|
||||
ra, err := provider.ReaderAt(ctx, desc)
|
||||
if err != nil {
|
||||
if errdefs.IsNotFound(err) {
|
||||
missing = append(missing, desc)
|
||||
continue
|
||||
} else {
|
||||
return false, nil, nil, nil, fmt.Errorf("failed to check image %v: %w", desc.Digest, err)
|
||||
}
|
||||
}
|
||||
ra.Close()
|
||||
present = append(present, desc)
|
||||
|
||||
}
|
||||
|
||||
return true, required, present, missing, nil
|
||||
}
|
||||
|
||||
// Children returns the immediate children of content described by the descriptor.
|
||||
func Children(ctx context.Context, provider content.Provider, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
if IsManifestType(desc.MediaType) {
|
||||
p, err := content.ReadBlob(ctx, provider, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateMediaType(p, desc.MediaType); err != nil {
|
||||
return nil, fmt.Errorf("children: invalid desc %s: %w", desc.Digest, err)
|
||||
}
|
||||
|
||||
// TODO(stevvooe): We just assume oci manifest, for now. There may be
|
||||
// subtle differences from the docker version.
|
||||
var manifest ocispec.Manifest
|
||||
if err := json.Unmarshal(p, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil
|
||||
} else if IsIndexType(desc.MediaType) {
|
||||
p, err := content.ReadBlob(ctx, provider, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateMediaType(p, desc.MediaType); err != nil {
|
||||
return nil, fmt.Errorf("children: invalid desc %s: %w", desc.Digest, err)
|
||||
}
|
||||
|
||||
var index ocispec.Index
|
||||
if err := json.Unmarshal(p, &index); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return append([]ocispec.Descriptor{}, index.Manifests...), nil
|
||||
} else if !IsLayerType(desc.MediaType) && !IsKnownConfig(desc.MediaType) {
|
||||
// Layers and configs are childless data types and should not be logged.
|
||||
log.G(ctx).Debugf("encountered unknown type %v; children may not be fetched", desc.MediaType)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// unknownDocument represents a manifest, manifest list, or index that has not
|
||||
// yet been validated.
|
||||
type unknownDocument struct {
|
||||
MediaType string `json:"mediaType,omitempty"`
|
||||
Config json.RawMessage `json:"config,omitempty"`
|
||||
Layers json.RawMessage `json:"layers,omitempty"`
|
||||
Manifests json.RawMessage `json:"manifests,omitempty"`
|
||||
FSLayers json.RawMessage `json:"fsLayers,omitempty"` // schema 1
|
||||
}
|
||||
|
||||
// validateMediaType returns an error if the byte slice is invalid JSON,
|
||||
// if the format of the blob is not supported, or if the media type
|
||||
// identifies the blob as one format, but it identifies itself as, or
|
||||
// contains elements of another format.
|
||||
func validateMediaType(b []byte, mt string) error {
|
||||
var doc unknownDocument
|
||||
if err := json.Unmarshal(b, &doc); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(doc.FSLayers) != 0 {
|
||||
return fmt.Errorf("media-type: schema 1 not supported")
|
||||
}
|
||||
if IsManifestType(mt) && (len(doc.Manifests) != 0 || IsIndexType(doc.MediaType)) {
|
||||
return fmt.Errorf("media-type: expected manifest but found index (%s)", mt)
|
||||
} else if IsIndexType(mt) && (len(doc.Config) != 0 || len(doc.Layers) != 0 || IsManifestType(doc.MediaType)) {
|
||||
return fmt.Errorf("media-type: expected index but found manifest (%s)", mt)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RootFS returns the unpacked diffids that make up and images rootfs.
|
||||
//
|
||||
// These are used to verify that a set of layers unpacked to the expected
|
||||
// values.
|
||||
func RootFS(ctx context.Context, provider content.Provider, configDesc ocispec.Descriptor) ([]digest.Digest, error) {
|
||||
p, err := content.ReadBlob(ctx, provider, configDesc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var config ocispec.Image
|
||||
if err := json.Unmarshal(p, &config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.RootFS.DiffIDs, nil
|
||||
}
|
||||
|
||||
// ConfigPlatform returns a normalized platform from an image manifest config.
|
||||
func ConfigPlatform(ctx context.Context, provider content.Provider, configDesc ocispec.Descriptor) (ocispec.Platform, error) {
|
||||
p, err := content.ReadBlob(ctx, provider, configDesc)
|
||||
if err != nil {
|
||||
return ocispec.Platform{}, err
|
||||
}
|
||||
|
||||
// Technically, this should be ocispec.Image, but we only need the
|
||||
// ocispec.Platform that is embedded in the image struct.
|
||||
var imagePlatform ocispec.Platform
|
||||
if err := json.Unmarshal(p, &imagePlatform); err != nil {
|
||||
return ocispec.Platform{}, err
|
||||
}
|
||||
return platforms.Normalize(imagePlatform), nil
|
||||
}
|
||||
127
core/images/image_test.go
Normal file
127
core/images/image_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package images
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestValidateMediaType(t *testing.T) {
|
||||
docTests := []struct {
|
||||
mt string
|
||||
index bool
|
||||
}{
|
||||
{MediaTypeDockerSchema2Manifest, false},
|
||||
{ocispec.MediaTypeImageManifest, false},
|
||||
{MediaTypeDockerSchema2ManifestList, true},
|
||||
{ocispec.MediaTypeImageIndex, true},
|
||||
}
|
||||
for _, tc := range docTests {
|
||||
t.Run("manifest-"+tc.mt, func(t *testing.T) {
|
||||
manifest := ocispec.Manifest{
|
||||
Config: ocispec.Descriptor{Size: 1},
|
||||
Layers: []ocispec.Descriptor{{Size: 2}},
|
||||
}
|
||||
b, err := json.Marshal(manifest)
|
||||
require.NoError(t, err, "failed to marshal manifest")
|
||||
|
||||
err = validateMediaType(b, tc.mt)
|
||||
if tc.index {
|
||||
assert.Error(t, err, "manifest should not be a valid index")
|
||||
} else {
|
||||
assert.NoError(t, err, "manifest should be valid")
|
||||
}
|
||||
})
|
||||
t.Run("index-"+tc.mt, func(t *testing.T) {
|
||||
index := ocispec.Index{
|
||||
Manifests: []ocispec.Descriptor{{Size: 1}},
|
||||
}
|
||||
b, err := json.Marshal(index)
|
||||
require.NoError(t, err, "failed to marshal index")
|
||||
|
||||
err = validateMediaType(b, tc.mt)
|
||||
if tc.index {
|
||||
assert.NoError(t, err, "index should be valid")
|
||||
} else {
|
||||
assert.Error(t, err, "index should not be a valid manifest")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
mtTests := []struct {
|
||||
mt string
|
||||
valid []string
|
||||
invalid []string
|
||||
}{{
|
||||
MediaTypeDockerSchema2Manifest,
|
||||
[]string{MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest},
|
||||
[]string{MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex},
|
||||
}, {
|
||||
ocispec.MediaTypeImageManifest,
|
||||
[]string{MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest},
|
||||
[]string{MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex},
|
||||
}, {
|
||||
MediaTypeDockerSchema2ManifestList,
|
||||
[]string{MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex},
|
||||
[]string{MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest},
|
||||
}, {
|
||||
ocispec.MediaTypeImageIndex,
|
||||
[]string{MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex},
|
||||
[]string{MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest},
|
||||
}}
|
||||
for _, tc := range mtTests {
|
||||
for _, v := range tc.valid {
|
||||
t.Run("valid-"+tc.mt+"-"+v, func(t *testing.T) {
|
||||
doc := struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
}{MediaType: v}
|
||||
b, err := json.Marshal(doc)
|
||||
require.NoError(t, err, "failed to marshal document")
|
||||
|
||||
err = validateMediaType(b, tc.mt)
|
||||
assert.NoError(t, err, "document should be valid")
|
||||
})
|
||||
}
|
||||
for _, iv := range tc.invalid {
|
||||
t.Run("invalid-"+tc.mt+"-"+iv, func(t *testing.T) {
|
||||
doc := struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
}{MediaType: iv}
|
||||
b, err := json.Marshal(doc)
|
||||
require.NoError(t, err, "failed to marshal document")
|
||||
|
||||
err = validateMediaType(b, tc.mt)
|
||||
assert.Error(t, err, "document should not be valid")
|
||||
})
|
||||
}
|
||||
}
|
||||
t.Run("schema1", func(t *testing.T) {
|
||||
doc := struct {
|
||||
FSLayers []string `json:"fsLayers"`
|
||||
}{FSLayers: []string{"1"}}
|
||||
b, err := json.Marshal(doc)
|
||||
require.NoError(t, err, "failed to marshal document")
|
||||
|
||||
err = validateMediaType(b, "")
|
||||
assert.Error(t, err, "document should not be valid")
|
||||
})
|
||||
}
|
||||
276
core/images/imagetest/content.go
Normal file
276
core/images/imagetest/content.go
Normal file
@@ -0,0 +1,276 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package imagetest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/plugins/content/local"
|
||||
"github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
// Content represents a piece of image content in the content store along with
|
||||
// its relevant children and size.
|
||||
type Content struct {
|
||||
Descriptor ocispec.Descriptor
|
||||
Labels map[string]string
|
||||
Size Size
|
||||
Children []Content
|
||||
}
|
||||
|
||||
// ContentStore is a temporary content store which provides simple
|
||||
// helper functions for quickly altering content in the store.
|
||||
// Directly modifying the content store without using the helper functions
|
||||
// may result in out of sync test content values, be careful of this
|
||||
// when writing tests.
|
||||
type ContentStore struct {
|
||||
content.Store
|
||||
|
||||
ctx context.Context
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
// NewContentStore creates a new content store in the testing's temporary directory
|
||||
func NewContentStore(ctx context.Context, t *testing.T) ContentStore {
|
||||
cs, err := local.NewLabeledStore(t.TempDir(), newMemoryLabelStore())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return ContentStore{
|
||||
Store: cs,
|
||||
ctx: ctx,
|
||||
t: t,
|
||||
}
|
||||
}
|
||||
|
||||
// Index creates an index with the provided manifests and stores it
|
||||
// in the content store.
|
||||
func (tc ContentStore) Index(manifests ...Content) Content {
|
||||
var index ocispec.Index
|
||||
for _, m := range manifests {
|
||||
index.Manifests = append(index.Manifests, m.Descriptor)
|
||||
}
|
||||
|
||||
idx := tc.JSONObject(ocispec.MediaTypeImageIndex, index)
|
||||
idx.Children = manifests
|
||||
return idx
|
||||
|
||||
}
|
||||
|
||||
// Manifest creates a manifest with the given config and layers then
|
||||
// stores it in the content store.
|
||||
func (tc ContentStore) Manifest(config Content, layers ...Content) Content {
|
||||
var manifest ocispec.Manifest
|
||||
manifest.Config = config.Descriptor
|
||||
for _, l := range layers {
|
||||
manifest.Layers = append(manifest.Layers, l.Descriptor)
|
||||
}
|
||||
m := tc.JSONObject(ocispec.MediaTypeImageManifest, manifest)
|
||||
m.Children = append(m.Children, config)
|
||||
m.Children = append(m.Children, layers...)
|
||||
return m
|
||||
}
|
||||
|
||||
// Blob creates a generic blob with the given data and media type
|
||||
// and stores the data in the content store.
|
||||
func (tc ContentStore) Blob(mediaType string, data []byte) Content {
|
||||
tc.t.Helper()
|
||||
|
||||
descriptor := ocispec.Descriptor{
|
||||
MediaType: mediaType,
|
||||
Digest: digest.SHA256.FromBytes(data),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
|
||||
ref := string(descriptor.Digest) // TODO: Add random component?
|
||||
if err := content.WriteBlob(tc.ctx, tc.Store, ref, bytes.NewReader(data), descriptor); err != nil {
|
||||
tc.t.Fatal(err)
|
||||
}
|
||||
|
||||
return Content{
|
||||
Descriptor: descriptor,
|
||||
Size: Size{
|
||||
Manifest: descriptor.Size,
|
||||
Content: descriptor.Size,
|
||||
},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// RandomBlob creates a blob object in the content store with random data.
|
||||
func (tc ContentStore) RandomBlob(mediaType string, n int) Content {
|
||||
tc.t.Helper()
|
||||
|
||||
data := make([]byte, int64(n))
|
||||
rand.New(rand.NewSource(int64(n))).Read(data)
|
||||
|
||||
return tc.Blob(mediaType, data)
|
||||
}
|
||||
|
||||
// JSONObject creates an object in the content store by first marshaling
|
||||
// to JSON and then storing the data.
|
||||
func (tc ContentStore) JSONObject(mediaType string, i interface{}) Content {
|
||||
tc.t.Helper()
|
||||
|
||||
data, err := json.Marshal(i)
|
||||
if err != nil {
|
||||
tc.t.Fatal(err)
|
||||
}
|
||||
|
||||
return tc.Blob(mediaType, data)
|
||||
}
|
||||
|
||||
// Walk walks all the children of the provided content and calls the provided
|
||||
// function with the associated content store.
|
||||
// Walk can be used to update an object and reflect that change in the content
|
||||
// store.
|
||||
func (tc ContentStore) Walk(c Content, fn func(context.Context, *Content, content.Store) error) Content {
|
||||
tc.t.Helper()
|
||||
|
||||
if err := fn(tc.ctx, &c, tc.Store); err != nil {
|
||||
tc.t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(c.Children) > 0 {
|
||||
children := make([]Content, len(c.Children))
|
||||
for i, child := range c.Children {
|
||||
children[i] = tc.Walk(child, fn)
|
||||
}
|
||||
c.Children = children
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// AddPlatform alters the content desciptor by setting the platform
|
||||
func AddPlatform(c Content, p ocispec.Platform) Content {
|
||||
c.Descriptor.Platform = &p
|
||||
return c
|
||||
}
|
||||
|
||||
// LimitChildren limits the amount of children in the content.
|
||||
// This function is non-recursive and uses the natural ordering.
|
||||
func LimitChildren(c Content, limit int) Content {
|
||||
if images.IsIndexType(c.Descriptor.MediaType) {
|
||||
if len(c.Children) > limit {
|
||||
c.Children = c.Children[0:limit]
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// ContentCreator is a simple interface for generating content for tests
|
||||
type ContentCreator func(ContentStore) Content
|
||||
|
||||
// SimpleManifest generates a simple manifest with small config and random layer
|
||||
// The layer produced is not a valid compressed tar, do not unpack it.
|
||||
func SimpleManifest(layerSize int) ContentCreator {
|
||||
return func(tc ContentStore) Content {
|
||||
config := ocispec.ImageConfig{
|
||||
Env: []string{"random"}, // TODO: Make random string
|
||||
}
|
||||
return tc.Manifest(
|
||||
tc.JSONObject(ocispec.MediaTypeImageConfig, config),
|
||||
tc.RandomBlob(ocispec.MediaTypeImageLayerGzip, layerSize))
|
||||
}
|
||||
}
|
||||
|
||||
// SimpleIndex generates a simple index with the number of simple
|
||||
// manifests specified.
|
||||
func SimpleIndex(manifests, layerSize int) ContentCreator {
|
||||
manifestFn := SimpleManifest(layerSize)
|
||||
return func(tc ContentStore) Content {
|
||||
var m []Content
|
||||
for i := 0; i < manifests; i++ {
|
||||
m = append(m, manifestFn(tc))
|
||||
}
|
||||
return tc.Index(m...)
|
||||
}
|
||||
}
|
||||
|
||||
// StripLayers deletes all layer content from the content store
|
||||
// and updates the content size to 0.
|
||||
func StripLayers(cc ContentCreator) ContentCreator {
|
||||
return func(tc ContentStore) Content {
|
||||
return tc.Walk(cc(tc), func(ctx context.Context, c *Content, store content.Store) error {
|
||||
if images.IsLayerType(c.Descriptor.MediaType) {
|
||||
if err := store.Delete(tc.ctx, c.Descriptor.Digest); err != nil {
|
||||
return err
|
||||
}
|
||||
c.Size.Content = 0
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
type memoryLabelStore struct {
|
||||
l sync.Mutex
|
||||
labels map[digest.Digest]map[string]string
|
||||
}
|
||||
|
||||
// newMemoryLabelStore creates an inmemory label store for testing
|
||||
func newMemoryLabelStore() local.LabelStore {
|
||||
return &memoryLabelStore{
|
||||
labels: map[digest.Digest]map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func (mls *memoryLabelStore) Get(d digest.Digest) (map[string]string, error) {
|
||||
mls.l.Lock()
|
||||
labels := mls.labels[d]
|
||||
mls.l.Unlock()
|
||||
|
||||
return labels, nil
|
||||
}
|
||||
|
||||
func (mls *memoryLabelStore) Set(d digest.Digest, labels map[string]string) error {
|
||||
mls.l.Lock()
|
||||
mls.labels[d] = labels
|
||||
mls.l.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mls *memoryLabelStore) Update(d digest.Digest, update map[string]string) (map[string]string, error) {
|
||||
mls.l.Lock()
|
||||
labels, ok := mls.labels[d]
|
||||
if !ok {
|
||||
labels = map[string]string{}
|
||||
}
|
||||
for k, v := range update {
|
||||
if v == "" {
|
||||
delete(labels, k)
|
||||
} else {
|
||||
labels[k] = v
|
||||
}
|
||||
}
|
||||
mls.labels[d] = labels
|
||||
mls.l.Unlock()
|
||||
|
||||
return labels, nil
|
||||
}
|
||||
58
core/images/imagetest/size.go
Normal file
58
core/images/imagetest/size.go
Normal file
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package imagetest
|
||||
|
||||
// Size represents the size of an object from different methods of counting, in bytes
|
||||
type Size struct {
|
||||
// Manifest is the total Manifest reported size of current object and children
|
||||
Manifest int64
|
||||
|
||||
// Content is the total of store Content of current object and children
|
||||
Content int64
|
||||
|
||||
// Unpacked is the total size of Unpacked data for the given snapshotter
|
||||
// The key is the snapshotter name
|
||||
// The value is the usage reported by the snapshotter for the image
|
||||
Unpacked map[string]int64
|
||||
|
||||
// Uncompressed is the total Uncompressed tar stream data (used for size estimate)
|
||||
Uncompressed int64
|
||||
}
|
||||
|
||||
// ContentSizeCalculator calculates a size property for the test content
|
||||
type ContentSizeCalculator func(Content) int64
|
||||
|
||||
// SizeOfManifest recursively calculates the manifest reported size,
|
||||
// not accounting for duplicate entries.
|
||||
func SizeOfManifest(t Content) int64 {
|
||||
accumulated := t.Size.Manifest
|
||||
for _, child := range t.Children {
|
||||
accumulated = accumulated + SizeOfManifest(child)
|
||||
}
|
||||
return accumulated
|
||||
}
|
||||
|
||||
// SizeOfContent recursively calculates the size of all stored content
|
||||
// The calculation is a simple accumulation based on the structure,
|
||||
// duplicate blobs are not accounted for.
|
||||
func SizeOfContent(t Content) int64 {
|
||||
accumulated := t.Size.Content
|
||||
for _, child := range t.Children {
|
||||
accumulated = accumulated + SizeOfContent(child)
|
||||
}
|
||||
return accumulated
|
||||
}
|
||||
37
core/images/importexport.go
Normal file
37
core/images/importexport.go
Normal file
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package images
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
// Importer is the interface for image importer.
|
||||
type Importer interface {
|
||||
// Import imports an image from a tar stream.
|
||||
Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error)
|
||||
}
|
||||
|
||||
// Exporter is the interface for image exporter.
|
||||
type Exporter interface {
|
||||
// Export exports an image to a tar stream.
|
||||
Export(ctx context.Context, store content.Provider, desc ocispec.Descriptor, writer io.Writer) error
|
||||
}
|
||||
21
core/images/labels.go
Normal file
21
core/images/labels.go
Normal file
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package images
|
||||
|
||||
const (
|
||||
ConvertedDockerSchema1LabelKey = "io.containerd.image/converted-docker-schema1"
|
||||
)
|
||||
215
core/images/mediatypes.go
Normal file
215
core/images/mediatypes.go
Normal file
@@ -0,0 +1,215 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package images
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
// mediatype definitions for image components handled in containerd.
|
||||
//
|
||||
// oci components are generally referenced directly, although we may centralize
|
||||
// here for clarity.
|
||||
const (
|
||||
MediaTypeDockerSchema2Layer = "application/vnd.docker.image.rootfs.diff.tar"
|
||||
MediaTypeDockerSchema2LayerForeign = "application/vnd.docker.image.rootfs.foreign.diff.tar"
|
||||
MediaTypeDockerSchema2LayerGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip"
|
||||
MediaTypeDockerSchema2LayerForeignGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
|
||||
MediaTypeDockerSchema2Config = "application/vnd.docker.container.image.v1+json"
|
||||
MediaTypeDockerSchema2Manifest = "application/vnd.docker.distribution.manifest.v2+json"
|
||||
MediaTypeDockerSchema2ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
|
||||
|
||||
// Checkpoint/Restore Media Types
|
||||
|
||||
MediaTypeContainerd1Checkpoint = "application/vnd.containerd.container.criu.checkpoint.criu.tar"
|
||||
MediaTypeContainerd1CheckpointPreDump = "application/vnd.containerd.container.criu.checkpoint.predump.tar"
|
||||
MediaTypeContainerd1Resource = "application/vnd.containerd.container.resource.tar"
|
||||
MediaTypeContainerd1RW = "application/vnd.containerd.container.rw.tar"
|
||||
MediaTypeContainerd1CheckpointConfig = "application/vnd.containerd.container.checkpoint.config.v1+proto"
|
||||
MediaTypeContainerd1CheckpointOptions = "application/vnd.containerd.container.checkpoint.options.v1+proto"
|
||||
MediaTypeContainerd1CheckpointRuntimeName = "application/vnd.containerd.container.checkpoint.runtime.name"
|
||||
MediaTypeContainerd1CheckpointRuntimeOptions = "application/vnd.containerd.container.checkpoint.runtime.options+proto"
|
||||
|
||||
// MediaTypeDockerSchema1Manifest is the legacy Docker schema1 manifest
|
||||
MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws"
|
||||
|
||||
// Encrypted media types
|
||||
|
||||
MediaTypeImageLayerEncrypted = ocispec.MediaTypeImageLayer + "+encrypted"
|
||||
MediaTypeImageLayerGzipEncrypted = ocispec.MediaTypeImageLayerGzip + "+encrypted"
|
||||
)
|
||||
|
||||
// DiffCompression returns the compression as defined by the layer diff media
|
||||
// type. For Docker media types without compression, "unknown" is returned to
|
||||
// indicate that the media type may be compressed. If the media type is not
|
||||
// recognized as a layer diff, then it returns errdefs.ErrNotImplemented
|
||||
func DiffCompression(ctx context.Context, mediaType string) (string, error) {
|
||||
base, ext := parseMediaTypes(mediaType)
|
||||
switch base {
|
||||
case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerForeign:
|
||||
if len(ext) > 0 {
|
||||
// Type is wrapped
|
||||
return "", nil
|
||||
}
|
||||
// These media types may have been compressed but failed to
|
||||
// use the correct media type. The decompression function
|
||||
// should detect and handle this case.
|
||||
return "unknown", nil
|
||||
case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2LayerForeignGzip:
|
||||
if len(ext) > 0 {
|
||||
// Type is wrapped
|
||||
return "", nil
|
||||
}
|
||||
return "gzip", nil
|
||||
case ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerNonDistributable: //nolint:staticcheck // Non-distributable layers are deprecated
|
||||
if len(ext) > 0 {
|
||||
switch ext[len(ext)-1] {
|
||||
case "gzip":
|
||||
return "gzip", nil
|
||||
case "zstd":
|
||||
return "zstd", nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
default:
|
||||
return "", fmt.Errorf("unrecognised mediatype %s: %w", mediaType, errdefs.ErrNotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
// parseMediaTypes splits the media type into the base type and
|
||||
// an array of sorted extensions
|
||||
func parseMediaTypes(mt string) (mediaType string, suffixes []string) {
|
||||
if mt == "" {
|
||||
return "", []string{}
|
||||
}
|
||||
mediaType, ext, ok := strings.Cut(mt, "+")
|
||||
if !ok {
|
||||
return mediaType, []string{}
|
||||
}
|
||||
|
||||
// Splitting the extensions following the mediatype "(+)gzip+encrypted".
|
||||
// We expect this to be a limited list, so add an arbitrary limit (50).
|
||||
//
|
||||
// Note that DiffCompression is only using the last element, so perhaps we
|
||||
// should split on the last "+" only.
|
||||
suffixes = strings.SplitN(ext, "+", 50)
|
||||
sort.Strings(suffixes)
|
||||
return mediaType, suffixes
|
||||
}
|
||||
|
||||
// IsNonDistributable returns true if the media type is non-distributable.
|
||||
func IsNonDistributable(mt string) bool {
|
||||
return strings.HasPrefix(mt, "application/vnd.oci.image.layer.nondistributable.") ||
|
||||
strings.HasPrefix(mt, "application/vnd.docker.image.rootfs.foreign.")
|
||||
}
|
||||
|
||||
// IsLayerType returns true if the media type is a layer
|
||||
func IsLayerType(mt string) bool {
|
||||
if strings.HasPrefix(mt, "application/vnd.oci.image.layer.") {
|
||||
return true
|
||||
}
|
||||
|
||||
// Parse Docker media types, strip off any + suffixes first
|
||||
switch base, _ := parseMediaTypes(mt); base {
|
||||
case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
|
||||
MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsDockerType returns true if the media type has "application/vnd.docker." prefix
|
||||
func IsDockerType(mt string) bool {
|
||||
return strings.HasPrefix(mt, "application/vnd.docker.")
|
||||
}
|
||||
|
||||
// IsManifestType returns true if the media type is an OCI-compatible manifest.
|
||||
// No support for schema1 manifest.
|
||||
func IsManifestType(mt string) bool {
|
||||
switch mt {
|
||||
case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// IsIndexType returns true if the media type is an OCI-compatible index.
|
||||
func IsIndexType(mt string) bool {
|
||||
switch mt {
|
||||
case ocispec.MediaTypeImageIndex, MediaTypeDockerSchema2ManifestList:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// IsConfigType returns true if the media type is an OCI-compatible image config.
|
||||
// No support for containerd checkpoint configs.
|
||||
func IsConfigType(mt string) bool {
|
||||
switch mt {
|
||||
case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// IsKnownConfig returns true if the media type is a known config type,
|
||||
// including containerd checkpoint configs
|
||||
func IsKnownConfig(mt string) bool {
|
||||
switch mt {
|
||||
case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig,
|
||||
MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ChildGCLabels returns the label for a given descriptor to reference it
|
||||
func ChildGCLabels(desc ocispec.Descriptor) []string {
|
||||
mt := desc.MediaType
|
||||
if IsKnownConfig(mt) {
|
||||
return []string{"containerd.io/gc.ref.content.config"}
|
||||
}
|
||||
|
||||
switch mt {
|
||||
case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
|
||||
return []string{"containerd.io/gc.ref.content.m."}
|
||||
}
|
||||
|
||||
if IsLayerType(mt) {
|
||||
return []string{"containerd.io/gc.ref.content.l."}
|
||||
}
|
||||
|
||||
return []string{"containerd.io/gc.ref.content."}
|
||||
}
|
||||
|
||||
// ChildGCLabelsFilterLayers returns the labels for a given descriptor to
|
||||
// reference it, skipping layer media types
|
||||
func ChildGCLabelsFilterLayers(desc ocispec.Descriptor) []string {
|
||||
if IsLayerType(desc.MediaType) {
|
||||
return nil
|
||||
}
|
||||
return ChildGCLabels(desc)
|
||||
}
|
||||
167
core/images/usage/calculator.go
Normal file
167
core/images/usage/calculator.go
Normal file
@@ -0,0 +1,167 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package usage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
"github.com/containerd/containerd/v2/platforms"
|
||||
"github.com/containerd/containerd/v2/snapshots"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
type usageOptions struct {
|
||||
platform platforms.MatchComparer
|
||||
manifestLimit int
|
||||
manifestOnly bool
|
||||
snapshots func(name string) snapshots.Snapshotter
|
||||
}
|
||||
|
||||
type Opt func(*usageOptions) error
|
||||
|
||||
// WithManifestLimit sets the limit to the number of manifests which will
|
||||
// be walked for usage. Setting this value to 0 will require all manifests to
|
||||
// be walked, returning ErrNotFound if manifests are missing.
|
||||
// NOTE: By default all manifests which exist will be walked
|
||||
// and any non-existent manifests and their subobjects will be ignored.
|
||||
func WithManifestLimit(platform platforms.MatchComparer, i int) Opt {
|
||||
// If 0 then don't filter any manifests
|
||||
// By default limits to current platform
|
||||
return func(o *usageOptions) error {
|
||||
o.manifestLimit = i
|
||||
o.platform = platform
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithSnapshotters will check for referenced snapshots from the image objects
|
||||
// and include the snapshot size in the total usage.
|
||||
func WithSnapshotters(f func(string) snapshots.Snapshotter) Opt {
|
||||
return func(o *usageOptions) error {
|
||||
o.snapshots = f
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifestUsage is used to get the usage for an image based on what is
|
||||
// reported by the manifests rather than what exists in the content store.
|
||||
// NOTE: This function is best used with the manifest limit set to get a
|
||||
// consistent value, otherwise non-existent manifests will be excluded.
|
||||
func WithManifestUsage() Opt {
|
||||
return func(o *usageOptions) error {
|
||||
o.manifestOnly = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func CalculateImageUsage(ctx context.Context, i images.Image, provider content.InfoReaderProvider, opts ...Opt) (int64, error) {
|
||||
var config usageOptions
|
||||
for _, opt := range opts {
|
||||
if err := opt(&config); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
handler = images.ChildrenHandler(provider)
|
||||
size int64
|
||||
mustExist bool
|
||||
)
|
||||
|
||||
if config.platform != nil {
|
||||
handler = images.LimitManifests(handler, config.platform, config.manifestLimit)
|
||||
mustExist = true
|
||||
}
|
||||
|
||||
var wh images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
var usage int64
|
||||
children, err := handler(ctx, desc)
|
||||
if err != nil {
|
||||
if !errdefs.IsNotFound(err) || mustExist {
|
||||
return nil, err
|
||||
}
|
||||
if !config.manifestOnly {
|
||||
// Do not count size of non-existent objects
|
||||
desc.Size = 0
|
||||
}
|
||||
} else if config.snapshots != nil || !config.manifestOnly {
|
||||
info, err := provider.Info(ctx, desc.Digest)
|
||||
if err != nil {
|
||||
if !errdefs.IsNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
if !config.manifestOnly {
|
||||
// Do not count size of non-existent objects
|
||||
desc.Size = 0
|
||||
}
|
||||
} else {
|
||||
if info.Size > desc.Size {
|
||||
// Count actual usage, Size may be unset or -1
|
||||
desc.Size = info.Size
|
||||
}
|
||||
|
||||
if config.snapshots != nil {
|
||||
for k, v := range info.Labels {
|
||||
const prefix = "containerd.io/gc.ref.snapshot."
|
||||
if !strings.HasPrefix(k, prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
sn := config.snapshots(k[len(prefix):])
|
||||
if sn == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
u, err := sn.Usage(ctx, v)
|
||||
if err != nil {
|
||||
if !errdefs.IsNotFound(err) && !errdefs.IsInvalidArgument(err) {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
usage += u.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ignore unknown sizes. Generally unknown sizes should
|
||||
// never be set in manifests, however, the usage
|
||||
// calculation does not need to enforce this.
|
||||
if desc.Size >= 0 {
|
||||
usage += desc.Size
|
||||
}
|
||||
|
||||
atomic.AddInt64(&size, usage)
|
||||
|
||||
return children, nil
|
||||
}
|
||||
|
||||
l := semaphore.NewWeighted(3)
|
||||
if err := images.Dispatch(ctx, wh, l, i.Target); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return size, nil
|
||||
}
|
||||
100
core/images/usage/calculator_test.go
Normal file
100
core/images/usage/calculator_test.go
Normal file
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package usage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/core/images/imagetest"
|
||||
"github.com/containerd/containerd/v2/platforms"
|
||||
"github.com/containerd/log/logtest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
func TestUsageCalculation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
target imagetest.ContentCreator
|
||||
expected imagetest.ContentSizeCalculator
|
||||
opts []Opt
|
||||
}{
|
||||
{
|
||||
name: "simple",
|
||||
target: imagetest.SimpleManifest(50),
|
||||
expected: imagetest.SizeOfManifest,
|
||||
opts: []Opt{WithManifestUsage()},
|
||||
},
|
||||
{
|
||||
name: "simpleIndex",
|
||||
target: imagetest.SimpleIndex(5, 50),
|
||||
expected: imagetest.SizeOfContent,
|
||||
},
|
||||
{
|
||||
name: "stripLayersManifestOnly",
|
||||
target: imagetest.StripLayers(imagetest.SimpleIndex(3, 51)),
|
||||
expected: imagetest.SizeOfManifest,
|
||||
opts: []Opt{WithManifestUsage()},
|
||||
},
|
||||
{
|
||||
name: "stripLayers",
|
||||
target: imagetest.StripLayers(imagetest.SimpleIndex(4, 60)),
|
||||
expected: imagetest.SizeOfContent,
|
||||
},
|
||||
{
|
||||
name: "manifestlimit",
|
||||
target: func(tc imagetest.ContentStore) imagetest.Content {
|
||||
return tc.Index(
|
||||
imagetest.AddPlatform(imagetest.SimpleManifest(5)(tc), ocispec.Platform{Architecture: "amd64", OS: "linux"}),
|
||||
imagetest.AddPlatform(imagetest.SimpleManifest(10)(tc), ocispec.Platform{Architecture: "arm64", OS: "linux"}),
|
||||
)
|
||||
},
|
||||
expected: func(t imagetest.Content) int64 { return imagetest.SizeOfManifest(imagetest.LimitChildren(t, 1)) },
|
||||
opts: []Opt{
|
||||
WithManifestLimit(platforms.Only(ocispec.Platform{Architecture: "amd64", OS: "linux"}), 1),
|
||||
WithManifestUsage(),
|
||||
},
|
||||
},
|
||||
// TODO: Add test with snapshot
|
||||
} {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := logtest.WithT(context.Background(), t)
|
||||
|
||||
cs := imagetest.NewContentStore(ctx, t)
|
||||
content := tc.target(cs)
|
||||
img := images.Image{
|
||||
Name: tc.name,
|
||||
Target: content.Descriptor,
|
||||
Labels: content.Labels,
|
||||
}
|
||||
|
||||
usage, err := CalculateImageUsage(ctx, img, cs, tc.opts...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expected := tc.expected(content)
|
||||
|
||||
if expected != usage {
|
||||
t.Fatalf("unexpected usage: %d, expected %d", usage, expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user