Add docker importer
Update ctr to support all formats by default Signed-off-by: Derek McGowan <derek@mcgstyle.net>
This commit is contained in:
Derek McGowan
parent
f57c5cdefb
commit
9e6db71954
@ -24,6 +24,7 @@ import (
|
||||
|
||||
"github.com/containerd/containerd"
|
||||
"github.com/containerd/containerd/cmd/ctr/commands"
|
||||
"github.com/containerd/containerd/images/docker"
|
||||
oci "github.com/containerd/containerd/images/oci"
|
||||
"github.com/containerd/containerd/log"
|
||||
"github.com/urfave/cli"
|
||||
@ -35,11 +36,14 @@ var importCommand = cli.Command{
|
||||
ArgsUsage: "[flags] <in>",
|
||||
Description: `Import images from a tar stream.
|
||||
Implemented formats:
|
||||
- oci.v1 (default)
|
||||
- oci.v1
|
||||
- docker.v1.1
|
||||
- docker.v1.2
|
||||
|
||||
|
||||
For oci.v1 format, you need to specify --oci-name because an OCI archive contains image refs (tags)
|
||||
but does not contain the base image name.
|
||||
For OCI v1, you may need to specify --base-name because an OCI archive
|
||||
contains only partial image references (tags without the base image name).
|
||||
If no base image name is provided, a name will be generated as "import-%{date}".
|
||||
|
||||
e.g.
|
||||
$ ctr images import --format oci.v1 --oci-name foo/bar foobar.tar
|
||||
@ -50,18 +54,22 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
|
||||
Flags: append([]cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "format",
|
||||
Value: "oci.v1",
|
||||
Usage: "image format. See DESCRIPTION.",
|
||||
Value: "",
|
||||
Usage: "image format, by default supports OCI v1, Docker v1.1, Docker v1.2",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "prefix,oci-name",
|
||||
Name: "base-name,oci-name",
|
||||
Value: "",
|
||||
Usage: "prefix image name for added images",
|
||||
Usage: "base image name for added images, when provided images without this name prefix are filtered out",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "digests",
|
||||
Usage: "whether to create digest images",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "index-name",
|
||||
Usage: "image name to keep index as, by default index is discarded",
|
||||
},
|
||||
}, commands.SnapshotterFlags...),
|
||||
|
||||
Action: func(context *cli.Context) error {
|
||||
@ -70,23 +78,30 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
|
||||
opts []containerd.ImportOpt
|
||||
)
|
||||
|
||||
prefix := context.String("base-name")
|
||||
if prefix == "" {
|
||||
prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02"))
|
||||
}
|
||||
|
||||
switch format := context.String("format"); format {
|
||||
case "oci.v1":
|
||||
case "", "docker", "docker.v1.1", "docker.v1.2":
|
||||
opts = append(opts, containerd.WithImporter(&docker.V1Importer{}))
|
||||
opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, context.String("base-name") != "")))
|
||||
case "oci", "oci.v1":
|
||||
opts = append(opts, containerd.WithImporter(&oci.V1Importer{}))
|
||||
|
||||
prefix := context.String("prefix")
|
||||
if prefix == "" {
|
||||
prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02"))
|
||||
}
|
||||
|
||||
opts = append(opts, containerd.WithImageRefTranslator(oci.RefTranslator(prefix)))
|
||||
if context.Bool("digests") {
|
||||
opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix)))
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown format %s", format)
|
||||
}
|
||||
|
||||
if context.Bool("digests") {
|
||||
opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix)))
|
||||
}
|
||||
|
||||
if idxName := context.String("index-name"); idxName != "" {
|
||||
opts = append(opts, containerd.WithIndexName(idxName))
|
||||
}
|
||||
|
||||
client, ctx, cancel, err := commands.NewClient(context)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
397
images/docker/importer.go
Normal file
397
images/docker/importer.go
Normal file
@ -0,0 +1,397 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package docker provides a Docker compatible importer capable of
|
||||
// importing both Docker and OCI formats.
|
||||
package docker
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/containerd/containerd/content"
|
||||
"github.com/containerd/containerd/images"
|
||||
"github.com/containerd/containerd/log"
|
||||
"github.com/containerd/cri/pkg/util"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
specs "github.com/opencontainers/image-spec/specs-go"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// V1Importer implements OCI Image Spec v1.
|
||||
type V1Importer struct {
|
||||
// TODO: Add option to compress layers on ingest
|
||||
}
|
||||
|
||||
var _ images.Importer = &V1Importer{}
|
||||
|
||||
// Import implements Importer.
|
||||
func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) {
|
||||
var (
|
||||
desc ocispec.Descriptor
|
||||
tr = tar.NewReader(reader)
|
||||
|
||||
mfsts []manifestDotJSON
|
||||
symlinkLayers = make(map[string]string) // key: filename (foobar/layer.tar), value: linkname (targetlayerid/layer.tar)
|
||||
layers = make(map[string]ocispec.Descriptor) // key: filename (foobar/layer.tar)
|
||||
configs = make(map[string]imageConfig) // key: filename (foobar.json)
|
||||
|
||||
)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeSymlink && isLayerTar(hdr.Name) {
|
||||
linkname, err := followSymlinkLayer(hdr.Linkname)
|
||||
if err != nil {
|
||||
log.G(ctx).WithError(err).WithField("file", hdr.Name).Debugf("symlink to %s ignored", hdr.Linkname)
|
||||
} else {
|
||||
symlinkLayers[hdr.Name] = linkname
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
|
||||
log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
|
||||
continue
|
||||
}
|
||||
hdrName := path.Clean(hdr.Name)
|
||||
if hdrName == "index.json" {
|
||||
if desc.Digest != "" {
|
||||
return ocispec.Descriptor{}, errors.New("duplicated index.json")
|
||||
}
|
||||
desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
} else if strings.HasPrefix(hdrName, "blobs/") {
|
||||
if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
} else if hdrName == ocispec.ImageLayoutFile {
|
||||
// TODO Validate
|
||||
} else if hdr.Name == "manifest.json" {
|
||||
mfsts, err = onUntarManifestJSON(tr)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name)
|
||||
}
|
||||
} else if isLayerTar(hdr.Name) {
|
||||
desc, err := onUntarLayerTar(ctx, tr, store, hdr.Name, hdr.Size)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, errors.Wrapf(err, "untar layer %q", hdr.Name)
|
||||
}
|
||||
layers[hdr.Name] = desc
|
||||
} else if isDotJSON(hdr.Name) {
|
||||
c, err := onUntarDotJSON(ctx, tr, store, hdr.Name, hdr.Size)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, errors.Wrapf(err, "untar config %q", hdr.Name)
|
||||
}
|
||||
configs[hdr.Name] = c
|
||||
} else {
|
||||
log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored")
|
||||
}
|
||||
|
||||
}
|
||||
if desc.Digest != "" {
|
||||
return desc, nil
|
||||
}
|
||||
|
||||
for name, linkname := range symlinkLayers {
|
||||
desc, ok := layers[linkname]
|
||||
if !ok {
|
||||
return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname)
|
||||
}
|
||||
layers[name] = desc
|
||||
}
|
||||
|
||||
var idx ocispec.Index
|
||||
for _, mfst := range mfsts {
|
||||
config, ok := configs[mfst.Config]
|
||||
if !ok {
|
||||
return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config)
|
||||
}
|
||||
|
||||
layers, err := resolveLayers(mfst.Layers, layers)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers")
|
||||
}
|
||||
|
||||
manifest := ocispec.Manifest{
|
||||
Versioned: specs.Versioned{
|
||||
SchemaVersion: 2,
|
||||
},
|
||||
Config: config.desc,
|
||||
Layers: layers,
|
||||
}
|
||||
|
||||
desc, err := writeManifest(ctx, store, manifest, config.img.Architecture, config.img.OS)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest")
|
||||
}
|
||||
|
||||
if len(mfst.RepoTags) == 0 {
|
||||
idx.Manifests = append(idx.Manifests, desc)
|
||||
} else {
|
||||
// Add descriptor per tag
|
||||
for _, ref := range mfst.RepoTags {
|
||||
msftdesc := desc
|
||||
|
||||
// TODO: Replace this function to not depend on reference package
|
||||
normalized, err := util.NormalizeImageRef(ref)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, errors.Wrapf(err, "normalize image ref %q", ref)
|
||||
}
|
||||
|
||||
msftdesc.Annotations = map[string]string{
|
||||
ocispec.AnnotationRefName: normalized.String(),
|
||||
}
|
||||
|
||||
idx.Manifests = append(idx.Manifests, msftdesc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return writeIndex(ctx, store, idx)
|
||||
}
|
||||
|
||||
// RefTranslator creates a reference which only has a tag or verifies
|
||||
// a full reference.
|
||||
func RefTranslator(image string, checkPrefix bool) func(string) string {
|
||||
return func(ref string) string {
|
||||
// Check if ref is full reference
|
||||
if strings.ContainsAny(ref, "/:@") {
|
||||
// If not prefixed, don't include image
|
||||
if checkPrefix && !isImagePrefix(ref, image) {
|
||||
return ""
|
||||
}
|
||||
return ref
|
||||
}
|
||||
return image + ":" + ref
|
||||
}
|
||||
}
|
||||
|
||||
func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) {
|
||||
b, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
desc := ocispec.Descriptor{
|
||||
MediaType: ocispec.MediaTypeImageIndex,
|
||||
Digest: digest.FromBytes(b),
|
||||
Size: size,
|
||||
}
|
||||
if int64(len(b)) != size {
|
||||
return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size)
|
||||
}
|
||||
|
||||
if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
return desc, err
|
||||
}
|
||||
|
||||
func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error {
|
||||
// name is like "blobs/sha256/deadbeef"
|
||||
split := strings.Split(name, "/")
|
||||
if len(split) != 3 {
|
||||
return errors.Errorf("unexpected name: %q", name)
|
||||
}
|
||||
algo := digest.Algorithm(split[1])
|
||||
if !algo.Available() {
|
||||
return errors.Errorf("unsupported algorithm: %s", algo)
|
||||
}
|
||||
dgst := digest.NewDigestFromHex(algo.String(), split[2])
|
||||
return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst})
|
||||
}
|
||||
|
||||
// manifestDotJSON is an entry in manifest.json.
|
||||
type manifestDotJSON struct {
|
||||
Config string
|
||||
RepoTags []string
|
||||
Layers []string
|
||||
// Parent is unsupported
|
||||
Parent string
|
||||
}
|
||||
|
||||
// isLayerTar returns true if name is like "foobar/layer.tar"
|
||||
func isLayerTar(name string) bool {
|
||||
slashes := len(strings.Split(name, "/"))
|
||||
return slashes == 2 && strings.HasSuffix(name, "/layer.tar")
|
||||
}
|
||||
|
||||
// followSymlinkLayer returns actual layer name of the symlink layer.
|
||||
// It returns "foobar/layer.tar" if the name is like
|
||||
// "../foobar/layer.tar", and returns error if the name
|
||||
// is not in "../foobar/layer.tar" format.
|
||||
func followSymlinkLayer(name string) (string, error) {
|
||||
parts := strings.Split(name, "/")
|
||||
if len(parts) != 3 || parts[0] != ".." {
|
||||
return "", errors.New("invalid symlink layer")
|
||||
}
|
||||
name = strings.TrimPrefix(name, "../")
|
||||
if !isLayerTar(name) {
|
||||
return "", errors.New("invalid layer tar")
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// isDotJSON returns true if name is like "foobar.json"
|
||||
func isDotJSON(name string) bool {
|
||||
slashes := len(strings.Split(name, "/"))
|
||||
return slashes == 1 && strings.HasSuffix(name, ".json")
|
||||
}
|
||||
|
||||
func resolveLayers(layerIDs []string, layerIDMap map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
||||
var layers []ocispec.Descriptor
|
||||
for _, f := range layerIDs {
|
||||
desc, ok := layerIDMap[f]
|
||||
if !ok {
|
||||
return nil, errors.Errorf("layer %q not found", f)
|
||||
}
|
||||
layers = append(layers, desc)
|
||||
}
|
||||
return layers, nil
|
||||
}
|
||||
|
||||
func writeManifest(ctx context.Context, cs content.Ingester, manifest ocispec.Manifest, arch, os string) (ocispec.Descriptor, error) {
|
||||
manifestBytes, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
desc := ocispec.Descriptor{
|
||||
MediaType: ocispec.MediaTypeImageManifest,
|
||||
Digest: digest.FromBytes(manifestBytes),
|
||||
Size: int64(len(manifestBytes)),
|
||||
}
|
||||
if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
if arch != "" || os != "" {
|
||||
desc.Platform = &ocispec.Platform{
|
||||
Architecture: arch,
|
||||
OS: os,
|
||||
}
|
||||
}
|
||||
return desc, nil
|
||||
}
|
||||
|
||||
func writeIndex(ctx context.Context, cs content.Ingester, manifest ocispec.Index) (ocispec.Descriptor, error) {
|
||||
manifestBytes, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
desc := ocispec.Descriptor{
|
||||
MediaType: ocispec.MediaTypeImageIndex,
|
||||
Digest: digest.FromBytes(manifestBytes),
|
||||
Size: int64(len(manifestBytes)),
|
||||
}
|
||||
if err := content.WriteBlob(ctx, cs, "index-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
|
||||
return desc, nil
|
||||
}
|
||||
|
||||
func onUntarManifestJSON(r io.Reader) ([]manifestDotJSON, error) {
|
||||
// name: "manifest.json"
|
||||
b, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var mfsts []manifestDotJSON
|
||||
if err := json.Unmarshal(b, &mfsts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mfsts, nil
|
||||
}
|
||||
|
||||
func onUntarLayerTar(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (ocispec.Descriptor, error) {
|
||||
// name is like "foobar/layer.tar" ( guaranteed by isLayerTar() )
|
||||
split := strings.Split(name, "/")
|
||||
// note: split[0] is not expected digest here
|
||||
cw, err := cs.Writer(ctx, content.WithRef("layer-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size}))
|
||||
if err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
// TODO: support compression and committing with labels
|
||||
defer cw.Close()
|
||||
if err := content.Copy(ctx, cw, r, size, ""); err != nil {
|
||||
return ocispec.Descriptor{}, err
|
||||
}
|
||||
return ocispec.Descriptor{
|
||||
MediaType: ocispec.MediaTypeImageLayer,
|
||||
Size: size,
|
||||
Digest: cw.Digest(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type imageConfig struct {
|
||||
desc ocispec.Descriptor
|
||||
img ocispec.Image
|
||||
}
|
||||
|
||||
func onUntarDotJSON(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (imageConfig, error) {
|
||||
config := imageConfig{}
|
||||
config.desc.MediaType = ocispec.MediaTypeImageConfig
|
||||
config.desc.Size = size
|
||||
// name is like "foobar.json" ( guaranteed by is DotJSON() )
|
||||
split := strings.Split(name, ".")
|
||||
cw, err := cs.Writer(ctx, content.WithRef("config-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size}))
|
||||
if err != nil {
|
||||
return imageConfig{}, err
|
||||
}
|
||||
defer cw.Close()
|
||||
var buf bytes.Buffer
|
||||
tr := io.TeeReader(r, &buf)
|
||||
if err := content.Copy(ctx, cw, tr, size, ""); err != nil {
|
||||
return imageConfig{}, err
|
||||
}
|
||||
config.desc.Digest = cw.Digest()
|
||||
if err := json.Unmarshal(buf.Bytes(), &config.img); err != nil {
|
||||
return imageConfig{}, err
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func isImagePrefix(s, prefix string) bool {
|
||||
if !strings.HasPrefix(s, prefix) {
|
||||
return false
|
||||
}
|
||||
if len(s) > len(prefix) {
|
||||
switch s[len(prefix)] {
|
||||
case '/', ':', '@':
|
||||
// Prevent matching partial namespaces
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user