github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

client: add Import() and Export() for importing/exporting image in OCI format

Browse Source 
Export as a tar (Note: "-" can be used for stdout):

    $ ctr images export /tmp/oci-busybox.tar docker.io/library/busybox:latest

Import a tar (Note: "-" can be used for stdin):

    $ ctr images import foo/new:latest /tmp/oci-busybox.tar

Note: media types are not converted at the moment: e.g.
  application/vnd.docker.image.rootfs.diff.tar.gzip
  -> application/vnd.oci.image.layer.v1.tar+gzip

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>
This commit is contained in:
Akihiro Suda 2017-06-15 08:50:20 +00:00
parent 856b038437
commit b518f11dba
13 changed files with 904 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
client.go
View File

@ -3,6 +3,7 @@ package containerd
import (
"context"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
@ -25,6 +26,7 @@ import (
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/plugin"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/remotes/docker/schema1"
@ -552,3 +554,120 @@ func (c *Client) Version(ctx context.Context) (Version, error) {
Revision: response.Revision,
}, nil
}
type imageFormat string
const (
ociImageFormat imageFormat = "oci"
)
type importOpts struct {
format imageFormat
refObject string
}
type ImportOpt func(c *importOpts) error
func WithOCIImportFormat() ImportOpt {
return func(c *importOpts) error {
if c.format != "" {
return errors.New("format already set")
}
c.format = ociImageFormat
return nil
}
}
// WithRefObject specifies the ref object to import.
// If refObject is empty, it is copied from the ref argument of Import().
func WithRefObject(refObject string) ImportOpt {
return func(c *importOpts) error {
c.refObject = refObject
return nil
}
}
func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) {
var iopts importOpts
for _, o := range opts {
if err := o(&iopts); err != nil {
return iopts, err
}
}
// use OCI as the default format
if iopts.format == "" {
iopts.format = ociImageFormat
}
// if refObject is not explicitly specified, use the one specified in ref
if iopts.refObject == "" {
refSpec, err := reference.Parse(ref)
if err != nil {
return iopts, err
}
iopts.refObject = refSpec.Object
}
return iopts, nil
}
// Import imports an image from a Tar stream using reader.
// OCI format is assumed by default.
//
// Note that unreferenced blobs are imported to the content store as well.
func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) {
iopts, err := resolveImportOpt(ref, opts...)
if err != nil {
return nil, err
}
switch iopts.format {
case ociImageFormat:
return c.importFromOCITar(ctx, ref, reader, iopts)
default:
return nil, errors.Errorf("unsupported format: %s", iopts.format)
}
}
type exportOpts struct {
format imageFormat
}
type ExportOpt func(c *exportOpts) error
func WithOCIExportFormat() ExportOpt {
return func(c *exportOpts) error {
if c.format != "" {
return errors.New("format already set")
}
c.format = ociImageFormat
return nil
}
}
// TODO: add WithMediaTypeTranslation that transforms media types according to the format.
// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip
// -> application/vnd.oci.image.layer.v1.tar+gzip
// Export exports an image to a Tar stream.
// OCI format is used by default.
// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc.
func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
var eopts exportOpts
for _, o := range opts {
if err := o(&eopts); err != nil {
return nil, err
}
}
// use OCI as the default format
if eopts.format == "" {
eopts.format = ociImageFormat
}
pr, pw := io.Pipe()
switch eopts.format {
case ociImageFormat:
go func() {
pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts))
}()
default:
return nil, errors.Errorf("unsupported format: %s", eopts.format)
}
return pr, nil
}

106
cmd/ctr/export.go Normal file
View File

@ -0,0 +1,106 @@
package main
import (
"io"
"os"
"github.com/containerd/containerd/reference"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/urfave/cli"
)
var imagesExportCommand = cli.Command{
Name: "export",
Usage: "export an image",
ArgsUsage: "[flags] <out> <image>",
Description: `Export an image to a tar stream
`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "oci-ref-name",
Value: "",
Usage: "Override org.opencontainers.image.ref.name annotation",
},
cli.StringFlag{
Name: "manifest",
Usage: "Digest of manifest",
},
cli.StringFlag{
Name: "manifest-type",
Usage: "Media type of manifest digest",
Value: ocispec.MediaTypeImageManifest,
},
},
Action: func(clicontext *cli.Context) error {
var (
out = clicontext.Args().First()
local = clicontext.Args().Get(1)
desc ocispec.Descriptor
)
ctx, cancel := appContext(clicontext)
defer cancel()
client, err := newClient(clicontext)
if err != nil {
return err
}
if manifest := clicontext.String("manifest"); manifest != "" {
desc.Digest, err = digest.Parse(manifest)
if err != nil {
return errors.Wrap(err, "invalid manifest digest")
}
desc.MediaType = clicontext.String("manifest-type")
} else {
img, err := client.ImageService().Get(ctx, local)
if err != nil {
return errors.Wrap(err, "unable to resolve image to manifest")
}
desc = img.Target
}
if desc.Annotations == nil {
desc.Annotations = make(map[string]string)
}
if s, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok || s == "" {
if ociRefName := determineOCIRefName(local); ociRefName != "" {
desc.Annotations[ocispec.AnnotationRefName] = ociRefName
}
if ociRefName := clicontext.String("oci-ref-name"); ociRefName != "" {
desc.Annotations[ocispec.AnnotationRefName] = ociRefName
}
}
var w io.WriteCloser
if out == "-" {
w = os.Stdout
} else {
w, err = os.Create(out)
if err != nil {
return nil
}
}
r, err := client.Export(ctx, desc)
if err != nil {
return err
}
if _, err := io.Copy(w, r); err != nil {
return err
}
if err := w.Close(); err != nil {
return err
}
return r.Close()
},
}
func determineOCIRefName(local string) string {
refspec, err := reference.Parse(local)
if err != nil {
return ""
}
tag, _ := reference.SplitObject(refspec.Object)
return tag
}

2
cmd/ctr/images.go
View File

@ -22,6 +22,8 @@ var imageCommand = cli.Command{
imagesListCommand,
imageRemoveCommand,
imagesSetLabelsCommand,
imagesImportCommand,
imagesExportCommand,
},
}

70
cmd/ctr/import.go Normal file
View File

@ -0,0 +1,70 @@
package main
import (
"fmt"
"io"
"os"
"github.com/containerd/containerd"
"github.com/containerd/containerd/log"
"github.com/urfave/cli"
)
var imagesImportCommand = cli.Command{
Name: "import",
Usage: "import an image",
ArgsUsage: "[flags] <ref> <in>",
Description: `Import an image from a tar stream
`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "ref-object",
Value: "",
Usage: "reference object e.g. tag@digest (default: use the object specified in ref)",
},
},
Action: func(clicontext *cli.Context) error {
var (
ref = clicontext.Args().First()
in = clicontext.Args().Get(1)
refObject = clicontext.String("ref-object")
)
ctx, cancel := appContext(clicontext)
defer cancel()
client, err := newClient(clicontext)
if err != nil {
return err
}
var r io.ReadCloser
if in == "-" {
r = os.Stdin
} else {
r, err = os.Open(in)
if err != nil {
return err
}
}
img, err := client.Import(ctx,
ref,
r,
containerd.WithRefObject(refObject),
)
if err != nil {
return err
}
if err = r.Close(); err != nil {
return err
}
log.G(ctx).WithField("image", ref).Debug("unpacking")
// TODO: Show unpack status
fmt.Printf("unpacking %s...", img.Target().Digest)
err = img.Unpack(ctx, clicontext.String("snapshotter"))
fmt.Println("done")
return err
},
}

5
content/content.go
View File

@ -5,6 +5,7 @@ import (
"io"
"time"
"github.com/containerd/containerd/oci"
"github.com/opencontainers/go-digest"
)
@ -77,10 +78,8 @@ type IngestManager interface {
}
type Writer interface {
io.WriteCloser
oci.BlobWriter
Status() (Status, error)
Digest() digest.Digest
Commit(size int64, expected digest.Digest) error
Truncate(size int64) error
}

5
content/local/writer.go
View File

@ -8,6 +8,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/oci"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
@ -78,7 +79,7 @@ func (w *writer) Commit(size int64, expected digest.Digest) error {
}
if size > 0 && size != fi.Size() {
return errors.Errorf("%q failed size validation: %v != %v", w.ref, fi.Size(), size)
return oci.ErrUnexpectedSize{Expected: size, Actual: fi.Size()}
}
if err := w.fp.Close(); err != nil {
@ -87,7 +88,7 @@ func (w *writer) Commit(size int64, expected digest.Digest) error {
dgst := w.digester.Digest()
if expected != "" && expected != dgst {
return errors.Errorf("unexpected digest: %v != %v", dgst, expected)
return oci.ErrUnexpectedDigest{Expected: expected, Actual: dgst}
}
var (

65
export.go Normal file
View File

@ -0,0 +1,65 @@
package containerd
import (
"archive/tar"
"context"
"io"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/oci"
ocispecs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error {
tw := tar.NewWriter(writer)
img := oci.Tar(tw)
// For tar, we defer creating index until end of the function.
if err := oci.Init(img, oci.InitOpts{SkipCreateIndex: true}); err != nil {
return err
}
cs := c.ContentStore()
handlers := images.Handlers(
images.ChildrenHandler(cs),
exportHandler(cs, img),
)
// For tar, we need to use Walk instead of Dispatch for ensuring sequential write
if err := images.Walk(ctx, handlers, desc); err != nil {
return err
}
// For tar, we don't use oci.PutManifestDescriptorToIndex() which allows appending desc to existing index.json
// but requires img to support random read access so as to read index.json.
return oci.WriteIndex(img,
ocispec.Index{
Versioned: ocispecs.Versioned{
SchemaVersion: 2,
},
Manifests: []ocispec.Descriptor{desc},
},
)
}
func exportHandler(cs content.Store, img oci.ImageDriver) images.HandlerFunc {
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
r, err := cs.Reader(ctx, desc.Digest)
if err != nil {
return nil, err
}
w, err := oci.NewBlobWriter(img, desc.Digest.Algorithm())
if err != nil {
return nil, err
}
if _, err = io.Copy(w, r); err != nil {
return nil, err
}
if err = w.Commit(desc.Size, desc.Digest); err != nil {
return nil, err
}
if err = w.Close(); err != nil {
return nil, err
}
return nil, nil
}
}

64
export_test.go Normal file
View File

@ -0,0 +1,64 @@
package containerd
import (
"archive/tar"
"io"
"runtime"
"testing"
)
// TestExport exports testImage as a tar stream
func TestExport(t *testing.T) {
// TODO: support windows
if testing.Short() || runtime.GOOS == "windows" {
t.Skip()
}
ctx, cancel := testContext()
defer cancel()
client, err := New(address)
if err != nil {
t.Fatal(err)
}
defer client.Close()
pulled, err := client.Pull(ctx, testImage)
if err != nil {
t.Fatal(err)
}
exportedStream, err := client.Export(ctx, pulled.Target())
if err != nil {
t.Fatal(err)
}
assertOCITar(t, exportedStream)
}
func assertOCITar(t *testing.T, r io.Reader) {
// TODO: add more assertion
tr := tar.NewReader(r)
foundOCILayout := false
foundIndexJSON := false
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Error(err)
continue
}
if h.Name == "oci-layout" {
foundOCILayout = true
}
if h.Name == "index.json" {
foundIndexJSON = true
}
}
if !foundOCILayout {
t.Error("oci-layout not found")
}
if !foundIndexJSON {
t.Error("index.json not found")
}
}

119
import.go Normal file
View File

@ -0,0 +1,119 @@
package containerd
import (
"archive/tar"
"context"
"encoding/json"
"io"
"io/ioutil"
"strings"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/reference"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) {
tag, dgst := reference.SplitObject(refObject)
if tag == "" && dgst == "" {
return nil, errors.Errorf("unexpected object: %q", refObject)
}
for _, m := range idx.Manifests {
if m.Digest == dgst {
return &m, nil
}
annot, ok := m.Annotations[ocispec.AnnotationRefName]
if ok && annot == tag && tag != "" {
return &m, nil
}
}
return nil, errors.Errorf("not found: %q", refObject)
}
func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) {
tr := tar.NewReader(reader)
store := c.ContentStore()
var desc *ocispec.Descriptor
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
continue
}
if hdr.Name == "index.json" {
desc, err = onUntarIndexJSON(tr, iopts.refObject)
if err != nil {
return nil, err
}
continue
}
if strings.HasPrefix(hdr.Name, "blobs/") {
if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil {
return nil, err
}
}
}
if desc == nil {
return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject)
}
imgrec := images.Image{
Name: ref,
Target: *desc,
}
is := c.ImageService()
if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
if !errdefs.IsNotFound(err) {
return nil, err
}
created, err := is.Create(ctx, imgrec)
if err != nil {
return nil, err
}
imgrec = created
} else {
imgrec = updated
}
img := &image{
client: c,
i: imgrec,
}
return img, nil
}
func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) {
b, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
var idx ocispec.Index
if err := json.Unmarshal(b, &idx); err != nil {
return nil, err
}
return resolveOCIIndex(idx, refObject)
}
func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error {
// name is like "blobs/sha256/deadbeef"
split := strings.Split(name, "/")
if len(split) != 3 {
return errors.Errorf("unexpected name: %q", name)
}
algo := digest.Algorithm(split[1])
if !algo.Available() {
return errors.Errorf("unsupported algorithm: %s", algo)
}
dgst := digest.NewDigestFromHex(algo.String(), split[2])
return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst)
}

44
import_test.go Normal file
View File

@ -0,0 +1,44 @@
package containerd
import (
"runtime"
"testing"
)
// TestExportAndImport exports testImage as a tar stream,
// and import the tar stream as a new image.
func TestExportAndImport(t *testing.T) {
// TODO: support windows
if testing.Short() || runtime.GOOS == "windows" {
t.Skip()
}
ctx, cancel := testContext()
defer cancel()
client, err := New(address)
if err != nil {
t.Fatal(err)
}
defer client.Close()
pulled, err := client.Pull(ctx, testImage)
if err != nil {
t.Fatal(err)
}
exported, err := client.Export(ctx, pulled.Target())
if err != nil {
t.Fatal(err)
}
importRef := "test/export-and-import:tmp"
_, err = client.Import(ctx, importRef, exported, WithRefObject("@"+pulled.Target().Digest.String()))
if err != nil {
t.Fatal(err)
}
err = client.ImageService().Delete(ctx, importRef)
if err != nil {
t.Fatal(err)
}
}

263
oci/oci.go
View File

@ -1,9 +1,12 @@
// Package oci provides basic operations for manipulating OCI images.
// This package can be used even outside of containerd, and contains some
// functions not used in containerd itself.
package oci
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
@ -14,53 +17,107 @@ import (
spec "github.com/opencontainers/image-spec/specs-go/v1"
)
// Init initializes the img directory as an OCI image.
// i.e. Creates oci-layout, index.json, and blobs.
// BlobWriter writes an OCI blob and returns a digest when committed.
type BlobWriter interface {
// Close is expected to be called after Commit() when commission is needed.
io.WriteCloser
// Digest may return empty digest or panics until committed.
Digest() digest.Digest
// Commit commits the blob (but no roll-back is guaranteed on an error).
// size and expected can be zero-value when unknown.
Commit(size int64, expected digest.Digest) error
}
// ErrUnexpectedSize can be returned from BlobWriter.Commit()
type ErrUnexpectedSize struct {
Expected int64
Actual int64
}
func (e ErrUnexpectedSize) Error() string {
if e.Expected > 0 && e.Expected != e.Actual {
return fmt.Sprintf("unexpected size: %d != %d", e.Expected, e.Actual)
}
return fmt.Sprintf("malformed ErrUnexpectedSize(%+v)", e)
}
// ErrUnexpectedDigest can be returned from BlobWriter.Commit()
type ErrUnexpectedDigest struct {
Expected digest.Digest
Actual digest.Digest
}
func (e ErrUnexpectedDigest) Error() string {
if e.Expected.String() != "" && e.Expected.String() != e.Actual.String() {
return fmt.Sprintf("unexpected digest: %v != %v", e.Expected, e.Actual)
}
return fmt.Sprintf("malformed ErrUnexpectedDigest(%+v)", e)
}
// ImageDriver corresponds to the representation of an image.
// Path uses os.PathSeparator as the separator.
// The methods of ImageDriver should only be called from oci package.
type ImageDriver interface {
Init() error
Remove(path string) error
Reader(path string) (io.ReadCloser, error)
Writer(path string, perm os.FileMode) (io.WriteCloser, error)
BlobWriter(algo digest.Algorithm) (BlobWriter, error)
}
type InitOpts struct {
// imageLayoutVersion can be an empty string for specifying the default version.
ImageLayoutVersion string
// skip creating oci-layout
SkipCreateImageLayout bool
// skip creating index.json
SkipCreateIndex bool
}
// Init initializes an OCI image structure.
// Init calls img.Init, creates `oci-layout`(0444), and creates `index.json`(0644).
//
// img directory must not exist before calling this function.
//
// imageLayoutVersion can be an empty string for specifying the default version.
func Init(img, imageLayoutVersion string) error {
func Init(img ImageDriver, opts InitOpts) error {
if err := img.Init(); err != nil {
return err
}
// Create oci-layout
if !opts.SkipCreateImageLayout {
imageLayoutVersion := opts.ImageLayoutVersion
if imageLayoutVersion == "" {
imageLayoutVersion = spec.ImageLayoutVersion
}
if _, err := os.Stat(img); err == nil {
return os.ErrExist
}
// Create the directory
if err := os.MkdirAll(img, 0755); err != nil {
return err
}
// Create blobs/sha256
if err := os.MkdirAll(
filepath.Join(img, "blobs", string(digest.Canonical)),
0755); err != nil {
return nil
}
// Create oci-layout
if err := WriteImageLayout(img, spec.ImageLayout{Version: imageLayoutVersion}); err != nil {
return err
}
}
// Create index.json
return WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}})
if !opts.SkipCreateIndex {
if err := WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}}); err != nil {
return err
}
}
return nil
}
func blobPath(img string, d digest.Digest) string {
return filepath.Join(img, "blobs", d.Algorithm().String(), d.Hex())
func blobPath(d digest.Digest) string {
return filepath.Join("blobs", d.Algorithm().String(), d.Hex())
}
func indexPath(img string) string {
return filepath.Join(img, "index.json")
}
const (
indexPath = "index.json"
)
// GetBlobReader returns io.ReadCloser for a blob.
func GetBlobReader(img string, d digest.Digest) (io.ReadCloser, error) {
func GetBlobReader(img ImageDriver, d digest.Digest) (io.ReadCloser, error) {
// we return a reader rather than the full *os.File here so as to prohibit write operations.
return os.Open(blobPath(img, d))
return img.Reader(blobPath(d))
}
// ReadBlob reads an OCI blob.
func ReadBlob(img string, d digest.Digest) ([]byte, error) {
func ReadBlob(img ImageDriver, d digest.Digest) ([]byte, error) {
r, err := GetBlobReader(img, d)
if err != nil {
return nil, err
@ -71,90 +128,47 @@ func ReadBlob(img string, d digest.Digest) ([]byte, error) {
// WriteBlob writes bytes as an OCI blob and returns its digest using the canonical digest algorithm.
// If you need to specify certain algorithm, you can use NewBlobWriter(img string, algo digest.Algorithm).
func WriteBlob(img string, b []byte) (digest.Digest, error) {
d := digest.FromBytes(b)
return d, ioutil.WriteFile(blobPath(img, d), b, 0444)
}
// BlobWriter writes an OCI blob and returns a digest when closed.
type BlobWriter interface {
io.Writer
io.Closer
// Digest returns the digest when closed.
// Digest panics when the writer is not closed.
Digest() digest.Digest
}
// blobWriter implements BlobWriter.
type blobWriter struct {
img string
digester digest.Digester
f *os.File
closed bool
func WriteBlob(img ImageDriver, b []byte) (digest.Digest, error) {
w, err := img.BlobWriter(digest.Canonical)
if err != nil {
return "", err
}
n, err := w.Write(b)
if err != nil {
return "", err
}
if n < len(b) {
return "", io.ErrShortWrite
}
if err := w.Close(); err != nil {
return "", err
}
return w.Digest(), err
}
// NewBlobWriter returns a BlobWriter.
func NewBlobWriter(img string, algo digest.Algorithm) (BlobWriter, error) {
// use img rather than the default tmp, so as to make sure rename(2) can be applied
f, err := ioutil.TempFile(img, "tmp.blobwriter")
if err != nil {
return nil, err
}
return &blobWriter{
img: img,
digester: algo.Digester(),
f: f,
}, nil
}
// Write implements io.Writer.
func (bw *blobWriter) Write(b []byte) (int, error) {
n, err := bw.f.Write(b)
if err != nil {
return n, err
}
return bw.digester.Hash().Write(b)
}
// Close implements io.Closer.
func (bw *blobWriter) Close() error {
oldPath := bw.f.Name()
if err := bw.f.Close(); err != nil {
return err
}
newPath := blobPath(bw.img, bw.digester.Digest())
if err := os.MkdirAll(filepath.Dir(newPath), 0755); err != nil {
return err
}
if err := os.Chmod(oldPath, 0444); err != nil {
return err
}
if err := os.Rename(oldPath, newPath); err != nil {
return err
}
bw.closed = true
return nil
}
// Digest returns the digest when closed.
func (bw *blobWriter) Digest() digest.Digest {
if !bw.closed {
panic("blobWriter is unclosed")
}
return bw.digester.Digest()
func NewBlobWriter(img ImageDriver, algo digest.Algorithm) (BlobWriter, error) {
return img.BlobWriter(algo)
}
// DeleteBlob deletes an OCI blob.
func DeleteBlob(img string, d digest.Digest) error {
return os.Remove(blobPath(img, d))
func DeleteBlob(img ImageDriver, d digest.Digest) error {
return img.Remove(blobPath(d))
}
// ReadImageLayout returns the image layout.
func ReadImageLayout(img string) (spec.ImageLayout, error) {
b, err := ioutil.ReadFile(filepath.Join(img, spec.ImageLayoutFile))
func ReadImageLayout(img ImageDriver) (spec.ImageLayout, error) {
r, err := img.Reader(spec.ImageLayoutFile)
if err != nil {
return spec.ImageLayout{}, err
}
b, err := ioutil.ReadAll(r)
if err != nil {
return spec.ImageLayout{}, err
}
if err := r.Close(); err != nil {
return spec.ImageLayout{}, err
}
var layout spec.ImageLayout
if err := json.Unmarshal(b, &layout); err != nil {
return spec.ImageLayout{}, err
@ -163,20 +177,38 @@ func ReadImageLayout(img string) (spec.ImageLayout, error) {
}
// WriteImageLayout writes the image layout.
func WriteImageLayout(img string, layout spec.ImageLayout) error {
func WriteImageLayout(img ImageDriver, layout spec.ImageLayout) error {
b, err := json.Marshal(layout)
if err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(img, spec.ImageLayoutFile), b, 0644)
w, err := img.Writer(spec.ImageLayoutFile, 0444)
if err != nil {
return err
}
n, err := w.Write(b)
if err != nil {
return err
}
if n < len(b) {
return io.ErrShortWrite
}
return w.Close()
}
// ReadIndex returns the index.
func ReadIndex(img string) (spec.Index, error) {
b, err := ioutil.ReadFile(indexPath(img))
func ReadIndex(img ImageDriver) (spec.Index, error) {
r, err := img.Reader(indexPath)
if err != nil {
return spec.Index{}, err
}
b, err := ioutil.ReadAll(r)
if err != nil {
return spec.Index{}, err
}
if err := r.Close(); err != nil {
return spec.Index{}, err
}
var idx spec.Index
if err := json.Unmarshal(b, &idx); err != nil {
return spec.Index{}, err
@ -185,17 +217,28 @@ func ReadIndex(img string) (spec.Index, error) {
}
// WriteIndex writes the index.
func WriteIndex(img string, idx spec.Index) error {
func WriteIndex(img ImageDriver, idx spec.Index) error {
b, err := json.Marshal(idx)
if err != nil {
return err
}
return ioutil.WriteFile(indexPath(img), b, 0644)
w, err := img.Writer(indexPath, 0644)
if err != nil {
return err
}
n, err := w.Write(b)
if err != nil {
return err
}
if n < len(b) {
return io.ErrShortWrite
}
return w.Close()
}
// RemoveManifestDescriptorFromIndex removes the manifest descriptor from the index.
// Returns nil error when the entry not found.
func RemoveManifestDescriptorFromIndex(img string, refName string) error {
func RemoveManifestDescriptorFromIndex(img ImageDriver, refName string) error {
if refName == "" {
return errors.New("empty refName specified")
}
@ -217,7 +260,7 @@ func RemoveManifestDescriptorFromIndex(img string, refName string) error {
// PutManifestDescriptorToIndex puts a manifest descriptor to the index.
// If ref name is set and conflicts with the existing descriptors, the old ones are removed.
func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error {
func PutManifestDescriptorToIndex(img ImageDriver, desc spec.Descriptor) error {
refName, ok := desc.Annotations[spec.AnnotationRefName]
if ok && refName != "" {
if err := RemoveManifestDescriptorFromIndex(img, refName); err != nil {
@ -233,7 +276,7 @@ func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error {
}
// WriteJSONBlob is an utility function that writes x as a JSON blob with the specified media type, and returns the descriptor.
func WriteJSONBlob(img string, x interface{}, mediaType string) (spec.Descriptor, error) {
func WriteJSONBlob(img ImageDriver, x interface{}, mediaType string) (spec.Descriptor, error) {
b, err := json.Marshal(x)
if err != nil {
return spec.Descriptor{}, err

168
oci/oci_test.go
View File

@ -1,168 +0,0 @@
package oci
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/containerd/containerd/fs/fstest"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
spec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/assert"
)
func TestInitError(t *testing.T) {
tmp, err := ioutil.TempDir("", "oci")
assert.Nil(t, err)
defer os.RemoveAll(tmp)
err = Init(tmp, "")
assert.Error(t, err, "file exists")
}
func TestInit(t *testing.T) {
tmp, err := ioutil.TempDir("", "oci")
assert.Nil(t, err)
defer os.RemoveAll(tmp)
img := filepath.Join(tmp, "foo")
err = Init(img, "")
assert.Nil(t, err)
ociLayout, err := json.Marshal(spec.ImageLayout{Version: spec.ImageLayoutVersion})
assert.Nil(t, err)
indexJSON, err := json.Marshal(spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}})
applier := fstest.Apply(
fstest.CreateDir("/foo", 0755),
fstest.CreateDir("/foo/blobs", 0755),
fstest.CreateDir("/foo/blobs/"+string(digest.Canonical), 0755),
fstest.CreateFile("/foo/oci-layout", ociLayout, 0644),
fstest.CreateFile("/foo/index.json", indexJSON, 0644),
)
err = fstest.CheckDirectoryEqualWithApplier(tmp, applier)
assert.Nil(t, err)
}
func TestWriteReadDeleteBlob(t *testing.T) {
tmp, err := ioutil.TempDir("", "oci")
assert.Nil(t, err)
defer os.RemoveAll(tmp)
img := filepath.Join(tmp, "foo")
err = Init(img, "")
assert.Nil(t, err)
testBlob := []byte("test")
// Write
d, err := WriteBlob(img, testBlob)
applier := fstest.Apply(
fstest.CreateFile("/"+d.Hex(), testBlob, 0444),
)
err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
assert.Nil(t, err)
// Read
b, err := ReadBlob(img, d)
assert.Nil(t, err)
assert.Equal(t, testBlob, b)
// Delete
err = DeleteBlob(img, d)
assert.Nil(t, err)
applier = fstest.Apply()
err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
assert.Nil(t, err)
}
func TestBlobWriter(t *testing.T) {
tmp, err := ioutil.TempDir("", "oci")
assert.Nil(t, err)
defer os.RemoveAll(tmp)
img := filepath.Join(tmp, "foo")
err = Init(img, "")
assert.Nil(t, err)
testBlob := []byte("test")
w, err := NewBlobWriter(img, digest.Canonical)
_, err = w.Write(testBlob)
assert.Nil(t, err)
// blob is not written until closing
applier := fstest.Apply()
err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
// digest is unavailable until closing
assert.Panics(t, func() { w.Digest() })
// close and calculate the digest
err = w.Close()
assert.Nil(t, err)
d := w.Digest()
applier = fstest.Apply(
fstest.CreateFile("/"+d.Hex(), testBlob, 0444),
)
err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
assert.Nil(t, err)
}
func TestIndex(t *testing.T) {
tmp, err := ioutil.TempDir("", "oci")
assert.Nil(t, err)
defer os.RemoveAll(tmp)
img := filepath.Join(tmp, "foo")
err = Init(img, "")
assert.Nil(t, err)
descs := []spec.Descriptor{
{
MediaType: spec.MediaTypeImageManifest,
Annotations: map[string]string{
spec.AnnotationRefName: "foo",
"dummy": "desc0",
},
},
{
MediaType: spec.MediaTypeImageManifest,
Annotations: map[string]string{
// will be removed later
spec.AnnotationRefName: "bar",
"dummy": "desc1",
},
},
{
MediaType: spec.MediaTypeImageManifest,
Annotations: map[string]string{
// duplicated ref name
spec.AnnotationRefName: "foo",
"dummy": "desc2",
},
},
{
MediaType: spec.MediaTypeImageManifest,
Annotations: map[string]string{
// no ref name
"dummy": "desc3",
},
},
}
for _, desc := range descs {
err := PutManifestDescriptorToIndex(img, desc)
assert.Nil(t, err)
}
err = RemoveManifestDescriptorFromIndex(img, "bar")
assert.Nil(t, err)
expected := spec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
Manifests: []spec.Descriptor{
{
MediaType: spec.MediaTypeImageManifest,
Annotations: map[string]string{
// duplicated ref name
spec.AnnotationRefName: "foo",
"dummy": "desc2",
},
},
{
MediaType: spec.MediaTypeImageManifest,
Annotations: map[string]string{
// no ref name
"dummy": "desc3",
},
},
},
}
idx, err := ReadIndex(img)
assert.Nil(t, err)
assert.Equal(t, expected, idx)
}

156
oci/tar.go Normal file
View File

@ -0,0 +1,156 @@
package oci
import (
"archive/tar"
"bytes"
"errors"
"io"
"os"
"path/filepath"
"github.com/opencontainers/go-digest"
)
// TarWriter is an interface that is implemented by archive/tar.Writer.
// (Using an interface allows hooking)
type TarWriter interface {
io.WriteCloser
Flush() error
WriteHeader(hdr *tar.Header) error
}
// Tar is ImageDriver for TAR representation of an OCI image.
func Tar(w TarWriter) ImageDriver {
return &tarDriver{
w: w,
}
}
type tarDriver struct {
w TarWriter
}
func (d *tarDriver) Init() error {
headers := []tar.Header{
{
Name: "blobs/",
Mode: 0755,
Typeflag: tar.TypeDir,
},
{
Name: "blobs/" + string(digest.Canonical) + "/",
Mode: 0755,
Typeflag: tar.TypeDir,
},
}
for _, h := range headers {
if err := d.w.WriteHeader(&h); err != nil {
return err
}
}
return nil
}
func (d *tarDriver) Remove(path string) error {
return errors.New("Tar does not support Remove")
}
func (d *tarDriver) Reader(path string) (io.ReadCloser, error) {
// because tar does not support random access
return nil, errors.New("Tar does not support Reader")
}
func (d *tarDriver) Writer(path string, perm os.FileMode) (io.WriteCloser, error) {
name := filepath.ToSlash(path)
return &tarDriverWriter{
w: d.w,
name: name,
mode: int64(perm),
}, nil
}
// tarDriverWriter is used for writing non-blob files
// (e.g. oci-layout, index.json)
type tarDriverWriter struct {
bytes.Buffer
w TarWriter
name string
mode int64
}
func (w *tarDriverWriter) Close() error {
if err := w.w.WriteHeader(&tar.Header{
Name: w.name,
Mode: w.mode,
Size: int64(w.Len()),
Typeflag: tar.TypeReg,
}); err != nil {
return err
}
n, err := io.Copy(w.w, w)
if err != nil {
return err
}
if n < int64(w.Len()) {
return io.ErrShortWrite
}
return w.w.Flush()
}
func (d *tarDriver) BlobWriter(algo digest.Algorithm) (BlobWriter, error) {
return &tarBlobWriter{
w: d.w,
digester: algo.Digester(),
}, nil
}
// tarBlobWriter implements BlobWriter.
type tarBlobWriter struct {
w TarWriter
digester digest.Digester
buf bytes.Buffer // TODO: use tmp file for large buffer?
}
// Write implements io.Writer.
func (bw *tarBlobWriter) Write(b []byte) (int, error) {
n, err := bw.buf.Write(b)
if err != nil {
return n, err
}
return bw.digester.Hash().Write(b)
}
func (bw *tarBlobWriter) Commit(size int64, expected digest.Digest) error {
path := "blobs/" + bw.digester.Digest().Algorithm().String() + "/" + bw.digester.Digest().Hex()
if err := bw.w.WriteHeader(&tar.Header{
Name: path,
Mode: 0444,
Size: int64(bw.buf.Len()),
Typeflag: tar.TypeReg,
}); err != nil {
return err
}
n, err := io.Copy(bw.w, &bw.buf)
if err != nil {
return err
}
if n < int64(bw.buf.Len()) {
return io.ErrShortWrite
}
if size > 0 && size != n {
return ErrUnexpectedSize{Expected: size, Actual: n}
}
if expected != "" && bw.digester.Digest() != expected {
return ErrUnexpectedDigest{Expected: expected, Actual: bw.digester.Digest()}
}
return bw.w.Flush()
}
func (bw *tarBlobWriter) Close() error {
// we don't close bw.w (reused for writing another blob)
return bw.w.Flush()
}
func (bw *tarBlobWriter) Digest() digest.Digest {
return bw.digester.Digest()
}