diff --git a/client.go b/client.go index 92ff67092..2664abb88 100644 --- a/client.go +++ b/client.go @@ -3,6 +3,7 @@ package containerd import ( "context" "fmt" + "io" "io/ioutil" "log" "net/http" @@ -25,6 +26,7 @@ import ( "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/reference" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/remotes/docker/schema1" @@ -552,3 +554,120 @@ func (c *Client) Version(ctx context.Context) (Version, error) { Revision: response.Revision, }, nil } + +type imageFormat string + +const ( + ociImageFormat imageFormat = "oci" +) + +type importOpts struct { + format imageFormat + refObject string +} + +type ImportOpt func(c *importOpts) error + +func WithOCIImportFormat() ImportOpt { + return func(c *importOpts) error { + if c.format != "" { + return errors.New("format already set") + } + c.format = ociImageFormat + return nil + } +} + +// WithRefObject specifies the ref object to import. +// If refObject is empty, it is copied from the ref argument of Import(). +func WithRefObject(refObject string) ImportOpt { + return func(c *importOpts) error { + c.refObject = refObject + return nil + } +} + +func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) { + var iopts importOpts + for _, o := range opts { + if err := o(&iopts); err != nil { + return iopts, err + } + } + // use OCI as the default format + if iopts.format == "" { + iopts.format = ociImageFormat + } + // if refObject is not explicitly specified, use the one specified in ref + if iopts.refObject == "" { + refSpec, err := reference.Parse(ref) + if err != nil { + return iopts, err + } + iopts.refObject = refSpec.Object + } + return iopts, nil +} + +// Import imports an image from a Tar stream using reader. +// OCI format is assumed by default. +// +// Note that unreferenced blobs are imported to the content store as well. +func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) { + iopts, err := resolveImportOpt(ref, opts...) + if err != nil { + return nil, err + } + switch iopts.format { + case ociImageFormat: + return c.importFromOCITar(ctx, ref, reader, iopts) + default: + return nil, errors.Errorf("unsupported format: %s", iopts.format) + } +} + +type exportOpts struct { + format imageFormat +} + +type ExportOpt func(c *exportOpts) error + +func WithOCIExportFormat() ExportOpt { + return func(c *exportOpts) error { + if c.format != "" { + return errors.New("format already set") + } + c.format = ociImageFormat + return nil + } +} + +// TODO: add WithMediaTypeTranslation that transforms media types according to the format. +// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip +// -> application/vnd.oci.image.layer.v1.tar+gzip + +// Export exports an image to a Tar stream. +// OCI format is used by default. +// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc. +func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) { + var eopts exportOpts + for _, o := range opts { + if err := o(&eopts); err != nil { + return nil, err + } + } + // use OCI as the default format + if eopts.format == "" { + eopts.format = ociImageFormat + } + pr, pw := io.Pipe() + switch eopts.format { + case ociImageFormat: + go func() { + pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts)) + }() + default: + return nil, errors.Errorf("unsupported format: %s", eopts.format) + } + return pr, nil +} diff --git a/cmd/ctr/export.go b/cmd/ctr/export.go new file mode 100644 index 000000000..0e081159c --- /dev/null +++ b/cmd/ctr/export.go @@ -0,0 +1,106 @@ +package main + +import ( + "io" + "os" + + "github.com/containerd/containerd/reference" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +var imagesExportCommand = cli.Command{ + Name: "export", + Usage: "export an image", + ArgsUsage: "[flags] ", + Description: `Export an image to a tar stream +`, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "oci-ref-name", + Value: "", + Usage: "Override org.opencontainers.image.ref.name annotation", + }, + cli.StringFlag{ + Name: "manifest", + Usage: "Digest of manifest", + }, + cli.StringFlag{ + Name: "manifest-type", + Usage: "Media type of manifest digest", + Value: ocispec.MediaTypeImageManifest, + }, + }, + Action: func(clicontext *cli.Context) error { + var ( + out = clicontext.Args().First() + local = clicontext.Args().Get(1) + desc ocispec.Descriptor + ) + + ctx, cancel := appContext(clicontext) + defer cancel() + + client, err := newClient(clicontext) + if err != nil { + return err + } + + if manifest := clicontext.String("manifest"); manifest != "" { + desc.Digest, err = digest.Parse(manifest) + if err != nil { + return errors.Wrap(err, "invalid manifest digest") + } + desc.MediaType = clicontext.String("manifest-type") + } else { + img, err := client.ImageService().Get(ctx, local) + if err != nil { + return errors.Wrap(err, "unable to resolve image to manifest") + } + desc = img.Target + } + + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + if s, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok || s == "" { + if ociRefName := determineOCIRefName(local); ociRefName != "" { + desc.Annotations[ocispec.AnnotationRefName] = ociRefName + } + if ociRefName := clicontext.String("oci-ref-name"); ociRefName != "" { + desc.Annotations[ocispec.AnnotationRefName] = ociRefName + } + } + var w io.WriteCloser + if out == "-" { + w = os.Stdout + } else { + w, err = os.Create(out) + if err != nil { + return nil + } + } + r, err := client.Export(ctx, desc) + if err != nil { + return err + } + if _, err := io.Copy(w, r); err != nil { + return err + } + if err := w.Close(); err != nil { + return err + } + return r.Close() + }, +} + +func determineOCIRefName(local string) string { + refspec, err := reference.Parse(local) + if err != nil { + return "" + } + tag, _ := reference.SplitObject(refspec.Object) + return tag +} diff --git a/cmd/ctr/images.go b/cmd/ctr/images.go index c047834af..b2287092e 100644 --- a/cmd/ctr/images.go +++ b/cmd/ctr/images.go @@ -22,6 +22,8 @@ var imageCommand = cli.Command{ imagesListCommand, imageRemoveCommand, imagesSetLabelsCommand, + imagesImportCommand, + imagesExportCommand, }, } diff --git a/cmd/ctr/import.go b/cmd/ctr/import.go new file mode 100644 index 000000000..ebdcee5d7 --- /dev/null +++ b/cmd/ctr/import.go @@ -0,0 +1,70 @@ +package main + +import ( + "fmt" + "io" + "os" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/log" + "github.com/urfave/cli" +) + +var imagesImportCommand = cli.Command{ + Name: "import", + Usage: "import an image", + ArgsUsage: "[flags] ", + Description: `Import an image from a tar stream +`, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "ref-object", + Value: "", + Usage: "reference object e.g. tag@digest (default: use the object specified in ref)", + }, + }, + Action: func(clicontext *cli.Context) error { + var ( + ref = clicontext.Args().First() + in = clicontext.Args().Get(1) + refObject = clicontext.String("ref-object") + ) + + ctx, cancel := appContext(clicontext) + defer cancel() + + client, err := newClient(clicontext) + if err != nil { + return err + } + + var r io.ReadCloser + if in == "-" { + r = os.Stdin + } else { + r, err = os.Open(in) + if err != nil { + return err + } + } + img, err := client.Import(ctx, + ref, + r, + containerd.WithRefObject(refObject), + ) + if err != nil { + return err + } + if err = r.Close(); err != nil { + return err + } + + log.G(ctx).WithField("image", ref).Debug("unpacking") + + // TODO: Show unpack status + fmt.Printf("unpacking %s...", img.Target().Digest) + err = img.Unpack(ctx, clicontext.String("snapshotter")) + fmt.Println("done") + return err + }, +} diff --git a/content/content.go b/content/content.go index 770a37b8f..12ed2fe7f 100644 --- a/content/content.go +++ b/content/content.go @@ -5,6 +5,7 @@ import ( "io" "time" + "github.com/containerd/containerd/oci" "github.com/opencontainers/go-digest" ) @@ -77,10 +78,8 @@ type IngestManager interface { } type Writer interface { - io.WriteCloser + oci.BlobWriter Status() (Status, error) - Digest() digest.Digest - Commit(size int64, expected digest.Digest) error Truncate(size int64) error } diff --git a/content/local/writer.go b/content/local/writer.go index e7a74b290..30a7e7172 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -8,6 +8,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/oci" "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -78,7 +79,7 @@ func (w *writer) Commit(size int64, expected digest.Digest) error { } if size > 0 && size != fi.Size() { - return errors.Errorf("%q failed size validation: %v != %v", w.ref, fi.Size(), size) + return oci.ErrUnexpectedSize{Expected: size, Actual: fi.Size()} } if err := w.fp.Close(); err != nil { @@ -87,7 +88,7 @@ func (w *writer) Commit(size int64, expected digest.Digest) error { dgst := w.digester.Digest() if expected != "" && expected != dgst { - return errors.Errorf("unexpected digest: %v != %v", dgst, expected) + return oci.ErrUnexpectedDigest{Expected: expected, Actual: dgst} } var ( diff --git a/export.go b/export.go new file mode 100644 index 000000000..22e56f0f5 --- /dev/null +++ b/export.go @@ -0,0 +1,65 @@ +package containerd + +import ( + "archive/tar" + "context" + "io" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/oci" + ocispecs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error { + tw := tar.NewWriter(writer) + img := oci.Tar(tw) + + // For tar, we defer creating index until end of the function. + if err := oci.Init(img, oci.InitOpts{SkipCreateIndex: true}); err != nil { + return err + } + cs := c.ContentStore() + handlers := images.Handlers( + images.ChildrenHandler(cs), + exportHandler(cs, img), + ) + // For tar, we need to use Walk instead of Dispatch for ensuring sequential write + if err := images.Walk(ctx, handlers, desc); err != nil { + return err + } + // For tar, we don't use oci.PutManifestDescriptorToIndex() which allows appending desc to existing index.json + // but requires img to support random read access so as to read index.json. + return oci.WriteIndex(img, + ocispec.Index{ + Versioned: ocispecs.Versioned{ + SchemaVersion: 2, + }, + Manifests: []ocispec.Descriptor{desc}, + }, + ) +} + +func exportHandler(cs content.Store, img oci.ImageDriver) images.HandlerFunc { + return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + r, err := cs.Reader(ctx, desc.Digest) + if err != nil { + return nil, err + } + w, err := oci.NewBlobWriter(img, desc.Digest.Algorithm()) + if err != nil { + return nil, err + } + if _, err = io.Copy(w, r); err != nil { + return nil, err + } + if err = w.Commit(desc.Size, desc.Digest); err != nil { + return nil, err + } + if err = w.Close(); err != nil { + return nil, err + } + return nil, nil + } +} diff --git a/export_test.go b/export_test.go new file mode 100644 index 000000000..dfe1d33f8 --- /dev/null +++ b/export_test.go @@ -0,0 +1,64 @@ +package containerd + +import ( + "archive/tar" + "io" + "runtime" + "testing" +) + +// TestExport exports testImage as a tar stream +func TestExport(t *testing.T) { + // TODO: support windows + if testing.Short() || runtime.GOOS == "windows" { + t.Skip() + } + ctx, cancel := testContext() + defer cancel() + + client, err := New(address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + pulled, err := client.Pull(ctx, testImage) + if err != nil { + t.Fatal(err) + } + + exportedStream, err := client.Export(ctx, pulled.Target()) + if err != nil { + t.Fatal(err) + } + assertOCITar(t, exportedStream) +} + +func assertOCITar(t *testing.T, r io.Reader) { + // TODO: add more assertion + tr := tar.NewReader(r) + foundOCILayout := false + foundIndexJSON := false + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Error(err) + continue + } + if h.Name == "oci-layout" { + foundOCILayout = true + } + if h.Name == "index.json" { + foundIndexJSON = true + } + } + if !foundOCILayout { + t.Error("oci-layout not found") + } + if !foundIndexJSON { + t.Error("index.json not found") + } +} diff --git a/import.go b/import.go new file mode 100644 index 000000000..a8fe63ffe --- /dev/null +++ b/import.go @@ -0,0 +1,119 @@ +package containerd + +import ( + "archive/tar" + "context" + "encoding/json" + "io" + "io/ioutil" + "strings" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/reference" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) { + tag, dgst := reference.SplitObject(refObject) + if tag == "" && dgst == "" { + return nil, errors.Errorf("unexpected object: %q", refObject) + } + for _, m := range idx.Manifests { + if m.Digest == dgst { + return &m, nil + } + annot, ok := m.Annotations[ocispec.AnnotationRefName] + if ok && annot == tag && tag != "" { + return &m, nil + } + } + return nil, errors.Errorf("not found: %q", refObject) +} + +func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) { + tr := tar.NewReader(reader) + store := c.ContentStore() + var desc *ocispec.Descriptor + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + continue + } + if hdr.Name == "index.json" { + desc, err = onUntarIndexJSON(tr, iopts.refObject) + if err != nil { + return nil, err + } + continue + } + if strings.HasPrefix(hdr.Name, "blobs/") { + if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil { + return nil, err + } + } + } + if desc == nil { + return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject) + } + imgrec := images.Image{ + Name: ref, + Target: *desc, + } + is := c.ImageService() + if updated, err := is.Update(ctx, imgrec, "target"); err != nil { + if !errdefs.IsNotFound(err) { + return nil, err + } + + created, err := is.Create(ctx, imgrec) + if err != nil { + return nil, err + } + + imgrec = created + } else { + imgrec = updated + } + + img := &image{ + client: c, + i: imgrec, + } + return img, nil +} + +func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) { + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + var idx ocispec.Index + if err := json.Unmarshal(b, &idx); err != nil { + return nil, err + } + return resolveOCIIndex(idx, refObject) +} + +func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error { + // name is like "blobs/sha256/deadbeef" + split := strings.Split(name, "/") + if len(split) != 3 { + return errors.Errorf("unexpected name: %q", name) + } + algo := digest.Algorithm(split[1]) + if !algo.Available() { + return errors.Errorf("unsupported algorithm: %s", algo) + } + dgst := digest.NewDigestFromHex(algo.String(), split[2]) + return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst) +} diff --git a/import_test.go b/import_test.go new file mode 100644 index 000000000..78640afda --- /dev/null +++ b/import_test.go @@ -0,0 +1,44 @@ +package containerd + +import ( + "runtime" + "testing" +) + +// TestExportAndImport exports testImage as a tar stream, +// and import the tar stream as a new image. +func TestExportAndImport(t *testing.T) { + // TODO: support windows + if testing.Short() || runtime.GOOS == "windows" { + t.Skip() + } + ctx, cancel := testContext() + defer cancel() + + client, err := New(address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + pulled, err := client.Pull(ctx, testImage) + if err != nil { + t.Fatal(err) + } + + exported, err := client.Export(ctx, pulled.Target()) + if err != nil { + t.Fatal(err) + } + + importRef := "test/export-and-import:tmp" + _, err = client.Import(ctx, importRef, exported, WithRefObject("@"+pulled.Target().Digest.String())) + if err != nil { + t.Fatal(err) + } + + err = client.ImageService().Delete(ctx, importRef) + if err != nil { + t.Fatal(err) + } +} diff --git a/oci/oci.go b/oci/oci.go index 90894962a..6094d3ea9 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -1,9 +1,12 @@ // Package oci provides basic operations for manipulating OCI images. +// This package can be used even outside of containerd, and contains some +// functions not used in containerd itself. package oci import ( "encoding/json" "errors" + "fmt" "io" "io/ioutil" "os" @@ -14,53 +17,107 @@ import ( spec "github.com/opencontainers/image-spec/specs-go/v1" ) -// Init initializes the img directory as an OCI image. -// i.e. Creates oci-layout, index.json, and blobs. -// -// img directory must not exist before calling this function. -// -// imageLayoutVersion can be an empty string for specifying the default version. -func Init(img, imageLayoutVersion string) error { - if imageLayoutVersion == "" { - imageLayoutVersion = spec.ImageLayoutVersion +// BlobWriter writes an OCI blob and returns a digest when committed. +type BlobWriter interface { + // Close is expected to be called after Commit() when commission is needed. + io.WriteCloser + // Digest may return empty digest or panics until committed. + Digest() digest.Digest + // Commit commits the blob (but no roll-back is guaranteed on an error). + // size and expected can be zero-value when unknown. + Commit(size int64, expected digest.Digest) error +} + +// ErrUnexpectedSize can be returned from BlobWriter.Commit() +type ErrUnexpectedSize struct { + Expected int64 + Actual int64 +} + +func (e ErrUnexpectedSize) Error() string { + if e.Expected > 0 && e.Expected != e.Actual { + return fmt.Sprintf("unexpected size: %d != %d", e.Expected, e.Actual) } - if _, err := os.Stat(img); err == nil { - return os.ErrExist + return fmt.Sprintf("malformed ErrUnexpectedSize(%+v)", e) +} + +// ErrUnexpectedDigest can be returned from BlobWriter.Commit() +type ErrUnexpectedDigest struct { + Expected digest.Digest + Actual digest.Digest +} + +func (e ErrUnexpectedDigest) Error() string { + if e.Expected.String() != "" && e.Expected.String() != e.Actual.String() { + return fmt.Sprintf("unexpected digest: %v != %v", e.Expected, e.Actual) } - // Create the directory - if err := os.MkdirAll(img, 0755); err != nil { + return fmt.Sprintf("malformed ErrUnexpectedDigest(%+v)", e) +} + +// ImageDriver corresponds to the representation of an image. +// Path uses os.PathSeparator as the separator. +// The methods of ImageDriver should only be called from oci package. +type ImageDriver interface { + Init() error + Remove(path string) error + Reader(path string) (io.ReadCloser, error) + Writer(path string, perm os.FileMode) (io.WriteCloser, error) + BlobWriter(algo digest.Algorithm) (BlobWriter, error) +} + +type InitOpts struct { + // imageLayoutVersion can be an empty string for specifying the default version. + ImageLayoutVersion string + // skip creating oci-layout + SkipCreateImageLayout bool + // skip creating index.json + SkipCreateIndex bool +} + +// Init initializes an OCI image structure. +// Init calls img.Init, creates `oci-layout`(0444), and creates `index.json`(0644). +// +func Init(img ImageDriver, opts InitOpts) error { + if err := img.Init(); err != nil { return err } - // Create blobs/sha256 - if err := os.MkdirAll( - filepath.Join(img, "blobs", string(digest.Canonical)), - 0755); err != nil { - return nil - } + // Create oci-layout - if err := WriteImageLayout(img, spec.ImageLayout{Version: imageLayoutVersion}); err != nil { - return err + if !opts.SkipCreateImageLayout { + imageLayoutVersion := opts.ImageLayoutVersion + if imageLayoutVersion == "" { + imageLayoutVersion = spec.ImageLayoutVersion + } + if err := WriteImageLayout(img, spec.ImageLayout{Version: imageLayoutVersion}); err != nil { + return err + } } + // Create index.json - return WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}}) + if !opts.SkipCreateIndex { + if err := WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}}); err != nil { + return err + } + } + return nil } -func blobPath(img string, d digest.Digest) string { - return filepath.Join(img, "blobs", d.Algorithm().String(), d.Hex()) +func blobPath(d digest.Digest) string { + return filepath.Join("blobs", d.Algorithm().String(), d.Hex()) } -func indexPath(img string) string { - return filepath.Join(img, "index.json") -} +const ( + indexPath = "index.json" +) // GetBlobReader returns io.ReadCloser for a blob. -func GetBlobReader(img string, d digest.Digest) (io.ReadCloser, error) { +func GetBlobReader(img ImageDriver, d digest.Digest) (io.ReadCloser, error) { // we return a reader rather than the full *os.File here so as to prohibit write operations. - return os.Open(blobPath(img, d)) + return img.Reader(blobPath(d)) } // ReadBlob reads an OCI blob. -func ReadBlob(img string, d digest.Digest) ([]byte, error) { +func ReadBlob(img ImageDriver, d digest.Digest) ([]byte, error) { r, err := GetBlobReader(img, d) if err != nil { return nil, err @@ -71,90 +128,47 @@ func ReadBlob(img string, d digest.Digest) ([]byte, error) { // WriteBlob writes bytes as an OCI blob and returns its digest using the canonical digest algorithm. // If you need to specify certain algorithm, you can use NewBlobWriter(img string, algo digest.Algorithm). -func WriteBlob(img string, b []byte) (digest.Digest, error) { - d := digest.FromBytes(b) - return d, ioutil.WriteFile(blobPath(img, d), b, 0444) -} - -// BlobWriter writes an OCI blob and returns a digest when closed. -type BlobWriter interface { - io.Writer - io.Closer - // Digest returns the digest when closed. - // Digest panics when the writer is not closed. - Digest() digest.Digest -} - -// blobWriter implements BlobWriter. -type blobWriter struct { - img string - digester digest.Digester - f *os.File - closed bool +func WriteBlob(img ImageDriver, b []byte) (digest.Digest, error) { + w, err := img.BlobWriter(digest.Canonical) + if err != nil { + return "", err + } + n, err := w.Write(b) + if err != nil { + return "", err + } + if n < len(b) { + return "", io.ErrShortWrite + } + if err := w.Close(); err != nil { + return "", err + } + return w.Digest(), err } // NewBlobWriter returns a BlobWriter. -func NewBlobWriter(img string, algo digest.Algorithm) (BlobWriter, error) { - // use img rather than the default tmp, so as to make sure rename(2) can be applied - f, err := ioutil.TempFile(img, "tmp.blobwriter") - if err != nil { - return nil, err - } - return &blobWriter{ - img: img, - digester: algo.Digester(), - f: f, - }, nil -} - -// Write implements io.Writer. -func (bw *blobWriter) Write(b []byte) (int, error) { - n, err := bw.f.Write(b) - if err != nil { - return n, err - } - return bw.digester.Hash().Write(b) -} - -// Close implements io.Closer. -func (bw *blobWriter) Close() error { - oldPath := bw.f.Name() - if err := bw.f.Close(); err != nil { - return err - } - newPath := blobPath(bw.img, bw.digester.Digest()) - if err := os.MkdirAll(filepath.Dir(newPath), 0755); err != nil { - return err - } - if err := os.Chmod(oldPath, 0444); err != nil { - return err - } - if err := os.Rename(oldPath, newPath); err != nil { - return err - } - bw.closed = true - return nil -} - -// Digest returns the digest when closed. -func (bw *blobWriter) Digest() digest.Digest { - if !bw.closed { - panic("blobWriter is unclosed") - } - return bw.digester.Digest() +func NewBlobWriter(img ImageDriver, algo digest.Algorithm) (BlobWriter, error) { + return img.BlobWriter(algo) } // DeleteBlob deletes an OCI blob. -func DeleteBlob(img string, d digest.Digest) error { - return os.Remove(blobPath(img, d)) +func DeleteBlob(img ImageDriver, d digest.Digest) error { + return img.Remove(blobPath(d)) } // ReadImageLayout returns the image layout. -func ReadImageLayout(img string) (spec.ImageLayout, error) { - b, err := ioutil.ReadFile(filepath.Join(img, spec.ImageLayoutFile)) +func ReadImageLayout(img ImageDriver) (spec.ImageLayout, error) { + r, err := img.Reader(spec.ImageLayoutFile) if err != nil { return spec.ImageLayout{}, err } + b, err := ioutil.ReadAll(r) + if err != nil { + return spec.ImageLayout{}, err + } + if err := r.Close(); err != nil { + return spec.ImageLayout{}, err + } var layout spec.ImageLayout if err := json.Unmarshal(b, &layout); err != nil { return spec.ImageLayout{}, err @@ -163,20 +177,38 @@ func ReadImageLayout(img string) (spec.ImageLayout, error) { } // WriteImageLayout writes the image layout. -func WriteImageLayout(img string, layout spec.ImageLayout) error { +func WriteImageLayout(img ImageDriver, layout spec.ImageLayout) error { b, err := json.Marshal(layout) if err != nil { return err } - return ioutil.WriteFile(filepath.Join(img, spec.ImageLayoutFile), b, 0644) + w, err := img.Writer(spec.ImageLayoutFile, 0444) + if err != nil { + return err + } + n, err := w.Write(b) + if err != nil { + return err + } + if n < len(b) { + return io.ErrShortWrite + } + return w.Close() } // ReadIndex returns the index. -func ReadIndex(img string) (spec.Index, error) { - b, err := ioutil.ReadFile(indexPath(img)) +func ReadIndex(img ImageDriver) (spec.Index, error) { + r, err := img.Reader(indexPath) if err != nil { return spec.Index{}, err } + b, err := ioutil.ReadAll(r) + if err != nil { + return spec.Index{}, err + } + if err := r.Close(); err != nil { + return spec.Index{}, err + } var idx spec.Index if err := json.Unmarshal(b, &idx); err != nil { return spec.Index{}, err @@ -185,17 +217,28 @@ func ReadIndex(img string) (spec.Index, error) { } // WriteIndex writes the index. -func WriteIndex(img string, idx spec.Index) error { +func WriteIndex(img ImageDriver, idx spec.Index) error { b, err := json.Marshal(idx) if err != nil { return err } - return ioutil.WriteFile(indexPath(img), b, 0644) + w, err := img.Writer(indexPath, 0644) + if err != nil { + return err + } + n, err := w.Write(b) + if err != nil { + return err + } + if n < len(b) { + return io.ErrShortWrite + } + return w.Close() } // RemoveManifestDescriptorFromIndex removes the manifest descriptor from the index. // Returns nil error when the entry not found. -func RemoveManifestDescriptorFromIndex(img string, refName string) error { +func RemoveManifestDescriptorFromIndex(img ImageDriver, refName string) error { if refName == "" { return errors.New("empty refName specified") } @@ -217,7 +260,7 @@ func RemoveManifestDescriptorFromIndex(img string, refName string) error { // PutManifestDescriptorToIndex puts a manifest descriptor to the index. // If ref name is set and conflicts with the existing descriptors, the old ones are removed. -func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error { +func PutManifestDescriptorToIndex(img ImageDriver, desc spec.Descriptor) error { refName, ok := desc.Annotations[spec.AnnotationRefName] if ok && refName != "" { if err := RemoveManifestDescriptorFromIndex(img, refName); err != nil { @@ -233,7 +276,7 @@ func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error { } // WriteJSONBlob is an utility function that writes x as a JSON blob with the specified media type, and returns the descriptor. -func WriteJSONBlob(img string, x interface{}, mediaType string) (spec.Descriptor, error) { +func WriteJSONBlob(img ImageDriver, x interface{}, mediaType string) (spec.Descriptor, error) { b, err := json.Marshal(x) if err != nil { return spec.Descriptor{}, err diff --git a/oci/oci_test.go b/oci/oci_test.go deleted file mode 100644 index 356de45a0..000000000 --- a/oci/oci_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package oci - -import ( - "encoding/json" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/containerd/containerd/fs/fstest" - "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" - spec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/stretchr/testify/assert" -) - -func TestInitError(t *testing.T) { - tmp, err := ioutil.TempDir("", "oci") - assert.Nil(t, err) - defer os.RemoveAll(tmp) - err = Init(tmp, "") - assert.Error(t, err, "file exists") -} - -func TestInit(t *testing.T) { - tmp, err := ioutil.TempDir("", "oci") - assert.Nil(t, err) - defer os.RemoveAll(tmp) - img := filepath.Join(tmp, "foo") - err = Init(img, "") - assert.Nil(t, err) - ociLayout, err := json.Marshal(spec.ImageLayout{Version: spec.ImageLayoutVersion}) - assert.Nil(t, err) - indexJSON, err := json.Marshal(spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}}) - applier := fstest.Apply( - fstest.CreateDir("/foo", 0755), - fstest.CreateDir("/foo/blobs", 0755), - fstest.CreateDir("/foo/blobs/"+string(digest.Canonical), 0755), - fstest.CreateFile("/foo/oci-layout", ociLayout, 0644), - fstest.CreateFile("/foo/index.json", indexJSON, 0644), - ) - err = fstest.CheckDirectoryEqualWithApplier(tmp, applier) - assert.Nil(t, err) -} - -func TestWriteReadDeleteBlob(t *testing.T) { - tmp, err := ioutil.TempDir("", "oci") - assert.Nil(t, err) - defer os.RemoveAll(tmp) - img := filepath.Join(tmp, "foo") - err = Init(img, "") - assert.Nil(t, err) - testBlob := []byte("test") - // Write - d, err := WriteBlob(img, testBlob) - applier := fstest.Apply( - fstest.CreateFile("/"+d.Hex(), testBlob, 0444), - ) - err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier) - assert.Nil(t, err) - // Read - b, err := ReadBlob(img, d) - assert.Nil(t, err) - assert.Equal(t, testBlob, b) - // Delete - err = DeleteBlob(img, d) - assert.Nil(t, err) - applier = fstest.Apply() - err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier) - assert.Nil(t, err) -} - -func TestBlobWriter(t *testing.T) { - tmp, err := ioutil.TempDir("", "oci") - assert.Nil(t, err) - defer os.RemoveAll(tmp) - img := filepath.Join(tmp, "foo") - err = Init(img, "") - assert.Nil(t, err) - testBlob := []byte("test") - w, err := NewBlobWriter(img, digest.Canonical) - _, err = w.Write(testBlob) - assert.Nil(t, err) - // blob is not written until closing - applier := fstest.Apply() - err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier) - // digest is unavailable until closing - assert.Panics(t, func() { w.Digest() }) - // close and calculate the digest - err = w.Close() - assert.Nil(t, err) - d := w.Digest() - applier = fstest.Apply( - fstest.CreateFile("/"+d.Hex(), testBlob, 0444), - ) - err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier) - assert.Nil(t, err) -} - -func TestIndex(t *testing.T) { - tmp, err := ioutil.TempDir("", "oci") - assert.Nil(t, err) - defer os.RemoveAll(tmp) - img := filepath.Join(tmp, "foo") - err = Init(img, "") - assert.Nil(t, err) - descs := []spec.Descriptor{ - { - MediaType: spec.MediaTypeImageManifest, - Annotations: map[string]string{ - spec.AnnotationRefName: "foo", - "dummy": "desc0", - }, - }, - { - MediaType: spec.MediaTypeImageManifest, - Annotations: map[string]string{ - // will be removed later - spec.AnnotationRefName: "bar", - "dummy": "desc1", - }, - }, - { - MediaType: spec.MediaTypeImageManifest, - Annotations: map[string]string{ - // duplicated ref name - spec.AnnotationRefName: "foo", - "dummy": "desc2", - }, - }, - { - MediaType: spec.MediaTypeImageManifest, - Annotations: map[string]string{ - // no ref name - "dummy": "desc3", - }, - }, - } - for _, desc := range descs { - err := PutManifestDescriptorToIndex(img, desc) - assert.Nil(t, err) - } - err = RemoveManifestDescriptorFromIndex(img, "bar") - assert.Nil(t, err) - expected := spec.Index{ - Versioned: specs.Versioned{SchemaVersion: 2}, - Manifests: []spec.Descriptor{ - { - MediaType: spec.MediaTypeImageManifest, - Annotations: map[string]string{ - // duplicated ref name - spec.AnnotationRefName: "foo", - "dummy": "desc2", - }, - }, - { - MediaType: spec.MediaTypeImageManifest, - Annotations: map[string]string{ - // no ref name - "dummy": "desc3", - }, - }, - }, - } - idx, err := ReadIndex(img) - assert.Nil(t, err) - assert.Equal(t, expected, idx) -} diff --git a/oci/tar.go b/oci/tar.go new file mode 100644 index 000000000..01614fa53 --- /dev/null +++ b/oci/tar.go @@ -0,0 +1,156 @@ +package oci + +import ( + "archive/tar" + "bytes" + "errors" + "io" + "os" + "path/filepath" + + "github.com/opencontainers/go-digest" +) + +// TarWriter is an interface that is implemented by archive/tar.Writer. +// (Using an interface allows hooking) +type TarWriter interface { + io.WriteCloser + Flush() error + WriteHeader(hdr *tar.Header) error +} + +// Tar is ImageDriver for TAR representation of an OCI image. +func Tar(w TarWriter) ImageDriver { + return &tarDriver{ + w: w, + } +} + +type tarDriver struct { + w TarWriter +} + +func (d *tarDriver) Init() error { + headers := []tar.Header{ + { + Name: "blobs/", + Mode: 0755, + Typeflag: tar.TypeDir, + }, + { + Name: "blobs/" + string(digest.Canonical) + "/", + Mode: 0755, + Typeflag: tar.TypeDir, + }, + } + for _, h := range headers { + if err := d.w.WriteHeader(&h); err != nil { + return err + } + } + return nil +} + +func (d *tarDriver) Remove(path string) error { + return errors.New("Tar does not support Remove") +} + +func (d *tarDriver) Reader(path string) (io.ReadCloser, error) { + // because tar does not support random access + return nil, errors.New("Tar does not support Reader") +} + +func (d *tarDriver) Writer(path string, perm os.FileMode) (io.WriteCloser, error) { + name := filepath.ToSlash(path) + return &tarDriverWriter{ + w: d.w, + name: name, + mode: int64(perm), + }, nil +} + +// tarDriverWriter is used for writing non-blob files +// (e.g. oci-layout, index.json) +type tarDriverWriter struct { + bytes.Buffer + w TarWriter + name string + mode int64 +} + +func (w *tarDriverWriter) Close() error { + if err := w.w.WriteHeader(&tar.Header{ + Name: w.name, + Mode: w.mode, + Size: int64(w.Len()), + Typeflag: tar.TypeReg, + }); err != nil { + return err + } + n, err := io.Copy(w.w, w) + if err != nil { + return err + } + if n < int64(w.Len()) { + return io.ErrShortWrite + } + return w.w.Flush() +} + +func (d *tarDriver) BlobWriter(algo digest.Algorithm) (BlobWriter, error) { + return &tarBlobWriter{ + w: d.w, + digester: algo.Digester(), + }, nil +} + +// tarBlobWriter implements BlobWriter. +type tarBlobWriter struct { + w TarWriter + digester digest.Digester + buf bytes.Buffer // TODO: use tmp file for large buffer? +} + +// Write implements io.Writer. +func (bw *tarBlobWriter) Write(b []byte) (int, error) { + n, err := bw.buf.Write(b) + if err != nil { + return n, err + } + return bw.digester.Hash().Write(b) +} + +func (bw *tarBlobWriter) Commit(size int64, expected digest.Digest) error { + path := "blobs/" + bw.digester.Digest().Algorithm().String() + "/" + bw.digester.Digest().Hex() + if err := bw.w.WriteHeader(&tar.Header{ + Name: path, + Mode: 0444, + Size: int64(bw.buf.Len()), + Typeflag: tar.TypeReg, + }); err != nil { + return err + } + n, err := io.Copy(bw.w, &bw.buf) + if err != nil { + return err + } + if n < int64(bw.buf.Len()) { + return io.ErrShortWrite + } + if size > 0 && size != n { + return ErrUnexpectedSize{Expected: size, Actual: n} + } + if expected != "" && bw.digester.Digest() != expected { + return ErrUnexpectedDigest{Expected: expected, Actual: bw.digester.Digest()} + } + return bw.w.Flush() +} + +func (bw *tarBlobWriter) Close() error { + // we don't close bw.w (reused for writing another blob) + return bw.w.Flush() +} + +func (bw *tarBlobWriter) Digest() digest.Digest { + return bw.digester.Digest() +}