github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

client: add Import() and Export() for importing/exporting image in OCI format

Browse Source 
Export as a tar (Note: "-" can be used for stdout):

    $ ctr images export /tmp/oci-busybox.tar docker.io/library/busybox:latest

Import a tar (Note: "-" can be used for stdin):

    $ ctr images import foo/new:latest /tmp/oci-busybox.tar

Note: media types are not converted at the moment: e.g.
  application/vnd.docker.image.rootfs.diff.tar.gzip
  -> application/vnd.oci.image.layer.v1.tar+gzip

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>
This commit is contained in:
Akihiro Suda
2017-06-15 08:50:20 +00:00
parent 856b038437
commit b518f11dba
13 changed files with 904 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

265
oci/oci.go
View File

@@ -1,9 +1,12 @@
// Package oci provides basic operations for manipulating OCI images.
// This package can be used even outside of containerd, and contains some
// functions not used in containerd itself.
package oci
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
@@ -14,53 +17,107 @@ import (
spec "github.com/opencontainers/image-spec/specs-go/v1"
)
// Init initializes the img directory as an OCI image.
// i.e. Creates oci-layout, index.json, and blobs.
//
// img directory must not exist before calling this function.
//
// imageLayoutVersion can be an empty string for specifying the default version.
func Init(img, imageLayoutVersion string) error {
if imageLayoutVersion == "" {
imageLayoutVersion = spec.ImageLayoutVersion
// BlobWriter writes an OCI blob and returns a digest when committed.
type BlobWriter interface {
// Close is expected to be called after Commit() when commission is needed.
io.WriteCloser
// Digest may return empty digest or panics until committed.
Digest() digest.Digest
// Commit commits the blob (but no roll-back is guaranteed on an error).
// size and expected can be zero-value when unknown.
Commit(size int64, expected digest.Digest) error
}
// ErrUnexpectedSize can be returned from BlobWriter.Commit()
type ErrUnexpectedSize struct {
Expected int64
Actual int64
}
func (e ErrUnexpectedSize) Error() string {
if e.Expected > 0 && e.Expected != e.Actual {
return fmt.Sprintf("unexpected size: %d != %d", e.Expected, e.Actual)
}
if _, err := os.Stat(img); err == nil {
return os.ErrExist
return fmt.Sprintf("malformed ErrUnexpectedSize(%+v)", e)
}
// ErrUnexpectedDigest can be returned from BlobWriter.Commit()
type ErrUnexpectedDigest struct {
Expected digest.Digest
Actual digest.Digest
}
func (e ErrUnexpectedDigest) Error() string {
if e.Expected.String() != "" && e.Expected.String() != e.Actual.String() {
return fmt.Sprintf("unexpected digest: %v != %v", e.Expected, e.Actual)
}
// Create the directory
if err := os.MkdirAll(img, 0755); err != nil {
return fmt.Sprintf("malformed ErrUnexpectedDigest(%+v)", e)
}
// ImageDriver corresponds to the representation of an image.
// Path uses os.PathSeparator as the separator.
// The methods of ImageDriver should only be called from oci package.
type ImageDriver interface {
Init() error
Remove(path string) error
Reader(path string) (io.ReadCloser, error)
Writer(path string, perm os.FileMode) (io.WriteCloser, error)
BlobWriter(algo digest.Algorithm) (BlobWriter, error)
}
type InitOpts struct {
// imageLayoutVersion can be an empty string for specifying the default version.
ImageLayoutVersion string
// skip creating oci-layout
SkipCreateImageLayout bool
// skip creating index.json
SkipCreateIndex bool
}
// Init initializes an OCI image structure.
// Init calls img.Init, creates `oci-layout`(0444), and creates `index.json`(0644).
//
func Init(img ImageDriver, opts InitOpts) error {
if err := img.Init(); err != nil {
return err
}
// Create blobs/sha256
if err := os.MkdirAll(
filepath.Join(img, "blobs", string(digest.Canonical)),
0755); err != nil {
return nil
}
// Create oci-layout
if err := WriteImageLayout(img, spec.ImageLayout{Version: imageLayoutVersion}); err != nil {
return err
if !opts.SkipCreateImageLayout {
imageLayoutVersion := opts.ImageLayoutVersion
if imageLayoutVersion == "" {
imageLayoutVersion = spec.ImageLayoutVersion
}
if err := WriteImageLayout(img, spec.ImageLayout{Version: imageLayoutVersion}); err != nil {
return err
}
}
// Create index.json
return WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}})
if !opts.SkipCreateIndex {
if err := WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}}); err != nil {
return err
}
}
return nil
}
func blobPath(img string, d digest.Digest) string {
return filepath.Join(img, "blobs", d.Algorithm().String(), d.Hex())
func blobPath(d digest.Digest) string {
return filepath.Join("blobs", d.Algorithm().String(), d.Hex())
}
func indexPath(img string) string {
return filepath.Join(img, "index.json")
}
const (
indexPath = "index.json"
)
// GetBlobReader returns io.ReadCloser for a blob.
func GetBlobReader(img string, d digest.Digest) (io.ReadCloser, error) {
func GetBlobReader(img ImageDriver, d digest.Digest) (io.ReadCloser, error) {
// we return a reader rather than the full *os.File here so as to prohibit write operations.
return os.Open(blobPath(img, d))
return img.Reader(blobPath(d))
}
// ReadBlob reads an OCI blob.
func ReadBlob(img string, d digest.Digest) ([]byte, error) {
func ReadBlob(img ImageDriver, d digest.Digest) ([]byte, error) {
r, err := GetBlobReader(img, d)
if err != nil {
return nil, err
@@ -71,90 +128,47 @@ func ReadBlob(img string, d digest.Digest) ([]byte, error) {
// WriteBlob writes bytes as an OCI blob and returns its digest using the canonical digest algorithm.
// If you need to specify certain algorithm, you can use NewBlobWriter(img string, algo digest.Algorithm).
func WriteBlob(img string, b []byte) (digest.Digest, error) {
d := digest.FromBytes(b)
return d, ioutil.WriteFile(blobPath(img, d), b, 0444)
}
// BlobWriter writes an OCI blob and returns a digest when closed.
type BlobWriter interface {
io.Writer
io.Closer
// Digest returns the digest when closed.
// Digest panics when the writer is not closed.
Digest() digest.Digest
}
// blobWriter implements BlobWriter.
type blobWriter struct {
img string
digester digest.Digester
f *os.File
closed bool
func WriteBlob(img ImageDriver, b []byte) (digest.Digest, error) {
w, err := img.BlobWriter(digest.Canonical)
if err != nil {
return "", err
}
n, err := w.Write(b)
if err != nil {
return "", err
}
if n < len(b) {
return "", io.ErrShortWrite
}
if err := w.Close(); err != nil {
return "", err
}
return w.Digest(), err
}
// NewBlobWriter returns a BlobWriter.
func NewBlobWriter(img string, algo digest.Algorithm) (BlobWriter, error) {
// use img rather than the default tmp, so as to make sure rename(2) can be applied
f, err := ioutil.TempFile(img, "tmp.blobwriter")
if err != nil {
return nil, err
}
return &blobWriter{
img: img,
digester: algo.Digester(),
f: f,
}, nil
}
// Write implements io.Writer.
func (bw *blobWriter) Write(b []byte) (int, error) {
n, err := bw.f.Write(b)
if err != nil {
return n, err
}
return bw.digester.Hash().Write(b)
}
// Close implements io.Closer.
func (bw *blobWriter) Close() error {
oldPath := bw.f.Name()
if err := bw.f.Close(); err != nil {
return err
}
newPath := blobPath(bw.img, bw.digester.Digest())
if err := os.MkdirAll(filepath.Dir(newPath), 0755); err != nil {
return err
}
if err := os.Chmod(oldPath, 0444); err != nil {
return err
}
if err := os.Rename(oldPath, newPath); err != nil {
return err
}
bw.closed = true
return nil
}
// Digest returns the digest when closed.
func (bw *blobWriter) Digest() digest.Digest {
if !bw.closed {
panic("blobWriter is unclosed")
}
return bw.digester.Digest()
func NewBlobWriter(img ImageDriver, algo digest.Algorithm) (BlobWriter, error) {
return img.BlobWriter(algo)
}
// DeleteBlob deletes an OCI blob.
func DeleteBlob(img string, d digest.Digest) error {
return os.Remove(blobPath(img, d))
func DeleteBlob(img ImageDriver, d digest.Digest) error {
return img.Remove(blobPath(d))
}
// ReadImageLayout returns the image layout.
func ReadImageLayout(img string) (spec.ImageLayout, error) {
b, err := ioutil.ReadFile(filepath.Join(img, spec.ImageLayoutFile))
func ReadImageLayout(img ImageDriver) (spec.ImageLayout, error) {
r, err := img.Reader(spec.ImageLayoutFile)
if err != nil {
return spec.ImageLayout{}, err
}
b, err := ioutil.ReadAll(r)
if err != nil {
return spec.ImageLayout{}, err
}
if err := r.Close(); err != nil {
return spec.ImageLayout{}, err
}
var layout spec.ImageLayout
if err := json.Unmarshal(b, &layout); err != nil {
return spec.ImageLayout{}, err
@@ -163,20 +177,38 @@ func ReadImageLayout(img string) (spec.ImageLayout, error) {
}
// WriteImageLayout writes the image layout.
func WriteImageLayout(img string, layout spec.ImageLayout) error {
func WriteImageLayout(img ImageDriver, layout spec.ImageLayout) error {
b, err := json.Marshal(layout)
if err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(img, spec.ImageLayoutFile), b, 0644)
w, err := img.Writer(spec.ImageLayoutFile, 0444)
if err != nil {
return err
}
n, err := w.Write(b)
if err != nil {
return err
}
if n < len(b) {
return io.ErrShortWrite
}
return w.Close()
}
// ReadIndex returns the index.
func ReadIndex(img string) (spec.Index, error) {
b, err := ioutil.ReadFile(indexPath(img))
func ReadIndex(img ImageDriver) (spec.Index, error) {
r, err := img.Reader(indexPath)
if err != nil {
return spec.Index{}, err
}
b, err := ioutil.ReadAll(r)
if err != nil {
return spec.Index{}, err
}
if err := r.Close(); err != nil {
return spec.Index{}, err
}
var idx spec.Index
if err := json.Unmarshal(b, &idx); err != nil {
return spec.Index{}, err
@@ -185,17 +217,28 @@ func ReadIndex(img string) (spec.Index, error) {
}
// WriteIndex writes the index.
func WriteIndex(img string, idx spec.Index) error {
func WriteIndex(img ImageDriver, idx spec.Index) error {
b, err := json.Marshal(idx)
if err != nil {
return err
}
return ioutil.WriteFile(indexPath(img), b, 0644)
w, err := img.Writer(indexPath, 0644)
if err != nil {
return err
}
n, err := w.Write(b)
if err != nil {
return err
}
if n < len(b) {
return io.ErrShortWrite
}
return w.Close()
}
// RemoveManifestDescriptorFromIndex removes the manifest descriptor from the index.
// Returns nil error when the entry not found.
func RemoveManifestDescriptorFromIndex(img string, refName string) error {
func RemoveManifestDescriptorFromIndex(img ImageDriver, refName string) error {
if refName == "" {
return errors.New("empty refName specified")
}
@@ -217,7 +260,7 @@ func RemoveManifestDescriptorFromIndex(img string, refName string) error {
// PutManifestDescriptorToIndex puts a manifest descriptor to the index.
// If ref name is set and conflicts with the existing descriptors, the old ones are removed.
func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error {
func PutManifestDescriptorToIndex(img ImageDriver, desc spec.Descriptor) error {
refName, ok := desc.Annotations[spec.AnnotationRefName]
if ok && refName != "" {
if err := RemoveManifestDescriptorFromIndex(img, refName); err != nil {
@@ -233,7 +276,7 @@ func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error {
}
// WriteJSONBlob is an utility function that writes x as a JSON blob with the specified media type, and returns the descriptor.
func WriteJSONBlob(img string, x interface{}, mediaType string) (spec.Descriptor, error) {
func WriteJSONBlob(img ImageDriver, x interface{}, mediaType string) (spec.Descriptor, error) {
b, err := json.Marshal(x)
if err != nil {
return spec.Descriptor{}, err