Merge pull request #929 from AkihiroSuda/ociimage

package for manipulating OCI images

oci/oci.go

@@ -0,0 +1,250 @@
+// Package oci provides basic operations for manipulating OCI images.
+package oci
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go"
+	spec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Init initializes the img directory as an OCI image.
+// i.e. Creates oci-layout, index.json, and blobs.
+//
+// img directory must not exist before calling this function.
+//
+// imageLayoutVersion can be an empty string for specifying the default version.
+func Init(img, imageLayoutVersion string) error {
+	if imageLayoutVersion == "" {
+		imageLayoutVersion = spec.ImageLayoutVersion
+	}
+	if _, err := os.Stat(img); err == nil {
+		return os.ErrExist
+	}
+	// Create the directory
+	if err := os.MkdirAll(img, 0755); err != nil {
+		return err
+	}
+	// Create blobs/sha256
+	if err := os.MkdirAll(
+		filepath.Join(img, "blobs", string(digest.Canonical)),
+		0755); err != nil {
+		return nil
+	}
+	// Create oci-layout
+	if err := WriteImageLayout(img, spec.ImageLayout{Version: imageLayoutVersion}); err != nil {
+		return err
+	}
+	// Create index.json
+	return WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}})
+}
+
+func blobPath(img string, d digest.Digest) string {
+	return filepath.Join(img, "blobs", d.Algorithm().String(), d.Hex())
+}
+
+func indexPath(img string) string {
+	return filepath.Join(img, "index.json")
+}
+
+// GetBlobReader returns io.ReadCloser for a blob.
+func GetBlobReader(img string, d digest.Digest) (io.ReadCloser, error) {
+	// we return a reader rather than the full *os.File here so as to prohibit write operations.
+	return os.Open(blobPath(img, d))
+}
+
+// ReadBlob reads an OCI blob.
+func ReadBlob(img string, d digest.Digest) ([]byte, error) {
+	r, err := GetBlobReader(img, d)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+	return ioutil.ReadAll(r)
+}
+
+// WriteBlob writes bytes as an OCI blob and returns its digest using the canonical digest algorithm.
+// If you need to specify certain algorithm, you can use NewBlobWriter(img string, algo digest.Algorithm).
+func WriteBlob(img string, b []byte) (digest.Digest, error) {
+	d := digest.FromBytes(b)
+	return d, ioutil.WriteFile(blobPath(img, d), b, 0444)
+}
+
+// BlobWriter writes an OCI blob and returns a digest when closed.
+type BlobWriter interface {
+	io.Writer
+	io.Closer
+	// Digest returns the digest when closed.
+	// Digest panics when the writer is not closed.
+	Digest() digest.Digest
+}
+
+// blobWriter implements BlobWriter.
+type blobWriter struct {
+	img      string
+	digester digest.Digester
+	f        *os.File
+	closed   bool
+}
+
+// NewBlobWriter returns a BlobWriter.
+func NewBlobWriter(img string, algo digest.Algorithm) (BlobWriter, error) {
+	// use img rather than the default tmp, so as to make sure rename(2) can be applied
+	f, err := ioutil.TempFile(img, "tmp.blobwriter")
+	if err != nil {
+		return nil, err
+	}
+	return &blobWriter{
+		img:      img,
+		digester: algo.Digester(),
+		f:        f,
+	}, nil
+}
+
+// Write implements io.Writer.
+func (bw *blobWriter) Write(b []byte) (int, error) {
+	n, err := bw.f.Write(b)
+	if err != nil {
+		return n, err
+	}
+	return bw.digester.Hash().Write(b)
+}
+
+// Close implements io.Closer.
+func (bw *blobWriter) Close() error {
+	oldPath := bw.f.Name()
+	if err := bw.f.Close(); err != nil {
+		return err
+	}
+	newPath := blobPath(bw.img, bw.digester.Digest())
+	if err := os.MkdirAll(filepath.Dir(newPath), 0755); err != nil {
+		return err
+	}
+	if err := os.Chmod(oldPath, 0444); err != nil {
+		return err
+	}
+	if err := os.Rename(oldPath, newPath); err != nil {
+		return err
+	}
+	bw.closed = true
+	return nil
+}
+
+// Digest returns the digest when closed.
+func (bw *blobWriter) Digest() digest.Digest {
+	if !bw.closed {
+		panic("blobWriter is unclosed")
+	}
+	return bw.digester.Digest()
+}
+
+// DeleteBlob deletes an OCI blob.
+func DeleteBlob(img string, d digest.Digest) error {
+	return os.Remove(blobPath(img, d))
+}
+
+// ReadImageLayout returns the image layout.
+func ReadImageLayout(img string) (spec.ImageLayout, error) {
+	b, err := ioutil.ReadFile(filepath.Join(img, spec.ImageLayoutFile))
+	if err != nil {
+		return spec.ImageLayout{}, err
+	}
+	var layout spec.ImageLayout
+	if err := json.Unmarshal(b, &layout); err != nil {
+		return spec.ImageLayout{}, err
+	}
+	return layout, nil
+}
+
+// WriteImageLayout writes the image layout.
+func WriteImageLayout(img string, layout spec.ImageLayout) error {
+	b, err := json.Marshal(layout)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(filepath.Join(img, spec.ImageLayoutFile), b, 0644)
+}
+
+// ReadIndex returns the index.
+func ReadIndex(img string) (spec.Index, error) {
+	b, err := ioutil.ReadFile(indexPath(img))
+	if err != nil {
+		return spec.Index{}, err
+	}
+	var idx spec.Index
+	if err := json.Unmarshal(b, &idx); err != nil {
+		return spec.Index{}, err
+	}
+	return idx, nil
+}
+
+// WriteIndex writes the index.
+func WriteIndex(img string, idx spec.Index) error {
+	b, err := json.Marshal(idx)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(indexPath(img), b, 0644)
+}
+
+// RemoveManifestDescriptorFromIndex removes the manifest descriptor from the index.
+// Returns nil error when the entry not found.
+func RemoveManifestDescriptorFromIndex(img string, refName string) error {
+	if refName == "" {
+		return errors.New("empty refName specified")
+	}
+	src, err := ReadIndex(img)
+	if err != nil {
+		return err
+	}
+	dst := src
+	dst.Manifests = nil
+	for _, m := range src.Manifests {
+		mRefName, ok := m.Annotations[spec.AnnotationRefName]
+		if ok && mRefName == refName {
+			continue
+		}
+		dst.Manifests = append(dst.Manifests, m)
+	}
+	return WriteIndex(img, dst)
+}
+
+// PutManifestDescriptorToIndex puts a manifest descriptor to the index.
+// If ref name is set and conflicts with the existing descriptors, the old ones are removed.
+func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error {
+	refName, ok := desc.Annotations[spec.AnnotationRefName]
+	if ok && refName != "" {
+		if err := RemoveManifestDescriptorFromIndex(img, refName); err != nil {
+			return err
+		}
+	}
+	idx, err := ReadIndex(img)
+	if err != nil {
+		return err
+	}
+	idx.Manifests = append(idx.Manifests, desc)
+	return WriteIndex(img, idx)
+}
+
+// WriteJSONBlob is an utility function that writes x as a JSON blob with the specified media type, and returns the descriptor.
+func WriteJSONBlob(img string, x interface{}, mediaType string) (spec.Descriptor, error) {
+	b, err := json.Marshal(x)
+	if err != nil {
+		return spec.Descriptor{}, err
+	}
+	d, err := WriteBlob(img, b)
+	if err != nil {
+		return spec.Descriptor{}, err
+	}
+	return spec.Descriptor{
+		MediaType: mediaType,
+		Digest:    d,
+		Size:      int64(len(b)),
+	}, nil
+}

oci/oci_test.go

@@ -0,0 +1,168 @@
+package oci
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/containerd/fs/fstest"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go"
+	spec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInitError(t *testing.T) {
+	tmp, err := ioutil.TempDir("", "oci")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmp)
+	err = Init(tmp, "")
+	assert.Error(t, err, "file exists")
+}
+
+func TestInit(t *testing.T) {
+	tmp, err := ioutil.TempDir("", "oci")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmp)
+	img := filepath.Join(tmp, "foo")
+	err = Init(img, "")
+	assert.Nil(t, err)
+	ociLayout, err := json.Marshal(spec.ImageLayout{Version: spec.ImageLayoutVersion})
+	assert.Nil(t, err)
+	indexJSON, err := json.Marshal(spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}})
+	applier := fstest.Apply(
+		fstest.CreateDir("/foo", 0755),
+		fstest.CreateDir("/foo/blobs", 0755),
+		fstest.CreateDir("/foo/blobs/"+string(digest.Canonical), 0755),
+		fstest.CreateFile("/foo/oci-layout", ociLayout, 0644),
+		fstest.CreateFile("/foo/index.json", indexJSON, 0644),
+	)
+	err = fstest.CheckDirectoryEqualWithApplier(tmp, applier)
+	assert.Nil(t, err)
+}
+
+func TestWriteReadDeleteBlob(t *testing.T) {
+	tmp, err := ioutil.TempDir("", "oci")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmp)
+	img := filepath.Join(tmp, "foo")
+	err = Init(img, "")
+	assert.Nil(t, err)
+	testBlob := []byte("test")
+	// Write
+	d, err := WriteBlob(img, testBlob)
+	applier := fstest.Apply(
+		fstest.CreateFile("/"+d.Hex(), testBlob, 0444),
+	)
+	err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
+	assert.Nil(t, err)
+	// Read
+	b, err := ReadBlob(img, d)
+	assert.Nil(t, err)
+	assert.Equal(t, testBlob, b)
+	// Delete
+	err = DeleteBlob(img, d)
+	assert.Nil(t, err)
+	applier = fstest.Apply()
+	err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
+	assert.Nil(t, err)
+}
+
+func TestBlobWriter(t *testing.T) {
+	tmp, err := ioutil.TempDir("", "oci")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmp)
+	img := filepath.Join(tmp, "foo")
+	err = Init(img, "")
+	assert.Nil(t, err)
+	testBlob := []byte("test")
+	w, err := NewBlobWriter(img, digest.Canonical)
+	_, err = w.Write(testBlob)
+	assert.Nil(t, err)
+	// blob is not written until closing
+	applier := fstest.Apply()
+	err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
+	// digest is unavailable until closing
+	assert.Panics(t, func() { w.Digest() })
+	// close and calculate the digest
+	err = w.Close()
+	assert.Nil(t, err)
+	d := w.Digest()
+	applier = fstest.Apply(
+		fstest.CreateFile("/"+d.Hex(), testBlob, 0444),
+	)
+	err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
+	assert.Nil(t, err)
+}
+
+func TestIndex(t *testing.T) {
+	tmp, err := ioutil.TempDir("", "oci")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmp)
+	img := filepath.Join(tmp, "foo")
+	err = Init(img, "")
+	assert.Nil(t, err)
+	descs := []spec.Descriptor{
+		{
+			MediaType: spec.MediaTypeImageManifest,
+			Annotations: map[string]string{
+				spec.AnnotationRefName: "foo",
+				"dummy":                "desc0",
+			},
+		},
+		{
+			MediaType: spec.MediaTypeImageManifest,
+			Annotations: map[string]string{
+				// will be removed later
+				spec.AnnotationRefName: "bar",
+				"dummy":                "desc1",
+			},
+		},
+		{
+			MediaType: spec.MediaTypeImageManifest,
+			Annotations: map[string]string{
+				// duplicated ref name
+				spec.AnnotationRefName: "foo",
+				"dummy":                "desc2",
+			},
+		},
+		{
+			MediaType: spec.MediaTypeImageManifest,
+			Annotations: map[string]string{
+				// no ref name
+				"dummy": "desc3",
+			},
+		},
+	}
+	for _, desc := range descs {
+		err := PutManifestDescriptorToIndex(img, desc)
+		assert.Nil(t, err)
+	}
+	err = RemoveManifestDescriptorFromIndex(img, "bar")
+	assert.Nil(t, err)
+	expected := spec.Index{
+		Versioned: specs.Versioned{SchemaVersion: 2},
+		Manifests: []spec.Descriptor{
+			{
+				MediaType: spec.MediaTypeImageManifest,
+				Annotations: map[string]string{
+					// duplicated ref name
+					spec.AnnotationRefName: "foo",
+					"dummy":                "desc2",
+				},
+			},
+			{
+				MediaType: spec.MediaTypeImageManifest,
+				Annotations: map[string]string{
+					// no ref name
+					"dummy": "desc3",
+				},
+			},
+		},
+	}
+	idx, err := ReadIndex(img)
+	assert.Nil(t, err)
+	assert.Equal(t, expected, idx)
+}

vendor.conf

@@ -27,7 +27,7 @@ google.golang.org/grpc v1.3.0
 github.com/pkg/errors v0.8.0
 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448
 golang.org/x/sys f3918c30c5c2cb527c0b071a27c35120a6c0719a
-github.com/opencontainers/image-spec v1.0.0-rc6
+github.com/opencontainers/image-spec 372ad780f63454fbbbbcc7cf80e5b90245c13e13
 github.com/containerd/continuity 86cec1535a968310e7532819f699ff2830ed7463
 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
 github.com/BurntSushi/toml v0.2.0-21-g9906417

vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go

@@ -0,0 +1,56 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+const (
+	// AnnotationCreated is the annotation key for the date and time on which the image was built (date-time string as defined by RFC 3339).
+	AnnotationCreated = "org.opencontainers.image.created"
+
+	// AnnotationAuthors is the annotation key for the contact details of the people or organization responsible for the image (freeform string).
+	AnnotationAuthors = "org.opencontainers.image.authors"
+
+	// AnnotationURL is the annotation key for the URL to find more information on the image.
+	AnnotationURL = "org.opencontainers.image.url"
+
+	// AnnotationDocumentation is the annotation key for the URL to get documentation on the image.
+	AnnotationDocumentation = "org.opencontainers.image.documentation"
+
+	// AnnotationSource is the annotation key for the URL to get source code for building the image.
+	AnnotationSource = "org.opencontainers.image.source"
+
+	// AnnotationVersion is the annotation key for the version of the packaged software.
+	// The version MAY match a label or tag in the source code repository.
+	// The version MAY be Semantic versioning-compatible.
+	AnnotationVersion = "org.opencontainers.image.version"
+
+	// AnnotationRevision is the annotation key for the source control revision identifier for the packaged software.
+	AnnotationRevision = "org.opencontainers.image.revision"
+
+	// AnnotationVendor is the annotation key for the name of the distributing entity, organization or individual.
+	AnnotationVendor = "org.opencontainers.image.vendor"
+
+	// AnnotationLicenses is the annotation key for the license(s) under which contained software is distributed as an SPDX License Expression.
+	AnnotationLicenses = "org.opencontainers.image.licenses"
+
+	// AnnotationRefName is the annotation key for the name of the reference for a target.
+	// SHOULD only be considered valid when on descriptors on `index.json` within image layout.
+	AnnotationRefName = "org.opencontainers.image.ref.name"
+
+	// AnnotationTitle is the annotation key for the human-readable title of the image.
+	AnnotationTitle = "org.opencontainers.image.title"
+
+	// AnnotationDescription is the annotation key for the human-readable description of the software packaged in the image.
+	AnnotationDescription = "org.opencontainers.image.description"
+)