From e4bc59a2952cfce1c4189e0324146b0a4a9e110e Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 27 May 2017 12:23:52 +0000 Subject: [PATCH] package for manipulating OCI images Signed-off-by: Akihiro Suda --- oci/oci.go | 250 ++++++++++++++++++ oci/oci_test.go | 168 ++++++++++++ vendor.conf | 2 +- .../image-spec/specs-go/v1/annotations.go | 56 ++++ 4 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 oci/oci.go create mode 100644 oci/oci_test.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go diff --git a/oci/oci.go b/oci/oci.go new file mode 100644 index 000000000..90894962a --- /dev/null +++ b/oci/oci.go @@ -0,0 +1,250 @@ +// Package oci provides basic operations for manipulating OCI images. +package oci + +import ( + "encoding/json" + "errors" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + spec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Init initializes the img directory as an OCI image. +// i.e. Creates oci-layout, index.json, and blobs. +// +// img directory must not exist before calling this function. +// +// imageLayoutVersion can be an empty string for specifying the default version. +func Init(img, imageLayoutVersion string) error { + if imageLayoutVersion == "" { + imageLayoutVersion = spec.ImageLayoutVersion + } + if _, err := os.Stat(img); err == nil { + return os.ErrExist + } + // Create the directory + if err := os.MkdirAll(img, 0755); err != nil { + return err + } + // Create blobs/sha256 + if err := os.MkdirAll( + filepath.Join(img, "blobs", string(digest.Canonical)), + 0755); err != nil { + return nil + } + // Create oci-layout + if err := WriteImageLayout(img, spec.ImageLayout{Version: imageLayoutVersion}); err != nil { + return err + } + // Create index.json + return WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}}) +} + +func blobPath(img string, d digest.Digest) string { + return filepath.Join(img, "blobs", d.Algorithm().String(), d.Hex()) +} + +func indexPath(img string) string { + return filepath.Join(img, "index.json") +} + +// GetBlobReader returns io.ReadCloser for a blob. +func GetBlobReader(img string, d digest.Digest) (io.ReadCloser, error) { + // we return a reader rather than the full *os.File here so as to prohibit write operations. + return os.Open(blobPath(img, d)) +} + +// ReadBlob reads an OCI blob. +func ReadBlob(img string, d digest.Digest) ([]byte, error) { + r, err := GetBlobReader(img, d) + if err != nil { + return nil, err + } + defer r.Close() + return ioutil.ReadAll(r) +} + +// WriteBlob writes bytes as an OCI blob and returns its digest using the canonical digest algorithm. +// If you need to specify certain algorithm, you can use NewBlobWriter(img string, algo digest.Algorithm). +func WriteBlob(img string, b []byte) (digest.Digest, error) { + d := digest.FromBytes(b) + return d, ioutil.WriteFile(blobPath(img, d), b, 0444) +} + +// BlobWriter writes an OCI blob and returns a digest when closed. +type BlobWriter interface { + io.Writer + io.Closer + // Digest returns the digest when closed. + // Digest panics when the writer is not closed. + Digest() digest.Digest +} + +// blobWriter implements BlobWriter. +type blobWriter struct { + img string + digester digest.Digester + f *os.File + closed bool +} + +// NewBlobWriter returns a BlobWriter. +func NewBlobWriter(img string, algo digest.Algorithm) (BlobWriter, error) { + // use img rather than the default tmp, so as to make sure rename(2) can be applied + f, err := ioutil.TempFile(img, "tmp.blobwriter") + if err != nil { + return nil, err + } + return &blobWriter{ + img: img, + digester: algo.Digester(), + f: f, + }, nil +} + +// Write implements io.Writer. +func (bw *blobWriter) Write(b []byte) (int, error) { + n, err := bw.f.Write(b) + if err != nil { + return n, err + } + return bw.digester.Hash().Write(b) +} + +// Close implements io.Closer. +func (bw *blobWriter) Close() error { + oldPath := bw.f.Name() + if err := bw.f.Close(); err != nil { + return err + } + newPath := blobPath(bw.img, bw.digester.Digest()) + if err := os.MkdirAll(filepath.Dir(newPath), 0755); err != nil { + return err + } + if err := os.Chmod(oldPath, 0444); err != nil { + return err + } + if err := os.Rename(oldPath, newPath); err != nil { + return err + } + bw.closed = true + return nil +} + +// Digest returns the digest when closed. +func (bw *blobWriter) Digest() digest.Digest { + if !bw.closed { + panic("blobWriter is unclosed") + } + return bw.digester.Digest() +} + +// DeleteBlob deletes an OCI blob. +func DeleteBlob(img string, d digest.Digest) error { + return os.Remove(blobPath(img, d)) +} + +// ReadImageLayout returns the image layout. +func ReadImageLayout(img string) (spec.ImageLayout, error) { + b, err := ioutil.ReadFile(filepath.Join(img, spec.ImageLayoutFile)) + if err != nil { + return spec.ImageLayout{}, err + } + var layout spec.ImageLayout + if err := json.Unmarshal(b, &layout); err != nil { + return spec.ImageLayout{}, err + } + return layout, nil +} + +// WriteImageLayout writes the image layout. +func WriteImageLayout(img string, layout spec.ImageLayout) error { + b, err := json.Marshal(layout) + if err != nil { + return err + } + return ioutil.WriteFile(filepath.Join(img, spec.ImageLayoutFile), b, 0644) +} + +// ReadIndex returns the index. +func ReadIndex(img string) (spec.Index, error) { + b, err := ioutil.ReadFile(indexPath(img)) + if err != nil { + return spec.Index{}, err + } + var idx spec.Index + if err := json.Unmarshal(b, &idx); err != nil { + return spec.Index{}, err + } + return idx, nil +} + +// WriteIndex writes the index. +func WriteIndex(img string, idx spec.Index) error { + b, err := json.Marshal(idx) + if err != nil { + return err + } + return ioutil.WriteFile(indexPath(img), b, 0644) +} + +// RemoveManifestDescriptorFromIndex removes the manifest descriptor from the index. +// Returns nil error when the entry not found. +func RemoveManifestDescriptorFromIndex(img string, refName string) error { + if refName == "" { + return errors.New("empty refName specified") + } + src, err := ReadIndex(img) + if err != nil { + return err + } + dst := src + dst.Manifests = nil + for _, m := range src.Manifests { + mRefName, ok := m.Annotations[spec.AnnotationRefName] + if ok && mRefName == refName { + continue + } + dst.Manifests = append(dst.Manifests, m) + } + return WriteIndex(img, dst) +} + +// PutManifestDescriptorToIndex puts a manifest descriptor to the index. +// If ref name is set and conflicts with the existing descriptors, the old ones are removed. +func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error { + refName, ok := desc.Annotations[spec.AnnotationRefName] + if ok && refName != "" { + if err := RemoveManifestDescriptorFromIndex(img, refName); err != nil { + return err + } + } + idx, err := ReadIndex(img) + if err != nil { + return err + } + idx.Manifests = append(idx.Manifests, desc) + return WriteIndex(img, idx) +} + +// WriteJSONBlob is an utility function that writes x as a JSON blob with the specified media type, and returns the descriptor. +func WriteJSONBlob(img string, x interface{}, mediaType string) (spec.Descriptor, error) { + b, err := json.Marshal(x) + if err != nil { + return spec.Descriptor{}, err + } + d, err := WriteBlob(img, b) + if err != nil { + return spec.Descriptor{}, err + } + return spec.Descriptor{ + MediaType: mediaType, + Digest: d, + Size: int64(len(b)), + }, nil +} diff --git a/oci/oci_test.go b/oci/oci_test.go new file mode 100644 index 000000000..356de45a0 --- /dev/null +++ b/oci/oci_test.go @@ -0,0 +1,168 @@ +package oci + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/containerd/containerd/fs/fstest" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + spec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" +) + +func TestInitError(t *testing.T) { + tmp, err := ioutil.TempDir("", "oci") + assert.Nil(t, err) + defer os.RemoveAll(tmp) + err = Init(tmp, "") + assert.Error(t, err, "file exists") +} + +func TestInit(t *testing.T) { + tmp, err := ioutil.TempDir("", "oci") + assert.Nil(t, err) + defer os.RemoveAll(tmp) + img := filepath.Join(tmp, "foo") + err = Init(img, "") + assert.Nil(t, err) + ociLayout, err := json.Marshal(spec.ImageLayout{Version: spec.ImageLayoutVersion}) + assert.Nil(t, err) + indexJSON, err := json.Marshal(spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}}) + applier := fstest.Apply( + fstest.CreateDir("/foo", 0755), + fstest.CreateDir("/foo/blobs", 0755), + fstest.CreateDir("/foo/blobs/"+string(digest.Canonical), 0755), + fstest.CreateFile("/foo/oci-layout", ociLayout, 0644), + fstest.CreateFile("/foo/index.json", indexJSON, 0644), + ) + err = fstest.CheckDirectoryEqualWithApplier(tmp, applier) + assert.Nil(t, err) +} + +func TestWriteReadDeleteBlob(t *testing.T) { + tmp, err := ioutil.TempDir("", "oci") + assert.Nil(t, err) + defer os.RemoveAll(tmp) + img := filepath.Join(tmp, "foo") + err = Init(img, "") + assert.Nil(t, err) + testBlob := []byte("test") + // Write + d, err := WriteBlob(img, testBlob) + applier := fstest.Apply( + fstest.CreateFile("/"+d.Hex(), testBlob, 0444), + ) + err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier) + assert.Nil(t, err) + // Read + b, err := ReadBlob(img, d) + assert.Nil(t, err) + assert.Equal(t, testBlob, b) + // Delete + err = DeleteBlob(img, d) + assert.Nil(t, err) + applier = fstest.Apply() + err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier) + assert.Nil(t, err) +} + +func TestBlobWriter(t *testing.T) { + tmp, err := ioutil.TempDir("", "oci") + assert.Nil(t, err) + defer os.RemoveAll(tmp) + img := filepath.Join(tmp, "foo") + err = Init(img, "") + assert.Nil(t, err) + testBlob := []byte("test") + w, err := NewBlobWriter(img, digest.Canonical) + _, err = w.Write(testBlob) + assert.Nil(t, err) + // blob is not written until closing + applier := fstest.Apply() + err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier) + // digest is unavailable until closing + assert.Panics(t, func() { w.Digest() }) + // close and calculate the digest + err = w.Close() + assert.Nil(t, err) + d := w.Digest() + applier = fstest.Apply( + fstest.CreateFile("/"+d.Hex(), testBlob, 0444), + ) + err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier) + assert.Nil(t, err) +} + +func TestIndex(t *testing.T) { + tmp, err := ioutil.TempDir("", "oci") + assert.Nil(t, err) + defer os.RemoveAll(tmp) + img := filepath.Join(tmp, "foo") + err = Init(img, "") + assert.Nil(t, err) + descs := []spec.Descriptor{ + { + MediaType: spec.MediaTypeImageManifest, + Annotations: map[string]string{ + spec.AnnotationRefName: "foo", + "dummy": "desc0", + }, + }, + { + MediaType: spec.MediaTypeImageManifest, + Annotations: map[string]string{ + // will be removed later + spec.AnnotationRefName: "bar", + "dummy": "desc1", + }, + }, + { + MediaType: spec.MediaTypeImageManifest, + Annotations: map[string]string{ + // duplicated ref name + spec.AnnotationRefName: "foo", + "dummy": "desc2", + }, + }, + { + MediaType: spec.MediaTypeImageManifest, + Annotations: map[string]string{ + // no ref name + "dummy": "desc3", + }, + }, + } + for _, desc := range descs { + err := PutManifestDescriptorToIndex(img, desc) + assert.Nil(t, err) + } + err = RemoveManifestDescriptorFromIndex(img, "bar") + assert.Nil(t, err) + expected := spec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + Manifests: []spec.Descriptor{ + { + MediaType: spec.MediaTypeImageManifest, + Annotations: map[string]string{ + // duplicated ref name + spec.AnnotationRefName: "foo", + "dummy": "desc2", + }, + }, + { + MediaType: spec.MediaTypeImageManifest, + Annotations: map[string]string{ + // no ref name + "dummy": "desc3", + }, + }, + }, + } + idx, err := ReadIndex(img) + assert.Nil(t, err) + assert.Equal(t, expected, idx) +} diff --git a/vendor.conf b/vendor.conf index 1bd280d2a..cf8391165 100644 --- a/vendor.conf +++ b/vendor.conf @@ -27,7 +27,7 @@ google.golang.org/grpc v1.3.0 github.com/pkg/errors v0.8.0 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448 golang.org/x/sys f3918c30c5c2cb527c0b071a27c35120a6c0719a -github.com/opencontainers/image-spec v1.0.0-rc6 +github.com/opencontainers/image-spec 372ad780f63454fbbbbcc7cf80e5b90245c13e13 github.com/containerd/continuity 86cec1535a968310e7532819f699ff2830ed7463 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/BurntSushi/toml v0.2.0-21-g9906417 diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go new file mode 100644 index 000000000..35d810895 --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go @@ -0,0 +1,56 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +const ( + // AnnotationCreated is the annotation key for the date and time on which the image was built (date-time string as defined by RFC 3339). + AnnotationCreated = "org.opencontainers.image.created" + + // AnnotationAuthors is the annotation key for the contact details of the people or organization responsible for the image (freeform string). + AnnotationAuthors = "org.opencontainers.image.authors" + + // AnnotationURL is the annotation key for the URL to find more information on the image. + AnnotationURL = "org.opencontainers.image.url" + + // AnnotationDocumentation is the annotation key for the URL to get documentation on the image. + AnnotationDocumentation = "org.opencontainers.image.documentation" + + // AnnotationSource is the annotation key for the URL to get source code for building the image. + AnnotationSource = "org.opencontainers.image.source" + + // AnnotationVersion is the annotation key for the version of the packaged software. + // The version MAY match a label or tag in the source code repository. + // The version MAY be Semantic versioning-compatible. + AnnotationVersion = "org.opencontainers.image.version" + + // AnnotationRevision is the annotation key for the source control revision identifier for the packaged software. + AnnotationRevision = "org.opencontainers.image.revision" + + // AnnotationVendor is the annotation key for the name of the distributing entity, organization or individual. + AnnotationVendor = "org.opencontainers.image.vendor" + + // AnnotationLicenses is the annotation key for the license(s) under which contained software is distributed as an SPDX License Expression. + AnnotationLicenses = "org.opencontainers.image.licenses" + + // AnnotationRefName is the annotation key for the name of the reference for a target. + // SHOULD only be considered valid when on descriptors on `index.json` within image layout. + AnnotationRefName = "org.opencontainers.image.ref.name" + + // AnnotationTitle is the annotation key for the human-readable title of the image. + AnnotationTitle = "org.opencontainers.image.title" + + // AnnotationDescription is the annotation key for the human-readable description of the software packaged in the image. + AnnotationDescription = "org.opencontainers.image.description" +)