diff --git a/api/next.pb.txt b/api/next.pb.txt index e028bd5e6..c58ef4d08 100755 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -1120,6 +1120,34 @@ file { type: TYPE_STRING json_name: "ref" } + field { + name: "labels" + number: 5 + label: LABEL_REPEATED + type: TYPE_MESSAGE + type_name: ".containerd.services.diff.v1.DiffRequest.LabelsEntry" + json_name: "labels" + } + nested_type { + name: "LabelsEntry" + field { + name: "key" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "key" + } + field { + name: "value" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "value" + } + options { + map_entry: true + } + } } message_type { name: "DiffResponse" diff --git a/api/services/diff/v1/diff.pb.go b/api/services/diff/v1/diff.pb.go index 764c5b17d..b19a37731 100644 --- a/api/services/diff/v1/diff.pb.go +++ b/api/services/diff/v1/diff.pb.go @@ -30,6 +30,7 @@ import ( import strings "strings" import reflect "reflect" +import github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" import io "io" @@ -78,6 +79,9 @@ type DiffRequest struct { // Ref identifies the pre-commit content store object. This // reference can be used to get the status from the content store. Ref string `protobuf:"bytes,4,opt,name=ref,proto3" json:"ref,omitempty"` + // Labels are the labels to apply to the generated content + // on content store commit. + Labels map[string]string `protobuf:"bytes,5,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *DiffRequest) Reset() { *m = DiffRequest{} } @@ -334,6 +338,23 @@ func (m *DiffRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintDiff(dAtA, i, uint64(len(m.Ref))) i += copy(dAtA[i:], m.Ref) } + if len(m.Labels) > 0 { + for k, _ := range m.Labels { + dAtA[i] = 0x2a + i++ + v := m.Labels[k] + mapSize := 1 + len(k) + sovDiff(uint64(len(k))) + 1 + len(v) + sovDiff(uint64(len(v))) + i = encodeVarintDiff(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintDiff(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x12 + i++ + i = encodeVarintDiff(dAtA, i, uint64(len(v))) + i += copy(dAtA[i:], v) + } + } return i, nil } @@ -441,6 +462,14 @@ func (m *DiffRequest) Size() (n int) { if l > 0 { n += 1 + l + sovDiff(uint64(l)) } + if len(m.Labels) > 0 { + for k, v := range m.Labels { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovDiff(uint64(len(k))) + 1 + len(v) + sovDiff(uint64(len(v))) + n += mapEntrySize + 1 + sovDiff(uint64(mapEntrySize)) + } + } return n } @@ -492,11 +521,22 @@ func (this *DiffRequest) String() string { if this == nil { return "nil" } + keysForLabels := make([]string, 0, len(this.Labels)) + for k, _ := range this.Labels { + keysForLabels = append(keysForLabels, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForLabels) + mapStringForLabels := "map[string]string{" + for _, k := range keysForLabels { + mapStringForLabels += fmt.Sprintf("%v: %v,", k, this.Labels[k]) + } + mapStringForLabels += "}" s := strings.Join([]string{`&DiffRequest{`, `Left:` + strings.Replace(fmt.Sprintf("%v", this.Left), "Mount", "containerd_types.Mount", 1) + `,`, `Right:` + strings.Replace(fmt.Sprintf("%v", this.Right), "Mount", "containerd_types.Mount", 1) + `,`, `MediaType:` + fmt.Sprintf("%v", this.MediaType) + `,`, `Ref:` + fmt.Sprintf("%v", this.Ref) + `,`, + `Labels:` + mapStringForLabels + `,`, `}`, }, "") return s @@ -865,6 +905,122 @@ func (m *DiffRequest) Unmarshal(dAtA []byte) error { } m.Ref = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDiff + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthDiff + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDiff + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDiff + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthDiff + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + if m.Labels == nil { + m.Labels = make(map[string]string) + } + if iNdEx < postIndex { + var valuekey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDiff + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + valuekey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDiff + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthDiff + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue := string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + m.Labels[mapkey] = mapvalue + } else { + var mapvalue string + m.Labels[mapkey] = mapvalue + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipDiff(dAtA[iNdEx:]) @@ -1079,31 +1235,34 @@ func init() { } var fileDescriptorDiff = []byte{ - // 401 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x93, 0x41, 0x8b, 0xda, 0x40, - 0x14, 0xc7, 0x9d, 0x26, 0x5a, 0x1c, 0x2d, 0x94, 0xa1, 0xd0, 0x90, 0xb6, 0x41, 0x72, 0x8a, 0x2d, - 0x9d, 0x54, 0x0b, 0x1e, 0xea, 0xc5, 0x16, 0xa1, 0xa7, 0x5e, 0x82, 0xa7, 0x16, 0x5a, 0x62, 0x32, - 0x89, 0x03, 0x9a, 0x19, 0x33, 0xa3, 0xe0, 0xad, 0x9f, 0x63, 0xbf, 0xce, 0x5e, 0x3c, 0xee, 0x71, - 0x8f, 0x6b, 0x3e, 0xc9, 0x92, 0x49, 0xb2, 0x1b, 0x58, 0x70, 0xb3, 0x7b, 0x9a, 0xc7, 0xbc, 0xdf, - 0xff, 0xbd, 0xff, 0xbc, 0xbc, 0xc0, 0x59, 0x4c, 0xe5, 0x6a, 0xb7, 0xc4, 0x01, 0xdb, 0xb8, 0x01, - 0x4b, 0xa4, 0x4f, 0x13, 0x92, 0x86, 0xf5, 0xd0, 0xe7, 0xd4, 0x15, 0x24, 0xdd, 0xd3, 0x80, 0x08, - 0x37, 0xa4, 0x51, 0xe4, 0xee, 0x47, 0xea, 0xc4, 0x3c, 0x65, 0x92, 0xa1, 0x77, 0xf7, 0x2c, 0xae, - 0x38, 0xac, 0xf2, 0xfb, 0x91, 0xf9, 0x26, 0x66, 0x31, 0x53, 0x9c, 0x9b, 0x47, 0x85, 0xc4, 0x9c, - 0x34, 0x6a, 0x2a, 0x0f, 0x9c, 0x08, 0x77, 0xc3, 0x76, 0x89, 0x2c, 0x75, 0xd3, 0x27, 0xe8, 0x42, - 0x22, 0x82, 0x94, 0x72, 0xc9, 0xd2, 0x42, 0x6c, 0x6f, 0x61, 0xff, 0x3b, 0xe7, 0xeb, 0x83, 0x47, - 0xb6, 0x3b, 0x22, 0x24, 0xfa, 0x02, 0xf5, 0xdc, 0xa5, 0x01, 0x06, 0xc0, 0xe9, 0x8d, 0xdf, 0xe3, - 0xda, 0x33, 0x54, 0x05, 0x3c, 0xbf, 0xab, 0xe0, 0x29, 0x12, 0xb9, 0xb0, 0xa3, 0xdc, 0x08, 0xe3, - 0xc5, 0x40, 0x73, 0x7a, 0xe3, 0xb7, 0x0f, 0x35, 0xbf, 0xf2, 0xbc, 0x57, 0x62, 0xf6, 0x4f, 0xf8, - 0xaa, 0x6c, 0x29, 0x38, 0x4b, 0x04, 0x41, 0x13, 0xf8, 0xd2, 0xe7, 0x7c, 0x4d, 0x49, 0xd8, 0xa8, - 0x6d, 0x05, 0xdb, 0x17, 0x00, 0xf6, 0xe6, 0x34, 0x8a, 0x2a, 0xef, 0x9f, 0xa0, 0xbe, 0x26, 0x91, - 0x34, 0xc0, 0x79, 0x1f, 0x0a, 0x42, 0x9f, 0x61, 0x3b, 0xa5, 0xf1, 0x4a, 0x3e, 0xe6, 0xba, 0xa0, - 0xd0, 0x07, 0x08, 0x37, 0x24, 0xa4, 0xfe, 0xbf, 0x3c, 0x67, 0x68, 0x03, 0xe0, 0x74, 0xbd, 0xae, - 0xba, 0x59, 0x1c, 0x38, 0x41, 0xaf, 0xa1, 0x96, 0x92, 0xc8, 0xd0, 0xd5, 0x7d, 0x1e, 0xda, 0x33, - 0xd8, 0x2f, 0xbc, 0x95, 0x8f, 0xac, 0x06, 0xab, 0x35, 0x1d, 0xec, 0xf8, 0x12, 0x40, 0x3d, 0x2f, - 0x81, 0xfe, 0xc2, 0xb6, 0x1a, 0x18, 0x1a, 0xe2, 0x33, 0x5b, 0x85, 0xeb, 0xdf, 0xd1, 0xfc, 0xd8, - 0x04, 0x2d, 0xad, 0xfd, 0x29, 0xfb, 0x38, 0x67, 0x35, 0xb5, 0x49, 0x9b, 0xc3, 0x06, 0x64, 0x51, - 0xfc, 0xc7, 0xe2, 0x78, 0xb2, 0x5a, 0xd7, 0x27, 0xab, 0xf5, 0x3f, 0xb3, 0xc0, 0x31, 0xb3, 0xc0, - 0x55, 0x66, 0x81, 0x9b, 0xcc, 0x02, 0xbf, 0xbf, 0x3d, 0xeb, 0x27, 0x9b, 0xe6, 0xe7, 0xb2, 0xa3, - 0xb6, 0xf7, 0xeb, 0x6d, 0x00, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x16, 0x1d, 0x04, 0xa9, 0x03, 0x00, - 0x00, + // 454 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x53, 0x4f, 0x6f, 0xd3, 0x30, + 0x14, 0x9f, 0xfb, 0x0f, 0xf5, 0x75, 0x48, 0xc8, 0x9a, 0x44, 0x14, 0x20, 0xaa, 0x7a, 0xea, 0x40, + 0x38, 0xac, 0xa0, 0x09, 0xb6, 0xcb, 0x40, 0x43, 0x5c, 0xc6, 0x25, 0xda, 0x09, 0x24, 0x50, 0xda, + 0xbc, 0x74, 0x16, 0x69, 0xec, 0xd9, 0x6e, 0xa5, 0xdc, 0xf8, 0x2e, 0x7c, 0x14, 0x2e, 0x3b, 0x72, + 0xe4, 0x48, 0xfb, 0x49, 0x90, 0x9d, 0x14, 0x22, 0x21, 0x95, 0xc0, 0x29, 0x2f, 0xcf, 0xbf, 0x7f, + 0xf6, 0xb3, 0xe1, 0x6c, 0xce, 0xcd, 0xd5, 0x72, 0xca, 0x66, 0x62, 0x11, 0xce, 0x44, 0x6e, 0x62, + 0x9e, 0xa3, 0x4a, 0xea, 0x65, 0x2c, 0x79, 0xa8, 0x51, 0xad, 0xf8, 0x0c, 0x75, 0x98, 0xf0, 0x34, + 0x0d, 0x57, 0x47, 0xee, 0xcb, 0xa4, 0x12, 0x46, 0xd0, 0x7b, 0xbf, 0xb1, 0x6c, 0x8b, 0x63, 0x6e, + 0x7d, 0x75, 0xe4, 0x1f, 0xcc, 0xc5, 0x5c, 0x38, 0x5c, 0x68, 0xab, 0x92, 0xe2, 0x1f, 0x37, 0x32, + 0x35, 0x85, 0x44, 0x1d, 0x2e, 0xc4, 0x32, 0x37, 0x15, 0xef, 0xf4, 0x1f, 0x78, 0x09, 0xea, 0x99, + 0xe2, 0xd2, 0x08, 0x55, 0x92, 0x47, 0xd7, 0xb0, 0xff, 0x52, 0xca, 0xac, 0x88, 0xf0, 0x7a, 0x89, + 0xda, 0xd0, 0x27, 0xd0, 0xb1, 0x29, 0x3d, 0x32, 0x24, 0xe3, 0xc1, 0xe4, 0x3e, 0xab, 0x6d, 0xc3, + 0x29, 0xb0, 0xf3, 0x5f, 0x0a, 0x91, 0x43, 0xd2, 0x10, 0x7a, 0x2e, 0x8d, 0xf6, 0x5a, 0xc3, 0xf6, + 0x78, 0x30, 0xb9, 0xfb, 0x27, 0xe7, 0xad, 0x5d, 0x8f, 0x2a, 0xd8, 0xe8, 0x0d, 0xdc, 0xae, 0x2c, + 0xb5, 0x14, 0xb9, 0x46, 0x7a, 0x0c, 0xb7, 0x62, 0x29, 0x33, 0x8e, 0x49, 0x23, 0xdb, 0x2d, 0x78, + 0xf4, 0xa5, 0x05, 0x83, 0x73, 0x9e, 0xa6, 0xdb, 0xec, 0x8f, 0xa0, 0x93, 0x61, 0x6a, 0x3c, 0xb2, + 0x3b, 0x87, 0x03, 0xd1, 0xc7, 0xd0, 0x55, 0x7c, 0x7e, 0x65, 0xfe, 0x96, 0xba, 0x44, 0xd1, 0x07, + 0x00, 0x0b, 0x4c, 0x78, 0xfc, 0xd1, 0xae, 0x79, 0xed, 0x21, 0x19, 0xf7, 0xa3, 0xbe, 0xeb, 0x5c, + 0x16, 0x12, 0xe9, 0x1d, 0x68, 0x2b, 0x4c, 0xbd, 0x8e, 0xeb, 0xdb, 0x92, 0x5e, 0x40, 0x2f, 0x8b, + 0xa7, 0x98, 0x69, 0xaf, 0xeb, 0x0c, 0x9e, 0xb1, 0x1d, 0x37, 0x82, 0xd5, 0xb6, 0xc1, 0x2e, 0x1c, + 0xed, 0x75, 0x6e, 0x54, 0x11, 0x55, 0x1a, 0xfe, 0x0b, 0x18, 0xd4, 0xda, 0xd6, 0xee, 0x13, 0x16, + 0xee, 0xb4, 0xfa, 0x91, 0x2d, 0xe9, 0x01, 0x74, 0x57, 0x71, 0xb6, 0x44, 0xaf, 0xe5, 0x7a, 0xe5, + 0xcf, 0x49, 0xeb, 0x39, 0x19, 0x9d, 0xc1, 0x7e, 0xa9, 0x5e, 0x9d, 0xf6, 0x76, 0xc2, 0xed, 0xa6, + 0x13, 0x9e, 0x7c, 0x25, 0xd0, 0xb1, 0x12, 0xf4, 0x03, 0x74, 0xdd, 0xe4, 0xe8, 0xe1, 0xce, 0xcd, + 0xd4, 0x2f, 0x94, 0xff, 0xb0, 0x09, 0xb4, 0x8a, 0xf6, 0xbe, 0xf2, 0x19, 0x37, 0x3d, 0x2b, 0xff, + 0xb0, 0x01, 0xb2, 0x14, 0x7f, 0x75, 0x79, 0xb3, 0x0e, 0xf6, 0xbe, 0xaf, 0x83, 0xbd, 0xcf, 0x9b, + 0x80, 0xdc, 0x6c, 0x02, 0xf2, 0x6d, 0x13, 0x90, 0x1f, 0x9b, 0x80, 0xbc, 0x3b, 0xf9, 0xaf, 0xd7, + 0x7e, 0x6a, 0xbf, 0xd3, 0x9e, 0x7b, 0x46, 0x4f, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0xd6, 0x01, + 0x51, 0xf0, 0x32, 0x04, 0x00, 0x00, } diff --git a/api/services/diff/v1/diff.proto b/api/services/diff/v1/diff.proto index f42b5ed42..d1d2a8244 100644 --- a/api/services/diff/v1/diff.proto +++ b/api/services/diff/v1/diff.proto @@ -50,6 +50,10 @@ message DiffRequest { // Ref identifies the pre-commit content store object. This // reference can be used to get the status from the content store. string ref = 4; + + // Labels are the labels to apply to the generated content + // on content store commit. + map labels = 5; } message DiffResponse { diff --git a/client.go b/client.go index 508dc6b91..6033f32c7 100644 --- a/client.go +++ b/client.go @@ -22,6 +22,7 @@ import ( versionservice "github.com/containerd/containerd/api/services/version/v1" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" @@ -31,7 +32,6 @@ import ( "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/remotes/docker/schema1" contentservice "github.com/containerd/containerd/services/content" - "github.com/containerd/containerd/services/diff" diffservice "github.com/containerd/containerd/services/diff" imagesservice "github.com/containerd/containerd/services/images" snapshotservice "github.com/containerd/containerd/services/snapshot" @@ -439,8 +439,8 @@ func (c *Client) ImageService() images.Store { return imagesservice.NewStoreFromClient(imagesapi.NewImagesClient(c.conn)) } -// DiffService returns the underlying DiffService -func (c *Client) DiffService() diff.DiffService { +// DiffService returns the underlying Differ +func (c *Client) DiffService() diff.Differ { return diffservice.NewDiffServiceFromClient(diffapi.NewDiffClient(c.conn)) } diff --git a/cmd/containerd/builtins.go b/cmd/containerd/builtins.go index 36e3bbd08..6ee3f8de7 100644 --- a/cmd/containerd/builtins.go +++ b/cmd/containerd/builtins.go @@ -2,7 +2,7 @@ package main // register containerd builtins here import ( - _ "github.com/containerd/containerd/differ" + _ "github.com/containerd/containerd/diff/walking" _ "github.com/containerd/containerd/services/containers" _ "github.com/containerd/containerd/services/content" _ "github.com/containerd/containerd/services/diff" diff --git a/cmd/ctr/utils.go b/cmd/ctr/utils.go index e84b19cb9..fa5c8000f 100644 --- a/cmd/ctr/utils.go +++ b/cmd/ctr/utils.go @@ -28,6 +28,7 @@ import ( "github.com/containerd/containerd/api/services/tasks/v1" versionservice "github.com/containerd/containerd/api/services/version/v1" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/images" "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/platforms" @@ -35,7 +36,7 @@ import ( "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/rootfs" contentservice "github.com/containerd/containerd/services/content" - "github.com/containerd/containerd/services/diff" + diffservice "github.com/containerd/containerd/services/diff" imagesservice "github.com/containerd/containerd/services/images" namespacesservice "github.com/containerd/containerd/services/namespaces" snapshotservice "github.com/containerd/containerd/services/snapshot" @@ -169,12 +170,12 @@ func getImageStore(clicontext *cli.Context) (images.Store, error) { return imagesservice.NewStoreFromClient(imagesapi.NewImagesClient(conn)), nil } -func getDiffService(context *cli.Context) (diff.DiffService, error) { +func getDiffService(context *cli.Context) (diff.Differ, error) { conn, err := getGRPCConnection(context) if err != nil { return nil, err } - return diff.NewDiffServiceFromClient(diffapi.NewDiffClient(conn)), nil + return diffservice.NewDiffServiceFromClient(diffapi.NewDiffClient(conn)), nil } func getVersionService(context *cli.Context) (versionservice.VersionClient, error) { diff --git a/container_opts_unix.go b/container_opts_unix.go index 51fca9df5..75283ef83 100644 --- a/container_opts_unix.go +++ b/container_opts_unix.go @@ -18,6 +18,7 @@ import ( digest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) // WithCheckpoint allows a container to be created from the checkpointed information @@ -41,11 +42,11 @@ func WithCheckpoint(desc v1.Descriptor, snapshotKey string) NewContainerOpts { case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList: config, err := images.Config(ctx, store, m, platforms.Default()) if err != nil { - return err + return errors.Wrap(err, "unable to resolve image config") } diffIDs, err := images.RootFS(ctx, store, config) if err != nil { - return err + return errors.Wrap(err, "unable to get rootfs") } setSnapshotterIfEmpty(c) if _, err := client.SnapshotService(c.Snapshotter).Prepare(ctx, snapshotKey, identity.ChainID(diffIDs).String()); err != nil { @@ -57,7 +58,7 @@ func WithCheckpoint(desc v1.Descriptor, snapshotKey string) NewContainerOpts { case images.MediaTypeContainerd1CheckpointConfig: data, err := content.ReadBlob(ctx, store, m.Digest) if err != nil { - return err + return errors.Wrap(err, "unable to read checkpoint config") } var any protobuf.Any if err := proto.Unmarshal(data, &any); err != nil { @@ -70,10 +71,10 @@ func WithCheckpoint(desc v1.Descriptor, snapshotKey string) NewContainerOpts { // apply the rw snapshot to the new rw layer mounts, err := client.SnapshotService(c.Snapshotter).Mounts(ctx, snapshotKey) if err != nil { - return err + return errors.Wrapf(err, "unable to get mounts for %s", snapshotKey) } if _, err := client.DiffService().Apply(ctx, *rw, mounts); err != nil { - return err + return errors.Wrap(err, "unable to apply rw diff") } } c.SnapshotKey = snapshotKey diff --git a/content/content.go b/content/content.go index 6770b3464..05fd4aebe 100644 --- a/content/content.go +++ b/content/content.go @@ -90,6 +90,7 @@ type IngestManager interface { // Writer handles the write of content into a content store type Writer interface { // Close is expected to be called after Commit() when commission is needed. + // Closing a writer without commit allows resuming or aborting. io.WriteCloser // Digest may return empty digest or panics until committed. diff --git a/content/local/writer.go b/content/local/writer.go index 362c0e209..c4f1a94f3 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -130,10 +130,10 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, // tact. // // If one needs to resume the transaction, a new writer can be obtained from -// `ContentStore.Resume` using the same key. The write can then be continued +// `Ingester.Writer` using the same key. The write can then be continued // from it was left off. // -// To abandon a transaction completely, first call close then `Store.Remove` to +// To abandon a transaction completely, first call close then `IngestManager.Abort` to // clean up the associated resources. func (w *writer) Close() (err error) { if w.fp != nil { diff --git a/diff/diff.go b/diff/diff.go new file mode 100644 index 000000000..85cef3583 --- /dev/null +++ b/diff/diff.go @@ -0,0 +1,69 @@ +package diff + +import ( + "github.com/containerd/containerd/mount" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/net/context" +) + +// Config is used to hold parameters needed for a diff operation +type Config struct { + // MediaType is the type of diff to generate + // Default depends on the differ, + // i.e. application/vnd.oci.image.layer.v1.tar+gzip + MediaType string + + // Reference is the content upload reference + // Default will use a random reference string + Reference string + + // Labels are the labels to apply to the generated content + Labels map[string]string +} + +// Opt is used to configure a diff operation +type Opt func(*Config) error + +// Differ allows the apply and creation of filesystem diffs between mounts +type Differ interface { + // Apply applies the content referred to by the given descriptor to + // the provided mount. The method of applying is based on the + // implementation and content descriptor. For example, in the common + // case the descriptor is a file system difference in tar format, + // that tar would be applied on top of the mounts. + Apply(ctx context.Context, desc ocispec.Descriptor, mount []mount.Mount) (ocispec.Descriptor, error) + + // DiffMounts computes the difference between two mounts and returns a + // descriptor for the computed diff. The options can provide + // a ref which can be used to track the content creation of the diff. + // The media type which is used to determine the format of the created + // content can also be provided as an option. + DiffMounts(ctx context.Context, lower, upper []mount.Mount, opts ...Opt) (ocispec.Descriptor, error) +} + +// WithMediaType sets the media type to use for creating the diff, without +// specifying the differ will choose a default. +func WithMediaType(m string) Opt { + return func(c *Config) error { + c.MediaType = m + return nil + } +} + +// WithReference is used to set the content upload reference used by +// the diff operation. This allows the caller to track the upload through +// the content store. +func WithReference(ref string) Opt { + return func(c *Config) error { + c.Reference = ref + return nil + } +} + +// WithLabels is used to set content labels on the created diff content. +func WithLabels(labels map[string]string) Opt { + return func(c *Config) error { + c.Labels = labels + return nil + } +} diff --git a/differ/differ.go b/diff/walking/differ.go similarity index 75% rename from differ/differ.go rename to diff/walking/differ.go index f8195c7fd..70ee06533 100644 --- a/differ/differ.go +++ b/diff/walking/differ.go @@ -1,16 +1,22 @@ -package differ +package walking import ( + "crypto/rand" + "encoding/base64" + "fmt" "io" "io/ioutil" "os" "strings" + "time" "github.com/containerd/containerd/archive" "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/log" "github.com/containerd/containerd/metadata" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/platforms" @@ -46,9 +52,9 @@ type walkingDiff struct { var emptyDesc = ocispec.Descriptor{} -// NewWalkingDiff is a generic implementation of plugin.Differ. +// NewWalkingDiff is a generic implementation of diff.Differ. // NewWalkingDiff is expected to work with any filesystem. -func NewWalkingDiff(store content.Store) (plugin.Differ, error) { +func NewWalkingDiff(store content.Store) (diff.Differ, error) { return &walkingDiff{ store: store, }, nil @@ -122,17 +128,25 @@ func (s *walkingDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts // DiffMounts creates a diff between the given mounts and uploads the result // to the content store. -func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount, media, ref string) (ocispec.Descriptor, error) { +func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) { + var config diff.Config + for _, opt := range opts { + if err := opt(&config); err != nil { + return emptyDesc, err + } + } + + if config.MediaType == "" { + config.MediaType = ocispec.MediaTypeImageLayerGzip + } + var isCompressed bool - switch media { + switch config.MediaType { case ocispec.MediaTypeImageLayer: case ocispec.MediaTypeImageLayerGzip: isCompressed = true - case "": - media = ocispec.MediaTypeImageLayerGzip - isCompressed = true default: - return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", media) + return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", config.MediaType) } aDir, err := ioutil.TempDir("", "left-") if err != nil { @@ -156,12 +170,32 @@ func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount } defer mount.Unmount(bDir, 0) - cw, err := s.store.Writer(ctx, ref, 0, "") + var newReference bool + if config.Reference == "" { + newReference = true + config.Reference = uniqueRef() + } + + cw, err := s.store.Writer(ctx, config.Reference, 0, "") if err != nil { return emptyDesc, errors.Wrap(err, "failed to open writer") } + defer func() { + if err != nil { + cw.Close() + if newReference { + if err := s.store.Abort(ctx, config.Reference); err != nil { + log.G(ctx).WithField("ref", config.Reference).Warnf("failed to delete diff upload") + } + } + } + }() + if !newReference { + if err := cw.Truncate(0); err != nil { + return emptyDesc, err + } + } - var opts []content.Opt if isCompressed { dgstr := digest.SHA256.Digester() compressed, err := compression.CompressStream(cw, compression.Gzip) @@ -173,17 +207,24 @@ func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount if err != nil { return emptyDesc, errors.Wrap(err, "failed to write compressed diff") } - opts = append(opts, content.WithLabels(map[string]string{ - "containerd.io/uncompressed": dgstr.Digest().String(), - })) + + if config.Labels == nil { + config.Labels = map[string]string{} + } + config.Labels["containerd.io/uncompressed"] = dgstr.Digest().String() } else { if err = archive.WriteDiff(ctx, cw, aDir, bDir); err != nil { return emptyDesc, errors.Wrap(err, "failed to write diff") } } + var commitopts []content.Opt + if config.Labels != nil { + commitopts = append(commitopts, content.WithLabels(config.Labels)) + } + dgst := cw.Digest() - if err := cw.Commit(ctx, 0, dgst, opts...); err != nil { + if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil { return emptyDesc, errors.Wrap(err, "failed to commit") } @@ -193,7 +234,7 @@ func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount } return ocispec.Descriptor{ - MediaType: media, + MediaType: config.MediaType, Size: info.Size, Digest: info.Digest, }, nil @@ -209,3 +250,11 @@ func (rc *readCounter) Read(p []byte) (n int, err error) { rc.c += int64(n) return } + +func uniqueRef() string { + t := time.Now() + var b [3]byte + // Ignore read failures, just decreases uniqueness + rand.Read(b[:]) + return fmt.Sprintf("%d-%s", t.UnixNano(), base64.URLEncoding.EncodeToString(b[:])) +} diff --git a/plugin/differ.go b/plugin/differ.go deleted file mode 100644 index 5fb3553dc..000000000 --- a/plugin/differ.go +++ /dev/null @@ -1,13 +0,0 @@ -package plugin - -import ( - "github.com/containerd/containerd/mount" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "golang.org/x/net/context" -) - -// Differ allows the apply and creation of filesystem diffs between mounts -type Differ interface { - Apply(ctx context.Context, desc ocispec.Descriptor, mount []mount.Mount) (ocispec.Descriptor, error) - DiffMounts(ctx context.Context, lower, upper []mount.Mount, media, ref string) (ocispec.Descriptor, error) -} diff --git a/rootfs/apply.go b/rootfs/apply.go index b398228d4..4e21fd10f 100644 --- a/rootfs/apply.go +++ b/rootfs/apply.go @@ -6,9 +6,9 @@ import ( "fmt" "time" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" - "github.com/containerd/containerd/mount" "github.com/containerd/containerd/snapshot" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" @@ -17,11 +17,6 @@ import ( "golang.org/x/net/context" ) -// Applier is used to apply a descriptor of a layer diff on top of mounts. -type Applier interface { - Apply(context.Context, ocispec.Descriptor, []mount.Mount) (ocispec.Descriptor, error) -} - // Layer represents the descriptors for a layer diff. These descriptions // include the descriptor for the uncompressed tar diff as well as a blob // used to transport that tar. The blob descriptor may or may not describe @@ -35,7 +30,7 @@ type Layer struct { // The returned result is a chain id digest representing all the applied layers. // Layers are applied in order they are given, making the first layer the // bottom-most layer in the layer chain. -func ApplyLayers(ctx context.Context, layers []Layer, sn snapshot.Snapshotter, a Applier) (digest.Digest, error) { +func ApplyLayers(ctx context.Context, layers []Layer, sn snapshot.Snapshotter, a diff.Differ) (digest.Digest, error) { var chain []digest.Digest for _, layer := range layers { if _, err := ApplyLayer(ctx, layer, chain, sn, a); err != nil { @@ -51,7 +46,7 @@ func ApplyLayers(ctx context.Context, layers []Layer, sn snapshot.Snapshotter, a // ApplyLayer applies a single layer on top of the given provided layer chain, // using the provided snapshotter and applier. If the layer was unpacked true // is returned, if the layer already exists false is returned. -func ApplyLayer(ctx context.Context, layer Layer, chain []digest.Digest, sn snapshot.Snapshotter, a Applier) (bool, error) { +func ApplyLayer(ctx context.Context, layer Layer, chain []digest.Digest, sn snapshot.Snapshotter, a diff.Differ, opts ...snapshot.Opt) (bool, error) { var ( parent = identity.ChainID(chain) chainID = identity.ChainID(append(chain, layer.Diff.Digest)) diff --git a/rootfs/diff.go b/rootfs/diff.go index 1a16fb9d9..035eb3026 100644 --- a/rootfs/diff.go +++ b/rootfs/diff.go @@ -3,25 +3,18 @@ package rootfs import ( "fmt" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/snapshot" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/net/context" ) -// MountDiffer computes the difference between two mounts and returns a -// descriptor for the computed diff. The provided ref can be used to track -// the content creation of the diff and media type is used to determine the -// format of the created content. -type MountDiffer interface { - DiffMounts(ctx context.Context, lower, upper []mount.Mount, media, ref string) (ocispec.Descriptor, error) -} - // Diff creates a layer diff for the given snapshot identifier from the parent // of the snapshot. A content ref is provided to track the progress of the // content creation and the provided snapshotter and mount differ are used // for calculating the diff. The descriptor for the layer diff is returned. -func Diff(ctx context.Context, snapshotID, contentRef string, sn snapshot.Snapshotter, md MountDiffer) (ocispec.Descriptor, error) { +func Diff(ctx context.Context, snapshotID string, sn snapshot.Snapshotter, d diff.Differ, opts ...diff.Opt) (ocispec.Descriptor, error) { info, err := sn.Stat(ctx, snapshotID) if err != nil { return ocispec.Descriptor{}, err @@ -49,5 +42,5 @@ func Diff(ctx context.Context, snapshotID, contentRef string, sn snapshot.Snapsh defer sn.Remove(ctx, lowerKey) } - return md.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayerGzip, contentRef) + return d.DiffMounts(ctx, lower, upper, opts...) } diff --git a/services/diff/client.go b/services/diff/client.go index 81b9bf34a..5267f9707 100644 --- a/services/diff/client.go +++ b/services/diff/client.go @@ -3,20 +3,15 @@ package diff import ( diffapi "github.com/containerd/containerd/api/services/diff/v1" "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/mount" - "github.com/containerd/containerd/rootfs" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/net/context" ) -type DiffService interface { - rootfs.Applier - rootfs.MountDiffer -} - // NewApplierFromClient returns a new Applier which communicates // over a GRPC connection. -func NewDiffServiceFromClient(client diffapi.DiffClient) DiffService { +func NewDiffServiceFromClient(client diffapi.DiffClient) diff.Differ { return &remote{ client: client, } @@ -38,12 +33,19 @@ func (r *remote) Apply(ctx context.Context, diff ocispec.Descriptor, mounts []mo return toDescriptor(resp.Applied), nil } -func (r *remote) DiffMounts(ctx context.Context, a, b []mount.Mount, media, ref string) (ocispec.Descriptor, error) { +func (r *remote) DiffMounts(ctx context.Context, a, b []mount.Mount, opts ...diff.Opt) (ocispec.Descriptor, error) { + var config diff.Config + for _, opt := range opts { + if err := opt(&config); err != nil { + return ocispec.Descriptor{}, err + } + } req := &diffapi.DiffRequest{ Left: fromMounts(a), Right: fromMounts(b), - MediaType: media, - Ref: ref, + MediaType: config.MediaType, + Ref: config.Reference, + Labels: config.Labels, } resp, err := r.client.Diff(ctx, req) if err != nil { diff --git a/services/diff/service.go b/services/diff/service.go index 46491fa54..81e44dcf6 100644 --- a/services/diff/service.go +++ b/services/diff/service.go @@ -3,6 +3,7 @@ package diff import ( diffapi "github.com/containerd/containerd/api/services/diff/v1" "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/plugin" @@ -39,7 +40,7 @@ func init() { } orderedNames := ic.Config.(*config).Order - ordered := make([]plugin.Differ, len(orderedNames)) + ordered := make([]diff.Differ, len(orderedNames)) for i, n := range orderedNames { differp, ok := differs[n] if !ok { @@ -50,7 +51,7 @@ func init() { return nil, errors.Wrapf(err, "could not load required differ due plugin init error: %s", n) } - ordered[i] = differ.(plugin.Differ) + ordered[i] = differ.(diff.Differ) } return &service{ @@ -61,7 +62,7 @@ func init() { } type service struct { - differs []plugin.Differ + differs []diff.Differ } func (s *service) Register(gs *grpc.Server) error { @@ -102,8 +103,19 @@ func (s *service) Diff(ctx context.Context, dr *diffapi.DiffRequest) (*diffapi.D bMounts = toMounts(dr.Right) ) + var opts []diff.Opt + if dr.MediaType != "" { + opts = append(opts, diff.WithMediaType(dr.MediaType)) + } + if dr.Ref != "" { + opts = append(opts, diff.WithReference(dr.Ref)) + } + if dr.Labels != nil { + opts = append(opts, diff.WithLabels(dr.Labels)) + } + for _, differ := range s.differs { - ocidesc, err = differ.DiffMounts(ctx, aMounts, bMounts, dr.MediaType, dr.Ref) + ocidesc, err = differ.DiffMounts(ctx, aMounts, bMounts, opts...) if !errdefs.IsNotImplemented(err) { break } diff --git a/task.go b/task.go index d3636c925..440b0e504 100644 --- a/task.go +++ b/task.go @@ -15,6 +15,7 @@ import ( "github.com/containerd/containerd/api/services/tasks/v1" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/plugin" @@ -482,7 +483,7 @@ func (t *task) checkpointTask(ctx context.Context, index *v1.Index, request *tas } func (t *task) checkpointRWSnapshot(ctx context.Context, index *v1.Index, snapshotterName string, id string) error { - rw, err := rootfs.Diff(ctx, id, fmt.Sprintf("checkpoint-rw-%s", id), t.client.SnapshotService(snapshotterName), t.client.DiffService()) + rw, err := rootfs.Diff(ctx, id, t.client.SnapshotService(snapshotterName), t.client.DiffService(), diff.WithReference(fmt.Sprintf("checkpoint-rw-%s", id))) if err != nil { return err }