github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #9630 from AkihiroSuda/transfer-desc

Browse Source 
transfer: add OCI descriptors to Progress structure
This commit is contained in:
Fu Wei 2024-01-12 02:40:13 +00:00 committed by GitHub
commit 0a7c192ca9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 97 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/next.pb.txt
View File

@ -7231,6 +7231,7 @@ file {
file {
name: "github.com/containerd/containerd/api/types/transfer/progress.proto"
package: "containerd.types.transfer"
dependency: "github.com/containerd/containerd/api/types/descriptor.proto"
message_type {
name: "Progress"
field {
@ -7268,6 +7269,14 @@ file {
type: TYPE_INT64
json_name: "total"
}
field {
name: "desc"
number: 6
label: LABEL_OPTIONAL
type: TYPE_MESSAGE
type_name: ".containerd.types.Descriptor"
json_name: "desc"
}
}
options {
go_package: "github.com/containerd/containerd/v2/api/types/transfer"

68
api/types/transfer/progress.pb.go
View File

@ -22,6 +22,7 @@
package transfer
import (
types "github.com/containerd/containerd/v2/api/types"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
@ -40,11 +41,12 @@ type Progress struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Event string `protobuf:"bytes,1,opt,name=event,proto3" json:"event,omitempty"`
Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
Parents []string `protobuf:"bytes,3,rep,name=parents,proto3" json:"parents,omitempty"`
Progress int64 `protobuf:"varint,4,opt,name=progress,proto3" json:"progress,omitempty"`
Total int64 `protobuf:"varint,5,opt,name=total,proto3" json:"total,omitempty"`
Event string `protobuf:"bytes,1,opt,name=event,proto3" json:"event,omitempty"`
Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
Parents []string `protobuf:"bytes,3,rep,name=parents,proto3" json:"parents,omitempty"`
Progress int64 `protobuf:"varint,4,opt,name=progress,proto3" json:"progress,omitempty"`
Total int64 `protobuf:"varint,5,opt,name=total,proto3" json:"total,omitempty"`
Desc *types.Descriptor `protobuf:"bytes,6,opt,name=desc,proto3" json:"desc,omitempty"`
}
func (x *Progress) Reset() {
@ -114,6 +116,13 @@ func (x *Progress) GetTotal() int64 {
return 0
}
func (x *Progress) GetDesc() *types.Descriptor {
if x != nil {
return x.Desc
}
return nil
}
var File_github_com_containerd_containerd_api_types_transfer_progress_proto protoreflect.FileDescriptor
var file_github_com_containerd_containerd_api_types_transfer_progress_proto_rawDesc = []byte{
@ -122,20 +131,27 @@ var file_github_com_containerd_containerd_api_types_transfer_progress_proto_rawD
0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61,
0x6e, 0x73, 0x66, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x12, 0x19, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64,
0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x22,
0x80, 0x01, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05,
0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x76, 0x65,
0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74,
0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73,
0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01,
0x28, 0x03, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05,
0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74,
0x61, 0x6c, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74,
0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79,
0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x1a,
0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74,
0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x64, 0x65, 0x73, 0x63,
0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb2, 0x01, 0x0a,
0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x76, 0x65,
0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12,
0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03,
0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1a, 0x0a,
0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52,
0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74,
0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12,
0x30, 0x0a, 0x04, 0x64, 0x65, 0x73, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e,
0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73,
0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x04, 0x64, 0x65, 0x73,
0x63, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61,
0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70,
0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
}
var (
@ -152,14 +168,16 @@ func file_github_com_containerd_containerd_api_types_transfer_progress_proto_raw
var file_github_com_containerd_containerd_api_types_transfer_progress_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_github_com_containerd_containerd_api_types_transfer_progress_proto_goTypes = []interface{}{
(*Progress)(nil), // 0: containerd.types.transfer.Progress
(*Progress)(nil), // 0: containerd.types.transfer.Progress
(*types.Descriptor)(nil), // 1: containerd.types.Descriptor
}
var file_github_com_containerd_containerd_api_types_transfer_progress_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
1, // 0: containerd.types.transfer.Progress.desc:type_name -> containerd.types.Descriptor
1, // [1:1] is the sub-list for method output_type
1, // [1:1] is the sub-list for method input_type
1, // [1:1] is the sub-list for extension type_name
1, // [1:1] is the sub-list for extension extendee
0, // [0:1] is the sub-list for field type_name
}
func init() { file_github_com_containerd_containerd_api_types_transfer_progress_proto_init() }

3
api/types/transfer/progress.proto
View File

@ -18,6 +18,8 @@ syntax = "proto3";
package containerd.types.transfer;
import "github.com/containerd/containerd/api/types/descriptor.proto";
option go_package = "github.com/containerd/containerd/v2/api/types/transfer";
message Progress {
@ -26,4 +28,5 @@ message Progress {
repeated string parents = 3;
int64 progress = 4;
int64 total = 5;
containerd.types.Descriptor desc = 6;
}

17
cmd/ctr/commands/images/pull.go
View File

@ -206,6 +206,18 @@ type progressNode struct {
root bool
}
func (n *progressNode) mainDesc() *ocispec.Descriptor {
if n.Desc != nil {
return n.Desc
}
for _, c := range n.children {
if desc := c.mainDesc(); desc != nil {
return desc
}
}
return nil
}
// ProgressHandler continuously updates the output with job progress
// by checking status in the content store.
func ProgressHandler(ctx context.Context, out io.Writer) (transfer.ProgressFunc, func()) {
@ -329,6 +341,11 @@ func ProgressHandler(ctx context.Context, out io.Writer) (transfer.ProgressFunc,
func DisplayHierarchy(w io.Writer, status string, roots []*progressNode, start time.Time) {
total := displayNode(w, "", roots)
for _, r := range roots {
if desc := r.mainDesc(); desc != nil {
fmt.Fprintf(w, "%s %s\n", desc.MediaType, desc.Digest)
}
}
// Print the Status line
fmt.Fprintf(w, "%s\telapsed: %-4.1fs\ttotal: %7.6v\t(%v)\t\n",
status,

2
pkg/transfer/local/import.go
View File

@ -128,6 +128,7 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
}
for _, desc := range descriptors {
desc := desc
imgs, err := is.Store(ctx, desc, ts.images)
if err != nil {
if errdefs.IsNotFound(err) {
@ -142,6 +143,7 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
tops.Progress(transfer.Progress{
Event: "saved",
Name: img.Name,
Desc: &desc,
})
}
}

4
pkg/transfer/local/progress.go
View File

@ -106,6 +106,7 @@ func (j *ProgressTracker) HandleProgress(ctx context.Context, pf transfer.Progre
Parents: job.parents,
Progress: status.Offset,
Total: status.Total,
Desc: &job.desc,
})
job.progress = status.Offset
job.state = jobInProgress
@ -122,6 +123,7 @@ func (j *ProgressTracker) HandleProgress(ctx context.Context, pf transfer.Progre
Parents: job.parents,
Progress: job.desc.Size,
Total: job.desc.Size,
Desc: &job.desc,
})
}
@ -165,6 +167,7 @@ func (j *ProgressTracker) HandleProgress(ctx context.Context, pf transfer.Progre
//Digest: desc.Digest.String(),
Progress: 0,
Total: update.desc.Size,
Desc: &job.desc,
})
}
if update.exists {
@ -173,6 +176,7 @@ func (j *ProgressTracker) HandleProgress(ctx context.Context, pf transfer.Progre
Name: remotes.MakeRefKey(ctx, update.desc),
Progress: update.desc.Size,
Total: update.desc.Size,
Desc: &job.desc,
})
job.state = jobComplete
job.progress = job.desc.Size

3
pkg/transfer/local/push.go
View File

@ -63,6 +63,7 @@ func (ts *localTransferService) push(ctx context.Context, ig transfer.ImageGette
Event: "pushing content",
Name: img.Name,
//Digest: img.Target.Digest.String(),
Desc: &img.Target,
})
}
@ -109,9 +110,11 @@ func (ts *localTransferService) push(ctx context.Context, ig transfer.ImageGette
Event: "pushed content",
Name: img.Name,
//Digest: img.Target.Digest.String(),
Desc: &img.Target,
})
tops.Progress(transfer.Progress{
Event: fmt.Sprintf("Completed push to %s", p),
Desc: &img.Target,
})
}

8
pkg/transfer/proxy/transfer.go
View File

@ -25,11 +25,13 @@ import (
transferapi "github.com/containerd/containerd/v2/api/services/transfer/v1"
transfertypes "github.com/containerd/containerd/v2/api/types/transfer"
"github.com/containerd/containerd/v2/oci"
"github.com/containerd/containerd/v2/pkg/streaming"
"github.com/containerd/containerd/v2/pkg/transfer"
tstreaming "github.com/containerd/containerd/v2/pkg/transfer/streaming"
"github.com/containerd/log"
"github.com/containerd/typeurl/v2"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type proxyTransferrer struct {
@ -74,12 +76,18 @@ func (p *proxyTransferrer) Transfer(ctx context.Context, src interface{}, dst in
}
switch v := i.(type) {
case *transfertypes.Progress:
var descp *ocispec.Descriptor
if v.Desc != nil {
desc := oci.DescriptorFromProto(v.Desc)
descp = &desc
}
o.Progress(transfer.Progress{
Event: v.Event,
Name: v.Name,
Parents: v.Parents,
Progress: v.Progress,
Total: v.Total,
Desc: descp,
})
default:
log.G(ctx).Warnf("unhandled progress object %T: %v", i, a.GetTypeUrl())

2
pkg/transfer/transfer.go
View File

@ -132,5 +132,5 @@ type Progress struct {
Parents []string
Progress int64
Total int64
// Descriptor?
Desc *ocispec.Descriptor // since containerd v2.0
}

7
services/transfer/service.go
View File

@ -20,8 +20,10 @@ import (
"context"
transferapi "github.com/containerd/containerd/v2/api/services/transfer/v1"
"github.com/containerd/containerd/v2/api/types"
transferTypes "github.com/containerd/containerd/v2/api/types/transfer"
"github.com/containerd/containerd/v2/errdefs"
"github.com/containerd/containerd/v2/oci"
"github.com/containerd/containerd/v2/pkg/streaming"
"github.com/containerd/containerd/v2/pkg/transfer"
tplugins "github.com/containerd/containerd/v2/pkg/transfer/plugins"
@ -92,12 +94,17 @@ func (s *service) Transfer(ctx context.Context, req *transferapi.TransferRequest
defer stream.Close()
pf := func(p transfer.Progress) {
var descp *types.Descriptor
if p.Desc != nil {
descp = oci.DescriptorToProto(*p.Desc)
}
progress, err := typeurl.MarshalAny(&transferTypes.Progress{
Event: p.Event,
Name: p.Name,
Parents: p.Parents,
Progress: p.Progress,
Total: p.Total,
Desc: descp,
})
if err != nil {
log.G(ctx).WithError(err).Warnf("event could not be marshaled: %v/%v", p.Event, p.Name)