Merge pull request #9630 from AkihiroSuda/transfer-desc

transfer: add OCI descriptors to Progress structure

api/next.pb.txt

@@ -7231,6 +7231,7 @@ file {
 file {
   name: "github.com/containerd/containerd/api/types/transfer/progress.proto"
   package: "containerd.types.transfer"
+  dependency: "github.com/containerd/containerd/api/types/descriptor.proto"
   message_type {
     name: "Progress"
     field {
@@ -7268,6 +7269,14 @@ file {
       type: TYPE_INT64
       json_name: "total"
     }
+    field {
+      name: "desc"
+      number: 6
+      label: LABEL_OPTIONAL
+      type: TYPE_MESSAGE
+      type_name: ".containerd.types.Descriptor"
+      json_name: "desc"
+    }
   }
   options {
     go_package: "github.com/containerd/containerd/v2/api/types/transfer"

api/types/transfer/progress.pb.go

@@ -22,6 +22,7 @@
 package transfer
 
 import (
+	types "github.com/containerd/containerd/v2/api/types"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
@@ -40,11 +41,12 @@ type Progress struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Event    string   `protobuf:"bytes,1,opt,name=event,proto3" json:"event,omitempty"`
+	Event    string            `protobuf:"bytes,1,opt,name=event,proto3" json:"event,omitempty"`
-	Name     string   `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
+	Name     string            `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
-	Parents  []string `protobuf:"bytes,3,rep,name=parents,proto3" json:"parents,omitempty"`
+	Parents  []string          `protobuf:"bytes,3,rep,name=parents,proto3" json:"parents,omitempty"`
-	Progress int64    `protobuf:"varint,4,opt,name=progress,proto3" json:"progress,omitempty"`
+	Progress int64             `protobuf:"varint,4,opt,name=progress,proto3" json:"progress,omitempty"`
-	Total    int64    `protobuf:"varint,5,opt,name=total,proto3" json:"total,omitempty"`
+	Total    int64             `protobuf:"varint,5,opt,name=total,proto3" json:"total,omitempty"`
+	Desc     *types.Descriptor `protobuf:"bytes,6,opt,name=desc,proto3" json:"desc,omitempty"`
 }
 
 func (x *Progress) Reset() {
@@ -114,6 +116,13 @@ func (x *Progress) GetTotal() int64 {
 	return 0
 }
 
+func (x *Progress) GetDesc() *types.Descriptor {
+	if x != nil {
+		return x.Desc
+	}
+	return nil
+}
+
 var File_github_com_containerd_containerd_api_types_transfer_progress_proto protoreflect.FileDescriptor
 
 var file_github_com_containerd_containerd_api_types_transfer_progress_proto_rawDesc = []byte{
@@ -122,20 +131,27 @@ var file_github_com_containerd_containerd_api_types_transfer_progress_proto_rawD
 	0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61,
 	0x6e, 0x73, 0x66, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x70,
 	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x19, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64,
-	0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x22,
+	0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x1a,
-	0x80, 0x01, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05,
+	0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74,
-	0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x76, 0x65,
+	0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
-	0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x64, 0x65, 0x73, 0x63,
-	0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74,
+	0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb2, 0x01, 0x0a,
-	0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73,
+	0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x76, 0x65,
-	0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01,
+	0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x12,
-	0x28, 0x03, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05,
+	0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
-	0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74,
+	0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03,
-	0x61, 0x6c, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
+	0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1a, 0x0a,
-	0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74,
+	0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52,
-	0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79,
+	0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74,
-	0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72,
+	0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12,
-	0x6f, 0x74, 0x6f, 0x33,
+	0x30, 0x0a, 0x04, 0x64, 0x65, 0x73, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e,
+	0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73,
+	0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x04, 0x64, 0x65, 0x73,
+	0x63, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
+	0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61,
+	0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70,
+	0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
 }
 
 var (
@@ -152,14 +168,16 @@ func file_github_com_containerd_containerd_api_types_transfer_progress_proto_raw
 
 var file_github_com_containerd_containerd_api_types_transfer_progress_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
 var file_github_com_containerd_containerd_api_types_transfer_progress_proto_goTypes = []interface{}{
-	(*Progress)(nil), // 0: containerd.types.transfer.Progress
+	(*Progress)(nil),         // 0: containerd.types.transfer.Progress
+	(*types.Descriptor)(nil), // 1: containerd.types.Descriptor
 }
 var file_github_com_containerd_containerd_api_types_transfer_progress_proto_depIdxs = []int32{
-	0, // [0:0] is the sub-list for method output_type
+	1, // 0: containerd.types.transfer.Progress.desc:type_name -> containerd.types.Descriptor
-	0, // [0:0] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for method output_type
-	0, // [0:0] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for method input_type
-	0, // [0:0] is the sub-list for extension extendee
+	1, // [1:1] is the sub-list for extension type_name
-	0, // [0:0] is the sub-list for field type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
 }
 
 func init() { file_github_com_containerd_containerd_api_types_transfer_progress_proto_init() }

api/types/transfer/progress.proto

@@ -18,6 +18,8 @@ syntax = "proto3";
 
 package containerd.types.transfer;
 
+import "github.com/containerd/containerd/api/types/descriptor.proto";
+
 option go_package = "github.com/containerd/containerd/v2/api/types/transfer";
 
 message Progress {
@@ -26,4 +28,5 @@ message Progress {
 	repeated string parents = 3;
 	int64 progress = 4;
 	int64 total = 5;
+	containerd.types.Descriptor desc = 6;
 }

cmd/ctr/commands/images/pull.go

@@ -206,6 +206,18 @@ type progressNode struct {
 	root     bool
 }
 
+func (n *progressNode) mainDesc() *ocispec.Descriptor {
+	if n.Desc != nil {
+		return n.Desc
+	}
+	for _, c := range n.children {
+		if desc := c.mainDesc(); desc != nil {
+			return desc
+		}
+	}
+	return nil
+}
+
 // ProgressHandler continuously updates the output with job progress
 // by checking status in the content store.
 func ProgressHandler(ctx context.Context, out io.Writer) (transfer.ProgressFunc, func()) {
@@ -329,6 +341,11 @@ func ProgressHandler(ctx context.Context, out io.Writer) (transfer.ProgressFunc,
 
 func DisplayHierarchy(w io.Writer, status string, roots []*progressNode, start time.Time) {
 	total := displayNode(w, "", roots)
+	for _, r := range roots {
+		if desc := r.mainDesc(); desc != nil {
+			fmt.Fprintf(w, "%s %s\n", desc.MediaType, desc.Digest)
+		}
+	}
 	// Print the Status line
 	fmt.Fprintf(w, "%s\telapsed: %-4.1fs\ttotal: %7.6v\t(%v)\t\n",
 		status,

pkg/transfer/local/import.go

@@ -128,6 +128,7 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
 	}
 
 	for _, desc := range descriptors {
+		desc := desc
 		imgs, err := is.Store(ctx, desc, ts.images)
 		if err != nil {
 			if errdefs.IsNotFound(err) {
@@ -142,6 +143,7 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
 				tops.Progress(transfer.Progress{
 					Event: "saved",
 					Name:  img.Name,
+					Desc:  &desc,
 				})
 			}
 		}

pkg/transfer/local/progress.go

@@ -106,6 +106,7 @@ func (j *ProgressTracker) HandleProgress(ctx context.Context, pf transfer.Progre
 							Parents:  job.parents,
 							Progress: status.Offset,
 							Total:    status.Total,
+							Desc:     &job.desc,
 						})
 						job.progress = status.Offset
 						job.state = jobInProgress
@@ -122,6 +123,7 @@ func (j *ProgressTracker) HandleProgress(ctx context.Context, pf transfer.Progre
 							Parents:  job.parents,
 							Progress: job.desc.Size,
 							Total:    job.desc.Size,
+							Desc:     &job.desc,
 						})
 
 					}
@@ -165,6 +167,7 @@ func (j *ProgressTracker) HandleProgress(ctx context.Context, pf transfer.Progre
 					//Digest:   desc.Digest.String(),
 					Progress: 0,
 					Total:    update.desc.Size,
+					Desc:     &job.desc,
 				})
 			}
 			if update.exists {
@@ -173,6 +176,7 @@ func (j *ProgressTracker) HandleProgress(ctx context.Context, pf transfer.Progre
 					Name:     remotes.MakeRefKey(ctx, update.desc),
 					Progress: update.desc.Size,
 					Total:    update.desc.Size,
+					Desc:     &job.desc,
 				})
 				job.state = jobComplete
 				job.progress = job.desc.Size

pkg/transfer/local/push.go

@@ -63,6 +63,7 @@ func (ts *localTransferService) push(ctx context.Context, ig transfer.ImageGette
 			Event: "pushing content",
 			Name:  img.Name,
 			//Digest: img.Target.Digest.String(),
+			Desc: &img.Target,
 		})
 	}
 
@@ -109,9 +110,11 @@ func (ts *localTransferService) push(ctx context.Context, ig transfer.ImageGette
 			Event: "pushed content",
 			Name:  img.Name,
 			//Digest: img.Target.Digest.String(),
+			Desc: &img.Target,
 		})
 		tops.Progress(transfer.Progress{
 			Event: fmt.Sprintf("Completed push to %s", p),
+			Desc:  &img.Target,
 		})
 	}
 

pkg/transfer/proxy/transfer.go

@@ -25,11 +25,13 @@ import (
 
 	transferapi "github.com/containerd/containerd/v2/api/services/transfer/v1"
 	transfertypes "github.com/containerd/containerd/v2/api/types/transfer"
+	"github.com/containerd/containerd/v2/oci"
 	"github.com/containerd/containerd/v2/pkg/streaming"
 	"github.com/containerd/containerd/v2/pkg/transfer"
 	tstreaming "github.com/containerd/containerd/v2/pkg/transfer/streaming"
 	"github.com/containerd/log"
 	"github.com/containerd/typeurl/v2"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 type proxyTransferrer struct {
@@ -74,12 +76,18 @@ func (p *proxyTransferrer) Transfer(ctx context.Context, src interface{}, dst in
 				}
 				switch v := i.(type) {
 				case *transfertypes.Progress:
+					var descp *ocispec.Descriptor
+					if v.Desc != nil {
+						desc := oci.DescriptorFromProto(v.Desc)
+						descp = &desc
+					}
 					o.Progress(transfer.Progress{
 						Event:    v.Event,
 						Name:     v.Name,
 						Parents:  v.Parents,
 						Progress: v.Progress,
 						Total:    v.Total,
+						Desc:     descp,
 					})
 				default:
 					log.G(ctx).Warnf("unhandled progress object %T: %v", i, a.GetTypeUrl())

pkg/transfer/transfer.go

@@ -132,5 +132,5 @@ type Progress struct {
 	Parents  []string
 	Progress int64
 	Total    int64
-	// Descriptor?
+	Desc     *ocispec.Descriptor // since containerd v2.0
 }

services/transfer/service.go

@@ -20,8 +20,10 @@ import (
 	"context"
 
 	transferapi "github.com/containerd/containerd/v2/api/services/transfer/v1"
+	"github.com/containerd/containerd/v2/api/types"
 	transferTypes "github.com/containerd/containerd/v2/api/types/transfer"
 	"github.com/containerd/containerd/v2/errdefs"
+	"github.com/containerd/containerd/v2/oci"
 	"github.com/containerd/containerd/v2/pkg/streaming"
 	"github.com/containerd/containerd/v2/pkg/transfer"
 	tplugins "github.com/containerd/containerd/v2/pkg/transfer/plugins"
@@ -92,12 +94,17 @@ func (s *service) Transfer(ctx context.Context, req *transferapi.TransferRequest
 			defer stream.Close()
 
 			pf := func(p transfer.Progress) {
+				var descp *types.Descriptor
+				if p.Desc != nil {
+					descp = oci.DescriptorToProto(*p.Desc)
+				}
 				progress, err := typeurl.MarshalAny(&transferTypes.Progress{
 					Event:    p.Event,
 					Name:     p.Name,
 					Parents:  p.Parents,
 					Progress: p.Progress,
 					Total:    p.Total,
+					Desc:     descp,
 				})
 				if err != nil {
 					log.G(ctx).WithError(err).Warnf("event could not be marshaled: %v/%v", p.Event, p.Name)