Transfer: Registry: Enable plain HTTP

Currenlty transfer service doesn't handle plain HTTP connection.
This commit fixes this issue by propagating
`(core/remotes/docker/config).HostOptions.DefaultScheme` from client to the
transfer service.
This commit also fixes ctr to use this feature for "--plain-http" flag.

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

api/next.pb.txt

@@ -7468,6 +7468,13 @@ file {
       type: TYPE_STRING
       json_name: "hostDir"
     }
+    field {
+      name: "default_scheme"
+      number: 4
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "defaultScheme"
+    }
     nested_type {
       name: "HeadersEntry"
       field {

api/types/transfer/registry.pb.go

@@ -159,8 +159,9 @@ type RegistryResolver struct {
 	// made on.
 	AuthStream string `protobuf:"bytes,1,opt,name=auth_stream,json=authStream,proto3" json:"auth_stream,omitempty"`
 	// Headers
-	Headers map[string]string `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	Headers       map[string]string `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	HostDir string            `protobuf:"bytes,3,opt,name=host_dir,json=hostDir,proto3" json:"host_dir,omitempty"`
+	HostDir       string            `protobuf:"bytes,3,opt,name=host_dir,json=hostDir,proto3" json:"host_dir,omitempty"`
+	DefaultScheme string            `protobuf:"bytes,4,opt,name=default_scheme,json=defaultScheme,proto3" json:"default_scheme,omitempty"`
 }
 
 func (x *RegistryResolver) Reset() {
@@ -216,6 +217,13 @@ func (x *RegistryResolver) GetHostDir() string {
 	return ""
 }
 
+func (x *RegistryResolver) GetDefaultScheme() string {
+	if x != nil {
+		return x.DefaultScheme
+	}
+	return ""
+}
+
 // AuthRequest is sent as a callback on a stream
 type AuthRequest struct {
 	state         protoimpl.MessageState
@@ -372,7 +380,7 @@ var file_github_com_containerd_containerd_api_types_transfer_registry_proto_rawD
 	0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70,
 	0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x67, 0x69,
 	0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x52, 0x08, 0x72, 0x65,
-	0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x22, 0xde, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x67, 0x69, 0x73,
+	0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x22, 0x85, 0x02, 0x0a, 0x10, 0x52, 0x65, 0x67, 0x69, 0x73,
 	0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x61,
 	0x75, 0x74, 0x68, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
 	0x52, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x52, 0x0a, 0x07,
@@ -382,38 +390,40 @@ var file_github_com_containerd_containerd_api_types_transfer_registry_proto_rawD
 	0x72, 0x79, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65,
 	0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73,
 	0x12, 0x19, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x03, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x07, 0x68, 0x6f, 0x73, 0x74, 0x44, 0x69, 0x72, 0x1a, 0x3a, 0x0a, 0x0c, 0x48,
+	0x28, 0x09, 0x52, 0x07, 0x68, 0x6f, 0x73, 0x74, 0x44, 0x69, 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x64,
-	0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
+	0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65, 0x18, 0x04, 0x20,
-	0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a,
+	0x01, 0x28, 0x09, 0x52, 0x0d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61,
+	0x6d, 0x65, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74,
-	0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x69, 0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x52,
+	0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01,
+	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x69,
-	0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72,
+	0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a,
-	0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x77, 0x77, 0x77, 0x61,
+	0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73,
-	0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28,
+	0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x02,
-	0x09, 0x52, 0x0f, 0x77, 0x77, 0x77, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12,
-	0x74, 0x65, 0x22, 0xbc, 0x01, 0x0a, 0x0c, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x28, 0x0a, 0x0f, 0x77, 0x77, 0x77, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
-	0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x08, 0x61, 0x75, 0x74, 0x68, 0x54, 0x79, 0x70, 0x65, 0x18,
+	0x74, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x77, 0x77, 0x61, 0x75, 0x74,
-	0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65,
+	0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x22, 0xbc, 0x01, 0x0a, 0x0c, 0x41, 0x75,
-	0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65,
+	0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x08, 0x61, 0x75,
-	0x72, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x61, 0x75, 0x74, 0x68,
+	0x74, 0x68, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x63,
-	0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02,
+	0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x1a, 0x0a, 0x08,
+	0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x54, 0x79, 0x70,
-	0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,
+	0x65, 0x52, 0x08, 0x61, 0x75, 0x74, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73,
-	0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x70, 0x69,
+	0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63,
-	0x72, 0x65, 0x5f, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
+	0x72, 0x65, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18,
-	0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69,
+	0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12,
-	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41,
+	0x37, 0x0a, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x5f, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01,
-	0x74, 0x2a, 0x3e, 0x0a, 0x08, 0x41, 0x75, 0x74, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a,
+	0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
-	0x04, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x52, 0x45, 0x44, 0x45,
+	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08,
-	0x4e, 0x54, 0x49, 0x41, 0x4c, 0x53, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x46, 0x52,
+	0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x2a, 0x3e, 0x0a, 0x08, 0x41, 0x75, 0x74, 0x68,
-	0x45, 0x53, 0x48, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x0f,
-	0x03, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
+	0x0a, 0x0b, 0x43, 0x52, 0x45, 0x44, 0x45, 0x4e, 0x54, 0x49, 0x41, 0x4c, 0x53, 0x10, 0x01, 0x12,
-	0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61,
+	0x0b, 0x0a, 0x07, 0x52, 0x45, 0x46, 0x52, 0x45, 0x53, 0x48, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06,
-	0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70,
+	0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x03, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68,
-	0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
-	0x74, 0x6f, 0x33,
+	0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f,
+	0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66,
+	0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

api/types/transfer/registry.proto

@@ -36,8 +36,9 @@ message RegistryResolver {
 	map<string, string> headers = 2;
 
 	string host_dir = 3;
+
+	string default_scheme = 4;
 	// Force skip verify
-	// Force HTTP
 	// CA callback? Client TLS callback?
 }
 

cmd/ctr/commands/images/pull.go

@@ -132,7 +132,11 @@ command. As part of this process, we do the following:
 				sopts = append(sopts, image.WithAllMetadata)
 			}
 
-			reg, err := registry.NewOCIRegistry(ctx, ref, registry.WithCredentials(ch), registry.WithHostDir(context.String("hosts-dir")))
+			opts := []registry.Opt{registry.WithCredentials(ch), registry.WithHostDir(context.String("hosts-dir"))}
+			if context.Bool("plain-http") {
+				opts = append(opts, registry.WithDefaultScheme("http"))
+			}
+			reg, err := registry.NewOCIRegistry(ctx, ref, opts...)
 			if err != nil {
 				return err
 			}

cmd/ctr/commands/images/push.go

@@ -104,7 +104,11 @@ var pushCommand = &cli.Command{
 			if local == "" {
 				local = ref
 			}
-			reg, err := registry.NewOCIRegistry(ctx, ref, registry.WithCredentials(ch), registry.WithHostDir(context.String("hosts-dir")))
+			opts := []registry.Opt{registry.WithCredentials(ch), registry.WithHostDir(context.String("hosts-dir"))}
+			if context.Bool("plain-http") {
+				opts = append(opts, registry.WithDefaultScheme("http"))
+			}
+			reg, err := registry.NewOCIRegistry(ctx, ref, opts...)
 			if err != nil {
 				return err
 			}

core/transfer/registry/registry.go

@@ -44,9 +44,10 @@ func init() {
 }
 
 type registryOpts struct {
-	headers http.Header
+	headers       http.Header
-	creds   CredentialHelper
+	creds         CredentialHelper
-	hostDir string
+	hostDir       string
+	defaultScheme string
 }
 
 // Opt sets registry-related configurations.
@@ -76,6 +77,14 @@ func WithHostDir(hostDir string) Opt {
 	}
 }
 
+// WithDefaultScheme specifies the default scheme for registry configuration
+func WithDefaultScheme(s string) Opt {
+	return func(o *registryOpts) error {
+		o.defaultScheme = s
+		return nil
+	}
+}
+
 // NewOCIRegistry initializes with hosts, authorizer callback, and headers
 func NewOCIRegistry(ctx context.Context, ref string, opts ...Opt) (*OCIRegistry, error) {
 	var ropts registryOpts
@@ -99,16 +108,20 @@ func NewOCIRegistry(ctx context.Context, ref string, opts ...Opt) (*OCIRegistry,
 			return c.Username, c.Secret, nil
 		}
 	}
+	if ropts.defaultScheme != "" {
+		hostOptions.DefaultScheme = ropts.defaultScheme
+	}
 	resolver := docker.NewResolver(docker.ResolverOptions{
 		Hosts:   config.ConfigureHosts(ctx, hostOptions),
 		Headers: ropts.headers,
 	})
 	return &OCIRegistry{
-		reference: ref,
+		reference:     ref,
-		headers:   ropts.headers,
+		headers:       ropts.headers,
-		creds:     ropts.creds,
+		creds:         ropts.creds,
-		resolver:  resolver,
+		resolver:      resolver,
-		hostDir:   ropts.hostDir,
+		hostDir:       ropts.hostDir,
+		defaultScheme: ropts.defaultScheme,
 	}, nil
 }
 
@@ -135,6 +148,8 @@ type OCIRegistry struct {
 
 	hostDir string
 
+	defaultScheme string
+
 	// This could be an interface which returns resolver?
 	// Resolver could also be a plug-able interface, to call out to a program to fetch?
 }
@@ -234,6 +249,7 @@ func (r *OCIRegistry) MarshalAny(ctx context.Context, sm streaming.StreamCreator
 		res.AuthStream = sid
 	}
 	res.HostDir = r.hostDir
+	res.DefaultScheme = r.defaultScheme
 	s := &transfertypes.OCIRegistry{
 		Reference: r.reference,
 		Resolver:  res,
@@ -253,6 +269,9 @@ func (r *OCIRegistry) UnmarshalAny(ctx context.Context, sm streaming.StreamGette
 		if s.Resolver.HostDir != "" {
 			hostOptions.HostDir = config.HostDirFromRoot(s.Resolver.HostDir)
 		}
+		if s.Resolver.DefaultScheme != "" {
+			hostOptions.DefaultScheme = s.Resolver.DefaultScheme
+		}
 		if sid := s.Resolver.AuthStream; sid != "" {
 			stream, err := sm.Get(ctx, sid)
 			if err != nil {