github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Update transfer packages

Browse Source 
Signed-off-by: Derek McGowan <derek@mcg.dev>
This commit is contained in:
Derek McGowan
2022-08-19 15:34:08 -07:00
parent 2a8d7a744b
commit 6b5df1ee16
13 changed files with 1007 additions and 890 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
pkg/transfer/image/imagestore.go Normal file
View File

@@ -0,0 +1,140 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package image
import (
"context"
"fmt"
transfertypes "github.com/containerd/containerd/api/types/transfer"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/pkg/streaming"
"github.com/containerd/containerd/pkg/transfer/plugins"
"github.com/containerd/containerd/pkg/unpack"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/containerd/typeurl"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
func init() {
// TODO: Move this to seperate package?
plugins.Register(&transfertypes.ImageStore{}, &ImageStore{}) // TODO: Rename ImageStoreDestination
}
type ImageStore struct {
// TODO: Put these configurations in object which can convert to/from any
// Embed generated type
imageName string
imageLabels map[string]string
platforms platforms.MatchComparer
allMetadata bool
labelMap func(ocispec.Descriptor) []string
manifestLimit int
// TODO: Convert these to unpack platforms
unpacks []unpack.Platform
}
func NewImageStore(image string) *ImageStore {
return &ImageStore{
imageName: image,
}
}
func (is *ImageStore) String() string {
return fmt.Sprintf("Local Image Store (%s)", is.imageName)
}
func (is *ImageStore) FilterHandler(h images.HandlerFunc, cs content.Store) images.HandlerFunc {
h = images.SetChildrenMappedLabels(cs, h, is.labelMap)
if is.allMetadata {
// Filter manifests by platforms but allow to handle manifest
// and configuration for not-target platforms
h = remotes.FilterManifestByPlatformHandler(h, is.platforms)
} else {
// Filter children by platforms if specified.
h = images.FilterPlatforms(h, is.platforms)
}
// Sort and limit manifests if a finite number is needed
if is.manifestLimit > 0 {
h = images.LimitManifests(h, is.platforms, is.manifestLimit)
}
return h
}
func (is *ImageStore) Store(ctx context.Context, desc ocispec.Descriptor, store images.Store) (images.Image, error) {
img := images.Image{
Name: is.imageName,
Target: desc,
Labels: is.imageLabels,
}
for {
if created, err := store.Create(ctx, img); err != nil {
if !errdefs.IsAlreadyExists(err) {
return images.Image{}, err
}
updated, err := store.Update(ctx, img)
if err != nil {
// if image was removed, try create again
if errdefs.IsNotFound(err) {
continue
}
return images.Image{}, err
}
img = updated
} else {
img = created
}
return img, nil
}
}
func (is *ImageStore) Get(ctx context.Context, store images.Store) (images.Image, error) {
return store.Get(ctx, is.imageName)
}
func (is *ImageStore) UnpackPlatforms() []unpack.Platform {
return is.unpacks
}
func (is *ImageStore) MarshalAny(ctx context.Context, sm streaming.StreamCreator) (typeurl.Any, error) {
s := &transfertypes.ImageStore{
Name: is.imageName,
// TODO: Support other fields
}
return typeurl.MarshalAny(s)
}
func (is *ImageStore) UnmarshalAny(ctx context.Context, sm streaming.StreamGetter, a typeurl.Any) error {
var s transfertypes.ImageStore
if err := typeurl.UnmarshalTo(a, &s); err != nil {
return err
}
is.imageName = s.Name
// TODO: Support other fields
return nil
}

322
pkg/transfer/image/local.go
View File

@@ -1,322 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package transfer
import (
"context"
"fmt"
"net/http"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/pkg/streaming"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/unpack"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/typeurl"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
// TODO: Should a factory be exposed here as a service??
/*
func NewOCIRegistryFromProto(p *transferapi.OCIRegistry, resolver remotes.Resolver, sm streaming.StreamManager) *OCIRegistry {
//transfer.OCIRegistry
// Create resolver
// Convert auth stream to credential manager
return &OCIRegistry{
reference: p.Reference,
resolver: resolver,
}
}
*/
// Initialize with hosts, authorizer callback, and headers
func NewOCIRegistry(ref string, headers http.Header, creds CredentialHelper) *OCIRegistry {
// Create an authorizer
var ropts []docker.RegistryOpt
if creds != nil {
// TODO: Support bearer
authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(func(host string) (string, string, error) {
c, err := creds.GetCredentials(context.Background(), ref, host)
if err != nil {
return "", "", err
}
return c.Username, c.Secret, nil
}))
ropts = append(ropts, docker.WithAuthorizer(authorizer))
}
// TODO: Apply local configuration, maybe dynamically create resolver when requested
resolver := docker.NewResolver(docker.ResolverOptions{
Hosts: docker.ConfigureDefaultRegistries(ropts...),
Headers: headers,
})
return &OCIRegistry{
reference: ref,
headers: headers,
creds: creds,
resolver: resolver,
}
}
// From stream
type CredentialHelper interface {
GetCredentials(ctx context.Context, ref, host string) (Credentials, error)
}
type Credentials struct {
Host string
Username string
Secret string
Bearer string
}
// OCI
type OCIRegistry struct {
reference string
headers http.Header
creds CredentialHelper
resolver remotes.Resolver
// This could be an interface which returns resolver?
// Resolver could also be a plug-able interface, to call out to a program to fetch?
}
func (r *OCIRegistry) String() string {
return fmt.Sprintf("OCI Registry (%s)", r.reference)
}
func (r *OCIRegistry) Image() string {
return r.reference
}
func (r *OCIRegistry) Resolve(ctx context.Context) (name string, desc ocispec.Descriptor, err error) {
return r.resolver.Resolve(ctx, r.reference)
}
func (r *OCIRegistry) Fetcher(ctx context.Context, ref string) (transfer.Fetcher, error) {
return r.resolver.Fetcher(ctx, ref)
}
func (r *OCIRegistry) MarshalAny(ctx context.Context, sm streaming.StreamManager) (typeurl.Any, error) {
if r.creds != nil {
// TODO: Unique stream ID
stream, err := sm.Get(ctx, "")
if err != nil {
return nil, err
}
go func() {
// Check for context cancellation as well
for {
select {
case <-ctx.Done():
return
default:
}
_, err := stream.Recv()
if err != nil {
// If not EOF, log error
return
}
// If closed, return
// Call creds helper
// Send response
}
}()
// link creds to stream
}
// Create API OCI Registry type
// Marshal and return
return nil, nil
}
type ImageStore struct {
// TODO: Put these configurations in object which can convert to/from any
// Embed generated type
imageName string
imageLabels map[string]string
platforms platforms.MatchComparer
allMetadata bool
labelMap func(ocispec.Descriptor) []string
manifestLimit int
images images.Store
content content.Store
// TODO: Convert these to unpack platforms
unpacks []unpack.Platform
}
func NewImageStore(image string, cs content.Store, is images.Store) *ImageStore {
return &ImageStore{
imageName: image,
images: is,
content: cs,
}
}
func (is *ImageStore) String() string {
return fmt.Sprintf("Local Image Store (%s)", is.imageName)
}
func (is *ImageStore) FilterHandler(h images.HandlerFunc) images.HandlerFunc {
h = images.SetChildrenMappedLabels(is.content, h, is.labelMap)
if is.allMetadata {
// Filter manifests by platforms but allow to handle manifest
// and configuration for not-target platforms
h = remotes.FilterManifestByPlatformHandler(h, is.platforms)
} else {
// Filter children by platforms if specified.
h = images.FilterPlatforms(h, is.platforms)
}
// Sort and limit manifests if a finite number is needed
if is.manifestLimit > 0 {
h = images.LimitManifests(h, is.platforms, is.manifestLimit)
}
return h
}
func (is *ImageStore) Store(ctx context.Context, desc ocispec.Descriptor) (images.Image, error) {
img := images.Image{
Name: is.imageName,
Target: desc,
Labels: is.imageLabels,
}
for {
if created, err := is.images.Create(ctx, img); err != nil {
if !errdefs.IsAlreadyExists(err) {
return images.Image{}, err
}
updated, err := is.images.Update(ctx, img)
if err != nil {
// if image was removed, try create again
if errdefs.IsNotFound(err) {
continue
}
return images.Image{}, err
}
img = updated
} else {
img = created
}
return img, nil
}
}
func (is *ImageStore) UnpackPlatforms() []unpack.Platform {
return is.unpacks
}
/*
type RemoteContext struct {
// Resolver is used to resolve names to objects, fetchers, and pushers.
// If no resolver is provided, defaults to Docker registry resolver.
Resolver remotes.Resolver
// PlatformMatcher is used to match the platforms for an image
// operation and define the preference when a single match is required
// from multiple platforms.
PlatformMatcher platforms.MatchComparer
// Unpack is done after an image is pulled to extract into a snapshotter.
// It is done simultaneously for schema 2 images when they are pulled.
// If an image is not unpacked on pull, it can be unpacked any time
// afterwards. Unpacking is required to run an image.
Unpack bool
// UnpackOpts handles options to the unpack call.
UnpackOpts []UnpackOpt
// Snapshotter used for unpacking
Snapshotter string
// SnapshotterOpts are additional options to be passed to a snapshotter during pull
SnapshotterOpts []snapshots.Opt
// Labels to be applied to the created image
Labels map[string]string
// BaseHandlers are a set of handlers which get are called on dispatch.
// These handlers always get called before any operation specific
// handlers.
BaseHandlers []images.Handler
// HandlerWrapper wraps the handler which gets sent to dispatch.
// Unlike BaseHandlers, this can run before and after the built
// in handlers, allowing operations to run on the descriptor
// after it has completed transferring.
HandlerWrapper func(images.Handler) images.Handler
// Platforms defines which platforms to handle when doing the image operation.
// Platforms is ignored when a PlatformMatcher is set, otherwise the
// platforms will be used to create a PlatformMatcher with no ordering
// preference.
Platforms []string
// MaxConcurrentDownloads is the max concurrent content downloads for each pull.
MaxConcurrentDownloads int
// MaxConcurrentUploadedLayers is the max concurrent uploaded layers for each push.
MaxConcurrentUploadedLayers int
// AllMetadata downloads all manifests and known-configuration files
AllMetadata bool
// ChildLabelMap sets the labels used to reference child objects in the content
// store. By default, all GC reference labels will be set for all fetched content.
ChildLabelMap func(ocispec.Descriptor) []string
}
*/
/*
// What should streamhandler look like?
type StreamHandler interface {
Authorize() error
Progress(key string, int64)
}
// Distribution options
// Stream handler
// Progress rate
// Unpack options
// Remote options
// Cases:
// Registry -> Content/ImageStore (pull)
// Registry -> Registry
// Content/ImageStore -> Registry (push)
// Content/ImageStore -> Content/ImageStore (tag)
// Common fetch/push interface for registry, content/imagestore, OCI index
// Always starts with string for source and destination, on client side, does not need to resolve
// Higher level implementation just takes strings and options
// Lower level implementation takes pusher/fetcher?
*/

290
pkg/transfer/image/registry.go Normal file
View File

@@ -0,0 +1,290 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package image
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
transfertypes "github.com/containerd/containerd/api/types/transfer"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/streaming"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/transfer/plugins"
tstreaming "github.com/containerd/containerd/pkg/transfer/streaming"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/typeurl"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
func init() {
// TODO: Move this to seperate package?
plugins.Register(&transfertypes.OCIRegistry{}, &OCIRegistry{})
}
// Initialize with hosts, authorizer callback, and headers
func NewOCIRegistry(ref string, headers http.Header, creds CredentialHelper) *OCIRegistry {
// Create an authorizer
var ropts []docker.RegistryOpt
if creds != nil {
// TODO: Support bearer
authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(func(host string) (string, string, error) {
c, err := creds.GetCredentials(context.Background(), ref, host)
if err != nil {
return "", "", err
}
return c.Username, c.Secret, nil
}))
ropts = append(ropts, docker.WithAuthorizer(authorizer))
}
// TODO: Apply local configuration, maybe dynamically create resolver when requested
resolver := docker.NewResolver(docker.ResolverOptions{
Hosts: docker.ConfigureDefaultRegistries(ropts...),
Headers: headers,
})
return &OCIRegistry{
reference: ref,
headers: headers,
creds: creds,
resolver: resolver,
}
}
// From stream
type CredentialHelper interface {
GetCredentials(ctx context.Context, ref, host string) (Credentials, error)
}
type Credentials struct {
Host string
Username string
Secret string
Header string
}
// OCI
type OCIRegistry struct {
reference string
headers http.Header
creds CredentialHelper
resolver remotes.Resolver
// This could be an interface which returns resolver?
// Resolver could also be a plug-able interface, to call out to a program to fetch?
}
func (r *OCIRegistry) String() string {
return fmt.Sprintf("OCI Registry (%s)", r.reference)
}
func (r *OCIRegistry) Image() string {
return r.reference
}
func (r *OCIRegistry) Resolve(ctx context.Context) (name string, desc ocispec.Descriptor, err error) {
return r.resolver.Resolve(ctx, r.reference)
}
func (r *OCIRegistry) Fetcher(ctx context.Context, ref string) (transfer.Fetcher, error) {
return r.resolver.Fetcher(ctx, ref)
}
func (r *OCIRegistry) Pusher(ctx context.Context, desc ocispec.Descriptor) (transfer.Pusher, error) {
var ref = r.reference
// Annotate ref with digest to push only push tag for single digest
if !strings.Contains(ref, "@") {
ref = ref + "@" + desc.Digest.String()
}
return r.resolver.Pusher(ctx, ref)
}
func (r *OCIRegistry) MarshalAny(ctx context.Context, sm streaming.StreamCreator) (typeurl.Any, error) {
res := &transfertypes.RegistryResolver{}
if r.headers != nil {
res.Headers = map[string]string{}
for k := range r.headers {
res.Headers[k] = r.headers.Get(k)
}
}
if r.creds != nil {
sid := tstreaming.GenerateID("creds")
stream, err := sm.Create(ctx, sid)
if err != nil {
return nil, err
}
go func() {
// Check for context cancellation as well
for {
select {
case <-ctx.Done():
return
default:
}
req, err := stream.Recv()
if err != nil {
// If not EOF, log error
return
}
var s transfertypes.AuthRequest
if err := typeurl.UnmarshalTo(req, &s); err != nil {
log.G(ctx).WithError(err).Error("failed to unmarshal credential request")
continue
}
creds, err := r.creds.GetCredentials(ctx, s.Reference, s.Host)
if err != nil {
log.G(ctx).WithError(err).Error("failed to get credentials")
continue
}
var resp transfertypes.AuthResponse
if creds.Header != "" {
resp.AuthType = transfertypes.AuthType_HEADER
resp.Secret = creds.Header
} else if creds.Username != "" {
resp.AuthType = transfertypes.AuthType_CREDENTIALS
resp.Username = creds.Username
resp.Secret = creds.Secret
} else {
resp.AuthType = transfertypes.AuthType_REFRESH
resp.Secret = creds.Secret
}
a, err := typeurl.MarshalAny(&resp)
if err != nil {
log.G(ctx).WithError(err).Error("failed to marshal credential response")
continue
}
if err := stream.Send(a); err != nil {
if !errors.Is(err, io.EOF) {
log.G(ctx).WithError(err).Error("unexpected send failure")
}
return
}
}
}()
res.AuthStream = sid
}
s := &transfertypes.OCIRegistry{
Reference: r.reference,
Resolver: res,
}
return typeurl.MarshalAny(s)
}
func (r *OCIRegistry) UnmarshalAny(ctx context.Context, sm streaming.StreamGetter, a typeurl.Any) error {
var (
s transfertypes.OCIRegistry
ropts []docker.RegistryOpt
aopts []docker.AuthorizerOpt
)
if err := typeurl.UnmarshalTo(a, &s); err != nil {
return err
}
if s.Resolver != nil {
if sid := s.Resolver.AuthStream; sid != "" {
stream, err := sm.Get(ctx, sid)
if err != nil {
log.G(ctx).WithError(err).WithField("stream", sid).Debug("failed to get auth stream")
return err
}
r.creds = &credCallback{
stream: stream,
}
aopts = append(aopts, docker.WithAuthCreds(func(host string) (string, string, error) {
c, err := r.creds.GetCredentials(context.Background(), s.Reference, host)
if err != nil {
return "", "", err
}
return c.Username, c.Secret, nil
}))
}
r.headers = http.Header{}
for k, v := range s.Resolver.Headers {
r.headers.Add(k, v)
}
}
authorizer := docker.NewDockerAuthorizer(aopts...)
ropts = append(ropts, docker.WithAuthorizer(authorizer))
r.reference = s.Reference
r.resolver = docker.NewResolver(docker.ResolverOptions{
Hosts: docker.ConfigureDefaultRegistries(ropts...),
Headers: r.headers,
})
return nil
}
type credCallback struct {
sync.Mutex
stream streaming.Stream
}
func (cc *credCallback) GetCredentials(ctx context.Context, ref, host string) (Credentials, error) {
cc.Lock()
defer cc.Unlock()
ar := &transfertypes.AuthRequest{
Host: host,
Reference: ref,
}
any, err := typeurl.MarshalAny(ar)
if err != nil {
return Credentials{}, err
}
if err := cc.stream.Send(any); err != nil {
return Credentials{}, err
}
resp, err := cc.stream.Recv()
if err != nil {
return Credentials{}, err
}
var s transfertypes.AuthResponse
if err := typeurl.UnmarshalTo(resp, &s); err != nil {
return Credentials{}, err
}
creds := Credentials{
Host: host,
}
switch s.AuthType {
case transfertypes.AuthType_CREDENTIALS:
creds.Username = s.Username
creds.Secret = s.Secret
case transfertypes.AuthType_REFRESH:
creds.Secret = s.Secret
case transfertypes.AuthType_HEADER:
creds.Header = s.Secret
}
return creds, nil
}