Merge pull request #1403 from stevvooe/platform-selectors

platforms: define selectors for platforms

differ/differ.go

@@ -71,7 +71,7 @@ func (s *walkingDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts
 		if strings.HasSuffix(desc.MediaType, ".tar.gzip") || strings.HasSuffix(desc.MediaType, ".tar+gzip") {
 			isCompressed = true
 		} else if !strings.HasSuffix(desc.MediaType, ".tar") {
-			return emptyDesc, errors.Wrapf(errdefs.ErrNotSupported, "unsupported diff media type: %v", desc.MediaType)
+			return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", desc.MediaType)
 		}
 	}
 
@@ -135,7 +135,7 @@ func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount
 		media = ocispec.MediaTypeImageLayerGzip
 		isCompressed = true
 	default:
-		return emptyDesc, errors.Wrapf(errdefs.ErrNotSupported, "unsupported diff media type: %v", media)
+		return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", media)
 	}
 	aDir, err := ioutil.TempDir("", "left-")
 	if err != nil {

errdefs/errors.go

@@ -26,7 +26,7 @@ var (
 	ErrAlreadyExists      = errors.New("already exists")
 	ErrFailedPrecondition = errors.New("failed precondition")
 	ErrUnavailable        = errors.New("unavailable")
-	ErrNotSupported       = errors.New("not supported") // represents not supported and unimplemented
+	ErrNotImplemented     = errors.New("not implemented") // represents not supported and unimplemented
 )
 
 func IsInvalidArgument(err error) bool {
@@ -54,6 +54,6 @@ func IsUnavailable(err error) bool {
 	return errors.Cause(err) == ErrUnavailable
 }
 
-func IsNotSupported(err error) bool {
-	return errors.Cause(err) == ErrNotSupported
+func IsNotImplemented(err error) bool {
+	return errors.Cause(err) == ErrNotImplemented
 }

errdefs/grpc.go

@@ -38,7 +38,7 @@ func ToGRPC(err error) error {
 		return grpc.Errorf(codes.FailedPrecondition, err.Error())
 	case IsUnavailable(err):
 		return grpc.Errorf(codes.Unavailable, err.Error())
-	case IsNotSupported(err):
+	case IsNotImplemented(err):
 		return grpc.Errorf(codes.Unimplemented, err.Error())
 	}
 
@@ -72,7 +72,7 @@ func FromGRPC(err error) error {
 	case codes.FailedPrecondition:
 		cls = ErrFailedPrecondition
 	case codes.Unimplemented:
-		cls = ErrNotSupported
+		cls = ErrNotImplemented
 	default:
 		cls = ErrUnknown
 	}

platforms/database.go

@@ -0,0 +1,77 @@
+package platforms
+
+import (
+	"runtime"
+	"strings"
+)
+
+// These function are generated from from https://golang.org/src/go/build/syslist.go.
+//
+// We use switch statements because they are slightly faster than map lookups
+// and use a little less memory.
+
+// isKnownOS returns true if we know about the operating system.
+//
+// The OS value should be normalized before calling this function.
+func isKnownOS(os string) bool {
+	switch os {
+	case "android", "darwin", "dragonfly", "freebsd", "linux", "nacl", "netbsd", "openbsd", "plan9", "solaris", "windows", "zos":
+		return true
+	}
+	return false
+}
+
+// isKnownArch returns true if we know about the architecture.
+//
+// The arch value should be normalized before being passed to this function.
+func isKnownArch(arch string) bool {
+	switch arch {
+	case "386", "amd64", "amd64p32", "arm", "armbe", "arm64", "arm64be", "ppc64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "mips64p32", "mips64p32le", "ppc", "s390", "s390x", "sparc", "sparc64":
+		return true
+	}
+	return false
+}
+
+func normalizeOS(os string) string {
+	if os == "" {
+		return runtime.GOOS
+	}
+	os = strings.ToLower(os)
+
+	switch os {
+	case "macos":
+		os = "darwin"
+	}
+	return os
+}
+
+// normalizeArch normalizes the architecture.
+func normalizeArch(arch, variant string) (string, string) {
+	arch, variant = strings.ToLower(arch), strings.ToLower(variant)
+	switch arch {
+	case "i386":
+		arch = "386"
+		variant = ""
+	case "x86_64", "x86-64":
+		arch = "amd64"
+		variant = ""
+	case "aarch64":
+		arch = "arm64"
+		variant = "" // v8 is implied
+	case "armhf":
+		arch = "arm"
+		variant = ""
+	case "armel":
+		arch = "arm"
+		variant = "v6"
+	case "arm":
+		switch variant {
+		case "v7", "7":
+			variant = "v7"
+		case "5", "6", "8":
+			variant = "v" + variant
+		}
+	}
+
+	return arch, variant
+}

platforms/defaults.go

@@ -0,0 +1,16 @@
+package platforms
+
+import (
+	"runtime"
+
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Default returns the current platform's default platform specification.
+func Default() specs.Platform {
+	return specs.Platform{
+		OS:           runtime.GOOS,
+		Architecture: runtime.GOARCH,
+		// TODO(stevvooe): Need to resolve GOARM for arm hosts.
+	}
+}

platforms/defaults_test.go

@@ -0,0 +1,20 @@
+package platforms
+
+import (
+	"reflect"
+	"runtime"
+	"testing"
+
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func TestDefault(t *testing.T) {
+	expected := specs.Platform{
+		OS:           runtime.GOOS,
+		Architecture: runtime.GOARCH,
+	}
+	p := Default()
+	if !reflect.DeepEqual(p, expected) {
+		t.Fatalf("default platform not as expected: %#v != %#v", p, expected)
+	}
+}

platforms/platforms.go

@@ -0,0 +1,236 @@
+// Package platforms provides a toolkit for normalizing, matching and
+// specifying container platforms.
+//
+// Centered around OCI platform specifications, we define a string-based
+// specifier syntax that can be used for user input. With a specifier, users
+// only need to specify the parts of the platform that are relevant to their
+// context, providing an operating system or architecture or both.
+//
+// How do I use this package?
+//
+// The vast majority of use cases should simply use the match function with
+// user input. The first step is to parse a specifier into a matcher:
+//
+//   m, err := Parse("linux")
+//   if err != nil { ... }
+//
+// Once you have a matcher, use it to match against the platform declared by a
+// component, typically from an image or runtime. Since extracting an images
+// platform is a little more involved, we'll use an example against the
+// platform default:
+//
+//   if ok := m.Match(Default()); !ok { /* doesn't match */ }
+//
+// This can be composed in loops for resolving runtimes or used as a filter for
+// fetch and select images.
+//
+// More details of the specifier syntax and platform spec follow.
+//
+// Declaring Platform Support
+//
+// Components that have strict platform requirements should use the OCI
+// platform specification to declare their support. Typically, this will be
+// images and runtimes that should make these declaring which platform they
+// support specifically. This looks roughly as follows:
+//
+//   type Platform struct {
+//	   Architecture string
+//	   OS           string
+//	   Variant      string
+//   }
+//
+// Most images and runtimes should at least set Architecture and OS, according
+// to their GOARCH and GOOS values, respectively (follow the OCI image
+// specification when in doubt). ARM should set variant under certain
+// discussions, which are outlined below.
+//
+// Platform Specifiers
+//
+// While the OCI platform specifications provide a tool for components to
+// specify structured information, user input typically doesn't need the full
+// context and much can be inferred. To solve this problem, we introduced
+// "specifiers". A specifier has the format
+// `<os>|<arch>|<os>/<arch>[/<variant>]`.  The user can provide either the
+// operating system or the architecture or both.
+//
+// An example of a common specifier is `linux/amd64`. If the host has a default
+// of runtime that matches this, the user can simply provide the component that
+// matters. For example, if a image provides amd64 and arm64 support, the
+// operating system, `linux` can be inferred, so they only have to provide
+// `arm64` or `amd64`. Similar behavior is implemented for operating systems,
+// where the architecture may be known but a runtime may support images from
+// different operating systems.
+//
+// Normalization
+//
+// Because not all users are familiar with the way the Go runtime represents
+// platforms, several normalizations have been provided to make this package
+// easier to user.
+//
+// The following are performed for architectures:
+//
+//   Value    Normalized
+//   aarch64  arm64
+//   armhf    arm
+//   armel    arm/v6
+//   i386     386
+//   x86_64   amd64
+//   x86-64   amd64
+//
+// We also normalize the operating system `macos` to `darwin`.
+//
+// ARM Support
+//
+// To qualify ARM architecture, the Variant field is used to qualify the arm
+// version. The most common arm version, v7, is represented without the variant
+// unless it is explicitly provided. This is treated as equivalent to armhf. A
+// previous architecture, armel, will be normalized to arm/v6.
+//
+// While these normalizations are provided, their support on arm platforms has
+// not yet been fully implemented and tested.
+package platforms
+
+import (
+	"regexp"
+	"runtime"
+	"strings"
+
+	"github.com/containerd/containerd/errdefs"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+var (
+	specifierRe = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
+)
+
+// Matcher matches platforms specifications, provided by an image or runtime.
+type Matcher interface {
+	Spec() specs.Platform
+	Match(platform specs.Platform) bool
+}
+
+type matcher struct {
+	specs.Platform
+}
+
+func (m *matcher) Spec() specs.Platform {
+	return m.Platform
+}
+
+func (m *matcher) Match(platform specs.Platform) bool {
+	normalized := Normalize(platform)
+	return m.OS == normalized.OS &&
+		m.Architecture == normalized.Architecture &&
+		m.Variant == normalized.Variant
+}
+
+func (m *matcher) String() string {
+	return Format(m.Platform)
+}
+
+// Parse parses the platform specifier syntax into a platform declaration.
+//
+// Platform specifiers are in the format `<os>|<arch>|<os>/<arch>[/<variant>]`.
+// The minimum required information for a platform specifier is the operating
+// system or architecture. If there is only a single string (no slashes), the
+// value will be matched against the known set of operating systems, then fall
+// back to the known set of architectures. The missing component will be
+// inferred based on the local environment.
+//
+// Applications should opt to use `Match` over directly parsing specifiers.
+func Parse(specifier string) (Matcher, error) {
+	if strings.Contains(specifier, "*") {
+		// TODO(stevvooe): need to work out exact wildcard handling
+		return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: wildcards not yet supported", specifier)
+	}
+
+	parts := strings.Split(specifier, "/")
+
+	for _, part := range parts {
+		if !specifierRe.MatchString(part) {
+			return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%q is an invalid component of %q: platform specifier component must match %q", part, specifier, specifierRe.String())
+		}
+	}
+
+	var p specs.Platform
+	switch len(parts) {
+	case 1:
+		// in this case, we will test that the value might be an OS, then look
+		// it up. If it is not known, we'll treat it as an architecture. Since
+		// we have very little information about the platform here, we are
+		// going to be a little more strict if we don't know about the argument
+		// value.
+		p.OS = normalizeOS(parts[0])
+		if isKnownOS(p.OS) {
+			// picks a default architecture
+			p.Architecture = runtime.GOARCH
+			if p.Architecture == "arm" {
+				// TODO(stevvooe): Resolve arm variant, if not v6 (default)
+				return nil, errors.Wrapf(errdefs.ErrNotImplemented, "arm support not fully implemented")
+			}
+
+			return &matcher{p}, nil
+		}
+
+		p.Architecture, p.Variant = normalizeArch(parts[0], "")
+		if isKnownArch(p.Architecture) {
+			p.OS = runtime.GOOS
+			return &matcher{p}, nil
+		}
+
+		return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: unknown operating system or architecture", specifier)
+	case 2:
+		// In this case, we treat as a regular os/arch pair. We don't care
+		// about whether or not we know of the platform.
+		p.OS = normalizeOS(parts[0])
+		p.Architecture, p.Variant = normalizeArch(parts[1], "")
+
+		return &matcher{p}, nil
+	case 3:
+		// we have a fully specified variant, this is rare
+		p.OS = normalizeOS(parts[0])
+		p.Architecture, p.Variant = normalizeArch(parts[1], parts[2])
+
+		return &matcher{p}, nil
+	}
+
+	return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: cannot parse platform specifier", specifier)
+}
+
+// Format returns a string specifier from the provided platform specification.
+func Format(platform specs.Platform) string {
+	if platform.OS == "" {
+		return "unknown"
+	}
+
+	return joinNotEmpty(platform.OS, platform.Architecture, platform.Variant)
+}
+
+func joinNotEmpty(s ...string) string {
+	var ss []string
+	for _, s := range s {
+		if s == "" {
+			continue
+		}
+
+		ss = append(ss, s)
+	}
+
+	return strings.Join(ss, "/")
+}
+
+// Normalize validates and translate the platform to the canonical value.
+//
+// For example, if "Aarch64" is encountered, we change it to "arm64" or if
+// "x86_64" is encountered, it becomes "amd64".
+func Normalize(platform specs.Platform) specs.Platform {
+	platform.OS = normalizeOS(platform.OS)
+	platform.Architecture, platform.Variant = normalizeArch(platform.Architecture, platform.Variant)
+
+	// these fields are deprecated, remove them
+	platform.OSFeatures = nil
+	platform.OSVersion = ""
+
+	return platform
+}

platforms/platforms_test.go

@@ -0,0 +1,243 @@
+package platforms
+
+import (
+	"fmt"
+	"reflect"
+	"runtime"
+	"testing"
+
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func TestParseSelector(t *testing.T) {
+	var (
+		defaultOS   = runtime.GOOS
+		defaultArch = runtime.GOARCH
+	)
+
+	for _, testcase := range []struct {
+		skip      bool
+		input     string
+		expected  specs.Platform
+		formatted string
+	}{
+		// While wildcards are a valid use case for platform selection,
+		// addressing these cases is outside the initial scope for this
+		// package. When we do add platform wildcards, we should add in these
+		// testcases to ensure that they are correctly represented.
+		{
+			skip:  true,
+			input: "*",
+			expected: specs.Platform{
+				OS:           "*",
+				Architecture: "*",
+			},
+			formatted: "*/*",
+		},
+		{
+			skip:  true,
+			input: "linux/*",
+			expected: specs.Platform{
+				OS:           "linux",
+				Architecture: "*",
+			},
+			formatted: "linux/*",
+		},
+		{
+			skip:  true,
+			input: "*/arm64",
+			expected: specs.Platform{
+				OS:           "*",
+				Architecture: "arm64",
+			},
+			formatted: "*/arm64",
+		},
+		{
+			// NOTE(stevvooe): In this case, the consumer can assume this is v7
+			// but we leave the variant blank. This will represent the vast
+			// majority of arm images.
+			input: "linux/arm",
+			expected: specs.Platform{
+				OS:           "linux",
+				Architecture: "arm",
+			},
+			formatted: "linux/arm",
+		},
+		{
+			input: "linux/arm/v6",
+			expected: specs.Platform{
+				OS:           "linux",
+				Architecture: "arm",
+				Variant:      "v6",
+			},
+			formatted: "linux/arm/v6",
+		},
+		{
+			input: "linux/arm/v7",
+			expected: specs.Platform{
+				OS:           "linux",
+				Architecture: "arm",
+				Variant:      "v7",
+			},
+			formatted: "linux/arm/v7",
+		},
+		{
+			input: "arm",
+			expected: specs.Platform{
+				OS:           defaultOS,
+				Architecture: "arm",
+			},
+			formatted: "linux/arm",
+		},
+		{
+			input: "armel",
+			expected: specs.Platform{
+				OS:           defaultOS,
+				Architecture: "arm",
+				Variant:      "v6",
+			},
+			formatted: "linux/arm/v6",
+		},
+		{
+			input: "armhf",
+			expected: specs.Platform{
+				OS:           defaultOS,
+				Architecture: "arm",
+			},
+			formatted: "linux/arm",
+		},
+		{
+			input: "Aarch64",
+			expected: specs.Platform{
+				OS:           defaultOS,
+				Architecture: "arm64",
+			},
+			formatted: joinNotEmpty(defaultOS, "arm64"),
+		},
+		{
+			input: "x86_64",
+			expected: specs.Platform{
+				OS:           defaultOS,
+				Architecture: "amd64",
+			},
+			formatted: joinNotEmpty(defaultOS, "amd64"),
+		},
+		{
+			input: "Linux/x86_64",
+			expected: specs.Platform{
+				OS:           "linux",
+				Architecture: "amd64",
+			},
+			formatted: "linux/amd64",
+		},
+		{
+			input: "i386",
+			expected: specs.Platform{
+				OS:           defaultOS,
+				Architecture: "386",
+			},
+			formatted: joinNotEmpty(defaultOS, "386"),
+		},
+		{
+			input: "linux",
+			expected: specs.Platform{
+				OS:           "linux",
+				Architecture: defaultArch,
+			},
+			formatted: joinNotEmpty("linux", defaultArch),
+		},
+		{
+			input: "s390x",
+			expected: specs.Platform{
+				OS:           defaultOS,
+				Architecture: "s390x",
+			},
+			formatted: joinNotEmpty(defaultOS, "s390x"),
+		},
+		{
+			input: "linux/s390x",
+			expected: specs.Platform{
+				OS:           "linux",
+				Architecture: "s390x",
+			},
+			formatted: "linux/s390x",
+		},
+		{
+			input: "macOS",
+			expected: specs.Platform{
+				OS:           "darwin",
+				Architecture: defaultArch,
+			},
+			formatted: joinNotEmpty("darwin", defaultArch),
+		},
+	} {
+		t.Run(testcase.input, func(t *testing.T) {
+			if testcase.skip {
+				t.Skip("this case is not yet supported")
+			}
+			m, err := Parse(testcase.input)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !reflect.DeepEqual(m.Spec(), testcase.expected) {
+				t.Fatalf("platform did not match expected: %#v != %#v", m.Spec(), testcase.expected)
+			}
+
+			// ensure that match works on the input to the output.
+			if ok := m.Match(testcase.expected); !ok {
+				t.Fatalf("expected specifier %q matches %v", testcase.input, testcase.expected)
+			}
+
+			if fmt.Sprint(m) != testcase.formatted {
+				t.Fatalf("unexpected matcher string:  %q != %q", fmt.Sprint(m), testcase.formatted)
+			}
+
+			formatted := Format(m.Spec())
+			if formatted != testcase.formatted {
+				t.Fatalf("unexpected format: %q != %q", formatted, testcase.formatted)
+			}
+
+			// re-parse the formatted output and ensure we are stable
+			reparsed, err := Parse(formatted)
+			if err != nil {
+				t.Fatalf("error parsing formatted output: %v", err)
+			}
+
+			if Format(reparsed.Spec()) != formatted {
+				t.Fatalf("normalized output did not survive the round trip: %v != %v", Format(reparsed.Spec()), formatted)
+			}
+		})
+	}
+}
+
+func TestParseSelectorInvalid(t *testing.T) {
+	for _, testcase := range []struct {
+		input string
+	}{
+		{
+			input: "", // empty
+		},
+		{
+			input: "/linux/arm", // leading slash
+		},
+		{
+			input: "linux/arm/", // trailing slash
+		},
+		{
+			input: "linux /arm", // spaces
+		},
+		{
+			input: "linux/&arm", // invalid character
+		},
+		{
+			input: "linux/arm/foo/bar", // too many components
+		},
+	} {
+		t.Run(testcase.input, func(t *testing.T) {
+			if _, err := Parse(testcase.input); err == nil {
+				t.Fatalf("should have received an error")
+			}
+		})
+	}
+}

services/diff/service.go

@@ -74,7 +74,7 @@ func (s *service) Apply(ctx context.Context, er *diffapi.ApplyRequest) (*diffapi
 
 	for _, differ := range s.differs {
 		ocidesc, err = differ.Apply(ctx, desc, mounts)
-		if !errdefs.IsNotSupported(err) {
+		if !errdefs.IsNotImplemented(err) {
 			break
 		}
 	}
@@ -99,7 +99,7 @@ func (s *service) Diff(ctx context.Context, dr *diffapi.DiffRequest) (*diffapi.D
 
 	for _, differ := range s.differs {
 		ocidesc, err = differ.DiffMounts(ctx, aMounts, bMounts, dr.MediaType, dr.Ref)
-		if !errdefs.IsNotSupported(err) {
+		if !errdefs.IsNotImplemented(err) {
 			break
 		}
 	}