diff --git a/differ/differ.go b/differ/differ.go index fc5a690e6..3aa264429 100644 --- a/differ/differ.go +++ b/differ/differ.go @@ -71,7 +71,7 @@ func (s *walkingDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts if strings.HasSuffix(desc.MediaType, ".tar.gzip") || strings.HasSuffix(desc.MediaType, ".tar+gzip") { isCompressed = true } else if !strings.HasSuffix(desc.MediaType, ".tar") { - return emptyDesc, errors.Wrapf(errdefs.ErrNotSupported, "unsupported diff media type: %v", desc.MediaType) + return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", desc.MediaType) } } @@ -135,7 +135,7 @@ func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount media = ocispec.MediaTypeImageLayerGzip isCompressed = true default: - return emptyDesc, errors.Wrapf(errdefs.ErrNotSupported, "unsupported diff media type: %v", media) + return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", media) } aDir, err := ioutil.TempDir("", "left-") if err != nil { diff --git a/errdefs/errors.go b/errdefs/errors.go index c8f0a6c46..44ec72e7b 100644 --- a/errdefs/errors.go +++ b/errdefs/errors.go @@ -26,7 +26,7 @@ var ( ErrAlreadyExists = errors.New("already exists") ErrFailedPrecondition = errors.New("failed precondition") ErrUnavailable = errors.New("unavailable") - ErrNotSupported = errors.New("not supported") // represents not supported and unimplemented + ErrNotImplemented = errors.New("not implemented") // represents not supported and unimplemented ) func IsInvalidArgument(err error) bool { @@ -54,6 +54,6 @@ func IsUnavailable(err error) bool { return errors.Cause(err) == ErrUnavailable } -func IsNotSupported(err error) bool { - return errors.Cause(err) == ErrNotSupported +func IsNotImplemented(err error) bool { + return errors.Cause(err) == ErrNotImplemented } diff --git a/errdefs/grpc.go b/errdefs/grpc.go index 53e7ee2d4..76fb3eb3f 100644 --- a/errdefs/grpc.go +++ b/errdefs/grpc.go @@ -38,7 +38,7 @@ func ToGRPC(err error) error { return grpc.Errorf(codes.FailedPrecondition, err.Error()) case IsUnavailable(err): return grpc.Errorf(codes.Unavailable, err.Error()) - case IsNotSupported(err): + case IsNotImplemented(err): return grpc.Errorf(codes.Unimplemented, err.Error()) } @@ -72,7 +72,7 @@ func FromGRPC(err error) error { case codes.FailedPrecondition: cls = ErrFailedPrecondition case codes.Unimplemented: - cls = ErrNotSupported + cls = ErrNotImplemented default: cls = ErrUnknown } diff --git a/platforms/database.go b/platforms/database.go new file mode 100644 index 000000000..bd66e2517 --- /dev/null +++ b/platforms/database.go @@ -0,0 +1,77 @@ +package platforms + +import ( + "runtime" + "strings" +) + +// These function are generated from from https://golang.org/src/go/build/syslist.go. +// +// We use switch statements because they are slightly faster than map lookups +// and use a little less memory. + +// isKnownOS returns true if we know about the operating system. +// +// The OS value should be normalized before calling this function. +func isKnownOS(os string) bool { + switch os { + case "android", "darwin", "dragonfly", "freebsd", "linux", "nacl", "netbsd", "openbsd", "plan9", "solaris", "windows", "zos": + return true + } + return false +} + +// isKnownArch returns true if we know about the architecture. +// +// The arch value should be normalized before being passed to this function. +func isKnownArch(arch string) bool { + switch arch { + case "386", "amd64", "amd64p32", "arm", "armbe", "arm64", "arm64be", "ppc64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "mips64p32", "mips64p32le", "ppc", "s390", "s390x", "sparc", "sparc64": + return true + } + return false +} + +func normalizeOS(os string) string { + if os == "" { + return runtime.GOOS + } + os = strings.ToLower(os) + + switch os { + case "macos": + os = "darwin" + } + return os +} + +// normalizeArch normalizes the architecture. +func normalizeArch(arch, variant string) (string, string) { + arch, variant = strings.ToLower(arch), strings.ToLower(variant) + switch arch { + case "i386": + arch = "386" + variant = "" + case "x86_64", "x86-64": + arch = "amd64" + variant = "" + case "aarch64": + arch = "arm64" + variant = "" // v8 is implied + case "armhf": + arch = "arm" + variant = "" + case "armel": + arch = "arm" + variant = "v6" + case "arm": + switch variant { + case "v7", "7": + variant = "v7" + case "5", "6", "8": + variant = "v" + variant + } + } + + return arch, variant +} diff --git a/platforms/defaults.go b/platforms/defaults.go new file mode 100644 index 000000000..d49912052 --- /dev/null +++ b/platforms/defaults.go @@ -0,0 +1,16 @@ +package platforms + +import ( + "runtime" + + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Default returns the current platform's default platform specification. +func Default() specs.Platform { + return specs.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + // TODO(stevvooe): Need to resolve GOARM for arm hosts. + } +} diff --git a/platforms/defaults_test.go b/platforms/defaults_test.go new file mode 100644 index 000000000..08c40be63 --- /dev/null +++ b/platforms/defaults_test.go @@ -0,0 +1,20 @@ +package platforms + +import ( + "reflect" + "runtime" + "testing" + + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestDefault(t *testing.T) { + expected := specs.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + } + p := Default() + if !reflect.DeepEqual(p, expected) { + t.Fatalf("default platform not as expected: %#v != %#v", p, expected) + } +} diff --git a/platforms/platforms.go b/platforms/platforms.go new file mode 100644 index 000000000..56c6ddc51 --- /dev/null +++ b/platforms/platforms.go @@ -0,0 +1,236 @@ +// Package platforms provides a toolkit for normalizing, matching and +// specifying container platforms. +// +// Centered around OCI platform specifications, we define a string-based +// specifier syntax that can be used for user input. With a specifier, users +// only need to specify the parts of the platform that are relevant to their +// context, providing an operating system or architecture or both. +// +// How do I use this package? +// +// The vast majority of use cases should simply use the match function with +// user input. The first step is to parse a specifier into a matcher: +// +// m, err := Parse("linux") +// if err != nil { ... } +// +// Once you have a matcher, use it to match against the platform declared by a +// component, typically from an image or runtime. Since extracting an images +// platform is a little more involved, we'll use an example against the +// platform default: +// +// if ok := m.Match(Default()); !ok { /* doesn't match */ } +// +// This can be composed in loops for resolving runtimes or used as a filter for +// fetch and select images. +// +// More details of the specifier syntax and platform spec follow. +// +// Declaring Platform Support +// +// Components that have strict platform requirements should use the OCI +// platform specification to declare their support. Typically, this will be +// images and runtimes that should make these declaring which platform they +// support specifically. This looks roughly as follows: +// +// type Platform struct { +// Architecture string +// OS string +// Variant string +// } +// +// Most images and runtimes should at least set Architecture and OS, according +// to their GOARCH and GOOS values, respectively (follow the OCI image +// specification when in doubt). ARM should set variant under certain +// discussions, which are outlined below. +// +// Platform Specifiers +// +// While the OCI platform specifications provide a tool for components to +// specify structured information, user input typically doesn't need the full +// context and much can be inferred. To solve this problem, we introduced +// "specifiers". A specifier has the format +// `||/[/]`. The user can provide either the +// operating system or the architecture or both. +// +// An example of a common specifier is `linux/amd64`. If the host has a default +// of runtime that matches this, the user can simply provide the component that +// matters. For example, if a image provides amd64 and arm64 support, the +// operating system, `linux` can be inferred, so they only have to provide +// `arm64` or `amd64`. Similar behavior is implemented for operating systems, +// where the architecture may be known but a runtime may support images from +// different operating systems. +// +// Normalization +// +// Because not all users are familiar with the way the Go runtime represents +// platforms, several normalizations have been provided to make this package +// easier to user. +// +// The following are performed for architectures: +// +// Value Normalized +// aarch64 arm64 +// armhf arm +// armel arm/v6 +// i386 386 +// x86_64 amd64 +// x86-64 amd64 +// +// We also normalize the operating system `macos` to `darwin`. +// +// ARM Support +// +// To qualify ARM architecture, the Variant field is used to qualify the arm +// version. The most common arm version, v7, is represented without the variant +// unless it is explicitly provided. This is treated as equivalent to armhf. A +// previous architecture, armel, will be normalized to arm/v6. +// +// While these normalizations are provided, their support on arm platforms has +// not yet been fully implemented and tested. +package platforms + +import ( + "regexp" + "runtime" + "strings" + + "github.com/containerd/containerd/errdefs" + specs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +var ( + specifierRe = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) +) + +// Matcher matches platforms specifications, provided by an image or runtime. +type Matcher interface { + Spec() specs.Platform + Match(platform specs.Platform) bool +} + +type matcher struct { + specs.Platform +} + +func (m *matcher) Spec() specs.Platform { + return m.Platform +} + +func (m *matcher) Match(platform specs.Platform) bool { + normalized := Normalize(platform) + return m.OS == normalized.OS && + m.Architecture == normalized.Architecture && + m.Variant == normalized.Variant +} + +func (m *matcher) String() string { + return Format(m.Platform) +} + +// Parse parses the platform specifier syntax into a platform declaration. +// +// Platform specifiers are in the format `||/[/]`. +// The minimum required information for a platform specifier is the operating +// system or architecture. If there is only a single string (no slashes), the +// value will be matched against the known set of operating systems, then fall +// back to the known set of architectures. The missing component will be +// inferred based on the local environment. +// +// Applications should opt to use `Match` over directly parsing specifiers. +func Parse(specifier string) (Matcher, error) { + if strings.Contains(specifier, "*") { + // TODO(stevvooe): need to work out exact wildcard handling + return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: wildcards not yet supported", specifier) + } + + parts := strings.Split(specifier, "/") + + for _, part := range parts { + if !specifierRe.MatchString(part) { + return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%q is an invalid component of %q: platform specifier component must match %q", part, specifier, specifierRe.String()) + } + } + + var p specs.Platform + switch len(parts) { + case 1: + // in this case, we will test that the value might be an OS, then look + // it up. If it is not known, we'll treat it as an architecture. Since + // we have very little information about the platform here, we are + // going to be a little more strict if we don't know about the argument + // value. + p.OS = normalizeOS(parts[0]) + if isKnownOS(p.OS) { + // picks a default architecture + p.Architecture = runtime.GOARCH + if p.Architecture == "arm" { + // TODO(stevvooe): Resolve arm variant, if not v6 (default) + return nil, errors.Wrapf(errdefs.ErrNotImplemented, "arm support not fully implemented") + } + + return &matcher{p}, nil + } + + p.Architecture, p.Variant = normalizeArch(parts[0], "") + if isKnownArch(p.Architecture) { + p.OS = runtime.GOOS + return &matcher{p}, nil + } + + return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: unknown operating system or architecture", specifier) + case 2: + // In this case, we treat as a regular os/arch pair. We don't care + // about whether or not we know of the platform. + p.OS = normalizeOS(parts[0]) + p.Architecture, p.Variant = normalizeArch(parts[1], "") + + return &matcher{p}, nil + case 3: + // we have a fully specified variant, this is rare + p.OS = normalizeOS(parts[0]) + p.Architecture, p.Variant = normalizeArch(parts[1], parts[2]) + + return &matcher{p}, nil + } + + return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: cannot parse platform specifier", specifier) +} + +// Format returns a string specifier from the provided platform specification. +func Format(platform specs.Platform) string { + if platform.OS == "" { + return "unknown" + } + + return joinNotEmpty(platform.OS, platform.Architecture, platform.Variant) +} + +func joinNotEmpty(s ...string) string { + var ss []string + for _, s := range s { + if s == "" { + continue + } + + ss = append(ss, s) + } + + return strings.Join(ss, "/") +} + +// Normalize validates and translate the platform to the canonical value. +// +// For example, if "Aarch64" is encountered, we change it to "arm64" or if +// "x86_64" is encountered, it becomes "amd64". +func Normalize(platform specs.Platform) specs.Platform { + platform.OS = normalizeOS(platform.OS) + platform.Architecture, platform.Variant = normalizeArch(platform.Architecture, platform.Variant) + + // these fields are deprecated, remove them + platform.OSFeatures = nil + platform.OSVersion = "" + + return platform +} diff --git a/platforms/platforms_test.go b/platforms/platforms_test.go new file mode 100644 index 000000000..a88dfbf42 --- /dev/null +++ b/platforms/platforms_test.go @@ -0,0 +1,243 @@ +package platforms + +import ( + "fmt" + "reflect" + "runtime" + "testing" + + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestParseSelector(t *testing.T) { + var ( + defaultOS = runtime.GOOS + defaultArch = runtime.GOARCH + ) + + for _, testcase := range []struct { + skip bool + input string + expected specs.Platform + formatted string + }{ + // While wildcards are a valid use case for platform selection, + // addressing these cases is outside the initial scope for this + // package. When we do add platform wildcards, we should add in these + // testcases to ensure that they are correctly represented. + { + skip: true, + input: "*", + expected: specs.Platform{ + OS: "*", + Architecture: "*", + }, + formatted: "*/*", + }, + { + skip: true, + input: "linux/*", + expected: specs.Platform{ + OS: "linux", + Architecture: "*", + }, + formatted: "linux/*", + }, + { + skip: true, + input: "*/arm64", + expected: specs.Platform{ + OS: "*", + Architecture: "arm64", + }, + formatted: "*/arm64", + }, + { + // NOTE(stevvooe): In this case, the consumer can assume this is v7 + // but we leave the variant blank. This will represent the vast + // majority of arm images. + input: "linux/arm", + expected: specs.Platform{ + OS: "linux", + Architecture: "arm", + }, + formatted: "linux/arm", + }, + { + input: "linux/arm/v6", + expected: specs.Platform{ + OS: "linux", + Architecture: "arm", + Variant: "v6", + }, + formatted: "linux/arm/v6", + }, + { + input: "linux/arm/v7", + expected: specs.Platform{ + OS: "linux", + Architecture: "arm", + Variant: "v7", + }, + formatted: "linux/arm/v7", + }, + { + input: "arm", + expected: specs.Platform{ + OS: defaultOS, + Architecture: "arm", + }, + formatted: "linux/arm", + }, + { + input: "armel", + expected: specs.Platform{ + OS: defaultOS, + Architecture: "arm", + Variant: "v6", + }, + formatted: "linux/arm/v6", + }, + { + input: "armhf", + expected: specs.Platform{ + OS: defaultOS, + Architecture: "arm", + }, + formatted: "linux/arm", + }, + { + input: "Aarch64", + expected: specs.Platform{ + OS: defaultOS, + Architecture: "arm64", + }, + formatted: joinNotEmpty(defaultOS, "arm64"), + }, + { + input: "x86_64", + expected: specs.Platform{ + OS: defaultOS, + Architecture: "amd64", + }, + formatted: joinNotEmpty(defaultOS, "amd64"), + }, + { + input: "Linux/x86_64", + expected: specs.Platform{ + OS: "linux", + Architecture: "amd64", + }, + formatted: "linux/amd64", + }, + { + input: "i386", + expected: specs.Platform{ + OS: defaultOS, + Architecture: "386", + }, + formatted: joinNotEmpty(defaultOS, "386"), + }, + { + input: "linux", + expected: specs.Platform{ + OS: "linux", + Architecture: defaultArch, + }, + formatted: joinNotEmpty("linux", defaultArch), + }, + { + input: "s390x", + expected: specs.Platform{ + OS: defaultOS, + Architecture: "s390x", + }, + formatted: joinNotEmpty(defaultOS, "s390x"), + }, + { + input: "linux/s390x", + expected: specs.Platform{ + OS: "linux", + Architecture: "s390x", + }, + formatted: "linux/s390x", + }, + { + input: "macOS", + expected: specs.Platform{ + OS: "darwin", + Architecture: defaultArch, + }, + formatted: joinNotEmpty("darwin", defaultArch), + }, + } { + t.Run(testcase.input, func(t *testing.T) { + if testcase.skip { + t.Skip("this case is not yet supported") + } + m, err := Parse(testcase.input) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(m.Spec(), testcase.expected) { + t.Fatalf("platform did not match expected: %#v != %#v", m.Spec(), testcase.expected) + } + + // ensure that match works on the input to the output. + if ok := m.Match(testcase.expected); !ok { + t.Fatalf("expected specifier %q matches %v", testcase.input, testcase.expected) + } + + if fmt.Sprint(m) != testcase.formatted { + t.Fatalf("unexpected matcher string: %q != %q", fmt.Sprint(m), testcase.formatted) + } + + formatted := Format(m.Spec()) + if formatted != testcase.formatted { + t.Fatalf("unexpected format: %q != %q", formatted, testcase.formatted) + } + + // re-parse the formatted output and ensure we are stable + reparsed, err := Parse(formatted) + if err != nil { + t.Fatalf("error parsing formatted output: %v", err) + } + + if Format(reparsed.Spec()) != formatted { + t.Fatalf("normalized output did not survive the round trip: %v != %v", Format(reparsed.Spec()), formatted) + } + }) + } +} + +func TestParseSelectorInvalid(t *testing.T) { + for _, testcase := range []struct { + input string + }{ + { + input: "", // empty + }, + { + input: "/linux/arm", // leading slash + }, + { + input: "linux/arm/", // trailing slash + }, + { + input: "linux /arm", // spaces + }, + { + input: "linux/&arm", // invalid character + }, + { + input: "linux/arm/foo/bar", // too many components + }, + } { + t.Run(testcase.input, func(t *testing.T) { + if _, err := Parse(testcase.input); err == nil { + t.Fatalf("should have received an error") + } + }) + } +} diff --git a/services/diff/service.go b/services/diff/service.go index 6d65fd951..af6326b60 100644 --- a/services/diff/service.go +++ b/services/diff/service.go @@ -74,7 +74,7 @@ func (s *service) Apply(ctx context.Context, er *diffapi.ApplyRequest) (*diffapi for _, differ := range s.differs { ocidesc, err = differ.Apply(ctx, desc, mounts) - if !errdefs.IsNotSupported(err) { + if !errdefs.IsNotImplemented(err) { break } } @@ -99,7 +99,7 @@ func (s *service) Diff(ctx context.Context, dr *diffapi.DiffRequest) (*diffapi.D for _, differ := range s.differs { ocidesc, err = differ.DiffMounts(ctx, aMounts, bMounts, dr.MediaType, dr.Ref) - if !errdefs.IsNotSupported(err) { + if !errdefs.IsNotImplemented(err) { break } }