Merge pull request #360 from Random-Liu/add-image-load

Add image load

Makefile

@@ -48,6 +48,7 @@ help:
 	@echo " * 'test-e2e-node'    - Test cri-containerd with Kubernetes node e2e test"
 	@echo " * 'clean'            - Clean artifacts"
 	@echo " * 'verify'           - Execute the source code verification tools"
+	@echo " * 'proto'            - Update protobuf of cri-containerd api"
 	@echo " * 'install.tools'    - Install tools used by verify"
 	@echo " * 'install.deps'     - Install dependencies of cri-containerd (containerd, runc, cni) Note: BUILDTAGS defaults to 'seccomp apparmor' for runc build"
 	@echo " * 'uninstall'        - Remove installed binaries from system locations"
@@ -117,6 +118,9 @@ release: $(BUILD_DIR)/$(TARBALL)
 push: $(BUILD_DIR)/$(TARBALL)
 	@BUILD_DIR=$(BUILD_DIR) TARBALL=$(TARBALL) VERSION=$(VERSION) ./hack/push.sh
 
+proto:
+	@hack/update-proto.sh
+
 .PHONY: install.deps
 
 install.deps:
@@ -160,4 +164,5 @@ install.tools: .install.gitvalidation .install.gometalinter
 	test-cri \
 	test-e2e-node \
 	uninstall \
-	version
+	version \
+	proto

cmd/cri-containerd/cri_containerd.go

@@ -18,15 +18,20 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/golang/glog"
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/util/interrupt"
 
 	"github.com/kubernetes-incubator/cri-containerd/cmd/cri-containerd/options"
+	api "github.com/kubernetes-incubator/cri-containerd/pkg/api/v1"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/client"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/server"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/version"
 )
@@ -72,6 +77,35 @@ func versionCommand() *cobra.Command {
 	}
 }
 
+func loadImageCommand() *cobra.Command {
+	c := &cobra.Command{
+		Use:   "load TAR",
+		Short: "Load an image from a tar archive.",
+		Args:  cobra.ExactArgs(1),
+	}
+	endpoint, timeout := options.AddGRPCFlags(c.Flags())
+	c.RunE = func(cmd *cobra.Command, args []string) error {
+		cl, err := client.NewCRIContainerdClient(*endpoint, *timeout)
+		if err != nil {
+			return fmt.Errorf("failed to create grpc client: %v", err)
+		}
+		path, err := filepath.Abs(args[0])
+		if err != nil {
+			return fmt.Errorf("failed to get absolute path: %v", err)
+		}
+		res, err := cl.LoadImage(context.Background(), &api.LoadImageRequest{FilePath: path})
+		if err != nil {
+			return fmt.Errorf("failed to load image: %v", err)
+		}
+		images := res.GetImages()
+		for _, image := range images {
+			fmt.Println("Loaded image:", image)
+		}
+		return nil
+	}
+	return c
+}
+
 func main() {
 	if reexec.Init() {
 		return
@@ -81,10 +115,11 @@ func main() {
 	o.AddFlags(cmd.Flags())
 	cmd.AddCommand(defaultConfigCommand())
 	cmd.AddCommand(versionCommand())
+	cmd.AddCommand(loadImageCommand())
 
-	cmd.Run = func(cmd *cobra.Command, args []string) {
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		if err := o.InitFlags(cmd.Flags()); err != nil {
-			glog.Exitf("Failed to init CRI containerd flags: %v", err)
+			return fmt.Errorf("failed to init CRI containerd flags: %v", err)
 		}
 		validateConfig(o)
 
@@ -93,19 +128,21 @@ func main() {
 		glog.V(2).Infof("Run cri-containerd grpc server on socket %q", o.SocketPath)
 		s, err := server.NewCRIContainerdService(o.Config)
 		if err != nil {
-			glog.Exitf("Failed to create CRI containerd service: %v", err)
+			return fmt.Errorf("failed to create CRI containerd service: %v", err)
 		}
 		// Use interrupt handler to make sure the server is stopped properly.
 		// Pass in non-empty final function to avoid os.Exit(1). We expect `Run`
 		// to return itself.
 		h := interrupt.New(func(os.Signal) {}, s.Stop)
 		if err := h.Run(func() error { return s.Run() }); err != nil {
-			glog.Exitf("Failed to run cri-containerd grpc server: %v", err)
+			return fmt.Errorf("failed to run cri-containerd grpc server: %v", err)
 		}
+		return nil
 	}
 
 	if err := cmd.Execute(); err != nil {
-		glog.Exitf("Failed to execute cri-containerd: %v", err)
+		// Error should have been reported.
+		os.Exit(1)
 	}
 }
 

cmd/cri-containerd/options/options.go

@@ -19,6 +19,7 @@ package options
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/BurntSushi/toml"
 	"github.com/containerd/containerd"
@@ -30,6 +31,8 @@ const (
 	configFilePathArgName = "config"
 	// defaultConfigFilePath is the default config file path.
 	defaultConfigFilePath = "/etc/cri-containerd/config.toml"
+	// connectionTimeout is the grpc connection timeout.
+	connectionTimeout = 10 * time.Second
 )
 
 // ContainerdConfig contains config related to containerd
@@ -178,6 +181,13 @@ func PrintDefaultTomlConfig() {
 	}
 }
 
+// AddGRPCFlags add flags for grpc connection.
+func AddGRPCFlags(fs *pflag.FlagSet) (*string, *time.Duration) {
+	endpoint := fs.String("endpoint", defaultConfig().SocketPath, "cri-containerd endpoint.")
+	timeout := fs.Duration("timeout", connectionTimeout, "cri-containerd connection timeout.")
+	return endpoint, timeout
+}
+
 // defaultConfig returns default configurations of cri-containerd.
 func defaultConfig() Config {
 	return Config{

hack/update-proto.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
+API_ROOT="${ROOT}/pkg/api/v1"
+
+go get k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
+if ! which protoc-gen-gogo >/dev/null; then
+  echo "GOPATH is not in PATH"
+  exit 1
+fi 
+
+function cleanup {
+	rm -f ${API_ROOT}/api.pb.go.bak
+}
+
+trap cleanup EXIT
+
+protoc \
+  --proto_path="${API_ROOT}" \
+  --proto_path="${ROOT}/vendor" \
+  --gogo_out=plugins=grpc:${API_ROOT} ${API_ROOT}/api.proto
+
+# Update boilerplate for the generated file.
+echo "$(cat hack/boilerplate/boilerplate.go.txt ${API_ROOT}/api.pb.go)" > ${API_ROOT}/api.pb.go
+sed -i".bak" "s/Copyright YEAR/Copyright $(date '+%Y')/g" ${API_ROOT}/api.pb.go
+
+gofmt -l -s -w ${API_ROOT}/api.pb.go

hack/verify-lint.sh

@@ -17,7 +17,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-for d in $(find . -type d -a \( -iwholename './pkg*' -o -iwholename './cmd*' \)); do
+for d in $(find . -type d -a \( -iwholename './pkg*' -o -iwholename './cmd*' \) -not -iwholename './pkg/api*'); do
 	echo for directory ${d} ...
 	gometalinter \
 		 --exclude='error return value not checked.*(Close|Log|Print).*\(errcheck\)$' \

integration/image_load_test.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"golang.org/x/net/context"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+
+	api "github.com/kubernetes-incubator/cri-containerd/pkg/api/v1"
+)
+
+// Test to load an image from tarball.
+func TestImageLoad(t *testing.T) {
+	testImage := "busybox:latest"
+	loadedImage := "docker.io/library/" + testImage
+	_, err := exec.LookPath("docker")
+	if err != nil {
+		t.Skip("Docker is not available: %v", err)
+	}
+	t.Logf("docker save image into tarball")
+	output, err := exec.Command("docker", "pull", testImage).CombinedOutput()
+	require.NoError(t, err, "output: %q", output)
+	tarF, err := ioutil.TempFile("", "image-load")
+	tar := tarF.Name()
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, os.RemoveAll(tar))
+	}()
+	output, err = exec.Command("docker", "save", testImage, "-o", tar).CombinedOutput()
+	require.NoError(t, err, "output: %q", output)
+
+	t.Logf("make sure no such image in cri-containerd")
+	img, err := imageService.ImageStatus(&runtime.ImageSpec{Image: testImage})
+	require.NoError(t, err)
+	if img != nil {
+		require.NoError(t, imageService.RemoveImage(&runtime.ImageSpec{Image: testImage}))
+	}
+
+	t.Logf("load image in cri-containerd")
+	res, err := criContainerdClient.LoadImage(context.Background(), &api.LoadImageRequest{FilePath: tar})
+	require.NoError(t, err)
+	require.Equal(t, []string{loadedImage}, res.GetImages())
+
+	t.Logf("make sure image is loaded")
+	img, err = imageService.ImageStatus(&runtime.ImageSpec{Image: testImage})
+	require.NoError(t, err)
+	require.NotNil(t, img)
+	require.Equal(t, []string{loadedImage}, img.RepoTags)
+
+	t.Logf("create a container with the loaded image")
+	sbConfig := PodSandboxConfig("sandbox", Randomize("image-load"))
+	sb, err := runtimeService.RunPodSandbox(sbConfig)
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, runtimeService.StopPodSandbox(sb))
+		assert.NoError(t, runtimeService.RemovePodSandbox(sb))
+	}()
+	containerConfig := ContainerConfig(
+		"container",
+		testImage,
+		WithCommand("tail", "-f", "/dev/null"),
+	)
+	// Rely on sandbox clean to do container cleanup.
+	cn, err := runtimeService.CreateContainer(sb, containerConfig, sbConfig)
+	require.NoError(t, err)
+	require.NoError(t, runtimeService.StartContainer(cn))
+
+	t.Logf("make sure container is running")
+	status, err := runtimeService.ContainerStatus(cn)
+	require.NoError(t, err)
+	require.Equal(t, runtime.ContainerState_CONTAINER_RUNNING, status.State)
+}

integration/test_utils.go

@@ -26,21 +26,25 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 	"k8s.io/kubernetes/pkg/kubelet/remote"
 
+	api "github.com/kubernetes-incubator/cri-containerd/pkg/api/v1"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/client"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
 )
 
 const (
-	sock               = "/var/run/cri-containerd.sock"
-	timeout            = 1 * time.Minute
-	pauseImage         = "gcr.io/google_containers/pause:3.0" // This is the same with default sandbox image.
-	k8sNamespace       = "k8s.io"                             // This is the same with server.k8sContainerdNamespace.
-	containerdEndpoint = "/run/containerd/containerd.sock"
+	sock                  = "/var/run/cri-containerd.sock"
+	timeout               = 1 * time.Minute
+	pauseImage            = "gcr.io/google_containers/pause:3.0" // This is the same with default sandbox image.
+	k8sNamespace          = "k8s.io"                             // This is the same with server.k8sContainerdNamespace.
+	containerdEndpoint    = "/run/containerd/containerd.sock"
+	criContainerdEndpoint = "/var/run/cri-containerd.sock"
 )
 
 var (
-	runtimeService   cri.RuntimeService
-	imageService     cri.ImageManagerService
-	containerdClient *containerd.Client
+	runtimeService      cri.RuntimeService
+	imageService        cri.ImageManagerService
+	containerdClient    *containerd.Client
+	criContainerdClient api.CRIContainerdServiceClient
 )
 
 func init() {
@@ -57,6 +61,10 @@ func init() {
 	if err != nil {
 		glog.Exitf("Failed to connect containerd: %v", err)
 	}
+	criContainerdClient, err = client.NewCRIContainerdClient(criContainerdEndpoint, timeout)
+	if err != nil {
+		glog.Exitf("Failed to connect cri-containerd: %v", err)
+	}
 }
 
 // Opts sets specific information in pod sandbox config.
@@ -70,7 +78,7 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb
 			// Using random id as uuid is good enough for local
 			// integration test.
 			Uid:       util.GenerateID(),
-			Namespace: ns,
+			Namespace: Randomize(ns),
 		},
 		Linux: &runtime.LinuxPodSandboxConfig{},
 	}
@@ -106,6 +114,14 @@ func WithResources(r *runtime.LinuxContainerResources) ContainerOpts {
 	}
 }
 
+// Add container command.
+func WithCommand(c string, args ...string) ContainerOpts {
+	return func(cf *runtime.ContainerConfig) {
+		cf.Command = []string{c}
+		cf.Args = args
+	}
+}
+
 // ContainerConfig creates a container config given a name and image name
 // and additional container config options
 func ContainerConfig(name, image string, opts ...ContainerOpts) *runtime.ContainerConfig {
@@ -143,3 +159,8 @@ func Eventually(f CheckFunc, period, timeout time.Duration) error {
 		time.Sleep(period)
 	}
 }
+
+// Randomize adds uuid after a string.
+func Randomize(str string) string {
+	return str + "-" + util.GenerateID()
+}

pkg/api/v1/api.pb.go

@@ -0,0 +1,598 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by protoc-gen-gogo.
+// source: api.proto
+// DO NOT EDIT!
+
+/*
+Package api_v1 is a generated protocol buffer package.
+
+It is generated from these files:
+	api.proto
+
+It has these top-level messages:
+	LoadImageRequest
+	LoadImageResponse
+*/
+package api_v1
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+import (
+	context "golang.org/x/net/context"
+	grpc "google.golang.org/grpc"
+)
+
+import strings "strings"
+import reflect "reflect"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+
+type LoadImageRequest struct {
+	// FilePath is the absolute path of docker image tarball.
+	FilePath string `protobuf:"bytes,1,opt,name=FilePath,proto3" json:"FilePath,omitempty"`
+}
+
+func (m *LoadImageRequest) Reset()                    { *m = LoadImageRequest{} }
+func (*LoadImageRequest) ProtoMessage()               {}
+func (*LoadImageRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{0} }
+
+func (m *LoadImageRequest) GetFilePath() string {
+	if m != nil {
+		return m.FilePath
+	}
+	return ""
+}
+
+type LoadImageResponse struct {
+	// Images have been loaded.
+	Images []string `protobuf:"bytes,1,rep,name=Images" json:"Images,omitempty"`
+}
+
+func (m *LoadImageResponse) Reset()                    { *m = LoadImageResponse{} }
+func (*LoadImageResponse) ProtoMessage()               {}
+func (*LoadImageResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{1} }
+
+func (m *LoadImageResponse) GetImages() []string {
+	if m != nil {
+		return m.Images
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*LoadImageRequest)(nil), "api.v1.LoadImageRequest")
+	proto.RegisterType((*LoadImageResponse)(nil), "api.v1.LoadImageResponse")
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for CRIContainerdService service
+
+type CRIContainerdServiceClient interface {
+	// LoadImage loads a image into containerd.
+	LoadImage(ctx context.Context, in *LoadImageRequest, opts ...grpc.CallOption) (*LoadImageResponse, error)
+}
+
+type cRIContainerdServiceClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewCRIContainerdServiceClient(cc *grpc.ClientConn) CRIContainerdServiceClient {
+	return &cRIContainerdServiceClient{cc}
+}
+
+func (c *cRIContainerdServiceClient) LoadImage(ctx context.Context, in *LoadImageRequest, opts ...grpc.CallOption) (*LoadImageResponse, error) {
+	out := new(LoadImageResponse)
+	err := grpc.Invoke(ctx, "/api.v1.CRIContainerdService/LoadImage", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// Server API for CRIContainerdService service
+
+type CRIContainerdServiceServer interface {
+	// LoadImage loads a image into containerd.
+	LoadImage(context.Context, *LoadImageRequest) (*LoadImageResponse, error)
+}
+
+func RegisterCRIContainerdServiceServer(s *grpc.Server, srv CRIContainerdServiceServer) {
+	s.RegisterService(&_CRIContainerdService_serviceDesc, srv)
+}
+
+func _CRIContainerdService_LoadImage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(LoadImageRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(CRIContainerdServiceServer).LoadImage(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/api.v1.CRIContainerdService/LoadImage",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(CRIContainerdServiceServer).LoadImage(ctx, req.(*LoadImageRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _CRIContainerdService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "api.v1.CRIContainerdService",
+	HandlerType: (*CRIContainerdServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "LoadImage",
+			Handler:    _CRIContainerdService_LoadImage_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "api.proto",
+}
+
+func (m *LoadImageRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *LoadImageRequest) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.FilePath) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintApi(dAtA, i, uint64(len(m.FilePath)))
+		i += copy(dAtA[i:], m.FilePath)
+	}
+	return i, nil
+}
+
+func (m *LoadImageResponse) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *LoadImageResponse) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Images) > 0 {
+		for _, s := range m.Images {
+			dAtA[i] = 0xa
+			i++
+			l = len(s)
+			for l >= 1<<7 {
+				dAtA[i] = uint8(uint64(l)&0x7f | 0x80)
+				l >>= 7
+				i++
+			}
+			dAtA[i] = uint8(l)
+			i++
+			i += copy(dAtA[i:], s)
+		}
+	}
+	return i, nil
+}
+
+func encodeFixed64Api(dAtA []byte, offset int, v uint64) int {
+	dAtA[offset] = uint8(v)
+	dAtA[offset+1] = uint8(v >> 8)
+	dAtA[offset+2] = uint8(v >> 16)
+	dAtA[offset+3] = uint8(v >> 24)
+	dAtA[offset+4] = uint8(v >> 32)
+	dAtA[offset+5] = uint8(v >> 40)
+	dAtA[offset+6] = uint8(v >> 48)
+	dAtA[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Api(dAtA []byte, offset int, v uint32) int {
+	dAtA[offset] = uint8(v)
+	dAtA[offset+1] = uint8(v >> 8)
+	dAtA[offset+2] = uint8(v >> 16)
+	dAtA[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintApi(dAtA []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return offset + 1
+}
+func (m *LoadImageRequest) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.FilePath)
+	if l > 0 {
+		n += 1 + l + sovApi(uint64(l))
+	}
+	return n
+}
+
+func (m *LoadImageResponse) Size() (n int) {
+	var l int
+	_ = l
+	if len(m.Images) > 0 {
+		for _, s := range m.Images {
+			l = len(s)
+			n += 1 + l + sovApi(uint64(l))
+		}
+	}
+	return n
+}
+
+func sovApi(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozApi(x uint64) (n int) {
+	return sovApi(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (this *LoadImageRequest) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&LoadImageRequest{`,
+		`FilePath:` + fmt.Sprintf("%v", this.FilePath) + `,`,
+		`}`,
+	}, "")
+	return s
+}
+func (this *LoadImageResponse) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&LoadImageResponse{`,
+		`Images:` + fmt.Sprintf("%v", this.Images) + `,`,
+		`}`,
+	}, "")
+	return s
+}
+func valueToStringApi(v interface{}) string {
+	rv := reflect.ValueOf(v)
+	if rv.IsNil() {
+		return "nil"
+	}
+	pv := reflect.Indirect(rv).Interface()
+	return fmt.Sprintf("*%v", pv)
+}
+func (m *LoadImageRequest) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowApi
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: LoadImageRequest: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: LoadImageRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field FilePath", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.FilePath = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipApi(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthApi
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *LoadImageResponse) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowApi
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: LoadImageResponse: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: LoadImageResponse: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Images", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Images = append(m.Images, string(dAtA[iNdEx:postIndex]))
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipApi(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthApi
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipApi(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowApi
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthApi
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowApi
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipApi(dAtA[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthApi = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowApi   = fmt.Errorf("proto: integer overflow")
+)
+
+func init() { proto.RegisterFile("api.proto", fileDescriptorApi) }
+
+var fileDescriptorApi = []byte{
+	// 223 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0x2c, 0xc8, 0xd4,
+	0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x03, 0x31, 0xcb, 0x0c, 0xa5, 0x74, 0xd3, 0x33, 0x4b,
+	0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0xd3, 0xf3, 0xd3, 0xf3, 0xf5, 0xc1, 0xd2, 0x49,
+	0xa5, 0x69, 0x60, 0x1e, 0x98, 0x03, 0x66, 0x41, 0xb4, 0x29, 0xe9, 0x71, 0x09, 0xf8, 0xe4, 0x27,
+	0xa6, 0x78, 0xe6, 0x26, 0xa6, 0xa7, 0x06, 0xa5, 0x16, 0x96, 0xa6, 0x16, 0x97, 0x08, 0x49, 0x71,
+	0x71, 0xb8, 0x65, 0xe6, 0xa4, 0x06, 0x24, 0x96, 0x64, 0x48, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06,
+	0xc1, 0xf9, 0x4a, 0xda, 0x5c, 0x82, 0x48, 0xea, 0x8b, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xc4,
+	0xb8, 0xd8, 0xc0, 0x02, 0xc5, 0x12, 0x8c, 0x0a, 0xcc, 0x1a, 0x9c, 0x41, 0x50, 0x9e, 0x51, 0x14,
+	0x97, 0x88, 0x73, 0x90, 0xa7, 0x73, 0x7e, 0x5e, 0x49, 0x62, 0x66, 0x5e, 0x6a, 0x51, 0x4a, 0x70,
+	0x6a, 0x51, 0x59, 0x66, 0x72, 0xaa, 0x90, 0x13, 0x17, 0x27, 0xdc, 0x10, 0x21, 0x09, 0x3d, 0x88,
+	0xcb, 0xf5, 0xd0, 0xdd, 0x21, 0x25, 0x89, 0x45, 0x06, 0x62, 0xa3, 0x12, 0x83, 0x93, 0xcc, 0x89,
+	0x87, 0x72, 0x8c, 0x37, 0x1e, 0xca, 0x31, 0x34, 0x3c, 0x92, 0x63, 0x3c, 0xf1, 0x48, 0x8e, 0xf1,
+	0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0x48, 0x62, 0x03, 0xfb,
+	0xce, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x6a, 0xfe, 0x35, 0x81, 0x21, 0x01, 0x00, 0x00,
+}

pkg/api/v1/api.proto

@@ -0,0 +1,30 @@
+// To regenerate api.pb.go run `make proto`hack/update-generated-runtime.sh
+syntax = 'proto3';
+
+package api.v1;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) =  true;
+option (gogoproto.goproto_getters_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_unrecognized_all) = false;
+
+// CRIContainerdService defines non-CRI APIs for cri-containerd.
+service CRIContainerdService{
+    // LoadImage loads a image into containerd.
+    rpc LoadImage(LoadImageRequest) returns (LoadImageResponse) {}
+}
+
+message LoadImageRequest {
+    // FilePath is the absolute path of docker image tarball.
+    string FilePath = 1;
+}
+
+message LoadImageResponse {
+    // Images have been loaded.
+    repeated string Images = 1;
+}

pkg/client/client.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc"
+	"k8s.io/kubernetes/pkg/kubelet/util"
+
+	api "github.com/kubernetes-incubator/cri-containerd/pkg/api/v1"
+)
+
+// NewCRIContainerdClient creates grpc client of cri-containerd
+// TODO(random-liu): Wrap grpc functions.
+func NewCRIContainerdClient(endpoint string, timeout time.Duration) (api.CRIContainerdServiceClient, error) {
+	addr, dailer, err := util.GetAddressAndDialer(endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get dialer: %v", err)
+	}
+	conn, err := grpc.Dial(addr,
+		grpc.WithInsecure(),
+		grpc.WithTimeout(timeout),
+		grpc.WithDialer(dailer),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to dial: %v", err)
+	}
+	return api.NewCRIContainerdServiceClient(conn), nil
+}

pkg/containerd/importer/importer.go

@@ -0,0 +1,298 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package importer
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/log"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+
+	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
+)
+
+// This code reuses the docker import code from containerd/containerd#1602.
+// It has been simplified a bit and garbage collection support was added.
+// If a library/helper is added to containerd in the future, we should switch to it.
+
+// manifestDotJSON is an entry in manifest.json.
+type manifestDotJSON struct {
+	Config   string
+	RepoTags []string
+	Layers   []string
+	// Parent is unsupported
+	Parent string
+}
+
+// isLayerTar returns true if name is like "deadbeeddeadbeef/layer.tar"
+func isLayerTar(name string) bool {
+	slashes := len(strings.Split(name, "/"))
+	return slashes == 2 && strings.HasSuffix(name, "/layer.tar")
+}
+
+// isDotJSON returns true if name is like "deadbeefdeadbeef.json"
+func isDotJSON(name string) bool {
+	slashes := len(strings.Split(name, "/"))
+	return slashes == 1 && strings.HasSuffix(name, ".json")
+}
+
+type imageConfig struct {
+	desc ocispec.Descriptor
+	img  ocispec.Image
+}
+
+// Import implements Docker Image Spec v1.1.
+// An image MUST have `manifest.json`.
+// `repositories` file in Docker Image Spec v1.0 is not supported (yet).
+// Also, the current implementation assumes the implicit file name convention,
+// which is not explicitly documented in the spec. (e.g. deadbeef/layer.tar)
+// It returns a group of image references successfully loaded.
+func Import(ctx context.Context, cs content.Store, is images.Store, reader io.Reader) (_ []string, retErr error) {
+	tr := tar.NewReader(reader)
+	var (
+		mfsts   []manifestDotJSON
+		layers  = make(map[string]ocispec.Descriptor) // key: filename (deadbeeddeadbeef/layer.tar)
+		configs = make(map[string]imageConfig)        // key: filename (deadbeeddeadbeef.json)
+	)
+	// Either image is successfully imported or not, we should cleanup gc.root
+	// for all image layers.
+	defer func() {
+		for _, desc := range layers {
+			// Remove root tag from layers now that manifest refers to it
+			if _, err := cs.Update(ctx, content.Info{Digest: desc.Digest}, "labels.containerd.io/gc.root"); err != nil {
+				log.G(ctx).WithError(err).Error("Failed to remove layer %q root tag", desc.Digest)
+			}
+		}
+		for _, cfg := range configs {
+			// Remove root tag from config now that manifest refers to it
+			if _, err := cs.Update(ctx, content.Info{Digest: cfg.desc.Digest}, "labels.containerd.io/gc.root"); err != nil {
+				log.G(ctx).WithError(err).Error("Failed to remove config %q root tag", cfg.desc.Digest)
+			}
+		}
+	}()
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, errors.Wrap(err, "get next file")
+		}
+		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
+			continue
+		}
+		if hdr.Name == "manifest.json" {
+			mfsts, err = onUntarManifestJSON(tr)
+			if err != nil {
+				return nil, errors.Wrapf(err, "untar manifest %q", hdr.Name)
+			}
+			continue
+		}
+		if isLayerTar(hdr.Name) {
+			desc, err := onUntarLayerTar(ctx, tr, cs, hdr.Name, hdr.Size)
+			if err != nil {
+				return nil, errors.Wrapf(err, "untar layer %q", hdr.Name)
+			}
+			layers[hdr.Name] = *desc
+			continue
+		}
+		if isDotJSON(hdr.Name) {
+			c, err := onUntarDotJSON(ctx, tr, cs, hdr.Name, hdr.Size)
+			if err != nil {
+				return nil, errors.Wrapf(err, "untar config %q", hdr.Name)
+			}
+			configs[hdr.Name] = *c
+			continue
+		}
+	}
+	var refs []string
+	defer func() {
+		if retErr == nil {
+			return
+		}
+		// TODO(random-liu): Consider whether we should keep images already imported
+		// even when there is an error.
+		for _, ref := range refs {
+			if err := is.Delete(ctx, ref); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove image %q", ref)
+			}
+		}
+	}()
+	for _, mfst := range mfsts {
+		config, ok := configs[mfst.Config]
+		if !ok {
+			return refs, errors.Errorf("image config %q not found", mfst.Config)
+		}
+		schema2Manifest, err := makeDockerSchema2Manifest(mfst, config, layers)
+		if err != nil {
+			return refs, errors.Wrap(err, "create docker manifest")
+		}
+		desc, err := writeDockerSchema2Manifest(ctx, cs, *schema2Manifest, config.img.Architecture, config.img.OS)
+		if err != nil {
+			return refs, errors.Wrap(err, "write docker manifest")
+		}
+		defer func() {
+			// Remove root tag from manifest.
+			if _, err := cs.Update(ctx, content.Info{Digest: desc.Digest}, "labels.containerd.io/gc.root"); err != nil {
+				log.G(ctx).WithError(err).Error("Failed to remove manifest root tag")
+			}
+		}()
+
+		for _, ref := range mfst.RepoTags {
+			normalized, err := util.NormalizeImageRef(ref)
+			if err != nil {
+				return refs, errors.Wrapf(err, "normalize image ref %q", ref)
+			}
+			ref = normalized.String()
+			imgrec := images.Image{
+				Name:   ref,
+				Target: *desc,
+			}
+			if _, err := is.Create(ctx, imgrec); err != nil {
+				if !errdefs.IsAlreadyExists(err) {
+					return refs, errors.Wrapf(err, "create image ref %+v", imgrec)
+				}
+
+				_, err := is.Update(ctx, imgrec)
+				if err != nil {
+					return refs, errors.Wrapf(err, "update image ref %+v", imgrec)
+				}
+			}
+			refs = append(refs, ref)
+		}
+	}
+	return refs, nil
+}
+
+func makeDockerSchema2Manifest(mfst manifestDotJSON, config imageConfig, layers map[string]ocispec.Descriptor) (*ocispec.Manifest, error) {
+	manifest := ocispec.Manifest{
+		Versioned: specs.Versioned{
+			SchemaVersion: 2,
+		},
+		Config: config.desc,
+	}
+	for _, f := range mfst.Layers {
+		desc, ok := layers[f]
+		if !ok {
+			return nil, errors.Errorf("layer %q not found", f)
+		}
+		manifest.Layers = append(manifest.Layers, desc)
+	}
+	return &manifest, nil
+}
+
+func writeDockerSchema2Manifest(ctx context.Context, cs content.Ingester, manifest ocispec.Manifest, arch, os string) (*ocispec.Descriptor, error) {
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		return nil, err
+	}
+	manifestBytesR := bytes.NewReader(manifestBytes)
+	manifestDigest := digest.FromBytes(manifestBytes)
+	labels := map[string]string{}
+	labels["containerd.io/gc.root"] = time.Now().UTC().Format(time.RFC3339)
+	labels["containerd.io/gc.ref.content.0"] = manifest.Config.Digest.String()
+	for i, ch := range manifest.Layers {
+		labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = ch.Digest.String()
+	}
+	if err := content.WriteBlob(ctx, cs, "manifest-"+manifestDigest.String(), manifestBytesR,
+		int64(len(manifestBytes)), manifestDigest, content.WithLabels(labels)); err != nil {
+		return nil, err
+	}
+
+	desc := &ocispec.Descriptor{
+		MediaType: images.MediaTypeDockerSchema2Manifest,
+		Digest:    manifestDigest,
+		Size:      int64(len(manifestBytes)),
+	}
+	if arch != "" || os != "" {
+		desc.Platform = &ocispec.Platform{
+			Architecture: arch,
+			OS:           os,
+		}
+	}
+	return desc, nil
+}
+
+func onUntarManifestJSON(r io.Reader) ([]manifestDotJSON, error) {
+	// name: "manifest.json"
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	var mfsts []manifestDotJSON
+	if err := json.Unmarshal(b, &mfsts); err != nil {
+		return nil, err
+	}
+	return mfsts, nil
+}
+
+func onUntarLayerTar(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (*ocispec.Descriptor, error) {
+	// name is like "deadbeeddeadbeef/layer.tar" ( guaranteed by isLayerTar() )
+	split := strings.Split(name, "/")
+	// note: split[0] is not expected digest here
+	cw, err := cs.Writer(ctx, "layer-"+split[0], size, "")
+	if err != nil {
+		return nil, err
+	}
+	defer cw.Close()
+	if err := content.Copy(ctx, cw, r, size, "", content.WithLabels(map[string]string{
+		"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339),
+	})); err != nil {
+		return nil, err
+	}
+	return &ocispec.Descriptor{
+		MediaType: images.MediaTypeDockerSchema2Layer,
+		Size:      size,
+		Digest:    cw.Digest(),
+	}, nil
+}
+
+func onUntarDotJSON(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (*imageConfig, error) {
+	config := imageConfig{}
+	config.desc.MediaType = images.MediaTypeDockerSchema2Config
+	config.desc.Size = size
+	// name is like "deadbeeddeadbeef.json" ( guaranteed by is DotJSON() )
+	split := strings.Split(name, ".")
+	cw, err := cs.Writer(ctx, "config-"+split[0], size, "")
+	if err != nil {
+		return nil, err
+	}
+	defer cw.Close()
+	var buf bytes.Buffer
+	tr := io.TeeReader(r, &buf)
+	if err := content.Copy(ctx, cw, tr, size, "", content.WithLabels(map[string]string{
+		"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339),
+	})); err != nil {
+		return nil, err
+	}
+	config.desc.Digest = cw.Digest()
+	if err := json.Unmarshal(buf.Bytes(), &config.img); err != nil {
+		return nil, err
+	}
+	return &config, nil
+}

pkg/server/container_create.go

@@ -43,7 +43,7 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	customopts "github.com/kubernetes-incubator/cri-containerd/pkg/opts"
+	customopts "github.com/kubernetes-incubator/cri-containerd/pkg/containerd/opts"
 	cio "github.com/kubernetes-incubator/cri-containerd/pkg/server/io"
 	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/util"

pkg/server/helpers.go

@@ -41,6 +41,7 @@ import (
 
 	"github.com/kubernetes-incubator/cri-containerd/pkg/store"
 	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
 )
 
 const (
@@ -187,35 +188,6 @@ func criContainerStateToString(state runtime.ContainerState) string {
 	return runtime.ContainerState_name[int32(state)]
 }
 
-// normalizeImageRef normalizes the image reference following the docker convention. This is added
-// mainly for backward compatibility.
-// The reference returned can only be either tagged or digested. For reference contains both tag
-// and digest, the function returns digested reference, e.g. docker.io/library/busybox:latest@
-// sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa will be returned as
-// docker.io/library/busybox@sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa.
-func normalizeImageRef(ref string) (reference.Named, error) {
-	named, err := reference.ParseNormalizedNamed(ref)
-	if err != nil {
-		return nil, err
-	}
-	if _, ok := named.(reference.NamedTagged); ok {
-		if canonical, ok := named.(reference.Canonical); ok {
-			// The reference is both tagged and digested, only
-			// return digested.
-			newNamed, err := reference.WithName(canonical.Name())
-			if err != nil {
-				return nil, err
-			}
-			newCanonical, err := reference.WithDigest(newNamed, canonical.Digest())
-			if err != nil {
-				return nil, err
-			}
-			return newCanonical, nil
-		}
-	}
-	return reference.TagNameOnly(named), nil
-}
-
 // getRepoDigestAngTag returns image repoDigest and repoTag of the named image reference.
 func getRepoDigestAndTag(namedRef reference.Named, digest imagedigest.Digest, schema1 bool) (string, string) {
 	var repoTag, repoDigest string
@@ -237,7 +209,7 @@ func (c *criContainerdService) localResolve(ctx context.Context, ref string) (*i
 	_, err := imagedigest.Parse(ref)
 	if err != nil {
 		// ref is not image id, try to resolve it locally.
-		normalized, err := normalizeImageRef(ref)
+		normalized, err := util.NormalizeImageRef(ref)
 		if err != nil {
 			return nil, fmt.Errorf("invalid image reference %q: %v", ref, err)
 		}

pkg/server/helpers_test.go

@@ -19,70 +19,11 @@ package server
 import (
 	"testing"
 
-	"github.com/containerd/containerd/reference"
 	imagedigest "github.com/opencontainers/go-digest"
 	"github.com/stretchr/testify/assert"
-)
 
-func TestNormalizeImageRef(t *testing.T) {
-	for _, test := range []struct {
-		input  string
-		expect string
-	}{
-		{ // has nothing
-			input:  "busybox",
-			expect: "docker.io/library/busybox:latest",
-		},
-		{ // only has tag
-			input:  "busybox:latest",
-			expect: "docker.io/library/busybox:latest",
-		},
-		{ // only has digest
-			input:  "busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
-			expect: "docker.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
-		},
-		{ // only has path
-			input:  "library/busybox",
-			expect: "docker.io/library/busybox:latest",
-		},
-		{ // only has hostname
-			input:  "docker.io/busybox",
-			expect: "docker.io/library/busybox:latest",
-		},
-		{ // has no tag
-			input:  "docker.io/library/busybox",
-			expect: "docker.io/library/busybox:latest",
-		},
-		{ // has no path
-			input:  "docker.io/busybox:latest",
-			expect: "docker.io/library/busybox:latest",
-		},
-		{ // has no hostname
-			input:  "library/busybox:latest",
-			expect: "docker.io/library/busybox:latest",
-		},
-		{ // full reference
-			input:  "docker.io/library/busybox:latest",
-			expect: "docker.io/library/busybox:latest",
-		},
-		{ // gcr reference
-			input:  "gcr.io/library/busybox",
-			expect: "gcr.io/library/busybox:latest",
-		},
-		{ // both tag and digest
-			input:  "gcr.io/library/busybox:latest@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
-			expect: "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
-		},
-	} {
-		t.Logf("TestCase %q", test.input)
-		normalized, err := normalizeImageRef(test.input)
-		assert.NoError(t, err)
-		output := normalized.String()
-		assert.Equal(t, test.expect, output)
-		_, err = reference.Parse(output)
-		assert.NoError(t, err, "%q should be containerd supported reference", output)
-	}
-}
+	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
+)
 
 // TestGetUserFromImage tests the logic of getting image uid or user name of image user.
 func TestGetUserFromImage(t *testing.T) {
@@ -154,7 +95,7 @@ func TestGetRepoDigestAndTag(t *testing.T) {
 		},
 	} {
 		t.Logf("TestCase %q", desc)
-		named, err := normalizeImageRef(test.ref)
+		named, err := util.NormalizeImageRef(test.ref)
 		assert.NoError(t, err)
 		repoDigest, repoTag := getRepoDigestAndTag(named, digest, test.schema1)
 		assert.Equal(t, test.expectedRepoDigest, repoDigest)

pkg/server/image_load.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"golang.org/x/net/context"
+	"os"
+	"path/filepath"
+
+	"github.com/golang/glog"
+
+	api "github.com/kubernetes-incubator/cri-containerd/pkg/api/v1"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/containerd/importer"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
+)
+
+// LoadImage loads a image into containerd.
+func (c *criContainerdService) LoadImage(ctx context.Context, r *api.LoadImageRequest) (*api.LoadImageResponse, error) {
+	path := r.GetFilePath()
+	if !filepath.IsAbs(path) {
+		return nil, fmt.Errorf("path %q is not an absolute path", path)
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %v", err)
+	}
+	repoTags, err := importer.Import(ctx, c.client.ContentStore(), c.client.ImageService(), f)
+	if err != nil {
+		return nil, fmt.Errorf("failed to import image: %v", err)
+	}
+	for _, repoTag := range repoTags {
+		image, err := c.client.GetImage(ctx, repoTag)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get image %q: %v", repoTag, err)
+		}
+		if err := image.Unpack(ctx, c.config.ContainerdConfig.Snapshotter); err != nil {
+			glog.Warningf("Failed to unpack image %q: %v", repoTag, err)
+			// Do not fail image importing. Unpack will be retried when container creation.
+		}
+		info, err := getImageInfo(ctx, image, c.client.ContentStore())
+		if err != nil {
+			return nil, fmt.Errorf("failed to get image %q info: %v", repoTag, err)
+		}
+		id := info.id
+
+		if err := c.createImageReference(ctx, id, image.Target()); err != nil {
+			return nil, fmt.Errorf("failed to create image reference %q: %v", id, err)
+		}
+		img := imagestore.Image{
+			ID:       id,
+			RepoTags: []string{repoTag},
+			ChainID:  info.chainID.String(),
+			Size:     info.size,
+			Config:   &info.config,
+			Image:    image,
+		}
+
+		if err := c.imageStore.Add(img); err != nil {
+			return nil, fmt.Errorf("failed to add image %q into store: %v", id, err)
+		}
+		glog.V(4).Infof("Imported image with id %q, repo tag %q", id, repoTag)
+	}
+	return &api.LoadImageResponse{Images: repoTags}, nil
+}

pkg/server/image_pull.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
 	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
 )
 
 // For image management:
@@ -75,7 +76,7 @@ import (
 // PullImage pulls an image with authentication config.
 func (c *criContainerdService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (*runtime.PullImageResponse, error) {
 	imageRef := r.GetImage().GetImage()
-	namedRef, err := normalizeImageRef(imageRef)
+	namedRef, err := util.NormalizeImageRef(imageRef)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse image reference %q: %v", imageRef, err)
 	}

pkg/server/instrumented_service.go

@@ -20,6 +20,8 @@ import (
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+
+	api "github.com/kubernetes-incubator/cri-containerd/pkg/api/v1"
 )
 
 // instrumentedService wraps service and logs each operation.
@@ -316,3 +318,15 @@ func (in *instrumentedService) ListContainerStats(ctx context.Context, r *runtim
 	}()
 	return in.criContainerdService.ListContainerStats(ctx, r)
 }
+
+func (in *instrumentedService) LoadImage(ctx context.Context, r *api.LoadImageRequest) (res *api.LoadImageResponse, err error) {
+	glog.V(4).Infof("LoadImage from file %q", r.GetFilePath())
+	defer func() {
+		if err != nil {
+			glog.Errorf("LoadImage failed, error: %v", err)
+		} else {
+			glog.V(4).Infof("LoadImage returns images %+v", res.GetImages())
+		}
+	}()
+	return in.criContainerdService.LoadImage(ctx, r)
+}

pkg/server/sandbox_run.go

@@ -33,7 +33,7 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	customopts "github.com/kubernetes-incubator/cri-containerd/pkg/opts"
+	customopts "github.com/kubernetes-incubator/cri-containerd/pkg/containerd/opts"
 	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
 )

pkg/server/service.go

@@ -41,6 +41,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 
 	"github.com/kubernetes-incubator/cri-containerd/cmd/cri-containerd/options"
+	api "github.com/kubernetes-incubator/cri-containerd/pkg/api/v1"
 	osinterface "github.com/kubernetes-incubator/cri-containerd/pkg/os"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/registrar"
 	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
@@ -62,6 +63,7 @@ type CRIContainerdService interface {
 	Stop()
 	runtime.RuntimeServiceServer
 	runtime.ImageServiceServer
+	api.CRIContainerdServiceServer
 }
 
 // criContainerdService implements CRIContainerdService.
@@ -167,8 +169,10 @@ func NewCRIContainerdService(config options.Config) (CRIContainerdService, error
 
 	// Create the grpc server and register runtime and image services.
 	c.server = grpc.NewServer()
-	runtime.RegisterRuntimeServiceServer(c.server, newInstrumentedService(c))
-	runtime.RegisterImageServiceServer(c.server, newInstrumentedService(c))
+	instrumented := newInstrumentedService(c)
+	runtime.RegisterRuntimeServiceServer(c.server, instrumented)
+	runtime.RegisterImageServiceServer(c.server, instrumented)
+	api.RegisterCRIContainerdServiceServer(c.server, instrumented)
 
 	return newInstrumentedService(c), nil
 }

pkg/util/image.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"github.com/docker/distribution/reference"
+)
+
+// NormalizeImageRef normalizes the image reference following the docker convention. This is added
+// mainly for backward compatibility.
+// The reference returned can only be either tagged or digested. For reference contains both tag
+// and digest, the function returns digested reference, e.g. docker.io/library/busybox:latest@
+// sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa will be returned as
+// docker.io/library/busybox@sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa.
+func NormalizeImageRef(ref string) (reference.Named, error) {
+	named, err := reference.ParseNormalizedNamed(ref)
+	if err != nil {
+		return nil, err
+	}
+	if _, ok := named.(reference.NamedTagged); ok {
+		if canonical, ok := named.(reference.Canonical); ok {
+			// The reference is both tagged and digested, only
+			// return digested.
+			newNamed, err := reference.WithName(canonical.Name())
+			if err != nil {
+				return nil, err
+			}
+			newCanonical, err := reference.WithDigest(newNamed, canonical.Digest())
+			if err != nil {
+				return nil, err
+			}
+			return newCanonical, nil
+		}
+	}
+	return reference.TagNameOnly(named), nil
+}

pkg/util/image_test.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	"github.com/containerd/containerd/reference"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizeImageRef(t *testing.T) {
+	for _, test := range []struct {
+		input  string
+		expect string
+	}{
+		{ // has nothing
+			input:  "busybox",
+			expect: "docker.io/library/busybox:latest",
+		},
+		{ // only has tag
+			input:  "busybox:latest",
+			expect: "docker.io/library/busybox:latest",
+		},
+		{ // only has digest
+			input:  "busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+			expect: "docker.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		},
+		{ // only has path
+			input:  "library/busybox",
+			expect: "docker.io/library/busybox:latest",
+		},
+		{ // only has hostname
+			input:  "docker.io/busybox",
+			expect: "docker.io/library/busybox:latest",
+		},
+		{ // has no tag
+			input:  "docker.io/library/busybox",
+			expect: "docker.io/library/busybox:latest",
+		},
+		{ // has no path
+			input:  "docker.io/busybox:latest",
+			expect: "docker.io/library/busybox:latest",
+		},
+		{ // has no hostname
+			input:  "library/busybox:latest",
+			expect: "docker.io/library/busybox:latest",
+		},
+		{ // full reference
+			input:  "docker.io/library/busybox:latest",
+			expect: "docker.io/library/busybox:latest",
+		},
+		{ // gcr reference
+			input:  "gcr.io/library/busybox",
+			expect: "gcr.io/library/busybox:latest",
+		},
+		{ // both tag and digest
+			input:  "gcr.io/library/busybox:latest@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+			expect: "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		},
+	} {
+		t.Logf("TestCase %q", test.input)
+		normalized, err := NormalizeImageRef(test.input)
+		assert.NoError(t, err)
+		output := normalized.String()
+		assert.Equal(t, test.expect, output)
+		_, err = reference.Parse(output)
+		assert.NoError(t, err, "%q should be containerd supported reference", output)
+	}
+}

vendor.conf

@@ -49,8 +49,8 @@ github.com/PuerkitoBio/purell v1.0.0
 github.com/PuerkitoBio/urlesc 5bd2802263f21d8788851d5305584c82a5c75d7e
 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 
 github.com/sirupsen/logrus v1.0.0
-github.com/spf13/cobra f62e98d28ab7ad31d707ba837a966378465c7b57
-github.com/spf13/pflag 9ff6c6923cfffbcd502984b8e0c80539a94968b7
+github.com/spf13/cobra v0.0.1
+github.com/spf13/pflag v1.0.0
 github.com/stretchr/testify v1.1.4
 github.com/syndtr/gocapability e7cb7fa329f456b3855136a2642b197bad7366ba
 github.com/tchap/go-patricia 5ad6cdb7538b0097d5598c7e57f0a24072adf7dc

vendor/github.com/docker/docker/hack/README.md

@@ -0,0 +1,60 @@
+## About
+
+This directory contains a collection of scripts used to build and manage this
+repository. If there are any issues regarding the intention of a particular
+script (or even part of a certain script), please reach out to us.
+It may help us either refine our current scripts, or add on new ones
+that are appropriate for a given use case.
+
+## DinD (dind.sh)
+
+DinD is a wrapper script which allows Docker to be run inside a Docker
+container. DinD requires the container to
+be run with privileged mode enabled.
+
+## Generate Authors (generate-authors.sh)
+
+Generates AUTHORS; a file with all the names and corresponding emails of
+individual contributors. AUTHORS can be found in the home directory of
+this repository.
+
+## Make
+
+There are two make files, each with different extensions. Neither are supposed
+to be called directly; only invoke `make`. Both scripts run inside a Docker
+container.
+
+### make.ps1
+
+- The Windows native build script that uses PowerShell semantics; it is limited
+unlike `hack\make.sh` since it does not provide support for the full set of
+operations provided by the Linux counterpart, `make.sh`. However, `make.ps1`
+does provide support for local Windows development and Windows to Windows CI.
+More information is found within `make.ps1` by the author, @jhowardmsft
+
+### make.sh
+
+- Referenced via `make test` when running tests on a local machine,
+or directly referenced when running tests inside a Docker development container.  
+- When running on a local machine, `make test` to run all tests found in
+`test`, `test-unit`, `test-integration`, and `test-docker-py` on
+your local machine. The default timeout is set in `make.sh` to 60 minutes
+(`${TIMEOUT:=60m}`), since it currently takes up to an hour to run
+all of the tests.
+- When running inside a Docker development container, `hack/make.sh` does
+not have a single target that runs all the tests. You need to provide a
+single command line with multiple targets that performs the same thing.
+An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration test-docker-py`
+- For more information related to testing outside the scope of this README,
+refer to
+[Run tests and test documentation](https://docs.docker.com/opensource/project/test-and-docs/)
+
+## Release (release.sh)
+
+Releases any bundles built by `make` on a public AWS S3 bucket.
+For information regarding configuration, please view `release.sh`.
+
+## Vendor (vendor.sh)
+
+A shell script that is a wrapper around Vndr. For information on how to use
+this, please refer to [vndr's README](https://github.com/LK4D4/vndr/blob/master/README.md)

vendor/github.com/docker/docker/hack/integration-cli-on-swarm/README.md

@@ -0,0 +1,69 @@
+# Integration Testing on Swarm
+
+IT on Swarm allows you to execute integration test in parallel across a Docker Swarm cluster
+
+## Architecture
+
+### Master service
+
+  - Works as a funker caller
+  - Calls a worker funker (`-worker-service`) with a chunk of `-check.f` filter strings (passed as a file via `-input` flag, typically `/mnt/input`)
+
+### Worker service
+
+  - Works as a funker callee
+  - Executes an equivalent of `TESTFLAGS=-check.f TestFoo|TestBar|TestBaz ... make test-integration-cli` using the bind-mounted API socket (`docker.sock`)
+
+### Client
+
+  - Controls master and workers via `docker stack`
+  - No need to have a local daemon
+
+Typically, the master and workers are supposed to be running on a cloud environment,
+while the client is supposed to be running on a laptop, e.g. Docker for Mac/Windows.
+
+## Requirement
+
+  - Docker daemon 1.13 or later
+  - Private registry for distributed execution with multiple nodes
+
+## Usage
+
+### Step 1: Prepare images
+
+    $ make build-integration-cli-on-swarm
+
+Following environment variables are known to work in this step:
+
+ - `BUILDFLAGS`
+ - `DOCKER_INCREMENTAL_BINARY`
+
+Note: during the transition into Moby Project, you might need to create a symbolic link `$GOPATH/src/github.com/docker/docker` to `$GOPATH/src/github.com/moby/moby`. 
+
+### Step 2: Execute tests
+
+    $ ./hack/integration-cli-on-swarm/integration-cli-on-swarm -replicas 40 -push-worker-image YOUR_REGISTRY.EXAMPLE.COM/integration-cli-worker:latest 
+
+Following environment variables are known to work in this step:
+
+ - `DOCKER_GRAPHDRIVER`
+ - `DOCKER_EXPERIMENTAL`
+
+#### Flags
+
+Basic flags:
+
+ - `-replicas N`: the number of worker service replicas. i.e. degree of parallelism.
+ - `-chunks N`: the number of chunks. By default, `chunks` == `replicas`.
+ - `-push-worker-image REGISTRY/IMAGE:TAG`: push the worker image to the registry. Note that if you have only single node and hence you do not need a private registry, you do not need to specify `-push-worker-image`.
+
+Experimental flags for mitigating makespan nonuniformity:
+
+ - `-shuffle`: Shuffle the test filter strings
+
+Flags for debugging IT on Swarm itself:
+
+ - `-rand-seed N`: the random seed. This flag is useful for deterministic replaying. By default(0), the timestamp is used.
+ - `-filters-file FILE`: the file contains `-check.f` strings. By default, the file is automatically generated.
+ - `-dry-run`: skip the actual workload
+ - `keep-executor`: do not auto-remove executor containers, which is used for running privileged programs on Swarm

vendor/github.com/docker/docker/hack/integration-cli-on-swarm/agent/vendor.conf

@@ -0,0 +1,2 @@
+# dependencies specific to worker (i.e. github.com/docker/docker/...) are not vendored here
+github.com/bfirsh/funker-go eaa0a2e06f30e72c9a0b7f858951e581e26ef773

vendor/github.com/spf13/cobra/args.go

@@ -0,0 +1,98 @@
+package cobra
+
+import (
+	"fmt"
+)
+
+type PositionalArgs func(cmd *Command, args []string) error
+
+// Legacy arg validation has the following behaviour:
+// - root commands with no subcommands can take arbitrary arguments
+// - root commands with subcommands will do subcommand validity checking
+// - subcommands will always accept arbitrary arguments
+func legacyArgs(cmd *Command, args []string) error {
+	// no subcommand, always take args
+	if !cmd.HasSubCommands() {
+		return nil
+	}
+
+	// root command with subcommands, do subcommand checking
+	if !cmd.HasParent() && len(args) > 0 {
+		return fmt.Errorf("unknown command %q for %q%s", args[0], cmd.CommandPath(), cmd.findSuggestions(args[0]))
+	}
+	return nil
+}
+
+// NoArgs returns an error if any args are included
+func NoArgs(cmd *Command, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("unknown command %q for %q", args[0], cmd.CommandPath())
+	}
+	return nil
+}
+
+// OnlyValidArgs returns an error if any args are not in the list of ValidArgs
+func OnlyValidArgs(cmd *Command, args []string) error {
+	if len(cmd.ValidArgs) > 0 {
+		for _, v := range args {
+			if !stringInSlice(v, cmd.ValidArgs) {
+				return fmt.Errorf("invalid argument %q for %q%s", v, cmd.CommandPath(), cmd.findSuggestions(args[0]))
+			}
+		}
+	}
+	return nil
+}
+
+func stringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}
+
+// ArbitraryArgs never returns an error
+func ArbitraryArgs(cmd *Command, args []string) error {
+	return nil
+}
+
+// MinimumNArgs returns an error if there is not at least N args
+func MinimumNArgs(n int) PositionalArgs {
+	return func(cmd *Command, args []string) error {
+		if len(args) < n {
+			return fmt.Errorf("requires at least %d arg(s), only received %d", n, len(args))
+		}
+		return nil
+	}
+}
+
+// MaximumNArgs returns an error if there are more than N args
+func MaximumNArgs(n int) PositionalArgs {
+	return func(cmd *Command, args []string) error {
+		if len(args) > n {
+			return fmt.Errorf("accepts at most %d arg(s), received %d", n, len(args))
+		}
+		return nil
+	}
+}
+
+// ExactArgs returns an error if there are not exactly n args
+func ExactArgs(n int) PositionalArgs {
+	return func(cmd *Command, args []string) error {
+		if len(args) != n {
+			return fmt.Errorf("accepts %d arg(s), received %d", n, len(args))
+		}
+		return nil
+	}
+}
+
+// RangeArgs returns an error if the number of args is not within the expected range
+func RangeArgs(min int, max int) PositionalArgs {
+	return func(cmd *Command, args []string) error {
+		if len(args) < min || len(args) > max {
+			return fmt.Errorf("accepts between %d and %d arg(s), received %d", min, max, len(args))
+		}
+		return nil
+	}
+}

vendor/github.com/spf13/cobra/bash_completions.go

@@ -1,6 +1,7 @@
 package cobra
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"os"
@@ -10,19 +11,17 @@ import (
 	"github.com/spf13/pflag"
 )
 
+// Annotations for Bash completion.
 const (
-	BashCompFilenameExt     = "cobra_annotation_bash_completion_filename_extentions"
+	BashCompFilenameExt     = "cobra_annotation_bash_completion_filename_extensions"
 	BashCompCustom          = "cobra_annotation_bash_completion_custom"
 	BashCompOneRequiredFlag = "cobra_annotation_bash_completion_one_required_flag"
 	BashCompSubdirsInDir    = "cobra_annotation_bash_completion_subdirs_in_dir"
 )
 
-func preamble(out io.Writer, name string) error {
-	_, err := fmt.Fprintf(out, "# bash completion for %-36s -*- shell-script -*-\n", name)
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprint(out, `
+func writePreamble(buf *bytes.Buffer, name string) {
+	buf.WriteString(fmt.Sprintf("# bash completion for %-36s -*- shell-script -*-\n", name))
+	buf.WriteString(`
 __debug()
 {
     if [[ -n ${BASH_COMP_DEBUG_FILE} ]]; then
@@ -87,13 +86,13 @@ __handle_reply()
                 local index flag
                 flag="${cur%%=*}"
                 __index_of_word "${flag}" "${flags_with_completion[@]}"
+                COMPREPLY=()
                 if [[ ${index} -ge 0 ]]; then
-                    COMPREPLY=()
                     PREFIX=""
                     cur="${cur#*=}"
                     ${flags_completion[${index}]}
                     if [ -n "${ZSH_VERSION}" ]; then
-                        # zfs completion needs --flag= prefix
+                        # zsh completion needs --flag= prefix
                         eval "COMPREPLY=( \"\${COMPREPLY[@]/#/${flag}=}\" )"
                     fi
                 fi
@@ -133,7 +132,10 @@ __handle_reply()
         declare -F __custom_func >/dev/null && __custom_func
     fi
 
-    __ltrim_colon_completions "$cur"
+    # available in bash-completion >= 2, not always present on macOS
+    if declare -F __ltrim_colon_completions >/dev/null; then
+        __ltrim_colon_completions "$cur"
+    fi
 }
 
 # The arguments should be in the form "ext1|ext2|extn"
@@ -224,7 +226,7 @@ __handle_command()
     fi
     c=$((c+1))
     __debug "${FUNCNAME[0]}: looking for ${next_command}"
-    declare -F $next_command >/dev/null && $next_command
+    declare -F "$next_command" >/dev/null && $next_command
 }
 
 __handle_word()
@@ -247,16 +249,12 @@ __handle_word()
 }
 
 `)
-	return err
 }
 
-func postscript(w io.Writer, name string) error {
+func writePostscript(buf *bytes.Buffer, name string) {
 	name = strings.Replace(name, ":", "__", -1)
-	_, err := fmt.Fprintf(w, "__start_%s()\n", name)
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprintf(w, `{
+	buf.WriteString(fmt.Sprintf("__start_%s()\n", name))
+	buf.WriteString(fmt.Sprintf(`{
     local cur prev words cword
     declare -A flaghash 2>/dev/null || :
     if declare -F _init_completion >/dev/null 2>&1; then
@@ -280,318 +278,227 @@ func postscript(w io.Writer, name string) error {
     __handle_word
 }
 
-`, name)
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprintf(w, `if [[ $(type -t compopt) = "builtin" ]]; then
+`, name))
+	buf.WriteString(fmt.Sprintf(`if [[ $(type -t compopt) = "builtin" ]]; then
     complete -o default -F __start_%s %s
 else
     complete -o default -o nospace -F __start_%s %s
 fi
 
-`, name, name, name, name)
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprintf(w, "# ex: ts=4 sw=4 et filetype=sh\n")
-	return err
+`, name, name, name, name))
+	buf.WriteString("# ex: ts=4 sw=4 et filetype=sh\n")
 }
 
-func writeCommands(cmd *Command, w io.Writer) error {
-	if _, err := fmt.Fprintf(w, "    commands=()\n"); err != nil {
-		return err
-	}
+func writeCommands(buf *bytes.Buffer, cmd *Command) {
+	buf.WriteString("    commands=()\n")
 	for _, c := range cmd.Commands() {
 		if !c.IsAvailableCommand() || c == cmd.helpCommand {
 			continue
 		}
-		if _, err := fmt.Fprintf(w, "    commands+=(%q)\n", c.Name()); err != nil {
-			return err
-		}
+		buf.WriteString(fmt.Sprintf("    commands+=(%q)\n", c.Name()))
 	}
-	_, err := fmt.Fprintf(w, "\n")
-	return err
+	buf.WriteString("\n")
 }
 
-func writeFlagHandler(name string, annotations map[string][]string, w io.Writer) error {
+func writeFlagHandler(buf *bytes.Buffer, name string, annotations map[string][]string) {
 	for key, value := range annotations {
 		switch key {
 		case BashCompFilenameExt:
-			_, err := fmt.Fprintf(w, "    flags_with_completion+=(%q)\n", name)
-			if err != nil {
-				return err
-			}
+			buf.WriteString(fmt.Sprintf("    flags_with_completion+=(%q)\n", name))
 
+			var ext string
 			if len(value) > 0 {
-				ext := "__handle_filename_extension_flag " + strings.Join(value, "|")
-				_, err = fmt.Fprintf(w, "    flags_completion+=(%q)\n", ext)
+				ext = "__handle_filename_extension_flag " + strings.Join(value, "|")
 			} else {
-				ext := "_filedir"
-				_, err = fmt.Fprintf(w, "    flags_completion+=(%q)\n", ext)
-			}
-			if err != nil {
-				return err
+				ext = "_filedir"
 			}
+			buf.WriteString(fmt.Sprintf("    flags_completion+=(%q)\n", ext))
 		case BashCompCustom:
-			_, err := fmt.Fprintf(w, "    flags_with_completion+=(%q)\n", name)
-			if err != nil {
-				return err
-			}
+			buf.WriteString(fmt.Sprintf("    flags_with_completion+=(%q)\n", name))
 			if len(value) > 0 {
 				handlers := strings.Join(value, "; ")
-				_, err = fmt.Fprintf(w, "    flags_completion+=(%q)\n", handlers)
+				buf.WriteString(fmt.Sprintf("    flags_completion+=(%q)\n", handlers))
 			} else {
-				_, err = fmt.Fprintf(w, "    flags_completion+=(:)\n")
-			}
-			if err != nil {
-				return err
+				buf.WriteString("    flags_completion+=(:)\n")
 			}
 		case BashCompSubdirsInDir:
-			_, err := fmt.Fprintf(w, "    flags_with_completion+=(%q)\n", name)
+			buf.WriteString(fmt.Sprintf("    flags_with_completion+=(%q)\n", name))
 
+			var ext string
 			if len(value) == 1 {
-				ext := "__handle_subdirs_in_dir_flag " + value[0]
-				_, err = fmt.Fprintf(w, "    flags_completion+=(%q)\n", ext)
+				ext = "__handle_subdirs_in_dir_flag " + value[0]
 			} else {
-				ext := "_filedir -d"
-				_, err = fmt.Fprintf(w, "    flags_completion+=(%q)\n", ext)
-			}
-			if err != nil {
-				return err
+				ext = "_filedir -d"
 			}
+			buf.WriteString(fmt.Sprintf("    flags_completion+=(%q)\n", ext))
 		}
 	}
-	return nil
 }
 
-func writeShortFlag(flag *pflag.Flag, w io.Writer) error {
-	b := (len(flag.NoOptDefVal) > 0)
+func writeShortFlag(buf *bytes.Buffer, flag *pflag.Flag) {
 	name := flag.Shorthand
 	format := "    "
-	if !b {
+	if len(flag.NoOptDefVal) == 0 {
 		format += "two_word_"
 	}
 	format += "flags+=(\"-%s\")\n"
-	if _, err := fmt.Fprintf(w, format, name); err != nil {
-		return err
-	}
-	return writeFlagHandler("-"+name, flag.Annotations, w)
+	buf.WriteString(fmt.Sprintf(format, name))
+	writeFlagHandler(buf, "-"+name, flag.Annotations)
 }
 
-func writeFlag(flag *pflag.Flag, w io.Writer) error {
-	b := (len(flag.NoOptDefVal) > 0)
+func writeFlag(buf *bytes.Buffer, flag *pflag.Flag) {
 	name := flag.Name
 	format := "    flags+=(\"--%s"
-	if !b {
+	if len(flag.NoOptDefVal) == 0 {
 		format += "="
 	}
 	format += "\")\n"
-	if _, err := fmt.Fprintf(w, format, name); err != nil {
-		return err
-	}
-	return writeFlagHandler("--"+name, flag.Annotations, w)
+	buf.WriteString(fmt.Sprintf(format, name))
+	writeFlagHandler(buf, "--"+name, flag.Annotations)
 }
 
-func writeLocalNonPersistentFlag(flag *pflag.Flag, w io.Writer) error {
-	b := (len(flag.NoOptDefVal) > 0)
+func writeLocalNonPersistentFlag(buf *bytes.Buffer, flag *pflag.Flag) {
 	name := flag.Name
 	format := "    local_nonpersistent_flags+=(\"--%s"
-	if !b {
+	if len(flag.NoOptDefVal) == 0 {
 		format += "="
 	}
 	format += "\")\n"
-	if _, err := fmt.Fprintf(w, format, name); err != nil {
-		return err
-	}
-	return nil
+	buf.WriteString(fmt.Sprintf(format, name))
 }
 
-func writeFlags(cmd *Command, w io.Writer) error {
-	_, err := fmt.Fprintf(w, `    flags=()
+func writeFlags(buf *bytes.Buffer, cmd *Command) {
+	buf.WriteString(`    flags=()
     two_word_flags=()
     local_nonpersistent_flags=()
     flags_with_completion=()
     flags_completion=()
 
 `)
-	if err != nil {
-		return err
-	}
 	localNonPersistentFlags := cmd.LocalNonPersistentFlags()
-	var visitErr error
 	cmd.NonInheritedFlags().VisitAll(func(flag *pflag.Flag) {
-		if err := writeFlag(flag, w); err != nil {
-			visitErr = err
+		if nonCompletableFlag(flag) {
 			return
 		}
+		writeFlag(buf, flag)
 		if len(flag.Shorthand) > 0 {
-			if err := writeShortFlag(flag, w); err != nil {
-				visitErr = err
-				return
-			}
+			writeShortFlag(buf, flag)
 		}
 		if localNonPersistentFlags.Lookup(flag.Name) != nil {
-			if err := writeLocalNonPersistentFlag(flag, w); err != nil {
-				visitErr = err
-				return
-			}
+			writeLocalNonPersistentFlag(buf, flag)
 		}
 	})
-	if visitErr != nil {
-		return visitErr
-	}
 	cmd.InheritedFlags().VisitAll(func(flag *pflag.Flag) {
-		if err := writeFlag(flag, w); err != nil {
-			visitErr = err
+		if nonCompletableFlag(flag) {
 			return
 		}
+		writeFlag(buf, flag)
 		if len(flag.Shorthand) > 0 {
-			if err := writeShortFlag(flag, w); err != nil {
-				visitErr = err
-				return
-			}
+			writeShortFlag(buf, flag)
 		}
 	})
-	if visitErr != nil {
-		return visitErr
-	}
 
-	_, err = fmt.Fprintf(w, "\n")
-	return err
+	buf.WriteString("\n")
 }
 
-func writeRequiredFlag(cmd *Command, w io.Writer) error {
-	if _, err := fmt.Fprintf(w, "    must_have_one_flag=()\n"); err != nil {
-		return err
-	}
+func writeRequiredFlag(buf *bytes.Buffer, cmd *Command) {
+	buf.WriteString("    must_have_one_flag=()\n")
 	flags := cmd.NonInheritedFlags()
-	var visitErr error
 	flags.VisitAll(func(flag *pflag.Flag) {
+		if nonCompletableFlag(flag) {
+			return
+		}
 		for key := range flag.Annotations {
 			switch key {
 			case BashCompOneRequiredFlag:
 				format := "    must_have_one_flag+=(\"--%s"
-				b := (flag.Value.Type() == "bool")
-				if !b {
+				if flag.Value.Type() != "bool" {
 					format += "="
 				}
 				format += "\")\n"
-				if _, err := fmt.Fprintf(w, format, flag.Name); err != nil {
-					visitErr = err
-					return
-				}
+				buf.WriteString(fmt.Sprintf(format, flag.Name))
 
 				if len(flag.Shorthand) > 0 {
-					if _, err := fmt.Fprintf(w, "    must_have_one_flag+=(\"-%s\")\n", flag.Shorthand); err != nil {
-						visitErr = err
-						return
-					}
+					buf.WriteString(fmt.Sprintf("    must_have_one_flag+=(\"-%s\")\n", flag.Shorthand))
 				}
 			}
 		}
 	})
-	return visitErr
 }
 
-func writeRequiredNouns(cmd *Command, w io.Writer) error {
-	if _, err := fmt.Fprintf(w, "    must_have_one_noun=()\n"); err != nil {
-		return err
-	}
+func writeRequiredNouns(buf *bytes.Buffer, cmd *Command) {
+	buf.WriteString("    must_have_one_noun=()\n")
 	sort.Sort(sort.StringSlice(cmd.ValidArgs))
 	for _, value := range cmd.ValidArgs {
-		if _, err := fmt.Fprintf(w, "    must_have_one_noun+=(%q)\n", value); err != nil {
-			return err
-		}
+		buf.WriteString(fmt.Sprintf("    must_have_one_noun+=(%q)\n", value))
 	}
-	return nil
 }
 
-func writeArgAliases(cmd *Command, w io.Writer) error {
-	if _, err := fmt.Fprintf(w, "    noun_aliases=()\n"); err != nil {
-		return err
-	}
+func writeArgAliases(buf *bytes.Buffer, cmd *Command) {
+	buf.WriteString("    noun_aliases=()\n")
 	sort.Sort(sort.StringSlice(cmd.ArgAliases))
 	for _, value := range cmd.ArgAliases {
-		if _, err := fmt.Fprintf(w, "    noun_aliases+=(%q)\n", value); err != nil {
-			return err
-		}
+		buf.WriteString(fmt.Sprintf("    noun_aliases+=(%q)\n", value))
 	}
-	return nil
 }
 
-func gen(cmd *Command, w io.Writer) error {
+func gen(buf *bytes.Buffer, cmd *Command) {
 	for _, c := range cmd.Commands() {
 		if !c.IsAvailableCommand() || c == cmd.helpCommand {
 			continue
 		}
-		if err := gen(c, w); err != nil {
-			return err
-		}
+		gen(buf, c)
 	}
 	commandName := cmd.CommandPath()
 	commandName = strings.Replace(commandName, " ", "_", -1)
 	commandName = strings.Replace(commandName, ":", "__", -1)
-	if _, err := fmt.Fprintf(w, "_%s()\n{\n", commandName); err != nil {
-		return err
-	}
-	if _, err := fmt.Fprintf(w, "    last_command=%q\n", commandName); err != nil {
-		return err
-	}
-	if err := writeCommands(cmd, w); err != nil {
-		return err
-	}
-	if err := writeFlags(cmd, w); err != nil {
-		return err
-	}
-	if err := writeRequiredFlag(cmd, w); err != nil {
-		return err
-	}
-	if err := writeRequiredNouns(cmd, w); err != nil {
-		return err
-	}
-	if err := writeArgAliases(cmd, w); err != nil {
-		return err
-	}
-	if _, err := fmt.Fprintf(w, "}\n\n"); err != nil {
-		return err
-	}
-	return nil
+	buf.WriteString(fmt.Sprintf("_%s()\n{\n", commandName))
+	buf.WriteString(fmt.Sprintf("    last_command=%q\n", commandName))
+	writeCommands(buf, cmd)
+	writeFlags(buf, cmd)
+	writeRequiredFlag(buf, cmd)
+	writeRequiredNouns(buf, cmd)
+	writeArgAliases(buf, cmd)
+	buf.WriteString("}\n\n")
 }
 
-func (cmd *Command) GenBashCompletion(w io.Writer) error {
-	if err := preamble(w, cmd.Name()); err != nil {
-		return err
+// GenBashCompletion generates bash completion file and writes to the passed writer.
+func (c *Command) GenBashCompletion(w io.Writer) error {
+	buf := new(bytes.Buffer)
+	writePreamble(buf, c.Name())
+	if len(c.BashCompletionFunction) > 0 {
+		buf.WriteString(c.BashCompletionFunction + "\n")
 	}
-	if len(cmd.BashCompletionFunction) > 0 {
-		if _, err := fmt.Fprintf(w, "%s\n", cmd.BashCompletionFunction); err != nil {
-			return err
-		}
-	}
-	if err := gen(cmd, w); err != nil {
-		return err
-	}
-	return postscript(w, cmd.Name())
+	gen(buf, c)
+	writePostscript(buf, c.Name())
+
+	_, err := buf.WriteTo(w)
+	return err
 }
 
-func (cmd *Command) GenBashCompletionFile(filename string) error {
+func nonCompletableFlag(flag *pflag.Flag) bool {
+	return flag.Hidden || len(flag.Deprecated) > 0
+}
+
+// GenBashCompletionFile generates bash completion file.
+func (c *Command) GenBashCompletionFile(filename string) error {
 	outFile, err := os.Create(filename)
 	if err != nil {
 		return err
 	}
 	defer outFile.Close()
 
-	return cmd.GenBashCompletion(outFile)
+	return c.GenBashCompletion(outFile)
 }
 
 // MarkFlagRequired adds the BashCompOneRequiredFlag annotation to the named flag, if it exists.
-func (cmd *Command) MarkFlagRequired(name string) error {
-	return MarkFlagRequired(cmd.Flags(), name)
+func (c *Command) MarkFlagRequired(name string) error {
+	return MarkFlagRequired(c.Flags(), name)
 }
 
 // MarkPersistentFlagRequired adds the BashCompOneRequiredFlag annotation to the named persistent flag, if it exists.
-func (cmd *Command) MarkPersistentFlagRequired(name string) error {
-	return MarkFlagRequired(cmd.PersistentFlags(), name)
+func (c *Command) MarkPersistentFlagRequired(name string) error {
+	return MarkFlagRequired(c.PersistentFlags(), name)
 }
 
 // MarkFlagRequired adds the BashCompOneRequiredFlag annotation to the named flag in the flag set, if it exists.
@@ -601,20 +508,20 @@ func MarkFlagRequired(flags *pflag.FlagSet, name string) error {
 
 // MarkFlagFilename adds the BashCompFilenameExt annotation to the named flag, if it exists.
 // Generated bash autocompletion will select filenames for the flag, limiting to named extensions if provided.
-func (cmd *Command) MarkFlagFilename(name string, extensions ...string) error {
-	return MarkFlagFilename(cmd.Flags(), name, extensions...)
+func (c *Command) MarkFlagFilename(name string, extensions ...string) error {
+	return MarkFlagFilename(c.Flags(), name, extensions...)
 }
 
 // MarkFlagCustom adds the BashCompCustom annotation to the named flag, if it exists.
 // Generated bash autocompletion will call the bash function f for the flag.
-func (cmd *Command) MarkFlagCustom(name string, f string) error {
-	return MarkFlagCustom(cmd.Flags(), name, f)
+func (c *Command) MarkFlagCustom(name string, f string) error {
+	return MarkFlagCustom(c.Flags(), name, f)
 }
 
 // MarkPersistentFlagFilename adds the BashCompFilenameExt annotation to the named persistent flag, if it exists.
 // Generated bash autocompletion will select filenames for the flag, limiting to named extensions if provided.
-func (cmd *Command) MarkPersistentFlagFilename(name string, extensions ...string) error {
-	return MarkFlagFilename(cmd.PersistentFlags(), name, extensions...)
+func (c *Command) MarkPersistentFlagFilename(name string, extensions ...string) error {
+	return MarkFlagFilename(c.PersistentFlags(), name, extensions...)
 }
 
 // MarkFlagFilename adds the BashCompFilenameExt annotation to the named flag in the flag set, if it exists.

vendor/github.com/spf13/cobra/cobra.go

@@ -27,48 +27,59 @@ import (
 )
 
 var templateFuncs = template.FuncMap{
-	"trim":               strings.TrimSpace,
-	"trimRightSpace":     trimRightSpace,
-	"appendIfNotPresent": appendIfNotPresent,
-	"rpad":               rpad,
-	"gt":                 Gt,
-	"eq":                 Eq,
+	"trim":                    strings.TrimSpace,
+	"trimRightSpace":          trimRightSpace,
+	"trimTrailingWhitespaces": trimRightSpace,
+	"appendIfNotPresent":      appendIfNotPresent,
+	"rpad":                    rpad,
+	"gt":                      Gt,
+	"eq":                      Eq,
 }
 
 var initializers []func()
 
-// automatic prefix matching can be a dangerous thing to automatically enable in CLI tools.
-// Set this to true to enable it
+// EnablePrefixMatching allows to set automatic prefix matching. Automatic prefix matching can be a dangerous thing
+// to automatically enable in CLI tools.
+// Set this to true to enable it.
 var EnablePrefixMatching = false
 
-//EnableCommandSorting controls sorting of the slice of commands, which is turned on by default.
-//To disable sorting, set it to false.
+// EnableCommandSorting controls sorting of the slice of commands, which is turned on by default.
+// To disable sorting, set it to false.
 var EnableCommandSorting = true
 
-//AddTemplateFunc adds a template function that's available to Usage and Help
-//template generation.
+// MousetrapHelpText enables an information splash screen on Windows
+// if the CLI is started from explorer.exe.
+// To disable the mousetrap, just set this variable to blank string ("").
+// Works only on Microsoft Windows.
+var MousetrapHelpText string = `This is a command line tool.
+
+You need to open cmd.exe and run it from there.
+`
+
+// AddTemplateFunc adds a template function that's available to Usage and Help
+// template generation.
 func AddTemplateFunc(name string, tmplFunc interface{}) {
 	templateFuncs[name] = tmplFunc
 }
 
-//AddTemplateFuncs adds multiple template functions availalble to Usage and
-//Help template generation.
+// AddTemplateFuncs adds multiple template functions that are available to Usage and
+// Help template generation.
 func AddTemplateFuncs(tmplFuncs template.FuncMap) {
 	for k, v := range tmplFuncs {
 		templateFuncs[k] = v
 	}
 }
 
-//OnInitialize takes a series of func() arguments and appends them to a slice of func().
+// OnInitialize takes a series of func() arguments and appends them to a slice of func().
 func OnInitialize(y ...func()) {
-	for _, x := range y {
-		initializers = append(initializers, x)
-	}
+	initializers = append(initializers, y...)
 }
 
-//Gt takes two types and checks whether the first type is greater than the second. In case of types Arrays, Chans,
-//Maps and Slices, Gt will compare their lengths. Ints are compared directly while strings are first parsed as
-//ints and then compared.
+// FIXME Gt is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra.
+
+// Gt takes two types and checks whether the first type is greater than the second. In case of types Arrays, Chans,
+// Maps and Slices, Gt will compare their lengths. Ints are compared directly while strings are first parsed as
+// ints and then compared.
 func Gt(a interface{}, b interface{}) bool {
 	var left, right int64
 	av := reflect.ValueOf(a)
@@ -96,7 +107,9 @@ func Gt(a interface{}, b interface{}) bool {
 	return left > right
 }
 
-//Eq takes two types and checks whether they are equal. Supported types are int and string. Unsupported types will panic.
+// FIXME Eq is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra.
+
+// Eq takes two types and checks whether they are equal. Supported types are int and string. Unsupported types will panic.
 func Eq(a interface{}, b interface{}) bool {
 	av := reflect.ValueOf(a)
 	bv := reflect.ValueOf(b)
@@ -116,7 +129,9 @@ func trimRightSpace(s string) string {
 	return strings.TrimRightFunc(s, unicode.IsSpace)
 }
 
-// appendIfNotPresent will append stringToAppend to the end of s, but only if it's not yet present in s
+// FIXME appendIfNotPresent is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra.
+
+// appendIfNotPresent will append stringToAppend to the end of s, but only if it's not yet present in s.
 func appendIfNotPresent(s, stringToAppend string) string {
 	if strings.Contains(s, stringToAppend) {
 		return s
@@ -124,7 +139,7 @@ func appendIfNotPresent(s, stringToAppend string) string {
 	return s + " " + stringToAppend
 }
 
-//rpad adds padding to the right of a string
+// rpad adds padding to the right of a string.
 func rpad(s string, padding int) string {
 	template := fmt.Sprintf("%%-%ds", padding)
 	return fmt.Sprintf(template, s)
@@ -138,7 +153,7 @@ func tmpl(w io.Writer, text string, data interface{}) error {
 	return t.Execute(w, data)
 }
 
-// ld compares two strings and returns the levenshtein distance between them
+// ld compares two strings and returns the levenshtein distance between them.
 func ld(s, t string, ignoreCase bool) int {
 	if ignoreCase {
 		s = strings.ToLower(s)

vendor/github.com/spf13/cobra/command_win.go

@@ -11,14 +11,8 @@ import (
 
 var preExecHookFn = preExecHook
 
-// enables an information splash screen on Windows if the CLI is started from explorer.exe.
-var MousetrapHelpText string = `This is a command line tool
-
-You need to open cmd.exe and run it from there.
-`
-
 func preExecHook(c *Command) {
-	if mousetrap.StartedByExplorer() {
+	if MousetrapHelpText != "" && mousetrap.StartedByExplorer() {
 		c.Print(MousetrapHelpText)
 		time.Sleep(5 * time.Second)
 		os.Exit(1)

vendor/github.com/spf13/cobra/zsh_completions.go

@@ -0,0 +1,126 @@
+package cobra
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// GenZshCompletionFile generates zsh completion file.
+func (c *Command) GenZshCompletionFile(filename string) error {
+	outFile, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer outFile.Close()
+
+	return c.GenZshCompletion(outFile)
+}
+
+// GenZshCompletion generates a zsh completion file and writes to the passed writer.
+func (c *Command) GenZshCompletion(w io.Writer) error {
+	buf := new(bytes.Buffer)
+
+	writeHeader(buf, c)
+	maxDepth := maxDepth(c)
+	writeLevelMapping(buf, maxDepth)
+	writeLevelCases(buf, maxDepth, c)
+
+	_, err := buf.WriteTo(w)
+	return err
+}
+
+func writeHeader(w io.Writer, cmd *Command) {
+	fmt.Fprintf(w, "#compdef %s\n\n", cmd.Name())
+}
+
+func maxDepth(c *Command) int {
+	if len(c.Commands()) == 0 {
+		return 0
+	}
+	maxDepthSub := 0
+	for _, s := range c.Commands() {
+		subDepth := maxDepth(s)
+		if subDepth > maxDepthSub {
+			maxDepthSub = subDepth
+		}
+	}
+	return 1 + maxDepthSub
+}
+
+func writeLevelMapping(w io.Writer, numLevels int) {
+	fmt.Fprintln(w, `_arguments \`)
+	for i := 1; i <= numLevels; i++ {
+		fmt.Fprintf(w, `  '%d: :->level%d' \`, i, i)
+		fmt.Fprintln(w)
+	}
+	fmt.Fprintf(w, `  '%d: :%s'`, numLevels+1, "_files")
+	fmt.Fprintln(w)
+}
+
+func writeLevelCases(w io.Writer, maxDepth int, root *Command) {
+	fmt.Fprintln(w, "case $state in")
+	defer fmt.Fprintln(w, "esac")
+
+	for i := 1; i <= maxDepth; i++ {
+		fmt.Fprintf(w, "  level%d)\n", i)
+		writeLevel(w, root, i)
+		fmt.Fprintln(w, "  ;;")
+	}
+	fmt.Fprintln(w, "  *)")
+	fmt.Fprintln(w, "    _arguments '*: :_files'")
+	fmt.Fprintln(w, "  ;;")
+}
+
+func writeLevel(w io.Writer, root *Command, i int) {
+	fmt.Fprintf(w, "    case $words[%d] in\n", i)
+	defer fmt.Fprintln(w, "    esac")
+
+	commands := filterByLevel(root, i)
+	byParent := groupByParent(commands)
+
+	for p, c := range byParent {
+		names := names(c)
+		fmt.Fprintf(w, "      %s)\n", p)
+		fmt.Fprintf(w, "        _arguments '%d: :(%s)'\n", i, strings.Join(names, " "))
+		fmt.Fprintln(w, "      ;;")
+	}
+	fmt.Fprintln(w, "      *)")
+	fmt.Fprintln(w, "        _arguments '*: :_files'")
+	fmt.Fprintln(w, "      ;;")
+
+}
+
+func filterByLevel(c *Command, l int) []*Command {
+	cs := make([]*Command, 0)
+	if l == 0 {
+		cs = append(cs, c)
+		return cs
+	}
+	for _, s := range c.Commands() {
+		cs = append(cs, filterByLevel(s, l-1)...)
+	}
+	return cs
+}
+
+func groupByParent(commands []*Command) map[string][]*Command {
+	m := make(map[string][]*Command)
+	for _, c := range commands {
+		parent := c.Parent()
+		if parent == nil {
+			continue
+		}
+		m[parent.Name()] = append(m[parent.Name()], c)
+	}
+	return m
+}
+
+func names(commands []*Command) []string {
+	ns := make([]string, len(commands))
+	for i, c := range commands {
+		ns[i] = c.Name()
+	}
+	return ns
+}

vendor/github.com/spf13/pflag/README.md

@@ -246,6 +246,25 @@ It is possible to mark a flag as hidden, meaning it will still function as norma
 flags.MarkHidden("secretFlag")
 ```
 
+## Disable sorting of flags
+`pflag` allows you to disable sorting of flags for help and usage message.
+
+**Example**:
+```go
+flags.BoolP("verbose", "v", false, "verbose output")
+flags.String("coolflag", "yeaah", "it's really cool flag")
+flags.Int("usefulflag", 777, "sometimes it's very useful")
+flags.SortFlags = false
+flags.PrintDefaults()
+```
+**Output**:
+```
+  -v, --verbose           verbose output
+      --coolflag string   it's really cool flag (default "yeaah")
+      --usefulflag int    sometimes it's very useful (default 777)
+```
+
+
 ## Supporting Go flags when using pflag
 In order to support flags defined using Go's `flag` package, they must be added to the `pflag` flagset. This is usually necessary
 to support flags defined by third-party dependencies (e.g. `golang/glog`).
@@ -270,8 +289,8 @@ func main() {
 You can see the full reference documentation of the pflag package
 [at godoc.org][3], or through go's standard documentation system by
 running `godoc -http=:6060` and browsing to
-[http://localhost:6060/pkg/github.com/ogier/pflag][2] after
+[http://localhost:6060/pkg/github.com/spf13/pflag][2] after
 installation.
 
-[2]: http://localhost:6060/pkg/github.com/ogier/pflag
-[3]: http://godoc.org/github.com/ogier/pflag
+[2]: http://localhost:6060/pkg/github.com/spf13/pflag
+[3]: http://godoc.org/github.com/spf13/pflag

vendor/github.com/spf13/pflag/count.go

@@ -83,7 +83,9 @@ func (f *FlagSet) CountP(name, shorthand string, usage string) *int {
 	return p
 }
 
-// Count like Count only the flag is placed on the CommandLine isntead of a given flag set
+// Count defines a count flag with specified name, default value, and usage string.
+// The return value is the address of an int variable that stores the value of the flag.
+// A count flag will add 1 to its value evey time it is found on the command line
 func Count(name string, usage string) *int {
 	return CommandLine.CountP(name, "", usage)
 }

vendor/github.com/spf13/pflag/flag.go

@@ -16,9 +16,9 @@ pflag is a drop-in replacement of Go's native flag package. If you import
 pflag under the name "flag" then all code should continue to function
 with no changes.
 
-	import flag "github.com/ogier/pflag"
+	import flag "github.com/spf13/pflag"
 
-	There is one exception to this: if you directly instantiate the Flag struct
+There is one exception to this: if you directly instantiate the Flag struct
 there is one more field "Shorthand" that you will need to set.
 Most code never instantiates this struct directly, and instead uses
 functions such as String(), BoolVar(), and Var(), and is therefore
@@ -134,14 +134,21 @@ type FlagSet struct {
 	// a custom error handler.
 	Usage func()
 
+	// SortFlags is used to indicate, if user wants to have sorted flags in
+	// help/usage messages.
+	SortFlags bool
+
 	name              string
 	parsed            bool
 	actual            map[NormalizedName]*Flag
+	orderedActual     []*Flag
+	sortedActual      []*Flag
 	formal            map[NormalizedName]*Flag
+	orderedFormal     []*Flag
+	sortedFormal      []*Flag
 	shorthands        map[byte]*Flag
 	args              []string // arguments after flags
 	argsLenAtDash     int      // len(args) when a '--' was located when parsing, or -1 if no --
-	exitOnError       bool     // does the program exit if there's an error?
 	errorHandling     ErrorHandling
 	output            io.Writer // nil means stderr; use out() accessor
 	interspersed      bool      // allow interspersed option/non-option args
@@ -156,7 +163,7 @@ type Flag struct {
 	Value               Value               // value as set
 	DefValue            string              // default value (as text); for usage message
 	Changed             bool                // If the user set the value (or if left to default)
-	NoOptDefVal         string              //default value (as text); if the flag is on the command line without any options
+	NoOptDefVal         string              // default value (as text); if the flag is on the command line without any options
 	Deprecated          string              // If this flag is deprecated, this string is the new or now thing to use
 	Hidden              bool                // used by cobra.Command to allow flags to be hidden from help/usage text
 	ShorthandDeprecated string              // If the shorthand of this flag is deprecated, this string is the new or now thing to use
@@ -194,11 +201,13 @@ func sortFlags(flags map[NormalizedName]*Flag) []*Flag {
 // "--getUrl" which may also be translated to "geturl" and everything will work.
 func (f *FlagSet) SetNormalizeFunc(n func(f *FlagSet, name string) NormalizedName) {
 	f.normalizeNameFunc = n
-	for k, v := range f.formal {
-		delete(f.formal, k)
-		nname := f.normalizeFlagName(string(k))
-		f.formal[nname] = v
+	f.sortedFormal = f.sortedFormal[:0]
+	for k, v := range f.orderedFormal {
+		delete(f.formal, NormalizedName(v.Name))
+		nname := f.normalizeFlagName(v.Name)
 		v.Name = string(nname)
+		f.formal[nname] = v
+		f.orderedFormal[k] = v
 	}
 }
 
@@ -229,10 +238,25 @@ func (f *FlagSet) SetOutput(output io.Writer) {
 	f.output = output
 }
 
-// VisitAll visits the flags in lexicographical order, calling fn for each.
+// VisitAll visits the flags in lexicographical order or
+// in primordial order if f.SortFlags is false, calling fn for each.
 // It visits all flags, even those not set.
 func (f *FlagSet) VisitAll(fn func(*Flag)) {
-	for _, flag := range sortFlags(f.formal) {
+	if len(f.formal) == 0 {
+		return
+	}
+
+	var flags []*Flag
+	if f.SortFlags {
+		if len(f.formal) != len(f.sortedFormal) {
+			f.sortedFormal = sortFlags(f.formal)
+		}
+		flags = f.sortedFormal
+	} else {
+		flags = f.orderedFormal
+	}
+
+	for _, flag := range flags {
 		fn(flag)
 	}
 }
@@ -253,22 +277,39 @@ func (f *FlagSet) HasAvailableFlags() bool {
 	return false
 }
 
-// VisitAll visits the command-line flags in lexicographical order, calling
-// fn for each.  It visits all flags, even those not set.
+// VisitAll visits the command-line flags in lexicographical order or
+// in primordial order if f.SortFlags is false, calling fn for each.
+// It visits all flags, even those not set.
 func VisitAll(fn func(*Flag)) {
 	CommandLine.VisitAll(fn)
 }
 
-// Visit visits the flags in lexicographical order, calling fn for each.
+// Visit visits the flags in lexicographical order or
+// in primordial order if f.SortFlags is false, calling fn for each.
 // It visits only those flags that have been set.
 func (f *FlagSet) Visit(fn func(*Flag)) {
-	for _, flag := range sortFlags(f.actual) {
+	if len(f.actual) == 0 {
+		return
+	}
+
+	var flags []*Flag
+	if f.SortFlags {
+		if len(f.actual) != len(f.sortedActual) {
+			f.sortedActual = sortFlags(f.actual)
+		}
+		flags = f.sortedActual
+	} else {
+		flags = f.orderedActual
+	}
+
+	for _, flag := range flags {
 		fn(flag)
 	}
 }
 
-// Visit visits the command-line flags in lexicographical order, calling fn
-// for each.  It visits only those flags that have been set.
+// Visit visits the command-line flags in lexicographical order or
+// in primordial order if f.SortFlags is false, calling fn for each.
+// It visits only those flags that have been set.
 func Visit(fn func(*Flag)) {
 	CommandLine.Visit(fn)
 }
@@ -278,6 +319,22 @@ func (f *FlagSet) Lookup(name string) *Flag {
 	return f.lookup(f.normalizeFlagName(name))
 }
 
+// ShorthandLookup returns the Flag structure of the short handed flag,
+// returning nil if none exists.
+// It panics, if len(name) > 1.
+func (f *FlagSet) ShorthandLookup(name string) *Flag {
+	if name == "" {
+		return nil
+	}
+	if len(name) > 1 {
+		msg := fmt.Sprintf("can not look up shorthand which is more than one ASCII character: %q", name)
+		fmt.Fprintf(f.out(), msg)
+		panic(msg)
+	}
+	c := name[0]
+	return f.shorthands[c]
+}
+
 // lookup returns the Flag structure of the named flag, returning nil if none exists.
 func (f *FlagSet) lookup(name NormalizedName) *Flag {
 	return f.formal[name]
@@ -319,7 +376,7 @@ func (f *FlagSet) MarkDeprecated(name string, usageMessage string) error {
 	if flag == nil {
 		return fmt.Errorf("flag %q does not exist", name)
 	}
-	if len(usageMessage) == 0 {
+	if usageMessage == "" {
 		return fmt.Errorf("deprecated message for flag %q must be set", name)
 	}
 	flag.Deprecated = usageMessage
@@ -334,7 +391,7 @@ func (f *FlagSet) MarkShorthandDeprecated(name string, usageMessage string) erro
 	if flag == nil {
 		return fmt.Errorf("flag %q does not exist", name)
 	}
-	if len(usageMessage) == 0 {
+	if usageMessage == "" {
 		return fmt.Errorf("deprecated message for flag %q must be set", name)
 	}
 	flag.ShorthandDeprecated = usageMessage
@@ -358,6 +415,12 @@ func Lookup(name string) *Flag {
 	return CommandLine.Lookup(name)
 }
 
+// ShorthandLookup returns the Flag structure of the short handed flag,
+// returning nil if none exists.
+func ShorthandLookup(name string) *Flag {
+	return CommandLine.ShorthandLookup(name)
+}
+
 // Set sets the value of the named flag.
 func (f *FlagSet) Set(name, value string) error {
 	normalName := f.normalizeFlagName(name)
@@ -365,17 +428,28 @@ func (f *FlagSet) Set(name, value string) error {
 	if !ok {
 		return fmt.Errorf("no such flag -%v", name)
 	}
+
 	err := flag.Value.Set(value)
 	if err != nil {
-		return err
+		var flagName string
+		if flag.Shorthand != "" && flag.ShorthandDeprecated == "" {
+			flagName = fmt.Sprintf("-%s, --%s", flag.Shorthand, flag.Name)
+		} else {
+			flagName = fmt.Sprintf("--%s", flag.Name)
+		}
+		return fmt.Errorf("invalid argument %q for %q flag: %v", value, flagName, err)
 	}
+
 	if f.actual == nil {
 		f.actual = make(map[NormalizedName]*Flag)
 	}
 	f.actual[normalName] = flag
+	f.orderedActual = append(f.orderedActual, flag)
+
 	flag.Changed = true
-	if len(flag.Deprecated) > 0 {
-		fmt.Fprintf(os.Stderr, "Flag --%s has been deprecated, %s\n", flag.Name, flag.Deprecated)
+
+	if flag.Deprecated != "" {
+		fmt.Fprintf(f.out(), "Flag --%s has been deprecated, %s\n", flag.Name, flag.Deprecated)
 	}
 	return nil
 }
@@ -557,28 +631,28 @@ func wrap(i, w int, s string) string {
 // for all flags in the FlagSet. Wrapped to `cols` columns (0 for no
 // wrapping)
 func (f *FlagSet) FlagUsagesWrapped(cols int) string {
-	x := new(bytes.Buffer)
+	buf := new(bytes.Buffer)
 
 	lines := make([]string, 0, len(f.formal))
 
 	maxlen := 0
 	f.VisitAll(func(flag *Flag) {
-		if len(flag.Deprecated) > 0 || flag.Hidden {
+		if flag.Deprecated != "" || flag.Hidden {
 			return
 		}
 
 		line := ""
-		if len(flag.Shorthand) > 0 && len(flag.ShorthandDeprecated) == 0 {
+		if flag.Shorthand != "" && flag.ShorthandDeprecated == "" {
 			line = fmt.Sprintf("  -%s, --%s", flag.Shorthand, flag.Name)
 		} else {
 			line = fmt.Sprintf("      --%s", flag.Name)
 		}
 
 		varname, usage := UnquoteUsage(flag)
-		if len(varname) > 0 {
+		if varname != "" {
 			line += " " + varname
 		}
-		if len(flag.NoOptDefVal) > 0 {
+		if flag.NoOptDefVal != "" {
 			switch flag.Value.Type() {
 			case "string":
 				line += fmt.Sprintf("[=\"%s\"]", flag.NoOptDefVal)
@@ -601,7 +675,7 @@ func (f *FlagSet) FlagUsagesWrapped(cols int) string {
 		line += usage
 		if !flag.defaultIsZeroValue() {
 			if flag.Value.Type() == "string" {
-				line += fmt.Sprintf(" (default \"%s\")", flag.DefValue)
+				line += fmt.Sprintf(" (default %q)", flag.DefValue)
 			} else {
 				line += fmt.Sprintf(" (default %s)", flag.DefValue)
 			}
@@ -614,10 +688,10 @@ func (f *FlagSet) FlagUsagesWrapped(cols int) string {
 		sidx := strings.Index(line, "\x00")
 		spacing := strings.Repeat(" ", maxlen-sidx)
 		// maxlen + 2 comes from + 1 for the \x00 and + 1 for the (deliberate) off-by-one in maxlen-sidx
-		fmt.Fprintln(x, line[:sidx], spacing, wrap(maxlen+2, cols, line[sidx+1:]))
+		fmt.Fprintln(buf, line[:sidx], spacing, wrap(maxlen+2, cols, line[sidx+1:]))
 	}
 
-	return x.String()
+	return buf.String()
 }
 
 // FlagUsages returns a string containing the usage information for all flags in
@@ -714,11 +788,10 @@ func (f *FlagSet) VarP(value Value, name, shorthand, usage string) {
 
 // AddFlag will add the flag to the FlagSet
 func (f *FlagSet) AddFlag(flag *Flag) {
-	// Call normalizeFlagName function only once
 	normalizedFlagName := f.normalizeFlagName(flag.Name)
 
-	_, alreadythere := f.formal[normalizedFlagName]
-	if alreadythere {
+	_, alreadyThere := f.formal[normalizedFlagName]
+	if alreadyThere {
 		msg := fmt.Sprintf("%s flag redefined: %s", f.name, flag.Name)
 		fmt.Fprintln(f.out(), msg)
 		panic(msg) // Happens only if flags are declared with identical names
@@ -729,28 +802,31 @@ func (f *FlagSet) AddFlag(flag *Flag) {
 
 	flag.Name = string(normalizedFlagName)
 	f.formal[normalizedFlagName] = flag
+	f.orderedFormal = append(f.orderedFormal, flag)
 
-	if len(flag.Shorthand) == 0 {
+	if flag.Shorthand == "" {
 		return
 	}
 	if len(flag.Shorthand) > 1 {
-		fmt.Fprintf(f.out(), "%s shorthand more than ASCII character: %s\n", f.name, flag.Shorthand)
-		panic("shorthand is more than one character")
+		msg := fmt.Sprintf("%q shorthand is more than one ASCII character", flag.Shorthand)
+		fmt.Fprintf(f.out(), msg)
+		panic(msg)
 	}
 	if f.shorthands == nil {
 		f.shorthands = make(map[byte]*Flag)
 	}
 	c := flag.Shorthand[0]
-	old, alreadythere := f.shorthands[c]
-	if alreadythere {
-		fmt.Fprintf(f.out(), "%s shorthand reused: %q for %s already used for %s\n", f.name, c, flag.Name, old.Name)
-		panic("shorthand redefinition")
+	used, alreadyThere := f.shorthands[c]
+	if alreadyThere {
+		msg := fmt.Sprintf("unable to redefine %q shorthand in %q flagset: it's already used for %q flag", c, f.name, used.Name)
+		fmt.Fprintf(f.out(), msg)
+		panic(msg)
 	}
 	f.shorthands[c] = flag
 }
 
 // AddFlagSet adds one FlagSet to another. If a flag is already present in f
-// the flag from newSet will be ignored
+// the flag from newSet will be ignored.
 func (f *FlagSet) AddFlagSet(newSet *FlagSet) {
 	if newSet == nil {
 		return
@@ -798,34 +874,6 @@ func (f *FlagSet) usage() {
 	}
 }
 
-func (f *FlagSet) setFlag(flag *Flag, value string, origArg string) error {
-	if err := flag.Value.Set(value); err != nil {
-		return f.failf("invalid argument %q for %s: %v", value, origArg, err)
-	}
-	// mark as visited for Visit()
-	if f.actual == nil {
-		f.actual = make(map[NormalizedName]*Flag)
-	}
-	f.actual[f.normalizeFlagName(flag.Name)] = flag
-	flag.Changed = true
-	if len(flag.Deprecated) > 0 {
-		fmt.Fprintf(os.Stderr, "Flag --%s has been deprecated, %s\n", flag.Name, flag.Deprecated)
-	}
-	if len(flag.ShorthandDeprecated) > 0 && containsShorthand(origArg, flag.Shorthand) {
-		fmt.Fprintf(os.Stderr, "Flag shorthand -%s has been deprecated, %s\n", flag.Shorthand, flag.ShorthandDeprecated)
-	}
-	return nil
-}
-
-func containsShorthand(arg, shorthand string) bool {
-	// filter out flags --<flag_name>
-	if strings.HasPrefix(arg, "-") {
-		return false
-	}
-	arg = strings.SplitN(arg, "=", 2)[0]
-	return strings.Contains(arg, shorthand)
-}
-
 func (f *FlagSet) parseLongArg(s string, args []string, fn parseFunc) (a []string, err error) {
 	a = args
 	name := s[2:]
@@ -833,10 +881,11 @@ func (f *FlagSet) parseLongArg(s string, args []string, fn parseFunc) (a []strin
 		err = f.failf("bad flag syntax: %s", s)
 		return
 	}
+
 	split := strings.SplitN(name, "=", 2)
 	name = split[0]
-	flag, alreadythere := f.formal[f.normalizeFlagName(name)]
-	if !alreadythere {
+	flag, exists := f.formal[f.normalizeFlagName(name)]
+	if !exists {
 		if name == "help" { // special case for nice help message.
 			f.usage()
 			return a, ErrHelp
@@ -844,11 +893,12 @@ func (f *FlagSet) parseLongArg(s string, args []string, fn parseFunc) (a []strin
 		err = f.failf("unknown flag: --%s", name)
 		return
 	}
+
 	var value string
 	if len(split) == 2 {
 		// '--flag=arg'
 		value = split[1]
-	} else if len(flag.NoOptDefVal) > 0 {
+	} else if flag.NoOptDefVal != "" {
 		// '--flag' (arg was optional)
 		value = flag.NoOptDefVal
 	} else if len(a) > 0 {
@@ -860,7 +910,8 @@ func (f *FlagSet) parseLongArg(s string, args []string, fn parseFunc) (a []strin
 		err = f.failf("flag needs an argument: %s", s)
 		return
 	}
-	err = fn(flag, value, s)
+
+	err = fn(flag, value)
 	return
 }
 
@@ -868,38 +919,49 @@ func (f *FlagSet) parseSingleShortArg(shorthands string, args []string, fn parse
 	if strings.HasPrefix(shorthands, "test.") {
 		return
 	}
+
 	outArgs = args
 	outShorts = shorthands[1:]
 	c := shorthands[0]
 
-	flag, alreadythere := f.shorthands[c]
-	if !alreadythere {
+	flag, exists := f.shorthands[c]
+	if !exists {
 		if c == 'h' { // special case for nice help message.
 			f.usage()
 			err = ErrHelp
 			return
 		}
-		//TODO continue on error
 		err = f.failf("unknown shorthand flag: %q in -%s", c, shorthands)
 		return
 	}
+
 	var value string
 	if len(shorthands) > 2 && shorthands[1] == '=' {
+		// '-f=arg'
 		value = shorthands[2:]
 		outShorts = ""
-	} else if len(flag.NoOptDefVal) > 0 {
+	} else if flag.NoOptDefVal != "" {
+		// '-f' (arg was optional)
 		value = flag.NoOptDefVal
 	} else if len(shorthands) > 1 {
+		// '-farg'
 		value = shorthands[1:]
 		outShorts = ""
 	} else if len(args) > 0 {
+		// '-f arg'
 		value = args[0]
 		outArgs = args[1:]
 	} else {
+		// '-f' (arg was required)
 		err = f.failf("flag needs an argument: %q in -%s", c, shorthands)
 		return
 	}
-	err = fn(flag, value, shorthands)
+
+	if flag.ShorthandDeprecated != "" {
+		fmt.Fprintf(f.out(), "Flag shorthand -%s has been deprecated, %s\n", flag.Shorthand, flag.ShorthandDeprecated)
+	}
+
+	err = fn(flag, value)
 	return
 }
 
@@ -907,6 +969,7 @@ func (f *FlagSet) parseShortArg(s string, args []string, fn parseFunc) (a []stri
 	a = args
 	shorthands := s[1:]
 
+	// "shorthands" can be a series of shorthand letters of flags (e.g. "-vvv").
 	for len(shorthands) > 0 {
 		shorthands, a, err = f.parseSingleShortArg(shorthands, args, fn)
 		if err != nil {
@@ -954,13 +1017,18 @@ func (f *FlagSet) parseArgs(args []string, fn parseFunc) (err error) {
 // The return value will be ErrHelp if -help was set but not defined.
 func (f *FlagSet) Parse(arguments []string) error {
 	f.parsed = true
-	f.args = make([]string, 0, len(arguments))
 
-	assign := func(flag *Flag, value, origArg string) error {
-		return f.setFlag(flag, value, origArg)
+	if len(arguments) < 0 {
+		return nil
 	}
 
-	err := f.parseArgs(arguments, assign)
+	f.args = make([]string, 0, len(arguments))
+
+	set := func(flag *Flag, value string) error {
+		return f.Set(flag.Name, value)
+	}
+
+	err := f.parseArgs(arguments, set)
 	if err != nil {
 		switch f.errorHandling {
 		case ContinueOnError:
@@ -974,7 +1042,7 @@ func (f *FlagSet) Parse(arguments []string) error {
 	return nil
 }
 
-type parseFunc func(flag *Flag, value, origArg string) error
+type parseFunc func(flag *Flag, value string) error
 
 // ParseAll parses flag definitions from the argument list, which should not
 // include the command name. The arguments for fn are flag and value. Must be
@@ -985,11 +1053,7 @@ func (f *FlagSet) ParseAll(arguments []string, fn func(flag *Flag, value string)
 	f.parsed = true
 	f.args = make([]string, 0, len(arguments))
 
-	assign := func(flag *Flag, value, origArg string) error {
-		return fn(flag, value)
-	}
-
-	err := f.parseArgs(arguments, assign)
+	err := f.parseArgs(arguments, fn)
 	if err != nil {
 		switch f.errorHandling {
 		case ContinueOnError:
@@ -1036,14 +1100,15 @@ func Parsed() bool {
 // CommandLine is the default set of command-line flags, parsed from os.Args.
 var CommandLine = NewFlagSet(os.Args[0], ExitOnError)
 
-// NewFlagSet returns a new, empty flag set with the specified name and
-// error handling property.
+// NewFlagSet returns a new, empty flag set with the specified name,
+// error handling property and SortFlags set to true.
 func NewFlagSet(name string, errorHandling ErrorHandling) *FlagSet {
 	f := &FlagSet{
 		name:          name,
 		errorHandling: errorHandling,
 		argsLenAtDash: -1,
 		interspersed:  true,
+		SortFlags:     true,
 	}
 	return f
 }