Pull sandbox image and apply image config

Signed-off-by: Lantao Liu <lantaol@google.com>

pkg/server/helpers.go

@@ -55,6 +55,9 @@ const (
 )
 
 const (
+	// defaultSandboxImage is the image used by sandbox container.
+	// TODO(random-liu): [P1] Build schema 2 pause image and use it here.
+	defaultSandboxImage = "gcr.io/google.com/noogler-kubernetes/pause-amd64:3.0"
 	// relativeRootfsPath is the rootfs path relative to bundle path.
 	relativeRootfsPath = "rootfs"
 	// defaultRuntime is the runtime to use in containerd. We may support
@@ -305,31 +308,39 @@ func getRepoDigestAndTag(namedRef reference.Named, digest imagedigest.Digest) (s
 	return repoDigest, repoTag
 }
 
-// localResolve resolves image reference to image id locally. It returns empty string
+// localResolve resolves image reference locally and returns corresponding image metadata. It returns
-// without error if the reference doesn't exist.
+// nil without error if the reference doesn't exist.
-func (c *criContainerdService) localResolve(ctx context.Context, ref string) (string, error) {
+func (c *criContainerdService) localResolve(ctx context.Context, ref string) (*metadata.ImageMetadata, error) {
 	_, err := imagedigest.Parse(ref)
-	if err == nil {
+	if err != nil {
-		return ref, nil
-	}
 		// ref is not image id, try to resolve it locally.
 		normalized, err := normalizeImageRef(ref)
 		if err != nil {
-		return "", fmt.Errorf("invalid image reference %q: %v", ref, err)
+			return nil, fmt.Errorf("invalid image reference %q: %v", ref, err)
 		}
 		image, err := c.imageStoreService.Get(ctx, normalized.String())
 		if err != nil {
 			if images.IsNotFound(err) {
-			return "", nil
+				return nil, nil
 			}
-		return "", fmt.Errorf("an error occurred when getting image %q from containerd image store: %v",
+			return nil, fmt.Errorf("an error occurred when getting image %q from containerd image store: %v",
 				normalized.String(), err)
 		}
 		desc, err := image.Config(ctx, c.contentStoreService)
 		if err != nil {
-		return "", fmt.Errorf("failed to get image config descriptor: %v", err)
+			return nil, fmt.Errorf("failed to get image config descriptor: %v", err)
 		}
-	return desc.Digest.String(), nil
+		ref = desc.Digest.String()
+	}
+	imageID := ref
+	meta, err := c.imageMetadataStore.Get(imageID)
+	if err != nil {
+		if metadata.IsNotExistError(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to get image %q metadata: %v", imageID, err)
+	}
+	return meta, nil
 }
 
 // getUserFromImage gets uid or user name of the image user.
@@ -350,3 +361,27 @@ func getUserFromImage(user string) (*int64, string) {
 	// If user is a numeric uid.
 	return &uid, ""
 }
+
+// ensureImageExists returns corresponding metadata of the image reference, if image is not
+// pulled yet, the function will pull the image.
+func (c *criContainerdService) ensureImageExists(ctx context.Context, ref string) (*metadata.ImageMetadata, error) {
+	meta, err := c.localResolve(ctx, ref)
+	if err != nil {
+		return nil, fmt.Errorf("failed to resolve image %q: %v", ref, err)
+	}
+	if meta != nil {
+		return meta, nil
+	}
+	// Pull image to ensure the image exists
+	resp, err := c.PullImage(ctx, &runtime.PullImageRequest{Image: &runtime.ImageSpec{Image: ref}})
+	if err != nil {
+		return nil, fmt.Errorf("failed to pull image %q: %v", ref, err)
+	}
+	imageID := resp.GetImageRef()
+	meta, err = c.imageMetadataStore.Get(imageID)
+	if err != nil {
+		// It's still possible that someone removed the image right after it is pulled.
+		return nil, fmt.Errorf("failed to get image %q metadata after pulling: %v", imageID, err)
+	}
+	return meta, nil
+}

pkg/server/image_remove.go

@@ -41,21 +41,14 @@ func (c *criContainerdService) RemoveImage(ctx context.Context, r *runtime.Remov
 			glog.V(2).Infof("RemoveImage %q returns successfully", r.GetImage().GetImage())
 		}
 	}()
-	imageID, err := c.localResolve(ctx, r.GetImage().GetImage())
+	meta, err := c.localResolve(ctx, r.GetImage().GetImage())
 	if err != nil {
 		return nil, fmt.Errorf("can not resolve %q locally: %v", r.GetImage().GetImage(), err)
 	}
-	if imageID == "" {
+	if meta == nil {
 		// return empty without error when image not found.
 		return &runtime.RemoveImageResponse{}, nil
 	}
-	meta, err := c.imageMetadataStore.Get(imageID)
-	if err != nil {
-		if metadata.IsNotExistError(err) {
-			return &runtime.RemoveImageResponse{}, nil
-		}
-		return nil, fmt.Errorf("an error occurred when get image %q metadata: %v", imageID, err)
-	}
 	// Also include repo digest, because if user pull image with digest,
 	// there will also be a corresponding repo digest reference.
 	for _, ref := range append(meta.RepoTags, meta.RepoDigests...) {
@@ -65,14 +58,14 @@ func (c *criContainerdService) RemoveImage(ctx context.Context, r *runtime.Remov
 		if err == nil || images.IsNotFound(err) {
 			continue
 		}
-		return nil, fmt.Errorf("failed to delete image reference %q for image %q: %v", ref, imageID, err)
+		return nil, fmt.Errorf("failed to delete image reference %q for image %q: %v", ref, meta.ID, err)
 	}
-	err = c.imageMetadataStore.Delete(imageID)
+	err = c.imageMetadataStore.Delete(meta.ID)
 	if err != nil {
 		if metadata.IsNotExistError(err) {
 			return &runtime.RemoveImageResponse{}, nil
 		}
-		return nil, fmt.Errorf("an error occurred when delete image %q matadata: %v", imageID, err)
+		return nil, fmt.Errorf("an error occurred when delete image %q matadata: %v", meta.ID, err)
 	}
 	return &runtime.RemoveImageResponse{}, nil
 }

pkg/server/image_status.go

@@ -22,8 +22,6 @@ import (
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
-
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 )
 
 // ImageStatus returns the status of the image, returns nil if the image isn't present.
@@ -37,23 +35,14 @@ func (c *criContainerdService) ImageStatus(ctx context.Context, r *runtime.Image
 				r.GetImage().GetImage(), retRes.GetImage())
 		}
 	}()
-	imageID, err := c.localResolve(ctx, r.GetImage().GetImage())
+	meta, err := c.localResolve(ctx, r.GetImage().GetImage())
 	if err != nil {
 		return nil, fmt.Errorf("can not resolve %q locally: %v", r.GetImage().GetImage(), err)
 	}
-	if imageID == "" {
+	if meta == nil {
 		// return empty without error when image not found.
 		return &runtime.ImageStatusResponse{}, nil
 	}
-
-	meta, err := c.imageMetadataStore.Get(imageID)
-	if err != nil {
-		if metadata.IsNotExistError(err) {
-			return &runtime.ImageStatusResponse{}, nil
-		}
-		return nil, fmt.Errorf("an error occurred during get image %q metadata: %v",
-			imageID, err)
-	}
 	// TODO(random-liu): [P0] Make sure corresponding snapshot exists. What if snapshot
 	// doesn't exist?
 	runtimeImage := &runtime.Image{

pkg/server/sandbox_run.go

@@ -21,17 +21,18 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strings"
 	"time"
 
+	"github.com/containerd/containerd/api/services/execution"
+	rootfsapi "github.com/containerd/containerd/api/services/rootfs"
 	prototypes "github.com/gogo/protobuf/types"
 	"github.com/golang/glog"
+	imagedigest "github.com/opencontainers/go-digest"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"golang.org/x/net/context"
-
-	"github.com/containerd/containerd/api/services/execution"
-	"github.com/containerd/containerd/api/types/mount"
-
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
 
 	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
@@ -81,10 +82,22 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 		Config: config,
 	}
 
-	// TODO(random-liu): [P0] Ensure pause image snapshot, apply default image config
+	// Ensure sandbox container image snapshot.
-	// and get snapshot mounts.
+	imageMeta, err := c.ensureImageExists(ctx, c.sandboxImage)
-	// Use fixed rootfs path and sleep command.
+	if err != nil {
-	const rootPath = "/"
+		return nil, fmt.Errorf("failed to get sandbox image %q: %v", defaultSandboxImage, err)
+	}
+	prepareResp, err := c.rootfsService.Prepare(ctx, &rootfsapi.PrepareRequest{
+		Name: id,
+		// We are sure that ChainID must be a digest.
+		ChainID:  imagedigest.Digest(imageMeta.ChainID),
+		Readonly: true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to prepare sandbox rootfs %q: %v", imageMeta.ChainID, err)
+	}
+	// TODO(random-liu): [P0] Cleanup snapshot on failure after switching to new rootfs api.
+	rootfsMounts := prepareResp.Mounts
 
 	// Create sandbox container root directory.
 	// Prepare streaming named pipe.
@@ -124,7 +137,10 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	}
 
 	// Start sandbox container.
-	spec := c.generateSandboxContainerSpec(id, config)
+	spec, err := c.generateSandboxContainerSpec(id, config, imageMeta.Config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate sandbox container spec: %v", err)
+	}
 	rawSpec, err := json.Marshal(spec)
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal oci spec %+v: %v", spec, err)
@@ -137,16 +153,7 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 			Value:   rawSpec,
 		},
 		// TODO(random-liu): [P0] Get rootfs mount from containerd.
-		Rootfs: []*mount.Mount{
+		Rootfs:  rootfsMounts,
-			{
-				Type:   "bind",
-				Source: rootPath,
-				Options: []string{
-					"rw",
-					"rbind",
-				},
-			},
-		},
 		Runtime: defaultRuntime,
 		// No stdin for sandbox container.
 		Stdout: stdout,
@@ -205,20 +212,35 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil
 }
 
-func (c *criContainerdService) generateSandboxContainerSpec(id string, config *runtime.PodSandboxConfig) *runtimespec.Spec {
+func (c *criContainerdService) generateSandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
-	// TODO(random-liu): [P0] Get command from image config.
+	imageConfig *imagespec.ImageConfig) (*runtimespec.Spec, error) {
-	pauseCommand := []string{"sh", "-c", "while true; do sleep 1000000000; done"}
-
 	// Creates a spec Generator with the default spec.
 	// TODO(random-liu): [P1] Compare the default settings with docker and containerd default.
 	g := generate.New()
 
+	// Apply default config from image config.
+	for _, e := range imageConfig.Env {
+		kv := strings.Split(e, "=")
+		if len(kv) != 2 {
+			return nil, fmt.Errorf("invalid environment variable in image config %+v", imageConfig)
+		}
+		g.AddProcessEnv(kv[0], kv[1])
+	}
+
+	if imageConfig.WorkingDir != "" {
+		g.SetProcessCwd(imageConfig.WorkingDir)
+	}
+
+	if len(imageConfig.Entrypoint) == 0 {
+		// Pause image must have entrypoint.
+		return nil, fmt.Errorf("invalid empty entrypoint in image config %+v", imageConfig)
+	}
+	// Set process commands.
+	g.SetProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...))
+
 	// Set relative root path.
 	g.SetRootPath(relativeRootfsPath)
 
-	// Set process commands.
-	g.SetProcessArgs(pauseCommand)
-
 	// Make root of sandbox container read-only.
 	g.SetRootReadonly(true)
 
@@ -276,5 +298,5 @@ func (c *criContainerdService) generateSandboxContainerSpec(id string, config *r
 
 	// TODO(random-liu): [P1] Set default sandbox container resource limit.
 
-	return g.Spec()
+	return g.Spec(), nil
 }

pkg/server/service.go

@@ -65,6 +65,9 @@ type criContainerdService struct {
 	os osinterface.OS
 	// rootDir is the directory for managing cri-containerd files.
 	rootDir string
+	// sandboxImage is the image to use for sandbox container.
+	// TODO(random-liu): Make this configurable via flag.
+	sandboxImage string
 	// sandboxStore stores all sandbox metadata.
 	sandboxStore metadata.SandboxStore
 	// imageMetadataStore stores all image metadata.
@@ -100,6 +103,7 @@ func NewCRIContainerdService(conn *grpc.ClientConn, rootDir, networkPluginBinDir
 	c := &criContainerdService{
 		os:                 osinterface.RealOS{},
 		rootDir:            rootDir,
+		sandboxImage:       defaultSandboxImage,
 		sandboxStore:       metadata.NewSandboxStore(store.NewMetadataStore()),
 		containerStore:     metadata.NewContainerStore(store.NewMetadataStore()),
 		imageMetadataStore: metadata.NewImageMetadataStore(store.NewMetadataStore()),