node_e2e: configure gce images via config file

This file provides the abiliy to specify image project on a per-image
basis and is more extensible for future changes.

For backwards compatibility and local development convenience, the
existing flags are kept and should work.

hack/verify-flags/known-flags.txt

@@ -203,6 +203,7 @@ http-check-frequency
 http-port
 ignore-daemonsets
 ignore-not-found
+image-config-file
 image-gc-high-threshold
 image-gc-low-threshold
 image-project

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -35,7 +35,7 @@ ARTIFACTS=${WORKSPACE}/_artifacts
 
 mkdir -p ${ARTIFACTS}
 go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="gce" \
-  --zone="$GCE_ZONE" --project="$GCE_PROJECT" --image-project="$GCE_IMAGE_PROJECT" \
+  --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
-  --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP" \
+  --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
   --results-dir="$ARTIFACTS" --ginkgo-flags="$GINKGO_FLAGS" \
   --setup-node="$SETUP_NODE"

test/e2e_node/jenkins/image-config.yaml

@@ -0,0 +1,16 @@
+# To copy an image between projects:
+# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
+# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
+images:
+  ubuntu-docker9:
+    image: e2e-node-ubuntu-trusty-docker9-v1-image
+    project: kubernetes-node-e2e-images
+  ubuntu-docker10:
+    image: e2e-node-ubuntu-trusty-docker10-v1-image
+    project: kubernetes-node-e2e-images
+  coreos-stable:
+    image: e2e-node-coreos-alpha-1068-20160707-image
+    project: kubernetes-node-e2e-images
+  containervm:
+    image: e2e-node-containervm-v20160321-image
+    project: kubernetes-node-e2e-images

test/e2e_node/jenkins/jenkins-ci.properties

@@ -1,12 +1,7 @@
 GCE_HOSTS=
-# Keep GCE_IMAGES consistent with those in jenkins-pull.properties.
+GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml
-# To copy an image between projects:
-# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
-# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
-GCE_IMAGES=e2e-node-ubuntu-trusty-docker9-v1-image,e2e-node-ubuntu-trusty-docker10-v1-image,e2e-node-coreos-alpha-1068-20160707-image,e2e-node-containervm-v20160321-image
 GCE_ZONE=us-central1-f
 GCE_PROJECT=kubernetes-jenkins
-GCE_IMAGE_PROJECT=kubernetes-node-e2e-images
 CLEANUP=true
 GINKGO_FLAGS=--skip=FLAKY
 SETUP_NODE=false

test/e2e_node/jenkins/jenkins-pull.properties

@@ -1,12 +1,7 @@
 GCE_HOSTS=
-# Keep GCE_IMAGES consistent with those in jenkins-ci.properties
+GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml
-# To copy an image between projects:
-# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
-# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
-GCE_IMAGES=e2e-node-ubuntu-trusty-docker9-v1-image,e2e-node-ubuntu-trusty-docker10-v1-image,e2e-node-coreos-alpha-1068-20160707-image,e2e-node-containervm-v20160321-image
 GCE_ZONE=us-central1-f
 GCE_PROJECT=kubernetes-jenkins-pull
-GCE_IMAGE_PROJECT=kubernetes-node-e2e-images
 CLEANUP=true
 GINKGO_FLAGS=--skip=FLAKY
 SETUP_NODE=false

test/e2e_node/jenkins/template.properties

@@ -1,17 +1,19 @@
 # Copy this file to your home directory and modify
-# Names of gce hosts to test against (must be resolvable) or empty (one or more of GCE_IMAGES, GCE_HOSTS is required)
+# Path to a yaml or json file describing images to run or empty
+GCE_IMAGE_CONFIG_PATH=
+# Names of gce hosts to test against (must be resolvable) or empty
 GCE_HOSTS=
-# Names of gce images to test or empty (one or more of GCE_IMAGES, GCE_HOSTS is required)
+# Comma-separated names of gce images to test or empty (one or more of GCE_IMAGE_CONFIG_PATH, GCE_IMAGES, GCE_HOSTS is required)
 GCE_IMAGES=
 # Gce zone to use - required when using GCE_IMAGES
 GCE_ZONE=
-# Gce project to use for creating instances 
+# Gce project to use for creating instances
-# required when using GCE_IMAGES
+# required when using GCE_IMAGES or GCE_IMAGE_CONFIG_PATH
 GCE_PROJECT=
 # Gce project to use for GCE_IMAGES
 # required when using GCE_IMAGES
 GCE_IMAGE_PROJECT=
-# If true, delete instances created from GCE_IMAGES and files copied to GCE_HOSTS
+# If true, delete instances created from GCE_IMAGES/GCE_IMAGE_CONFIG_PATH and files copied to GCE_HOSTS
 CLEANUP=true
 # If true, current user will be added to the docker group on test node
 SETUP_NODE=false

test/e2e_node/runner/run_e2e.go

@@ -23,6 +23,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"os"
@@ -32,6 +33,7 @@ import (
 
 	"k8s.io/kubernetes/test/e2e_node"
 
+	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
 	"github.com/pborman/uuid"
 	"golang.org/x/oauth2"
@@ -42,6 +44,7 @@ import (
 var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
 var zone = flag.String("zone", "", "gce zone the hosts live in")
 var project = flag.String("project", "", "gce project the hosts live in")
+var imageConfigFile = flag.String("image-config-file", "", "yaml file describing images to run")
 var imageProject = flag.String("image-project", "", "gce project the hosts live in")
 var images = flag.String("images", "", "images to test")
 var hosts = flag.String("hosts", "", "hosts to test")
@@ -67,6 +70,24 @@ type TestResult struct {
 	exitOk bool
 }
 
+// ImageConfig specifies what images should be run and how for these tests.
+// It can be created via the `--images` and `--image-project` flags, or by
+// specifying the `--image-config-file` flag, pointing to a json or yaml file
+// of the form:
+//
+//     images:
+//       short-name:
+//         image: gce-image-name
+//         project: gce-image-project
+type ImageConfig struct {
+	Images map[string]GCEImage `json:"images"`
+}
+
+type GCEImage struct {
+	Image   string `json:"image"`
+	Project string `json:"project"`
+}
+
 func main() {
 	flag.Parse()
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -76,18 +97,50 @@ func main() {
 		return
 	}
 
-	if *hosts == "" && *images == "" {
+	if *hosts == "" && *imageConfigFile == "" && *images == "" {
-		glog.Fatalf("Must specify one of --images or --hosts flag.")
+		glog.Fatalf("Must specify one of --image-config-file, --hosts, --images.")
 	}
-	if *images != "" && *zone == "" {
+	gceImages := &ImageConfig{
-		glog.Fatal("Must specify --zone flag")
+		Images: make(map[string]GCEImage),
 	}
+	if *imageConfigFile != "" {
+		// parse images
+		imageConfigData, err := ioutil.ReadFile(*imageConfigFile)
+		if err != nil {
+			glog.Fatalf("Could not read image config file provided: %v", err)
+		}
+		err = yaml.Unmarshal(imageConfigData, gceImages)
+		if err != nil {
+			glog.Fatalf("Could not parse image config file: %v", err)
+		}
+	}
+
+	// Allow users to specify additional images via cli flags for local testing
+	// convenience; merge in with config file
 	if *images != "" {
 		if *imageProject == "" {
-			glog.Fatal("Must specify --image-project flag")
+			glog.Fatal("Must specify --image-project if you specify --images")
 		}
+		cliImages := strings.Split(*images, ",")
+		for _, img := range cliImages {
+			gceImages.Images[img] = GCEImage{
+				Image:   img,
+				Project: *imageProject,
+			}
+		}
+	}
+
+	if len(gceImages.Images) != 0 && *zone == "" {
+		glog.Fatal("Must specify --zone flag")
+	}
+	for shortName, image := range gceImages.Images {
+		if image.Project == "" {
+			glog.Fatalf("Invalid config for %v; must specify a project", shortName)
+		}
+	}
+	if len(gceImages.Images) != 0 {
 		if *project == "" {
-			glog.Fatal("Must specify --project flag")
+			glog.Fatal("Must specify --project flag to launch images into")
 		}
 	}
 	if *instanceNamePrefix == "" {
@@ -115,12 +168,12 @@ func main() {
 
 	results := make(chan *TestResult)
 	running := 0
-	if *images != "" {
+	for shortName, image := range gceImages.Images {
-		for _, image := range strings.Split(*images, ",") {
+		running++
-			running++
+		fmt.Printf("Initializing e2e tests using image %s.\n", shortName)
-			fmt.Printf("Initializing e2e tests using image %s.\n", image)
+		go func(image, imageProject string, junitFileNum int) {
-			go func(image string, junitFileNum int) { results <- testImage(image, junitFileNum) }(image, running)
+			results <- testImage(image, imageProject, junitFileNum)
-		}
+		}(image.Image, image.Project, running)
 	}
 	if *hosts != "" {
 		for _, host := range strings.Split(*hosts, ",") {
@@ -211,8 +264,8 @@ func testHost(host string, deleteFiles bool, junitFileNum int, setupNode bool) *
 
 // Provision a gce instance using image and run the tests in archive against the instance.
 // Delete the instance afterward.
-func testImage(image string, junitFileNum int) *TestResult {
+func testImage(image, imageProject string, junitFileNum int) *TestResult {
-	host, err := createInstance(image)
+	host, err := createInstance(image, imageProject)
 	if *deleteInstances {
 		defer deleteInstance(image)
 	}
@@ -229,7 +282,7 @@ func testImage(image string, junitFileNum int) *TestResult {
 }
 
 // Provision a gce instance using image
-func createInstance(image string) (string, error) {
+func createInstance(image, imageProject string) (string, error) {
 	name := imageToInstanceName(image)
 	i := &compute.Instance{
 		Name:        name,
@@ -249,7 +302,7 @@ func createInstance(image string) (string, error) {
 				Boot:       true,
 				Type:       "PERSISTENT",
 				InitializeParams: &compute.AttachedDiskInitializeParams{
-					SourceImage: sourceImage(image),
+					SourceImage: sourceImage(image, imageProject),
 				},
 			},
 		},
@@ -347,8 +400,8 @@ func imageToInstanceName(image string) string {
 	return *instanceNamePrefix + "-" + image
 }
 
-func sourceImage(image string) string {
+func sourceImage(image, imageProject string) string {
-	return fmt.Sprintf("projects/%s/global/images/%s", *imageProject, image)
+	return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image)
 }
 
 func machineType() string {