Merge pull request #117017 from ike-ma/arm-ci

Setup e2e_node to support testing on ARM64

test/e2e_node/builder/build.go

@@ -29,6 +29,8 @@ import (
 )
 
 var k8sBinDir = flag.String("k8s-bin-dir", "", "Directory containing k8s kubelet binaries.")
+var useDockerizedBuild = flag.Bool("use-dockerized-build", false, "Use dockerized build for test artifacts")
+var targetBuildArch = flag.String("target-build-arch", "linux/amd64", "Target architecture for the test artifacts for dockerized build")
 
 var buildTargets = []string{
 	"cmd/kubelet",
@@ -47,6 +49,11 @@ func BuildGo() error {
 	}
 	targets := strings.Join(buildTargets, " ")
 	cmd := exec.Command("make", "-C", k8sRoot, fmt.Sprintf("WHAT=%s", targets))
+	if IsDockerizedBuild() {
+		klog.Infof("Building dockerized k8s binaries targets %s for architecture %s", targets, GetTargetBuildArch())
+		// Multi-architecture build is only supported in dockerized build
+		cmd = exec.Command(filepath.Join(k8sRoot, "build/run.sh"), "make", fmt.Sprintf("WHAT=%s", targets), fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", GetTargetBuildArch()))
+	}
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()
@@ -56,6 +63,21 @@ func BuildGo() error {
 	return nil
 }
 
+// IsDockerizedBuild returns if test needs to use dockerized build
+func IsDockerizedBuild() bool {
+	return *useDockerizedBuild
+}
+
+// GetTargetBuildArch returns the target build architecture for dockerized build
+func GetTargetBuildArch() string {
+	return *targetBuildArch
+}
+
+// IsTargetArchArm64 returns if the target is for linux/arm64 platform
+func IsTargetArchArm64() bool {
+	return GetTargetBuildArch() == "linux/arm64"
+}
+
 func getK8sBin(bin string) (string, error) {
 	// Use commandline specified path
 	if *k8sBinDir != "" {
@@ -77,7 +99,7 @@ func getK8sBin(bin string) (string, error) {
 		return filepath.Join(path, bin), nil
 	}
 
-	buildOutputDir, err := utils.GetK8sBuildOutputDir()
+	buildOutputDir, err := utils.GetK8sBuildOutputDir(IsDockerizedBuild(), GetTargetBuildArch())
 	if err != nil {
 		return "", err
 	}

test/e2e_node/image_list.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 	"os/user"
+	"runtime"
 	"sync"
 	"time"
 
@@ -61,7 +62,6 @@ var NodePrePullImageList = sets.NewString(
 	imageutils.GetPauseImageName(),
 	imageutils.GetE2EImage(imageutils.NodePerfNpbEp),
 	imageutils.GetE2EImage(imageutils.NodePerfNpbIs),
-	imageutils.GetE2EImage(imageutils.NodePerfTfWideDeep),
 	imageutils.GetE2EImage(imageutils.Etcd),
 )
 
@@ -70,6 +70,11 @@ var NodePrePullImageList = sets.NewString(
 // 2. the ones passed in from framework.TestContext.ExtraEnvs
 // So this function needs to be called after the extra envs are applied.
 func updateImageAllowList(ctx context.Context) {
+	// Architecture-specific image
+	if !isRunningOnArm64() {
+		// NodePerfTfWideDeep is only supported on x86_64, pulling in arm64 will fail
+		NodePrePullImageList = NodePrePullImageList.Insert(imageutils.GetE2EImage(imageutils.NodePerfTfWideDeep))
+	}
 	// Union NodePrePullImageList and PrePulledImages into the framework image pre-pull list.
 	e2epod.ImagePrePullList = NodePrePullImageList.Union(commontest.PrePulledImages)
 	// Images from extra envs
@@ -96,6 +101,10 @@ func updateImageAllowList(ctx context.Context) {
 	}
 }
 
+func isRunningOnArm64() bool {
+	return runtime.GOARCH == "arm64"
+}
+
 func getNodeProblemDetectorImage() string {
 	const defaultImage string = "registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.7"
 	image := os.Getenv("NODE_PROBLEM_DETECTOR_IMAGE")

test/e2e_node/remote/node_conformance.go

@@ -107,7 +107,7 @@ func (c *ConformanceRemote) SetupTestPackage(tardir, systemSpecName string) erro
 	}
 
 	// Make sure we can find the newly built binaries
-	buildOutputDir, err := utils.GetK8sBuildOutputDir()
+	buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch())
 	if err != nil {
 		return fmt.Errorf("failed to locate kubernetes build output directory %v", err)
 	}

test/e2e_node/remote/node_e2e.go

@@ -48,7 +48,7 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
 	}
 
 	// Make sure we can find the newly built binaries
-	buildOutputDir, err := utils.GetK8sBuildOutputDir()
+	buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch())
 	if err != nil {
 		return fmt.Errorf("failed to locate kubernetes build output directory: %w", err)
 	}
@@ -62,6 +62,7 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
 	requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo", "mounter", "gcp-credential-provider"}
 	for _, bin := range requiredBins {
 		source := filepath.Join(buildOutputDir, bin)
+		klog.V(2).Infof("Copying binaries from %s", source)
 		if _, err := os.Stat(source); err != nil {
 			return fmt.Errorf("failed to locate test binary %s: %w", bin, err)
 		}

test/e2e_node/remote/utils.go

@@ -21,16 +21,16 @@ import (
 	"path/filepath"
 
 	"k8s.io/klog/v2"
+
+	"k8s.io/kubernetes/test/e2e_node/builder"
 )
 
 // utils.go contains functions used across test suites.
 
 const (
 	cniVersion       = "v1.2.0"
-	cniArch          = "amd64"
 	cniDirectory     = "cni/bin" // The CNI tarball places binaries under directory under "cni/bin".
 	cniConfDirectory = "cni/net.d"
-	cniURL           = "https://storage.googleapis.com/k8s-artifacts-cni/release/" + cniVersion + "/" + "cni-plugins-linux-" + cniArch + "-" + cniVersion + ".tgz"
 )
 
 const cniConfig = `{
@@ -60,14 +60,25 @@ providers:
     - "*.pkg.dev"
     defaultCacheDuration: 1m`
 
+func getCNIURL() string {
+	cniArch := "amd64"
+	if builder.IsTargetArchArm64() {
+		cniArch = "arm64"
+	}
+	cniURL := fmt.Sprintf("https://storage.googleapis.com/k8s-artifacts-cni/release/%s/cni-plugins-linux-%s-%s.tgz", cniVersion, cniArch, cniVersion)
+	return cniURL
+
+}
+
 // Install the cni plugin and add basic bridge configuration to the
 // configuration directory.
 func setupCNI(host, workspace string) error {
 	klog.V(2).Infof("Install CNI on %q", host)
 	cniPath := filepath.Join(workspace, cniDirectory)
+	klog.V(2).Infof("Install CNI on path %q", cniPath)
 	cmd := getSSHCommand(" ; ",
 		fmt.Sprintf("mkdir -p %s", cniPath),
-		fmt.Sprintf("curl -s -L %s | tar -xz -C %s", cniURL, cniPath),
+		fmt.Sprintf("curl -s -L %s | tar -xz -C %s", getCNIURL(), cniPath),
 	)
 	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
 		return fmt.Errorf("failed to install cni plugin on %q: %v output: %q", host, err, output)

test/e2e_node/runner/local/run_local.go

@@ -51,7 +51,7 @@ func main() {
 	}
 
 	// Run node e2e test
-	outputDir, err := utils.GetK8sBuildOutputDir()
+	outputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch())
 	if err != nil {
 		klog.Fatalf("Failed to get build output directory: %v", err)
 	}

test/utils/paths.go

@@ -64,12 +64,17 @@ func RootDir() (string, error) {
 }
 
 // GetK8sBuildOutputDir returns the build output directory for k8s
-func GetK8sBuildOutputDir() (string, error) {
+// For dockerized build, targetArch (eg: 'linux/arm64', 'linux/amd64') must be explicitly specified
+// For non dockerized build, targetArch is ignored
+func GetK8sBuildOutputDir(isDockerizedBuild bool, targetArch string) (string, error) {
 	k8sRoot, err := GetK8sRootDir()
 	if err != nil {
 		return "", err
 	}
 	buildOutputDir := filepath.Join(k8sRoot, "_output/local/go/bin")
+	if isDockerizedBuild {
+		buildOutputDir = filepath.Join(k8sRoot, "_output/dockerized/bin/", targetArch)
+	}
 	if _, err := os.Stat(buildOutputDir); err != nil {
 		return "", err
 	}