diff --git a/test/e2e_node/builder/build.go b/test/e2e_node/builder/build.go index ad84233208c..0c07db16229 100644 --- a/test/e2e_node/builder/build.go +++ b/test/e2e_node/builder/build.go @@ -29,6 +29,8 @@ import ( ) var k8sBinDir = flag.String("k8s-bin-dir", "", "Directory containing k8s kubelet binaries.") +var useDockerizedBuild = flag.Bool("use-dockerized-build", false, "Use dockerized build for test artifacts") +var targetBuildArch = flag.String("target-build-arch", "linux/amd64", "Target architecture for the test artifacts for dockerized build") var buildTargets = []string{ "cmd/kubelet", @@ -47,6 +49,11 @@ func BuildGo() error { } targets := strings.Join(buildTargets, " ") cmd := exec.Command("make", "-C", k8sRoot, fmt.Sprintf("WHAT=%s", targets)) + if IsDockerizedBuild() { + klog.Infof("Building dockerized k8s binaries targets %s for architecture %s", targets, GetTargetBuildArch()) + // Multi-architecture build is only supported in dockerized build + cmd = exec.Command(filepath.Join(k8sRoot, "build/run.sh"), "make", fmt.Sprintf("WHAT=%s", targets), fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", GetTargetBuildArch())) + } cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err = cmd.Run() @@ -56,6 +63,21 @@ func BuildGo() error { return nil } +// IsDockerizedBuild returns if test needs to use dockerized build +func IsDockerizedBuild() bool { + return *useDockerizedBuild +} + +// GetTargetBuildArch returns the target build architecture for dockerized build +func GetTargetBuildArch() string { + return *targetBuildArch +} + +// IsTargetArchArm64 returns if the target is for linux/arm64 platform +func IsTargetArchArm64() bool { + return GetTargetBuildArch() == "linux/arm64" +} + func getK8sBin(bin string) (string, error) { // Use commandline specified path if *k8sBinDir != "" { @@ -77,7 +99,7 @@ func getK8sBin(bin string) (string, error) { return filepath.Join(path, bin), nil } - buildOutputDir, err := utils.GetK8sBuildOutputDir() + buildOutputDir, err := utils.GetK8sBuildOutputDir(IsDockerizedBuild(), GetTargetBuildArch()) if err != nil { return "", err } diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index aabe5a25da7..500e54ed2d4 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "os/user" + "runtime" "sync" "time" @@ -61,7 +62,6 @@ var NodePrePullImageList = sets.NewString( imageutils.GetPauseImageName(), imageutils.GetE2EImage(imageutils.NodePerfNpbEp), imageutils.GetE2EImage(imageutils.NodePerfNpbIs), - imageutils.GetE2EImage(imageutils.NodePerfTfWideDeep), imageutils.GetE2EImage(imageutils.Etcd), ) @@ -70,6 +70,11 @@ var NodePrePullImageList = sets.NewString( // 2. the ones passed in from framework.TestContext.ExtraEnvs // So this function needs to be called after the extra envs are applied. func updateImageAllowList(ctx context.Context) { + // Architecture-specific image + if !isRunningOnArm64() { + // NodePerfTfWideDeep is only supported on x86_64, pulling in arm64 will fail + NodePrePullImageList = NodePrePullImageList.Insert(imageutils.GetE2EImage(imageutils.NodePerfTfWideDeep)) + } // Union NodePrePullImageList and PrePulledImages into the framework image pre-pull list. e2epod.ImagePrePullList = NodePrePullImageList.Union(commontest.PrePulledImages) // Images from extra envs @@ -96,6 +101,10 @@ func updateImageAllowList(ctx context.Context) { } } +func isRunningOnArm64() bool { + return runtime.GOARCH == "arm64" +} + func getNodeProblemDetectorImage() string { const defaultImage string = "registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.7" image := os.Getenv("NODE_PROBLEM_DETECTOR_IMAGE") diff --git a/test/e2e_node/remote/node_conformance.go b/test/e2e_node/remote/node_conformance.go index 7c9130b974c..551499972d6 100644 --- a/test/e2e_node/remote/node_conformance.go +++ b/test/e2e_node/remote/node_conformance.go @@ -107,7 +107,7 @@ func (c *ConformanceRemote) SetupTestPackage(tardir, systemSpecName string) erro } // Make sure we can find the newly built binaries - buildOutputDir, err := utils.GetK8sBuildOutputDir() + buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch()) if err != nil { return fmt.Errorf("failed to locate kubernetes build output directory %v", err) } diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index 419766a5a7c..a05aae75b7b 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -48,7 +48,7 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { } // Make sure we can find the newly built binaries - buildOutputDir, err := utils.GetK8sBuildOutputDir() + buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch()) if err != nil { return fmt.Errorf("failed to locate kubernetes build output directory: %w", err) } @@ -62,6 +62,7 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo", "mounter", "gcp-credential-provider"} for _, bin := range requiredBins { source := filepath.Join(buildOutputDir, bin) + klog.V(2).Infof("Copying binaries from %s", source) if _, err := os.Stat(source); err != nil { return fmt.Errorf("failed to locate test binary %s: %w", bin, err) } diff --git a/test/e2e_node/remote/utils.go b/test/e2e_node/remote/utils.go index 10e7cba5cef..488c568a5f2 100644 --- a/test/e2e_node/remote/utils.go +++ b/test/e2e_node/remote/utils.go @@ -21,16 +21,16 @@ import ( "path/filepath" "k8s.io/klog/v2" + + "k8s.io/kubernetes/test/e2e_node/builder" ) // utils.go contains functions used across test suites. const ( cniVersion = "v1.2.0" - cniArch = "amd64" cniDirectory = "cni/bin" // The CNI tarball places binaries under directory under "cni/bin". cniConfDirectory = "cni/net.d" - cniURL = "https://storage.googleapis.com/k8s-artifacts-cni/release/" + cniVersion + "/" + "cni-plugins-linux-" + cniArch + "-" + cniVersion + ".tgz" ) const cniConfig = `{ @@ -60,14 +60,25 @@ providers: - "*.pkg.dev" defaultCacheDuration: 1m` +func getCNIURL() string { + cniArch := "amd64" + if builder.IsTargetArchArm64() { + cniArch = "arm64" + } + cniURL := fmt.Sprintf("https://storage.googleapis.com/k8s-artifacts-cni/release/%s/cni-plugins-linux-%s-%s.tgz", cniVersion, cniArch, cniVersion) + return cniURL + +} + // Install the cni plugin and add basic bridge configuration to the // configuration directory. func setupCNI(host, workspace string) error { klog.V(2).Infof("Install CNI on %q", host) cniPath := filepath.Join(workspace, cniDirectory) + klog.V(2).Infof("Install CNI on path %q", cniPath) cmd := getSSHCommand(" ; ", fmt.Sprintf("mkdir -p %s", cniPath), - fmt.Sprintf("curl -s -L %s | tar -xz -C %s", cniURL, cniPath), + fmt.Sprintf("curl -s -L %s | tar -xz -C %s", getCNIURL(), cniPath), ) if output, err := SSH(host, "sh", "-c", cmd); err != nil { return fmt.Errorf("failed to install cni plugin on %q: %v output: %q", host, err, output) diff --git a/test/e2e_node/runner/local/run_local.go b/test/e2e_node/runner/local/run_local.go index 5787fe1de88..a35ba1ef5b3 100644 --- a/test/e2e_node/runner/local/run_local.go +++ b/test/e2e_node/runner/local/run_local.go @@ -51,7 +51,7 @@ func main() { } // Run node e2e test - outputDir, err := utils.GetK8sBuildOutputDir() + outputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch()) if err != nil { klog.Fatalf("Failed to get build output directory: %v", err) } diff --git a/test/utils/paths.go b/test/utils/paths.go index a980cda733c..eaedb02aed2 100644 --- a/test/utils/paths.go +++ b/test/utils/paths.go @@ -64,12 +64,17 @@ func RootDir() (string, error) { } // GetK8sBuildOutputDir returns the build output directory for k8s -func GetK8sBuildOutputDir() (string, error) { +// For dockerized build, targetArch (eg: 'linux/arm64', 'linux/amd64') must be explicitly specified +// For non dockerized build, targetArch is ignored +func GetK8sBuildOutputDir(isDockerizedBuild bool, targetArch string) (string, error) { k8sRoot, err := GetK8sRootDir() if err != nil { return "", err } buildOutputDir := filepath.Join(k8sRoot, "_output/local/go/bin") + if isDockerizedBuild { + buildOutputDir = filepath.Join(k8sRoot, "_output/dockerized/bin/", targetArch) + } if _, err := os.Stat(buildOutputDir); err != nil { return "", err }