github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reorganize node e2e output directories.

Browse Source
This commit is contained in:
Random-Liu
2016-10-28 22:56:23 -07:00
parent a0758f6767
commit d9ddd64c9c
4 changed files with 44 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
hack/make-rules/test-e2e-node.sh
View File

@@ -26,8 +26,7 @@ skip=${SKIP:-""}
# Currently, parallelism only affects when REMOTE=true. For local test,
# ginkgo default parallelism (cores - 1) is used.
parallelism=${PARALLELISM:-8}
report=${REPORT:-"/tmp/"}
artifacts=${ARTIFACTS:-"/tmp/_artifacts"}
artifacts=${ARTIFACTS:-"/tmp/_artifacts/`date +%y%m%dT%H%M%S`"}
remote=${REMOTE:-"false"}
run_until_failure=${RUN_UNTIL_FAILURE:-"false"}
test_args=${TEST_ARGS:-""}
@@ -50,6 +49,12 @@ if [[ $run_until_failure != "" ]]; then
ginkgoflags="$ginkgoflags -untilItFails=$run_until_failure "
fi
# Setup the directory to copy test artifacts (logs, junit.xml, etc) from remote host to local host
if [ ! -d "${artifacts}" ]; then
echo "Creating artifacts directory at ${artifacts}"
mkdir -p ${artifacts}
fi
echo "Test artifacts will be written to ${artifacts}"
if [ $remote = true ] ; then
# The following options are only valid in remote run.
@@ -74,18 +79,6 @@ if [ $remote = true ] ; then
cleanup=${CLEANUP:-"true"}
delete_instances=${DELETE_INSTANCES:-"false"}
# Setup the directory to copy test artifacts (logs, junit.xml, etc) from remote host to local host
if [[ $gubernator = true && -d "${artifacts}" ]]; then
echo "Removing artifacts directory at ${artifacts}"
rm -r ${artifacts}
fi
if [ ! -d "${artifacts}" ]; then
echo "Creating artifacts directory at ${artifacts}"
mkdir -p ${artifacts}
fi
echo "Test artifacts will be written to ${artifacts}"
# Get the compute zone
zone=$(gcloud info --format='value(config.properties.compute.zone)')
if [[ $zone == "" ]]; then
@@ -155,7 +148,7 @@ else
# Test using the host the script was run on
# Provided for backwards compatibility
go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \
--test-flags="--alsologtostderr --v 4 --report-dir=${report} --node-name $(hostname) \
$test_args" --build-dependencies=true
--test-flags="--alsologtostderr --v 4 --report-dir=${artifacts} --node-name $(hostname) \
$test_args" --build-dependencies=true 2>&1 | tee "${artifacts}/build-log.txt"
exit $?
fi

48
test/e2e_node/remote/remote.go
View File

@@ -20,7 +20,6 @@ import (
"flag"
"fmt"
"io/ioutil"
"math/rand"
"os"
"os/exec"
"os/user"
@@ -169,26 +168,24 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
// Create the temp staging directory
glog.Infof("Staging test binaries on %s", host)
dirName := fmt.Sprintf("gcloud-e2e-%d", rand.Int31())
tmp := fmt.Sprintf("/tmp/%s", dirName)
workspace := fmt.Sprintf("/tmp/node-e2e-%s", getTimestamp())
// Do not sudo here, so that we can use scp to copy test archive to the directdory.
_, err := SSHNoSudo(host, "mkdir", tmp)
_, err := SSHNoSudo(host, "mkdir", workspace)
if err != nil {
// Exit failure with the error
return "", false, err
}
if cleanup {
defer func() {
output, err := SSH(host, "rm", "-rf", tmp)
output, err := SSH(host, "rm", "-rf", workspace)
if err != nil {
glog.Errorf("failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output)
glog.Errorf("failed to cleanup workspace %s on host %v. Output:\n%s", workspace, err, output)
}
}()
}
// Install the cni plugin.
cniPath := filepath.Join(tmp, CNIDirectory)
cniPath := filepath.Join(workspace, CNIDirectory)
cmd := getSSHCommand(" ; ",
fmt.Sprintf("mkdir -p %s", cniPath),
fmt.Sprintf("wget -O - %s | tar -xz -C %s", CNIURL, cniPath),
@@ -220,7 +217,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
}
// Copy the archive to the staging directory
_, err = runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), tmp))
_, err = runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), workspace))
if err != nil {
// Exit failure with the error
return "", false, err
@@ -240,7 +237,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
// Extract the archive
cmd = getSSHCommand(" && ",
fmt.Sprintf("cd %s", tmp),
fmt.Sprintf("cd %s", workspace),
fmt.Sprintf("tar -xzvf ./%s", archiveName),
)
glog.Infof("Extracting tar on %s", host)
@@ -251,7 +248,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
}
// If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args.
// We do this here because the local var `tmp` tells us which /tmp/gcloud-e2e-%d is relevant to the current test run.
// We do this here because the local var `workspace` tells us which /tmp/node-e2e-%d is relevant to the current test run.
// Determine if the GCI mounter script exists locally.
k8sDir, err := builder.GetK8sRootDir()
@@ -275,7 +272,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
if strings.Contains(output, "ID=gci") {
// Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error
// here will tell us to update the remote test runner.
mounterPath := filepath.Join(tmp, "cluster/gce/gci/mounter/mounter")
mounterPath := filepath.Join(workspace, "cluster/gce/gci/mounter/mounter")
output, err = SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath))
if err != nil {
glog.Errorf("Unable to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output)
@@ -291,9 +288,9 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
// Run the tests
cmd = getSSHCommand(" && ",
fmt.Sprintf("cd %s", tmp),
fmt.Sprintf("cd %s", workspace),
fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s/results --report-prefix=%s %s",
testTimeoutSeconds.Seconds(), ginkgoFlags, host, tmp, junitFilePrefix, testArgs),
testTimeoutSeconds.Seconds(), ginkgoFlags, host, workspace, junitFilePrefix, testArgs),
)
aggErrs := []error{}
@@ -311,8 +308,8 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
// TODO: This is a best-effort, temporary hack that only works for
// journald nodes. We should have a more robust way to collect logs.
var (
logName = fmt.Sprintf("%s-system.log", dirName)
logPath = fmt.Sprintf("/tmp/%s-system.log", dirName)
logName = "system.log"
logPath = filepath.Join(workspace, logName)
destPath = fmt.Sprintf("%s/%s-%s", *resultsDir, host, logName)
)
glog.Infof("Test failed unexpectedly. Attempting to retreiving system logs (only works for nodes with journald)")
@@ -331,7 +328,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
}
glog.Infof("Copying test artifacts from %s", host)
scpErr := getTestArtifacts(host, tmp)
scpErr := getTestArtifacts(host, workspace)
if scpErr != nil {
aggErrs = append(aggErrs, scpErr)
}
@@ -339,14 +336,25 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
return output, len(aggErrs) == 0, utilerrors.NewAggregate(aggErrs)
}
// timestampFormat is the timestamp format used in the node e2e directory name.
const timestampFormat = "20060102T150405"
func getTimestamp() string {
return fmt.Sprintf(time.Now().Format(timestampFormat))
}
func getTestArtifacts(host, testDir string) error {
_, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/", GetHostnameOrIp(host), testDir), fmt.Sprintf("%s/%s", *resultsDir, host))
logPath := filepath.Join(*resultsDir, host)
if err := os.MkdirAll(logPath, 0755); err != nil {
return fmt.Errorf("failed to create log directory %q: %v", logPath, err)
}
// Copy logs to artifacts/hostname
_, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.log", GetHostnameOrIp(host), testDir), logPath)
if err != nil {
return err
}
// Copy junit to the top of artifacts
_, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIp(host), testDir), fmt.Sprintf("%s/", *resultsDir))
_, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIp(host), testDir), *resultsDir)
if err != nil {
return err
}

3
test/e2e_node/services/internal_services.go
View File

@@ -115,7 +115,8 @@ func (es *e2eServices) stop() {
// startEtcd starts the embedded etcd instance or returns an error.
func (es *e2eServices) startEtcd() error {
glog.Info("Starting etcd")
dataDir, err := ioutil.TempDir("", "node-e2e")
// Create data directory in current working space.
dataDir, err := ioutil.TempDir(".", "etcd")
if err != nil {
return err
}

6
test/e2e_node/services/services.go
View File

@@ -82,7 +82,11 @@ func (e *E2EServices) Start() error {
// Start kubelet
// Create the manifest path for kubelet.
// TODO(random-liu): Remove related logic when we move kubelet starting logic out of the test.
framework.TestContext.ManifestPath, err = ioutil.TempDir("", "node-e2e-pod")
cwd, err := os.Getwd()
if err != nil {
return fmt.Errorf("failed to get current working directory: %v", err)
}
framework.TestContext.ManifestPath, err = ioutil.TempDir(cwd, "pod-manifest")
if err != nil {
return fmt.Errorf("failed to create static pod manifest directory: %v", err)
}