github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor node e2e tests

Browse Source 
- Add Makefile targets
- Start services in the test harness and connect locally
- Build test into binary and copy to remote host to start services
- Use tar to copy binaries to remote hosts to simplify design
This commit is contained in:
Phillip Wittrock
2016-02-22 10:52:20 -08:00
parent 423215fdc6
commit a3623c0c14
13 changed files with 623 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
test/e2e_node/runner/run_e2e.go
View File

@@ -19,189 +19,72 @@ limitations under the License.
package main
import (
"bufio"
"flag"
"fmt"
"os"
"os/exec"
"strings"
"sync"
"time"
"runtime"
"github.com/golang/glog"
"k8s.io/kubernetes/test/e2e_node/gcloud"
"path/filepath"
"k8s.io/kubernetes/test/e2e_node"
)
type RunFunc func(host string, port string) ([]byte, error)
type Result struct {
host string
output []byte
err error
}
const gray = "\033[1;30m"
const blue = "\033[0;34m"
const noColour = "\033[0m"
var u = sync.WaitGroup{}
var zone = flag.String("zone", "", "gce zone the hosts live in")
var hosts = flag.String("hosts", "", "hosts to test")
var wait = flag.Bool("wait", false, "if true, wait for input before running tests")
var kubeOutputRelPath = flag.String("k8s-build-output", "_output/local/bin/linux/amd64", "Where k8s binary files are written")
var kubeRoot = ""
const buildScriptRelPath = "hack/build-go.sh"
const ginkgoTestRelPath = "test/e2e_node"
const healthyTimeoutDuration = time.Minute * 3
func main() {
// Setup coloring
stat, _ := os.Stdout.Stat()
useColor := (stat.Mode() & os.ModeCharDevice) != 0
blue := ""
noColour := ""
if useColor {
blue = "\033[0;34m"
noColour = "\033[0m"
}
flag.Parse()
if *hosts == "" {
glog.Fatalf("Must specific --hosts flag")
}
// Figure out the kube root
_, path, _, _ := runtime.Caller(0)
kubeRoot, _ = filepath.Split(path)
kubeRoot = strings.Split(kubeRoot, "/test/e2e_node")[0]
// Build the go code
out, err := exec.Command(filepath.Join(kubeRoot, buildScriptRelPath)).CombinedOutput()
if err != nil {
glog.Fatalf("Failed to build go packages %s: %v", out, err)
}
// Copy kubelet to each host and run test
if *wait {
u.Add(1)
fmt.Printf("Must specific --hosts flag")
}
archive := e2e_node.CreateTestArchive()
defer os.Remove(archive)
results := make(chan *TestResult)
hs := strings.Split(*hosts, ",")
for _, h := range hs {
go func(host string) { results <- runTests(host) }(h)
}
// Maybe wait for user input before running tests
if *wait {
WaitForUser()
fmt.Printf("Starting tests on host %s.", h)
go func(host string) {
output, err := e2e_node.RunRemote(archive, host)
results <- &TestResult{
output: output,
err: err,
host: host,
}
}(h)
}
// Wait for all tests to complete and emit the results
errCount := 0
for i := 0; i < len(hs); i++ {
tr := <-results
host := tr.fullhost
host := tr.host
fmt.Printf("%s================================================================%s\n", blue, noColour)
if tr.err != nil {
errCount++
glog.Infof("%s================================================================%s", blue, noColour)
glog.Infof("Failure Finished Host %s Test Suite %s %v", host, tr.testCombinedOutput, tr.err)
glog.V(2).Infof("----------------------------------------------------------------")
glog.V(5).Infof("Host %s Etcd Logs\n%s%s%s", host, gray, tr.etcdCombinedOutput, noColour)
glog.V(5).Infof("----------------------------------------------------------------")
glog.V(5).Infof("Host %s Apiserver Logs\n%s%s%s", host, gray, tr.apiServerCombinedOutput, noColour)
glog.V(5).Infof("----------------------------------------------------------------")
glog.V(2).Infof("Host %s Kubelet Logs\n%s%s%s", host, gray, tr.kubeletCombinedOutput, noColour)
glog.Infof("%s================================================================%s", blue, noColour)
fmt.Printf("Failure Finished Host %s Test Suite %s %v\n", host, tr.output, tr.err)
} else {
glog.Infof("================================================================")
glog.Infof("Success Finished Host %s Test Suite %s", host, tr.testCombinedOutput)
glog.Infof("================================================================")
fmt.Printf("Success Finished Host %s Test Suite %s\n", host, tr.output)
}
fmt.Printf("%s================================================================%s\n", blue, noColour)
}
// Set the exit code if there were failures
if errCount > 0 {
glog.Errorf("Failure: %d errors encountered.", errCount)
fmt.Printf("Failure: %d errors encountered.", errCount)
os.Exit(1)
}
}
func WaitForUser() {
scanner := bufio.NewScanner(os.Stdin)
fmt.Printf("Enter \"y\" to run tests\n")
for scanner.Scan() {
if strings.ToUpper(scanner.Text()) != "Y\n" {
break
}
fmt.Printf("Enter \"y\" to run tests\n")
}
u.Done()
}
type TestResult struct {
fullhost string
err error
testCombinedOutput string
etcdCombinedOutput string
apiServerCombinedOutput string
kubeletCombinedOutput string
}
func runTests(fullhost string) *TestResult {
result := &TestResult{fullhost: fullhost}
host := strings.Split(fullhost, ".")[0]
c := gcloud.NewGCloudClient(host, *zone)
// TODO(pwittrock): Come up with something better for bootstrapping the environment.
eh, err := c.RunAndWaitTillHealthy(
false, false, "4001", healthyTimeoutDuration, "v2/keys/", "etcd", "--data-dir", "./", "--name", "e2e-node")
defer func() {
eh.TearDown()
result.etcdCombinedOutput = fmt.Sprintf("%s", eh.CombinedOutput.Bytes())
}()
if err != nil {
result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
return result
}
apiBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kube-apiserver")
ah, err := c.RunAndWaitTillHealthy(
true, true, "8080", healthyTimeoutDuration, "healthz", apiBin, "--service-cluster-ip-range",
"10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://127.0.0.1:4001",
"--v", "2", "--alsologtostderr", "--kubelet-port", "10250")
defer func() {
ah.TearDown()
result.apiServerCombinedOutput = fmt.Sprintf("%s", ah.CombinedOutput.Bytes())
}()
if err != nil {
result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
return result
}
kubeletBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kubelet")
// TODO: Used --v 4 or higher and upload to gcs instead of printing to the console
// TODO: Copy /var/log/messages and upload to GCS for failed tests
kh, err := c.RunAndWaitTillHealthy(
true, true, "10255", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://127.0.0.1:8080",
"--v", "2", "--alsologtostderr", "--address", "0.0.0.0", "--port", "10250")
defer func() {
kh.TearDown()
result.kubeletCombinedOutput = fmt.Sprintf("%s", kh.CombinedOutput.Bytes())
}()
if err != nil {
result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
}
// Run the tests
glog.Infof("Kubelet healthy on host %s", host)
glog.Infof("Kubelet host %s tunnel running on port %s", host, ah.LPort)
u.Wait()
glog.Infof("Running ginkgo tests against host %s", host)
ginkgoTests := filepath.Join(kubeRoot, ginkgoTestRelPath)
out, err := exec.Command(
"ginkgo", ginkgoTests, "--",
"--kubelet-address", fmt.Sprintf("http://127.0.0.1:%s", kh.LPort),
"--api-server-address", fmt.Sprintf("http://127.0.0.1:%s", ah.LPort),
"--node-name", fullhost,
"--v", "2", "--alsologtostderr").CombinedOutput()
result.err = err
result.testCombinedOutput = fmt.Sprintf("%s", out)
return result
output string
err error
host string
}