Merge pull request #29764 from coufon/node_density_and_performance_test

Automatic merge from submit-queue

Add density (batch pods creation latency and resource) and resource performance tests to `test-e2e-node'

This PR contains two new tests (migrate from e2e test):
1. Density test: verify startup latency and resource usage when create a batch of pod with throughput control. Throughput control is done by sleep for an interval between firing concurrently create pod operations. 
It tests both batch creation and sequential (back-to-back) creation and report the throughputs. 

2. Verify resource usage of steady state kubelet.

The test creates a new resource controller for `test-node-e2e' (resource_controller.go) which monitors resource through a standalone Cadvisor pod (port 8090) with 1s housekeeping interval.

vendor/github.com/google/cadvisor/client/v2/README.md

@@ -0,0 +1,69 @@
+# Example REST API Client
+
+This is an implementation of a cAdvisor REST API in Go.  You can use it like this:
+
+```go
+client, err := client.NewClient("http://192.168.59.103:8080/")
+```
+
+Obviously, replace the URL with the path to your actual cAdvisor REST endpoint.
+
+
+### MachineInfo
+
+```go
+client.MachineInfo()
+```
+
+There is no v2 MachineInfo API, so the v2 client exposes the [v1 MachineInfo](../../info/v1/machine.go#L131)
+
+```
+(*v1.MachineInfo)(0xc208022b10)({
+ NumCores: (int) 4,
+ MemoryCapacity: (int64) 2106028032,
+ Filesystems: ([]v1.FsInfo) (len=1 cap=4) {
+  (v1.FsInfo) {
+   Device: (string) (len=9) "/dev/sda1",
+   Capacity: (uint64) 19507089408
+  }
+ }
+})
+```
+
+You can see the full specification of the [MachineInfo struct in the source](../../info/v1/machine.go#L131)
+
+### VersionInfo
+
+```go
+client.VersionInfo()
+```
+
+This method returns the cAdvisor version.
+
+### Attributes
+
+```go
+client.Attributes()
+```
+
+This method returns a [cadvisor/info/v2/Attributes](../../info/v2/machine.go#L24) struct with all the fields filled in. Attributes includes hardware attributes (as returned by MachineInfo) as well as software attributes (eg. software versions). Here is an example return value:
+
+```
+(*v2.Attributes)({
+ KernelVersion: (string) (len=17) "3.13.0-44-generic"
+ ContainerOsVersion: (string) (len=18) "Ubuntu 14.04.1 LTS"
+ DockerVersion: (string) (len=9) "1.5.0-rc4"
+ CadvisorVersion: (string) (len=6) "0.10.1"
+ NumCores: (int) 4,
+ MemoryCapacity: (int64) 2106028032,
+ Filesystems: ([]v2.FsInfo) (len=1 cap=4) {
+  (v2.FsInfo) {
+   Device: (string) (len=9) "/dev/sda1",
+   Capacity: (uint64) 19507089408
+  }
+ }
+})
+```
+
+You can see the full specification of the [Attributes struct in the source](../../info/v2/machine.go#L24)
+

vendor/github.com/google/cadvisor/client/v2/client.go

@@ -0,0 +1,176 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Client library to programmatically access cAdvisor API.
+package v2
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+
+	v1 "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/info/v2"
+)
+
+// Client represents the base URL for a cAdvisor client.
+type Client struct {
+	baseUrl string
+}
+
+// NewClient returns a new client with the specified base URL.
+func NewClient(url string) (*Client, error) {
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
+	}
+
+	return &Client{
+		baseUrl: fmt.Sprintf("%sapi/v2.1/", url),
+	}, nil
+}
+
+// MachineInfo returns the JSON machine information for this client.
+// A non-nil error result indicates a problem with obtaining
+// the JSON machine information data.
+func (self *Client) MachineInfo() (minfo *v1.MachineInfo, err error) {
+	u := self.machineInfoUrl()
+	ret := new(v1.MachineInfo)
+	if err = self.httpGetJsonData(ret, nil, u, "machine info"); err != nil {
+		return
+	}
+	minfo = ret
+	return
+}
+
+// MachineStats returns the JSON machine statistics for this client.
+// A non-nil error result indicates a problem with obtaining
+// the JSON machine information data.
+func (self *Client) MachineStats() ([]v2.MachineStats, error) {
+	var ret []v2.MachineStats
+	u := self.machineStatsUrl()
+	err := self.httpGetJsonData(&ret, nil, u, "machine stats")
+	return ret, err
+}
+
+// VersionInfo returns the version info for cAdvisor.
+func (self *Client) VersionInfo() (version string, err error) {
+	u := self.versionInfoUrl()
+	version, err = self.httpGetString(u, "version info")
+	return
+}
+
+// Attributes returns hardware and software attributes of the machine.
+func (self *Client) Attributes() (attr *v2.Attributes, err error) {
+	u := self.attributesUrl()
+	ret := new(v2.Attributes)
+	if err = self.httpGetJsonData(ret, nil, u, "attributes"); err != nil {
+		return
+	}
+	attr = ret
+	return
+}
+
+// Stats returns stats for the requested container.
+func (self *Client) Stats(name string, request *v2.RequestOptions) (map[string]v2.ContainerInfo, error) {
+	u := self.statsUrl(name)
+	ret := make(map[string]v2.ContainerInfo)
+	data := url.Values{
+		"type":      []string{request.IdType},
+		"count":     []string{strconv.Itoa(request.Count)},
+		"recursive": []string{strconv.FormatBool(request.Recursive)},
+	}
+
+	u = fmt.Sprintf("%s?%s", u, data.Encode())
+	if err := self.httpGetJsonData(&ret, nil, u, "stats"); err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
+func (self *Client) machineInfoUrl() string {
+	return self.baseUrl + path.Join("machine")
+}
+
+func (self *Client) machineStatsUrl() string {
+	return self.baseUrl + path.Join("machinestats")
+}
+
+func (self *Client) versionInfoUrl() string {
+	return self.baseUrl + path.Join("version")
+}
+
+func (self *Client) attributesUrl() string {
+	return self.baseUrl + path.Join("attributes")
+}
+
+func (self *Client) statsUrl(name string) string {
+	return self.baseUrl + path.Join("stats", name)
+}
+
+func (self *Client) httpGetResponse(postData interface{}, urlPath, infoName string) ([]byte, error) {
+	var resp *http.Response
+	var err error
+
+	if postData != nil {
+		data, marshalErr := json.Marshal(postData)
+		if marshalErr != nil {
+			return nil, fmt.Errorf("unable to marshal data: %v", marshalErr)
+		}
+		resp, err = http.Post(urlPath, "application/json", bytes.NewBuffer(data))
+	} else {
+		resp, err = http.Get(urlPath)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("unable to post %q to %q: %v", infoName, urlPath, err)
+	}
+	if resp == nil {
+		return nil, fmt.Errorf("received empty response for %q from %q", infoName, urlPath)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		err = fmt.Errorf("unable to read all %q from %q: %v", infoName, urlPath, err)
+		return nil, err
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("request %q failed with error: %q", urlPath, strings.TrimSpace(string(body)))
+	}
+	return body, nil
+}
+
+func (self *Client) httpGetString(url, infoName string) (string, error) {
+	body, err := self.httpGetResponse(nil, url, infoName)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
+func (self *Client) httpGetJsonData(data, postData interface{}, url, infoName string) error {
+	body, err := self.httpGetResponse(postData, url, infoName)
+	if err != nil {
+		return err
+	}
+	if err = json.Unmarshal(body, data); err != nil {
+		err = fmt.Errorf("unable to unmarshal %q (Body: %q) from %q with error: %v", infoName, string(body), url, err)
+		return err
+	}
+	return nil
+}