udpate cadvisor dependency to v0.28.2

This commit is contained in:
David Ashpole
2017-11-21 17:13:50 -08:00
parent 65f5c1e847
commit 6893e3c24f
88 changed files with 24943 additions and 214 deletions

View File

@@ -21,6 +21,7 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"time"
info "github.com/google/cadvisor/info/v1"
@@ -30,6 +31,8 @@ import (
)
type NvidiaManager struct {
sync.RWMutex
// true if the NVML library (libnvidia-ml.so.1) was loaded successfully
nvmlInitialized bool
@@ -48,12 +51,12 @@ func (nm *NvidiaManager) Setup() {
return
}
nm.initializeNVML()
if nm.nvmlInitialized {
return
}
go func() {
glog.Info("Starting goroutine to initialize NVML")
nm.initializeNVML()
if nm.nvmlInitialized {
return
}
// TODO: use globalHousekeepingInterval
for range time.Tick(time.Minute) {
nm.initializeNVML()
@@ -95,10 +98,13 @@ func (nm *NvidiaManager) initializeNVML() {
glog.V(3).Infof("Could not initialize NVML: %v", err)
return
}
nm.nvmlInitialized = true
numDevices, err := gonvml.DeviceCount()
if err != nil {
glog.Warningf("GPU metrics would not be available. Failed to get the number of nvidia devices: %v", err)
nm.Lock()
// Even though we won't have GPU metrics, the library was initialized and should be shutdown when exiting.
nm.nvmlInitialized = true
nm.Unlock()
return
}
glog.Infof("NVML initialized. Number of nvidia devices: %v", numDevices)
@@ -116,6 +122,10 @@ func (nm *NvidiaManager) initializeNVML() {
}
nm.nvidiaDevices[int(minorNumber)] = device
}
nm.Lock()
// Doing this at the end to avoid race in accessing nvidiaDevices in GetCollector.
nm.nvmlInitialized = true
nm.Unlock()
}
// Destroy shuts down NVML.
@@ -129,9 +139,12 @@ func (nm *NvidiaManager) Destroy() {
// present in the devices.list file in the given devicesCgroupPath.
func (nm *NvidiaManager) GetCollector(devicesCgroupPath string) (AcceleratorCollector, error) {
nc := &NvidiaCollector{}
nm.RLock()
if !nm.nvmlInitialized || len(nm.nvidiaDevices) == 0 {
nm.RUnlock()
return nc, nil
}
nm.RUnlock()
nvidiaMinorNumbers, err := parseDevicesCgroup(devicesCgroupPath)
if err != nil {
return nc, err

View File

@@ -27,6 +27,7 @@ filegroup(
srcs = [
":package-srcs",
"//vendor/github.com/google/cadvisor/container/common:all-srcs",
"//vendor/github.com/google/cadvisor/container/containerd:all-srcs",
"//vendor/github.com/google/cadvisor/container/crio:all-srcs",
"//vendor/github.com/google/cadvisor/container/docker:all-srcs",
"//vendor/github.com/google/cadvisor/container/libcontainer:all-srcs",

View File

@@ -11,12 +11,12 @@ go_library(
importpath = "github.com/google/cadvisor/container/common",
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/fsnotify/fsnotify:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/google/cadvisor/container:go_default_library",
"//vendor/github.com/google/cadvisor/fs:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
"//vendor/github.com/google/cadvisor/utils:go_default_library",
"//vendor/golang.org/x/exp/inotify:go_default_library",
],
)

View File

@@ -17,15 +17,15 @@ package common
import (
"sync"
"github.com/fsnotify/fsnotify"
"golang.org/x/exp/inotify"
)
// Watcher for container-related fsnotify events in the cgroup hierarchy.
// Watcher for container-related inotify events in the cgroup hierarchy.
//
// Implementation is thread-safe.
type InotifyWatcher struct {
// Underlying fsnotify watcher.
watcher *fsnotify.Watcher
// Underlying inotify watcher.
watcher *inotify.Watcher
// Map of containers being watched to cgroup paths watched for that container.
containersWatched map[string]map[string]bool
@@ -35,7 +35,7 @@ type InotifyWatcher struct {
}
func NewInotifyWatcher() (*InotifyWatcher, error) {
w, err := fsnotify.NewWatcher()
w, err := inotify.NewWatcher()
if err != nil {
return nil, err
}
@@ -53,9 +53,9 @@ func (iw *InotifyWatcher) AddWatch(containerName, dir string) (bool, error) {
cgroupsWatched, alreadyWatched := iw.containersWatched[containerName]
// Register an fsnotify notification.
// Register an inotify notification.
if !cgroupsWatched[dir] {
err := iw.watcher.Add(dir)
err := iw.watcher.AddWatch(dir, inotify.IN_CREATE|inotify.IN_DELETE|inotify.IN_MOVE)
if err != nil {
return alreadyWatched, err
}
@@ -84,9 +84,9 @@ func (iw *InotifyWatcher) RemoveWatch(containerName, dir string) (bool, error) {
return false, nil
}
// Remove the fsnotify watch if it exists.
// Remove the inotify watch if it exists.
if cgroupsWatched[dir] {
err := iw.watcher.Remove(dir)
err := iw.watcher.RemoveWatch(dir)
if err != nil {
return false, nil
}
@@ -104,15 +104,15 @@ func (iw *InotifyWatcher) RemoveWatch(containerName, dir string) (bool, error) {
// Errors are returned on this channel.
func (iw *InotifyWatcher) Error() chan error {
return iw.watcher.Errors
return iw.watcher.Error
}
// Events are returned on this channel.
func (iw *InotifyWatcher) Event() chan fsnotify.Event {
return iw.watcher.Events
func (iw *InotifyWatcher) Event() chan *inotify.Event {
return iw.watcher.Event
}
// Closes the fsnotify watcher.
// Closes the inotify watcher.
func (iw *InotifyWatcher) Close() error {
return iw.watcher.Close()
}

View File

@@ -35,6 +35,7 @@ const (
ContainerTypeRkt
ContainerTypeSystemd
ContainerTypeCrio
ContainerTypeContainerd
)
// Interface for container operation handlers.

View File

@@ -0,0 +1,50 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"client.go",
"factory.go",
"grpc.go",
"handler.go",
],
importpath = "github.com/google/cadvisor/container/containerd",
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/containerd/containerd/api/services/containers/v1:go_default_library",
"//vendor/github.com/containerd/containerd/api/services/tasks/v1:go_default_library",
"//vendor/github.com/containerd/containerd/api/services/version/v1:go_default_library",
"//vendor/github.com/containerd/containerd/containers:go_default_library",
"//vendor/github.com/containerd/containerd/dialer:go_default_library",
"//vendor/github.com/containerd/containerd/errdefs:go_default_library",
"//vendor/github.com/containerd/containerd/namespaces:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/golang/protobuf/ptypes/empty:go_default_library",
"//vendor/github.com/google/cadvisor/container:go_default_library",
"//vendor/github.com/google/cadvisor/container/common:go_default_library",
"//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library",
"//vendor/github.com/google/cadvisor/fs:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
"//vendor/github.com/google/cadvisor/manager/watcher:go_default_library",
"//vendor/github.com/opencontainers/runc/libcontainer/cgroups:go_default_library",
"//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library",
"//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library",
"//vendor/github.com/opencontainers/runtime-spec/specs-go:go_default_library",
"//vendor/golang.org/x/net/context:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,122 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package containerd
import (
"context"
"time"
containersapi "github.com/containerd/containerd/api/services/containers/v1"
tasksapi "github.com/containerd/containerd/api/services/tasks/v1"
versionapi "github.com/containerd/containerd/api/services/version/v1"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/dialer"
"github.com/containerd/containerd/errdefs"
pempty "github.com/golang/protobuf/ptypes/empty"
"google.golang.org/grpc"
)
const (
// k8sNamespace is the namespace we use to connect containerd.
k8sNamespace = "k8s.io"
)
type client struct {
containerService containersapi.ContainersClient
taskService tasksapi.TasksClient
versionService versionapi.VersionClient
}
type containerdClient interface {
LoadContainer(ctx context.Context, id string) (*containers.Container, error)
TaskPid(ctx context.Context, id string) (uint32, error)
Version(ctx context.Context) (string, error)
}
// Client creates a containerd client
func Client() (containerdClient, error) {
gopts := []grpc.DialOption{
grpc.WithInsecure(),
grpc.FailOnNonTempDialError(true),
grpc.WithDialer(dialer.Dialer),
grpc.WithBlock(),
grpc.WithTimeout(2 * time.Second),
grpc.WithBackoffMaxDelay(3 * time.Second),
}
unary, stream := newNSInterceptors(k8sNamespace)
gopts = append(gopts,
grpc.WithUnaryInterceptor(unary),
grpc.WithStreamInterceptor(stream),
)
conn, err := grpc.Dial(dialer.DialAddress("/var/run/containerd/containerd.sock"), gopts...)
if err != nil {
return nil, err
}
c := &client{
containerService: containersapi.NewContainersClient(conn),
taskService: tasksapi.NewTasksClient(conn),
versionService: versionapi.NewVersionClient(conn),
}
return c, err
}
func (c *client) LoadContainer(ctx context.Context, id string) (*containers.Container, error) {
r, err := c.containerService.Get(ctx, &containersapi.GetContainerRequest{
ID: id,
})
if err != nil {
return nil, errdefs.FromGRPC(err)
}
return containerFromProto(r.Container), nil
}
func (c *client) TaskPid(ctx context.Context, id string) (uint32, error) {
response, err := c.taskService.Get(ctx, &tasksapi.GetRequest{
ContainerID: id,
})
if err != nil {
return 0, errdefs.FromGRPC(err)
}
return response.Process.Pid, nil
}
func (c *client) Version(ctx context.Context) (string, error) {
response, err := c.versionService.Version(ctx, &pempty.Empty{})
if err != nil {
return "", errdefs.FromGRPC(err)
}
return response.Version, nil
}
func containerFromProto(containerpb containersapi.Container) *containers.Container {
var runtime containers.RuntimeInfo
if containerpb.Runtime != nil {
runtime = containers.RuntimeInfo{
Name: containerpb.Runtime.Name,
Options: containerpb.Runtime.Options,
}
}
return &containers.Container{
ID: containerpb.ID,
Labels: containerpb.Labels,
Image: containerpb.Image,
Runtime: runtime,
Spec: containerpb.Spec,
Snapshotter: containerpb.Snapshotter,
SnapshotKey: containerpb.SnapshotKey,
Extensions: containerpb.Extensions,
}
}

View File

@@ -0,0 +1,148 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package containerd
import (
"flag"
"fmt"
"path"
"regexp"
"strings"
"github.com/golang/glog"
"golang.org/x/net/context"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
"github.com/google/cadvisor/manager/watcher"
)
var ArgContainerdEndpoint = flag.String("containerd", "unix:///var/run/containerd.sock", "containerd endpoint")
// The namespace under which containerd aliases are unique.
const k8sContainerdNamespace = "containerd"
// Regexp that identifies containerd cgroups, containers started with
// --cgroup-parent have another prefix than 'containerd'
var containerdCgroupRegexp = regexp.MustCompile(`([a-z0-9]{64})`)
type containerdFactory struct {
machineInfoFactory info.MachineInfoFactory
client containerdClient
version string
// Information about the mounted cgroup subsystems.
cgroupSubsystems libcontainer.CgroupSubsystems
// Information about mounted filesystems.
fsInfo fs.FsInfo
ignoreMetrics container.MetricSet
}
func (self *containerdFactory) String() string {
return k8sContainerdNamespace
}
func (self *containerdFactory) NewContainerHandler(name string, inHostNamespace bool) (handler container.ContainerHandler, err error) {
client, err := Client()
if err != nil {
return
}
metadataEnvs := []string{}
return newContainerdContainerHandler(
client,
name,
self.machineInfoFactory,
self.fsInfo,
&self.cgroupSubsystems,
inHostNamespace,
metadataEnvs,
self.ignoreMetrics,
)
}
// Returns the containerd ID from the full container name.
func ContainerNameToContainerdID(name string) string {
id := path.Base(name)
if matches := containerdCgroupRegexp.FindStringSubmatch(id); matches != nil {
return matches[1]
}
return id
}
// isContainerName returns true if the cgroup with associated name
// corresponds to a containerd container.
func isContainerName(name string) bool {
// TODO: May be check with HasPrefix ContainerdNamespace
if strings.HasSuffix(name, ".mount") {
return false
}
return containerdCgroupRegexp.MatchString(path.Base(name))
}
// Containerd can handle and accept all containerd created containers
func (self *containerdFactory) CanHandleAndAccept(name string) (bool, bool, error) {
// if the container is not associated with containerd, we can't handle it or accept it.
if !isContainerName(name) {
return false, false, nil
}
// Check if the container is known to containerd and it is running.
id := ContainerNameToContainerdID(name)
// If container and task lookup in containerd fails then we assume
// that the container state is not known to containerd
ctx := context.Background()
_, err := self.client.LoadContainer(ctx, id)
if err != nil {
return false, false, fmt.Errorf("failed to load container: %v", err)
}
return true, true, nil
}
func (self *containerdFactory) DebugInfo() map[string][]string {
return map[string][]string{}
}
// Register root container before running this function!
func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, ignoreMetrics container.MetricSet) error {
client, err := Client()
if err != nil {
return fmt.Errorf("unable to create containerd client: %v", err)
}
containerdVersion, err := client.Version(context.Background())
if err != nil {
return fmt.Errorf("failed to fetch containerd client version: %v", err)
}
cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
if err != nil {
return fmt.Errorf("failed to get cgroup subsystems: %v", err)
}
glog.Infof("Registering containerd factory")
f := &containerdFactory{
cgroupSubsystems: cgroupSubsystems,
client: client,
fsInfo: fsInfo,
machineInfoFactory: factory,
version: containerdVersion,
ignoreMetrics: ignoreMetrics,
}
container.RegisterContainerHandlerFactory(f, []watcher.ContainerWatchSource{watcher.Raw})
return nil
}

View File

@@ -0,0 +1,49 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//This code has been taken from containerd repo to avoid large library import
package containerd
import (
"github.com/containerd/containerd/namespaces"
"golang.org/x/net/context"
"google.golang.org/grpc"
)
type namespaceInterceptor struct {
namespace string
}
func (ni namespaceInterceptor) unary(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
_, ok := namespaces.Namespace(ctx)
if !ok {
ctx = namespaces.WithNamespace(ctx, ni.namespace)
}
return invoker(ctx, method, req, reply, cc, opts...)
}
func (ni namespaceInterceptor) stream(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
_, ok := namespaces.Namespace(ctx)
if !ok {
ctx = namespaces.WithNamespace(ctx, ni.namespace)
}
return streamer(ctx, desc, cc, method, opts...)
}
func newNSInterceptors(ns string) (grpc.UnaryClientInterceptor, grpc.StreamClientInterceptor) {
ni := namespaceInterceptor{
namespace: ns,
}
return grpc.UnaryClientInterceptor(ni.unary), grpc.StreamClientInterceptor(ni.stream)
}

View File

@@ -0,0 +1,246 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Handler for containerd containers.
package containerd
import (
"encoding/json"
"fmt"
"path"
"strings"
"time"
"github.com/opencontainers/runc/libcontainer/cgroups"
cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs"
"golang.org/x/net/context"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/common"
containerlibcontainer "github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
specs "github.com/opencontainers/runtime-spec/specs-go"
)
type containerdContainerHandler struct {
client containerdClient
name string
id string
aliases []string
machineInfoFactory info.MachineInfoFactory
// Absolute path to the cgroup hierarchies of this container.
// (e.g.: "cpu" -> "/sys/fs/cgroup/cpu/test")
cgroupPaths map[string]string
// Manager of this container's cgroups.
cgroupManager cgroups.Manager
fsInfo fs.FsInfo
poolName string
// Time at which this container was created.
creationTime time.Time
// Metadata associated with the container.
labels map[string]string
envs map[string]string
// The container PID used to switch namespaces as required
pid int
// Image name used for this container.
image string
// The host root FS to read
rootFs string
// Filesystem handler.
ignoreMetrics container.MetricSet
}
var _ container.ContainerHandler = &containerdContainerHandler{}
// newContainerdContainerHandler returns a new container.ContainerHandler
func newContainerdContainerHandler(
client containerdClient,
name string,
machineInfoFactory info.MachineInfoFactory,
fsInfo fs.FsInfo,
cgroupSubsystems *containerlibcontainer.CgroupSubsystems,
inHostNamespace bool,
metadataEnvs []string,
ignoreMetrics container.MetricSet,
) (container.ContainerHandler, error) {
// Create the cgroup paths.
cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints))
for key, val := range cgroupSubsystems.MountPoints {
cgroupPaths[key] = path.Join(val, name)
}
// Generate the equivalent cgroup manager for this container.
cgroupManager := &cgroupfs.Manager{
Cgroups: &libcontainerconfigs.Cgroup{
Name: name,
},
Paths: cgroupPaths,
}
id := ContainerNameToContainerdID(name)
// We assume that if load fails then the container is not known to containerd.
ctx := context.Background()
cntr, err := client.LoadContainer(ctx, id)
if err != nil {
return nil, err
}
var spec specs.Spec
if err := json.Unmarshal(cntr.Spec.Value, &spec); err != nil {
return nil, err
}
taskPid, err := client.TaskPid(ctx, id)
if err != nil {
return nil, err
}
rootfs := "/"
if !inHostNamespace {
rootfs = "/rootfs"
}
handler := &containerdContainerHandler{
id: id,
client: client,
name: name,
machineInfoFactory: machineInfoFactory,
cgroupPaths: cgroupPaths,
cgroupManager: cgroupManager,
rootFs: rootfs,
fsInfo: fsInfo,
envs: make(map[string]string),
labels: make(map[string]string),
ignoreMetrics: ignoreMetrics,
pid: int(taskPid),
creationTime: cntr.CreatedAt,
}
// Add the name and bare ID as aliases of the container.
handler.labels = cntr.Labels
handler.image = cntr.Image
handler.aliases = []string{id, name}
for _, envVar := range spec.Process.Env {
if envVar != "" {
splits := strings.SplitN(envVar, "=", 2)
if len(splits) == 2 {
handler.envs[splits[0]] = splits[1]
}
}
}
return handler, nil
}
func (self *containerdContainerHandler) ContainerReference() (info.ContainerReference, error) {
return info.ContainerReference{
Id: self.id,
Name: self.name,
Namespace: k8sContainerdNamespace,
Labels: self.labels,
Aliases: self.aliases,
}, nil
}
func (self *containerdContainerHandler) needNet() bool {
// Since containerd does not handle networking ideally we need to return based
// on ignoreMetrics list. Here the assumption is the presence of cri-containerd
// label
if !self.ignoreMetrics.Has(container.NetworkUsageMetrics) {
//TODO change it to exported cri-containerd constants
return self.labels["io.cri-containerd.kind"] == "sandbox"
}
return false
}
func (self *containerdContainerHandler) GetSpec() (info.ContainerSpec, error) {
// TODO: Since we dont collect disk usage stats for containerd, we set hasFilesystem
// to false. Revisit when we support disk usage stats for containerd
hasFilesystem := false
spec, err := common.GetSpec(self.cgroupPaths, self.machineInfoFactory, self.needNet(), hasFilesystem)
spec.Labels = self.labels
spec.Envs = self.envs
spec.Image = self.image
return spec, err
}
func (self *containerdContainerHandler) getFsStats(stats *info.ContainerStats) error {
mi, err := self.machineInfoFactory.GetMachineInfo()
if err != nil {
return err
}
if !self.ignoreMetrics.Has(container.DiskIOMetrics) {
common.AssignDeviceNamesToDiskStats((*common.MachineInfoNamer)(mi), &stats.DiskIo)
}
return nil
}
func (self *containerdContainerHandler) GetStats() (*info.ContainerStats, error) {
stats, err := containerlibcontainer.GetStats(self.cgroupManager, self.rootFs, self.pid, self.ignoreMetrics)
if err != nil {
return stats, err
}
// Clean up stats for containers that don't have their own network - this
// includes containers running in Kubernetes pods that use the network of the
// infrastructure container. This stops metrics being reported multiple times
// for each container in a pod.
if !self.needNet() {
stats.Network = info.NetworkStats{}
}
// Get filesystem stats.
err = self.getFsStats(stats)
return stats, err
}
func (self *containerdContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) {
return []info.ContainerReference{}, nil
}
func (self *containerdContainerHandler) GetCgroupPath(resource string) (string, error) {
path, ok := self.cgroupPaths[resource]
if !ok {
return "", fmt.Errorf("could not find path for resource %q for container %q\n", resource, self.name)
}
return path, nil
}
func (self *containerdContainerHandler) GetContainerLabels() map[string]string {
return self.labels
}
func (self *containerdContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
return containerlibcontainer.GetProcesses(self.cgroupManager)
}
func (self *containerdContainerHandler) Exists() bool {
return common.CgroupExists(self.cgroupPaths)
}
func (self *containerdContainerHandler) Type() container.ContainerType {
return container.ContainerTypeContainerd
}
func (self *containerdContainerHandler) Start() {
}
func (self *containerdContainerHandler) Cleanup() {
}
func (self *containerdContainerHandler) GetContainerIPAddress() string {
// containerd doesnt take care of networking.So it doesnt maintain networking states
return ""
}

View File

@@ -24,7 +24,7 @@ import (
)
const (
CrioSocket = "/var/run/crio.sock"
CrioSocket = "/var/run/crio/crio.sock"
maxUnixSocketPathSize = len(syscall.RawSockaddrUnix{}.Path)
)

View File

@@ -55,19 +55,36 @@ func GetCgroupSubsystems() (CgroupSubsystems, error) {
if err != nil {
return CgroupSubsystems{}, err
}
return getCgroupSubsystemsHelper(allCgroups)
}
func getCgroupSubsystemsHelper(allCgroups []cgroups.Mount) (CgroupSubsystems, error) {
if len(allCgroups) == 0 {
return CgroupSubsystems{}, fmt.Errorf("failed to find cgroup mounts")
}
// Trim the mounts to only the subsystems we care about.
supportedCgroups := make([]cgroups.Mount, 0, len(allCgroups))
recordedMountpoints := make(map[string]struct{}, len(allCgroups))
mountPoints := make(map[string]string, len(allCgroups))
for _, mount := range allCgroups {
for _, subsystem := range mount.Subsystems {
if _, ok := supportedSubsystems[subsystem]; ok {
supportedCgroups = append(supportedCgroups, mount)
mountPoints[subsystem] = mount.Mountpoint
if _, ok := supportedSubsystems[subsystem]; !ok {
// Unsupported subsystem
continue
}
if _, ok := mountPoints[subsystem]; ok {
// duplicate mount for this subsystem; use the first one we saw
glog.V(5).Infof("skipping %s, already using mount at %s", mount.Mountpoint, mountPoints[subsystem])
continue
}
if _, ok := recordedMountpoints[mount.Mountpoint]; !ok {
// avoid appending the same mount twice in e.g. `cpu,cpuacct` case
supportedCgroups = append(supportedCgroups, mount)
recordedMountpoints[mount.Mountpoint] = struct{}{}
}
mountPoints[subsystem] = mount.Mountpoint
}
}

View File

@@ -307,7 +307,7 @@ type CpuStats struct {
}
type PerDiskStats struct {
Device string `json:"-"`
Device string `json:"device"`
Major uint64 `json:"major"`
Minor uint64 `json:"minor"`
Stats map[string]uint64 `json:"stats"`

View File

@@ -236,6 +236,9 @@ type RequestOptions struct {
Count int `json:"count"`
// Whether to include stats for child subcontainers.
Recursive bool `json:"recursive"`
// Update stats if they are older than MaxAge
// nil indicates no update, and 0 will always trigger an update.
MaxAge *time.Duration `json:"max_age"`
}
type ProcessInfo struct {

View File

@@ -206,9 +206,6 @@ func InstCpuStats(last, cur *v1.ContainerStats) (*CpuInstStats, error) {
return nil, fmt.Errorf("different number of cpus")
}
timeDelta := cur.Timestamp.Sub(last.Timestamp)
if timeDelta <= 100*time.Millisecond {
return nil, fmt.Errorf("time delta unexpectedly small")
}
// Nanoseconds to gain precision and avoid having zero seconds if the
// difference between the timestamps is just under a second
timeDeltaNs := uint64(timeDelta.Nanoseconds())

View File

@@ -15,6 +15,7 @@ go_library(
"//vendor/github.com/google/cadvisor/cache/memory:go_default_library",
"//vendor/github.com/google/cadvisor/collector:go_default_library",
"//vendor/github.com/google/cadvisor/container:go_default_library",
"//vendor/github.com/google/cadvisor/container/containerd:go_default_library",
"//vendor/github.com/google/cadvisor/container/crio:go_default_library",
"//vendor/github.com/google/cadvisor/container/docker:go_default_library",
"//vendor/github.com/google/cadvisor/container/raw:go_default_library",
@@ -34,6 +35,7 @@ go_library(
"//vendor/github.com/google/cadvisor/utils/sysfs:go_default_library",
"//vendor/github.com/google/cadvisor/version:go_default_library",
"//vendor/github.com/opencontainers/runc/libcontainer/cgroups:go_default_library",
"//vendor/k8s.io/utils/clock:go_default_library",
],
)

View File

@@ -40,6 +40,7 @@ import (
units "github.com/docker/go-units"
"github.com/golang/glog"
"k8s.io/utils/clock"
)
// Housekeeping interval.
@@ -65,8 +66,11 @@ type containerData struct {
housekeepingInterval time.Duration
maxHousekeepingInterval time.Duration
allowDynamicHousekeeping bool
lastUpdatedTime time.Time
infoLastUpdatedTime time.Time
statsLastUpdatedTime time.Time
lastErrorTime time.Time
// used to track time
clock clock.Clock
// Decay value used for load average smoothing. Interval length of 10 seconds is used.
loadDecay float64
@@ -77,6 +81,9 @@ type containerData struct {
// Tells the container to stop.
stop chan bool
// Tells the container to immediately collect stats
onDemandChan chan chan struct{}
// Runs custom metric collectors.
collectorManager collector.CollectorManager
@@ -110,16 +117,43 @@ func (c *containerData) Stop() error {
}
func (c *containerData) allowErrorLogging() bool {
if time.Since(c.lastErrorTime) > time.Minute {
c.lastErrorTime = time.Now()
if c.clock.Since(c.lastErrorTime) > time.Minute {
c.lastErrorTime = c.clock.Now()
return true
}
return false
}
// OnDemandHousekeeping performs housekeeping on the container and blocks until it has completed.
// It is designed to be used in conjunction with periodic housekeeping, and will cause the timer for
// periodic housekeeping to reset. This should be used sparingly, as calling OnDemandHousekeeping frequently
// can have serious performance costs.
func (c *containerData) OnDemandHousekeeping(maxAge time.Duration) {
if c.clock.Since(c.statsLastUpdatedTime) > maxAge {
housekeepingFinishedChan := make(chan struct{})
c.onDemandChan <- housekeepingFinishedChan
select {
case <-c.stop:
case <-housekeepingFinishedChan:
}
}
}
// notifyOnDemand notifies all calls to OnDemandHousekeeping that housekeeping is finished
func (c *containerData) notifyOnDemand() {
for {
select {
case finishedChan := <-c.onDemandChan:
close(finishedChan)
default:
return
}
}
}
func (c *containerData) GetInfo(shouldUpdateSubcontainers bool) (*containerInfo, error) {
// Get spec and subcontainers.
if time.Since(c.lastUpdatedTime) > 5*time.Second {
if c.clock.Since(c.infoLastUpdatedTime) > 5*time.Second {
err := c.updateSpec()
if err != nil {
return nil, err
@@ -130,7 +164,7 @@ func (c *containerData) GetInfo(shouldUpdateSubcontainers bool) (*containerInfo,
return nil, err
}
}
c.lastUpdatedTime = time.Now()
c.infoLastUpdatedTime = c.clock.Now()
}
// Make a copy of the info for the user.
c.lock.Lock()
@@ -310,7 +344,7 @@ func (c *containerData) GetProcessList(cadvisorContainer string, inHostNamespace
return processes, nil
}
func newContainerData(containerName string, memoryCache *memory.InMemoryCache, handler container.ContainerHandler, logUsage bool, collectorManager collector.CollectorManager, maxHousekeepingInterval time.Duration, allowDynamicHousekeeping bool) (*containerData, error) {
func newContainerData(containerName string, memoryCache *memory.InMemoryCache, handler container.ContainerHandler, logUsage bool, collectorManager collector.CollectorManager, maxHousekeepingInterval time.Duration, allowDynamicHousekeeping bool, clock clock.Clock) (*containerData, error) {
if memoryCache == nil {
return nil, fmt.Errorf("nil memory storage")
}
@@ -332,6 +366,8 @@ func newContainerData(containerName string, memoryCache *memory.InMemoryCache, h
loadAvg: -1.0, // negative value indicates uninitialized.
stop: make(chan bool, 1),
collectorManager: collectorManager,
onDemandChan: make(chan chan struct{}, 100),
clock: clock,
}
cont.info.ContainerReference = ref
@@ -362,7 +398,7 @@ func newContainerData(containerName string, memoryCache *memory.InMemoryCache, h
}
// Determine when the next housekeeping should occur.
func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Time {
func (self *containerData) nextHousekeepingInterval() time.Duration {
if self.allowDynamicHousekeeping {
var empty time.Time
stats, err := self.memoryCache.RecentStats(self.info.Name, empty, empty, 2)
@@ -385,7 +421,7 @@ func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Tim
}
}
return lastHousekeeping.Add(jitter(self.housekeepingInterval, 1.0))
return jitter(self.housekeepingInterval, 1.0)
}
// TODO(vmarmol): Implement stats collecting as a custom collector.
@@ -411,24 +447,19 @@ func (c *containerData) housekeeping() {
// Housekeep every second.
glog.V(3).Infof("Start housekeeping for container %q\n", c.info.Name)
lastHousekeeping := time.Now()
houseKeepingTimer := c.clock.NewTimer(0 * time.Second)
defer houseKeepingTimer.Stop()
for {
select {
case <-c.stop:
// Stop housekeeping when signaled.
if !c.housekeepingTick(houseKeepingTimer.C(), longHousekeeping) {
return
default:
// Perform housekeeping.
start := time.Now()
c.housekeepingTick()
// Log if housekeeping took too long.
duration := time.Since(start)
if duration >= longHousekeeping {
glog.V(3).Infof("[%s] Housekeeping took %s", c.info.Name, duration)
}
// Stop and drain the timer so that it is safe to reset it
if !houseKeepingTimer.Stop() {
select {
case <-houseKeepingTimer.C():
default:
}
}
// Log usage if asked to do so.
if c.logUsage {
const numSamples = 60
@@ -455,26 +486,35 @@ func (c *containerData) housekeeping() {
glog.Infof("[%s] %.3f cores (average: %.3f cores), %s of memory", c.info.Name, instantUsageInCores, usageInCores, usageInHuman)
}
}
next := c.nextHousekeeping(lastHousekeeping)
// Schedule the next housekeeping. Sleep until that time.
if time.Now().Before(next) {
time.Sleep(next.Sub(time.Now()))
} else {
next = time.Now()
}
lastHousekeeping = next
houseKeepingTimer.Reset(c.nextHousekeepingInterval())
}
}
func (c *containerData) housekeepingTick() {
func (c *containerData) housekeepingTick(timer <-chan time.Time, longHousekeeping time.Duration) bool {
select {
case <-c.stop:
// Stop housekeeping when signaled.
return false
case finishedChan := <-c.onDemandChan:
// notify the calling function once housekeeping has completed
defer close(finishedChan)
case <-timer:
}
start := c.clock.Now()
err := c.updateStats()
if err != nil {
if c.allowErrorLogging() {
glog.Infof("Failed to update stats for container \"%s\": %s", c.info.Name, err)
}
}
// Log if housekeeping took too long.
duration := c.clock.Since(start)
if duration >= longHousekeeping {
glog.V(3).Infof("[%s] Housekeeping took %s", c.info.Name, duration)
}
c.notifyOnDemand()
c.statsLastUpdatedTime = c.clock.Now()
return true
}
func (c *containerData) updateSpec() error {
@@ -550,7 +590,7 @@ func (c *containerData) updateStats() error {
var customStatsErr error
cm := c.collectorManager.(*collector.GenericCollectorManager)
if len(cm.Collectors) > 0 {
if cm.NextCollectionTime.Before(time.Now()) {
if cm.NextCollectionTime.Before(c.clock.Now()) {
customStats, err := c.updateCustomStats()
if customStats != nil {
stats.CustomMetrics = customStats

View File

@@ -30,6 +30,7 @@ import (
"github.com/google/cadvisor/cache/memory"
"github.com/google/cadvisor/collector"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/containerd"
"github.com/google/cadvisor/container/crio"
"github.com/google/cadvisor/container/docker"
"github.com/google/cadvisor/container/raw"
@@ -49,6 +50,7 @@ import (
"github.com/golang/glog"
"github.com/opencontainers/runc/libcontainer/cgroups"
"k8s.io/utils/clock"
)
var globalHousekeepingInterval = flag.Duration("global_housekeeping_interval", 1*time.Minute, "Interval between global housekeepings")
@@ -279,6 +281,11 @@ func (self *manager) Start() error {
self.containerWatchers = append(self.containerWatchers, watcher)
}
err = containerd.Register(self, self.fsInfo, self.ignoreMetrics)
if err != nil {
glog.Warningf("Registration of the containerd container factory failed: %v", err)
}
err = crio.Register(self, self.fsInfo, self.ignoreMetrics)
if err != nil {
glog.Warningf("Registration of the crio container factory failed: %v", err)
@@ -707,6 +714,18 @@ func (self *manager) getRequestedContainers(containerName string, options v2.Req
default:
return containersMap, fmt.Errorf("invalid request type %q", options.IdType)
}
if options.MaxAge != nil {
// update stats for all containers in containersMap
var waitGroup sync.WaitGroup
waitGroup.Add(len(containersMap))
for _, container := range containersMap {
go func(cont *containerData) {
cont.OnDemandHousekeeping(*options.MaxAge)
waitGroup.Done()
}(container)
}
waitGroup.Wait()
}
return containersMap, nil
}
@@ -804,6 +823,8 @@ func (m *manager) Exists(containerName string) bool {
func (m *manager) GetProcessList(containerName string, options v2.RequestOptions) ([]v2.ProcessInfo, error) {
// override recursive. Only support single container listing.
options.Recursive = false
// override MaxAge. ProcessList does not require updated stats.
options.MaxAge = nil
conts, err := m.getRequestedContainers(containerName, options)
if err != nil {
return nil, err
@@ -919,7 +940,7 @@ func (m *manager) createContainerLocked(containerName string, watchSource watche
}
logUsage := *logCadvisorUsage && containerName == m.cadvisorContainer
cont, err := newContainerData(containerName, m.memoryCache, handler, logUsage, collectorManager, m.maxHousekeepingInterval, m.allowDynamicHousekeeping)
cont, err := newContainerData(containerName, m.memoryCache, handler, logUsage, collectorManager, m.maxHousekeepingInterval, m.allowDynamicHousekeeping, clock.RealClock{})
if err != nil {
return err
}

View File

@@ -6,11 +6,11 @@ go_library(
importpath = "github.com/google/cadvisor/manager/watcher/raw",
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/fsnotify/fsnotify:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/google/cadvisor/container/common:go_default_library",
"//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library",
"//vendor/github.com/google/cadvisor/manager/watcher:go_default_library",
"//vendor/golang.org/x/exp/inotify:go_default_library",
],
)

View File

@@ -27,8 +27,8 @@ import (
"github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/manager/watcher"
"github.com/fsnotify/fsnotify"
"github.com/golang/glog"
"golang.org/x/exp/inotify"
)
type rawContainerWatcher struct {
@@ -121,7 +121,7 @@ func (self *rawContainerWatcher) watchDirectory(dir string, containerName string
if cleanup {
_, err := self.watcher.RemoveWatch(containerName, dir)
if err != nil {
glog.Warningf("Failed to remove fsnotify watch for %q: %v", dir, err)
glog.Warningf("Failed to remove inotify watch for %q: %v", dir, err)
}
}
}()
@@ -152,16 +152,18 @@ func (self *rawContainerWatcher) watchDirectory(dir string, containerName string
return alreadyWatching, nil
}
func (self *rawContainerWatcher) processEvent(event fsnotify.Event, events chan watcher.ContainerEvent) error {
// Convert the fsnotify event type to a container create or delete.
func (self *rawContainerWatcher) processEvent(event *inotify.Event, events chan watcher.ContainerEvent) error {
// Convert the inotify event type to a container create or delete.
var eventType watcher.ContainerEventType
switch {
case event.Op == fsnotify.Create:
case (event.Mask & inotify.IN_CREATE) > 0:
eventType = watcher.ContainerAdd
case event.Op == fsnotify.Remove:
case (event.Mask & inotify.IN_DELETE) > 0:
eventType = watcher.ContainerDelete
case event.Op == fsnotify.Rename:
case (event.Mask & inotify.IN_MOVED_FROM) > 0:
eventType = watcher.ContainerDelete
case (event.Mask & inotify.IN_MOVED_TO) > 0:
eventType = watcher.ContainerAdd
default:
// Ignore other events.
return nil

File diff suppressed because one or more lines are too long

View File

@@ -28,25 +28,27 @@ import (
const StaticResource = "/static/"
var bootstrapJs, _ = Asset("pages/assets/js/bootstrap-3.1.1.min.js")
var popper, _ = Asset("pages/assets/js/popper.min.js")
var bootstrapJs, _ = Asset("pages/assets/js/bootstrap-4.0.0-beta.2.min.js")
var containersJs, _ = Asset("pages/assets/js/containers.js")
var gchartsJs, _ = Asset("pages/assets/js/gcharts.js")
var googleJsapiJs, _ = Asset("pages/assets/js/google-jsapi.js")
var jqueryJs, _ = Asset("pages/assets/js/jquery-1.10.2.min.js")
var jqueryJs, _ = Asset("pages/assets/js/jquery-3.0.0.min.js")
var bootstrapCss, _ = Asset("pages/assets/styles/bootstrap-3.1.1.min.css")
var bootstrapCss, _ = Asset("pages/assets/styles/bootstrap-4.0.0-beta.2.min.css")
var bootstrapThemeCss, _ = Asset("pages/assets/styles/bootstrap-theme-3.1.1.min.css")
var containersCss, _ = Asset("pages/assets/styles/containers.css")
var staticFiles = map[string][]byte{
"bootstrap-3.1.1.min.css": bootstrapCss,
"bootstrap-3.1.1.min.js": bootstrapJs,
"bootstrap-theme-3.1.1.min.css": bootstrapThemeCss,
"containers.css": containersCss,
"containers.js": containersJs,
"gcharts.js": gchartsJs,
"google-jsapi.js": googleJsapiJs,
"jquery-1.10.2.min.js": jqueryJs,
"popper.min.js": popper,
"bootstrap-4.0.0-beta.2.min.css": bootstrapCss,
"bootstrap-4.0.0-beta.2.min.js": bootstrapJs,
"bootstrap-theme-3.1.1.min.css": bootstrapThemeCss,
"containers.css": containersCss,
"containers.js": containersJs,
"gcharts.js": gchartsJs,
"google-jsapi.js": googleJsapiJs,
"jquery-3.0.0.min.js": jqueryJs,
}
func HandleRequest(w http.ResponseWriter, u *url.URL) {

File diff suppressed because one or more lines are too long