Bump cAdvisor (and dependencies) godeps version

vendor/github.com/google/cadvisor/manager/container.go

@@ -42,6 +42,7 @@ import (
 )
 
 // Housekeeping interval.
+var enableLoadReader = flag.Bool("enable_load_reader", false, "Whether to enable cpu load reader")
 var HousekeepingInterval = flag.Duration("housekeeping_interval", 1*time.Second, "Interval between container housekeepings")
 
 var cgroupPathRegExp = regexp.MustCompile(`devices[^:]*:(.*?)[,;$]`)
@@ -303,7 +304,7 @@ func (c *containerData) GetProcessList(cadvisorContainer string, inHostNamespace
 	return processes, nil
 }
 
-func newContainerData(containerName string, memoryCache *memory.InMemoryCache, handler container.ContainerHandler, loadReader cpuload.CpuLoadReader, logUsage bool, collectorManager collector.CollectorManager, maxHousekeepingInterval time.Duration, allowDynamicHousekeeping bool) (*containerData, error) {
+func newContainerData(containerName string, memoryCache *memory.InMemoryCache, handler container.ContainerHandler, logUsage bool, collectorManager collector.CollectorManager, maxHousekeepingInterval time.Duration, allowDynamicHousekeeping bool) (*containerData, error) {
 	if memoryCache == nil {
 		return nil, fmt.Errorf("nil memory storage")
 	}
@@ -321,7 +322,6 @@ func newContainerData(containerName string, memoryCache *memory.InMemoryCache, h
 		housekeepingInterval:     *HousekeepingInterval,
 		maxHousekeepingInterval:  maxHousekeepingInterval,
 		allowDynamicHousekeeping: allowDynamicHousekeeping,
-		loadReader:               loadReader,
 		logUsage:                 logUsage,
 		loadAvg:                  -1.0, // negative value indicates uninitialized.
 		stop:                     make(chan bool, 1),
@@ -331,6 +331,17 @@ func newContainerData(containerName string, memoryCache *memory.InMemoryCache, h
 
 	cont.loadDecay = math.Exp(float64(-cont.housekeepingInterval.Seconds() / 10))
 
+	if *enableLoadReader {
+		// Create cpu load reader.
+		loadReader, err := cpuload.New()
+		if err != nil {
+			// TODO(rjnagal): Promote to warning once we support cpu load inside namespaces.
+			glog.Infof("Could not initialize cpu load reader for %q: %s", ref.Name, err)
+		} else {
+			cont.loadReader = loadReader
+		}
+	}
+
 	err = cont.updateSpec()
 	if err != nil {
 		return nil, err
@@ -375,6 +386,16 @@ func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Tim
 func (c *containerData) housekeeping() {
 	// Start any background goroutines - must be cleaned up in c.handler.Cleanup().
 	c.handler.Start()
+	defer c.handler.Cleanup()
+
+	// Initialize cpuload reader - must be cleaned up in c.loadReader.Stop()
+	if c.loadReader != nil {
+		err := c.loadReader.Start()
+		if err != nil {
+			glog.Warningf("Could not start cpu load stat collector for %q: %s", c.info.Name, err)
+		}
+		defer c.loadReader.Stop()
+	}
 
 	// Long housekeeping is either 100ms or half of the housekeeping interval.
 	longHousekeeping := 100 * time.Millisecond
@@ -388,8 +409,6 @@ func (c *containerData) housekeeping() {
 	for {
 		select {
 		case <-c.stop:
-			// Cleanup container resources before stopping housekeeping.
-			c.handler.Cleanup()
 			// Stop housekeeping when signaled.
 			return
 		default:

vendor/github.com/google/cadvisor/manager/machine.go

@@ -1,183 +0,0 @@
-// Copyright 2014 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package manager
-
-import (
-	"bytes"
-	"flag"
-	"io/ioutil"
-	"path/filepath"
-	"strings"
-	"syscall"
-
-	"github.com/google/cadvisor/container/docker"
-	"github.com/google/cadvisor/fs"
-	info "github.com/google/cadvisor/info/v1"
-	"github.com/google/cadvisor/utils/cloudinfo"
-	"github.com/google/cadvisor/utils/machine"
-	"github.com/google/cadvisor/utils/sysfs"
-	"github.com/google/cadvisor/utils/sysinfo"
-	version "github.com/google/cadvisor/version"
-
-	"github.com/golang/glog"
-)
-
-var machineIdFilePath = flag.String("machine_id_file", "/etc/machine-id,/var/lib/dbus/machine-id", "Comma-separated list of files to check for machine-id. Use the first one that exists.")
-var bootIdFilePath = flag.String("boot_id_file", "/proc/sys/kernel/random/boot_id", "Comma-separated list of files to check for boot-id. Use the first one that exists.")
-
-func getInfoFromFiles(filePaths string) string {
-	if len(filePaths) == 0 {
-		return ""
-	}
-	for _, file := range strings.Split(filePaths, ",") {
-		id, err := ioutil.ReadFile(file)
-		if err == nil {
-			return strings.TrimSpace(string(id))
-		}
-	}
-	glog.Infof("Couldn't collect info from any of the files in %q", filePaths)
-	return ""
-}
-
-func getMachineInfo(sysFs sysfs.SysFs, fsInfo fs.FsInfo, inHostNamespace bool) (*info.MachineInfo, error) {
-	rootFs := "/"
-	if !inHostNamespace {
-		rootFs = "/rootfs"
-	}
-
-	cpuinfo, err := ioutil.ReadFile(filepath.Join(rootFs, "/proc/cpuinfo"))
-	clockSpeed, err := machine.GetClockSpeed(cpuinfo)
-	if err != nil {
-		return nil, err
-	}
-
-	memoryCapacity, err := machine.GetMachineMemoryCapacity()
-	if err != nil {
-		return nil, err
-	}
-
-	filesystems, err := fsInfo.GetGlobalFsInfo()
-	if err != nil {
-		glog.Errorf("Failed to get global filesystem information: %v", err)
-	}
-
-	diskMap, err := sysinfo.GetBlockDeviceInfo(sysFs)
-	if err != nil {
-		glog.Errorf("Failed to get disk map: %v", err)
-	}
-
-	netDevices, err := sysinfo.GetNetworkDevices(sysFs)
-	if err != nil {
-		glog.Errorf("Failed to get network devices: %v", err)
-	}
-
-	topology, numCores, err := machine.GetTopology(sysFs, string(cpuinfo))
-	if err != nil {
-		glog.Errorf("Failed to get topology information: %v", err)
-	}
-
-	systemUUID, err := sysinfo.GetSystemUUID(sysFs)
-	if err != nil {
-		glog.Errorf("Failed to get system UUID: %v", err)
-	}
-
-	realCloudInfo := cloudinfo.NewRealCloudInfo()
-	cloudProvider := realCloudInfo.GetCloudProvider()
-	instanceType := realCloudInfo.GetInstanceType()
-	instanceID := realCloudInfo.GetInstanceID()
-
-	machineInfo := &info.MachineInfo{
-		NumCores:       numCores,
-		CpuFrequency:   clockSpeed,
-		MemoryCapacity: memoryCapacity,
-		DiskMap:        diskMap,
-		NetworkDevices: netDevices,
-		Topology:       topology,
-		MachineID:      getInfoFromFiles(filepath.Join(rootFs, *machineIdFilePath)),
-		SystemUUID:     systemUUID,
-		BootID:         getInfoFromFiles(filepath.Join(rootFs, *bootIdFilePath)),
-		CloudProvider:  cloudProvider,
-		InstanceType:   instanceType,
-		InstanceID:     instanceID,
-	}
-
-	for _, fs := range filesystems {
-		machineInfo.Filesystems = append(machineInfo.Filesystems, info.FsInfo{Device: fs.Device, Type: fs.Type.String(), Capacity: fs.Capacity, Inodes: fs.Inodes})
-	}
-
-	return machineInfo, nil
-}
-
-func getVersionInfo() (*info.VersionInfo, error) {
-
-	kernel_version := getKernelVersion()
-	container_os := getContainerOsVersion()
-	docker_version := getDockerVersion()
-
-	return &info.VersionInfo{
-		KernelVersion:      kernel_version,
-		ContainerOsVersion: container_os,
-		DockerVersion:      docker_version,
-		CadvisorVersion:    version.Info["version"],
-		CadvisorRevision:   version.Info["revision"],
-	}, nil
-}
-
-func getContainerOsVersion() string {
-	container_os := "Unknown"
-	os_release, err := ioutil.ReadFile("/etc/os-release")
-	if err == nil {
-		// We might be running in a busybox or some hand-crafted image.
-		// It's useful to know why cadvisor didn't come up.
-		for _, line := range strings.Split(string(os_release), "\n") {
-			parsed := strings.Split(line, "\"")
-			if len(parsed) == 3 && parsed[0] == "PRETTY_NAME=" {
-				container_os = parsed[1]
-				break
-			}
-		}
-	}
-	return container_os
-}
-
-func getDockerVersion() string {
-	docker_version := "Unknown"
-	client, err := docker.Client()
-	if err == nil {
-		version, err := client.Version()
-		if err == nil {
-			docker_version = version.Get("Version")
-		}
-	}
-	return docker_version
-}
-
-func getKernelVersion() string {
-	uname := &syscall.Utsname{}
-
-	if err := syscall.Uname(uname); err != nil {
-		return "Unknown"
-	}
-
-	release := make([]byte, len(uname.Release))
-	i := 0
-	for _, c := range uname.Release {
-		release[i] = byte(c)
-		i++
-	}
-	release = release[:bytes.IndexByte(release, 0)]
-
-	return string(release)
-}

vendor/github.com/google/cadvisor/manager/manager.go

@@ -36,9 +36,13 @@ import (
 	"github.com/google/cadvisor/fs"
 	info "github.com/google/cadvisor/info/v1"
 	"github.com/google/cadvisor/info/v2"
-	"github.com/google/cadvisor/utils/cpuload"
+	"github.com/google/cadvisor/machine"
+	"github.com/google/cadvisor/manager/watcher"
+	rawwatcher "github.com/google/cadvisor/manager/watcher/raw"
+	rktwatcher "github.com/google/cadvisor/manager/watcher/rkt"
 	"github.com/google/cadvisor/utils/oomparser"
 	"github.com/google/cadvisor/utils/sysfs"
+	"github.com/google/cadvisor/version"
 
 	"github.com/golang/glog"
 	"github.com/opencontainers/runc/libcontainer/cgroups"
@@ -46,7 +50,6 @@ import (
 
 var globalHousekeepingInterval = flag.Duration("global_housekeeping_interval", 1*time.Minute, "Interval between global housekeepings")
 var logCadvisorUsage = flag.Bool("log_cadvisor_usage", false, "Whether to log the usage of the cAdvisor container")
-var enableLoadReader = flag.Bool("enable_load_reader", false, "Whether to enable cpu load reader")
 var eventStorageAgeLimit = flag.String("event_storage_age_limit", "default=24h", "Max length of time for which to store events (per type). Value is a comma separated list of key values, where the keys are event types (e.g.: creation, oom) or \"default\" and the value is a duration. Default is applied to all non-specified event types")
 var eventStorageEventLimit = flag.String("event_storage_event_limit", "default=100000", "Max number of events to store (per type). Value is a comma separated list of key values, where the keys are event types (e.g.: creation, oom) or \"default\" and the value is an integer. Default is applied to all non-specified event types")
 var applicationMetricsCountLimit = flag.Int("application_metrics_count_limit", 100, "Max number of application metrics to store (per container)")
@@ -65,6 +68,8 @@ type Manager interface {
 	GetContainerInfo(containerName string, query *info.ContainerInfoRequest) (*info.ContainerInfo, error)
 
 	// Get V2 information about a container.
+	// Recursive (subcontainer) requests are best-effort, and may return a partial result alongside an
+	// error in the partial failure case.
 	GetContainerInfoV2(containerName string, options v2.RequestOptions) (map[string]v2.ContainerInfo, error)
 
 	// Get information about all subcontainers of the specified container (includes self).
@@ -110,10 +115,10 @@ type Manager interface {
 	CloseEventChannel(watch_id int)
 
 	// Get status information about docker.
-	DockerInfo() (DockerStatus, error)
+	DockerInfo() (info.DockerStatus, error)
 
 	// Get details about interesting docker images.
-	DockerImages() ([]DockerImage, error)
+	DockerImages() ([]info.DockerImage, error)
 
 	// Returns debugging information. Map of lines per category.
 	DebugInfo() map[string][]string
@@ -132,7 +137,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn
 	}
 	glog.Infof("cAdvisor running in container: %q", selfContainer)
 
-	dockerInfo, err := dockerInfo()
+	dockerStatus, err := docker.Status()
 	if err != nil {
 		glog.Warningf("Unable to connect to Docker: %v", err)
 	}
@@ -144,8 +149,8 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn
 	context := fs.Context{
 		Docker: fs.DockerContext{
 			Root:         docker.RootDir(),
-			Driver:       dockerInfo.Driver,
-			DriverStatus: dockerInfo.DriverStatus,
+			Driver:       dockerStatus.Driver,
+			DriverStatus: dockerStatus.DriverStatus,
 		},
 		RktPath: rktPath,
 	}
@@ -161,6 +166,9 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn
 		inHostNamespace = true
 	}
 
+	// Register for new subcontainers.
+	eventsChannel := make(chan watcher.ContainerEvent, 16)
+
 	newManager := &manager{
 		containers:               make(map[namespacedContainerName]*containerData),
 		quitChannels:             make([]chan error, 0, 2),
@@ -172,9 +180,11 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn
 		maxHousekeepingInterval:  maxHousekeepingInterval,
 		allowDynamicHousekeeping: allowDynamicHousekeeping,
 		ignoreMetrics:            ignoreMetricsSet,
+		containerWatchers:        []watcher.ContainerWatcher{},
+		eventsChannel:            eventsChannel,
 	}
 
-	machineInfo, err := getMachineInfo(sysfs, fsInfo, inHostNamespace)
+	machineInfo, err := machine.Info(sysfs, fsInfo, inHostNamespace)
 	if err != nil {
 		return nil, err
 	}
@@ -209,12 +219,13 @@ type manager struct {
 	quitChannels             []chan error
 	cadvisorContainer        string
 	inHostNamespace          bool
-	loadReader               cpuload.CpuLoadReader
 	eventHandler             events.EventManager
 	startupTime              time.Time
 	maxHousekeepingInterval  time.Duration
 	allowDynamicHousekeeping bool
 	ignoreMetrics            container.MetricSet
+	containerWatchers        []watcher.ContainerWatcher
+	eventsChannel            chan watcher.ContainerEvent
 }
 
 // Start the container manager.
@@ -227,6 +238,12 @@ func (self *manager) Start() error {
 	err = rkt.Register(self, self.fsInfo, self.ignoreMetrics)
 	if err != nil {
 		glog.Errorf("Registration of the rkt container factory failed: %v", err)
+	} else {
+		watcher, err := rktwatcher.NewRktContainerWatcher()
+		if err != nil {
+			return err
+		}
+		self.containerWatchers = append(self.containerWatchers, watcher)
 	}
 
 	err = systemd.Register(self, self.fsInfo, self.ignoreMetrics)
@@ -239,24 +256,11 @@ func (self *manager) Start() error {
 		glog.Errorf("Registration of the raw container factory failed: %v", err)
 	}
 
-	self.DockerInfo()
-	self.DockerImages()
-
-	if *enableLoadReader {
-		// Create cpu load reader.
-		cpuLoadReader, err := cpuload.New()
-		if err != nil {
-			// TODO(rjnagal): Promote to warning once we support cpu load inside namespaces.
-			glog.Infof("Could not initialize cpu load reader: %s", err)
-		} else {
-			err = cpuLoadReader.Start()
-			if err != nil {
-				glog.Warningf("Could not start cpu load stat collector: %s", err)
-			} else {
-				self.loadReader = cpuLoadReader
-			}
-		}
+	rawWatcher, err := rawwatcher.NewRawContainerWatcher()
+	if err != nil {
+		return err
 	}
+	self.containerWatchers = append(self.containerWatchers, rawWatcher)
 
 	// Watch for OOMs.
 	err = self.watchForNewOoms()
@@ -270,7 +274,7 @@ func (self *manager) Start() error {
 	}
 
 	// Create root and then recover all containers.
-	err = self.createContainer("/")
+	err = self.createContainer("/", watcher.Raw)
 	if err != nil {
 		return err
 	}
@@ -310,10 +314,6 @@ func (self *manager) Stop() error {
 		}
 	}
 	self.quitChannels = make([]chan error, 0, 2)
-	if self.loadReader != nil {
-		self.loadReader.Stop()
-		self.loadReader = nil
-	}
 	return nil
 }
 
@@ -373,15 +373,16 @@ func (self *manager) GetDerivedStats(containerName string, options v2.RequestOpt
 	if err != nil {
 		return nil, err
 	}
+	var errs partialFailure
 	stats := make(map[string]v2.DerivedStats)
 	for name, cont := range conts {
 		d, err := cont.DerivedStats()
 		if err != nil {
-			return nil, err
+			errs.append(name, "DerivedStats", err)
 		}
 		stats[name] = d
 	}
-	return stats, nil
+	return stats, errs.OrNil()
 }
 
 func (self *manager) GetContainerSpec(containerName string, options v2.RequestOptions) (map[string]v2.ContainerSpec, error) {
@@ -389,16 +390,17 @@ func (self *manager) GetContainerSpec(containerName string, options v2.RequestOp
 	if err != nil {
 		return nil, err
 	}
+	var errs partialFailure
 	specs := make(map[string]v2.ContainerSpec)
 	for name, cont := range conts {
 		cinfo, err := cont.GetInfo()
 		if err != nil {
-			return nil, err
+			errs.append(name, "GetInfo", err)
 		}
 		spec := self.getV2Spec(cinfo)
 		specs[name] = spec
 	}
-	return specs, nil
+	return specs, errs.OrNil()
 }
 
 // Get V2 container spec from v1 container info.
@@ -434,26 +436,32 @@ func (self *manager) GetContainerInfoV2(containerName string, options v2.Request
 		return nil, err
 	}
 
+	var errs partialFailure
+	var nilTime time.Time // Ignored.
+
 	infos := make(map[string]v2.ContainerInfo, len(containers))
 	for name, container := range containers {
+		result := v2.ContainerInfo{}
 		cinfo, err := container.GetInfo()
 		if err != nil {
-			return nil, err
+			errs.append(name, "GetInfo", err)
+			infos[name] = result
+			continue
 		}
+		result.Spec = self.getV2Spec(cinfo)
 
-		var nilTime time.Time // Ignored.
 		stats, err := self.memoryCache.RecentStats(name, nilTime, nilTime, options.Count)
 		if err != nil {
-			return nil, err
+			errs.append(name, "RecentStats", err)
+			infos[name] = result
+			continue
 		}
 
-		infos[name] = v2.ContainerInfo{
-			Spec:  self.getV2Spec(cinfo),
-			Stats: v2.ContainerStatsFromV1(&cinfo.Spec, stats),
-		}
+		result.Stats = v2.ContainerStatsFromV1(&cinfo.Spec, stats)
+		infos[name] = result
 	}
 
-	return infos, nil
+	return infos, errs.OrNil()
 }
 
 func (self *manager) containerDataToContainerInfo(cont *containerData, query *info.ContainerInfoRequest) (*info.ContainerInfo, error) {
@@ -594,6 +602,7 @@ func (self *manager) GetRequestedContainersInfo(containerName string, options v2
 	if err != nil {
 		return nil, err
 	}
+	var errs partialFailure
 	containersMap := make(map[string]*info.ContainerInfo)
 	query := info.ContainerInfoRequest{
 		NumStats: options.Count,
@@ -601,12 +610,11 @@ func (self *manager) GetRequestedContainersInfo(containerName string, options v2
 	for name, data := range containers {
 		info, err := self.containerDataToContainerInfo(data, &query)
 		if err != nil {
-			// Skip containers with errors, we try to degrade gracefully.
-			continue
+			errs.append(name, "containerDataToContainerInfo", err)
 		}
 		containersMap[name] = info
 	}
-	return containersMap, nil
+	return containersMap, errs.OrNil()
 }
 
 func (self *manager) getRequestedContainers(containerName string, options v2.RequestOptions) (map[string]*containerData, error) {
@@ -769,8 +777,12 @@ func (m *manager) registerCollectors(collectorConfigs map[string]string, cont *c
 	return nil
 }
 
-// Create a container.
-func (m *manager) createContainer(containerName string) error {
+// Enables overwriting an existing containerData/Handler object for a given containerName.
+// Can't use createContainer as it just returns if a given containerName has a handler already.
+// Ex: rkt handler will want to take priority over the raw handler, but the raw handler might be created first.
+
+// Only allow raw handler to be overridden
+func (m *manager) overrideContainer(containerName string, watchSource watcher.ContainerWatchSource) error {
 	m.containersLock.Lock()
 	defer m.containersLock.Unlock()
 
@@ -778,12 +790,41 @@ func (m *manager) createContainer(containerName string) error {
 		Name: containerName,
 	}
 
+	if _, ok := m.containers[namespacedName]; ok {
+		containerData := m.containers[namespacedName]
+
+		if containerData.handler.Type() != container.ContainerTypeRaw {
+			return nil
+		}
+
+		err := m.destroyContainerLocked(containerName)
+		if err != nil {
+			return fmt.Errorf("overrideContainer: failed to destroy containerData/handler for %v: %v", containerName, err)
+		}
+	}
+
+	return m.createContainerLocked(containerName, watchSource)
+}
+
+// Create a container.
+func (m *manager) createContainer(containerName string, watchSource watcher.ContainerWatchSource) error {
+	m.containersLock.Lock()
+	defer m.containersLock.Unlock()
+
+	return m.createContainerLocked(containerName, watchSource)
+}
+
+func (m *manager) createContainerLocked(containerName string, watchSource watcher.ContainerWatchSource) error {
+	namespacedName := namespacedContainerName{
+		Name: containerName,
+	}
+
 	// Check that the container didn't already exist.
 	if _, ok := m.containers[namespacedName]; ok {
 		return nil
 	}
 
-	handler, accept, err := container.NewContainerHandler(containerName, m.inHostNamespace)
+	handler, accept, err := container.NewContainerHandler(containerName, watchSource, m.inHostNamespace)
 	if err != nil {
 		return err
 	}
@@ -798,7 +839,7 @@ func (m *manager) createContainer(containerName string) error {
 	}
 
 	logUsage := *logCadvisorUsage && containerName == m.cadvisorContainer
-	cont, err := newContainerData(containerName, m.memoryCache, handler, m.loadReader, logUsage, collectorManager, m.maxHousekeepingInterval, m.allowDynamicHousekeeping)
+	cont, err := newContainerData(containerName, m.memoryCache, handler, logUsage, collectorManager, m.maxHousekeepingInterval, m.allowDynamicHousekeeping)
 	if err != nil {
 		return err
 	}
@@ -850,6 +891,10 @@ func (m *manager) destroyContainer(containerName string) error {
 	m.containersLock.Lock()
 	defer m.containersLock.Unlock()
 
+	return m.destroyContainerLocked(containerName)
+}
+
+func (m *manager) destroyContainerLocked(containerName string) error {
 	namespacedName := namespacedContainerName{
 		Name: containerName,
 	}
@@ -947,7 +992,7 @@ func (m *manager) detectSubcontainers(containerName string) error {
 
 	// Add the new containers.
 	for _, cont := range added {
-		err = m.createContainer(cont.Name)
+		err = m.createContainer(cont.Name, watcher.Raw)
 		if err != nil {
 			glog.Errorf("Failed to create existing container: %s: %s", cont.Name, err)
 		}
@@ -966,28 +1011,15 @@ func (m *manager) detectSubcontainers(containerName string) error {
 
 // Watches for new containers started in the system. Runs forever unless there is a setup error.
 func (self *manager) watchForNewContainers(quit chan error) error {
-	var root *containerData
-	var ok bool
-	func() {
-		self.containersLock.RLock()
-		defer self.containersLock.RUnlock()
-		root, ok = self.containers[namespacedContainerName{
-			Name: "/",
-		}]
-	}()
-	if !ok {
-		return fmt.Errorf("root container does not exist when watching for new containers")
-	}
-
-	// Register for new subcontainers.
-	eventsChannel := make(chan container.SubcontainerEvent, 16)
-	err := root.handler.WatchSubcontainers(eventsChannel)
-	if err != nil {
-		return err
+	for _, watcher := range self.containerWatchers {
+		err := watcher.Start(self.eventsChannel)
+		if err != nil {
+			return err
+		}
 	}
 
 	// There is a race between starting the watch and new container creation so we do a detection before we read new containers.
-	err = self.detectSubcontainers("/")
+	err := self.detectSubcontainers("/")
 	if err != nil {
 		return err
 	}
@@ -996,21 +1028,37 @@ func (self *manager) watchForNewContainers(quit chan error) error {
 	go func() {
 		for {
 			select {
-			case event := <-eventsChannel:
+			case event := <-self.eventsChannel:
 				switch {
-				case event.EventType == container.SubcontainerAdd:
-					err = self.createContainer(event.Name)
-				case event.EventType == container.SubcontainerDelete:
+				case event.EventType == watcher.ContainerAdd:
+					switch event.WatchSource {
+					// the Rkt and Raw watchers can race, and if Raw wins, we want Rkt to override and create a new handler for Rkt containers
+					case watcher.Rkt:
+						err = self.overrideContainer(event.Name, event.WatchSource)
+					default:
+						err = self.createContainer(event.Name, event.WatchSource)
+					}
+				case event.EventType == watcher.ContainerDelete:
 					err = self.destroyContainer(event.Name)
 				}
 				if err != nil {
-					glog.Warningf("Failed to process watch event: %v", err)
+					glog.Warningf("Failed to process watch event %+v: %v", event, err)
 				}
 			case <-quit:
+				var errs partialFailure
+
 				// Stop processing events if asked to quit.
-				err := root.handler.StopWatchingSubcontainers()
-				quit <- err
-				if err == nil {
+				for i, watcher := range self.containerWatchers {
+					err := watcher.Stop()
+					if err != nil {
+						errs.append(fmt.Sprintf("watcher %d", i), "Stop", err)
+					}
+				}
+
+				if len(errs) > 0 {
+					quit <- errs
+				} else {
+					quit <- nil
 					glog.Infof("Exiting thread watching subcontainers")
 					return
 				}
@@ -1125,79 +1173,12 @@ func parseEventsStoragePolicy() events.StoragePolicy {
 	return policy
 }
 
-type DockerStatus struct {
-	Version       string            `json:"version"`
-	KernelVersion string            `json:"kernel_version"`
-	OS            string            `json:"os"`
-	Hostname      string            `json:"hostname"`
-	RootDir       string            `json:"root_dir"`
-	Driver        string            `json:"driver"`
-	DriverStatus  map[string]string `json:"driver_status"`
-	ExecDriver    string            `json:"exec_driver"`
-	NumImages     int               `json:"num_images"`
-	NumContainers int               `json:"num_containers"`
+func (m *manager) DockerImages() ([]info.DockerImage, error) {
+	return docker.Images()
 }
 
-type DockerImage struct {
-	ID          string   `json:"id"`
-	RepoTags    []string `json:"repo_tags"` // repository name and tags.
-	Created     int64    `json:"created"`   // unix time since creation.
-	VirtualSize int64    `json:"virtual_size"`
-	Size        int64    `json:"size"`
-}
-
-func (m *manager) DockerImages() ([]DockerImage, error) {
-	images, err := docker.DockerImages()
-	if err != nil {
-		return nil, err
-	}
-	out := []DockerImage{}
-	const unknownTag = "<none>:<none>"
-	for _, image := range images {
-		if len(image.RepoTags) == 1 && image.RepoTags[0] == unknownTag {
-			// images with repo or tags are uninteresting.
-			continue
-		}
-		di := DockerImage{
-			ID:          image.ID,
-			RepoTags:    image.RepoTags,
-			Created:     image.Created,
-			VirtualSize: image.VirtualSize,
-			Size:        image.Size,
-		}
-		out = append(out, di)
-	}
-	return out, nil
-}
-
-func (m *manager) DockerInfo() (DockerStatus, error) {
-	return dockerInfo()
-}
-
-func dockerInfo() (DockerStatus, error) {
-	dockerInfo, err := docker.DockerInfo()
-	if err != nil {
-		return DockerStatus{}, err
-	}
-	versionInfo, err := getVersionInfo()
-	if err != nil {
-		return DockerStatus{}, err
-	}
-	out := DockerStatus{}
-	out.Version = versionInfo.DockerVersion
-	out.KernelVersion = dockerInfo.KernelVersion
-	out.OS = dockerInfo.OperatingSystem
-	out.Hostname = dockerInfo.Name
-	out.RootDir = dockerInfo.DockerRootDir
-	out.Driver = dockerInfo.Driver
-	out.ExecDriver = dockerInfo.ExecutionDriver
-	out.NumImages = dockerInfo.Images
-	out.NumContainers = dockerInfo.Containers
-	out.DriverStatus = make(map[string]string, len(dockerInfo.DriverStatus))
-	for _, v := range dockerInfo.DriverStatus {
-		out.DriverStatus[v[0]] = v[1]
-	}
-	return out, nil
+func (m *manager) DockerInfo() (info.DockerStatus, error) {
+	return docker.Status()
 }
 
 func (m *manager) DebugInfo() map[string][]string {
@@ -1234,3 +1215,36 @@ func (m *manager) DebugInfo() map[string][]string {
 	debugInfo["Managed containers"] = lines
 	return debugInfo
 }
+
+func getVersionInfo() (*info.VersionInfo, error) {
+
+	kernel_version := machine.KernelVersion()
+	container_os := machine.ContainerOsVersion()
+	docker_version := docker.VersionString()
+
+	return &info.VersionInfo{
+		KernelVersion:      kernel_version,
+		ContainerOsVersion: container_os,
+		DockerVersion:      docker_version,
+		CadvisorVersion:    version.Info["version"],
+		CadvisorRevision:   version.Info["revision"],
+	}, nil
+}
+
+// Helper for accumulating partial failures.
+type partialFailure []string
+
+func (f *partialFailure) append(id, operation string, err error) {
+	*f = append(*f, fmt.Sprintf("[%q: %s: %s]", id, operation, err))
+}
+
+func (f partialFailure) Error() string {
+	return fmt.Sprintf("partial failures: %s", strings.Join(f, ", "))
+}
+
+func (f partialFailure) OrNil() error {
+	if len(f) == 0 {
+		return nil
+	}
+	return f
+}

vendor/github.com/google/cadvisor/manager/manager_mock.go

@@ -1,119 +0,0 @@
-// Copyright 2014 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build test
-
-package manager
-
-import (
-	"github.com/google/cadvisor/events"
-	info "github.com/google/cadvisor/info/v1"
-	"github.com/google/cadvisor/info/v2"
-
-	"github.com/stretchr/testify/mock"
-)
-
-type ManagerMock struct {
-	mock.Mock
-}
-
-func (c *ManagerMock) Start() error {
-	args := c.Called()
-	return args.Error(0)
-}
-
-func (c *ManagerMock) Stop() error {
-	args := c.Called()
-	return args.Error(0)
-}
-
-func (c *ManagerMock) GetContainerInfo(name string, query *info.ContainerInfoRequest) (*info.ContainerInfo, error) {
-	args := c.Called(name, query)
-	return args.Get(0).(*info.ContainerInfo), args.Error(1)
-}
-
-func (c *ManagerMock) SubcontainersInfo(containerName string, query *info.ContainerInfoRequest) ([]*info.ContainerInfo, error) {
-	args := c.Called(containerName, query)
-	return args.Get(0).([]*info.ContainerInfo), args.Error(1)
-}
-
-func (c *ManagerMock) AllDockerContainers(query *info.ContainerInfoRequest) (map[string]info.ContainerInfo, error) {
-	args := c.Called(query)
-	return args.Get(0).(map[string]info.ContainerInfo), args.Error(1)
-}
-
-func (c *ManagerMock) DockerContainer(name string, query *info.ContainerInfoRequest) (info.ContainerInfo, error) {
-	args := c.Called(name, query)
-	return args.Get(0).(info.ContainerInfo), args.Error(1)
-}
-
-func (c *ManagerMock) GetContainerSpec(containerName string, options v2.RequestOptions) (map[string]v2.ContainerSpec, error) {
-	args := c.Called(containerName, options)
-	return args.Get(0).(map[string]v2.ContainerSpec), args.Error(1)
-}
-
-func (c *ManagerMock) GetDerivedStats(containerName string, options v2.RequestOptions) (map[string]v2.DerivedStats, error) {
-	args := c.Called(containerName, options)
-	return args.Get(0).(map[string]v2.DerivedStats), args.Error(1)
-}
-
-func (c *ManagerMock) GetRequestedContainersInfo(containerName string, options v2.RequestOptions) (map[string]*info.ContainerInfo, error) {
-	args := c.Called(containerName, options)
-	return args.Get(0).(map[string]*info.ContainerInfo), args.Error(1)
-}
-
-func (c *ManagerMock) Exists(name string) bool {
-	args := c.Called(name)
-	return args.Get(0).(bool)
-}
-
-func (c *ManagerMock) WatchForEvents(queryuest *events.Request, passedChannel chan *info.Event) error {
-	args := c.Called(queryuest, passedChannel)
-	return args.Error(0)
-}
-
-func (c *ManagerMock) GetPastEvents(queryuest *events.Request) ([]*info.Event, error) {
-	args := c.Called(queryuest)
-	return args.Get(0).([]*info.Event), args.Error(1)
-}
-
-func (c *ManagerMock) GetMachineInfo() (*info.MachineInfo, error) {
-	args := c.Called()
-	return args.Get(0).(*info.MachineInfo), args.Error(1)
-}
-
-func (c *ManagerMock) GetVersionInfo() (*info.VersionInfo, error) {
-	args := c.Called()
-	return args.Get(0).(*info.VersionInfo), args.Error(1)
-}
-
-func (c *ManagerMock) GetFsInfo() ([]v2.FsInfo, error) {
-	args := c.Called()
-	return args.Get(0).([]v2.FsInfo), args.Error(1)
-}
-
-func (c *ManagerMock) GetProcessList(name string, options v2.RequestOptions) ([]v2.ProcessInfo, error) {
-	args := c.Called()
-	return args.Get(0).([]v2.ProcessInfo), args.Error(1)
-}
-
-func (c *ManagerMock) DockerInfo() (DockerStatus, error) {
-	args := c.Called()
-	return args.Get(0).(DockerStatus), args.Error(1)
-}
-
-func (c *ManagerMock) DockerImages() ([]DockerImage, error) {
-	args := c.Called()
-	return args.Get(0).([]DockerImage), args.Error(1)
-}

vendor/github.com/google/cadvisor/manager/watcher/raw/raw.go

@@ -0,0 +1,214 @@
+// Copyright 2014 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package container defines types for sub-container events and also
+// defines an interface for container operation handlers.
+package raw
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"strings"
+
+	"github.com/google/cadvisor/container/common"
+	"github.com/google/cadvisor/container/libcontainer"
+	"github.com/google/cadvisor/manager/watcher"
+
+	"github.com/golang/glog"
+	"golang.org/x/exp/inotify"
+)
+
+type rawContainerWatcher struct {
+	// Absolute path to the root of the cgroup hierarchies
+	cgroupPaths map[string]string
+
+	cgroupSubsystems *libcontainer.CgroupSubsystems
+
+	// Inotify event watcher.
+	watcher *common.InotifyWatcher
+
+	// Signal for watcher thread to stop.
+	stopWatcher chan error
+}
+
+func NewRawContainerWatcher() (watcher.ContainerWatcher, error) {
+	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cgroup subsystems: %v", err)
+	}
+	if len(cgroupSubsystems.Mounts) == 0 {
+		return nil, fmt.Errorf("failed to find supported cgroup mounts for the raw factory")
+	}
+
+	watcher, err := common.NewInotifyWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	rawWatcher := &rawContainerWatcher{
+		cgroupPaths:      common.MakeCgroupPaths(cgroupSubsystems.MountPoints, "/"),
+		cgroupSubsystems: &cgroupSubsystems,
+		watcher:          watcher,
+		stopWatcher:      make(chan error),
+	}
+
+	return rawWatcher, nil
+}
+
+func (self *rawContainerWatcher) Start(events chan watcher.ContainerEvent) error {
+	// Watch this container (all its cgroups) and all subdirectories.
+	for _, cgroupPath := range self.cgroupPaths {
+		_, err := self.watchDirectory(cgroupPath, "/")
+		if err != nil {
+			return err
+		}
+	}
+
+	// Process the events received from the kernel.
+	go func() {
+		for {
+			select {
+			case event := <-self.watcher.Event():
+				err := self.processEvent(event, events)
+				if err != nil {
+					glog.Warningf("Error while processing event (%+v): %v", event, err)
+				}
+			case err := <-self.watcher.Error():
+				glog.Warningf("Error while watching %q:", "/", err)
+			case <-self.stopWatcher:
+				err := self.watcher.Close()
+				if err == nil {
+					self.stopWatcher <- err
+					return
+				}
+			}
+		}
+	}()
+
+	return nil
+}
+
+func (self *rawContainerWatcher) Stop() error {
+	// Rendezvous with the watcher thread.
+	self.stopWatcher <- nil
+	return <-self.stopWatcher
+}
+
+// Watches the specified directory and all subdirectories. Returns whether the path was
+// already being watched and an error (if any).
+func (self *rawContainerWatcher) watchDirectory(dir string, containerName string) (bool, error) {
+	alreadyWatching, err := self.watcher.AddWatch(containerName, dir)
+	if err != nil {
+		return alreadyWatching, err
+	}
+
+	// Remove the watch if further operations failed.
+	cleanup := true
+	defer func() {
+		if cleanup {
+			_, err := self.watcher.RemoveWatch(containerName, dir)
+			if err != nil {
+				glog.Warningf("Failed to remove inotify watch for %q: %v", dir, err)
+			}
+		}
+	}()
+
+	// TODO(vmarmol): We should re-do this once we're done to ensure directories were not added in the meantime.
+	// Watch subdirectories as well.
+	entries, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return alreadyWatching, err
+	}
+	for _, entry := range entries {
+		if entry.IsDir() {
+			// TODO(vmarmol): We don't have to fail here, maybe we can recover and try to get as many registrations as we can.
+			_, err = self.watchDirectory(path.Join(dir, entry.Name()), path.Join(containerName, entry.Name()))
+			if err != nil {
+				return alreadyWatching, err
+			}
+		}
+	}
+
+	cleanup = false
+	return alreadyWatching, nil
+}
+
+func (self *rawContainerWatcher) processEvent(event *inotify.Event, events chan watcher.ContainerEvent) error {
+	// Convert the inotify event type to a container create or delete.
+	var eventType watcher.ContainerEventType
+	switch {
+	case (event.Mask & inotify.IN_CREATE) > 0:
+		eventType = watcher.ContainerAdd
+	case (event.Mask & inotify.IN_DELETE) > 0:
+		eventType = watcher.ContainerDelete
+	case (event.Mask & inotify.IN_MOVED_FROM) > 0:
+		eventType = watcher.ContainerDelete
+	case (event.Mask & inotify.IN_MOVED_TO) > 0:
+		eventType = watcher.ContainerAdd
+	default:
+		// Ignore other events.
+		return nil
+	}
+
+	// Derive the container name from the path name.
+	var containerName string
+	for _, mount := range self.cgroupSubsystems.Mounts {
+		mountLocation := path.Clean(mount.Mountpoint) + "/"
+		if strings.HasPrefix(event.Name, mountLocation) {
+			containerName = event.Name[len(mountLocation)-1:]
+			break
+		}
+	}
+	if containerName == "" {
+		return fmt.Errorf("unable to detect container from watch event on directory %q", event.Name)
+	}
+
+	// Maintain the watch for the new or deleted container.
+	switch eventType {
+	case watcher.ContainerAdd:
+		// New container was created, watch it.
+		alreadyWatched, err := self.watchDirectory(event.Name, containerName)
+		if err != nil {
+			return err
+		}
+
+		// Only report container creation once.
+		if alreadyWatched {
+			return nil
+		}
+	case watcher.ContainerDelete:
+		// Container was deleted, stop watching for it.
+		lastWatched, err := self.watcher.RemoveWatch(containerName, event.Name)
+		if err != nil {
+			return err
+		}
+
+		// Only report container deletion once.
+		if !lastWatched {
+			return nil
+		}
+	default:
+		return fmt.Errorf("unknown event type %v", eventType)
+	}
+
+	// Deliver the event.
+	events <- watcher.ContainerEvent{
+		EventType:   eventType,
+		Name:        containerName,
+		WatchSource: watcher.Raw,
+	}
+
+	return nil
+}

vendor/github.com/google/cadvisor/manager/watcher/rkt/rkt.go

@@ -0,0 +1,154 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package rkt implements the watcher interface for rkt
+package rkt
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/google/cadvisor/container/rkt"
+	"github.com/google/cadvisor/manager/watcher"
+
+	rktapi "github.com/coreos/rkt/api/v1alpha"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+)
+
+type rktContainerWatcher struct {
+	// Signal for watcher thread to stop.
+	stopWatcher chan error
+}
+
+func NewRktContainerWatcher() (watcher.ContainerWatcher, error) {
+	watcher := &rktContainerWatcher{
+		stopWatcher: make(chan error),
+	}
+
+	return watcher, nil
+}
+
+func (self *rktContainerWatcher) Start(events chan watcher.ContainerEvent) error {
+	go self.detectRktContainers(events)
+
+	return nil
+}
+
+func (self *rktContainerWatcher) Stop() error {
+	// Rendezvous with the watcher thread.
+	self.stopWatcher <- nil
+	return nil
+}
+
+func (self *rktContainerWatcher) detectRktContainers(events chan watcher.ContainerEvent) {
+	glog.Infof("starting detectRktContainers thread")
+	ticker := time.Tick(10 * time.Second)
+	curpods := make(map[string]*rktapi.Pod)
+
+	for {
+		select {
+		case <-ticker:
+			pods, err := listRunningPods()
+			if err != nil {
+				glog.Errorf("detectRktContainers: listRunningPods failed: %v", err)
+				continue
+			}
+			curpods = self.syncRunningPods(pods, events, curpods)
+
+		case <-self.stopWatcher:
+			glog.Infof("Exiting rktContainer Thread")
+			return
+		}
+	}
+}
+
+func (self *rktContainerWatcher) syncRunningPods(pods []*rktapi.Pod, events chan watcher.ContainerEvent, curpods map[string]*rktapi.Pod) map[string]*rktapi.Pod {
+	newpods := make(map[string]*rktapi.Pod)
+
+	for _, pod := range pods {
+		newpods[pod.Id] = pod
+		// if pods become mutable, have to handle this better
+		if _, ok := curpods[pod.Id]; !ok {
+			// should create all cgroups not including system.slice
+			// i.e. /system.slice/rkt-test.service and /system.slice/rkt-test.service/system.slice/pause.service
+			for _, cgroup := range podToCgroup(pod) {
+				self.sendUpdateEvent(cgroup, events)
+			}
+		}
+	}
+
+	for id, pod := range curpods {
+		if _, ok := newpods[id]; !ok {
+			for _, cgroup := range podToCgroup(pod) {
+				glog.Infof("cgroup to delete = %v", cgroup)
+				self.sendDestroyEvent(cgroup, events)
+			}
+		}
+	}
+
+	return newpods
+}
+
+func (self *rktContainerWatcher) sendUpdateEvent(cgroup string, events chan watcher.ContainerEvent) {
+	events <- watcher.ContainerEvent{
+		EventType:   watcher.ContainerAdd,
+		Name:        cgroup,
+		WatchSource: watcher.Rkt,
+	}
+}
+
+func (self *rktContainerWatcher) sendDestroyEvent(cgroup string, events chan watcher.ContainerEvent) {
+	events <- watcher.ContainerEvent{
+		EventType:   watcher.ContainerDelete,
+		Name:        cgroup,
+		WatchSource: watcher.Rkt,
+	}
+}
+
+func listRunningPods() ([]*rktapi.Pod, error) {
+	client, err := rkt.Client()
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.ListPods(context.Background(), &rktapi.ListPodsRequest{
+		// Specify the request: Fetch and print only running pods and their details.
+		Detail: true,
+		Filters: []*rktapi.PodFilter{
+			{
+				States: []rktapi.PodState{rktapi.PodState_POD_STATE_RUNNING},
+			},
+		},
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return resp.Pods, nil
+}
+
+func podToCgroup(pod *rktapi.Pod) []string {
+	cgroups := make([]string, 1+len(pod.Apps), 1+len(pod.Apps))
+
+	baseCgroup := pod.Cgroup
+	cgroups[0] = baseCgroup
+
+	for i, app := range pod.Apps {
+		cgroups[i+1] = filepath.Join(baseCgroup, "system.slice", app.Name+".service")
+	}
+
+	return cgroups
+}

vendor/github.com/google/cadvisor/manager/watcher/watcher.go

@@ -0,0 +1,52 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package container defines types for sub-container events and also
+// defines an interface for container operation handlers.
+package watcher
+
+// SubcontainerEventType indicates an addition or deletion event.
+type ContainerEventType int
+
+const (
+	ContainerAdd ContainerEventType = iota
+	ContainerDelete
+)
+
+type ContainerWatchSource int
+
+const (
+	Raw ContainerWatchSource = iota
+	Rkt
+)
+
+// ContainerEvent represents a
+type ContainerEvent struct {
+	// The type of event that occurred.
+	EventType ContainerEventType
+
+	// The full container name of the container where the event occurred.
+	Name string
+
+	// The watcher that detected this change event
+	WatchSource ContainerWatchSource
+}
+
+type ContainerWatcher interface {
+	// Registers a channel to listen for events affecting subcontainers (recursively).
+	Start(events chan ContainerEvent) error
+
+	// Stops watching for subcontainer changes.
+	Stop() error
+}