update cadvisor, docker, and runc godeps

This commit is contained in:
David Ashpole
2017-09-05 12:38:57 -07:00
parent 2f543f321d
commit e5a6a79fd7
618 changed files with 47108 additions and 13510 deletions

View File

@@ -420,23 +420,30 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma
}
return writeResult(specs, w)
case storageApi:
var err error
fi := []v2.FsInfo{}
label := r.URL.Query().Get("label")
if len(label) == 0 {
// Get all global filesystems info.
fi, err = m.GetFsInfo("")
uuid := r.URL.Query().Get("uuid")
switch {
case uuid != "":
fi, err := m.GetFsInfoByFsUUID(uuid)
if err != nil {
return err
}
} else {
return writeResult(fi, w)
case label != "":
// Get a specific label.
fi, err = m.GetFsInfo(label)
fi, err := m.GetFsInfo(label)
if err != nil {
return err
}
return writeResult(fi, w)
default:
// Get all global filesystems info.
fi, err := m.GetFsInfo("")
if err != nil {
return err
}
return writeResult(fi, w)
}
return writeResult(fi, w)
case eventsApi:
return handleEventRequest(request, m, w, r)
case psApi:

View File

@@ -72,7 +72,7 @@ func NewPrometheusCollector(collectorName string, configFile []byte, metricCount
}
if metricCountLimit < 0 {
return nil, fmt.Errorf("Metric count limit must be greater than 0")
return nil, fmt.Errorf("Metric count limit must be greater than or equal to 0")
}
var metricsSet map[string]bool

View File

@@ -14,6 +14,7 @@ go_library(
"//vendor/github.com/docker/engine-api/client:go_default_library",
"//vendor/github.com/docker/engine-api/types:go_default_library",
"//vendor/github.com/docker/engine-api/types/container:go_default_library",
"//vendor/github.com/docker/go-connections/tlsconfig:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/google/cadvisor/container:go_default_library",
"//vendor/github.com/google/cadvisor/container/common:go_default_library",

View File

@@ -18,9 +18,11 @@
package docker
import (
"net/http"
"sync"
dclient "github.com/docker/engine-api/client"
"github.com/docker/go-connections/tlsconfig"
)
var (
@@ -29,9 +31,32 @@ var (
dockerClientOnce sync.Once
)
// Client creates a Docker API client based on the given Docker flags
func Client() (*dclient.Client, error) {
dockerClientOnce.Do(func() {
dockerClient, dockerClientErr = dclient.NewClient(*ArgDockerEndpoint, "", nil, nil)
var client *http.Client
if *ArgDockerTLS {
client = &http.Client{}
options := tlsconfig.Options{
CAFile: *ArgDockerCA,
CertFile: *ArgDockerCert,
KeyFile: *ArgDockerKey,
InsecureSkipVerify: false,
}
tlsc, err := tlsconfig.Client(options)
if err != nil {
dockerClientErr = err
return
}
client.Transport = &http.Transport{
TLSClientConfig: tlsc,
}
}
dockerClient, dockerClientErr = dclient.NewClient(*ArgDockerEndpoint,
"",
client,
nil)
})
return dockerClient, dockerClientErr
}

View File

@@ -41,6 +41,10 @@ import (
)
var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "docker endpoint")
var ArgDockerTLS = flag.Bool("docker-tls", false, "use TLS to connect to docker")
var ArgDockerCert = flag.String("docker-tls-cert", "cert.pem", "path to client certificate")
var ArgDockerKey = flag.String("docker-tls-key", "key.pem", "path to private key")
var ArgDockerCA = flag.String("docker-tls-ca", "ca.pem", "path to trusted CA")
// The namespace under which Docker aliases are unique.
const DockerNamespace = "docker"

View File

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["helpers.go"],
cgo = True,
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/golang/glog:go_default_library",

View File

@@ -33,6 +33,11 @@ import (
"github.com/opencontainers/runc/libcontainer/cgroups"
)
/*
#include <unistd.h>
*/
import "C"
type CgroupSubsystems struct {
// Cgroup subsystem mounts.
// e.g.: "/sys/fs/cgroup/cpu" -> ["cpu", "cpuacct"]
@@ -99,7 +104,7 @@ func GetStats(cgroupManager cgroups.Manager, rootFs string, pid int, ignoreMetri
if !ignoreMetrics.Has(container.NetworkUsageMetrics) {
netStats, err := networkStatsFromProc(rootFs, pid)
if err != nil {
glog.V(2).Infof("Unable to get network stats from pid %d: %v", pid, err)
glog.V(4).Infof("Unable to get network stats from pid %d: %v", pid, err)
} else {
stats.Network.Interfaces = append(stats.Network.Interfaces, netStats...)
}
@@ -107,14 +112,14 @@ func GetStats(cgroupManager cgroups.Manager, rootFs string, pid int, ignoreMetri
if !ignoreMetrics.Has(container.NetworkTcpUsageMetrics) {
t, err := tcpStatsFromProc(rootFs, pid, "net/tcp")
if err != nil {
glog.V(2).Infof("Unable to get tcp stats from pid %d: %v", pid, err)
glog.V(4).Infof("Unable to get tcp stats from pid %d: %v", pid, err)
} else {
stats.Network.Tcp = t
}
t6, err := tcpStatsFromProc(rootFs, pid, "net/tcp6")
if err != nil {
glog.V(2).Infof("Unable to get tcp6 stats from pid %d: %v", pid, err)
glog.V(4).Infof("Unable to get tcp6 stats from pid %d: %v", pid, err)
} else {
stats.Network.Tcp6 = t6
}
@@ -122,14 +127,14 @@ func GetStats(cgroupManager cgroups.Manager, rootFs string, pid int, ignoreMetri
if !ignoreMetrics.Has(container.NetworkUdpUsageMetrics) {
u, err := udpStatsFromProc(rootFs, pid, "net/udp")
if err != nil {
glog.V(2).Infof("Unable to get udp stats from pid %d: %v", pid, err)
glog.V(4).Infof("Unable to get udp stats from pid %d: %v", pid, err)
} else {
stats.Network.Udp = u
}
u6, err := udpStatsFromProc(rootFs, pid, "net/udp6")
if err != nil {
glog.V(2).Infof("Unable to get udp6 stats from pid %d: %v", pid, err)
glog.V(4).Infof("Unable to get udp6 stats from pid %d: %v", pid, err)
} else {
stats.Network.Udp6 = u6
}
@@ -433,15 +438,40 @@ func DiskStatsCopy(blkio_stats []cgroups.BlkioStatEntry) (stat []info.PerDiskSta
return DiskStatsCopy1(disk_stat)
}
func minUint32(x, y uint32) uint32 {
if x < y {
return x
}
return y
}
// var to allow unit tests to stub it out
var numCpusFunc = getNumberOnlineCPUs
// Convert libcontainer stats to info.ContainerStats.
func setCpuStats(s *cgroups.Stats, ret *info.ContainerStats) {
ret.Cpu.Usage.User = s.CpuStats.CpuUsage.UsageInUsermode
ret.Cpu.Usage.System = s.CpuStats.CpuUsage.UsageInKernelmode
n := len(s.CpuStats.CpuUsage.PercpuUsage)
ret.Cpu.Usage.PerCpu = make([]uint64, n)
numPossible := uint32(len(s.CpuStats.CpuUsage.PercpuUsage))
// Note that as of https://patchwork.kernel.org/patch/8607101/ (kernel v4.7),
// the percpu usage information includes extra zero values for all additional
// possible CPUs. This is to allow statistic collection after CPU-hotplug.
// We intentionally ignore these extra zeroes.
numActual, err := numCpusFunc()
if err != nil {
glog.Errorf("unable to determine number of actual cpus; defaulting to maximum possible number: errno %v", err)
numActual = numPossible
}
if numActual > numPossible {
// The real number of cores should never be greater than the number of
// datapoints reported in cpu usage.
glog.Errorf("PercpuUsage had %v cpus, but the actual number is %v; ignoring extra CPUs", numPossible, numActual)
}
numActual = minUint32(numPossible, numActual)
ret.Cpu.Usage.PerCpu = make([]uint64, numActual)
ret.Cpu.Usage.Total = 0
for i := 0; i < n; i++ {
for i := uint32(0); i < numActual; i++ {
ret.Cpu.Usage.PerCpu[i] = s.CpuStats.CpuUsage.PercpuUsage[i]
ret.Cpu.Usage.Total += s.CpuStats.CpuUsage.PercpuUsage[i]
}
@@ -451,6 +481,21 @@ func setCpuStats(s *cgroups.Stats, ret *info.ContainerStats) {
ret.Cpu.CFS.ThrottledTime = s.CpuStats.ThrottlingData.ThrottledTime
}
// Copied from
// https://github.com/moby/moby/blob/8b1adf55c2af329a4334f21d9444d6a169000c81/daemon/stats/collector_unix.go#L73
// Apache 2.0, Copyright Docker, Inc.
func getNumberOnlineCPUs() (uint32, error) {
i, err := C.sysconf(C._SC_NPROCESSORS_ONLN)
// According to POSIX - errno is undefined after successful
// sysconf, and can be non-zero in several cases, so look for
// error in returned value not in errno.
// (https://sourceware.org/bugzilla/show_bug.cgi?id=21536)
if i == -1 {
return 0, err
}
return uint32(i), nil
}
func setDiskIoStats(s *cgroups.Stats, ret *info.ContainerStats) {
ret.DiskIo.IoServiceBytes = DiskStatsCopy(s.BlkioStats.IoServiceBytesRecursive)
ret.DiskIo.IoServiced = DiskStatsCopy(s.BlkioStats.IoServicedRecursive)
@@ -466,8 +511,14 @@ func setMemoryStats(s *cgroups.Stats, ret *info.ContainerStats) {
ret.Memory.Usage = s.MemoryStats.Usage.Usage
ret.Memory.Failcnt = s.MemoryStats.Usage.Failcnt
ret.Memory.Cache = s.MemoryStats.Stats["cache"]
ret.Memory.RSS = s.MemoryStats.Stats["rss"]
ret.Memory.Swap = s.MemoryStats.Stats["swap"]
if s.MemoryStats.UseHierarchy {
ret.Memory.RSS = s.MemoryStats.Stats["total_rss"]
ret.Memory.Swap = s.MemoryStats.Stats["total_swap"]
} else {
ret.Memory.RSS = s.MemoryStats.Stats["rss"]
ret.Memory.Swap = s.MemoryStats.Stats["swap"]
}
if v, ok := s.MemoryStats.Stats["pgfault"]; ok {
ret.Memory.ContainerData.Pgfault = v
ret.Memory.HierarchicalData.Pgfault = v

View File

@@ -83,6 +83,8 @@ type RealFsInfo struct {
mounts map[string]*mount.Info
// devicemapper client
dmsetup devicemapper.DmsetupClient
// fsUUIDToDeviceName is a map from the filesystem UUID to its device name.
fsUUIDToDeviceName map[string]string
}
type Context struct {
@@ -103,13 +105,19 @@ func NewFsInfo(context Context) (FsInfo, error) {
return nil, err
}
fsUUIDToDeviceName, err := getFsUUIDToDeviceNameMap()
if err != nil {
return nil, err
}
// Avoid devicemapper container mounts - these are tracked by the ThinPoolWatcher
excluded := []string{fmt.Sprintf("%s/devicemapper/mnt", context.Docker.Root)}
fsInfo := &RealFsInfo{
partitions: processMounts(mounts, excluded),
labels: make(map[string]string, 0),
mounts: make(map[string]*mount.Info, 0),
dmsetup: devicemapper.NewDmsetupClient(),
partitions: processMounts(mounts, excluded),
labels: make(map[string]string, 0),
mounts: make(map[string]*mount.Info, 0),
dmsetup: devicemapper.NewDmsetupClient(),
fsUUIDToDeviceName: fsUUIDToDeviceName,
}
for _, mount := range mounts {
@@ -121,11 +129,44 @@ func NewFsInfo(context Context) (FsInfo, error) {
// add a "partition" for devicemapper to fsInfo.partitions
fsInfo.addDockerImagesLabel(context, mounts)
glog.Infof("Filesystem UUIDs: %+v", fsInfo.fsUUIDToDeviceName)
glog.Infof("Filesystem partitions: %+v", fsInfo.partitions)
fsInfo.addSystemRootLabel(mounts)
return fsInfo, nil
}
// getFsUUIDToDeviceNameMap creates the filesystem uuid to device name map
// using the information in /dev/disk/by-uuid. If the directory does not exist,
// this function will return an empty map.
func getFsUUIDToDeviceNameMap() (map[string]string, error) {
const dir = "/dev/disk/by-uuid"
if _, err := os.Stat(dir); os.IsNotExist(err) {
return make(map[string]string), nil
}
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil, err
}
fsUUIDToDeviceName := make(map[string]string)
for _, file := range files {
path := filepath.Join(dir, file.Name())
target, err := os.Readlink(path)
if err != nil {
glog.Infof("Failed to resolve symlink for %q", path)
continue
}
device, err := filepath.Abs(filepath.Join(dir, target))
if err != nil {
return nil, fmt.Errorf("failed to resolve the absolute path of %q", filepath.Join(dir, target))
}
fsUUIDToDeviceName[file.Name()] = device
}
return fsUUIDToDeviceName, nil
}
func processMounts(mounts []*mount.Info, excludedMountpointPrefixes []string) map[string]partition {
partitions := make(map[string]partition, 0)
@@ -433,6 +474,18 @@ func minor(devNumber uint64) uint {
return uint((devNumber & 0xff) | ((devNumber >> 12) & 0xfff00))
}
func (self *RealFsInfo) GetDeviceInfoByFsUUID(uuid string) (*DeviceInfo, error) {
deviceName, found := self.fsUUIDToDeviceName[uuid]
if !found {
return nil, ErrNoSuchDevice
}
p, found := self.partitions[deviceName]
if !found {
return nil, fmt.Errorf("cannot find device %q in partitions", deviceName)
}
return &DeviceInfo{deviceName, p.major, p.minor}, nil
}
func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) {
buf := new(syscall.Stat_t)
err := syscall.Stat(dir, buf)

View File

@@ -14,7 +14,10 @@
package fs
import "time"
import (
"errors"
"time"
)
type DeviceInfo struct {
Device string
@@ -59,6 +62,9 @@ type DiskStats struct {
WeightedIoTime uint64
}
// ErrNoSuchDevice is the error indicating the requested device does not exist.
var ErrNoSuchDevice = errors.New("cadvisor: no such device")
type FsInfo interface {
// Returns capacity and free space, in bytes, of all the ext2, ext3, ext4 filesystems on the host.
GetGlobalFsInfo() ([]Fs, error)
@@ -72,6 +78,11 @@ type FsInfo interface {
// Returns number of inodes used by 'dir'.
GetDirInodeUsage(dir string, timeout time.Duration) (uint64, error)
// GetDeviceInfoByFsUUID returns the information of the device with the
// specified filesystem uuid. If no such device exists, this function will
// return the ErrNoSuchDevice error.
GetDeviceInfoByFsUUID(uuid string) (*DeviceInfo, error)
// Returns the block device info of the filesystem on which 'dir' resides.
GetDirFsDevice(dir string) (*DeviceInfo, error)

View File

@@ -199,6 +199,9 @@ type DerivedStats struct {
}
type FsInfo struct {
// Time of generation of these stats.
Timestamp time.Time `json:"timestamp"`
// The block device name associated with the filesystem.
Device string `json:"device"`

View File

@@ -101,6 +101,11 @@ type Manager interface {
// Get version information about different components we depend on.
GetVersionInfo() (*info.VersionInfo, error)
// GetFsInfoByFsUUID returns the information of the device having the
// specified filesystem uuid. If no such device with the UUID exists, this
// function will return the fs.ErrNoSuchDevice error.
GetFsInfoByFsUUID(uuid string) (v2.FsInfo, error)
// Get filesystem information for the filesystem that contains the given directory
GetDirFsInfo(dir string) (v2.FsInfo, error)
@@ -136,7 +141,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn
}
// Detect the container we are running on.
selfContainer, err := cgroups.GetThisCgroupDir("cpu")
selfContainer, err := cgroups.GetOwnCgroupPath("cpu")
if err != nil {
return nil, err
}
@@ -676,24 +681,19 @@ func (self *manager) getRequestedContainers(containerName string, options v2.Req
}
func (self *manager) GetDirFsInfo(dir string) (v2.FsInfo, error) {
dirDevice, err := self.fsInfo.GetDirFsDevice(dir)
device, err := self.fsInfo.GetDirFsDevice(dir)
if err != nil {
return v2.FsInfo{}, fmt.Errorf("error trying to get filesystem Device for dir %v: err: %v", dir, err)
return v2.FsInfo{}, fmt.Errorf("failed to get device for dir %q: %v", dir, err)
}
dirMountpoint, err := self.fsInfo.GetMountpointForDevice(dirDevice.Device)
if err != nil {
return v2.FsInfo{}, fmt.Errorf("error trying to get MountPoint for Root Device: %v, err: %v", dirDevice, err)
}
infos, err := self.GetFsInfo("")
return self.getFsInfoByDeviceName(device.Device)
}
func (self *manager) GetFsInfoByFsUUID(uuid string) (v2.FsInfo, error) {
device, err := self.fsInfo.GetDeviceInfoByFsUUID(uuid)
if err != nil {
return v2.FsInfo{}, err
}
for _, info := range infos {
if info.Mountpoint == dirMountpoint {
return info, nil
}
}
return v2.FsInfo{}, fmt.Errorf("did not find fs info for dir: %v", dir)
return self.getFsInfoByDeviceName(device.Device)
}
func (self *manager) GetFsInfo(label string) ([]v2.FsInfo, error) {
@@ -726,6 +726,7 @@ func (self *manager) GetFsInfo(label string) ([]v2.FsInfo, error) {
}
fi := v2.FsInfo{
Timestamp: stats[0].Timestamp,
Device: fs.Device,
Mountpoint: mountpoint,
Capacity: fs.Limit,
@@ -1265,6 +1266,23 @@ func (m *manager) DebugInfo() map[string][]string {
return debugInfo
}
func (self *manager) getFsInfoByDeviceName(deviceName string) (v2.FsInfo, error) {
mountPoint, err := self.fsInfo.GetMountpointForDevice(deviceName)
if err != nil {
return v2.FsInfo{}, fmt.Errorf("failed to get mount point for device %q: %v", deviceName, err)
}
infos, err := self.GetFsInfo("")
if err != nil {
return v2.FsInfo{}, err
}
for _, info := range infos {
if info.Mountpoint == mountPoint {
return info, nil
}
}
return v2.FsInfo{}, fmt.Errorf("cannot find filesystem info for device %q", deviceName)
}
func getVersionInfo() (*info.VersionInfo, error) {
kernel_version := machine.KernelVersion()

View File

@@ -27,7 +27,6 @@ filegroup(
"//vendor/github.com/google/cadvisor/utils/oomparser:all-srcs",
"//vendor/github.com/google/cadvisor/utils/sysfs:all-srcs",
"//vendor/github.com/google/cadvisor/utils/sysinfo:all-srcs",
"//vendor/github.com/google/cadvisor/utils/tail:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],

View File

@@ -5,9 +5,8 @@ go_library(
srcs = ["oomparser.go"],
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/euank/go-kmsg-parser/kmsgparser:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/google/cadvisor/utils:go_default_library",
"//vendor/github.com/google/cadvisor/utils/tail:go_default_library",
],
)

View File

@@ -1,44 +0,0 @@
Jan 5 15:19:01 CRON[14500]: (root) CMD (touch /var/run/crond.sittercheck)
Jan 5 15:19:04 cookie_monster[1249]: uid 0, pid 14504, "/var/lib/certs/machine_cert.crt" accessed by exe "/usr/bin/nsscacheclient", cwd "/root", comm "/usr/bin/nsscacheclient"
Jan 5 15:19:04 cookie_monster[1249]: uid 0, pid 14504, "/var/lib/certs/machine_cert.key" accessed by exe "/usr/bin/nsscacheclient", cwd "/root", comm "/usr/bin/nsscacheclient"
Jan 5 15:19:05 nsscacheclient[14504]: SUCCESS: Completed run (v29/c20 rtime:0.334299 utime:0.136923 stime:0.011736 maxrss:5260k dials:1 sent:1793 rcvd:5143).
Jan 5 15:19:27 kernel: [ 5864.708440] memorymonster invoked oom-killer: gfp_mask=0xd0, order=0, oom_score_adj=0
Jan 5 15:19:27 kernel: [ 5864.708443] memorymonster cpuset=/ mems_allowed=0
Jan 5 15:19:27 kernel: [ 5864.708446] CPU: 5 PID: 13536 Comm: memorymonster Tainted: P OX 3.13.0-43-generic #72-Ubuntu
Jan 5 15:19:27 kernel: [ 5864.708447] Hardware name: Hewlett-Packard HP Z420 Workstation/1589, BIOS J61 v03.65 12/19/2013
Jan 5 15:19:27 kernel: [ 5864.708448] ffff88072ae10800 ffff8807a4835c48 ffffffff81720bf6 ffff8807a8e86000
Jan 5 15:19:27 kernel: [ 5864.708451] ffff8807a4835cd0 ffffffff8171b4b1 0000000000000246 ffff88072ae10800
Jan 5 15:19:27 kernel: [ 5864.708453] ffff8807a4835c90 ffff8807a4835ca0 ffffffff811522a7 0000000000000001
Jan 5 15:19:27 kernel: [ 5864.708455] Call Trace:
Jan 5 15:19:27 kernel: [ 5864.708460] [<ffffffff81720bf6>] dump_stack+0x45/0x56
Jan 5 15:19:27 kernel: [ 5864.708463] [<ffffffff8171b4b1>] dump_header+0x7f/0x1f1
Jan 5 15:19:27 kernel: [ 5864.708465] [<ffffffff811522a7>] ? find_lock_task_mm+0x27/0x70
Jan 5 15:19:27 kernel: [ 5864.708467] [<ffffffff811526de>] oom_kill_process+0x1ce/0x330
Jan 5 15:19:27 kernel: [ 5864.708470] [<ffffffff812d6ce5>] ? security_capable_noaudit+0x15/0x20
Jan 5 15:19:27 kernel: [ 5864.708474] [<ffffffff811b491c>] mem_cgroup_oom_synchronize+0x51c/0x560
Jan 5 15:19:27 kernel: [ 5864.708476] [<ffffffff811b3e50>] ? mem_cgroup_charge_common+0xa0/0xa0
Jan 5 15:19:27 kernel: [ 5864.708478] [<ffffffff81152e64>] pagefault_out_of_memory+0x14/0x80
Jan 5 15:19:27 kernel: [ 5864.708480] [<ffffffff81719aa1>] mm_fault_error+0x8e/0x180
Jan 5 15:19:27 kernel: [ 5864.708482] [<ffffffff8172cf31>] __do_page_fault+0x4a1/0x560
Jan 5 15:19:27 kernel: [ 5864.708485] [<ffffffff810a0255>] ? set_next_entity+0x95/0xb0
Jan 5 15:19:27 kernel: [ 5864.708489] [<ffffffff81012609>] ? __switch_to+0x169/0x4c0
Jan 5 15:19:27 kernel: [ 5864.708490] [<ffffffff8172d00a>] do_page_fault+0x1a/0x70
Jan 5 15:19:27 kernel: [ 5864.708492] [<ffffffff81729468>] page_fault+0x28/0x30
Jan 5 15:19:27 kernel: [ 5864.708493] Task in /mem2 killed as a result of limit of /mem2
Jan 5 15:19:27 kernel: [ 5864.708495] memory: usage 980kB, limit 980kB, failcnt 4152239
Jan 5 15:19:27 kernel: [ 5864.708495] memory+swap: usage 0kB, limit 18014398509481983kB, failcnt 0
Jan 5 15:19:27 kernel: [ 5864.708496] kmem: usage 0kB, limit 18014398509481983kB, failcnt 0
Jan 5 15:19:27 kernel: [ 5864.708497] Memory cgroup stats for /mem2: cache:0KB rss:980KB rss_huge:0KB mapped_file:0KB writeback:20KB inactive_anon:560KB active_anon:420KB inactive_file:0KB active_file:0KB unevictable:0KB
Jan 5 15:19:27 kernel: [ 5864.708505] [ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
Jan 5 15:19:27 kernel: [ 5864.708600] [13536] 275858 13536 8389663 343 16267 8324326 0 memorymonster
Jan 5 15:19:27 kernel: [ 5864.708607] Memory cgroup out of memory: Kill process 13536 (memorymonster) score 996 or sacrifice child
Jan 5 15:19:27 kernel: [ 5864.708608] Killed process 13536 (memorymonster) total-vm:33558652kB, anon-rss:920kB, file-rss:452kB
Jan 5 15:20:01 CRON[14608]: (root) CMD (touch /var/run/crond.sittercheck)
Jan 5 15:20:01 CRON[14609]: (root) CMD (/usr/bin/alarm 6000 /usr/share/update-notifier/reevaluate.py)
Jan 5 15:20:01 CRON[14610]: (root) CMD (/usr/bin/corp_cronwrap -j 80 -t 600 -A -K -L -l 'nsscache-client' /usr/bin/nsscacheclient all)
Jan 5 15:20:01 /usr/bin/lock: called by /bin/bash for . uid 0, euid 0.
Jan 5 15:21:01 CRON[14639]: (root) CMD (touch /var/run/crond.sittercheck)
Jan 5 15:21:05 cookie_monster[1249]: uid 0, pid 14643, "/var/lib/certs/machine_cert.crt" accessed by exe "/usr/bin/nsscacheclient", cwd "/root", comm "/usr/bin/nsscacheclient"
Jan 5 15:21:05 cookie_monster[1249]: uid 0, pid 14643, "/var/lib/certs/machine_cert.key" accessed by exe "/usr/bin/nsscacheclient", cwd "/root", comm "/usr/bin/nsscacheclient"
Jan 5 15:21:05 nsscacheclient[14643]: auto.auto(no change) time:0.042264697000000004 retries:0
Jan 5 15:21:05 nsscacheclient[14643]: auto.home(63c07d09->8686499b write:3631382) time:0.318774602 retries:0

View File

@@ -15,30 +15,26 @@
package oomparser
import (
"bufio"
"fmt"
"io"
"os/exec"
"path"
"regexp"
"strconv"
"time"
"github.com/google/cadvisor/utils"
"github.com/google/cadvisor/utils/tail"
"github.com/euank/go-kmsg-parser/kmsgparser"
"github.com/golang/glog"
)
var (
containerRegexp = regexp.MustCompile(`Task in (.*) killed as a result of limit of (.*)`)
lastLineRegexp = regexp.MustCompile(`(^[A-Z][a-z]{2} .*[0-9]{1,2} [0-9]{1,2}:[0-9]{2}:[0-9]{2}) .* Killed process ([0-9]+) \((.+)\)`)
lastLineRegexp = regexp.MustCompile(`Killed process ([0-9]+) \((.+)\)`)
firstLineRegexp = regexp.MustCompile(`invoked oom-killer:`)
)
// struct to hold file from which we obtain OomInstances
// OomParser wraps a kmsgparser in order to extract OOM events from the
// individual kernel ring buffer messages.
type OomParser struct {
ioreader *bufio.Reader
parser kmsgparser.Parser
}
// struct that contains information related to an OOM kill instance
@@ -75,20 +71,13 @@ func getProcessNamePid(line string, currentOomInstance *OomInstance) (bool, erro
if reList == nil {
return false, nil
}
const longForm = "Jan _2 15:04:05 2006"
stringYear := strconv.Itoa(time.Now().Year())
linetime, err := time.ParseInLocation(longForm, reList[1]+" "+stringYear, time.Local)
if err != nil {
return false, err
}
currentOomInstance.TimeOfDeath = linetime
pid, err := strconv.Atoi(reList[2])
pid, err := strconv.Atoi(reList[1])
if err != nil {
return false, err
}
currentOomInstance.Pid = pid
currentOomInstance.ProcessName = reList[3]
currentOomInstance.ProcessName = reList[2]
return true, nil
}
@@ -101,132 +90,61 @@ func checkIfStartOfOomMessages(line string) bool {
return false
}
// reads the file and sends only complete lines over a channel to analyzeLines.
// Should prevent EOF errors that occur when lines are read before being fully
// written to the log. It reads line by line splitting on
// the "\n" character.
func readLinesFromFile(lineChannel chan string, ioreader *bufio.Reader) error {
linefragment := ""
var line string
var err error
for true {
line, err = ioreader.ReadString('\n')
if err != nil && err != io.EOF {
glog.Errorf("exiting analyzeLinesHelper with error %v", err)
close(lineChannel)
break
}
if line == "" {
time.Sleep(100 * time.Millisecond)
continue
}
if err == nil {
lineChannel <- linefragment + line
linefragment = ""
} else { // err == io.EOF
linefragment += line
}
}
return err
}
// StreamOoms writes to a provided a stream of OomInstance objects representing
// OOM events that are found in the logs.
// It will block and should be called from a goroutine.
func (self *OomParser) StreamOoms(outStream chan<- *OomInstance) {
kmsgEntries := self.parser.Parse()
defer self.parser.Close()
// Calls goroutine for readLinesFromFile, which feeds it complete lines.
// Lines are checked against a regexp to check for the pid, process name, etc.
// At the end of an oom message group, StreamOoms adds the new oomInstance to
// oomLog
func (self *OomParser) StreamOoms(outStream chan *OomInstance) {
lineChannel := make(chan string, 10)
go func() {
readLinesFromFile(lineChannel, self.ioreader)
}()
for line := range lineChannel {
in_oom_kernel_log := checkIfStartOfOomMessages(line)
for msg := range kmsgEntries {
in_oom_kernel_log := checkIfStartOfOomMessages(msg.Message)
if in_oom_kernel_log {
oomCurrentInstance := &OomInstance{
ContainerName: "/",
TimeOfDeath: msg.Timestamp,
}
for line := range lineChannel {
err := getContainerName(line, oomCurrentInstance)
for msg := range kmsgEntries {
err := getContainerName(msg.Message, oomCurrentInstance)
if err != nil {
glog.Errorf("%v", err)
}
finished, err := getProcessNamePid(line, oomCurrentInstance)
finished, err := getProcessNamePid(msg.Message, oomCurrentInstance)
if err != nil {
glog.Errorf("%v", err)
}
if finished {
oomCurrentInstance.TimeOfDeath = msg.Timestamp
break
}
}
outStream <- oomCurrentInstance
}
}
glog.Infof("exiting analyzeLines. OOM events will not be reported.")
}
func callJournalctl() (io.ReadCloser, error) {
cmd := exec.Command("journalctl", "-k", "-f")
readcloser, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
if err := cmd.Start(); err != nil {
return nil, err
}
return readcloser, err
}
func trySystemd() (*OomParser, error) {
readcloser, err := callJournalctl()
if err != nil {
return nil, err
}
glog.Infof("oomparser using systemd")
return &OomParser{
ioreader: bufio.NewReader(readcloser),
}, nil
}
// List of possible kernel log files. These are prioritized in order so that
// we will use the first one that is available.
var kernelLogFiles = []string{"/var/log/kern.log", "/var/log/messages", "/var/log/syslog"}
// looks for system files that contain kernel messages and if one is found, sets
// the systemFile attribute of the OomParser object
func getLogFile() (string, error) {
for _, logFile := range kernelLogFiles {
if utils.FileExists(logFile) {
glog.Infof("OOM parser using kernel log file: %q", logFile)
return logFile, nil
}
}
return "", fmt.Errorf("unable to find any kernel log file available from our set: %v", kernelLogFiles)
}
func tryLogFile() (*OomParser, error) {
logFile, err := getLogFile()
if err != nil {
return nil, err
}
tail, err := tail.NewTail(logFile)
if err != nil {
return nil, err
}
return &OomParser{
ioreader: bufio.NewReader(tail),
}, nil
// Should not happen
glog.Errorf("exiting analyzeLines. OOM events will not be reported.")
}
// initializes an OomParser object. Returns an OomParser object and an error.
func New() (*OomParser, error) {
parser, err := trySystemd()
if err == nil {
return parser, nil
parser, err := kmsgparser.NewParser()
if err != nil {
return nil, err
}
parser, err = tryLogFile()
if err == nil {
return parser, nil
}
return nil, err
parser.SetLogger(glogAdapter{})
return &OomParser{parser: parser}, nil
}
type glogAdapter struct{}
var _ kmsgparser.Logger = glogAdapter{}
func (glogAdapter) Infof(format string, args ...interface{}) {
glog.V(4).Infof(format, args)
}
func (glogAdapter) Warningf(format string, args ...interface{}) {
glog.Infof(format, args)
}
func (glogAdapter) Errorf(format string, args ...interface{}) {
glog.Warningf(format, args)
}

View File

@@ -1,362 +0,0 @@
[ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1
[ 0.000000] Hierarchical RCU implementation.
[ 0.000000] RCU dyntick-idle grace-period acceleration is enabled.
[ 0.000000] RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=1.
[ 0.000000] Offload RCU callbacks from all CPUs
[ 0.000000] Offload RCU callbacks from CPUs: 0.
[ 0.000000] NR_IRQS:16640 nr_irqs:256 16
[ 0.000000] Console: colour dummy device 80x25
[ 0.000000] console [ttyS0] enabled
[ 0.000000] allocated 7340032 bytes of page_cgroup
[ 0.000000] please try 'cgroup_disable=memory' option if you don't want memory cgroups
[ 0.000000] tsc: Detected 2500.000 MHz processor
[ 0.008000] Calibrating delay loop (skipped) preset value.. 5000.00 BogoMIPS (lpj=10000000)
[ 0.008000] pid_max: default: 32768 minimum: 301
[ 0.008000] Security Framework initialized
[ 0.008000] AppArmor: AppArmor initialized
[ 0.008000] Yama: becoming mindful.
[ 0.008200] Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes)
[ 0.011365] Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes)
[ 0.013066] Mount-cache hash table entries: 4096 (order: 3, 32768 bytes)
[ 0.014030] Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes)
[ 0.016266] Initializing cgroup subsys memory
[ 0.016898] Initializing cgroup subsys devices
[ 0.017546] Initializing cgroup subsys freezer
[ 0.018193] Initializing cgroup subsys blkio
[ 0.018793] Initializing cgroup subsys perf_event
[ 0.019416] Initializing cgroup subsys hugetlb
[ 0.020067] Disabled fast string operations
[ 0.020681] CPU: Physical Processor ID: 0
[ 0.021238] CPU: Processor Core ID: 0
[ 0.022587] mce: CPU supports 32 MCE banks
[ 0.023260] Last level iTLB entries: 4KB 512, 2MB 0, 4MB 0
[ 0.023260] Last level dTLB entries: 4KB 512, 2MB 0, 4MB 0
[ 0.023260] tlb_flushall_shift: 6
[ 0.043758] Freeing SMP alternatives memory: 32K (ffffffff81e6c000 - ffffffff81e74000)
[ 0.048361] ACPI: Core revision 20131115
[ 0.049516] ACPI: All ACPI Tables successfully acquired
[ 0.050342] ftrace: allocating 28458 entries in 112 pages
[ 0.060327] Enabling x2apic
[ 0.060740] Enabled x2apic
[ 0.064005] Switched APIC routing to physical x2apic.
[ 0.065489] ..TIMER: vector=0x30 apic1=0 pin1=0 apic2=-1 pin2=-1
[ 0.066331] smpboot: CPU0: Intel(R) Xeon(R) CPU @ 2.50GHz (fam: 06, model: 3e, stepping: 04)
[ 0.072000] APIC calibration not consistent with PM-Timer: 227ms instead of 100ms
[ 0.072000] APIC delta adjusted to PM-Timer: 6250028 (14249259)
[ 0.074382] Performance Events: unsupported p6 CPU model 62 no PMU driver, software events only.
[ 0.077174] x86: Booted up 1 node, 1 CPUs
[ 0.077738] smpboot: Total of 1 processors activated (5000.00 BogoMIPS)
[ 0.078932] NMI watchdog: disabled (cpu0): hardware events not enabled
[ 0.079945] devtmpfs: initialized
[ 0.081784] EVM: security.selinux
[ 0.082251] EVM: security.SMACK64
[ 0.082720] EVM: security.ima
[ 0.083135] EVM: security.capability
[ 0.084729] pinctrl core: initialized pinctrl subsystem
[ 0.085517] regulator-dummy: no parameters
[ 0.086187] RTC time: 19:51:09, date: 01/28/15
[ 0.086869] NET: Registered protocol family 16
[ 0.087613] cpuidle: using governor ladder
[ 0.088009] cpuidle: using governor menu
[ 0.088580] ACPI: bus type PCI registered
[ 0.089191] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[ 0.090220] PCI: Using configuration type 1 for base access
[ 0.091749] bio: create slab <bio-0> at 0
[ 0.092215] ACPI: Added _OSI(Module Device)
[ 0.092799] ACPI: Added _OSI(Processor Device)
[ 0.093410] ACPI: Added _OSI(3.0 _SCP Extensions)
[ 0.094173] ACPI: Added _OSI(Processor Aggregator Device)
[ 0.096962] ACPI: Interpreter enabled
[ 0.097483] ACPI Exception: AE_NOT_FOUND, While evaluating Sleep State [\_S1_] (20131115/hwxface-580)
[ 0.098762] ACPI Exception: AE_NOT_FOUND, While evaluating Sleep State [\_S2_] (20131115/hwxface-580)
[ 0.100011] ACPI: (supports S0 S3 S4 S5)
[ 0.100555] ACPI: Using IOAPIC for interrupt routing
[ 0.101252] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[ 0.102545] ACPI: No dock devices found.
[ 0.105210] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[ 0.106060] acpi PNP0A03:00: _OSC: OS supports [ASPM ClockPM Segments MSI]
[ 0.108025] acpi PNP0A03:00: _OSC failed (AE_NOT_FOUND); disabling ASPM
[ 0.109116] acpi PNP0A03:00: fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.
[ 0.112685] PCI host bridge to bus 0000:00
[ 0.113294] pci_bus 0000:00: root bus resource [bus 00-ff]
[ 0.114054] pci_bus 0000:00: root bus resource [io 0x0000-0x0cf7]
[ 0.115065] pci_bus 0000:00: root bus resource [io 0x0d00-0xffff]
[ 0.116004] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff]
[ 0.116955] pci_bus 0000:00: root bus resource [mem 0x6cc00000-0xfebfffff]
[ 0.117916] pci 0000:00:01.0: [8086:7110] type 00 class 0x060100
[ 0.122089] pci 0000:00:01.3: [8086:7113] type 00 class 0x068000
[ 0.125713] pci 0000:00:01.3: quirk: [io 0xb000-0xb03f] claimed by PIIX4 ACPI
[ 0.127117] pci 0000:00:03.0: [1af4:1004] type 00 class 0x000000
[ 0.128752] pci 0000:00:03.0: reg 0x10: [io 0xc000-0xc03f]
[ 0.130322] pci 0000:00:03.0: reg 0x14: [mem 0xfebfe000-0xfebfe07f]
[ 0.133571] pci 0000:00:04.0: [1af4:1000] type 00 class 0x020000
[ 0.135267] pci 0000:00:04.0: reg 0x10: [io 0xc040-0xc07f]
[ 0.136777] pci 0000:00:04.0: reg 0x14: [mem 0xfebff000-0xfebff03f]
[ 0.140811] ACPI: PCI Interrupt Link [LNKA] (IRQs 5 *10 11)
[ 0.141879] ACPI: PCI Interrupt Link [LNKB] (IRQs 5 *10 11)
[ 0.142886] ACPI: PCI Interrupt Link [LNKC] (IRQs 5 10 *11)
[ 0.144086] ACPI: PCI Interrupt Link [LNKD] (IRQs 5 10 *11)
[ 0.145067] ACPI: PCI Interrupt Link [LNKS] (IRQs *9)
[ 0.146245] ACPI: Enabled 16 GPEs in block 00 to 0F
[ 0.147038] ACPI: \_SB_.PCI0: notify handler is installed
[ 0.147840] Found 1 acpi root devices
[ 0.148136] vgaarb: loaded
[ 0.148780] SCSI subsystem initialized
[ 0.149472] libata version 3.00 loaded.
[ 0.150070] ACPI: bus type USB registered
[ 0.150659] usbcore: registered new interface driver usbfs
[ 0.151536] usbcore: registered new interface driver hub
[ 0.152055] usbcore: registered new device driver usb
[ 0.153144] PCI: Using ACPI for IRQ routing
[ 0.153756] PCI: pci_cache_line_size set to 64 bytes
[ 0.154617] e820: reserve RAM buffer [mem 0x0009fc00-0x0009ffff]
[ 0.156004] e820: reserve RAM buffer [mem 0x6cbfe000-0x6fffffff]
[ 0.156993] NetLabel: Initializing
[ 0.157498] NetLabel: domain hash size = 128
[ 0.158082] NetLabel: protocols = UNLABELED CIPSOv4
[ 0.158815] NetLabel: unlabeled traffic allowed by default
[ 0.160005] Switched to clocksource kvm-clock
[ 0.168695] AppArmor: AppArmor Filesystem Enabled
[ 0.169361] pnp: PnP ACPI init
[ 0.169853] ACPI: bus type PNP registered
[ 0.170499] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[ 0.171591] pnp 00:01: Plug and Play ACPI device, IDs PNP0501 (active)
[ 0.172574] pnp 00:02: Plug and Play ACPI device, IDs PNP0501 (active)
[ 0.173782] pnp: PnP ACPI: found 3 devices
[ 0.174430] ACPI: bus type PNP unregistered
[ 0.181364] pci_bus 0000:00: resource 4 [io 0x0000-0x0cf7]
[ 0.182172] pci_bus 0000:00: resource 5 [io 0x0d00-0xffff]
[ 0.183049] pci_bus 0000:00: resource 6 [mem 0x000a0000-0x000bffff]
[ 0.184120] pci_bus 0000:00: resource 7 [mem 0x6cc00000-0xfebfffff]
[ 0.185051] NET: Registered protocol family 2
[ 0.185859] TCP established hash table entries: 16384 (order: 5, 131072 bytes)
[ 0.187117] TCP bind hash table entries: 16384 (order: 6, 262144 bytes)
[ 0.188393] TCP: Hash tables configured (established 16384 bind 16384)
[ 0.189429] TCP: reno registered
[ 0.189929] UDP hash table entries: 1024 (order: 3, 32768 bytes)
[ 0.190824] UDP-Lite hash table entries: 1024 (order: 3, 32768 bytes)
[ 0.191830] NET: Registered protocol family 1
[ 0.192585] PCI: CLS 0 bytes, default 64
[ 0.193412] Trying to unpack rootfs image as initramfs...
[ 0.897565] Freeing initrd memory: 18780K (ffff880035b42000 - ffff880036d99000)
[ 0.898982] microcode: CPU0 sig=0x306e4, pf=0x1, revision=0x1
[ 0.899884] microcode: Microcode Update Driver: v2.00 <tigran@aivazian.fsnet.co.uk>, Peter Oruba
[ 0.901196] Scanning for low memory corruption every 60 seconds
[ 0.902497] Initialise system trusted keyring
[ 0.903169] audit: initializing netlink socket (disabled)
[ 0.904016] type=2000 audit(1422474669.702:1): initialized
[ 0.926617] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[ 0.928567] zbud: loaded
[ 0.929030] VFS: Disk quotas dquot_6.5.2
[ 0.929685] Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[ 0.931113] fuse init (API version 7.22)
[ 0.931781] msgmni has been set to 3390
[ 0.932595] Key type big_key registered
[ 0.933680] Key type asymmetric registered
[ 0.934332] Asymmetric key parser 'x509' registered
[ 0.935078] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 252)
[ 0.936224] io scheduler noop registered
[ 0.936858] io scheduler deadline registered (default)
[ 0.937675] io scheduler cfq registered
[ 0.938307] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[ 0.939158] pciehp: PCI Express Hot Plug Controller Driver version: 0.4
[ 0.940239] efifb: probing for efifb
[ 0.940788] efifb: framebuffer at 0xa0000, mapped to 0xffff8800000a0000, using 64k, total 64k
[ 0.942044] efifb: mode is 640x480x1, linelength=80, pages=1
[ 0.942964] efifb: scrolling: redraw
[ 0.943525] efifb: Truecolor: size=8:8:8:8, shift=24:16:8:0
[ 0.945209] Console: switching to colour frame buffer device 80x30
[ 0.946826] fb0: EFI VGA frame buffer device
[ 0.947485] intel_idle: does not run on family 6 model 62
[ 0.948380] ipmi message handler version 39.2
[ 0.949036] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input0
[ 0.950135] ACPI: Power Button [PWRF]
[ 0.950722] input: Sleep Button as /devices/LNXSYSTM:00/LNXSLPBN:00/input/input1
[ 0.951773] ACPI: Sleep Button [SLPF]
[ 0.952529] GHES: HEST is not enabled!
[ 0.953921] ACPI: PCI Interrupt Link [LNKC] enabled at IRQ 11
[ 0.955783] ACPI: PCI Interrupt Link [LNKD] enabled at IRQ 10
[ 0.957395] Serial: 8250/16550 driver, 32 ports, IRQ sharing enabled
[ 1.112167] 00:01: ttyS0 at I/O 0x3f8 (irq = 4, base_baud = 115200) is a 16550A
[ 1.134843] 00:02: ttyS1 at I/O 0x2f8 (irq = 3, base_baud = 115200) is a 16550A
[ 1.137110] Linux agpgart interface v0.103
[ 1.138975] brd: module loaded
[ 1.140117] loop: module loaded
[ 1.140923] libphy: Fixed MDIO Bus: probed
[ 1.141640] tun: Universal TUN/TAP device driver, 1.6
[ 1.142342] tun: (C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>
[ 1.144063] virtio-pci 0000:00:04.0: irq 40 for MSI/MSI-X
[ 1.144871] virtio-pci 0000:00:04.0: irq 41 for MSI/MSI-X
[ 1.145670] virtio-pci 0000:00:04.0: irq 42 for MSI/MSI-X
[ 1.151673] PPP generic driver version 2.4.2
[ 1.152344] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver
[ 1.153399] ehci-pci: EHCI PCI platform driver
[ 1.154021] ehci-platform: EHCI generic platform driver
[ 1.154939] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver
[ 1.155973] ohci-pci: OHCI PCI platform driver
[ 1.156675] ohci-platform: OHCI generic platform driver
[ 1.157423] uhci_hcd: USB Universal Host Controller Interface driver
[ 1.158352] i8042: PNP: No PS/2 controller found. Probing ports directly.
[ 3.646820] i8042: No controller found
[ 3.647493] tsc: Refined TSC clocksource calibration: 2500.002 MHz
[ 3.648490] mousedev: PS/2 mouse device common for all mice
[ 3.649499] rtc_cmos 00:00: RTC can wake from S4
[ 3.650595] rtc_cmos 00:00: rtc core: registered rtc_cmos as rtc0
[ 3.651521] rtc_cmos 00:00: alarms up to one day, 114 bytes nvram
[ 3.652422] device-mapper: uevent: version 1.0.3
[ 3.653131] device-mapper: ioctl: 4.27.0-ioctl (2013-10-30) initialised: dm-devel@redhat.com
[ 3.654281] ledtrig-cpu: registered to indicate activity on CPUs
[ 3.655182] TCP: cubic registered
[ 3.655704] NET: Registered protocol family 10
[ 3.656551] NET: Registered protocol family 17
[ 3.657183] Key type dns_resolver registered
[ 3.657931] Loading compiled-in X.509 certificates
[ 3.659264] Loaded X.509 cert 'Magrathea: Glacier signing key: 23984ac203784325ccf7b95b51f6c119380eb933'
[ 3.660726] registered taskstats version 1
[ 3.663211] Key type trusted registered
[ 3.665462] Key type encrypted registered
[ 3.667679] AppArmor: AppArmor sha1 policy hashing enabled
[ 3.668454] IMA: No TPM chip found, activating TPM-bypass!
[ 3.669388] regulator-dummy: disabling
[ 3.669971] Magic number: 15:428:901
[ 3.670625] clocksource clocksource0: hash matches
[ 3.671311] acpi PNP0501:01: hash matches
[ 3.671953] rtc_cmos 00:00: setting system clock to 2015-01-28 19:51:13 UTC (1422474673)
[ 3.673268] BIOS EDD facility v0.16 2004-Jun-25, 0 devices found
[ 3.674088] EDD information not available.
[ 3.674668] PM: Hibernation image not present or could not be loaded.
[ 3.676577] Freeing unused kernel memory: 1332K (ffffffff81d1f000 - ffffffff81e6c000)
[ 3.678370] Write protecting the kernel read-only data: 12288k
[ 3.681251] Freeing unused kernel memory: 828K (ffff880001731000 - ffff880001800000)
[ 3.684444] Freeing unused kernel memory: 700K (ffff880001b51000 - ffff880001c00000)
[ 3.700162] systemd-udevd[90]: starting version 204
[ 3.866262] virtio-pci 0000:00:03.0: irq 43 for MSI/MSI-X
[ 3.867187] virtio-pci 0000:00:03.0: irq 44 for MSI/MSI-X
[ 3.867997] virtio-pci 0000:00:03.0: irq 45 for MSI/MSI-X
[ 3.876214] virtio-pci 0000:00:03.0: irq 46 for MSI/MSI-X
[ 3.880005] scsi0 : Virtio SCSI HBA
[ 3.912410] scsi 0:0:1:0: Direct-Access Google PersistentDisk 1 PQ: 0 ANSI: 6
[ 3.938957] sd 0:0:1:0: Attached scsi generic sg0 type 0
[ 3.939845] sd 0:0:1:0: [sda] 20971520 512-byte logical blocks: (10.7 GB/10.0 GiB)
[ 3.941149] sd 0:0:1:0: [sda] 4096-byte physical blocks
[ 3.942233] sd 0:0:1:0: [sda] Write Protect is off
[ 3.942988] sd 0:0:1:0: [sda] Mode Sense: 1f 00 00 08
[ 3.944398] sd 0:0:1:0: [sda] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
[ 3.961885] sda: sda1
[ 3.963152] sd 0:0:1:0: [sda] Attached SCSI disk
[ 4.414649] EXT4-fs (sda1): mounted filesystem with ordered data mode. Opts: (null)
[ 5.293574] random: init urandom read with 73 bits of entropy available
[ 6.418187] random: nonblocking pool is initialized
[ 6.692508] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro
[ 7.121847] IPv6: ADDRCONF(NETDEV_UP): eth0: link is not ready
[ 7.681714] systemd-udevd[293]: starting version 204
[ 8.437234] lp: driver loaded but no devices found
[ 9.164195] piix4_smbus 0000:00:01.3: SMBus base address uninitialized - upgrade BIOS or use force_addr=0xaddr
[ 9.648096] device-mapper: multipath: version 1.6.0 loaded
[ 10.434575] type=1400 audit(1422474680.256:2): apparmor="STATUS" operation="profile_load" profile="unconfined" name="/sbin/dhclient" pid=368 comm="apparmor_parser"
[ 10.437242] type=1400 audit(1422474680.260:3): apparmor="STATUS" operation="profile_load" profile="unconfined" name="/usr/lib/NetworkManager/nm-dhcp-client.action" pid=368 comm="apparmor_parser"
[ 10.439901] type=1400 audit(1422474680.260:4): apparmor="STATUS" operation="profile_load" profile="unconfined" name="/usr/lib/connman/scripts/dhclient-script" pid=368 comm="apparmor_parser"
[ 11.126295] type=1400 audit(1422474680.948:5): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/sbin/dhclient" pid=412 comm="apparmor_parser"
[ 11.129123] type=1400 audit(1422474680.952:6): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/NetworkManager/nm-dhcp-client.action" pid=412 comm="apparmor_parser"
[ 11.132139] type=1400 audit(1422474680.956:7): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/connman/scripts/dhclient-script" pid=412 comm="apparmor_parser"
[ 11.196173] type=1400 audit(1422474681.020:8): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/sbin/dhclient" pid=458 comm="apparmor_parser"
[ 11.198887] type=1400 audit(1422474681.020:9): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/NetworkManager/nm-dhcp-client.action" pid=458 comm="apparmor_parser"
[ 11.201484] type=1400 audit(1422474681.028:10): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/connman/scripts/dhclient-script" pid=458 comm="apparmor_parser"
[ 11.361371] init: udev-fallback-graphics main process (454) terminated with status 1
[ 11.378437] type=1400 audit(1422474681.200:11): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/NetworkManager/nm-dhcp-client.action" pid=458 comm="apparmor_parser"
[ 14.366411] init: failsafe main process (491) killed by TERM signal
kateknister@kateknister-test3:~$ tail -f /var/log/syslog
Jan 28 19:51:47 localhost ntpdate[1240]: adjust time server 169.254.169.254 offset -0.383723 sec
Jan 28 19:51:47 localhost ntpd[1312]: ntpd 4.2.6p5@1.2349-o Wed Oct 9 19:08:06 UTC 2013 (1)
Jan 28 19:51:47 localhost ntpd[1313]: proto: precision = 0.449 usec
Jan 28 19:51:47 localhost ntpd[1313]: ntp_io: estimated max descriptors: 1024, initial socket boundary: 16
Jan 28 19:51:47 localhost ntpd[1313]: Listen and drop on 0 v4wildcard 0.0.0.0 UDP 123
Jan 28 19:51:47 localhost ntpd[1313]: Listen and drop on 1 v6wildcard :: UDP 123
Jan 28 19:51:47 localhost ntpd[1313]: Listen normally on 2 lo 127.0.0.1 UDP 123
Jan 28 19:51:47 localhost ntpd[1313]: Listen normally on 3 eth0 10.240.192.196 UDP 123
Jan 28 19:51:47 localhost ntpd[1313]: peers refreshed
Jan 28 19:51:47 localhost ntpd[1313]: Listening on routing socket on fd #20 for interface updates
Jan 28 19:58:45 localhost kernel: [ 455.498827] badsysprogram invoked oom-killer: gfp_mask=0x280da, order=0, oom_score_adj=0
Jan 28 19:58:45 localhost kernel: [ 455.500173] badsysprogram cpuset=/ mems_allowed=0
Jan 28 19:58:45 localhost kernel: [ 455.501007] CPU: 0 PID: 1532 Comm: badsysprogram Not tainted 3.13.0-27-generic #50-Ubuntu
Jan 28 19:58:45 localhost kernel: [ 455.502301] Hardware name: Google Google, BIOS Google 01/01/2011
Jan 28 19:58:45 localhost kernel: [ 455.503298] 0000000000000000 ffff880069715a90 ffffffff817199c4 ffff8800680d8000
Jan 28 19:58:45 localhost kernel: [ 455.504563] ffff880069715b18 ffffffff817142ff 0000000000000000 0000000000000000
Jan 28 19:58:45 localhost kernel: [ 455.505779] 0000000000000000 0000000000000000 0000000000000000 0000000000000000
Jan 28 19:58:45 localhost kernel: [ 455.506971] Call Trace:
Jan 28 19:58:45 localhost kernel: [ 455.507353] [<ffffffff817199c4>] dump_stack+0x45/0x56
Jan 28 19:58:45 localhost kernel: [ 455.508289] [<ffffffff817142ff>] dump_header+0x7f/0x1f1
Jan 28 19:58:45 localhost kernel: [ 455.509112] [<ffffffff8115196e>] oom_kill_process+0x1ce/0x330
Jan 28 19:58:45 localhost kernel: [ 455.510023] [<ffffffff812d3395>] ? security_capable_noaudit+0x15/0x20
Jan 28 19:58:45 localhost kernel: [ 455.510994] [<ffffffff811520a4>] out_of_memory+0x414/0x450
Jan 28 19:58:45 localhost kernel: [ 455.511820] [<ffffffff81158377>] __alloc_pages_nodemask+0xa87/0xb20
Jan 28 19:58:45 localhost kernel: [ 455.512815] [<ffffffff811985da>] alloc_pages_vma+0x9a/0x140
Jan 28 19:58:45 localhost kernel: [ 455.513647] [<ffffffff8117909b>] handle_mm_fault+0xb2b/0xf10
Jan 28 19:58:45 localhost kernel: [ 455.514498] [<ffffffff81725924>] __do_page_fault+0x184/0x560
Jan 28 19:58:45 localhost kernel: [ 455.515415] [<ffffffff8101b7d9>] ? sched_clock+0x9/0x10
Jan 28 19:58:45 localhost kernel: [ 455.516318] [<ffffffff8109d13d>] ? sched_clock_local+0x1d/0x80
Jan 28 19:58:45 localhost kernel: [ 455.517242] [<ffffffff811112ec>] ? acct_account_cputime+0x1c/0x20
Jan 28 19:58:45 localhost kernel: [ 455.518141] [<ffffffff8109d76b>] ? account_user_time+0x8b/0xa0
Jan 28 19:58:45 localhost kernel: [ 455.519014] [<ffffffff8109dd84>] ? vtime_account_user+0x54/0x60
Jan 28 19:58:45 localhost kernel: [ 455.519910] [<ffffffff81725d1a>] do_page_fault+0x1a/0x70
Jan 28 19:58:45 localhost kernel: [ 455.520712] [<ffffffff81722188>] page_fault+0x28/0x30
Jan 28 19:58:45 localhost kernel: [ 455.521498] Mem-Info:
Jan 28 19:58:45 localhost kernel: [ 455.521873] Node 0 DMA per-cpu:
Jan 28 19:58:45 localhost kernel: [ 455.522388] CPU 0: hi: 0, btch: 1 usd: 0
Jan 28 19:58:45 localhost kernel: [ 455.598342] Node 0 DMA32 per-cpu:
Jan 28 19:58:45 localhost kernel: [ 455.598890] CPU 0: hi: 186, btch: 31 usd: 86
Jan 28 19:58:45 localhost kernel: [ 455.599687] active_anon:405991 inactive_anon:57 isolated_anon:0
Jan 28 19:58:45 localhost kernel: [ 455.599687] active_file:35 inactive_file:69 isolated_file:0
Jan 28 19:58:45 localhost kernel: [ 455.599687] unevictable:0 dirty:0 writeback:0 unstable:0
Jan 28 19:58:45 localhost kernel: [ 455.599687] free:12929 slab_reclaimable:1635 slab_unreclaimable:1919
Jan 28 19:58:45 localhost kernel: [ 455.599687] mapped:34 shmem:70 pagetables:1423 bounce:0
Jan 28 19:58:45 localhost kernel: [ 455.599687] free_cma:0
Jan 28 19:58:45 localhost kernel: [ 455.604585] Node 0 DMA free:7124kB min:412kB low:512kB high:616kB active_anon:8508kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:4kB slab_reclaimable:16kB slab_unreclaimable:16kB kernel_stack:0kB pagetables:12kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
Jan 28 19:58:45 localhost kernel: [ 455.610811] lowmem_reserve[]: 0 1679 1679 1679
Jan 28 19:58:45 localhost kernel: [ 455.611600] Node 0 DMA32 free:44592kB min:44640kB low:55800kB high:66960kB active_anon:1615456kB inactive_anon:224kB active_file:140kB inactive_file:276kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:1765368kB managed:1722912kB mlocked:0kB dirty:0kB writeback:0kB mapped:136kB shmem:276kB slab_reclaimable:6524kB slab_unreclaimable:7660kB kernel_stack:592kB pagetables:5680kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:819 all_unreclaimable? yes
Jan 28 19:58:45 localhost kernel: [ 455.618372] lowmem_reserve[]: 0 0 0 0
Jan 28 19:58:45 localhost kernel: [ 455.619041] Node 0 DMA: 5*4kB (UM) 6*8kB (UEM) 7*16kB (UEM) 1*32kB (M) 2*64kB (UE) 3*128kB (UEM) 1*256kB (E) 2*512kB (EM) 3*1024kB (UEM) 1*2048kB (R) 0*4096kB = 7124kB
Jan 28 19:58:45 localhost kernel: [ 455.621861] Node 0 DMA32: 74*4kB (UEM) 125*8kB (UEM) 78*16kB (UEM) 26*32kB (UE) 12*64kB (UEM) 4*128kB (UE) 4*256kB (UE) 2*512kB (E) 11*1024kB (UE) 7*2048kB (UE) 3*4096kB (UR) = 44592kB
Jan 28 19:58:45 localhost kernel: [ 455.625174] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
Jan 28 19:58:45 localhost kernel: [ 455.626394] 204 total pagecache pages
Jan 28 19:58:45 localhost kernel: [ 455.626954] 0 pages in swap cache
Jan 28 19:58:45 localhost kernel: [ 455.627455] Swap cache stats: add 0, delete 0, find 0/0
Jan 28 19:58:45 localhost kernel: [ 455.628242] Free swap = 0kB
Jan 28 19:58:45 localhost kernel: [ 455.628686] Total swap = 0kB
Jan 28 19:58:45 localhost kernel: [ 455.629147] 445340 pages RAM
Jan 28 19:58:45 localhost kernel: [ 455.629577] 0 pages HighMem/MovableOnly
Jan 28 19:58:45 localhost kernel: [ 455.630301] 10614 pages reserved
Jan 28 19:58:45 localhost kernel: [ 455.630787] [ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
Jan 28 19:58:45 localhost kernel: [ 455.631937] [ 273] 0 273 4869 50 13 0 0 upstart-udev-br
Jan 28 19:58:45 localhost kernel: [ 455.633290] [ 293] 0 293 12802 154 28 0 -1000 systemd-udevd
Jan 28 19:58:45 localhost kernel: [ 455.634671] [ 321] 0 321 3819 54 12 0 0 upstart-file-br
Jan 28 19:58:45 localhost kernel: [ 455.636070] [ 326] 102 326 9805 109 24 0 0 dbus-daemon
Jan 28 19:58:45 localhost kernel: [ 455.637373] [ 334] 101 334 63960 94 26 0 0 rsyslogd
Jan 28 19:58:45 localhost kernel: [ 455.638761] [ 343] 0 343 10863 102 26 0 0 systemd-logind
Jan 28 19:58:45 localhost kernel: [ 455.640158] [ 546] 0 546 3815 60 13 0 0 upstart-socket-
Jan 28 19:58:45 localhost kernel: [ 455.641534] [ 710] 0 710 2556 587 8 0 0 dhclient
Jan 28 19:58:45 localhost kernel: [ 455.642834] [ 863] 0 863 3955 48 13 0 0 getty
Jan 28 19:58:45 localhost kernel: [ 455.644139] [ 865] 0 865 3955 50 13 0 0 getty
Jan 28 19:58:45 localhost kernel: [ 455.645325] [ 867] 0 867 3955 51 13 0 0 getty
Jan 28 19:58:45 localhost kernel: [ 455.646621] [ 868] 0 868 3955 51 12 0 0 getty
Jan 28 19:58:45 localhost kernel: [ 455.647963] [ 870] 0 870 3955 49 13 0 0 getty
Jan 28 19:58:45 localhost kernel: [ 455.649234] [ 915] 0 915 5914 61 16 0 0 cron
Jan 28 19:58:45 localhost kernel: [ 455.650439] [ 1015] 0 1015 10885 1524 25 0 0 manage_addresse
Jan 28 19:58:45 localhost kernel: [ 455.651817] [ 1028] 0 1028 3955 49 13 0 0 getty
Jan 28 19:58:45 localhost kernel: [ 455.653091] [ 1033] 0 1033 3197 48 12 0 0 getty
Jan 28 19:58:45 localhost kernel: [ 455.654783] [ 1264] 0 1264 11031 1635 26 0 0 manage_accounts
Jan 28 19:58:45 localhost kernel: [ 455.656657] [ 1268] 0 1268 15341 180 33 0 -1000 sshd
Jan 28 19:58:45 localhost kernel: [ 455.657865] [ 1313] 104 1313 6804 154 17 0 0 ntpd
Jan 28 19:58:45 localhost kernel: [ 455.659085] [ 1389] 0 1389 25889 255 55 0 0 sshd
Jan 28 19:58:45 localhost kernel: [ 455.660440] [ 1407] 1020 1407 25889 255 52 0 0 sshd
Jan 28 19:58:45 localhost kernel: [ 455.661595] [ 1408] 1020 1408 5711 581 17 0 0 bash
Jan 28 19:58:45 localhost kernel: [ 455.662887] [ 1425] 0 1425 25889 256 53 0 0 sshd
Jan 28 19:58:45 localhost kernel: [ 455.664075] [ 1443] 1020 1443 25889 257 52 0 0 sshd
Jan 28 19:58:45 localhost kernel: [ 455.665330] [ 1444] 1020 1444 5711 581 16 0 0 bash
Jan 28 19:58:45 localhost kernel: [ 455.666450] [ 1476] 1020 1476 1809 25 9 0 0 tail
Jan 28 19:58:45 localhost kernel: [ 455.667682] [ 1532] 1020 1532 410347 398810 788 0 0 badsysprogram
Jan 28 19:58:45 localhost kernel: [ 455.669006] Out of memory: Kill process 1532 (badsysprogram) score 919 or sacrifice child
Jan 28 19:58:45 localhost kernel: [ 455.670291] Killed process 1532 (badsysprogram) total-vm:1641388kB, anon-rss:1595164kB, file-rss:76kB
[ 0.170499] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[ 0.171591] pnp 00:01: Plug and Play ACPI device, IDs PNP0501 (active)
[ 0.172574] pnp 00:02: Plug and Play ACPI device, IDs PNP0501 (active)

View File

@@ -1,25 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["tail.go"],
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/golang.org/x/exp/inotify:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -1,162 +0,0 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package tail implements "tail -F" functionality following rotated logs
package tail
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"time"
"github.com/golang/glog"
"golang.org/x/exp/inotify"
)
type Tail struct {
reader *bufio.Reader
readerErr error
readerLock sync.RWMutex
filename string
file *os.File
stop chan bool
watcher *inotify.Watcher
}
const (
defaultRetryInterval = 100 * time.Millisecond
maxRetryInterval = 30 * time.Second
)
// NewTail starts opens the given file and watches it for deletion/rotation
func NewTail(filename string) (*Tail, error) {
t, err := newTail(filename)
if err != nil {
return nil, err
}
go t.watchLoop()
return t, nil
}
// newTail creates a Tail object.
func newTail(filename string) (*Tail, error) {
t := &Tail{
filename: filename,
}
var err error
t.stop = make(chan bool)
t.watcher, err = inotify.NewWatcher()
if err != nil {
return nil, fmt.Errorf("inotify init failed on %s: %v", t.filename, err)
}
// Initialize readerErr as io.EOF, so that the reader can work properly
// during initialization.
t.readerErr = io.EOF
return t, nil
}
// Read implements the io.Reader interface for Tail
func (t *Tail) Read(p []byte) (int, error) {
t.readerLock.RLock()
defer t.readerLock.RUnlock()
if t.readerErr != nil {
return 0, t.readerErr
}
return t.reader.Read(p)
}
var _ io.ReadCloser = &Tail{}
// Close stops watching and closes the file
func (t *Tail) Close() error {
close(t.stop)
return nil
}
func (t *Tail) attemptOpen() error {
t.readerLock.Lock()
defer t.readerLock.Unlock()
t.readerErr = nil
attempt := 0
var lastErr error
for interval := defaultRetryInterval; ; interval *= 2 {
attempt++
glog.V(4).Infof("Opening %s (attempt %d)", t.filename, attempt)
var err error
t.file, err = os.Open(t.filename)
if err == nil {
// TODO: not interested in old events?
// t.file.Seek(0, os.SEEK_END)
t.reader = bufio.NewReader(t.file)
return nil
}
lastErr = err
glog.V(4).Infof("open log file %s error: %v", t.filename, err)
if interval >= maxRetryInterval {
break
}
select {
case <-time.After(interval):
case <-t.stop:
t.readerErr = io.EOF
return fmt.Errorf("watch was cancelled")
}
}
err := fmt.Errorf("can't open log file %s: %v", t.filename, lastErr)
t.readerErr = err
return err
}
func (t *Tail) watchLoop() {
for {
err := t.watchFile()
if err != nil {
glog.Errorf("Tail failed on %s: %v", t.filename, err)
break
}
}
}
func (t *Tail) watchFile() error {
err := t.attemptOpen()
if err != nil {
return err
}
defer t.file.Close()
watchDir := filepath.Dir(t.filename)
err = t.watcher.AddWatch(watchDir, inotify.IN_MOVED_FROM|inotify.IN_DELETE)
if err != nil {
return fmt.Errorf("Failed to add watch to directory %s: %v", watchDir, err)
}
defer t.watcher.RemoveWatch(watchDir)
for {
select {
case event := <-t.watcher.Event:
eventPath := filepath.Clean(event.Name) // Directory events have an extra '/'
if eventPath == t.filename {
glog.V(4).Infof("Log file %s moved/deleted", t.filename)
return nil
}
case <-t.stop:
return fmt.Errorf("watch was cancelled")
}
}
}