233 lines
6.3 KiB
Go
233 lines
6.3 KiB
Go
// Copyright 2014 Google Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package oomparser
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"io"
|
|
"os/exec"
|
|
"path"
|
|
"regexp"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/google/cadvisor/utils"
|
|
"github.com/google/cadvisor/utils/tail"
|
|
|
|
"github.com/golang/glog"
|
|
)
|
|
|
|
var (
|
|
containerRegexp = regexp.MustCompile(`Task in (.*) killed as a result of limit of (.*)`)
|
|
lastLineRegexp = regexp.MustCompile(`(^[A-Z][a-z]{2} .*[0-9]{1,2} [0-9]{1,2}:[0-9]{2}:[0-9]{2}) .* Killed process ([0-9]+) \(([\w]+)\)`)
|
|
firstLineRegexp = regexp.MustCompile(`invoked oom-killer:`)
|
|
)
|
|
|
|
// struct to hold file from which we obtain OomInstances
|
|
type OomParser struct {
|
|
ioreader *bufio.Reader
|
|
}
|
|
|
|
// struct that contains information related to an OOM kill instance
|
|
type OomInstance struct {
|
|
// process id of the killed process
|
|
Pid int
|
|
// the name of the killed process
|
|
ProcessName string
|
|
// the time that the process was reported to be killed,
|
|
// accurate to the minute
|
|
TimeOfDeath time.Time
|
|
// the absolute name of the container that OOMed
|
|
ContainerName string
|
|
// the absolute name of the container that was killed
|
|
// due to the OOM.
|
|
VictimContainerName string
|
|
}
|
|
|
|
// gets the container name from a line and adds it to the oomInstance.
|
|
func getContainerName(line string, currentOomInstance *OomInstance) error {
|
|
parsedLine := containerRegexp.FindStringSubmatch(line)
|
|
if parsedLine == nil {
|
|
return nil
|
|
}
|
|
currentOomInstance.ContainerName = path.Join("/", parsedLine[1])
|
|
currentOomInstance.VictimContainerName = path.Join("/", parsedLine[2])
|
|
return nil
|
|
}
|
|
|
|
// gets the pid, name, and date from a line and adds it to oomInstance
|
|
func getProcessNamePid(line string, currentOomInstance *OomInstance) (bool, error) {
|
|
reList := lastLineRegexp.FindStringSubmatch(line)
|
|
|
|
if reList == nil {
|
|
return false, nil
|
|
}
|
|
const longForm = "Jan _2 15:04:05 2006"
|
|
stringYear := strconv.Itoa(time.Now().Year())
|
|
linetime, err := time.ParseInLocation(longForm, reList[1]+" "+stringYear, time.Local)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
currentOomInstance.TimeOfDeath = linetime
|
|
pid, err := strconv.Atoi(reList[2])
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
currentOomInstance.Pid = pid
|
|
currentOomInstance.ProcessName = reList[3]
|
|
return true, nil
|
|
}
|
|
|
|
// uses regex to see if line is the start of a kernel oom log
|
|
func checkIfStartOfOomMessages(line string) bool {
|
|
potential_oom_start := firstLineRegexp.MatchString(line)
|
|
if potential_oom_start {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// reads the file and sends only complete lines over a channel to analyzeLines.
|
|
// Should prevent EOF errors that occur when lines are read before being fully
|
|
// written to the log. It reads line by line splitting on
|
|
// the "\n" character.
|
|
func readLinesFromFile(lineChannel chan string, ioreader *bufio.Reader) error {
|
|
linefragment := ""
|
|
var line string
|
|
var err error
|
|
for true {
|
|
line, err = ioreader.ReadString('\n')
|
|
if err != nil && err != io.EOF {
|
|
glog.Errorf("exiting analyzeLinesHelper with error %v", err)
|
|
close(lineChannel)
|
|
break
|
|
}
|
|
if line == "" {
|
|
time.Sleep(100 * time.Millisecond)
|
|
continue
|
|
}
|
|
if err == nil {
|
|
lineChannel <- linefragment + line
|
|
linefragment = ""
|
|
} else { // err == io.EOF
|
|
linefragment += line
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Calls goroutine for readLinesFromFile, which feeds it complete lines.
|
|
// Lines are checked against a regexp to check for the pid, process name, etc.
|
|
// At the end of an oom message group, StreamOoms adds the new oomInstance to
|
|
// oomLog
|
|
func (self *OomParser) StreamOoms(outStream chan *OomInstance) {
|
|
lineChannel := make(chan string, 10)
|
|
go func() {
|
|
readLinesFromFile(lineChannel, self.ioreader)
|
|
}()
|
|
|
|
for line := range lineChannel {
|
|
in_oom_kernel_log := checkIfStartOfOomMessages(line)
|
|
if in_oom_kernel_log {
|
|
oomCurrentInstance := &OomInstance{
|
|
ContainerName: "/",
|
|
}
|
|
for line := range lineChannel {
|
|
err := getContainerName(line, oomCurrentInstance)
|
|
if err != nil {
|
|
glog.Errorf("%v", err)
|
|
}
|
|
finished, err := getProcessNamePid(line, oomCurrentInstance)
|
|
if err != nil {
|
|
glog.Errorf("%v", err)
|
|
}
|
|
if finished {
|
|
break
|
|
}
|
|
}
|
|
outStream <- oomCurrentInstance
|
|
}
|
|
}
|
|
glog.Infof("exiting analyzeLines. OOM events will not be reported.")
|
|
}
|
|
|
|
func callJournalctl() (io.ReadCloser, error) {
|
|
cmd := exec.Command("journalctl", "-k", "-f")
|
|
readcloser, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := cmd.Start(); err != nil {
|
|
return nil, err
|
|
}
|
|
return readcloser, err
|
|
}
|
|
|
|
func trySystemd() (*OomParser, error) {
|
|
readcloser, err := callJournalctl()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
glog.Infof("oomparser using systemd")
|
|
return &OomParser{
|
|
ioreader: bufio.NewReader(readcloser),
|
|
}, nil
|
|
}
|
|
|
|
// List of possible kernel log files. These are prioritized in order so that
|
|
// we will use the first one that is available.
|
|
var kernelLogFiles = []string{"/var/log/kern.log", "/var/log/messages", "/var/log/syslog"}
|
|
|
|
// looks for system files that contain kernel messages and if one is found, sets
|
|
// the systemFile attribute of the OomParser object
|
|
func getLogFile() (string, error) {
|
|
for _, logFile := range kernelLogFiles {
|
|
if utils.FileExists(logFile) {
|
|
glog.Infof("OOM parser using kernel log file: %q", logFile)
|
|
return logFile, nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("unable to find any kernel log file available from our set: %v", kernelLogFiles)
|
|
}
|
|
|
|
func tryLogFile() (*OomParser, error) {
|
|
logFile, err := getLogFile()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tail, err := tail.NewTail(logFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &OomParser{
|
|
ioreader: bufio.NewReader(tail),
|
|
}, nil
|
|
}
|
|
|
|
// initializes an OomParser object. Returns an OomParser object and an error.
|
|
func New() (*OomParser, error) {
|
|
parser, err := trySystemd()
|
|
if err == nil {
|
|
return parser, nil
|
|
}
|
|
parser, err = tryLogFile()
|
|
if err == nil {
|
|
return parser, nil
|
|
}
|
|
return nil, err
|
|
}
|