// Copyright 2014 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package oomparser import ( "bufio" "fmt" "io" "os/exec" "path" "regexp" "strconv" "time" "github.com/google/cadvisor/utils" "github.com/google/cadvisor/utils/tail" "github.com/golang/glog" ) var ( containerRegexp = regexp.MustCompile(`Task in (.*) killed as a result of limit of (.*)`) lastLineRegexp = regexp.MustCompile(`(^[A-Z][a-z]{2} .*[0-9]{1,2} [0-9]{1,2}:[0-9]{2}:[0-9]{2}) .* Killed process ([0-9]+) \(([\w]+)\)`) firstLineRegexp = regexp.MustCompile(`invoked oom-killer:`) ) // struct to hold file from which we obtain OomInstances type OomParser struct { ioreader *bufio.Reader } // struct that contains information related to an OOM kill instance type OomInstance struct { // process id of the killed process Pid int // the name of the killed process ProcessName string // the time that the process was reported to be killed, // accurate to the minute TimeOfDeath time.Time // the absolute name of the container that OOMed ContainerName string // the absolute name of the container that was killed // due to the OOM. VictimContainerName string } // gets the container name from a line and adds it to the oomInstance. func getContainerName(line string, currentOomInstance *OomInstance) error { parsedLine := containerRegexp.FindStringSubmatch(line) if parsedLine == nil { return nil } currentOomInstance.ContainerName = path.Join("/", parsedLine[1]) currentOomInstance.VictimContainerName = path.Join("/", parsedLine[2]) return nil } // gets the pid, name, and date from a line and adds it to oomInstance func getProcessNamePid(line string, currentOomInstance *OomInstance) (bool, error) { reList := lastLineRegexp.FindStringSubmatch(line) if reList == nil { return false, nil } const longForm = "Jan _2 15:04:05 2006" stringYear := strconv.Itoa(time.Now().Year()) linetime, err := time.ParseInLocation(longForm, reList[1]+" "+stringYear, time.Local) if err != nil { return false, err } currentOomInstance.TimeOfDeath = linetime pid, err := strconv.Atoi(reList[2]) if err != nil { return false, err } currentOomInstance.Pid = pid currentOomInstance.ProcessName = reList[3] return true, nil } // uses regex to see if line is the start of a kernel oom log func checkIfStartOfOomMessages(line string) bool { potential_oom_start := firstLineRegexp.MatchString(line) if potential_oom_start { return true } return false } // reads the file and sends only complete lines over a channel to analyzeLines. // Should prevent EOF errors that occur when lines are read before being fully // written to the log. It reads line by line splitting on // the "\n" character. func readLinesFromFile(lineChannel chan string, ioreader *bufio.Reader) error { linefragment := "" var line string var err error for true { line, err = ioreader.ReadString('\n') if err != nil && err != io.EOF { glog.Errorf("exiting analyzeLinesHelper with error %v", err) close(lineChannel) break } if line == "" { time.Sleep(100 * time.Millisecond) continue } if err == nil { lineChannel <- linefragment + line linefragment = "" } else { // err == io.EOF linefragment += line } } return err } // Calls goroutine for readLinesFromFile, which feeds it complete lines. // Lines are checked against a regexp to check for the pid, process name, etc. // At the end of an oom message group, StreamOoms adds the new oomInstance to // oomLog func (self *OomParser) StreamOoms(outStream chan *OomInstance) { lineChannel := make(chan string, 10) go func() { readLinesFromFile(lineChannel, self.ioreader) }() for line := range lineChannel { in_oom_kernel_log := checkIfStartOfOomMessages(line) if in_oom_kernel_log { oomCurrentInstance := &OomInstance{ ContainerName: "/", } for line := range lineChannel { err := getContainerName(line, oomCurrentInstance) if err != nil { glog.Errorf("%v", err) } finished, err := getProcessNamePid(line, oomCurrentInstance) if err != nil { glog.Errorf("%v", err) } if finished { break } } outStream <- oomCurrentInstance } } glog.Infof("exiting analyzeLines. OOM events will not be reported.") } func callJournalctl() (io.ReadCloser, error) { cmd := exec.Command("journalctl", "-k", "-f") readcloser, err := cmd.StdoutPipe() if err != nil { return nil, err } if err := cmd.Start(); err != nil { return nil, err } return readcloser, err } func trySystemd() (*OomParser, error) { readcloser, err := callJournalctl() if err != nil { return nil, err } glog.Infof("oomparser using systemd") return &OomParser{ ioreader: bufio.NewReader(readcloser), }, nil } // List of possible kernel log files. These are prioritized in order so that // we will use the first one that is available. var kernelLogFiles = []string{"/var/log/kern.log", "/var/log/messages", "/var/log/syslog"} // looks for system files that contain kernel messages and if one is found, sets // the systemFile attribute of the OomParser object func getLogFile() (string, error) { for _, logFile := range kernelLogFiles { if utils.FileExists(logFile) { glog.Infof("OOM parser using kernel log file: %q", logFile) return logFile, nil } } return "", fmt.Errorf("unable to find any kernel log file available from our set: %v", kernelLogFiles) } func tryLogFile() (*OomParser, error) { logFile, err := getLogFile() if err != nil { return nil, err } tail, err := tail.NewTail(logFile) if err != nil { return nil, err } return &OomParser{ ioreader: bufio.NewReader(tail), }, nil } // initializes an OomParser object. Returns an OomParser object and an error. func New() (*OomParser, error) { parser, err := trySystemd() if err == nil { return parser, nil } parser, err = tryLogFile() if err == nil { return parser, nil } return nil, err }