rkt: Implement pod FinishedAt

This is implemented via touching a file on stop as a hook in the systemd
unit. The ctime of this file is then used to get the `finishedAt` time
in the future.
In addition, this changes the `startedAt` and `createdAt` to use the api
server's results rather than the annotations it previously used.

It's possible we might want to move this into the api in the future.

Fixes #23887

pkg/kubelet/container/helpers.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/kubernetes/pkg/client/record"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	"k8s.io/kubernetes/pkg/runtime"
+	"k8s.io/kubernetes/pkg/types"
 	hashutil "k8s.io/kubernetes/pkg/util/hash"
 	"k8s.io/kubernetes/third_party/golang/expansion"
 
@@ -42,6 +43,7 @@ type RuntimeHelper interface {
 	GenerateRunContainerOptions(pod *api.Pod, container *api.Container, podIP string) (*RunContainerOptions, error)
 	GetClusterDNS(pod *api.Pod) (dnsServers []string, dnsSearches []string, err error)
 	GeneratePodHostNameAndDomain(pod *api.Pod) (hostname string, hostDomain string)
+	GetPodDir(podUID types.UID) string
 }
 
 // ShouldContainerBeRestarted checks whether a container needs to be restarted.

pkg/kubelet/container/os.go

@@ -25,6 +25,7 @@ import (
 type OSInterface interface {
 	Mkdir(path string, perm os.FileMode) error
 	Symlink(oldname string, newname string) error
+	Stat(path string) (os.FileInfo, error)
 }
 
 // RealOS is used to dispatch the real system level operaitons.
@@ -39,3 +40,8 @@ func (RealOS) Mkdir(path string, perm os.FileMode) error {
 func (RealOS) Symlink(oldname string, newname string) error {
 	return os.Symlink(oldname, newname)
 }
+
+// Stat will call os.Stat to get the FileInfo for a given path
+func (RealOS) Stat(path string) (os.FileInfo, error) {
+	return os.Stat(path)
+}

pkg/kubelet/container/testing/os.go

@@ -17,12 +17,17 @@ limitations under the License.
 package testing
 
 import (
+	"errors"
 	"os"
 )
 
 // FakeOS mocks out certain OS calls to avoid perturbing the filesystem
 // on the test machine.
-type FakeOS struct{}
+// If a member of the form `*Fn` is set, that function will be called in place
+// of the real call.
+type FakeOS struct {
+	StatFn func(string) (os.FileInfo, error)
+}
 
 // Mkdir is a fake call that just returns nil.
 func (FakeOS) Mkdir(path string, perm os.FileMode) error {
@@ -33,3 +38,11 @@ func (FakeOS) Mkdir(path string, perm os.FileMode) error {
 func (FakeOS) Symlink(oldname string, newname string) error {
 	return nil
 }
+
+// Stat is a fake that returns an error
+func (f FakeOS) Stat(path string) (os.FileInfo, error) {
+	if f.StatFn != nil {
+		return f.StatFn(path)
+	}
+	return nil, errors.New("unimplemented testing mock")
+}

pkg/kubelet/dockertools/manager_test.go

@@ -93,6 +93,10 @@ func (f *fakeRuntimeHelper) GeneratePodHostNameAndDomain(pod *api.Pod) (string,
 	return "", ""
 }
 
+func (f *fakeRuntimeHelper) GetPodDir(types.UID) string {
+	return ""
+}
+
 func createTestDockerManager(fakeHTTPClient *fakeHTTP, fakeDocker *FakeDockerClient) (*DockerManager, *FakeDockerClient) {
 	if fakeHTTPClient == nil {
 		fakeHTTPClient = &fakeHTTP{}

pkg/kubelet/kubelet.go

@@ -72,6 +72,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/atomic"
 	"k8s.io/kubernetes/pkg/util/bandwidth"
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
+	utilexec "k8s.io/kubernetes/pkg/util/exec"
 	"k8s.io/kubernetes/pkg/util/flowcontrol"
 	kubeio "k8s.io/kubernetes/pkg/util/io"
 	"k8s.io/kubernetes/pkg/util/mount"
@@ -426,6 +427,8 @@ func NewMainKubelet(
 			klet.livenessManager,
 			klet.volumeManager,
 			klet.httpClient,
+			utilexec.New(),
+			kubecontainer.RealOS{},
 			imageBackOff,
 			serializeImagePulls,
 		)
@@ -830,8 +833,12 @@ func (kl *Kubelet) getPluginDir(pluginName string) string {
 	return path.Join(kl.getPluginsDir(), pluginName)
 }
 
-// getPodDir returns the full path to the per-pod data directory for the
+// GetPodDir returns the full path to the per-pod data directory for the
-// specified pod.  This directory may not exist if the pod does not exist.
+// specified pod. This directory may not exist if the pod does not exist.
+func (kl *Kubelet) GetPodDir(podUID types.UID) string {
+	return kl.getPodDir(podUID)
+}
+
 func (kl *Kubelet) getPodDir(podUID types.UID) string {
 	// Backwards compat.  The "old" stuff should be removed before 1.0
 	// release.  The thinking here is this:

pkg/kubelet/rkt/fake_rkt_interface_test.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/types"
 
 	"github.com/coreos/go-systemd/dbus"
 	rktapi "github.com/coreos/rkt/api/v1alpha"
@@ -166,3 +167,7 @@ func (f *fakeRuntimeHelper) GetClusterDNS(pod *api.Pod) ([]string, []string, err
 func (f *fakeRuntimeHelper) GeneratePodHostNameAndDomain(pod *api.Pod) (string, string) {
 	return f.hostName, f.hostDomain
 }
+
+func (f *fakeRuntimeHelper) GetPodDir(podUID types.UID) string {
+	return "/poddir/" + string(podUID)
+}

pkg/kubelet/rkt/mock_os/mockfileinfo.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Generated via: mockgen os FileInfo
+// Edited to include required boilerplate
+// Source: os (interfaces: FileInfo)
+
+package mock_os
+
+import (
+	os "os"
+	time "time"
+
+	gomock "github.com/golang/mock/gomock"
+)
+
+// Mock of FileInfo interface
+type MockFileInfo struct {
+	ctrl     *gomock.Controller
+	recorder *_MockFileInfoRecorder
+}
+
+// Recorder for MockFileInfo (not exported)
+type _MockFileInfoRecorder struct {
+	mock *MockFileInfo
+}
+
+func NewMockFileInfo(ctrl *gomock.Controller) *MockFileInfo {
+	mock := &MockFileInfo{ctrl: ctrl}
+	mock.recorder = &_MockFileInfoRecorder{mock}
+	return mock
+}
+
+func (_m *MockFileInfo) EXPECT() *_MockFileInfoRecorder {
+	return _m.recorder
+}
+
+func (_m *MockFileInfo) IsDir() bool {
+	ret := _m.ctrl.Call(_m, "IsDir")
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+func (_mr *_MockFileInfoRecorder) IsDir() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "IsDir")
+}
+
+func (_m *MockFileInfo) ModTime() time.Time {
+	ret := _m.ctrl.Call(_m, "ModTime")
+	ret0, _ := ret[0].(time.Time)
+	return ret0
+}
+
+func (_mr *_MockFileInfoRecorder) ModTime() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "ModTime")
+}
+
+func (_m *MockFileInfo) Mode() os.FileMode {
+	ret := _m.ctrl.Call(_m, "Mode")
+	ret0, _ := ret[0].(os.FileMode)
+	return ret0
+}
+
+func (_mr *_MockFileInfoRecorder) Mode() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Mode")
+}
+
+func (_m *MockFileInfo) Name() string {
+	ret := _m.ctrl.Call(_m, "Name")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+func (_mr *_MockFileInfoRecorder) Name() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Name")
+}
+
+func (_m *MockFileInfo) Size() int64 {
+	ret := _m.ctrl.Call(_m, "Size")
+	ret0, _ := ret[0].(int64)
+	return ret0
+}
+
+func (_mr *_MockFileInfoRecorder) Size() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Size")
+}
+
+func (_m *MockFileInfo) Sys() interface{} {
+	ret := _m.ctrl.Call(_m, "Sys")
+	ret0, _ := ret[0].(interface{})
+	return ret0
+}
+
+func (_mr *_MockFileInfoRecorder) Sys() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Sys")
+}

pkg/kubelet/rkt/rkt.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -76,13 +77,11 @@ const (
 	unitRktID             = "RktID"
 	unitRestartCount      = "RestartCount"
 
-	k8sRktKubeletAnno      = "rkt.kubernetes.io/managed-by-kubelet"
+	k8sRktKubeletAnno                = "rkt.kubernetes.io/managed-by-kubelet"
-	k8sRktKubeletAnnoValue = "true"
+	k8sRktKubeletAnnoValue           = "true"
-	k8sRktUIDAnno          = "rkt.kubernetes.io/uid"
+	k8sRktUIDAnno                    = "rkt.kubernetes.io/uid"
-	k8sRktNameAnno         = "rkt.kubernetes.io/name"
+	k8sRktNameAnno                   = "rkt.kubernetes.io/name"
-	k8sRktNamespaceAnno    = "rkt.kubernetes.io/namespace"
+	k8sRktNamespaceAnno              = "rkt.kubernetes.io/namespace"
-	//TODO: remove the creation time annotation once this is closed: https://github.com/coreos/rkt/issues/1789
-	k8sRktCreationTimeAnno           = "rkt.kubernetes.io/created"
 	k8sRktContainerHashAnno          = "rkt.kubernetes.io/container-hash"
 	k8sRktRestartCountAnno           = "rkt.kubernetes.io/restart-count"
 	k8sRktTerminationMessagePathAnno = "rkt.kubernetes.io/termination-message-path"
@@ -128,6 +127,11 @@ type Runtime struct {
 	volumeGetter        volumeGetter
 	imagePuller         kubecontainer.ImagePuller
 	runner              kubecontainer.HandlerRunner
+	execer              utilexec.Interface
+	os                  kubecontainer.OSInterface
+
+	// used for a systemd Exec, which requires the full path.
+	touchPath string
 
 	versions versions
 }
@@ -151,6 +155,8 @@ func New(
 	livenessManager proberesults.Manager,
 	volumeGetter volumeGetter,
 	httpClient kubetypes.HttpGetter,
+	execer utilexec.Interface,
+	os kubecontainer.OSInterface,
 	imageBackOff *flowcontrol.Backoff,
 	serializeImagePulls bool,
 ) (*Runtime, error) {
@@ -170,12 +176,17 @@ func New(
 	if config.Path == "" {
 		// No default rkt path was set, so try to find one in $PATH.
 		var err error
-		config.Path, err = exec.LookPath("rkt")
+		config.Path, err = execer.LookPath("rkt")
 		if err != nil {
 			return nil, fmt.Errorf("cannot find rkt binary: %v", err)
 		}
 	}
 
+	touchPath, err := execer.LookPath("touch")
+	if err != nil {
+		return nil, fmt.Errorf("cannot find touch binary: %v", err)
+	}
+
 	rkt := &Runtime{
 		systemd:             systemd,
 		apisvcConn:          apisvcConn,
@@ -187,6 +198,8 @@ func New(
 		recorder:            recorder,
 		livenessManager:     livenessManager,
 		volumeGetter:        volumeGetter,
+		execer:              execer,
+		touchPath:           touchPath,
 	}
 
 	rkt.config, err = rkt.getConfig(rkt.config)
@@ -541,7 +554,6 @@ func (r *Runtime) makePodManifest(pod *api.Pod, pullSecrets []api.Secret) (*appc
 	manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktUIDAnno), string(pod.UID))
 	manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktNameAnno), pod.Name)
 	manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktNamespaceAnno), pod.Namespace)
-	manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktCreationTimeAnno), strconv.FormatInt(time.Now().Unix(), 10))
 	manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktRestartCountAnno), strconv.Itoa(restartCount))
 
 	for _, c := range pod.Spec.Containers {
@@ -587,6 +599,34 @@ func makeHostNetworkMount(opts *kubecontainer.RunContainerOptions) (*kubecontain
 	return &hostsMount, &resolvMount
 }
 
+// podFinishedMarkerPath returns the path to a file which should be used to
+// indicate the pod exiting, and the time thereof.
+// If the file at the path does not exist, the pod should not be exited. If it
+// does exist, then the ctime of the file should indicate the time the pod
+// exited.
+func podFinishedMarkerPath(podDir string, rktUID string) string {
+	return filepath.Join(podDir, "finished-"+rktUID)
+}
+
+func podFinishedMarkCommand(touchPath, podDir, rktUID string) string {
+	// TODO, if the path has a `'` character in it, this breaks.
+	return touchPath + " " + podFinishedMarkerPath(podDir, rktUID)
+}
+
+// podFinishedAt returns the time that a pod exited, or a zero time if it has
+// not.
+func (r *Runtime) podFinishedAt(podUID types.UID, rktUID string) time.Time {
+	markerFile := podFinishedMarkerPath(r.runtimeHelper.GetPodDir(podUID), rktUID)
+	stat, err := r.os.Stat(markerFile)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			glog.Warningf("rkt: unexpected fs error checking pod finished marker: %v", err)
+		}
+		return time.Time{}
+	}
+	return stat.ModTime()
+}
+
 func makeContainerLogMount(opts *kubecontainer.RunContainerOptions, container *api.Container) (*kubecontainer.Mount, error) {
 	if opts.PodContainerDir == "" || container.TerminationMessagePath == "" {
 		return nil, nil
@@ -884,8 +924,11 @@ func (r *Runtime) preparePod(pod *api.Pod, pullSecrets []api.Secret) (string, *k
 	// TODO handle pod.Spec.HostPID
 	// TODO handle pod.Spec.HostIPC
 
+	// TODO per container finishedAt, not just per pod
+	markPodFinished := podFinishedMarkCommand(r.touchPath, r.runtimeHelper.GetPodDir(pod.UID), uuid)
 	units := []*unit.UnitOption{
 		newUnitOption("Service", "ExecStart", runPrepared),
+		newUnitOption("Service", "ExecStopPost", markPodFinished),
 		// This enables graceful stop.
 		newUnitOption("Service", "KillMode", "mixed"),
 	}
@@ -1045,14 +1088,6 @@ func (r *Runtime) convertRktPod(rktpod *rktapi.Pod) (*kubecontainer.Pod, error)
 	if !ok {
 		return nil, fmt.Errorf("pod is missing annotation %s", k8sRktNamespaceAnno)
 	}
-	podCreatedString, ok := manifest.Annotations.Get(k8sRktCreationTimeAnno)
-	if !ok {
-		return nil, fmt.Errorf("pod is missing annotation %s", k8sRktCreationTimeAnno)
-	}
-	podCreated, err := strconv.ParseInt(podCreatedString, 10, 64)
-	if err != nil {
-		return nil, fmt.Errorf("couldn't parse pod creation timestamp: %v", err)
-	}
 
 	kubepod := &kubecontainer.Pod{
 		ID:        types.UID(podUID),
@@ -1078,8 +1113,8 @@ func (r *Runtime) convertRktPod(rktpod *rktapi.Pod) (*kubecontainer.Pod, error)
 			// By default, the version returned by rkt API service will be "latest" if not specified.
 			Image:   fmt.Sprintf("%s:%s", app.Image.Name, app.Image.Version),
 			Hash:    containerHash,
-			Created: podCreated,
 			State:   appStateToContainerState(app.State),
+			Created: time.Unix(0, rktpod.CreatedAt).Unix(), // convert ns to s
 		})
 	}
 
@@ -1526,7 +1561,7 @@ func appStateToContainerState(state rktapi.AppState) kubecontainer.ContainerStat
 }
 
 // getPodInfo returns the pod manifest, creation time and restart count of the pod.
-func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, creationTime time.Time, restartCount int, err error) {
+func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, restartCount int, err error) {
 	// TODO(yifan): The manifest is only used for getting the annotations.
 	// Consider to let the server to unmarshal the annotations.
 	var manifest appcschema.PodManifest
@@ -1534,16 +1569,6 @@ func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, creationT
 		return
 	}
 
-	creationTimeStr, ok := manifest.Annotations.Get(k8sRktCreationTimeAnno)
-	if !ok {
-		err = fmt.Errorf("no creation timestamp in pod manifest")
-		return
-	}
-	unixSec, err := strconv.ParseInt(creationTimeStr, 10, 64)
-	if err != nil {
-		return
-	}
-
 	if countString, ok := manifest.Annotations.Get(k8sRktRestartCountAnno); ok {
 		restartCount, err = strconv.Atoi(countString)
 		if err != nil {
@@ -1551,11 +1576,11 @@ func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, creationT
 		}
 	}
 
-	return &manifest, time.Unix(unixSec, 0), restartCount, nil
+	return &manifest, restartCount, nil
 }
 
 // populateContainerStatus fills the container status according to the app's information.
-func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcschema.RuntimeApp, restartCount int, creationTime time.Time) (*kubecontainer.ContainerStatus, error) {
+func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcschema.RuntimeApp, restartCount int, finishedTime time.Time) (*kubecontainer.ContainerStatus, error) {
 	hashStr, ok := runtimeApp.Annotations.Get(k8sRktContainerHashAnno)
 	if !ok {
 		return nil, fmt.Errorf("No container hash in pod manifest")
@@ -1584,14 +1609,17 @@ func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcsche
 		}
 	}
 
+	createdTime := time.Unix(0, pod.CreatedAt)
+	startedTime := time.Unix(0, pod.StartedAt)
+
 	return &kubecontainer.ContainerStatus{
-		ID:    buildContainerID(&containerID{uuid: pod.Id, appName: app.Name}),
+		ID:         buildContainerID(&containerID{uuid: pod.Id, appName: app.Name}),
-		Name:  app.Name,
+		Name:       app.Name,
-		State: appStateToContainerState(app.State),
+		State:      appStateToContainerState(app.State),
-		// TODO(yifan): Use the creation/start/finished timestamp when it's implemented.
+		CreatedAt:  createdTime,
-		CreatedAt: creationTime,
+		StartedAt:  startedTime,
-		StartedAt: creationTime,
+		FinishedAt: finishedTime,
-		ExitCode:  int(app.ExitCode),
+		ExitCode:   int(app.ExitCode),
 		// By default, the version returned by rkt API service will be "latest" if not specified.
 		Image:   fmt.Sprintf("%s:%s", app.Image.Name, app.Image.Version),
 		ImageID: "rkt://" + app.Image.Id, // TODO(yifan): Add the prefix only in api.PodStatus.
@@ -1605,6 +1633,13 @@ func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcsche
 	}, nil
 }
 
+// GetPodStatus returns the status for a pod specified by a given UID, name,
+// and namespace.  It will attempt to find pod's information via a request to
+// the rkt api server.
+// An error will be returned if the api server returns an error. If the api
+// server doesn't error, but doesn't provide meaningful information about the
+// pod, a status with no information (other than the passed in arguments) is
+// returned anyways.
 func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecontainer.PodStatus, error) {
 	podStatus := &kubecontainer.PodStatus{
 		ID:        uid,
@@ -1626,7 +1661,7 @@ func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecont
 	// In this loop, we group all containers from all pods together,
 	// also we try to find the latest pod, so we can fill other info of the pod below.
 	for _, pod := range listResp.Pods {
-		manifest, creationTime, restartCount, err := getPodInfo(pod)
+		manifest, restartCount, err := getPodInfo(pod)
 		if err != nil {
 			glog.Warningf("rkt: Couldn't get necessary info from the rkt pod, (uuid %q): %v", pod.Id, err)
 			continue
@@ -1637,11 +1672,10 @@ func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecont
 			latestRestartCount = restartCount
 		}
 
+		finishedTime := r.podFinishedAt(uid, pod.Id)
 		for i, app := range pod.Apps {
 			// The order of the apps is determined by the rkt pod manifest.
-			// TODO(yifan): Save creationTime, restartCount in each app's annotation,
+			cs, err := populateContainerStatus(*pod, *app, manifest.Apps[i], restartCount, finishedTime)
-			// so we don't need to pass them.
-			cs, err := populateContainerStatus(*pod, *app, manifest.Apps[i], restartCount, creationTime)
 			if err != nil {
 				glog.Warningf("rkt: Failed to populate container status(uuid %q, app %q): %v", pod.Id, app.Name, err)
 				continue

pkg/kubelet/rkt/rkt_test.go

@@ -27,11 +27,14 @@ import (
 	appcschema "github.com/appc/spec/schema"
 	appctypes "github.com/appc/spec/schema/types"
 	rktapi "github.com/coreos/rkt/api/v1alpha"
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	containertesting "k8s.io/kubernetes/pkg/kubelet/container/testing"
 	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+	"k8s.io/kubernetes/pkg/kubelet/rkt/mock_os"
 	"k8s.io/kubernetes/pkg/util/errors"
 	utiltesting "k8s.io/kubernetes/pkg/util/testing"
 )
@@ -62,9 +65,10 @@ func mustRktHash(hash string) *appctypes.Hash {
 
 func makeRktPod(rktPodState rktapi.PodState,
 	rktPodID, podUID, podName, podNamespace,
-	podIP, podCreationTs, podRestartCount string,
+	podIP string, podCreatedAt, podStartedAt int64,
-	appNames, imgIDs, imgNames, containerHashes []string,
+	podRestartCount string, appNames, imgIDs, imgNames,
-	appStates []rktapi.AppState, exitcodes []int32) *rktapi.Pod {
+	containerHashes []string, appStates []rktapi.AppState,
+	exitcodes []int32) *rktapi.Pod {
 
 	podManifest := &appcschema.PodManifest{
 		ACKind:    appcschema.PodManifestKind,
@@ -86,10 +90,6 @@ func makeRktPod(rktPodState rktapi.PodState,
 				Name:  *appctypes.MustACIdentifier(k8sRktNamespaceAnno),
 				Value: podNamespace,
 			},
-			appctypes.Annotation{
-				Name:  *appctypes.MustACIdentifier(k8sRktCreationTimeAnno),
-				Value: podCreationTs,
-			},
 			appctypes.Annotation{
 				Name:  *appctypes.MustACIdentifier(k8sRktRestartCountAnno),
 				Value: podRestartCount,
@@ -143,11 +143,13 @@ func makeRktPod(rktPodState rktapi.PodState,
 	}
 
 	return &rktapi.Pod{
-		Id:       rktPodID,
+		Id:        rktPodID,
-		State:    rktPodState,
+		State:     rktPodState,
-		Networks: []*rktapi.Network{{Name: defaultNetworkName, Ipv4: podIP}},
+		Networks:  []*rktapi.Network{{Name: defaultNetworkName, Ipv4: podIP}},
-		Apps:     apps,
+		Apps:      apps,
-		Manifest: mustMarshalPodManifest(podManifest),
+		Manifest:  mustMarshalPodManifest(podManifest),
+		StartedAt: podStartedAt,
+		CreatedAt: podCreatedAt,
 	}
 }
 
@@ -346,6 +348,10 @@ func TestGetPods(t *testing.T) {
 	fs := newFakeSystemd()
 	r := &Runtime{apisvc: fr, systemd: fs}
 
+	ns := func(seconds int64) int64 {
+		return seconds * 1e9
+	}
+
 	tests := []struct {
 		pods   []*rktapi.Pod
 		result []*kubecontainer.Pod
@@ -357,7 +363,7 @@ func TestGetPods(t *testing.T) {
 			[]*rktapi.Pod{
 				makeRktPod(rktapi.PodState_POD_STATE_RUNNING,
 					"uuid-4002", "42", "guestbook", "default",
-					"10.10.10.42", "100000", "7",
+					"10.10.10.42", ns(10), ns(10), "7",
 					[]string{"app-1", "app-2"},
 					[]string{"img-id-1", "img-id-2"},
 					[]string{"img-name-1", "img-name-2"},
@@ -377,7 +383,7 @@ func TestGetPods(t *testing.T) {
 							Name:    "app-1",
 							Image:   "img-name-1:latest",
 							Hash:    1001,
-							Created: 100000,
+							Created: 10,
 							State:   "running",
 						},
 						{
@@ -385,7 +391,7 @@ func TestGetPods(t *testing.T) {
 							Name:    "app-2",
 							Image:   "img-name-2:latest",
 							Hash:    1002,
-							Created: 100000,
+							Created: 10,
 							State:   "exited",
 						},
 					},
@@ -397,7 +403,7 @@ func TestGetPods(t *testing.T) {
 			[]*rktapi.Pod{
 				makeRktPod(rktapi.PodState_POD_STATE_RUNNING,
 					"uuid-4002", "42", "guestbook", "default",
-					"10.10.10.42", "100000", "7",
+					"10.10.10.42", ns(10), ns(20), "7",
 					[]string{"app-1", "app-2"},
 					[]string{"img-id-1", "img-id-2"},
 					[]string{"img-name-1", "img-name-2"},
@@ -407,7 +413,7 @@ func TestGetPods(t *testing.T) {
 				),
 				makeRktPod(rktapi.PodState_POD_STATE_EXITED,
 					"uuid-4003", "43", "guestbook", "default",
-					"10.10.10.43", "90000", "7",
+					"10.10.10.43", ns(30), ns(40), "7",
 					[]string{"app-11", "app-22"},
 					[]string{"img-id-11", "img-id-22"},
 					[]string{"img-name-11", "img-name-22"},
@@ -417,7 +423,7 @@ func TestGetPods(t *testing.T) {
 				),
 				makeRktPod(rktapi.PodState_POD_STATE_EXITED,
 					"uuid-4004", "43", "guestbook", "default",
-					"10.10.10.44", "100000", "8",
+					"10.10.10.44", ns(50), ns(60), "8",
 					[]string{"app-11", "app-22"},
 					[]string{"img-id-11", "img-id-22"},
 					[]string{"img-name-11", "img-name-22"},
@@ -437,7 +443,7 @@ func TestGetPods(t *testing.T) {
 							Name:    "app-1",
 							Image:   "img-name-1:latest",
 							Hash:    1001,
-							Created: 100000,
+							Created: 10,
 							State:   "running",
 						},
 						{
@@ -445,7 +451,7 @@ func TestGetPods(t *testing.T) {
 							Name:    "app-2",
 							Image:   "img-name-2:latest",
 							Hash:    1002,
-							Created: 100000,
+							Created: 10,
 							State:   "exited",
 						},
 					},
@@ -460,7 +466,7 @@ func TestGetPods(t *testing.T) {
 							Name:    "app-11",
 							Image:   "img-name-11:latest",
 							Hash:    10011,
-							Created: 90000,
+							Created: 30,
 							State:   "exited",
 						},
 						{
@@ -468,7 +474,7 @@ func TestGetPods(t *testing.T) {
 							Name:    "app-22",
 							Image:   "img-name-22:latest",
 							Hash:    10022,
-							Created: 90000,
+							Created: 30,
 							State:   "exited",
 						},
 						{
@@ -476,7 +482,7 @@ func TestGetPods(t *testing.T) {
 							Name:    "app-11",
 							Image:   "img-name-11:latest",
 							Hash:    10011,
-							Created: 100000,
+							Created: 50,
 							State:   "running",
 						},
 						{
@@ -484,7 +490,7 @@ func TestGetPods(t *testing.T) {
 							Name:    "app-22",
 							Image:   "img-name-22:latest",
 							Hash:    10022,
-							Created: 100000,
+							Created: 50,
 							State:   "running",
 						},
 					},
@@ -557,7 +563,18 @@ func TestGetPodsFilters(t *testing.T) {
 func TestGetPodStatus(t *testing.T) {
 	fr := newFakeRktInterface()
 	fs := newFakeSystemd()
-	r := &Runtime{apisvc: fr, systemd: fs}
+	fos := &containertesting.FakeOS{}
+	frh := &fakeRuntimeHelper{}
+	r := &Runtime{
+		apisvc:        fr,
+		systemd:       fs,
+		runtimeHelper: frh,
+		os:            fos,
+	}
+
+	ns := func(seconds int64) int64 {
+		return seconds * 1e9
+	}
 
 	tests := []struct {
 		pods   []*rktapi.Pod
@@ -573,7 +590,7 @@ func TestGetPodStatus(t *testing.T) {
 			[]*rktapi.Pod{
 				makeRktPod(rktapi.PodState_POD_STATE_RUNNING,
 					"uuid-4002", "42", "guestbook", "default",
-					"10.10.10.42", "100000", "7",
+					"10.10.10.42", ns(10), ns(20), "7",
 					[]string{"app-1", "app-2"},
 					[]string{"img-id-1", "img-id-2"},
 					[]string{"img-name-1", "img-name-2"},
@@ -592,8 +609,9 @@ func TestGetPodStatus(t *testing.T) {
 						ID:           kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"),
 						Name:         "app-1",
 						State:        kubecontainer.ContainerStateRunning,
-						CreatedAt:    time.Unix(100000, 0),
+						CreatedAt:    time.Unix(10, 0),
-						StartedAt:    time.Unix(100000, 0),
+						StartedAt:    time.Unix(20, 0),
+						FinishedAt:   time.Unix(0, 30),
 						Image:        "img-name-1:latest",
 						ImageID:      "rkt://img-id-1",
 						Hash:         1001,
@@ -603,8 +621,9 @@ func TestGetPodStatus(t *testing.T) {
 						ID:           kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"),
 						Name:         "app-2",
 						State:        kubecontainer.ContainerStateExited,
-						CreatedAt:    time.Unix(100000, 0),
+						CreatedAt:    time.Unix(10, 0),
-						StartedAt:    time.Unix(100000, 0),
+						StartedAt:    time.Unix(20, 0),
+						FinishedAt:   time.Unix(0, 30),
 						Image:        "img-name-2:latest",
 						ImageID:      "rkt://img-id-2",
 						Hash:         1002,
@@ -619,7 +638,7 @@ func TestGetPodStatus(t *testing.T) {
 			[]*rktapi.Pod{
 				makeRktPod(rktapi.PodState_POD_STATE_EXITED,
 					"uuid-4002", "42", "guestbook", "default",
-					"10.10.10.42", "90000", "7",
+					"10.10.10.42", ns(10), ns(20), "7",
 					[]string{"app-1", "app-2"},
 					[]string{"img-id-1", "img-id-2"},
 					[]string{"img-name-1", "img-name-2"},
@@ -629,7 +648,7 @@ func TestGetPodStatus(t *testing.T) {
 				),
 				makeRktPod(rktapi.PodState_POD_STATE_RUNNING, // The latest pod is running.
 					"uuid-4003", "42", "guestbook", "default",
-					"10.10.10.42", "100000", "10",
+					"10.10.10.42", ns(10), ns(20), "10",
 					[]string{"app-1", "app-2"},
 					[]string{"img-id-1", "img-id-2"},
 					[]string{"img-name-1", "img-name-2"},
@@ -649,8 +668,9 @@ func TestGetPodStatus(t *testing.T) {
 						ID:           kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"),
 						Name:         "app-1",
 						State:        kubecontainer.ContainerStateRunning,
-						CreatedAt:    time.Unix(90000, 0),
+						CreatedAt:    time.Unix(10, 0),
-						StartedAt:    time.Unix(90000, 0),
+						StartedAt:    time.Unix(20, 0),
+						FinishedAt:   time.Unix(0, 30),
 						Image:        "img-name-1:latest",
 						ImageID:      "rkt://img-id-1",
 						Hash:         1001,
@@ -660,8 +680,9 @@ func TestGetPodStatus(t *testing.T) {
 						ID:           kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"),
 						Name:         "app-2",
 						State:        kubecontainer.ContainerStateExited,
-						CreatedAt:    time.Unix(90000, 0),
+						CreatedAt:    time.Unix(10, 0),
-						StartedAt:    time.Unix(90000, 0),
+						StartedAt:    time.Unix(20, 0),
+						FinishedAt:   time.Unix(0, 30),
 						Image:        "img-name-2:latest",
 						ImageID:      "rkt://img-id-2",
 						Hash:         1002,
@@ -672,8 +693,9 @@ func TestGetPodStatus(t *testing.T) {
 						ID:           kubecontainer.BuildContainerID("rkt", "uuid-4003:app-1"),
 						Name:         "app-1",
 						State:        kubecontainer.ContainerStateRunning,
-						CreatedAt:    time.Unix(100000, 0),
+						CreatedAt:    time.Unix(10, 0),
-						StartedAt:    time.Unix(100000, 0),
+						StartedAt:    time.Unix(20, 0),
+						FinishedAt:   time.Unix(0, 30),
 						Image:        "img-name-1:latest",
 						ImageID:      "rkt://img-id-1",
 						Hash:         1001,
@@ -683,8 +705,9 @@ func TestGetPodStatus(t *testing.T) {
 						ID:           kubecontainer.BuildContainerID("rkt", "uuid-4003:app-2"),
 						Name:         "app-2",
 						State:        kubecontainer.ContainerStateExited,
-						CreatedAt:    time.Unix(100000, 0),
+						CreatedAt:    time.Unix(10, 0),
-						StartedAt:    time.Unix(100000, 0),
+						StartedAt:    time.Unix(20, 0),
+						FinishedAt:   time.Unix(0, 30),
 						Image:        "img-name-2:latest",
 						ImageID:      "rkt://img-id-2",
 						Hash:         1002,
@@ -697,10 +720,28 @@ func TestGetPodStatus(t *testing.T) {
 		},
 	}
 
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
 	for i, tt := range tests {
 		testCaseHint := fmt.Sprintf("test case #%d", i)
 		fr.pods = tt.pods
 
+		podTimes := map[string]time.Time{}
+		for _, pod := range tt.pods {
+			podTimes[podFinishedMarkerPath(r.runtimeHelper.GetPodDir(tt.result.ID), pod.Id)] = tt.result.ContainerStatuses[0].FinishedAt
+		}
+
+		r.os.(*containertesting.FakeOS).StatFn = func(name string) (os.FileInfo, error) {
+			podTime, ok := podTimes[name]
+			if !ok {
+				t.Errorf("osStat called with %v, but only knew about %#v", name, podTimes)
+			}
+			mockFI := mock_os.NewMockFileInfo(ctrl)
+			mockFI.EXPECT().ModTime().Return(podTime)
+			return mockFI, nil
+		}
+
 		status, err := r.GetPodStatus("42", "guestbook", "default")
 		if err != nil {
 			t.Errorf("test case #%d: unexpected error: %v", i, err)