github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Fix update container resources

Browse Source 
Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu
2017-10-03 06:03:39 +00:00
parent 29d9a788e6
commit a81a47bf9b
14 changed files with 324 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/server/container_attach.go
View File

@@ -61,7 +61,7 @@ func (c *criContainerdService) attachContainer(ctx context.Context, id string, s
return fmt.Errorf("container is in %s state", criContainerStateToString(state))
}
task, err := cntr.Container.Task(ctx, nil)
task, err := cntr.Container.Get().Task(ctx, nil)
if err != nil {
return fmt.Errorf("failed to load task: %v", err)
}

2
pkg/server/container_create.go
View File

@@ -647,6 +647,8 @@ func setOCILinuxResource(g *generate.Generator, resources *runtime.LinuxContaine
g.SetLinuxResourcesCPUShares(uint64(resources.GetCpuShares()))
g.SetLinuxResourcesMemoryLimit(resources.GetMemoryLimitInBytes())
g.SetProcessOOMScoreAdj(int(resources.GetOomScoreAdj()))
g.SetLinuxResourcesCPUCpus(resources.GetCpusetCpus())
g.SetLinuxResourcesCPUMems(resources.GetCpusetMems())
}
// getOCICapabilitiesList returns a list of all available capabilities.

4
pkg/server/container_create_test.go
View File

@@ -96,6 +96,8 @@ func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandbox
CpuShares: 300,
MemoryLimitInBytes: 400,
OomScoreAdj: 500,
CpusetCpus: "0-1",
CpusetMems: "2-3",
},
SecurityContext: &runtime.LinuxContainerSecurityContext{
SupplementalGroups: []int64{1111, 2222},
@@ -138,6 +140,8 @@ func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandbox
assert.EqualValues(t, *spec.Linux.Resources.CPU.Period, 100)
assert.EqualValues(t, *spec.Linux.Resources.CPU.Quota, 200)
assert.EqualValues(t, *spec.Linux.Resources.CPU.Shares, 300)
assert.EqualValues(t, spec.Linux.Resources.CPU.Cpus, "0-1")
assert.EqualValues(t, spec.Linux.Resources.CPU.Mems, "2-3")
assert.EqualValues(t, *spec.Linux.Resources.Memory.Limit, 400)
assert.EqualValues(t, *spec.Process.OOMScoreAdj, 500)

2
pkg/server/container_execsync.go
View File

@@ -87,7 +87,7 @@ func (c *criContainerdService) execInContainer(ctx context.Context, id string, o
return nil, fmt.Errorf("container is in %s state", criContainerStateToString(state))
}
container := cntr.Container
container := cntr.Container.Get()
spec, err := container.Spec()
if err != nil {
return nil, fmt.Errorf("failed to get container spec: %v", err)

2
pkg/server/container_remove.go
View File

@@ -75,7 +75,7 @@ func (c *criContainerdService) RemoveContainer(ctx context.Context, r *runtime.R
}
// Delete containerd container.
if err := container.Container.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
if err := container.Container.Get().Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
if !errdefs.IsNotFound(err) {
return nil, fmt.Errorf("failed to delete containerd container %q: %v", id, err)
}

2
pkg/server/container_start.go
View File

@@ -59,7 +59,7 @@ func (c *criContainerdService) startContainer(ctx context.Context,
status *containerstore.Status) (retErr error) {
id := cntr.ID
meta := cntr.Metadata
container := cntr.Container
container := cntr.Container.Get()
config := meta.Config
// Return error if container is not in created state.

4
pkg/server/container_stop.go
View File

@@ -88,7 +88,7 @@ func (c *criContainerdService) stopContainer(ctx context.Context, container cont
}
}
glog.V(2).Infof("Stop container %q with signal %v", id, stopSignal)
task, err := container.Container.Task(ctx, nil)
task, err := container.Container.Get().Task(ctx, nil)
if err != nil {
if !errdefs.IsNotFound(err) {
return fmt.Errorf("failed to stop container, task not found for container %q: %v", id, err)
@@ -111,7 +111,7 @@ func (c *criContainerdService) stopContainer(ctx context.Context, container cont
glog.Errorf("Stop container %q timed out: %v", id, err)
}
task, err := container.Container.Task(ctx, nil)
task, err := container.Container.Get().Task(ctx, nil)
if err != nil {
if !errdefs.IsNotFound(err) {
return fmt.Errorf("failed to stop container, task not found for container %q: %v", id, err)

151
pkg/server/container_update_resources.go
View File

@@ -20,41 +20,150 @@ import (
"fmt"
"github.com/containerd/containerd"
"github.com/golang/protobuf/proto"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/typeurl"
"github.com/golang/glog"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/generate"
"golang.org/x/net/context"
"k8s.io/apimachinery/pkg/conversion"
"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
)
// UpdateContainerResources updates ContainerConfig of the container.
func (c *criContainerdService) UpdateContainerResources(ctx context.Context, r *runtime.UpdateContainerResourcesRequest) (retRes *runtime.UpdateContainerResourcesResponse, retErr error) {
cntr, err := c.containerStore.Get(r.GetContainerId())
container, err := c.containerStore.Get(r.GetContainerId())
if err != nil {
return nil, fmt.Errorf("failed to find container: %v", err)
}
task, err := cntr.Container.Task(ctx, nil)
if err != nil {
return nil, fmt.Errorf("failed to find task: %v", err)
}
resources := toOCIResources(r.GetLinux())
if err := task.Update(ctx, containerd.WithResources(resources)); err != nil {
// Update resources in status update transaction, so that:
// 1) There won't be race condition with container start.
// 2) There won't be concurrent resource update to the same container.
if err := container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
return status, c.updateContainerResources(ctx, container, r.GetLinux(), status)
}); err != nil {
return nil, fmt.Errorf("failed to update resources: %v", err)
}
return &runtime.UpdateContainerResourcesResponse{}, nil
}
// toOCIResources converts CRI resource constraints to OCI.
func toOCIResources(r *runtime.LinuxContainerResources) *runtimespec.LinuxResources {
return &runtimespec.LinuxResources{
CPU: &runtimespec.LinuxCPU{
Shares: proto.Uint64(uint64(r.GetCpuShares())),
Quota: proto.Int64(r.GetCpuQuota()),
Period: proto.Uint64(uint64(r.GetCpuPeriod())),
Cpus: r.GetCpusetCpus(),
Mems: r.GetCpusetMems(),
},
Memory: &runtimespec.LinuxMemory{
Limit: proto.Int64(r.GetMemoryLimitInBytes()),
},
func (c *criContainerdService) updateContainerResources(ctx context.Context,
cntr containerstore.Container,
resources *runtime.LinuxContainerResources,
status containerstore.Status) (retErr error) {
id := cntr.ID
// Do not update the container when there is a removal in progress.
if status.Removing {
return fmt.Errorf("container %q is in removing state", id)
}
// Update container spec. If the container is not started yet, updating
// spec makes sure that the resource limits are correct when start;
// if the container is already started, updating spec is still required,
// the spec will become our source of truth for resource limits.
oldSpec, err := cntr.Container.Get().Spec()
if err != nil {
return fmt.Errorf("failed to get container spec: %v", err)
}
newSpec, err := updateOCILinuxResource(oldSpec, resources)
if err != nil {
return fmt.Errorf("failed to update resource in spec: %v", err)
}
info := cntr.Container.Get().Info()
any, err := typeurl.MarshalAny(newSpec)
if err != nil {
return fmt.Errorf("failed to marshal spec %+v: %v", newSpec, err)
}
info.Spec = any
// TODO(random-liu): Add helper function in containerd to do the update.
if _, err := c.client.ContainerService().Update(ctx, info, "spec"); err != nil {
return fmt.Errorf("failed to update container spec: %v", err)
}
defer func() {
if retErr != nil {
// Reset spec on error.
any, err := typeurl.MarshalAny(oldSpec)
if err != nil {
glog.Errorf("Failed to marshal spec %+v for container %q: %v", oldSpec, id, err)
return
}
info.Spec = any
if _, err := c.client.ContainerService().Update(ctx, info, "spec"); err != nil {
glog.Errorf("Failed to recover spec %+v for container %q: %v", oldSpec, id, err)
}
}
}()
container, err := c.client.LoadContainer(ctx, id)
if err != nil {
return fmt.Errorf("failed to load container: %v", err)
}
defer func() {
if retErr == nil {
// Update container client if no error is returned.
// NOTE(random-liu): By updating container client, we'll be able
// to get latest OCI spec from it, which includes the up-to-date
// container resource limits. This will be useful after the debug
// api is introduced.
cntr.Container.Set(container)
}
}()
// If container is not running, only update spec is enough, new resource
// limit will be applied when container start.
if status.State() != runtime.ContainerState_CONTAINER_RUNNING {
return nil
}
task, err := container.Task(ctx, nil)
if err != nil {
if errdefs.IsNotFound(err) {
// Task exited already.
return nil
}
return fmt.Errorf("failed to get task: %v", err)
}
// newSpec.Linux won't be nil
if err := task.Update(ctx, containerd.WithResources(newSpec.Linux.Resources)); err != nil {
if errdefs.IsNotFound(err) {
// Task exited already.
return nil
}
return fmt.Errorf("failed to update resources: %v", err)
}
return nil
}
// updateOCILinuxResource updates container resource limit.
func updateOCILinuxResource(spec *runtimespec.Spec, new *runtime.LinuxContainerResources) (*runtimespec.Spec, error) {
// Copy to make sure old spec is not changed.
cloned, err := conversion.NewCloner().DeepCopy(spec)
if err != nil {
return nil, fmt.Errorf("failed to deep copy: %v", err)
}
g := generate.NewFromSpec(cloned.(*runtimespec.Spec))
if new.GetCpuPeriod() != 0 {
g.SetLinuxResourcesCPUPeriod(uint64(new.GetCpuPeriod()))
}
if new.GetCpuQuota() != 0 {
g.SetLinuxResourcesCPUQuota(new.GetCpuQuota())
}
if new.GetCpuShares() != 0 {
g.SetLinuxResourcesCPUShares(uint64(new.GetCpuShares()))
}
if new.GetMemoryLimitInBytes() != 0 {
g.SetLinuxResourcesMemoryLimit(new.GetMemoryLimitInBytes())
}
// OOMScore is not updatable.
if new.GetCpusetCpus() != "" {
g.SetLinuxResourcesCPUCpus(new.GetCpusetCpus())
}
if new.GetCpusetMems() != "" {
g.SetLinuxResourcesCPUMems(new.GetCpusetMems())
}
return g.Spec(), nil
}

150
pkg/server/container_update_resources_test.go
View File

@@ -25,27 +25,137 @@ import (
"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
)
func TestToOCIResources(t *testing.T) {
resources := &runtime.LinuxContainerResources{
CpuPeriod: 10000,
CpuQuota: 20000,
CpuShares: 300,
MemoryLimitInBytes: 4000000,
OomScoreAdj: -500,
CpusetCpus: "6,7",
CpusetMems: "8,9",
}
expected := &runtimespec.LinuxResources{
CPU: &runtimespec.LinuxCPU{
Period: proto.Uint64(10000),
Quota: proto.Int64(20000),
Shares: proto.Uint64(300),
Cpus: "6,7",
Mems: "8,9",
func TestUpdateOCILinuxResource(t *testing.T) {
oomscoreadj := new(int)
*oomscoreadj = -500
for desc, test := range map[string]struct {
spec *runtimespec.Spec
resources *runtime.LinuxContainerResources
expected *runtimespec.Spec
expectErr bool
}{
"should be able to update each resource": {
spec: &runtimespec.Spec{
Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
Linux: &runtimespec.Linux{
Resources: &runtimespec.LinuxResources{
Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
CPU: &runtimespec.LinuxCPU{
Shares: proto.Uint64(1111),
Quota: proto.Int64(2222),
Period: proto.Uint64(3333),
Cpus: "0-1",
Mems: "2-3",
},
},
},
},
resources: &runtime.LinuxContainerResources{
CpuPeriod: 6666,
CpuQuota: 5555,
CpuShares: 4444,
MemoryLimitInBytes: 54321,
OomScoreAdj: 500,
CpusetCpus: "4-5",
CpusetMems: "6-7",
},
expected: &runtimespec.Spec{
Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
Linux: &runtimespec.Linux{
Resources: &runtimespec.LinuxResources{
Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
CPU: &runtimespec.LinuxCPU{
Shares: proto.Uint64(4444),
Quota: proto.Int64(5555),
Period: proto.Uint64(6666),
Cpus: "4-5",
Mems: "6-7",
},
},
},
},
},
Memory: &runtimespec.LinuxMemory{
Limit: proto.Int64(4000000),
"should skip empty fields": {
spec: &runtimespec.Spec{
Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
Linux: &runtimespec.Linux{
Resources: &runtimespec.LinuxResources{
Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
CPU: &runtimespec.LinuxCPU{
Shares: proto.Uint64(1111),
Quota: proto.Int64(2222),
Period: proto.Uint64(3333),
Cpus: "0-1",
Mems: "2-3",
},
},
},
},
resources: &runtime.LinuxContainerResources{
CpuQuota: 5555,
CpuShares: 4444,
MemoryLimitInBytes: 54321,
OomScoreAdj: 500,
CpusetMems: "6-7",
},
expected: &runtimespec.Spec{
Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
Linux: &runtimespec.Linux{
Resources: &runtimespec.LinuxResources{
Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
CPU: &runtimespec.LinuxCPU{
Shares: proto.Uint64(4444),
Quota: proto.Int64(5555),
Period: proto.Uint64(3333),
Cpus: "0-1",
Mems: "6-7",
},
},
},
},
},
"should be able to fill empty fields": {
spec: &runtimespec.Spec{
Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
Linux: &runtimespec.Linux{
Resources: &runtimespec.LinuxResources{
Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
},
},
},
resources: &runtime.LinuxContainerResources{
CpuPeriod: 6666,
CpuQuota: 5555,
CpuShares: 4444,
MemoryLimitInBytes: 54321,
OomScoreAdj: 500,
CpusetCpus: "4-5",
CpusetMems: "6-7",
},
expected: &runtimespec.Spec{
Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
Linux: &runtimespec.Linux{
Resources: &runtimespec.LinuxResources{
Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
CPU: &runtimespec.LinuxCPU{
Shares: proto.Uint64(4444),
Quota: proto.Int64(5555),
Period: proto.Uint64(6666),
Cpus: "4-5",
Mems: "6-7",
},
},
},
},
},
} {
t.Logf("TestCase %q", desc)
got, err := updateOCILinuxResource(test.spec, test.resources)
if test.expectErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
assert.Equal(t, test.expected, got)
}
assert.Equal(t, expected, toOCIResources(resources))
}

2
pkg/server/events.go
View File

@@ -104,7 +104,7 @@ func (em *eventMonitor) handleEvent(evt *events.Envelope) {
return
}
// Attach container IO so that `Delete` could cleanup the stream properly.
task, err := cntr.Container.Task(context.Background(),
task, err := cntr.Container.Get().Task(context.Background(),
func(*containerd.FIFOSet) (containerd.IO, error) {
return cntr.IO, nil
},