github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #43089 from krousey/upgrades

Browse Source 
Automatic merge from submit-queue

Add guards for StatefulSet and AppArmor upgrade testing

This PR adds automated upgrade infrastructure to allow test suites to know what versions and node images are going to be testing and whether or not they should be skipped. It also adds a guard to prevent StatefulSets from being tested with versions prior to 1.5.0, and a guard to prevent AppArmor from running on distros other than gci and ubuntu.
This commit is contained in:
Kubernetes Submit Queue 2017-03-16 09:26:56 -07:00 committed by GitHub
commit 67a6c68d77
7 changed files with 162 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/e2e/BUILD
View File

@ -196,6 +196,7 @@ go_library(
"//vendor:k8s.io/apimachinery/pkg/watch",
"//vendor:k8s.io/apiserver/pkg/authentication/serviceaccount",
"//vendor:k8s.io/apiserver/pkg/registry/generic/registry",
"//vendor:k8s.io/client-go/discovery",
"//vendor:k8s.io/client-go/kubernetes",
"//vendor:k8s.io/client-go/pkg/api/v1",
"//vendor:k8s.io/client-go/pkg/apis/extensions/v1beta1",

141
test/e2e/cluster_upgrade.go
View File

@ -17,6 +17,8 @@ limitations under the License.
package e2e
import (
"k8s.io/client-go/discovery"
"k8s.io/kubernetes/pkg/util/version"
"k8s.io/kubernetes/test/e2e/chaosmonkey"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/upgrades"
@ -50,18 +52,22 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
framework.KubeDescribe("master upgrade", func() {
It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() {
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
framework.ExpectNoError(err)
cm := chaosmonkey.New(func() {
v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
framework.ExpectNoError(err)
framework.ExpectNoError(framework.MasterUpgrade(v))
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
target := upgCtx.Versions[1].Version.String()
framework.ExpectNoError(framework.MasterUpgrade(target))
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
})
for _, t := range upgradeTests {
cm.RegisterInterface(&chaosMonkeyAdapter{
cma := chaosMonkeyAdapter{
test: t,
framework: testFrameworks[t.Name()],
upgradeType: upgrades.MasterUpgrade,
})
upgCtx: *upgCtx,
}
cm.Register(cma.Test)
}
cm.Do()
@ -70,18 +76,22 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
framework.KubeDescribe("node upgrade", func() {
It("should maintain a functioning cluster [Feature:NodeUpgrade]", func() {
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
framework.ExpectNoError(err)
cm := chaosmonkey.New(func() {
v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
framework.ExpectNoError(err)
framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
target := upgCtx.Versions[1].Version.String()
framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
})
for _, t := range upgradeTests {
cm.RegisterInterface(&chaosMonkeyAdapter{
cma := chaosMonkeyAdapter{
test: t,
framework: testFrameworks[t.Name()],
upgradeType: upgrades.NodeUpgrade,
})
upgCtx: *upgCtx,
}
cm.Register(cma.Test)
}
cm.Do()
})
@ -89,20 +99,24 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
framework.KubeDescribe("cluster upgrade", func() {
It("should maintain a functioning cluster [Feature:ClusterUpgrade]", func() {
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
framework.ExpectNoError(err)
cm := chaosmonkey.New(func() {
v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
framework.ExpectNoError(err)
framework.ExpectNoError(framework.MasterUpgrade(v))
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
target := upgCtx.Versions[1].Version.String()
framework.ExpectNoError(framework.MasterUpgrade(target))
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
})
for _, t := range upgradeTests {
cm.RegisterInterface(&chaosMonkeyAdapter{
cma := chaosMonkeyAdapter{
test: t,
framework: testFrameworks[t.Name()],
upgradeType: upgrades.ClusterUpgrade,
})
upgCtx: *upgCtx,
}
cm.Register(cma.Test)
}
cm.Do()
})
@ -121,21 +135,25 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
framework.KubeDescribe("cluster downgrade", func() {
It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() {
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
framework.ExpectNoError(err)
cm := chaosmonkey.New(func() {
// Yes this really is a downgrade. And nodes must downgrade first.
v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
framework.ExpectNoError(err)
framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
framework.ExpectNoError(framework.MasterUpgrade(v))
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
target := upgCtx.Versions[1].Version.String()
framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
framework.ExpectNoError(framework.MasterUpgrade(target))
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
})
for _, t := range upgradeTests {
cm.RegisterInterface(&chaosMonkeyAdapter{
cma := chaosMonkeyAdapter{
test: t,
framework: testFrameworks[t.Name()],
upgradeType: upgrades.ClusterUpgrade,
})
upgCtx: *upgCtx,
}
cm.Register(cma.Test)
}
cm.Do()
})
@ -143,6 +161,8 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
})
var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
f := framework.NewDefaultFramework("etc-upgrade")
// Create the frameworks here because we can only create them
// in a "Describe".
testFrameworks := map[string]*framework.Framework{}
@ -152,16 +172,21 @@ var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
framework.KubeDescribe("etcd upgrade", func() {
It("should maintain a functioning cluster", func() {
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "")
framework.ExpectNoError(err)
cm := chaosmonkey.New(func() {
framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion))
// TODO(mml): verify the etcd version
})
for _, t := range upgradeTests {
cm.RegisterInterface(&chaosMonkeyAdapter{
cma := chaosMonkeyAdapter{
test: t,
framework: testFrameworks[t.Name()],
upgradeType: upgrades.EtcdUpgrade,
})
upgCtx: *upgCtx,
}
cm.Register(cma.Test)
}
cm.Do()
@ -173,16 +198,60 @@ type chaosMonkeyAdapter struct {
test upgrades.Test
framework *framework.Framework
upgradeType upgrades.UpgradeType
upgCtx upgrades.UpgradeContext
}
func (cma *chaosMonkeyAdapter) Setup() {
func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) {
if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.upgCtx) {
By("skipping test " + cma.test.Name())
sem.Ready()
return
}
cma.test.Setup(cma.framework)
defer cma.test.Teardown(cma.framework)
sem.Ready()
cma.test.Test(cma.framework, sem.StopCh, cma.upgradeType)
}
func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) {
cma.test.Test(cma.framework, stopCh, cma.upgradeType)
}
func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget string) (*upgrades.UpgradeContext, error) {
current, err := c.ServerVersion()
if err != nil {
return nil, err
}
func (cma *chaosMonkeyAdapter) Teardown() {
cma.test.Teardown(cma.framework)
curVer, err := version.ParseSemantic(current.String())
if err != nil {
return nil, err
}
upgCtx := &upgrades.UpgradeContext{
Versions: []upgrades.VersionContext{
{
Version: *curVer,
NodeImage: framework.TestContext.NodeOSDistro,
},
},
}
if len(upgradeTarget) == 0 {
return upgCtx, nil
}
next, err := framework.RealVersion(upgradeTarget)
if err != nil {
return nil, err
}
nextVer, err := version.ParseSemantic(next)
if err != nil {
return nil, err
}
upgCtx.Versions = append(upgCtx.Versions, upgrades.VersionContext{
Version: *nextVer,
NodeImage: framework.TestContext.UpgradeImage,
})
return upgCtx, nil
}

5
test/e2e/common/apparmor.go
View File

@ -32,8 +32,11 @@ const (
appArmorDeniedPath = "/expect_permission_denied"
)
// AppArmorDistros are distros with AppArmor support
var AppArmorDistros = []string{"gci", "ubuntu"}
func SkipIfAppArmorNotSupported() {
framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
framework.SkipUnlessNodeOSDistroIs(AppArmorDistros...)
}
func LoadAppArmorProfiles(f *framework.Framework) {

1
test/e2e/upgrades/BUILD
View File

@ -33,6 +33,7 @@ go_library(
"//pkg/controller:go_default_library",
"//pkg/controller/deployment/util:go_default_library",
"//pkg/kubelet/sysctl:go_default_library",
"//pkg/util/version:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//vendor:github.com/onsi/ginkgo",

15
test/e2e/upgrades/apparmor.go
View File

@ -34,9 +34,22 @@ type AppArmorUpgradeTest struct {
func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" }
func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool {
var supportedImages map[string]bool
for _, d := range common.AppArmorDistros {
supportedImages[d] = true
}
for _, vCtx := range upgCtx.Versions {
if !supportedImages[vCtx.NodeImage] {
return true
}
}
return false
}
// Setup creates a secret and then verifies that a pod can consume it.
func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) {
common.SkipIfAppArmorNotSupported()
By("Loading AppArmor profiles to nodes")
common.LoadAppArmorProfiles(f)

12
test/e2e/upgrades/statefulset.go
View File

@ -22,6 +22,7 @@ import (
"k8s.io/kubernetes/pkg/api/v1"
apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
"k8s.io/kubernetes/pkg/util/version"
"k8s.io/kubernetes/test/e2e/framework"
)
@ -35,6 +36,17 @@ type StatefulSetUpgradeTest struct {
func (StatefulSetUpgradeTest) Name() string { return "statefulset-upgrade" }
func (StatefulSetUpgradeTest) Skip(upgCtx UpgradeContext) bool {
minVersion := version.MustParseSemantic("1.5.0")
for _, vCtx := range upgCtx.Versions {
if vCtx.Version.LessThan(minVersion) {
return true
}
}
return false
}
// Setup creates a StatefulSet and a HeadlessService. It verifies the basic SatefulSet properties
func (t *StatefulSetUpgradeTest) Setup(f *framework.Framework) {
ssName := "ss"

26
test/e2e/upgrades/upgrade.go
View File

@ -18,7 +18,10 @@ limitations under the License.
// features before, during, and after different types of upgrades.
package upgrades
import "k8s.io/kubernetes/test/e2e/framework"
import (
"k8s.io/kubernetes/pkg/util/version"
"k8s.io/kubernetes/test/e2e/framework"
)
// UpgradeType represents different types of upgrades.
type UpgradeType int
@ -57,3 +60,24 @@ type Test interface {
// aren't already cleaned up by the framework.
Teardown(f *framework.Framework)
}
// Skippable is an interface that an upgrade test can implement to be
// able to indicate that it should be skipped.
type Skippable interface {
// Skip should return true if test should be skipped. upgCtx
// provides information about the upgrade that is going to
// occur.
Skip(upgCtx UpgradeContext) bool
}
// UpgradeContext contains information about all the stages of the
// upgrade that is going to occur.
type UpgradeContext struct {
Versions []VersionContext
}
// VersionContext represents a stage of the upgrade.
type VersionContext struct {
Version version.Version
NodeImage string
}