From d0887aec2b8190bb19eecabf7282c88732c1c6e3 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Tue, 1 Sep 2015 20:19:37 +0100 Subject: [PATCH 01/46] cadvisor bump to 0.16.0.2 --- Godeps/Godeps.json | 72 ++--- .../google/cadvisor/api/versions.go | 65 ++++- .../google/cadvisor/api/versions_test.go | 169 ++++++++++++ .../cadvisor/container/docker/factory.go | 3 +- .../cadvisor/container/docker/handler.go | 91 +++---- .../google/cadvisor/container/factory.go | 6 +- .../google/cadvisor/container/factory_test.go | 14 +- .../container/libcontainer/helpers.go | 90 ++++++- .../google/cadvisor/container/mock.go | 2 +- .../google/cadvisor/container/raw/factory.go | 8 +- .../google/cadvisor/container/raw/handler.go | 15 +- .../google/cadvisor/info/v1/container.go | 3 + .../google/cadvisor/info/v2/container.go | 34 ++- .../google/cadvisor/manager/manager.go | 3 +- .../google/cadvisor/metrics/prometheus.go | 133 +++++++--- .../cadvisor/metrics/prometheus_test.go | 16 ++ .../metrics/testdata/prometheus_metrics | 155 ----------- .../cadvisor/utils/machine/testdata/cpuinfo | 251 ------------------ 18 files changed, 569 insertions(+), 561 deletions(-) delete mode 100644 Godeps/_workspace/src/github.com/google/cadvisor/metrics/testdata/prometheus_metrics delete mode 100644 Godeps/_workspace/src/github.com/google/cadvisor/utils/machine/testdata/cpuinfo diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1a0e19fb676..153623aee71 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -261,93 +261,93 @@ }, { "ImportPath": "github.com/google/cadvisor/api", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/cache/memory", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/collector", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/container", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/events", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/fs", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/healthz", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/http", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/info/v1", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/info/v2", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/manager", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/metrics", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/pages", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/storage", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/summary", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/utils", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/validate", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/cadvisor/version", - "Comment": "0.16.0-81-g27fb6d5", - "Rev": "27fb6d593c6bffe274718119659815771e79e198" + "Comment": "0.16.0.2", + "Rev": "cefada41b87c35294533638733c563a349b95f05" }, { "ImportPath": "github.com/google/gofuzz", diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go b/Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go index ab5d93eef73..43839038576 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go @@ -19,6 +19,7 @@ import ( "net/http" "path" "strconv" + "time" "github.com/golang/glog" info "github.com/google/cadvisor/info/v1" @@ -449,8 +450,63 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma } } +func instCpuStats(last, cur *info.ContainerStats) (*v2.CpuInstStats, error) { + if last == nil { + return nil, nil + } + if !cur.Timestamp.After(last.Timestamp) { + return nil, fmt.Errorf("container stats move backwards in time") + } + if len(last.Cpu.Usage.PerCpu) != len(cur.Cpu.Usage.PerCpu) { + return nil, fmt.Errorf("different number of cpus") + } + timeDelta := cur.Timestamp.Sub(last.Timestamp) + if timeDelta <= 100*time.Millisecond { + return nil, fmt.Errorf("time delta unexpectedly small") + } + // Nanoseconds to gain precision and avoid having zero seconds if the + // difference between the timestamps is just under a second + timeDeltaNs := uint64(timeDelta.Nanoseconds()) + convertToRate := func(lastValue, curValue uint64) (uint64, error) { + if curValue < lastValue { + return 0, fmt.Errorf("cumulative stats decrease") + } + valueDelta := curValue - lastValue + return (valueDelta * 1e9) / timeDeltaNs, nil + } + total, err := convertToRate(last.Cpu.Usage.Total, cur.Cpu.Usage.Total) + if err != nil { + return nil, err + } + percpu := make([]uint64, len(last.Cpu.Usage.PerCpu)) + for i := range percpu { + var err error + percpu[i], err = convertToRate(last.Cpu.Usage.PerCpu[i], cur.Cpu.Usage.PerCpu[i]) + if err != nil { + return nil, err + } + } + user, err := convertToRate(last.Cpu.Usage.User, cur.Cpu.Usage.User) + if err != nil { + return nil, err + } + system, err := convertToRate(last.Cpu.Usage.System, cur.Cpu.Usage.System) + if err != nil { + return nil, err + } + return &v2.CpuInstStats{ + Usage: v2.CpuInstUsage{ + Total: total, + PerCpu: percpu, + User: user, + System: system, + }, + }, nil +} + func convertStats(cont *info.ContainerInfo) []v2.ContainerStats { - stats := []v2.ContainerStats{} + stats := make([]v2.ContainerStats, 0, len(cont.Stats)) + var last *info.ContainerStats for _, val := range cont.Stats { stat := v2.ContainerStats{ Timestamp: val.Timestamp, @@ -463,6 +519,13 @@ func convertStats(cont *info.ContainerInfo) []v2.ContainerStats { } if stat.HasCpu { stat.Cpu = val.Cpu + cpuInst, err := instCpuStats(last, val) + if err != nil { + glog.Warningf("Could not get instant cpu stats: %v", err) + } else { + stat.CpuInst = cpuInst + } + last = val } if stat.HasMemory { stat.Memory = val.Memory diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/api/versions_test.go b/Godeps/_workspace/src/github.com/google/cadvisor/api/versions_test.go index 0a107858a1b..82675703f29 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/api/versions_test.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/api/versions_test.go @@ -19,9 +19,11 @@ import ( "net/http" "reflect" "testing" + "time" "github.com/google/cadvisor/events" info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" ) @@ -78,3 +80,170 @@ func TestGetEventRequestDoubleArgument(t *testing.T) { assert.True(t, stream) assert.Nil(t, err) } + +func TestInstCpuStats(t *testing.T) { + tests := []struct { + last *info.ContainerStats + cur *info.ContainerStats + want *v2.CpuInstStats + }{ + // Last is missing + { + nil, + &info.ContainerStats{}, + nil, + }, + // Goes back in time + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(time.Second), + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + }, + nil, + }, + // Zero time delta + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + }, + nil, + }, + // Unexpectedly small time delta + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(30 * time.Millisecond), + }, + nil, + }, + // Different number of cpus + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + PerCpu: []uint64{100, 200}, + }, + }, + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(time.Second), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + PerCpu: []uint64{100, 200, 300}, + }, + }, + }, + nil, + }, + // Stat numbers decrease + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 300, + PerCpu: []uint64{100, 200}, + User: 250, + System: 50, + }, + }, + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(time.Second), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 200, + PerCpu: []uint64{100, 100}, + User: 150, + System: 50, + }, + }, + }, + nil, + }, + // One second elapsed + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 300, + PerCpu: []uint64{100, 200}, + User: 250, + System: 50, + }, + }, + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(time.Second), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 500, + PerCpu: []uint64{200, 300}, + User: 400, + System: 100, + }, + }, + }, + &v2.CpuInstStats{ + Usage: v2.CpuInstUsage{ + Total: 200, + PerCpu: []uint64{100, 100}, + User: 150, + System: 50, + }, + }, + }, + // Two seconds elapsed + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 300, + PerCpu: []uint64{100, 200}, + User: 250, + System: 50, + }, + }, + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(2 * time.Second), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 500, + PerCpu: []uint64{200, 300}, + User: 400, + System: 100, + }, + }, + }, + &v2.CpuInstStats{ + Usage: v2.CpuInstUsage{ + Total: 100, + PerCpu: []uint64{50, 50}, + User: 75, + System: 25, + }, + }, + }, + } + for _, c := range tests { + got, err := instCpuStats(c.last, c.cur) + if err != nil { + if c.want == nil { + continue + } + t.Errorf("Unexpected error: %v", err) + } + assert.Equal(t, c.want, got) + } +} diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go index 7dd3b1edb89..e449586edf2 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go @@ -96,7 +96,7 @@ func (self *dockerFactory) String() string { return DockerNamespace } -func (self *dockerFactory) NewContainerHandler(name string) (handler container.ContainerHandler, err error) { +func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool) (handler container.ContainerHandler, err error) { client, err := docker.NewClient(*ArgDockerEndpoint) if err != nil { return @@ -108,6 +108,7 @@ func (self *dockerFactory) NewContainerHandler(name string) (handler container.C self.fsInfo, self.usesAufsDriver, &self.cgroupSubsystems, + inHostNamespace, ) return } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go index 2ad1f306c1f..c2ff1af697d 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go @@ -71,6 +71,18 @@ type dockerContainerHandler struct { // Metadata labels associated with the container. labels map[string]string + + // The container PID used to switch namespaces as required + pid int + + // Image name used for this container. + image string + + // The host root FS to read + rootFs string + + // The network mode of the container + networkMode string } func newDockerContainerHandler( @@ -80,6 +92,7 @@ func newDockerContainerHandler( fsInfo fs.FsInfo, usesAufsDriver bool, cgroupSubsystems *containerLibcontainer.CgroupSubsystems, + inHostNamespace bool, ) (container.ContainerHandler, error) { // Create the cgroup paths. cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) @@ -95,6 +108,11 @@ func newDockerContainerHandler( Paths: cgroupPaths, } + rootFs := "/" + if !inHostNamespace { + rootFs = "/rootfs" + } + id := ContainerNameToDockerId(name) handler := &dockerContainerHandler{ id: id, @@ -105,6 +123,7 @@ func newDockerContainerHandler( cgroupManager: cgroupManager, usesAufsDriver: usesAufsDriver, fsInfo: fsInfo, + rootFs: rootFs, } handler.storageDirs = append(handler.storageDirs, path.Join(*dockerRootDir, pathToAufsDir, id)) @@ -114,11 +133,14 @@ func newDockerContainerHandler( return nil, fmt.Errorf("failed to inspect container %q: %v", id, err) } handler.creationTime = ctnr.Created + handler.pid = ctnr.State.Pid // Add the name and bare ID as aliases of the container. handler.aliases = append(handler.aliases, strings.TrimPrefix(ctnr.Name, "/")) handler.aliases = append(handler.aliases, id) handler.labels = ctnr.Config.Labels + handler.image = ctnr.Config.Image + handler.networkMode = ctnr.HostConfig.NetworkMode return handler, nil } @@ -167,21 +189,23 @@ func libcontainerConfigToContainerSpec(config *libcontainerConfigs.Config, mi *i } spec.Cpu.Mask = utils.FixCpuMask(config.Cgroups.CpusetCpus, mi.NumCores) - // Docker reports a loop device for containers with --net=host. Ignore - // those too. - networkCount := 0 - for _, n := range config.Networks { - if n.Type != "loopback" { - networkCount += 1 - } - } - - spec.HasNetwork = networkCount > 0 spec.HasDiskIo = true return spec } +var ( + hasNetworkModes = map[string]bool{ + "host": true, + "bridge": true, + "default": true, + } +) + +func hasNet(networkMode string) bool { + return hasNetworkModes[networkMode] +} + func (self *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) { mi, err := self.machineInfoFactory.GetMachineInfo() if err != nil { @@ -198,6 +222,8 @@ func (self *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) { spec.HasFilesystem = true } spec.Labels = self.labels + spec.Image = self.image + spec.HasNetwork = hasNet(self.networkMode) return spec, err } @@ -247,32 +273,16 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error // TODO(vmarmol): Get from libcontainer API instead of cgroup manager when we don't have to support older Dockers. func (self *dockerContainerHandler) GetStats() (*info.ContainerStats, error) { - config, err := self.readLibcontainerConfig() - if err != nil { - return nil, err - } - - var networkInterfaces []string - if len(config.Networks) > 0 { - // ContainerStats only reports stat for one network device. - // TODO(vmarmol): Handle multiple physical network devices. - for _, n := range config.Networks { - // Take the first non-loopback. - if n.Type != "loopback" { - networkInterfaces = []string{n.HostInterfaceName} - break - } - } - } - stats, err := containerLibcontainer.GetStats(self.cgroupManager, networkInterfaces) + stats, err := containerLibcontainer.GetStats(self.cgroupManager, self.rootFs, self.pid) if err != nil { return stats, err } - - // TODO(rjnagal): Remove the conversion when network stats are read from libcontainer. - convertInterfaceStats(&stats.Network.InterfaceStats) - for i := range stats.Network.Interfaces { - convertInterfaceStats(&stats.Network.Interfaces[i]) + // Clean up stats for containers that don't have their own network - this + // includes containers running in Kubernetes pods that use the network of the + // infrastructure container. This stops metrics being reported multiple times + // for each container in a pod. + if !hasNet(self.networkMode) { + stats.Network = info.NetworkStats{} } // Get filesystem stats. @@ -284,21 +294,6 @@ func (self *dockerContainerHandler) GetStats() (*info.ContainerStats, error) { return stats, nil } -func convertInterfaceStats(stats *info.InterfaceStats) { - net := *stats - - // Ingress for host veth is from the container. - // Hence tx_bytes stat on the host veth is actually number of bytes received by the container. - stats.RxBytes = net.TxBytes - stats.RxPackets = net.TxPackets - stats.RxErrors = net.TxErrors - stats.RxDropped = net.TxDropped - stats.TxBytes = net.RxBytes - stats.TxPackets = net.RxPackets - stats.TxErrors = net.RxErrors - stats.TxDropped = net.RxDropped -} - func (self *dockerContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) { if self.name != "/docker" { return []info.ContainerReference{}, nil diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/factory.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/factory.go index d5ab8290359..9a120f7caa2 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/factory.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/factory.go @@ -23,7 +23,7 @@ import ( type ContainerHandlerFactory interface { // Create a new ContainerHandler using this factory. CanHandleAndAccept() must have returned true. - NewContainerHandler(name string) (c ContainerHandler, err error) + NewContainerHandler(name string, inHostNamespace bool) (c ContainerHandler, err error) // Returns whether this factory can handle and accept the specified container. CanHandleAndAccept(name string) (handle bool, accept bool, err error) @@ -60,7 +60,7 @@ func HasFactories() bool { } // Create a new ContainerHandler for the specified container. -func NewContainerHandler(name string) (ContainerHandler, bool, error) { +func NewContainerHandler(name string, inHostNamespace bool) (ContainerHandler, bool, error) { factoriesLock.RLock() defer factoriesLock.RUnlock() @@ -76,7 +76,7 @@ func NewContainerHandler(name string) (ContainerHandler, bool, error) { return nil, false, nil } glog.V(3).Infof("Using factory %q for container %q", factory, name) - handle, err := factory.NewContainerHandler(name) + handle, err := factory.NewContainerHandler(name, inHostNamespace) return handle, canAccept, err } else { glog.V(4).Infof("Factory %q was unable to handle container %q", factory, name) diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/factory_test.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/factory_test.go index 991c365ab3b..8c4bb0429d7 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/factory_test.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/factory_test.go @@ -39,7 +39,7 @@ func (self *mockContainerHandlerFactory) CanHandleAndAccept(name string) (bool, return self.CanHandleValue, self.CanAcceptValue, nil } -func (self *mockContainerHandlerFactory) NewContainerHandler(name string) (ContainerHandler, error) { +func (self *mockContainerHandlerFactory) NewContainerHandler(name string, isHostNamespace bool) (ContainerHandler, error) { args := self.Called(name) return args.Get(0).(ContainerHandler), args.Error(1) } @@ -60,13 +60,13 @@ func TestNewContainerHandler_FirstMatches(t *testing.T) { RegisterContainerHandlerFactory(allwaysYes) // The yes factory should be asked to create the ContainerHandler. - mockContainer, err := mockFactory.NewContainerHandler(testContainerName) + mockContainer, err := mockFactory.NewContainerHandler(testContainerName, true) if err != nil { t.Error(err) } allwaysYes.On("NewContainerHandler", testContainerName).Return(mockContainer, nil) - cont, _, err := NewContainerHandler(testContainerName) + cont, _, err := NewContainerHandler(testContainerName, true) if err != nil { t.Error(err) } @@ -93,13 +93,13 @@ func TestNewContainerHandler_SecondMatches(t *testing.T) { RegisterContainerHandlerFactory(allwaysYes) // The yes factory should be asked to create the ContainerHandler. - mockContainer, err := mockFactory.NewContainerHandler(testContainerName) + mockContainer, err := mockFactory.NewContainerHandler(testContainerName, true) if err != nil { t.Error(err) } allwaysYes.On("NewContainerHandler", testContainerName).Return(mockContainer, nil) - cont, _, err := NewContainerHandler(testContainerName) + cont, _, err := NewContainerHandler(testContainerName, true) if err != nil { t.Error(err) } @@ -125,7 +125,7 @@ func TestNewContainerHandler_NoneMatch(t *testing.T) { } RegisterContainerHandlerFactory(allwaysNo2) - _, _, err := NewContainerHandler(testContainerName) + _, _, err := NewContainerHandler(testContainerName, true) if err == nil { t.Error("Expected NewContainerHandler to fail") } @@ -148,7 +148,7 @@ func TestNewContainerHandler_Accept(t *testing.T) { } RegisterContainerHandlerFactory(cannotAccept) - _, accept, err := NewContainerHandler(testContainerName) + _, accept, err := NewContainerHandler(testContainerName, true) if err != nil { t.Error("Expected NewContainerHandler to succeed") } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go index 0cd4c119b9d..a3b2e50a1f6 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go @@ -15,14 +15,19 @@ package libcontainer import ( + "bufio" "fmt" + "io/ioutil" "path" + "regexp" + "strconv" + "strings" "time" "github.com/docker/libcontainer" "github.com/docker/libcontainer/cgroups" + "github.com/golang/glog" info "github.com/google/cadvisor/info/v1" - "github.com/google/cadvisor/utils/sysinfo" ) type CgroupSubsystems struct { @@ -74,7 +79,7 @@ var supportedSubsystems map[string]struct{} = map[string]struct{}{ } // Get cgroup and networking stats of the specified container -func GetStats(cgroupManager cgroups.Manager, networkInterfaces []string) (*info.ContainerStats, error) { +func GetStats(cgroupManager cgroups.Manager, rootFs string, pid int) (*info.ContainerStats, error) { cgroupStats, err := cgroupManager.GetStats() if err != nil { return nil, err @@ -84,23 +89,90 @@ func GetStats(cgroupManager cgroups.Manager, networkInterfaces []string) (*info. } stats := toContainerStats(libcontainerStats) - // TODO(rjnagal): Use networking stats directly from libcontainer. - stats.Network.Interfaces = make([]info.InterfaceStats, len(networkInterfaces)) - for i := range networkInterfaces { - interfaceStats, err := sysinfo.GetNetworkStats(networkInterfaces[i]) + // If we know the pid then get network stats from /proc//net/dev + if pid > 0 { + netStats, err := networkStatsFromProc(rootFs, pid) if err != nil { - return stats, err + glog.V(2).Infof("Unable to get network stats from pid %d: %v", pid, err) + } else { + stats.Network.Interfaces = append(stats.Network.Interfaces, netStats...) } - stats.Network.Interfaces[i] = interfaceStats } + // For backwards compatibility. - if len(networkInterfaces) > 0 { + if len(stats.Network.Interfaces) > 0 { stats.Network.InterfaceStats = stats.Network.Interfaces[0] } return stats, nil } +func networkStatsFromProc(rootFs string, pid int) ([]info.InterfaceStats, error) { + netStatsFile := path.Join(rootFs, "proc", strconv.Itoa(pid), "/net/dev") + + ifaceStats, err := scanInterfaceStats(netStatsFile) + if err != nil { + return []info.InterfaceStats{}, fmt.Errorf("couldn't read network stats: %v", err) + } + + return ifaceStats, nil +} + +var ( + ignoredDevicePrefixes = []string{"lo", "veth", "docker"} + netStatLineRE = regexp.MustCompile("[ ]*(.+):([ ]+[0-9]+){16}") +) + +func isIgnoredDevice(ifName string) bool { + for _, prefix := range ignoredDevicePrefixes { + if strings.HasPrefix(strings.ToLower(ifName), prefix) { + return true + } + } + return false +} + +func scanInterfaceStats(netStatsFile string) ([]info.InterfaceStats, error) { + var ( + bkt uint64 + ) + + stats := []info.InterfaceStats{} + + data, err := ioutil.ReadFile(netStatsFile) + if err != nil { + return stats, fmt.Errorf("failure opening %s: %v", netStatsFile, err) + } + + reader := strings.NewReader(string(data)) + scanner := bufio.NewScanner(reader) + + scanner.Split(bufio.ScanLines) + + for scanner.Scan() { + line := scanner.Text() + if netStatLineRE.MatchString(line) { + line = strings.Replace(line, ":", "", -1) + + i := info.InterfaceStats{} + + _, err := fmt.Sscanf(line, "%s %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d", + &i.Name, &i.RxBytes, &i.RxPackets, &i.RxErrors, &i.RxDropped, &bkt, &bkt, &bkt, + &bkt, &i.TxBytes, &i.TxPackets, &i.TxErrors, &i.TxDropped, &bkt, &bkt, &bkt, &bkt) + + if err != nil { + return stats, fmt.Errorf("failure opening %s: %v", netStatsFile, err) + } + + if !isIgnoredDevice(i.Name) { + stats = append(stats, i) + } + } + } + + return stats, nil +} + func GetProcesses(cgroupManager cgroups.Manager) ([]int, error) { pids, err := cgroupManager.GetPids() if err != nil { diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/mock.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/mock.go index 7422b3ddbc1..f949c57908a 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/mock.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/mock.go @@ -109,7 +109,7 @@ func (self *FactoryForMockContainerHandler) String() string { return self.Name } -func (self *FactoryForMockContainerHandler) NewContainerHandler(name string) (ContainerHandler, error) { +func (self *FactoryForMockContainerHandler) NewContainerHandler(name string, inHostNamespace bool) (ContainerHandler, error) { handler := &MockContainerHandler{} if self.PrepareContainerHandlerFunc != nil { self.PrepareContainerHandlerFunc(name, handler) diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/raw/factory.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/raw/factory.go index be1c799b75f..5b45ee923c8 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/raw/factory.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/raw/factory.go @@ -45,8 +45,12 @@ func (self *rawFactory) String() string { return "raw" } -func (self *rawFactory) NewContainerHandler(name string) (container.ContainerHandler, error) { - return newRawContainerHandler(name, self.cgroupSubsystems, self.machineInfoFactory, self.fsInfo, self.watcher) +func (self *rawFactory) NewContainerHandler(name string, inHostNamespace bool) (container.ContainerHandler, error) { + rootFs := "/" + if !inHostNamespace { + rootFs = "/rootfs" + } + return newRawContainerHandler(name, self.cgroupSubsystems, self.machineInfoFactory, self.fsInfo, self.watcher, rootFs) } // The raw factory can handle any container. If --docker_only is set to false, non-docker containers are ignored. diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/raw/handler.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/raw/handler.go index 51405db74f7..8223b6e50f5 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/raw/handler.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/raw/handler.go @@ -61,9 +61,11 @@ type rawContainerHandler struct { fsInfo fs.FsInfo externalMounts []mount + + rootFs string } -func newRawContainerHandler(name string, cgroupSubsystems *libcontainer.CgroupSubsystems, machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, watcher *InotifyWatcher) (container.ContainerHandler, error) { +func newRawContainerHandler(name string, cgroupSubsystems *libcontainer.CgroupSubsystems, machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, watcher *InotifyWatcher, rootFs string) (container.ContainerHandler, error) { // Create the cgroup paths. cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) for key, val := range cgroupSubsystems.MountPoints { @@ -108,6 +110,7 @@ func newRawContainerHandler(name string, cgroupSubsystems *libcontainer.CgroupSu hasNetwork: hasNetwork, externalMounts: externalMounts, watcher: watcher, + rootFs: rootFs, }, nil } @@ -326,15 +329,7 @@ func (self *rawContainerHandler) getFsStats(stats *info.ContainerStats) error { } func (self *rawContainerHandler) GetStats() (*info.ContainerStats, error) { - nd, err := self.GetRootNetworkDevices() - if err != nil { - return new(info.ContainerStats), err - } - networkInterfaces := make([]string, len(nd)) - for i := range nd { - networkInterfaces[i] = nd[i].Name - } - stats, err := libcontainer.GetStats(self.cgroupManager, networkInterfaces) + stats, err := libcontainer.GetStats(self.cgroupManager, self.rootFs, os.Getpid()) if err != nil { return stats, err } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go b/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go index 91512eca296..3ca17c461f6 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go @@ -61,6 +61,9 @@ type ContainerSpec struct { HasCustomMetrics bool `json:"has_custom_metrics"` CustomMetrics []MetricSpec `json:"custom_metrics,omitempty"` + + // Image name used for this container. + Image string `json:"image,omitempty"` } // Container reference contains enough information to uniquely identify a container diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go b/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go index 74446890cac..cbf8c2525fa 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go @@ -80,14 +80,20 @@ type ContainerSpec struct { HasNetwork bool `json:"has_network"` HasFilesystem bool `json:"has_filesystem"` HasDiskIo bool `json:"has_diskio"` + + // Image name used for this container. + Image string `json:"image,omitempty"` } type ContainerStats struct { // The time of this stat point. Timestamp time.Time `json:"timestamp"` // CPU statistics - HasCpu bool `json:"has_cpu"` - Cpu v1.CpuStats `json:"cpu,omitempty"` + HasCpu bool `json:"has_cpu"` + // In nanoseconds (aggregated) + Cpu v1.CpuStats `json:"cpu,omitempty"` + // In nanocores per second (instantaneous) + CpuInst *CpuInstStats `json:"cpu_inst,omitempty"` // Disk IO statistics HasDiskIo bool `json:"has_diskio"` DiskIo v1.DiskIoStats `json:"diskio,omitempty"` @@ -204,3 +210,27 @@ type NetworkStats struct { // Network stats by interface. Interfaces []v1.InterfaceStats `json:"interfaces,omitempty"` } + +// Instantaneous CPU stats +type CpuInstStats struct { + Usage CpuInstUsage `json:"usage"` +} + +// CPU usage time statistics. +type CpuInstUsage struct { + // Total CPU usage. + // Units: nanocores per second + Total uint64 `json:"total"` + + // Per CPU/core usage of the container. + // Unit: nanocores per second + PerCpu []uint64 `json:"per_cpu_usage,omitempty"` + + // Time spent in user space. + // Unit: nanocores per second + User uint64 `json:"user"` + + // Time spent in kernel space. + // Unit: nanocores per second + System uint64 `json:"system"` +} diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/manager/manager.go b/Godeps/_workspace/src/github.com/google/cadvisor/manager/manager.go index df360263016..8526bc450ce 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/manager/manager.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/manager/manager.go @@ -380,6 +380,7 @@ func (self *manager) getV2Spec(cinfo *containerInfo) v2.ContainerSpec { HasNetwork: specV1.HasNetwork, HasDiskIo: specV1.HasDiskIo, HasCustomMetrics: specV1.HasCustomMetrics, + Image: specV1.Image, } if specV1.HasCpu { specV2.Cpu.Limit = specV1.Cpu.Limit @@ -736,7 +737,7 @@ func (m *manager) registerCollectors(collectorConfigs map[string]string, cont *c // Create a container. func (m *manager) createContainer(containerName string) error { - handler, accept, err := container.NewContainerHandler(containerName) + handler, accept, err := container.NewContainerHandler(containerName, m.inHostNamespace) if err != nil { return err } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus.go b/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus.go index f957c7c9cbb..a0c05779b45 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus.go @@ -61,7 +61,7 @@ type containerMetric struct { } func (cm *containerMetric) desc() *prometheus.Desc { - return prometheus.NewDesc(cm.name, cm.help, append([]string{"name", "id"}, cm.extraLabels...), nil) + return prometheus.NewDesc(cm.name, cm.help, append([]string{"name", "id", "image"}, cm.extraLabels...), nil) } // PrometheusCollector implements prometheus.Collector. @@ -287,60 +287,124 @@ func NewPrometheusCollector(infoProvider subcontainersInfoProvider) *PrometheusC }) }, }, { - name: "container_network_receive_bytes_total", - help: "Cumulative count of bytes received", - valueType: prometheus.CounterValue, + name: "container_network_receive_bytes_total", + help: "Cumulative count of bytes received", + valueType: prometheus.CounterValue, + extraLabels: []string{"interface"}, getValues: func(s *info.ContainerStats) metricValues { - return metricValues{{value: float64(s.Network.RxBytes)}} + values := make(metricValues, 0, len(s.Network.Interfaces)) + for _, value := range s.Network.Interfaces { + values = append(values, metricValue{ + value: float64(value.RxBytes), + labels: []string{value.Name}, + }) + } + return values }, }, { - name: "container_network_receive_packets_total", - help: "Cumulative count of packets received", - valueType: prometheus.CounterValue, + name: "container_network_receive_packets_total", + help: "Cumulative count of packets received", + valueType: prometheus.CounterValue, + extraLabels: []string{"interface"}, getValues: func(s *info.ContainerStats) metricValues { - return metricValues{{value: float64(s.Network.RxPackets)}} + values := make(metricValues, 0, len(s.Network.Interfaces)) + for _, value := range s.Network.Interfaces { + values = append(values, metricValue{ + value: float64(value.RxPackets), + labels: []string{value.Name}, + }) + } + return values }, }, { - name: "container_network_receive_packets_dropped_total", - help: "Cumulative count of packets dropped while receiving", - valueType: prometheus.CounterValue, + name: "container_network_receive_packets_dropped_total", + help: "Cumulative count of packets dropped while receiving", + valueType: prometheus.CounterValue, + extraLabels: []string{"interface"}, getValues: func(s *info.ContainerStats) metricValues { - return metricValues{{value: float64(s.Network.RxDropped)}} + values := make(metricValues, 0, len(s.Network.Interfaces)) + for _, value := range s.Network.Interfaces { + values = append(values, metricValue{ + value: float64(value.RxDropped), + labels: []string{value.Name}, + }) + } + return values }, }, { - name: "container_network_receive_errors_total", - help: "Cumulative count of errors encountered while receiving", - valueType: prometheus.CounterValue, + name: "container_network_receive_errors_total", + help: "Cumulative count of errors encountered while receiving", + valueType: prometheus.CounterValue, + extraLabels: []string{"interface"}, getValues: func(s *info.ContainerStats) metricValues { - return metricValues{{value: float64(s.Network.RxErrors)}} + values := make(metricValues, 0, len(s.Network.Interfaces)) + for _, value := range s.Network.Interfaces { + values = append(values, metricValue{ + value: float64(value.RxErrors), + labels: []string{value.Name}, + }) + } + return values }, }, { - name: "container_network_transmit_bytes_total", - help: "Cumulative count of bytes transmitted", - valueType: prometheus.CounterValue, + name: "container_network_transmit_bytes_total", + help: "Cumulative count of bytes transmitted", + valueType: prometheus.CounterValue, + extraLabels: []string{"interface"}, getValues: func(s *info.ContainerStats) metricValues { - return metricValues{{value: float64(s.Network.TxBytes)}} + values := make(metricValues, 0, len(s.Network.Interfaces)) + for _, value := range s.Network.Interfaces { + values = append(values, metricValue{ + value: float64(value.TxBytes), + labels: []string{value.Name}, + }) + } + return values }, }, { - name: "container_network_transmit_packets_total", - help: "Cumulative count of packets transmitted", - valueType: prometheus.CounterValue, + name: "container_network_transmit_packets_total", + help: "Cumulative count of packets transmitted", + valueType: prometheus.CounterValue, + extraLabels: []string{"interface"}, getValues: func(s *info.ContainerStats) metricValues { - return metricValues{{value: float64(s.Network.TxPackets)}} + values := make(metricValues, 0, len(s.Network.Interfaces)) + for _, value := range s.Network.Interfaces { + values = append(values, metricValue{ + value: float64(value.TxPackets), + labels: []string{value.Name}, + }) + } + return values }, }, { - name: "container_network_transmit_packets_dropped_total", - help: "Cumulative count of packets dropped while transmitting", - valueType: prometheus.CounterValue, + name: "container_network_transmit_packets_dropped_total", + help: "Cumulative count of packets dropped while transmitting", + valueType: prometheus.CounterValue, + extraLabels: []string{"interface"}, getValues: func(s *info.ContainerStats) metricValues { - return metricValues{{value: float64(s.Network.TxDropped)}} + values := make(metricValues, 0, len(s.Network.Interfaces)) + for _, value := range s.Network.Interfaces { + values = append(values, metricValue{ + value: float64(value.TxDropped), + labels: []string{value.Name}, + }) + } + return values }, }, { - name: "container_network_transmit_errors_total", - help: "Cumulative count of errors encountered while transmitting", - valueType: prometheus.CounterValue, + name: "container_network_transmit_errors_total", + help: "Cumulative count of errors encountered while transmitting", + valueType: prometheus.CounterValue, + extraLabels: []string{"interface"}, getValues: func(s *info.ContainerStats) metricValues { - return metricValues{{value: float64(s.Network.TxErrors)}} + values := make(metricValues, 0, len(s.Network.Interfaces)) + for _, value := range s.Network.Interfaces { + values = append(values, metricValue{ + value: float64(value.TxErrors), + labels: []string{value.Name}, + }) + } + return values }, }, { name: "container_tasks_state", @@ -401,12 +465,13 @@ func (c *PrometheusCollector) Collect(ch chan<- prometheus.Metric) { if len(container.Aliases) > 0 { name = container.Aliases[0] } + image := container.Spec.Image stats := container.Stats[0] for _, cm := range c.containerMetrics { desc := cm.desc() for _, metricValue := range cm.getValues(stats) { - ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append([]string{name, id}, metricValue.labels...)...) + ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append([]string{name, id, image}, metricValue.labels...)...) } } } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus_test.go b/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus_test.go index 3efbe043cc7..0bb6c5e1c87 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus_test.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus_test.go @@ -34,6 +34,9 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container ContainerReference: info.ContainerReference{ Name: "testcontainer", }, + Spec: info.ContainerSpec{ + Image: "test", + }, Stats: []*info.ContainerStats{ { Cpu: info.CpuStats{ @@ -68,6 +71,19 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container TxErrors: 20, TxDropped: 21, }, + Interfaces: []info.InterfaceStats{ + { + Name: "eth0", + RxBytes: 14, + RxPackets: 15, + RxErrors: 16, + RxDropped: 17, + TxBytes: 18, + TxPackets: 19, + TxErrors: 20, + TxDropped: 21, + }, + }, }, Filesystem: []info.FsStats{ { diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/metrics/testdata/prometheus_metrics b/Godeps/_workspace/src/github.com/google/cadvisor/metrics/testdata/prometheus_metrics deleted file mode 100644 index 70d5d977a11..00000000000 --- a/Godeps/_workspace/src/github.com/google/cadvisor/metrics/testdata/prometheus_metrics +++ /dev/null @@ -1,155 +0,0 @@ -# HELP container_cpu_system_seconds_total Cumulative system cpu time consumed in seconds. -# TYPE container_cpu_system_seconds_total counter -container_cpu_system_seconds_total{id="testcontainer",name="testcontainer"} 7e-09 -# HELP container_cpu_usage_seconds_total Cumulative cpu time consumed per cpu in seconds. -# TYPE container_cpu_usage_seconds_total counter -container_cpu_usage_seconds_total{cpu="cpu00",id="testcontainer",name="testcontainer"} 2e-09 -container_cpu_usage_seconds_total{cpu="cpu01",id="testcontainer",name="testcontainer"} 3e-09 -container_cpu_usage_seconds_total{cpu="cpu02",id="testcontainer",name="testcontainer"} 4e-09 -container_cpu_usage_seconds_total{cpu="cpu03",id="testcontainer",name="testcontainer"} 5e-09 -# HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds. -# TYPE container_cpu_user_seconds_total counter -container_cpu_user_seconds_total{id="testcontainer",name="testcontainer"} 6e-09 -# HELP container_fs_io_current Number of I/Os currently in progress -# TYPE container_fs_io_current gauge -container_fs_io_current{device="sda1",id="testcontainer",name="testcontainer"} 42 -container_fs_io_current{device="sda2",id="testcontainer",name="testcontainer"} 47 -# HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os -# TYPE container_fs_io_time_seconds_total counter -container_fs_io_time_seconds_total{device="sda1",id="testcontainer",name="testcontainer"} 4.3e-08 -container_fs_io_time_seconds_total{device="sda2",id="testcontainer",name="testcontainer"} 4.8e-08 -# HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds -# TYPE container_fs_io_time_weighted_seconds_total counter -container_fs_io_time_weighted_seconds_total{device="sda1",id="testcontainer",name="testcontainer"} 4.4e-08 -container_fs_io_time_weighted_seconds_total{device="sda2",id="testcontainer",name="testcontainer"} 4.9e-08 -# HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem. -# TYPE container_fs_limit_bytes gauge -container_fs_limit_bytes{device="sda1",id="testcontainer",name="testcontainer"} 22 -container_fs_limit_bytes{device="sda2",id="testcontainer",name="testcontainer"} 37 -# HELP container_fs_read_seconds_total Cumulative count of seconds spent reading -# TYPE container_fs_read_seconds_total counter -container_fs_read_seconds_total{device="sda1",id="testcontainer",name="testcontainer"} 2.7e-08 -container_fs_read_seconds_total{device="sda2",id="testcontainer",name="testcontainer"} 4.2e-08 -# HELP container_fs_reads_merged_total Cumulative count of reads merged -# TYPE container_fs_reads_merged_total counter -container_fs_reads_merged_total{device="sda1",id="testcontainer",name="testcontainer"} 25 -container_fs_reads_merged_total{device="sda2",id="testcontainer",name="testcontainer"} 40 -# HELP container_fs_reads_total Cumulative count of reads completed -# TYPE container_fs_reads_total counter -container_fs_reads_total{device="sda1",id="testcontainer",name="testcontainer"} 24 -container_fs_reads_total{device="sda2",id="testcontainer",name="testcontainer"} 39 -# HELP container_fs_sector_reads_total Cumulative count of sector reads completed -# TYPE container_fs_sector_reads_total counter -container_fs_sector_reads_total{device="sda1",id="testcontainer",name="testcontainer"} 26 -container_fs_sector_reads_total{device="sda2",id="testcontainer",name="testcontainer"} 41 -# HELP container_fs_sector_writes_total Cumulative count of sector writes completed -# TYPE container_fs_sector_writes_total counter -container_fs_sector_writes_total{device="sda1",id="testcontainer",name="testcontainer"} 40 -container_fs_sector_writes_total{device="sda2",id="testcontainer",name="testcontainer"} 45 -# HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem. -# TYPE container_fs_usage_bytes gauge -container_fs_usage_bytes{device="sda1",id="testcontainer",name="testcontainer"} 23 -container_fs_usage_bytes{device="sda2",id="testcontainer",name="testcontainer"} 38 -# HELP container_fs_write_seconds_total Cumulative count of seconds spent writing -# TYPE container_fs_write_seconds_total counter -container_fs_write_seconds_total{device="sda1",id="testcontainer",name="testcontainer"} 4.1e-08 -container_fs_write_seconds_total{device="sda2",id="testcontainer",name="testcontainer"} 4.6e-08 -# HELP container_fs_writes_merged_total Cumulative count of writes merged -# TYPE container_fs_writes_merged_total counter -container_fs_writes_merged_total{device="sda1",id="testcontainer",name="testcontainer"} 39 -container_fs_writes_merged_total{device="sda2",id="testcontainer",name="testcontainer"} 44 -# HELP container_fs_writes_total Cumulative count of writes completed -# TYPE container_fs_writes_total counter -container_fs_writes_total{device="sda1",id="testcontainer",name="testcontainer"} 28 -container_fs_writes_total{device="sda2",id="testcontainer",name="testcontainer"} 43 -# HELP container_last_seen Last time a container was seen by the exporter -# TYPE container_last_seen gauge -container_last_seen{id="testcontainer",name="testcontainer"} 1.426203694e+09 -# HELP container_memory_failures_total Cumulative count of memory allocation failures. -# TYPE container_memory_failures_total counter -container_memory_failures_total{id="testcontainer",name="testcontainer",scope="container",type="pgfault"} 10 -container_memory_failures_total{id="testcontainer",name="testcontainer",scope="container",type="pgmajfault"} 11 -container_memory_failures_total{id="testcontainer",name="testcontainer",scope="hierarchy",type="pgfault"} 12 -container_memory_failures_total{id="testcontainer",name="testcontainer",scope="hierarchy",type="pgmajfault"} 13 -# HELP container_memory_usage_bytes Current memory usage in bytes. -# TYPE container_memory_usage_bytes gauge -container_memory_usage_bytes{id="testcontainer",name="testcontainer"} 8 -# HELP container_memory_working_set_bytes Current working set in bytes. -# TYPE container_memory_working_set_bytes gauge -container_memory_working_set_bytes{id="testcontainer",name="testcontainer"} 9 -# HELP container_network_receive_bytes_total Cumulative count of bytes received -# TYPE container_network_receive_bytes_total counter -container_network_receive_bytes_total{id="testcontainer",name="testcontainer"} 14 -# HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving -# TYPE container_network_receive_errors_total counter -container_network_receive_errors_total{id="testcontainer",name="testcontainer"} 16 -# HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving -# TYPE container_network_receive_packets_dropped_total counter -container_network_receive_packets_dropped_total{id="testcontainer",name="testcontainer"} 17 -# HELP container_network_receive_packets_total Cumulative count of packets received -# TYPE container_network_receive_packets_total counter -container_network_receive_packets_total{id="testcontainer",name="testcontainer"} 15 -# HELP container_network_transmit_bytes_total Cumulative count of bytes transmitted -# TYPE container_network_transmit_bytes_total counter -container_network_transmit_bytes_total{id="testcontainer",name="testcontainer"} 18 -# HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting -# TYPE container_network_transmit_errors_total counter -container_network_transmit_errors_total{id="testcontainer",name="testcontainer"} 20 -# HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting -# TYPE container_network_transmit_packets_dropped_total counter -container_network_transmit_packets_dropped_total{id="testcontainer",name="testcontainer"} 21 -# HELP container_network_transmit_packets_total Cumulative count of packets transmitted -# TYPE container_network_transmit_packets_total counter -container_network_transmit_packets_total{id="testcontainer",name="testcontainer"} 19 -# HELP container_scrape_error 1 if there was an error while getting container metrics, 0 otherwise -# TYPE container_scrape_error gauge -container_scrape_error 0 -# HELP container_tasks_state Number of tasks in given state -# TYPE container_tasks_state gauge -container_tasks_state{id="testcontainer",name="testcontainer",state="iowaiting"} 54 -container_tasks_state{id="testcontainer",name="testcontainer",state="running"} 51 -container_tasks_state{id="testcontainer",name="testcontainer",state="sleeping"} 50 -container_tasks_state{id="testcontainer",name="testcontainer",state="stopped"} 52 -container_tasks_state{id="testcontainer",name="testcontainer",state="uninterruptible"} 53 -# HELP http_request_duration_microseconds The HTTP request latencies in microseconds. -# TYPE http_request_duration_microseconds summary -http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 0 -http_request_duration_microseconds{handler="prometheus",quantile="0.9"} 0 -http_request_duration_microseconds{handler="prometheus",quantile="0.99"} 0 -http_request_duration_microseconds_sum{handler="prometheus"} 0 -http_request_duration_microseconds_count{handler="prometheus"} 0 -# HELP http_request_size_bytes The HTTP request sizes in bytes. -# TYPE http_request_size_bytes summary -http_request_size_bytes{handler="prometheus",quantile="0.5"} 0 -http_request_size_bytes{handler="prometheus",quantile="0.9"} 0 -http_request_size_bytes{handler="prometheus",quantile="0.99"} 0 -http_request_size_bytes_sum{handler="prometheus"} 0 -http_request_size_bytes_count{handler="prometheus"} 0 -# HELP http_response_size_bytes The HTTP response sizes in bytes. -# TYPE http_response_size_bytes summary -http_response_size_bytes{handler="prometheus",quantile="0.5"} 0 -http_response_size_bytes{handler="prometheus",quantile="0.9"} 0 -http_response_size_bytes{handler="prometheus",quantile="0.99"} 0 -http_response_size_bytes_sum{handler="prometheus"} 0 -http_response_size_bytes_count{handler="prometheus"} 0 -# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. -# TYPE process_cpu_seconds_total counter -process_cpu_seconds_total 0 -# HELP process_goroutines Number of goroutines that currently exist. -# TYPE process_goroutines gauge -process_goroutines 16 -# HELP process_max_fds Maximum number of open file descriptors. -# TYPE process_max_fds gauge -process_max_fds 1024 -# HELP process_open_fds Number of open file descriptors. -# TYPE process_open_fds gauge -process_open_fds 4 -# HELP process_resident_memory_bytes Resident memory size in bytes. -# TYPE process_resident_memory_bytes gauge -process_resident_memory_bytes 7.74144e+06 -# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. -# TYPE process_start_time_seconds gauge -process_start_time_seconds 1.42620369439e+09 -# HELP process_virtual_memory_bytes Virtual memory size in bytes. -# TYPE process_virtual_memory_bytes gauge -process_virtual_memory_bytes 1.16420608e+08 diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/utils/machine/testdata/cpuinfo b/Godeps/_workspace/src/github.com/google/cadvisor/utils/machine/testdata/cpuinfo deleted file mode 100644 index ca2b722a560..00000000000 --- a/Godeps/_workspace/src/github.com/google/cadvisor/utils/machine/testdata/cpuinfo +++ /dev/null @@ -1,251 +0,0 @@ -processor : 0 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 1596.000 -cache size : 12288 KB -physical id : 0 -siblings : 6 -core id : 0 -cpu cores : 6 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 1 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 1596.000 -cache size : 12288 KB -physical id : 0 -siblings : 6 -core id : 1 -cpu cores : 6 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 2 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 1596.000 -cache size : 12288 KB -physical id : 0 -siblings : 6 -core id : 2 -cpu cores : 6 -apicid : 4 -initial apicid : 4 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 3 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 1596.000 -cache size : 12288 KB -physical id : 1 -siblings : 6 -core id : 3 -cpu cores : 6 -apicid : 16 -initial apicid : 16 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 4 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 1596.000 -cache size : 12288 KB -physical id : 1 -siblings : 6 -core id : 4 -cpu cores : 6 -apicid : 18 -initial apicid : 18 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 5 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 1596.000 -cache size : 12288 KB -physical id : 1 -siblings : 6 -core id : 5 -cpu cores : 6 -apicid : 20 -initial apicid : 20 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 6 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 2661.000 -cache size : 12288 KB -physical id : 0 -siblings : 6 -core id : 0 -cpu cores : 6 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 7 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 2661.000 -cache size : 12288 KB -physical id : 0 -siblings : 6 -core id : 1 -cpu cores : 6 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 8 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 1596.000 -cache size : 12288 KB -physical id : 0 -siblings : 6 -core id : 2 -cpu cores : 6 -apicid : 5 -initial apicid : 5 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 9 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 2661.000 -cache size : 12288 KB -physical id : 1 -siblings : 6 -core id : 3 -cpu cores : 6 -apicid : 17 -initial apicid : 17 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - -processor : 10 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 1596.000 -cache size : 12288 KB -physical id : 1 -siblings : 6 -core id : 4 -cpu cores : 6 -apicid : 19 -initial apicid : 19 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual -processor : 11 -cpu family : 6 -stepping : 2 -microcode : 0x10 -cpu MHz : 2661.000 -cache size : 12288 KB -physical id : 1 -siblings : 6 -core id : 5 -cpu cores : 6 -apicid : 21 -initial apicid : 21 -fpu : yes -fpu_exception : yes -cpuid level : 11 -wp : yes -bogomips : 5333.60 -clflush size : 64 -cache_alignment : 64 -address sizes : 40 bits physical, 48 bits virtual - From 4b47cb93f3af430d9a76df00f5ba0499ae8bc919 Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Thu, 3 Sep 2015 17:50:23 +0000 Subject: [PATCH 02/46] Reduce the default apiserver timeout back down to 30 seconds. It was originally raised due to slow load balancer creation (#5180), but that was fixed months ago. --- pkg/apiserver/apiserver.go | 3 +-- pkg/apiserver/apiserver_test.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 2f2f11d6921..c87234edd3a 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -345,8 +345,7 @@ func parseTimeout(str string) time.Duration { } glog.Errorf("Failed to parse %q: %v", str, err) } - // TODO: change back to 30s once #5180 is fixed - return 2 * time.Minute + return 30 * time.Second } func readBody(req *http.Request) ([]byte, error) { diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go index 80b9dc218e1..85c51a1508a 100644 --- a/pkg/apiserver/apiserver_test.go +++ b/pkg/apiserver/apiserver_test.go @@ -2145,10 +2145,10 @@ func TestUpdateChecksDecode(t *testing.T) { } func TestParseTimeout(t *testing.T) { - if d := parseTimeout(""); d != 2*time.Minute { + if d := parseTimeout(""); d != 30*time.Second { t.Errorf("blank timeout produces %v", d) } - if d := parseTimeout("not a timeout"); d != 2*time.Minute { + if d := parseTimeout("not a timeout"); d != 30*time.Second { t.Errorf("bad timeout produces %v", d) } if d := parseTimeout("10s"); d != 10*time.Second { From a5feac5739756a79e42056525c7258a3c0ff2d57 Mon Sep 17 00:00:00 2001 From: markturansky Date: Mon, 7 Sep 2015 14:44:49 -0400 Subject: [PATCH 03/46] improved recycler unit test --- pkg/volume/host_path/host_path_test.go | 23 ++--------------------- pkg/volume/testing.go | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/pkg/volume/host_path/host_path_test.go b/pkg/volume/host_path/host_path_test.go index 2189f1e1cbb..9e7fe52c642 100644 --- a/pkg/volume/host_path/host_path_test.go +++ b/pkg/volume/host_path/host_path_test.go @@ -63,7 +63,8 @@ func TestGetAccessModes(t *testing.T) { func TestRecycler(t *testing.T) { plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins([]volume.VolumePlugin{&hostPathPlugin{nil, newMockRecycler}}, volume.NewFakeVolumeHost("/tmp/fake", nil, nil)) + volumeHost := volume.NewFakeVolumeHost("/tmp/fake", nil, nil) + plugMgr.InitPlugins([]volume.VolumePlugin{&hostPathPlugin{nil, volume.NewFakeRecycler}}, volumeHost) spec := &volume.Spec{PersistentVolume: &api.PersistentVolume{Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/foo"}}}}} plug, err := plugMgr.FindRecyclablePluginBySpec(spec) @@ -82,26 +83,6 @@ func TestRecycler(t *testing.T) { } } -func newMockRecycler(spec *volume.Spec, host volume.VolumeHost) (volume.Recycler, error) { - return &mockRecycler{ - path: spec.PersistentVolume.Spec.HostPath.Path, - }, nil -} - -type mockRecycler struct { - path string - host volume.VolumeHost -} - -func (r *mockRecycler) GetPath() string { - return r.path -} - -func (r *mockRecycler) Recycle() error { - // return nil means recycle passed - return nil -} - func TestPlugin(t *testing.T) { plugMgr := volume.VolumePluginMgr{} plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volume.NewFakeVolumeHost("fake", nil, nil)) diff --git a/pkg/volume/testing.go b/pkg/volume/testing.go index 7ceb94d4f25..b347f5461f3 100644 --- a/pkg/volume/testing.go +++ b/pkg/volume/testing.go @@ -17,6 +17,7 @@ limitations under the License. package volume import ( + "fmt" "os" "path" @@ -125,7 +126,7 @@ func (plugin *FakeVolumePlugin) NewCleaner(volName string, podUID types.UID, mou } func (plugin *FakeVolumePlugin) NewRecycler(spec *Spec) (Recycler, error) { - return &FakeRecycler{"/attributesTransferredFromSpec"}, nil + return &fakeRecycler{"/attributesTransferredFromSpec"}, nil } func (plugin *FakeVolumePlugin) GetAccessModes() []api.PersistentVolumeAccessMode { @@ -162,15 +163,24 @@ func (fv *FakeVolume) TearDownAt(dir string) error { return os.RemoveAll(dir) } -type FakeRecycler struct { +type fakeRecycler struct { path string } -func (fr *FakeRecycler) Recycle() error { +func (fr *fakeRecycler) Recycle() error { // nil is success, else error return nil } -func (fr *FakeRecycler) GetPath() string { +func (fr *fakeRecycler) GetPath() string { return fr.path } + +func NewFakeRecycler(spec *Spec, host VolumeHost) (Recycler, error) { + if spec.PersistentVolume == nil || spec.PersistentVolume.Spec.HostPath == nil { + return nil, fmt.Errorf("fakeRecycler only supports spec.PersistentVolume.Spec.HostPath") + } + return &fakeRecycler{ + path: spec.PersistentVolume.Spec.HostPath.Path, + }, nil +} From d093fe0c4bf6f33cf2c6b98aec5439ff038fcd0a Mon Sep 17 00:00:00 2001 From: Carsten Clasohm Date: Mon, 7 Sep 2015 17:43:09 -0700 Subject: [PATCH 04/46] add raw flag for GitHub download links --- cmd/mungedocs/example_syncer.go | 4 ++-- cmd/mungedocs/example_syncer_test.go | 4 ++-- docs/admin/namespaces/README.md | 2 +- docs/getting-started-guides/logging.md | 4 ++-- docs/user-guide/downward-api.md | 4 ++-- docs/user-guide/logging.md | 2 +- docs/user-guide/simple-yaml.md | 4 ++-- docs/user-guide/walkthrough/README.md | 2 +- docs/user-guide/walkthrough/k8s201.md | 8 ++++---- examples/cassandra/README.md | 6 +++--- examples/celery-rabbitmq/README.md | 10 +++++----- examples/guestbook/README.md | 12 ++++++------ examples/hazelcast/README.md | 4 ++-- examples/mysql-wordpress-pd/README.md | 8 ++++---- examples/phabricator/README.md | 6 +++--- 15 files changed, 40 insertions(+), 40 deletions(-) diff --git a/cmd/mungedocs/example_syncer.go b/cmd/mungedocs/example_syncer.go index 8df6a968e45..ba153b3c46d 100644 --- a/cmd/mungedocs/example_syncer.go +++ b/cmd/mungedocs/example_syncer.go @@ -43,7 +43,7 @@ var exampleMungeTagRE = regexp.MustCompile(beginMungeTag(fmt.Sprintf("%s %s", ex // bar: // ``` // -// [Download example](../../examples/guestbook/frontend-controller.yaml) +// [Download example](../../examples/guestbook/frontend-controller.yaml?raw=true) // func syncExamples(filePath string, mlines mungeLines) (mungeLines, error) { var err error @@ -108,7 +108,7 @@ func exampleContent(filePath, linkPath, fileType string) (mungeLines, error) { // remove leading and trailing spaces and newlines trimmedFileContent := strings.TrimSpace(string(dat)) - content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s)", fileType, trimmedFileContent, fileRel) + content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s?raw=true)", fileType, trimmedFileContent, fileRel) out := getMungeLines(content) return out, nil } diff --git a/cmd/mungedocs/example_syncer_test.go b/cmd/mungedocs/example_syncer_test.go index 84fd8854a1e..72c1514a1ff 100644 --- a/cmd/mungedocs/example_syncer_test.go +++ b/cmd/mungedocs/example_syncer_test.go @@ -41,11 +41,11 @@ spec: {"", ""}, { "\n\n", - "\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml)\n\n", + "\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml?raw=true)\n\n", }, { "\n\n", - "\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml)\n\n", + "\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml?raw=true)\n\n", }, } repoRoot = "" diff --git a/docs/admin/namespaces/README.md b/docs/admin/namespaces/README.md index f15c29baa00..c866b1e20d5 100644 --- a/docs/admin/namespaces/README.md +++ b/docs/admin/namespaces/README.md @@ -98,7 +98,7 @@ Use the file [`namespace-dev.json`](namespace-dev.json) which describes a develo } ``` -[Download example](namespace-dev.json) +[Download example](namespace-dev.json?raw=true) Create the development namespace using kubectl. diff --git a/docs/getting-started-guides/logging.md b/docs/getting-started-guides/logging.md index 95460f9de1d..de308b9e84c 100644 --- a/docs/getting-started-guides/logging.md +++ b/docs/getting-started-guides/logging.md @@ -73,7 +73,7 @@ spec: 'for ((i = 0; ; i++)); do echo "$i: $(date)"; sleep 1; done'] ``` -[Download example](../../examples/blog-logging/counter-pod.yaml) +[Download example](../../examples/blog-logging/counter-pod.yaml?raw=true) This pod specification has one container which runs a bash script when the container is born. This script simply writes out the value of a counter and the date once per second and runs indefinitely. Let’s create the pod in the default @@ -192,7 +192,7 @@ spec: path: /var/lib/docker/containers ``` -[Download example](../../cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml) +[Download example](../../cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml?raw=true) This pod specification maps the directory on the host containing the Docker log files, `/var/lib/docker/containers`, to a directory inside the container which has the same path. The pod runs one image, `gcr.io/google_containers/fluentd-gcp:1.6`, which is configured to collect the Docker log files from the logs directory and ingest them into Google Cloud Logging. One instance of this pod runs on each node of the cluster. Kubernetes will notice if this pod fails and automatically restart it. diff --git a/docs/user-guide/downward-api.md b/docs/user-guide/downward-api.md index 9551ec0876f..8f625bd6c84 100644 --- a/docs/user-guide/downward-api.md +++ b/docs/user-guide/downward-api.md @@ -108,7 +108,7 @@ spec: restartPolicy: Never ``` -[Download example](downward-api/dapi-pod.yaml) +[Download example](downward-api/dapi-pod.yaml?raw=true) @@ -178,7 +178,7 @@ spec: fieldPath: metadata.annotations ``` -[Download example](downward-api/volume/dapi-volume.yaml) +[Download example](downward-api/volume/dapi-volume.yaml?raw=true) Some more thorough examples: diff --git a/docs/user-guide/logging.md b/docs/user-guide/logging.md index 49af4f527d6..c48f9ea7c5b 100644 --- a/docs/user-guide/logging.md +++ b/docs/user-guide/logging.md @@ -58,7 +58,7 @@ spec: 'for ((i = 0; ; i++)); do echo "$i: $(date)"; sleep 1; done'] ``` -[Download example](../../examples/blog-logging/counter-pod.yaml) +[Download example](../../examples/blog-logging/counter-pod.yaml?raw=true) we can run the pod: diff --git a/docs/user-guide/simple-yaml.md b/docs/user-guide/simple-yaml.md index cf5f0c4a1fa..aa1bf743421 100644 --- a/docs/user-guide/simple-yaml.md +++ b/docs/user-guide/simple-yaml.md @@ -64,7 +64,7 @@ spec: - containerPort: 80 ``` -[Download example](pod.yaml) +[Download example](pod.yaml?raw=true) You can see your cluster's pods: @@ -116,7 +116,7 @@ spec: - containerPort: 80 ``` -[Download example](replication.yaml) +[Download example](replication.yaml?raw=true) To delete the replication controller (and the pods it created): diff --git a/docs/user-guide/walkthrough/README.md b/docs/user-guide/walkthrough/README.md index 0740e1674f2..e5fe13d13cb 100644 --- a/docs/user-guide/walkthrough/README.md +++ b/docs/user-guide/walkthrough/README.md @@ -165,7 +165,7 @@ spec: emptyDir: {} ``` -[Download example](pod-redis.yaml) +[Download example](pod-redis.yaml?raw=true) Notes: diff --git a/docs/user-guide/walkthrough/k8s201.md b/docs/user-guide/walkthrough/k8s201.md index 24f9c1d8001..229eef033d8 100644 --- a/docs/user-guide/walkthrough/k8s201.md +++ b/docs/user-guide/walkthrough/k8s201.md @@ -86,7 +86,7 @@ spec: - containerPort: 80 ``` -[Download example](pod-nginx-with-label.yaml) +[Download example](pod-nginx-with-label.yaml?raw=true) Create the labeled pod ([pod-nginx-with-label.yaml](pod-nginx-with-label.yaml)): @@ -142,7 +142,7 @@ spec: - containerPort: 80 ``` -[Download example](replication-controller.yaml) +[Download example](replication-controller.yaml?raw=true) #### Replication Controller Management @@ -195,7 +195,7 @@ spec: app: nginx ``` -[Download example](service.yaml) +[Download example](service.yaml?raw=true) #### Service Management @@ -311,7 +311,7 @@ spec: - containerPort: 80 ``` -[Download example](pod-with-http-healthcheck.yaml) +[Download example](pod-with-http-healthcheck.yaml?raw=true) For more information about health checking, see [Container Probes](../pod-states.md#container-probes). diff --git a/examples/cassandra/README.md b/examples/cassandra/README.md index ff51ddd499f..7adb5c84aea 100644 --- a/examples/cassandra/README.md +++ b/examples/cassandra/README.md @@ -100,7 +100,7 @@ spec: emptyDir: {} ``` -[Download example](cassandra-controller.yaml) +[Download example](cassandra-controller.yaml?raw=true) There are a few things to note in this description. First is that we are running the ```kubernetes/cassandra``` image. This is a standard Cassandra installation on top of Debian. However it also adds a custom [```SeedProvider```](https://svn.apache.org/repos/asf/cassandra/trunk/src/java/org/apache/cassandra/locator/SeedProvider.java) to Cassandra. In Cassandra, a ```SeedProvider``` bootstraps the gossip protocol that Cassandra uses to find other nodes. The ```KubernetesSeedProvider``` discovers the Kubernetes API Server using the built in Kubernetes discovery service, and then uses the Kubernetes API to find new nodes (more on this later) @@ -131,7 +131,7 @@ spec: name: cassandra ``` -[Download example](cassandra-service.yaml) +[Download example](cassandra-service.yaml?raw=true) The important thing to note here is the ```selector```. It is a query over labels, that identifies the set of _Pods_ contained by the _Service_. In this case the selector is ```name=cassandra```. If you look back at the Pod specification above, you'll see that the pod has the corresponding label, so it will be selected for membership in this Service. @@ -241,7 +241,7 @@ spec: emptyDir: {} ``` -[Download example](cassandra-controller.yaml) +[Download example](cassandra-controller.yaml?raw=true) Most of this replication controller definition is identical to the Cassandra pod definition above, it simply gives the replication controller a recipe to use when it creates new Cassandra pods. The other differentiating parts are the ```selector``` attribute which contains the controller's selector query, and the ```replicas``` attribute which specifies the desired number of replicas, in this case 1. diff --git a/examples/celery-rabbitmq/README.md b/examples/celery-rabbitmq/README.md index 04cc646a53f..cc2a4db85c5 100644 --- a/examples/celery-rabbitmq/README.md +++ b/examples/celery-rabbitmq/README.md @@ -81,7 +81,7 @@ spec: component: rabbitmq ``` -[Download example](rabbitmq-service.yaml) +[Download example](rabbitmq-service.yaml?raw=true) To start the service, run: @@ -126,7 +126,7 @@ spec: cpu: 100m ``` -[Download example](rabbitmq-controller.yaml) +[Download example](rabbitmq-controller.yaml?raw=true) Running `$ kubectl create -f examples/celery-rabbitmq/rabbitmq-controller.yaml` brings up a replication controller that ensures one pod exists which is running a RabbitMQ instance. @@ -167,7 +167,7 @@ spec: cpu: 100m ``` -[Download example](celery-controller.yaml) +[Download example](celery-controller.yaml?raw=true) There are several things to point out here... @@ -238,7 +238,7 @@ spec: type: LoadBalancer ``` -[Download example](flower-service.yaml) +[Download example](flower-service.yaml?raw=true) It is marked as external (LoadBalanced). However on many platforms you will have to add an explicit firewall rule to open port 5555. @@ -279,7 +279,7 @@ spec: cpu: 100m ``` -[Download example](flower-controller.yaml) +[Download example](flower-controller.yaml?raw=true) This will bring up a new pod with Flower installed and port 5555 (Flower's default port) exposed through the service endpoint. This image uses the following command to start Flower: diff --git a/examples/guestbook/README.md b/examples/guestbook/README.md index c93cbece6b4..89d9981bc92 100644 --- a/examples/guestbook/README.md +++ b/examples/guestbook/README.md @@ -100,7 +100,7 @@ spec: - containerPort: 6379 ``` -[Download example](redis-master-controller.yaml) +[Download example](redis-master-controller.yaml?raw=true) Change to the `/examples/guestbook` directory if you're not already there. Create the redis master pod in your Kubernetes cluster by running: @@ -221,7 +221,7 @@ spec: name: redis-master ``` -[Download example](redis-master-service.yaml) +[Download example](redis-master-service.yaml?raw=true) Create the service by running: @@ -296,7 +296,7 @@ spec: - containerPort: 6379 ``` -[Download example](redis-slave-controller.yaml) +[Download example](redis-slave-controller.yaml?raw=true) and create the replication controller by running: @@ -347,7 +347,7 @@ spec: name: redis-slave ``` -[Download example](redis-slave-service.yaml) +[Download example](redis-slave-service.yaml?raw=true) This time the selector for the service is `name=redis-slave`, because that identifies the pods running redis slaves. It may also be helpful to set labels on your service itself as we've done here to make it easy to locate them with the `kubectl get services -l "label=value"` command. @@ -398,7 +398,7 @@ spec: - containerPort: 80 ``` -[Download example](frontend-controller.yaml) +[Download example](frontend-controller.yaml?raw=true) Using this file, you can turn up your frontend with: @@ -501,7 +501,7 @@ spec: name: frontend ``` -[Download example](frontend-service.yaml) +[Download example](frontend-service.yaml?raw=true) #### Using 'type: LoadBalancer' for the frontend service (cloud-provider-specific) diff --git a/examples/hazelcast/README.md b/examples/hazelcast/README.md index 5ae17f5c696..269755f45f2 100644 --- a/examples/hazelcast/README.md +++ b/examples/hazelcast/README.md @@ -83,7 +83,7 @@ spec: name: hazelcast ``` -[Download example](hazelcast-service.yaml) +[Download example](hazelcast-service.yaml?raw=true) The important thing to note here is the `selector`. It is a query over labels, that identifies the set of _Pods_ contained by the _Service_. In this case the selector is `name: hazelcast`. If you look at the Replication Controller specification below, you'll see that the pod has the corresponding label, so it will be selected for membership in this Service. @@ -138,7 +138,7 @@ spec: name: hazelcast ``` -[Download example](hazelcast-controller.yaml) +[Download example](hazelcast-controller.yaml?raw=true) There are a few things to note in this description. First is that we are running the `quay.io/pires/hazelcast-kubernetes` image, tag `0.5`. This is a `busybox` installation with JRE 8 Update 45. However it also adds a custom [`application`](https://github.com/pires/hazelcast-kubernetes-bootstrapper) that finds any Hazelcast nodes in the cluster and bootstraps an Hazelcast instance accordingly. The `HazelcastDiscoveryController` discovers the Kubernetes API Server using the built in Kubernetes discovery service, and then uses the Kubernetes API to find new nodes (more on this later). diff --git a/examples/mysql-wordpress-pd/README.md b/examples/mysql-wordpress-pd/README.md index 7a496eeedc4..34d2fd1a3da 100644 --- a/examples/mysql-wordpress-pd/README.md +++ b/examples/mysql-wordpress-pd/README.md @@ -131,7 +131,7 @@ spec: fsType: ext4 ``` -[Download example](mysql.yaml) +[Download example](mysql.yaml?raw=true) Note that we've defined a volume mount for `/var/lib/mysql`, and specified a volume that uses the persistent disk (`mysql-disk`) that you created. @@ -186,7 +186,7 @@ spec: name: mysql ``` -[Download example](mysql-service.yaml) +[Download example](mysql-service.yaml?raw=true) Start the service like this: @@ -241,7 +241,7 @@ spec: fsType: ext4 ``` -[Download example](wordpress.yaml) +[Download example](wordpress.yaml?raw=true) Create the pod: @@ -282,7 +282,7 @@ spec: type: LoadBalancer ``` -[Download example](wordpress-service.yaml) +[Download example](wordpress-service.yaml?raw=true) Note the `type: LoadBalancer` setting. This will set up the wordpress service behind an external IP. diff --git a/examples/phabricator/README.md b/examples/phabricator/README.md index 4aa15260ab4..d9fca4bbe16 100644 --- a/examples/phabricator/README.md +++ b/examples/phabricator/README.md @@ -98,7 +98,7 @@ To start Phabricator server use the file [`examples/phabricator/phabricator-cont } ``` -[Download example](phabricator-controller.json) +[Download example](phabricator-controller.json?raw=true) Create the phabricator pod in your Kubernetes cluster by running: @@ -188,7 +188,7 @@ To automate this process and make sure that a proper host is authorized even if } ``` -[Download example](authenticator-controller.json) +[Download example](authenticator-controller.json?raw=true) To create the pod run: @@ -237,7 +237,7 @@ Use the file [`examples/phabricator/phabricator-service.json`](phabricator-servi } ``` -[Download example](phabricator-service.json) +[Download example](phabricator-service.json?raw=true) To create the service run: From 1402a63150db8cc6f66a28d38166f9abcc8790a4 Mon Sep 17 00:00:00 2001 From: qiaolei Date: Tue, 8 Sep 2015 15:17:42 +0800 Subject: [PATCH 05/46] change 'random' to 'round robin' for the choice of a service backend --- docs/user-guide/services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/services.md b/docs/user-guide/services.md index 2badff776ae..53a328e3c47 100644 --- a/docs/user-guide/services.md +++ b/docs/user-guide/services.md @@ -219,7 +219,7 @@ appropriate backend without the clients knowing anything about Kubernetes or ![Services overview diagram](services-overview.png) -By default, the choice of backend is random. Client-IP based session affinity +By default, the choice of backend is round robin. Client-IP based session affinity can be selected by setting `service.spec.sessionAffinity` to `"ClientIP"` (the default is `"None"`). From f7384827b80d809e6bdabd7801e34328e29a027d Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Tue, 8 Sep 2015 10:46:49 +0200 Subject: [PATCH 06/46] Fixed error handling of cloud init. Avoid creating a new 'err' variable in the 'if'-branch, shadowing the one in the outer scope. Any error from subsequent 'cloud, err = GetCloudProvider()' was not propagated to 'err' variable in the outer scope and thus errors were never returned from this function. This is hard to debug error on OpenStack, when content of --cloud-config= file is wrong or connection to OpenStack fails. Such error is never logged and Kubernetes thinks everything is OK. --- pkg/cloudprovider/plugins.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/plugins.go b/pkg/cloudprovider/plugins.go index 86e5f8dbd32..ad39e34051c 100644 --- a/pkg/cloudprovider/plugins.go +++ b/pkg/cloudprovider/plugins.go @@ -65,15 +65,16 @@ func GetCloudProvider(name string, config io.Reader) (Interface, error) { // InitCloudProvider creates an instance of the named cloud provider. func InitCloudProvider(name string, configFilePath string) (Interface, error) { var cloud Interface + var err error if name == "" { glog.Info("No cloud provider specified.") return nil, nil } - var err error if configFilePath != "" { - config, err := os.Open(configFilePath) + var config *os.File + config, err = os.Open(configFilePath) if err != nil { glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", configFilePath, err) From d22a29cf6646c91c9ea507345f78f3df4c8a20bd Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Tue, 8 Sep 2015 11:35:32 -0500 Subject: [PATCH 07/46] Block apiserver startup on certificate With some regularity, if the root certificate file needs to be generated the apiserver could come up on the non-secure port before the cert was generated. `hack/local-up-cluster.sh` requires that apiserver.crt exists before the replication controller starts. Otherwise service accounts and secrets don't work. This change just takes the certificate handling code out of the `go`. --- cmd/kube-apiserver/app/server.go | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 158ddde7a12..f14fea3b5d0 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -505,23 +505,24 @@ func (s *APIServer) Run(_ []string) error { } glog.Infof("Serving securely on %s", secureLocation) + if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" { + s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt") + s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key") + // TODO (cjcullen): Is PublicAddress the right address to sign a cert with? + alternateIPs := []net.IP{config.ServiceReadWriteIP} + alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} + // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless + // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") + if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { + glog.Errorf("Unable to generate self signed cert: %v", err) + } else { + glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile) + } + } + go func() { defer util.HandleCrash() for { - if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" { - s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt") - s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key") - // TODO (cjcullen): Is PublicAddress the right address to sign a cert with? - alternateIPs := []net.IP{config.ServiceReadWriteIP} - alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} - // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless - // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") - if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { - glog.Errorf("Unable to generate self signed cert: %v", err) - } else { - glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile) - } - } // err == systemd.SdNotifyNoSocket when not running on a systemd system if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) From f94c57ac303f40f83a93b6aeb003ccfedf40043c Mon Sep 17 00:00:00 2001 From: Avesh Agarwal Date: Tue, 8 Sep 2015 19:54:07 -0400 Subject: [PATCH 08/46] If ForceUserspaceProxy is true, check for iptables based proxy should not be performed. --- cmd/kube-proxy/app/server.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index c0d573493a0..b5334d6a9cf 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -168,12 +168,17 @@ func (s *ProxyServer) Run(_ []string) error { var proxier proxy.ProxyProvider var endpointsHandler config.EndpointsConfigHandler - // guaranteed false on error, error only necessary for debugging - shouldUseIptables, err := iptables.ShouldUseIptablesProxier() - if err != nil { - glog.Errorf("Can't determine whether to use iptables or userspace, using userspace proxier: %v", err) + shouldUseIptables := false + if !s.ForceUserspaceProxy { + var err error + // guaranteed false on error, error only necessary for debugging + shouldUseIptables, err = iptables.ShouldUseIptablesProxier() + if err != nil { + glog.Errorf("Can't determine whether to use iptables proxy, using userspace proxier: %v", err) + } } - if !s.ForceUserspaceProxy && shouldUseIptables { + + if shouldUseIptables { glog.V(2).Info("Using iptables Proxier.") execer := exec.New() From b2268574c5ebd94e4fc9ad402f81f00464bf571a Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Tue, 8 Sep 2015 23:58:36 -0400 Subject: [PATCH 09/46] Add pods/attach to long running requests, protect in admission for privileged pods --- cmd/kube-apiserver/app/server.go | 2 +- cmd/kube-apiserver/app/server_test.go | 4 +++ .../exec/denyprivileged/admission.go | 6 ++-- .../exec/denyprivileged/admission_test.go | 29 ++++++++++++++----- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 085fb376929..80d8269e6a7 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -57,7 +57,7 @@ const ( // Set to a value larger than the timeouts in each watch server. ReadWriteTimeout = time.Minute * 60 //TODO: This can be tightened up. It still matches objects named watch or proxy. - defaultLongRunningRequestRE = "(/|^)((watch|proxy)(/|$)|(logs|portforward|exec)/?$)" + defaultLongRunningRequestRE = "(/|^)((watch|proxy)(/|$)|(logs?|portforward|exec|attach)/?$)" ) // APIServer runs a kubernetes api server. diff --git a/cmd/kube-apiserver/app/server_test.go b/cmd/kube-apiserver/app/server_test.go index 04ab483c5a6..c6bca513293 100644 --- a/cmd/kube-apiserver/app/server_test.go +++ b/cmd/kube-apiserver/app/server_test.go @@ -38,12 +38,16 @@ func TestLongRunningRequestRegexp(t *testing.T) { "/api/v1/watch/stuff", "/api/v1/default/service/proxy", "/api/v1/pods/proxy/path/to/thing", + "/api/v1/namespaces/myns/pods/mypod/log", "/api/v1/namespaces/myns/pods/mypod/logs", "/api/v1/namespaces/myns/pods/mypod/portforward", "/api/v1/namespaces/myns/pods/mypod/exec", + "/api/v1/namespaces/myns/pods/mypod/attach", + "/api/v1/namespaces/myns/pods/mypod/log/", "/api/v1/namespaces/myns/pods/mypod/logs/", "/api/v1/namespaces/myns/pods/mypod/portforward/", "/api/v1/namespaces/myns/pods/mypod/exec/", + "/api/v1/namespaces/myns/pods/mypod/attach/", "/api/v1/watch/namespaces/myns/pods", } for _, path := range dontMatch { diff --git a/plugin/pkg/admission/exec/denyprivileged/admission.go b/plugin/pkg/admission/exec/denyprivileged/admission.go index 481eb557baa..9fa8ffb202a 100644 --- a/plugin/pkg/admission/exec/denyprivileged/admission.go +++ b/plugin/pkg/admission/exec/denyprivileged/admission.go @@ -45,8 +45,8 @@ func (d *denyExecOnPrivileged) Admit(a admission.Attributes) (err error) { if !ok { return errors.NewBadRequest("a connect request was received, but could not convert the request object.") } - // Only handle exec requests on pods - if connectRequest.ResourcePath != "pods/exec" { + // Only handle exec or attach requests on pods + if connectRequest.ResourcePath != "pods/exec" && connectRequest.ResourcePath != "pods/attach" { return nil } pod, err := d.client.Pods(a.GetNamespace()).Get(connectRequest.Name) @@ -54,7 +54,7 @@ func (d *denyExecOnPrivileged) Admit(a admission.Attributes) (err error) { return admission.NewForbidden(a, err) } if isPrivileged(pod) { - return admission.NewForbidden(a, fmt.Errorf("Cannot exec into a privileged container")) + return admission.NewForbidden(a, fmt.Errorf("Cannot exec into or attach to a privileged container")) } return nil } diff --git a/plugin/pkg/admission/exec/denyprivileged/admission_test.go b/plugin/pkg/admission/exec/denyprivileged/admission_test.go index 22d5b97613d..4c0bc7115ec 100644 --- a/plugin/pkg/admission/exec/denyprivileged/admission_test.go +++ b/plugin/pkg/admission/exec/denyprivileged/admission_test.go @@ -47,15 +47,30 @@ func testAdmission(t *testing.T, pod *api.Pod, shouldAccept bool) { handler := &denyExecOnPrivileged{ client: mockClient, } - req := &rest.ConnectRequest{Name: pod.Name, ResourcePath: "pods/exec"} - err := handler.Admit(admission.NewAttributesRecord(req, "Pod", "test", "name", "pods", "exec", admission.Connect, nil)) - if shouldAccept && err != nil { - t.Errorf("Unexpected error returned from admission handler: %v", err) - } - if !shouldAccept && err == nil { - t.Errorf("An error was expected from the admission handler. Received nil") + + // pods/exec + { + req := &rest.ConnectRequest{Name: pod.Name, ResourcePath: "pods/exec"} + err := handler.Admit(admission.NewAttributesRecord(req, "Pod", "test", "name", "pods", "exec", admission.Connect, nil)) + if shouldAccept && err != nil { + t.Errorf("Unexpected error returned from admission handler: %v", err) + } + if !shouldAccept && err == nil { + t.Errorf("An error was expected from the admission handler. Received nil") + } } + // pods/attach + { + req := &rest.ConnectRequest{Name: pod.Name, ResourcePath: "pods/attach"} + err := handler.Admit(admission.NewAttributesRecord(req, "Pod", "test", "name", "pods", "attach", admission.Connect, nil)) + if shouldAccept && err != nil { + t.Errorf("Unexpected error returned from admission handler: %v", err) + } + if !shouldAccept && err == nil { + t.Errorf("An error was expected from the admission handler. Received nil") + } + } } func acceptPod(name string) *api.Pod { From f1ac43aa4e21ecfbf9a4ae9626fdd7ece2f5d141 Mon Sep 17 00:00:00 2001 From: qiaolei Date: Wed, 9 Sep 2015 13:15:34 +0800 Subject: [PATCH 10/46] Fixed a typo in fedora_manual_config.md Change '--enablerepo=update-testing' to '--enablerepo=updates-testing' to keep align with the command output. --- docs/getting-started-guides/fedora/fedora_manual_config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started-guides/fedora/fedora_manual_config.md b/docs/getting-started-guides/fedora/fedora_manual_config.md index f306cd80549..bba7c2f3cfe 100644 --- a/docs/getting-started-guides/fedora/fedora_manual_config.md +++ b/docs/getting-started-guides/fedora/fedora_manual_config.md @@ -62,7 +62,7 @@ fed-node = 192.168.121.65 **Prepare the hosts:** * Install Kubernetes on all hosts - fed-{master,node}. This will also pull in docker. Also install etcd on fed-master. This guide has been tested with kubernetes-0.18 and beyond. -* The [--enablerepo=update-testing](https://fedoraproject.org/wiki/QA:Updates_Testing) directive in the yum command below will ensure that the most recent Kubernetes version that is scheduled for pre-release will be installed. This should be a more recent version than the Fedora "stable" release for Kubernetes that you would get without adding the directive. +* The [--enablerepo=updates-testing](https://fedoraproject.org/wiki/QA:Updates_Testing) directive in the yum command below will ensure that the most recent Kubernetes version that is scheduled for pre-release will be installed. This should be a more recent version than the Fedora "stable" release for Kubernetes that you would get without adding the directive. * If you want the very latest Kubernetes release [you can download and yum install the RPM directly from Fedora Koji](http://koji.fedoraproject.org/koji/packageinfo?packageID=19202) instead of using the yum install command below. ```sh From 287055930b2ca62bcf648d9067d2da0f4348e0f6 Mon Sep 17 00:00:00 2001 From: hurf Date: Wed, 9 Sep 2015 15:02:01 +0800 Subject: [PATCH 11/46] Aggregate errors when checking limitrange Return all errors at a time when resources violate limitrange to provide users better experience. --- plugin/pkg/admission/limitranger/admission.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/plugin/pkg/admission/limitranger/admission.go b/plugin/pkg/admission/limitranger/admission.go index bc544c3863d..ba98913d254 100644 --- a/plugin/pkg/admission/limitranger/admission.go +++ b/plugin/pkg/admission/limitranger/admission.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/watch" ) @@ -300,30 +301,31 @@ func sum(inputs []api.ResourceList) api.ResourceList { // the specified LimitRange. The pod may be modified to apply default resource // requirements if not specified, and enumerated on the LimitRange func PodLimitFunc(limitRange *api.LimitRange, pod *api.Pod) error { + var errs []error + defaultResources := defaultContainerResourceRequirements(limitRange) mergePodResourceRequirements(pod, &defaultResources) for i := range limitRange.Spec.Limits { limit := limitRange.Spec.Limits[i] limitType := limit.Type - // enforce container limits if limitType == api.LimitTypeContainer { for j := range pod.Spec.Containers { container := &pod.Spec.Containers[j] for k, v := range limit.Min { if err := minConstraint(limitType, k, v, container.Resources.Requests, container.Resources.Limits); err != nil { - return err + errs = append(errs, err) } } for k, v := range limit.Max { if err := maxConstraint(limitType, k, v, container.Resources.Requests, container.Resources.Limits); err != nil { - return err + errs = append(errs, err) } } for k, v := range limit.MaxLimitRequestRatio { if err := limitRequestRatioConstraint(limitType, k, v, container.Resources.Requests, container.Resources.Limits); err != nil { - return err + errs = append(errs, err) } } } @@ -341,20 +343,20 @@ func PodLimitFunc(limitRange *api.LimitRange, pod *api.Pod) error { podLimits := sum(containerLimits) for k, v := range limit.Min { if err := minConstraint(limitType, k, v, podRequests, podLimits); err != nil { - return err + errs = append(errs, err) } } for k, v := range limit.Max { if err := maxConstraint(limitType, k, v, podRequests, podLimits); err != nil { - return err + errs = append(errs, err) } } for k, v := range limit.MaxLimitRequestRatio { if err := limitRequestRatioConstraint(limitType, k, v, podRequests, podLimits); err != nil { - return err + errs = append(errs, err) } } } } - return nil + return errors.NewAggregate(errs) } From 294689ccff8d8e7359cea9811a649739961a7d17 Mon Sep 17 00:00:00 2001 From: qiaolei Date: Wed, 9 Sep 2015 16:43:55 +0800 Subject: [PATCH 12/46] Change `~/.kube/kubeconfig` to `~/.kube/config` Where `~/.kube/kubeconfig` should be `~/.kube/config` --- docs/getting-started-guides/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started-guides/aws.md b/docs/getting-started-guides/aws.md index a55c1f92b3d..72a6bb8d13c 100644 --- a/docs/getting-started-guides/aws.md +++ b/docs/getting-started-guides/aws.md @@ -76,7 +76,7 @@ using [cluster/aws/config-default.sh](http://releases.k8s.io/HEAD/cluster/aws/co This process takes about 5 to 10 minutes. Once the cluster is up, the IP addresses of your master and node(s) will be printed, as well as information about the default services running in the cluster (monitoring, logging, dns). User credentials and security -tokens are written in `~/.kube/kubeconfig`, they will be necessary to use the CLI or the HTTP Basic Auth. +tokens are written in `~/.kube/config`, they will be necessary to use the CLI or the HTTP Basic Auth. By default, the script will provision a new VPC and a 4 node k8s cluster in us-west-2a (Oregon) with `t2.micro` instances running on Ubuntu. You can override the variables defined in [config-default.sh](http://releases.k8s.io/HEAD/cluster/aws/config-default.sh) to change this behavior as follows: From 390530d345bf027dbf384e0003a6f13cb6a19579 Mon Sep 17 00:00:00 2001 From: qiaolei Date: Wed, 9 Sep 2015 16:53:32 +0800 Subject: [PATCH 13/46] Fixed a markdown error in azure.md --- docs/getting-started-guides/azure.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started-guides/azure.md b/docs/getting-started-guides/azure.md index 1983bc8773d..9918194a630 100644 --- a/docs/getting-started-guides/azure.md +++ b/docs/getting-started-guides/azure.md @@ -43,7 +43,7 @@ Getting started on Microsoft Azure ## Prerequisites -** Azure Prerequisites** +**Azure Prerequisites** 1. You need an Azure account. Visit http://azure.microsoft.com/ to get started. 2. Install and configure the Azure cross-platform command-line interface. http://azure.microsoft.com/en-us/documentation/articles/xplat-cli/ From 1460a1fb9ea59fdb182073040a0e5ea02dd7dab6 Mon Sep 17 00:00:00 2001 From: jiangyaoguo Date: Wed, 9 Sep 2015 16:57:21 +0800 Subject: [PATCH 14/46] Rate limit events in kubelet 1. Add EvnetRecordQps and EventBurst parameter in kubelet. 2. If EvnetRecordQps and EventBurst was set, rate limit events in kubelet with a independent ratelimiter as setted. --- cmd/kubelet/app/server.go | 18 +- contrib/mesos/pkg/executor/service/service.go | 2 + hack/verify-flags/known-flags.txt | 542 +++++++++--------- pkg/kubelet/kubelet.go | 2 + 4 files changed, 294 insertions(+), 270 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 3eb671c148f..0d66b8e8bc0 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -84,6 +84,8 @@ type KubeletServer struct { HostNetworkSources string RegistryPullQPS float64 RegistryBurst int + EventRecordQPS float32 + EventBurst int RunOnce bool EnableDebuggingHandlers bool MinimumGCAge time.Duration @@ -220,6 +222,8 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.HostNetworkSources, "host-network-sources", s.HostNetworkSources, "Comma-separated list of sources from which the Kubelet allows pods to use of host network. For all sources use \"*\" [default=\"file\"]") fs.Float64Var(&s.RegistryPullQPS, "registry-qps", s.RegistryPullQPS, "If > 0, limit registry pull QPS to this value. If 0, unlimited. [default=0.0]") fs.IntVar(&s.RegistryBurst, "registry-burst", s.RegistryBurst, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry-qps. Only used if --registry-qps > 0") + fs.Float32Var(&s.EventRecordQPS, "event-qps", s.EventRecordQPS, "If > 0, limit event creations per second to this value. If 0, unlimited. [default=0.0]") + fs.IntVar(&s.EventBurst, "event-burst", s.EventBurst, "Maximum size of a bursty event records, temporarily allows event records to burst to this number, while still not exceeding event-qps. Only used if --event-qps > 0") fs.BoolVar(&s.RunOnce, "runonce", s.RunOnce, "If true, exit after spawning pods from local manifests or remote urls. Exclusive with --api-servers, and --enable-server") fs.BoolVar(&s.EnableDebuggingHandlers, "enable-debugging-handlers", s.EnableDebuggingHandlers, "Enables server endpoints for log collection and local running of containers and commands") fs.DurationVar(&s.MinimumGCAge, "minimum-container-ttl-duration", s.MinimumGCAge, "Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'") @@ -327,6 +331,8 @@ func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) { SyncFrequency: s.SyncFrequency, RegistryPullQPS: s.RegistryPullQPS, RegistryBurst: s.RegistryBurst, + EventRecordQPS: s.EventRecordQPS, + EventBurst: s.EventBurst, MinimumGCAge: s.MinimumGCAge, MaxPerPodContainerCount: s.MaxPerPodContainerCount, MaxContainerCount: s.MaxContainerCount, @@ -646,7 +652,13 @@ func RunKubelet(kcfg *KubeletConfig, builder KubeletBuilder) error { eventBroadcaster.StartLogging(glog.V(3).Infof) if kcfg.KubeClient != nil { glog.V(4).Infof("Sending events to api server.") - eventBroadcaster.StartRecordingToSink(kcfg.KubeClient.Events("")) + if kcfg.EventRecordQPS == 0.0 { + eventBroadcaster.StartRecordingToSink(kcfg.KubeClient.Events("")) + } else { + eventClient := *kcfg.KubeClient + eventClient.Throttle = util.NewTokenBucketRateLimiter(kcfg.EventRecordQPS, kcfg.EventBurst) + eventBroadcaster.StartRecordingToSink(eventClient.Events("")) + } } else { glog.Warning("No api server defined - no events will be sent to API server.") } @@ -742,6 +754,8 @@ type KubeletConfig struct { SyncFrequency time.Duration RegistryPullQPS float64 RegistryBurst int + EventRecordQPS float32 + EventBurst int MinimumGCAge time.Duration MaxPerPodContainerCount int MaxContainerCount int @@ -809,6 +823,8 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.SyncFrequency, float32(kc.RegistryPullQPS), kc.RegistryBurst, + kc.EventRecordQPS, + kc.EventBurst, gcPolicy, pc.SeenAllSources, kc.RegisterNode, diff --git a/contrib/mesos/pkg/executor/service/service.go b/contrib/mesos/pkg/executor/service/service.go index fe5cab5b89a..5508d78ee6e 100644 --- a/contrib/mesos/pkg/executor/service/service.go +++ b/contrib/mesos/pkg/executor/service/service.go @@ -301,6 +301,8 @@ func (ks *KubeletExecutorServer) createAndInitKubelet( kc.SyncFrequency, float32(kc.RegistryPullQPS), kc.RegistryBurst, + kc.EventRecordQPS, + kc.EventBurst, gcPolicy, pc.SeenAllSources, kc.RegisterNode, diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 32ac4143353..cd7cfad7bac 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -1,269 +1,273 @@ -accept-hosts -accept-paths -account-for-pod-resources -admission-control -admission-control-config-file -advertise-address -advertised-address -algorithm-provider -all-namespaces -allocate-node-cidrs -allow-privileged -api-prefix -api-servers -api-token -api-version -authorization-mode -authorization-policy-file -auth-path -basic-auth-file -bench-pods -bench-quiet -bench-tasks -bench-workers -bind-address -bind-pods-burst -bind-pods-qps -cadvisor-port -cert-dir -certificate-authority -cgroup-root -chaos-chance -cleanup-iptables -client-ca-file -client-certificate -client-key -cloud-config -cloud-provider -cluster-cidr -cluster-dns -cluster-domain -cluster-name -cluster-tag -concurrent-endpoint-syncs -configure-cbr0 -contain-pod-resources -container-port -container-runtime -cors-allowed-origins -create-external-load-balancer -current-release-pr -current-replicas -default-container-cpu-limit -default-container-mem-limit -delay-shutdown -deleting-pods-burst -deleting-pods-qps -deployment-label-key -dest-file -disable-filter -docker-endpoint -docker-exec-handler -dockercfg-path -driver-port -dry-run -duration-sec -e2e-output-dir -enable-debugging-handlers -enable-horizontal-pod-autoscaler -enable-server -etcd-config -etcd-prefix -etcd-server -etcd-servers -event-ttl -executor-bindall -executor-logv -executor-path -executor-suicide-timeout -experimental-keystone-url -experimental-prefix -external-hostname -external-ip -failover-timeout -file-check-frequency -file-suffix -forward-services -framework-name -framework-weburi -func-dest -fuzz-iters -gce-project -gce-zone -gke-cluster -google-json-key -grace-period -ha-domain -healthz-bind-address -healthz-port -horizontal-pod-autoscaler-sync-period -hostname-override -host-network-sources -http-check-frequency -http-port -ignore-not-found -image-gc-high-threshold -image-gc-low-threshold -insecure-bind-address -insecure-port -insecure-skip-tls-verify -iptables-sync-period -ir-data-source -ir-dbname -ir-influxdb-host -ir-password -ir-user -jenkins-host -jenkins-jobs -km-path -kubectl-path -kubelet-cadvisor-port -kubelet-certificate-authority -kubelet-client-certificate -kubelet-client-key -kubelet-docker-endpoint -kubelet-host-network-sources -kubelet-https -kubelet-network-plugin -kubelet-pod-infra-container-image -kubelet-port -kubelet-root-dir -kubelet-sync-frequency -kubelet-timeout -kube-master -label-columns -last-release-pr -legacy-userspace-proxy -log-flush-frequency -long-running-request-regexp -low-diskspace-threshold-mb -manifest-url -manifest-url-header -masquerade-all -master-service-namespace -max-concurrency -max-connection-bytes-per-sec -maximum-dead-containers -maximum-dead-containers-per-container -max-log-age -max-log-backups -max-log-size -max-outgoing-burst -max-outgoing-qps -max-pods -max-requests-inflight -mesos-authentication-principal -mesos-authentication-provider -mesos-authentication-secret-file -mesos-cgroup-prefix -mesos-executor-cpus -mesos-executor-mem -mesos-master -mesos-role -mesos-user -minimum-container-ttl-duration -minion-max-log-age -minion-max-log-backups -minion-max-log-size -minion-path-override -min-pr-number -min-request-timeout -namespace-sync-period -network-plugin -network-plugin-dir -node-instance-group -node-monitor-grace-period -node-monitor-period -node-startup-grace-period -node-status-update-frequency -node-sync-period -no-headers -num-nodes -oidc-ca-file -oidc-client-id -oidc-issuer-url -oidc-username-claim -oom-score-adj -output-version -out-version -path-override -pod-cidr -pod-eviction-timeout -pod-infra-container-image -policy-config-file -poll-interval -portal-net -private-mountns -prom-push-gateway -proxy-bindall -proxy-logv -proxy-port-range -public-address-override -pvclaimbinder-sync-period -read-only-port -really-crash-for-testing -reconcile-cooldown -reconcile-interval -register-node -register-retry-count -registry-burst -registry-qps -reject-methods -reject-paths -repo-root -report-dir -required-contexts -resolv-conf -resource-container -resource-quota-sync-period -resource-version -rkt-path -root-ca-file -root-dir -run-proxy -runtime-config -scheduler-config -secure-port -service-account-key-file -service-account-lookup -service-account-private-key-file -service-address -service-cluster-ip-range -service-node-port-range -service-node-ports -service-sync-period -session-affinity -show-all -shutdown-fd -shutdown-fifo -skip-munges -sort-by -source-file -ssh-keyfile -ssh-user -static-pods-config -stats-port -storage-version -streaming-connection-idle-timeout -suicide-timeout -sync-frequency -system-container -target-port -tcp-services -tls-cert-file -tls-private-key-file -token-auth-file -ttl-secs -type-src -unix-socket -update-period -upgrade-target -use-kubernetes-cluster-service -user-whitelist -watch-cache -watch-only -whitelist-override-label -www-prefix -retry_time -file_content_in_loop -cpu-cfs-quota +accept-hosts +accept-paths +account-for-pod-resources +admission-control +admission-control-config-file +advertise-address +advertised-address +algorithm-provider +all-namespaces +allocate-node-cidrs +allow-privileged +api-burst +api-prefix +api-rate +api-servers +api-token +api-version +authorization-mode +authorization-policy-file +auth-path +basic-auth-file +bench-pods +bench-quiet +bench-tasks +bench-workers +bind-address +bind-pods-burst +bind-pods-qps +cadvisor-port +cert-dir +certificate-authority +cgroup-root +chaos-chance +cleanup-iptables +client-ca-file +client-certificate +client-key +cloud-config +cloud-provider +cluster-cidr +cluster-dns +cluster-domain +cluster-name +cluster-tag +concurrent-endpoint-syncs +configure-cbr0 +contain-pod-resources +container-port +container-runtime +cors-allowed-origins +create-external-load-balancer +current-release-pr +current-replicas +default-container-cpu-limit +default-container-mem-limit +delay-shutdown +deleting-pods-burst +deleting-pods-qps +deployment-label-key +dest-file +disable-filter +docker-endpoint +docker-exec-handler +dockercfg-path +driver-port +dry-run +duration-sec +e2e-output-dir +enable-debugging-handlers +enable-horizontal-pod-autoscaler +enable-server +etcd-config +etcd-prefix +etcd-server +etcd-servers +event-burst +event-qps +event-ttl +executor-bindall +executor-logv +executor-path +executor-suicide-timeout +experimental-keystone-url +experimental-prefix +external-hostname +external-ip +failover-timeout +file-check-frequency +file-suffix +forward-services +framework-name +framework-weburi +func-dest +fuzz-iters +gce-project +gce-zone +gke-cluster +google-json-key +grace-period +ha-domain +healthz-bind-address +healthz-port +horizontal-pod-autoscaler-sync-period +hostname-override +host-network-sources +http-check-frequency +http-port +ignore-not-found +image-gc-high-threshold +image-gc-low-threshold +insecure-bind-address +insecure-port +insecure-skip-tls-verify +iptables-sync-period +ir-data-source +ir-dbname +ir-influxdb-host +ir-password +ir-user +jenkins-host +jenkins-jobs +km-path +kubectl-path +kubelet-cadvisor-port +kubelet-certificate-authority +kubelet-client-certificate +kubelet-client-key +kubelet-docker-endpoint +kubelet-host-network-sources +kubelet-https +kubelet-network-plugin +kubelet-pod-infra-container-image +kubelet-port +kubelet-root-dir +kubelet-sync-frequency +kubelet-timeout +kube-master +label-columns +last-release-pr +legacy-userspace-proxy +log-flush-frequency +long-running-request-regexp +low-diskspace-threshold-mb +manifest-url +manifest-url-header +masquerade-all +master-service-namespace +max-concurrency +max-connection-bytes-per-sec +maximum-dead-containers +maximum-dead-containers-per-container +max-log-age +max-log-backups +max-log-size +max-outgoing-burst +max-outgoing-qps +max-pods +max-requests-inflight +mesos-authentication-principal +mesos-authentication-provider +mesos-authentication-secret-file +mesos-cgroup-prefix +mesos-executor-cpus +mesos-executor-mem +mesos-master +mesos-role +mesos-user +minimum-container-ttl-duration +minion-max-log-age +minion-max-log-backups +minion-max-log-size +minion-path-override +min-pr-number +min-request-timeout +namespace-sync-period +network-plugin +network-plugin-dir +node-instance-group +node-monitor-grace-period +node-monitor-period +node-startup-grace-period +node-status-update-frequency +node-sync-period +no-headers +num-nodes +oidc-ca-file +oidc-client-id +oidc-issuer-url +oidc-username-claim +oom-score-adj +output-version +out-version +path-override +pod-cidr +pod-eviction-timeout +pod-infra-container-image +policy-config-file +poll-interval +portal-net +private-mountns +prom-push-gateway +proxy-bindall +proxy-logv +proxy-port-range +public-address-override +pvclaimbinder-sync-period +read-only-port +really-crash-for-testing +reconcile-cooldown +reconcile-interval +register-node +register-retry-count +registry-burst +registry-qps +reject-methods +reject-paths +repo-root +report-dir +required-contexts +resolv-conf +resource-container +resource-quota-sync-period +resource-version +rkt-path +root-ca-file +root-dir +run-proxy +runtime-config +scheduler-config +secure-port +service-account-key-file +service-account-lookup +service-account-private-key-file +service-address +service-cluster-ip-range +service-node-port-range +service-node-ports +service-sync-period +session-affinity +show-all +shutdown-fd +shutdown-fifo +skip-munges +sort-by +source-file +ssh-keyfile +ssh-user +static-pods-config +stats-port +storage-version +streaming-connection-idle-timeout +suicide-timeout +sync-frequency +system-container +target-port +tcp-services +tls-cert-file +tls-private-key-file +token-auth-file +ttl-secs +type-src +unix-socket +update-period +upgrade-target +use-kubernetes-cluster-service +user-whitelist +watch-cache +watch-only +whitelist-override-label +www-prefix +retry_time +file_content_in_loop +cpu-cfs-quota diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 243ccc8b446..d82f29ff26e 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -139,6 +139,8 @@ func NewMainKubelet( resyncInterval time.Duration, pullQPS float32, pullBurst int, + eventQPS float32, + eventBurst int, containerGCPolicy ContainerGCPolicy, sourcesReady SourcesReadyFn, registerNode bool, From 6998247e1b3dc07d3162030b4c914207cd2086ae Mon Sep 17 00:00:00 2001 From: Jerzy Szczepkowski Date: Thu, 3 Sep 2015 16:50:53 +0200 Subject: [PATCH 15/46] Implemented removal of Deployments, Daemons & HorizontalPodAutoscalers when Namespace is removed. Implemented removal of Deployments, Daemons & HorizontalPodAutoscalers when Namespace is removed. Added unittest. Fixes #12735. --- .../app/controllermanager.go | 19 ++--- .../controllermanager/controllermanager.go | 2 +- .../testclient/fake_deployments.go | 82 +++++++++++++++++++ .../fake_horizontal_pod_autoscalers.go | 82 +++++++++++++++++++ .../unversioned/testclient/fake_scales.go | 52 ++++++++++++ .../unversioned/testclient/testclient.go | 12 +-- .../horizontalpodautoscaler_controller.go | 12 ++- ...horizontalpodautoscaler_controller_test.go | 6 +- .../namespace/namespace_controller.go | 70 ++++++++++++++-- .../namespace/namespace_controller_test.go | 30 ++++++- 10 files changed, 327 insertions(+), 40 deletions(-) create mode 100644 pkg/client/unversioned/testclient/fake_deployments.go create mode 100644 pkg/client/unversioned/testclient/fake_horizontal_pod_autoscalers.go create mode 100644 pkg/client/unversioned/testclient/fake_scales.go diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 30d7fba666f..7a7e7aecd63 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -242,9 +242,16 @@ func (s *CMServer) Run(_ []string) error { resourceQuotaController := resourcequotacontroller.NewResourceQuotaController(kubeClient) resourceQuotaController.Run(s.ResourceQuotaSyncPeriod) - namespaceController := namespacecontroller.NewNamespaceController(kubeClient, s.NamespaceSyncPeriod) + // An OR of all flags to enable/disable experimental features + experimentalMode := s.EnableHorizontalPodAutoscaler + namespaceController := namespacecontroller.NewNamespaceController(kubeClient, experimentalMode, s.NamespaceSyncPeriod) namespaceController.Run() + if s.EnableHorizontalPodAutoscaler { + horizontalPodAutoscalerController := autoscalercontroller.New(kubeClient, metrics.NewHeapsterMetricsClient(kubeClient)) + horizontalPodAutoscalerController.Run(s.HorizontalPodAutoscalerSyncPeriod) + } + pvclaimBinder := volumeclaimbinder.NewPersistentVolumeClaimBinder(kubeClient, s.PVClaimBinderSyncPeriod) pvclaimBinder.Run() pvRecycler, err := volumeclaimbinder.NewPersistentVolumeRecycler(kubeClient, s.PVClaimBinderSyncPeriod, ProbeRecyclableVolumePlugins(s.VolumeConfigFlags)) @@ -287,15 +294,5 @@ func (s *CMServer) Run(_ []string) error { serviceaccount.DefaultServiceAccountsControllerOptions(), ).Run() - if s.EnableHorizontalPodAutoscaler { - expClient, err := client.NewExperimental(kubeconfig) - if err != nil { - glog.Fatalf("Invalid API configuration: %v", err) - } - horizontalPodAutoscalerController := autoscalercontroller.New(kubeClient, expClient, - metrics.NewHeapsterMetricsClient(kubeClient)) - horizontalPodAutoscalerController.Run(s.HorizontalPodAutoscalerSyncPeriod) - } - select {} } diff --git a/contrib/mesos/pkg/controllermanager/controllermanager.go b/contrib/mesos/pkg/controllermanager/controllermanager.go index e7ac60e92e4..ad44a1c0e2b 100644 --- a/contrib/mesos/pkg/controllermanager/controllermanager.go +++ b/contrib/mesos/pkg/controllermanager/controllermanager.go @@ -144,7 +144,7 @@ func (s *CMServer) Run(_ []string) error { resourceQuotaController := resourcequotacontroller.NewResourceQuotaController(kubeClient) resourceQuotaController.Run(s.ResourceQuotaSyncPeriod) - namespaceController := namespacecontroller.NewNamespaceController(kubeClient, s.NamespaceSyncPeriod) + namespaceController := namespacecontroller.NewNamespaceController(kubeClient, false, s.NamespaceSyncPeriod) namespaceController.Run() pvclaimBinder := volumeclaimbinder.NewPersistentVolumeClaimBinder(kubeClient, s.PVClaimBinderSyncPeriod) diff --git a/pkg/client/unversioned/testclient/fake_deployments.go b/pkg/client/unversioned/testclient/fake_deployments.go new file mode 100644 index 00000000000..38942476ffd --- /dev/null +++ b/pkg/client/unversioned/testclient/fake_deployments.go @@ -0,0 +1,82 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testclient + +import ( + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/expapi" + "k8s.io/kubernetes/pkg/fields" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/watch" +) + +// FakeDeployments implements DeploymentsInterface. Meant to be embedded into a struct to get a default +// implementation. This makes faking out just the methods you want to test easier. +type FakeDeployments struct { + Fake *FakeExperimental + Namespace string +} + +func (c *FakeDeployments) Get(name string) (*expapi.Deployment, error) { + obj, err := c.Fake.Invokes(NewGetAction("deployments", c.Namespace, name), &expapi.Deployment{}) + if obj == nil { + return nil, err + } + + return obj.(*expapi.Deployment), err +} + +func (c *FakeDeployments) List(label labels.Selector, field fields.Selector) (*expapi.DeploymentList, error) { + obj, err := c.Fake.Invokes(NewListAction("deployments", c.Namespace, label, field), &expapi.DeploymentList{}) + if obj == nil { + return nil, err + } + list := &expapi.DeploymentList{} + for _, deployment := range obj.(*expapi.DeploymentList).Items { + if label.Matches(labels.Set(deployment.Labels)) { + list.Items = append(list.Items, deployment) + } + } + return list, err +} + +func (c *FakeDeployments) Create(deployment *expapi.Deployment) (*expapi.Deployment, error) { + obj, err := c.Fake.Invokes(NewCreateAction("deployments", c.Namespace, deployment), deployment) + if obj == nil { + return nil, err + } + + return obj.(*expapi.Deployment), err +} + +func (c *FakeDeployments) Update(deployment *expapi.Deployment) (*expapi.Deployment, error) { + obj, err := c.Fake.Invokes(NewUpdateAction("deployments", c.Namespace, deployment), deployment) + if obj == nil { + return nil, err + } + + return obj.(*expapi.Deployment), err +} + +func (c *FakeDeployments) Delete(name string, options *api.DeleteOptions) error { + _, err := c.Fake.Invokes(NewDeleteAction("deployments", c.Namespace, name), &expapi.Deployment{}) + return err +} + +func (c *FakeDeployments) Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { + return c.Fake.InvokesWatch(NewWatchAction("deployments", c.Namespace, label, field, resourceVersion)) +} diff --git a/pkg/client/unversioned/testclient/fake_horizontal_pod_autoscalers.go b/pkg/client/unversioned/testclient/fake_horizontal_pod_autoscalers.go new file mode 100644 index 00000000000..4c3b7369d9d --- /dev/null +++ b/pkg/client/unversioned/testclient/fake_horizontal_pod_autoscalers.go @@ -0,0 +1,82 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testclient + +import ( + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/expapi" + "k8s.io/kubernetes/pkg/fields" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/watch" +) + +// FakeHorizontalPodAutoscalers implements HorizontalPodAutoscalerInterface. Meant to be embedded into a struct to get a default +// implementation. This makes faking out just the methods you want to test easier. +type FakeHorizontalPodAutoscalers struct { + Fake *FakeExperimental + Namespace string +} + +func (c *FakeHorizontalPodAutoscalers) Get(name string) (*expapi.HorizontalPodAutoscaler, error) { + obj, err := c.Fake.Invokes(NewGetAction("horizontalpodautoscalers", c.Namespace, name), &expapi.HorizontalPodAutoscaler{}) + if obj == nil { + return nil, err + } + + return obj.(*expapi.HorizontalPodAutoscaler), err +} + +func (c *FakeHorizontalPodAutoscalers) List(label labels.Selector, field fields.Selector) (*expapi.HorizontalPodAutoscalerList, error) { + obj, err := c.Fake.Invokes(NewListAction("horizontalpodautoscalers", c.Namespace, label, field), &expapi.HorizontalPodAutoscalerList{}) + if obj == nil { + return nil, err + } + list := &expapi.HorizontalPodAutoscalerList{} + for _, a := range obj.(*expapi.HorizontalPodAutoscalerList).Items { + if label.Matches(labels.Set(a.Labels)) { + list.Items = append(list.Items, a) + } + } + return list, err +} + +func (c *FakeHorizontalPodAutoscalers) Create(a *expapi.HorizontalPodAutoscaler) (*expapi.HorizontalPodAutoscaler, error) { + obj, err := c.Fake.Invokes(NewCreateAction("horizontalpodautoscalers", c.Namespace, a), a) + if obj == nil { + return nil, err + } + + return obj.(*expapi.HorizontalPodAutoscaler), err +} + +func (c *FakeHorizontalPodAutoscalers) Update(a *expapi.HorizontalPodAutoscaler) (*expapi.HorizontalPodAutoscaler, error) { + obj, err := c.Fake.Invokes(NewUpdateAction("horizontalpodautoscalers", c.Namespace, a), a) + if obj == nil { + return nil, err + } + + return obj.(*expapi.HorizontalPodAutoscaler), err +} + +func (c *FakeHorizontalPodAutoscalers) Delete(name string, options *api.DeleteOptions) error { + _, err := c.Fake.Invokes(NewDeleteAction("horizontalpodautoscalers", c.Namespace, name), &expapi.HorizontalPodAutoscaler{}) + return err +} + +func (c *FakeHorizontalPodAutoscalers) Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { + return c.Fake.InvokesWatch(NewWatchAction("horizontalpodautoscalers", c.Namespace, label, field, resourceVersion)) +} diff --git a/pkg/client/unversioned/testclient/fake_scales.go b/pkg/client/unversioned/testclient/fake_scales.go new file mode 100644 index 00000000000..95d7220791f --- /dev/null +++ b/pkg/client/unversioned/testclient/fake_scales.go @@ -0,0 +1,52 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testclient + +import ( + "k8s.io/kubernetes/pkg/expapi" +) + +// FakeScales implements ScaleInterface. Meant to be embedded into a struct to get a default +// implementation. This makes faking out just the methods you want to test easier. +type FakeScales struct { + Fake *FakeExperimental + Namespace string +} + +func (c *FakeScales) Get(kind string, name string) (result *expapi.Scale, err error) { + action := GetActionImpl{} + action.Verb = "get" + action.Namespace = c.Namespace + action.Resource = kind + action.Subresource = "scale" + action.Name = name + obj, err := c.Fake.Invokes(action, &expapi.Scale{}) + result = obj.(*expapi.Scale) + return +} + +func (c *FakeScales) Update(kind string, scale *expapi.Scale) (result *expapi.Scale, err error) { + action := UpdateActionImpl{} + action.Verb = "update" + action.Namespace = c.Namespace + action.Resource = kind + action.Subresource = "scale" + action.Object = scale + obj, err := c.Fake.Invokes(action, scale) + result = obj.(*expapi.Scale) + return +} diff --git a/pkg/client/unversioned/testclient/testclient.go b/pkg/client/unversioned/testclient/testclient.go index 2773890482b..9949a0d1985 100644 --- a/pkg/client/unversioned/testclient/testclient.go +++ b/pkg/client/unversioned/testclient/testclient.go @@ -251,13 +251,13 @@ func (c *FakeExperimental) Daemons(namespace string) client.DaemonInterface { } func (c *FakeExperimental) HorizontalPodAutoscalers(namespace string) client.HorizontalPodAutoscalerInterface { - panic("unimplemented") -} - -func (c *FakeExperimental) Scales(namespace string) client.ScaleInterface { - panic("unimplemented") + return &FakeHorizontalPodAutoscalers{Fake: c, Namespace: namespace} } func (c *FakeExperimental) Deployments(namespace string) client.DeploymentInterface { - panic("unimplemented") + return &FakeDeployments{Fake: c, Namespace: namespace} +} + +func (c *FakeExperimental) Scales(namespace string) client.ScaleInterface { + return &FakeScales{Fake: c, Namespace: namespace} } diff --git a/pkg/controller/autoscaler/horizontalpodautoscaler_controller.go b/pkg/controller/autoscaler/horizontalpodautoscaler_controller.go index abdb061e99d..11e98ccadca 100644 --- a/pkg/controller/autoscaler/horizontalpodautoscaler_controller.go +++ b/pkg/controller/autoscaler/horizontalpodautoscaler_controller.go @@ -42,17 +42,15 @@ const ( type HorizontalPodAutoscalerController struct { client client.Interface - expClient client.ExperimentalInterface metricsClient metrics.MetricsClient } var downscaleForbiddenWindow, _ = time.ParseDuration("20m") var upscaleForbiddenWindow, _ = time.ParseDuration("3m") -func New(client client.Interface, expClient client.ExperimentalInterface, metricsClient metrics.MetricsClient) *HorizontalPodAutoscalerController { +func New(client client.Interface, metricsClient metrics.MetricsClient) *HorizontalPodAutoscalerController { return &HorizontalPodAutoscalerController{ client: client, - expClient: expClient, metricsClient: metricsClient, } } @@ -67,14 +65,14 @@ func (a *HorizontalPodAutoscalerController) Run(syncPeriod time.Duration) { func (a *HorizontalPodAutoscalerController) reconcileAutoscalers() error { ns := api.NamespaceAll - list, err := a.expClient.HorizontalPodAutoscalers(ns).List(labels.Everything(), fields.Everything()) + list, err := a.client.Experimental().HorizontalPodAutoscalers(ns).List(labels.Everything(), fields.Everything()) if err != nil { return fmt.Errorf("error listing nodes: %v", err) } for _, hpa := range list.Items { reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Namespace, hpa.Spec.ScaleRef.Name) - scale, err := a.expClient.Scales(hpa.Spec.ScaleRef.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name) + scale, err := a.client.Experimental().Scales(hpa.Spec.ScaleRef.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name) if err != nil { glog.Warningf("Failed to query scale subresource for %s: %v", reference, err) continue @@ -127,7 +125,7 @@ func (a *HorizontalPodAutoscalerController) reconcileAutoscalers() error { if rescale { scale.Spec.Replicas = desiredReplicas - _, err = a.expClient.Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale) + _, err = a.client.Experimental().Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale) if err != nil { glog.Warningf("Failed to rescale %s: %v", reference, err) continue @@ -147,7 +145,7 @@ func (a *HorizontalPodAutoscalerController) reconcileAutoscalers() error { hpa.Status.LastScaleTimestamp = &now } - _, err = a.expClient.HorizontalPodAutoscalers(hpa.Namespace).Update(&hpa) + _, err = a.client.Experimental().HorizontalPodAutoscalers(hpa.Namespace).Update(&hpa) if err != nil { glog.Warningf("Failed to update HorizontalPodAutoscaler %s: %v", hpa.Name, err) continue diff --git a/pkg/controller/autoscaler/horizontalpodautoscaler_controller_test.go b/pkg/controller/autoscaler/horizontalpodautoscaler_controller_test.go index 1147cc3087c..bdffbbb18ba 100644 --- a/pkg/controller/autoscaler/horizontalpodautoscaler_controller_test.go +++ b/pkg/controller/autoscaler/horizontalpodautoscaler_controller_test.go @@ -177,14 +177,12 @@ func TestSyncEndpointsItemsPreserveNoSelector(t *testing.T) { defer testServer.Close() kubeClient := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Experimental.Version()}) - expClient := client.NewExperimentalOrDie(&client.Config{Host: testServer.URL, Version: testapi.Experimental.Version()}) - fakeRC := fakeResourceConsumptionClient{metrics: map[api.ResourceName]expapi.ResourceConsumption{ api.ResourceCPU: {Resource: api.ResourceCPU, Quantity: resource.MustParse("650m")}, }} fake := fakeMetricsClient{consumption: &fakeRC} - hpaController := New(kubeClient, expClient, &fake) + hpaController := New(kubeClient, &fake) err := hpaController.reconcileAutoscalers() if err != nil { @@ -193,7 +191,7 @@ func TestSyncEndpointsItemsPreserveNoSelector(t *testing.T) { for _, h := range handlers { h.ValidateRequestCount(t, 1) } - obj, err := expClient.Codec.Decode([]byte(handlers[updateHpaHandler].RequestBody)) + obj, err := kubeClient.Codec.Decode([]byte(handlers[updateHpaHandler].RequestBody)) if err != nil { t.Fatal("Failed to decode: %v %v", err) } diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go index 425cf830c34..9cbc79c7dd4 100644 --- a/pkg/controller/namespace/namespace_controller.go +++ b/pkg/controller/namespace/namespace_controller.go @@ -41,7 +41,7 @@ type NamespaceController struct { } // NewNamespaceController creates a new NamespaceController -func NewNamespaceController(kubeClient client.Interface, resyncPeriod time.Duration) *NamespaceController { +func NewNamespaceController(kubeClient client.Interface, experimentalMode bool, resyncPeriod time.Duration) *NamespaceController { var controller *framework.Controller _, controller = framework.NewInformer( &cache.ListWatch{ @@ -57,7 +57,7 @@ func NewNamespaceController(kubeClient client.Interface, resyncPeriod time.Durat framework.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { namespace := obj.(*api.Namespace) - if err := syncNamespace(kubeClient, *namespace); err != nil { + if err := syncNamespace(kubeClient, experimentalMode, *namespace); err != nil { if estimate, ok := err.(*contentRemainingError); ok { go func() { // Estimate is the aggregate total of TerminationGracePeriodSeconds, which defaults to 30s @@ -79,7 +79,7 @@ func NewNamespaceController(kubeClient client.Interface, resyncPeriod time.Durat }, UpdateFunc: func(oldObj, newObj interface{}) { namespace := newObj.(*api.Namespace) - if err := syncNamespace(kubeClient, *namespace); err != nil { + if err := syncNamespace(kubeClient, experimentalMode, *namespace); err != nil { if estimate, ok := err.(*contentRemainingError); ok { go func() { t := estimate.Estimate/2 + 1 @@ -152,7 +152,7 @@ func (e *contentRemainingError) Error() string { // deleteAllContent will delete all content known to the system in a namespace. It returns an estimate // of the time remaining before the remaining resources are deleted. If estimate > 0 not all resources // are guaranteed to be gone. -func deleteAllContent(kubeClient client.Interface, namespace string, before util.Time) (estimate int64, err error) { +func deleteAllContent(kubeClient client.Interface, experimentalMode bool, namespace string, before util.Time) (estimate int64, err error) { err = deleteServiceAccounts(kubeClient, namespace) if err != nil { return estimate, err @@ -189,12 +189,26 @@ func deleteAllContent(kubeClient client.Interface, namespace string, before util if err != nil { return estimate, err } - + // If experimental mode, delete all experimental resources for the namespace. + if experimentalMode { + err = deleteHorizontalPodAutoscalers(kubeClient.Experimental(), namespace) + if err != nil { + return estimate, err + } + err = deleteDaemons(kubeClient.Experimental(), namespace) + if err != nil { + return estimate, err + } + err = deleteDeployments(kubeClient.Experimental(), namespace) + if err != nil { + return estimate, err + } + } return estimate, nil } // syncNamespace makes namespace life-cycle decisions -func syncNamespace(kubeClient client.Interface, namespace api.Namespace) (err error) { +func syncNamespace(kubeClient client.Interface, experimentalMode bool, namespace api.Namespace) (err error) { if namespace.DeletionTimestamp == nil { return nil } @@ -224,7 +238,7 @@ func syncNamespace(kubeClient client.Interface, namespace api.Namespace) (err er } // there may still be content for us to remove - estimate, err := deleteAllContent(kubeClient, namespace.Name, *namespace.DeletionTimestamp) + estimate, err := deleteAllContent(kubeClient, experimentalMode, namespace.Name, *namespace.DeletionTimestamp) if err != nil { return err } @@ -389,3 +403,45 @@ func deletePersistentVolumeClaims(kubeClient client.Interface, ns string) error } return nil } + +func deleteHorizontalPodAutoscalers(expClient client.ExperimentalInterface, ns string) error { + items, err := expClient.HorizontalPodAutoscalers(ns).List(labels.Everything(), fields.Everything()) + if err != nil { + return err + } + for i := range items.Items { + err := expClient.HorizontalPodAutoscalers(ns).Delete(items.Items[i].Name, nil) + if err != nil && !errors.IsNotFound(err) { + return err + } + } + return nil +} + +func deleteDaemons(expClient client.ExperimentalInterface, ns string) error { + items, err := expClient.Daemons(ns).List(labels.Everything()) + if err != nil { + return err + } + for i := range items.Items { + err := expClient.Daemons(ns).Delete(items.Items[i].Name) + if err != nil && !errors.IsNotFound(err) { + return err + } + } + return nil +} + +func deleteDeployments(expClient client.ExperimentalInterface, ns string) error { + items, err := expClient.Deployments(ns).List(labels.Everything(), fields.Everything()) + if err != nil { + return err + } + for i := range items.Items { + err := expClient.Deployments(ns).Delete(items.Items[i].Name, nil) + if err != nil && !errors.IsNotFound(err) { + return err + } + } + return nil +} diff --git a/pkg/controller/namespace/namespace_controller_test.go b/pkg/controller/namespace/namespace_controller_test.go index b98e038c5b3..d1807e4e5df 100644 --- a/pkg/controller/namespace/namespace_controller_test.go +++ b/pkg/controller/namespace/namespace_controller_test.go @@ -69,7 +69,7 @@ func TestFinalize(t *testing.T) { } } -func TestSyncNamespaceThatIsTerminating(t *testing.T) { +func testSyncNamespaceThatIsTerminating(t *testing.T, experimentalMode bool) { mockClient := &testclient.Fake{} now := util.Now() testNamespace := api.Namespace{ @@ -85,7 +85,7 @@ func TestSyncNamespaceThatIsTerminating(t *testing.T) { Phase: api.NamespaceTerminating, }, } - err := syncNamespace(mockClient, testNamespace) + err := syncNamespace(mockClient, experimentalMode, testNamespace) if err != nil { t.Errorf("Unexpected error when synching namespace %v", err) } @@ -98,9 +98,20 @@ func TestSyncNamespaceThatIsTerminating(t *testing.T) { strings.Join([]string{"list", "secrets", ""}, "-"), strings.Join([]string{"list", "limitranges", ""}, "-"), strings.Join([]string{"list", "events", ""}, "-"), + strings.Join([]string{"list", "serviceaccounts", ""}, "-"), + strings.Join([]string{"list", "persistentvolumeclaims", ""}, "-"), strings.Join([]string{"create", "namespaces", "finalize"}, "-"), strings.Join([]string{"delete", "namespaces", ""}, "-"), ) + + if experimentalMode { + expectedActionSet.Insert( + strings.Join([]string{"list", "horizontalpodautoscalers", ""}, "-"), + strings.Join([]string{"list", "daemons", ""}, "-"), + strings.Join([]string{"list", "deployments", ""}, "-"), + ) + } + actionSet := util.NewStringSet() for _, action := range mockClient.Actions() { actionSet.Insert(strings.Join([]string{action.GetVerb(), action.GetResource(), action.GetSubresource()}, "-")) @@ -108,6 +119,17 @@ func TestSyncNamespaceThatIsTerminating(t *testing.T) { if !actionSet.HasAll(expectedActionSet.List()...) { t.Errorf("Expected actions: %v, but got: %v", expectedActionSet, actionSet) } + if !expectedActionSet.HasAll(actionSet.List()...) { + t.Errorf("Expected actions: %v, but got: %v", expectedActionSet, actionSet) + } +} + +func TestSyncNamespaceThatIsTerminatingNonExperimental(t *testing.T) { + testSyncNamespaceThatIsTerminating(t, false) +} + +func TestSyncNamespaceThatIsTerminatingExperimental(t *testing.T) { + testSyncNamespaceThatIsTerminating(t, true) } func TestSyncNamespaceThatIsActive(t *testing.T) { @@ -124,7 +146,7 @@ func TestSyncNamespaceThatIsActive(t *testing.T) { Phase: api.NamespaceActive, }, } - err := syncNamespace(mockClient, testNamespace) + err := syncNamespace(mockClient, false, testNamespace) if err != nil { t.Errorf("Unexpected error when synching namespace %v", err) } @@ -135,7 +157,7 @@ func TestSyncNamespaceThatIsActive(t *testing.T) { func TestRunStop(t *testing.T) { mockClient := &testclient.Fake{} - nsController := NewNamespaceController(mockClient, 1*time.Second) + nsController := NewNamespaceController(mockClient, false, 1*time.Second) if nsController.StopEverything != nil { t.Errorf("Non-running manager should not have a stop channel. Got %v", nsController.StopEverything) From 70c74fbe4b9c14511f87e57f80715d7bf58c2219 Mon Sep 17 00:00:00 2001 From: Jan Chaloupka Date: Wed, 9 Sep 2015 12:02:22 +0200 Subject: [PATCH 16/46] Commit cd7d78b6969a9ccd9fc370e5c8d79b7ae9114095 replaced use of util.CheckErr(err) with if err != nil { return err } One replacement is incorrent. The current call of resource.NewBuilder is returned in r variable. Which was tested with util.CheckErr(r.Err()) before cd7d78b6969a9ccd9fc370e5c8d79b7ae9114095 commit. It was replaced by if err != nil { return err } which is incorrect as err is not set by resource.NewBuilder call. The correct use is err := r.Err() if err != nil { return err } --- pkg/kubectl/cmd/get.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/kubectl/cmd/get.go b/pkg/kubectl/cmd/get.go index e0780a5b2bd..1c4d0654cc7 100644 --- a/pkg/kubectl/cmd/get.go +++ b/pkg/kubectl/cmd/get.go @@ -130,6 +130,7 @@ func RunGet(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string SingleResourceType(). Latest(). Do() + err := r.Err() if err != nil { return err } From 7b4f91c4ee1286736aaa2bc4c16d394aeff44b65 Mon Sep 17 00:00:00 2001 From: hurf Date: Wed, 9 Sep 2015 21:23:03 +0800 Subject: [PATCH 17/46] Doc fix for admission/plugin.go Complete unfinished doc, changed some comments to be more readable. --- pkg/admission/plugins.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/admission/plugins.go b/pkg/admission/plugins.go index 7824396e619..3c9e9f1bc0d 100644 --- a/pkg/admission/plugins.go +++ b/pkg/admission/plugins.go @@ -37,7 +37,7 @@ var ( plugins = make(map[string]Factory) ) -// GetPlugins enumerates the +// GetPlugins enumerates the names of all registered plugins. func GetPlugins() []string { pluginsMutex.Lock() defer pluginsMutex.Unlock() @@ -48,7 +48,7 @@ func GetPlugins() []string { return keys } -// RegisterPlugin registers a plugin Factory by name. This +// RegisterPlugin registers a plugin Factory by name. This // is expected to happen during app startup. func RegisterPlugin(name string, plugin Factory) { pluginsMutex.Lock() @@ -61,11 +61,10 @@ func RegisterPlugin(name string, plugin Factory) { plugins[name] = plugin } -// GetPlugin creates an instance of the named plugin, or nil if -// the name is not known. The error return is only used if the named provider -// was known but failed to initialize. The config parameter specifies the -// io.Reader handler of the configuration file for the cloud provider, or nil -// for no configuration. +// GetPlugin creates an instance of the named plugin, or nil if the name is not +// known. The error is returned only when the named provider was known but failed +// to initialize. The config parameter specifies the io.Reader handler of the +// configuration file for the cloud provider, or nil for no configuration. func GetPlugin(name string, client client.Interface, config io.Reader) (Interface, error) { pluginsMutex.Lock() defer pluginsMutex.Unlock() From cd1ac360deb30a5fba24858b8d71066f3dc3a17f Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Wed, 9 Sep 2015 16:18:17 +0200 Subject: [PATCH 18/46] Rename "minion" to "node" in few places. --- pkg/controller/node/doc.go | 2 +- pkg/kubectl/resource_printer_test.go | 28 ++--- pkg/kubelet/kubelet.go | 4 +- pkg/master/master.go | 12 +- pkg/master/master_test.go | 2 +- pkg/registry/{minion => node}/doc.go | 4 +- pkg/registry/{minion => node}/etcd/etcd.go | 14 +-- .../{minion => node}/etcd/etcd_test.go | 0 pkg/registry/{minion => node}/registry.go | 26 ++-- pkg/registry/{minion => node}/strategy.go | 2 +- .../{minion => node}/strategy_test.go | 2 +- pkg/registry/registrytest/minion.go | 117 ------------------ pkg/registry/registrytest/node.go | 117 ++++++++++++++++++ 13 files changed, 165 insertions(+), 165 deletions(-) rename pkg/registry/{minion => node}/doc.go (85%) rename pkg/registry/{minion => node}/etcd/etcd.go (90%) rename pkg/registry/{minion => node}/etcd/etcd_test.go (100%) rename pkg/registry/{minion => node}/registry.go (60%) rename pkg/registry/{minion => node}/strategy.go (99%) rename pkg/registry/{minion => node}/strategy_test.go (98%) delete mode 100644 pkg/registry/registrytest/minion.go create mode 100644 pkg/registry/registrytest/node.go diff --git a/pkg/controller/node/doc.go b/pkg/controller/node/doc.go index 0cc00b6fff8..3174bef7c09 100644 --- a/pkg/controller/node/doc.go +++ b/pkg/controller/node/doc.go @@ -15,5 +15,5 @@ limitations under the License. */ // Package nodecontroller contains code for syncing cloud instances with -// minion registry +// node registry package nodecontroller diff --git a/pkg/kubectl/resource_printer_test.go b/pkg/kubectl/resource_printer_test.go index 4197ea98e84..b7d67eaffd7 100644 --- a/pkg/kubectl/resource_printer_test.go +++ b/pkg/kubectl/resource_printer_test.go @@ -545,21 +545,21 @@ func TestPrintEventsResultSorted(t *testing.T) { VerifyDatesInOrder(out, "\n" /* rowDelimiter */, " " /* columnDelimiter */, t) } -func TestPrintMinionStatus(t *testing.T) { +func TestPrintNodeStatus(t *testing.T) { printer := NewHumanReadablePrinter(false, false, false, false, []string{}) table := []struct { - minion api.Node + node api.Node status string }{ { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo1"}, Status: api.NodeStatus{Conditions: []api.NodeCondition{{Type: api.NodeReady, Status: api.ConditionTrue}}}, }, status: "Ready", }, { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo2"}, Spec: api.NodeSpec{Unschedulable: true}, Status: api.NodeStatus{Conditions: []api.NodeCondition{{Type: api.NodeReady, Status: api.ConditionTrue}}}, @@ -567,7 +567,7 @@ func TestPrintMinionStatus(t *testing.T) { status: "Ready,SchedulingDisabled", }, { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo3"}, Status: api.NodeStatus{Conditions: []api.NodeCondition{ {Type: api.NodeReady, Status: api.ConditionTrue}, @@ -576,14 +576,14 @@ func TestPrintMinionStatus(t *testing.T) { status: "Ready", }, { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo4"}, Status: api.NodeStatus{Conditions: []api.NodeCondition{{Type: api.NodeReady, Status: api.ConditionFalse}}}, }, status: "NotReady", }, { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo5"}, Spec: api.NodeSpec{Unschedulable: true}, Status: api.NodeStatus{Conditions: []api.NodeCondition{{Type: api.NodeReady, Status: api.ConditionFalse}}}, @@ -591,21 +591,21 @@ func TestPrintMinionStatus(t *testing.T) { status: "NotReady,SchedulingDisabled", }, { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo6"}, Status: api.NodeStatus{Conditions: []api.NodeCondition{{Type: "InvalidValue", Status: api.ConditionTrue}}}, }, status: "Unknown", }, { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo7"}, Status: api.NodeStatus{Conditions: []api.NodeCondition{{}}}, }, status: "Unknown", }, { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo8"}, Spec: api.NodeSpec{Unschedulable: true}, Status: api.NodeStatus{Conditions: []api.NodeCondition{{Type: "InvalidValue", Status: api.ConditionTrue}}}, @@ -613,7 +613,7 @@ func TestPrintMinionStatus(t *testing.T) { status: "Unknown,SchedulingDisabled", }, { - minion: api.Node{ + node: api.Node{ ObjectMeta: api.ObjectMeta{Name: "foo9"}, Spec: api.NodeSpec{Unschedulable: true}, Status: api.NodeStatus{Conditions: []api.NodeCondition{{}}}, @@ -624,12 +624,12 @@ func TestPrintMinionStatus(t *testing.T) { for _, test := range table { buffer := &bytes.Buffer{} - err := printer.PrintObj(&test.minion, buffer) + err := printer.PrintObj(&test.node, buffer) if err != nil { - t.Fatalf("An error occurred printing Minion: %#v", err) + t.Fatalf("An error occurred printing Node: %#v", err) } if !contains(strings.Fields(buffer.String()), test.status) { - t.Fatalf("Expect printing minion %s with status %#v, got: %#v", test.minion.Name, test.status, buffer.String()) + t.Fatalf("Expect printing node %s with status %#v, got: %#v", test.node.Name, test.status, buffer.String()) } } } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 243ccc8b446..ca2fea10727 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -214,8 +214,8 @@ func NewMainKubelet( } nodeLister := &cache.StoreToNodeLister{Store: nodeStore} - // TODO: get the real minion object of ourself, - // and use the real minion name and UID. + // TODO: get the real node object of ourself, + // and use the real node name and UID. // TODO: what is namespace for node? nodeRef := &api.ObjectReference{ Kind: "Node", diff --git a/pkg/master/master.go b/pkg/master/master.go index 0c6569a7e3a..d3498aec8c4 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -56,10 +56,10 @@ import ( eventetcd "k8s.io/kubernetes/pkg/registry/event/etcd" expcontrolleretcd "k8s.io/kubernetes/pkg/registry/experimental/controller/etcd" limitrangeetcd "k8s.io/kubernetes/pkg/registry/limitrange/etcd" - "k8s.io/kubernetes/pkg/registry/minion" - nodeetcd "k8s.io/kubernetes/pkg/registry/minion/etcd" "k8s.io/kubernetes/pkg/registry/namespace" namespaceetcd "k8s.io/kubernetes/pkg/registry/namespace/etcd" + "k8s.io/kubernetes/pkg/registry/node" + nodeetcd "k8s.io/kubernetes/pkg/registry/node/etcd" pvetcd "k8s.io/kubernetes/pkg/registry/persistentvolume/etcd" pvcetcd "k8s.io/kubernetes/pkg/registry/persistentvolumeclaim/etcd" podetcd "k8s.io/kubernetes/pkg/registry/pod/etcd" @@ -100,7 +100,7 @@ type Config struct { DatabaseStorage storage.Interface ExpDatabaseStorage storage.Interface EventTTL time.Duration - MinionRegexp string + NodeRegexp string KubeletClient client.KubeletClient // allow downstream consumers to disable the core controller loops EnableCoreControllers bool @@ -219,7 +219,7 @@ type Master struct { // registries are internal client APIs for accessing the storage layer // TODO: define the internal typed interface in a way that clients can // also be replaced - nodeRegistry minion.Registry + nodeRegistry node.Registry namespaceRegistry namespace.Registry serviceRegistry service.Registry endpointRegistry endpoint.Registry @@ -446,7 +446,7 @@ func (m *Master) init(c *Config) { m.endpointRegistry = endpoint.NewRegistry(endpointsStorage) nodeStorage, nodeStatusStorage := nodeetcd.NewREST(c.DatabaseStorage, c.EnableWatchCache, c.KubeletClient) - m.nodeRegistry = minion.NewRegistry(nodeStorage) + m.nodeRegistry = node.NewRegistry(nodeStorage) serviceStorage := serviceetcd.NewREST(c.DatabaseStorage) m.serviceRegistry = service.NewRegistry(serviceStorage) @@ -911,7 +911,7 @@ func (m *Master) needToReplaceTunnels(addrs []string) bool { } func (m *Master) getNodeAddresses() ([]string, error) { - nodes, err := m.nodeRegistry.ListMinions(api.NewDefaultContext(), labels.Everything(), fields.Everything()) + nodes, err := m.nodeRegistry.ListNodes(api.NewDefaultContext(), labels.Everything(), fields.Everything()) if err != nil { return nil, err } diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index f7ac4ebc381..898d92385ab 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -45,7 +45,7 @@ func TestGetServersToValidate(t *testing.T) { config.DatabaseStorage = etcdstorage.NewEtcdStorage(fakeClient, latest.Codec, etcdtest.PathPrefix()) config.ExpDatabaseStorage = etcdstorage.NewEtcdStorage(fakeClient, explatest.Codec, etcdtest.PathPrefix()) - master.nodeRegistry = registrytest.NewMinionRegistry([]string{"node1", "node2"}, api.NodeResources{}) + master.nodeRegistry = registrytest.NewNodeRegistry([]string{"node1", "node2"}, api.NodeResources{}) servers := master.getServersToValidate(&config) diff --git a/pkg/registry/minion/doc.go b/pkg/registry/node/doc.go similarity index 85% rename from pkg/registry/minion/doc.go rename to pkg/registry/node/doc.go index b67a96d2b13..cd604b4ab41 100644 --- a/pkg/registry/minion/doc.go +++ b/pkg/registry/node/doc.go @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package minion provides Registry interface and implementation for storing Minions. -package minion +// Package node provides Registry interface and implementation for storing Nodes. +package node diff --git a/pkg/registry/minion/etcd/etcd.go b/pkg/registry/node/etcd/etcd.go similarity index 90% rename from pkg/registry/minion/etcd/etcd.go rename to pkg/registry/node/etcd/etcd.go index 46f321513f0..ec66b5cf46f 100644 --- a/pkg/registry/minion/etcd/etcd.go +++ b/pkg/registry/node/etcd/etcd.go @@ -24,7 +24,7 @@ import ( "k8s.io/kubernetes/pkg/api/rest" client "k8s.io/kubernetes/pkg/client/unversioned" etcdgeneric "k8s.io/kubernetes/pkg/registry/generic/etcd" - "k8s.io/kubernetes/pkg/registry/minion" + "k8s.io/kubernetes/pkg/registry/node" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/storage" ) @@ -79,17 +79,17 @@ func NewREST(s storage.Interface, useCacher bool, connection client.ConnectionIn ObjectNameFunc: func(obj runtime.Object) (string, error) { return obj.(*api.Node).Name, nil }, - PredicateFunc: minion.MatchNode, + PredicateFunc: node.MatchNode, EndpointName: "node", - CreateStrategy: minion.Strategy, - UpdateStrategy: minion.Strategy, + CreateStrategy: node.Strategy, + UpdateStrategy: node.Strategy, Storage: storageInterface, } statusStore := *store - statusStore.UpdateStrategy = minion.StatusStrategy + statusStore.UpdateStrategy = node.StatusStrategy return &REST{store, connection}, &StatusREST{store: &statusStore} } @@ -97,7 +97,7 @@ func NewREST(s storage.Interface, useCacher bool, connection client.ConnectionIn // Implement Redirector. var _ = rest.Redirector(&REST{}) -// ResourceLocation returns a URL to which one can send traffic for the specified minion. +// ResourceLocation returns a URL to which one can send traffic for the specified node. func (r *REST) ResourceLocation(ctx api.Context, id string) (*url.URL, http.RoundTripper, error) { - return minion.ResourceLocation(r, r.connection, ctx, id) + return node.ResourceLocation(r, r.connection, ctx, id) } diff --git a/pkg/registry/minion/etcd/etcd_test.go b/pkg/registry/node/etcd/etcd_test.go similarity index 100% rename from pkg/registry/minion/etcd/etcd_test.go rename to pkg/registry/node/etcd/etcd_test.go diff --git a/pkg/registry/minion/registry.go b/pkg/registry/node/registry.go similarity index 60% rename from pkg/registry/minion/registry.go rename to pkg/registry/node/registry.go index d67f983e286..e587b6c0f14 100644 --- a/pkg/registry/minion/registry.go +++ b/pkg/registry/node/registry.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package minion +package node import ( "k8s.io/kubernetes/pkg/api" @@ -26,12 +26,12 @@ import ( // Registry is an interface for things that know how to store node. type Registry interface { - ListMinions(ctx api.Context, label labels.Selector, field fields.Selector) (*api.NodeList, error) - CreateMinion(ctx api.Context, minion *api.Node) error - UpdateMinion(ctx api.Context, minion *api.Node) error - GetMinion(ctx api.Context, minionID string) (*api.Node, error) - DeleteMinion(ctx api.Context, minionID string) error - WatchMinions(ctx api.Context, label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) + ListNodes(ctx api.Context, label labels.Selector, field fields.Selector) (*api.NodeList, error) + CreateNode(ctx api.Context, node *api.Node) error + UpdateNode(ctx api.Context, node *api.Node) error + GetNode(ctx api.Context, nodeID string) (*api.Node, error) + DeleteNode(ctx api.Context, nodeID string) error + WatchNodes(ctx api.Context, label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) } // storage puts strong typing around storage calls @@ -45,7 +45,7 @@ func NewRegistry(s rest.StandardStorage) Registry { return &storage{s} } -func (s *storage) ListMinions(ctx api.Context, label labels.Selector, field fields.Selector) (*api.NodeList, error) { +func (s *storage) ListNodes(ctx api.Context, label labels.Selector, field fields.Selector) (*api.NodeList, error) { obj, err := s.List(ctx, label, field) if err != nil { return nil, err @@ -54,21 +54,21 @@ func (s *storage) ListMinions(ctx api.Context, label labels.Selector, field fiel return obj.(*api.NodeList), nil } -func (s *storage) CreateMinion(ctx api.Context, node *api.Node) error { +func (s *storage) CreateNode(ctx api.Context, node *api.Node) error { _, err := s.Create(ctx, node) return err } -func (s *storage) UpdateMinion(ctx api.Context, node *api.Node) error { +func (s *storage) UpdateNode(ctx api.Context, node *api.Node) error { _, _, err := s.Update(ctx, node) return err } -func (s *storage) WatchMinions(ctx api.Context, label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { +func (s *storage) WatchNodes(ctx api.Context, label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { return s.Watch(ctx, label, field, resourceVersion) } -func (s *storage) GetMinion(ctx api.Context, name string) (*api.Node, error) { +func (s *storage) GetNode(ctx api.Context, name string) (*api.Node, error) { obj, err := s.Get(ctx, name) if err != nil { return nil, err @@ -76,7 +76,7 @@ func (s *storage) GetMinion(ctx api.Context, name string) (*api.Node, error) { return obj.(*api.Node), nil } -func (s *storage) DeleteMinion(ctx api.Context, name string) error { +func (s *storage) DeleteNode(ctx api.Context, name string) error { _, err := s.Delete(ctx, name, nil) return err } diff --git a/pkg/registry/minion/strategy.go b/pkg/registry/node/strategy.go similarity index 99% rename from pkg/registry/minion/strategy.go rename to pkg/registry/node/strategy.go index 0c775ee66ee..939fc106dca 100644 --- a/pkg/registry/minion/strategy.go +++ b/pkg/registry/node/strategy.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package minion +package node import ( "fmt" diff --git a/pkg/registry/minion/strategy_test.go b/pkg/registry/node/strategy_test.go similarity index 98% rename from pkg/registry/minion/strategy_test.go rename to pkg/registry/node/strategy_test.go index 7708a5abadd..4b0f998ea51 100644 --- a/pkg/registry/minion/strategy_test.go +++ b/pkg/registry/node/strategy_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package minion +package node import ( "testing" diff --git a/pkg/registry/registrytest/minion.go b/pkg/registry/registrytest/minion.go deleted file mode 100644 index 7c0ba439e81..00000000000 --- a/pkg/registry/registrytest/minion.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package registrytest - -import ( - "sync" - - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/errors" - "k8s.io/kubernetes/pkg/fields" - "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/watch" -) - -// MinionRegistry implements minion.Registry interface. -type MinionRegistry struct { - Err error - Minion string - Minions api.NodeList - - sync.Mutex -} - -// MakeMinionList constructs api.MinionList from list of minion names and a NodeResource. -func MakeMinionList(minions []string, nodeResources api.NodeResources) *api.NodeList { - list := api.NodeList{ - Items: make([]api.Node, len(minions)), - } - for i := range minions { - list.Items[i].Name = minions[i] - list.Items[i].Status.Capacity = nodeResources.Capacity - } - return &list -} - -func NewMinionRegistry(minions []string, nodeResources api.NodeResources) *MinionRegistry { - return &MinionRegistry{ - Minions: *MakeMinionList(minions, nodeResources), - } -} - -func (r *MinionRegistry) SetError(err error) { - r.Lock() - defer r.Unlock() - r.Err = err -} - -func (r *MinionRegistry) ListMinions(ctx api.Context, label labels.Selector, field fields.Selector) (*api.NodeList, error) { - r.Lock() - defer r.Unlock() - return &r.Minions, r.Err -} - -func (r *MinionRegistry) CreateMinion(ctx api.Context, minion *api.Node) error { - r.Lock() - defer r.Unlock() - r.Minion = minion.Name - r.Minions.Items = append(r.Minions.Items, *minion) - return r.Err -} - -func (r *MinionRegistry) UpdateMinion(ctx api.Context, minion *api.Node) error { - r.Lock() - defer r.Unlock() - for i, node := range r.Minions.Items { - if node.Name == minion.Name { - r.Minions.Items[i] = *minion - return r.Err - } - } - return r.Err -} - -func (r *MinionRegistry) GetMinion(ctx api.Context, minionID string) (*api.Node, error) { - r.Lock() - defer r.Unlock() - if r.Err != nil { - return nil, r.Err - } - for _, node := range r.Minions.Items { - if node.Name == minionID { - return &node, nil - } - } - return nil, errors.NewNotFound("node", minionID) -} - -func (r *MinionRegistry) DeleteMinion(ctx api.Context, minionID string) error { - r.Lock() - defer r.Unlock() - var newList []api.Node - for _, node := range r.Minions.Items { - if node.Name != minionID { - newList = append(newList, api.Node{ObjectMeta: api.ObjectMeta{Name: node.Name}}) - } - } - r.Minions.Items = newList - return r.Err -} - -func (r *MinionRegistry) WatchMinions(ctx api.Context, label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { - return nil, r.Err -} diff --git a/pkg/registry/registrytest/node.go b/pkg/registry/registrytest/node.go new file mode 100644 index 00000000000..c2d2476f568 --- /dev/null +++ b/pkg/registry/registrytest/node.go @@ -0,0 +1,117 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registrytest + +import ( + "sync" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/errors" + "k8s.io/kubernetes/pkg/fields" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/watch" +) + +// NodeRegistry implements node.Registry interface. +type NodeRegistry struct { + Err error + Node string + Nodes api.NodeList + + sync.Mutex +} + +// MakeNodeList constructs api.NodeList from list of node names and a NodeResource. +func MakeNodeList(nodes []string, nodeResources api.NodeResources) *api.NodeList { + list := api.NodeList{ + Items: make([]api.Node, len(nodes)), + } + for i := range nodes { + list.Items[i].Name = nodes[i] + list.Items[i].Status.Capacity = nodeResources.Capacity + } + return &list +} + +func NewNodeRegistry(nodes []string, nodeResources api.NodeResources) *NodeRegistry { + return &NodeRegistry{ + Nodes: *MakeNodeList(nodes, nodeResources), + } +} + +func (r *NodeRegistry) SetError(err error) { + r.Lock() + defer r.Unlock() + r.Err = err +} + +func (r *NodeRegistry) ListNodes(ctx api.Context, label labels.Selector, field fields.Selector) (*api.NodeList, error) { + r.Lock() + defer r.Unlock() + return &r.Nodes, r.Err +} + +func (r *NodeRegistry) CreateNode(ctx api.Context, node *api.Node) error { + r.Lock() + defer r.Unlock() + r.Node = node.Name + r.Nodes.Items = append(r.Nodes.Items, *node) + return r.Err +} + +func (r *NodeRegistry) UpdateNode(ctx api.Context, node *api.Node) error { + r.Lock() + defer r.Unlock() + for i, item := range r.Nodes.Items { + if item.Name == node.Name { + r.Nodes.Items[i] = *node + return r.Err + } + } + return r.Err +} + +func (r *NodeRegistry) GetNode(ctx api.Context, nodeID string) (*api.Node, error) { + r.Lock() + defer r.Unlock() + if r.Err != nil { + return nil, r.Err + } + for _, node := range r.Nodes.Items { + if node.Name == nodeID { + return &node, nil + } + } + return nil, errors.NewNotFound("node", nodeID) +} + +func (r *NodeRegistry) DeleteNode(ctx api.Context, nodeID string) error { + r.Lock() + defer r.Unlock() + var newList []api.Node + for _, node := range r.Nodes.Items { + if node.Name != nodeID { + newList = append(newList, api.Node{ObjectMeta: api.ObjectMeta{Name: node.Name}}) + } + } + r.Nodes.Items = newList + return r.Err +} + +func (r *NodeRegistry) WatchNodes(ctx api.Context, label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { + return nil, r.Err +} From 561cbcd7c9ce9fe4d0571287f3490b71546a420e Mon Sep 17 00:00:00 2001 From: Paul Morie Date: Wed, 9 Sep 2015 11:48:11 -0400 Subject: [PATCH 19/46] Fix typo in downward api volume e2e --- test/e2e/downwardapi_volume.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/downwardapi_volume.go b/test/e2e/downwardapi_volume.go index 7bb05b75016..5bc26907eee 100644 --- a/test/e2e/downwardapi_volume.go +++ b/test/e2e/downwardapi_volume.go @@ -25,7 +25,7 @@ import ( . "github.com/onsi/ginkgo" ) -var _ = Describe("Downwar dAPI volume", func() { +var _ = Describe("Downward API volume", func() { f := NewFramework("downward-api") It("should provide labels and annotations files", func() { From 6d6f338fac7f8b7a910ae0f2b92d46c0ee774c86 Mon Sep 17 00:00:00 2001 From: Avesh Agarwal Date: Tue, 1 Sep 2015 12:19:17 -0400 Subject: [PATCH 20/46] Adds servicesaccounts to kubectl get/describe cli help and docs. --- docs/man/man1/kubectl-describe.1 | 2 +- docs/man/man1/kubectl-get.1 | 2 +- docs/user-guide/kubectl/kubectl_describe.md | 2 +- docs/user-guide/kubectl/kubectl_get.md | 2 +- pkg/kubectl/cmd/describe.go | 2 +- pkg/kubectl/cmd/get.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/man/man1/kubectl-describe.1 b/docs/man/man1/kubectl-describe.1 index 9633eb1da5f..6dfa016aafc 100644 --- a/docs/man/man1/kubectl-describe.1 +++ b/docs/man/man1/kubectl-describe.1 @@ -30,7 +30,7 @@ exists, it will output details for every resource that has a name prefixed with Possible resource types include (case insensitive): pods (po), services (svc), replicationcontrollers (rc), nodes (no), events (ev), limitranges (limits), persistentvolumes (pv), persistentvolumeclaims (pvc), resourcequotas (quota), -namespaces (ns) or secrets. +namespaces (ns), serviceaccounts or secrets. .SH OPTIONS diff --git a/docs/man/man1/kubectl-get.1 b/docs/man/man1/kubectl-get.1 index 63ecbb104d6..2a8965284ec 100644 --- a/docs/man/man1/kubectl-get.1 +++ b/docs/man/man1/kubectl-get.1 @@ -19,7 +19,7 @@ Display one or many resources. Possible resource types include (case insensitive): pods (po), services (svc), replicationcontrollers (rc), nodes (no), events (ev), componentstatuses (cs), limitranges (limits), persistentvolumes (pv), persistentvolumeclaims (pvc), -resourcequotas (quota), namespaces (ns), endpoints (ep) or secrets. +resourcequotas (quota), namespaces (ns), endpoints (ep), serviceaccounts or secrets. .PP By specifying the output as 'template' and providing a Go template as the value diff --git a/docs/user-guide/kubectl/kubectl_describe.md b/docs/user-guide/kubectl/kubectl_describe.md index 5fa18144d32..7eac8d62820 100644 --- a/docs/user-guide/kubectl/kubectl_describe.md +++ b/docs/user-guide/kubectl/kubectl_describe.md @@ -51,7 +51,7 @@ exists, it will output details for every resource that has a name prefixed with Possible resource types include (case insensitive): pods (po), services (svc), replicationcontrollers (rc), nodes (no), events (ev), limitranges (limits), persistentvolumes (pv), persistentvolumeclaims (pvc), resourcequotas (quota), -namespaces (ns) or secrets. +namespaces (ns), serviceaccounts or secrets. ``` kubectl describe (-f FILENAME | TYPE [NAME_PREFIX | -l label] | TYPE/NAME) diff --git a/docs/user-guide/kubectl/kubectl_get.md b/docs/user-guide/kubectl/kubectl_get.md index 659f6ce7b78..ca42aba4b53 100644 --- a/docs/user-guide/kubectl/kubectl_get.md +++ b/docs/user-guide/kubectl/kubectl_get.md @@ -43,7 +43,7 @@ Display one or many resources. Possible resource types include (case insensitive): pods (po), services (svc), replicationcontrollers (rc), nodes (no), events (ev), componentstatuses (cs), limitranges (limits), persistentvolumes (pv), persistentvolumeclaims (pvc), -resourcequotas (quota), namespaces (ns), endpoints (ep) or secrets. +resourcequotas (quota), namespaces (ns), endpoints (ep), serviceaccounts or secrets. By specifying the output as 'template' and providing a Go template as the value of the --template flag, you can filter the attributes of the fetched resource(s). diff --git a/pkg/kubectl/cmd/describe.go b/pkg/kubectl/cmd/describe.go index c5b6fc4668a..4a026f883e5 100644 --- a/pkg/kubectl/cmd/describe.go +++ b/pkg/kubectl/cmd/describe.go @@ -52,7 +52,7 @@ exists, it will output details for every resource that has a name prefixed with Possible resource types include (case insensitive): pods (po), services (svc), replicationcontrollers (rc), nodes (no), events (ev), limitranges (limits), persistentvolumes (pv), persistentvolumeclaims (pvc), resourcequotas (quota), -namespaces (ns) or secrets.` +namespaces (ns), serviceaccounts or secrets.` describe_example = `# Describe a node $ kubectl describe nodes kubernetes-minion-emt8.c.myproject.internal diff --git a/pkg/kubectl/cmd/get.go b/pkg/kubectl/cmd/get.go index 1c4d0654cc7..fea289d95dd 100644 --- a/pkg/kubectl/cmd/get.go +++ b/pkg/kubectl/cmd/get.go @@ -39,7 +39,7 @@ const ( Possible resource types include (case insensitive): pods (po), services (svc), replicationcontrollers (rc), nodes (no), events (ev), componentstatuses (cs), limitranges (limits), persistentvolumes (pv), persistentvolumeclaims (pvc), -resourcequotas (quota), namespaces (ns), endpoints (ep) or secrets. +resourcequotas (quota), namespaces (ns), endpoints (ep), serviceaccounts or secrets. By specifying the output as 'template' and providing a Go template as the value of the --template flag, you can filter the attributes of the fetched resource(s).` From 0b08fc8595d4a6e779d30140320ac53e75b370e0 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 10 Sep 2015 00:22:43 +0800 Subject: [PATCH 21/46] fix a typo in development.md and update git_workflow.png --- docs/devel/development.md | 2 +- docs/devel/git_workflow.png | Bin 90004 -> 114745 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/devel/development.md b/docs/devel/development.md index fc14333b83b..75cb2365e85 100644 --- a/docs/devel/development.md +++ b/docs/devel/development.md @@ -96,7 +96,7 @@ git push -f origin myfeature ### Creating a pull request -1. Visit http://github.com/$YOUR_GITHUB_USERNAME/kubernetes +1. Visit https://github.com/$YOUR_GITHUB_USERNAME/kubernetes 2. Click the "Compare and pull request" button next to your "myfeature" branch. 3. Check out the pull request [process](pull-requests.md) for more details diff --git a/docs/devel/git_workflow.png b/docs/devel/git_workflow.png index e3bd70da02c847da17fa1d645e81d22be9fce0bd..80a66248fb87d62344eae6b673df4246369974af 100644 GIT binary patch literal 114745 zcmb5W1yo$wwk`|=2o{_Q9xS*9cZWc5C%C)226qka5Ikt%?iSqL-QD#SN%y&Z-hcaz z@iR71d$X6#HGO`IU>PaVcW~HnU|?YH#6Anjfq_BJfPq06!a#$bd@C^=2K|CG;+NzH z0|Q3FKk7h%?g{ih%SnQPxsrl`dHI2X-Gd%^?Sg?h(1U^PYlDGtB!Ypxw@z)4%*&->=x4aTBRX$`A-z*%}Zq(=yT06Y;Kk&%35opG9CXJ`WNdG5%|S=! zfjWln zp-}gZ*&$?%o*D~J?Ac4M?*~f=DZxis_6_G4i0Yc2Ne|+;RcPM zoBn6jJpTA8C2ri0fMq+;LDO_IdPd_!z1$%3su}o7pUxG}c zIDX&MuvIG}iT;vt(=_EW-y8dib*|c1Wf7g#^9$uujs&yX>savM4W0MUM#jY$AEE(K^VA3LL0aK5OKd{IrO`Q3aI220LR z?W(+LEd7)DTtSbZdGb;fKBH1xI8NuKqFpWl<=d)? z6|Y4W8yBbAjYYg^nH5jS#8f4QAHTNpe6*6RRH|+Qc-qU)VEN@dFp7^Q7jF_}@MlSt zvf+U=zbE)N0rE!F(!bzMzEVRib^d2ywm#dro8B)4o;Ux0=X%Ghi=Bh`d*U=wAd1U= znoLi?MpARFQVMVfe6mmjk;&E%fkJ)=z|0NWI?t2M`oR??tQvEB0B8dyoo6e$U-z_H z80nRe`jIE$-1@WYIX!|y0MTF8n>RhS~zmLf7MERxc1oyF8oMU`4hJZu~s$%(Vn|DEq&<)O`|O`5tC1pqL_>Nfiq z5qcFZ9LQpk(z(x_SicNR^R_kEqQT(RLFO87IT@zHkU3jO5=ZIEqrvPE%D_80`N4ptGf5o9X7f#rOrwzf2Oe zIIncCp=h%l!GBe)B-U;a_N7bdDDwZNpY3HK;NR2waZN&`nVITiO8;wDwS?N$+w(sa zi^?sRIG9*imPx6sV*ZW5d^4n{XkRWmJ36SdohARZPZDikuc$zWOb-|}V)T~>8gudtfQ7~F*Bz<|f=dbmsO8_10>-{&Y zw6S;(snulFTvKW$W4zT`2{b3OZC9DAVRQK@0X#~v8m0u4TdesYeW#3R(;zyPcXHyo zl;O?2EFG=Mv^%2p7&}xfF20QL&zvNJ#O}|+K;kZW(%1c+B_Y5;L_yg=MtMf}yspfclj!{X_ehkG z5@}}RhRP1*K#MWO7GPBjWNN6_%+rnDc#aXhxpC%1Ee*n#0t|HYSwG_~^`hcpAw@+D z0YO0vL+#C_j|8GWx3<3d2nN~}quA&pqBw8<;zWBzi1=aVU+rFev0 z`)sVQn*+4B)Hf`%SlHQxbBdc@${F7%muWEW6XTh-VsfFjI=hp&UX2YIu`F-?R-UuL zZT^)8buwodl*6g?G2}DGZYI;AyHiE^s$6t`2~#E&a`~2ejANsZZ%Nf<&UnqOvY_vo zEYLzns2vSN)3+$bw=>_e?;Gvdo@N9dt6<{#k2NNEb&Wq&sg}u>x*aAnoF%LGC2x}%i^9Y6^0G6>y#dG6yksmENRJ-JdKJU) z@lxw)Zi2)9LJ}Y(c{*^YhipRmgpQztn~&P}8UEM6lEl@#^Crz31(SQczOD z?C8V0xe=Jb%Y__?h#PkC18skM=%wf7apWa4XA=_oEmr_O_wn+dzt0PkP}Hh84>+%te^$D>*n_t^dCU0~FJ&t(E^h67 zYhZUGKXT~dW;V|h!ZRLN5f29yjvZ+;(q&svBhQ6L$qzw7kRL8_e^eAjw;fP^ zu(T9qR`<8}%BTGCRQ|?|f!h$^dgac?PcXxz)=H>oj(xMFAxy-1WFVSI=*P`1X`+T% zdMEa()0|?w)CDzbXf}<%3Pza?uN< zddN)#f?-X*I;`7iPEyu2C*h{TY;^ce>sxRG5s-kn{GYJRi1 z8W+cAu+-;t(rC~od;*`VakjjN$o3r?WhZoEX<6XE@88S9T#*-)ie@M#8L&NKfcG=(a_SC3R|UCVvi?X-fN$Hi0dh= z5K8mB^mdmg>yCG&7Crz5I2*H4AvG_ecN~bmi{>mi#+Z}2fgXEZRClxLE)l-0AtD#U@xxGs!aR(YT zg5Y#u3@uDok`o~;TFUku?F+Y3$ya}egXt_FaO@mVh9|LFInqiW-xTHC+U1+^&aiI3|4Y{%bj_ctjxFXe~iiw?iS_=7ku%F7oP)3E1NSBeLEtGm&V8z=gHsRm@k z#~b3Apz1NxFi}$@l(jxLh@qxCLW0-Rj#0!AMjWc{!|YT0g7`reQbKgOf6Ocb#=e-{ zhtlAWXI7Qq)wrEs(r;>Aezkw+^AjCdcC(!PuZA52U7zIEl^MV>GjCiq|5;ZF9GpSpSLq^I4S`(zcF)+y z|2b}FnB4dKrt6h2^ixNnM15~z)m}%xyVKc^F;^uo5%gDmXsrX0*J_}+!u--xUoNJQ z_LsIe6mMJAuDn4Xt0cv{d3p;yuV@7KN(%2r{x*ALx%%C@lU?gZ)pU@X+TSmZ4gfe- zrE9b_g@o9NK#x-m{GqZ`X3Dl^Htb5O1%8@Y*dL4gTW)dJv||qHew}K!X2j6H>I-`~ zh$Ze5)_GaCtDk30*;4E4>jX&EAX`4r*T?7VT=&(r2c&=0L;H|7S>}Gn{-anhiL~kN z)ioPbZNqI*EWi7OwSNb3ZD0OtPwvkr%87I$mx+{^ zJ5@@SQs{Nhj!3Mp=BpjI?Y+$%zO`2Xd0;Rzf^19RcfVkWWp=`-^Zy#2OXRvHgoF%! zL9s!I6cb7ke333Ca7%6}p5X};4Be-Xpe`svF31J|9#s??(=1}Pnj8zT4kslzFdnmQqa{OZHlr;SDlvBSfF+7!HXFOy+s%?pE)a$%z zhu+MMwxzVuKp~%h0cVN1=gTYOs34cu`VJMDbx3er=pAb&jX(`n<1-S_^zWVHI7^4f%i+& zHk;BJx-Z*UyDOSWgQ5@`eqPOZ<{FSVL!cpc1t1X)#2s^f85kHu1%A!6lMF}L_$%__ z^Y^-5Kmyv{tE{OsmVWmcBN?57UF5T9j%d%WL`8nOB|dezML}?#tt|4*dLkL6gs?BK zhC@Sp)c|~fC@OW{MSGW&hVASX$UgxKJt#unqS-cLbNXk zUnAiy4VgCp#`p?8-Gqw`sR~M>Nf_cMbp5F_=h~qfqTXEqwxjwj?bvOG9W@YkXr&oUfJ&_B*dFH zgul5xrqLgy)cz;86%W~veG+6$^7u^hTtSRN$XySg@{v%}$pIpL_XS?-_L3QEY)(Tn zbmM(wTXsrBKr7Fv`f|!6h$|F_KyrM5Uz!JJMU1f@j=sMAF0eFeWhNeNLzn> zUVPLU$!IRD@kRM615y~EH`Vr-mVY^ zud7@nUE<@J!Rb8Pd<3nwete>#xpcn5SwdzWKpf@qi=%m{#@{${kfBO6gk!%s&I0}* ze?ksn0YEdhcX+d_esCzAc+H;ZbqlV+2_Y?=5_))!gGd{?ek8^B@X_{*CHOHR%!e+N z&yG8PETy(@oBvZ8Y;R)I*jxE;FND*)IDVS0R@HeY=>Bg@!<;>E<_ zT1Rlf0$q5>*&1W;h>I27g^s(7F+AhM=xA_I{sEMx=%Y`E9E9QW(plR;!R3IJzD;at zdI^dV9mvvB>Y;v1mzv>Km21oe^WB$zf*|sd*TB%dZYuE)@5TU@RU%-A%=@}CRU+Hvc?QF8Twtu`C%VdCd3R{8R@Z}8Rq}BGP*ZOoKU-1p zsZWsRvFp6&e=-GMm)DtL?KL2h2@|7hS`X$|C}^zs6fJr7{VjYmaKIy_Oo+pYUd=Wc zvCIvUd)|j&7Gw9Qdu$W#jIO%pHVPosN$$vtcf%f1QvMOXA3wKRP088-hVxnsTsM%^ zbB{r9yKK;)DmBeXtc?QJ-F_3Sg_74`(qcQ5arNLK>MWUzRzRDNI^H7E0mY!u@BpUV zVvIwoH05V*nn121$Rd`P)6Z1LC%;Bcd_}MdS+|v@(XQk~ZyY8fzox~({?VTEqM6S` z`?0AvnfbMI*6-k5(s#{vuL>GGgt-IPqah|{UPBLyY$URI;;Cu5kRIc3he#2S3m3&uT zPcZ8CT1UaGEZe%l63yoqR)Yx@+}ome?+`-W5PLP*XN3IrX*wNnV8bMZa#8ip@{gHp zr+c*TV{E}aRD0eea&hA~hW;$ZdB>cTSSTH%qU+W!SC9Nt>IJ1wcDfWuF0DD8Cam*Q zEQ$XOgWWOeW~{1Zi#BnY>7+sGFDW72!Hq=ZOg-kW{TTAj=lvzDZll2_d50e~z7}^z;fMPJ<3P`N=RDW6bnFk_ zQ*K|(`<40|6#O1T*UkA50&E)DzC1ROzB(vU{B1^Z9M@#w0gGx`dzWUnD}gNs=#FDS zrJ`OD%toHu39=1N_<*pRZp^JGjhpla3gxDJ)6ghPl9dDxU}4i;+On&lZ%qvib|&{X zP!t=W{hdfF7A5oB2i_I+2=C;YT`Z3WVhH~Re8f(gr+nubsVS?Fyy^I83cRrzWQ&}R z+6@NLQ^u;Ho-jObzv?75J$dD0WiLED5%o`dl-b{yil!2-BshRnh?oKRwhztPk<#L$V>^h(sSQoze$Qt zNMreVp}&|t>A8&iXs^{QIdOm|r`0;D)M2cMTHB-fVVIBE3W?w+V2&u^n z{~?B`QV&GVk%UOp5I+L%4|)^GaXX2OIhx4~UY}<83mM=xB!CdbP6?O6F2{rpKuqU? zJFLzYGPp%ta@gcbByZ#6Dk_rcvW04g;UfIovDP+%uU}Hg zC4;J0!KGEF-l;6c6QwkNT<7UXCk{keQcGxi*y@~h+Wc}jhi|o2Y;ymVV;Wy%w&+AJ zf&3yS<~`78h69r7fNe!UxZfT9XE={M#Y@c?Yly$z5&rGVwZ|bv>CBwVEm$66v$B4( zf0-QVhGd+qO^rc@@HDlr-jR+F34V80jBEWj9xrF%V`lWSApU6=U%vrzb@VK!VlfRz zxCaN92#l9^6lDPvW@*eG&8iB;YTYo*f;=%JR2|?HxMuiSVk!^$cDE|!oR`m}#Es(^ zgh3b))je62(U;FyOHaH(I>i~dCzlmV z71zQ*(H%;r-jEal9+61X4sGZ-=k&2?`7D<~#`DtcXhsdy$&YIP#u8jn)1Otj7P2&W zX6ZV*PFvZt%GC3KZllu&X8B8tC|i9pgJjO-NjQryP0b4c-8`uu?Z9a_uI(cZUumI6 ziJGCXllynw&SsWG!Y`lF1^w}o zxP$2p(P{rMmjnj!ahH(NX3nvdL$_K+Mvz6wgoM$T@yY1Z@iM3Ev*5O{qX(2MHuwRc zTW#s5iTID9vbGI)#gCdas0|W;>w}Vg9_02)e_z`TXN~N=1ED6D$!U7a=;txX*?Mm9 zEZ_S!!SV~ZFwt=woqnbdtE9>IG;I-Sg+VZvQ%fJ1utHg=y)3mmOwd&;N;th!QO7!W zv3!D_O|wZaUC%*@g4f(p1SEcYosmNtwj|ki;MEs3hn08exgR&oauvQw;28vV&p$jv zAMD&yp{7U$?PYy9n6WRPdJS5B|t!my$|slU-KPpwCJ4`KDx2%I4R z>a?e;OH4X-EY#`~4sj!h1@d%IMH8X?hV?1#k zWIhtSLx1K8<#hG-;%x2X!e+D9pkT6b0_5!_@=*oPY2;!^F51;mTR_Tb;w_3}F~CCj z$)ojDzgUImGuh#zD4P+x2FPz+$mCQtS69dV6LDc-VZk6FDSsJpMzvOj_zEOd2A(6n z(61-VVv{bu&H946wgUtIFw_ap5`bjphepn~2UYK1v{F=}JT2ufBz)EHey|sX%$XsT z^r6^J!lN0ZSCJC$LQdF9LMeScvCHeD^RED!U3{N}cru;+qLgy-ipaiYFFQBhamgwB z3?E@id6{OSm@}XC!^V%(QsfsqZ(Bt8I3r4G4W-lb2}Vw?<0I9+D6K-u&c)T^3-0zpn2w;$WDsm6uAi|P{z zyu+I&Jwu8>7;{?Gs&xcQ$}yWMX{|P7-%>hh0*r6fOUVY_%9oEopkz{JV~si zJ>jEv=EHl;eNj6N#%fHx1B^8^gD&>dMJ${%`T&gTAVg<1Tlq*XBP80WDmPHFF1en9LToj1_W+oBTaTu+z3W>5v?|BW=`2QgP`YSm#S4W?j%#G4~Vc#NyT)TfgJ@p&V1mf zLcc{Z61O+At%s6QIx3DkS@bN4nw};>N)9{!G*onkT*(#-LsJ!A;@rzR!sU|btf60zcWzKs@f5FyIcB`m(%QN zc#D0Mt*AFqG}ZpVFmZ@T1e|i1Yoc*oop^eBT9i3?2^5MUh`w(D`EuJkJ2UHCLsq&yYDmuRkoDXsgI6MORn5r50j|DSsNUT z-9h#9(_x8?-XAUn2b%A1Owt6!51YQKmdUTSw50d+_kI27ml^C-}1)-KD82dfA^~#2j_>GR=n-$;107G;bYNwhZ%LU{UCa z>9TM|=c73+rmgja$s8kbvnp;jtPjwosvx=BC&{fw_2WWa8;XGYir@VsHlto(bOGq`m!{I8NQJ z!E@q-@<$NPBn6V=9rzHr`OAXEG~OteU(_DOx^cSE3D z%p8<1gY#}{<4YlVg-J+sgR(x5kX-lsu0Lhn05g-GL)E<4qH64czi8M-g(6kI#A(Y9 zMc(5R;=~7~XB3HFHlOa#CHL3Yzun#UKVWfMmhB4O%brTOyxGcocVF7@>C}lm^b} z$oU5cact4axzRZ(S!Lb{i}h7J3P4(Qj6}%)AR4?m3Hf^G(?%QNUAEjxEqADamKD$3 z)A86nV@fraj^?T2A8KTSMM9ZF!dWlLuFch$q_vvEK2T57G14fj5zhmlo5PO$7D^Op z3>SB2K**k|G!uR&>*1Piyf!;v@qa2*C7GFEAV5?AC|euj6^d%z>#K)9Bgt8BBj9n9 znvBM_hrlS*mFl@ zTAnIRdmQ}8W+(?fSGtb@x8hGc1mCfho>dVSDRYrTIlnZk4hw2}yQE_%znEm07Wc z3?+2qZGa!+0l(~;`$y{;} zCEemIwOJPg`$bBA_%Mms(6#bV#vCtzWh#_~P_^UHY#RP}O|l`(`tl-r?S016zPa%G z0P3s(R5J~sU&?l9XXz~vxVU&f8dL?SVw*frfs&4;xI+h65u?*=7*=%A6scZ(8cPo7 z7~j?*lVzZOm(9FO#aF#Gk#tu_K3G)V*%w!1CB^7l_Y%wNjb5}Z6f}R2unBEk<+%1p$k zi*;_YQFdt6bHCW0&pi*8q(@GA-72Kj%>`H_%muc+d0;$_b#kok0Fu3E(V777Zg1%P zEdHAVaw$n6e_M8#F+JSHq~RKWaQB2;g%m_n`sM!m8q+DtJ`m~yRJuM!No>?7hx;y_ z#k1owIA$(7{-|>MWk+tvWYZ$NcmK^jiUQHkEq~f~_FQXO(kc(C9B>2cYh(ez4~XX$ zOFv$q8R9dNyX~Qlx6eH>GXcezH<;3htw1# zjhDiZC-9#sYjOWDp3p-qij~F+L)VOAa6E__QEMTlCOb#Yf`yq7W-vB{@qMOFp@~VG zZG1$T*hmSz!D2S#wvRBxSB`3$Vc;uPsJZa5Ow%V`A){~FL;(NeUQ-=WVR0D1be`VW zfb3}kd_o9CPHx9q4e3@;X{{+ADnPX*HZjGnSJ^h@ka7K;D5HRfay}=m9|T8gTloC} zo8#jIB3%O1UzjH^3h(^R3_)$aC=-Ry9nii8_3l}jLesP_j!JBC*Iri3kW2^PsL#n# zuhfrx=y9Xk04lF|hbfiW9A?(|k1#rS*fH8#E(+|8|C;qHNhEaKYm$~wP>T2^mV z<1NZHIJ`Dft0Ok{q=eoX#_)xW&~`EBiZ9XfJ>{#n`TC|bUVnYape$SN=Y$oHCRV;3 z^u449JX$4@u-dkozBCg5hUpwWRw4b&bSwcYGk%J#{ZV~-tE6y_=geq)iRp_*4bA*W zDdTpPZK;C@u?ZS1CXBQ6Qef;k3<>TnAnxt9ZXTP2?}8WottL5Nu60}0Fej;qeK)Ju2fr?)|c3tS0H zdwVK)c+pqy3^!L64hS?cbfMikMr+myxaA3QW<;9V7FjFVm^qzloS?tZ=xpAWt`#-X z+qj0jT|ucdbg3D^wx(_T%yxdvZ+*E@;?`6H#V{=fb9;N)o%>~lGnhr;PcW2OgjOee z^HPfqj%QXPvegCK?RI?M$;Rq|dmU24$8Vd%HaKBekgC7p`oiIB=)&%0v zer`|wz&_O=$wxV(EuBH_EO04_oad@U&b#gkKV+05IFWM-Z=n}%=c_Mf{dBXx zMRmA;UFWx7I@Cp>y)3|Z<>vKRvr(GCxu1LSBK*^2JZH0ydwlvfayJ7qzIQ>N8PCk3V{0+EF|v#5VMKdY^UN|eE|BqkS+LWF+bRPA1lkVS z22_Sb*K?qKsmbcmShCXzU>;z*-HQiVb}Dh`#*4Ohl9OGqi*DNer72~H00D>f!`Tr341+LZ)`USpXo49-2eFLa`{jySwKUUZLD6PMC8#eJ6lp00Vg?*L)d-7YZA zf^U;LF$}>;RyQGQUNY`P-f=cq=*&_mSjfLAS`x@*wLF{!iuk(-K%@n<-3dt`o_5LW zW$<6SEGTJk-vfrpYXb=U;FlKyx>SB!w%39JD80;rwhPnyH6>0rz!9x38PA6ylG{sc zq8s9#1ytXSh;h5HpP+urFEjc?wtelQsEoe*#s|X zNC>9d%w{<+$Y@^y0mqH(PNh_vyDYViHluHf-%_f5re1;_nTtO1Sz# zTZJk*tUVie*{S{G*~nY^lRNs`tCre&cUEBC^C(5oiZjAnr(vt*=sb3)kw5JI*UwBz z&~RpGwC}$xzTqE`UL7r!3_`?%v%1;Nt5#S4W_=4m?bZs{7+*fI+>K8?^$`RMd(Dm0 zQVa?~&ko=SK~d3Wi8-YX^EVjluZr<+Bm)gvre28wPt%vzOmwLT!7ExR4z#DXwWEK~ zGr;b!rvFbX5^X6^#la50>6M<|!+e=T!^}zT^ou2oV43g!+5PM1QIuCY!}PzxTf87s zBVTc>yu7@xp2>d^=oMD!0Qv0r6LVgqgyC%Py%B-_Ao&Kf^q;pl=)AXSBst2$tHS>8 zcDUl8*#xfVo+=@~GVU1oNM_w!??Q;WgaBxGY=}Pveh@W(7kPiL`=67fL0b)~j=wuC zO@lQql1XmH_^=IFlTc+Jg(Ts@@zEhpZ|}moK;dXx8ita?g|(M_g#jxdSY26WebUj< zAq0W^=a_@XO#dNEza`4}Eof`z!NXPOCHdSa(9CpqeyoPn!yI+XlgIaZ*CMAT25uI< z$zqTmH!0iPZ8lO)^|>kf*4Z-mhO;D%sdxi6CpWiRrC_Q`W=~zM(a|C_2?X|5kN09h zg2%sTGqxE7#SG4j`-plGElp6u?;_|%gvAT!285Kx49F&~IYM?^LrwzdrW4Lo+2=a5 zIR0AkYAmGd^&za2(^4M+$8_pBii2iuRX*_OW3%3n1u`k=w(doduU~V!N@@)(%A1_M z8JSdEe+!N6rRxUFZYS~CqwrB=Em$va$U_kVdV1t@8F&zjDHbV#fL5}v8ClRSt|G#~ zyq%s$adw?o*H2ifd&7>{_wxz0{$nlJ4YD87`l-uXoN!5vmKY!Ts6 zRmAE2_=&Ke5jX4Uj>qalLG1Wf2+*vi4fQ!*H^rFK`-9E7alJZR$DYq?0l3?Xxi%Vj zdP2FeopwwiL-^OuS#v{-w{L^$pa&FE;TnmcLSsbwXtO+$iVuZ13$ECb;HkjnQ%Lxd z-^8Bj``JxCwe;YaA6Z#g3|Nt;9?Sl%E2V`TtW4HJJ0c|@G8~HYHDzvYXKVK1J}$yI z&nmL~#)tiu+EIlhK|b2W4-S_+4;4U$jxhZ7=} zg&R&`6B@L>zCIb$E|Bq&=!-UE0n&qvNv0V}fxwy3RL5pk5W?YDwM2gPKiQgA6qJB? zPOE+-R+ua-C1X`J}&E8=wBULZ6tf^$hG}iSM+FMkdA0e&7~mcvZB($sJL{cCU1x+`WUQ2 zucW+%~aER;4sm?EvC{fcbsb6^&dD^9;dx96R)r|=mP9TpO<_+vRpl?^Kod(CQmDNv)J4+n zE^T?o;O+)Q=(GU6GEdg**D`Y4*{qNPOQ^zdRl<2x+@(Y%Po&E_MLQ*fL&p8U8NttjEe#)=%P_h?}VU$&mCwF^}Y5k#=TG z?bARdb{qK0pq9@RztK4t7Z-xY#s#n04L-7u)>!^u1c4k{*N%&ktIT9kxJPC)YhAqh zk^qb`-OaRnrIXA?|NGGHzJ*WkCVG3b%X^^b8&D6^k3*BnEJVBqjZ}(@v#{fOoi<9? z!EcAgJqncUhE1fk)ef09-D!?c`8SIS4!j@imHV}hZXTl9Jej7ySwqH&Ls9%lDb2cf zj*og_0~n@OugNqOn@gM?4tDp^azS+#*sqeBw(|HKVr-n6PokpZSlIrl3-2rqPA=gr zKj8Q%itUY+^mgWujtQztv%1-SInh27rtP)3 zSI|;&85dAAi~s{IZbHq^UHMM>ek!Q(|81I5Q@lv&Q3g63%zjSojC5aS_C}L0FO21M zPhqQNw;gIrFX@=8xC*I?ce|}i9a5I|0$W#*Mc*Mdx5DbASA^N>mq{=?T~8gNy@3K` zD;yeaU+*GNyS?D@+%*oWMM^)XhIXTU0Nia3=b@;?Sly+UO*|CE4y}7E;qZ@;=`0sj zqFi_BH__)s9_QVvdFZx&f*Yax1gY7hk{-p^ivp^My}Cfhm>>kGMxd89`bS}Hmk3;R z58jSxbi(f82qV2>`vLDCdNwlxPmoDI?xIzTU9O9}cjb?f% zIPQGT?iK10eGTd&80+(eb6C??OhGY=n&mai|)<8hV)A0gt%rnp|L1t zh+};;2x1SP7z}c6(-Hr+ce45Go^|5xETyJOI+Q3InA>#Cd3TA*;3RU+TGGvo!_U!Ae}*Of$@~N>J<^Q3g==031}7J7lXue3lEo_~k5L z969=vU|P)CYRj@4xP9hM>zLO~+}B%v5nWtPrU#rgbT{QCEGfjavSeoK-?S7H7DvK{ zCea-ORaZYnAfJH0^p$+}l;y=gdO&^DE)1LHrpOnvzDD4OtUavB>HM$5d-)$n-?o3@ z>ENF`ORoddd6S%SGam}oCl=9F$)L*;JT~SK7+}nG#V933%sJ!g7z%OrgcDf{DZI@fhCeGw6fSCks_oQh{Lssl~(w(ZV%GPFo( z-9!l%^VyS)NJgU}lV2V{A*A?M19pXj+kB8DaLuA44)%7j5<*}EKkDhW&eRDB zQD)bd67p{NdFsbuO4dFERRFxzc2WJ1=znS3?laSZXNA1%q1#HSv()Rc;rUU>2VlAd zAOHbELrs`HOg zb?r@9OYhIW7HF_sVjSCG+So!~Xv8-OSgoh$LM6_R(EWN&I>2{DVxX!A&KL7quQ_nR1v6*T@!zb5)&n+RrhO3TD#_aQz%#Cl}~an`IS z5$eHUWM{J?Yjs3Kw5&pUXz91R!xm2#rsLhuGDIw_K`db*6Zx08;NNf%x#ig{Jp@lP zP5jo>4{0|);yG1(VnQ5hiTF8k8L~7RuF9VjFsYOQ-JIOQOK2WaaYW7d>uTg2tq1O^ zmtpqdd(l6kFG-)@jXb(|Sl+cT4u^At5kh!lPe zbqz{WpnIH6RqmU5KuFmr9xmO;fR6FlCN>a_Q>i_+aQ|30H6V$wc~&}T&-*B#hs)-zi;B&D<`{{R!w67bCSObWnge?*hD}zrk{ux`rOT1fOt!le^4BrBs{MkMkZrr%Z zp=hgfi1eR{TxLkE)ccOeeFRoXEW^sSsQ4HWt?bb3oz6!aaHRph0$vX++{Ca&dD*>vwT#>(R`mFpSUzob3up#^LGoQzRt~St#tmF-@)=$CTfNNjD=F21(N`))r<-B5p`qBGYhwB+ryGn$K|)h+u&`EeIt3?STIAM2}g(fjVu+ z!WYqMN>F86A7@eKdTT_Id3ut_!yQPEFgDz3#8Q*|peJkDrbTM;&(VRT=jcggrEo?( zoMEZWg?;QFr#u7c4Y^#Sf~8#oxw}=_X*b`ItFFzv(o~`iv%t#n)lHMo^m&{i zMly?SG~$;1DM~tq@%@K^26g;54#YzU@}Go@KP^q<4K||e!O{sL#U>u#3A(AwDJsZ3mEFZ>thAIPUkhee_)So~&h35|GalFYGft{6dT zZ&`P@!BY`V@mCY*_}f3y1jGD%68_Yj{`)vHU8Of#`z+TLxrx@UiU%nKH+M|ZMTevu zmvJ&LdKVJN6OTkIVmK*q>NLN)jwfAIGGuj^uTd%uAc%JK=F+*k28Mct3Z4(Eek9~s zLAo#5+mP`pXoc3dkON|T+!@$O0qiXO(_-b%YBVClilN4>#tax!_bhs!L8mo9r)fBf z^#MWG$WM3UVfpqfb^PS7n{5!*?UZgbs%k%mXM-McsY4vTzx z5Wey{NepyYFU1uBnsz$@RArG?3xUM+WK%!R6=i$OMv z5Lu%DsUs>EHod9BC9nQN{a6;&A{DK8pGzkvVlrkdf2F&@iAIN$?3CyU56C<}OnXT_ z*WfEdDCDE8Lx;Dr3)b(ZR@^#HSjFUmnmxWx?=bQT zkFR-eaNZYv*t(*5F8?IIM^Qn&W$H;ZJ`(y4yELf3wDpd|D1dnt+YD#Wd>p|QHtQa- zI;1+dhU)N3(rnF*nz5AvywNt_lR)7*6v2kW;gdG&wdGW}P~~LrWrG^s6Pq*8z^U}n z;u1mB_?yQ`nu$Pq3?Dq?_{%9Ri}|B8Ns0QB0Oj&OGNhu&yWW|1!KcbX869dsU>{}u z;-e*vNIFI{QBcO(I>pD?0o#m=3NRX}R#RFtt5$71Y zGrbN`)u(C;t8|m%m+Xiy?~a8}&xIf{^Gm)S{2$)lGODVr?H`s+cZYOINFyaJ2q=x9 zbT^ys?v$495)kR`lJ4$q>F#k>CB6$c0 zl{9S_>yJyRMGdN+V;JwP{Uj^v$+_s*Cfy;A1Yd*T#UEe8|52dsy@G7ts8q5jUpup5 zW0r)e>(T6JAx)z#k;=ZELfn2^m2!rXn;PS)Q<5bGBK(;Ci?4@;=se!BpQW%2Uf|LN zUxI(gGq>p>gNnPcB#F|alxtr=*-Rb?`%M$c!|Sa12+18Y@RGv52}ZD4RSlfX{YXxD zZ79ea%U_)IcE;9((O^LQ&9Rd(>bRYP-cVbzpO|foOEa{RUKNZlHSby0Gq8Y!nD&gZ zLUgdYbaqo>WT1zzfOp9XX{<~v6a0;f4sLqB{`R{N^^oFdjf#vw4?ZD*(Lz+AEtyd> zg`;yFdbLDa=jB9~!=0fm?YT@R9Wq7q`5(;szJu_23`^m!gklZa^>{~T5!8CMex5*m z)!jd3`izb8XksMjbj_)r3J>v=kNWjv%j^ZABV1fT`;#`f(F)39z!fS=5iU30()EQe zG^w^k=}Fi=2OE46C*ESxIrq!M)0-KZr=pn3Gb*HAKloM9(L|LIznZUoVMUug`g0JPQb3yu(F7pc1Y6W&t$#Mm*M3&ct6TB-A*3^ zg!g;Qkz(sEJ5xgx>c$cakdL^i6g>s|!}LEUmECF`4{&ylvgMyyVi6 z{m#9f=$L`Kx&fWTie^4q{9D?G$Pi*2PiR^(TQE9rDLjuP?rfY$PhpV*@i5PTdB&=b za{Tjxd|y#m)UFd$>Fs1|^25yuGH0Lmm?FF6S9r;Goc8!$ zypkfG;GW9u;JtTiZ{vaI(`KkUQ)Dw;-d<<6iXQJtq)8||8XU!3vvhA6WzxNTfcz*M zd3awwU8$!>Fw3Sh3%-CKA+E!HqeEm?P8A@{`uN3D9Uck>Mu^y?)uehDNG$)V?O?-S zNlA&hURbhZqhFbr@s@S1lc{i9`DTGTQ(qkoz3X%#A)d70bYlV>V6QC=>CR|NWa}&y z@|;!*rY-HFc=tNDT~(U9o5XK$Z?ic`K`T~4f#DnloCaiBicmNt`~iV`3VQnk!dA8M z7)Tl^l`3%f#AG(Ywha54T*k^d=vMx6Z8@;5+nJi^n)lSnmj zit6DTJPs-xrUt@rQj+p&*hpn0PFyCq+s^FWGg>;_n{L-j!7-P>Zd@OuNTz8@%|-aq zD0FZ1L56y0SpP|2=%kQ6IvyxY{r2b5Dd3G@-fH{*{F+9FP`Jngikjm0j2;a~Gd(9Z z&x_%?BaiBUvFM5Aj0P9KFtjoa8c-cZil^_z3fQ09DP0I4)p?)8H69heblef$c3wyH ze!wD-&H*a>K2Z15aR|^LCfri^-nU+!O@bXQLjE(ySJMxSKBA#r-4XBF7@h{^BRnNA zuPlcQo+h6^BLlTfD`i!BBJe_;@$mFF0_$2fZ59~h8^69;(T?AZdEB@VRSeuyr>4Q| zisiybre6%{Qov@B{S}nUgulz0BFTkj+UyMD-)R3T0Rd;#+E#MmzI)9(y!+0i43^dTJ) z?XSU9_e^`Ae+`a)gqsi2-LDUhd&XH^#5x@U( zs@0P`=hM=&wb&UH47&UJz_h4`Pjj=tk6JY-M~9!T`UzAqg+qa?oD>7uuTocwJ5*7YqZyROZ8{-p(+wy0)E;c? z7nS$R(uCl$z_hxh;K(;wD$4j#e;;rT&eb9B%V1>{neGmwl&4uRmaJzC^~YD&f~ z<+yu5=6zbGVm}hnsO_inp_PeikKc4t^A%K4Q3*~|F*71f0fVQC%vdo0zFEpM|7jFuA{Ebe5Nk?) zt`y&+p1`I{AgI>DJp{ii_U-mlNqLFyuaYt_oc#f6-Bz}eHUq==_YGf0rlxxzB8=$a z<1uSO8vL5@?&AZvhy{i0u`9&A%cuD?I-RseXUA^1PLt7s69(l@oV27PXRktF)EFeX zcDBRUXkQdkQ-y@L>{O#>eNc?>lWp6uQ&^`Wm)k@f-e%!8BFz~}3@9M?3G=TwDTlxY z!@j(cR#Nf)x8l(@bI~emS(a_t=DH^Ajx>d#8B&hGc3!za3vQHQM}D^jg@0y1nvim zwlSk@kJN&h{j(+uh?6gFeP}-1-D!X+AM)k&;%Xo&zG^I76KPwvgHjm~z7ZJaIh*_4yXPAqTE! zq{*I$?^t+JPm+M<%lfs(wq9^nRS-lmL0+8*dYV zMGyseRoYaO@A+a?4&1Nu&u;e!PX+18$MN$PH>%N!r9-jRs}36Gh4}B=_}{|~;jeFH z<1InPR*(!zYB&U#Z^_=6N@@|CXI|`5bZd}>h0MI0N%*YQLFHh(yNlZF-hzsoZ-+GB zxUwMOV}9>8GC=bcVxso=SFHelZM}!V2s+=}p`-K?HE^jG^%Y3U!5KR^fhzRv4=*N!tzaKojWltUj*(Lf{~@6;3OhDuK=b$cu1E=N>L&0^j0)16Tk z|28<()GbwhHWduBvj!s`g%vNRabBUg45_*f=n-s>KTQbQAK2@ehx=8FU})I$fj&`8 z)e(bKBtbBsRWVLlP@m0L-A#92_%4;tDMaA1uO{0{yL|Ac_9<=Dz{p!^b6VRWCUr(` zgVPJdpx{yM1N`qoRjs9)#36YE;VTheWY!rT2tHtR(m8=`47ANnf`Z1R5yt&)T(_0# zVHY#X(bFP)e;zTv?F?nMiT?QYUDs?IoCKFAA2j>b*DFH%>=$J3o{L%|YGyOGT9uDr z=wPXQ2UZVVRupN{F%IWt)RCD5A;mfkHHZ2t({{=^xm%HF4@I^G9u$rZ{M*o7LG+G9 zEFKYf@pn_Pgiz)SZ!{Lhj6Zmiaf+o4z_vtOVJJ`sX{up~NS1xR!3mV?)@ zQe0ad-Ap2-qzAq@PAWf9{-w+bLKs!oif8iHPvW3+x>O&15}$k}A-=zir=Xap8RC|a zYDMutWjC-iqvWT`t zd6C+#xD-IwF_@w~FxSFw;YVqe9H-6z8QT1hhzcf5dC`jzp6VM?EfXqxbHZXZ&fnz| zdGX!TxMeB*b`@rv!@f4UNa1CD*@9a^sIGlnr?zO4%UM*62=J;YD`iyLA`?!KMvOv# zphU}RMR*CYZH3EW|1P7f?*3IqS;{?ggLjW`X0bH^e3IJ1wZdC4>nHaU zri>#Q^e;6qYtf4^F1sGk`T<``jb>>=@DF@ENv%Tg!+MHeP+Ss>k=19;*P)`*9l}Zp z+V3Mr?F|4L|L+SC2z<&8}%c#(w;o zm*)XJgjqeD z+2(7Qaz)s@7mc5lvIbFC6vqrm&(~ft?|R5lvZd3r2pX#Pn@<;|7HL_842G=)Wiy@3W*C_5DdL)YGivL6De+8qt+V9IPCvw-28}WU!34=*tIj-y~WSF^O|1# z7Hwlvlw)f`<4sS~+Xk#xYF%{ic>9dYkHhYW$K47ZrSHXEb0y0NlQKOnhE~2w%^=rA zaHz!@xUw>`8M3EV%4l8_&3Nm-@|d`LBt6GoYSiZu==8JqxC}8l_JykB0C3~*1hJl_JvRUVU7lL_1Vl=W=c?*~5 z%VC}&fmU?qshYp}Z-u2QY0o$Dtong7@7VL-{;DdC(Z<$cCB7MIk$K(dmtQ%6Qnr(| z{Vn1+l_ij5O6&fbYt9MpjAuS2!lV|nIJ@^_at%qrTivQI zt~ubryO&xWHcvs?s6NVT4BVR%)xPmJO~&A^cQ_{<$^@*IcR%#mYVNU84^&#Ia8+?aME6j{xKQTu50SA7oy1)& zILWtcMNS_fZI>DeS-$Y@%qUuo|AaE#a5!Y-mBD!r8hj6+&?aL! zx>*9l0(s$Kq7RWtrqRc#DXqig|~0_sfN+x>PUSv(!O5RV;g{Lbv$csPIi7V}3A z&Dg?l$X zn*VYHjc-_RONrppT&x*Y8ZM&*&%*Bn)+IhvhcHjP9dz;FG(!)cjD96Jc~=JS)r@XB8^{`yxe89D!8IAn}@46vbt>f3Qk#2-VM{5@~`bi ze^V=;MBH(&eyckcMN#qSIJKYZ_oyVFb7wiqdH>?>xOy9pa0}^ZJZ0VoRnr~mwCdx^ zhi$d~rBk{F{-%1GHk$ROa#&pZC1}Il%8{$$%TKx?y8;h`A7C}8wvPE93g`Sk){Dqm ztef6#`)!xJ?t0ZiZE*+lSh_CnT7PSGcbh+@Dgb%G=4i$6+je(EV7K<%U=jAeAxY#VZv=$QBWC>Ct9xeWGI{&e@nEB$8 z{x~xr;wKSXK3ZJZeX4WPQ~H*JNM?Yy;e+STg1w_eT^)u zRvvbnOhHGuWPQrj5$@Dk0<_<`*s=SWf#b6o5zS6q3Z;@UFh>z`@;$ZlM zD{{sl&o48=0|)ixDo~OOl;*uK%e_>bTA}shl{8EnAI8}DJ%TN4n-}9_-kVRw)~T>L z(j-=CDx<&3Bfd9WqhdNYyMs^1S<1VybrZ0;Ee23ejU`RX|THOQrz)14W-2` zoHwkJ*zl7X?~5sNRjvKH^J)3B(oaEiRP)etl@t|IYVfbT5O0rW^EGMwcGymgC1tE( z;=Pr7>-KGqzhImNwh}Q}ygg7b)RIcgl1Mv_Pu*%E67MT!rFWSV-X-s2a6E3b>mEq- z+ExxeuET4VgL$@FB@legTH;S|d^WdxRD-V+_1u9Vkma(lrQSx2PW?hAsK^@}vz$8b zgh0aZD0|%qdS8BgQ~q23z6{|{9s z6D_HOJ-;Mk6~C!mZjX5G?45uklu_)Cak+Ikq3Cy$VP!cEI zBiKK*HC9W)I%Sn7!a#&GOd;jfc!Bm+1j9)tg<8;EU zc&F6YUKhw1X8u|!?Ha>E22ne|j*&mp!hB;y1H}b4Y4Ku#TuR~)EYhf2`gLt1vzqzG zkj^>YDTchUJg;c_T0HD{RmR=(ss2@7b# zaL`D9pM3VD>+-JjB9U%|+S2(}sNiE~!spiLC253KN*Ep2l1qCUhV}?%$Yuft70+6x z%#VYI1_MHB*>Lno>>S6HA&CdtNL=WoE-p#seGfmytX!fd%|oewzJ8FKH<<}#IM>u# zSd!6JA(;{SP9VyE%NQF?EY>y=#alhTJBmutffnh3&4D|}ap{u%Lf?eG{Iar-MSBwR z9kVN=s+P-7IV9y%E|kwLV>>d<9s5zdN-jeUb{7}@Es(;Y`t?WW2&T4`=E;nmLt~c} z69|(VDteMqzH8z7-mBhC6 z(RC_(43Ooj!zV!fs_Tr2iU?zPPo!x8o&U8z07N9`d5P2e5jD;Q@tgr~m`?(!=zaip znr)w%A4grd+O_66NwB$yp?>OlK*`~KZu{DcyMA``ROgrc4w4WsQ{0<&hJ|*ZN$Pge z2kVf^B+WNFlP+>J{4|=Fuk6Fz4&{`PInnAM`3gUKrA|yoi-uv94qT?|sR$y~Ju|_Xu9^XP$nv7ehTC6t=y*+w_Jjg3i{$QA;4TaBg4D$gJOD z0q$%H$slSWr`APNf7`Jp&_04I?drW6zSuUa@6pLWvXpyM9KI?uMx|4w{UP9)bcgT{ z@#RJ1W+9Odt7I-K4BWT!Jq{0B$n&>-0cej*MLqOCzE==6GGkBQT$md!=MUMUA06G* zPLsc25uFy(M2pbuOsLf<()d;*6MK9jtG#VMzD}M?#R*Cb^-(k0wT?(tBIUNxBlqWtkz9i9wWE3?PBkhk zOir}$s_62kk?*_daI!=(h5JJL8@Y7AHxQE$6FX0Pb?52&sja?1Cy3t>){X`RLkS;jkE+ z&0$8@yA@T(vB>aV3HB?;i?wN|T^GFkMG|PC7Tq5jq;!4PeexO_sa<9Gb$-s09ido8 zT_E+_8gr~Vsm{YKF2lWcfz%(#AB%XX3lCPOG!$*h%5(XU4CqJIxoZ2m=%d-M%HB@x z%k`-e-__#zE;(UNPjHOSeE%*%{sl8Rh;dJS4ZlXyBLaMXrg~-8+f2%5Ms8hQ^AL4N zrhLTFq17@_#M35EaC8h`o{Xw{zV6Y=uqL`%&E&R9arCo4t7~~zU8)#0sONcDf}UD9 z(-v|E1Fg?n#UVx6qItK;)jm|s*0jxg6c6j9gbInJBxd_r2}h>^47J>KB_G!ET`g-- z^DrbYx}qA;<5o}CNOiFdGbMY=r*j$fk_`+E5H(`SEVdxNwPqdO5It63HP<$NG~dBa zYA0#Jki5Mp=YAZNx(*B7`uoB+2~G}fZc|s>mpDdtnyYP`25f`$ug$rrmTu|v1`;T! zUQe5`mgf{99ptytM7g9M6xZKFnPgF4s+UP7QMF(m5*N{s!Y#L)xn~Fi!ZgF$P>vSp-PtG;jUtFAjh1*kiVW@K z=v;I2>l$6HxkVO>i<2qe@l!{Si1G}kXzV3xFe5HM_ZN9tFx4EjD zO`tGF6a!A{2{W zCz-Q;}ofA$X#m>eTTYmnS#Am)L-|o)VcHV zBg-%QrD3^a44s-#&2VjuY#_|XDSVG%m$kHEw>fHarq)ntD6j}5gDzCf z+u2pX!76vU)sUyr3Fe%wV~Unh)<@9LG|L|WVApU;+(QD`*>e3kXybv~0%fx4(XgRu zb}D{eeONfjsqTH{RVER9HxAgKa8i#Qxt0BOM(gH4(i}QOz7T{5)T))(fke@yRN6{& zx#k$O@!Pc9Idv~amgCj}YR%Rr%*q1fO6m`V`7r*C38MkKI9Qr zRX@D4khID^SC7ESsoB)#y+$;2+#fXzU(N`?S=BWY98!rcyeD!``nZggeC~qO+^kM= zwi{Mdgy()PR;+E~*G(V>NgrbO*&g;g@V+#NB z`q5YnkORNl4qm@TX#GJUDm^W|qr|hCj}1IGH7fESJ=CWtBbF#v{1B44w)0h+4+0q% zmJ{bOsUI2ZTz2fQ)v%+u`iE3Jht>___1O#>8X!e(UzQlQ**Bb@=#39GprvK?&NmoZ%?^372 z!*M9N%D3tKsKM=ourpP@GF_f4b$N{-ir=*We(!YtV%M*AjihA_aZ>v}40F0#o9`sG zf@0=ecr{N-W*HS0zgg^D%sK^xWmNR+kRg%I@5ve@T2!+*Y9*d(-2FD2xC&$RN|uwOmIqlZ!@`}!2fXD3bqO^IFxZ&9dt3w zlnqIOQ48hKQ1?Mvh&NU5YVfXYu2DI7yP-+)#=u0zO41=)GK%UEl+PcvU|!tWF3{v3 zWZ=U@D|VW8LE7?66;HDRQ4?q@H@v>d$#x_j7r^Vl#e|$+zkAnG4f|PAbtT!+0)Le1 z5sRXe&kJK;U{C{LSfA9qiu-cPBasQFiu1i{BYrb6C)yhIlHXPp9f?|vr6YUQ%K<7)1 zo^U{iODkoaJrLv$sk>D|G&gq`D_)M~P~+;>#nicnjP1iUH0XQ@m+Ctj4VGEXUXZFk zwBM%a97y8EEpEQYNPa~8-jM-0cv*Nbu9p1_`VJd#LSA(U*;$|)HQ}h;UL~zA%J@sR z*O(w$bd9g9y05D^S&U%Hlei*~^o5Tc7 zDsHWXt+y7PbbG28+HSKW31D_HV08_oF1xF}vyC^#gJW*Nfx&GRMrU+Y_x+H4zP+L2 zKj=>U0SYYdpyqLDv}L$IqVFixuVbwAR-z2sf<&G}RWv&Z<=xU*jYf^rPUxVNUH(z> zl?HQXOH&7Rag*=RPE!}{NM_#9MA4)g?l_KZCQn9Jg4X(86q9y7ej5g+4V(HnQxKw2 zT|TtFi8F|)*rfF0ql`Lqb4QUPE>Hcn3e~p3S_P@J8M?hQk2n&;kGJZa^lwW~G>)ui zKTS{Q4@1^WOgQc@dpk{=(B#v!_c$B}T!v*d6(LZMa6Kd^lL}s|?~ihnF15M50h+WL zZt}WRY=a1XCK|2LnVzC`*U8o)^@)cDz-TNYa@|znO<4@Q~RDUt~9j6KUV zt_6qRy2v;@Q=}Dy2nT}Mqi&sfyzef-f~VMgCd7`1%o^^SoL4gui-R(6^mPcV9v!5< zpK2I(xzw~-7V8$^S66s=NyoL1KIA`NWaqX0hQ-9F_7%&z{Ttgn+vdd)BeR4U1cGdO z0%&*9{rM8%qxk!MrKW-eF=&CATAQsSO{S)={p2rg4#_(OR_QZa9n8(c?B))IdNEn2 zEr~Yo%rJA25sLM0f%-Xp;*3oijma?4L_#WHH$=F$+{@&76wFA+<}CT z@Fv|Ta>5O3>f^pT9z7(Romk2F#Ph5BM5gH2js@z*U@051*Q(!oCQSuA)Ychdp?G2& zZjd^^xk=`4P{Iza*0QI4uI*OqJ1@>Kx~43&oAZNw#k{jL`T0JM;`-8~(_@hoX5`R~ z0b|_>k|D~OF>0&Rh8y#&PM;WO;0D{YY+gslzmjRFW$%u)wEUGnUoC-!)c#WZ*i>-j zMIZDasdO4`{Hsf=dG2>m`1y~;@eKBio4Y$E2^GXHo8QYrKJRiUt8ud28Wrb3PoJ$< z4kqERaDKcZ!a7eYIYYb~8LKqrtp^u^=nkY?g{%tea*Mkz3@2>Zq#S`{YW!I{N3Pq@ z+;Kai48>aK!)n7hm)o*>f7&@)6;0u3Q5)uJ<0_$z)Fzo~#e(O{-^ns;jzlCb3HsZj>{NM>frz%-* zCY}(VpS{E}K^^+>L#PnxCWd5Nv)eAga%(Wf@t}5V-r<-XnnnY zai<@LzVS?KL0on80oIb;E~CsJjQA^_d z^SH2FVr8%TxIlE$aPFh-A)=q-3@{7$CdY+znUijMR1afy@TYnuIX7_MNAT2kM_mYE z98~B!O>c=iJQ_$Gg6VG`n&6->gFHW_>svEbZ1485P(pCk>>JNh6HDQuYu%l{yn9cg z6a`=8{%k0>IXK`%L|wY5!xSAaxsa!1?7WY~dI zLyKXBGOmFZOtp(?Hs>j;{hidA%Ct#xy06*c1mRmtbK#t+$^LpQS=Q9& zdJ8aG(_fX9%CremNeT&kH5U$dmDg`8x-a)<>?P4!F@s}|H-5U@@0ScQv|K2XSnisb zv@QosW@xR@TS+#+^*xK*d8>tdTw}1##(QiX$+h`hGJ);KYnb?-pKH#+>TgyLeq=UC zQeM{1SIXTf9;|P7n&&~pypL<1nsBe7vp`+Ey2nD=-R5o+S=eM9mpv3(vvzr?b9=H~0{pvR$u;i=(MfWY= z#?spZ$LeNy%|pJ+)xXsxod|x~l%qRf=7)KP0c&ARid3~KdQxuYGia5Mq-u43J9Gn# zna;>X@-5YZx(Cd2f5tZo0%fUlDb4GMNU^6HS z`&q>ES+q`r_ceeMpDx%Amt2V0yqwq$Huf|-4MRP?{1Q&F$z76;Wfau%$a9YTkoKij znQ*2WGH$%5O!bQ~EKjUI(VlmJoZWSs8=g864)7khM8U_0jAXM#8x|szfU-98Y8T02 zu6WMT*;^*RFlN%y_5|*ZP|)vo3CWS-fsg|py(*LX zj(y^Fw5QwWSQzhgHd?nsu%ztYB-gKmCSvBuX`3^s*0A&5I{8l4zcgN5h4~{Syk}v^Nf;zw6hpZ-Y>o;ZE9Nb&W%{AG^O@!Q{M01stf%!6yn5F{ ztuSW3bI#)kTQbZeOw0AY;>s)iD0Uq=#3Np*ECVow2r)X2XxJn!n(dt`>J&E2^pEr2 z6169~&@30X9BGGgrrwa$-wgz8KQYnQNAUh~OV)i|BAdoKXBYzVCP26}G1{xOl8Vsm zQl~v|MK(7o#sdF(b4BQB+aI9U9l^-RgEQg!)eg^VKKuGXRK2Sar>FRZdRG{D1-Ej$ zNeK4H=NldOfXq4~{`8>4&SCKq^Y;sc5yoPkQ^fY3!4E6#cX{6-wEg z4(i$Ctd4lS$O^#seXkg-4}=f^DP%x=E}g9g1mE|#RTUxcDTPxjj${U9-nrv(lo4*&HYFH5vx2jqq06`mDY%2+Qz-#nK#(s=#=%xVpU65vkxB8}sx|^7Q8Fcg;4|)88GD zih3r&YO|utMLdrmd5&?(YdTA(*iec9%t0qjf7E|prE+`Su1b+UzA3_nSTZp&kx0_c z_lwbjUd-sjOMzvElJRO-I6goIpvv=3Zn=LuO5FKc*XwSCR1FspAT`8Ux!xD5#o1pe z=_X=)yox=xV!}SyO-6yR0xiAj9{GH{FEz(ZA{pK3BU*a=ZA=w$7#&iP-^#~G3;dZ?VMbQ* z`hB13wvuRij3*X~Ld^8qN5K{3z4E=-pMy0PNE?nP{V_EUX^O5vg!A-@XVBacEp4dx zN%j_U?(2xI71tizvir&(0a3(&UT+j>y)6b`;#hmfR^fxC&ZC6C_55N`OkA8U;652_ z*EV89txL`8hVy5pY#+s?rLRZr&w`(5hy}+9aV=3ZtEc+Fbu^vzh;A-oYUKn-1Pj@# zCuI@z;y{ad6|k5EK0fYfN+I+pL@t7qc@KfWrYU%_9HlGLD`F2DBY;tHzIy2kgAxFV zNmY?H31j940pe@kBj&!=w3tF+nr%WMF6W(_S9_=pMEg-^kc!wZ#jCZsFPU12`EIqw3+0EW5~uyMSMvZY4WqX_vxKjD zJSQ?Egfbq91c8aJXUh+Mi6+_%=-?8k@%H}houdayo%#et8C~b7JRs9_mOlkqE=-*? z>#HqMUY)*chnl41^#P-wTtqQY#y;wcM2(`ptw zkLhY29mELIbXQraeL^bW96vol;Hknce^mZ)6Jl)r*5SxC<}?4&J+iDk%O^`%>6z<- z__{NZu=(}8b3ixQ__|K@T#VtR2cDV@F8tb5SJ=np4aEoF#KcFl58}`D>bZ}(DUs@$ zdhDNGO@oF6@%rM8eupB>{c04IZhlV#CMN7l%o5)=F1Xfjbf>?B2m-+t%S$*wyrZ!L zV|;1~{4w@T)f-_8p;0@|3`17|Gaf)JAP}JB1*;TzT$iJ$1g87$HVV?MPrNM~s$~rk z80hKG-f6=X^%%{6#-=OL?LrBMwbY$Mta%*$8vD-uxkid3pEK$ee4V`m$L;}-R(f61 z&=Z|q5xNCc&hx}yEX-I8E$@~=`g-k&KumUB$Y_&)si zK-M()PMRaRkAiQNGh1&FAkjGiLJpus=;loXB?t`NokB@}B7Cz^N}+Nob8cxTI{Yq` zreD`m3TC=L)5>JydcF#jE2Fq(kyNd7nVLz}%u#8U=V%XDrAEYVrlufF^NxWm`5z~^ z*%t>xH*!z12Jn!K^pIh)aA{bNyiztfzzGP8u!a%4U4qYesH=lX_J9nwn@BD31L$Nb z$L&_4g{HO+?fB34co)<~2B0oF( zgd3(NplQGK@X3MukB`JqNDNcFUstI9>-Ycqc8kxmF11t7A5#DN&3`6H?fp)M|u2iM{J6gC=k zAX0|r(=_;B0vJg_)Gr*I+xoLiajAY&WIFgNSdT$-HHTB+e%Tl?zHJ>0UCgLV_@yC% z_Q`+vkKK@}K_M*&>01_NWRR)#c3A@?Aj5(a31+XXS(=*4&v*S>cK$lrun8zY998&Y zbVUfbZUAVjF)PG8<$dyN;3eO`7i~_pVqhG1sSlQQ%E{@tW`_KamwAF75E||WO-vW6 zi0*GLIuLCPZe6&>?%O$x!{)rSNCT`Jtq> z@ts{tyjs7HBbi@)L61OFmK4papn(3jWAmC2C4_);ke;vlPjcc_f)No_*)wT!BN7JL zXgyJXBc+xrC

>^NBz$??3}4IhfSz^kP%zet9|nDIV#9({SBAh?B*Sf^QSC^ z=7bsgWS2GQXxSc{m@d;#iVZf+-ZZXt zTZeP~;EZ=JaOc^kFIECzVhA3~y!>d%3>BoG!8v?I#9g3 zFK>sR|Ic!QsZ_h8Zie9#=>sEF_2INaO}lg%m;C5rFf66_ASQp3l-3xnsAd&Grl(Cu z)i#bX2Y7`is|;htQ|nqxEBlOdDB}MpFCb5V&$dPhld4*nw;}!fi*MT5L+QzflzG90 zE<*$;VW4V_D4Np`_`84E-hVr`uaKZXP^%edn5SvVCir^eTbl9v^>vssRa#sGC<)Kj zO0D;-B>$zpPs<3sT;>mIWoRq1n7qC@-+9gwMZlIXBsu;aR2G+8tv!y^`h;eC^V@sD z^WR@=^!zvCJSl-^kf*5`m1;^}aI?=tUk@^UoO%u+BQ@vr`3NyP9lv}_zUP8@01zAb zFDJIGwD`hf_K(`|{OSBgc7O_81=~woMBy0)&3_v2K^`B-Z~qDhIJ@$iCkdb&kUQ+u zIt1`9);R_uc>c5*K`_GJTs28qWrCXCZn@^?=Yo*RXHf*MnwsHu*CbG1ki#)QK|^pq8>VpOd)3AOjv3~W)cp<)?Eu~fZ2zfv&kn>3rV+Eeladb6ZL=+)y`qAkEwqya0L;%;Rm_EE$4hGF@nJ5gajT zXlcFv3LGu}L>04NEMgM7r>EI??r?8cv(pP)lFn88M)uvD9Ti|ue}fYpJ$?J>kbuQM z!BalmPimGY&H6c&ZzwxRS0A&}Wwy2s^G4gOI-y%B;TL~YoSa_U?@ep}(<0r_^VBRg zZtH>Rsblt1AU#g`T1#u6^ilhoek7n0ullXpNG$+U$L|jSY4i~SXp!8t;Ht2xvc-U8 zps2>a>_J8$dTA%o8i#enpaoZS&F@S5N1pzKn%wY6ub_%DgzKb)F?hi)Z9dFJmp8*Q zFAd=USeto~tHl51B5;E)mwf?9wc2cIu-HO9#|&xpv%9s=8ntx#vW2R5JWsg^D#%^N zXqfor0QbM?(%&D^5Nm9(`Q?qlS8dXN`Kps!Tc z?s&x<5dp}2!`~lEQhWGLO3H4jed>=3eg&0PPQO%_-HpqVCsmE9oIL3p$+BEu=lu4# z0*i%>y|yVwi^t2AS1CPo`nJR~VD8cg5hgqbhPTJGAgADlf~=>CaR-VYd=R3 zRO~(Q=gWs^4GQkI8FS)N?r^jqLFu_&6EXds<*GN)MiV%{0!jj9{|gn zO7^IlL7a{yK^rWsqxA^_5Tdj4xJ071{e9II^J-!Kejs@?uwtq8!xtV1kpHlAARrMD zl-4P6zlT-PD4S3M)=5v9HWQIHe=^$oyMWPN8_xJpjpaHM33Z23)C0+3u{a+YqNMnT z{mcq36S468ygWbeO(-tE2ry;GK?zQtU#=n$GZGnbP@Ukg_5(5g3vg&R{>TIgXgzRA zW~R;yf*EniDjdYzjVA-{5U!N{&d|i9#D3w%@+EN}hA0Tg1L4Jliu}R9PESE7P^%Qp zpQ+g`_AzC^G4ET*lCK~)n<+z1o)=vEDNFa_#R~u`HpC2gekEN0i=RNU?8%0B^?Qsh zUDBaiY<(#YT(lYsUHo!YS-egnc6?DK0S$uNf0Xz{{~FkQkF1G#f7X=~L{7 z1faMR9>ZDzI7w+94^H7hwUPfTGeE?lp!I0DS?Jk+!gzOwSgsCX#>B`szt@uJ>@B>C z$4DeY%EW|vqFn_T`llpER~XS#ji^6t1&Ff#%i#j{k5{H&eY_p0P zES<_aaETnsFnRMi2rDxy%Z-@ZHGaSlp9*kCIg&1S%>kfE)Tp>eFBSbCp$0aX)8Ywn zf2?qUIl!HL-?}{@KK11->oljM^KRo-i!y~}hvNzYn$N%Z{ zK(-y8ixB0>8U2Z45*zVHM-n*K@CcN+c!Nxf_P@X-Y_t95DRTU8Ed*4gg>?DSGy`x~ z{3U@F=0AM{c#atX4@lWA<3pN?(wP52dY+c{toH<*H5X9R|2l(z(ViEPxn^_5PxTM* z{z<~jdY&rIf7rMe3WHYlT;t>tnwkw4VZ49`KR!Nw195lC zKODg5@VDt-OsuPPV4e*Bhl37a194-yN?`;l<3B8v?&$y_ZM3`);PU5FaF+0BUl3dkz@*{I<$K>KCBSKY2}0q+iBhhMJ=JqZjf7IBYV23%L23 z1^bUL6NJUGBbYP1l!eTnvi<)RSw3-4Q0&Q9IabjZwv-ZG{ZydjMZ{z8$QQx$ zXF}2pFGAm?GK`y4>HzoO?=SeBKV|vzd z*F2CN{i7&fFg}U$rx8F3U@e6piw*win-C&^yZHZ;bnZxI0D&uUJ^*(;zB~>G?KVSv zTgT7Lzy1Rg#L$7eA5K+13RlZMR|6dE)G41DAXENFf)c%HF(?Ndm8UWdH8m_UGV(uJ z+9!Hh!UM5y^gn%O-Iu=OAC1gM?@ zx#eaGg(y3uNWjrj2^?%Q{gXNfwtyO4+F@MjQNIe(K;x+)W;34t0cHVTaYVfi>II1b z(8~&4R#ry(v|-h6|4*eXnK9MKH~rgG(EPtK<*!IA7R<^*&Bna8`X{@pBC&k7y%UkL zlizD06wCSQKnd_W0HXQ1TKEM0Lw5s*}N6=9(Y?-S+dH3y}Yy?E7!hWx|Qc z@w(Ew1hZZDuUp_Fa- z0#CT%2MHo0+vevd>6M`uKr-U}TjI0-`Gf$ekr(LQKwNEe95nfn9!Pfz4VE?r{m7&R z3X#*(@xOfel3cL=zgClKk_NmWXhg^%^BRlc=Vv|BWo9VQXCjBXH2(*?fIJQ!BAD*B zDRP@)2TILi8i0SQ5M6{avj69-2t+rVY)0$l=iKe4v*k?)*dM?9`0=4@IWXfR!hjr@ zf!&yN;L1NghiL87-2%qjVDuecOONCFeraU)2w_Ku` z3ZaiW`JY~m7#uQ^plnidDepL8*@H>da{POd$xNpYzcnAe>eI~u3Syr=juUu*;guX{ zQF$A*Tb&MLHJ)4=+A*Ot*n6W=A`8gtPs?q}5&vZ2`T)HgC9Fq@g&p&>hlevru%qEMd8NVg@3GGQ@tjczE z7OC)~K$y6y#P&e4y%2=+mT!)_@?ky~w`cLS|77Z^0jZzIY|-lzJ&c4xx6ncZ1#Qw*Ub*4^rGKgxUDWR&Au~lr5|e8p|{F(WM?*M zOsG(~7?Y84eZ0h62aWDvS`BPzk^k$Ti3R*<(2Xf_-g>B0*gfbz|4~^)3|_1#I`O?(9xg1BD4d z4g7o~R@cJO!hwp4%G7Hb;K|!&=}ciITx2-4{Mv#vDtV&U3VV+vBc@u-+?H>Ocj0Ia zRuda!c$6E#slJVian{DU-%pdHFzjhb*aD223+mLtfgGmXJi^_vS5Dvn==+T{um4mzGRWy_EOAg zLf1dBLn5kwWacY%zpmdkuJz7F{hCJXgCJ?KprAZFr%Q=z{_iDhQ_`~tFJ}!w>Z#8_D2qka=7N4C=(pXCKxQ# zcy7Kl@dlHpAjY>6%xopGiIVy*;`9({_(K$>6HPnAu-fAfo%41<+@J{X~f&eqU2;23^;7R)M{ zRag6?Lh5T66Ei&$A@1exmwu3@ZQ}>J{YgcQQgK;X*E)wv>uYj`WzTGa3ij0|i}jly z8*5B%8!MoSg$J_1^7i3fAoO2>5hzdL0KhxHm|0q^{=)+FiRYZtjY3p&0}~^mGu?_7^L3{53IOu^k1+l2a`H^4K(* zq6LMPAAgw?bbQ9rz%v=4+z(Rwu1Zs0fmG?1mRyx>$4ck~HT*+&@8Hs{HvdDjWTX7l zk&H`T9Zxj>0FG`?_nf(coJ<@zC9h)euQsRP_4r{!2~_%aI5=h{E^G2cv#>-Dih&l} zim?f?hL;np-&LOM=shthbS-Y&N*d)BgX0qed@PEA{)1y^I$V94-u8$r7>@db>4pl4 zpQc3T+&>w{MpJ;tlH0EDS-m#$<7rTI4t^e`LthhOjixKKtPj3~vBHt6z9Cdg6P&8# zy~_ljnp>zim6MzZAd`EfK#mUNmy^W zQ2+|7e$}=Zeqt2=8vlFBu&*l`N7abgNHHcZ&2W|QCw(;z*EcnP$(EOLQrME-yxw9X zL8K4Pgi+dZM87ccY8(_rf)(hVuZjByt@;(iwhExeZR&(;V&e8f=MSUezq>m!IgT6d zm<$ewnvP3d=k#9ZoeSzFj%{NJd5JO9B-CAo`CvkNSfouOEC=7XvB9d)9;i0`S$nD zyu#nLX79=I-wz>Z!{eazi$8ZtcT8M`--77b>+|ikwy#TDLTz5)-j>)mJDE?v^bTGi z4(Om;8cW8k*QB@NW`0|q?=%#dFak^Z?H7$BYoOAug%zdUkWapjZuxUA?Vzd8&)}re zV1Tk6{i6eb->DVA|A4Nuy%bLZ+;3NuQv5n8-&FV`gW@f#@%3%Pak?#gWkn&vIR6`{I!sf2rg zAoBjtsr$!Wm?!7>M7FwE^u?|yDr1)ODq&YNW(HfkbY)}IN#nd?$!SDnrT~>^;Jh-9 z86w7-z*tlQ6v~1E?=ix2; z38-rSrR~~c{)=^}y|HfKTSVx2Df@N-!PjjWK`x~9M}-`+q`n1vU1mJa9WgT?n|L(# z{|00N!B9tl5uFIQUbi`%+N#&KWO}E<^e!NHQSXkc=fY^WwWLv*WO=(@xUEk48goFC z@hRnVPs`9JtA)>y^@fykJZ5kS?Ij*};qaF0^g}1%6{0#)SC`IKP0;Ax39Z2UmEK8} zck1sK2^SaF@J+WVOx=)+)F%;tT9XpRG<|oP;5(9i3$tSz)~=R}j*j_fn3FylBw*Jp zuLxgR{H=L!SuJYuGw;^{0qYAj^3)HhnHo++^o*{jNqZHhzV%1!QSJ+Q4ZRw;=JZFr z6gy)7=Xvo3k6|SCcr3nG-e(lsZs3rTkEW%xLu-%f<%^-`!+Y!lFp1)bJW(YYv-yy= z3=UYhBY@y+R|@VvvMB!{!Kx}?KrW;Ip8q8qoh7~H=P-DQbE45GlqZ)@4#}bh3qJ># z9y4*{j`b}I;n2`H$D$}Rz^vp+Vox=FIo2z_#TpQoq^U$-+x;NQp|;>=TDXh0os=?H zTgpT<{+qQgJVRp4yxFvxlx@hpFx5~l4w3Udfl)@-(2&t7Y(m7oJ$Y_skA+b;0#xS` z{3r!HH@%xJ*Wj#{p?Txc^l$ofqGQ=WPJ#q^2i)~I@3HnQk3JD0w zK1R&`n)#$164%Z~{N9dhKumkxta{T%6dqYo~py;(av_hOof`%lc% zTFdr0AvAPb6NMqFMOq7~Zx}=n3&&WuPR#ta698pvU_Vxb7@S*^20NyOSLycNJ;%jV zE7x^7cz>Ns+#}QSDyDaUa^E3-9Mdw)hF+k7PYe}M4@o9rg@usSoQWy?@`P$qUql9Q z4xuW8T~|rSwX-NR4`>{myBhVRzMqg3_0Y{>hcL4qvNSFzG-zhAKSJK2ld;7eMfGbu z33tjbG9#W*SpR(eaKp>#!&%aMRrjSp+A*1{gO)|am#K135o8p)`Hy`*8pn^-OR{J5GFBkv^r7AbaB*Tg@l8LCjqfR z1Brkn1|1fujfNGY*cdJCKm_WXc)z#Iu!t}3>FKXOO z>1ozrjS%8TL9!WQXZXf@V(JTPd`x7oSJ(IwhK0qf3`x)=av_IBfOfwEJy`L@Y`%U{dn+uY+oyD-kW^lN(K^d-| zQvh*s0ZQaHy|lX{=6KWP5i8B63{@a8kF{A<>(9gt<%+=Aw+n;cGvvx;RRHqps4}qV z=SbqVNO-}rCSvq59pfnp$mw5SV)r_2y$ztZC@V0m%N48klu4#DV%XnUw@}`j#Qa+| zsciYYnBhuXZJUL;@Dg>*zF8(|<_H+LGNB)j!udbQd==6Xo<%={?wpT@%p?W-&w!1! zX74iiRJI|DYH0ID?)g$UO_1Y3Mq$WFR8n@M_@_;=droZPT3ABkRWJXa_No|W5bIn_@ zvovpl6}75{s%u!4GW^(_WCe92we-hE)RzcSZb9Nf&0GI9@|$#Q2pxp-p^BK_=<0C^ z74u_;%aj#Cd&uDl=XkhvR!{fGMLh6sn258grECQ$re~H7nX%l|J>1 zl)#&Mht;Za>@524qOcWcj{hY4n;^@p_OT3QL-BjW!xQ*CSptvxhjqWpQ{ep&7im;X%kjMA}Yf9P`M{I9M@ixSo}XxjKynyOV|3=Z(3^=kfbxnxNUIEDhf| zoJ-T&-G5Sb0wW;Eytx#xyZF2(Ese9@YX6Jdy8Og&r=Uhk#WCB}yAH<8QiMpeTHEZD z=te^b$?LUVH9Sqc$o%+s6{*=bl_Ww#(q4T=vvF=p*62sXgNZ~y_npf1b4*L_*ZqIX zmL}N>J?TuPp=H~tD9TXQ>7*_jJ z;Z+HGNQnNbu^0N<^xqHEf1+NHD(=O4M3w$|#imVA3u#`G)AeJ5b2;ZasL~GW2lG@1 zNX`(TIF3F{?gWm)x~F*nn+&RT_(ui_NN;zx$_3Ir&=%EeyWmvFRYBLo&TY&GI48+# zENx>@)^Xn-&nf-e&T6txO(g-2wLf-%^qk!sCXxS`AQ;>O^25qrzbb%(90Bf0Iwgz9 zI(?8B-|S-o?~w>1h{u1Lyz+5%Ueh?u{`1P2;unb06^j3LYz5g8M1Ak>?sB@EVE0E7 z;B_6k!Tyi&qF*o&YXL!tc^RoMf1Zx(ZuNm)4iCYJa}$*7rNE$(;T+)tO$869cE$HK z8C&Y##5IGnPARSYhycoyfw)}?>Ey|p+AH(F8PmTeqE8C?Zf{1@tN4*YK?szE_B4s% zE_Nb*XWS#{6o;F$D=?k^01zaQi5%Q^l=s}2f<3NFW*@F|Ptd=zu8n;-~*g#mGkXpN6R}PINJ3Zfw z>IEuxr++BY|6fZj-$jjlqzc#<32SPvV93D?gEF!q#7MjeLQ1mOJwp4DA~cui>HT~+ zHQtjj3Cq-kiL18vUSaf~GxYC&ebe40e=_}=@S;p4t^jsuH7 ztq3wxa@hC6$v>|kxqy`QF0v)aVlXwCQuO?<6fdNT{@N1)2?0O`v*DsENi(I#o)ki( zX?nW&WDDE*7N0< z*HdU&&3WBtI7HfI3<_YL{ROvszw*yZ|ESN+8ueujXwa^&szL^h&&{gY<#3w(e~Mv( z@%^5_^F>8TBzYCD0SLPl^e)7e*~3-s-d4rG6w_+YnC}cZ_xG$?Q``Ua>(#}{wHUZf zn!zTd*uFSrxkP5kHHm5=HDbKF0U(5#Qo7YBJEux{^#y!DMF5 z|IGw})*eVLANGR|Z)-q$U>enE638YsjWS;Ut~F=i$yL5BrLF7&$xz@u>P^b($r8r1 z6+tKl8>C;KoInZxB+R$2dE4mx@F$2UK6iNDQ(K7ff=sUHU8}X<`KG%x*QuwcZQqBH z8B4vgla{;H$gyWH)zt!RF3_2tb;e(m&@SU`%Hdkt4L>YkMn@5)MrN>A&tmy?Ry;^Fw6IEwOcXV zyX;OC$3OF>Ft%E^v+TYPDF2mi80H9H*{T%J_Sx^~IKJ6^|C;hVZC#(99NrP1?_AvE z2M`YGSkYug;eP+_PC6&v!|KTQpJ!16oqba!@`U&%1&zK8a%cj(tXUrx_~e zxuX~g->*VaKq|1IFL|Kg#m)XVsma@pPdkCqS{6_JJ7DyKjW|Npy^+W=|#R56E93UkZR=j@j30}xig;@f7v e+exTv2%x9S~z%2I{70sto`=1qE$v8mAJ6V!hV*B)Oodzv&TnCB6F@CcQ5)Uk4Gg$ zJ&Rp;@hC~b>Fj>f)b-(9#-Q(OC)hsaJ=ORfFTX(R?5HzZk@mgyE58q~S3$yl^n$j_ zMm$h?F46SK@pu8v8N%vl;nw2j*s@XH9Z@fg|GZtG(%q47wo_E^5Q0945x$v!5>UZE ze!}}l{rLKNphrzpqd{OKhcQlr0OED^TIEatv_xKOd|m}eb@-;bPp2MwKEuChfu01n z`!{uGa$Gx?p#6~BpSPV!67H;!%hLnBuj2(A^lV%um@F)es6Bzcj)O(({pDf039QI0 z4pYPgosY&JK69o)h}M=%1uM?Fa+@H>lxbfI{p%yi%C&Pbd?rv6|r-2nQj5n>dmlo)A`o zWXBnk;^DO5rF^aBF&rwl*4aO~y1AOs7k`S{Lqp)f29(Xme{)nHe?Oq0NC(p$R*N$o{ zWObF$B$GaoOIz}6nSI`#(RydD5nzc5!8O0U-ZOr=N{0z=gSjTVi@JKBsfu7CdTZV? zcTM}cuUjOW@vb?RSZo!ynrt1&M5Bl7oQ~irBHEMEL)7yLtMRoYNQ+aoJ=MsuFYCa6 zy6s1oSpcUGs&p86wOsEQHkdb*p|_y4M_VA(EQI%KX}@hTrHOn&8ai`JOB!r-c!AiUzrz$z>EUpaDe)SHx>S=Hj*H4{PEHP( zGe%mqo22b#Hfz;*XS=qTnm-$3W!KfAlYfN(jNcq}Vucy&16dVKZ3HxQ=u1m2 zr|c}TFgGZ2tw^oZzR)711azl_)O9eyyO395hOJtj zy;|WqCbRW_UvWntB(l1|5oL(UEBQNi;T z%@$Rq2&0CvF)4b%P(3C;aLY#>vR4z>FDR!pWp_%xaO07-ID-(Jig%33r}sROM^fNBV#!^r@ z%d8}EuVn{GeG7-03|&{9jA|pVITWV46=&Zer8)>`8LSZ~V_wNyY}4iBBoAgkt#FSC zk2Bzn`-%BfI%^dr{iU>4`1A9IXXIa6>sN}3e<}`^zX{YW(@?oYMqtv~*hR#!nKg^R zHz=!Z1664ckWrFZsO7?C>X4P-6hnVZC&B<@dw)Vn8gZTsoG`b81O@(lp43-iBU9V^2 z3Mx+CPFR8FIftmroP=528fh*<6?NbKx4zrF2z<`j%wLz4YaATwzaxV>CruPd(Q9vKBOSuqUxgJQmzFbhfr*IhjO| ze}&KNmE16%kO==kpdA;7 zGbl*X3_3&vQrUY}7ybYOsp$HDwFdZWDhGDprhZ>BwsKpx25OV$jV>+Rsmg&}>uh1n zj*@VJoG?;oDGQsMAgKT|7mCE4SCdf8_w>4536l%+OIoSAa4|&O;CK>=)K=Gk>%po_mo(IwfT>~@QE7ehOM1mLkcj=ABpzaZF9ghlkLcLBtoU5f zlct<`oLgK}Lw}6QTvZLM@^L;XkJz>(sK$eDczgWGkjJq1HaN6~Bt9wsovg+gioj%+ z_%@~u*Mm$>C^Z!EpUC`YoIp6+x<-1bVlq2;S)~xNhu)LQX_$)2-rGbejVn|$H`)4f zxT?`n$}vDjD~l^`P}MSJP)8>?mWEl&a*3f_;{~?AwWU|JVpPPX`Zs^Fg+8Tv0(kGG^G3p5g<3^HT#E|PQ7xsoMXtazI!O7aH%U962Uy77$8ig+ry7(Ti0#jT0iP(Xk1S^ttHAzdd@!qxKKt~X zmFqw!gXfP!8zx`!>46sB8|5|^-nW2!mr*Cw2|!9QVF_wP3_i2)VrZuMa7SW7R8LM``YOV zJXw-fep?faC}R%JRXLh;WqDzppZ5=>3&)wltli*txrEp!1&(F4U6N}&eokMV%=V6Y z^N2K?7umt!9Uf_sZwKS|e9Qfg;!*R<)4>SfHjQb&Jk}HvJ~m@w;Bt|`-uQs-pWM=_ zc-i(3ilhyuM}!|O^Ol>4EF(d|l$)Rzx{RkzR#Zk#=g0?wsW{`ZVj7rY>FkuKH21iE z&2z(Hw^|A4*IC)5LN z^#`e>%HHX#QrI1>3pZemxt*(aBse_w1$d)OGqhP0Cm934p7S7U5Fmqr7T{kfBswbc z^XHkzhBzk+d#tU@D;8?2YuOy!chNW|hX{Dw_W*mUU#b%{+8ZTE#6GJ1SdrLcIL#1q@Kg%Kq;;1hBuc&;>e%w>cn&UU_o#3zv66cT`kJ@6 z&@tr>imjXe@eE_XGOfjpP+#w5y^_~8fTQw=js&me*RFA=mc|^;*~?*)hl9FpDwP*Q z(lwhC^O#KisN~KZ&yzxMEpJ>M2cx$?CaoomFkGB!PQb`zjta~g0yd58Ei^)6Ff|>9 zh758Yk2V2YZB|@>!QC2lvQum&3gMRtYPvH_DPL^L=Ob0NHz|YcH=`%2{p`86!+%#R z(tEtXa%RY0(5C*30q|aoFm{Hs>pnq&31$~R$r zhzGbkCh1G|%C#h0aU8}!*i>`(AjFP$L1NEp2xVpp1pS;xW%&wE^^;gO2JYix4jsvV z7Vwia@lelh_UVXf-Ck^K7SosxU+HN%DA1C}SW1U6uA`NwVM zj;>I`0$RNIpqFsfS%&oR^qjAtj>>6%yP@#j?4}hY(<8`M(uK|7VcxtXsbyID4&$(& z2)Mzy&`b?1tXbDOE4sGYE&%De${{pwf6`PZx_`z9QwITMhqiePSC#32M*LiwwZn*x z;7x25pPoipeda%9aNe~wVK8lqa^gG9@c1mur@gjo)w{58U>Cg-d1WWNJy8p9ZO!dl zE&>JZmI^LMoQvp5BbXp(qXVn~dML04Hg^DL8qE7)aZ){a`Sl zQ+XUB`O|YS$|x@oX7uy5v~>chi=n^+BlXUZNQwu3;J=NTR*YpG_jFp55)!b(?~;xD zj6BUvVwm}m*9R?LOsl4a5$^Q;KzSF3in_Q@`AsITnnItoyk!h{c#AfENe&mFM)=k*t1K)m`Y%+pM15fkx!oFp+P7S| zyXkxl2l`cW`hBBb#>JiC&Q7+|bD&J;^eCxv3x|>D&}U=Xdm>{wF+z$2`Oru)j*~)I zx}r47}zTbuR$ z!y8uSv-8o>@(7_n*=3~CF4_tUI2`WDN+&-Bw`XqeF^#`FiOCAF z>jQ?0YD>3+7CcYQe|iDj8GA$xcyKNGbn`;GpS#xOmbj4b&8c=PC{B_y&o^fyrZl?` z+FWN3=|$+vtFqtpeljg}4axBL*J{zmzN(qZ9o@yFn?z7$=fqVuBSszLUp}VL3L8`< z>fAXmh=l>(7HhRf>tZC=ln4HznZ>tGncR=ll|xH?ab&5fJv3z`Xa;ljvUVcVQm&g# z+LoS?^hi6d{{DzxUgQKJmCjZb$Xib?hR9|Dm8?XNP7rXau*a3Iep~~(Rr#a*x{8un zepQSR=iXdSo&|NNvCbM?| zq%c6jVsY~f(yA@jmp*|JKQ`LnVxPc=e9mfT5cP3rspA%JstY9-<3 zZK(AGX0aE#$%713Z%!bqcSS53;&mr9J|eH;V(I%>^mo1bf< z+$W`g$Ipuv8G+DPH1IDR<@D$-*{A>wr7C0Lyh7KFJ(J{L(De%jU~3gPe?0l( zNu3~@9s%R|Uw;2D_aD6j`=v@P74gvH3LOw^1qjOnXV_KNcOD-!Drj_t6(OB3r=X$P ze3O~t0qRUh&e_SYV_>ns-kgQO$ZA2f&42z}ckCnO&9$=q1KL|ddJhi#hrkgDlf%vu zGe@f9$ExCHD8mi`kpN|ll37Rw#8d@sC{J)y8MhPFaw}7#tE%KuzNJvAI)&;8-qvD$ z!3aQg7FC$J9XLG#Z8CCM|&Sij?QG3@! z(uSu`DSCKJE5c(;i6O4kLuGEXXq4#lSV9%oYramlCES`bwLy*tc@9%haVl-`kwu5C zyraw*bD05DsnUur5=z3lFXg-wsnH(v7tV633!Tr-#clBa!bYGq5}+X#$k+6%fH)ni zPZke(E-Oy;b7|eW?b492VBCvd;65ccGIgKe+fV}Y-?rMmB%T-Q5iDA_(;5Ehf3*uo zV{G@QsY$!rb^YGlgK?C@+}h$ZCRiUAvE2};R{=SUoPJi_yT^SajZSo_Y2f-!gAR|w zUN_?zp_79~Lhk}UI@e2Rl8{}>qL~@0uDRvcqgzQOP1}(3Md+Y#TpmuVn*ULZF6pd< zfd)$NaDq)$-9}X1sW6w3K&%W$W7WnbdpOU@<#phA{>C(6nUrFD4yU|WorNeYNhNny zufA64kXV|!(Y#rK`b`mdq49x-^r;M@yOr z1&2iP5mUv9$Zf%LW=XmBy;&i`W;oN!8w+X(LZd=%ysB$K-qK>|z@NCEwwa8BCx%E& z1SCm>X=!M>D$P;FDsUpiKy@>G|G|Q+kYbqxo)h(~K?pPQ%Z-d+?hCvLOiA<8al=-l zz*1jkluU{Au8jd;Oj0VzFamy(rreJr@l4#NB|t>J4W@fy%*ywCHIHrwsz^5{z|om3 zg&vLPIfa~{VU<49t|cL0b%r|LEjbh}K0#9+T}?wE{db|AR(m(BQc1hox_yRcd?Lsg{Tp+C2KSSW zBR~Kh_xzuwKb~qHpA!k<{Z4+RsJB_-VMvDF5i_G;-c7KPq5v9oJY}?n1#yP3HeT>t z-0x_zuiB3Z8sC@-J|NN=GwtTu_V?X&MNhuKGX#&|=GJ+_{>1pA?+~F!3VOd|dbR{x1660$yq97JGm91pu zo~XiM4zH(nudi&8jD{%J!RX*lPJCR2X#`#%&YZcmUKCgS8~uVsU|uQVST6K5fE{iS8pq0bGt(_v38`rzWj0=bYN_MQlivh`jm*d~(uzr5T@? z2Y7~<2nz`rg_v$Q1;hZnl_+aLPaW|bGAAV>*Mh_N=yb;?Btl+ny>G$tm*Qr*-+#M( z(!us_CU6upyujz?H-}@FXQMWSHA77r3uab>1<|j)&Q+65($5rfU$N@ z6IR|4l|=llBq$d9*{1tSzw81H@CW53ztyxFW*cid_^E&ilQUT5I)knb^YL2~v2}@a z7vwUi6$VmJ4_MJmh)94@5vHUe@IBH&_hX&lRdPQVM_MnENh?IL<(nhhejha(%Hqot zX1Ujv#F$$JTq>;LDG1qD8@`S#w*&-0CEGm~EZft3(OthNRUOpF2dxd2aTjF1Cb_bZ zP}EyzsNlc`Wc52-i=+hi%LI6)@`r~|;}m6<9$GA}71uwjknzpP4LW5P6wn{dF9J^7 zJb=`JF~CE~JRF3~j%kAot+@gv6!6_DC>_xl7U;}Oa@k|;{1tjeUWQH|-Bd8>J`;Mo znn?UbeD&>0;^nE?e77c&JjmU`s%}5DxRx=I<(tD9X2V|C(M&!(&@JF-hV&(s$=H;z z`Iz7^i1=MIa{NHAj*Ilz@yH+;)DnM)s3{&+Umfvd@chF+4nVC zL)zvQB_2r3^hx}Wx6n~FTOE_2!+!p>l?chSS`k;5yM*&sSk2Q-`5u_ua6jED}sNFmh5hbFp9vG;U~gfbAM zcsr*^a%5AbIV7|B(a5(Kl8;C0}%?K#;$($0@O+dRv1j4#66o-~^!PFit8racm;)|m6|Fk$$ z31XDPQ%E_rVVY3QMpB9U7`NuvT=7wT{e&k;bgmy2LjvDYMlUi&x%GOS=kl!eTuhnu z?uS(T9|fcS-?KSi;l6AM9HPGia$-n7B>u%PB%}ui{Pyd+`RrQhy36BfDaO8B(Xf`6 z7?dvxi0^MJKz``imeZHg*?)Vx%oG2um4wmX_e_JHtWs&y3+k5_Z!=!Y{_NRUs2%%9 zppvbh;i6ID;Jc^i=(_j0+a2``D~qfg*F?wq*|Xa?sFY2Ifk^CKpTm361;Od@g`K~q zRa-~)9TzAPcM%N&fp7C2gN6pPZ^qsPZ^Uv<6VxPp{OuunySSvoEs_tS1xqt2-^j)T z_<%Q0>h9FLili^RblUP!cSjD3c;Qun6emT|b62IA&%j4FemIWX{w>Mq$Qy9=uxa_$ z4g!K>pfdAJFq{$W?_UZ0KxJ6pvD=vtwgAmV)$Dm-;qUmH>2uL6CKw?Md zU@d_O(nM6&Nb_4s9ss$ z6q>OO0zRB#(USN3R`9oUT)a>?uK-LBV_f&S$EGqDPTI1~e36^A($%G1(nG4!svP;p zUfcewA+h*n9rH`uIjK1jYYH;VSS-Q>@UHmKD}CFF8|fnx67X@^1o5H5K=!Zm-Rio( zXQTqaO|j{#;zp9?fLQdhrCXYHzQqi&R>_)IHu zq2Uc4p6JCN88GDe*vleFOpc;Ip+<4?cpboR_HZ3I2)Iqy6uI$7TD{FAe8*fadwdrOgu*v);ZN z4m@18_9VST>?iC8_xS|W$p7^(qcxpQ~tZ>3)?vquf`O2 zTygh~;_cI|dgpgGOHB`b^%j(baq_PYCH0_QDtr0|o88#FZlDgp3?kM5XgEsh6TD0L z($)p$8>OZa>Isg*dJja3D59v#y;NVztbt^r)4C3H#C=2lZq=v+M+*HN`37gEz`aSN zBF#H7?4K`>Y*lIg5=@A+Y*ZVVr^Px<7~4Y>ackRJ61IMMreCify=E*&Q7%mY18S`s>_X0uUKZIW?8MtD2 z$HZNiBEF6_^bBbcz%wsom5@T)f!97t6MFAFA#P zvPi7@wh3*VnFqr24fKQlwx8XG($J5$xe#2}xl1$MOOz;zxI{1?n0 z`tiZDny>DKQGZJh<4!;9u1A=F>^<tuInj3kdKVXK3Lzc${;)&*{}HRfSfm zG|mN;_rIRfJ4+-N_VtqpDN#i*plqHuRCH|o8f1$a@Q=Xba4GpxP1$ZvHACy}2fe>M z8NJvCvu?SLiw*O9#e@TB<6G`rFcBWBDTSM(Z)xT37j3^8iT!*ZS zVc&J#y@);7r%4jK7&_?7`;i+$*57)I4G~RMnJJQkQRBb_MTCAkLcZ;Dm<%mQm2H}3 zIQlRm3BP=-X247z>Q6z97L9=w( zJ`0VI!X4>D2APUrYVhVf$1mPQ_`D!wIt@GCIXKNR@^4d9j6&jLb8y+-s)jLD0Xpd+Vl{_LOV zAHmVB3$2HF4EUIu0|D3nQsU6Gmh{EarGw+Eb! zzHb`7`fT=PR}izHzm%L~JxSsk*lw;-E-y~}!KbRyp= z0%}&Aatu60NH9d0wHlK!!_x~CP~~nL>yTZjmEb4{<@tdHe+UX@$U#=9FdeP2S>txs z=l}3VFqjyJ1ZqyW_@njg4|T(=^<20S_yN{NJBpgFUB`S1Z8!$Ppw!SAM*Pv<5ExJl7f|i#jBY+M=P`N=x0bN!0AxHr# z(0|>1LU@y?{N2zK!{&p7YnWyD^Z48`S@CC;C6BBF&O6_}(2aN zmn#KX)xw{?(5L_qyL>rw?j;pmkF5*Gu&AoGulR+<-hEAUBRuC$U!_|YzyX;RpO-H{ z9Ij2v$fyAhSfZ*tLSrgH{&(t13UEq-6@`vfH51y{VVX}Q74#4~{zRxg{r^Jh}q zURT;mCazs^H1%@p@9YPVrcymRGw!%yTI% z{vTW47+h!6w%e#tW81cE+qP{xY1BArY}<`(+qRR&_SxzCe&>8Y&di?akEA>EtY_iA zFRhv5hVPt9{jSC-3Ys~4s!fGGwt^Q=9O|8&xRs-(?p$2*9rlzzOu@M?pwRE*YxP}l z%-h19>sfPMpRpsERD%$#bz^WySNG<@sA_%1;)~+RZrWQ<3sYEl(UQ_2*c;g8)3^|k z4yi?4w%1(;m)NF^AIVvRJ}gQpgAK4rOsY3?S5De!neDWJDs;4y<#k3G5Q0z!wWYI~ z5F1t-PWi2TXjCWDr4!N03E{1E-OI*@CsM<~sH_yKfhE;iJ4#^}q|6N3HzBm-Y{JZQ zoDZWfe!$pp=OncJWg^eOPwbl4csjuVoS9#9pB-41@^V5Kms-1jyElR?2<|uTzrs%O z8_+}RLN=FZ?cIjkR`AfC1W@^a@E2qOLA(HbTUJZ9AfqPTKQ+D;tyQvH3$IlEGvrWR z>Rw)c^YP)FtESB%1{la_VqddexP5Uq*xLGDlz3^`i^gE~P}pKdaGn7&kpVJdiFNmC zCZFB&j>uYz1?@R7twy)B8LcNTi83YXR(?&A+Q1i5)q3+N+Y31qi%oxJ+&WEIvSX0FA@ciP z-_RLYE7ffpf@6c3o6NtfQfiG1j+>-g@C+thYn(NOU8Nccihw57TDY*J9jA@!HDE;5 z$MoT`7~Y1bD!;}S;*Lq3Uy+|*yj$q;!9kz357TJ`z_ju9?)2g1I^0r@PdvUeS?lSA z|AD9LVJ$`()L76@BWp|f-N%Nj`j^l1j{c$4Y3;0L+FTGhy2fj8gd8k@0RWD=c5J1Z z*H>R5M_yD2xB3jN`KXZHf(7`7@#`PY_m}r?bm{3DuO#5p`@;{zx%0z6J+lHz@=7RR z2ay~q#q(9WL!;}8RPyn$sB#z`3bim6mW{}0+~&tnY^!#&s?L`uyz|wE&&NN#%_`a` z)hsZF1$F>tIjnKnsL7B8No=^!kSrJ<^<-!754l8h3or`{$<>@JL6OG}b)6xPm#@iG z=$Hni6i{&Ri`dECS7)it6>m}Rs*H49aAVFXL@^#dYpT40#o4j?=Cp!UCF!s_B3e3- zb8?r>`=GUoT;ZVFqbW>~ae~|_-k7jrQ!kWV@3wE0TK?Tca+y4v4qxk>T=l8RwHX_C zf|SmqMP>fB{l~boGR0HD>%IKn@}F_}j9W`{`$26YJQBI3(ToYDPD`!&bTm1!id!C; zoe(AoYhnSRzZkWGrL>gh_>DIy-C(RG)k|PGVM!X6chob^u@v>d)Ve)a+i5XZZgr4R zM+w@IKgy*-JG&#p(}&s(qjQL*}05_2Lf*IUH9((ja&YHDHr=Qt$< z&hU~!c^F6eD#DiWyMv*8O}xY*KY!dSD@in*(RGsB8q2x4)_&&4)S53?(!?+!S@FG( z=uqgj^KZO5MWfXde%JP3$s#UtuPiMg`)m`mz~*D(v0p}c01qrmPYX&IJaADkkqk~0GH785chlgE+%G)bo53!*c``fuK~8gzAUqL0&26UJOog)#F8W)H zSi8zk7H)G^+Jt=cM=C$ZHk5aqLpSlLUlANf^%%+S@Y2#^p63b>rw;};)}sCb6qmJV zlqz&uMppXSZ9kD^smz|>>ULShb@0exVs1Nm@47S5C>5&F$F4~UeJJKQLhS*4u$zz_ z^UIRenL%frcY+2{Y~soHls9(pi;XpY;^M8ds*3Sn5M22FPns|0W3H`9lmVEavDkMI z0o~KnnKD_p6H%-ww_M`p=FH9odhq^;jD;aVj^rq6>q!5Y4*W!fVGO5E)#aS3D%!** z!bWJ|AsnEG1W%8?@~3MZ6Ukg)7VJ2fz1V(Zf`w(=$+b6#csV`qp3r^y9iqia4zmKs zj8nhnUp?j33zln7i-pPN=jv?zRy2#h9%S%}<@eN>IT-FhkAwkz2OX(w2D&HZ!%#*+ zS5Gl(Qwct!$4_ZwWCR4&;u`8qR{5V^Y+jXd^Aoc|W(sKtAI?v=H7q^n1he7XMe2?d zg~M9S-|i<=N}Si^OAl3`@y~ClAaX>b0wwT9WysLy_=N+}?>2ok~5 zDS}j^fI?!MosiygoE`gZ*o|#y(@N)^oFm$FboM9zEMfQKIG1ZP!f|d9rWZigVL#AZ zOODR;((GAU`f$Y+FYtx^xHt%p1IQM5Y@9hgVtQ=n-Q`>RhhI#)jV#xi5m@ydiq^I2 ztr=*dRSvR2;JEJmBUTFso!s4_;0)jr{6s5 zfec)QU3Bb3@?f5mBN9tP!l)E?0CH|QPNn=M1#(PC<0&~A`>BGf9L;$Md?UPGMVfP^u&9Z8T4-F~X#|#aDb#|m7d|HGk2| zAscXnD|~Rb-E)Au4Su1`37xrz5SCfp@B1v2V+B(FJlu~YG_;`&`GGNu#3I4dczB?e z0BmDT9G|`v)kSan5Wb|jo^#%?iyLpnO>mM$=`u2GY;eIUxxQ6b1_EkM8eV~H$5{?r zxlgcrgW6rPCoc)^VJaUGM6q}^J$a$~eLLKy943dZXKK%?74KF}xYn^}!S@H69>! z+mDt~nH#YER|n-0;_quJoA3936Z041#d2k%#I>A(V1We$kwC@Bf<5PVagRAFmr@C@ZQe6&E9B$E#rOUTU<`&acZQrlKVJ>f{zsSxWHgp-G2q2Ftmu?X~rcYo-E}yZUjgsqlpH_ z|30hpQ^u5DBC zTvAu+e(F&cB``X+fFf?hv)Y<>rpmw zxlQ%WqBn#$P;p@Zt-Nv^aJxT%KDS<85OJ!QW z_C5hPUq^Jhjs~IE;xB5)(#8T?4|Lsj(&IAPvj3W7DTfTM)_b5idew26u!d{%JJ;x< z*l=T_EkTg|u@ zwn;w+FhMi&;E|y!gV4mXqTi32DelKxVkStdTIo9iw1OEgy&+O!kMP^;5a0XQP3+Ii zvIDXu2LwZ=lP}l_m(`y}rd$b^UxMpf^qbQ3bNu`R!Qlq=aS&ca3ilDDbn|Vg?j*Az zW$!_^ymN2xxaj9Ut zs-%?oL*SjoTG02PDoQp5p&m(0P~pty3?a^xA8nSD4zD1JSv%F zNmEcgh2Mq8Y)V6e|8j{wk=qKb>&H@tC!53^cTsb*O<@bnL8P-W?NMC zd#|}u+gSEG<|CB+$JQdk(QrDMD`^p{2RG-hFw0kt7;Y-Ec97=?Qg__v%irs zcAeo?T%1orWImdUM!}uk2{EMFomKwzIlFlZ$cZEhi6kx zHwwB6`$MGsq0C-U{Oy85WD4fTjT>IbZMq%@EYi(yv$|_3Kae_L66sOK^RI5cS#kcF zuGR_jXi%|FK%VYeynu+)|Bs}vbfwt(OVa0Tl$eUO(??Y%S;lP1mg}6mb(V;WQNQR_ zGFn|0$Zc3DlG3V8a-brDY}^&d3w|_RZb-X0dt|3jGZl4J7yu%O^XMcj4|?oE6<{rY|w4!CY4fr!0B4{10M;6*jpP6x{(n z(ZN3v=YO=E(L}&_e%|ola5&-SJ)J(UCvIVvyir>|L)lgm7eum9C&rBD08)#dl1hr= zDbbo^UT_?S)A_Rw@=Mm!zVJkh#J|-sg3VmThXWcC_(#ovyHDW1B6%q9zmV)3&O&}a z84n&66%=L;Z~?gQ31oW{^E*8q!gK#ddTs_`CVr-ozXz#1UoRIdKw zQbC}QkQ(oGf#Er;qe(Uc0V^>&n%FRA5>P`;kJlwj!G%&4E1AdiOISm_;CMA*Z`&_C z*!tWa1=(zxF@?52jp=IKNTxS$j7`f1^6g8E2}WK8#lSk26+^SZ%=r26Of<;6G2d@4 z7<&tjBy^j{0Q3#C5~^CSXqJwor_g{YVKt3ZY2=HeG>NHAayj*{V?!AHR!bfm3Y%F& zWtHELn04khw#tbb^bx$pT1!&_rFq|~7|S@%()}ATZv^j=tX4><=V~AI_K=@gybx-mpzKwZcMh zrmRiUY5$SPO1O&L7<0s-U#D5)Erx@NDKl(s-|D{ZdEaX6YdO1IwI}s6^_%($O?M}fXF>L&uvc1K z&b*tTw6-oS@?)36&nJHl%N`*6IZ;bg-3!yobH*aKQBmuq{U*@6Ah_a1lbgquUuJ-| zNShi4pza3f0y-xCkR|F5u%b~$yz$i&q9^Lge>eNb?P1d=e^(Tlc%yKtHLL@t#ZuXe zpjSH|ZAc(o^NY1(nTwer{w%hO+#i0OC=XDiBRf zjH7%$kAj_)s1YRj$ZM^x`cuL~!`IrvT$B~cjCOEU6e=_7Z&oiu1SE&kII_@QW1GZmSLgtX>~^7D2LPC5}{!U zSH1f&vyIbj>|2v#@yI27I1qGf)}fN$NbEUAzkVdc)~$WgACihOy8Lp(=qyVj$C2KZ z#xPM{T5sN;#;9T^T-pKES&AbWE#oWPz3ZaZv(Qs1xXf%A6*H1vos-bMZ%S$K9!L)M zO=SC}e~mwAM{>Z3Cf^vuVjKw7j14S0?@HW&}45#8?($*98<$f$3&|G{Wi(LZ# zcvq0Mi*~d}4jT|&`3zx$JDE$qOnQ*lZv$?2{YHH4FmAHhW!wd{<6|-shEW}*)_%UM zzts1LU$MWhW{0tSc)Q;>Xr6B^*eF$nX}QSj7F2ZtfxqF#W1FU*&hTK`gOnf~wz#f8 z$m*&B@bC^ob$$EY;~9u|OSI3LI+Yoz^Wz;aTMW$Tq^#!Tt}3w{1S-N zMUNENx-;_eOq|ThJK#| zz9T2PdiGu8XEMAoh4wj@r#)G-j*dnWG&=D?dQm4I8l3#}tmpcha65_4VB>FB^M>*t zg8(`NM2*_028A0PBYf5P_0}Fa;RG?>ipBas^mi`OKIt7l2uKTRWVfH3MLjs?Smu(YRr zlQS1w$)8)s;v1-?mWRKA%PM$&EZ;sbTszh`03$fuAaFxA@>;Y9T>}kOk+IE{=F)Eu+@(=BJlt2k?*KLq>Iaa z|C)khURpt0mIqa%*`o|6(FdQqfr=0o_p9!2O=J9 z2sLZxqvFvIk;H;kB7!0>Uqz$P_8%!|mSpoKuQrDRup_IZmstSlp*ga#T^Kvp%86Da zm!77%X{GG$&yMk%`Iu=gUodxUe(o11)Uj;B4?$01uhngfllojCl4Q=dUM(Bt^MsWu zencuAxJp*R;~zM|NsP$CRSgOQv-msh)`=*{qbETT%T2aB0=TBEN0fvo)an{?k$4Ac zT3wC=9qPCE#s}#o-()CISQ%cZz?5VgLULqYoRq$6AUaNm@SC;7#ygV#k81WyDBv%! z4G=qCD_0@`bc^#3<1Gu)bGH05ENWIh?=1?qH&pIx9ZD;NUiFhLu8!JGZ2+|MdRD8m zcs%l6hAnNcHH-X`2iVpT>R9r7a29oOO%}jI7V0`y3)p>r$b(j;dM7sCb`uwAn$clt z>%N8GayeDkK5CXh-pc;9I`ERiJ%e1WDzt~;&7*(FAM$KaCQOUhT!IG8zE`XTGCFi0 z*KEdYfU0HvHOqULYd$M)~FIYqvpAFKd>Ng0WSyqQLx-tCT~!&Frk`TTpJAtu zjnnxO<2Au=Yn#6ftS&c$S7!Ts_igP(;WHwTuBydymh`MHG$V}XHz(XGK^>5ai0K_z z+daujR(+aBe=HDkGS9b{i&>iI+Kuex5b~boJik*sF5PmVR;opo`1(|NTN$6a%d}d(x@V{eDJLEf=hHC z&j{+|0V(1Cdf@=(48!p+7?%Z}G9Vm?RfqFHC90x%9b*nceWRU>eY z+ikh9kx55=lu9VgHPCAZ1G()z-G4~~}{WL|ix z5-94)QHCcOgmINAm2)}LqtjL;+$A?&;#bkhw<&gNum5jL_wS!$JqUZuzaH zPMLB99r|}9#QEgo1?b#fRkcSVjA3d_le~EL6DlesIyLZQzYE-nps6?j9OhT*>!6*qv6H}Fdn{oB!>+AE= zhO5$&z75s{hhwWp$kSKLQgKfU?5A-OE-8ixek7fi1Q(VdtJt~-oLc3#6q-^iFSq|l zsnkwQ5LWZQ8KqDb{#nq0$wZz|54etC{%m%)ENG=fH`zNu4LLZdmQbFG7?wxX9DqMsn#5xlN zlhR%z@K-jdv$KE;mEwk%PHXaj#}CB}xjV^# z%GGY-C%_s7BbA5yfNEKt3rgd#2~rp`xnDs@V(mZ!sK*rgJ)yJEta8(`Fti#04I>Ov|Lj zm^jt>7-?d1KBid+BzCAn-u^Ao3)9#WqdYrtpFm)CBk@>@+n=Zw3?4n4aVQECA2Ct# z-pQR&Jhg*1-v@+90+@Zz}{Y9lShEw9ZdUjV9_h8Yg}0aAdG29y}SJX3K`ch_1R2b9{6!B ze9fwi{z}9s4pENNj1!iPl35_Pm%Jq77~Ry$o!!x^vE(@PF(yclh$Zdr4$L+GG$Syj z)eY2vQK3J}FUfmGP3tlkkwz|y2fy}}s<8$^!6c`GSSf}LVI;#`q84@VP1;phoLLQ5 zHKuS%s<2URf1^GG#=}??=2!)8UXXS|;cinkxE>dT*@A6krNIgw2)YjOv0sbau}b1J z-^5H4TOyGX`F;4`ZpYj?|Ie_WO`f$%D?kzQf7iaa2lW|}1=_(G0&*kJ8&15zh3pB-Hjw_DGI3$$o&m?lvaFu<=G${_*orNK(}MKq}j*qp(q4 z07No_&zGY`@_!qciso98m&Nx5HL<5JK$QZ%I>@%TFX zo8Me{{HYE?vd}GSb*sUg?}92ZjM@mOUrw^6{zmNI;rT!B0zU-8 z{`e>zCQ6`XUiseplGkSy24q!AX5i;m3!^!H?O>t+l0b!ni+rNG5f?Fbg2J)hEC$tF z*8@r07bDG1(5*LF)_&J?ZplP?$#EwPfcOjYBC?5jj@$`IJ@@j7aau?v7q<U;e2P za~8hm+&%wqHKJwj`x82m=Kr1@@Sxw}fu$GUf*nIP8VpIY$~KTiD;zk{sy;Ykoyvkg zoURb8x)!+?W%*3R1t|j^Q!c`jS_hfZV2{^2Zu=D4-7`)yE}hg z)v$&yC;E1GqJUl717?34BRSo}yp`;^r-QpKz)1)4qFHcySgBfW*gAI-*&GZAPylyq zm==jZ70JNQ{MT00Jf@}P2X!nNroK5nSqL#4cbRqMx%a z|Al4Okug88ekHchc&1OIu1V6+AD@5bJot)_=!Xp;@~Z84QQ&T}W%{`Ah0FQz=TSD@ z|LvM>`$fxksNd;z=(Fwf?MUhEsqqBLwo7-X+hj1$)@XTVCO`=MvoMnHync@1so*0( zg87@S=If8F3U7D~v+Rf?T`xBCWwUJGpMWcZhr;wY5Q^crh>kO`0Sy3Pm5oC%_qSi8Cy36B#Vi(h6r_+w{I66`zoJ%Y1G*nsfx za9T2epa|D}8;D@3HAn{FO98zHSoBa5OgaUZ9 zX&KOF@}8_i(_rZ}p?*zs17MY?9(q^+zZSdC5w2ntbIHY50e7b^g}!$mixhnuhz8}UEA-Hlbg}*2aqG=p&IqTz7s~es&$7SC9abfIKBLz`({FW zW3;m(3LA)Yt{ETbNX{$gv|9aco=Fmx<-rw-WHCklYpdpd zIJVQzd?YHoDGu^;eTUlzh-?FzHn04jU?_W`s4>&!Z_i~y^y)R-@S-?26T6QKSWm?H5L9*0L2`N?ldcy)5uU{Kn!z^mmmN+EQ8 zjC6*=lkYh6prJ=eMelRuMV?^~Ba7<@xZKuZzk_92cKsX@$xd{V9}pi8nU*IEkIoK3 zzvJjn$i(~fgnxi;I|vRAXVKD3nA>d-)Y8Hcei%+c{y$j&p{ptDGwgu4jkpjAJ!H@n z^)s8Q&U!+#Do;TN_q&%)&yRT~oMZumDp3F?=D_fgEr%**XB4I?uJ zdAtkP)+0X*oDSgZAOHBMbl8X|M+5Au#-iO39o4pbcb45PZavr($acq|f;Lggo2gp7kZ}KQ(R#E_wJgVg;i1{0;ep2Moi^Yh$et4C4MdVPJtXnEZ7z&Q7&QCwld^O8RKj1|CAO zLqk>Bb>+)KZ{mx+M~7Si2Ad5Axxp_CkWYAfF7QKsDz{jKfW>*Z#fnV#Rl`%hv%$^n z1LpI2%IwK=afMy$lh0t-+g%^;;qtwB)^CPHG{e3Z(!glb(FPa|vUxi_dAB?e96a;W zY_H9H^%j>~rY(-a8gz1EL})VQHV$<_>9k83!DDD^Q{Fie(v!19p9^a<90D$ zBjic0Pr?k3M!F_#VWQHxx%po)ny}!2O8h(M(ZA#!wS6*tW$nw2UT~yQ zSXfGKR|r|G_>c`<-bB{tRU4itAr!WuHMu zSsDRSq`DE|eB5ZP7SFRH%U8WjlAot&O-LLMpFang-}iXl>=#+kukYjB4V3a-tL~@8 zo))J3f+nh~7GCBDEsC=PUXxZGt_#)5bXH%@vT+8_ldKYrXj$z?DT;T&w=$*oH$EwR zz>;-bq#e&cKQ6nP-9BjC^B+Vac`pUuK2Fq_-`62@h-zldk6Rdybo|%sI&eSN-(_QA z=iEADvNs>sa(M5zyFVTs9@jApoH`j!KXbiD_EtZ9pNHp~Pgg%xitkT4MnAOP4g=bH zoyR-Ql8>MG@LUbZO8c&l_(W;U;ll?Qst3T+uc1OovtMoYrFLe-?Td}K_|o|L;qYF4 zBv)xgi(!GK3!(rnupfB5KaSwz1}#*D4MV@FfgLC_wqZS*OL8DHE8trFt$l*v04s4^ zNR02N;0MdcYkzFkEYYnV>JLMD?O8N|{yOy_K-t4i_sgq6(Qx!VT*p+^0lu4)S#6$# za?*~ymC&iM{yn;hS5&&l6U;nmK;fnc1cWn5(Fs({`%27v&uM?z%eUhOw~0LvNaM~I zoGhn0%(8SwM}@1(sguUK4NbYPtff?mW9R4Ogx=ElynvQ?yCmDDfaP|)=Vvh3@ZH-L z5Dvq>ch=nC4D@G2>TJ5l&tPwCYVJ&9yzQRz?e61%eu8}FFqrkh>mlcct&4vs83*dw z4c5abGbsy!llrgw$-5g>|LRqBX7JWqCnj0;dP{LlkB+nVTeYb&cg?p~o+x&=J9Ta6 z5ft~6XS1sjK+uZfse?9WEd)|kE3Y^RlGP!_GrQ-$)74AUhdxlSPBf|AAI}OuUl!Z< ztty_)p9wCv{tTYGZe5C^Q`9Z-f0-$9U8~r1`))%0LIeUkXJuq&LS2qDu~#3++{Hx} zUE(Qjv&AWiVv`ozKAB3}&1!i*AdHPOWcOF>gN3D}xHJf6_IvE#9Lx+S9=59+%5vHb z3By<#`~yx`hJ{bv>==jq_WlUY;=>=dm@;Gb(EX>`ZjImvRAKH$QaGAgpEAVDq4oey zG{mW{jjKaz&$jahidwrt6B6z~arN&7&GS+#SSG;k{(=HbhH)XXvOu?OR$ae4r*^p8 zR?Qn!yYdO#$&DAvTNFD}N2q2y-u2`N7D`7^;dJy+DX?jb9sayU7cEbIjPET0ntT?e z#;|vY2fFic6XB;C9i_IBbN_k4b^ulT&`{x|@Au$Bg00NsF>%jywOlC=7ZS)WxjT7Xp`&8BUqhAIQc>y*wqUXy;BoUt!}DpDJG+$G zTVBe8@zWbpIr<|{zmu^^P2lt!0*Iv}_%t9}xG0Qb2m^?gxSJp?E~sNN1P-ex!|-Pk z&$N1qeEd%kC}hY_B*}^sa+aP6HB6%quY&TZ_nkr*O>z3q*OtA979E&mr+4@OddEt< z3x!7FFT{-BzfTCyr$SL9tb8w$lZmQm00tl`K~A1*d>=T{>3v=jRq}f@_~Y%+ny9Mtz;>KA3toA6J3&dV4{VQY z_uM9(TyK#|V1E@0%Lj!;%GVw4<7w@+*5!6g-;R}~8uQ4|%JPjX&L|a~=|K=Is^jvq z2ing^&_Zk2DC)K%c$CJ;H*!Jvr;o#YL~*uy2p)m~io~DE(ZSzCWvwz59iqCfF}&)c zg_LsLM1I;4c5iZ9z2UNNrxKFl7WyJHJ%Mt9A5UW=`IptpN6%uhE7*jur!Op$Ta|oI zVWJhKfwJ7!!t0DjRVlx%!qp*pwa)?CI6)y)6XG@4U@X?1d6V-+sXC1^uJ!8TuUW-&2+#1I4i;L)>-ysM^A=S z(=B$k_AF$=8%-8oZvXR9&2BJ_@BQedD(UWrd2sPUx9M|4V{>!3#3>}WAPDY->1uIA zm+2n02y0EBt7cBE*vby%$J~V z3KGVmGE}O}{4?084NP*+E7L2Rek9al``Nv|$);3gUe{rQ+53R%+Kbh@!U|AScIgi# zcT4Qq4VJ|3)+)_V03P~lxO70D-2BgbLLIonz7oj}FDA7sm_~N|I}Al$@PM&*Mykzc zJ=jk1cCOuPEar;L_X_pCTJ(Sth;`?D4G+yekmhEAJ#g^6=<{<(S#eI^J8~ONzg%DG z!ij&Q_KlFE(YeNnH?T25UN0%7T zU%dS|*W-Ls>6{vws=uX&s9sj(N+8aCtzG0cdkabpo$t55{}Dl%L&tDv7V5 znLE(nFgQqKMx^|Y9Cc4Az*)-K>-XR#Uj z{l<{%zTN=ctH-@C=F>h=J_$(&DVkE$?1-D z_v<rTN6~)YTnt88^No2c6&%8L`<1I!(~oxVC*~4V~WBB1#hi3*=MW zQq_^Kmwqw3w^M70t_6JHe6|X$b(6fwhoL-kk<28PTMb32&9_;hiMQHecLbCP-MTtq zN!8gW&~7_dD-WMf!=3J{_KvF*X)k>?pNp%IcHT8W=_yJ)k8A974yE3RoiZBo{}zI= zh(BJDlS19m(#tGMO(yot4byFtQQFzF`m34YIl>UMJXIhnDyq22v{3--x5?0rgOmFg z+C6dbZ=`=j0QK)N5wP|;f1=Fg8xQSHu5`_Y>p9DIK_NAuSR>bKJZ=5wRt6jzL|oVV z>yRl5?y$Z!lVX}8>jx64=tZ}g9G=Z{PElS^WKTEw4hVs84=lOY8#wHr>B$y}+*41Q zsFNeO`O>akD#iOodx2D>ui_jBpzj<~pL~``pO%}|4MJG2++GP)e-(H$bC0|{tsCDU zlJF8J=CZs@!BnZ+x{4`ZxIf10e6j*6jyt+8f0fv|b#NK_*>TG8a=-}5-g|`mxa-^5 zDfb&s<}Q8yG@VUw*#Ip=q5LnU7Ap?Q4ikCaqZb&FR>}y-muNmHlQ=aH+~lA?P zH@cQOpn&?lx zyZYmlzT!}m7#(J|H$>EE54tojACvi+kSs>{TSSy(wo{;xN7uaGj1ms}hvJsfl}t5v zwKIxL8v}Kd`pZe*ReqaaX@=kF^_Ko?diui2sMlH-+)jYk5#@Q@)^OozVN%Ym4~PCA zO92nC$R%mD`L~x&uG$?Dy_w+7Kx&6-&6Et_CToz2LSxo=2oeWoaf$x1$0eq+LBcUw)1aUDa^(48}WWYR#Ka z+6df|7tfmy6iQP2tv&n!oo!U^}B4-~L`^U1}vRk5#7`k@gWF?WMB|R^o>W(|KgG7%aTpDoh z(W`W`tzZGYXD9%SNnRcDy#&rA5Il0gB^%`ajlI^f%pn%7&5>z9!> z>MgsX+Gw&N9w|*P3N1#sqIL)q%?SyE9YR@I*|}xqACvC;EVUKCOl$UEkO1%nemmno zhPzbXMR8z#9iPzaJl)bD8vy`awag)upmv_tzdZp1_-HdalyY3&){L*T$4*MrU~q~; z`%;SRf;B6&E!k!OVf6+Qjk=!*PIj;qp%G7M_$VVo1?|6CCCR%5>|4iTnC5>iYE@Neb(p7Zj|YuLarR=U;+C^dL@) z<7wxHmEuQmzt z(Fni*+p>#`OL(K0YE@q_npLMdgJ$k$DaXxJLy6|KR)v>Kd6FKCuy2Ov1E_R6p#A7{ z01hE>U#kq|?e(4FWAGLxZO$V_k_S;)x3lT7Ve$bzcxmlGXMM9`2;A+`^A31S$Glpo z&I0vul^wrxEi^wtDb9Pzm#VQp`!0>Ux**Ph21p!`+EUmZ>+u9oTsiIS+*kEq35NXe z5JLWa{2=84j@N(EtC@T4ZC#g_g`F!rU5GZ<&!t#<&Pe*~B3P4g8lkwbWzo_!xrWOG z#K#!B%_&7RQo+D_Fc=Vl;wuCof`WsgDF5xl{g^;3-1QCjTcb+ZW?ReQ8frQSXC}x# zmIL^S`bKb#LJ7{!trnKri%5ll-?#UMXFpjlZ9Wk2oVl5piX4j6^M!%4@dHZ67Hi6j zG$87xL+GwOgaiw@rEMlI>c^zwiSsp&!sKDirJve*qBtaT7SZ^EP5}I1@HaIOormL_ zoIEg%K1Tbsh9&(ci=m@Z(HjIS8~c@TeeGrt=~5R-@YPQ~uOnbQ6_=`aFO-9EgSlyE5R`Q3<~hY^FG^%=Sj7?U`idU^X;10tb15JyV(*lJzTjgZ2hs8 zG}Y7C&OJ^=DJZLWq#nf<)svqDA+K!}20s=54bnNqAM_=`KN~trui&0U&X3siCoWYb zph0lnrmPv6t6p7IN_ig^qNREM4JwY03~?ua5l8(&a`hPgDf0Y@+rO+ux+e+VBv(tR zWu_UxsVhk?hgw1{EYVxa@6Shu<@D& z)(8&rO7#x3>!qw09$^_wtA_)PXh`dIE?|bTnn6PAO^6Rja7UOqRnScc9lv{C)=&!} zp|GG=MJ0$rwWirPWzA$umDYT{yK|0NQqaEr6`V-S51c8(zl%5=`dj_CMBF~e`|F+P zC*Ixp-`x5SVg0stFSx%S`_-#fBZQQ}!090YpRxSn2+fA`PAQ*;pu4TpmNW(ajfM=L zUdMD#_jP;=he{fA#W8rMxtwS>TcHoVQ8o2uqizv>I|bP*(gL(i&^i_{yTxA{GYUf4B-N z{E{0HRIors+{G1J8OR(St;J_qg5dS77dpKXNFQKn(Vs|TS#GdW3k5d=2Rt6}wz&RZ z86$b$yOMAkl`@qs&1Q_Lsf?YvC#$)kXaPQw?Gp`Q5&iPY%7ML@36oXpA*?* zyIudG?*yIW4~0cqvXqaWj)mw414a=RscA6>gcYn1)@*Fz6MIo|Nh}5>t`<*Z&UG3{ zHY~2>&D3Fd`axQces{COURqno(bsgz*#JG{W`rh_%@9^uIASke57nhv-<<;mZ#^re zt-+~$Lgt|RK-HLf`w6Voc1cJ_41@^b2K-sYm@`_GT(_0g)8&1n2akYIs^+l{;2B4) zM+Jeu#t^;{v|sc5Ud4{TGybfwJRWtJn;X&!C=rFZ*T24nNxw9Vzw8jIY|d$244<+r zy-XfqrQ%*v8)Se*jGLF&6={jsDAk(!-`k)9M)W&ZgGTKw%Tkumw9+GGk7fDQjk=Q0 zwduX}qa5*0R#jzZq}aCa10muEup-${fKuW( zH#DS;=RTy*sb3!%p1Xy$qee9NBQ6>%GEfYU%ES>}4roRTM$jw>unBPovs@N7K>L1qw`^0-D3|EBWqcfbDPe?MFB z?2ZN4UG9v~zm28o5Vg+3xX6f%X3upFzf(GKJRhhHd%VU3&{UjEYc4M#n-WxOV!8mt z4~Qvr^I>~nH?hoy2Gz@R#=?1^vR zf^F75WoTQNno`CP3r@6HuZ7oU0Wvg$Ao(#UG!$;D`ymEgOoS~Ln*g?i=5?7>)*Op~ zT_zS}=zYv@(m+{uKb(d1uucU+(-8f?W8_2l!tae0a$*HLHK(-Eoj{``s6czu!i_bgrhMy^<9%bTu{{VSqj34sU{gZyoq!71})$n^9 zcw8Q=;=@L#$xxP*^&4UY+78G4n$W((&kEyjiQZ=$^jm_XYTuNsjAjFw-+L8AlVqxR6pOb2QD~ zNr5OPM_?#|N;GOT8N%?`!|l{?MNdvkd#GeuOn+VNRYSTTZklcyMByP+bY=$_iKlMs zQ4R8Xlay`m~IUKwHg{ zEXnD8uPEg7qm9{q51)CA=!DA}&pRWFCug#nH>X;|1qDL1%;QXn0b#QxBTZdW}Y}=!6vazDJRLNE_hX?ZIeSg9ePyo#_r0v_`w=KZkYVB0ryAQ&!1ZO5 z&!yn63;odsKF7MUmw=xQ<|~#RoSmgL^mA|V`JWp4W5gMp%A_!c`B3k#R}<@L@T{*Z zZ+K8_Bn42*a9T2;R8T*dn50nCJACm7xwWl*?^!ks?L!;iB7|Npb-%j-Y}==lCRAJj zKAb~MYe5yGld%9@VE?*T(_+A~ePv%ezrrx@BS4M}m_8wp9>N|-TV11OBynp*3%oi* z;z)W)M*G!rplD_z3+8=TkLQ@p%HxSTWYC{D^qS3RESS7y{a%dsrk+Lpdxe5Izzj3<%0T*@~=;-*MOED6r8Z6h?Zn zv~Qy0_85!WP>yI1zPLsWr9P+zi_e&m!R)2=qgQn!uXf3z&EWBBd#yn)$bbCv=CgeW z$P8eraSegZkbRF%8=kQTet@sJOX#+`aB47;$#RhS0LRw|y#w`ZOJG2O7sQ6+UzKal z4PfHe)1ysT#AQiVuqH@Z5NsG$U_4uf-h%?)3$4UQHh0HDAcS!^12L4SjB_S_==w(} zCmD_<=PkZ%y=_#PGWyo`X1YlJ(-3 z>#I?9bwR@9Wg~KNE#vmbE79K-GV9h8wC!(Qz@-9awX->0fFH8&UG`r{Y&q(Qoj zH6u#{1-?^3H+%y|39lr_{IbJ(9p^XDl7e&X*9CMVB8dr_hNYq{=&gmgT@V{#UM-A0f zih7%le}JSHTWiddg6tq#VP2a9fKXFRM!>v7wBbE@v(c3?MCw@bs5lh??X@p$RIB?W z0{z(HPJGGSa&Ed=YMwRMk6&w&&{_ur7r3;fbvHBDNqN^GrD%W@<^5IjuU~4xSh(ob ztI&UHdZ?#|w6727a#LBYyZSXdCfvbRkU|lo#)TyeEW2CerI#NRcTn4h6GxrR>%&XI zV1QuBI z!bm;Gz>FWA-C2)lDUx^xQMWfzo7PbwL6<6zHJ*B1Hc=><7aVc3z405 z%XLxvXRck=sJqW&KJxZG{xN%R$968qpq~LPfG4Qs=RXaTDa)G$z(FCz(L@{wz z81y=iP7yM*uQN=xjb=&HJj-pXoQBKE6*44o4g6^ncIFH+?JGT;;Dl!W=tsa`gewemu>CIHIl}d6|ueH@! zcE`&xYkrlwZKBS;rKFfi@$od=*8&+qdEAB1w_8~)q<_0;XVdT*vIE<)90d*to%*B-}y88vQ; zlklIfz{MW=Rm#hEC$<9}BZ4Z?2Q~JJYjetq3*ce+73*Oi4L$uXW%6h&*55zmX);?3 zZX64JeAB(=qGWt2OkTW*__=Ic%;)^vctNZ1^@P795Hm^oaQ?2^!1@03nmATyuwa?C z!gYiYF;*CbMgi$J(dC}8;859|^-3^jtx`r&mwET)X^_V$_za4omQ1@+tz6dQ0l1woFGXc6A1}OJA19O*78El_A)j`EvM}I zO_pTl;SF;ES=~rVtd!ty)~3hr`ZZ4_jCnQi_K+yCSmQdI1BnmG2G!+ zlQdwsB1W!%89R&g%jH?|@P|!%ulMpWRzzdM>v_Js>ldqk!qB)16APcIR#xS|jUH&g zzS*GDi7lRO+!?6OF{*IJw_QS`O?^7%leBYD_MM#3Du~u48bFhn#sGZ@u1I*5yueWyx+e&mU5Wa@@;7E&M&V z^Q&R|9sdJslQpR&8S9S`F?qYXzWv_yN>NRSFHeAf&Y?zUx=iBNl6MqPPpEOftjgCl zjEQB9G#^|bwouF_3)J6&a!5&7XGdd+RDZ}Y&@c;H#qRI6yzqQ_jc0Bnr!kX&CFwh94n2oG@243%#U;))gfZ+G+Q z?R}?3G7w&MSibSFvvLP6W?_2Izu!pI zpZvfnJ!wU|rU`$mT2-j7QzYIs8{i8&P{Jke=vIBi30wVm6(zuVEY(vD)1U6Udi(mC zCx@*?L;dOUdKtte)S(aMSR+qL0=6V8h{&PBJ?6*M{m}jx1}E8)U_fV7c^#H%=JPX7 zSUny~(x#DA1QKLP`^6gJd7|GtMC+gfxiA7cPPtL#Yl2DQZXjKkGe~$*RE=df*f4mg zrCVzv@!7Wc)5!JbTPJeX#e(o0a_y5ni@&N$&ST#Sc-%$AMrssRf*0 zhJ$I;STstlfHEleeIaJ`sdE8G;fP(ur%Fe}f6FF`)4_&-;MLj+vpT2x>PD(ZPp(w8 z|6Du+PitA0V9Ydm6IY7cdUVA6+Xi>uSwga|Uy_xUTCx`xdy?J!;-t1p9M`l+UF_P& zUoR_b!}0l`HtjM$PH{(z>F6`B=QPkm`>5hbUym3R*~>nlx4Q+y1B=6*GEUT3`FcKma~#qn-?^0gtz@!x++;@ z(mGs={poVSO=w@E8*1@C>+5%{bvn>D z+?@Lhiq(NF(;1_41;wqn@r1~uvUEQrl($JYwK z3dahp^%oa;c@s5VJoR8l=VkY^T90>h@o!Rm=Z!jaSjhEDx?1m(OLc#QVpx1kt1~DU zA2{Y*YEpXbsQEON%5sHPu*yg1*hw(BMT+<?TZ1(MbGsw2O~&r~+6ZV9?YuDnvJ@}O8;T8Q zP+D5r5)P2=5Dn%17u$Rc3DEF=I8URlb(+JkDJj-ey5z9|Qvdhr>X@E*zGJ5cjpOGx z#?mHtMA2Wv?Ir{h>0a~<-#`eJAGQe$#w>TGGc$h*)~I7w*914F)KBJ zqb{%A-u-UqG)3P6&OiXKy#LPq5MWoOx%_>=Xo*;G@SVMyJ)_G)r>eCj`jpn*#D2q% z)@}&O4YEtwK@|4U&*q`4)MRgjgtBXc8}RFeE@M7v6m@Gg-xT>{ky`1)4;_cD_lKEs zlGfB?6B_t;2q>WV6-lC^eX)wLEq0pJ5M*D*mAjY#yU$c)={D%}O}ynCzEtsXLTpR= z82FI~@VIS>g#Jg z&Wvx8kt!|Wob`}Ib<2nm(&-T(rAOLSxmu|``Nm?(5VQymg zT{!?|CT8T(_Z3=UZJbuLMlbT5X@8Bp0wdS(?)2Ki*Z5 zO)A-exE0pc}7PpxS^!l`6>KrcBr(}ZyI-~+UhL$;zM;KBLs1AQ587E5`{z|M`Jwe_@qOIY* zKsZrf56Yb)tP68Q^W*faPp++Z%XC7?Y#_cMSQYWUw0IN81evu&TUcT2?Ba3iV7*Sm zE=xf+TxX>h_eLL4@o7{V$8W%|!kwd0)6I+uOeqzG9VKkA*(Cas#v&H!zq53g#*eZQ zn0+FABW%oi;cjo_L|^CBc0E-z&ku>qff!_{x=mc%ac|6A81rl2`d(2zUQAG*g^Y%$ zpGGzgr=!l?vk>mxbU>P2%Recq^Wj*=NRjuiUWe{mWqkZRTF1DlKzFhK*+_-%W@;@n z4}q(jBI`9XXec&&az%&`Hr{bww>%^gff+rwT}v0|qNxu+KgS`mg3M==mtLN=8en>8Vs$R=G5@qwbr6N2|MuH9`xrt;XmSxwrt3!v@9eIBviq^cNyP z5C=iSBM0;wI=f(C0trP8(FZjf$A>E9RMUtaa!v8V@a4UsY*on2_wpkwTP5b+SkvV$ z$>U(Tv-@oQ{fSo%Sjvc`zn-G;oJzI`Vr-FNhcE$<2{9Go-jEkA#Jtr*_%( zJxOCPW%4a6j_--ZhWcHAJ(y&O1HatQt26RSiyxUmJ6NI1@`J6F!yJjZVP!j)KRwfz zd2F&f#MEaB}#pJ-L7Kek?)RWOt$9J~#2i{E1b5_4m5M9E1vQVy#I!8^ahSV{* zC)-tQ<|aN>Ckt51hwThS1_cFZ@QUku!ropTM?cctkIap-F}?}m7-}bF%JIK-$i`Y4 zs1=YOpiY*}kp=BYh71Kt_sHoW%|~NeP`-(_1^h`ftk~7pvK13QS+*vdNoH=H=cTZ^u|JvvftxD-n;xSt!T{hFud zk&s}I^Od?D?+5-DipNDqhs|MUtOXAiuhkwqkq4ZUJksi4u~WZlLd@BrDE+JV< zr%bjM?TspuzHUAZ2kS_|b-wV+XZmlxWmSaVgRo z4P5W<>Td*|ZGx_msu2Y0BJN7wjbKVw%GQeqv&}-m+0FesY`VZBgr@tSYK^@5w0Sx!A{i|H@%65$*5&fPl{yT> zx>wX+{C_dMxql#|1M^2|S~)ZFqbG07g)pi4jmo)vL0=Y7>|04Yaz5*KbDf)1^|&FV z{M6X4zm0s`PDztdbsXzT3^sqnIWT2QDV1xT`&l@2I^ZFNEqf(>ER~Z9L(!!*5!vlJ zy5f;(7*Lno-Kty>P4!?x;pXRqu*-B`pUJ*u+C@cDm7evfUwB4_Mae7QhC173b%m!r*RC2E@{AZI11rZ7L=HdVPzXNZaTR@2H3FM!Jbh!synm$4 z@Ge1b(_7`lfgRHm?~UOvx!=h{w>JWqpxkDU5MGL#<)9}Qj4pyxn)&IGZtVy}*zUdq zYpx#&%t?()rc-o83xug$yS%dPz4bh9g=e()+3_ZCsy$%w=UV*vHO*%t$&Ap7b?*w$ zkKXAYuz2kfU&{@4$YqnYZeVUzGY@u=XO{*;vB^>xg;5)?bSfC_6dYu(nm1~`>Yv>% z|B!h_Z}ZM#Y-bUAsL=#PD?ZTedU;dA$v@O^?9+UW>Mq5FVw7<-TXwc*S}YgsqsVhb z=4@`uFzOhN4txD^6K&~hEtl)8tTCcFnj|S>dK#U=XQ6Tg5A?aMzW#og755W<;Uix| zMmhC$cyKPVNNq#k$5%n@@wZACvRAajvS&xTjg2(0okfz8X`(j2rlV9+EYK(FA?0B7 z;qh_XrhU^C6-S$1o7@-<4_ROVrD3*YYVt8NDj07+(W|^moPzqS*w9%a=5VJIVT(E1 zcQl*EUcz@_Tc4-xJ%l)Tuyjir|Rd5+b@(PG!+3p4Io5LN8m6o#j?sv_nkZ2 zAEuXW?%opiK+9>t(!0qhrv!pP3X~XJ`@W_cjDzEJGtB$RYzBN7Z227JnusY64 zMj_`?TXvk;l5a>EXrJDbee>EAj(Xi=Q*WzHs15L3fN5z+3h>bf>MA3&T}!^)u_BPk z$=WtcdWgM})PheA!=Q{)=27V@& z*a+F(H6A5wN@c0D`OS73mf3z*&S_t|v9fLW{^OVnWe$=JQtvUj8RnS@HghLIV*9k#MuFo8KXv#_*83FEp761y27QG zGDX6T_@R_@O=d$oD9Y7yjiC$;`|9GIupzwrl1dj5(}R042tjD2X;g{exO4As{X{Nm6Q!a+=*4H5z=A=cum?QG!?-32oaNZ)6yiqBX(TP-qKmRIA7l<@wlJHTx7Tf6&xP3KHDGU1pD7 zDGhx0)y7}HXKCL6UDV0Ws+ANZRoQ%uVH_wZzx@trD#S`oWFsOFJ$?@i4Ex-h_tI9- z;+OSZCW;#};$i61i4W`}wd9}&XD)Pu>O(Rm zVTCNvaQ4`|;Q7?JA|&ftj58%iSo!i+i;KnjHy8uGdATqO#b&t;u)W`{6vDP(m{#Q0_HpldW?w_ zx`R#zU5&_r{*a$-%(fTZ4nJfFmRL|CoQ{5Mr^d$zonwJX@?oqT>vb@m6zv5H0^1NK zBA?Unw+R8M6=QCqt-*Fa?(75!M8hfJd-PQXvP}~(L=?Wy1Pj}uyV#|8gUOGKl6C~e z0Q~_V8c>iNKOLw7ff5cQ<#QPjx|yQ)7rCil-+PdubdS>^yH9~pDPh^WV{swT#x2C= zjixPaQX%&Ct0Lsx5A_!X>4)W(2Prz zbuw?Cr{{px@M38;H5ho+!d!7biFhKfME}TR&DUs6qT1i=?JwIK~A+!`xNt& zb+0?HyCN_Rf*5fNT6Gb_Y8}4tWq7(GVkUPpYrYFjPs}Zb!v-tw4&Pq>)?*g@9`p5g zMgVo-(MqfH(u|h>_mIth^tt4o`Qq;zUz~c4NCO>6Nj@&+ey3|(ljv&sZ!Z81D|Ebp zMo|fWV)9Uw`K}vD$)rg-P8!Z-+37aQ0?(SIx=Jo_*dN4YCKU4sMwt_NgP_QwN>^pD zVU^>z%PFI{__Uxm4}~e6s;x!I*1K?|lp`C3Np+M7uErT!x`fh<(iH4(wbE&!--@8W z(<4L+!Ero!lP_4l=F2}CiAOtEDgHiDO7KTh_W};|u~8ufH-fAqBRjithP$0z)$)De zM@YH^Lj^#4Ac1lRwGHq5Wb!)Bewv+*k@B^^aN>b9}JPO~krUkXt6Dz3I`7Y;Rc!MAwY?_oKejh%hPa11_> zX-1%r{3K+UyqqG>ORtTY!C$<&DygXL?h(@!wk~rLEUQ-c_f~b)4s|>!l(AarZ#+D- zVbJT)K#H5EWQXMZNeDJzT->LsyUJ8q7rAl^3T6d1fw(`w#{V`a7;NK4m+R?(%jB%3 zVN&p}%J|to0p$@l&^hlDfHo)fTzj`SwrP~9&t>)L(s{Q~v>B2H8c;i&k5xCERKy?6 zB!r|_RO@$_b2fkfOdd*pSuy)=y?5|KWR`-PaOEn(H+y64j91^^MK7*(`$yoP7V@$% zPQPQEQO@L! z5snCcO3*U&<`64fhQA%O#J(sfep`8kU-01`i9l%tajU(o7(M1=dS;N5ZB(pcbt1i4 zbpA+Er|_n*uW7zm+CjvGFCm)gB*WIv#C(&&@!;{z7vOa1S( zgY6&M$@L;sfbZ**Wk?xu`b6cIUb3tRa_Cs3_sq{S&9pte8vRt}30{4sq-S6!1HBQNl) zYm0><>u3cn(Lj!ujG^&8TX7`clUNiQfOHXBw;)804qD2wd7UO#DB|*Ls<6GlM6rcX z&Aut}u*mOV*i=^O3j`(UY~Xy`ktSK&aO}ED_F3?8%KJQ7@l-N2rc0Nkk(O7-kU{`m zhNP%cS{FOm&M~EzT;7j}I9$culbRfs_XQ@_`qaggWUe*BI*@~RBIFOlI6B~%jSg%B zKguAFh-$)E^?sQX*cd9f*U<+=1jDVB^{Y#(Wnh1-RHH&s1LhJ;z*68k8v^)KzTpLn<-#y0xw*adYQW{|Ag8JYRGK zGByqQu%V2_@r=}&C0t#J>R#4=*s_8YvU%*79I5cKE~~lZvsp)vA5FJMr-^jad7Fz{ zQM?3m<}4*f=T*GFVaVS8Jv>zx;3lG!u2}DKu_uj}@G(l%AjV2Nnekv$;**_f5elvK z*ov99whi7iM4MCQ!7BFd*R0E{bU}S7uc5e1!}saDgm5XzoS&Spf54fl7RM9QOjU2@ zx4qgZPI)tb50n1&IZVffsx_OzAd$3bLJ?Rw*y5t&5Vc5CS>$NKj(72=IWF9LvhyU13sp?ZX9`rd zdVX?EH>tI++S3#WEBQc4nwglxi%`h}_@nm|q8Mg&);$wxh?3>pq}=?7tkP$t%*>+l z;%}2_L>RO!D_^~V)zv9y8Wl0Lgo6yg&Mm~py$LKh)S=kzRxQ;jE*Ui+ zhKAni8n*q2@)@XEb2N(beJ*rWj{9DX3 zMT~?8VK}JXyguuF@1XmiV*j)eVdBMgHj4zJ&3y%O%pHzReWGlZ>ba1bX1;1Ed=vUj zH6^JWn8rxU=icDe-g-9J(zb11I+zW!H=YAa23b9SC1B}C}0p?taVLgSF^1O5>@#6iEP_x z@x4CUAVDhk9J%SJ6@h?Sz)-^6&Jwl}tD_~F?D$JpJ8Vv6w2%TFb}*RQ&@2!=(0yVk z(ePq*c}V{2l3GRY!~3VX2HxDjaO-%+L`eFR9l8V{T?oXW^DS*IwB`Q1Bgx;2lnsQz z15U$;AAKz2dq#)}ZOgi-uIY3t`?=6oyl#Gm@R;roLbO3y!HxO?7X@x8zI<%n|58X5 z&^a!XT^_x9nt^JS$9Wo~<3iDDYIuL{S>RjS3Hks;$sKvmNjfYGrNcWc9X`So3dfoL zab<=XxK}u+T#NjKxGqbMDj+9m`nbfD=r0{)iA2h*3zM&>M3m#QGJ_a`5C;{>s>C=l zL~|?!MprqC5$J!;d`{92ZHxRV8$6d>ki%u50>{5#Hwp6qC?V2wyl=19>1w8_3^tX`S$3)uN3}SZLF5G=a^rqmqK6n#EGz^s%1MYSL#= z^DDaW;=*6+r+@y5*Z$34`G+KwXwNt!N5IZt=^uzj18p*abT1M-(1by)0O^lNh$9)T z63a>++o83n>~}mU)=?^ye8=6Xv)0+(saHL7qka=(wt@ruHM=i;BFpm<_6_{hS-A!nGb^;&#h9wuB1Y}CFj#@17ua9W34U9-L6#Pa6^KngV-+6_ z7%8o}Gjx%EFqRH5l;<@5kbF&D(hY1uK~Ac%SP zIa)G3s=!Wa*A-NC5v~~sc8%!XBEI|AY8*Eo#oa*)kc{So_l0`>19$dPAI~J{U_J0Z z0cbYR$8eWo7E!xnt^OgX*PvO)`m4%f;-A83dQWrl0koH2CM$2mSFYX9np!&L_s#25 zZHXpgNZ_&-Nep|DFLf+GB^S$r8@n5OIcvy`&z7o4QL){lxoQwBK7an4K33aQ91f=B zrWh@W#};a3OlALAX!-iRu|n%P?|pJY5%^7?Cr4VxBqA zE1YH=gZN<~=hLlRVnQn%VyMoH@vVMEkP&-m0ADTY%L4rE%6 z#V=|4AS>o&hBb|WV^Qhs0k6PpmULg=3Yl_>=0#2y8Fc;=RO`UFoXMFT$ps))VR8y; zN2uOF#$?j%x17ML8VS^s3h%B<*4b>O-b0Vfj{iv)az%3>!$a5G!9gulk(l%sVo2Bq z0|(`+kQ#m#KZ4t*oa#(O6kt@K6d_q9IH$S6_n=D3$??YAU$IRE!>xK>=bxYZ4DeR~ z++b^T<4SvDHU$5Qu``L?EW(TjR*Io#MS{x4_%++hs6s>*to)hBrk=>_0}?aIK{AgG zx?2yxX!sEvXbqr{08C4|K~)tpKHB=>uL9OkZ^&t2j6wTjq0Ji^+v4g9-er6`Q*>R) z{_4!|{4aYhAfraLyI8Br_5S{T`4d5$w*=lZIE`8thnwXir=durhq;s?VVcP*tNfQJ~-b}?* zF{+55FX?6k6F~75U9DXR=!y_7o~Axug+Q?K6zDr0_D4&$+5R~Q*0;f1 zP$_z9olTO0TR=+zFp(3lMDDEQ@z8V$vI>!f8T#5LjiV;vOJ2>H53h$ z`Lo=Uni2!OlJZhNuPuS!YH@jn(o1OaYaQ%`5oY-2gep91d-rT@ z{umq*hb>87Js-~quK9BBn@|nVo|ujUXv#<~JG-k{gJCJmT|^{E&2^d3@|Ws_^aIDu z9|d-R$;#p*q-=kx7?)XD-sTR`*sN=a=~=8dKg+<@7fuSATVnYt^yTeE<%9FL1~0LL zx5o5;A1F^vvi`jgCt5t#w_UR;;`enhBE~x|vpt~z$T%Tt zjJT8CwO~%%QJ{g!1P+@?XK%~0hVP#q?7ao`8rAMdO0J0Ld+k$tt4>t!y!Y#=`*SG? zT3olx`3oUtdbJ7*u&ucLP^US_NS7bT(2wl$vb1ZHp9NQvWD=nT$N;&7C&K>x4jAM2 z8({50_z?P%VoozNGo!&M+-v1?Xt!hN5m_e#>ONrQ1FOhLMv#bFF5tKgqZftw{3Hg~ zIK;=O@CY9sV%!Ib{z~=z`v6F~K8rKz-{JbB;jmGxJ@)HoJ+Axfr+0^5kMN*Z2{Q;^ zIsuwn6calHj&{MAOLKUxZR>-6TkC!OHj96{xQ@IGpWLnd27Ji#)`!Jm<3-2my-)vu zZvK@i2l9bK_Te8}A(`_4v&Gj6wCZWUhqiyM`Bw{sK)s?H<<(YG&Gy*_=}z^|oEhVQ zYK3HN3E@4Ru7{E_KA5L+Tm_m{Z%d({QU+dt;V#3&FSk(-`GgR{OS+vF$t0wb4Hogdo_=*-}^DutF`ri1j!%>I@1NF09g}4sm;zv)pFYQkVI-aJ?mmQFoq7paY zCi%`CA#Au@u{736{e&f*UFs-VchqZ8-fM`TI>RL|X76NP_I zL}h`wJVhq!&}2vsNW-Rv=!14Eb{&882^y+;^SUwkdQ1G!f85qTz2Y`^c)W)lQce#C z(-hM(j#K6Fa@AxjC#gSJ6JPF4t)6kj1|CYyCZG6KOuIO6dthb+OO4{OV5Sxf8os%4A9?4A#@HIh@4ZFPoYlD9S1Gul1A5*;&vLfBQ=p5rT{AgFS$6H3TmXKe| zusR_~Mp_Dko=eWk`VA0ORrg`i(SFo3VRUuXKVWXRS?|^tG^KTK@ zoQh1z8lYZ1>cozXB3qFymntJ47c!~YXevB`V!$Ry3nT}$ z^J*DlS$fp>6>8;=)5YGaZ{_uYmIB3q3tBff`F|@zDh5cX}oRPh6zp1s+h571!6 zubCRGcmbH&#?et}0zp55&B5r4*(}%pjfMhs7KVeKhB8ko?*&@4sE~CvY&z4I2m>>W zi9{JQV=P%mW?ZuIboW-2G^j~FgPsyK=u-KA+8 zxf5bQ<@>72P4mNP`ai7z!8~_peR}TDDtE+~Pi34Xr&jx4V>b*b6xwvZl2kAU+SHP9 z1Qw7~@>iwx^BmA0r`Me z%+bynK#8eGG?x}a5*w>GLp@dDL$o;V9}Urf^hCtbW7$_?uJ-rIt(muNh^I`ut-mOdbt#<#t$9oWDN7HRGFTxV& zM{RET_^iG61{wHS2$(Kkt0gBJT%m)Ml8ouy4wu~I`mke`J(O{?Q$eD_1QOSHUt(+C%iF{sA{FDmD zN}^0@3kP?K&=030egPbEaU9O6R3k@c6VE@81Q5QwjTOqM50Qoo=ClSI%vb4VawA=f z{%070SenA&`fySFpy4HD8DT$f`P^nwTPl~)FNbm8nD;>vvK6n`KoI$|RzCW%rt|Zc z9J|s>fPs;-z-L_L0;*u%=L{D&GIM^THDv(DHH5&GC#5{u39Vk~X%x z7VoR4h!g!aF^7wQJc(ZCdlr9Z(B z=7Lnv-rM?bKtedEs4`zrZjvn=XWtEcz;IWHpygE3EQu!IC3_X_5J5310E#CO#wbyZ zkA2C!=tSGl7|U4wZzO?Ds5vcQEG-+E|7LKK3DmZ1?s#^`qp7+LhKJ?krl z)34~U5r})|i@1r4@Q6GZ7SWSnzPETIBh8RO*8F|Dqo~;GzwxPLPqa|Rz`r^j4iH;A z97JLMpCy^Z13}8vbUL6V_*=@vfWqkrhMg+wY-kX!fVN12?Q_7XMc(k5TQ3&h9*Bq( zn*1sB{$@eIMjSFd<4$sQ!J|}UK-c>J7PTRy2cGy?%=`9RP{|;R zn=$_S75#rx0S%}qQBKXL|2#^LDTu5L)81bnV{W4Lp6BFwBZ@0$V22p3<8VgcqPE0m zBC-|OCW|cw83-uq%`gJL{3#+m-?4`sxLsCj&EFj3`3t1L9nnUfD-j=n7dCk{gT$wn zAVh)@2Sb6szPdG)F6WMmY2B2??+tg(#>4&Z)c~ABN=iyA8(n-{8L+f;^z@GRzZ|3R zxhIsp!G81#fZ+j0 zRY;81+C9L1sVH;ON^J38189mkJPrZM{QA$sr-J&eeHu8*rlq9xvq182EG0#VkDD@d$}`OnT#rU$7`MVNLUOAmyM2o|5G?X{43_xRvQLNxvA(K z`g*Z}zl@`zIz9r_ymm?foC*m0gn`eX+3?xl*Ak*Zb*R@#xXi2bRNj1VIHd#qpP~gs z2T)sc{(nq;WmFtp(`^{sf@=sqxH|-QcbDK0+%32}!QBZK+}&M+ySrN;1kD{F&->lA z=GQbspRQB2%l4^m_PJUG)ej~<17iR13)h1@YesXU!?4j9Ts?>@CVK-)yx#7e^Acsk z2a4Ds{#(li8emQmebM(sd( zrI3j4hw7fABMdRym$6OADg7|?h=4*(15#R@`W4E0+_G7~{G4H>_t?Va zvuq&{1UnT?VZTq82Z}T4{}Zq}EC{scJ`K42OPyl)fmf(o7p^I|)ypuWW9*>U&fqIl z<`&EZjMfe)$d`@xjSJej;l~2hbwAl=OG{~PB*Qume~e46?EH34_Fe?ADcY&TT|c5y z9q+C}Q(wg%3CoVBP5&xpeq_|a(a{-`jI^|4yWi;Z+y?sp)jn8WnicxupM$xC4VDTd zcM3TkyfsH0^zhgS3}zA=e?yF#G0I>;PfThZUsZXE#X*J|0X(;=nj9zLvfX=&qxmV& zD8AaX*ki0Bl+2pz7N}Q+mQd(So{GEv%S(J9mS|q)OBh~m^)=dV3cjNYR{GEBd81W! z$Krfb7)dbGJkdU({~4IJUpVVm`bR-MH&yuOlK*fk1Vrp*YNN?+>mMz0_6d%JF1Y~K z=j@x?htTa9epj^U4l!!AHn8yB=#LPFRf`H2uc?SQp*XC^fYTZXkjhow4ugX>QD2%6 z(7z&?K29a(<{8maSrAi|+dhdbRN4h7b{&S2WS`(~7D|VI=^mq8GZXy9A>HiaN>EWQ zU67rf-Rw(B((5kUwgb(#xd3#}{tK2Mz?uVtdaZVd-?p_mmfuqWcdHPcqG9sOC`EYy zCBI*JXl3HBE$WHwl1ao`>UAI<=u~27f)A-0$}4a7wDA7+IJnf*OQ1yag+lxqJwv=8 ze3xw!huB->(Mh6LCMJ^)DdNUCsM)RrYcULTT!K=+QbK`2neU}Cy7^#nvT}dZf?Gr2 z1p_*qQ#t>ZnE`zH|MgfOlt~u_K<7NJ&J!|VUJB-*4%_DvVl@G>d$y) zhv%HUn5~hfH!cOMu78?z6il^U!nwlmG7t#@9jvO>(`oTs0dx|}^E0SU9=#}h%JOfs zClVm8kR{uMW|6)cH;+>jG<*^?Sx+ik{k(d8Se6_qmC;h&c zTib5I*rZtsBqP%U5m3HzGiCU6Sj(vB5?ASepf`cqK`pUhuq}pt=>Zhxjkr z69BN@iQ_V$0nN?)G4Tc6R9n|E5AR7{ESdpC$M*V=1^(hReduAt@#@!$A_i1TRE{$N zY4j8|a&I~@PPl>TP{z#lIt}S%X~{rKW{lFEvPc!VMX{hoh1ud3F5xf5<=z!Nkn`R>POjoZQdyUi<%Q_{k^?gg~ok zt1;unLCPDWI9310QarqAO&~D|uzx?I*PcYINh-s_F)6k*`*uBP@0T6ttL6Mj15MvJ zOcCz$vpTHat=0Q=8wr}B^dg=9|7#gYFum6OEPj6dZ#zB#E^(TYQD{fX`>(#E8(}2o zwZ{m0x}n%BMkY6v36$V8Ga#ACE@=X{m4eJo=A8AJ0JA7X+9?aQ?Px!j8vmkXa+>`?GeaX0t?D3>@CrQ89C@P$Nn@+{X# z6V{Xd(HvIzb7vI+iz@2Bx~@rmed$UiqR$tR_=NSxpwkyYo`PQwb%J)`niSey>bL&x zu?@tY5$C_sdLedWI4@jiLqY}(9^}-e@#)X$Ow^e z6Q6j0dSw_%;)9xC4;KIME}GvIl>s!y(d8w$Yndfzb>|+E{I@rRzMVvbDFQbTP@>xm zAR}|I;6MXi;zPscw1*IupB}ltvGKz*4r?a3U=ci8}R$E+d=Q3~pi2zg- z;pZ=9d9TSkCpxz((4vQn_ev`O1o^1{kSH*syxI_8`oDdnhpCOHz5Z10PX+`wsa~?a zJ|k#D`LWQ{8@&8Ip=tQ#${zY4!rb{yvZD+$yG0O1M}qh`k@f-s19iOa4Ki=GRP=!Sr4kXg=S{ zbL#8l#}~Ur0u>fg`Ji^O|MyU!$$JtA^)}w?YwD#e|NeI>j5yTfOS4W*+(Yue%=3aA zk{=WDCJL~fM-6N@P_a!H-Paeb+9Y_E^1MTS6v=Zj#de2ZPy^mS8> zkP8N?a(L6OCYC3iZuubifAT$Ob8c zk$CIsR~Z_wb?P;gbokBEx9M(xC{TLH+LYbc$gzRh4f`+Q;BQ2+ji_V08K>N=_b5+# zv6dxi&}>Si#fq4YoE)1rlR)5z52$0=PzfaJSc?k!C*urhP)Z0o@yc>6wluU|^Is2# z5DW?8JsyR_F!EA^4FOV_y2xu(x*^X0tSadtNx7-tW+LKdR!0KBe~aoI!FIgEntF~w5I3fHYI6{@&`ms~w=0i=*-!-#X`jTDQ*l)187)Q! zN0{^0>e_2`wZfxk4pd{6ci6(m@ympsG^D0c^2TOET8G~|?G}b77z^mrU48I=S7$Pj zw~L|4EMdg(yMVE*DZuJw**-6R|MM{@uZ#>dde3A39nssvKT^+=lg5eZ@XPn_Kq4Zd zt`STmbBiu#2TonkM)4DaD&?>s&jIp#mTcR1`&s{IgEWRb4Yi;D3gDR#vE(b}x{Z0j$~Gq9_ocfc z6W@ERMM-gp1&vg+pXPuITe#Psh6M|3SR3EIBfm&=iYwItNOzWPll! zsxl_Vmbwz;H@_I4YN|ncvoF)5Nbwpcr(y^M)_iZn3XpP%L`J%Fus%K~G6HN6N1i4)l^p6LuA;POEZAy~P#-4Zv%8+n(aNiLHIWildCRuSZvl6>{Y z{DfN@T5C*X<|B1ib73Ljc(QMIsw{5oDoD1V56F?)jger!7Et*gVAV%{#Uv!Y6N;ym z5V$RJCC%UO*3Mhuc8ENnlZ!{Y`iP3&NX}B3sx%w7{~5cbd@`fb@;wv_)CGIpA-p}J zq>P5RREU^#051VJ>Qr^KO|gwhjH>zwe8xP=q^pjb!?(_3{UNn&U~%Wr`)wR0rf97h z!Ij3Z@Nf=OG3W&T6jo)cDEbVy3ZlwHDC-~QcwhtZ_jB(@nHH9?t1T~WhvHO1O!ZT}Xf zn776nezCk*r^=HG#*2$ho+5hl9BlLtW;V05s;fN$Li!Od$#GAmryC_pO21RVp3=Oqm#E#|qQMEOm!Sfc z4b!xKrx-c5*|9slB z-{Q#((au~0?cu$$1@HPpMB&huyS=oVq<=9%ZLAMy`r%9pWOVT0YS4J!(vz+^BMp6e zd1pc3Hi~Bz!ZF;*x8$@_*6t#j)nVE}ISX_X(Vg7dR=nLrkMCT2+!sqyoc2lu%fe&` z-4Xfjwp-VbR=kDOZ2J1S*IL0=4vH~aXKO=MPLLd!Jovxn&IJ0BUXCuoooFx$-H`4w zB$_72>m(W$E9a`9?j=!(&@w6LG!T$0Kgxz~x^0dKEnH6t_x{+MJvJex9O*MJ=?|!l z2U4Ih(m4}E z21{p1lSS3vfF8VE{9C61?) zscka-i${=AHTGo=|_>p`o4*sUT0wc^^$5U6U zr`ZwaNh1bR5i@5@=i&U-enSznoN8|7g}}n(0E8W^v6|4Yvu=ZzCW-|#Q}&9nmBMqS zd+o|&t3w7dDq2!CF;!&(X|of7`3(C70)3wK7%6$pVu^KVIYTMN8iOaf?^qEL_i@Sd z6H$xD`u`WES&<;~ag2Kj2>G8fcYa2B4~;_ADkzx8BVt@Ec6@2u*~$7+Lhz}8WqHoG z!X=}WXre8S_%ArJB9u22%`8Bsbdv}bfmJx0$5mdbLttFLaw`@q{>nxB6(h)m=u=;h zQSQw8{YqeRgDs`~v}0hgnZcylB0lX7!Vz=#?|0gLsx>XLbJ-LwX>6YxLaEj?J(T#O z#2AOKXl4tvyNs-r_j+Y&_^h@uwvmn7I?Gg1%6d`jtm);M`NOXEI zdZOvez&hesTPLZiU%@4mWm3g3N2eVw+S8EHLTmVcpA&`9E{3>>P=aEY+yVF*jKAT6dRE1q6eTpuIJLaAuGnM^iiEE z2ZY%rB8Tom@_N}@mkBF0k(Ci~322nMHDN#{%n%MNb3z|84v-mik?BqzD|2|2{08cT z>M&(mT{bq-&RP@gyJmQKOtR4`r>fZdUt6~S_Nl-jjPc1k&*&3!kE42#i2yKDwhgtH>yW|RpT8z-ZN-;E?ZSi8l1?arDz z`HD-vS(9PdN|=S%9ra4C3xEx{5(-QGyS~;E%&xNk@NDDu8@%e5`d93YAtmh6jF`{} zCY0J~C3K6nlXvbyMmCI0FgWS@YJ2J=e0`ZHRa?6dj?7hOkvP9Yz%Z!JD1C8=KGX>Z zWQG|IF~~-8D3K&Re|_gJxStYxCB~U`-n?6{x*U)G_0Gd>s?=lt_Uf*Ej@V+ihBqk) z_s<97kkqhE8hX~owE<+S$YHSsnpi}{P5&_|Lx8jothOk?EYTuK0W&idmH~{6;R|f9 zrd9&sBf7Bqmn=tk$VT?H;^jP>1O;Ku-At3=*2ScUFG^%C>s6}4ES~yU&Eozh*NXFK z`Gljny%f_a9wX}?+3S+l+=C0$(cG8oxfF`Al4&tT)bFff#0EN(L&XKN@jB~OS4VTc zD%yTT(|cJ1f;2#g-PstSEIy;fVAs5~yhxb8>8$W0Otxoe;_V zHZU;sa+ge0X2W&jq<%Q|sGZh-J$wH`O#%irmVquO$1Ak*i?{+<>vf^R!6 zZ;4oP;CcHO2gnR5@Za^Gx}W6haHVWS0grynD`yH<83uR;v2I%Lz$!IIp-WG*+p(tBjz7 zp4TrgD*xD+jhBW|noa|K4)pad+?vVZFHfqd&CdC#}Wi9Xi9_AI!%Xj-; zp*A&Jp$+p8ebmTOr%?On$F*U9fW<~R)|goKjK0i+ z9U&6w^Ip!(c8`aI#(V_g_z%!)<6!6PTTXbVt1uEidx~U+=x!%gW%uGBy|>CGoT%>A zhr?gR%~D(p3k-2E)T~n_A`ppz5G-L@sr2x!-sD~Df_SqC!chLtj8v2o6VH*<4BnbV z-=(s@M8|!jG?uEeK4%Q@h_|FDM)Q8PX$D_QG;yVJV|M`lw^!}phDp}HZIq|LTyD+8 zS8Uu+BRNp&{0mh9-UshYErC=NX$~_>i-&fFUP8liw{gQR-#@~j>7J9#xXe?ON(@zq zZN#SdI2*tcUZGCHuG8(g_=7vj5 z%P^Tn4Tnh%)-I+KCYq!K4&@>6le3{faU6iYfoho3J`eHhCTMtc+?US>#7*vh+@ekW z2iUcq?F0t86fo4TCC->@cC$W4QilU?ab!g8V-UZdP<$@M%cHc|up=E+(F>Mg@;DaE zn5Mh5#DM!b$Kbc4s-)eDA0{iSqyCtot+?B>Dqv-SjGz!S69PxR7SZrejt>yLNZv6| z0?;cV*YabBqB7t>SVhAOQ4$!)R@($zAY1OSjf`?=+|oPKZlFyk?Ken+SB-iHAhJV} zqtELeW+h?L{`s;kgV8=iN1}S$mP4Sy1a{z<(jVztOm<|uN5eF0cjEP&+qu3})LJks zhb0tFM5=e_k%fgxDqx%iI!9`=QiHctDt~!s#;ww~L+OWSNBO!dn;Aaq)Xh_eEajrm zS`APs2zO=I*etk9l!B|LW3g6`ARc2iroyBsr{pxAg>jXNEBgtTegUnjm)c0oYSUMp z5*JN9lo~GKqL!_Cu+u*}(C_m>V$%Sjmvy=AB5^3GVejU>vKeo7^}}d0Qw}h0)szLt zfqeTcp+y2i;9pgZdN4e{@yhUB*gXY`blt)CZhqfUnAKJnPy$}0kz0VsHj?NIPi1$q%whU3_= zc0_a05oL(@2l|Z57RYNI3cU4_9Vmtc*_MZ*cMF;k=mn7IhEe08v40o`a~7;c>n+se z?xX4(qoN#^*lhm!)jbH*IP8cIpJoiojmQ1n|9!g-SF+7`;?W0573y9>K{gIg z-%6MQLq7M5?ajq$)A+))&J7h25)3?I1S30Zx$l4cS~YF|S3qRL#{1!e*YibQ>}VXh zZn&;8FK^L~ss5|H!9zRY&#>&S50YZV|2_}XUzXW;S`dZ~RKMN26PYNRw`M+(B;I0w zNG%KgpGW5RL51KLBOwB;e=F0Sf`BD`zWC&I@&{R) zh(7cMdu$J3io#gYj6G!^z1kd6FQi{E8x-r{!F;Ej59-Nvi+ly~QnYMrQ|DtC$2!)$ zxkLW^EI&AlIAN6cu6pz;;J=dcs*_bLaPil~NeF4?ggM0|&}6fC$(IloP<}D`kj^T@f!g13 z?P%Xji&d_m-lW6e`x*7(&I1&R{Qm$E?=$!nhtu!W0+Wm!IU6tbPHDK(ag`1aHoWuE zvM!NwX>;zGYvi^L7~JtkaCr~$CCqwf!29)PCi<2c%n&fIV>;>Vg~I#8>es&|UT5-e z-4+5jb41R?HWJ@9q7%Xh(4|+Yg${~DAt50fPwSVPPkTSTS>=m+1*mbY=M@(~*#!og zjX~x<3^Agc1@dPbCjsF5AmTS zJ6Z3Gqy~w>N@n3+18>$C+UxR`GbZz&H~(#5awm|O!Bg`Q@L6}=r{n3=Ch3bwX3(pZ zalnS05A{S6$^6l>8EUtu{b|tixL5-^8XmzN$A*E7m{-tFlHqA<8>6-FRTtR-vV~|{#>${w$-P`_bN}xd`IGo*4)`-|K~h7|EwPTljA+b3+8t9xie9azZ@1#b1|a3wo^zGX_`x zmxX2vZ73<0W`qyT>s~%B5@-mHIom&6KWyG|$!C**s~M;9<4dlxUZ6SXHbI8ok6Qlh z-DZ5zmU~G;nI=_v`FAml@&8bY0i^mUu2#$nRdx8ZOG5W_y;oOs%QENkPKb985c3oM z(0>~)i8guOr00|I$-6|M@kX&Zz`W)^Q$`SbQQ=J(-}!&0g@YObv0)Ye5RsxA*%lnN zaS$T}qpCq4XK!`T92CO80>W47F3eEG-`h9l7CzYu(_OXCC984|FRz!tfK{|983&*&RT@Z%X*8xx{uB9{9Q%o!p%=P-mquOSz z+@?aJRhP?Sa$qJtOFNGwQ|$o!ZvN8vpI>QRT{~=6jzPvJNK>*BUa%mHu&w6pmNcX zBT*42G+COi219xd!EzF$XaF6{zFU4A?j>!g&*DEO4q*`u1U_93^^KMdqqX-NCzD>r z&c_c1H@sTHA|WzdAmL}x-SY@%+ke>?0~iT_F^rmI{R39Fib~Yf=SMF_-0)*}GTWc6 z-n~m$^HPX`DnTKAa|GqGJW}0nunivP6H-@iwY;2ayX9U}hzjqnPhZZTVd5*p)4VjH z-2jW>v=h`=+n5x_sA!^5kj;QAP-8I8L)?K3Nv5!_Mc$7si2q4aZH`C*4;98r#%V$L zCe4GolKnu`EVH?q0(ZrEL3RH4x_ChK$NCXv4_BDfUiHxk%I`&ejkXWfl8X2ZCHW|I zCHutKOHo0?FzJ2N3hVjfl*p`k);&zpK-$HaZ1}HI5rM6hoT}tCr(Xn~<`R+@k-OG> zK7u{0s;{FHs8!l5wc*425m^!e^Lo-N>7C-xo%#X;LxfW&iH8#X9|f7E$>3SIi#`7^ zzj%%fVwj>7LI1AoGYSv{=Wc?+qP*o~>i+<^2x4b{n5;QQ#o|v}2Mo+{xbQJ+RNI{H zcQAF<5>DKjD# zlQ)ZCiB>kc(2FOr?;JP+^o-AGn)ATOEyE0G3ymP_2{=Wn2e0s!8j%AHRtjz^lLida zL|eZ{fQluC!*is!C>Db`yjD)?8s=g$C8X-ZS!AVHVRhYi>SD)y&ObgDMxFKH7Lz!Y z>AWkEFWpVhANwuDF-Q0NT9o$hVPgT)Kj^PeiTh_v5dhAL{nMzIqaR7-5vFCcT;oG~pp(zHimsobnRvnbmbl zTgYM5kaaaN@B$Wa{Pt4WGe?(#+xM%siq7S($ex?!`a!dM0m}rk9Vk+^o)=T{ zRkny!A7P(<(NH)wZqDDN$(ve>_*bAt8HDdKF)DI1Iw|Lp#kUjJVU3CYx{KRqrhID7 zpy&0gPpEt!!$3?;ho{d@Gj&OHi}=)HZFa9d0b4Uuj$LtLD;Y{{AI?U>$B|lTUwr~# zn%WXoM^m?)xcot6Z8Voy*q91qt!m$J2|;`~K$T--;g6KC<43$=5a1sAnEOk-vES7{ zWH6PdSnOvx5hDx7*cG-aJu&RBUiR&Bnjt#wl=Q4_@P4UsR@nV{@z|z?_em8*XP?os zSn|q*sUvRJ9(RgBY13|A1!kwcCW9lKQZX`jdC{b-Mm(>am^7-H#Ztk4`T{0caUssxZR8;1tOEH`a z)<|_cRE~(PP#<%zOm-VHSNUF%UQsnc4a(`PtH@#8(h_MdIydIbj^f7bvj8QPr?u7T zN1bVT84oI8gF;mY_AW2M-iRgx%b&>deZxu?e(M}Z+J=cnvgY;bmNxaAgWYB>dj@wY zWY+ROUTGyz?M!$3W>Obh6=@pW{M_g-2KElb)%l8(6W!5<`dd{+MM6#j^@nTRi`R(; z^Hu$mp^8qs?9;r*c3AKYUm!QX9rG9qYZ9@9UV<@}!!{zGr@m!APqzMCOic==?1zRj z4TPK-SQut$|6Fr_0O380pDjtza@Z_vO60sWt*v9y*eI zsXvV`YT0@Ho)dSv6>1SaIJ28w7180T1{1_YL+3-f7#a&!gN+5|&7ggYF|g~*=N2x% zCGAy}NM#Glo~K{GNNT+JHm|Oip%~Kr_-B_SSX&ovh}AGGA->pLH%xzTB`H0HIe%2pk%*;4+B*;H+pY37~)U1F#j`;ZO zI&roZ*;)2#uR?L1Q5h{PC9E+-jw_8-JT&nJo_9vxz3F$>&Epno^krE?n(rG*1*VSf zt^DzO?^62?IFogEXPjziY+dEs3=rq6N&_w~B4QJ?`L77Zg2T5GF%Do%t0#I&^9{fA zd}PHBaI|)?r|3@^OP^Y_6scN4q{s_4D)x^%2Cl|R0GG@!q8Zm=)mFS!tZD8w5*~u_ zzFx$e7>I_+G$%zv7hIrmT@8P_qkqvJy$zI*zg)Pk)m6fvhyg(oK)awb?}ukEx8PK( z<7~#^1f!KG8>2d1?~ZeV{}G;fNvp%22NEzYwy4Rfos zpvP#%T6o@Rw~AovlAYyWQHq{CH-Gs1W;%P%qJ^-%$Hi(3e_I6(XNZ=$u|gD)oG0{j zc4Dg%@9v8QOuabA9<7kT*@q#ZU}nS90bQR}E@Dg4qJWen^_Z5$BzFMjpR1^^s88na+ zh1$cd7yI(Q0Ak)iP<| z;3CHNwC6}D4r9`-h6oMBJA~ZBt6|Jy^ z=0D4Hb8r_$LF}S$uP#;}B$lc}iL;pLa5KuWR^huoc8}i*muSGvhIUraC)G_-F1 z%}IGDm?^%6mPnu%MF|YAbKK3Qsf1M(D?Tw_#DKzyEX05}8=7(-5aQj@y1tELOjSmp zH%nAZo;Jy|sK1|xr`dp;A`c3UWwL!MDvAP1wLe6NMH-s`z%zVo%`QgHGGdF|R5@A& zPIM?r3{!tlr-v%LHOpO@=Dcu3Q3%xvKl&&khLoVQV)CY&wZf4zxIFRsk z&fQuyqETXiqpSnwWDf8qUQ07VcA4fF%yK02dg_j&ya{#xpgGrIin1KV<~y<`X9c#Z zD+7D@gxfSY%cAQCBKl%jYq{!Jaq|SFAX*}O)|eIfKq-i(fLM!QZ;elIEpYsqm?(rTScfCDFW`)MURc_SJKL!Cz#Ky{KH1t2miF*f zO^;s|6h)7?vVjsfl5@Ien$m@IJnH6h0qzA6JJ(Z^a?EU5(dTZCIblJS;c|BH_IQ>xs*Y`4>v zgfJGH)C*~eGF0HFs)cYUDv~fzos)v$14Yf3jv;MA>VkV5OBP}&e-x8uzQa|c)IMw- zH5HL0YdKASZqmTD2}Vd}r)3R>><6q>-DP$)86VE zDYC7Xe=FYcgIKnJZk_V&spw!u`%hWbbpRQimQ6%a{f5F70XQtbt#~dXNS-n$Xy-1R zDB_fO+OWlD%sc~;lI-6O8!Qb6Z-~ageXUPv(0gc2+vJaG1)Dn(SOnsIM?OSn z!O0a1Cw$*U>n4qFRaADH>%_}(rRYn&7q?*OSIB23pm^4%)bNZ6r(9hMkG6`sE?>g_ z`eC)nmL=3tceT-WCbn9>g$T=k8dn<~!IeF~+UEB?vai4k(vamFMx&1@wCLPb#WjWe zI!_4(hd!R1`|WM_*b6&zwg9UzpCzI`=#>6xK1=Ch;RPHS0rMK|>*v%<%~qe<&CsxT z#U*qty}i{c_9G-{DcQ^`+^uK_fPtYj){<7}^CS^^J_cF9m2@+RfLwpS zhO|1kC2RpzbJf*l_UKR?$sy>P(1-~^;%pv`Xka~_03C!~pDbvYpcJD9I0pG#jD0>U zaT7_92wOQnqTf)MXqtkV&s8u0ef???uDGpu)*n^}Z9cKMjcB~& z*0^KjGz$T6Rgu<*hUlNdGvUddkL^CS(BYAxx1`i z!+ocY14r2yxYMT7a+;c-_4$Dghn`q~3roC-EK@)25HCAOZ!b}l_u_kzj{NL796wDdf}7XMLtv&>Ty|5F`5S}FJ?+|9=F8XH!$Bn} zgS|3lg#Gs9k6B(zA!^U7?uj z>_`rCB`ulqv|yLy+KS};GVn`hzMH&Zb|-`*EQG(n>eo6ZnIdfIZoTt;FS>n7d zRNPw@>jFAF;x9FMw%~K@4cYx@pm)(3n6>FwBoX&0P&#%MQ>~!xOss-nE6bVJNkE0- zuwI4ASgI!g6CJ*=|Jb*JECgz**SrVF(u9|Gzi;};jl$$~W8L*#IlteNvp7C5d7Y`9>aUF!TPNAKYtG=JFgx#zWB6VO?;{dBuJY3d^H1cqMTttwviInB2+iz(%k z<&+S;Q^O;E-k;o@ogZHJM%C(+2Bti#_<{K+yHNqrFiY{1guu_q;j7nrKa55g{~Sup zgdoz!Pg7FdP|mfNN3gVKgUU;f6(2n$isgs|n!~CXVc-o9$ky-!dmm8`R}L z^Fki0DgIXIfJUssjaNM3vO#=0%4QlfE<_$lNdh%>QK;w#iwtRmaSX%Ngy0s*Zt#2= z12MVAzW3EyN6>78t>W_(~hMd#kKBtRllA*$cM@Kn))FeEAR<10z*YVE4 z$a82>^DD8s%Xq2AiEUWAqi1=pb-+>Ps5?|VsnLEq*cqB9MstzBE0W)O*L+r15H2BR zh2fzO7o#W9R4?jL^keApGKxE_Qu_Sql3gR+c4u0MZaDh1BG*W)>#BAFrp(dveY62$ zG5|U-cW}g^OLP;%S?RaEoH61sCCOKT;rOabq;eT6MXRU`rsP5z#{4_NCkjlFflwb) zACKtrggz5c_(%BCNkcT!*I)x#QnhraGJ@_kx5w&xsYS$}EiG0twQ<^iw&rvt+zUzIbJagET5)>BUvyf_RFQ?>M}EDn`rIfduhvX%m9K6S6ePL> zg=046pMs0D=ROArD^{(x$2s~0xL^~eY{%7ri47#;AqXD%Ovv_CO}dvjICaXieCJGJ zxbCwlFm_cN$Kxy%1c6#P$cn>mtd??Fv&aGA|vS`B^9b;!+E$>)k&dLKGD(fhwrym%WM6!aqa#k6`77zLdUs1cI+}(f=xeQ$xK(rP*mar4of}~k5Q~@B z;yLDK9M_{G3p>ftc1-iQxTj=+tzw(@SF8KVP!VNp3fekb6l})EUSl&*DFyoiqM1U= zbb;R&1(uMIhyNp3iNYgu24v6^?;;P%^g(~9HI)*E#SA$nEOG8{*5uEtt0gWda^u7e zmKHEnmy%QNqOELjx!m|od-m0TB-i=An2*LI)=F!ma#2W;HyPeP#wKKoOX@M3R>Gkf zgR3+;L0BL&8B84sQJUq$ftUkL9V)4Wn7PtFVg+p=Oq5h7hFSV>SxJn^0>8wCPc96f z^|GUU6QdHmE1CbsG+?$(=6qt8rkxRJ6GK7kHLOf+b@)hxWodyQI9!!mt4Ll`D*J6s z6&5nBas_!8kEh9{!V{Jl3=Hd~?9nMtazW7H^)Xn;J30~AK2PBGXaCP!L;Gxlv7#V> zjqOLt-u#NP_a*zXqGzCzHzyNT+5&?0g2sG$F12 zt@&sM*XLvA1i`N}@I5W*sHL#?n>6C-RtJ`8R`KzYnzAHw^vR*Q#FA{rjs#qeiDT5D z7<+>W`K_c*+#cQp!Z47@fk@hGk{#obM$}DQjk^n0oZmq>Hs<*?HFfl{&JylAKyfmL z^cL0dL70NQcu)cmt7Y^|iPX1iJcUS|%54jscPJE{pD;kuBD=SMmo#HQsgoRDmLr{! z;Cu*G*P|OK*)id_*LfDU-g|2e#ca6?v{!<=(3NFenR3!lfGGJPzdq-ixTmUEnX>!a zXpy2Je+H@?#ltFEh9kGp5zoza{UzwwU&>VlGtU7Gm(I}@0t$+~{scq*AFgIR1JM%d5D z!@1Ok+n{t>R^v&@>wdzZaOELL&7XG1%n?$#d;Uxyo+A8{tac=#x9TAs!)i%d6}oj# zwhoElXIvTb;?e;CRqDXvoFoN>j=H)hm1}=7Zs2D3Ytw+*x4gNJ!r66OGGY$uPkO$( zJasKGiTsvbUagBu@yuEPAWg-MN<&T#-hA+ej!q*!7O(MCm1eEXTw8b{G63qb0LLcf z6kl0FE?cW|$i%-S$Pb@7uXhmcr&WWm0jyFAh<;GK1s8kC0$w2WnXxG&B=}lK<_mgr z9pigNzaw+zmW0KV3%$Uropv);vymmWMgq9SxARsAq5}SobJ7V3SX}i`M(rR z_V>Mw)E`=Olo~$K?q1N1C8avf##uy-$;_oK`3+?RYrA8fB%(2+muHg_?^y>QkI3pb zUGinQmWn|xdxP3mMvMJ$_3iR|ydXhp(*Xpl3i=b!nK;f`u%-*;Mxr>}`Y3LpkaOYt z?|$tc&Pw8p@hB2!72WGtliiL7jO64q(KJN5D#q(L&MR*C1WV@_PO0ysSQU*@Ses?e zg1JPd59uTZuPA?byx<5&#PBflJkVDm71qZ;xKG5Dl=U;ICMp`lss++W10}70WQOS3 z;crUr5|^FDt^L~g5`8;hj(YArN`W+VqX6VoRAg^fqACj=KEX|ImQaA#{%Sd9OZ7`6 zZVxSQx<~$yS19xL@ean^(rsw^iu?tG_%UIw&B~7zTMbj#(w}d-vo{(cBm^lWH6iF#LGS`=!E|n-AU(frK1$DcS zuJdSBL0YRF#R5H0mO6#ZSfovqE-JxSM_p4K&4tyVXNx!@Nt;EK*yT#Qv>XktO^{}F z7djC0!(*Uz!AJ%uN2VcLUmVyNV2Mg=q)mzv!@^~!5uJ41)yC$-fM;WogsV=kJ|exv zOg@Aj!rdKEoQ4|B0JW4pnmS~11c!mRNIcTU)befGc*s;`-?`%&zIUuP2Z_Tm)Y zrAgz&*>-SX^kT|LW3etZLS&WrjXmU}-35`8|M z2CLJPA699|!PVRTf%mxVQ;_ttwYH^LSi0TtJw&tPY4iF@Nc}g@-H3zXL+`{%lf;&7 z+bO62=oq-tRjJhMGi7~QGqf@D&abWe!#Mf+>i4G_SrqSLG_rx!3|Z&qf#kdda8;TK}38yh1EJbuWWpqdJYq%^D0YRb*Bx?oUZWsCY*B zf#|Sucq=QbkX;#pp>Ea%ukRR$_~a*&CnfbhKRM=-E=lj*N7-D31}YoAQVtjNu5wcZ z=CjIh`w@e?*g=AOrL^+1(W!aF z31z+~94-Eh6PMgcNMsA5diV>gV!jrrK@H9n{spV9nOP%a4l9zV*bz|yzMlgTp8!e-J=hbjIm#VK;4MT6ei2ctsHa=aEm2|Ct(PHq3^(%l`C(XH74Z6!D$i-5`)>?zgxW#}F`IAL zE2KuyLsh*L?p(8y9_Qw8TW`HG=5*Cdjm4@85F`@K@Zo|ws@tW~NId;ZryHO*RSYZg z**9=spyP!piR#NT!n*hBN%R0c@%$_>4e{Cf5W^Hr0$mJ!4Csj&CSA^-nQweuzdQRr zhZ55){P=sS;^9bWJCXz zOWTdzBkyzFzEpLFKn7b#ExRuy8Z1fZf+MEuI6I#d z`nr6epJFiK>c#M%IaSNNpfZSu1~9H-zq9A)wQK=Yh7u~Qt|UI$wB;L3RHR`sIKmFr zC*?7i?@Z`y+v+aP>k{--=>FRL6N-i#*7TPH8dQoY^c+7E#V?g*H$Uim<$u0Ft_uV= zzMM-7CAPFA%5WX%055NTA|KN$r=nAy6xpIo;Lfipxup@1h8t)xicNFfJQ+-yBZTrZ zTXz~nkhokA4;QZ@qn;!UXRVUR>f@XBlc7r)FPK9Sd19XXSR)t-_b)g8Eu13 zs5ljHXl|h9;?wZ^Al%Y4`^EY%YF)|jju*QQ{Ibx`&Fuh~KN;kdgilM}UEmHCNe>j; z>ntvqi52fkpm;PIYWSWmJcDbbu%<<)D56yG*3cb}W5$8%|U-=vat*Wy9 zRMD*Upy45VT;lOXRj?W&b-Uo0@jM64Qd+$*S(dk>oxCL=EbPLtQ75(mp>FE#3dlyq zlvws8ZAhAGo?jy?wQ$LA&>i*rqvR6}?lFy^4V|xy^*P_Ist z=(8vg#Eei;0p^Rb9d-WWiZD3BB##=cm6F1YQPT8;>BN7GESn1&2P;T;ysY6fJ8s(Y zD>YK0h6?usQGjGe?}y_4tP-@VGJ&~lo(#!dv>@{Jthiyg^2vY|lwLgAwf>#qQE%HX zS~>6LOeX|Nm7{|Fb<~0EyUg{uUS99pj8o{-P`y4CJlP?r@_sN>XZh)?$?2F8M*F;W zXKG2-*Y9u9kRxy`SItV$X*0JLtnuw+LZPzn;x6s|=%4*Q*Gcbp-khGYDQbteBM-I@ zxkixLZ2J4AuEvKl>ZE%XQ6uiZzjNs#ZvE!ftO#G>2)xZ|19u?M4ZiI*Co>^BhQ8qO zg7#|vCB<^CP5d!y9WQsA#2pWfkBir0U%zWfZd)?b;zZpbrjPT{B;);us&g7%FVk(;R8&YYF-`d5$;k z_tPppTB35pldZ`LZl~_OTn@g$4xx(2!z(=Q%G3TJ==k^xb>42)MKAwuz~XO4d9Uj| zch!aF&hW@c_qy!XQ7?GM<2!Ov&s04xyu5P7tr3O2nMf&{&bE?PSGmUy`9_jCNhX_~ z@Ne;Yb3ZC}2M^mj=?g!=Qf3oDbgHjsC2m8^v|3^*p zB1i5}I=n4*1-wBlwk-sRIVv_Tg`|?Z`EEss?AM_WiKhkxYT}o54-yYgk#tt7R3?6p z=n*~n$_$PRScC1o4LchW-luB_-iKNg8(v1}?LJi5`K`%>w3`IU!iJoCesr?`SKD>Q zHPL))1(70x6lu~_dM_e9pfu?akdAakKtcy05To>NXi_6ZIzd1r^d1ldL@-e)ood{2G zTgpjaV3~=*lofn%Dep>_(V$lBzb=kno8`PC05y1Cla-GWHCvHpq-^%N2DLP->A$%p zQ$-jD*yCX8zO_Ntdt|az=bx3)QcypIN~l8QXX&q6X`;QnBfvbo7ah`qHYjZY;%ol1 z&n~SbdA`7`1suH#_AdRJSeh#^FcV|RX5Fa0NYPv6mp)J0u*E+yN{{+l2RvpD0kv}@ z@f>^H3^b|G&l(W<+ay$$E8Hhh`kOapW`O-OmpaD&B?3%}-x|O_Y#TB+oE27q$%U-t z_xnf#qA*Ukp0*AR$b|{V1{FoHQ`5^~KRa0j?r@aFN}h)1lN`~D;Hl`y<+>IDwzH)1 z1ZB{9vEllh6<3qot??%KV6lZ?O5oZT%vy>uC%=kEdFN+h2+{8A z8{cIV>6FwcW;ybIKK_xh*W%hRQ^y)9%FLzHNUYoB)Mo6UphRfe6G*!!E(0>!g+!!z z|L!p|P5y?#t%#Wpa(|Lw8tE%Rm4B)p{P-s>JaGLM@4FrmEAuLWhBu&qHgGN?HKa>! zo>hJ8SHV7o`j+;{JflQ4K{Hs|W+rfyBNWt}(c-p3jf9WAVvJ*BeapcDnKJ}1M5w9Mm}*o;i}a_E-+2g8SwK_>ymil-MsE` zWXVG&_d`MelD@D@-7r8?DggPFJ=6s3Vrs(hw| z4e}k!1IUC)I+qY;kJg8h>!=^KA)Sa{66FBE)C0J;sb!eq1i9H!&~56C-w zhb{tfCCN_rIq)844*R=(`^S(_Rj}MSwF~M+0D$Wjd&y+}kS)T?I_gZ2uoKMr1b+I? zb&Uc7o$I=qafwsg++vb$IAcMz(F681MT5aH+%N36O}h_4a4$g*_SgMuQh zO4GA>ljePH>m9z8$AhJRw2?=~euBrEz%)yEsmTx*Lrh!I^hFBr)eMCYotZ}kd~qM{ zN?MQRD2GH>-rglf&N0RR2o=udi+$A5Y1WC7^9tk~Gg4?V~aOS<2VK>DE`VP(_ZM9RK44W{*cm}lF zVmU!pPGXN>D|O}GSaX<#*YxuneNjf7Zc?GU(}&7THXoibmOgFpMi1`uI{i_3$07;% zAfQcA+vRxZ_?l&Ich7BQo3&y5evgC(S)PeXGpQhvHtbxyu=lMlW0jyYWmj z7ea{@X?2|TSYSw6Qg^O@@B7v8xZR#YZerC0P3m240sGSL~9S)kLa zX~0SA6ceW8=pvV4|1Q5@2u62!H0gVv!}bh_IbkrcQ+J!-7ve5c4UE`s&X@o2B)LSS zNWHwg^yDfaC%UB8RPkxYsJ+>!+%X^B8|F2QF~7AEnXd;pjm2TN^bA!sgc`+G0+607 zNti#a(j9G(9EL~YK-#C%_XZ^n(7L%?7IAEY{6$__oj;Nla`#Jwsc`meQ3ni?jJU-+&H&BO+Ec3Xk#&{<7XiCRp^w+R_W2v%G0HZX7qG9+l%Kt3Dc~ zevilE&hOe?;vaRN^V}0D1+nY3@6B>O{Jv=;XLz5>9)VJB^^B0fIf(MS9+40^F*d@? zeY2!hmtIIw0@|}*1%Wdfu&_YB4EJX;ulncqmamuwUm?zTdq@lB_n zRf1WsQ@^?UOEuisSB2+$2c=atSbIE2ClN0Skn%Z1=Dne*A&TrO9NVUDN+3|aI*Sl@ zUaJX#;VU<|1J3?qcTup8oEiJ%GkTksS=s z=?lUv$x^h5xRxB1yFYACep&D!J1x=vDV3bVhLFdtLmDZ(o!Khl-98k8S?D71buI8nIoz#O;U1hQWwy^WM&&V ze>Vj;L^2;_&Mi;i5G7xfSl=-Sksu!e5irytqhj4`1>* z@fDZlp?4P2*4ZRBZ?bHcED)VJkP`zjK{GO#O$4S#@CDbBq-t3o+Y!+8Lj{Tn->>V- zKm4{=@>D*sb!<^ig*}_ED@x|~C`WlG|IU7IeL3>6R#)(=>wazlyJ+WB-sD*$mz9iH zGxAjue;%uhv{hJ(ft%$btFU)(4e#?GhTB#>GY}vu5UBits`(jYFTMu(sW?G~z@mDl z7#!vg>E#qj8E5-HB<@$m3tTGSXN`2_Yo{zgt%DST7{UT19$k*%P?%dAkGmuHd?9E$ zFo#u^UTY;N{$i!?=Df4bKF(f?J4At-kAa;yr1KPD5rLLcpJ=nZy;!6Fdd!mn)2nfk4unaY5gzk;TLg$`;X8nBxos{ZjqpZZ#{^v1E#cMgn<=?cSlaPm)dT zG~HT4UTM8sg zur#2tR?iqG6Im+S2_ZTVcHi7RDMd+44)M%EtEO#cgTZpZ4F$}CIM(sG^4~1;#ieG60-^INmZT9U*0>SHQOYryD2PuZK{wq5SDRtA!p4b8mkel0?K|ZSTrI5zM z(umkhseP4_L6X2?Ht#-4b~SEnRpLDZ-)lYA``ZoF6G2#Ix%XYxlXWb_rVA!Lm(4g_ zm3VVkWOwGX>eVV)Y3?0>`q&TAh2Y3kF57zj{WlW<;|qNsntu`Fh#V3>wEUTa=)*KF z^m&7NT?>&H2O18U7xVbgj)^)NLheA8vXMG8vcv@|>7Fj-&9E^w3S?uGbXU_I&qZ{W z5qacdxP!p197Cy0a&Fjap7a39EJXeG(%|)ZwJfDiq*BRdy2OciCB|t`w7&V>qZUt5 z4M}oVwMU-nI8V*NaP1Oqb4(Lco+}|;u_ZmMvGe%z+OrGbrz*7DRuO#?JuMSu;)i69 z4qjD)In`S_+ZeoK2&*`+ewS`IUK!W$cr~xDn~*-Q26IUe4Jg*hc`?8EYH!GqRxp!u z^fqecQAaa6A{uTy+poe2!&C9-RY}?D29Q9fpqUl}zXci=2cxAYg{Hegox@SH%9=uC zoC}uZnYSLm4Yj(z#qj%T$mkS16^q}L4w=xcV9fRR=BN$2!^K0^hIt+6s57Om-GQtW z(|3h`n%+QqxAu92{dwkh&^G{Fq^s|roiny-oB87@rD7CMvrVmIFUDDODhxY)&HXvz z^DRn>i`%+i*ZDHYQ=qP%rOAV6zf8m)P9Kdpl3U+-jIao~!TGb=$c3LC=IE`dmjyt_ZzRpN1Z)Q&`htxG6fjuG>8U zqDHeqs`S=2Qm$5Qw2ZOrSv&;PKq7WWWY!IuUA(3J(K1qqPxfA}y=RCTK5bv7TKAJX zQLE5-YIBJ5;AtxV){x*~-r!3-Tw|r~evpB?avOI3?>@Wm`LlxLWJ`91vVjn?!sA4m z>IFasa-FmovPo!|zA55w&)pBJjx$8{wa#fmW1OcuM=_$2Sk-D%5?2Ytszbv@hbb_e zxn-2ld7Zrx7**sb5p*;h{aFABB-RJ`W{rmJQh@KnRWC%LZ9XXiKSUo~WZwF`*LS1x zmg8XqClVnX%`MQ{43de03tNV@=M`f6(w=W?5Qz~OuoglCj3$^wc9WlMMCJIGa;c0I z$y2s}T%wBZ?^9W_$_-aXzZ!nhs)^j%jmxfMl=7VS0!CL=ByP8Dqv~mKt8we{07}68 z41DT9KI^7Rx!#M1E;ZRQT0Tx}U)=MJZTE#@WYriYE1HbQ^YQ{$zX>jb9Ys5@uHGN| z6Xq_AAIx6!>#{8#ZfzkJvx(jH42(hG&-T(~~+8&i?o?)_e@VjDCLSDs2nsm!bK752V zvwZtUjwgF$e+kzd%9MX*e?Y+&OG7asJ?Es}Rz>U#v&~qTYM^Ki@}g)eVH{8Wf~ZS_ z5x9)z`@5J3p^_(!LdJjk^)eeanCQ+W&+t7R2H+UYL%FA)cBGeNZXeG&y_r8|xPwW0 zv)3-|1{m2rz7d!hUVjE&;Vbp%O9-T*g9**|&NJ6}5* z{bu^Oz@o)&K+sp-*D`U{${=|l<(gkFG$NIv&KZ|&OL;THqFQ~~IutaVTm)ybYiw1j{BpnKC zfWtV(O3&qZliR}QVmJJFJucO0K4zy&&$rmx2m8&0(Br>^v44~J75%C5>#Re(mTaHI zD`>83C~gEueW7}OhRvu9DCSq}eF$J6&4}{;KAd_i&YYjSw-|cQmFZ`K&JGeSH*2L9(GEHZaLQJFW3`lzSk4u^=x*M}W&yX9P?UgVNY7TslNLe_;=}$xQX&(L- zP72m`M+2|$qk$3YPyVJYe3xFfvUl_rE~6~!-DY0u2O#itlE0P<<%RAlOb`z742f9E z3#a2je`SK>rR`(*9#yG{W?L)UnVqcPP$6H^J}p(h@t8O><=hFstvP>?$Hx~Z@GLE; z&4U-$o!i&nZo$rGjy>W+m&Jd&IVmbA6ATqhk|acweWOlYKau^q6ZlT)qeRiOQhrfq z(;3Qp@uS6!eB!K>|6$YfNy8iXi}=NqUQUTkKe@UXZ^4mh7*lv_M^gtiSxDX`l3DqR z?OleIHtHqX2nMm&xe8q(vxz+A7&8>(Zmc2P7y}x!JN=^P|A2jwi4h;DEk2#Mfi}c`ooiYC3ZxYeTTr zEbfOQ0Ccd~SWttK3fZHB2e{MOmdK+0wZ!E?z^-d>ahBUV0eK2t4>C31buPtd+z)CV zUW$=F6Ha3J&a| zhwXCuZ=zv!x|UI_+|^dI0r_?e75&Pw!=ljwXhZ2yK|{{g69VYzJN~cd@$YIti(QGJ z&NZ7cw)O8ODgWz{E$>fMy*AG=&j=jh{6oJAa8D*LJ*V3`Ks zO{sqkHcHgc*~@YXUqiz*?_Xa~%GSakL#fpG5CA((j;Z^thVq?1ad4eSwfX6Zkb$rh z!1MOY_3pXVdcRkLv!}<&H4k)SH`5Q1A^=kq8T|1P>&fHeAABflgqD)oQg@WusV z{d12fG7IT|=!<{vNs92rONo6Jwz@ZTbcIb#o)@zb5fMLs{da z;}~Z8mYrgjsO^!TS2@`E$@xR4yVViXg~Ir3)QVo~--%g=8Tah+Y)dz70(+)CYi`9A zx;rAKH7*)jHJhVNY0=66IRvF@R5-%xLcp&l?kFRxFQi{0U8 zn0@o&b^g=DP+d|I0kFTn@43yZy|>;K6dMym%lw~iSfzY-4E;CDI- z&RhSd_>}bE|5+)~;Qg@%rT^bfXF+An%~@zE2sr(>N<)HfUzQFItXiGY8Q5uPXt=np zCvs)qqJ96jvp!?-b}1%~@c#x%=9{8Q{J(*Y#Q%K?MCF32{&&!7=C{w%Q<3(6*Ya}M zm@fXTcS%c2v#Z4Y*E`AW_MG9|w%&j0jN%b>-WrL&zdo(6qicV93_5LpTJI(IdpPUO z^lk82ToQS8{vP{BQ$= zGStQ$w}4l-QWu^Goky3(udYi)#)|n*huz;QO%X z8Sse!e~3!^(e1)PEo+w=9y%0Fn@8&M^F^#e^|j2){hJx`G0_ ze{e8zt5XVPZ2 zm}&3Tlc~A6bx#4fE{_v~q910&Yuef$g!)82sD0> zMf=`7^@3Yf%CEAn&dte*RhQwfyv9|gRCz|2U-ee~gnL4B`Ig|B^&HU+i-=Eap2gYt(=Ifs+L-6O+uDb+FD;@Nw<7!@Qe zQ91wX-?6Dr$Je5o%vZBQP@ej>wqGq2uSEM=uHa8+*C`f{ZEgYI35y6)MfJkA%tmiF!rQTBVTXt@C?auJuods@Vj|xDm^XLw^tvA+#-a!qvd)te zXMR-N-t&0?N4jP{f$Z*qwn?moCIby+b@1(vqeLjUlbn4fz)jX=s*{!YY*_W0TrA7P zl4rqgsz$degi!Tc_YS0kwOclbTW+{yfrp>t4-{0BOY4}TeGg_S;%d2fj}N!3SG%A} zRH;KTtLOeKlIFx0jJNb$xKxdSdnKokp73&Y7iI4DH))1v#9b8TM<$!uB}2zg0BFDN zH=fi|_$a}3T9u0mj?nlKP-ZX{{~gnLUEpl1$={(4>(bRYwh@tQykU&a!qU=IiuED1 zYY`k|2R$gNm*_NvhUKmLWZ{i=4=676H3 zn-{8GxUN4Meex+AV~D365Y1nUjkbGZ$6T7dAB@b>oU70|UitD#RoV2!0ols?nK!!kB}SDA_CK)~e6Y4;OEv>7Y(W%t09B3NkWxx4W70 zZHCJhl;`^Z%)uuU&wpW@7m7Crz;dU!`U(6;=(0o zo7XnmdZt8LPA)BMa@76V0($-|OgUKxj}pS;R3+`(TQm#_I$DaYvtOHnEWdK9=O`Bz z7r7QvZ3<%@Jdwbs3*kD^dyf2|J)=vZQF6Z~uuQw!Y&eDEZG@45g{`fzxvR62BZoUf z04Ay1l_E5>>)x*>h1_nn&F|?kR&k`J7&fSDdy!{I2DHTI7FAU>cmmDd8@6V}5;bk> z)>lE)ip_E~8lrt^gisA%fkZK}@Vo6UE(&4U;uz?!{_f?LXiQNw0V)^Ol>)f2g$2&P zk8iyBkSuBCzsMs~RL2I_)zwxCaibb0eI!?reI@R=$V;V;Kc7G zFRw0Dz*`s}S5{SJ!_Ut=jT_KC!eD29MG^TL{d)EYqtsUBuOxV|p^lFJ_r77&Oji=EcOSI3} zZXv`%R`!r2ND?`j)W=yhBt;F6=+!I7{fXR0ZfMID^gX%I$$KwUimmbAWm4A##CIDV zI&BuMC4nsNZ%o#2ypEI8v#m4n8qXx`9`MCX!mI!8-+7&#-cWKqhTWHq<;M?INW=|p z4r^NkMD4T94o&lF7sSS5*9I#1Gi3$vSAiR-x=}3Y=V=(WTY^$7W3qfo!bMgoxzEIu zP#S$v@~W28Ffo>Pl37gf!h_kq-ZyZfh2(O4jflev>KHwuO#X8~XY_4k`*R?AV-S^@(zX9R_2GlF=Puo@oE$omrR= z5Pl)$Zz*?Jt;M}n3J{!|uW+kipdlQNr@iOn%1$fv!5GF zgT5;$-?r}{wPVlj9P8j6Q#zBC5?ozzPBoLkCwGoo)oATbZwlAVFawlFJygJzMKqe& zaH}r6#op!XsWs|mj_>tGn4LFxvF5IOthLJ7588a9%+AF~k;BJN^k)?jU#|dHnZidIN?k{kFoO%D}sKmOwEuY6>3@bOdOAPtM zDs+3lb=Mriyt>D)$FFWzFRFCw3K$u~)0Ca>A4bOFWUV?_-sQhGg{wUh8u@1u#Nnva z&LM?!kgbrw7X5fQj~vhjA8s%f4enmLCcdfu-b}gAtjUB2btFXsTCkY(7+RgqnEj%( zV8t|779UTkNr4OunYKUMhF%LPUMWHl!InoJpj6`3#O9^v>|$Hki(B`G(UNd)^VMIa zhqH)*&c#i`cC z&XU1N)&=7;Y~lvbDQja36UV`OG z*G&&t<1Je`e+(=U<^%lhWxaj#9}PxZJk}qi=01qtmYSYo1=o_J zoX-O^=@6(Hy2NYh>rb=2qAjy!rb>Ru#7S@cBLoRg6I;%O*iIR-o>03l0up~st?Mk*cT z20SeC)z7{Wee1uj!|9c)L+F+BKDZ>~?U`o(KSvT4mIs={a(ym-GG8!)Tn>GC3+g9C z5A7sA9ML{1lfA`3QlP>cibA#j<9M)*a;kRSUbBr9#qaJHA}1takw_?`iRZ&%cWOGY z)_5#0BNEb974|gC#b{V9)PCDu_X1u~u|*&`&O8^1HFpR7BTSA=TnIt>YkW`by%Y{> zxmTzZCT;~~1{yF1Fk~}35#Sb}RepL059lpYOJYkxr0PaEF5J_zv+)$QBW7Pa?C!ij zUH*2Bb>8}fQ|FRENsa1nfI@#}V7u}OIhmwLJUMx!VGN8tNd&#;ZBiN`O**rz-r7mVPAA=g3cU@uq58&J@qi75uhhuzylqU0~ z#5xV@uuwfdQV6m0vub8f6w+J{tm2&i7Huoh@T29KBxqFV!$xftW%Y zx0mepS(vk}7>94N#~C!Cup`#<=#mPKRd=6jP=gp&OhLV_Qc+_$$m5FwmCsbKth9Pq@!F$65+THHF@zj!)(hT4X9y!E3DJ*$aBzhobNsiQ3 z1%*42(akfcb22&CI^{Px=frm~!1toZrHq?(nejg%q1KlQ>?FHQ&|0*drWEW(YXQH8 zo(7-M{4QL!@G5R5L5G{|WfIEaX(40)Tp)7L;!KYDTt@lHwK-i}d!@U^lLu%d$bNlG zRMN4@jiR46K3&#?)FJx1V$&%NA&o`XVzK(VG||xOUE!~+B83cXO|8Fn%e}9D?7J_J z7s;W@H_Qm07wfoIA8ZX5m7Kfd?!P+05VEn~XR)-9zE8rcJ=h7H3P3#{Fb>83^<`Z& zuaKMb&nNZKq+PaxBFI27WMDKzRw(x47)x3HPL!ru|K4EVk}c!lqJB2N20rS18JwxR z6j{S;8Li?)8$&+v`zlqSjv{;FyqL zM%9}WE&g#{7>si9%S0(z*-*8dcI){jneOn;(0cQUC9Z0jxF(G{cdvtCmP;{P^J+>i z6=jA<0bL?aRz^(c=07(PrQ4parA4t^#R_;qxV|oL#w4E_T=P3k5iksTM5ecowg!ix zyW0vpkM}Z?hKPB^g|wSeJ-gbty!d|s98bFTn+`*=J98^KgUo~yxf+Ze+`?7LP~+IH5go4jUWU(t`K z{*AW=F2iyflBcSA*1Ll3r4?&Uc^ZYv-WpBQgD6HsE2#2p7D^mr3GM0fB7rc{9QbYH zQ}i66%rZGGCH`dHpQ9d5=Z2fCKSColFM*}q`tAO53#<5)g+uxM(;G&_8u>!yMK<1x z$0nBnHrco^pV(EQRpm?h>%dO5t9}E%*U@COlb*-YOg6=LdH0>SEm@{O)`0w&&nrZ` zI4+nk@mMfm9KbNR+ma4sZGUY2;01TrMt=$by@?sATgw6`4sCZX4o+2kz;W*HfQMwg z_yJR&)^cc!J$tRpaNu8Uk-;(bcpv9Zk03Ig5S4hGLK|pZ-gKNE zxsskepR*i2;J@=l?3|D7g~!|Y0)xiy@*<>M-2;m_GV-M=YN@~#mI1}DUipqtu_4` zNOZ>`Cr87rYUN{)D0WLomtU+w7LH{o9mo<3_UcI<`jokSg{Nmt9N8x(?Mng>O!4;m zc)y)q6*Kx(Tc=G%8wO-IDOwZG$vmW6dOC^*li{z9-5xSx@z z{mf^F>G-Rwq3DA`*vl1h>_g^%HPH5Sb97pbNtc{Si;zy1N6k@cg}LTQrkWNU#jp4r=!^+oAV!jiNH zbpCi(>EC*6y7?;E>8%2g#RjQ&1B81U1*p@(ZP5ooM~IxMee-KS^y0N3pl;{VZwt*q zT`77mL~zhFl`>e1k7#m)Q3VTJu5DyhMMl1#ccN*J3;1dW4(4WkB)|ijN+t1U3HZ>c z@SIc@!^VjYVF}mPoCX4-9X(sAj(j#unArH6{$`XmL8%)C#q@cAq(_A=k)+t3vqjR< z$wXYe>k5k(NW!W;9A6I?2;jKRAvHik_wm?Ew1gN^q}ckl*jL0x7#4P*pNiTcN@P98 z*ZuajX)znHv%n+KK0>t=Zy)%*HOz9I!4Uu&Y{E$REV*Z+8Oe{xJ(W^TbV7-0w6>^S zv-i94<4y-7ProlTGhW@8PmPJ2b@FUME2x3CVJ~OnjoUi|M9@6mIah^8=e7bQeu*R( zL+t27&gf5KKe2%wCmC;^Qj}Y*A2=>e>_r<@OhDblGE9@aLqbd+cRyNW8@KRI4&U!_ z5e98E0IvSKOK4J!rRI;Myp19<09{w*!3A!UX%+k4PET#3-$;sVOhfete8-vKE4rCO z9-CHrrfzH?C9GJj3B$v7nX|5^+c_o}0KUvpy6m*nYM-lbW3E? zXH^zAF_5i=SrGie<^aFq{0JZ zZne%28XA(WwdpVwb6v82!mt%@Ui)5;y!Wg04`#Q-9tf-9->;fq@kfH8yl+`n(& zGe(*;b;~%AfRePd+otUn36i~uf&s|M=R=}3$=OkaKtijQi~ik2CW8j2!9H)b0J)6q z1NTwy8pjt|t;v}cXYq+(tk%$=Ywp@0 zXi;#St?tj_jO`Bv;iN7HSVlJ8ID8=(d`$aJsUw*Zy3w(AY9#23MRX3XQ?@HxPV>A| zNsl9HA)ascOtKXe6sGWG-sahzA_BVOlwHhBiKpzs0q|LF@<15vybM2{6;!VkhSo&MUS*pb}HGI5rJIxoQJ^xYY0IYm4X(7+iXSF@|{)6lVyA+eQaMJ$L!6s27|8Uhh+QOPAPl$~(Nj=1h z_p>QQ8GGvue+~PCjP4D0q@zBD!4Z$eghSs$uYS5mt9e>bMnXVl)c0!-;vU_$C(lLS z91Q5&8)kuDJ#J_38&(qD)xPWZmZIAmvNU&)^}#Yz%S?8#`spaI7m_lWFB@NQ`=he* zl|L9l8t@E9%jWfTj*|g*3nxSdI!PJmldhEP2@b4qlI7V8D;Zwja>a^#fr_fi^fg#s zZ=|wzpN_uxIOXL=fev6zr#igjninY?A**qA0`B_?cNzjTF-*<>wXo=97&e%&fRPa~ zhnZdI#{=&8!k0OT?6A_DH`ysf`Ai-(9%~@VRM5}O&ELaiOg+`z=hz}Ri-|H}Oe(_k zwFfL>k7|^t2Y!G2N6-dctHurv4u1Z8qZ%l_LzVHho)}5kFg^Nq#`BPfcUKUeDG?Qr zYyY)fRV-oppkUw?_()GD?5CfM0*FIslE}g#TI*mWr&{xP{&2OKA5btWOJ2YI7}e*?=Yb za<-VDYkK+xn2ul8h=)Cg87cJ2Zxb?}#ZIeik}7dn6dM~`9v3&+kAa`khhczzdbp7g z7p525%YO25DpR46BZR5~PNvp}2r+2{ArRtF0Zi=`SZUl#lfl1Ji%u+NZwqVMo8p=8 zePvu`C>C0Rh5$s&W~gId_ExwpZ}b)xPt|1%6~~hi@VOGJre+qySM>2s1rBy4$cM>R zofd89k96;+A@4CKZr8&3n^*jH2+i3iD1z9-+d#c7JkJ_g*x9oh?iw3&UV|aT?|NrU zfota2`52Y4`7Y;qtc@tYs$#Z7HT5Mam@5pj4=Lw@1!;1bbA%c{8tN#C<6N^#^9uPs z?SBQ55@J2%H;>qkWpHL8eX~VXCKbip+Y7o9ynnBKHRCivBSN@l2IH0a`oj*A%(1Dq zM$XeBneibfoTjQN7fc7^oL54OBzVrHeK6EZP{8xN0im)Wj8sYzRzK}KscsMTw=>I( zFP6H8d2twV7hqcDaS5Ye+;+K-_Z%LrBZcXvDE1`R8qvw5yuJeMHCY_Zg zLKt35?9kft8o0B1fLrSHsupRjX4^!m%}+YN>b2evftXi>uj|&o?3XeshVvUtz;%4k zwd$x=^~~7(Au|PHQ8bjQbtbYOJj1|9?Oen6C%ceSO z({M&yMAJk&FSns>--6?Vz$)F2k6{9pzo-&j%Pba?%Ojy)|J4FaFj*Q&d+CdmG`!Y0 zMGpMcg)P5upRc|Yjw08*H0i*r=Bcy$Kyoc`YTiSv``eU=y*?E8x+5Nb@?xXfeqW_` zv0i?xG!0X9$lhV^DdQ8IjL5Ek>r>@~?z*5yY0-fA(*R2=2EvK~^yuR!Y!;1>l!@EM zjlQ2zDa#V>wXO)28(+qq7Ul0ZKjow25WX-1UH9CH>(QFy)@l})AX%X z@W5E(_Ph<~|55jcbyj0#;)^lQ;}^)BKA7DvKJtx>+1zl#|1EBrVMLzU`$1bdwsu9r+d2q{-9gh9>Dx8IuC>r3iNO`cyKhjUu6+h)1H! zrMSA+$6YAvwMA#VI9h>aN}+H&shKJ`{GPE)W$B0nv~Tm)@uwN2?j#`sdPsXEV$;5epxi=s5)IBQnWOHY+lbDb&vq>We zU-9wMv+_;H(mH?VHyQU#^gW1|W;y4#Yi%%Y*`p^C z4iyNEIh1XX`J1Ndff-KJ!B5Ux2mkm`f`Wp1C?P|NvQYiTq1KUs%}OtNou;7jA7Bzj zZ>tOrg|UMBJ?DB!^>bN*hd6l5<>N6X&KXq{Gr6ulmfS6Np^mV$*5U@iUh5OUH$dFK zjZr3Sv1GF`xqx^Mh!U}=zA?#IlW`-DvY%-@*edMgtV@9b`Hv!cEV zV*V2&nLF+4^wu-M?8|eBNbtTAiWSxj1bs$7%DiR$kh+4P?46@cUbcw;YL2h>P?zvKe)j94_QVhNk9WJP4kWl9JNL zhkpC^Pr(NNTTp!gL!@0CI$B`l3p$}HDp;k<{}hLub7$dz`TzX4@qv=!;{T-n!RwKr zO?W}E^CLEpm*-iM>Heqk<0Iv4d)t6T5WZdx*y`g)(BXToRp~PhQW--iZ`Y!9TXu$lTRlhWJ zU^04fRz_iZWM%S}3F2EyyVE-1o30YhIxTeY^+&2tm!I{C#fu9KmzjiAdunpEkh6K=WB)6JYj~-ymjKv9FRo z>=P6)m%+gcVtsC;qL|!We-;qyo20r1{g=4Flmy9S>;|YnARf@b)D~TiaMR3-is#6F zedozbBn$alUGDm$nBJgCpGt_B+55sNgiwWRz%4Lu1r|VVxrRrI0f6zLO~*$1C)kCM6h`L{HrcW1ri2w(Dt zNl}wctpZDalKQWOSBKUMe1dlKJ2arj^chtOH*;2~l+sHxF1v9~^(;9m%qZ!bQW+iY zZp!(D;J+rqw^Fc@xG>;EaE*IwV&1}8*Zx@TfP3Fwr>c=jMBv&)DE~rg1R*0FGnsLw zaZHzXF~buT=(Rpfo><;s@z|+8DZ{e-gEzl2HkGP`C*FOpH>AVDOf=j8o2_+ZqSP|L zY%=W>a`DxyAWR`Xzhr^c-Te~r6dla**!LU!t;npd9*Qc;)_c*=x^vkl`5r`N6VlU5 z8^vBbi{um)O~Pv7;qPRp)@jSsA^wFpdvS}oLgfmE`7Ni7bhcjVFr1^fCey&g(q;2F z4F?qt%fziB7**0WzB}aD6ze(G{Ed5j4ePI)&qmfg?lZ~daA|SD!2yUw8KCfXc2aHs zW&yJ+nGb4>15+>~tQihrHq=OfT|8Ie?ztQq9H18_b<^{qd}C6hgXgz%A^5m}<^vDK zH4_nFC`G#?JPUA?N6Fp!J27=TS*hS$tmlZKH)>Sxfz8VD7OYF7q3M0DCaYG4sL1fb zIB1gf|EW(k5)%==h}d0*!hWF+yC9!%p=FJ*xn5Y~k1V_1i9ZGMm#<) zcYo%ujHRg_~_gKD@?;V*g%6I4UhhSFNUnq_KF8|<{^DyQ6gyhWe z;x-kIi$7#D#>L1JPnkmc zR`~J^@BH_nq6=EBPt&S3Y4f(5HZ&@aN}c@yi>0WT2Sh5WU!9B9pWeEBB4Ydpe^6R` zwxt@WVm}*u1dR`RI(7ZO*0*=D8$4t90R+aI7B2xtQ^_6db)CZsAL zlSiX!#8XmY*uEA0Kpef=ynN6bRU)N}u*uY<8^*@GM29g?rt9W@!^PCGbXcOg1)fFKD^Z#)0(m^2E&CVh1@0D-JRmhb!!N19 z7Y(&^E}ibu<>Q6*qUv=U7r#VsU3Fn5-}%jPXo^CIKCcj>N%R?E*?8xKC912F6Ig4@ zBEcPC*}TvRbeu<74!>H^$3FCHDMDHkY-d#Tl~667nliAL@$0o`$+DFRU&t6g%UYI; zerw9cUhi-GzqrOTiUQjZPu_I6_ti^ZZZO0LTt2{jK4a_ea;=Q~F_-N#t}!zcZ}`a~ zQ!AV`US0$t#$5)r%o;>JbWGiU6>=-0W?h;LU13uxTQ)IO$J&K?Cyd_U&8lUr#Vz%} zFmZQCcSpz%+OUMCDN`bJ{Cd#Ce9iF!iP z0@AnXOKLJL%x39E>7uAN(3rD{1f$VeD*W``t9`FAG&AqLHzO{_|Y`vQ0_y$ z%=AIq#^d?VK=;o$S8E*c(a3A8O86Z3#^{!6$vg>kHk0ie$g1FoZ-EnoU~l8VREGSQ zI(hBxJ%2aQB$k*C|&!~ZfzI8wfDwZm_8igmS>H7HWU5Ddpw9;I* zQiNLnmjK~N%5kAa6~A2nZ`UgJA{DF*JXh%gzAM?fQVV7$Q@DjdBGBKGEE^&ht$^1A zmLR84&59FDzlS9FvxT=3t4655gmv(cIy?KNM6uXmL8nt&ePpwcCjq3GSgLmkQ1Fc$ z)9u5*{v>ALNUJr3F1^bX51$X2_<94VHI7r)&)YUztEk$+I4u1=n^*JW6xu@3IA-9B zedF;dt$w>#feTI}rX~fO51Rh1+S(e#03t^-KamMr4>({UT)1BWfYjQ%OT|nbb|8BHBeyju9c3}sHHHZsXT)Tgi)d5 zci_Et+|t!r^e+T6@QqOYl?^gRJ{qdtb?~ocP%y`wt`N50zaI}t=RHpNQcpjD@vVue zcE{;B5QgynYY8AM(7;`Av`M8>x75!}y`Du3)FdnuO>zWV(zO&J_EMQVqRBrWYdZ*D znV^GP|3Cs*fKldGf680+L)SwtQoj7X%##aBIoa`-j6na%*X+)9E8@`}Dpm^lCW5th z@?77Z2CNTMH*u{xcut1qW6M3}s{M8wWvPGl&eP8uZF?0WhNb<5+o~le4Z&DUPH}o` zZSv~#$+vSe;f8_K84K+S3%!EmcVCZ;_?c!ac-}i3KkfMor~9q**r2BE%rsUt0}MYa zU)_-s+rX;S@xmFErK?`R3G2#-V1cKb_|L~bvWM!9bQ!>yo?b4jzkx)*DWbo*OPT@d zam*UFNCbw5f_qVw9c;zRkPD7I;}$id9Rz>NKFtCc;T+R)k-Hq6ybtuC1x@NHDYgPS zOUECZfDs9?dlF z6#)6;0uLOIu|Iy3+lTwR76(~H8OJbAnw_yVj_h)dsi@{cH3%Ei_UJZiyr%>R_WYs( zt(uDHPhrb`)kvZ~M@NT8n%(g}POWqTbR+ije^w8?jzKg1NG!HdreF{CD8oepQbGq^ z?E;;KWSWHqZ;jH zsgk~(suVK!v8fDzc}Gx|31;RRba7o2-m$v~yEYO6cUAS}QN)5J)3E+^)n(+L@zxlt zwqVhcJ|~ad^RWlgaOc+K7gz3+0jOsvUm>sEJ+dG)y4f66vo|+PZ4;&Aj*r-v!aMW) z5bevOq%YYg5#SM&yDw=gA&(8n9jA4H2aZR@=Z;B_G$mUJg(faKZjvjr(2i;4>oSzY zLEn+yfMw@W8H7)_jgv%-&q3W%1y1Qg4CNL*J;oUpYwo~O+?PT6nv5m*rAa{zJif3YPl2t4u;|)LFZJ@eMayfEM?y5B~8LY2NbtyhoQtfVx-QEr*E-ms> z4f7y*wj4&bYzF3*Onzx6^Li1FvFsKH?fS~&=-fdiKbupZ)tR=r5S(i-EBc_?T9>I| zb*SQ`E4q?#(1NCSy?oAZt7ak%`PybC;ag2f4$S@%7_*l((ee*y4II8Gl;l-sSva%S zw_0B4p_GDrm1i$<3dCEAXAfKdvId4)HF|y#OCX~r)9-Qq4TZo;gJ{2#+6)Y%Dc3hq z&3%pQM%iWk-kI?r&%CK%t~6K$;}$x0Ua2Iq^3I~_&loS~el1#$TTTnq=-ZMb8}N$! z&C1)l8G)SrW8Ye70k5qgxBq=K3+qyNEdu4TB|NL^d zew8T05${&)4Da%$g15?*A0m0u3qcXY`5lu>F5G2!5^tzw6%6RB{8sx^X5Y`RXm9IZ zlwFB5NJD00`BU<@K9Oe@`@5>jnyQwP9VZE~``3Kfz6Pqw1|u14=BmBlx3F3q+bjn{ z%0_M%=?@Tpoq2_Wh7OaGlJa}PQ|yXlja*paT=^U&4B)Dqv0m~HdsBx`8|Y{q5ixd8 zVZYV^5fBw4mOI0D(Db|57#IMt`3~?m?lpvWu$eE6X=?U-SwD5L%IO)PtkETJp6;q2 z`+l!sFgJq)7^>Cl77sILwe8N|j=z1;f7%~T?nbYO34EKR>53R5svRlr*Zp0Gc~$iN z`;K`Jx=WgXoUM+U=vTaoKVbuQMJnjNZj~pSLnZa@GCurA`Occ3IdU&`;42@Tuct92@Q6a2+og{mx`Q|GkH>Jn3?u>;)Tk{D8 zc#kuftjn!vXWNcFQ5X97aS3snQ@8P21tX8W8ja}Jf4@D-F#b;-E0UDhP& z$hbKiuyFbNy3K)y+`q1}&nSZhC@OXp0M-~B*I*NV#E$?D#VsT6t7eM}ERX;s=2!S@ zy!Xw3|G7@_ugfsY<{9=g&+HjkdkVlM!GMF6!dBP!wcoN%xqcy4Se}+l9l~E23+7&P zj)>IN6}m{~eRx1b>*pDrdj2^)jKkc1)|9~Yg+UXA`d7O-bh48$py1GI;pIyDVkj|o z?#Sftnm%hVs-y1RT6{y9o))FdQ9L4qmAfvM_$h)z;E z^5O{dP#9_!#pc$_<;^RIN@zIxSCX#&M13xmD!WJ01hK3~UdNkG*c(o(+|nqXdkB22i6;x4pVQ9n3zkgl!Yv=l992gKjE#Dx8K@ z(LC{I-W0(;1PLIK^+E?^%ip~q7WPgsXpOL>_bo4wUCueEaUych1!n#bG5Q+RNDjh| z|CbNcF5uCA&nl?!aS~2q1wGoNwEf6J($Vol_>Y<&+>mRfzEo!2_DU*|@fSc?Ypn6t zp!O&|4DFI~T>sQl>5nS1@iwPQlIoe*RRVQKU~0I(5~j;Jir1Eha0^*j8{|Atj3>ZtBm0sdBXN3V*Qf>%_Bts0(~ts!@BaI3jtWga?(=`-c(AuHMkP-$ zuz|4|3Ax4n9Mgiq>8f@Rh1m;fRqa-b}tjAeOI&5_-~Yva%N79GjI zYl>fUp_G6HJyV%d9lPuIRq0PM;HJn?j)$DUmWf2}6ESMf`w2ciA_ZYZCh>tG0|jc# zWvIpcUqC7GZT!xx8^)?TnTkHtc&%(w1_?t}_rmdpCW&qr;LjHr2Nozrg=T*`m6yl}WM zl&B_P!U3z#kJuYM!Sd0fva+(mLXc^}uC5laTa#Nb#) zlWWk3_=l@UOI-IH*h!{q_B8bPjG6hg&9+_W&<)bCd_s(t$BclbiK)4BKTKu}94|5I zm-7h>8Odtclk>O>W7|cHxMf@3M*FUQq($xbZxT=>Xuc15-uYjKe#Pw z+vfe44<1GTJe_B_5nfEJQ=>3mszWJ;zU#cD>-=3%&1Kkd9I^I$tCB)~5q{!EAvv=6 za`9$<9<>a9m7y~k1&0xZky*>lniqX{dv0rG7za&E;!^s-Jzv9fA&c9g+V+!Gs3ATy zvUeh+I*L6dg8h}-%pW~ZieQW?-IsiKCpH^pE^05~jwaW6?_ThnBOb5SUc;8dd~shy zq3UUN5V~+IXfl1~71sN@j9TxA@ez6qhr| z>m(oo!P){#soyu_AI2S@_NRi6e$w?qI*r3l>YmYxbikqKJEoHdSmn}WbU)E|J&JyW zs%SArq|x6n+OF=o_o2RJn(>^t~@A5;UxdMYrb^7Tk) zqs?t!o_no$n7l-bnEq(aEUo1v>ZUrKTebYnS>1WLYorD@O$PmslkU)o@h|*O(m4}W z%s9Da6UWCL`t0?x$h{Pw+;Tv8+?S_5|DH9z{7e6c;l-!?inSk0t8=!DsE!X(iM*_l z-3XtWd9=Rs%c8G#Lre}ULt|u3F{DNEAK*(hNeUtF!~QzsSNz3gbV`74(LL**w=n*? zR;uq#FGx4!*)<#P!%HH?+zzG@Sg(hMF=a9w%q98@O*lDi4M+8n^;+D2o7MX`y`_FF z+(IDS+_KXgJ$x*x%DSmhC1BMdVSX(x;GBQ{IT^biLca(Wwq(5Jo~1Hl2T>L&v%bNt?uolaT@U#$4ZFI=2$S(pn+qM>t?L3je!S4J zaQXgs2+QP8oV6fhn+zK)aLwV{ZTS*$K6yB(q1+lSNBdVO$m=1w3QJ;Bw4XrkI6Bc| zg}J#OZTIG}tfm3iKfai&^tD`{Nw~ZS&!mBA{BweWmnqTgCru>N0N6q=deo_`PEotj z@33&7E?T<|i5G|PY~n^2NTaIv9K#Kdk|s?3Vt0YMvB;CeMF!fc$ao2Zs;$kpN!OpF zLjI2`P!Y`EQ{oe&AR|hJ>l50teB$AW!Tarv$>5%jWS_}iZWdV?r0sUqFqP}gb0@eb zDtL7($4ly?iY=}&OAyJJDvLKJ1AvN%$3F|XIK1y2{WR*R4oRRb_1YIDJ79>xb zS;^4tT?;MsF#CRpX#npo!E6k zAdt9hBg67vEr22SwFo$5+TiV^rE`!lvd7dv8M@5$J&?eiha3cE*8}?Vvp}Ml>N9aX z59IDyonV4o*?0wAkoEK|$&?g-U#eDK2X4KobyH`>V7 zu|>q&+hV>Su)Lg+M0bA2>4>yymey9)%)Apye!~u~KVrTYSj2xM%H%io2 zA95VC1BRkM7fV5}w>kM*CdKi`5G;#G9W&RUIxQbBmwLeoIUn9Hl7dA5Jd*JRfjMCP zR*z(m3Z6_K$-ZTQ)l3Q>GOLt@q~5bKN5C%EjcA^kK{L<6VsUY?wUreXSXqRJ2a@x^ zYCZ*j%SY(JAJ|K=I?vlKW?5JUd90}ZcdL+4ruje{-4PD=HyG}RE^82SmyR>{OuxF- zc+@Orp^NEO`HH1aJE=8^c%o8m+MJmy;D8=AKlAg68D~Kkt*Pk_#IgQS<_-;5`PD62 zJoLoKWd)UuKJ}ZiJ`wIa%Heum(T=XJuDy?p|MM>ZER*qA*_LrxU_lB6-uY&jIvP;3 zUA7CdiiV|930iHFL-j>LCc|Rx7y*gYEz%IuOv|o)aSK7BLRwly*`+0e9JzXyw8HhZ`=uS$1EamR_8su~w7m;-Qzhn(f zk-e+;UpLVX%ZKmY$$)T2q)%R|n5ZB_+3~DUx|abBP$1}lkLs~eTNP2M2z8Tq%$D!k z#nLI&J!O_Vpqb1U?~Zn$A~8FL69jJ-Ue0`=8k9sBnQTCm8%bJmhJzJcSJbOl(s@fvpYb4`>t+4nA_`@9`if`^xTYLgj zfr)(7Kw=JK{>f`N{k*SW?nfERhh=L`wG0iljKK(A)rMhXm$H`6#X-MWZt2sVYc&$; zHHbXu`j&EVTvQqqy&44mKepaFEUNE&13e&uw4ih;ARQ{*2qTSjw{#@AC zFgOM7>od#ABjkyfLzsnq?@;$*X=8NU(v_H^)-X^pG*Eft5hJYDu##R%7jJrVw5ND4 zK5C~h;PNy&4``l$)*eKrFM^U+@bHvU^7SeeR(6i&|Y_f4`fO=N36X@tfrSA`kgW-)ROb@{`YQ z46#0$vi|PZR5?Ry6hCX3(8iOrAk|C-^HL~kw@~)o@mqGhxS|oPtIxeV==t0NiFsm} zsRo&x49Wfx20U^vtlE2hIQdiN#51=f)KpFRN5_|+oanby{pxrBmL|TLdS%wJ5ECzv3EFe3SboF4 z?eDX{YbghA5o8gyCtKfi*7MnV%VqpTwossaWZt??KD|29sHbyHNfNxw!M z!~lSxZR3~Jk3U60n`HI7lsSkb002FPnb9;E_113an%ZmMTF>K<|8`_TnOBQVi9<`~kI zJoaCn_+QQTj~Mw3;6NF`U8*yN?9|rkCe1hxZg&E|(73=Ds>+2kf&CxLt=gn=uXu)2 z<9!lv$FW)PQpefFgNJ_Wd;Gvu?%prL=d9rJtd~PP3>$QjOh;WPTZP2A(`T;*d*k(( z#-c17k5>;Bdq>8r-|j-Fe7CLp;3b4O?8k-rP2NmIq2>24r73=rkOH`aOHgPj+jjzS zC3C!Kw=ATMLp<@qS;x6ZA-plvu_^qnUxU(SgDcG&Y-|@kAKxLJ292UhJ8rjg21da6 z0{OW#=>O%}wHs{mxqo5U$h1jof$Q z7s)B@l2g1ox<@?TA~qks;uHN9ummlfmtoFpqZJp3GpHM?ab9(kd?_4ymlU?zg2C;D9AE*&Nf<7uc3#Z(?IJcUS~cTcf|9Jd5V~Pm%ox3h+BHo zY{QR{ve)-y<;HT?ur;x0820|A5=`+-41aux#rxDqr2!pp z=tjvtmq}++leYYp*?)>gm3_}9_<2=DC-KbGqktS*ZRJOIXBC6*FlP@=5qp(=L!81c zeB6hpD0Qdk6+O>xIK+g7WycyK*~#hNW(EId}UvC3KPlJ$o)sM3kbUP{hw9@#ok59hHoK;;g-EQ^9gMlQec)G?{LsqP`5i^l;&5|b!afwkPx%d2C^7N7GB%y^>7ZvF3%7Y{oGPe@6H z39V4=#v#Y6(*1z=c9^tf^Sn+tdo<&QQ>p3a$zPqr543sri(}l2J^#Z=Ap-gKxnkI*kpxnaiNs}a!Fn~b~}#7Ph~&8F-#Y({d_yfRD7vJMBvcp zykq@^n?e4_HXB6d65FwPFHRWk`)TLA&N-63(*|4XW^GLGq~Nlw>!hvM5qR*61M0q0 zw~K4U#}Nd_6kpcbq&7@6!c>s+1b5w=moE~2S9Qr**j6lbOA$Gl+Kqbbb@++--S^Yf zB7^1;w1r25eIFoK)GGF? zz6jo4du}Tq5_0ot+Q+T}-%`CT@Kv+V=ems{!`A4U)RBhY1fj?5$<1gn4Tq!ligj4i z@AR(1C9s#ZU=_>wTvDjtHJ0Bi%^4maU8%T9NeQ2|Mbfycx`j1uW;VeTUWk~B)I&F! z0m-FQFNI%2Qc-);uJI9|iE#}JXzW+|B$C^HX{zjvtcXK)r;FgEeV z-kpD+BWzenmJ}MhdbT}8!siHu&M|%W+}S31r-}MlUhHCTwQ9m?+WF(4wS0=#=?IZ( z+UkV7Uig$0V`wS-9Re$Gt@&$gyzom>{fA1r6^qHU@*jjY%PBnXe_Xv>G_3|QQao$6 zZ=O7QBzV5s2TufmSfzJ|$&U~2b7nL^2}Uc4KajxS$4b~AfVA|>wv?GzY9CqwySbk_ zg*TT{L-0DLDkbAk{ezjfyNRdlXP0xihXn($8rrq~ff1ikn5|-=UZHK(%t&cd%wACg^ua}96#jVt)v^+Bzo ze|`3td;Ns7PiYC^_pT@+*^>b%;!;fbVzVh0?#h{$()Ne|Tb^1$AT>qSV=H!zHb@g{ zuUbDY7UAViFe*7~|9CGZnudgp59+aZbtl#PWHo|I3xzwGx!5dL|KaYzEMH@)m*-?aD|c3-&Aek)smtyyyH!!Lm_znqi|6!@~+b*u|q)-s$YZ( zT^nG>B~n7kS<7*n7o}f&7Y%!yCz`QDdzWhcRqQu{$U8tPLDjr3 z@X?p}Ep{b6r7T4uj)angg3O{-rY93;SLeNJ{fW&y-mYg0I%&`t9TJq&w8oOz3Z@?B z0i1Y`+=ovrlEf~OdFqA4Y&=H3%2sYyzUEH&^rLmxP*-?gV+~^>h9XI59<9D^{}SBM z6e#!QU9iy6*!kkw7trby|kY$~Lebibs#wJ79l_qre2BkBX!gHw(QgS8G{p3`*R}5-4GP*pHsQ-EOsZ(J5u@&QXbDX zku%~yB{*)&&J>UW;vxinGj_dbhP&4!PcF4`xJHw&+}_DI?eQ1x9|!!J-#5~84v%9= zc!U>m?Wws`5uoMharjXwHFVFiejRj$WX zYkPik>1yE6Kkv5kwg#a|Dg3Fj+Z&QO#E<`}rCw0Q#3a2`BI7;d7GHd6Kj8Nuk!#9L z3a`n0NEpcTK3=bOOlnehtR6Gs^*;NRiMbi}_WFU(HnITYW>3dj1kmnPvWXRKxIQE{ zZksL5dO6+91A3=}G8~{E#=E-@_t9qXUgS*(zV<9-$)yw@{05(2Q}>;rgX@;_@_>^b zL=-N1(Gy!Ux_xX;bG5iX?|Vt-*R52dgSd{Sdg^=+&=;tHbt$WA>N?zg6lPGqm_h&~ zoCrg2Bn3MC;mk3DttV?Vx~>IxzZVe=%<&KcErgovR48d?(Kg%2c|?`{-f`)`oc>{e z8t|D-CV90_3?D(pkg;=d{b@%Jv@FT1>(uE0YKy-~A$BMS2Xhe#r^Y*8f#Oj2sHg7Q z=dSI?(hWt7MyglG5bUSV_mv!>`=jw8(x+5DzP+0RA_<;rRf7)nJ|bHswu(d?DV4{_ zVL{+tbPaoPl|f^Dovm6a{{ z4<#EDGuCoN&YkDxS~j@urZ%5n2G=hGY0mR%rOAu@nCGON86|E6qgTXIee!zLC~245 zUa8|gK9eU@R`uSOp>Q*AP;hZX{d$P6;gja+s`={a%4f`kiQA5%ugJUDP} z5h!5eLXo?(9xYf9j;W?Aj)7p9^64`I*MQ^1kCuS3>ag7auN4Xq$k_G@)o+TcGYKP~ zV_U~Z+Igj_Ef>ciOK*wUB^ww#((uVYoV9VGmP>kmGxuZ@-xvS8O^<*(ncrC$NsS1v zW$-M*(6$|Ov2RzfdQ;YIeCVL>aiyA+y4O~lIq+KN5^YgWn(eY={s?B;KRH3{k*`LYi%?%{Op;kWI7$p6Z{1~-`(!S6sf;XY?| zdLyCq+Un`UrmKydT*IrS*iH90XFr8@ecyCq=vknME3U^oH1R;@+QH?m4{?Rx?Pgcn z@uro+QTM$bHjvMed+mrB7>c8RFsPO~j{5ktXu2ubocc0sQ=+>!tJmHrB=@>uiOxi@ znV(_%PUJ5o{J#4(hmVZyehxFval;O0KD=sPXj7v`vbl9)W54deK>^-;-AXZ!?>wB> zAy|K^#d=h7GiB++c!-bvnpS$khaEpKuKH-PuViP7XeqFf`-#UcynLl!$JBl(N#qM@!I(!OB}-k>-_H4%_-js#0@!$0a7GApZfLzpzS{-T|QBc`g!$oP75Dv{p+j`YK?3TRsV5T<7VEuRbeaI(C2~LW(TyV z&+Cgdi|HYxm`&aj#(acU0`T2vI`-f8OL4Hgy?y#+x5 zW<~V0m#0(INUe?w#gj49F!lfi%)5TOc$2yQn#p4%m_LaMn9L5`@WEc1s1nJk<-Ig9RA{?*Z+-&B)Et}mYQ1%)MFhUz5}vuj ztYT|h2siz`DJLJAXn0QH7r!rCen@;SL`m!Xc=sc_7EAM;H0bH}+-7!ty~lEn7NTkbDCObsq_N}A8DlPvV*G1751yKJ zv~o68u$;%J_*EY`orJI&2~FMmsZ8>a4IhL2kUAmK2X#qkj@fII7l~dx-*mA+HLVD= zJ;S$~+&laBTSa`i{-fSaoyB-Q@uQWwa4f?wGk$OjPPOdWJuo_J#iT8iAXBTs9K>tTDI!%J{O;E_Siemmptmj=x0dFsoB_Sh*@ zYLDGqX`_b)>nUEyIf2r6g6r?;AF68uQ4qgo#C%;brDm`CC;Mt$_c`=`8hTF4GkjvZ zI@%NLY-!>N82P5rzyK_(ra%|#5z`F8Ciw!9$l(zsUr(~{Uo5e)rpC;4aT0g5Yc_?9 z5k>Q`=EeP$?$_F!ls?P;(bI6^5^i7E{+`M74ewawX5>zPekE)-ybtc7uI9hEH_Zua z8euXUU)Ci0osv4!N=gpb7PtwzWH26v={?tAKDdwR*MS`{@AIPm%9iyr{k39jB~#WR zpPQ4{O)C{qv;|;xj*GWe7n4OQVE%e3GY!)hX9vj)w%sL1Q3mT&j+A6K2Vt)E0wyv& zm4%gRHAx?=k-_KF0cZhXHu?GXsD5A1GL}YfqyY159H1X7p3CYcD-#R=KaO#cWvcrY{OLXL& zXIJrneNxq7ya>_kM$et}!&=zQWzUVbik6qaO}6x*QAOkaFGOv{h>yod`h#i@M_%^x ztLEKTO@#_Rr?E(XC0PBzmfJggqJhiShNzDPms2rnLgRDgl~>+Q-*lgN*c6Z)%PbiM*N{%N&m@poJ)J0tCkRI6cmfU)qZ$*ZH^v1?G9_$1r?o{h|*uEY&Q8 z65tW2+_dG_&zf2n*#mwN51IrV9zOSNO{o-0^>ueZpiLi@FC<#dB4_M1~gqV!Y(DbM;?6j34`)m*}ui9{*=#ZZ%E!rD%Xpb7n;s+L*^X2n@R zDs;23RWB9J>0xmXaM}5B^5n~7`Rg1V9xa<*Srr}AZ5=m*#4?D>+qIEii%^QY03Bl& zp_bj{!X~~3o~)htR)gf-L`}?RcSxC&VJqw5cyMIg%yeO+;NzfXG31`%cIu8|RH&_< z!_%E{m)PdAk#>r=Bgvf9f=6R6Zu2YEuIm9iNW(}EYXPrr=>4@kAbK3 zXW$bK-Lr|%dHOZ2GUPL~pL%Y;2lY6!I&K7;?7LgL-f-<`U$lVZk3STesjTFS4Ge5} z!4ccL>b!QC6yXI;QjkELo?A#c2)*iiLSvPPJ!Rs#Ap=LbWs=d0H!)J7ok~rH%*;|s^29yX4BSJUYBkBk^+;!m7U3| zinDQZh+HM?1K|)bl4+}X9i@ceGk6|*Y=hJ-)|7!z%`7C=X?_z4Rvo-nV@sA${Gw)>vUtf&BjhW{EuNFW$H3>L)y#eP=y=9rBpqm>|Zs@0ZIw@KuJMfZzdilBaUeAzr}cvvSjf_YOsGwS{khBm`b( z#$jt+4g`Kl8cu#G2MY*dB8aH`T{DkSs=tz@503MyHl2)X1%~jK%LYMo|608NJ1wMF ztI2jgyPDBzdc-*p{et?}l2G%&Y=f)5DsJo2s+TiOz5~fBx zKjMZiF)cu!fvfmaBFJhlfsd~l`fAdozxT;OAt(-0j2mW3u`=&?qu0j7Q(PcL#qhAV zxG-P%XdmMmFTm<+g}yDv+1k5w?wzy!MOkc;Ga=e>L52(n!H2n}w~2f$M2- ze7LHh7Bu+o+~>$xIk1M+(yFH^Gd;xeKlEs1L)h_>5D^{QP+rY~bsJixo)Y}LP?s5} z%QD~P$K*+Pz7_IuQnV2=nxz*oc`$hEXnrdqA2^>&GtFec|?^meWtI zj|&(jBqc8?hFx71AWsNo-&Xyc2anJ?@4TI)B?jO^$?X}zlY=L{)xTz_;;DQ)Pk(%1 zsMqI+ZJcNqM}QwRn1>VJjuf`tnY|7TuzKoL7pRLw_@19yzOhK( z**v2nrzEHLU4t%XM_l6uSph7?hOoQ+qhDz$BMyo~WinUFcxZvB&ElDjQA zY`?kYQy`F)pzwo}9pu@AMfa}z-Bmh4BMyoSKcCa?I@bkafztIH-GvIU%NLHKDzil6 zfy)p)2J~_%hHtLP^Elvy3vBsSaFU<^OGmg00WJW}dMu}prLb2Ml!(c%tBbi%OJoS+ z)ci4RMhXi%kAufDdq_KPw|UonSwxlgBw}K)4D%ZIixt2mqGl)P(j;<&_9#|KZ~r%) zXmcQ80p9wtXTTvyMQIlH_4jkIu^pKh-+^?_AEHTqmSK3=@g=0r!%Y0C-#&rbFIx~Ac`VG|LvyIo6K^NkN}syudmPJoQ6i&tae@8$?B{wFtL^K z6&WpA2P;#CgUw}nN%_8w!Ets^$Qnm%s;yDmryA!vs10?zDRIH5O){ujVF?LSSE*E& zOTWucQjY?=SU-G_ny+5yuXnLx9-#6V@3e9FB@Q?>rW>ov>$q$-+0lM+t zgv3Opl;=r;&?#g7zutN{?&c~$AkY^sE{74EVbC{66Cf{q9p<0s zyNM)G`6=Nw)E$}I!>NfF8yy`RLjgZnuwCaJ6(~q92A&#QW&$w0ihF7|F=(Zuo|?cX z0fC5)%*@zvCxB1r7v-I;=h`m3e5evDXbAEDgeKjaadBPa1R4PF$dN$Mv;N1vUe8|= z;DGA@yl_?DnWo!Rd$xYYrKEZSZsMS&%`kM9?DV}iU4Q@YEysi<*nj__onLHF{&NrQ z0hLAipCbSvD^M)|mem5KM%Q$U%KD}z;m41ENWUOB*8_)0A`QP>&rKD!h7c2VC$sl;pL*2DOAo_cxMW^CD-1YU$6Yn=$&aO6{jtv0#ZQ0S^vx^uIY2Jd+o9YY?P^SqRs$FxD1_H~9< z!bWbKKjH23Dn{kYRK-qTNx69Fjia2J-u7>?d);7sZw?oW7h)8xTBF$V$OFUI5mw(r zOCyjY&8)-OG6v1@J8Xr*v$850Zq4YO1MuEr8i?_w}SnyjqR~p2NB!v-5LHwaNNO zMy0+afz~w?)bDi*kJcs{90y1`ImU4_9wIy*svF<>Xe^^Pa4~?F*)C0^i+={B60BPa z!*{E5t%DrUxLEA6u%YgS&(idmseo0Sr_ishZLilRJdd0>(MNnI`?@rk9@-p2v9rPsY(Tm~gwiry>C!*tKd3y(fVj=`y3|32Cue;qd>ft0c9)ev zy@1{)Jx2exFhJ`cLOAnZh`BA2SSMtgo&}f2Mp6+xBO3Y0qs%uj6kf`Oy56OVuUQSOT?hS*YSATo#Q+8AN zQ9>mnu`@LJYPW(~gqPk}{h*lZ)AFsp`#7!OqJR8h{FFb{lK8I*H;;-s`FBYN(B|)6>Yll|1BFLa6Gpwr&EMoZl%aHK4Vxl zQ#vA|)X9M9&$>B(Wq92=&PH6P0ZI`XpMfue8|M8%=ii?)9_Q^J1Msjrw_D?=Z8+7` z->iRgxIrnck~JJE-uNpA2rL(>6}_To`3ddIfm|Zk)qq`@hhe8XS8?XS7oz_GQeXeS z%6VsEO*#zgWsp=jX$mRVXb8J>7=9{C^1X$G_KJ|{fD^J?)H%eT3Qu1a163uwhgIn4VQ3X^8e` ziSB7>k)V0`SG0F#OJ?gq70VwEidb%i4w}k*&uer{WTvbgK+6$r=fDQbCn*2;?9C7F6Px(J86m0}-!lFLR-8{eerpAPlkIKIH8;5}nw_@mO~d9RY+J-hcy!N_@R znvz=C~S9Vl%@}0qC z?h3APUAu6qzgQr7*1qFy{j|YX#uexB(k27qDV@g#f1b;r4hoDrv+GbZn0ayD*x58X zey2C~rF3u8DETq3CgY!bXldt7{w+?(T<$1nSc(rl#=8DF`dycLMQq9r3GjCr`U6?v ze}?L-YoSz$MEafz|GyuY1T4~TCs_t3vrtOIHsu|Mm*k`+uXFvwV81{fFliNlJhJB=C@7`dau$na$HZVa<7Y zR#z9tqS*hY2%R3x;vY24{vU$v&vUp|&h;~^7VLjg%D;Ho{|W}-^`FHK^bYuEUkN-? zHE#uY8}9$@cEqVGD>nE`P*Bi+MkWxQ10+qGKvlf$e8ShS3z`2kqR9kA!`FE4J-<7^ zmHj2sm6N2QVm<`=yG zDP@=BZT~$Q%auX z3t%MjQ(wFORilH%y#-@@JyJ2z8PBsMcQ<1tAZ2w8un?l4c=z z%e?9TdMx8+i)~O)eCR;G!u2nZ=o)948uRz$&?R0i&heb)u?>Ms?lBkN&Svre*FTeN z)M!x-kb-i5hwZAUIEURS&9aF;er8`=p!`bkIpn?Lla~6A3t{BQ`J$!7N?EC zp%I-CGI9L~K730qY7E%J#xGMNBdVILWpYExP{yL zFiWEC?>bA)*uTFF{5R>|xpJw|DfF@E!4QxJA!JZv;&Qdvjp15H_Vo0le4TM$b_!~P z5!+DMX9r1mS12FmrurPb;Jg$5r;XBYZ~n5M1~PPn}+en2C?Ql5^J;99razI5A zqx*$SRm&a~*a$r*V|IQ#CvL4DET^Zx>Rrl_;#kg0$J2lE>E-+3I>Z-B5_vs#sbr~m zvrTDw%qZrhUCR(IWgWZnI9Kv)2?pV5aUIo$v3EuxFABj@P1;cTKEx@L?yUq5f>{h2 z?Ch47r%z&|_yC}2G$&)r@*89K2sjxRML9W|{G>D+*KLSGP+WB0&*^GsypTOOEY7#g zjL92*XL=^HaCmm-z@A7n;jVU`I(V$S z%a<_yJr=I5alT$rUMp6?d|J1SnVEj+zLT>$V$Q64$mv!|TA`XX)NNaFI(_lJ`iFW{ zqxJbwSsf1SkSmX~U%BhsxEV?OC(CK$Dfo88x@EWJ_JT;5J#6kz6BUIS+RZYDx?qkK|NbU0r?Lvwo`Ck>k6rgCgMVW_<;vHsao-3V{mYpu2w(=UUzsZ>0HnbwTE zjJ`xFE=`PjWQ$eR)YKTVvTG;a2ed=Rvht3`uV{MnWOXb0&>peaZ^mksRNvmws9g5! zF_1ThlnL(O-P4UuEpYdC zODy*8uzywWd5nM7=vxbe$YIpL-x*-Z{pTTs)=xwE=d

u zPmiRC?2hq&wVx)wGfLx&cAre3-PXiasf+2fQrTv0-Zh;#lNV0;oz&c-0I?Gz?k~vo zqjjt^t3a(~t6zpF_!*$QtAfg47`MJh^*Evsf%zKfGs}xa-FCoK z&*PyOcbF-V#l6G!x#-dcK(Qu|o>}Z=V&mbW!}4?WjcHx#pTKDpk~fom4v2qJB^ocU zd|EFI02_sW9q@UePA(TppsZ4a;4Cqdk_L;KTBOwU9VHNrgo>lCzX z952yYnCOWapE`ldb1t<)dk!OY#VM=m6HLs-HqiI=U#EYw2s0 zs=50Y&8-?u1{LLR0rEktXJ-FBV}&GPoa1&~jo@OR90+-AhOI(g;xmt(BnYU8z>AogxeL9v{2s~0C9@8+h=af%FtUmbq36oQy8mjcIe=3Uqs)JkR zm**{PJSIDqsw5JRt72i5^+eL?olI?5f~I75w!*vNQqWZZ<9=p0JtgFGqfx@|nN`-7 z7gcOM*w2n3D`!r%G14Zc`h}-gAF)5n$(l!CvzWVY%8FY{g-uKnnZQa<%cF z9Jfa4f?gh0PYuXDR~z4QyuV>-c)l2boqNd35e6uyGzv3kx+|}w=hO^C4vZAvHAeI# z=mLa6^oWlzL(i=KLVthnRAHpNo12Zbz5TI^A#H38oxn?UQYwXy7@l-~`tK?pkw5H3~V8jPFoEV441n=6GTwQW`^HY)d-9G_| zBPH_l?y040rwfsr5^UU%5Q;nBZ3fAb4~jh{n};wSPoUpVmOn;N;EtBiynahyA1ykM z22|7(OmiT=v6unM21RbK8vAr)BN;-e$zu%dFe7_AfimK?x~cbwq7kh-em|`zS#P^} ztWkqq8>gUH0QLV`7Ny{d15moLi3;pRk=u{F$(5a*$Ir~L*H$GaTKA7@NUyKg;U{I- zu>%VAST^k<@Il}=5_chRW06ki;c0{BcTj{A0F`F@{PW9|fdh#R9b0mhdc8oc79j&p zC;8zvpJO>GcYLRihk~SoR1B0_w^XYz1PFYfF5B-KAc#g0+pLU}28qnC9q?ptClAFG z7gsINr;W(X{em+1b$fz9IXW`J8r2UiJEO4V7AEEGTCr*qMr3^zoF&ash5&x`?rEHO zBA!R`i;pZ0=xZqiToyk#J$31L3tClNkkGq_AsI09I~pC@)R|9X4Mvfgo8EJ99VYsN zFF)H+{9JDi>JV(u=(~Rqg>0~54UJp_^)B{9(ZSV*qRZDGe+(YtmvBF^#sM~c%!64v zmCehw3%*gpYz8pvvR;=mU3+Sw%1xz zxi?6G)#-2FrV7(_em4A!%3=p^20Ff%n3Nkq$p-=>)G(^qG!T~+qy z+atN1`JPRAdW5+P}k8Z^@Rh4z+6J&c5a|d+acN=iD66-FD zkNb)7p>5r2x&9yd@hyg93<@3+`}dfMCWM9HJqQwcd)!Hbf_69bM9%i=!!3<{+`mTE zDFM|@mbp(FtK*$+?4m1yAbkb|cDAt@CTi&><=+Dx>H<>^bU;yEdGkI|N*R ztH9C1sx8qcePEKPCCb3^*gn$~o3LKalA~`rF6O?e*E^~qICtHR;^7I@?Hmi(Zglh# z;|KMKTztbde|TjEJ@COC*GBYk7$1||+)&tmwE*VblcV=!Sc`^P&^L^&KvMCT!Vv8# z2mkMSM7Bfip(^-oMIIl+$)j%rHP#+posFGgM6I)|QZL$1dE&i9Q9&=hI{78HiOfiB zWz98RV~RhBNx7iQi3MG+bYPeV2$YTBx_isoWcYAAWmb!PgkJPA9Mt!z6a75Cl`Ph8 zm0*3$2dUY|cU?71vW$cd+Mg|ck6f?oBM$lc0UexG9W1Q0nqn45cbT&3P<`8S`;q5w zKJ49IlNfu23gh%VNx4;RTlb$?{dPAW9YG9HjCIp>&MCtyVs&=JGhktMX?MwxD1|3L(is~<2 zYrDQwEOxrT^nzu2H}43YaQ6$`W)yQnS0P{XBVPHmf`mBZ408 zEanWjDvDuY5Ihhd4yF*imMqqEoGANI%lh&T5qWELRFOicW-ebGc@fZNnz8hdSx)2g zbB9O1a1<6J_a!f%!M4|JXs;>l@3O_k-FxQH*yjeNK#l-k%pGXW2%DFoiWP~aw^$iVEZn>JYTaOZG_2( z@!X%XD^GfweCJVw%&)%CQ(f;_)92Bc7>g-HT|^zuonB=seZYULU{gLw3F+NQZ_j2} z{C;{T{r>=3S?lZ*)cOY)lNrhY=p_qB1^<^FnTx7*OVvV`K>0o3O-FfF$3XyO4#faU zWNSm>06+2OIWI@sj+SmV1;8y^m1qzUhPYtWrBlJ z&9oPEFgw#$nz|Lw1l9`b;BE3`?!0MPo9SQHu-;@4x*+zZ_^O&uT_L zAqj$Hh?|#Z$9hfe_j|Rx024dHl=rTV3L{&7wA^)bC)XUP=EO=~(y`9iWhuRf0oMMA zgj%PS_5Rz^_($4Fh)wl_vT2&`=Ln;l=8vslT3f+<4fsO+On9hLA?@gJdgHh&ojPN9)v{&lSfXWk((n00{IX@mNHRbf`HB;|V!oERdEX}0#|1nD zD(SMT35FECM&^Y38_sxh+&L~?>{dfhXBz5T?&$I5nOwbU%wIw#oV*_d3zdv|yxTid z^)tI;Dm`D3!Tiv#DDp+`k8X!sV&1pR_OzE1j`N7V?KsdzMy3mP$|kY)5e|G3dOIf3 z+37haPtq_iPmSO%7KJJ z+u*eCxW}@9239|N1BcN-oxP5+D@$SZh8~lRR%Xhu`m0UsNMGD!uk}}X(5%?z!wR>K zd80Vu%2%2Fh+ib}Xq#CL~+3#7X3vn=tCFxGX_DI)>uO@dqP(HaicW-kSR|bQO;DNECW_&xDjIv0f#Dc zh9)H~8}0BQN}9Qm0LT7mM(@hEyd)n$8*tGC(tNZYiQW5e6{h6mPk19k*tj3>VnF56 zR3f7i%()YLC;&7F#9x;FxrUEV1HzTlUdO6s3Q?dq$pE03cBC~lpaA#3mg$tLbx#jd zndY!vkUgN)Oy#v%c_*N(N1UMSq?k%SqgpB-%jn4JHe?CWH7w_CkD1dn@9q>eM15nI z(2@j$Pc4=#P4uC|vcuh0Arx_JGw_o#Gmkw`>eT{6yFyQFjz-xu~ zJC0KUPHR;E4_E&HT-W#gkHa@+V>GsHqiN9CR+Gkd+So>8+qN64v2ELC|0nOy?>qC% z)5&z^p4^@u?7jASVXb{bTJcbTnCxkttyN?0kd@LP^MB!;K7*Q=Zh<2w~jIVALtADlN5r+Zw3| z9|V08vy7d-JI7dVI8+_B-s?*4*ABb{h(B-cc0LAp;a@IpLF)$kSSMGfZ*z(xockFa z4rj{^0DH#8Lr73*xJ2bbfgux2|u8j@js~-nl4l*YWrDo{8Yq@o%uUXIz1BzXn9Z<3 zFDZo_Gl&3UXZq$Y?FX2OX<7X~A(siu{z(~FUopp#$Rh}RuSAo_k_^n5)=mtT;_EA$A8KP-vX)7n&lpDN1dSr8K*r%snf^U1ebrH@F%LA;c z02>wNy%~;xVRNwj!6{UoU6)u6(*JlOK$M^v|44C_ObY{+*3c4TOQMbv1kqX9woj=# zhU1U>4M7yk9**+bao+-F>qOI(SM8})NhjEw8XMJ)*zgh3#7kIX!Ov2E7Y&4hX_Sqg zBM!X(cnCg|rWQ!~L(6^NDZV23>r37nD>gO(r_)t6qvA`ZcT9}(YpmP@6#R#(#QniG zkC`|sD#k?rm!^?kQ-v$4uY$fuh@kE}{YVx;okH*&$RFUj!=#pvf$&i@J+&vKmX-xN zitJ3R3i8Ti=8ygzW;&-$!x#Znb)a9iFsNF?jUTIY#e4Ixh6xOC{@5E|_hbsLeyQtM znly-3K-P9#DVZq>gorNM;XTLTg{AqBsg69>89&r9!1kC6R@GQD6c?P7h}(|&A?7{F zwAa9cdW?L>jen6F)Cw2H5uyi6W}xdq20?{^X`Ga~2~w$*4l`+FpnyPg6fbpL0v)^e zl6hhV*BGja=-hS1x?QyDOx}{ zvR~GFw%q=kO!FZul?;D>co;VvUD$PUJZ$RVmM;1IKcR&@rtsK}f9$W}ym>96z(3tZ zK#9Tt{y~en9z(chQVOoAJErI- zw$QdqT>tNq-xNY|?yL8rxH4iZgq`gVUfO^qVI$^S+%5o+`}@Z?jsaG6{c*%{l#9Cn zU|65TLBER@IZxxikpp~ie2vzqTisgd^ESS05UoI7FH<|3xsFt;xcaLeI7kIgho4K{ zdgCgytpvP5 zpAkQl-|1{lt3~FjdLVf_NO0j*PN2)0F3tt1CtL2rmo$VmZjxMg5Mm_Q(5=4 z3+N8`Cih*(O$y`_E7Q;$km2T8*qYDB@C|50J$Ke4_{n`v>xD-waa-oji;h@Mfdc4D zygXor7v_AQR;035c60WRQ!65sBdyVDC4Zd7bvlQ9bOvMUi@O*pi0(q~@M)fY3`_du zoSr89;_3&FCX?KtfMfSP-oSS~40z%HXL5>~3k4eFd19g|5-AHMlO?rMR3$BE>XEW%ePu@>;0aY~NFKz#_RjK}|4F97rL3LToj?_?An!POg1 zAs%}rYAJRvkDt>$P?=nZjex-h4(6sv_AWmI$nb|-c|kja!NEa6KS^r6q(2BHw}YFg zA31=o?l1K%2-Np$mpf~s&s0_ihRcnw z_eV23DhvAlG*O`h$^c{0C;3&3G{(&7HkM(qLRdvBlQFM1J*Qe_Y(N8G_H^U8&2J}j z@~GB2JZ9dnljkn1>o*42nQ7=ujWIyV_he1c1hi<9qNG!Yhr6<3YQ(e*5^2ZntX)AjU*pJ588$VzxaAi*F#YN~?9bmk)ai^xwNL6L4&wQCG_c5JeoQX|Vw?dzYS z?gq~M3iOpFY4)N3eBYNp<>c*{2s`*h=fGVq=CY+j-Y{~nXDjk3QoR2XAQ`cAtB;Lj%eVgC zHX;N!(PyDggoACuEVBFwh~Lvj;?iw|XkObecjVeAGzZCF57NtpiBKhMYOJQPWge|- z)lx`Av-lqoI0k?hJE^ew3lc`HfD0YlK)?nNc3c0;a7pK=ui>3?#by$pONTZq6R#OS zx=+^?ZwRRsHI)t4W7M=4N*~nUp_c@ ziM?w6lm0oP4v~tLa6J1Q zzdS&OkG>ITQYJRIoZmv@q06Qx6T|H(q}nx0yf^tfWPbcM??EoJdQd?vEWN9RU2^?m zN0#w7ongxTyunJY8j^s=>TnuoN^%<^s^9aSM@(hjis(SfU1wxmrvLL1Idt7rQy+ zhx4_zhp(!NCaf#!we9)+Qo=fcQ~TA&+hD|M2K7AnoZ*zUhVrhXxPghlV2i@{2}Wiiw#{r(!V?4dHRPcGqCpRes6@{!Zu`^QqATJDk~-HPAMf)|eK~j?Qk#E^i$N{a}7vHUqA64Kj*}HiKJ1 zzJYDZwzMkr$=0m~a0n?}JEj@p98?t5Fkmx=5EL{M)xq@tUIoq&ThXY!@3~=m;MYTpFVdUpOX}D-=gG z4lE1SkL0RLu?@cpTe-=V8JZI6@k;tZ4{kFi3st%Qut(KNqV_U`NqRQ@5uHdA*H55x zOhPFdxWBtwtx)W(iY@7N% zWurssParIls)pHGAswo^T%TO5Tf}2lwypv}%pP4CN`tIY@iTAnAKLe_=z)m5ZWizLp7=Z zy)_vf77{#1J3C+ND=)H-=S!@leRQ=3Jy!VL-<=utT|VDr9_>*#bN%(7r{r+(-zSn1 zh3U_GX5)*?mu&n_(44D}ZHe*K=`$B28BE=fcr)aRW;g^0b47sVqt*GOb(kDHM}m33 z2I)ye;uGEgV#f=kTS*RMyNK#`F!M}t_uG>S@K9v2F|oWGwxt zg3cYw&jsSPX25W{T(sfLHM2ITDw~>*FXN!P(Aik@}$n8TBP~B zG5@jPYesOslLGwDhk2vf&({eqr(OcXHfA_l+OF0fDu}rRT2EoDLfc%pdFgJ4#=E+R zS$JQTm2|uihj|<5XwK|^;Rbnib}pH(?~9rJKvuFrH($bMnCGyUeK7)|l*xj%Z( z)s6!#YDMZ>qz(|U7Ha+|bQi~ra+&Mu>N+7h;2^qys;g3N|E9uTsGIcFu!3BqN*2hi zcd{UGL4=18r&d2xOz~Uq&l#HL9$o%*JEb@!fKaOD){x)#hCl5`R!UmZ8cZZe$=E>% z*R~ECBmQb%-`+Ni8#1cV_`~qm4T#W)9)ej*sq&JNTlL;t;lhU%@!p=VGrCWs+^uN! zDAvhJ0zOAKbGLt28~y z)>T6W^@a44pv9qC%7_64qpxyBM8c=n>4y!5C)`-!6dWARe}9tz&Q>Ej_qa_7@Vs_OI|(3vQ{f2xy#f>{cG5@5ez|43qq^N24N;ZyerlI5a$Lf$PZk%L2{EeuoMPSLT%mvInBsZOHSk7O(3l zZqL{MzhVpK2DWe8Xj?g^gX6k`@+!6e6P}R9mkH_SDsleuR`1@qR z+f08%0VE$0@zP1BZad@T>FLSdA(Z!P7cOoYL8&d*0;h1?-yO;=hqZj*V3L3P@Nh1?Ysi;5duB_9D1u z>QFS`w$2e8-?_1|vdT~@Z~pE2!ZO5$O+JyH#c%nA3s;=@R?lYrGP9DNiO;3_bkSFv z%M%0y+I~Yt+63)JD-xF0fZ*8YXE*EfHs5{6A$d8CbPQP<=k1NC;eoCiVomp=E8x~Mw7Y;`vBa`w zjuI53lthS-%_>d1kebr0v1~j)OinkP%7pDi6!sO&XCjwN%c?OkY)L$bY{oop3AFla zrhwLe4!v5KQFjLtB`$ZQL;`BQQKo?R&ugo-mZhB8 zi54RKoo%1@v&z!d<5%f^evr>vAaD-Sfw5m9dj$0tgt6csXEH#8JwU z26n(A6RE03sg^B3k{>$G0QB+~CNKS$q-j=@@^9Mg#>2bRjO#XX8I3GalT_jZ0{1(p z1UeP9HN5BbMWJN3-M{}RfNy!#C5Ahn*#}1vu4jvs;XZV=eqJ_aq-SJg9OrT;WKSTQ z({3cM^G%KAa3t~$I11%-m6epWx=z;sneiM4ktGg72+JA_+yZtS8bXh5yGPnM;~%Ve zKA!Fa%JvTomtdy>R&^xBpQ!tjLYF`u6q=4yvZ>^z&pDR~l#VXocy{|RSRrX0xs#vz z3eQ+A^VF5H%Gfy9M zlCpBj?u4(uVXfYZ5v{Xeo1HmVvIsH%AO1GGl9JgcfESLy0s;&0y!y~bcmK}|!1@_+ z%^KSmg+nR|e`M{JFmJHf>y1@8Q}(rlI_yOMwS#lQ={F7)rIeAQiJ1LWl!DIc_qjj% zGDn+ARPSfskI^V7rCQsMZL55WKOSMq>Is)^UVK@CnFv%Foh*ythWL5~MQ z!4sl#T)>g|3hdj*n|7&M=r?La=xBnte2NIw6=O!9$p*D#KGXw^fYK}dZD>_>tJZ>o zME!s^qVSYQWB{r=DE!L9k8Dm$0JDu%84VY&tsfSBx)z~SXN)So5qV%XJ#Ty}o3Kb=4DNx5CMlHjIlb}l&mlpH!r#oUWlrmK+1AcF~lG9K45DyV| zRI}0H?0h?JM-)AU3;I~|nv;I-?e zS4Hc$c@cD>lkM<&jZoovL&}kDHRmi^HHKV~(WCiE6VJ}iSB!go`uIkz(h7RtV{o)k z!Ns~~KyIVwtyg`;39eb17CX&E{*urnRFQA+BFY0GmzBSCqF z<+A;`F-uP;G`}gd)L^OZNFxs^GEr1IeDyvbZNr7-9^yty?cji-ZjNatw?GzQyC+Pq%~^t z>%T{R6JmK1Oq?DAskZ;}L|sO8b8~ZSO~~b(tvU_*#BV2urPB9KW_9uXhBX>WH~t72 zlWG)6z=u9J^#ircmK9)Wa0nxGqDPo%gJlBEN zrYORfuwPGof182#9{EW1BhNj78LpMmT~(esY(xP_)cYeQ-BRBVeqn>catmd!T!Dj&W30)JPI;PWL45RTwhT>B7F@Z4l4~u(p5r4C2?(SZF1J1xFSkaCTibF-9I^N-Mb;I6|t==_~1FJvkMCsi!F4nGwu<1 zC|y0Gl4{3()Gm;pst=^4vtji1ht=JpAfXf-b~P{va_(;!DNmfQjqE4v@9#anIp?m! z_s0w?D=AGIp=wK`ElXt!E8gvuopHa_5JhJ{-$86T-JXJr#cPbkue9jn!bU>5T0PrX zyGfb>G@b7fT?Agg`$1m&nl+N|q$>`4^3Ax!OLA!oWn8G$HoBXj{=c`!(mBhhtm5x_ zf-I~yZ#U8=GGP}HmYy!~TOb=x*DB^4D2x3Fi;WEfpr-sy(WCFJ-` z46ns)pNQ+RW0Y)_T$)gkkx65I8@zZ);tmH@87@8c3UE@+M>jwg}^&q@N4(g z%U0?1>U{2xmz&N43G%6FEom^{_eb33n+ufvGEuw&qr`J5wbL0S$nk|{0Pp6W&ZZ!> zXu$Q6D?;Ft^T>e&2O)lD{(gI?2m+rXv!?is>vqh3WFmKn0W9;th5W)=vI7QGmMJ!4 zBwjznrbElO9V`*#{KiJci#KFPu6FF-*#H$&F8Sh|m5^=o;7TQ@0jBWwJoeH*!PoKq zH<^>yp!$h&K=EXHIzCqT9L>U&RwQmqz(fv!!}^n`>nX9Sw%6Ry)3wZ=48B&bq^$GpC&7?q)LA@Bg%4Q8NdsvqdlK%9mOsY#VJ3O4D^T&VB zJ}^1LRU$X5a2A?ZJ$b2dORL%IOkJr_IIYc(%Cy6JqQYvw6SrUg-JR&W3^Db?^qxyP zp-|!kVtRXyC3OxLT5u^LGLryiA>+`gwJ=H{ zKx~)wOeYt~dS;m1a`-i}>g{~EhodAgD1H~#A-M5q51>~#9-<#DPmQ1MYdv&_IqMWD z3PE=H7QbZdb0*B0x!ISi{~aLtcf%2@<;3!H&nz4K4xh(>soZx}$Er-RtijaeU@Rb~ z`(D>Mpc$LQvVL4qTPzk?HqGcS604|#UYiXZS4-AbGh-%l`E2y%%7HLWDz#Oh*&%X5 zS2MXSfAagPF$c|4clgf;T6?dpw-zsx!X+((DM>;Me+fP(rKeQZ4bSjXiWT+v&tp(X zair8|(JnX%(-eW{#4DZ=XKrueb&@w-2m($a0U{>uN(0q5iTx@UY)seb3tmAs3`mu=WHSdxy9>bJN! z^gX(t475wBtU>_T-q}KEy7-`)f&qdE&dT)4vtTx4u=sZ+bqN@s!Bb^e+5djMeuhx6 zG94%^dj(%I-3O~Km8;eVZioA+Q2~a+dIm28^ixWej#qBcaRFlicBT=(vh}(5jEFJb zMkWeOK{=T=(I~-w--@2s@$McttYRg-114-N7sZjCPQd$1-2M{^+sCB{t^TlTcPk`j z^Zk@fsZVfP?0jex0f|{?YuQ}0f{kPgoXi4%(5(ttm291pIDAxX+1owHO}78+atzg5 zQ9!4^X#G#VH0{#q)f7g^X>^p;hw-#@6YJMVq3`0~Yt4R~%bKoz@-n}QHO|x6GXEMz z|0w5&#}A9cC4}=tQ_kT0n#uE$8PnV!=)RNRh!>fDQ9db45Z}^Ma@iopWb24Lpj~=H zlT*)CZLDU=0~;r7IHf=qP?wrYO+G(AXJma&zVa5xSiGw3>O)=r!YjZuYpigv01DYK zV=*`KfCq5{WR;cPI6{`RHv6$55%FlF)~ys=(vy>d2Ou*Fq!S`M!p~`mJ1ZF`_`!5p zBSyv*Si9xn`6v`v>8|u9$CHLDGN9Y@0QB74=X#msKNDHiV;1v&`*p&$FDYpKNf@Ag zCM+DLK4DKEoq`N!WW{B(6|D%!9m;%jRVvgRg!rIlthiVDxbK-uZ(fyXaul) zbj#|3LXMJopg8N)NVXiQ!+MjbKNUtsAaQ7siO&c1@C%iM@Mz|+;T=?iJ&yWj>n-P~ zTDwBjj7V3AC&X`D=C_~)jvJ0)fxojl%OQY`AybiwBO>Pq&2tekuY+IxATos>j8?Db zj}tbdGd4CJB{ruVf~7!l!_2ZX8 znwWTGke<9hQ~b82Oo13w8={u^&s|yx@-QgS^2+z*g;K5;us%x%qntRNY5cW;)5sO4 zu-`~HHDT%1i8vl5>{y^b72lR+CIbY7qBI!cSu$Oik$&We&-A%Tl`Hi_T&DD_je3H` zRCg0t2whkV9`*i!j>hN20HMd@iS@#I;wO#t2kb8TZ`jT@hUjUeE% zlsI)qE?_`KV|#%i5W@Ki{Z&J?ZINziY7_ZBoX3lh&LII-V2XJ})u>@l3&ZGvLSC5+y3)`~X?{3A*?B z?gWUW*bO6Y0(wFqyC+nsH9Ik#%?|hifyx0h<<5-_C%2tgFjHdv9J#;4#pNtV*1Vws z;D4~%hhT^`87$8x=AiADZn>K{{&|=u^=HrEjH;#875)m}GPa4J+itw3%i$@@bz(iU zuRoW35YRt-ydn+DHA{UrZS%+hZ4U~L7f#iF_%d^bSH@jS(aMoYZoi00sf= zv^2L8(6UQCQ*83L3+$#)?&!ZEMX6N)STv23Y&ON-QSr`ynG;sZWJFLHb5OP?Bz~i* zQnkGXJB^QicQ_n7@!QoW0XJ++yVtt5aW~o=uhxS)G_54cuI}LdqxXb4 z@rn2GiQEa&*UZp0O_&%(%dc2LWVWQL%9Xo}EKD9)`XFO2Q_R>2%>4y&_UzEnDLOr$ z`Wf5Yw1Mw1F$(^T&`)M*-_hH4OI`o>fIgbcEy5=f0>5|jq*%#3zU8p!LqwPa1FN6n z@~f8$La>>D5x0Vlp)aB2pn62g0j022Y@e$~SyAyo^w82XX#9G0eiK_^$xn#wUqV2xGzo6ONpluJ5XlW^#YMB8j$Boo7V! z-V5um0e$Dv(2-rwudqN39>E@Qj_SaULHG{LYPQD%DD@Y$5TcPH0LR$v*H?6O2xU*d3h za&QI*c}qrds+de6`4@7tQ^k-eju?U-!{>Mmc;g$7<^pZ>BWLg!g8e!Tuc^!1ja6{P(CB-va0gs zO6!wFOS1F_Ea=A^QgJIwYAksI62w>-ty}JTQUHu@mszuelH^7W`;8aDvMK+SNPIwu zrPuw?1!|xGiaxY^r{Q>)6e%3Q2!<`3#n-iix#}Yn9>4(rg5Yfz_ zFhkNWo%~Fu9&xYn3AaFHwV5%ze=FRWw!MF@PpxV$Tk#uSq7i)vH*&TvfIQ|e`!p~l zW?%O{#93i8sEGa7=Zis{rIzi4e_qbUYUg|xSVWKfGQW^;xWAX$t+Fna`#heeZhs}2 zpRTVw?ApM^n~Ee9I9`**b8n`aJbPzst!W0m&wmc*a~xn3Xv9#K2dXN{=8g*ozmhsI zU`cCgE&*y(?oWeCk*tc0_!Fmn$pU6v2jK2*!sS{92nCZqXwNaa;n(v&M@nuN<~MV8gi_A+(izH_P%U1N05(;<}I$O zn!(pf>NE3l(~*P|hwZ_tnzp79qxdg?m%$O&Yk~H96@*sLqnnl~Dd})aGcaqIRmH68 z#{gX+HC!a@UJl)0D3(GbED|L*Z~EZOM;QMy+D8STU@&gC+mXz9j&PU5y;w{NFzBr0H$7hTQ zIZsmaUEF1)?8QT0-gj$zR%=E4?UUw;5lgAh8j1^Q`P~oS6DO>hK7{`7i-vW@5R`2Z z4nwiqsX^*iT^Ek(ShiL4>y;Dq4TtB0cQcH6O0D^mp59+NC;mf2#S6CVY3#bF9$$!2 zU<1~_SX=Ar{^&9Cx#(zn%=-5V)8+ksll8w3t0rN4^&e}eI5H8Z&F>j9X%#eLgq`mG-Gd(gs)Dy8X00gYH0a@bcte-`-dwUW~CAYT}!MO5*5bHVPd|>V{tl zgNs;@$O;#t&`FyY4}7AK5X=SwshUoDPzKbA*+a;tQD7O5-w^ zZ#pLfb%%dq&nOr&hg3gX!$*I?Pco1w?=fThrd8)?IH3#CM_EK?Ov+8qSdxzH(`fz~ zZdiJ$c%jsY6bV$-o3BdaICA>S(Rh`%j8bK*HC!4%OL>Y_iB~i~t;Hi9hFsib?Spiv z;70FKRj>Kmh)@_ej7JqiS};IWBZjlo>KLtW3AhcK=82A=E=whva7&PZo3COKiX+Gb zTf8H;dLXlCy3|U`M6pV2?3;qYGH1IQ&7LyotR}VI!dTCE3AZHuq?-E4`n3~SXJMyy zfFyLGQvbq+4pwm!$Q|;4f*Apil)H z7)8`8fW@-fqa#G>dHVv{Z+sq#%hsvSc{w5 z!1`?KQyU_lAtcBbQY!Fv_>N*(XrMTOrWI3WpG-d79mWcK%f89+;AN6zmR7bSs6!1{ zD~XVxd;EqL$$HBkEa3Q$*G&8x5nCF-_4|EdihcYpxgk&x1P5sM1Agv%hy%A8 zAg{C*lBJYJoC+_{&@wa}ywA22E@iE`cCV1`{|_ot{XwK3MyGg4KW{~{$)tRs@oWUn z0bmML6;+-P$g@bmrt$IWlmmD6-#xB@=zKT9^mIA5r&)r4MfJvdRg_Ce`lXCXcOL7V zf%prQpZ$f+@**9%nQDT+Ik1E0Q`NpG1=a(QAj@cOPUT75)iXEg4M!95Q_w1qrYsX9 zAi!a1lQvQAgTk@N^Cq4^(Vl05o+J88v!%CX-aE0+%4#r6LaIZ-K(0z2HGZx`y# z+DJf0{;qP{FZBKptbND1vb4P1ni<2t$%+Hu-|6~N1$8nCHS?KzN4~}H)MzZM4g5~> zy~fk(;32BJf77g7E)$M-%PEY#!q>MGM*Lke3}<>}_rEzhY~QFazV$~s@+f&88Ki|{ z4Luo*D0$EjJ#l112-wjeK|Ow-!6T^64x};vD@3=P!^H~*%aKZ%&_q)sg_)IR{E&jk zskKZy$P8E(oNxj1GS9t_2)D;`ATk0WA{Ysm-QzMl=*t933IobMxR`BmAYOJy$NO6a z5z9`CP!@iI#_Hxd{RD&L$)g{nQwBVMl`$n}yq*^PL{bZZ_2ljOsPFzPS2pW$_Z?jg zB-Rh(COU%ke?AQIavw6q5Je5FJIgR5gP}aV;FJ=(dfgrt_KXWtrEh>n?x#Ynji2L% ztxmeveI7J+n!9HRkD*SU02Tc8rvTCih|^>+QKpz&q{jZ_`?_+o+7XMnLo^gBA46xv zS|UTa6})6o*9z8^LXk^qHP77mjE}@b?#b)s1Kh1po&@5Al}>+#C>v0I-8r|g6*bu=GsFPTiT`xD8yvJhn2%BkBA=WJwJ%Cak$1}907s3obT-Nc zDm_(Jy+l82mi-4X&Vo4EdjVwaXwTPpGg9g1?(&AU*Tc`ce%ppWSGNt7{P*^JA z#%}^G6%S%w6K6zR&|xKp?3i0jT)0>B0zaWAU#ad4XGJ`p`B3UBTH6YYOF6S34~2l^ zPGxRb7=vshDs_j5?Tuaq^YwXcrk@WaChS0a!+Yd9#TWSuUTgZ*!HB#BB>3+S>Yri^>rBTEy`n{! zM2lv`=;QD{>_6u52)k&o?TM&lK?smLlfDBvpu#~R$-5JnO`r?~$OqWZEMf2&u|cnd z@^Q;Tpfh_hAv*S4)Ck8}Uu*f)=s!`5a zZ+GE<_@JkR(QvIgXPG+JE^XhkwsH2_*Y)4cY~BoeGmC`+Vj0?{JW8%eGyK@l&=I9H z;Eo)e>69^o*3Wn|eNj8qJXeU)po9bk56uJJwmFq|e})$0kr+nUrHxh=Uhp~@A_X%g zT9lKZ-FZ2$5bKyYegZWnn`araC9i@OX`pxBhT9P9M|yP1>_0{KXU`R@v}TvtJ)#9_ z+16NLfcwl?-GJ-VBr6s`ODk)emudBFUH3caD$f8z3JAhaiyZ?0+}`!?Jyfy&g6lGS z>)Fx-;K=}92x0=K-RhhvDVR@v+u$1aO3+8+Xu9tk{zWLTo(cKpo%z)8U6Kbayc?1C z4AXI|IixMLgg#ZwgD8dR|7Vc^O%lKzV-6)mOpYkx=;YY`l1asHn>c9vilGK+dRX5hz7sC^7#sava(y=yqoulgztH_ z#nY=9171jf2G0(TQatrB3MSyW={wbAOZJ`Hvf6;}KA-Hw!Vn)Uzb92Xo(4*UOeFWT z*wE3W+eWogjKnl=_FT_rvM)Wb+NTVGl3>(rBW;5Ksm^qmF8xH381peC6I4Jai7mu> z0m0Tehh3)Dy~k3Z1+EW&`w~5u9bbs`405TJ8(~+>5oKio0pu!f9o6P-JI{J@U#A_ceq=LIm!NkwsN)gVL$aRlPb0Pci*<|aD*Sh6`y z`01bD%RdVRd_cXgl*U%B;F|@Q$qictp@#c_>3D}x=Sl_t0|;(s4*zupKMjb)ii_P~ z4lbLtmH~m>*H@b*sDT1#gPLthh9SDVeIwfu3~=G>Lk4wo_WXWuua|p@xa|*YYFHB_ zqgA_xL@c;cYv^>C601 z7h=f{JCb>=K&6Fktj(kJ{FW+^$sU$yi zVYew0qLzD16N?X7)!?6bo&IW{Y!F9H%56(@Gd%)!!HJ9iA~XCYa9z-0RBJ zc}Am{Jk>s?H&S+Xw%xFiy{kvbuQAqh9XcvKgZe|DyvPWTcJw+QZF`OEzj>&xthfhw z;bHi*5W%~fXXBrcgH5XdE{XVy1QH&u7#!fJ(h~Rc>)Uo=1ssOHm*16?acifMxm;!} zVEm@?GIyiF0M;d31}5)^=E`QsuHmD_=?bQl?hjSsNxd{lvben#F!JKLEt-`7!XAnn z6tLv4J=|TX5_#j|dms?AC<1-m&h#?=;Q%qbo%2Cg+z+5Vh{MF}Pt2cpdL@#t{GM(I zqXy^B7Ognb7)1EGh&Tj9iBWn!3+Z8Cx*k?#z3;V*s2{btMteTZ0noDYpUV#2opwZQ zd2Q-;c}u^h0RTO*W3>4G7k611z4R@e)v^RG;Olh_KAtNOP>X=6!_0I084mYwq}Jm+ z-3+rI3a}U;+#pIs6E-k&P-uHLQR`r$zRQqS+jf`R|wR1iI(qBpp_RN zJCO9h%R(z<(yz(toK*1u|0WYJ^J2W+doYX;_um=-T+HPj&WmOe9!N}aL~l4`z83O7 zz~2n7+vz2sm;))IoV$mV!9ufsTn<#IJ~sf{Wng2v4$9&EXB*IE^!blzB6f-Z7C%Sz z%5;GXUe_lnyD<{znxEUdl5wpi_oC|6H|SrfBqukWCO7CMcdiCREsgy&=%aq}lQXeX zQBeV#hyV&UCGAoUNbjfkFd`B@|E`Po0!toj0JVLured|l*^Tm^vLPCzil2{Ea+mfW zv-h->2QTH7ak-X^$W9M0;4sQ4W82hLgU#XyVyAYy+4d^j$Cpw1Mw2t+Q`(Qdsm|Dl z-f}Jdfdosa{yvz1q~F*+Svl4Nd?a4kwcy{a(&9*ZBEmlEG!I4B_t)rU$%QU5Bv+RO5lIDceq1Yni-smCK+Z$gNy=rywobk6Q4GW<-K&-9Gxrqc) zWCWOqKs>$o^nA76`B%Q8TzybhTgT{H4T%{})sY!Pj@|QV`A)jQ`G8Ts zMa8wkyo&+w^N5mm14CgTJOEfFF5Jl4k_q@+34|28BeK#m@x|c6CY>e^67$9EjYSat z%S3#EPZipoU_Slt6M$0Q$Vc)H-2Wm|b8_=)C)TO%J*%V;u@mK3A2_3%kk-@G;&O3B2CyzN?%7CJTV z2d|VL=&{9jlsJl_GumfvFAKe^iN@?j_P8#wfU-o5VlJgF(nY)MiLkN`To?%0`fXu`b}1Yj~ai2yi+Q zQ*z)r&XXIjzXbETgT`0uCEFZ&HnX|B(S%6MYYYq49;K!eHW*ypI;c6$0GM^%~$`6d0%~{OvQfI9M9~2 z|0lnpwqL?_dvy>`j>c8~h+X%v^XLy02)uvTSlMe@Q~ijVOOCJ>BF{vKqxItKEh!qk0cj<*Jby?{#W4J6*<$Bv^T#nO{N zsQfeVe#U0X|Me9tL)4MO;cF>6;2_!E-DRu^1}yp?uBiKRoY6r*$Bxiz~%&M zmYun> z(PTl6o5%{xBvN&@R?P-3eF1lSQsp;llXJf%TN|OxFdfW6$I?yL6xQ|npp@`+zTMlG zkp;ONG{`>>rGzRBLl^Wb9adt!N3->PC3Ll137&W^SZ`y4X%k>R!ZjUFX4){xAP!!<&O0pJ$I_&eV|L%)1$wg;SW}{KU z>Z{n&wuwrF_*+Bu4zMnzt)DO~I9~J7iCAzb&Y)3MDpY+cofB36r~Jz>151QS%Y)Y5^5_u@9Js~uZGOYc+9i?Z z)MJQ^+L_lyLPiAhWZk+MM4XcnF zddvdJ&Ow&4sJ*?2CG6Gzw!K=G1x#7(L4hSS-a=m6&v!)Qx-Awj_kRE%OQKLX5qQ14S6PC-AD3z(!%F2tjzoL3^1mJe@H(Z+!;bMA@)|(X zTj*GGcri=6@7@J=%TZo%tzesWXcMX!^m?x;di_=OqV`ssr+UHY_G-aPAnIU$se2LA zep?`bL8wDYobdQ%MhD;WT;}q6Rgf^>`XX^Yn<1dsr7>O1`#MfGdKRtA zTh(7YWq|N-ru*16(zpk`;@4>r=Qedx1cCe%^`fu;=Y5dLg-X_1>(4{?>Yq`Jx;;dK z)l>X`1#6brOmtNUJU@#npKr0#lq z_5UO5t)trdg0A6EtUwDzS}5A4I23m+5Zv9}-3dY4QYaKFrMML@?k=Il-Q8V-yM*`l z_q@-y-nG80`(JXCn{(!z*|TS#nR#&=;2aXWIVFxDpWk))YI_KZ_xY$a%$yRH&F|WU z0Ckew)L$%N=)2cOofgE4T;%T;33Z}4ApgF@qGa=&n)T-g8$5tLo4?JZY)fZ-zb+N& z08Pq5AW(+!2mPIVnu*?V~;7Ur=Q)nd8i*`M9U zZ2fd&t_eREFA0aHe1)nN&L*l*MIT4#)Q_AP2ZN%ymUq6oqZMhZntmy2O;E%cQwi^= zD01^av?PsYs}kig_f$eZFhO6#u+`~Ozdeo2$`!N{a+cB$Vq+6wC`Hs!N$ES4zh3~V z9`0U@LqD!o8I1<~l3A-*EfJ{}OL-8x>8&l>D&yMHMs535&}*J@*XKrRtgB^&`7HUO zFCOm|?Fk0Tv2S6txmwVPsYR%EM*bQkR?bJZ|x& zcFB)?KNkwZ1cXR|&BBV!IA+M|`FsjWI^^V3J=?C8e>a6U5+@ZMl=^V6SVhuAzh6^R z)zOUPRXUtzzTL?=x+M-zplmPsQ)Q$JU!A2kebl=f9c0}ISPk)SSDGJtj&((Rl_6?V zf||$jlB=xC7yqB$09e=`LTP~@4%kq4<}UVkzO?>&mO;TJ4Bcb|@XC}$* z!(ct+f0_qj8O;l{n+Nb!TlpS0ba?#=aV*UIRNuw8|v)irS(4NiXBOyTvbTrV)`nBhkR)pdHnL6KdDCgDg*7KYY`9i z*9RdJy5{4#`%fU2Bjml%^OZcuq6=D3y4hq_`MLb1a{Pl4KF-QiX31m#sM`1OPZ21D z)!);|Mr`hnY`+$b@J^J{D~ci%HrVD7+A42nQn3LM=E~`(>;!->>7IU3JG)9C;Mnko zXO(3+@YRRMBzs~cb%!TEy(yw!Jw>qJxb-LDr6|kp_TK~}L;s#tOaQLrLA7{Np7m1J z_>3R6INL&-liRR-?N@YGRY1R-Yi^|GE zt=8m}9#pTpk0Y=r;{Tg8kvW_ z0jv-EPyQb==hH?o!P63pj?mT9pEm>ER|)aAQJWk0Kw1M3#b#BgsRCpukplG#y^j>* zRG^&{N8*>rGwB7;8*fInvQDuz$tR%SH+g+HBiiuNR#X#jwH$gCyM^YQPoP7-k_6C% zpZYGwh;g&G0bv>!a{#hFez4Ia=av?fIz*t9@`CVkc|elN0R7|MUcV9O06BnIA9`+l z##9@^vuL7!EXXv33UZ#Cp!E!EAk4#>v$J`ukUcXJG{f%y{>+@sS^V$kSsZI#0j%AW zgw*AqM~iU(N&P^PwhhzTY&JE(X5V7)YPZEFytu@j%yN0#2hNjB*=R|RYCL(H`sc4HpZ80o{kHvgqP*wc z-=h0|1Wgpc_dG8ePw!~loF*2YfQITrZ9N@~7T&DTI!)e!h6eRmLC;=2i}+0+a)yCY zx6Y$nn zA%o}>^f*r}G@}AHNgiY3lJ%n(zDvLQpYX;3dou8a^^DC@`P34%f38~{dtlw96?D3$ z{pj_lT0fJ+J336MdlEM5XW#Cr@9^>|y%RoBzNZ+*1gKWkOdvG|5PYbLV)@wJo;g&$ z?gsyJ$cf=wN+SJRN!O;d7ccv^&u;tzcy$5kkX*X{k6Nb(43OpHy_8TqE6KykLd_P* z2i5-Vq0uDVG3b#%(QwRsrtjFYa&m&e`R$})+90f?$9yDLS z69bqXNjsklG@wYqlT3+YZ}V3J8Ql?rlnHkLiOT#qNp9tL59y3bYZ9=d`ZPj*5O$W= z32N&d%m|#XRlmcFqIe2ga>t)nS~@mp!UCXB2Z)60bAhZr1%JkI$Qdc9qvLs2+t%?{ z)V7*_?ii~^DwP#$mxDJTm!=e39a3y%4rLH%8%k+cfwEguR`<%k;;yRBc*Zt*fjsv) zuXl!w_PMV_$IQx%*uy8Q^scmi#O}oDhf+kKsbdToh}E;3uDc~JKh^h=V&QlTqq8}u z$7x_{G!IWyz`Xiv}nsrc&CIs>e9zOy63j}+hY3D`k4YF#oKRc?Qel!5bSr2Z)D}rzco({Gp zHM_mW0}Tb_%ucu=QUwD9bly~S4{f1*xG0D{%!_r_Lfm11%;+MvK^hBOVu2vfO`w9I z9P8~uW-Jt70hCCdzw!Fa_o^+e@ZBNpcJ00dcf zU4e6D9f8%%ol#%`*nLyLbmML5o&b|_xO}u4{o9f8Eh`|-1WQi!pO977IG#o7f7MkQ zzdOxz;Q{{INsu=}Ld?DlK?MoEQ=1|u&eE9L^1zY8HxPi8)V!JQ76gpw1=gkVRBH?i z2z0m4`u=t?#`&GV&O-h(d1Id&lh6;O?Dne~pzGf@lx|< z>(99bMe=|DxIZ)dqQGyx$NI(;mfmc!sR4qvE`4{#xiMaU^t$@mofTX0+R9l1;t{Ci zeSaKzIxJ%4<2i=>#l$;-EXbxZQ3Q7Kw7y9R>Cz|bTioxj0mim53Xu6XUx_k{9P^)@ z0s;#Au>gt-J%|lhY}%rXl-cjP-v=kG=N z;h=b$G(`-;%YZvS3YLips}Odb*If?4I}|QdYOfinH*eaKI!Jr#T4D z%>;84yWtiEE!B~W5y0TQZM6K657uQ>xm`s8GjMLT%+%K-BxbeHIBqk(ugI}VM@VR$mF%aTmdS?$;fzEQr%Vr{VMzE ze7in2xt$z{P4PzSO%yt)d+t5@rBsZ9alry`+d)={mj==v`s4>+NZ9Mv+lwfUl$OTA z0#v~d|0XhYUeD;cf?xc(&D$4^(S?lNWcy13=2e`mu^Fk~;~GsslME>;zOB`0@+xDb z)%q{~rN-)aPrmO#fH{N!Xa(`kc8*m!gC97F-{OCD*!?xyqAvrnqP~^{k36#nJPao3J;|@cW>$vtxO5aDqI!nO0ur*Uyn<8AF42BWIoQn%P-3 z0?LDNMEUuTBPGj-HP^RhPouTt1IcT_zNu#2f`NMEQ%|`fY8T%8n`oInZM~!G;v*Ir z-z((9;CN|cp)O2dW9_8u@cv+Lj5x|jMEBaxZ}ou}t%_(ub$#|#q#fqOmpFZE3Vpxo zA?;Qc)2+tD%=Q9(Pi`pa1Do?hWl^A!|7}#ap?tO}=V#8hpD@3YZ9`jIz3Fa;A9p5N z83yTldHhWT!skxk;hG#|_2vNE{)i%Q=mX8*-_bvLYne{b*YmQ|w+A}Ay4n)2+5pV% zS&Md|_gpXSk41%m{XIx9gf6~MlQdFI+^Z$|KYXcbJ}duP;wvtS%gtO3{Glal9Vc(t z&#JIIQ*sShb3n#Npq2L12wIfMYLsxkL52g2$OZP0G=_1W+cvYAR zFp3aUS*inmWDyEv^bQH|@!7bomk1EdBbem+gHL`r>r> z`20#_770=5g*W=5j12P)CpSxVgoCded1dU(lK_gxOu0So7 z-zEJO&n0G)x2MRP6f*;nw1MR`$)H*_Bdzn>BBR*8p`(D(GZ?>?(#Gba^{>~kxP(37{gue7-Qx;9L#m<(?GcdBWAb$jJxTQEf)Z zR^>fZ>mKPL7-N21G`#E^MhgPXgmK=WA2|e0&xG#SMjh2o!cVU*N%Q8DJUSN7r9N|$ z+Wp>4Vg$QI$bH*P6zU9n`4u1Eb|cpurd9f1dG@XCJ5M^GhL)+RsaD}rlk>UjzFVOE zMq}gckRVjel8&lH9Uq@P)>l-}SO!wKXWz?k6iyiURk+OLr)z5KCgS>jdc2G$b?N+e zSCLj3>9-zcDBpctObhympVgr23-ol>^c?odT6KazP#ycsa--^*l@3x)nx(Y3hW#2u z`a3&gdOqCN@__FzIYwE7vHxX4O|s=cQgOVzEhm?k&5cLdTK9=G zJbwRZp`fH_zS1*=#If77+BNB-yFZ_pLPl9sPfx3$!SYSk$stmC(5OPAzOz|^02ro} z)<~DIe_(gNZX8GOr%-Z!A&82*XzQaqXyp&g zyWUxTp;FK?+NjLBv09fp^fd4`XFf-kU>55kO)$;J->ombzJ7A%54zOrx?j)X&*x0` z89~6E8k!7vls{^#YdP?!GIpvjXVKev(Gyhf{M`!=Alqqz7S?y4dHOnRNr9PBc(Ed= z;l87U#%3_%uuQzB#o29HxcB2xO>d)8MD)ZPioh9PFbMmtUiNtIa%-Jot+&w!0$zBT z4Z2N&H!VM}kY*9TOqE5si;F(I{uXFlH~h3vTh1)l`Wb}idn!49p2TmULrE9}nCr-3 zpwDNdR8VdvD4qZ?L1}4et10$9!!g^K7X4HmX4UgkxG}S-mkYnXOKD!-C2xvRH-{?l zW977IN_74FuC*(vCj-NIiVu?u{J_JJ9O@_BC;Yf^k%TNQ8O;;a&GcfvZBZj7Yi=0% zuu*?quFyf&)!Xw9lrc<1?T8Ye)3x-S$E6lQuiS8qi}M_E}0df>DV5gIX)He4f};Hn`7c`3apfoL$##u=KG%=2L19?x}p`0z8pe8uj)?X zE+{A{usbLJ$IO=^jGPKspJt@JURhs7p6qfCaEt=uO3-Sv8qV6Gx3Tu`w^18iDs*TC zO;ZE=NqL>b+QxC7q*G3@K%S);J#Kc&cReK~AZP}@xZQvJt=&(Wr)DBr_%UZ|EpADn zqP1VejtZ6=RdBxs5n;2D^E5XP&CiF2Cce?T^RLP+Jvc(tI9`+m@HjG_9XtGKs9T0< zXBSjVT%dqJ7#g`6re}NXJAXUZmL2h_Zn+jo#H`&PYDL~yRCB)!9sssg-|wtrdBpKI;sZ{i;#&Q@3IbuVtKoZ`E)O%%UTQG zWC5!kA?PJpqVtNdd=bXsdYtGaGZ8JG`RkL`*DPt4k8x~b z9*O^)g6;_C+P1ccNQJd3MCnlW*|)ztRw@oUYs1*58UD>fZqv;AXT_pSJ)-A1G>=|$ z-;MmejFE6hL-Yt1P0c=NEga}CAPpIK{QKKl1bI_stG*neSIch|JPF?6@0^@SjrE^j zol`_}jJ=Xl@$w51k9s;c{M~1xZ}SxB>Wi4K(=THDSYfFn4GHVnrlSO$-rfZ8R$zd< zKZfMHzZo^LGLSx=z63>T9EDWsUEi4hN)V7@Hi3 zLbRwJslmYf*OhH*Pm*qB8rHPafNQt}ek6Up+qzN1s>R@@iIKq&a!wZiXZXO}`-KRG z3RLJ6iEp1~Y>?mF-s9DILVm{jp00KN02cpX@=Ke7rU-_?!MUg?T*2AEXDcWR#Uj29 zY2QWvhy%{ykHi=g&kiM>?He0IF=}5szDXY#HfsUvCTbMY5sRtKk?8UX6)oB-GaL=gadm3Y~3lj>w+ZmSBgp2ZwhOv&R1}EV~fqPuZ_?=^EFXc{1^3 zZF&0fvSxvW`17TTHEjm_3;Y;b|HaSepNu<{iq+^i=cjijVY>XmkQ`+hGL!f{ozKA9 z=PQm^_G#z1l5l6h7ycgZ{vjZtcgtk=9FTt zMUV7Xb4$fv=-uiyqSqi`5Q2|ZNv-rMiARG*<8+dHl(VPEiFRD-TO!5 zi1Ab0xRAYB)9FU4Vw?oShDLVRYb&jILq_!rF5xym0$1+2D^hNayLILZgrzy$SQ!eP zq4o`KCM5DDw($+Q4Hk)}IdN z8&Y#w33#N3jARPH16#gccJFzVGqJckI2))!-%eZy(55{xcCqLijCSe#aZLl&M7#hI zkx|<}5&1#+?j_|n-x>gzCVImNzqoYVdYd6?-rm1|a4HclZ(#(zLT!1ghcs~}xyeyu zz{d&}k39pXf~OsAbeN-3PEIatr|K;D;{E$!ymk#jq;kgjS+ZB_X~s6wV*%VfU%hO& zS2!ud_#^Y}uNM%+;$RlMc|!EFNKT%(>&OsJ^3qoTaz#@#_#JE%pnkn9O`4+n{kjnM z9=WNQJJ$q&0BxqqGA)b17me?r+6EsURr06FjpI%OREhCpGBpcaQ$6bbhTxD*If&Xd z(<~O2@;!KU?wZ@f>3y<~<}y<(cgv&?;12b^Tk%_S!9l*Pm5+;>I?=L=LWq+1C0ybKL}_P9j%ZBB1=^2|j@HKDd~&BL>E7d0QLLXDq%ziU!G zy6X|RShq}~l)iOc3iJA;$bqFpPAI~}NVJ9H5BQ0#<9aQl``PJ&{^eApG{5_RSw&WM zyfh1LohJM74u*W@|Kd_j0?_x^*zqpysjpwqIPo^M;lgSzM$^_D5uHvQX76!)wiatx z2exk6trM9WE%$33bXjUC;AOyU4v4w-=AvTa&DWBU%nVlI8Gq|h6Y)W#0V|H84|d!L z(rnN56<$~_|3pNji9t%x$pTn47ZsfcjhipV^3>i(;(2c(SSKVWZ!7KmM;q=tNCLC+ zvJm)!!?hRigUNSC#GVabSr+ThE+XweXUr4rEp8Veq`8gyjUUSo&UqarJL7(c3W>(? zKR!k+?WoVV(tc3FHf*Xp@Th4m)-%3T%-z82&nMuvDD(GMPWlwu!zu1|6V7^hjgPO0 zwd{Kn-rP^s()Xq7t^BEqT$Ml(Tls!%rVqucLgd%SD_q2KhbhJ#EheNn<;d!(w{X+$Tx*@& zdQ`9?E!h7iy+pDuG=0Hs$GPd=_bWO@i-dL_kzy`L)oXlK6roGzbHxD8lI@3Xk%u3@ z0e|CdSkKuMgzvz>BFCb(*qXG3l7mr!56Es`rJSo4{e zb9Rtr6L@Jzk34r#&UiUh?^-5OHxu&C{CFohrA98UFiSjBqfp>3aC{JeWnHo4DISy@ zu1c;K=bO#|CgvLEE>u_;41%++t+BJoY2?VzY||zx|x) z&myc6_Ub)LgNA`y*!~eQeua(J%D`&~f9_Sd?NkQ~@0t7RSiZReel zh}3&M=YC7=>2BWUW~W;Z9B3cNH^?&+bP4uA8q}5zAN~{u8qot)Xf$TS;?BgGm&$s2 z5+#QW6KQ^|AIu|aw;u7+X@~?A^V)EN;m+IVr7lr`7dabcu|9wyzczPsLAOK zcv@1w{rwBEu&&{!ycr|=?1CfVrIYFx=#g(1IxASwR z`B2Pfvzb<5HPaJs#)m#|!TPAP=AoX3+~;51$QDAUM@EW zeyf4YHrJcY{Own7+ZJJ`g(D5}*29b?YH9FW<-BMod3$@M!`{?|Sc5<~>`eTL_36o7 z8~uhTOytlB2r1E2n??f=KEtZ^mmzrTjGfV~c-Iz})Mq5yJAacP5D)!%B?v}Io;98) zhlgiUOZ{|-{sRkpum1=`an#|K5@R6pRu%R5?5yXnF0 zX^Yf!IW|VeydsPYXd-MA zVS-+PUAmvDY!+62UI8mw9kq_H)ZW%WdiA7z z?^-Sby5w~@f8BF-J$T!C9aFkOQm*Faxf`GLSKT=EJ)gc_!7g~O(a2-(Axr%ZSHum6 zzWm*6O<~kZX;ul_=W&xEp()AyVK7*DchDW7HfN`RzKn^HwgS7w_q_jkEd%b`f1- zv5E9@AbnZ;W0->)u2#KgBJqIfjDU&kM~hjxI{#|3wITC{99xaQKZ_0FsfiEmLC9Kz z*FAJ2^OTgQu*{=aoMJMO-B&~_&6ATe2pnx$+~>VICnQx zacpkCK$-QdoK7I^aOG`5h4vm0)-Tpm|6LOApP5)Fo~*650UF8HN){pb*^F0br@45w z;kM(=i;MRyd9w(+C3(%h@r3?J-%n;QcYgEgOurpJggp7whCK;t^Ot17%@|tGJG?8e zx%di%*ti>MF9*E$8mz5!SKB`|oo!f(4Kw5T)5C)r%u&5CkrKhn zqhX7olxe8VYB)X2-W%_EldA`IJ5L$(q6U{ey}ZUFZTt;x+R?iHCi}qZQe-ojJSYi} zr-n^eJC%O~r$k7kA6Mvh9cUI??>o6Ae(!oapN&pu)p9A|K1D9#yI#Lp45Qq@4r#ns z&p$1yS zp(SSQ-Vb0~6cyjQ7i$zI2HoW;W_QyGG$^JY@H!UOHKk-sa|u4){sKmXHJtVJKU@2@ zDX-lHnJuR-&kSJr*}e)?x+l|o0?JFY%<7tL04&m|rPS!GxXq5@%s*faA8zv&p8@IL zaw>RHb4xa`Lytc>H3?j4@A;%#!hk{`-P7G)2QO=s9536bLtCr-x1D)fMDmbWGp`eL z1BG4~m9HI?>@lB|TJpE(=kQhLD}fs&_dXZZ{gpWC>P)$j9xK9Z_(|)l>~x>0>5p+s z3g`?+UySCwtnecSXZ2Q@OQgx;%r#-xQqto*II3izXN2^v2MSEN0Nr$x%zJes)7{s^ z3o1jaF=KPda4!xXdu^DO>x#tZmFBt%l$RoZt%rr48-{#s{LFLJc895ND>t3By^O`7 zJ5)#qPCkf06UYKUz)(+**{jb$fOfape%Ve~x@@yMW}+gR!ZYbP265))rNV=4q6UTd7-~@EZiBhC<)tDK?%Z8wnJr!@Z^>zff`__hy>q*M z62LG<-mDz5{EAYLb6_3j(-MJo<;Xe&Iwxcr|A~P!#6w-+mAe!BMGHT!N}g{kfx!~% zb@{ceakCJJdVLB)H*z4rQbD*F7PKr!HUA`(D3iDM?JSK_mrU65tyih0<>;`owqcHx=A$GoA zHgxZzKV2~WE_5zl|GKr~=Em8W%|=Am-DP)^H?JzOp2VQKpt`8#{NZKMGH+{q3l~g2 z$Xh-5Mw2$U`gC!lxFE|qYfuB!(B&{KBye&AAV ziy80MY7lOFSzqCMrCB@&l%3h>Aw!W4wGmQ1dbSl-wS0FL5Bf3^Ba=T)li%!ozjypU zq_hyES@(9J*gZ~Lu<1~zv2AqNc}^pi6KOP_7!*UsjcHMd2GoO<{3068P;dTx{GB#S zLQ3jUg7~Hz%+z%C5;0$*^$PS19{?o#i-_(jcQ@r!aGvF8#rm)?r##y_L&|-Zn^{E! zYK{W9WW3g(X>`QxEYEeS6#vY1o3i9Sq8=I0u8NS@-`OGW>-P&-s{rUo>PkXhJw1Gg z;~=M5H|A^%8DnE%H1#2lXzk>3Ob?K?f*qdtA5pN5oY@CnX?=l?+OK^o zUF|Wdz4~x{Qiqqftu;|Me$5ad*01g4XK(r4wlKA0`BqgHf~51U#QE0RvNRzHiJcL} z(`$8GkGJ%bmsSap?r?UW#P7bcu-Y&#ENoqrgbW?b;+y_Qf9$}8;wY*fd!C=n&n9pi zLI`Eu!+Q{7r7PFt&kMG#Q|~u|)CV$3!)@+~ctpX>htpP9pJT+b3uKjM%@?99fuJn` zUHHr;0Z5fp&Yiv>aaA~_!w>#~1}CYTE6tPMl@Ctk-&G_!9)*5G8(KVb1dej+=g^OF zh_+wAVxr&kOAy6B1PS?!v@}X_o(b!b;Q9-${UbRa#%=5(g0dhR0V?KhpZ&{^4IUk@`>eUsjE2!v!9Ee4l_oDm*~g?qb(aOL0*#ii`V|K z(HZXgcE_P~xoj(VX4@H?Bol-Dv3mP{Hi%9k7R%AFV^^FRYdXB z6jkWXxoDq!k;d>~I(PR_;9;~Kt(eDMl;Qq#=U2VDe0C5h8~7{$d4Mku$N5X?B=`#C zF3@5Wa4|o|IN8&mVIujz*~1$uT{sb_{FiRk6X>D0b?0gBK>O5u-J@6OG2(ofMU%l% ze@3SZZS)@n(yok`Pi#nE>nICi!?5gkyYf%$f{XwzJ}kpH>?!uEOh-8EK>XzLT}B47 zd-1JGU?5!4nfhL=c@h_FlP&Zf$Yl1qxwyEPz)rIzljK^ADEHS^R#q6)&ixp&Szr1H z@bU2-k552yv_tuDamUrUAdo`GaSC#Ba=x{@z<4|Lo*%(a((}WEfEQ|bd-N+=`S}nv z^8yVbvH|m@?Ulc)(zG(C6F#xTxhMbbLaXGlQ_WBlb`A70myq8(@S(vJdjGT;1Emb&Dv&yKTJ0B+lZJFD!dTB zy|@W1v=7}GZ{LM{!B+*9l7rA|V9GafXv3|%1 zphVWe8$SYkRs8}!weM&N8E}*1pe8t=~muR>hh8`deO(`dm{w{46m z1sJaYS8_76_J~Vr1fap^$g7^<6Hx{$m5Jup-+J9z7w`j_8el8`x8Ee3(v3O?II`mZ zB()T$B~}WX^F=ld9RjZqrmx1&-zW}sTV5Rqb`Wkd{DfgSfWf7F$^{yX7XG|0?C?0Q z_=cDB-bIHY@>G(FZ|ztY$ltS2P@nI;Ti1sU71{8bQPdEIWl`A(H?iO7@ii^qeB1c0_Hdz95$oc1TVbJ1kg@X)7{s?gAO+9 ztiB1FnBG3$O&9xp-dBu7*n`^#PuiYx_zO7SdKhWOaR_dIsc=;V(A0DoVTI8I?HciW z>a6}k`+s&o5+F(ZjgzA?QMVr0xu%4e%8N>xO*&K!9!rbmJT*&peSiGt*P8guVhjh z|H{W<70@j`eimTnu50;&p7rgpQ*F2y6#P*&v^NX*o0j|y3hQ92sKedMh!4L>aZ|^AW znYr#CGg*zfAE{oE8ifi!&LcAoGId5+g)6T|h2)&U66WS)F?9F^mbR7 zx~Y?iXrn_zL!J2H4u(Xa2rJQr=E>Pf%Y8ZYIIUTt zbT(F3or>l{epP6cz6E!$=$z3crT-a4(a_;e&=ReJcZPk4vqLIcu6=~s6 zmW6ag61faycw$9usw=(aj$@n)4dhi2a2qfhbUX2&wWHfhvY5{sD53`2-SW*y+67n! z0rc)K7DsCPe3cj(2n48^xgn%R-$vhUG1}_xA$YV9WBi~MaG?*kRS)t;Cg#~w_y^fY zZ4nod`Agfak0+0Y#O@M5S#3Nn&7_8yfIwvX=rkUfs^qs(VQsC zYhiK!cdU+KJ~UK_A9a(NI30a_y2aUZn3t!P5WnFV)9+n)yKCNGFS~t;)dPNn+cJZL z;N^+Rsx>D#LXV?Mko zm*NMR;80Q?k{Rnhnl0=BfP*+(K9MH7D^rz~x6UN}F@laSG`oFO<`2x=_|L`u8X_<4 z4Jd*KxTR@n5!w4kIq8VJUt`j9E}k2wyF+>@cq{A;Xlw zP|>Lpau;IGQB)}k|1=c(>c?mKO1xTU02k(d?q>W*Y?$!-vpp?0;|B$ICfPaLyXf&)0H`T8y^ou(f+!A zSgsRm3gF)@m!6=Hqvq>O`!EBR@#5dDxip$yYc%Km6zu>sGbnw0 zCKn23OnLTAC|6{RI;@7lLQa0^+c1|zeJ-c7qTSB28KydGv6a590=@s~wkmsLr(0u@P8-E@H@=Ud4-wZC{ucI(IHT>OzwjXEq)r4(57JTGxo>bV z&Y1A0s;xziz_y1zHCF-P-hTJMcm_ydVx$=b{xg?bvWvNRdtOouzvmgELEW2C_s6y! zYjhozqDqrCV;vl>M$4H}K>F8jD}6rr(xi*^BJxUKq!(K$)N4r2d0Hm+w4*Y>&^9iq zHyE-(AG+C;sJw6yq3|@jQk+qqV7t@ma=urxOy<=lNoXhqnf=n)PJs8Q_+hOhqZ$p3 zjQM8E{KN}<)og`SjoBroxDK*%=Q77P!A~MQm6Vi#5)nY+7{~^9{R_62#{fqPhTkI0 z*Sl_Dif3oZAt+;*pPyobYCEm&4i`?8_F5klppD~?^l&YpX6=us%!Cwi|3T?y7+_L* z56@#Mv?3Cn>c(NnciF2WsAeafy{cpNsh6Cte<~c+!che`ax})w2wXSZie=^eQaaKMETV&*``{Ii#rXztE81+1)GpL zO;|6dx`v{z{O8^5YKzQ|?O;1~3tF0|>Y5NP%Noc%+j?B{mul_tSANCpM5Q=U4nO4x zz%UkwBX$9wm!Xm%zAGUOs=^aWuK^!rpv#pTKpMvNZvu#azRtN$3pkG^4e+G^{XnXy z2;{3AZ9u04Br&%ThX5nV*d=q6>5K+cCv|r6p|BT2X}e8L zO+~(0@(~iGyXs&7H+p5i<=^j*lm{Q9`sIw*E*IO|7EISpaNF-!zYpqE0=s$or*^O3 z-d;y!nGXN)?vgtr`pnM|ms~vk%A&>V3hQv~R~M%x;tJLzU;a@sj#SN`oOj#Y4JaEy zOZe>7Q+{#XbfC{yu0mX^lhSQttFof5KOZZsyfcY8pz^+`wugpFyHLT78tgS84IBHk zzVF}gl=?J+N6a8TZ+D=UX1wkKe!Lg)N1jwmVo!-Wq5uENBDyMOTODeTfbalF#K);= zx^8|Gcb&X;*hkC*YN*QeBcfg)AO@%%XDDx7Z|noWLFSp=nG=WsM6+p?zvDkIWXNQm zHPb4Bi7R{I;}6_eP)W$N9sK2O~KgyV@RVKdM%wVvmoBF628ZMjbBjDn@hXD(gv;C?}mD>g5m8a+Wc#>~uwz_lOp<2FS(D-P(Hq-I>D zT!C6)VUn$3kZT&>sN*TMXqjzkENvq|8v;QlN$MZ_lc$U?o?_$RgxNff-s_5G#upHP znDhUC;UhgjO15RTcfnZ13?6I3U(M7~zscGIctSD5s@kfe@K8OcCA${SqU{J>x_J~} zXl`zA>zf)Bw_VvUx>A8)q~T4`A}*BUbx}DB5h&z~tz`!}XIOP!ZEarORDI~rQGiYZ zhMabJRD{PVweG7R{>=q$ya-P~bKR_Vm620vwn+hj6kxATz(fN=F~9hjNho0gZ+3!k z$MDi#<1MhtsX|DlOK>HoKR(z}MFF@0z`;VL@TMlg`DskJ4N2sf$&f&hG9=Qu!(%ZxesQ0;T+-f&xqN0_AiEJoI?rbc6pXlOjvDLGJ$!(RFsm6HY z`+h+F76WWv=bBDbNjSOv|EpCvmI0RRZHO9p%*IYYK%nqvVN}#m@|+&H@rv5$ZW^`? zH0|1KWvB)MbB$RntG%C=(qfT<{L#OZ8$srBHK%(I#umz8;;&Wy{%ThUpmB2=^e|^D z^?q}KALoYC!Ejn<6@lP3{rjcW+RxvQR>3~SC82@~3rY|>JJoK% z64Rrj>?=E^4Xfe5{56@%LW0Z6uPss9)FPH&&%yy*as;pQh4xWyz%K z!JSzsmvp*he0W#cia#+wzdxwxtW*0o^TV5&3iGe%tWQDlnueOaI0Xp8anoom7;ks0>k*h)H= zeIOJ3X&>tij=QR8W&j1j0o8DLCpZg$6TnP#!pr9W;ls@UkH-WMsO`@r}OCV zstZt<2wziMbLscE&iyUun7*)xk!+6(LiFmD)xI938sNGCo3C#aQGx)IMcyOmyLZZ4 z%YqG(;si|h<5!Ecr+sX=!K()hQkLBT%jfsOgop0sr~m-(mdsP%;C< zSMr%pKAGHIBBmIzKqe4I;F|%RRFe5VL`6zXjlkVyC&*B%o&xl&b7@gODF`J$HBtq@ z-%j!iYY2qozd`czYCV=gsXajj$#W=Xb}93NKqjTmXwR{*mM#vxfbRi+i;D8{@}kW; zt<3scr9**j;+r()uV4)3Nf92)Ur#ov@&|ka78aHjB@kvk{=9)Y-YL#&p@Xt9rhIRmT7S;V5LID8UfS!dpzN43*C*sf-FNeGo~j@=v!`gEQ3AyUJfKiAs^9+vx}0v%rYE5C z;zCBtKHtE%U;uvK#=qLiDFM?(#mq&i7(EXkxYLT;5GRRC`*)E?yj!>FapB8)}5Y+C};F)q$K>u(~m=ccx@>0xHd4ZoX%*(&*%nTIQ zi&|qn1A!=$(|~bZJG<$EHPA&7kyWVGd-@7=C+f-ygZ%9eLP%6@V20LYBWjERQ_hR{LYL^{OSAU5jobVD0F{GV?L!q$M!9qX~((XFktPX@n% zLgpg{29tmPYQ^{XdC^JnZRExsm|9m)|AnbwP+Lz35pef!VB;C6Ljo)x{M@X#q<>eM zF^*EiFOi>xmFb&qk6>KnYy+sBhVvn)W>joaGj5>0;9A_LGzUwfx1AMg74#&4f`H&1tVZ1QyR@o$vzLSG_%YzIgIYijTNb>w29 zSnm1KyruvUJQdRuBn-`VJT2K;Xoz&|6)J6+}+*X?ct0%GgA=g zSY~c+?u)pol2X*kfQGZzOx8=&@OaE2+!|9;6a2Zn#uP-!dqvN=yjy)l$=uO?-p#Ef z{F+REn)_DMfQ6Bn&voy5v~_%OeSMvVpjXF68V$r&tf2l|gONMB5LhzX8*VT)H8oSy z%-$OS9jLd=02faIHl@bZMWl3{m%K0KmHN+z4yJ>nl*9FH8aXtgokr&v_e99v^W$#u#g6(5-rqbK0Y-? z9o-A2bIMS2FcZQHuzBBI?>ye%_-r!n9v+Uk%DL_9F$gvmB=>x2Xbd9h$(a8_61Xik zbTXx8J;~byX4yS;{d+b#I-1~v3K=*64j`08QaO}aQUXMKrXjApNZD?Cw;^kqnEwJ{ z?E?l+{Nr`R<8{V;T>FUgRnDT!_G?G`Z4dvnXffryIsyLhTB%8+5Qi%4jx-30qj(V% zC`l2Okn&j06bu7Fdu-m>;o;#m7UV`>pPY;mN{Sq%$TC#U(_}@@QLDYY{p2kVMwj#y2(+NA#x-tyRpnT)COZ${v^zYSOD#pU8k3A2^Pe z)9qgZb7hnH9Sr5a>6EC=&;9G@h~@kww6ugxEhy0pSJIUo+zyPYoB7jGhK!__KU9jT zC@as8j-JqxkM~SoIJmlAA!C!CU7V0-?g$?GA!8rD6hKr_bl3o}eO%nq=YMFy!9I@y zOCnh@gM(VsOxTL=-v=Endo$UC#Q-j@>d(y&Z>1X$O*LW*fRCVuK$bD$BtoO$=m5%>wQfa(JoZ~_FciixSKO%uG; zjW4BlSLDv#TF1a!lPL+I1-D#r^J*rL>ExChxi~pls$YAVtfHivlP#20%vD|e@@4Az z3ftu1EKcyMiz+>qA4XO=DHJmEd=EpKs z3ajSgKuJzUg#BeIr@+VG&uYpruz+_CjM8(n=7>MMZ(TPBd_lR)$2~`cpQ973Fx6RE z@8-vTdt$Nv;VSCV0rNc94mZD_C;9DD6lgj5B0yCD#0m@YELN6Qu~MpF$1%J4 z?(gpts&;_iy?VHLcqo*K&tTf`NF$T*3xKmFsP2#qe~D6If@TX?M}Q1odErklMF>Ac^O|mw|p!s!GNFN4zTZVCDt+Eg&&Kc8qTSX*6+QY9vZ1 zXBB{ru9i~0Xx6@F@5mM;#Sg*(KE^u~cnBf}3R_J_kv6uFA-JzyZ|&fqbL|o>@M9e4 zow87TK!~7TaLA?L8%FE!9%~ya_N9API?ax zlJEO=zu{n%PG2WmfZ_qF7~3W?#`;pwIU61WtG%hgD<8ELndBI7malvXpSChI!J-yW z&Ii)}o|!!xO5tOOjPR~G4g}<`4cy>h2K9lweF~sZX2}0rWI+%P8ajLs_y27?NE6bD zg0?=K)SJ%5Frrr7P779wsjN=PL0r*h%81Yb2WsXWk6(}AN0B1g0w|9n6QNIab1%}CHdaEV|3jm4qyti<3t*7DJ6P36xcu% z_+JwnAV_VT#lLR@bZ#wRaMz@ImVlSAZ>S(VH= zxy**}R8(kXxIZTnvm+Ksz(C(|feiy5q+DLV@U64A_uJ*pbWs0`fL#dyt_pO;WAK!?4GGc;@2zF!2WJrE>t(-7Bw93Jtqx~? z!-Cr00ox`9&=J`o;7J2Doi*y`@2^?T3eJZs&|tKd+6QfFr8b?Pikfk_PRXX9}>&%+cA-AsO<)hBJN%L7)BPrF`Q&9&HF%+DjRRANN z)zJXNe;aW4|2HeVMp*pl#}=;19t<9DK=-7X?bT}-Ywb`e{)bVDeuM3c(3{E9^oz*r z6AXlqr9c}PIJ}jd^8V@YQcwV?d`irsfY~|0NIotmGBGwqu)laVHG#xj_}8gELG-`q zLGuL8Y|?ti`lesM>_Lp7#)t+%)-2j_$Q+!Sc%MV8 zocBt=D+1?qQxO+}Bp9N}6=d4s2^tFNH-N1J>a2@HksS?rUH4@fDK_$bPbN}5or`~QQ&az6&E8xCCj>SAry8fG z;_&;me@u!g{~a#MQPU(S9G!TbxBn6Et05kW7$p4GI&`~h2TXEzFR5RFBp^_C z8X+wuiX}bL(s+4k78FSp{Mu23HvXy^AkQcKCS{1k!@~oR%3dZP2;9}e`;z_-=f*Ic z%&7cNkoK`FCe+3Iq-u*4@=ns-N-D0ft|rUYEuy9(egY`KfCQoKAIK8}G@s}SN4J%k zwAl&9fYBh%6KJ&Pv&Sf)(ma2v6)`X{@R5X1_5q)p^1QG0C8PP-dR^t5*4$&fR-f=~ z<8LZn_8AXen)S=DpXB82+CH`Jch@H!Zo&|OUr5LqNGhs7-W&`j*9L9}XI9i}3NYB` zmh6GMxjeOx1_w_p0_Z;mD`iRS%xM#gK#~q2r9N!l4kzMhq=u(jMkg}J4N51ut9r^u zZT`ExNAN0yF_77f7X>z0t3)CXL`kc;a;V;atJyQ@PU0uVKF$=uW8t*fUfk~md5c4= z+<XV}$@%V> zV4_Nr<{WohG?+#oE++O*qIoIPm5HT{Rbm9Bp0M7+>|+A96O#jKrIEdjt}dYF^QR6? z0ad1vCYqr@i?g{E1^l+}AF>!fei&$QcA(JAx{0>*9}^K<@4c#8@znWG+0Z#PZO{6f z|3)^b9r$MMR5|tE9TNUe!EBB~y-o}8#jRpNT-D$}VN@?BX7Zsohk7kFEb*@~E|#Lk z*Nqx0P=nJ;=fQvRnF0kF*?Xr`Lt2**`VNOJ25fg=eog5i;DZ{8;Xo3E{~tbLS^e+s z=IlRwBxa3D@^2ny6tyZQT^wCvP`yoG)4I904#*wA@5_ZW6N^qJX3lJHZ_mJV%N+rI z3#enmVhVP4W%z4q+UpSz)Y11kkQ(sG_JSdZ(W1>p3u!S3g1*PJF z5*amD9ItXvu84xwH5o*xK5^QDbv_ysug>~JX?Qvg6trzUW3FN5 z|636BKldmH7*~{R-nh+dLs()6apS+6p3c>pQY45}!ri^I1!Fv64Q%PCE*c~OQ=lRx zCYmv_F4zTjPom~qK(@@YjWe@XUtvRafch^5m;ni>;ma^#hC*TA%@t75(e<{sw`)-W z$1{tbrK~P9KYOXZu7(*ABK-hxH*hM%CJLq>otM3ziWHCY5%!{P*IF<@rFm>0i0q`V z1=mnF4LbTo84=+L;PpbaOH>NAs^NZh{2wlWwoiUZkw98%B`6%3!UgiB*IKUdMI;g& zQBbK4s=Y~uhb$~?s3-uX2QYIDtLo#ayr3zg<-NWohAKCLC?51{Zo zF0D2R8o$AYQUd@2@O%^3zSRp8;v%MpQ~ zY$JpjXTE_1poRdXL(sXHeTp1*L5l?j)n}k89DqK$1pwO#>HIB1+m?@mU7f-QMuGz` zsD=pvK#NHJqGvW*J+Ge{o^sLxkjV#lN33uez@6o!f+dK}mEU6P$VmNspLhrR--5Az zj}i43UA+N4RaLMcNRV+YcqAbox2EJ7^gY7iHmB08m6Wd|aHKj1n{$j+@BPi1UMS~wK zo$w9#Ue?;k5LzvG2o+V1n2L_4h4KE&LiN5_F|ma|W!a<5rt*!)1H&+F+d2QQ{{Ylj zU73W%uP6GGbg|CzQI#N*tZjLL>OVmk4s8Z`pGSNQIIn_{aUd&S(sOHi?>!0?`d%kq z&A(L0DW(SF06|NJQm$ZFX#V8b{_66QA`{q?sB%~HWTjC+g{3fpZ-)~>f+x^^0ytl@ z3ok!8pQkScTr`SaLj>(a9R5uOsCv(j~Tt38uAH z)5Ag9!ASalNC>vx&T#xka0vH=2{75yg4!;hO7tn%gh8cn^p78)Ndy#1z!+}etfHb) z)mC{Gs+UTmJX?`HnlGEEIpe*o9;;59@U1P8BcT2j^b55T_Gz?XV5ti2e(QV1EYVRp zu@p%mgf}q0Pwtg`H&rwaKouqy7i~&@0HoI*>$O>*F}npU#x2;NZ{Ps>MrGHO zh6gq^H;=r;H`rfs@Mv)c@zQ+Fk6Viv1~jpWG~C+t>WE<#ZbDGZC|mgZd{8=Vg{Ftd zMu!ZVLC#k2Ws@x%!0ASC(gCzI5)vIZ*bK3ye0AyTr-!w7$u9nLrWxzc(nOCK{9s2z zL9fj-`Lu~F&5wv7T}lPK+zH*DY=T4&Npee{I44r+X9+;wLViu$VHobq{yjW& zX&r+xI0u;_R>ZDZOqasJI_VInnIbYT=Z!`(Z}aOzFa~!GjwDF(!XZ?oxGj*jO@jis zq)G)(K)!FebShzgGHLt|)!dJ2lr=>t<2)=YNIwo=4(af=>oCtpxvvatI@ZHn?YYm; zO0USKRx(o+$#i=vBC_-0R!IK1-uMZHoK;hu~b&T9~0#|96W4|_DdqYDbX(GV9~xQ<}v`gFNOKaiMD zXQo?~{Wt|zc+NN((-}k<-*TH#c@}WQ{4#v&1(s?q_4)JS-753;8c{gD04h@*3jpGQ ze-9bEMa_!V6tbUFL>A0ml`9evCn}k~GJJpV=26!Su+&{$J8~j2b1|#&ZNu>}ST`lb zY;p|SzfMS1B1wf+XtZ1L$%=))wYyvb#JVyUmNZdgT4_y8IzqRVOFcm&4%|Ma4-nR9 z%#4h`QbA@)x<8K(s0X`Diq>cIs`<)gRXq3%hfr}yOobZn`Fvkq zc(~lA0iZi|d_skO$qU*JZ^{7JsitI|y!?8!9e9MhN>y~QzTR{yuP_br@O;@a6w_R~ zM8rOR_`C8@X~8G@0e1eYE0Fwt4opQ~Ub>3MRGnuZempKO-M{Bi-ExQXF&idvd?}aA z`}?)gXs)2)Z_29G?ioqv7bA-Zp5JqY-V?iwAm9*nO@5z^io=?+H^`;SdRHhg|0+wC zFeTS5E_Wd7IfuyD7f}XFv$LXA$i5n~rO zn6@KMO#vv|^7-g34CN%@6;5nSaur+j+)b)PjkOt%o{yIN*>dteghU?Qc2vO+BO+*k zGQuj6`QxN-E+D$8RzT3Kly@weeW&Y)^=3R2lGmy z=mm8WKQ$%Gbgx%@c;X_E`$lq}d{V0T3 zDRg5L)v%-lP@l`mQ-MK+o7syLj>%33a6%}c1FKT7P`tEz4NSjI&yNIa*F$#ET40im z(aX^{|MlCMHtzm+39nGFA>)^#Y`&MxyKlRgiNx~vkv>n!^>WA~k=K5W%8!MG%-IPH zTLkXqzLhqh`f6ZLr+&e7L>xEgBnqSUcHbwcx2};ws;^`d#lnZ0(w^Pp-|G$YPOYEJ zN=cUW$GksHU_80WEaT4&H9eKbejb0{heVhF6}}J4 zNv%nf)GTkE!$y5WK#-Tb&k{cTQmS{*6g!DbKCZr1Z593U_fLYvrG!EF6=r4nkP_=P ztm_oKR8h4#SP>m_Qr3ssWs|W&piKGOUpe?i3UmZkD1=EE)y~t!%ZYP^uS+~H#I^NbS9 zFhMNDmC6x>VDh-UA8^W4Q=cvl?c-j#!brCf80bOU^n~%fWFPbuRH2PUfu0dz9}tly zXTd?QS6fQZZiV%AP<7M$XW33E!|@X<{4pJC6P=g6zzbpeS%zeZ4Z$_+N;_>W_k+5* z6c79oIx7_D?;TM9o28gwS%;FYwzjrSj-nHC{rsPy2~lStW4L}&sVS&G^~gA6QJT1$ zE%J9nDG8us41pG&R zB)CVyi@8XyU|}9zB+_o|JGH|{oX=S)sGlY`={YwgxRqL`KPoZs)IykhXy5Lc|1yMa zl8{2K1^g)i?Ci}^sd%3%Wi+kBNFqA&f0Q80zMd1l3YE;kfsjZ>^FQI`fq|1Ay6Dkr z)C{t$(e&WY&V*4{g3glB;TRi;LgY*}g;g4IWvC}#jxJ^1y$WW7@hn?Hv-ld@4 zBmdX0n@ew9e%t;m6-hZR`krml!)&WuLS+a^GLh89_dEL&2DKi!&2F%* z>eS+!Q1Abw>nSovJ$n4q0tFJEZ9A%RH|EySR*WufzL`ZHB7tm8Kd$?V?oF>#(bD(O zmG!K_NHSTzw10bvCYU);xXlFrt=%B?faPY7d%bM_I6a{FU8=ok&&w@g49(dHpZjGw zITuDhOm-yzXfwxmvqTi4I`fhUxa>>bRfc0ac4Ju-mu?pRP8m?)&R7rHfZxc|RSQFc z>h3afWmXwvYTmdLX2`^xJXa({5y|J|W0~(qt>934*ZWh1ahRpR$|)(v$O~dtAxe#x zSI5~j33ks2JN;&zk%^InRqD=CN(M>obKggawb(JIiEUc>N-S&WJngS|9!qL#@D; z4Q&JE;M0t=1i|jz6vXoxvsb(%JfQRIBWfZjEQ>(A%ZTX+! zQ}etK?RjRT47yX&!c+Mzz?grc5IvL%6{Fff;9r;|ayFLWY1^tGR7G(aAhjtE{V`Io zy&A9uvwd-E(Fg8N8Z~_TZPoM!>|2Qb3bP#Jro<1>+;J`z0WTT3?#Jku^vfjchu z8?LU@B#P*2nIaetWb)c|fc)%-Ad>xLjKO42O?Ya)OwyWC0q&?q5jHulrTk-A*igY- zuC_rfUYs(M-&t1oZ++Ak6r`=&|ybx>Y2gm(58T`T3QbVxyGy0F8 zL%pzW_OV5KKE2nZCHIrzs5Bc+;-G<_ppB1g6v^g0OyQ#|D8LAJ7hwM82WpL z3|spxtLgxA-J;(r&ViQ%UH>4EpFuC6cm4Qz%?om4&QJWHWpIc-X{`Te!8DLj3_=OQ z6zw6n9~TztY~ekf&Wa`mLRAy&*+n+2#XXb^v^dW1n|g%gVPVhJ#_!XaQFrDy)Rh5p zDCBFRPdS3I>kUQC{8?)?S{!T~EONida;*y9bARt$m9;Qp9%>!}(H;{KItquMtZi3K;Mq>#jl66pQCnknW)#Uo#n%o;W4 zOuF89#8eKQRT9?;tDacV412d1aZ$7sbopLp)}1W|J? z9@;Xps}he~r}%~FRO>_Mvs*)Ytz>N#CKcKwFV*wewTjh>J}?obj~t?T z`nMZaYYl244@SxMJPS!J@ZS|hCRdK_&i1rcHu3VLJnVHatIBsQ%{H3boaQmH zN&gIt|Bke#nkVerBayG|+tCnFjQb6xvk+FLl#LcEfAB{GZ^0@Wq-14zev`_N70~ff zB}&mX^F7TeK@Q1K#8fpT@_xhU3Y59G>lx`ZkHH%s4^t-tzMI7K!~cFgDu&m5bw)KXmnyB~&4EA~B2SFV~0JO+g{I zm((TfRqtCQGuXH*4m9pai{3I}`>(0uYG;u$NrcI*yXF%Ny@^nDfe{&RAoK5_^jrAP ztWXn-J$hi1p^*E9ep%i+h4IzMo2Ey2eB+|tJ9uNUW9NhTv*vPG8c*7S-$V3}fX8+r zzp&6$e{1vZ(=S8s_zzU`pI(nOa*V7BRk)60g;a}h8%y;fQ4t!ZcP}o%`zF)lv4z?v zNECSIDdx<08jHLi6M5es)tnr$&xJqx5&=G4ciW0HKSp~mD`Zl=Wi3U?e*L{CUx+B?yE}dV)s~gsZ24Uti$;J2Y(2^E92My;c+M*v6eqU5Q z{_G^)LLeetAVs{JbHrAFqrKM%y>T_~4@bg9#QyF4yO1I47Cph&F51N-;8U$Hzp``PFT{BDnWVLHaPt0&sX0*$$F7@n6 z=mwrau8B$wO;zG4wKP>tw({H4u9pxYCsfm54K8*EWL-+4~Buy>l>rVB;yvk}~nwi&9jlpLg z#_m5e?EkJSq^lScPl$rvG%=8{EjsVlboklMO7@Yvz97+ETs5ec(1}7+QB?iGt!c3}(iVix$x8}UrK=!`* zzcpHEFi_Lo`05~<$Ktp6gAISasJ$WS!2!Br-qaN_w4K(6A}*iB?|06`Rmplc>5n(* zilwqhiNU2h^@WS)D7T|6jvKoMoubR^J+wjfx}5RGeuvb2m&XDbW%CoiHzjBLajo&@ z9-F!HkK>&;(-H*UX}eR4x^@^Gf7V66RZV}hk{tfLz!^h3Jl+mt(eF3Ut7At?=Jz&i zID+h1EtVZui!kr)`E*ni49MK9teB;q+wt#iIyO@$6W_n|uBX6;+}uGMJrDkLe7;wr zJqPdjXvmGQ>WMdv^SZX)e@zL7K~~sy{pu;NDB*);OMA3yvqUK$`LXTP>A~$B*4F$hT-AZPM$eb}tYgJR#nHqx&il zh0YW1Z|A;{4(2S^r~sB8vu*y)|F8ejs? z`sMXaql;=90ANI-J0YdpXis2;(>5!{4QU6nJ zz3w8?vr;+am-@qz+)Z`iUJmCopl9)!5gPF7xTw*qy)f{p}vus;Ea?j-65{9?#GAZ0+7efiP)RkQ)-%m8!oYzFMhp&s>V66&uZY zx5%tSRo*|VIEw8lAce!WJdz_zlM|5pD}#4>8KIBX#RL<>e0zcA8LYp#Pt-S||)rVL|M zddz0n;7ay87OAC;>RQ8wwR9F>MI!a5&8PZi=9wv#@U%;;KOq>MYXmm!)8`HTj0?I( z>%i~jG-YUfjC-&_{}!*j_G>%l1$p{v2u`Od+^t^&L!{=CIfEK7|KP$ZQzudk5p^TC z1ksEN-T>5}oyI|Zq0QWDQBa%m4$ix<=c|>wpDtiT^p>_+^K&xDC_ICd%xsGLHenC$ z6Pu<@^ABMo>$^?IBs*HguPOYeta||}#A}n)_aH%g*x)fRHx_s7ukyXSWkp&AA;3wV z+83|cmXjl6DD92ag?zfH{^RPKGq-Meq(oj|t^+-TRr!Qd-*`;@6!#(}gO0_1Khysa zs=#T^kvxie5X>M`RRq)7Bm}KUzmU(Q!Z~@Dli6aP_%?UJ(2w%vGv98q?AOmJMS5@y zxRVD>o$efoDB~lw$$NPI3^)&}Z(k6`CTqb~QwPZB5ZQOjO>P^H^4PZw7!-}yd3u^G zBw$h$AlXx{oZWoM=YcQ$yZxC@A~WVhYQAyh;u<1{^s{<_#3&Z4ckL@?@Wdg#@ho7p z#0IcoZmE2>0@BK%@9@sMzva)ePE5qHz{=I)_t%_Yg^P2kU-R!kVnnH)XG#BjDS_;>ru7c;&D!rC`LuekHXfl* zP6$OOJMDdeGY7##Wv#0Yw|LJ>>z3CJj{$7g%9HIq`;2}2J|qpl$e=rH_Pi6f{nt%l zGb|U{^0;)fuZ>YaI2I;>U3urh<`2e*9k|?ziHrASpYhqCtIHg=A5)>^#5nL-56xJN z5JZV`z522yHF;u$f?^V^QdQMqadxeGZ|6i6T*oct86-m&b@6y`_4&VE=bvU6!{1Wi zb#m*$UW*Ei2s3(TtH#LSr3(j7bbiOOj$RG97D2oNlZ%K_MwmDY42}6^Elo5%8avm= zh-b0!q$ZTO=E%*SU3gf_K}ZvYFSsjFo`q}-OdjBQIlFaq zxAZB5n0(##cp{o%>Osb9Z~kV#a+p{y6#=M`?40|b*P`Sr5^a0W-}sXB99?{((y@~H zJ|Bzk85K#^f`6mq5Frl(3B}_>*O_nWg^|f!v#toBW zEZ;s@L?qfxZN@bisrQubRnt-OjQH^=e`i^RJAf;^{cVDX+-}L9#E70ZzKA8ML1B7^ zd|EQ{{rY2)1D&Tw%5@0*4(eHUAC}5gI1)BGab;vFxhp*O)>!3+ENpP={SF_ zYg%onYg}WiXZc-)B8nDADsiApzbaSgM5}gO`8oypQ=+kv~D>r2A7ie>!{a$1Xkb;#Gn4K*mMy!bauXNv3{9ueAE?2 z{C%vJ0^1LL9uFUSrD(xXV{5xpIF)>Xve+hpPg=Rep1EahJsJQiKnxaJw>(Mz*xR4uN)5RrpNN{pQRwox0VVkI3liY9N z1qjs-Z9@n4QIoJ+&FtaV=1V0)3J#5L-j#&={aE#`YYg3Wu6_yeRE}8Ft9?Ke{CKGu zpy^7RF;%tXSCV;08WGQuV#t&!W(kx+!$0Qaf>9bHnK&27QCH`2PyCY z?t35GT^Rr5)IX@LpShf%y6;PTH^0t(2Q7B zVIluTJED>@SyQJ??&FboCgS5X@3Z$kshv&_t=$Z$_Z=RasLNi6D~Q5%wa2mP5%x?$ zT?uv?db6Wdni;3bviv!LUzUwHEV-h(ZQh}98SXeHFrv4_4C8-5w5WswbD1|29g7-w zd3sg6L!w^F4{veqP3(m~gWk3%GQs}jBwDYLz4$`8XG~u<+vYv`Pe<*Wn)2K;Uj`nc z80FhUybd$FPA1c6)k3@z{~uU93>1E$2w_puJxr3k`Gwu4dr(_!m+MOCW<`=V^lIFg zt1w5gmanyY6}g$nOC?m29L6_X;{6>|OB&72bg<;%H`hgXk)WPGX5#dPIk|V#yvF4h zM-`d?{R_&Ex9%={j6>HQCrP-XEs~idTFua%T;xe^8d z%i|@dYiljTAForPTd6$XiFvKk=LU_aB;C$rilf{g5b&o)HU>S-f%JQX54Tca`2J7W##KO@j;*VCy9(s71H08l^tzcx_XGeAjd2yG5#wD zavoFKvt3q-W&$k*MA(JPwzcB&25YW~7w;vMK$%ozKBpmMzqgiabfFb;!KABuz!dm$ z(&7_}{>=@5uLxda-QS&Uo=t2;M;5FC9ApM72(orlWn@#WX#Rg4>P`Z28{l#PP@Btf_DDdfRXmNses6Qp&QwuAT%#8?OAI|vI3*C_9@ zTn*TmS-2=04diu;rWFMTY#gLk{ucxevL(>pr}g)X9E}3R0*kMNp;;9Q&3ePD_SE#R zMGv2Sp|mo#RHmgl;ZEWTUf5->nH;s(#GzD@a)%Vk#x*I%zdp&hR6{QJM2?yD8)ZNG zEFn0NTfH15GlvRc#yisFw>gYfh>;c}47qAAP>xObt0AN+PG>aXchY_eHjI6cZ(Iqx z_`1UXoi~sIG!fbyXhgMyp+?+W)h!h-RjGz&l2=o-6sSkx*(s_=S3gdlbKzmknm1Onb|mpjMW`X=*LsJ-{h8BG|38_}z$J|LB##W^J$YCdzSo%+X) z${@uKwaxS9cK-^^o>5bOg#asSVue>8mU;L=yZ%BF{}pq7Ww-%tI&!yUb&|^EhCN4> zSc%E6NnF!cy9!6C>Dp5`5J-GPxfo&=hEXL#BT6-=@Ar2$?Y*naWi{VK>#FF{B0yyf zO5KTdig99A^ac#<4AXC(@^@DY$(J(?6`xz1vNK;f{|V?*CwQsqUSNQ7w4q*Xip z{1sPOpK0Ai(r)_;#rXaM7auR+BC=zT9yV-^!~gx%E=`l?O!e(C#i4*#_m$cqKhJE& z!jHq@!kUfJo)>}Ol5<3LYrZ8fZP)2p1?OL@mxqQ)1PNc6wW*0`1I6HOtfjtp;7e79 z&+re}WB93vf4yk{z&3qgpQt!RlxRjmwj@&GcY>&{zZFNl@p-4ve@P44S`VW4bGwq+ z<@Ng>ryGVa_+P;E1!3WLY5GdzyEdGbDpG8|=*fI1BudFHgVjhbmYxApJSL z`_aIiXK{@l9}V)mpg(8NNSGy>c*fXv6NqhmuQnEhwcmRh=a)#jYZL^{94&c!Dv{;VQ->)j;d&~&KeaSxV z-sY~;x+-++!kQ1C&v|b@7t;a8{ShaPDy>~K0^{9Te-{^wRQ zDda}$qwJk)yp&P}U&{nf1=$0*zoOPhSO`2tqR`1LnIZUe-SBN4N^*QsqOyi<>wY#4 zSTS5$oM68&o2<>J^xeh9g;R6>=((|B z^H@W%{nZ=&&?0N--L(%^)F9e z7=&2;+8feb2qC6*vn9yn#y<$cy8QaHc12&J2k4ch2LcrS8MJbxRaMWRUOHPYp79(~==3 z@3KRkPkX=H67T$XU{7{yhn;<#%;$r1z<(heOeDC)`xXZ^MV+WxRe^L^uTRH&N!_w!&MRZ_K3MZEelo@kID^ z9JbzCZxAO}#4+On)gZ0Dg-3Q*?9-H9QyaSj>(H=r)4+ht?Y%i!{OJp?*9KWcO9Q8F zq*tRohN)USx1k4rwJ-?o{_hZ{>;dOi=ppS<-q%%f2Q2uGz}$ zBM1_gNQt`={K}}#*i?QpuO;#ovSA_l6kU}qPG#|Jo3qY%{c;xG_6NdxY96fn!*_#y z)!LRq_ce$RY-*Y^mq}7AUm58^D8roV>h|SANkHo^%*gcO)TaC z?BU54g;9}+turGsoXw4V4BgOqxt0083r&{&IUfoYp^=>$ZI?Hd6~CzYEzWC670fn@ zZw3QgrgIJ-el(`Ld)oNGk}Pe;9~J8-)ml9wyV8D_!)TQ4cdd1FvESGpqEX-VzCXdo z7v1W%na=}SbRa)#UO)W1`hyU%b*`EVs1L;YEj;)YzChR9yECEN3Eqhhxw=%Dd(k|x0C-f?Iw zoh(!iA5LGbRKIZSz9Dz@4D6oBva0&K3+;w(CkjQXH?MHt#JnhUkd?!F6GNRAFWAL_ z&8uGI?p<3_67YrzzwQGX^vUP)@SlHSrmN5c=O@zk2A5a59&J%jdc_ylW6z-EdAQV5yDvo&6jGe;kz6%B@ z!XbUNiI}&0fj!E7Tab3E+WPq@q|!`a!K^iZDOU={zTop&T;I&y!PrCXzIrrXAvaD- z%c|x^^zi2J25f4azIA~uoGH*3SJnEB!!C8;M?P&qy-pBWB;vm0bl6ZSg7h}7R_M90 zpmt5`jf=?;rGisBg3bxQkExGE~03jURJEDvct{5!(% zcGw)4roq_vJ{5w#V9%>>Sl7bG*VER$%(%vblk&m#5IG?GjXUk3^SrxdDF>6Q~I&%H@+15 zFZH}N>qYYW-2mUBZkG)_vIvs+Ws8Zs&2ih6FRJtCF9pU|h33JJ_o;{`Z=ub8B~)rW zO@={so61fI;T;wWEg0O|A1PA`;+D6}sk;;&wV2fJ=|4_ixXaV>2qnUKG4F!GvtY()zhbUD78t$FaIc3+a$Td?A7JT~$8d1l=q&KN}4 z%x!bu?K8n-UP(ws_h4Z))gY(bU+%3G2PkN_*H%t58f!_irrf z{~eH$W;df3pHq32(I|i3*M4eqHhUoD=u>@GggxBsgr!evb0+*I1cp^$FP6qEg@5Ww zHTEF+4u4C$#-J`0?dV$L-oMDfzTmVXRL-J_WR3n|%mXY8(L;OY_}kz9bzK=GVvb|m zeTq$2E%mjvuCTDunvG{W+jGBW8T9&_munu!CL_>2FN+p6;iya3g-$XpEW5=QIo3hiKDB)srtzAxXVzIy>9* ziA1NE#`Ache`A}25k&no-pSHwcxyBG+f5Vx!(91L)yXkQvijkKBJ-Ks(nwsX9~H3Te9 zCG;D|Eg|u{LJd_m-b9g2>+u_^!&S~yCQN}$9*p!rCxK&0&KUoB$EnSnFpF^adrCz9iOlUF=v+YHyNlb|ufQLo; zW73|5>Dt5LrUOZ)CF7+^In;mP@NM}LLa;N}6w`6btXH8{eShkV3!|7JHmpb$D>6{wL5}&iP zxOuGfCdi*Lr`W@Zr0YgkbH2T45rXR6Hbq};1sFAxd_zI_hR>o$&vtitcY_+0G#qHs zZ(Pq;WHwA{v}Od%{~-zb``;S?xaH&r?30C$cr1EVZYP<3c8g8!XW%3IdzWYUwo6UO zE1QE$XZXyYT=Y=CYwaviH`RYYF6{Ynv<9=deUr=K%Xd%s?zDJDCc?Q>^VY@WpaAbm z{r=q=+LwM}ZsZ+P23E5>`^Y!cGp46?X53JltdyV!O1!|=MF*;zBC*>`cV5rNUK%^FujZ~pjq{UwMYjBpR}{Xqw7d;*#^9a*Qp4Z>r7vR0b9PQxV#20VwuR-h zK~`Jr@gy1l=0^z{(n=6G{890KSV6pc0HmCYKWqK@57C&SFM|6-xrk&vZRh~UMqu)) z_0cOxY@j8)$>3#S!JE6S2x>}N2wAD*S)#gYN148-X{*5bbm%Q<5{tWeDec~9F%{kO zvXR}j&WuSjd`&a+|^5CIx{GpWN7Z0|++H4DaWVsrj>x?SQ$`^d} zG*7P{Ph4n4SY7-0&PAK|$tPe*NWL7&l2$j)zJ0vsJD{nxhmg=dlbfPBn!cVNnh@ei zgnJacI~EMOUZo1bAH9AD*iW4Qk(`W`KI!hCbYBtdw|s(!er!@bkN)O-Zxe6YO56rl zR1aGNhS5J@!zIvKU0r zZIB1jz%IRH6C`u}fes!=32fzd6&5F^q?S786#nWHfs9{xG{rL6r|4VDh<>Qjx{um> zb@5+R_HF)*AlJ#Z``cJcgy_c%(T73*Hq68iu*mO?v}}uGh1LNN&DvV?!D35>V}^|P zcT?nA^=2-*9nRJnO5k?YO2uvfRbf2K^1NC|X&W3z_Nwpw@))gYNYHa6HFx=j|ORM@BG9+b3pz zA4UV(+;vp23L9NC8f8he#jd`l`al+J_rw=J1q*C__$^Hc=T9wyRR#KE=Ec*OD+Ir&z@UQK zs5&dWss;)W^@V_>+kN+2=T9dM(0`M9jf0!0XS)$mLGFqCz+f z&og{II{lnwv^*3Nkemyfv$@_BQ^rIur4v}+nKwU)3GLWk!U^J7lpS<$c{1O_9Bf{5 zW{I>Q1U*m_i?O{V4$osNxM=5e_M#5O)f*~;Aw;Xhwbz-`VP|B`FclBvp!7*($#4gb;a=mP-I za_O(l6v$^&Lud~&yo`J#U&*r+?X!ns4%jeV!UoZ-#NN}QM|*(-G_f>44*1gyV&=Vb z;x!3P4Zh0w!CJ@Efiv29&Wm?QJx^C?3lFHiRo_PZpO90wt}+lf1{*8$j zjvN^8IdO*xG}CgLu66lcW2u*pm7{r$^?2}ai||QcMMjiS zW?BMJz)u&Ie1U@jn66`r0C{DWsXWoOA`yqpi0s|o{{D)riu`NAXdoO%iFm43#(q0#M2)MN0#5rRKK4>nAS>ZheF`S)87dNtj-P`)iE!_y63e^VMpske8RQpg)N zUKD{J1t5nN_=$W|IaLQ0H&{1+A1& AlK=n! From 4a94ce46dbc52e75bb80b0461a2c28f67d073d51 Mon Sep 17 00:00:00 2001 From: Matt Moore Date: Wed, 9 Sep 2015 10:07:29 -0700 Subject: [PATCH 22/46] Switch from gcr.io/_b_ prefix to b.gcr.io/ to designate bring-your-own-bucket pulls --- test/e2e/rc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/rc.go b/test/e2e/rc.go index 9d25f37c100..409b91505de 100644 --- a/test/e2e/rc.go +++ b/test/e2e/rc.go @@ -41,7 +41,7 @@ var _ = Describe("ReplicationController", func() { // requires private images SkipUnlessProviderIs("gce", "gke") - ServeImageOrFail(framework, "private", "gcr.io/_b_k8s_authenticated_test/serve_hostname:1.1") + ServeImageOrFail(framework, "private", "b.gcr.io/k8s_authenticated_test/serve_hostname:1.1") }) }) From ea919f6d1e716a350d8db9530193574a05370aa6 Mon Sep 17 00:00:00 2001 From: derekwaynecarr Date: Tue, 8 Sep 2015 14:49:54 -0400 Subject: [PATCH 23/46] Fix precision handling in validating LimitRange --- pkg/api/resource/quantity.go | 21 +++ pkg/api/resource/quantity_test.go | 20 ++ pkg/api/validation/validation.go | 60 +++--- pkg/api/validation/validation_test.go | 256 ++++++++++++-------------- 4 files changed, 182 insertions(+), 175 deletions(-) diff --git a/pkg/api/resource/quantity.go b/pkg/api/resource/quantity.go index ef8eaef9e6b..577d5b6093c 100644 --- a/pkg/api/resource/quantity.go +++ b/pkg/api/resource/quantity.go @@ -301,6 +301,27 @@ func (q *Quantity) String() string { return number + string(suffix) } +// Cmp compares q and y and returns: +// +// -1 if q < y +// 0 if q == y +// +1 if q > y +// +func (q *Quantity) Cmp(y Quantity) int { + num1 := q.Value() + num2 := y.Value() + if num1 < MaxMilliValue && num2 < MaxMilliValue { + num1 = q.MilliValue() + num2 = y.MilliValue() + } + if num1 < num2 { + return -1 + } else if num1 > num2 { + return 1 + } + return 0 +} + func (q *Quantity) Add(y Quantity) error { if q.Format != y.Format { return fmt.Errorf("format mismatch: %v vs. %v", q.Format, y.Format) diff --git a/pkg/api/resource/quantity_test.go b/pkg/api/resource/quantity_test.go index 70f4836bb1d..da8858ea97d 100644 --- a/pkg/api/resource/quantity_test.go +++ b/pkg/api/resource/quantity_test.go @@ -77,6 +77,26 @@ func TestQuantityCanocicalizeZero(t *testing.T) { } } +func TestQuantityCmp(t *testing.T) { + table := []struct { + x string + y string + expect int + }{ + {"0", "0", 0}, + {"100m", "50m", 1}, + {"50m", "100m", -1}, + {"10000T", "100Gi", 1}, + } + for _, testCase := range table { + q1 := MustParse(testCase.x) + q2 := MustParse(testCase.y) + if result := q1.Cmp(q2); result != testCase.expect { + t.Errorf("X: %v, Y: %v, Expected: %v, Actual: %v", testCase.x, testCase.y, testCase.expect, result) + } + } +} + func TestQuantityParse(t *testing.T) { table := []struct { input string diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index 43ea1e7e1be..d5965bce358 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -1442,79 +1442,75 @@ func ValidateLimitRange(limitRange *api.LimitRange) errs.ValidationErrorList { limitTypeSet[limit.Type] = true keys := util.StringSet{} - min := map[string]int64{} - max := map[string]int64{} - defaults := map[string]int64{} - defaultRequests := map[string]int64{} + min := map[string]resource.Quantity{} + max := map[string]resource.Quantity{} + defaults := map[string]resource.Quantity{} + defaultRequests := map[string]resource.Quantity{} - for k := range limit.Max { + for k, q := range limit.Max { allErrs = append(allErrs, validateResourceName(string(k), fmt.Sprintf("spec.limits[%d].max[%s]", i, k))...) keys.Insert(string(k)) - q := limit.Max[k] - max[string(k)] = q.Value() + max[string(k)] = q } - for k := range limit.Min { + for k, q := range limit.Min { allErrs = append(allErrs, validateResourceName(string(k), fmt.Sprintf("spec.limits[%d].min[%s]", i, k))...) keys.Insert(string(k)) - q := limit.Min[k] - min[string(k)] = q.Value() + min[string(k)] = q } - for k := range limit.Default { + for k, q := range limit.Default { allErrs = append(allErrs, validateResourceName(string(k), fmt.Sprintf("spec.limits[%d].default[%s]", i, k))...) keys.Insert(string(k)) - q := limit.Default[k] - defaults[string(k)] = q.Value() + defaults[string(k)] = q } - for k := range limit.DefaultRequest { + for k, q := range limit.DefaultRequest { allErrs = append(allErrs, validateResourceName(string(k), fmt.Sprintf("spec.limits[%d].defaultRequest[%s]", i, k))...) keys.Insert(string(k)) - q := limit.DefaultRequest[k] - defaultRequests[string(k)] = q.Value() + defaultRequests[string(k)] = q } for k := range limit.MaxLimitRequestRatio { allErrs = append(allErrs, validateResourceName(string(k), fmt.Sprintf("spec.limits[%d].maxLimitRequestRatio[%s]", i, k))...) } for k := range keys { - minValue, minValueFound := min[k] - maxValue, maxValueFound := max[k] - defaultValue, defaultValueFound := defaults[k] - defaultRequestValue, defaultRequestValueFound := defaultRequests[k] + minQuantity, minQuantityFound := min[k] + maxQuantity, maxQuantityFound := max[k] + defaultQuantity, defaultQuantityFound := defaults[k] + defaultRequestQuantity, defaultRequestQuantityFound := defaultRequests[k] - if minValueFound && maxValueFound && minValue > maxValue { + if minQuantityFound && maxQuantityFound && minQuantity.Cmp(maxQuantity) > 0 { minQuantity := limit.Min[api.ResourceName(k)] maxQuantity := limit.Max[api.ResourceName(k)] - allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].min[%s]", i, k), minValue, fmt.Sprintf("min value %s is greater than max value %s", minQuantity.String(), maxQuantity.String()))) + allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].min[%s]", i, k), minQuantity, fmt.Sprintf("min value %s is greater than max value %s", minQuantity.String(), maxQuantity.String()))) } - if defaultRequestValueFound && minValueFound && minValue > defaultRequestValue { + if defaultRequestQuantityFound && minQuantityFound && minQuantity.Cmp(defaultRequestQuantity) > 0 { minQuantity := limit.Min[api.ResourceName(k)] defaultRequestQuantity := limit.DefaultRequest[api.ResourceName(k)] - allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].defaultRequest[%s]", i, k), defaultRequestValue, fmt.Sprintf("min value %s is greater than default request value %s", minQuantity.String(), defaultRequestQuantity.String()))) + allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].defaultRequest[%s]", i, k), defaultRequestQuantity, fmt.Sprintf("min value %s is greater than default request value %s", minQuantity.String(), defaultRequestQuantity.String()))) } - if defaultRequestValueFound && maxValueFound && defaultRequestValue > maxValue { + if defaultRequestQuantityFound && maxQuantityFound && defaultRequestQuantity.Cmp(maxQuantity) > 0 { maxQuantity := limit.Max[api.ResourceName(k)] defaultRequestQuantity := limit.DefaultRequest[api.ResourceName(k)] - allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].defaultRequest[%s]", i, k), defaultRequestValue, fmt.Sprintf("default request value %s is greater than max value %s", defaultRequestQuantity.String(), maxQuantity.String()))) + allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].defaultRequest[%s]", i, k), defaultRequestQuantity, fmt.Sprintf("default request value %s is greater than max value %s", defaultRequestQuantity.String(), maxQuantity.String()))) } - if defaultRequestValueFound && defaultValueFound && defaultRequestValue > defaultValue { + if defaultRequestQuantityFound && defaultQuantityFound && defaultRequestQuantity.Cmp(defaultQuantity) > 0 { defaultQuantity := limit.Default[api.ResourceName(k)] defaultRequestQuantity := limit.DefaultRequest[api.ResourceName(k)] - allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].defaultRequest[%s]", i, k), defaultRequestValue, fmt.Sprintf("default request value %s is greater than default limit value %s", defaultRequestQuantity.String(), defaultQuantity.String()))) + allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].defaultRequest[%s]", i, k), defaultRequestQuantity, fmt.Sprintf("default request value %s is greater than default limit value %s", defaultRequestQuantity.String(), defaultQuantity.String()))) } - if defaultValueFound && minValueFound && minValue > defaultValue { + if defaultQuantityFound && minQuantityFound && minQuantity.Cmp(defaultQuantity) > 0 { minQuantity := limit.Min[api.ResourceName(k)] defaultQuantity := limit.Default[api.ResourceName(k)] - allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].default[%s]", i, k), minValue, fmt.Sprintf("min value %s is greater than default value %s", minQuantity.String(), defaultQuantity.String()))) + allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].default[%s]", i, k), minQuantity, fmt.Sprintf("min value %s is greater than default value %s", minQuantity.String(), defaultQuantity.String()))) } - if defaultValueFound && maxValueFound && defaultValue > maxValue { + if defaultQuantityFound && maxQuantityFound && defaultQuantity.Cmp(maxQuantity) > 0 { maxQuantity := limit.Max[api.ResourceName(k)] defaultQuantity := limit.Default[api.ResourceName(k)] - allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].default[%s]", i, k), maxValue, fmt.Sprintf("default value %s is greater than max value %s", defaultQuantity.String(), maxQuantity.String()))) + allErrs = append(allErrs, errs.NewFieldInvalid(fmt.Sprintf("spec.limits[%d].default[%s]", i, k), maxQuantity, fmt.Sprintf("default value %s is greater than max value %s", defaultQuantity.String(), maxQuantity.String()))) } } } diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index 3f71b703950..7a1a0167899 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -2886,138 +2886,58 @@ func TestValidateResourceNames(t *testing.T) { } } +func getResourceList(cpu, memory string) api.ResourceList { + res := api.ResourceList{} + if cpu != "" { + res[api.ResourceCPU] = resource.MustParse(cpu) + } + if memory != "" { + res[api.ResourceMemory] = resource.MustParse(memory) + } + return res +} + func TestValidateLimitRange(t *testing.T) { - spec := api.LimitRangeSpec{ - Limits: []api.LimitRangeItem{ - { - Type: api.LimitTypePod, - Max: api.ResourceList{ - api.ResourceCPU: resource.MustParse("100"), - api.ResourceMemory: resource.MustParse("10000"), - }, - Min: api.ResourceList{ - api.ResourceCPU: resource.MustParse("5"), - api.ResourceMemory: resource.MustParse("100"), - }, - Default: api.ResourceList{ - api.ResourceCPU: resource.MustParse("50"), - api.ResourceMemory: resource.MustParse("500"), - }, - DefaultRequest: api.ResourceList{ - api.ResourceCPU: resource.MustParse("10"), - api.ResourceMemory: resource.MustParse("200"), - }, - MaxLimitRequestRatio: api.ResourceList{ - api.ResourceCPU: resource.MustParse("20"), - }, - }, - }, - } - - invalidSpecDuplicateType := api.LimitRangeSpec{ - Limits: []api.LimitRangeItem{ - { - Type: api.LimitTypePod, - Max: api.ResourceList{ - api.ResourceCPU: resource.MustParse("100"), - api.ResourceMemory: resource.MustParse("10000"), - }, - Min: api.ResourceList{ - api.ResourceCPU: resource.MustParse("0"), - api.ResourceMemory: resource.MustParse("100"), - }, - }, - { - Type: api.LimitTypePod, - Min: api.ResourceList{ - api.ResourceCPU: resource.MustParse("0"), - api.ResourceMemory: resource.MustParse("100"), - }, - }, - }, - } - - invalidSpecRangeMaxLessThanMin := api.LimitRangeSpec{ - Limits: []api.LimitRangeItem{ - { - Type: api.LimitTypePod, - Max: api.ResourceList{ - api.ResourceCPU: resource.MustParse("10"), - }, - Min: api.ResourceList{ - api.ResourceCPU: resource.MustParse("1000"), - }, - }, - }, - } - - invalidSpecRangeDefaultOutsideRange := api.LimitRangeSpec{ - Limits: []api.LimitRangeItem{ - { - Type: api.LimitTypePod, - Max: api.ResourceList{ - api.ResourceCPU: resource.MustParse("1000"), - }, - Min: api.ResourceList{ - api.ResourceCPU: resource.MustParse("100"), - }, - Default: api.ResourceList{ - api.ResourceCPU: resource.MustParse("2000"), - }, - }, - }, - } - - invalidSpecRangeDefaultRequestOutsideRange := api.LimitRangeSpec{ - Limits: []api.LimitRangeItem{ - { - Type: api.LimitTypePod, - Max: api.ResourceList{ - api.ResourceCPU: resource.MustParse("1000"), - }, - Min: api.ResourceList{ - api.ResourceCPU: resource.MustParse("100"), - }, - DefaultRequest: api.ResourceList{ - api.ResourceCPU: resource.MustParse("2000"), - }, - }, - }, - } - - invalidSpecRangeRequestMoreThanDefaultRange := api.LimitRangeSpec{ - Limits: []api.LimitRangeItem{ - { - Type: api.LimitTypePod, - Max: api.ResourceList{ - api.ResourceCPU: resource.MustParse("1000"), - }, - Min: api.ResourceList{ - api.ResourceCPU: resource.MustParse("100"), - }, - Default: api.ResourceList{ - api.ResourceCPU: resource.MustParse("500"), - }, - DefaultRequest: api.ResourceList{ - api.ResourceCPU: resource.MustParse("800"), - }, - }, - }, - } - - successCases := []api.LimitRange{ + successCases := []struct { + name string + spec api.LimitRangeSpec + }{ { - ObjectMeta: api.ObjectMeta{ - Name: "abc", - Namespace: "foo", + name: "all-fields-valid", + spec: api.LimitRangeSpec{ + Limits: []api.LimitRangeItem{ + { + Type: api.LimitTypePod, + Max: getResourceList("100m", "10000Mi"), + Min: getResourceList("5m", "100Mi"), + Default: getResourceList("50m", "500Mi"), + DefaultRequest: getResourceList("10m", "200Mi"), + MaxLimitRequestRatio: getResourceList("10", ""), + }, + }, + }, + }, + { + name: "all-fields-valid-big-numbers", + spec: api.LimitRangeSpec{ + Limits: []api.LimitRangeItem{ + { + Type: api.LimitTypePod, + Max: getResourceList("100m", "10000T"), + Min: getResourceList("5m", "100Mi"), + Default: getResourceList("50m", "500Mi"), + DefaultRequest: getResourceList("10m", "200Mi"), + MaxLimitRequestRatio: getResourceList("10", ""), + }, + }, }, - Spec: spec, }, } for _, successCase := range successCases { - if errs := ValidateLimitRange(&successCase); len(errs) != 0 { - t.Errorf("expected success: %v", errs) + limitRange := &api.LimitRange{ObjectMeta: api.ObjectMeta{Name: successCase.name, Namespace: "foo"}, Spec: successCase.spec} + if errs := ValidateLimitRange(limitRange); len(errs) != 0 { + t.Errorf("Case %v, unexpected error: %v", successCase.name, errs) } } @@ -3025,43 +2945,92 @@ func TestValidateLimitRange(t *testing.T) { R api.LimitRange D string }{ - "zero-length Name": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "", Namespace: "foo"}, Spec: spec}, + "zero-length-name": { + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "", Namespace: "foo"}, Spec: api.LimitRangeSpec{}}, "name or generateName is required", }, "zero-length-namespace": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: ""}, Spec: spec}, + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: ""}, Spec: api.LimitRangeSpec{}}, "", }, - "invalid Name": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "^Invalid", Namespace: "foo"}, Spec: spec}, + "invalid-name": { + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "^Invalid", Namespace: "foo"}, Spec: api.LimitRangeSpec{}}, DNSSubdomainErrorMsg, }, - "invalid Namespace": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "^Invalid"}, Spec: spec}, + "invalid-namespace": { + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "^Invalid"}, Spec: api.LimitRangeSpec{}}, DNS1123LabelErrorMsg, }, - "duplicate limit type": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: invalidSpecDuplicateType}, + "duplicate-limit-type": { + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: api.LimitRangeSpec{ + Limits: []api.LimitRangeItem{ + { + Type: api.LimitTypePod, + Max: getResourceList("100m", "10000m"), + Min: getResourceList("0m", "100m"), + }, + { + Type: api.LimitTypePod, + Min: getResourceList("0m", "100m"), + }, + }, + }}, "", }, - "min value 1k is greater than max value 10": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: invalidSpecRangeMaxLessThanMin}, - "min value 1k is greater than max value 10", + "min value 100m is greater than max value 10m": { + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: api.LimitRangeSpec{ + Limits: []api.LimitRangeItem{ + { + Type: api.LimitTypePod, + Max: getResourceList("10m", ""), + Min: getResourceList("100m", ""), + }, + }, + }}, + "min value 100m is greater than max value 10m", }, "invalid spec default outside range": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: invalidSpecRangeDefaultOutsideRange}, - "default value 2k is greater than max value 1k", + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: api.LimitRangeSpec{ + Limits: []api.LimitRangeItem{ + { + Type: api.LimitTypePod, + Max: getResourceList("1", ""), + Min: getResourceList("100m", ""), + Default: getResourceList("2000m", ""), + }, + }, + }}, + "default value 2 is greater than max value 1", }, "invalid spec defaultrequest outside range": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: invalidSpecRangeDefaultRequestOutsideRange}, - "default request value 2k is greater than max value 1k", + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: api.LimitRangeSpec{ + Limits: []api.LimitRangeItem{ + { + Type: api.LimitTypePod, + Max: getResourceList("1", ""), + Min: getResourceList("100m", ""), + DefaultRequest: getResourceList("2000m", ""), + }, + }, + }}, + "default request value 2 is greater than max value 1", }, "invalid spec defaultrequest more than default": { - api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: invalidSpecRangeRequestMoreThanDefaultRange}, - "default request value 800 is greater than default limit value 500", + api.LimitRange{ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: "foo"}, Spec: api.LimitRangeSpec{ + Limits: []api.LimitRangeItem{ + { + Type: api.LimitTypeContainer, + Max: getResourceList("2", ""), + Min: getResourceList("100m", ""), + Default: getResourceList("500m", ""), + DefaultRequest: getResourceList("800m", ""), + }, + }, + }}, + "default request value 800m is greater than default limit value 500m", }, } + for k, v := range errorCases { errs := ValidateLimitRange(&v.R) if len(errs) == 0 { @@ -3074,6 +3043,7 @@ func TestValidateLimitRange(t *testing.T) { } } } + } func TestValidateResourceQuota(t *testing.T) { From 08aae94deaacddfb33767ab74073baccb1dbc056 Mon Sep 17 00:00:00 2001 From: Maciej Szulik Date: Wed, 9 Sep 2015 17:23:39 +0200 Subject: [PATCH 24/46] Fixed passing RUNTIME_CONFIG flag, since currently it fails passing eg. RUNTIME_CONFIG="experimental/v1=true" to the server. --- hack/local-up-cluster.sh | 2 +- hack/verify-flags/exceptions.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 8065e692026..dda072a8360 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -212,7 +212,7 @@ function start_apiserver { fi runtime_config="" if [[ -n "${RUNTIME_CONFIG}" ]]; then - runtime_config="--runtime-config=\"${RUNTIME_CONFIG}\"" + runtime_config="--runtime-config=${RUNTIME_CONFIG}" fi APISERVER_LOG=/tmp/kube-apiserver.log diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 5036ad26938..a8816eff7e5 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -69,7 +69,7 @@ examples/elasticsearch/production_cluster/README.md: "cluster_name" : "myesdb", examples/elasticsearch/production_cluster/README.md: "cluster_name" : "myesdb", hack/lib/logging.sh: local source_file=${BASH_SOURCE[$frame_no]} hack/lib/logging.sh: local source_file=${BASH_SOURCE[$stack_skip]} -hack/local-up-cluster.sh: runtime_config="--runtime-config=\"${RUNTIME_CONFIG}\"" +hack/local-up-cluster.sh: runtime_config="--runtime-config=${RUNTIME_CONFIG}" hack/local-up-cluster.sh: runtime_config="" pkg/cloudprovider/providers/vagrant/vagrant_test.go: testSaltMinionsResponse = []byte(`{ "return": [{"kubernetes-minion-1": {"kernel": "Linux", "domain": "", "zmqversion": "3.2.4", "kernelrelease": "3.11.10-301.fc20.x86_64", "pythonpath": ["/usr/bin", "/usr/lib64/python27.zip", "/usr/lib64/python2.7", "/usr/lib64/python2.7/plat-linux2", "/usr/lib64/python2.7/lib-tk", "/usr/lib64/python2.7/lib-old", "/usr/lib64/python2.7/lib-dynload", "/usr/lib64/python2.7/site-packages", "/usr/lib/python2.7/site-packages"], "etcd_servers": "10.245.1.2", "ip_interfaces": {"lo": ["127.0.0.1"], "docker0": ["172.17.42.1"], "enp0s8": ["10.245.2.2"], "p2p1": ["10.0.2.15"]}, "shell": "/bin/sh", "mem_total": 491, "saltversioninfo": [2014, 1, 7], "osmajorrelease": ["20"], "node_ip": "10.245.2.2", "id": "kubernetes-minion-1", "osrelease": "20", "ps": "ps -efH", "server_id": 1005530826, "num_cpus": 1, "hwaddr_interfaces": {"lo": "00:00:00:00:00:00", "docker0": "56:84:7a:fe:97:99", "enp0s8": "08:00:27:17:c5:0f", "p2p1": "08:00:27:96:96:e1"}, "virtual": "VirtualBox", "osfullname": "Fedora", "master": "kubernetes-master", "ipv4": ["10.0.2.15", "10.245.2.2", "127.0.0.1", "172.17.42.1"], "ipv6": ["::1", "fe80::a00:27ff:fe17:c50f", "fe80::a00:27ff:fe96:96e1"], "cpu_flags": ["fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce", "cx8", "apic", "sep", "mtrr", "pge", "mca", "cmov", "pat", "pse36", "clflush", "mmx", "fxsr", "sse", "sse2", "syscall", "nx", "rdtscp", "lm", "constant_tsc", "rep_good", "nopl", "pni", "monitor", "ssse3", "lahf_lm"], "localhost": "kubernetes-minion-1", "lsb_distrib_id": "Fedora", "fqdn_ip4": ["127.0.0.1"], "fqdn_ip6": [], "nodename": "kubernetes-minion-1", "saltversion": "2014.1.7", "saltpath": "/usr/lib/python2.7/site-packages/salt", "pythonversion": [2, 7, 5, "final", 0], "host": "kubernetes-minion-1", "os_family": "RedHat", "oscodename": "Heisenbug", "defaultencoding": "UTF-8", "osfinger": "Fedora-20", "roles": ["kubernetes-pool"], "num_gpus": 1, "cpu_model": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", "fqdn": "kubernetes-minion-1", "osarch": "x86_64", "cpuarch": "x86_64", "gpus": [{"model": "VirtualBox Graphics Adapter", "vendor": "unknown"}], "path": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "os": "Fedora", "defaultlanguage": "en_US"}}]}`) pkg/kubelet/qos/memory_policy_test.go: t.Errorf("oom_score_adj should be between %d and %d, but was %d", test.lowOomScoreAdj, test.highOomScoreAdj, oomScoreAdj) From d531f0fd8b691e35aca89525449d054d9ee3409a Mon Sep 17 00:00:00 2001 From: Amy Unruh Date: Wed, 9 Sep 2015 12:40:26 -0700 Subject: [PATCH 25/46] Update the frontend image to a version that uses relative URLs. Update image names in the README. --- examples/guestbook/README.md | 8 ++++---- examples/guestbook/frontend-controller.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/guestbook/README.md b/examples/guestbook/README.md index f6e71023705..a430595440d 100644 --- a/examples/guestbook/README.md +++ b/examples/guestbook/README.md @@ -328,7 +328,7 @@ replicationcontrollers/redis-slave $ kubectl get rc CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS redis-master master redis name=redis-master 1 -redis-slave slave kubernetes/redis-slave:v2 name=redis-slave 2 +redis-slave slave gcr.io/google_samples/gb-redisslave:v1 name=redis-slave 2 ``` Once the replication controller is up, you can list the pods in the cluster, to verify that the master and slaves are running. You should see a list that includes something like the following: @@ -413,7 +413,7 @@ spec: spec: containers: - name: php-redis - image: gcr.io/google_samples/gb-frontend:v2 + image: gcr.io/google_samples/gb-frontend:v3 env: - name: GET_HOSTS_FROM value: dns @@ -441,9 +441,9 @@ Then, list all your replication controllers: ```console $ kubectl get rc CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS -frontend php-redis kubernetes/example-guestbook-php-redis:v2 name=frontend 3 +frontend php-redis kubernetes/example-guestbook-php-redis:v3 name=frontend 3 redis-master master redis name=redis-master 1 -redis-slave slave kubernetes/redis-slave:v2 name=redis-slave 2 +redis-slave slave gcr.io/google_samples/gb-redisslave:v1 name=redis-slave 2 ``` Once it's up (again, it may take up to thirty seconds to create the pods) you can list the pods in the cluster, to verify that the master, slaves and frontends are all running. You should see a list that includes something like the following: diff --git a/examples/guestbook/frontend-controller.yaml b/examples/guestbook/frontend-controller.yaml index ae8d24986bc..1a48f95b346 100644 --- a/examples/guestbook/frontend-controller.yaml +++ b/examples/guestbook/frontend-controller.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: php-redis - image: gcr.io/google_samples/gb-frontend:v2 + image: gcr.io/google_samples/gb-frontend:v3 env: - name: GET_HOSTS_FROM value: dns From 23133a734476f2590457e87240ab25c2dccb8522 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 9 Sep 2015 18:03:54 -0400 Subject: [PATCH 26/46] Define lock coding convention --- docs/devel/coding-conventions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/devel/coding-conventions.md b/docs/devel/coding-conventions.md index 1569d1aa0d8..8ddf000e205 100644 --- a/docs/devel/coding-conventions.md +++ b/docs/devel/coding-conventions.md @@ -50,6 +50,7 @@ Code conventions - so pkg/controllers/autoscaler/foo.go should say `package autoscaler` not `package autoscalercontroller`. - Unless there's a good reason, the `package foo` line should match the name of the directory in which the .go file exists. - Importers can use a different name if they need to disambiguate. + - Locks should be called `lock` and should never be embedded (always `lock sync.Mutex`). When multiple locks are present, give each lock a distinct name following Go conventions - `stateLock`, `mapLock` etc. - API conventions - [API changes](api_changes.md) - [API conventions](api-conventions.md) From b104f660da86c5f4dc0ebc31f35df2e9d882cc32 Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Wed, 9 Sep 2015 15:26:32 -0700 Subject: [PATCH 27/46] enable the experimental API group in e2e tests --- cluster/gce/config-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index ac537bf52f1..a45477a3e7d 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -58,7 +58,7 @@ ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}" TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:---v=4}" KUBELET_TEST_ARGS="--max-pods=100 $TEST_CLUSTER_LOG_LEVEL" -APISERVER_TEST_ARGS="${TEST_CLUSTER_LOG_LEVEL}" +APISERVER_TEST_ARGS="--runtime-config=experimental/v1 ${TEST_CLUSTER_LOG_LEVEL}" CONTROLLER_MANAGER_TEST_ARGS="${TEST_CLUSTER_LOG_LEVEL}" SCHEDULER_TEST_ARGS="${TEST_CLUSTER_LOG_LEVEL}" KUBEPROXY_TEST_ARGS="${TEST_CLUSTER_LOG_LEVEL}" From 73df8f30a3871970ffaf33fa08e4efd8c08b14e4 Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Wed, 19 Aug 2015 16:12:44 -0700 Subject: [PATCH 28/46] adding a proposal for api groups --- docs/proposals/api-group.md | 152 ++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 docs/proposals/api-group.md diff --git a/docs/proposals/api-group.md b/docs/proposals/api-group.md new file mode 100644 index 00000000000..53531d43260 --- /dev/null +++ b/docs/proposals/api-group.md @@ -0,0 +1,152 @@ + + + + +WARNING +WARNING +WARNING +WARNING +WARNING + +

PLEASE NOTE: This document applies to the HEAD of the source tree

+ +If you are using a released version of Kubernetes, you should +refer to the docs that go with that version. + + +The latest 1.0.x release of this document can be found +[here](http://releases.k8s.io/release-1.0/docs/proposals/api-group.md). + +Documentation for other releases can be found at +[releases.k8s.io](http://releases.k8s.io). + +-- + + + + + +# Supporting multiple API groups + +## Goal + +1. Breaking the monolithic v1 API into modular groups and allowing groups to be enabled/disabled individually. This allows us to break the monolithic API server to smaller components in the future. + +2. Supporting different versions in different groups. This allows different groups to evolve at different speed. + +3. Supporting identically named kinds to exist in different groups. This is useful when we experiment new features of an API in the experimental group while supporting the stable API in the original group at the same time. + +4. Exposing the API groups and versions supported by the server. This is required to develop a dynamic client. + +5. Laying the basis for [API Plugin](../../docs/design/extending-api.md). + +6. Keeping the user interaction easy. For example, we should allow users to omit group name when using kubectl if there is no ambiguity. + + +## Bookkeeping for groups + +1. No changes to TypeMeta: + + Currently many internal structures, such as RESTMapper and Scheme, are indexed and retrieved by APIVersion. For a fast implementation targeting the v1.1 deadline, we will concatenate group with version, in the form of "group/version", and use it where a version string is expected, so that many code can be reused. This implies we will not add a new field to TypeMeta, we will use TypeMeta.APIVersion to hold "group/version". + + For backward compatibility, v1 objects belong to the group with an empty name, so existing v1 config files will remain valid. + +2. /pkg/conversion#Scheme: + + The key of /pkg/conversion#Scheme.versionMap for versioned types will be "group/version". For now, the internal version types of all groups will be registered to versionMap[""], as we don't have any identically named kinds in different groups yet. In the near future, internal version types will be registered to versionMap["group/"], and pkg/conversion#Scheme.InternalVersion will have type []string. + + We will need a mechanism to express if two kinds in different groups (e.g., compute/pods and experimental/pods) are convertible, and auto-generate the conversions if they are. + +3. meta.RESTMapper: + + Each group will have its own RESTMapper (of type DefaultRESTMapper), and these mappers will be registered to pkg/api#RESTMapper (of type MultiRESTMapper). + + To support identically named kinds in different groups, We need to expand the input of RESTMapper.VersionAndKindForResource from (resource string) to (group, resource string). If group is not specified and there is ambiguity (i.e., the resource exists in multiple groups), an error should be returned to force the user to specify the group. + +## Server-side implementation + +1. resource handlers' URL: + + We will force the URL to be in the form of prefix/group/version/... + + Prefix is used to differentiate API paths from other paths like /healthz. All groups will use the same prefix="apis", except when backward compatibility requires otherwise. No "/" is allowed in prefix, group, or version. Specifically, + + * for /api/v1, we set the prefix="api" (which is populated from cmd/kube-apiserver/app#APIServer.APIPrefix), group="", version="v1", so the URL remains to be /api/v1. + + * for new kube API groups, we will set the prefix="apis" (we will add a field in type APIServer to hold this prefix), group=GROUP_NAME, version=VERSION. For example, the URL of the experimental resources will be /apis/experimental/v1alpha1. + + * for OpenShift v1 API, because it's currently registered at /oapi/v1, to be backward compatible, OpenShift may set prefix="oapi", group="". + + * for other new third-party API, they should also use the prefix="apis" and choose the group and version. This can be done through the thirdparty API plugin mechanism in [13000](http://pr.k8s.io/13000). + +2. supporting API discovery: + + * At /prefix (e.g., /apis), API server will return the supported groups and their versions using pkg/api/unversioned#APIVersions type, setting the Versions field to "group/version". This is backward compatible, because currently API server does return "v1" encoded in pkg/api/unversioned#APIVersions at /api. (We will also rename the JSON field name from `versions` to `apiVersions`, to be consistent with pkg/api#TypeMeta.APIVersion field) + + * At /prefix/group, API server will return all supported versions of the group. We will create a new type VersionList (name is open to discussion) in pkg/api/unversioned as the API. + + * At /prefix/group/version, API server will return all supported resources in this group, and whether each resource is namespaced. We will create a new type APIResourceList (name is open to discussion) in pkg/api/unversioned as the API. + + We will design how to handle deeper path in other proposals. + + * At /swaggerapi/swagger-version/prefix/group/version, API server will return the Swagger spec of that group/version in `swagger-version` (e.g. we may support both Swagger v1.2 and v2.0). + +3. handling common API objects: + + * top-level common API objects: + + To handle the top-level API objects that are used by all groups, we either have to register them to all schemes, or we can choose not to encode them to a version. We plan to take the latter approach and place such types in a new package called `unversioned`, because many of the common top-level objects, such as APIVersions, VersionList, and APIResourceList, which are used in the API discovery, and pkg/api#Status, are part of the protocol between client and server, and do not belong to the domain-specific parts of the API, which will evolve independently over time. + + Types in the unversioned package will not have the APIVersion field, but may retain the Kind field. + + For backward compatibility, when hanlding the Status, the server will encode it to v1 if the client expects the Status to be encoded in v1, otherwise the server will send the unversioned#Status. If an error occurs before the version can be determined, the server will send the unversioned#Status. + + * non-top-level common API objects: + + Assuming object o belonging to group X is used as a field in an object belonging to group Y, currently genconversion will generate the conversion functions for o in package Y. Hence, we don't need any special treatment for non-top-level common API objects. + + TypeMeta is an exception, because it is a common object that is used by objects in all groups but does not logically belong to any group. We plan to move it to the package `unversioned`. + +## Client-side implementation + +1. clients: + + Currently we have structured (pkg/client/unversioned#ExperimentalClient, pkg/client/unversioned#Client) and unstructured (pkg/kubectl/resource#Helper) clients. The structured clients are not scalable because each of them implements specific interface, e.g., [here](../../pkg/client/unversioned/client.go#L32). Only the unstructured clients are scalable. We should either auto-generate the code for structured clients or migrate to use the unstructured clients as much as possible. + + We should also move the unstructured client to pkg/client/. + +2. Spelling the URL: + + The URL is in the form of prefix/group/version/. The prefix is hard-coded in the client/unversioned.Config (see [here](../../pkg/client/unversioned/experimental.go#L101)). The client should be able to figure out `group` and `version` using the RESTMapper. For a third-party client which does not have access to the RESTMapper, it should discover the mapping of `group`, `version` and `kind` by querying the server as described in point 2 of #server-side-implementation. + +3. kubectl: + + kubectl should accept arguments like `group/resource`, `group/resource/name`. Nevertheless, the user can omit the `group`, then kubectl shall rely on RESTMapper.VersionAndKindForResource() to figure out the default group/version of the resource. For example, for resources (like `node`) that exist in both k8s v1 API and k8s modularized API (like `infra/v2`), we should set kubectl default to use one of them. If there is no default group, kubectl should return an error for the ambiguity. + + When kubectl is used with a single resource type, the --api-version and --output-version flag of kubectl should accept values in the form of `group/version`, and they should work as they do today. For multi-resource operations, we will disable these two flags initially. + + Currently, by setting pkg/client/unversioned/clientcmd/api/v1#Config.NamedCluster[x].Cluster.APIVersion ([here](../../pkg/client/unversioned/clientcmd/api/v1/types.go#L58)), user can configure the default apiVersion used by kubectl to talk to server. It does not make sense to set a global version used by kubectl when there are multiple groups, so we plan to deprecate this field. We may extend the version negotiation function to negotiate the preferred version of each group. Details will be in another proposal. + +## OpenShift integration + +OpenShift can take a similar approach to break monolithic v1 API: keeping the v1 where they are, and gradually adding groups. + +For the v1 objects in OpenShift, they should keep doing what they do now: they should remain registered to Scheme.versionMap["v1"] scheme, they should keep being added to originMapper. + +For new OpenShift groups, they should do the same as native Kubernetes groups would do: each group should register to Scheme.versionMap["group/version"], each should has separate RESTMapper and the register the MultiRESTMapper. + +To expose a list of the supported Openshift groups to clients, OpenShift just has to call to pkg/cmd/server/origin#call initAPIVersionRoute() as it does now, passing in the supported "group/versions" instead of "versions". + + +## Future work + +1. Dependencies between groups: we need an interface to register the dependencies between groups. It is not our priority now as the use cases are not clear yet. + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/proposals/api-group.md?pixel)]() + From 3f8953e23b25d5695306439abb9c953d83242ca5 Mon Sep 17 00:00:00 2001 From: gmarek Date: Wed, 9 Sep 2015 13:05:22 +0200 Subject: [PATCH 29/46] Add additional knobs to SimpleKubelet --- cmd/integration/integration.go | 12 ++++++++++-- cmd/kubelet/app/server.go | 16 ++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go index 1acd1a8cc8d..d17f7d2520d 100644 --- a/cmd/integration/integration.go +++ b/cmd/integration/integration.go @@ -204,7 +204,11 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string configFilePath := makeTempDirOrDie("config", testRootDir) glog.Infof("Using %s as root dir for kubelet #1", testRootDir) fakeDocker1.VersionInfo = docker.Env{"ApiVersion=1.15"} - kcfg := kubeletapp.SimpleKubelet(cl, &fakeDocker1, "localhost", testRootDir, firstManifestURL, "127.0.0.1", 10250, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, configFilePath, nil, kubecontainer.FakeOS{}) + kcfg := kubeletapp.SimpleKubelet(cl, &fakeDocker1, "localhost", testRootDir, firstManifestURL, "127.0.0.1", + 10250 /* KubeletPort */, 0 /* ReadOnlyPort */, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, + cadvisorInterface, configFilePath, nil, kubecontainer.FakeOS{}, 1*time.Second, /* FileCheckFrequency */ + 1*time.Second /* HTTPCheckFrequency */, 10*time.Second /* MinimumGCAge */, 3*time.Second, /* NodeStatusUpdateFrequency */ + 10*time.Second /* SyncFrequency */) kubeletapp.RunKubelet(kcfg, nil) // Kubelet (machine) // Create a second kubelet so that the guestbook example's two redis slaves both @@ -212,7 +216,11 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string testRootDir = makeTempDirOrDie("kubelet_integ_2.", "") glog.Infof("Using %s as root dir for kubelet #2", testRootDir) fakeDocker2.VersionInfo = docker.Env{"ApiVersion=1.15"} - kcfg = kubeletapp.SimpleKubelet(cl, &fakeDocker2, "127.0.0.1", testRootDir, secondManifestURL, "127.0.0.1", 10251, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, "", nil, kubecontainer.FakeOS{}) + kcfg = kubeletapp.SimpleKubelet(cl, &fakeDocker2, "127.0.0.1", testRootDir, secondManifestURL, "127.0.0.1", + 10251 /* KubeletPort */, 0 /* ReadOnlyPort */, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, + cadvisorInterface, "", nil, kubecontainer.FakeOS{}, 1*time.Second, /* FileCheckFrequency */ + 1*time.Second /* HTTPCheckFrequency */, 10*time.Second /* MinimumGCAge */, 3*time.Second, /* NodeStatusUpdateFrequency */ + 10*time.Second /* SyncFrequency */) kubeletapp.RunKubelet(kcfg, nil) return apiServer.URL, configFilePath } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 0d66b8e8bc0..2c4a9e4931c 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -371,6 +371,7 @@ func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) { DockerExecHandler: dockerExecHandler, ResolverConfig: s.ResolverConfig, CPUCFSQuota: s.CPUCFSQuota, + OSInterface: kubecontainer.RealOS{}, }, nil } @@ -560,13 +561,15 @@ func SimpleKubelet(client *client.Client, dockerClient dockertools.DockerInterface, hostname, rootDir, manifestURL, address string, port uint, + readOnlyPort uint, masterServiceNamespace string, volumePlugins []volume.VolumePlugin, tlsOptions *kubelet.TLSOptions, cadvisorInterface cadvisor.Interface, configFilePath string, cloud cloudprovider.Interface, - osInterface kubecontainer.OSInterface) *KubeletConfig { + osInterface kubecontainer.OSInterface, + fileCheckFrequency, httpCheckFrequency, minimumGCAge, nodeStatusUpdateFrequency, syncFrequency time.Duration) *KubeletConfig { imageGCPolicy := kubelet.ImageGCPolicy{ HighThresholdPercent: 90, @@ -584,13 +587,14 @@ func SimpleKubelet(client *client.Client, ManifestURL: manifestURL, PodInfraContainerImage: dockertools.PodInfraContainerImage, Port: port, + ReadOnlyPort: readOnlyPort, Address: net.ParseIP(address), EnableServer: true, EnableDebuggingHandlers: true, - HTTPCheckFrequency: 1 * time.Second, - FileCheckFrequency: 1 * time.Second, - SyncFrequency: 3 * time.Second, - MinimumGCAge: 10 * time.Second, + HTTPCheckFrequency: httpCheckFrequency, + FileCheckFrequency: fileCheckFrequency, + SyncFrequency: syncFrequency, + MinimumGCAge: minimumGCAge, MaxPerPodContainerCount: 2, MaxContainerCount: 100, RegisterNode: true, @@ -602,7 +606,7 @@ func SimpleKubelet(client *client.Client, ImageGCPolicy: imageGCPolicy, DiskSpacePolicy: diskSpacePolicy, Cloud: cloud, - NodeStatusUpdateFrequency: 10 * time.Second, + NodeStatusUpdateFrequency: nodeStatusUpdateFrequency, ResourceContainer: "/kubelet", OSInterface: osInterface, CgroupRoot: "", From fb14c8cdf331aa0321d94e978c3c7e0f898c43fc Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Wed, 9 Sep 2015 14:18:31 +0200 Subject: [PATCH 30/46] Allow to replace os.Exit() with panic when CLI command fatal error --- pkg/kubectl/cmd/util/helpers.go | 41 ++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/pkg/kubectl/cmd/util/helpers.go b/pkg/kubectl/cmd/util/helpers.go index 2adb8113e44..4b39f616f3b 100644 --- a/pkg/kubectl/cmd/util/helpers.go +++ b/pkg/kubectl/cmd/util/helpers.go @@ -62,13 +62,37 @@ func AddSourceToErr(verb string, source string, err error) error { return err } +var fatalErrHandler = fatal + +// BehaviorOnFatal allows you to override the default behavior when a fatal +// error occurs, which is call os.Exit(1). You can pass 'panic' as a function +// here if you prefer the panic() over os.Exit(1). +func BehaviorOnFatal(f func(string)) { + fatalErrHandler = f +} + +// fatal prints the message and then exits. If V(2) or greater, glog.Fatal +// is invoked for extended information. +func fatal(msg string) { + // add newline if needed + if !strings.HasSuffix(msg, "\n") { + msg += "\n" + } + + if glog.V(2) { + glog.FatalDepth(2, msg) + } + fmt.Fprint(os.Stderr, msg) + os.Exit(1) +} + // CheckErr prints a user friendly error to STDERR and exits with a non-zero // exit code. Unrecognized errors will be printed with an "error: " prefix. // // This method is generic to the command in use and may be used by non-Kubectl // commands. func CheckErr(err error) { - checkErr(err, fatal) + checkErr(err, fatalErrHandler) } func checkErr(err error, handleErr func(string)) { @@ -180,21 +204,6 @@ func messageForError(err error) string { return msg } -// fatal prints the message and then exits. If V(2) or greater, glog.Fatal -// is invoked for extended information. -func fatal(msg string) { - // add newline if needed - if !strings.HasSuffix(msg, "\n") { - msg += "\n" - } - - if glog.V(2) { - glog.FatalDepth(2, msg) - } - fmt.Fprint(os.Stderr, msg) - os.Exit(1) -} - func UsageError(cmd *cobra.Command, format string, args ...interface{}) error { msg := fmt.Sprintf(format, args...) return fmt.Errorf("%s\nsee '%s -h' for help.", msg, cmd.CommandPath()) From 44e6a566f643b4bf42e76db41f3814ec2861b0ab Mon Sep 17 00:00:00 2001 From: gmarek Date: Wed, 9 Sep 2015 13:10:08 +0200 Subject: [PATCH 31/46] alphabetize structs in cmd/kubelet/app/server.go --- cmd/integration/integration.go | 52 +++- cmd/kubelet/app/server.go | 440 ++++++++++++++++----------------- 2 files changed, 264 insertions(+), 228 deletions(-) diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go index d17f7d2520d..fd4ecf9fb75 100644 --- a/cmd/integration/integration.go +++ b/cmd/integration/integration.go @@ -204,11 +204,29 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string configFilePath := makeTempDirOrDie("config", testRootDir) glog.Infof("Using %s as root dir for kubelet #1", testRootDir) fakeDocker1.VersionInfo = docker.Env{"ApiVersion=1.15"} - kcfg := kubeletapp.SimpleKubelet(cl, &fakeDocker1, "localhost", testRootDir, firstManifestURL, "127.0.0.1", - 10250 /* KubeletPort */, 0 /* ReadOnlyPort */, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, - cadvisorInterface, configFilePath, nil, kubecontainer.FakeOS{}, 1*time.Second, /* FileCheckFrequency */ - 1*time.Second /* HTTPCheckFrequency */, 10*time.Second /* MinimumGCAge */, 3*time.Second, /* NodeStatusUpdateFrequency */ + + kcfg := kubeletapp.SimpleKubelet( + cl, + &fakeDocker1, + "localhost", + testRootDir, + firstManifestURL, + "127.0.0.1", + 10250, /* KubeletPort */ + 0, /* ReadOnlyPort */ + api.NamespaceDefault, + empty_dir.ProbeVolumePlugins(), + nil, + cadvisorInterface, + configFilePath, + nil, + kubecontainer.FakeOS{}, + 1*time.Second, /* FileCheckFrequency */ + 1*time.Second, /* HTTPCheckFrequency */ + 10*time.Second, /* MinimumGCAge */ + 3*time.Second, /* NodeStatusUpdateFrequency */ 10*time.Second /* SyncFrequency */) + kubeletapp.RunKubelet(kcfg, nil) // Kubelet (machine) // Create a second kubelet so that the guestbook example's two redis slaves both @@ -216,11 +234,29 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string testRootDir = makeTempDirOrDie("kubelet_integ_2.", "") glog.Infof("Using %s as root dir for kubelet #2", testRootDir) fakeDocker2.VersionInfo = docker.Env{"ApiVersion=1.15"} - kcfg = kubeletapp.SimpleKubelet(cl, &fakeDocker2, "127.0.0.1", testRootDir, secondManifestURL, "127.0.0.1", - 10251 /* KubeletPort */, 0 /* ReadOnlyPort */, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, - cadvisorInterface, "", nil, kubecontainer.FakeOS{}, 1*time.Second, /* FileCheckFrequency */ - 1*time.Second /* HTTPCheckFrequency */, 10*time.Second /* MinimumGCAge */, 3*time.Second, /* NodeStatusUpdateFrequency */ + + kcfg = kubeletapp.SimpleKubelet( + cl, + &fakeDocker2, + "127.0.0.1", + testRootDir, + secondManifestURL, + "127.0.0.1", + 10251, /* KubeletPort */ + 0, /* ReadOnlyPort */ + api.NamespaceDefault, + empty_dir.ProbeVolumePlugins(), + nil, + cadvisorInterface, + "", + nil, + kubecontainer.FakeOS{}, + 1*time.Second, /* FileCheckFrequency */ + 1*time.Second, /* HTTPCheckFrequency */ + 10*time.Second, /* MinimumGCAge */ + 3*time.Second, /* NodeStatusUpdateFrequency */ 10*time.Second /* SyncFrequency */) + kubeletapp.RunKubelet(kcfg, nil) return apiServer.URL, configFilePath } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 2c4a9e4931c..225d756f26a 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -66,75 +66,75 @@ const defaultRootDir = "/var/lib/kubelet" // KubeletServer encapsulates all of the parameters necessary for starting up // a kubelet. These can either be set via command line or directly. type KubeletServer struct { - Config string - SyncFrequency time.Duration - FileCheckFrequency time.Duration - HTTPCheckFrequency time.Duration - ManifestURL string - ManifestURLHeader string - EnableServer bool Address net.IP - Port uint - ReadOnlyPort uint - HostnameOverride string - PodInfraContainerImage string - DockerEndpoint string - RootDirectory string AllowPrivileged bool - HostNetworkSources string - RegistryPullQPS float64 - RegistryBurst int - EventRecordQPS float32 - EventBurst int - RunOnce bool - EnableDebuggingHandlers bool - MinimumGCAge time.Duration - MaxPerPodContainerCount int - MaxContainerCount int - AuthPath util.StringFlag // Deprecated -- use KubeConfig instead - KubeConfig util.StringFlag - CadvisorPort uint - HealthzPort int - HealthzBindAddress net.IP - OOMScoreAdj int APIServerList []string - RegisterNode bool - StandaloneMode bool - ClusterDomain string - MasterServiceNamespace string + AuthPath util.StringFlag // Deprecated -- use KubeConfig instead + CadvisorPort uint + CertDirectory string + CgroupRoot string + CloudConfigFile string + CloudProvider string ClusterDNS net.IP - StreamingConnectionIdleTimeout time.Duration + ClusterDomain string + Config string + ConfigureCBR0 bool + ContainerRuntime string + CPUCFSQuota bool + DockerDaemonContainer string + DockerEndpoint string + DockerExecHandlerName string + EnableDebuggingHandlers bool + EnableServer bool + EventBurst int + EventRecordQPS float32 + FileCheckFrequency time.Duration + HealthzBindAddress net.IP + HealthzPort int + HostnameOverride string + HostNetworkSources string + HTTPCheckFrequency time.Duration ImageGCHighThresholdPercent int ImageGCLowThresholdPercent int + KubeConfig util.StringFlag LowDiskSpaceThresholdMB int - NetworkPluginName string + ManifestURL string + ManifestURLHeader string + MasterServiceNamespace string + MaxContainerCount int + MaxPerPodContainerCount int + MaxPods int + MinimumGCAge time.Duration NetworkPluginDir string - CloudProvider string - CloudConfigFile string + NetworkPluginName string + NodeStatusUpdateFrequency time.Duration + OOMScoreAdj int + PodCIDR string + PodInfraContainerImage string + Port uint + ReadOnlyPort uint + RegisterNode bool + RegistryBurst int + RegistryPullQPS float64 + ResolverConfig string + ResourceContainer string + RktPath string + RootDirectory string + RunOnce bool + StandaloneMode bool + StreamingConnectionIdleTimeout time.Duration + SyncFrequency time.Duration + SystemContainer string TLSCertFile string TLSPrivateKeyFile string - CertDirectory string - NodeStatusUpdateFrequency time.Duration - ResourceContainer string - CgroupRoot string - ContainerRuntime string - RktPath string - DockerDaemonContainer string - SystemContainer string - ConfigureCBR0 bool - PodCIDR string - MaxPods int - DockerExecHandlerName string - ResolverConfig string - CPUCFSQuota bool - // Flags intended for testing - // Crash immediately, rather than eating panics. - ReallyCrashForTesting bool - // Insert a probability of random errors during calls to the master. - ChaosChance float64 + // Flags intended for testing // Is the kubelet containerized? Containerized bool + // Insert a probability of random errors during calls to the master. + ChaosChance float64 + // Crash immediately, rather than eating panics. + ReallyCrashForTesting bool } // bootstrapping interface for kubelet, targets the initialization protocol @@ -153,45 +153,45 @@ type KubeletBuilder func(kc *KubeletConfig) (KubeletBootstrap, *config.PodConfig // NewKubeletServer will create a new KubeletServer with default values. func NewKubeletServer() *KubeletServer { return &KubeletServer{ - SyncFrequency: 10 * time.Second, - FileCheckFrequency: 20 * time.Second, - HTTPCheckFrequency: 20 * time.Second, - EnableServer: true, Address: net.ParseIP("0.0.0.0"), - Port: ports.KubeletPort, - ReadOnlyPort: ports.KubeletReadOnlyPort, - PodInfraContainerImage: dockertools.PodInfraContainerImage, - RootDirectory: defaultRootDir, - RegistryBurst: 10, - EnableDebuggingHandlers: true, - MinimumGCAge: 1 * time.Minute, - MaxPerPodContainerCount: 2, - MaxContainerCount: 100, AuthPath: util.NewStringFlag("/var/lib/kubelet/kubernetes_auth"), // deprecated - KubeConfig: util.NewStringFlag("/var/lib/kubelet/kubeconfig"), CadvisorPort: 4194, - HealthzPort: 10248, + CertDirectory: "/var/run/kubernetes", + CgroupRoot: "", + ConfigureCBR0: false, + ContainerRuntime: "docker", + CPUCFSQuota: false, + DockerDaemonContainer: "/docker-daemon", + DockerExecHandlerName: "native", + EnableDebuggingHandlers: true, + EnableServer: true, + FileCheckFrequency: 20 * time.Second, HealthzBindAddress: net.ParseIP("127.0.0.1"), - RegisterNode: true, // will be ignored if no apiserver is configured - OOMScoreAdj: qos.KubeletOomScoreAdj, - MasterServiceNamespace: api.NamespaceDefault, + HealthzPort: 10248, + HostNetworkSources: kubelet.FileSource, + HTTPCheckFrequency: 20 * time.Second, ImageGCHighThresholdPercent: 90, ImageGCLowThresholdPercent: 80, + KubeConfig: util.NewStringFlag("/var/lib/kubelet/kubeconfig"), LowDiskSpaceThresholdMB: 256, - NetworkPluginName: "", + MasterServiceNamespace: api.NamespaceDefault, + MaxContainerCount: 100, + MaxPerPodContainerCount: 2, + MinimumGCAge: 1 * time.Minute, NetworkPluginDir: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/", - HostNetworkSources: kubelet.FileSource, - CertDirectory: "/var/run/kubernetes", + NetworkPluginName: "", NodeStatusUpdateFrequency: 10 * time.Second, - ResourceContainer: "/kubelet", - CgroupRoot: "", - ContainerRuntime: "docker", - RktPath: "", - DockerDaemonContainer: "/docker-daemon", - SystemContainer: "", - ConfigureCBR0: false, - DockerExecHandlerName: "native", - CPUCFSQuota: false, + OOMScoreAdj: qos.KubeletOomScoreAdj, + PodInfraContainerImage: dockertools.PodInfraContainerImage, + Port: ports.KubeletPort, + ReadOnlyPort: ports.KubeletReadOnlyPort, + RegisterNode: true, // will be ignored if no apiserver is configured + RegistryBurst: 10, + ResourceContainer: "/kubelet", + RktPath: "", + RootDirectory: defaultRootDir, + SyncFrequency: 10 * time.Second, + SystemContainer: "", } } @@ -317,61 +317,61 @@ func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) { } return &KubeletConfig{ - Address: s.Address, - AllowPrivileged: s.AllowPrivileged, - HostNetworkSources: hostNetworkSources, - HostnameOverride: s.HostnameOverride, - RootDirectory: s.RootDirectory, - ConfigFile: s.Config, - ManifestURL: s.ManifestURL, - ManifestURLHeader: manifestURLHeader, - FileCheckFrequency: s.FileCheckFrequency, - HTTPCheckFrequency: s.HTTPCheckFrequency, - PodInfraContainerImage: s.PodInfraContainerImage, - SyncFrequency: s.SyncFrequency, - RegistryPullQPS: s.RegistryPullQPS, - RegistryBurst: s.RegistryBurst, - EventRecordQPS: s.EventRecordQPS, - EventBurst: s.EventBurst, - MinimumGCAge: s.MinimumGCAge, - MaxPerPodContainerCount: s.MaxPerPodContainerCount, - MaxContainerCount: s.MaxContainerCount, - RegisterNode: s.RegisterNode, - StandaloneMode: (len(s.APIServerList) == 0), - ClusterDomain: s.ClusterDomain, - ClusterDNS: s.ClusterDNS, - Runonce: s.RunOnce, + Address: s.Address, + AllowPrivileged: s.AllowPrivileged, + CadvisorInterface: nil, // launches background processes, not set here + CgroupRoot: s.CgroupRoot, + Cloud: nil, // cloud provider might start background processes + ClusterDNS: s.ClusterDNS, + ClusterDomain: s.ClusterDomain, + ConfigFile: s.Config, + ConfigureCBR0: s.ConfigureCBR0, + ContainerRuntime: s.ContainerRuntime, + CPUCFSQuota: s.CPUCFSQuota, + DiskSpacePolicy: diskSpacePolicy, + DockerClient: dockertools.ConnectToDockerOrDie(s.DockerEndpoint), + DockerDaemonContainer: s.DockerDaemonContainer, + DockerExecHandler: dockerExecHandler, + EnableDebuggingHandlers: s.EnableDebuggingHandlers, + EnableServer: s.EnableServer, + EventBurst: s.EventBurst, + EventRecordQPS: s.EventRecordQPS, + FileCheckFrequency: s.FileCheckFrequency, + HostnameOverride: s.HostnameOverride, + HostNetworkSources: hostNetworkSources, + HTTPCheckFrequency: s.HTTPCheckFrequency, + ImageGCPolicy: imageGCPolicy, + KubeClient: nil, + ManifestURL: s.ManifestURL, + ManifestURLHeader: manifestURLHeader, + MasterServiceNamespace: s.MasterServiceNamespace, + MaxContainerCount: s.MaxContainerCount, + MaxPerPodContainerCount: s.MaxPerPodContainerCount, + MaxPods: s.MaxPods, + MinimumGCAge: s.MinimumGCAge, + Mounter: mounter, + NetworkPluginName: s.NetworkPluginName, + NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir), + NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency, + OSInterface: kubecontainer.RealOS{}, + PodCIDR: s.PodCIDR, + PodInfraContainerImage: s.PodInfraContainerImage, Port: s.Port, ReadOnlyPort: s.ReadOnlyPort, - CadvisorInterface: nil, // launches background processes, not set here - EnableServer: s.EnableServer, - EnableDebuggingHandlers: s.EnableDebuggingHandlers, - DockerClient: dockertools.ConnectToDockerOrDie(s.DockerEndpoint), - KubeClient: nil, - MasterServiceNamespace: s.MasterServiceNamespace, - VolumePlugins: ProbeVolumePlugins(), - NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir), - NetworkPluginName: s.NetworkPluginName, + RegisterNode: s.RegisterNode, + RegistryBurst: s.RegistryBurst, + RegistryPullQPS: s.RegistryPullQPS, + ResolverConfig: s.ResolverConfig, + ResourceContainer: s.ResourceContainer, + RktPath: s.RktPath, + RootDirectory: s.RootDirectory, + Runonce: s.RunOnce, + StandaloneMode: (len(s.APIServerList) == 0), StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout, + SyncFrequency: s.SyncFrequency, + SystemContainer: s.SystemContainer, TLSOptions: tlsOptions, - ImageGCPolicy: imageGCPolicy, - DiskSpacePolicy: diskSpacePolicy, - Cloud: nil, // cloud provider might start background processes - NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency, - ResourceContainer: s.ResourceContainer, - CgroupRoot: s.CgroupRoot, - ContainerRuntime: s.ContainerRuntime, - RktPath: s.RktPath, - Mounter: mounter, - DockerDaemonContainer: s.DockerDaemonContainer, - SystemContainer: s.SystemContainer, - ConfigureCBR0: s.ConfigureCBR0, - PodCIDR: s.PodCIDR, - MaxPods: s.MaxPods, - DockerExecHandler: dockerExecHandler, - ResolverConfig: s.ResolverConfig, - CPUCFSQuota: s.CPUCFSQuota, - OSInterface: kubecontainer.RealOS{}, + VolumePlugins: ProbeVolumePlugins(), }, nil } @@ -580,44 +580,44 @@ func SimpleKubelet(client *client.Client, RootFreeDiskMB: 256, } kcfg := KubeletConfig{ - KubeClient: client, - DockerClient: dockerClient, - HostnameOverride: hostname, - RootDirectory: rootDir, - ManifestURL: manifestURL, - PodInfraContainerImage: dockertools.PodInfraContainerImage, - Port: port, - ReadOnlyPort: readOnlyPort, - Address: net.ParseIP(address), - EnableServer: true, - EnableDebuggingHandlers: true, - HTTPCheckFrequency: httpCheckFrequency, - FileCheckFrequency: fileCheckFrequency, - SyncFrequency: syncFrequency, - MinimumGCAge: minimumGCAge, - MaxPerPodContainerCount: 2, - MaxContainerCount: 100, - RegisterNode: true, - MasterServiceNamespace: masterServiceNamespace, - VolumePlugins: volumePlugins, - TLSOptions: tlsOptions, - CadvisorInterface: cadvisorInterface, - ConfigFile: configFilePath, - ImageGCPolicy: imageGCPolicy, - DiskSpacePolicy: diskSpacePolicy, - Cloud: cloud, - NodeStatusUpdateFrequency: nodeStatusUpdateFrequency, - ResourceContainer: "/kubelet", - OSInterface: osInterface, + Address: net.ParseIP(address), + CadvisorInterface: cadvisorInterface, CgroupRoot: "", + Cloud: cloud, + ConfigFile: configFilePath, ContainerRuntime: "docker", - Mounter: mount.New(), - DockerDaemonContainer: "/docker-daemon", - SystemContainer: "", - MaxPods: 32, - DockerExecHandler: &dockertools.NativeExecHandler{}, - ResolverConfig: kubelet.ResolvConfDefault, CPUCFSQuota: false, + DiskSpacePolicy: diskSpacePolicy, + DockerClient: dockerClient, + DockerDaemonContainer: "/docker-daemon", + DockerExecHandler: &dockertools.NativeExecHandler{}, + EnableDebuggingHandlers: true, + EnableServer: true, + FileCheckFrequency: fileCheckFrequency, + HostnameOverride: hostname, + HTTPCheckFrequency: httpCheckFrequency, + ImageGCPolicy: imageGCPolicy, + KubeClient: client, + ManifestURL: manifestURL, + MasterServiceNamespace: masterServiceNamespace, + MaxContainerCount: 100, + MaxPerPodContainerCount: 2, + MaxPods: 32, + MinimumGCAge: minimumGCAge, + Mounter: mount.New(), + NodeStatusUpdateFrequency: nodeStatusUpdateFrequency, + OSInterface: osInterface, + PodInfraContainerImage: dockertools.PodInfraContainerImage, + Port: port, + ReadOnlyPort: readOnlyPort, + RegisterNode: true, + ResolverConfig: kubelet.ResolvConfDefault, + ResourceContainer: "/kubelet", + RootDirectory: rootDir, + SyncFrequency: syncFrequency, + SystemContainer: "", + TLSOptions: tlsOptions, + VolumePlugins: volumePlugins, } return &kcfg } @@ -739,64 +739,64 @@ func makePodSourceConfig(kc *KubeletConfig) *config.PodConfig { // KubeletConfig is all of the parameters necessary for running a kubelet. // TODO: This should probably be merged with KubeletServer. The extra object is a consequence of refactoring. type KubeletConfig struct { - KubeClient *client.Client - DockerClient dockertools.DockerInterface - CadvisorInterface cadvisor.Interface Address net.IP AllowPrivileged bool - HostNetworkSources []string - HostnameOverride string - RootDirectory string + CadvisorInterface cadvisor.Interface + CgroupRoot string + Cloud cloudprovider.Interface + ClusterDNS net.IP + ClusterDomain string ConfigFile string + ConfigureCBR0 bool + ContainerRuntime string + CPUCFSQuota bool + DiskSpacePolicy kubelet.DiskSpacePolicy + DockerClient dockertools.DockerInterface + DockerDaemonContainer string + DockerExecHandler dockertools.ExecHandler + EnableDebuggingHandlers bool + EnableServer bool + EventBurst int + EventRecordQPS float32 + FileCheckFrequency time.Duration + Hostname string + HostnameOverride string + HostNetworkSources []string + HTTPCheckFrequency time.Duration + ImageGCPolicy kubelet.ImageGCPolicy + KubeClient *client.Client ManifestURL string ManifestURLHeader http.Header - FileCheckFrequency time.Duration - HTTPCheckFrequency time.Duration - Hostname string - NodeName string - PodInfraContainerImage string - SyncFrequency time.Duration - RegistryPullQPS float64 - RegistryBurst int - EventRecordQPS float32 - EventBurst int - MinimumGCAge time.Duration - MaxPerPodContainerCount int + MasterServiceNamespace string MaxContainerCount int - RegisterNode bool - StandaloneMode bool - ClusterDomain string - ClusterDNS net.IP - EnableServer bool - EnableDebuggingHandlers bool + MaxPerPodContainerCount int + MaxPods int + MinimumGCAge time.Duration + Mounter mount.Interface + NetworkPluginName string + NetworkPlugins []network.NetworkPlugin + NodeName string + NodeStatusUpdateFrequency time.Duration + OSInterface kubecontainer.OSInterface + PodCIDR string + PodInfraContainerImage string Port uint ReadOnlyPort uint - Runonce bool - MasterServiceNamespace string - VolumePlugins []volume.VolumePlugin - NetworkPlugins []network.NetworkPlugin - NetworkPluginName string - StreamingConnectionIdleTimeout time.Duration Recorder record.EventRecorder - TLSOptions *kubelet.TLSOptions - ImageGCPolicy kubelet.ImageGCPolicy - DiskSpacePolicy kubelet.DiskSpacePolicy - Cloud cloudprovider.Interface - NodeStatusUpdateFrequency time.Duration - ResourceContainer string - OSInterface kubecontainer.OSInterface - CgroupRoot string - ContainerRuntime string - RktPath string - Mounter mount.Interface - DockerDaemonContainer string - SystemContainer string - ConfigureCBR0 bool - PodCIDR string - MaxPods int - DockerExecHandler dockertools.ExecHandler + RegisterNode bool + RegistryBurst int + RegistryPullQPS float64 ResolverConfig string - CPUCFSQuota bool + ResourceContainer string + RktPath string + RootDirectory string + Runonce bool + StandaloneMode bool + StreamingConnectionIdleTimeout time.Duration + SyncFrequency time.Duration + SystemContainer string + TLSOptions *kubelet.TLSOptions + VolumePlugins []volume.VolumePlugin } func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) { From ca30de0bde294bf60f22ce51b366e0e5a291c112 Mon Sep 17 00:00:00 2001 From: gmarek Date: Tue, 11 Aug 2015 16:58:24 +0200 Subject: [PATCH 32/46] Initial kubemark proposal --- docs/proposals/Kubemark_architecture.png | Bin 0 -> 30417 bytes docs/proposals/kubemark.md | 190 +++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 docs/proposals/Kubemark_architecture.png create mode 100644 docs/proposals/kubemark.md diff --git a/docs/proposals/Kubemark_architecture.png b/docs/proposals/Kubemark_architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..479ad8b11f490d49fe977e28a253e6f1a4c4b263 GIT binary patch literal 30417 zcmeFZRX~(o_clIsm%z}i0>V&AH;AB;V!+TnG)VUV5;9UEEh&N`%@9hf2$G}ZfP^3n z-JRb(@I3GHd*Ao>Klq=02j7VY<9+XY@3q%n>sr^kHqUjn)hS4sNkJeG#a)ft4?rM% z2nd9ePD}{=Qo7#V4+43o-My{+(06R5i7dg`=3|TO%V%Gox!te71{YPSC%8dP9QIr` z(Oe3ji@WpD%hUMG106cM^p}#FzPt_nX z%<*CL+M@>;gQ60d!S`PChKoK2wl-cm{hIxnKKL#|!nGTs?ykrKTj`Dxbl-w>i@uAOzhMnIJCXed% zz~u*}HN&dYA3bad>kU?%?_Rw09v4}EzP4_SK0l2ahAwl7Osl~D+JBUaV`&8V(i2D^ z$YCfM zAxPcJv0-^N!_fbKA02XqZSEAxZtVhZ?)M`4(pXS4lce7@R@20BkkPMfv#)G50IP1>|0R8s%{2ejgW~&`yIVEv z{_9oULgzWt`F^<(;-bjmu?$9F)QorG>-uV?LHSpa)PI}cp^?`{&9HYLc15L4!Z+T& z0<7bV(Zm{T>^L7Vi|hf^QPxg{tR)lbxEiDfSPd|&tlfXT(j%dTwZtSt@`HQWxBu&b z|7A>IL;rrh)}s(Rhs!Up1-ib3j!V-*iT^bXV4ueN(y@D5v#$p102!5aR+kS6B@3zgPDee0pzI&5x_6Sc1pI!yw z9RaRX&>=ZAaV*`PmR;oVH+yuCKd&7{)Wf45*XgP`)e`JCG)I~4kAkZrh#wO=9G6l; zIwK9A{=ig8n)zw!av@;>u-^%?r8Rra#egHje$b2PC~ncBUsx3;3vF-J5^b?Ofs+A4 zDv>#3?qT<0?6~;Bz4sO*kj|=wcVf4#=eEeI@p#uW73YLi2$qc8*33*B`KBHhcnaM3?tW{fWW@cUS&uwNJnN;-U*59eM z1=zio;5A#DUk@%dGKo1L_l?~*SbMEj^k?MRX+e#!R7d@pd*4am+Vh63W!es-J?W~` zp%0I}g9usm8$jH$Pq0UHc=8G0WHq8l&dya}UHIn8rO&yMxr78v%#C~BY2*5Y6xMg- zjKW*?r(9K@fOIMX50L!mG|`56oWny*phSpblM9?Qj;FR=42Oug!Y?bRGTtDL#etQF zI(+0OGg}8ofc4O}Ez62szx+2$hs1FK)1gTE#JAYq;nRbo6QAMuZHq%br7oS6_#ijU zlRvJtskoGpo)ubs^5!_>wmEc?vHj*V&*Zm9>V-~v#q9!&w;jYGGp})2oszbn&zk2f zid<1JtNs3z6>r(CL~ul(=CSOPTKj)RpDR%ELli#l&I~mTF0?Pc=tK=hgJx>?h-aCY za4hdVKigdTu-%dq=;>USpGhvsl~5)wch!^!kE{1-v=u+$2L6n-8wl(`52n~iRLOwj z;_q1teOCgA2FM`npYM9R`qlReHR= z3|Qqrz+yWoi~zTDapSirg%f}7c7T4RJuZ@*NK_@^R40EQy=#zNbkcZd7{~l~C52*F zHHicL3_ff_a$F!Yj%=YdA@LSmI4lv03gcq8L2dRlhe>_6DVLko zC8_V_1BqLLkvQPdAQs_2U2Uut*j$Sx7R=I2Aco8x~&0F!iabtcu#Y?5`c+`5>WL`&Jr1$+tZ(mh~Pa#pMs-H`)6H~og@>GjOaIxfmH&^pdwM+}sc;+!=)NAOxUKbb4_K51k9XNW z4(y-`YpZ};qPj{LUsD;!;JnMW3K6uW{A4&X6h8ay<$JUxalUec&kD%lU>&E zS|b?KCng_6m`Kv*pZNX3NyD&oY8FSwSIm`;HgWB0>lI`E3ox|);U2#Yk(nbliHO2c zxaJ*~$(1p3d@(>^f4JGVw(j8tP5|JQ3gTs0Jr>6O;vg2nW~Dp;KoLXk4P*E%$RmCN zV6MKfEM?#3U(b^{S_0?``@=;FXGItO<;jnMbcoIIi6{WU0U@}BMeC2<6r}Gl6j2&0~ zg;wI8fwA%T;uY*FvUsjx{bM!AkpJz(Ye4Ot7zy zOJDg5!T_JaF3&seGFDv9{{M4109Xw$0}OJ3z0_66u!9De{l{{s3Z_Er9BX$m`_>#M zyVuI2E^P+Pz19PZ@W^Wg&Q)dodeTg)nL0GPgBy>QT$GPrh4ap__gJxZ1~s2X@osIK z006_=e;vrjkg4?(zv?ZU+H(i1US|_Obn4bz_N1r!nMujM?Dt)hfNn{`9p-O-TP+V9 zKVp$k5RpUHIsgGyp_COjsXKb?KJNuU%OpqsWA17QWyI-zW&%<4dto3ipLk80eLVTh z6#=f+$w&;S$Rofxd6E*$6>!kZvEs?r)rvQ=-1_}^zD2S2xB8*ijGa!0TVKBZ5@{b1 z3pDU}>8(7w$4w=~r15|54TKOv{IGj6V87IEODXVACk~uSs@t3_%3?aJ->svk23nD66F8m+TQh2hRSmH$<2CGxJPRF_|*792;{dP%V6^i5{s3 zFZxvlgED}`*s7|Y%*73qOPL{On<1$x2r0#$N;eY&+oQys&dGq|k1n~t53J%>K2f?u zMqL6W+)c?J-D`1dpDp$-#o@B;nMulIVX|R& zar=p-NahWqm}zOG@E8r^K%!yRVLHPCwptr9sPH*BYkPkz?_^2pa0QUU4;sG*NAu3$ z7vpE)e@>*(HIB@|5yJw-3{==D%}!x?rSqO%ASnkXarogoum-KK`gQ#2!m*76e<11J zcGH>}!_xzi;Y1zbLh@5I+@p!7@Dw=8%xIu)+BhG6IsGF>12W14UoFAVsn3>`XPBw{l>oqzkCuh_pP0DkUZ|E zTWRw(8tuZi*0)92P@JclGmG-dPDdZ<2Tt$y$N%QDLE$QWPCBky$3au#udeLB7?@wd z*J6C|l5pJy07=(uI_TZP^7z_S@>um{Rtlh;S+-V##rAr_NFW7O-xwYvhEwO}3ZwF! zqKPYTVQu0B~pNA}jD&{8mtnIV!!5XQ`;b%&SnN9a> z;||6xUAg{AB}5e$cVM>rnavl~pN@T)9`SfhMEy6M5N@)9B9+%_ocfKmO&^CBy<+W+ zzSo*(4cZsRtv>rZtzG5@Ehz9YrFR67{KDHZ+r(%Z(Lf%^lEzU8sb;6K2AON&aS43_ zIHx#?>qyQy`3sBa{9Y0^;;-sueW7AIgb1?!!pN1vH~lEdv9{p$OAO`X)8Y0 ztO+BdTdblrtgqkRQ)WVf4qgRiFN&8D$S$SJ7wDn5hd<0!F<<}pRECg%gW*weU7B=y zoiB51of<^DiCW>UX)B1%Pa6A7SJ>`a zbbIFDw<*O|jZwGK#DwDBCocO=%|{6_*HD4LXlghe2tGKaw#ubIjwi)&p|6Hkqitqo z)b}*d$bwinIMJjH^HsWtiBApIX4ktb3@*nvEf|rCB)}nyIVKWwUt28U!vprFmTW=#RLU7PyJ>)HT8}$JXAYxQm|h=7h>OYxsyW zNV3s(%u3r|6${bUQ^1i>s*!s3c0r2QLG~5FVp9Ydf2lDQOc9F~pJB$=eQvL)4{P%) za*HHSPP2H^g5u8Fj2IcfeGeQngRfrNMRWur^wow3PHxaI&phS=X9C1fC~n(l{K2X9 zDegV;6c#_QNXWrJh0G;I?waK|w656ir36a99esfSZgZ~M?bJ4#tYgC<;%8=Na1jKJ}z0fkYwVoK#>9SzIKyn=KQT+H_r6dYh z_j5px(iNT4?_$5jbBQ~csd-49zIjn@(4qxTdS!4_V%!RaQQVQybU0cx(61F#a1zYd z7h-`ftSp!m$~Ro$IIV3Gp}J}P5&e99&LX)4H5xHfZSSZ)Z;s!&JjQA`k3|T5deVN# zgCl=_qSxe2)^-w=AHP6*Kjf}!AnZk#MfcxVrOW!__SberKH9jI*;-KoFwE3Wc(laV zznk(6C-IH*>TFB9WJg!Cw*_Y;Qv=`RTc@S#aHEaP0kKH2$ED8N8j7(Q)JhH6r=tm% z>G#tK=NdV_kP*iXv>DMpx{K5~bsyFenTn*OfIrNj};L>3TW;<1| zn=(VJhRk$hLVFXY7V-^UQkCLN$ml!4pB0X6sGNco#?M;L3cb1mdOt-~ zSI&N;7)82Y08qrd_!kw8?Ra|~zv4Bo9~VqyF+m=IAvj<~RKfMnde9&KaFdD0lgQm6 zAvYR*{ILti8Ji-vi*^#I7DQeUo(>i;nt^iBt6w>v)H!}5nA)tG3A!A9@76BkfpN?( zKk}0P1Ynk>g8veTaeB#nw@qImJEVUg%=vFDd^y-t7!AeAqp6SW(~s*)L!Sorvm6c% z{f+D|NYrqANb|n156iEmTJh?bTQcvE7nt>LVio}Sc@fSTmvrAn@xb`YAur-@?0-S) z0$CeibpQ28Rk)%jz@n;WcS$%0)mV(r8uZKUbK|)~kQM*48bCB-Ex7HBCj`6?%fqJr z3zpc1sKq{)V4M#4E#iXXxL66>iE>18~;+RY+tzU`^G*cn_&&gyE zUT5{|aX<7ASd-^sX*p&;T<7h}PHj4v55aVUz&9l0ekD=L)jI}pc0?!gcx=n8SI5!? zmNXx?Ijz*Y9|jz=iAbM+obK_!F2>@rr2Pxh3{g~o_!nj|n$$L0dnK)@^heHyWHq04L?oNlQ6r@7}0G0W331VHvPPiBD3%S#u91W1_-5N#1f&VhYFE}2l{mab+6#`q_-{O-9LuFiO ziQ&H6$WtW>Q@r4>j^!sK$A?bbJqoxRzUJH8CQM9?PkOchy8S=g`IZ0f1l2q|w1>D* z%EWi?@X(FY)W~c6Q`ubb)`&Gb{Gv9*Cm&7S>&pa zT|>es6`L00P7G>>V|RhNg;$Om+g4%bgDG2Rh8s@762XvW`$&CxBnXz*O!8Z3>PGn>Q$6hri+Wl&oR;fz@g6MmvoS^;QmMMTU z@4OfQyJleFKx1a;9*VoSHu?owWQ^`?*vxBaT}Js)9|Wx^)WR(boQ+S-BiLH^bajGZ z<$^{4m9M^274YS&U*a|MM|~_h4ApU4UX4u3jCW5=2Jl zaqB32l5pLraj^+GKbkxA@#8fAJ2XOoMchL4ZSyA&Yt2&OGvHAH9hAR=GwSp4Q^bT4 zLzd`B(PJo89;Pqd-kcMO0!SBZti^7xN`RRJA8L%SbMLOB^Y~?>C;~j0NA#$)qTewE?#hkJLMpO7qHc;SzddFxZ_Mfd5uejdK@bk zPz?RYTo(k8(R9AqH?|vwKZ$0GJEnZO-zY3;r@PUYsbhonNQEM5`w8-3ptpnu7z0NZ zP5>u*$Gt*XrYT`5EN3n9^=@Z&x<;T3M>7@mic;LGi{;fGy=m0sV6z8wJ*sf76gxn4 zA6=3+z;afR<>3I%vv5Vr)&eBZbPJ%K1Lu6&JvY8 z2Iz`W5Xqw&@Vgrq_Qq~35~^Eq`h=wc-d?054H-Wg%c)vDUv`DoHI9@7NU_Me7@tj? zb^W}2(d9<2|1i@we(}XI!{$@eXBL=FcF@lkZ0!hbGwa+B<9+dGGjE-~b|xPByv`$# zP*QnRno06bsCmKT(q(`(W8aoUL2+=##~s(3su|8U>|~QZLF8TAm=5<}Ly0e6YRjx` z9wQK1GYu&HXLq@f8-!5XZ-dO#3*P(b7}LQG!jV?*3HRCN^;)J_*MQe*{{BScZ|DwAlFT`f2xsH#-oN z2sPdn?{a3&ybkQ=F!o9M_c`1^0+Eu1bDXzyye_E98S9hew=QT(a#CO{M#fPE`^&_9 zG4`di0tqk&aH!RP8%u<5L5%B5-z4FheWj0@Gc1uD*h3>F-q%5l-g-dn@Uj&@+yh zQ}folHFJ-Ry;Mh@+IC(T1MxT%>^NMq=+J-!QbcDWPo#J~;Ng5o&fMv{n5KBSWX0qU zyRTmI`U^&3k(G~s&M7tP#xY)IPJilbO?GWZ6*&PvyS`^9`p!|XoU*k^iq&Bim#C51 zv3@gq_uDU;2EkZNGvuQ<>dcn?`P-p8drh|K<8dG3@T${08+0QLPC;k3EXEwc@w5E0 z9U?huf*sPSO&mE4Bh1yOwv?o}4_U=_bIb134sQ_>_KqtgK5zBcyk>YZxd~MC)2E?5 z^^APPGJ_~w;%6TqKBIaa)>a*{*p@XGIY9^@mO=l7mtlaX=SISmVVk{~F01}OnXcpI z5npvV`H_VN4#Vj4!OgF?!V*S1q#1;JWxE!i%IRr^yM)n31{ZX@vAR#52(BUEa=31B zmGx_0ijB@iG938#2alrX*8CRi4g>0@&J83i|8$mFS!SxUdl=U_ z9#jsX(YpQ_j#RTB&H*c_LGI3@t~J}*{aMb1ITRiGa_VJ4?wKd1b&aD|w)ezAhQFx1Vv2D%_83-OYx*G|K|;vjHY1YGNm4D)krH2d|1n{Z z6sPaz<%?(lwhL7qVddZI4oVrBiNt-c_b> zQB-{%FY8IlEF;OZVTxv?A6ef5>5e{3Ik#Wm_(4B9#M+U%u}GP&gNzeoTcIU0P^tCp zw|d9Nde^w)g<0*+cv;*;5RD+q@f`GyTib4~Rqv6ptb%0RO)*(qXoXhF9P?8MUW(g` z_ErLo&!W{P=C?QftA#r7eYs6uTQR-EEo!15Ee?$_@kZP5!t@+ZCi8F3e|x(Bm^m+R z$7#HX%2A;pwt$jkg_=MBJNoT=s`ljx-64|qxLrjhgN4*XG142}cJFMjp~TBj@WWW!p# z%m@GRE`D4%1w>me*uNo;;UxENYkNYSPN11#(c@ty^nia@VDlIJP5h5YDkP7S!0FUn zgO+*Exx)Dy_4Q8P4efI~T;eOa>fcnxym#>-2OL)uFa`(b zZ%p2BUSFlpEf}5LRMg(+JJ%>hXImonu25*(eeB#sOGg9`Rv{5 z3aL7S6vxipIcE2u!xuoqw7GQ>KJFwNc~)WCIyHKI)tA&2!4%&58{j#TY9;ovz_PJ> z&z3bad+mmEYPBy=)n6H}F_e-Tnfv9NzfIwrRKj{s(5$^&*0;gMfSRBFB;;MWYG~RP z;)BIeLXo`L?*rm88#er3Z2%gMV_|_IURGvPFEzYRSm#Rj&!|}vn?|r2E{U?I zalC(+aLQfT?V1PK14@FCEw|x`why1pe!tnlkYn$8qL26Z=ei!gdCih1`J+OD2zcj9 zMp4V#@gnfK0}lReKzd~LQ7iepJ8x_Fx&v`kV-TtDXy&^52^C%S4VbL23YB&A`IiG1 zbzRm%=ey(KReM)+)!Y|I&jW_W#KvMei9Y}YV}qWYg87}+mpcb=s`ev!d1VCrqy0P~ zHMgmN3RzVLEF4k&ykcrbBBBmc_+Sq%gHQaN`#*kxCp1Wk+bmI3~eSTeVB7o4RkGa~1V*csO-5>i1_8G^W_qB#o; z2EzJeb;t&knglzsT$2a>xp92edm=x7qG^899A9qM>j@U<1(A;C9P%iOq+% zz zd@Rf~N#E`BY2{>c>9>QamHKx16tlf~Mhwd_Z17dtB@Dd4_^{s;NRnIvow59ZC#95kUOb7TGvUNzdvq4}Z~|BkImpd3 z5}_)vkXEGt=QP)N9Uu-5RNEsX_Qw+#O}nbq ziM;ZYvG;%!q{^_^Ccw=Z*SsRI#xYTW!o?d#6MUXkie7pofE6H(7e`1%fFm+!o7mVN zEFWhK=kbLT28Dv#x1YCuDoL{{Ny2S|qVltU-(K9@44E*3(X8H=x&3U)$_Sua_abUz zdy?O&(m1=(Kws6^xd-XPa6Ab=C9u(~+MT=zs%~-EFW!Ns3~{}e_$3W+2cSA@a`NeYlgEsUBei@8xO4(O_zK2l$=JTF()cVlHxI@h8>rZ-%e>fwWnZmRA z0|8Sz7pEHNC8aq3veUL_Jb5D0oVWZ3T008yHA!9OZ1d_ZF_tcwZ>%2p#K38Lki$dY zxbhs35xk21PPpcF!moL+|5}8w@vQHI37||o!3v_V+A{QJ2RlC!_IrAEBNMD!;KJRE?jRyCARGRq}kqOVbMG%K`*s31I}H8dI6rauxxVDT_@*f^*j z#91cb9AcKIhTIg?F(Sz=)QT&K-r2s<{o*|#1oq^ENgWK)4T?d1R!F@l@W4=CGZ%MQ z=%U6pT?yl}4%(B>7;~m7!5CGx8GCE|WqYoVbQ&m{`B4z4gn@0T(_Di~nR&Eh@27hg zO$FV}Ys7t_0#dlKRej$0toCEl3E>`sagG=R>Gk?D_g`v!w1!vp8q3I`UxN`=e`f2u za|t?<6c%Hh@nM!GahXLK{JQtTn>Acr)nJJ-OQv%5YuAC+VvP_r*Sg#dQRLZ#;=vtIgJbdj? zBa8}@kX)l}-+qx7?QUvcYf#xVH%T)h)=X>Tb)&xMHjW;)jDr1mjk0Wyq5rGc$WuGd zBgmTb9lO(!z-u9zB1e%n`<_QpT1aNKkwcsxm-DMazJp>LfzC!gZqe zZ#S!r@kv*5;zT-#JEJwD0p=N};6w9o7x~>fE&ffLJXLOS66W#mbr4@Q##xrU;f!!y= zc5(@o-_FdLr`EQqNcMuyRv6`S+vo}>cXnTD+xO1knAIIHwBQ_|u+-=(|D4NFd?r~s z41s`gO9>z|QZcHtAi_mK#?uHe4i7GM(Kx>f!Xd)XUKQvw;z}{tna_-=8F6$Z*=D^R zk?0-N$aAL&E7IlfQ@@Csw-}J4%|QVY_CK%wRRp3eF~~6}!LNmtYR$??d^WpbglMm4 z=OT5X&e~yCkyRp1tC%(Yh722K)YbD7$*obs?tza*;MLxl7Qe;o<}6}i;NwE1@^fQdz{RT7CnRQF&WNx_U>An7EoU`iCe93En`Wi z;@6m#N$r(s)wk+PoV7Xj+&Fx_m{6@vClU(^52Afm5r_`{sYx{)zxvb!&a^1Ts#D54 z0YtoZfU-DgA<}$fGTqSQ18)^~V5cKHAmscLLYPZBN-p&s2*CPr0BNWccaZaTB7gM* zA>=^<6M^6CdG^T9^YZLmHFzA11<&9q>q$T13uk;}v|&b!>H?swn#NzfK61sw3FpoL zI(~_b{YsATNUX&3{udHLF_c9We7SfdN!!sDu2M}YPP_^?%a}UM2#2u+V4j+XG9rSE z!MjH7Uv8-X@v#=!#PK3K&!~es86?u|n64Uw@ASfnJNGt?R3Tgh_Y9P)RzZ>=fLF`P z)8;+s72Q*imIr7M_6-WhN@xZ9H@}^cyh3z?_o=A)2oJ4qRT$%MxKA!t5*U>2k9`Y{ zQJwHDP0PDtelsr1k;Pm(S9BR91IH6&F=VvakZfwM3<<`~vwX~7CVGxxrDYOmcLv$` z$;UCgGm_8}Zc5IyEYGla zw=M-o99~q#%#r9m;`c=2YBq*pnV_2r4-Y2zFui09^=c zRckzKkY!%H*{9IoSL$MV$zKr9>qknPO=nZ2cidcS?~Qm?;it-8C$Fh=wgIZ%CxHwhnyEn%TM`Pk=+bd-DSEmn}?e3kJ(E`GOi)}TBqxBSpaAx3Aakwa-)7Rprml@6CsZ+9y3|WG zk%5(Xf62pf2*B>f>H?a#7Y1`Tzi_m_NLxk;fWicPw%9>l#I(s_4&F-o&wpF=0Gue$ zTLS2xTIVXj*I_qdm6q+m6Tq@Bm`oN%tPL*vWAkC)@jBQ3>slFCv4rsNAat&hBOV^` z@THdRhcU|X_vZ+c8e3qY*vg584(5zyUH?*tW#B@c&{fyWTnFEoMEvhh5J8Q7CcF;) zITWq2EwR5CQVMdc`Ba_d!Yy2^5wT383QX@vzQ>mJW~n+{ z^jDr*Nx!~L@Ca1$TeWR3^VW;%-+^YPk3S|l0HiiX{C*M8Ubx`nGH9R$WrQNnz{6ir zpG+Jr|EB~bpCGqgh~rS>5&SQv86e*C)Qpr6K9hkru;MX;Cu(&iIe=ho51{X{w$ES9 zZ4Yo&Ol`t)wAh|NR?xhizxd$RAKG{tAX))Y1>pNp`CfShY`<@6XIfFvPk^(fEc%^x zi<8krAwl1_hBnA2RJUodAgM7Z**iX8__W)-+yU=@+i(zq7D1N~_%^Wa#a;mt1c4!R!7Z zaL}s43pH0s$B2*4_SVR-z7^iAVCdKyhRLoM2+6Tv7DzQqGYOnrL<1EUR;c!B^<&3@ zsWNejBG9P_Jhdw1fYOcy0RlO6gRA_&dItE3sh>^rB$B5P2%v!HV%3u53dmf2w$_<} zqSqO;y&wAH`1o9WU%wJ~i+0 z^yso-v#|1UUQ}?}5ca@dT>OEF!cbv?WVLmeN!ltbjg4nAk;23f*g+z*%1R*&YePj( zo$n>GQg58~PMt!%=WNWnaB#l}blzh;{vq0gOJn_3v?P4D9}wlCOVZWn%9GgIL>f}- zK=`+k=I;RJrO##B)4 zwJSJMe1t{VzdOc*Yyf9CiX7@R)hwTR%l_qGVF74gxuCe24?i?}AliV8r2jf;UE2Q8 zJ1(O-h#>dnTcujuRhAKKgca?#<@|s{O;h9JkH=learitiND1E%yb`sF$KCn4-hb{I zM0{>hR|U~bcnZjC$QXoHwl`jRM}d=gb^20-OrX1`>g;}3&E`KXmo4wsqJ@^fO*w_P z)H#n!@L{}HL#_J&(M^K;?gZtzVj(ohL8&<#6A7+$q%yF|u)p$>DhAF+0L-NFWRzC1a+Ehl%AseqGuX%x?46fKSH2w>mp#=MR{lzqtQlkWSWIB86nX!?KsrOUKsqB2(HlThvBL%p z+;5ob4UerduVKn-*VRcM(?;QnxUGG}*z(+Yv2Zjs?^F9FpeYm_uXyI`{gp=Dvz+&) zyXA-*WvwI-1sf9_{fz_9yW-^ca5OYR@mup(wgtk$fkkiJoYJfengjhLQLXC?#%{`W z%dHQM#*=7!Y;{_7v zhR4cirQb%gZEwcOdMt~i?Ld)Ee`>>WS;MEjj;H)0#P1ToM_)@22qoO;HtNI~kwk8! zIY$(`Ka7&Q?BjgK3feBGw`JTe+t0R{6~narO?vwj!(3r(oVg>1o-z$sP0r!P0GD{6 zJ!zsmQ&cV3c>9*{shz&CXsq49t4~sjbRqx_1!DCb^mOYb{$c99mU15 zeNh-a3)AM9sM(`(jv8x9Gb6v@AmAKEeqnvkiIWwt{B5_JQ~NBm(bv6hR;NG7c_V}eK;Rddx{vY7f^QI+Qeidr?(;YHf zg*YCU9xHuGj}v{MQuO5ESriV9;_^;+a!GVas%_TP!&-MUo6PPT$7x1rOxj}yvk0&% z6+~exG2ZjEFUbM77by{n%J<772$jSm8T>nOK1NevX$8J~6p9lI?U2 z3!kvkYcKDOH$QBlN`;`4Q&$cqU6^~Pg}N=!BF>O+wCNZ~yq{t+DFCA()c(WvyvvO` zFM?eVws}D>jc>eJS$IP^K|=Z3d+Gbzb4dU?P(cwB9&=ANeEp3XOBxC~ABuJ-|M*T{fLA8vd%UzM(fZ9rl{%AgBUk-QPOg(we5 zNFr4cISLJ*@bl66ix-7s~+B%k{$8o3Wq zpo57tC-p|1{&cPY{m(7Os9Kqr{m-1?p$>r~x zr_WbjdyDJ5^vf4S!h&G#KvKYj=r8Bh3zl!Pe^FQp1#dR-jk{V4eAT5@!f8`fy#Z$c za(9mzkpW1kbIIZzftQ4T2Gdox>ko|Q|0L;@kwO@c=+N}mMuTJ!DWNJz@eoCu{bK-6 zTqFpDsS09)CMrnIXn%HjyC@Jq0vO}QTV_53!%rG0==t#ofir(tB;1vX^LU34`c*+O zq;2Mem5WfANU z1@HH7@xR0It>6fLD%VOSrU&QXub+m3Bz~UKArhBDL&1t6PR$NOJN0)`20f3~T3 zo6#@jto_K179Pc!WR8@g#T7a;OR!hA{RSv-a!L&<^UY8edv;)ANCpV{%k2w=>{?>o zRP#Vp<8UH*IyT6&anI&CE%h^gqtTBQ`@^%&Qvyx?LS{>K7e|?nTxw*B%5}L`a2c1;fY&tO=&j;>w<*$7?MyPE;Ei=O`ACE#!PUEY1l{19@;j|62$Fzx z3zV^f;9E(h0?CRwJMmU6-vH}-f!i_HyzC4E0Tt3jq#Ww(5eqn!zZfd<%JRFKEUv{~ zR`c1gW9Aa`CmPj=vA5KAG!F~f77XpOhKh0(9r!e5EkB|1fkwm^MmS*)DpA|DYrXMX z08Pn0K4hjE0&c>Wt6-~Q|2gtLBy>PC{+$mF-E_mqfM;R51@~rCtWCZtp{mx;Ii?xp z`PngM`KzCPQg~3|uJA!tPkhz(3*0;lC2|bY7t_B?eQ#oXE%d%?9s!!prcGXz-}m%~ zS>xabv?`oDvGNBCj3&zfL_l3Iwvp%n=bSwLKHKjd4(5Li;1WaTP3A93zOnx7_9m3( zcia`N!!}H<^=}tOC|S7WwU@oQBXJahIbYm4&N?fPW2+8mH27YBFeU3v9#G%%Gx+3w zM;!m{^7T=@qM7?_O)Qq~8tSKCB==m^_?)=<0F~sBl3H5P&o_5sm-JB_%70vDJIJse zSbzqT`*D83*$z<86R`o(N>4bW#8(ljD!$GK(e}FOcW%k2q(!_jXuN5=JS}S=jMSJC z7NSkR0*JCnJ0LNeg+ zl{@F_{#&}NArs;X$5u)Aeg-N2TwSlU>iEONlxkcd(m%RDhK!!h7R1B-ETYQho0Z3- z22-w8{v>F7fDh*uWAi72fC$O^qdjL5&Z`%6aM((g`Cdg=J$FQDXO30=y$6(O z5Tk)G(ZK>8#r>})hfXA?@9mZEs47KQf#gXI3elu&@`FgePm&Q*XsUFM!9-#yILSkO zm>$01dmf{FW@Ib*akOR(pUOS0SURZN+R$Do56f+s=uU*xld}5U>YGMpCJ4_PopISw z&=d8JI4&w{W8LK$bR3&%x9P$`g+zM*1wP06lUk;m@lNlR;saVSARU*rNsgrT*?nOrviAZcP1yt+R)U~M8Kk%kX5$8J%q@jsPLK!sW1 zu!;o1YS0>Rg2AeHwWaUc^mGad3Oo6ZXPN2jX!9o3lwxPx#a>*EU(o2e z5x?Zw*Cx_yMiJm%3J5pn;X|8)DRZiR7(UY6zc((J-1W!b9EOI@wPP|!+FA7*v36ML z9Pyf=ki&iQhlK=8RDvDkSK&l4ex{qNk7RqnSKPvY+)8Bdd?t-Q5XfB#@>kfVJns#4 z9)SicT&wg(i7*_;^Wrxl8MILILR?$I=Yafn<(;)+u<@Q^OQ5z+77y-bL~;(*BQbi1 zQ22TE6;$<#j?Af25eF71=eC$iZDe*yoP8V+kO_KRWcLts$Lv-mF177L_-Di9Jr}jN zy@wVZefl*Lf&N2jGfC;U-~I47lg{ZaT581p#fTAB*sDJLE#U7~9LWbnq4EzM@9p@U z(}i3AtD|s1@HMC+I?bce68@xX4B0_va22vhZx9EFz<%S4d!eeNrF0dWCDZWl8vm#$ zAT-t{Y9~QoPu?oiOPj+1FK%6EM}K2^os}eB)-0lLrwgoXrX(k>CDyDb&3D|+mFI#u zzk1Q@RusJCch`T(r6A`pk(>w_6URn^u}S8kdS;^I@@Byr9zcepUd8W=bi63At? zA2;-b3^GOs6=5M<4mwG|{=o1- z5yKbi@C!0Q3lBnA>4AS$s=2B(qzn}8BjdoZeWG;@5hh5uWTL5A+d-YkMBiDNkE_96 zT(YgLei|WdRt8TNPd*LId;0oVQa>}rRz|mjx0MKmw=Hvaw%fgJ^ZqU_Txu36E&XaV zq=jvOJ$}mB=}RjSC&7m3X{XMgV=3pe?}rNSCF!Pl;soNhf%~{~9lauN0fFI%C*^&Q z%Z$_I0VWE9134ly`I`s@*kATIRDK4WioC|)F%D77!aMm#O99C%&t-2^%p|eRi3@ce zqyeM|v3k#gr>-`~u3m#rUT?1_3nrR*21?a9Xug+u`^M*H%&nmnF6;Xss>#5)O4jST zgHj+>5YaRqH7+C0>xFkc!^rXaZ=qDR9kq6>Hz7nEO5nY_wIm&G+deHR90TFYE=#x^ zI8Oc3f5O;m(#y~q7)2rKt% zM)^ftoX1a6cp7F{=wVgwUk3f3=Ds_g%Kz{GAR|sjA+jkE$EL{4ipXp@%61$($;u{D zvdi9OBqHZz&+M#YWF{jcTQ1!9BSdm!))kV@ z(wPY4KVPC2M!)SeJ*l@bXw{vXLdvLNsSc|JNe#P(;B&*xVN960G+)`#5_`kK4CMu` z@Ee)5CMTJ6GRRyB-7}|hlyrT}=E zJ~CR^WrYIdTP%Go0<&eVZXIkNsbx-7^#B z_(F-fN6gsn}t#bCzV#=TpL;;P-?SqYNPsFG*% zy-ST3w3eI>|93HMW6XX=b*yK&+}wvU@wIbjni`6n#j?y>G`I(!^8*T)t zqm+zy?$qN#ui=jd)tg&ZJ^gscP=p)|5uEWB*MuGgW-#gCGahFsbHNV?ybfl^SK8}a zE_VOY30Zee!#`rRfU$PT;+O0I0yGQ|z+y&G)O4iT@!^gm+Np0%ft2M0G&)fcWGJt8 z=xLUejuXVB%-kx?O4ygku2}B8aA1m)Of>R8Kab7-YV**pP(H%;8nuPG+>YFr=&^bU z%<*!63e`!6_GR07peymAZ>r_+d z#unjGiswrC9xnR12x5E4Ji==azgjD2_{HpB3(Yt1q(^WTDpIg#q;S;T_U})k4w-tgbu^+G#WSh72c>P~wU0AWIjCuyEf#diQM%GG zj7phLt}v3_3T>vF(x{$G^>p=ref^jFdsA@)$`=K}Ri~+sU@mMr5gP$);@caFF-o6F zD&MZzw3=K#cgx%UDeq-J-9Yd)NdH>?dmgk4~P!O2@d~dOgj$h zo)xv_moj&d{Y}yrJ`hvz1;6XG*BzrJ&0ToGX=lWC<68o4j|DePhO#%#tgw3f8Zq6+ zZmGq5+2vy>BPFcZM$`8X7{`~@|KtUI{4#NwNiG?np?9_Ab0SNlA%TZven_T2j3Pk$ zrYccpe5^Q1B!k*YtFKr@^BN|K|5HA`!T zW0R*gA=9=1V3UVE?Sc|aGguT`HW|$H{0!Y>XVP^m=s?hSlH!Dk*a418Oo4=%^@VgH0BI9>j1 zU_2nViBrT^`Md`x7ztT}E7rk9bjHak(fkz5NdzcylVrlZ7fgNw1?bdZsx$dT&i#%o zL-Ro_V>y6fTs%#oAw!DoigS!mL7)X5KiNKd~?9YmN$yZ;E-KvbFh0Ys?bPXQUU z43;u=@V?k$nVR>jwV1EVz!*YyF>LRTDjt4L!fpsh91={9gz>3a{h@7>whDm3y? zlfAMCHtLakLT`Bu7ZD8KC8H+)Le^DuoRPm(Ku_K=peOVUkQ(xjKbGfgQy@Xi9tBkFfJidCc=xAOtsRgxB3$!F zzf(DSe4%)>X_*`HKZ?;niPRVUD!_1Gftyefka1BQG=SeCA z6D3e!P^|Bc&++`ZK})Vj92$`}m~bqV4YbypynWtKCzXkmON|4$oje6p~) ziY)C^?;XL;95HCQDy9&udLp0Rv|Ku#TDvn9Dkve24cSV#I}B8=pFTPPAgPCmC#?G3 z&{tNezH+;0lG`V+HkKWmOE)d5e@4E3pY3?)gBOpQYFOg={tT1bUG>`UQ2z8X zjBAA0$%k^sd)-#;af01l(irPu0+HnwEXp2aCrb^5m~7S;;A_h!({^< z?z>BQp8x36{uh}eGi_0CF(5O(O645f#MfOXX#y6St3-o9Y4W5D>L&SVC&gYDyutU+ZQy zE0C(@Xy=09fxvN+J*A2tQ;X5&LM2d^Q|+pnAT|`ik0~kRjLAlaCqJdRQvhSG;`@sXZ64@q4Qzg8k ztK8js`@3(oAT=OW|J%tuH)0kva^mf6$MR4rWU0>}+av6^{rLRX&ye?v1r`!w-|wE?w0s#WrMc&c%kf8wphxE(ruu5?mh@^C+K`wbMF{=*aA?=Wa z>(8>E|Lb*veHmFXqgx+rXFKfL9@Hd`S79B*jfuI&ug9&H(sUU3HmG({0 zBcyif>80DNP-SzwbtEg!4>+0*wZ~uVj^;zxmy>UJ8kJ1tF;y&R3O)P1ago34GYCG$ z1~Ozx@Kq}>OI^5rD9XliDkE6#8iaSgw$8`ua*J1`uYwe&3bSyJjZz7+_kN^ zy)tD3NT^otD+U5{!X}QZ7S2yt?E=#v(OFiD^HH?ry>TFW2DmU1-P5?N_vp9x_E@PQ zslwv@?L-%>3c;iyt4AMpytt|gBAQZB(qrZ0o~z$wRG{MB5!J?#dvNl z#h|8JDh7=_?*%cMw}f|QYo(=X-Fx0CK4<}HRdbkF)X@sD#83b;#KH2b_aZG%8hekTjC;t%-q%>@LgN z`8ju>o0`MlCeVU0mxw>ae1cGO#5sf?ora~$#PlJvbN*--P}1g3S`PuTfqfbjruulR zHEjkLjEeu}Y~F_^GF%m{Q$Rhpy*!F(Ve^$f_MY#}8iqE7R~N3{uWnfdWQXA+{A$|q zUHh}_b&kIHQ3Zecx(fqI&;Lw%Q|4U~uF3-3?DWJpfg&-QkpRW;I6{%BIvM^ok`Dj_ z*5J|s3nSgf-!cQlWNV)ABpCcj2OQH{_#K3-rz3%9X9J$7=F#&S ze?Ur78kFXh_w=4+G~ih85EjIuQ?}tTb|bbQ76Lq%YFG!)76P}2lmYX+ADbge18T#?5M_lY{+61Z^HdFf zR?FvL4!3O4>gE@!LPeDO7U1oCa6@6@ZS zC{a{LD@HJ{9$qXm+nE$+<)jqcAea1nKk`Q-nix%^VE%P7aqnv(yc^^Wl?D`tUCoAD zGt*hm4AaSzux3XSd|QM?50T(CF_#xUTI=u&AW0All5t_Wl7t4Lu^IosSxjzEGqwuCF&h0nF|(r^MK(gjhO^f_*xNxvnb))5@NAK4Jz5l7Gnv*_TV0|KIghW;-mo@oX}IA8Ly zl3A7h<`ahH$-cya&2>J6s7RvGLZWS0>%R3Zh%4~+)*Biwm1&G8KLEhTNf*EP01$_R z*W+=}-`^O(4EgM^+mnwA$p2v$X~e8TL(C0i4#Lj24Y&zSI7RrT#6@U^9G|D^v`kHZ zFJhih24}Da?YtDxxx|5<5K;>mEsFV-DVwlGwjIgavb(ZkK-)niPHeb1Ne$6cOHPE& zFX#`+_UC+DO^;%bj8)A=Q6Kekwr?O7O<%Ia9(#8Ca=z~ld^d|fmmCm{Mb6- zD?AEr9U(M&H*yS!WfUmE@YXi|yf1Uiv5g)`YC#33++)W^q zOVzTi0KukL8L5!j`gGep(`InQOQcf#cwf(awj;qq0eAleyqjyVG(I5lAhnpa>I) z4<6^Q*5QA?{HUsM7*Zw|*V$jcKTjQ*n^nJPSGzP6nx39$j<=jkq{rpEb}_qsJ-p`` zLHOP;vDn%e;%DdLZb2mD5eDR1e{N2=nO%hCr+b2Ic73Lqee}*fJ6o?J|LtIJihV0z zl!9VnpM>=UG>5A&;P*J(vNkL1?~aru|GKi82%oF-e&KtC2~S&&+@W=ba#B(Uy=>0z zXZdAS0;69@ybdo3}fL%+uN{9=sBL|rsR!h@~;UFwQ(-%22gFBDRu{S$h{_KN_& zL3qqsT>dYhMo4(M-;vtNcDO4_`1Q#am$Fr5eU+kzmM#Cg>j?3GL>`M zSx5q8W4}d{nL!kB6su~7TDO?(&)WuZc~koGT7y=2*txvdY~DZ5?mZP&W`c2&2N3cB z^XnS|eu8zJT`$x0RdYJSY=8~UD70ROy1t+X>z@N_{N{j$&d6Xo3d+9dos!i}sY#vr2j~olARb7J*sW?$zOTN@hfUYy zN_w*tlKY00kv`pc375QsSiMucFN^EtsT_vZP&y8;`l*err~$Ep$L+?Q4lVZ<4H$mvScXxQG}NZrrxwd-2p_Y#0;)Oe*xjDAkWa{JcsKSD*qn-@;yGpfUKh; zMEvB(?j8`|Y~C0HWS}e@AzD6BRExqIlj5aks5$4~9IJJYvC17Og zkki6Qq)p$jXOJOa6dxY8tVIb8)6eT^h+N&pr@r(Vu$LqxNxfCb~ahmw`qxJykVO0dw*&W6j1< zuL+lw_lJfipV!>D+raM3l86Wx&nSnHPxnn*xd>QCoCH~{B0-cWb2)wBH- z$q^4(a}@$v8HQ7kQJORv5e{zndvh^o z*`OL1;GNUt{~p2hDZo7BP$^0rS~LD*y8<1oh{zC-FItW4e0S3f=M^awi<!hk`BSl#VXU8%*G1ZC6%i$`lnr+o`~ zB73T!uGUtbQTjrnZNcfa&Te@1$v!-!X8L&fXh?> z74}rMwCTh4bqP0KXrKCf=1+*!rKSo04DS4-&lmgBr{X!3(55CPMEx}YfEeW-iT)=_06AU%`Wnqg)IzkR39zP}H8wydizrk4RA| zPgDGMGx1`R1Y;=cD4a+L9ssonV5gtDEoJK9%uZ-&K(PYkyAo*OAr!+|gA4R&eIz6e z6i98+Z>vQP5A;IYPJEcqkJ0u0nJ2sWNecQ`+)J?dhp-G^&>(JgilzJ72hR32xT!RO z$`Np%A=~xEn42s1JBM|rw=1B(IQ?$+svdJ?8W+Qv0kCfvaLL#0t;h0=KuEB?zgS2C z{CAK<$VNL35$HGyri9PRC=f*bxdiVX`NBUFAY~nuGEC59T z2$U^0W?j-M3_(%>Vm3c^?i3#e(L5yV5pyG#NRhHJmEI4HTbj1fD|)Ge-#of$h!KA0 z*9GSr3lE2XgS0xQO8{=?=%;h{a-ZH9@P*o8ykz&^(u+0y=YWyDC!rF-8p8s7)I7$o zE>u~MsmuDd6)%Ale&q!5aoP!f0JlOrZMeC4MN)LjcHafIL#kHt=pN*9AK*&Q4#p=0 zLPk11aW01OggrpEg5}Ec`Z=`J`G6I3B0}5j15kenhn6dz{MCIy)Z&OG$0!sSR*?|9Lk7u$mp|S(a{lA`0qa$JRck~LKT#++4Ey>J zzX?A3#A0Kv^Qm2q17EkSwTdD~-pN$%8PUt$rb7Z>f0HzR0larbifA^k5FeJIS)m@% zQ-v{0W>TDPTae?BglXF0`sy}-98XMw^dNI-i%$;9b1`EPf26su`6kC<6utpj`r* zxl^yG(DkkP9R`s2Pb1bwn`-0UIngL8V;wH~g3m3HV1CE|9e{v=6405xpzkC^xSN4`G0FV(jo&^Syg@Xj# zE#E4-tM}9P#@!r--X3>KkaE?99p0=`nH` zd9-jHF0-~Sb2weuOXdHns2PAm-aRl%D1=hpAi>TMy#~x5XB7F2b{o<_I;5m#JLkY9 zIJ|1zc$JC)i#`Jkvnje#%zdYV#(z9|X)411z@9<%D=18^O9~{8hMeC{Eagfx)j@>D zp#AS!)EwdX9IZfI4f3GH!2VR3DMrx`7*yTx{M&iFLWgw+6jl@@fUv#EkXy7+;_?_j z;=Fox_{*`cr`sUsoWV;1>C5T5I$HhNTS1whgI>#^=lx0Fg;D0mD-ZNuktF+Uwh$0- zOHJv#6UEdWVr%_NoV5G`&CFyZn{`})R(~m`8zUxOGL*A!7_WG*FQG(4lf3bV6PZgL zqJ?UxxtizWI~%V?h%vu3sCCcr zvs;UW{CG`j&w~dQW|?w*ndG5ku{q#%)8#olZT1`E2g$!hD6)*q$zMN84dB8EukU(< zd1x+^q#w52HH~W5b)WX`vTapKDIdF7d+@Gcn91dlLlXXT%Y9H_IJr_|UEN$vNH=$y zwR$Mm{wm7Fe~w7aCTRJV@R2_CAldx-=*o;diW3`Ay7~1% zDS7Dh*VW@E4q3|TDH&IId3i<#hm2oGi*Naf&2s1_6H;f*c+BF52+kPT>~A zh^;##D$#LwKVVybXS?5?k2beLAc9jpU&pu(tE<0CF)~L5O%Ek1^vb##Df)E5-)8jT zN@haAP;kN(bpKYor!*UTWqk1+J*PSa)SeB-kEUgkxI+bN%L=X7h8JZJHn_6B^P$Kt zAvNo$^XUk}U=)A9M(|IoScCo=I$l;0Lyn+3GDfK9AJ}teXk6HM`EAl(b$oBN7meXc z^6k<^E^YpF_NdMgXR;{m$BXJyYKq{MtuwGU_?w!7LzX4=ZVVsESU z#?lYpewLHK--go&eZG?LxVUebpGpS_ONSl}zGW$k#(zG%qI+#KqEk2gQ5>gfbT4O; z+V=IKe2bz~a|~_fc5iNrCU2xA-?D+yLXGW4*O+=JE%Vo5tI7kj(q7I8$+q>#j$i0i zE~Xfibvw?s1OKky8(aD*OF&UsiIr>6>cZWupk6{4)Xf0}-B{}#nordcZ|Omk8X+yz zR7e&LVtd#^23gOE6HS2*lkB)lzj^1Uz zuPXc}cay^7)HGXq)vWC}hLI3V?JM*$fv%`hF~{78!Y|%wqPAC*)pTc+d}|}Fp^K-X zhK0#nb7W#~{F*I}h=;m)=uT*f-EIe_+a+)v{u9&B8JFTcbI_P0-3brmugt}FZzb$S z9&+ol_6H}vTYY?_=4OWyz~*e_&&BlF|)3Fl1;bNW0{kNSG<<}Q|&bB z6yWkLCF{u@5kK(LOHvoy167V9ix_snOCilQ$D1ZW#-*9uI2@`tG(K)w6V$mn4_g~6 z%|>5gQv(;9{_=ptzv*k{tp0hUcl59hOxX~1ly|2jn=^h&i^G z)8OI5KMtNg%Qs2No=&e1+$^oA9naB9V;!$_)3PRj!PF@dwyTXEKVE+QQneE;mRNxu z`uUSNP0GdQQ<>23TjA)T?atBcnuD*cyQZ1#&k$dv=rZ1ggwhv~D{a3(>_O$Yv zK(YSx65m{9r=;_Jny7}@cE>8oqu$=(zZbNsYVkL{K~DA~$^bMiG*&g8qISFz`D+fQ zG{CGV%gy5CRK0Rh;8A=9ys6W6Ar2ao{-1oiz>C>mj zP8^&YZD=vw(~6 + + + +WARNING +WARNING +WARNING +WARNING +WARNING + +

PLEASE NOTE: This document applies to the HEAD of the source tree

+ +If you are using a released version of Kubernetes, you should +refer to the docs that go with that version. + + +The latest 1.0.x release of this document can be found +[here](http://releases.k8s.io/release-1.0/docs/proposals/kubemark.md). + +Documentation for other releases can be found at +[releases.k8s.io](http://releases.k8s.io). + +-- + + + + + +# Kubemark proposal + +## Goal of this document + +This document describes a design of Kubemark - a system that allows performance testing of a Kubernetes cluster. It describes the +assumption, high level design and discusses possible solutions for lower-level problems. It is supposed to be a starting point for more +detailed discussion. + +## Current state and objective + +Currently performance testing happens on ‘live’ clusters of up to 100 Nodes. It takes quite a while to start such cluster or to push +updates to all Nodes, and it uses quite a lot of resources. At this scale the amount of wasted time and used resources is still acceptable. +In the next quarter or two we’re targeting 1000 Node cluster, which will push it way beyond ‘acceptable’ level. Additionally we want to +enable people without many resources to run scalability tests on bigger clusters than they can afford at given time. Having an ability to +cheaply run scalability tests will enable us to run some set of them on "normal" test clusters, which in turn would mean ability to run +them on every PR. + +This means that we need a system that will allow for realistic performance testing on (much) smaller number of “real” machines. First +assumption we make is that Nodes are independent, i.e. number of existing Nodes do not impact performance of a single Node. This is not +entirely true, as number of Nodes can increase latency of various components on Master machine, which in turn may increase latency of Node +operations, but we’re not interested in measuring this effect here. Instead we want to measure how number of Nodes and the load imposed by +Node daemons affects the performance of Master components. + +## Kubemark architecture overview + +The high-level idea behind Kubemark is to write library that allows running artificial "Hollow" Nodes that will be able to simulate a +behavior of real Kubelet and KubeProxy in a single, lightweight binary. Hollow components will need to correctly respond to Controllers +(via API server), and preferably, in the fullness of time, be able to ‘replay’ previously recorded real traffic (this is out of scope for +initial version). To teach Hollow components replaying recorded traffic they will need to store data specifying when given Pod/Container +should die (e.g. observed lifetime). Such data can be extracted e.g. from etcd Raft logs, or it can be reconstructed from Events. In the +initial version we only want them to be able to fool Master components and put some configurable (in what way TBD) load on them. + +When we have Hollow Node ready, we’ll be able to test performance of Master Components by creating a real Master Node, with API server, +Controllers, etcd and whatnot, and create number of Hollow Nodes that will register to the running Master. + +To make Kubemark easier to maintain when system evolves Hollow components will reuse real "production" code for Kubelet and KubeProxy, but +will mock all the backends with no-op or very simple mocks. We believe that this approach is better in the long run than writing special +"performance-test-aimed" separate version of them. This may take more time to create an initial version, but we think maintenance cost will +be noticeably smaller. + +### Option 1 + +For the initial version we will teach Master components to use port number to identify Kubelet/KubeProxy. This will allow running those +components on non-default ports, and in the same time will allow to run multiple Hollow Nodes on a single machine. During setup we will +generate credentials for cluster communication and pass them to HollowKubelet/HollowProxy to use. Master will treat all HollowNodes as +normal ones. + +![Kubmark architecture diagram for option 1](Kubemark_architecture.png?raw=true "Kubemark architecture overview") +*Kubmark architecture diagram for option 1* + +### Option 2 + +As a second (equivalent) option we will run Kubemark on top of 'real' Kubernetes cluster, where both Master and Hollow Nodes will be Pods. +In this option we'll be able to use Kubernetes mechanisms to streamline setup, e.g. by using Kubernetes networking to ensure unique IPs for +Hollow Nodes, or using Secrets to distribute Kubelet credentials. The downside of this configuration is that it's likely that some noise +will appear in Kubemark results from either CPU/Memory pressure from other things running on Nodes (e.g. FluentD, or Kubelet) or running +cluster over an overlay network. We believe that it'll be possible to turn off cluster monitoring for Kubemark runs, so that the impact +of real Node daemons will be minimized, but we don't know what will be the impact of using higher level networking stack. Running a +comparison will be an interesting test in itself. + +### Discussion + +Before taking a closer look at steps necessary to set up a minimal Hollow cluster it's hard to tell which approach will be simpler. It's +quite possible that the initial version will end up as hybrid between running the Hollow cluster directly on top of VMs and running the +Hollow cluster on top of a Kubernetes cluster that is running on top of VMs. E.g. running Nodes as Pods in Kubernetes cluster and Master +directly on top of VM. + +## Things to simulate + +In real Kubernetes on a single Node we run two daemons that communicate with Master in some way: Kubelet and KubeProxy. + +### KubeProxy + +As a replacement for KubeProxy we'll use HollowProxy, which will be a real KubeProxy with injected no-op mocks everywhere it makes sense. + +### Kubelet + +As a replacement for Kubelet we'll use HollowKubelet, which will be a real Kubelet with injected no-op or simple mocks everywhere it makes +sense. + +Kubelet also exposes cadvisor endpoint which is scraped by Heapster, healthz to be read by supervisord, and we have FluentD running as a +Pod on each Node that exports logs to Elasticsearch (or Google Cloud Logging). Both Heapster and Elasticsearch are running in Pods in the +cluster so do not add any load on a Master components by themselves. There can be other systems that scrape Heapster through proxy running +on Master, which adds additional load, but they're not the part of default setup, so in the first version we won't simulate this behavior. + +In the first version we’ll assume that all started Pods will run indefinitely if not explicitly deleted. In the future we can add a model +of short-running batch jobs, but in the initial version we’ll assume only serving-like Pods. + +### Heapster + +In addition to system components we run Heapster as a part of cluster monitoring setup. Heapster currently watches Events, Pods and Nodes +through the API server. In the test setup we can use real heapster for watching API server, with mocked out piece that scrapes cAdvisor +data from Kubelets. + +### Elasticsearch and Fluentd + +Similarly to Heapster Elasticsearch runs outside the Master machine but generates some traffic on it. Fluentd “daemon” running on Master +periodically sends Docker logs it gathered to the Elasticsearch running on one of the Nodes. In the initial version we omit Elasticsearch, +as it produces only a constant small load on Master Node that does not change with the size of the cluster. + +## Necessary work + +There are three more or less independent things that needs to be worked on: +- HollowNode implementation, creating a library/binary that will be able to listen to Watches and respond in a correct fashion with Status +updates. This also involves creation of a CloudProvider that can produce such Hollow Nodes, or making sure that HollowNodes can correctly +self-register in no-provider Master. +- Kubemark setup, including figuring networking model, number of Hollow Nodes that will be allowed to run on a single “machine”, writing +setup/run/teardown scripts (in [option 1](#option-1)), or figuring out how to run Master and Hollow Nodes on top of Kubernetes +(in [option 2](#option-2)) +- Creating a Player component that will send requests to the API server putting a load on a cluster. This involves creating a way to +specify desired workload. This task is +very well isolated from the rest, as it is about sending requests to the real API server. Because of that we can discuss requirements +separately. + +## Concerns + +Network performance most likely won't be a problem for the initial version if running on directly on VMs rather than on top of a Kubernetes +cluster, as Kubemark will be running on standard networking stack (no cloud-provider software routes, or overlay network is needed, as we +don't need custom routing between Pods). Similarly we don't think that running Kubemark on Kubernetes virtualized cluster networking will +cause noticeable performance impact, but it requires testing. + +On the other hand when adding additional features it may turn out that we need to simulate Kubernetes Pod network. In such, when running +'pure' Kubemark we may try one of the following: + - running overlay network like Flannel or OVS instead of using cloud providers routes, + - write simple network multiplexer to multiplex communications from the Hollow Kubelets/KubeProxies on the machine. + +In case of Kubemark on Kubernetes it may turn that we run into a problem with adding yet another layer of network virtualization, but we +don't need to solve this problem now. + +## Work plan + +- Teach/make sure that Master can talk to multiple Kubelets on the same Machine [option 1](#option-1): + - make sure that Master can talk to a Kubelet on non-default port, + - make sure that Master can talk to all Kubelets on different ports, +- Write HollowNode library: + - new HollowProxy, + - new HollowKubelet, + - new HollowNode combining the two, + - make sure that Master can talk to two HollowKubelets running on the same machine +- Make sure that we can run Hollow cluster on top of Kubernetes [option 2](#option-2) +- Write a player that will automatically put some predefined load on Master, <- this is the moment when it’s possible to play with it and is useful by itself for +scalability tests. Alternatively we can just use current density/load tests, +- Benchmark our machines - see how many Watch clients we can have before everything explodes, +- See how many HollowNodes we can run on a single machine by attaching them to the real master <- this is the moment it starts to useful +- Update kube-up/kube-down scripts to enable creating “HollowClusters”/write a new scripts/something, integrate HollowCluster with a Elasticsearch/Heapster equivalents, +- Allow passing custom configuration to the Player + +## Future work + +In the future we want to add following capabilities to the Kubemark system: +- replaying real traffic reconstructed from the recorded Events stream, +- simulating scraping things running on Nodes through Master proxy. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/proposals/kubemark.md?pixel)]() + From 569ebf7a9b7033149af2622b54da1aae3f9b8707 Mon Sep 17 00:00:00 2001 From: Ewa Socala Date: Thu, 10 Sep 2015 12:17:02 +0200 Subject: [PATCH 33/46] Resource Consumer Handler milicore changed to millicore --- .../resource-consumer/resource_consumer_handler.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/images/resource-consumer/resource_consumer_handler.go b/test/images/resource-consumer/resource_consumer_handler.go index 7bc7ee4e640..f01350cc42b 100644 --- a/test/images/resource-consumer/resource_consumer_handler.go +++ b/test/images/resource-consumer/resource_consumer_handler.go @@ -31,7 +31,7 @@ const ( consumeCPUAddress = "/ConsumeCPU" consumeMemAddress = "/ConsumeMem" getCurrentStatusAddress = "/GetCurrentStatus" - milicoresQuery = "milicores" + millicoresQuery = "millicores" megabytesQuery = "megabytes" durationSecQuery = "durationSec" ) @@ -68,21 +68,21 @@ func (handler ResourceConsumerHandler) ServeHTTP(w http.ResponseWriter, req *htt func (handler ResourceConsumerHandler) handleConsumeCPU(w http.ResponseWriter, query url.Values) { // geting string data for consumeCPU durationSecString := query.Get(durationSecQuery) - milicoresString := query.Get(milicoresQuery) - if durationSecString == "" || milicoresString == "" { + millicoresString := query.Get(millicoresQuery) + if durationSecString == "" || millicoresString == "" { http.Error(w, notGivenFunctionArgument, http.StatusBadRequest) return } else { // convert data (strings to ints) for consumeCPU durationSec, durationSecError := strconv.Atoi(durationSecString) - milicores, milicoresError := strconv.Atoi(milicoresString) - if durationSecError != nil || milicoresError != nil { + millicores, millicoresError := strconv.Atoi(millicoresString) + if durationSecError != nil || millicoresError != nil { http.Error(w, incorrectFunctionArgument, http.StatusBadRequest) return } - go ConsumeCPU(milicores, durationSec) + go ConsumeCPU(millicores, durationSec) fmt.Fprintln(w, consumeCPUAddress[1:]) - fmt.Fprintln(w, milicores, milicoresQuery) + fmt.Fprintln(w, millicores, millicoresQuery) fmt.Fprintln(w, durationSec, durationSecQuery) } From ff9d482c8261a437dd1bae9bfb28846278eb4681 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 19 Aug 2015 17:04:08 +0000 Subject: [PATCH 34/46] Add CentOS BareMetal deployment scripts. --- cluster/centos/.gitignore | 13 + cluster/centos/build.sh | 149 ++++++++ cluster/centos/config-build.sh | 38 ++ cluster/centos/config-default.sh | 50 +++ cluster/centos/master/scripts/apiserver.sh | 85 +++++ .../master/scripts/controller-manager.sh | 47 +++ cluster/centos/master/scripts/etcd.sh | 79 +++++ cluster/centos/master/scripts/scheduler.sh | 58 ++++ cluster/centos/minion/bin/mk-docker-opts.sh | 108 ++++++ cluster/centos/minion/bin/remove-docker0.sh | 27 ++ cluster/centos/minion/scripts/docker.sh | 49 +++ cluster/centos/minion/scripts/flannel.sh | 66 ++++ cluster/centos/minion/scripts/kubelet.sh | 75 ++++ cluster/centos/minion/scripts/proxy.sh | 51 +++ cluster/centos/util.sh | 324 ++++++++++++++++++ 15 files changed, 1219 insertions(+) create mode 100644 cluster/centos/.gitignore create mode 100755 cluster/centos/build.sh create mode 100755 cluster/centos/config-build.sh create mode 100755 cluster/centos/config-default.sh create mode 100755 cluster/centos/master/scripts/apiserver.sh create mode 100755 cluster/centos/master/scripts/controller-manager.sh create mode 100755 cluster/centos/master/scripts/etcd.sh create mode 100755 cluster/centos/master/scripts/scheduler.sh create mode 100755 cluster/centos/minion/bin/mk-docker-opts.sh create mode 100755 cluster/centos/minion/bin/remove-docker0.sh create mode 100755 cluster/centos/minion/scripts/docker.sh create mode 100755 cluster/centos/minion/scripts/flannel.sh create mode 100755 cluster/centos/minion/scripts/kubelet.sh create mode 100755 cluster/centos/minion/scripts/proxy.sh create mode 100755 cluster/centos/util.sh diff --git a/cluster/centos/.gitignore b/cluster/centos/.gitignore new file mode 100644 index 00000000000..56aad3778f4 --- /dev/null +++ b/cluster/centos/.gitignore @@ -0,0 +1,13 @@ +binaries + +master/bin/etcd +master/bin/etcdctl +master/bin/kube* + +minion/bin/brctl +minion/bin/docker +minion/bin/etcd +minion/bin/etcdctl +minion/bin/flanneld +minion/bin/kube* +test.sh \ No newline at end of file diff --git a/cluster/centos/build.sh b/cluster/centos/build.sh new file mode 100755 index 00000000000..a8b6a5aafdb --- /dev/null +++ b/cluster/centos/build.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Download the flannel, etcd, docker, bridge-utils and K8s binaries automatically +# and store into binaries directory. +# Run as root only + +# author @kevin-wangzefeng + +set -o errexit +set -o nounset +set -o pipefail + +readonly ROOT=$(dirname "${BASH_SOURCE}") +source ${ROOT}/config-build.sh + +# ensure $RELEASES_DIR is an absolute file path +mkdir -p ${RELEASES_DIR} +RELEASES_DIR=$(cd ${RELEASES_DIR}; pwd) + +# get absolute file path of binaries +BINARY_DIR=$(cd ${ROOT}; pwd)/binaries + +function clean-up() { + rm -rf ${RELEASES_DIR} + rm -rf ${BINARY_DIR} +} + +function download-releases() { + rm -rf ${RELEASES_DIR} + mkdir -p ${RELEASES_DIR} + + echo "Download flannel release v${FLANNEL_VERSION} ..." + curl -L ${FLANNEL_DOWNLOAD_URL} -o ${RELEASES_DIR}/flannel.tar.gz + + echo "Download etcd release v${ETCD_VERSION} ..." + curl -L ${ETCD_DOWNLOAD_URL} -o ${RELEASES_DIR}/etcd.tar.gz + + echo "Download kubernetes release v${K8S_VERSION} ..." + curl -L ${K8S_DOWNLOAD_URL} -o ${RELEASES_DIR}/kubernetes.tar.gz + + echo "Download docker-latest ..." + curl -L https://get.docker.com/builds/Linux/x86_64/docker-latest -o ${RELEASES_DIR}/docker + + echo "Download bridge-utils from yum repo ..." + yum --downloadonly --downloaddir=${RELEASES_DIR} install bridge-utils + + mkdir -p ${RELEASES_DIR}/brctl-tmp + local rpm_file=$(ls ${RELEASES_DIR}/bridge-utils-*.rpm) + pushd ${RELEASES_DIR}/brctl-tmp >/dev/null 2>&1 + rpm2cpio ${rpm_file} | cpio -id + popd >/dev/null 2>&1 + cp ${RELEASES_DIR}/brctl-tmp/usr/sbin/brctl ${RELEASES_DIR} +} + +function unpack-releases() { + rm -rf ${BINARY_DIR} + mkdir -p ${BINARY_DIR}/master/bin + mkdir -p ${BINARY_DIR}/minion/bin + + # flannel + if [[ -f ${RELEASES_DIR}/flannel.tar.gz ]] ; then + tar xzf ${RELEASES_DIR}/flannel.tar.gz -C ${RELEASES_DIR} + cp ${RELEASES_DIR}/flannel-${FLANNEL_VERSION}/flanneld ${BINARY_DIR}/master/bin + cp ${RELEASES_DIR}/flannel-${FLANNEL_VERSION}/flanneld ${BINARY_DIR}/minion/bin + fi + + # ectd + if [[ -f ${RELEASES_DIR}/etcd.tar.gz ]] ; then + tar xzf ${RELEASES_DIR}/etcd.tar.gz -C ${RELEASES_DIR} + ETCD="etcd-v${ETCD_VERSION}-linux-amd64" + cp ${RELEASES_DIR}/$ETCD/etcd \ + ${RELEASES_DIR}/$ETCD/etcdctl ${BINARY_DIR}/master/bin + cp ${RELEASES_DIR}/$ETCD/etcd \ + ${RELEASES_DIR}/$ETCD/etcdctl ${BINARY_DIR}/minion/bin + fi + + # k8s + if [[ -f ${RELEASES_DIR}/kubernetes.tar.gz ]] ; then + tar xzf ${RELEASES_DIR}/kubernetes.tar.gz -C ${RELEASES_DIR} + + pushd ${RELEASES_DIR}/kubernetes/server + tar xzf kubernetes-server-linux-amd64.tar.gz + popd + cp ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kube-apiserver \ + ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kube-controller-manager \ + ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kube-scheduler ${BINARY_DIR}/master/bin + + cp ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kubelet \ + ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kube-proxy ${BINARY_DIR}/minion/bin + + cp ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kubectl ${BINARY_DIR} + fi + + if [[ -f ${RELEASES_DIR}/docker ]]; then + cp ${RELEASES_DIR}/docker ${BINARY_DIR}/minion/bin + fi + + if [[ -f ${RELEASES_DIR}/brctl ]]; then + cp ${RELEASES_DIR}/brctl ${BINARY_DIR}/minion/bin + fi + + chmod -R +x ${BINARY_DIR} + echo "Done! All binaries are stored in ${BINARY_DIR}" +} + +function parse-opt() { + local opt=${1-} + + case $opt in + download) + download-releases + ;; + unpack) + unpack-releases + ;; + clean) + clean-up + ;; + all) + download-releases + unpack-releases + ;; + *) + echo "Usage: " + echo " build.sh " + echo "Commands:" + echo " clean Clean up downloaded releases and unpacked binaries." + echo " download Download releases to \"${RELEASES_DIR}\"." + echo " unpack Unpack releases downloaded in \"${RELEASES_DIR}\", and copy binaries to \"${BINARY_DIR}\"." + echo " all Download releases and unpack them." + ;; + esac +} + +parse-opt $@ diff --git a/cluster/centos/config-build.sh b/cluster/centos/config-build.sh new file mode 100755 index 00000000000..a3fc754752b --- /dev/null +++ b/cluster/centos/config-build.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Contains configuration values for the Binaries downloading and unpacking. + +# Directory to store release packages that will be downloaded. +RELEASES_DIR=${RELEASES_DIR:-/tmp/downloads} + +# Define flannel version to use. +FLANNEL_VERSION=${FLANNEL_VERSION:-"0.4.0"} + +# Define etcd version to use. +ETCD_VERSION=${ETCD_VERSION:-"2.0.12"} + +# Define k8s version to use. +K8S_VERSION=${K8S_VERSION:-"1.0.1"} + +FLANNEL_DOWNLOAD_URL=\ +"https://github.com/coreos/flannel/releases/download/v${FLANNEL_VERSION}/flannel-${FLANNEL_VERSION}-linux-amd64.tar.gz" + +ETCD_DOWNLOAD_URL=\ +"https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz" + +K8S_DOWNLOAD_URL=\ +"https://github.com/kubernetes/kubernetes/releases/download/v${K8S_VERSION}/kubernetes.tar.gz" diff --git a/cluster/centos/config-default.sh b/cluster/centos/config-default.sh new file mode 100755 index 00000000000..559300a7ba4 --- /dev/null +++ b/cluster/centos/config-default.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Contains configuration values for the CentOS cluster + +# Currently only support root user. +export MASTER=${MASTER:-"root@8.8.8.18"} +export MASTER_IP=${MASTER#*@} + +# Define all your minion nodes, +# And separated with blank space like . +# Currently only support root user. +export MINIONS=${MINIONS:-"root@8.8.8.20 root@8.8.8.21"} +# If it practically impossible to set an array as an environment variable +# from a script, so assume variable is a string then convert it to an array +export MINIONS_ARRAY=($MINIONS) + +# Number of nodes in your cluster. +export NUM_MINIONS=${NUM_MINIONS:-2} + +# By default, the cluster will use the etcd installed on master. +export ETCD_SERVERS=${ETCD_SERVERS:-"http://$MASTER_IP:4001"} + +# define the IP range used for service cluster IPs. +# according to rfc 1918 ref: https://tools.ietf.org/html/rfc1918 choose a private ip range here. +export SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-"192.168.3.0/24"} + +# define the IP range used for flannel overlay network, should not conflict with above SERVICE_CLUSTER_IP_RANGE +export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"} + +# Extra options to set on the Docker command line. +# This is useful for setting --insecure-registry for local registries. +export DOCKER_OPTS=${DOCKER_OPTS:-""} + + +# Timeouts for process checking on master and minion +export PROCESS_CHECK_TIMEOUT=${PROCESS_CHECK_TIMEOUT:-180} # seconds. diff --git a/cluster/centos/master/scripts/apiserver.sh b/cluster/centos/master/scripts/apiserver.sh new file mode 100755 index 00000000000..967bc1a5f31 --- /dev/null +++ b/cluster/centos/master/scripts/apiserver.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +MASTER_ADDRESS=${1:-"8.8.8.18"} +ETCD_SERVERS=${2:-"http://8.8.8.18:4001"} +SERVICE_CLUSTER_IP_RANGE=${3:-"10.10.10.0/24"} + +cat </opt/kubernetes/cfg/kube-apiserver +# --logtostderr=true: log to standard error instead of files +KUBE_LOGTOSTDERR="--logtostderr=true" + +# --v=0: log level for V logs +KUBE_LOG_LEVEL="--v=4" + +# --etcd-servers=[]: List of etcd servers to watch (http://ip:port), +# comma separated. Mutually exclusive with -etcd-config +KUBE_ETCD_SERVERS="--etcd-servers=${ETCD_SERVERS}" + +# --address=127.0.0.1: DEPRECATED: see --insecure-bind-address instead +KUBE_API_ADDRESS="--address=${MASTER_ADDRESS}" + +# --port=8080: DEPRECATED: see --insecure-port instead +KUBE_API_PORT="--port=8080" + +# --kubelet-port=10250: Kubelet port +MINION_PORT="--kubelet-port=10250" + +# --allow-privileged=false: If true, allow privileged containers. +KUBE_ALLOW_PRIV="--allow-privileged=false" + +# --service-cluster-ip-range=: A CIDR notation IP range from which to assign service cluster IPs. +# This must not overlap with any IP ranges assigned to nodes for pods. +KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" + +# --admission-control="AlwaysAdmit": Ordered list of plug-ins +# to do admission control of resources into cluster. +# Comma-delimited list of: +# LimitRanger, AlwaysDeny, SecurityContextDeny, NamespaceExists, +# NamespaceLifecycle, NamespaceAutoProvision, DenyExecOnPrivileged, +# AlwaysAdmit, ServiceAccount, ResourceQuota +#KUBE_ADMISSION_CONTROL="" + +EOF + +KUBE_APISERVER_OPTS=" \${KUBE_LOGTOSTDERR} \\ + \${KUBE_LOG_LEVEL} \\ + \${KUBE_ETCD_SERVERS} \\ + \${KUBE_API_ADDRESS} \\ + \${KUBE_API_PORT} \\ + \${MINION_PORT} \\ + \${KUBE_ALLOW_PRIV} \\ + \${KUBE_SERVICE_ADDRESSES}" + + +cat </usr/lib/systemd/system/kube-apiserver.service +[Unit] +Description=Kubernetes API Server +Documentation=https://github.com/GoogleCloudPlatform/kubernetes + +[Service] +EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver +ExecStart=/opt/kubernetes/bin/kube-apiserver ${KUBE_APISERVER_OPTS} +Restart=on-failure + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable kube-apiserver +systemctl start kube-apiserver diff --git a/cluster/centos/master/scripts/controller-manager.sh b/cluster/centos/master/scripts/controller-manager.sh new file mode 100755 index 00000000000..3631a88c751 --- /dev/null +++ b/cluster/centos/master/scripts/controller-manager.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +MASTER_ADDRESS=${1:-"8.8.8.18"} + +cat </opt/kubernetes/cfg/kube-controller-manager +KUBE_LOGTOSTDERR="--logtostderr=true" +KUBE_LOG_LEVEL="--v=4" +KUBE_MASTER="--master=${MASTER_ADDRESS}:8080" + +EOF + +KUBE_CONTROLLER_MANAGER_OPTS=" \${KUBE_LOGTOSTDERR} \\ + \${KUBE_LOG_LEVEL} \\ + \${KUBE_MASTER}" + +cat </usr/lib/systemd/system/kube-controller-manager.service +[Unit] +Description=Kubernetes Controller Manager +Documentation=https://github.com/GoogleCloudPlatform/kubernetes + +[Service] +EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager +ExecStart=/opt/kubernetes/bin/kube-controller-manager ${KUBE_CONTROLLER_MANAGER_OPTS} +Restart=on-failure + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable kube-controller-manager +systemctl start kube-controller-manager diff --git a/cluster/centos/master/scripts/etcd.sh b/cluster/centos/master/scripts/etcd.sh new file mode 100755 index 00000000000..31c458538d8 --- /dev/null +++ b/cluster/centos/master/scripts/etcd.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Create etcd.conf, etcd.service, and start etcd service. + + +etcd_data_dir=/var/lib/etcd/ +mkdir -p ${etcd_data_dir} + +cat </opt/kubernetes/cfg/etcd.conf +# [member] +ETCD_NAME=default +ETCD_DATA_DIR="${etcd_data_dir}/default.etcd" +#ETCD_SNAPSHOT_COUNTER="10000" +#ETCD_HEARTBEAT_INTERVAL="100" +#ETCD_ELECTION_TIMEOUT="1000" +#ETCD_LISTEN_PEER_URLS="http://localhost:2380,http://localhost:7001" +ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:4001" +#ETCD_MAX_SNAPSHOTS="5" +#ETCD_MAX_WALS="5" +#ETCD_CORS="" +# +#[cluster] +#ETCD_INITIAL_ADVERTISE_PEER_URLS="http://localhost:2380,http://localhost:7001" +# if you use different ETCD_NAME (e.g. test), +# set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..." +#ETCD_INITIAL_CLUSTER="default=http://localhost:2380,default=http://localhost:7001" +#ETCD_INITIAL_CLUSTER_STATE="new" +#ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" +ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379,http://localhost:4001" +#ETCD_DISCOVERY="" +#ETCD_DISCOVERY_SRV="" +#ETCD_DISCOVERY_FALLBACK="proxy" +#ETCD_DISCOVERY_PROXY="" +# +#[proxy] +#ETCD_PROXY="off" +# +#[security] +#ETCD_CA_FILE="" +#ETCD_CERT_FILE="" +#ETCD_KEY_FILE="" +#ETCD_PEER_CA_FILE="" +#ETCD_PEER_CERT_FILE="" +#ETCD_PEER_KEY_FILE="" +EOF + +cat <//usr/lib/systemd/system/etcd.service +[Unit] +Description=Etcd Server +After=network.target + +[Service] +Type=simple +WorkingDirectory=${etcd_data_dir} +EnvironmentFile=-/opt/kubernetes/cfg/etcd.conf +# set GOMAXPROCS to number of processors +ExecStart=/bin/bash -c "GOMAXPROCS=\$(nproc) /opt/kubernetes/bin/etcd" + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable etcd +systemctl start etcd diff --git a/cluster/centos/master/scripts/scheduler.sh b/cluster/centos/master/scripts/scheduler.sh new file mode 100755 index 00000000000..beafd8c9278 --- /dev/null +++ b/cluster/centos/master/scripts/scheduler.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +MASTER_ADDRESS=${1:-"8.8.8.18"} + +cat </opt/kubernetes/cfg/kube-scheduler +### +# kubernetes scheduler config + +# --logtostderr=true: log to standard error instead of files +KUBE_LOGTOSTDERR="--logtostderr=true" + +# --v=0: log level for V logs +KUBE_LOG_LEVEL="--v=4" + +KUBE_MASTER="--master=${MASTER_ADDRESS}:8080" + +# Add your own! +KUBE_SCHEDULER_ARGS="" + +EOF + +KUBE_SCHEDULER_OPTS=" \${KUBE_LOGTOSTDERR} \\ + \${KUBE_LOG_LEVEL} \\ + \${KUBE_MASTER} \\ + \${KUBE_SCHEDULER_ARGS}" + +cat </usr/lib/systemd/system/kube-scheduler.service +[Unit] +Description=Kubernetes Scheduler +Documentation=https://github.com/GoogleCloudPlatform/kubernetes + +[Service] +EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler +ExecStart=/opt/kubernetes/bin/kube-scheduler ${KUBE_SCHEDULER_OPTS} +Restart=on-failure + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable kube-scheduler +systemctl start kube-scheduler diff --git a/cluster/centos/minion/bin/mk-docker-opts.sh b/cluster/centos/minion/bin/mk-docker-opts.sh new file mode 100755 index 00000000000..f3c50531bda --- /dev/null +++ b/cluster/centos/minion/bin/mk-docker-opts.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate Docker daemon options based on flannel env file. + +# exit on any error +set -e + +usage() { + echo "$0 [-f FLANNEL-ENV-FILE] [-d DOCKER-ENV-FILE] [-i] [-c] [-m] [-k COMBINED-KEY] + +Generate Docker daemon options based on flannel env file +OPTIONS: + -f Path to flannel env file. Defaults to /run/flannel/subnet.env + -d Path to Docker env file to write to. Defaults to /run/docker_opts.env + -i Output each Docker option as individual var. e.g. DOCKER_OPT_MTU=1500 + -c Output combined Docker options into DOCKER_OPTS var + -k Set the combined options key to this value (default DOCKER_OPTS=) + -m Do not output --ip-masq (useful for older Docker version) +" >/dev/stderr + exit 1 +} + +flannel_env="/run/flannel/subnet.env" +docker_env="/run/docker_opts.env" +combined_opts_key="DOCKER_OPTS" +indiv_opts=false +combined_opts=false +ipmasq=true + +while getopts "f:d:ick:" opt; do + case $opt in + f) + flannel_env=$OPTARG + ;; + d) + docker_env=$OPTARG + ;; + i) + indiv_opts=true + ;; + c) + combined_opts=true + ;; + m) + ipmasq=false + ;; + k) + combined_opts_key=$OPTARG + ;; + \?) + usage + ;; + esac +done + +if [[ $indiv_opts = false ]] && [[ $combined_opts = false ]]; then + indiv_opts=true + combined_opts=true +fi + +if [[ -f "$flannel_env" ]]; then + source $flannel_env +fi + +if [[ -n "$FLANNEL_SUBNET" ]]; then + DOCKER_OPT_BIP="--bip=$FLANNEL_SUBNET" +fi + +if [[ -n "$FLANNEL_MTU" ]]; then + DOCKER_OPT_MTU="--mtu=$FLANNEL_MTU" +fi + +if [[ "$FLANNEL_IPMASQ" = true ]] && [[ $ipmasq = true ]]; then + DOCKER_OPT_IPMASQ="--ip-masq=false" +fi + +eval docker_opts="\$${combined_opts_key}" +docker_opts+=" " + +echo -n "" >$docker_env +for opt in $(compgen -v DOCKER_OPT_); do + eval val=\$$opt + + if [[ "$indiv_opts" = true ]]; then + echo "$opt=\"$val\"" >>$docker_env + fi + + docker_opts+="$val " +done + +if [[ "$combined_opts" = true ]]; then + echo "${combined_opts_key}=\"${docker_opts}\"" >>$docker_env +fi + diff --git a/cluster/centos/minion/bin/remove-docker0.sh b/cluster/centos/minion/bin/remove-docker0.sh new file mode 100755 index 00000000000..4d016fc9246 --- /dev/null +++ b/cluster/centos/minion/bin/remove-docker0.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Delete default docker bridge, so that docker can start with flannel network. + +# exit on any error +set -e + +rc=0 +ip link show docker0 >/dev/null 2>&1 || rc="$?" +if [[ "$rc" -eq "0" ]]; then + ip link set dev docker0 down + /opt/kubernetes/bin/brctl delbr docker0 +fi \ No newline at end of file diff --git a/cluster/centos/minion/scripts/docker.sh b/cluster/centos/minion/scripts/docker.sh new file mode 100755 index 00000000000..74039e601c1 --- /dev/null +++ b/cluster/centos/minion/scripts/docker.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +DOCKER_OPTS=${1:-""} + +DOCKER_CONFIG=/opt/kubernetes/cfg/docker + +cat <$DOCKER_CONFIG +DOCKER_OPTS="-H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock --selinux-enabled=false ${DOCKER_OPTS}" +EOF + +cat </usr/lib/systemd/system/docker.service +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +After=network.target flannel.service +Requires=flannel.service + +[Service] +Type=notify +EnvironmentFile=-/run/flannel/docker +EnvironmentFile=-/opt/kubernetes/cfg/docker +WorkingDirectory=/opt/kubernetes/bin +ExecStartPre=/opt/kubernetes/bin/remove-docker0.sh +ExecStart=/opt/kubernetes/bin/docker -d \$DOCKER_OPT_BIP \$DOCKER_OPT_MTU \$DOCKER_OPTS +LimitNOFILE=1048576 +LimitNPROC=1048576 + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable docker +systemctl start docker diff --git a/cluster/centos/minion/scripts/flannel.sh b/cluster/centos/minion/scripts/flannel.sh new file mode 100755 index 00000000000..c19480d93a3 --- /dev/null +++ b/cluster/centos/minion/scripts/flannel.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +ETCD_SERVERS=${1:-"http://8.8.8.18:4001"} +FLANNEL_NET=${2:-"172.16.0.0/16"} + + +cat </opt/kubernetes/cfg/flannel +FLANNEL_ETCD="-etcd-endpoints=${ETCD_SERVERS}" +FLANNEL_ETCD_KEY="-etcd-prefix=/coreos.com/network" +EOF + +cat </usr/lib/systemd/system/flannel.service +[Unit] +Description=Flanneld overlay address etcd agent +After=network.target +Before=docker.service + +[Service] +EnvironmentFile=-/opt/kubernetes/cfg/flannel +ExecStart=/opt/kubernetes/bin/flanneld \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY} +ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -d /run/flannel/docker + +Type=notify + +[Install] +WantedBy=multi-user.target +RequiredBy=docker.service +EOF + +# Store FLANNEL_NET to etcd. +attempt=0 +while true; do + /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \ + get /coreos.com/network/config >/dev/null 2>&1 + if [[ "$?" == 0 ]]; then + break + else + if (( attempt > 600 )); then + echo "timeout for waiting network config" > ~/kube/err.log + exit 2 + fi + + /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \ + mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}" >/dev/null 2>&1 + attempt=$((attempt+1)) + sleep 3 + fi +done +wait + +systemctl daemon-reload \ No newline at end of file diff --git a/cluster/centos/minion/scripts/kubelet.sh b/cluster/centos/minion/scripts/kubelet.sh new file mode 100755 index 00000000000..c186c36939e --- /dev/null +++ b/cluster/centos/minion/scripts/kubelet.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +MASTER_ADDRESS=${1:-"8.8.8.18"} +NODE_ADDRESS=${2:-"8.8.8.20"} + + +cat </opt/kubernetes/cfg/kubelet +# --logtostderr=true: log to standard error instead of files +KUBE_LOGTOSTDERR="--logtostderr=true" + +# --v=0: log level for V logs +KUBE_LOG_LEVEL="--v=4" + +# --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces) +MINION_ADDRESS="--address=${NODE_ADDRESS}" + +# --port=10250: The port for the Kubelet to serve on. Note that "kubectl logs" will not work if you set this flag. +MINION_PORT="--port=10250" + +# --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname. +MINION_HOSTNAME="--hostname-override=${NODE_ADDRESS}" + +# --api-servers=[]: List of Kubernetes API servers for publishing events, +# and reading pods and services. (ip:port), comma separated. +KUBELET_API_SERVER="--api-servers=${MASTER_ADDRESS}:8080" + +# --allow-privileged=false: If true, allow containers to request privileged mode. [default=false] +KUBE_ALLOW_PRIV="--allow-privileged=false" + +# Add your own! +KUBELET_ARGS="" +EOF + +KUBE_PROXY_OPTS=" \${KUBE_LOGTOSTDERR} \\ + \${KUBE_LOG_LEVEL} \\ + \${MINION_ADDRESS} \\ + \${MINION_PORT} \\ + \${MINION_HOSTNAME} \\ + \${KUBELET_API_SERVER} \\ + \${KUBE_ALLOW_PRIV} \\ + \${KUBELET_ARGS}" + +cat </usr/lib/systemd/system/kubelet.service +[Unit] +Description=Kubernetes Kubelet +After=docker.service +Requires=docker.service + +[Service] +EnvironmentFile=-/opt/kubernetes/cfg/kubelet +ExecStart=/opt/kubernetes/bin/kubelet ${KUBE_PROXY_OPTS} +Restart=on-failure + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable kubelet +systemctl start kubelet \ No newline at end of file diff --git a/cluster/centos/minion/scripts/proxy.sh b/cluster/centos/minion/scripts/proxy.sh new file mode 100755 index 00000000000..5e4181f2c1b --- /dev/null +++ b/cluster/centos/minion/scripts/proxy.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +MASTER_ADDRESS=${1:-"8.8.8.18"} + +cat </opt/kubernetes/cfg/kube-proxy +# --logtostderr=true: log to standard error instead of files +KUBE_LOGTOSTDERR="--logtostderr=true" + +# --v=0: log level for V logs +KUBE_LOG_LEVEL="--v=4" + +# --master="": The address of the Kubernetes API server (overrides any value in kubeconfig) +KUBE_MASTER="--master=http://${MASTER_ADDRESS}:8080" +EOF + +KUBE_PROXY_OPTS=" \${KUBE_LOGTOSTDERR} \\ + \${KUBE_LOG_LEVEL} \\ + \${KUBE_MASTER}" + +cat </usr/lib/systemd/system/kube-proxy.service +[Unit] +Description=Kubernetes Proxy +After=network.target + +[Service] +EnvironmentFile=-/opt/kubernetes/cfg/kube-proxy +ExecStart=/opt/kubernetes/bin/kube-proxy ${KUBE_PROXY_OPTS} +Restart=on-failure + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable kube-proxy +systemctl start kube-proxy \ No newline at end of file diff --git a/cluster/centos/util.sh b/cluster/centos/util.sh new file mode 100755 index 00000000000..0fd54d788bd --- /dev/null +++ b/cluster/centos/util.sh @@ -0,0 +1,324 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts. + +# exit on any error +set -e + +SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR" + +# Use the config file specified in $KUBE_CONFIG_FILE, or default to +# config-default.sh. +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +readonly ROOT=$(dirname "${BASH_SOURCE}") +source "${ROOT}/${KUBE_CONFIG_FILE:-"config-default.sh"}" +source "$KUBE_ROOT/cluster/common.sh" + + +KUBECTL_PATH=${KUBE_ROOT}/cluster/centos/binaries/kubectl + +# Directory to be used for master and minion provisioning. +KUBE_TEMP="~/kubernetes" + + +# Must ensure that the following ENV vars are set +function detect-master() { + KUBE_MASTER=$MASTER + KUBE_MASTER_IP=${MASTER#*@} + echo "KUBE_MASTER_IP: ${KUBE_MASTER_IP}" 1>&2 + echo "KUBE_MASTER: ${MASTER}" 1>&2 +} + +# Get minion IP addresses and store in KUBE_MINION_IP_ADDRESSES[] +function detect-minions() { + KUBE_MINION_IP_ADDRESSES=() + for minion in ${MINIONS}; do + KUBE_MINION_IP_ADDRESSES+=("${minion#*@}") + done + echo "KUBE_MINION_IP_ADDRESSES: [${KUBE_MINION_IP_ADDRESSES[*]}]" 1>&2 +} + +# Verify prereqs on host machine +function verify-prereqs() { + local rc + rc=0 + ssh-add -L 1> /dev/null 2> /dev/null || rc="$?" + # "Could not open a connection to your authentication agent." + if [[ "${rc}" -eq 2 ]]; then + eval "$(ssh-agent)" > /dev/null + trap-add "kill ${SSH_AGENT_PID}" EXIT + fi + rc=0 + ssh-add -L 1> /dev/null 2> /dev/null || rc="$?" + # "The agent has no identities." + if [[ "${rc}" -eq 1 ]]; then + # Try adding one of the default identities, with or without passphrase. + ssh-add || true + fi + rc=0 + # Expect at least one identity to be available. + if ! ssh-add -L 1> /dev/null 2> /dev/null; then + echo "Could not find or add an SSH identity." + echo "Please start ssh-agent, add your identity, and retry." + exit 1 + fi +} + +# Install handler for signal trap +function trap-add { + local handler="$1" + local signal="${2-EXIT}" + local cur + + cur="$(eval "sh -c 'echo \$3' -- $(trap -p ${signal})")" + if [[ -n "${cur}" ]]; then + handler="${cur}; ${handler}" + fi + + trap "${handler}" ${signal} +} + +# Validate a kubernetes cluster +function validate-cluster() { + # by default call the generic validate-cluster.sh script, customizable by + # any cluster provider if this does not fit. + "${KUBE_ROOT}/cluster/validate-cluster.sh" +} + +# Instantiate a kubernetes cluster +function kube-up() { + provision-master + + for minion in ${MINIONS}; do + provision-minion ${minion} + done + + verify-master + for minion in ${MINIONS}; do + verify-minion ${minion} + done + + detect-master + + # set CONTEXT and KUBE_SERVER values for create-kubeconfig() and get-password() + export CONTEXT="centos" + export KUBE_SERVER="http://${KUBE_MASTER_IP}:8080" + source "${KUBE_ROOT}/cluster/common.sh" + + # set kubernetes user and password + get-password + create-kubeconfig +} + +# Delete a kubernetes cluster +function kube-down() { + tear-down-master + for minion in ${MINIONS}; do + tear-down-minion ${minion} + done +} + + +function verify-master() { + # verify master has all required daemons + printf "[INFO] Validating master ${MASTER}" + local -a required_daemon=("kube-apiserver" "kube-controller-manager" "kube-scheduler") + local validated="1" + local try_count=0 + until [[ "$validated" == "0" ]]; do + validated="0" + local daemon + for daemon in "${required_daemon[@]}"; do + local rc=0 + kube-ssh "${MASTER}" "pgrep -f ${daemon}" >/dev/null 2>&1 || rc="$?" + if [[ "${rc}" -ne "0" ]]; then + printf "." + validated="1" + ((try_count=try_count+2)) + if [[ ${try_count} -gt ${PROCESS_CHECK_TIMEOUT} ]]; then + printf "\nWarning: Process \"${daemon}\" status check timeout, please check manually.\n" + exit 1 + fi + sleep 2 + fi + done + done + printf "\n" + +} + +function verify-minion() { + # verify minion has all required daemons + printf "[INFO] Validating minion ${1}" + local -a required_daemon=("kube-proxy" "kubelet" "docker") + local validated="1" + local try_count=0 + until [[ "$validated" == "0" ]]; do + validated="0" + local daemon + for daemon in "${required_daemon[@]}"; do + local rc=0 + kube-ssh "${1}" "pgrep -f ${daemon}" >/dev/null 2>&1 || rc="$?" + if [[ "${rc}" -ne "0" ]]; then + printf "." + validated="1" + ((try_count=try_count+2)) + if [[ ${try_count} -gt ${PROCESS_CHECK_TIMEOUT} ]] ; then + printf "\nWarning: Process \"${daemon}\" status check timeout, please check manually.\n" + exit 1 + fi + sleep 2 + fi + done + done + printf "\n" +} + +# Clean up on master +function tear-down-master() { +echo "[INFO] tear-down-master on ${MASTER}" + for service_name in etcd kube-apiserver kube-controller-manager kube-scheduler ; do + service_file="/usr/lib/systemd/system/${service_name}.service" + ( + echo "if [[ -f $service_file ]]; then" + echo "systemctl stop $service_name" + echo "systemctl disable $service_name" + echo "rm -f $service_file" + echo "fi" + ) | kube-ssh "$MASTER" + done + kube-ssh "${MASTER}" "rm -rf /opt/kubernetes" + kube-ssh "${MASTER}" "rm -rf ${KUBE_TEMP}" + kube-ssh "${MASTER}" "rm -rf /var/lib/etcd" +} + +# Clean up on minion +function tear-down-minion() { +echo "[INFO] tear-down-minion on $1" + for service_name in kube-proxy kubelet docker flannel ; do + service_file="/usr/lib/systemd/system/${service_name}.service" + ( + echo "if [[ -f $service_file ]]; then" + echo "systemctl stop $service_name" + echo "systemctl disable $service_name" + echo "rm -f $service_file" + echo "fi" + ) | kube-ssh "$1" + done + kube-ssh "$1" "rm -rf /run/flannel" + kube-ssh "$1" "rm -rf /opt/kubernetes" + kube-ssh "$1" "rm -rf ${KUBE_TEMP}" +} + +# Provision master +# +# Assumed vars: +# MASTER +# KUBE_TEMP +# ETCD_SERVERS +# SERVICE_CLUSTER_IP_RANGE +function provision-master() { + echo "[INFO] Provision master on ${MASTER}" + local master_ip=${MASTER#*@} + ensure-setup-dir ${MASTER} + + # scp -r ${SSH_OPTS} master config-default.sh copy-files.sh util.sh "${MASTER}:${KUBE_TEMP}" + kube-scp ${MASTER} "${ROOT}/binaries/master ${ROOT}/master ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}" + ( + echo "cp -r ${KUBE_TEMP}/master/bin /opt/kubernetes" + echo "chmod -R +x /opt/kubernetes/bin" + + echo "bash ${KUBE_TEMP}/master/scripts/etcd.sh" + echo "bash ${KUBE_TEMP}/master/scripts/apiserver.sh ${master_ip} ${ETCD_SERVERS} ${SERVICE_CLUSTER_IP_RANGE}" + echo "bash ${KUBE_TEMP}/master/scripts/controller-manager.sh ${master_ip}" + echo "bash ${KUBE_TEMP}/master/scripts/scheduler.sh ${master_ip}" + + ) | kube-ssh "${MASTER}" +} + + +# Provision minion +# +# Assumed vars: +# $1 (minion) +# MASTER +# KUBE_TEMP +# ETCD_SERVERS +# FLANNEL_NET +# DOCKER_OPTS +function provision-minion() { + echo "[INFO] Provision minion on $1" + local master_ip=${MASTER#*@} + local minion=$1 + local minion_ip=${minion#*@} + ensure-setup-dir ${minion_ip} + + # scp -r ${SSH_OPTS} minion config-default.sh copy-files.sh util.sh "${minion_ip}:${KUBE_TEMP}" + kube-scp ${minion_ip} "${ROOT}/binaries/minion ${ROOT}/minion ${ROOT}/config-default.sh ${ROOT}/util.sh" ${KUBE_TEMP} + ( + echo "cp -r ${KUBE_TEMP}/minion/bin /opt/kubernetes" + echo "chmod -R +x /opt/kubernetes/bin" + + echo "bash ${KUBE_TEMP}/minion/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}" + echo "bash ${KUBE_TEMP}/minion/scripts/docker.sh ${DOCKER_OPTS}" + echo "bash ${KUBE_TEMP}/minion/scripts/kubelet.sh ${master_ip} ${minion_ip}" + echo "bash ${KUBE_TEMP}/minion/scripts/proxy.sh ${master_ip}" + + ) | kube-ssh "${minion_ip}" +} + +# Create dirs that'll be used during setup on target machine. +# +# Assumed vars: +# KUBE_TEMP +function ensure-setup-dir() { + ( + echo "mkdir -p ${KUBE_TEMP}" + echo "mkdir -p /opt/kubernetes/bin" + echo "mkdir -p /opt/kubernetes/cfg" + ) | kube-ssh "${1}" +} + +# Run command over ssh +function kube-ssh() { + local host="$1" + shift + ssh ${SSH_OPTS-} "${host}" "$@" >/dev/null 2>&1 +} + +# Copy file recursively over ssh +function kube-scp() { + local host="$1" + local src=($2) + local dst="$3" + scp -r ${SSH_OPTS-} ${src[*]} "${host}:${dst}" +} + +# Ensure that we have a password created for validating to the master. Will +# read from kubeconfig if available. +# +# Vars set: +# KUBE_USER +# KUBE_PASSWORD +function get-password { + get-kubeconfig-basicauth + if [[ -z "${KUBE_USER}" || -z "${KUBE_PASSWORD}" ]]; then + KUBE_USER=admin + KUBE_PASSWORD=$(python -c 'import string,random; \ + print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') + fi +} From 5f985045f4f69945bfa378994977a16b9cbc1fc7 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 2 Sep 2015 11:49:27 +0000 Subject: [PATCH 35/46] set default docker storage driver to devicemapper and fix DOCKER_OPTS passing --- cluster/centos/minion/scripts/docker.sh | 2 +- cluster/centos/util.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster/centos/minion/scripts/docker.sh b/cluster/centos/minion/scripts/docker.sh index 74039e601c1..14fb52a4556 100755 --- a/cluster/centos/minion/scripts/docker.sh +++ b/cluster/centos/minion/scripts/docker.sh @@ -20,7 +20,7 @@ DOCKER_OPTS=${1:-""} DOCKER_CONFIG=/opt/kubernetes/cfg/docker cat <$DOCKER_CONFIG -DOCKER_OPTS="-H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock --selinux-enabled=false ${DOCKER_OPTS}" +DOCKER_OPTS="-H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock -s devicemapper --selinux-enabled=false ${DOCKER_OPTS}" EOF cat </usr/lib/systemd/system/docker.service diff --git a/cluster/centos/util.sh b/cluster/centos/util.sh index 0fd54d788bd..d9baa43ddaf 100755 --- a/cluster/centos/util.sh +++ b/cluster/centos/util.sh @@ -274,7 +274,7 @@ function provision-minion() { echo "chmod -R +x /opt/kubernetes/bin" echo "bash ${KUBE_TEMP}/minion/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}" - echo "bash ${KUBE_TEMP}/minion/scripts/docker.sh ${DOCKER_OPTS}" + echo "bash ${KUBE_TEMP}/minion/scripts/docker.sh \"${DOCKER_OPTS}\"" echo "bash ${KUBE_TEMP}/minion/scripts/kubelet.sh ${master_ip} ${minion_ip}" echo "bash ${KUBE_TEMP}/minion/scripts/proxy.sh ${master_ip}" From 4e2613575de5cdedf43f55431d2b28899d04da78 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 2 Sep 2015 15:35:48 +0000 Subject: [PATCH 36/46] update centos deployment scripts call make-ca-cert.sh to generate certs --- cluster/centos/config-default.sh | 3 +++ cluster/centos/master/scripts/apiserver.sh | 21 +++++++++++++++++-- .../master/scripts/controller-manager.sh | 11 +++++++++- cluster/centos/util.sh | 17 ++++++++------- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/cluster/centos/config-default.sh b/cluster/centos/config-default.sh index 559300a7ba4..da60816435f 100755 --- a/cluster/centos/config-default.sh +++ b/cluster/centos/config-default.sh @@ -41,6 +41,9 @@ export SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-"192.168.3.0/24"} # define the IP range used for flannel overlay network, should not conflict with above SERVICE_CLUSTER_IP_RANGE export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"} +# Admission Controllers to invoke prior to persisting objects in cluster +export ADMISSION_CONTROL=NamespaceLifecycle,NamespaceExists,LimitRanger,ServiceAccount,ResourceQuota,SecurityContextDeny + # Extra options to set on the Docker command line. # This is useful for setting --insecure-registry for local registries. export DOCKER_OPTS=${DOCKER_OPTS:-""} diff --git a/cluster/centos/master/scripts/apiserver.sh b/cluster/centos/master/scripts/apiserver.sh index 967bc1a5f31..53c4e5fcce3 100755 --- a/cluster/centos/master/scripts/apiserver.sh +++ b/cluster/centos/master/scripts/apiserver.sh @@ -18,6 +18,7 @@ MASTER_ADDRESS=${1:-"8.8.8.18"} ETCD_SERVERS=${2:-"http://8.8.8.18:4001"} SERVICE_CLUSTER_IP_RANGE=${3:-"10.10.10.0/24"} +ADMISSION_CONTROL=${4:-""} cat </opt/kubernetes/cfg/kube-apiserver # --logtostderr=true: log to standard error instead of files @@ -52,8 +53,21 @@ KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" # LimitRanger, AlwaysDeny, SecurityContextDeny, NamespaceExists, # NamespaceLifecycle, NamespaceAutoProvision, DenyExecOnPrivileged, # AlwaysAdmit, ServiceAccount, ResourceQuota -#KUBE_ADMISSION_CONTROL="" +#KUBE_ADMISSION_CONTROL="--admission-control=\"${ADMISSION_CONTROL}\"" +# --client-ca-file="": If set, any request presenting a client certificate signed +# by one of the authorities in the client-ca-file is authenticated with an identity +# corresponding to the CommonName of the client certificate. +KUBE_API_CLIENT_CA_FILE="--client-ca-file=/srv/kubernetes/ca.crt" + +# --tls-cert-file="": File containing x509 Certificate for HTTPS. (CA cert, if any, +# concatenated after server cert). If HTTPS serving is enabled, and --tls-cert-file +# and --tls-private-key-file are not provided, a self-signed certificate and key are +# generated for the public address and saved to /var/run/kubernetes. +KUBE_API_TLS_CERT_FILE="--tls-cert-file=/srv/kubernetes/server.cert" + +# --tls-private-key-file="": File containing x509 private key matching --tls-cert-file. +KUBE_API_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/srv/kubernetes/server.key" EOF KUBE_APISERVER_OPTS=" \${KUBE_LOGTOSTDERR} \\ @@ -63,7 +77,10 @@ KUBE_APISERVER_OPTS=" \${KUBE_LOGTOSTDERR} \\ \${KUBE_API_PORT} \\ \${MINION_PORT} \\ \${KUBE_ALLOW_PRIV} \\ - \${KUBE_SERVICE_ADDRESSES}" + \${KUBE_SERVICE_ADDRESSES} \\ + \${KUBE_API_CLIENT_CA_FILE} \\ + \${KUBE_API_TLS_CERT_FILE} \\ + \${KUBE_API_TLS_PRIVATE_KEY_FILE}" cat </usr/lib/systemd/system/kube-apiserver.service diff --git a/cluster/centos/master/scripts/controller-manager.sh b/cluster/centos/master/scripts/controller-manager.sh index 3631a88c751..b6fb216c8de 100755 --- a/cluster/centos/master/scripts/controller-manager.sh +++ b/cluster/centos/master/scripts/controller-manager.sh @@ -22,11 +22,20 @@ KUBE_LOGTOSTDERR="--logtostderr=true" KUBE_LOG_LEVEL="--v=4" KUBE_MASTER="--master=${MASTER_ADDRESS}:8080" +# --root-ca-file="": If set, this root certificate authority will be included in +# service account's token secret. This must be a valid PEM-encoded CA bundle. +KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE="--root-ca-file=/srv/kubernetes/ca.crt" + +# --service-account-private-key-file="": Filename containing a PEM-encoded private +# RSA key used to sign service account tokens. +KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE="--service-account-private-key-file=/srv/kubernetes/server.key" EOF KUBE_CONTROLLER_MANAGER_OPTS=" \${KUBE_LOGTOSTDERR} \\ \${KUBE_LOG_LEVEL} \\ - \${KUBE_MASTER}" + \${KUBE_MASTER} \\ + \${KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE} \\ + \${KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE}" cat </usr/lib/systemd/system/kube-controller-manager.service [Unit] diff --git a/cluster/centos/util.sh b/cluster/centos/util.sh index d9baa43ddaf..fdcb299101d 100755 --- a/cluster/centos/util.sh +++ b/cluster/centos/util.sh @@ -150,7 +150,7 @@ function verify-master() { validated="1" ((try_count=try_count+2)) if [[ ${try_count} -gt ${PROCESS_CHECK_TIMEOUT} ]]; then - printf "\nWarning: Process \"${daemon}\" status check timeout, please check manually.\n" + printf "\nWarning: Process \"${daemon}\" failed to run on ${MASTER}, please check.\n" exit 1 fi sleep 2 @@ -178,7 +178,7 @@ function verify-minion() { validated="1" ((try_count=try_count+2)) if [[ ${try_count} -gt ${PROCESS_CHECK_TIMEOUT} ]] ; then - printf "\nWarning: Process \"${daemon}\" status check timeout, please check manually.\n" + printf "\nWarning: Process \"${daemon}\" failed to run on ${1}, please check.\n" exit 1 fi sleep 2 @@ -237,13 +237,14 @@ function provision-master() { ensure-setup-dir ${MASTER} # scp -r ${SSH_OPTS} master config-default.sh copy-files.sh util.sh "${MASTER}:${KUBE_TEMP}" - kube-scp ${MASTER} "${ROOT}/binaries/master ${ROOT}/master ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}" + kube-scp ${MASTER} "${ROOT}/../saltbase/salt/generate-cert/make-ca-cert.sh ${ROOT}/binaries/master ${ROOT}/master ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}" ( echo "cp -r ${KUBE_TEMP}/master/bin /opt/kubernetes" echo "chmod -R +x /opt/kubernetes/bin" + echo "bash ${KUBE_TEMP}/make-ca-cert.sh ${master_ip} IP:${master_ip},IP:${SERVICE_CLUSTER_IP_RANGE%.*}.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local" echo "bash ${KUBE_TEMP}/master/scripts/etcd.sh" - echo "bash ${KUBE_TEMP}/master/scripts/apiserver.sh ${master_ip} ${ETCD_SERVERS} ${SERVICE_CLUSTER_IP_RANGE}" + echo "bash ${KUBE_TEMP}/master/scripts/apiserver.sh ${master_ip} ${ETCD_SERVERS} ${SERVICE_CLUSTER_IP_RANGE} ${ADMISSION_CONTROL}" echo "bash ${KUBE_TEMP}/master/scripts/controller-manager.sh ${master_ip}" echo "bash ${KUBE_TEMP}/master/scripts/scheduler.sh ${master_ip}" @@ -265,10 +266,10 @@ function provision-minion() { local master_ip=${MASTER#*@} local minion=$1 local minion_ip=${minion#*@} - ensure-setup-dir ${minion_ip} + ensure-setup-dir ${minion} # scp -r ${SSH_OPTS} minion config-default.sh copy-files.sh util.sh "${minion_ip}:${KUBE_TEMP}" - kube-scp ${minion_ip} "${ROOT}/binaries/minion ${ROOT}/minion ${ROOT}/config-default.sh ${ROOT}/util.sh" ${KUBE_TEMP} + kube-scp ${minion} "${ROOT}/binaries/minion ${ROOT}/minion ${ROOT}/config-default.sh ${ROOT}/util.sh" ${KUBE_TEMP} ( echo "cp -r ${KUBE_TEMP}/minion/bin /opt/kubernetes" echo "chmod -R +x /opt/kubernetes/bin" @@ -278,7 +279,7 @@ function provision-minion() { echo "bash ${KUBE_TEMP}/minion/scripts/kubelet.sh ${master_ip} ${minion_ip}" echo "bash ${KUBE_TEMP}/minion/scripts/proxy.sh ${master_ip}" - ) | kube-ssh "${minion_ip}" + ) | kube-ssh "${minion}" } # Create dirs that'll be used during setup on target machine. @@ -297,7 +298,7 @@ function ensure-setup-dir() { function kube-ssh() { local host="$1" shift - ssh ${SSH_OPTS-} "${host}" "$@" >/dev/null 2>&1 + ssh ${SSH_OPTS-} "${host}" "$@" # >/dev/null 2>&1 } # Copy file recursively over ssh From de9d7229935b110e946424d73628a2833fa36649 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 2 Sep 2015 17:36:10 +0000 Subject: [PATCH 37/46] update centos deployment scripts, add sudo user support. --- cluster/centos/build.sh | 4 +- cluster/centos/minion/scripts/docker.sh | 2 +- cluster/centos/util.sh | 92 +++++++++++-------------- 3 files changed, 44 insertions(+), 54 deletions(-) diff --git a/cluster/centos/build.sh b/cluster/centos/build.sh index a8b6a5aafdb..a624ade1310 100755 --- a/cluster/centos/build.sh +++ b/cluster/centos/build.sh @@ -16,7 +16,7 @@ # Download the flannel, etcd, docker, bridge-utils and K8s binaries automatically # and store into binaries directory. -# Run as root only +# Run as sudoers only # author @kevin-wangzefeng @@ -56,7 +56,7 @@ function download-releases() { curl -L https://get.docker.com/builds/Linux/x86_64/docker-latest -o ${RELEASES_DIR}/docker echo "Download bridge-utils from yum repo ..." - yum --downloadonly --downloaddir=${RELEASES_DIR} install bridge-utils + sudo yum --downloadonly --downloaddir=${RELEASES_DIR} install bridge-utils mkdir -p ${RELEASES_DIR}/brctl-tmp local rpm_file=$(ls ${RELEASES_DIR}/bridge-utils-*.rpm) diff --git a/cluster/centos/minion/scripts/docker.sh b/cluster/centos/minion/scripts/docker.sh index 14fb52a4556..5d1b6ebfd36 100755 --- a/cluster/centos/minion/scripts/docker.sh +++ b/cluster/centos/minion/scripts/docker.sh @@ -36,7 +36,7 @@ EnvironmentFile=-/run/flannel/docker EnvironmentFile=-/opt/kubernetes/cfg/docker WorkingDirectory=/opt/kubernetes/bin ExecStartPre=/opt/kubernetes/bin/remove-docker0.sh -ExecStart=/opt/kubernetes/bin/docker -d \$DOCKER_OPT_BIP \$DOCKER_OPT_MTU \$DOCKER_OPTS +ExecStart=/opt/kubernetes/bin/docker daemon \$DOCKER_OPT_BIP \$DOCKER_OPT_MTU \$DOCKER_OPTS LimitNOFILE=1048576 LimitNPROC=1048576 diff --git a/cluster/centos/util.sh b/cluster/centos/util.sh index fdcb299101d..27dfc0f4ef1 100755 --- a/cluster/centos/util.sh +++ b/cluster/centos/util.sh @@ -32,7 +32,7 @@ source "$KUBE_ROOT/cluster/common.sh" KUBECTL_PATH=${KUBE_ROOT}/cluster/centos/binaries/kubectl # Directory to be used for master and minion provisioning. -KUBE_TEMP="~/kubernetes" +KUBE_TEMP="~/kube_temp" # Must ensure that the following ENV vars are set @@ -144,7 +144,7 @@ function verify-master() { local daemon for daemon in "${required_daemon[@]}"; do local rc=0 - kube-ssh "${MASTER}" "pgrep -f ${daemon}" >/dev/null 2>&1 || rc="$?" + kube-ssh "${MASTER}" "sudo pgrep -f ${daemon}" >/dev/null 2>&1 || rc="$?" if [[ "${rc}" -ne "0" ]]; then printf "." validated="1" @@ -172,7 +172,7 @@ function verify-minion() { local daemon for daemon in "${required_daemon[@]}"; do local rc=0 - kube-ssh "${1}" "pgrep -f ${daemon}" >/dev/null 2>&1 || rc="$?" + kube-ssh "${1}" "sudo pgrep -f ${daemon}" >/dev/null 2>&1 || rc="$?" if [[ "${rc}" -ne "0" ]]; then printf "." validated="1" @@ -193,17 +193,16 @@ function tear-down-master() { echo "[INFO] tear-down-master on ${MASTER}" for service_name in etcd kube-apiserver kube-controller-manager kube-scheduler ; do service_file="/usr/lib/systemd/system/${service_name}.service" - ( - echo "if [[ -f $service_file ]]; then" - echo "systemctl stop $service_name" - echo "systemctl disable $service_name" - echo "rm -f $service_file" - echo "fi" - ) | kube-ssh "$MASTER" + kube-ssh "$MASTER" " \ + if [[ -f $service_file ]]; then \ + sudo systemctl stop $service_name; \ + sudo systemctl disable $service_name; \ + sudo rm -f $service_file; \ + fi" done - kube-ssh "${MASTER}" "rm -rf /opt/kubernetes" - kube-ssh "${MASTER}" "rm -rf ${KUBE_TEMP}" - kube-ssh "${MASTER}" "rm -rf /var/lib/etcd" + kube-ssh "${MASTER}" "sudo rm -rf /opt/kubernetes" + kube-ssh "${MASTER}" "sudo rm -rf ${KUBE_TEMP}" + kube-ssh "${MASTER}" "sudo rm -rf /var/lib/etcd" } # Clean up on minion @@ -211,17 +210,16 @@ function tear-down-minion() { echo "[INFO] tear-down-minion on $1" for service_name in kube-proxy kubelet docker flannel ; do service_file="/usr/lib/systemd/system/${service_name}.service" - ( - echo "if [[ -f $service_file ]]; then" - echo "systemctl stop $service_name" - echo "systemctl disable $service_name" - echo "rm -f $service_file" - echo "fi" - ) | kube-ssh "$1" + kube-ssh "$1" " \ + if [[ -f $service_file ]]; then \ + sudo systemctl stop $service_name; \ + sudo systemctl disable $service_name; \ + sudo rm -f $service_file; \ + fi" done - kube-ssh "$1" "rm -rf /run/flannel" - kube-ssh "$1" "rm -rf /opt/kubernetes" - kube-ssh "$1" "rm -rf ${KUBE_TEMP}" + kube-ssh "$1" "sudo rm -rf /run/flannel" + kube-ssh "$1" "sudo rm -rf /opt/kubernetes" + kube-ssh "$1" "sudo rm -rf ${KUBE_TEMP}" } # Provision master @@ -238,17 +236,14 @@ function provision-master() { # scp -r ${SSH_OPTS} master config-default.sh copy-files.sh util.sh "${MASTER}:${KUBE_TEMP}" kube-scp ${MASTER} "${ROOT}/../saltbase/salt/generate-cert/make-ca-cert.sh ${ROOT}/binaries/master ${ROOT}/master ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}" - ( - echo "cp -r ${KUBE_TEMP}/master/bin /opt/kubernetes" - echo "chmod -R +x /opt/kubernetes/bin" - - echo "bash ${KUBE_TEMP}/make-ca-cert.sh ${master_ip} IP:${master_ip},IP:${SERVICE_CLUSTER_IP_RANGE%.*}.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local" - echo "bash ${KUBE_TEMP}/master/scripts/etcd.sh" - echo "bash ${KUBE_TEMP}/master/scripts/apiserver.sh ${master_ip} ${ETCD_SERVERS} ${SERVICE_CLUSTER_IP_RANGE} ${ADMISSION_CONTROL}" - echo "bash ${KUBE_TEMP}/master/scripts/controller-manager.sh ${master_ip}" - echo "bash ${KUBE_TEMP}/master/scripts/scheduler.sh ${master_ip}" - - ) | kube-ssh "${MASTER}" + kube-ssh "${MASTER}" " \ + sudo cp -r ${KUBE_TEMP}/master/bin /opt/kubernetes; \ + sudo chmod -R +x /opt/kubernetes/bin; \ + sudo bash ${KUBE_TEMP}/make-ca-cert.sh ${master_ip} IP:${master_ip},IP:${SERVICE_CLUSTER_IP_RANGE%.*}.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local; \ + sudo bash ${KUBE_TEMP}/master/scripts/etcd.sh; \ + sudo bash ${KUBE_TEMP}/master/scripts/apiserver.sh ${master_ip} ${ETCD_SERVERS} ${SERVICE_CLUSTER_IP_RANGE} ${ADMISSION_CONTROL}; \ + sudo bash ${KUBE_TEMP}/master/scripts/controller-manager.sh ${master_ip}; \ + sudo bash ${KUBE_TEMP}/master/scripts/scheduler.sh ${master_ip}" } @@ -270,16 +265,13 @@ function provision-minion() { # scp -r ${SSH_OPTS} minion config-default.sh copy-files.sh util.sh "${minion_ip}:${KUBE_TEMP}" kube-scp ${minion} "${ROOT}/binaries/minion ${ROOT}/minion ${ROOT}/config-default.sh ${ROOT}/util.sh" ${KUBE_TEMP} - ( - echo "cp -r ${KUBE_TEMP}/minion/bin /opt/kubernetes" - echo "chmod -R +x /opt/kubernetes/bin" - - echo "bash ${KUBE_TEMP}/minion/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}" - echo "bash ${KUBE_TEMP}/minion/scripts/docker.sh \"${DOCKER_OPTS}\"" - echo "bash ${KUBE_TEMP}/minion/scripts/kubelet.sh ${master_ip} ${minion_ip}" - echo "bash ${KUBE_TEMP}/minion/scripts/proxy.sh ${master_ip}" - - ) | kube-ssh "${minion}" + kube-ssh "${minion}" " \ + sudo cp -r ${KUBE_TEMP}/minion/bin /opt/kubernetes; \ + sudo chmod -R +x /opt/kubernetes/bin; \ + sudo bash ${KUBE_TEMP}/minion/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \ + sudo bash ${KUBE_TEMP}/minion/scripts/docker.sh \"${DOCKER_OPTS}\"; \ + sudo bash ${KUBE_TEMP}/minion/scripts/kubelet.sh ${master_ip} ${minion_ip}; \ + sudo bash ${KUBE_TEMP}/minion/scripts/proxy.sh ${master_ip}" } # Create dirs that'll be used during setup on target machine. @@ -287,18 +279,16 @@ function provision-minion() { # Assumed vars: # KUBE_TEMP function ensure-setup-dir() { - ( - echo "mkdir -p ${KUBE_TEMP}" - echo "mkdir -p /opt/kubernetes/bin" - echo "mkdir -p /opt/kubernetes/cfg" - ) | kube-ssh "${1}" + kube-ssh "${1}" "mkdir -p ${KUBE_TEMP}; \ + sudo mkdir -p /opt/kubernetes/bin; \ + sudo mkdir -p /opt/kubernetes/cfg" } # Run command over ssh function kube-ssh() { local host="$1" shift - ssh ${SSH_OPTS-} "${host}" "$@" # >/dev/null 2>&1 + ssh ${SSH_OPTS} -t "${host}" "$@" >/dev/null 2>&1 } # Copy file recursively over ssh @@ -306,7 +296,7 @@ function kube-scp() { local host="$1" local src=($2) local dst="$3" - scp -r ${SSH_OPTS-} ${src[*]} "${host}:${dst}" + scp -r ${SSH_OPTS} ${src[*]} "${host}:${dst}" } # Ensure that we have a password created for validating to the master. Will From 82aaf118e2e97d8edecda71e0b95bdef0851ce53 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 10 Sep 2015 15:38:12 +0000 Subject: [PATCH 38/46] remove the dependency on brctl, update default config and rephrase minion to node --- cluster/centos/.gitignore | 13 +++++----- cluster/centos/build.sh | 24 ++++--------------- cluster/centos/config-build.sh | 4 ++-- cluster/centos/config-default.sh | 9 ++++--- .../{minion => node}/bin/mk-docker-opts.sh | 0 .../{minion => node}/bin/remove-docker0.sh | 4 ++-- .../centos/{minion => node}/scripts/docker.sh | 0 .../{minion => node}/scripts/flannel.sh | 0 .../{minion => node}/scripts/kubelet.sh | 0 .../centos/{minion => node}/scripts/proxy.sh | 0 cluster/centos/util.sh | 12 +++++----- 11 files changed, 25 insertions(+), 41 deletions(-) rename cluster/centos/{minion => node}/bin/mk-docker-opts.sh (100%) rename cluster/centos/{minion => node}/bin/remove-docker0.sh (95%) rename cluster/centos/{minion => node}/scripts/docker.sh (100%) rename cluster/centos/{minion => node}/scripts/flannel.sh (100%) rename cluster/centos/{minion => node}/scripts/kubelet.sh (100%) rename cluster/centos/{minion => node}/scripts/proxy.sh (100%) diff --git a/cluster/centos/.gitignore b/cluster/centos/.gitignore index 56aad3778f4..c97ce235be4 100644 --- a/cluster/centos/.gitignore +++ b/cluster/centos/.gitignore @@ -4,10 +4,9 @@ master/bin/etcd master/bin/etcdctl master/bin/kube* -minion/bin/brctl -minion/bin/docker -minion/bin/etcd -minion/bin/etcdctl -minion/bin/flanneld -minion/bin/kube* -test.sh \ No newline at end of file +node/bin/docker +node/bin/etcd +node/bin/etcdctl +node/bin/flanneld +node/bin/kube* +local-test.sh diff --git a/cluster/centos/build.sh b/cluster/centos/build.sh index a624ade1310..55a675fad0d 100755 --- a/cluster/centos/build.sh +++ b/cluster/centos/build.sh @@ -54,28 +54,18 @@ function download-releases() { echo "Download docker-latest ..." curl -L https://get.docker.com/builds/Linux/x86_64/docker-latest -o ${RELEASES_DIR}/docker - - echo "Download bridge-utils from yum repo ..." - sudo yum --downloadonly --downloaddir=${RELEASES_DIR} install bridge-utils - - mkdir -p ${RELEASES_DIR}/brctl-tmp - local rpm_file=$(ls ${RELEASES_DIR}/bridge-utils-*.rpm) - pushd ${RELEASES_DIR}/brctl-tmp >/dev/null 2>&1 - rpm2cpio ${rpm_file} | cpio -id - popd >/dev/null 2>&1 - cp ${RELEASES_DIR}/brctl-tmp/usr/sbin/brctl ${RELEASES_DIR} } function unpack-releases() { rm -rf ${BINARY_DIR} mkdir -p ${BINARY_DIR}/master/bin - mkdir -p ${BINARY_DIR}/minion/bin + mkdir -p ${BINARY_DIR}/node/bin # flannel if [[ -f ${RELEASES_DIR}/flannel.tar.gz ]] ; then tar xzf ${RELEASES_DIR}/flannel.tar.gz -C ${RELEASES_DIR} cp ${RELEASES_DIR}/flannel-${FLANNEL_VERSION}/flanneld ${BINARY_DIR}/master/bin - cp ${RELEASES_DIR}/flannel-${FLANNEL_VERSION}/flanneld ${BINARY_DIR}/minion/bin + cp ${RELEASES_DIR}/flannel-${FLANNEL_VERSION}/flanneld ${BINARY_DIR}/node/bin fi # ectd @@ -85,7 +75,7 @@ function unpack-releases() { cp ${RELEASES_DIR}/$ETCD/etcd \ ${RELEASES_DIR}/$ETCD/etcdctl ${BINARY_DIR}/master/bin cp ${RELEASES_DIR}/$ETCD/etcd \ - ${RELEASES_DIR}/$ETCD/etcdctl ${BINARY_DIR}/minion/bin + ${RELEASES_DIR}/$ETCD/etcdctl ${BINARY_DIR}/node/bin fi # k8s @@ -100,17 +90,13 @@ function unpack-releases() { ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kube-scheduler ${BINARY_DIR}/master/bin cp ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kubelet \ - ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kube-proxy ${BINARY_DIR}/minion/bin + ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kube-proxy ${BINARY_DIR}/node/bin cp ${RELEASES_DIR}/kubernetes/server/kubernetes/server/bin/kubectl ${BINARY_DIR} fi if [[ -f ${RELEASES_DIR}/docker ]]; then - cp ${RELEASES_DIR}/docker ${BINARY_DIR}/minion/bin - fi - - if [[ -f ${RELEASES_DIR}/brctl ]]; then - cp ${RELEASES_DIR}/brctl ${BINARY_DIR}/minion/bin + cp ${RELEASES_DIR}/docker ${BINARY_DIR}/node/bin fi chmod -R +x ${BINARY_DIR} diff --git a/cluster/centos/config-build.sh b/cluster/centos/config-build.sh index a3fc754752b..879a7e0cad3 100755 --- a/cluster/centos/config-build.sh +++ b/cluster/centos/config-build.sh @@ -20,13 +20,13 @@ RELEASES_DIR=${RELEASES_DIR:-/tmp/downloads} # Define flannel version to use. -FLANNEL_VERSION=${FLANNEL_VERSION:-"0.4.0"} +FLANNEL_VERSION=${FLANNEL_VERSION:-"0.5.3"} # Define etcd version to use. ETCD_VERSION=${ETCD_VERSION:-"2.0.12"} # Define k8s version to use. -K8S_VERSION=${K8S_VERSION:-"1.0.1"} +K8S_VERSION=${K8S_VERSION:-"1.0.4"} FLANNEL_DOWNLOAD_URL=\ "https://github.com/coreos/flannel/releases/download/v${FLANNEL_VERSION}/flannel-${FLANNEL_VERSION}-linux-amd64.tar.gz" diff --git a/cluster/centos/config-default.sh b/cluster/centos/config-default.sh index da60816435f..beb0841bf78 100755 --- a/cluster/centos/config-default.sh +++ b/cluster/centos/config-default.sh @@ -15,15 +15,14 @@ # limitations under the License. ## Contains configuration values for the CentOS cluster - -# Currently only support root user. -export MASTER=${MASTER:-"root@8.8.8.18"} +# The user should have sudo privilege +export MASTER=${MASTER:-"centos@172.10.0.11"} export MASTER_IP=${MASTER#*@} # Define all your minion nodes, # And separated with blank space like . -# Currently only support root user. -export MINIONS=${MINIONS:-"root@8.8.8.20 root@8.8.8.21"} +# The user should have sudo privilege +export MINIONS=${MINIONS:-"centos@172.10.0.12 centos@172.10.0.13"} # If it practically impossible to set an array as an environment variable # from a script, so assume variable is a string then convert it to an array export MINIONS_ARRAY=($MINIONS) diff --git a/cluster/centos/minion/bin/mk-docker-opts.sh b/cluster/centos/node/bin/mk-docker-opts.sh similarity index 100% rename from cluster/centos/minion/bin/mk-docker-opts.sh rename to cluster/centos/node/bin/mk-docker-opts.sh diff --git a/cluster/centos/minion/bin/remove-docker0.sh b/cluster/centos/node/bin/remove-docker0.sh similarity index 95% rename from cluster/centos/minion/bin/remove-docker0.sh rename to cluster/centos/node/bin/remove-docker0.sh index 4d016fc9246..31a90c50c79 100755 --- a/cluster/centos/minion/bin/remove-docker0.sh +++ b/cluster/centos/node/bin/remove-docker0.sh @@ -23,5 +23,5 @@ rc=0 ip link show docker0 >/dev/null 2>&1 || rc="$?" if [[ "$rc" -eq "0" ]]; then ip link set dev docker0 down - /opt/kubernetes/bin/brctl delbr docker0 -fi \ No newline at end of file + ip link delete docker0 +fi diff --git a/cluster/centos/minion/scripts/docker.sh b/cluster/centos/node/scripts/docker.sh similarity index 100% rename from cluster/centos/minion/scripts/docker.sh rename to cluster/centos/node/scripts/docker.sh diff --git a/cluster/centos/minion/scripts/flannel.sh b/cluster/centos/node/scripts/flannel.sh similarity index 100% rename from cluster/centos/minion/scripts/flannel.sh rename to cluster/centos/node/scripts/flannel.sh diff --git a/cluster/centos/minion/scripts/kubelet.sh b/cluster/centos/node/scripts/kubelet.sh similarity index 100% rename from cluster/centos/minion/scripts/kubelet.sh rename to cluster/centos/node/scripts/kubelet.sh diff --git a/cluster/centos/minion/scripts/proxy.sh b/cluster/centos/node/scripts/proxy.sh similarity index 100% rename from cluster/centos/minion/scripts/proxy.sh rename to cluster/centos/node/scripts/proxy.sh diff --git a/cluster/centos/util.sh b/cluster/centos/util.sh index 27dfc0f4ef1..9242d9b1904 100755 --- a/cluster/centos/util.sh +++ b/cluster/centos/util.sh @@ -264,14 +264,14 @@ function provision-minion() { ensure-setup-dir ${minion} # scp -r ${SSH_OPTS} minion config-default.sh copy-files.sh util.sh "${minion_ip}:${KUBE_TEMP}" - kube-scp ${minion} "${ROOT}/binaries/minion ${ROOT}/minion ${ROOT}/config-default.sh ${ROOT}/util.sh" ${KUBE_TEMP} + kube-scp ${minion} "${ROOT}/binaries/node ${ROOT}/node ${ROOT}/config-default.sh ${ROOT}/util.sh" ${KUBE_TEMP} kube-ssh "${minion}" " \ - sudo cp -r ${KUBE_TEMP}/minion/bin /opt/kubernetes; \ + sudo cp -r ${KUBE_TEMP}/node/bin /opt/kubernetes; \ sudo chmod -R +x /opt/kubernetes/bin; \ - sudo bash ${KUBE_TEMP}/minion/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \ - sudo bash ${KUBE_TEMP}/minion/scripts/docker.sh \"${DOCKER_OPTS}\"; \ - sudo bash ${KUBE_TEMP}/minion/scripts/kubelet.sh ${master_ip} ${minion_ip}; \ - sudo bash ${KUBE_TEMP}/minion/scripts/proxy.sh ${master_ip}" + sudo bash ${KUBE_TEMP}/node/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \ + sudo bash ${KUBE_TEMP}/node/scripts/docker.sh \"${DOCKER_OPTS}\"; \ + sudo bash ${KUBE_TEMP}/node/scripts/kubelet.sh ${master_ip} ${minion_ip}; \ + sudo bash ${KUBE_TEMP}/node/scripts/proxy.sh ${master_ip}" } # Create dirs that'll be used during setup on target machine. From 9e640b356c903cf651788fe97d7ee79289542000 Mon Sep 17 00:00:00 2001 From: Ewa Socala Date: Thu, 10 Sep 2015 13:50:49 +0200 Subject: [PATCH 39/46] Memory consumption added to Resource Consumer --- .../resource-consumer/resource_consumer_handler.go | 3 +-- test/images/resource-consumer/utils.go | 12 ++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/test/images/resource-consumer/resource_consumer_handler.go b/test/images/resource-consumer/resource_consumer_handler.go index 7bc7ee4e640..20292eb966d 100644 --- a/test/images/resource-consumer/resource_consumer_handler.go +++ b/test/images/resource-consumer/resource_consumer_handler.go @@ -104,8 +104,7 @@ func (handler ResourceConsumerHandler) handleConsumeMem(w http.ResponseWriter, q http.Error(w, incorrectFunctionArgument, http.StatusBadRequest) return } - ConsumeMem(megabytes, durationSec) - fmt.Fprintln(w, "Warning: not implemented!") + go ConsumeMem(megabytes, durationSec) fmt.Fprintln(w, consumeMemAddress[1:]) fmt.Fprintln(w, megabytes, megabytesQuery) fmt.Fprintln(w, durationSec, durationSecQuery) diff --git a/test/images/resource-consumer/utils.go b/test/images/resource-consumer/utils.go index 9557e49c952..3b3b646eb2a 100644 --- a/test/images/resource-consumer/utils.go +++ b/test/images/resource-consumer/utils.go @@ -20,9 +20,13 @@ import ( "fmt" "log" "os/exec" + "strconv" ) -const consumeCPUBinary = "./consume-cpu/consume-cpu" +const ( + consumeCPUBinary = "./consume-cpu/consume-cpu" + consumeMemBinary = "stress" +) func ConsumeCPU(millicores int, durationSec int) { log.Printf("ConsumeCPU millicores: %v, durationSec: %v", millicores, durationSec) @@ -35,7 +39,11 @@ func ConsumeCPU(millicores int, durationSec int) { func ConsumeMem(megabytes int, durationSec int) { log.Printf("ConsumeMem megabytes: %v, durationSec: %v", megabytes, durationSec) - // not implemented + megabytesString := strconv.Itoa(megabytes) + "M" + durationSecString := strconv.Itoa(durationSec) + // creating new consume memory process + consumeMem := exec.Command(consumeMemBinary, "-m", "1", "--vm-bytes", megabytesString, "--vm-hang", "0", "-t", durationSecString) + consumeMem.Start() } func GetCurrentStatus() { From b41862b6703bcf9036a61dc8453a5253534545c3 Mon Sep 17 00:00:00 2001 From: Jerzy Szczepkowski Date: Thu, 3 Sep 2015 10:55:26 +0200 Subject: [PATCH 40/46] Turning on pod autoscaler on GCE. Implemented optional turning on of pod autoscaler in kube-up script for GCE. --- cluster/gce/config-default.sh | 10 +++++++++- cluster/gce/config-test.sh | 11 ++++++++++- cluster/gce/configure-vm.sh | 6 ++++++ cluster/gce/coreos/helper.sh | 2 ++ cluster/gce/debian/helper.sh | 2 ++ cluster/gce/util.sh | 13 +++++++++++++ .../kube-controller-manager.manifest | 6 +++++- hack/verify-flags/exceptions.txt | 3 +-- 8 files changed, 48 insertions(+), 5 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index c0a18753c08..6aae9232ebb 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -44,6 +44,8 @@ MINION_TAG="${INSTANCE_PREFIX}-minion" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}" MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}" +RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" +ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}" # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 @@ -87,7 +89,6 @@ CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" # Optional: Create autoscaler for cluster's nodes. -# NOT WORKING YET! ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}" if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" @@ -95,6 +96,13 @@ if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" fi +# Optional: Enable feature for autoscaling number of pods +# Experimental feature, not ready for production use. +ENABLE_HORIZONTAL_POD_AUTOSCALER="${KUBE_ENABLE_HORIZONTAL_POD_AUTOSCALER:-false}" +if [[ "${ENABLE_HORIZONTAL_POD_AUTOSCALER}" == "true" ]]; then + ENABLE_EXPERIMENTAL_API=true +fi + # Admission Controllers to invoke prior to persisting objects in cluster ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index a45477a3e7d..ef6ffa6f44a 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -45,6 +45,9 @@ MINION_TAG="${INSTANCE_PREFIX}-minion" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.245.0.0/16}" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}" +RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" +ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}" + # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET @@ -92,7 +95,6 @@ CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" # Optional: Create autoscaler for cluster's nodes. -# NOT WORKING YET! ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}" if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" @@ -100,6 +102,13 @@ if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" fi +# Optional: Enable feature for autoscaling number of pods +# Experimental feature, not ready for production use. +ENABLE_HORIZONTAL_POD_AUTOSCALER="${KUBE_ENABLE_HORIZONTAL_POD_AUTOSCALER:-false}" +if [[ "${ENABLE_HORIZONTAL_POD_AUTOSCALER}" == "true" ]]; then + ENABLE_EXPERIMENTAL_API=true +fi + ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota # Optional: if set to true kube-up will automatically check for existing resources and clean them up. diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index c199a6643c5..c5ef423800a 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -277,6 +277,7 @@ dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")' +enable_horizontal_pod_autoscaler: '$(echo "$ENABLE_HORIZONTAL_POD_AUTOSCALER" | sed -e "s/'/''/g")' EOF if [ -n "${APISERVER_TEST_ARGS:-}" ]; then @@ -568,6 +569,11 @@ EOF # CIDR range. cat <>/etc/salt/minion.d/grains.conf cbr-cidr: ${MASTER_IP_RANGE} +EOF + fi + if [[ ! -z "${RUNTIME_CONFIG:-}" ]]; then + cat <>/etc/salt/minion.d/grains.conf + runtime_config: '$(echo "$RUNTIME_CONFIG" | sed -e "s/'/''/g")' EOF fi } diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index 5ef057acf4a..846bb2591b0 100755 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -54,6 +54,8 @@ KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) +ENABLE_HORIZONTAL_POD_AUTOSCALER: $(yaml-quote ${ENABLE_HORIZONTAL_POD_AUTOSCALER}) +RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME}) KUBERNETES_CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME}) RKT_VERSION: $(yaml-quote ${RKT_VERSION}) diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 6b929c2d9ee..694f29ec85e 100755 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -51,6 +51,8 @@ KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) +ENABLE_HORIZONTAL_POD_AUTOSCALER: $(yaml-quote ${ENABLE_HORIZONTAL_POD_AUTOSCALER}) +RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-}) KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-}) KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-}) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index c884c9c2195..604c855f236 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -53,6 +53,18 @@ function join_csv { # Verify prereqs function verify-prereqs { + if [[ "${ENABLE_EXPERIMENTAL_API}" == "true" ]]; then + if [[ -z "${RUNTIME_CONFIG}" ]]; then + RUNTIME_CONFIG="experimental/v1=true" + else + # TODO: add checking if RUNTIME_CONFIG contains "experimental/v1=false" and appending "experimental/v1=true" if not. + if echo "${RUNTIME_CONFIG}" | grep -q -v "experimental/v1=true"; then + echo "Experimental API should be turned on, but is not turned on in RUNTIME_CONFIG!" + exit 1 + fi + fi + fi + local cmd for cmd in gcloud gsutil; do if ! which "${cmd}" >/dev/null; then @@ -465,6 +477,7 @@ function write-master-env { if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then KUBELET_APISERVER="${MASTER_NAME}" fi + build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" } diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index 7c0e214213a..a4f82888a01 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -1,6 +1,7 @@ {% set cluster_name = "" -%} {% set cluster_cidr = "" -%} {% set allocate_node_cidrs = "" -%} +{% set enable_horizontal_pod_autoscaler = "" -%} {% if pillar['instance_prefix'] is defined -%} {% set cluster_name = "--cluster-name=" + pillar['instance_prefix'] -%} @@ -11,6 +12,9 @@ {% if pillar['allocate_node_cidrs'] is defined -%} {% set allocate_node_cidrs = "--allocate-node-cidrs=" + pillar['allocate_node_cidrs'] -%} {% endif -%} +{% if pillar['enable_horizontal_pod_autoscaler'] is defined -%} + {% set enable_horizontal_pod_autoscaler = "--enable-horizontal-pod-autoscaler=" + pillar['enable_horizontal_pod_autoscaler'] -%} +{% endif -%} {% set cloud_provider = "" -%} {% set cloud_config = "" -%} @@ -34,7 +38,7 @@ {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%} {% endif -%} -{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} +{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + enable_horizontal_pod_autoscaler + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration {% if pillar['controller_manager_test_args'] is defined -%} diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index a8816eff7e5..c541cc21188 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -1,4 +1,3 @@ -cluster/addons/cluster-monitoring/README.md:Heapster enables monitoring of Kubernetes Clusters using [cAdvisor](https://github.com/google/cadvisor). The kubelet will communicate with an instance of cAdvisor running on localhost and proxy container stats to Heapster. Kubelet will attempt to connect to cAdvisor on port 4194 by default but this port can be configured with kubelet's `--cadvisor-port` run flag. Detailed information about heapster can be found [here](https://github.com/GoogleCloudPlatform/heapster). cluster/addons/registry/images/Dockerfile:ADD run_proxy.sh /usr/bin/run_proxy cluster/addons/registry/images/Dockerfile:CMD ["/usr/bin/run_proxy"] cluster/aws/templates/salt-minion.sh:# We set the hostname_override to the full EC2 private dns name @@ -39,7 +38,7 @@ cluster/saltbase/salt/kube-addons/kube-addons.sh:# Create admission_control obje cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + etcd_servers + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + " " + basic_auth_file + " " + min_request_timeout -%} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = params + " " + cluster_name + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + " " + token_auth_file + " " + bind_address + " " + pillar['log_level'] + " " + advertise_address + " " + proxy_ssh_options -%} -cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} +cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + enable_horizontal_pod_autoscaler + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers + ":7080" -%} From bf881f187e29d111c0001777d84ac5ace0def031 Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Tue, 8 Sep 2015 16:58:25 -0700 Subject: [PATCH 41/46] rename expapi.Daemon to expapi.DaemonSet --- pkg/client/unversioned/cache/listers.go | 42 +++---- pkg/client/unversioned/cache/listers_test.go | 76 ++++++------ pkg/client/unversioned/daemon.go | 95 --------------- pkg/client/unversioned/daemon_sets.go | 95 +++++++++++++++ .../{daemon_test.go => daemon_sets_test.go} | 66 +++++----- pkg/client/unversioned/experimental.go | 6 +- .../testclient/fake_daemon_sets.go | 76 ++++++++++++ .../unversioned/testclient/fake_daemons.go | 76 ------------ .../unversioned/testclient/testclient.go | 4 +- .../namespace/namespace_controller.go | 4 +- .../namespace/namespace_controller_test.go | 2 +- pkg/expapi/deep_copy_generated.go | 24 ++-- pkg/expapi/register.go | 8 +- pkg/expapi/types.go | 42 +++---- pkg/expapi/v1/conversion_generated.go | 64 +++++----- pkg/expapi/v1/deep_copy_generated.go | 24 ++-- pkg/expapi/v1/defaults.go | 2 +- pkg/expapi/v1/defaults_test.go | 26 ++-- pkg/expapi/v1/register.go | 8 +- pkg/expapi/v1/types.go | 42 +++---- pkg/expapi/v1/types_swagger_doc_generated.go | 48 ++++---- pkg/expapi/validation/validation.go | 26 ++-- pkg/expapi/validation/validation_test.go | 114 +++++++++--------- pkg/master/master.go | 6 +- pkg/registry/{daemon => daemonset}/doc.go | 6 +- .../{daemon => daemonset}/etcd/etcd.go | 31 +++-- .../{daemon => daemonset}/etcd/etcd_test.go | 40 +++--- .../{daemon => daemonset}/strategy.go | 72 +++++------ 28 files changed, 562 insertions(+), 563 deletions(-) delete mode 100644 pkg/client/unversioned/daemon.go create mode 100644 pkg/client/unversioned/daemon_sets.go rename pkg/client/unversioned/{daemon_test.go => daemon_sets_test.go} (63%) create mode 100644 pkg/client/unversioned/testclient/fake_daemon_sets.go delete mode 100644 pkg/client/unversioned/testclient/fake_daemons.go rename pkg/registry/{daemon => daemonset}/doc.go (80%) rename pkg/registry/{daemon => daemonset}/etcd/etcd.go (72%) rename pkg/registry/{daemon => daemonset}/etcd/etcd_test.go (84%) rename pkg/registry/{daemon => daemonset}/strategy.go (53%) diff --git a/pkg/client/unversioned/cache/listers.go b/pkg/client/unversioned/cache/listers.go index 0f92daf59c0..cba3e5b3003 100644 --- a/pkg/client/unversioned/cache/listers.go +++ b/pkg/client/unversioned/cache/listers.go @@ -221,59 +221,59 @@ func (s *StoreToReplicationControllerLister) GetPodControllers(pod *api.Pod) (co controllers = append(controllers, rc) } if len(controllers) == 0 { - err = fmt.Errorf("Could not find controllers for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels) + err = fmt.Errorf("Could not find daemon set for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels) } return } -// StoreToDaemonLister gives a store List and Exists methods. The store must contain only Daemons. -type StoreToDaemonLister struct { +// StoreToDaemonSetLister gives a store List and Exists methods. The store must contain only DaemonSets. +type StoreToDaemonSetLister struct { Store } -// Exists checks if the given dc exists in the store. -func (s *StoreToDaemonLister) Exists(daemon *expapi.Daemon) (bool, error) { - _, exists, err := s.Store.Get(daemon) +// Exists checks if the given daemon set exists in the store. +func (s *StoreToDaemonSetLister) Exists(ds *expapi.DaemonSet) (bool, error) { + _, exists, err := s.Store.Get(ds) if err != nil { return false, err } return exists, nil } -// StoreToDaemonLister lists all daemons in the store. +// List lists all daemon sets in the store. // TODO: converge on the interface in pkg/client -func (s *StoreToDaemonLister) List() (daemons []expapi.Daemon, err error) { +func (s *StoreToDaemonSetLister) List() (dss []expapi.DaemonSet, err error) { for _, c := range s.Store.List() { - daemons = append(daemons, *(c.(*expapi.Daemon))) + dss = append(dss, *(c.(*expapi.DaemonSet))) } - return daemons, nil + return dss, nil } -// GetPodDaemons returns a list of daemons managing a pod. Returns an error iff no matching daemons are found. -func (s *StoreToDaemonLister) GetPodDaemons(pod *api.Pod) (daemons []expapi.Daemon, err error) { +// GetPodDaemonSets returns a list of daemon sets managing a pod. Returns an error iff no matching daemon sets are found. +func (s *StoreToDaemonSetLister) GetPodDaemonSets(pod *api.Pod) (daemonSets []expapi.DaemonSet, err error) { var selector labels.Selector - var daemonController expapi.Daemon + var daemonSet expapi.DaemonSet if len(pod.Labels) == 0 { - err = fmt.Errorf("No daemons found for pod %v because it has no labels", pod.Name) + err = fmt.Errorf("No daemon sets found for pod %v because it has no labels", pod.Name) return } for _, m := range s.Store.List() { - daemonController = *m.(*expapi.Daemon) - if daemonController.Namespace != pod.Namespace { + daemonSet = *m.(*expapi.DaemonSet) + if daemonSet.Namespace != pod.Namespace { continue } - selector = labels.Set(daemonController.Spec.Selector).AsSelector() + selector = labels.Set(daemonSet.Spec.Selector).AsSelector() - // If a daemonController with a nil or empty selector creeps in, it should match nothing, not everything. + // If a daemonSet with a nil or empty selector creeps in, it should match nothing, not everything. if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) { continue } - daemons = append(daemons, daemonController) + daemonSets = append(daemonSets, daemonSet) } - if len(daemons) == 0 { - err = fmt.Errorf("Could not find daemons for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels) + if len(daemonSets) == 0 { + err = fmt.Errorf("Could not find daemon set for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels) } return } diff --git a/pkg/client/unversioned/cache/listers_test.go b/pkg/client/unversioned/cache/listers_test.go index 112b10416d2..2ddd427539b 100644 --- a/pkg/client/unversioned/cache/listers_test.go +++ b/pkg/client/unversioned/cache/listers_test.go @@ -156,64 +156,64 @@ func TestStoreToReplicationControllerLister(t *testing.T) { } } -func TestStoreToDaemonLister(t *testing.T) { +func TestStoreToDaemonSetLister(t *testing.T) { store := NewStore(MetaNamespaceKeyFunc) - lister := StoreToDaemonLister{store} + lister := StoreToDaemonSetLister{store} testCases := []struct { - inDCs []*expapi.Daemon - list func() ([]expapi.Daemon, error) - outDCNames util.StringSet - expectErr bool + inDSs []*expapi.DaemonSet + list func() ([]expapi.DaemonSet, error) + outDaemonSetNames util.StringSet + expectErr bool }{ // Basic listing { - inDCs: []*expapi.Daemon{ + inDSs: []*expapi.DaemonSet{ {ObjectMeta: api.ObjectMeta{Name: "basic"}}, }, - list: func() ([]expapi.Daemon, error) { + list: func() ([]expapi.DaemonSet, error) { return lister.List() }, - outDCNames: util.NewStringSet("basic"), + outDaemonSetNames: util.NewStringSet("basic"), }, - // Listing multiple controllers + // Listing multiple daemon sets { - inDCs: []*expapi.Daemon{ + inDSs: []*expapi.DaemonSet{ {ObjectMeta: api.ObjectMeta{Name: "basic"}}, {ObjectMeta: api.ObjectMeta{Name: "complex"}}, {ObjectMeta: api.ObjectMeta{Name: "complex2"}}, }, - list: func() ([]expapi.Daemon, error) { + list: func() ([]expapi.DaemonSet, error) { return lister.List() }, - outDCNames: util.NewStringSet("basic", "complex", "complex2"), + outDaemonSetNames: util.NewStringSet("basic", "complex", "complex2"), }, // No pod labels { - inDCs: []*expapi.Daemon{ + inDSs: []*expapi.DaemonSet{ { ObjectMeta: api.ObjectMeta{Name: "basic", Namespace: "ns"}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: map[string]string{"foo": "baz"}, }, }, }, - list: func() ([]expapi.Daemon, error) { + list: func() ([]expapi.DaemonSet, error) { pod := &api.Pod{ ObjectMeta: api.ObjectMeta{Name: "pod1", Namespace: "ns"}, } - return lister.GetPodDaemons(pod) + return lister.GetPodDaemonSets(pod) }, - outDCNames: util.NewStringSet(), - expectErr: true, + outDaemonSetNames: util.NewStringSet(), + expectErr: true, }, - // No RC selectors + // No DS selectors { - inDCs: []*expapi.Daemon{ + inDSs: []*expapi.DaemonSet{ { ObjectMeta: api.ObjectMeta{Name: "basic", Namespace: "ns"}, }, }, - list: func() ([]expapi.Daemon, error) { + list: func() ([]expapi.DaemonSet, error) { pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: "pod1", @@ -221,28 +221,28 @@ func TestStoreToDaemonLister(t *testing.T) { Labels: map[string]string{"foo": "bar"}, }, } - return lister.GetPodDaemons(pod) + return lister.GetPodDaemonSets(pod) }, - outDCNames: util.NewStringSet(), - expectErr: true, + outDaemonSetNames: util.NewStringSet(), + expectErr: true, }, // Matching labels to selectors and namespace { - inDCs: []*expapi.Daemon{ + inDSs: []*expapi.DaemonSet{ { ObjectMeta: api.ObjectMeta{Name: "foo"}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: map[string]string{"foo": "bar"}, }, }, { ObjectMeta: api.ObjectMeta{Name: "bar", Namespace: "ns"}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: map[string]string{"foo": "bar"}, }, }, }, - list: func() ([]expapi.Daemon, error) { + list: func() ([]expapi.DaemonSet, error) { pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: "pod1", @@ -250,17 +250,17 @@ func TestStoreToDaemonLister(t *testing.T) { Namespace: "ns", }, } - return lister.GetPodDaemons(pod) + return lister.GetPodDaemonSets(pod) }, - outDCNames: util.NewStringSet("bar"), + outDaemonSetNames: util.NewStringSet("bar"), }, } for _, c := range testCases { - for _, r := range c.inDCs { + for _, r := range c.inDSs { store.Add(r) } - gotControllers, err := c.list() + daemonSets, err := c.list() if err != nil && c.expectErr { continue } else if c.expectErr { @@ -268,12 +268,12 @@ func TestStoreToDaemonLister(t *testing.T) { } else if err != nil { t.Fatalf("Unexpected error %#v", err) } - gotNames := make([]string, len(gotControllers)) - for ix := range gotControllers { - gotNames[ix] = gotControllers[ix].Name + daemonSetNames := make([]string, len(daemonSets)) + for ix := range daemonSets { + daemonSetNames[ix] = daemonSets[ix].Name } - if !c.outDCNames.HasAll(gotNames...) || len(gotNames) != len(c.outDCNames) { - t.Errorf("Unexpected got controllers %+v expected %+v", gotNames, c.outDCNames) + if !c.outDaemonSetNames.HasAll(daemonSetNames...) || len(daemonSetNames) != len(c.outDaemonSetNames) { + t.Errorf("Unexpected got controllers %+v expected %+v", daemonSetNames, c.outDaemonSetNames) } } } diff --git a/pkg/client/unversioned/daemon.go b/pkg/client/unversioned/daemon.go deleted file mode 100644 index 3b99fcdd3b0..00000000000 --- a/pkg/client/unversioned/daemon.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package unversioned - -import ( - "k8s.io/kubernetes/pkg/expapi" - "k8s.io/kubernetes/pkg/fields" - "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/watch" -) - -// DaemonsNamespacer has methods to work with Daemon resources in a namespace -type DaemonsNamespacer interface { - Daemons(namespace string) DaemonInterface -} - -type DaemonInterface interface { - List(selector labels.Selector) (*expapi.DaemonList, error) - Get(name string) (*expapi.Daemon, error) - Create(ctrl *expapi.Daemon) (*expapi.Daemon, error) - Update(ctrl *expapi.Daemon) (*expapi.Daemon, error) - Delete(name string) error - Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) -} - -// daemons implements DaemonsNamespacer interface -type daemons struct { - r *ExperimentalClient - ns string -} - -func newDaemons(c *ExperimentalClient, namespace string) *daemons { - return &daemons{c, namespace} -} - -// Ensure statically that daemons implements DaemonInterface. -var _ DaemonInterface = &daemons{} - -func (c *daemons) List(selector labels.Selector) (result *expapi.DaemonList, err error) { - result = &expapi.DaemonList{} - err = c.r.Get().Namespace(c.ns).Resource("daemons").LabelsSelectorParam(selector).Do().Into(result) - return -} - -// Get returns information about a particular daemon. -func (c *daemons) Get(name string) (result *expapi.Daemon, err error) { - result = &expapi.Daemon{} - err = c.r.Get().Namespace(c.ns).Resource("daemons").Name(name).Do().Into(result) - return -} - -// Create creates a new daemon. -func (c *daemons) Create(daemon *expapi.Daemon) (result *expapi.Daemon, err error) { - result = &expapi.Daemon{} - err = c.r.Post().Namespace(c.ns).Resource("daemons").Body(daemon).Do().Into(result) - return -} - -// Update updates an existing daemon. -func (c *daemons) Update(daemon *expapi.Daemon) (result *expapi.Daemon, err error) { - result = &expapi.Daemon{} - err = c.r.Put().Namespace(c.ns).Resource("daemons").Name(daemon.Name).Body(daemon).Do().Into(result) - return -} - -// Delete deletes an existing daemon. -func (c *daemons) Delete(name string) error { - return c.r.Delete().Namespace(c.ns).Resource("daemons").Name(name).Do().Error() -} - -// Watch returns a watch.Interface that watches the requested daemons. -func (c *daemons) Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { - return c.r.Get(). - Prefix("watch"). - Namespace(c.ns). - Resource("daemons"). - Param("resourceVersion", resourceVersion). - LabelsSelectorParam(label). - FieldsSelectorParam(field). - Watch() -} diff --git a/pkg/client/unversioned/daemon_sets.go b/pkg/client/unversioned/daemon_sets.go new file mode 100644 index 00000000000..321b34cc465 --- /dev/null +++ b/pkg/client/unversioned/daemon_sets.go @@ -0,0 +1,95 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unversioned + +import ( + "k8s.io/kubernetes/pkg/expapi" + "k8s.io/kubernetes/pkg/fields" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/watch" +) + +// DaemonsSetsNamespacer has methods to work with DaemonSet resources in a namespace +type DaemonSetsNamespacer interface { + DaemonSets(namespace string) DaemonSetInterface +} + +type DaemonSetInterface interface { + List(selector labels.Selector) (*expapi.DaemonSetList, error) + Get(name string) (*expapi.DaemonSet, error) + Create(ctrl *expapi.DaemonSet) (*expapi.DaemonSet, error) + Update(ctrl *expapi.DaemonSet) (*expapi.DaemonSet, error) + Delete(name string) error + Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) +} + +// daemonSets implements DaemonsSetsNamespacer interface +type daemonSets struct { + r *ExperimentalClient + ns string +} + +func newDaemonSets(c *ExperimentalClient, namespace string) *daemonSets { + return &daemonSets{c, namespace} +} + +// Ensure statically that daemonSets implements DaemonSetsInterface. +var _ DaemonSetInterface = &daemonSets{} + +func (c *daemonSets) List(selector labels.Selector) (result *expapi.DaemonSetList, err error) { + result = &expapi.DaemonSetList{} + err = c.r.Get().Namespace(c.ns).Resource("daemonsets").LabelsSelectorParam(selector).Do().Into(result) + return +} + +// Get returns information about a particular daemon set. +func (c *daemonSets) Get(name string) (result *expapi.DaemonSet, err error) { + result = &expapi.DaemonSet{} + err = c.r.Get().Namespace(c.ns).Resource("daemonsets").Name(name).Do().Into(result) + return +} + +// Create creates a new daemon set. +func (c *daemonSets) Create(daemon *expapi.DaemonSet) (result *expapi.DaemonSet, err error) { + result = &expapi.DaemonSet{} + err = c.r.Post().Namespace(c.ns).Resource("daemonsets").Body(daemon).Do().Into(result) + return +} + +// Update updates an existing daemon set. +func (c *daemonSets) Update(daemon *expapi.DaemonSet) (result *expapi.DaemonSet, err error) { + result = &expapi.DaemonSet{} + err = c.r.Put().Namespace(c.ns).Resource("daemonsets").Name(daemon.Name).Body(daemon).Do().Into(result) + return +} + +// Delete deletes an existing daemon set. +func (c *daemonSets) Delete(name string) error { + return c.r.Delete().Namespace(c.ns).Resource("daemonsets").Name(name).Do().Error() +} + +// Watch returns a watch.Interface that watches the requested daemon sets. +func (c *daemonSets) Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { + return c.r.Get(). + Prefix("watch"). + Namespace(c.ns). + Resource("daemonsets"). + Param("resourceVersion", resourceVersion). + LabelsSelectorParam(label). + FieldsSelectorParam(field). + Watch() +} diff --git a/pkg/client/unversioned/daemon_test.go b/pkg/client/unversioned/daemon_sets_test.go similarity index 63% rename from pkg/client/unversioned/daemon_test.go rename to pkg/client/unversioned/daemon_sets_test.go index 0f1f96b1cda..6df19940149 100644 --- a/pkg/client/unversioned/daemon_test.go +++ b/pkg/client/unversioned/daemon_sets_test.go @@ -25,20 +25,20 @@ import ( "k8s.io/kubernetes/pkg/labels" ) -func getDCResourceName() string { - return "daemons" +func getDSResourceName() string { + return "daemonsets" } -func TestListDaemons(t *testing.T) { +func TestListDaemonSets(t *testing.T) { ns := api.NamespaceAll c := &testClient{ Request: testRequest{ Method: "GET", - Path: testapi.Experimental.ResourcePath(getDCResourceName(), ns, ""), + Path: testapi.Experimental.ResourcePath(getDSResourceName(), ns, ""), }, Response: Response{StatusCode: 200, - Body: &expapi.DaemonList{ - Items: []expapi.Daemon{ + Body: &expapi.DaemonSetList{ + Items: []expapi.DaemonSet{ { ObjectMeta: api.ObjectMeta{ Name: "foo", @@ -47,7 +47,7 @@ func TestListDaemons(t *testing.T) { "name": "baz", }, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Template: &api.PodTemplateSpec{}, }, }, @@ -55,18 +55,18 @@ func TestListDaemons(t *testing.T) { }, }, } - receivedControllerList, err := c.Setup(t).Experimental().Daemons(ns).List(labels.Everything()) - c.Validate(t, receivedControllerList, err) + receivedDSs, err := c.Setup(t).Experimental().DaemonSets(ns).List(labels.Everything()) + c.Validate(t, receivedDSs, err) } -func TestGetDaemon(t *testing.T) { +func TestGetDaemonSet(t *testing.T) { ns := api.NamespaceDefault c := &testClient{ - Request: testRequest{Method: "GET", Path: testapi.Experimental.ResourcePath(getDCResourceName(), ns, "foo"), Query: buildQueryValues(nil)}, + Request: testRequest{Method: "GET", Path: testapi.Experimental.ResourcePath(getDSResourceName(), ns, "foo"), Query: buildQueryValues(nil)}, Response: Response{ StatusCode: 200, - Body: &expapi.Daemon{ + Body: &expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{ Name: "foo", Labels: map[string]string{ @@ -74,20 +74,20 @@ func TestGetDaemon(t *testing.T) { "name": "baz", }, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Template: &api.PodTemplateSpec{}, }, }, }, } - receivedController, err := c.Setup(t).Experimental().Daemons(ns).Get("foo") - c.Validate(t, receivedController, err) + receivedDaemonSet, err := c.Setup(t).Experimental().DaemonSets(ns).Get("foo") + c.Validate(t, receivedDaemonSet, err) } -func TestGetDaemonWithNoName(t *testing.T) { +func TestGetDaemonSetWithNoName(t *testing.T) { ns := api.NamespaceDefault c := &testClient{Error: true} - receivedPod, err := c.Setup(t).Experimental().Daemons(ns).Get("") + receivedPod, err := c.Setup(t).Experimental().DaemonSets(ns).Get("") if (err != nil) && (err.Error() != nameRequiredError) { t.Errorf("Expected error: %v, but got %v", nameRequiredError, err) } @@ -95,16 +95,16 @@ func TestGetDaemonWithNoName(t *testing.T) { c.Validate(t, receivedPod, err) } -func TestUpdateDaemon(t *testing.T) { +func TestUpdateDaemonSet(t *testing.T) { ns := api.NamespaceDefault - requestController := &expapi.Daemon{ + requestDaemonSet := &expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "foo", ResourceVersion: "1"}, } c := &testClient{ - Request: testRequest{Method: "PUT", Path: testapi.Experimental.ResourcePath(getDCResourceName(), ns, "foo"), Query: buildQueryValues(nil)}, + Request: testRequest{Method: "PUT", Path: testapi.Experimental.ResourcePath(getDSResourceName(), ns, "foo"), Query: buildQueryValues(nil)}, Response: Response{ StatusCode: 200, - Body: &expapi.Daemon{ + Body: &expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{ Name: "foo", Labels: map[string]string{ @@ -112,36 +112,36 @@ func TestUpdateDaemon(t *testing.T) { "name": "baz", }, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Template: &api.PodTemplateSpec{}, }, }, }, } - receivedController, err := c.Setup(t).Experimental().Daemons(ns).Update(requestController) - c.Validate(t, receivedController, err) + receivedDaemonSet, err := c.Setup(t).Experimental().DaemonSets(ns).Update(requestDaemonSet) + c.Validate(t, receivedDaemonSet, err) } func TestDeleteDaemon(t *testing.T) { ns := api.NamespaceDefault c := &testClient{ - Request: testRequest{Method: "DELETE", Path: testapi.Experimental.ResourcePath(getDCResourceName(), ns, "foo"), Query: buildQueryValues(nil)}, + Request: testRequest{Method: "DELETE", Path: testapi.Experimental.ResourcePath(getDSResourceName(), ns, "foo"), Query: buildQueryValues(nil)}, Response: Response{StatusCode: 200}, } - err := c.Setup(t).Experimental().Daemons(ns).Delete("foo") + err := c.Setup(t).Experimental().DaemonSets(ns).Delete("foo") c.Validate(t, nil, err) } -func TestCreateDaemon(t *testing.T) { +func TestCreateDaemonSet(t *testing.T) { ns := api.NamespaceDefault - requestController := &expapi.Daemon{ + requestDaemonSet := &expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "foo"}, } c := &testClient{ - Request: testRequest{Method: "POST", Path: testapi.Experimental.ResourcePath(getDCResourceName(), ns, ""), Body: requestController, Query: buildQueryValues(nil)}, + Request: testRequest{Method: "POST", Path: testapi.Experimental.ResourcePath(getDSResourceName(), ns, ""), Body: requestDaemonSet, Query: buildQueryValues(nil)}, Response: Response{ StatusCode: 200, - Body: &expapi.Daemon{ + Body: &expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{ Name: "foo", Labels: map[string]string{ @@ -149,12 +149,12 @@ func TestCreateDaemon(t *testing.T) { "name": "baz", }, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Template: &api.PodTemplateSpec{}, }, }, }, } - receivedController, err := c.Setup(t).Experimental().Daemons(ns).Create(requestController) - c.Validate(t, receivedController, err) + receivedDaemonSet, err := c.Setup(t).Experimental().DaemonSets(ns).Create(requestDaemonSet) + c.Validate(t, receivedDaemonSet, err) } diff --git a/pkg/client/unversioned/experimental.go b/pkg/client/unversioned/experimental.go index bd30ace7d8b..f23083bd571 100644 --- a/pkg/client/unversioned/experimental.go +++ b/pkg/client/unversioned/experimental.go @@ -34,7 +34,7 @@ type ExperimentalInterface interface { VersionInterface HorizontalPodAutoscalersNamespacer ScaleNamespacer - DaemonsNamespacer + DaemonSetsNamespacer DeploymentsNamespacer } @@ -82,8 +82,8 @@ func (c *ExperimentalClient) Scales(namespace string) ScaleInterface { return newScales(c, namespace) } -func (c *ExperimentalClient) Daemons(namespace string) DaemonInterface { - return newDaemons(c, namespace) +func (c *ExperimentalClient) DaemonSets(namespace string) DaemonSetInterface { + return newDaemonSets(c, namespace) } func (c *ExperimentalClient) Deployments(namespace string) DeploymentInterface { diff --git a/pkg/client/unversioned/testclient/fake_daemon_sets.go b/pkg/client/unversioned/testclient/fake_daemon_sets.go new file mode 100644 index 00000000000..f50ec708488 --- /dev/null +++ b/pkg/client/unversioned/testclient/fake_daemon_sets.go @@ -0,0 +1,76 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testclient + +import ( + kClientLib "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/expapi" + "k8s.io/kubernetes/pkg/fields" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/watch" +) + +// FakeDaemonSet implements DaemonInterface. Meant to be embedded into a struct to get a default +// implementation. This makes faking out just the method you want to test easier. +type FakeDaemonSets struct { + Fake *FakeExperimental + Namespace string +} + +// Ensure statically that FakeDaemonSets implements DaemonInterface. +var _ kClientLib.DaemonSetInterface = &FakeDaemonSets{} + +func (c *FakeDaemonSets) Get(name string) (*expapi.DaemonSet, error) { + obj, err := c.Fake.Invokes(NewGetAction("daemonsets", c.Namespace, name), &expapi.DaemonSet{}) + if obj == nil { + return nil, err + } + return obj.(*expapi.DaemonSet), err +} + +func (c *FakeDaemonSets) List(label labels.Selector) (*expapi.DaemonSetList, error) { + obj, err := c.Fake.Invokes(NewListAction("daemonsets", c.Namespace, label, nil), &expapi.DaemonSetList{}) + if obj == nil { + return nil, err + } + return obj.(*expapi.DaemonSetList), err +} + +func (c *FakeDaemonSets) Create(daemon *expapi.DaemonSet) (*expapi.DaemonSet, error) { + obj, err := c.Fake.Invokes(NewCreateAction("daemonsets", c.Namespace, daemon), &expapi.DaemonSet{}) + if obj == nil { + return nil, err + } + return obj.(*expapi.DaemonSet), err +} + +func (c *FakeDaemonSets) Update(daemon *expapi.DaemonSet) (*expapi.DaemonSet, error) { + obj, err := c.Fake.Invokes(NewUpdateAction("daemonsets", c.Namespace, daemon), &expapi.DaemonSet{}) + if obj == nil { + return nil, err + } + return obj.(*expapi.DaemonSet), err +} + +func (c *FakeDaemonSets) Delete(name string) error { + _, err := c.Fake.Invokes(NewDeleteAction("daemonsets", c.Namespace, name), &expapi.DaemonSet{}) + return err +} + +func (c *FakeDaemonSets) Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { + return c.Fake.InvokesWatch(NewWatchAction("daemonsets", c.Namespace, label, field, resourceVersion)) +} diff --git a/pkg/client/unversioned/testclient/fake_daemons.go b/pkg/client/unversioned/testclient/fake_daemons.go deleted file mode 100644 index 8b0b3bc8014..00000000000 --- a/pkg/client/unversioned/testclient/fake_daemons.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testclient - -import ( - kClientLib "k8s.io/kubernetes/pkg/client/unversioned" - "k8s.io/kubernetes/pkg/expapi" - "k8s.io/kubernetes/pkg/fields" - "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/watch" -) - -// FakeDaemons implements DaemonInterface. Meant to be embedded into a struct to get a default -// implementation. This makes faking out just the method you want to test easier. -type FakeDaemons struct { - Fake *FakeExperimental - Namespace string -} - -// Ensure statically that FakeDaemons implements DaemonInterface. -var _ kClientLib.DaemonInterface = &FakeDaemons{} - -func (c *FakeDaemons) Get(name string) (*expapi.Daemon, error) { - obj, err := c.Fake.Invokes(NewGetAction("daemons", c.Namespace, name), &expapi.Daemon{}) - if obj == nil { - return nil, err - } - return obj.(*expapi.Daemon), err -} - -func (c *FakeDaemons) List(label labels.Selector) (*expapi.DaemonList, error) { - obj, err := c.Fake.Invokes(NewListAction("daemons", c.Namespace, label, nil), &expapi.DaemonList{}) - if obj == nil { - return nil, err - } - return obj.(*expapi.DaemonList), err -} - -func (c *FakeDaemons) Create(daemon *expapi.Daemon) (*expapi.Daemon, error) { - obj, err := c.Fake.Invokes(NewCreateAction("daemons", c.Namespace, daemon), &expapi.Daemon{}) - if obj == nil { - return nil, err - } - return obj.(*expapi.Daemon), err -} - -func (c *FakeDaemons) Update(daemon *expapi.Daemon) (*expapi.Daemon, error) { - obj, err := c.Fake.Invokes(NewUpdateAction("daemons", c.Namespace, daemon), &expapi.Daemon{}) - if obj == nil { - return nil, err - } - return obj.(*expapi.Daemon), err -} - -func (c *FakeDaemons) Delete(name string) error { - _, err := c.Fake.Invokes(NewDeleteAction("daemons", c.Namespace, name), &expapi.Daemon{}) - return err -} - -func (c *FakeDaemons) Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) { - return c.Fake.InvokesWatch(NewWatchAction("daemons", c.Namespace, label, field, resourceVersion)) -} diff --git a/pkg/client/unversioned/testclient/testclient.go b/pkg/client/unversioned/testclient/testclient.go index 9949a0d1985..6bb59bfc9e1 100644 --- a/pkg/client/unversioned/testclient/testclient.go +++ b/pkg/client/unversioned/testclient/testclient.go @@ -246,8 +246,8 @@ type FakeExperimental struct { *Fake } -func (c *FakeExperimental) Daemons(namespace string) client.DaemonInterface { - return &FakeDaemons{Fake: c, Namespace: namespace} +func (c *FakeExperimental) DaemonSets(namespace string) client.DaemonSetInterface { + return &FakeDaemonSets{Fake: c, Namespace: namespace} } func (c *FakeExperimental) HorizontalPodAutoscalers(namespace string) client.HorizontalPodAutoscalerInterface { diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go index 9cbc79c7dd4..ec972ef6840 100644 --- a/pkg/controller/namespace/namespace_controller.go +++ b/pkg/controller/namespace/namespace_controller.go @@ -419,12 +419,12 @@ func deleteHorizontalPodAutoscalers(expClient client.ExperimentalInterface, ns s } func deleteDaemons(expClient client.ExperimentalInterface, ns string) error { - items, err := expClient.Daemons(ns).List(labels.Everything()) + items, err := expClient.DaemonSets(ns).List(labels.Everything()) if err != nil { return err } for i := range items.Items { - err := expClient.Daemons(ns).Delete(items.Items[i].Name) + err := expClient.DaemonSets(ns).Delete(items.Items[i].Name) if err != nil && !errors.IsNotFound(err) { return err } diff --git a/pkg/controller/namespace/namespace_controller_test.go b/pkg/controller/namespace/namespace_controller_test.go index d1807e4e5df..dd1c78701e0 100644 --- a/pkg/controller/namespace/namespace_controller_test.go +++ b/pkg/controller/namespace/namespace_controller_test.go @@ -107,7 +107,7 @@ func testSyncNamespaceThatIsTerminating(t *testing.T, experimentalMode bool) { if experimentalMode { expectedActionSet.Insert( strings.Join([]string{"list", "horizontalpodautoscalers", ""}, "-"), - strings.Join([]string{"list", "daemons", ""}, "-"), + strings.Join([]string{"list", "daemonsets", ""}, "-"), strings.Join([]string{"list", "deployments", ""}, "-"), ) } diff --git a/pkg/expapi/deep_copy_generated.go b/pkg/expapi/deep_copy_generated.go index 648d3ae414c..1879cc8e3ac 100644 --- a/pkg/expapi/deep_copy_generated.go +++ b/pkg/expapi/deep_copy_generated.go @@ -763,23 +763,23 @@ func deepCopy_expapi_APIVersion(in APIVersion, out *APIVersion, c *conversion.Cl return nil } -func deepCopy_expapi_Daemon(in Daemon, out *Daemon, c *conversion.Cloner) error { +func deepCopy_expapi_DaemonSet(in DaemonSet, out *DaemonSet, c *conversion.Cloner) error { if err := deepCopy_api_TypeMeta(in.TypeMeta, &out.TypeMeta, c); err != nil { return err } if err := deepCopy_api_ObjectMeta(in.ObjectMeta, &out.ObjectMeta, c); err != nil { return err } - if err := deepCopy_expapi_DaemonSpec(in.Spec, &out.Spec, c); err != nil { + if err := deepCopy_expapi_DaemonSetSpec(in.Spec, &out.Spec, c); err != nil { return err } - if err := deepCopy_expapi_DaemonStatus(in.Status, &out.Status, c); err != nil { + if err := deepCopy_expapi_DaemonSetStatus(in.Status, &out.Status, c); err != nil { return err } return nil } -func deepCopy_expapi_DaemonList(in DaemonList, out *DaemonList, c *conversion.Cloner) error { +func deepCopy_expapi_DaemonSetList(in DaemonSetList, out *DaemonSetList, c *conversion.Cloner) error { if err := deepCopy_api_TypeMeta(in.TypeMeta, &out.TypeMeta, c); err != nil { return err } @@ -787,9 +787,9 @@ func deepCopy_expapi_DaemonList(in DaemonList, out *DaemonList, c *conversion.Cl return err } if in.Items != nil { - out.Items = make([]Daemon, len(in.Items)) + out.Items = make([]DaemonSet, len(in.Items)) for i := range in.Items { - if err := deepCopy_expapi_Daemon(in.Items[i], &out.Items[i], c); err != nil { + if err := deepCopy_expapi_DaemonSet(in.Items[i], &out.Items[i], c); err != nil { return err } } @@ -799,7 +799,7 @@ func deepCopy_expapi_DaemonList(in DaemonList, out *DaemonList, c *conversion.Cl return nil } -func deepCopy_expapi_DaemonSpec(in DaemonSpec, out *DaemonSpec, c *conversion.Cloner) error { +func deepCopy_expapi_DaemonSetSpec(in DaemonSetSpec, out *DaemonSetSpec, c *conversion.Cloner) error { if in.Selector != nil { out.Selector = make(map[string]string) for key, val := range in.Selector { @@ -819,7 +819,7 @@ func deepCopy_expapi_DaemonSpec(in DaemonSpec, out *DaemonSpec, c *conversion.Cl return nil } -func deepCopy_expapi_DaemonStatus(in DaemonStatus, out *DaemonStatus, c *conversion.Cloner) error { +func deepCopy_expapi_DaemonSetStatus(in DaemonSetStatus, out *DaemonSetStatus, c *conversion.Cloner) error { out.CurrentNumberScheduled = in.CurrentNumberScheduled out.NumberMisscheduled = in.NumberMisscheduled out.DesiredNumberScheduled = in.DesiredNumberScheduled @@ -1193,10 +1193,10 @@ func init() { deepCopy_api_VolumeSource, deepCopy_resource_Quantity, deepCopy_expapi_APIVersion, - deepCopy_expapi_Daemon, - deepCopy_expapi_DaemonList, - deepCopy_expapi_DaemonSpec, - deepCopy_expapi_DaemonStatus, + deepCopy_expapi_DaemonSet, + deepCopy_expapi_DaemonSetList, + deepCopy_expapi_DaemonSetSpec, + deepCopy_expapi_DaemonSetStatus, deepCopy_expapi_Deployment, deepCopy_expapi_DeploymentList, deepCopy_expapi_DeploymentSpec, diff --git a/pkg/expapi/register.go b/pkg/expapi/register.go index 35d9246e43e..10e656da2cb 100644 --- a/pkg/expapi/register.go +++ b/pkg/expapi/register.go @@ -36,8 +36,8 @@ func addKnownTypes() { &Scale{}, &ThirdPartyResource{}, &ThirdPartyResourceList{}, - &DaemonList{}, - &Daemon{}, + &DaemonSetList{}, + &DaemonSet{}, &ThirdPartyResourceData{}, &ThirdPartyResourceDataList{}, ) @@ -51,7 +51,7 @@ func (*ReplicationControllerDummy) IsAnAPIObject() {} func (*Scale) IsAnAPIObject() {} func (*ThirdPartyResource) IsAnAPIObject() {} func (*ThirdPartyResourceList) IsAnAPIObject() {} -func (*Daemon) IsAnAPIObject() {} -func (*DaemonList) IsAnAPIObject() {} +func (*DaemonSet) IsAnAPIObject() {} +func (*DaemonSetList) IsAnAPIObject() {} func (*ThirdPartyResourceData) IsAnAPIObject() {} func (*ThirdPartyResourceDataList) IsAnAPIObject() {} diff --git a/pkg/expapi/types.go b/pkg/expapi/types.go index 1612f2e2768..e136b93a890 100644 --- a/pkg/expapi/types.go +++ b/pkg/expapi/types.go @@ -293,65 +293,65 @@ type DeploymentList struct { Items []Deployment `json:"items"` } -// DaemonSpec is the specification of a daemon. -type DaemonSpec struct { - // Selector is a label query over pods that are managed by the daemon. +// DaemonSetSpec is the specification of a daemon set. +type DaemonSetSpec struct { + // Selector is a label query over pods that are managed by the daemon set. // Must match in order to be controlled. // If empty, defaulted to labels on Pod template. // More info: http://releases.k8s.io/HEAD/docs/user-guide/labels.md#label-selectors Selector map[string]string `json:"selector,omitempty"` // Template is the object that describes the pod that will be created. - // The Daemon will create exactly one copy of this pod on every node + // The DaemonSet will create exactly one copy of this pod on every node // that matches the template's node selector (or on every node if no node // selector is specified). // More info: http://releases.k8s.io/HEAD/docs/user-guide/replication-controller.md#pod-template Template *api.PodTemplateSpec `json:"template,omitempty"` } -// DaemonStatus represents the current status of a daemon. -type DaemonStatus struct { - // CurrentNumberScheduled is the number of nodes that are running exactly 1 copy of the - // daemon and are supposed to run the daemon. +// DaemonSetStatus represents the current status of a daemon set. +type DaemonSetStatus struct { + // CurrentNumberScheduled is the number of nodes that are running exactly 1 + // daemon pod and are supposed to run the daemon pod. CurrentNumberScheduled int `json:"currentNumberScheduled"` - // NumberMisscheduled is the number of nodes that are running the daemon, but are - // not supposed to run the daemon. + // NumberMisscheduled is the number of nodes that are running the daemon pod, but are + // not supposed to run the daemon pod. NumberMisscheduled int `json:"numberMisscheduled"` // DesiredNumberScheduled is the total number of nodes that should be running the daemon - // (including nodes correctly running the daemon). + // pod (including nodes correctly running the daemon pod). DesiredNumberScheduled int `json:"desiredNumberScheduled"` } -// Daemon represents the configuration of a daemon. -type Daemon struct { +// DaemonSet represents the configuration of a daemon set. +type DaemonSet struct { api.TypeMeta `json:",inline"` // Standard object's metadata. // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata api.ObjectMeta `json:"metadata,omitempty"` - // Spec defines the desired behavior of this daemon. + // Spec defines the desired behavior of this daemon set. // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status - Spec DaemonSpec `json:"spec,omitempty"` + Spec DaemonSetSpec `json:"spec,omitempty"` - // Status is the current status of this daemon. This data may be + // Status is the current status of this daemon set. This data may be // out of date by some window of time. // Populated by the system. // Read-only. // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status - Status DaemonStatus `json:"status,omitempty"` + Status DaemonSetStatus `json:"status,omitempty"` } -// DaemonList is a collection of daemon. -type DaemonList struct { +// DaemonSetList is a collection of daemon sets. +type DaemonSetList struct { api.TypeMeta `json:",inline"` // Standard list metadata. // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata api.ListMeta `json:"metadata,omitempty"` - // Items is a list of daemons. - Items []Daemon `json:"items"` + // Items is a list of daemon sets. + Items []DaemonSet `json:"items"` } type ThirdPartyResourceDataList struct { diff --git a/pkg/expapi/v1/conversion_generated.go b/pkg/expapi/v1/conversion_generated.go index a05cdeb760c..12289ef2363 100644 --- a/pkg/expapi/v1/conversion_generated.go +++ b/pkg/expapi/v1/conversion_generated.go @@ -1571,9 +1571,9 @@ func convert_expapi_APIVersion_To_v1_APIVersion(in *expapi.APIVersion, out *APIV return nil } -func convert_expapi_Daemon_To_v1_Daemon(in *expapi.Daemon, out *Daemon, s conversion.Scope) error { +func convert_expapi_DaemonSet_To_v1_DaemonSet(in *expapi.DaemonSet, out *DaemonSet, s conversion.Scope) error { if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found { - defaulting.(func(*expapi.Daemon))(in) + defaulting.(func(*expapi.DaemonSet))(in) } if err := convert_api_TypeMeta_To_v1_TypeMeta(&in.TypeMeta, &out.TypeMeta, s); err != nil { return err @@ -1581,18 +1581,18 @@ func convert_expapi_Daemon_To_v1_Daemon(in *expapi.Daemon, out *Daemon, s conver if err := convert_api_ObjectMeta_To_v1_ObjectMeta(&in.ObjectMeta, &out.ObjectMeta, s); err != nil { return err } - if err := convert_expapi_DaemonSpec_To_v1_DaemonSpec(&in.Spec, &out.Spec, s); err != nil { + if err := convert_expapi_DaemonSetSpec_To_v1_DaemonSetSpec(&in.Spec, &out.Spec, s); err != nil { return err } - if err := convert_expapi_DaemonStatus_To_v1_DaemonStatus(&in.Status, &out.Status, s); err != nil { + if err := convert_expapi_DaemonSetStatus_To_v1_DaemonSetStatus(&in.Status, &out.Status, s); err != nil { return err } return nil } -func convert_expapi_DaemonList_To_v1_DaemonList(in *expapi.DaemonList, out *DaemonList, s conversion.Scope) error { +func convert_expapi_DaemonSetList_To_v1_DaemonSetList(in *expapi.DaemonSetList, out *DaemonSetList, s conversion.Scope) error { if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found { - defaulting.(func(*expapi.DaemonList))(in) + defaulting.(func(*expapi.DaemonSetList))(in) } if err := convert_api_TypeMeta_To_v1_TypeMeta(&in.TypeMeta, &out.TypeMeta, s); err != nil { return err @@ -1601,9 +1601,9 @@ func convert_expapi_DaemonList_To_v1_DaemonList(in *expapi.DaemonList, out *Daem return err } if in.Items != nil { - out.Items = make([]Daemon, len(in.Items)) + out.Items = make([]DaemonSet, len(in.Items)) for i := range in.Items { - if err := convert_expapi_Daemon_To_v1_Daemon(&in.Items[i], &out.Items[i], s); err != nil { + if err := convert_expapi_DaemonSet_To_v1_DaemonSet(&in.Items[i], &out.Items[i], s); err != nil { return err } } @@ -1613,9 +1613,9 @@ func convert_expapi_DaemonList_To_v1_DaemonList(in *expapi.DaemonList, out *Daem return nil } -func convert_expapi_DaemonSpec_To_v1_DaemonSpec(in *expapi.DaemonSpec, out *DaemonSpec, s conversion.Scope) error { +func convert_expapi_DaemonSetSpec_To_v1_DaemonSetSpec(in *expapi.DaemonSetSpec, out *DaemonSetSpec, s conversion.Scope) error { if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found { - defaulting.(func(*expapi.DaemonSpec))(in) + defaulting.(func(*expapi.DaemonSetSpec))(in) } if in.Selector != nil { out.Selector = make(map[string]string) @@ -1636,9 +1636,9 @@ func convert_expapi_DaemonSpec_To_v1_DaemonSpec(in *expapi.DaemonSpec, out *Daem return nil } -func convert_expapi_DaemonStatus_To_v1_DaemonStatus(in *expapi.DaemonStatus, out *DaemonStatus, s conversion.Scope) error { +func convert_expapi_DaemonSetStatus_To_v1_DaemonSetStatus(in *expapi.DaemonSetStatus, out *DaemonSetStatus, s conversion.Scope) error { if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found { - defaulting.(func(*expapi.DaemonStatus))(in) + defaulting.(func(*expapi.DaemonSetStatus))(in) } out.CurrentNumberScheduled = in.CurrentNumberScheduled out.NumberMisscheduled = in.NumberMisscheduled @@ -1959,9 +1959,9 @@ func convert_v1_APIVersion_To_expapi_APIVersion(in *APIVersion, out *expapi.APIV return nil } -func convert_v1_Daemon_To_expapi_Daemon(in *Daemon, out *expapi.Daemon, s conversion.Scope) error { +func convert_v1_DaemonSet_To_expapi_DaemonSet(in *DaemonSet, out *expapi.DaemonSet, s conversion.Scope) error { if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found { - defaulting.(func(*Daemon))(in) + defaulting.(func(*DaemonSet))(in) } if err := convert_v1_TypeMeta_To_api_TypeMeta(&in.TypeMeta, &out.TypeMeta, s); err != nil { return err @@ -1969,18 +1969,18 @@ func convert_v1_Daemon_To_expapi_Daemon(in *Daemon, out *expapi.Daemon, s conver if err := convert_v1_ObjectMeta_To_api_ObjectMeta(&in.ObjectMeta, &out.ObjectMeta, s); err != nil { return err } - if err := convert_v1_DaemonSpec_To_expapi_DaemonSpec(&in.Spec, &out.Spec, s); err != nil { + if err := convert_v1_DaemonSetSpec_To_expapi_DaemonSetSpec(&in.Spec, &out.Spec, s); err != nil { return err } - if err := convert_v1_DaemonStatus_To_expapi_DaemonStatus(&in.Status, &out.Status, s); err != nil { + if err := convert_v1_DaemonSetStatus_To_expapi_DaemonSetStatus(&in.Status, &out.Status, s); err != nil { return err } return nil } -func convert_v1_DaemonList_To_expapi_DaemonList(in *DaemonList, out *expapi.DaemonList, s conversion.Scope) error { +func convert_v1_DaemonSetList_To_expapi_DaemonSetList(in *DaemonSetList, out *expapi.DaemonSetList, s conversion.Scope) error { if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found { - defaulting.(func(*DaemonList))(in) + defaulting.(func(*DaemonSetList))(in) } if err := convert_v1_TypeMeta_To_api_TypeMeta(&in.TypeMeta, &out.TypeMeta, s); err != nil { return err @@ -1989,9 +1989,9 @@ func convert_v1_DaemonList_To_expapi_DaemonList(in *DaemonList, out *expapi.Daem return err } if in.Items != nil { - out.Items = make([]expapi.Daemon, len(in.Items)) + out.Items = make([]expapi.DaemonSet, len(in.Items)) for i := range in.Items { - if err := convert_v1_Daemon_To_expapi_Daemon(&in.Items[i], &out.Items[i], s); err != nil { + if err := convert_v1_DaemonSet_To_expapi_DaemonSet(&in.Items[i], &out.Items[i], s); err != nil { return err } } @@ -2001,9 +2001,9 @@ func convert_v1_DaemonList_To_expapi_DaemonList(in *DaemonList, out *expapi.Daem return nil } -func convert_v1_DaemonSpec_To_expapi_DaemonSpec(in *DaemonSpec, out *expapi.DaemonSpec, s conversion.Scope) error { +func convert_v1_DaemonSetSpec_To_expapi_DaemonSetSpec(in *DaemonSetSpec, out *expapi.DaemonSetSpec, s conversion.Scope) error { if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found { - defaulting.(func(*DaemonSpec))(in) + defaulting.(func(*DaemonSetSpec))(in) } if in.Selector != nil { out.Selector = make(map[string]string) @@ -2024,9 +2024,9 @@ func convert_v1_DaemonSpec_To_expapi_DaemonSpec(in *DaemonSpec, out *expapi.Daem return nil } -func convert_v1_DaemonStatus_To_expapi_DaemonStatus(in *DaemonStatus, out *expapi.DaemonStatus, s conversion.Scope) error { +func convert_v1_DaemonSetStatus_To_expapi_DaemonSetStatus(in *DaemonSetStatus, out *expapi.DaemonSetStatus, s conversion.Scope) error { if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found { - defaulting.(func(*DaemonStatus))(in) + defaulting.(func(*DaemonSetStatus))(in) } out.CurrentNumberScheduled = in.CurrentNumberScheduled out.NumberMisscheduled = in.NumberMisscheduled @@ -2379,10 +2379,10 @@ func init() { convert_api_VolumeSource_To_v1_VolumeSource, convert_api_Volume_To_v1_Volume, convert_expapi_APIVersion_To_v1_APIVersion, - convert_expapi_DaemonList_To_v1_DaemonList, - convert_expapi_DaemonSpec_To_v1_DaemonSpec, - convert_expapi_DaemonStatus_To_v1_DaemonStatus, - convert_expapi_Daemon_To_v1_Daemon, + convert_expapi_DaemonSetList_To_v1_DaemonSetList, + convert_expapi_DaemonSetSpec_To_v1_DaemonSetSpec, + convert_expapi_DaemonSetStatus_To_v1_DaemonSetStatus, + convert_expapi_DaemonSet_To_v1_DaemonSet, convert_expapi_DeploymentList_To_v1_DeploymentList, convert_expapi_DeploymentStatus_To_v1_DeploymentStatus, convert_expapi_Deployment_To_v1_Deployment, @@ -2407,10 +2407,10 @@ func init() { convert_v1_CinderVolumeSource_To_api_CinderVolumeSource, convert_v1_ContainerPort_To_api_ContainerPort, convert_v1_Container_To_api_Container, - convert_v1_DaemonList_To_expapi_DaemonList, - convert_v1_DaemonSpec_To_expapi_DaemonSpec, - convert_v1_DaemonStatus_To_expapi_DaemonStatus, - convert_v1_Daemon_To_expapi_Daemon, + convert_v1_DaemonSetList_To_expapi_DaemonSetList, + convert_v1_DaemonSetSpec_To_expapi_DaemonSetSpec, + convert_v1_DaemonSetStatus_To_expapi_DaemonSetStatus, + convert_v1_DaemonSet_To_expapi_DaemonSet, convert_v1_DeploymentList_To_expapi_DeploymentList, convert_v1_DeploymentStatus_To_expapi_DeploymentStatus, convert_v1_Deployment_To_expapi_Deployment, diff --git a/pkg/expapi/v1/deep_copy_generated.go b/pkg/expapi/v1/deep_copy_generated.go index a61f1a17114..0e5ba1cb976 100644 --- a/pkg/expapi/v1/deep_copy_generated.go +++ b/pkg/expapi/v1/deep_copy_generated.go @@ -765,23 +765,23 @@ func deepCopy_v1_APIVersion(in APIVersion, out *APIVersion, c *conversion.Cloner return nil } -func deepCopy_v1_Daemon(in Daemon, out *Daemon, c *conversion.Cloner) error { +func deepCopy_v1_DaemonSet(in DaemonSet, out *DaemonSet, c *conversion.Cloner) error { if err := deepCopy_v1_TypeMeta(in.TypeMeta, &out.TypeMeta, c); err != nil { return err } if err := deepCopy_v1_ObjectMeta(in.ObjectMeta, &out.ObjectMeta, c); err != nil { return err } - if err := deepCopy_v1_DaemonSpec(in.Spec, &out.Spec, c); err != nil { + if err := deepCopy_v1_DaemonSetSpec(in.Spec, &out.Spec, c); err != nil { return err } - if err := deepCopy_v1_DaemonStatus(in.Status, &out.Status, c); err != nil { + if err := deepCopy_v1_DaemonSetStatus(in.Status, &out.Status, c); err != nil { return err } return nil } -func deepCopy_v1_DaemonList(in DaemonList, out *DaemonList, c *conversion.Cloner) error { +func deepCopy_v1_DaemonSetList(in DaemonSetList, out *DaemonSetList, c *conversion.Cloner) error { if err := deepCopy_v1_TypeMeta(in.TypeMeta, &out.TypeMeta, c); err != nil { return err } @@ -789,9 +789,9 @@ func deepCopy_v1_DaemonList(in DaemonList, out *DaemonList, c *conversion.Cloner return err } if in.Items != nil { - out.Items = make([]Daemon, len(in.Items)) + out.Items = make([]DaemonSet, len(in.Items)) for i := range in.Items { - if err := deepCopy_v1_Daemon(in.Items[i], &out.Items[i], c); err != nil { + if err := deepCopy_v1_DaemonSet(in.Items[i], &out.Items[i], c); err != nil { return err } } @@ -801,7 +801,7 @@ func deepCopy_v1_DaemonList(in DaemonList, out *DaemonList, c *conversion.Cloner return nil } -func deepCopy_v1_DaemonSpec(in DaemonSpec, out *DaemonSpec, c *conversion.Cloner) error { +func deepCopy_v1_DaemonSetSpec(in DaemonSetSpec, out *DaemonSetSpec, c *conversion.Cloner) error { if in.Selector != nil { out.Selector = make(map[string]string) for key, val := range in.Selector { @@ -821,7 +821,7 @@ func deepCopy_v1_DaemonSpec(in DaemonSpec, out *DaemonSpec, c *conversion.Cloner return nil } -func deepCopy_v1_DaemonStatus(in DaemonStatus, out *DaemonStatus, c *conversion.Cloner) error { +func deepCopy_v1_DaemonSetStatus(in DaemonSetStatus, out *DaemonSetStatus, c *conversion.Cloner) error { out.CurrentNumberScheduled = in.CurrentNumberScheduled out.NumberMisscheduled = in.NumberMisscheduled out.DesiredNumberScheduled = in.DesiredNumberScheduled @@ -1215,10 +1215,10 @@ func init() { deepCopy_v1_VolumeMount, deepCopy_v1_VolumeSource, deepCopy_v1_APIVersion, - deepCopy_v1_Daemon, - deepCopy_v1_DaemonList, - deepCopy_v1_DaemonSpec, - deepCopy_v1_DaemonStatus, + deepCopy_v1_DaemonSet, + deepCopy_v1_DaemonSetList, + deepCopy_v1_DaemonSetSpec, + deepCopy_v1_DaemonSetStatus, deepCopy_v1_Deployment, deepCopy_v1_DeploymentList, deepCopy_v1_DeploymentSpec, diff --git a/pkg/expapi/v1/defaults.go b/pkg/expapi/v1/defaults.go index 105a49ee3de..cc141513c25 100644 --- a/pkg/expapi/v1/defaults.go +++ b/pkg/expapi/v1/defaults.go @@ -28,7 +28,7 @@ func addDefaultingFuncs() { obj.APIGroup = "experimental" } }, - func(obj *Daemon) { + func(obj *DaemonSet) { var labels map[string]string if obj.Spec.Template != nil { labels = obj.Spec.Template.Labels diff --git a/pkg/expapi/v1/defaults_test.go b/pkg/expapi/v1/defaults_test.go index 3463534fc9d..afc00939fa2 100644 --- a/pkg/expapi/v1/defaults_test.go +++ b/pkg/expapi/v1/defaults_test.go @@ -26,14 +26,14 @@ import ( "k8s.io/kubernetes/pkg/util" ) -func TestSetDefaultDaemon(t *testing.T) { +func TestSetDefaultDaemonSet(t *testing.T) { tests := []struct { - dc *Daemon + ds *DaemonSet expectLabelsChange bool }{ { - dc: &Daemon{ - Spec: DaemonSpec{ + ds: &DaemonSet{ + Spec: DaemonSetSpec{ Template: &v1.PodTemplateSpec{ ObjectMeta: v1.ObjectMeta{ Labels: map[string]string{ @@ -46,13 +46,13 @@ func TestSetDefaultDaemon(t *testing.T) { expectLabelsChange: true, }, { - dc: &Daemon{ + ds: &DaemonSet{ ObjectMeta: v1.ObjectMeta{ Labels: map[string]string{ "bar": "foo", }, }, - Spec: DaemonSpec{ + Spec: DaemonSetSpec{ Template: &v1.PodTemplateSpec{ ObjectMeta: v1.ObjectMeta{ Labels: map[string]string{ @@ -67,18 +67,18 @@ func TestSetDefaultDaemon(t *testing.T) { } for _, test := range tests { - dc := test.dc - obj2 := roundTrip(t, runtime.Object(dc)) - dc2, ok := obj2.(*Daemon) + ds := test.ds + obj2 := roundTrip(t, runtime.Object(ds)) + ds2, ok := obj2.(*DaemonSet) if !ok { - t.Errorf("unexpected object: %v", dc2) + t.Errorf("unexpected object: %v", ds2) t.FailNow() } - if test.expectLabelsChange != reflect.DeepEqual(dc2.Labels, dc2.Spec.Template.Labels) { + if test.expectLabelsChange != reflect.DeepEqual(ds2.Labels, ds2.Spec.Template.Labels) { if test.expectLabelsChange { - t.Errorf("expected: %v, got: %v", dc2.Spec.Template.Labels, dc2.Labels) + t.Errorf("expected: %v, got: %v", ds2.Spec.Template.Labels, ds2.Labels) } else { - t.Errorf("unexpected equality: %v", dc.Labels) + t.Errorf("unexpected equality: %v", ds.Labels) } } } diff --git a/pkg/expapi/v1/register.go b/pkg/expapi/v1/register.go index e6fab8c212e..30bb3d923d4 100644 --- a/pkg/expapi/v1/register.go +++ b/pkg/expapi/v1/register.go @@ -40,8 +40,8 @@ func addKnownTypes() { &Scale{}, &ThirdPartyResource{}, &ThirdPartyResourceList{}, - &DaemonList{}, - &Daemon{}, + &DaemonSetList{}, + &DaemonSet{}, &ThirdPartyResourceData{}, &ThirdPartyResourceDataList{}, ) @@ -55,7 +55,7 @@ func (*ReplicationControllerDummy) IsAnAPIObject() {} func (*Scale) IsAnAPIObject() {} func (*ThirdPartyResource) IsAnAPIObject() {} func (*ThirdPartyResourceList) IsAnAPIObject() {} -func (*Daemon) IsAnAPIObject() {} -func (*DaemonList) IsAnAPIObject() {} +func (*DaemonSet) IsAnAPIObject() {} +func (*DaemonSetList) IsAnAPIObject() {} func (*ThirdPartyResourceData) IsAnAPIObject() {} func (*ThirdPartyResourceDataList) IsAnAPIObject() {} diff --git a/pkg/expapi/v1/types.go b/pkg/expapi/v1/types.go index 19fb02b15a1..e3d120f9e12 100644 --- a/pkg/expapi/v1/types.go +++ b/pkg/expapi/v1/types.go @@ -292,65 +292,65 @@ type DeploymentList struct { Items []Deployment `json:"items"` } -// DaemonSpec is the specification of a daemon. -type DaemonSpec struct { - // Selector is a label query over pods that are managed by the daemon. +// DaemonSetSpec is the specification of a daemon set. +type DaemonSetSpec struct { + // Selector is a label query over pods that are managed by the daemon set. // Must match in order to be controlled. // If empty, defaulted to labels on Pod template. // More info: http://releases.k8s.io/HEAD/docs/user-guide/labels.md#label-selectors Selector map[string]string `json:"selector,omitempty"` // Template is the object that describes the pod that will be created. - // The Daemon will create exactly one copy of this pod on every node + // The DaemonSet will create exactly one copy of this pod on every node // that matches the template's node selector (or on every node if no node // selector is specified). // More info: http://releases.k8s.io/HEAD/docs/user-guide/replication-controller.md#pod-template Template *v1.PodTemplateSpec `json:"template,omitempty"` } -// DaemonStatus represents the current status of a daemon. -type DaemonStatus struct { - // CurrentNumberScheduled is the number of nodes that are running exactly 1 copy of the - // daemon and are supposed to run the daemon. +// DaemonSetStatus represents the current status of a daemon set. +type DaemonSetStatus struct { + // CurrentNumberScheduled is the number of nodes that are running exactly 1 + // daemon pod and are supposed to run the daemon pod. CurrentNumberScheduled int `json:"currentNumberScheduled"` - // NumberMisscheduled is the number of nodes that are running the daemon, but are - // not supposed to run the daemon. + // NumberMisscheduled is the number of nodes that are running the daemon pod, but are + // not supposed to run the daemon pod. NumberMisscheduled int `json:"numberMisscheduled"` // DesiredNumberScheduled is the total number of nodes that should be running the daemon - // (including nodes correctly running the daemon). + // pod (including nodes correctly running the daemon pod). DesiredNumberScheduled int `json:"desiredNumberScheduled"` } -// Daemon represents the configuration of a daemon. -type Daemon struct { +// DaemonSet represents the configuration of a daemon set. +type DaemonSet struct { v1.TypeMeta `json:",inline"` // Standard object's metadata. // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata v1.ObjectMeta `json:"metadata,omitempty"` - // Spec defines the desired behavior of this daemon. + // Spec defines the desired behavior of this daemon set. // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status - Spec DaemonSpec `json:"spec,omitempty"` + Spec DaemonSetSpec `json:"spec,omitempty"` - // Status is the current status of this daemon. This data may be + // Status is the current status of this daemon set. This data may be // out of date by some window of time. // Populated by the system. // Read-only. // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status - Status DaemonStatus `json:"status,omitempty"` + Status DaemonSetStatus `json:"status,omitempty"` } -// DaemonList is a list of Daemons. -type DaemonList struct { +// DaemonSetList is a collection of daemon sets. +type DaemonSetList struct { v1.TypeMeta `json:",inline"` // Standard list metadata. // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata v1.ListMeta `json:"metadata,omitempty"` - // Items is the list of Daemons. - Items []Daemon `json:"items"` + // Items is a list of daemon sets. + Items []DaemonSet `json:"items"` } // ThirdPartyResrouceDataList is a list of ThirdPartyResourceData. diff --git a/pkg/expapi/v1/types_swagger_doc_generated.go b/pkg/expapi/v1/types_swagger_doc_generated.go index 584d693f198..ced86254859 100644 --- a/pkg/expapi/v1/types_swagger_doc_generated.go +++ b/pkg/expapi/v1/types_swagger_doc_generated.go @@ -37,46 +37,46 @@ func (APIVersion) SwaggerDoc() map[string]string { return map_APIVersion } -var map_Daemon = map[string]string{ - "": "Daemon represents the configuration of a daemon.", +var map_DaemonSet = map[string]string{ + "": "DaemonSet represents the configuration of a daemon set.", "metadata": "Standard object's metadata. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata", - "spec": "Spec defines the desired behavior of this daemon. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status", - "status": "Status is the current status of this daemon. This data may be out of date by some window of time. Populated by the system. Read-only. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status", + "spec": "Spec defines the desired behavior of this daemon set. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status", + "status": "Status is the current status of this daemon set. This data may be out of date by some window of time. Populated by the system. Read-only. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status", } -func (Daemon) SwaggerDoc() map[string]string { - return map_Daemon +func (DaemonSet) SwaggerDoc() map[string]string { + return map_DaemonSet } -var map_DaemonList = map[string]string{ - "": "DaemonList is a list of Daemons.", +var map_DaemonSetList = map[string]string{ + "": "DaemonSetList is a collection of daemon sets.", "metadata": "Standard list metadata. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata", - "items": "Items is the list of Daemons.", + "items": "Items is a list of daemon sets.", } -func (DaemonList) SwaggerDoc() map[string]string { - return map_DaemonList +func (DaemonSetList) SwaggerDoc() map[string]string { + return map_DaemonSetList } -var map_DaemonSpec = map[string]string{ - "": "DaemonSpec is the specification of a daemon.", - "selector": "Selector is a label query over pods that are managed by the daemon. Must match in order to be controlled. If empty, defaulted to labels on Pod template. More info: http://releases.k8s.io/HEAD/docs/user-guide/labels.md#label-selectors", - "template": "Template is the object that describes the pod that will be created. The Daemon will create exactly one copy of this pod on every node that matches the template's node selector (or on every node if no node selector is specified). More info: http://releases.k8s.io/HEAD/docs/user-guide/replication-controller.md#pod-template", +var map_DaemonSetSpec = map[string]string{ + "": "DaemonSetSpec is the specification of a daemon set.", + "selector": "Selector is a label query over pods that are managed by the daemon set. Must match in order to be controlled. If empty, defaulted to labels on Pod template. More info: http://releases.k8s.io/HEAD/docs/user-guide/labels.md#label-selectors", + "template": "Template is the object that describes the pod that will be created. The DaemonSet will create exactly one copy of this pod on every node that matches the template's node selector (or on every node if no node selector is specified). More info: http://releases.k8s.io/HEAD/docs/user-guide/replication-controller.md#pod-template", } -func (DaemonSpec) SwaggerDoc() map[string]string { - return map_DaemonSpec +func (DaemonSetSpec) SwaggerDoc() map[string]string { + return map_DaemonSetSpec } -var map_DaemonStatus = map[string]string{ - "": "DaemonStatus represents the current status of a daemon.", - "currentNumberScheduled": "CurrentNumberScheduled is the number of nodes that are running exactly 1 copy of the daemon and are supposed to run the daemon.", - "numberMisscheduled": "NumberMisscheduled is the number of nodes that are running the daemon, but are not supposed to run the daemon.", - "desiredNumberScheduled": "DesiredNumberScheduled is the total number of nodes that should be running the daemon (including nodes correctly running the daemon).", +var map_DaemonSetStatus = map[string]string{ + "": "DaemonSetStatus represents the current status of a daemon set.", + "currentNumberScheduled": "CurrentNumberScheduled is the number of nodes that are running exactly 1 daemon pod and are supposed to run the daemon pod.", + "numberMisscheduled": "NumberMisscheduled is the number of nodes that are running the daemon pod, but are not supposed to run the daemon pod.", + "desiredNumberScheduled": "DesiredNumberScheduled is the total number of nodes that should be running the daemon pod (including nodes correctly running the daemon pod).", } -func (DaemonStatus) SwaggerDoc() map[string]string { - return map_DaemonStatus +func (DaemonSetStatus) SwaggerDoc() map[string]string { + return map_DaemonSetStatus } var map_Deployment = map[string]string{ diff --git a/pkg/expapi/validation/validation.go b/pkg/expapi/validation/validation.go index 7cc986691c9..82984add68a 100644 --- a/pkg/expapi/validation/validation.go +++ b/pkg/expapi/validation/validation.go @@ -93,25 +93,25 @@ func ValidateThirdPartyResource(obj *expapi.ThirdPartyResource) errs.ValidationE return allErrs } -// ValidateDaemon tests if required fields in the daemon are set. -func ValidateDaemon(controller *expapi.Daemon) errs.ValidationErrorList { +// ValidateDaemonSet tests if required fields in the DaemonSet are set. +func ValidateDaemonSet(controller *expapi.DaemonSet) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} allErrs = append(allErrs, apivalidation.ValidateObjectMeta(&controller.ObjectMeta, true, apivalidation.ValidateReplicationControllerName).Prefix("metadata")...) - allErrs = append(allErrs, ValidateDaemonSpec(&controller.Spec).Prefix("spec")...) + allErrs = append(allErrs, ValidateDaemonSetSpec(&controller.Spec).Prefix("spec")...) return allErrs } -// ValidateDaemonUpdate tests if required fields in the daemon are set. -func ValidateDaemonUpdate(oldController, controller *expapi.Daemon) errs.ValidationErrorList { +// ValidateDaemonSetUpdate tests if required fields in the DaemonSet are set. +func ValidateDaemonSetUpdate(oldController, controller *expapi.DaemonSet) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} allErrs = append(allErrs, apivalidation.ValidateObjectMetaUpdate(&controller.ObjectMeta, &oldController.ObjectMeta).Prefix("metadata")...) - allErrs = append(allErrs, ValidateDaemonSpec(&controller.Spec).Prefix("spec")...) - allErrs = append(allErrs, ValidateDaemonTemplateUpdate(oldController.Spec.Template, controller.Spec.Template).Prefix("spec.template")...) + allErrs = append(allErrs, ValidateDaemonSetSpec(&controller.Spec).Prefix("spec")...) + allErrs = append(allErrs, ValidateDaemonSetTemplateUpdate(oldController.Spec.Template, controller.Spec.Template).Prefix("spec.template")...) return allErrs } -// ValidateDaemonTemplateUpdate tests that certain fields in the daemon's pod template are not updated. -func ValidateDaemonTemplateUpdate(oldPodTemplate, podTemplate *api.PodTemplateSpec) errs.ValidationErrorList { +// ValidateDaemonSetTemplateUpdate tests that certain fields in the daemon set's pod template are not updated. +func ValidateDaemonSetTemplateUpdate(oldPodTemplate, podTemplate *api.PodTemplateSpec) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} podSpec := podTemplate.Spec // podTemplate.Spec is not a pointer, so we can modify NodeSelector and NodeName directly. @@ -125,8 +125,8 @@ func ValidateDaemonTemplateUpdate(oldPodTemplate, podTemplate *api.PodTemplateSp return allErrs } -// ValidateDaemonSpec tests if required fields in the daemon spec are set. -func ValidateDaemonSpec(spec *expapi.DaemonSpec) errs.ValidationErrorList { +// ValidateDaemonSetSpec tests if required fields in the DaemonSetSpec are set. +func ValidateDaemonSetSpec(spec *expapi.DaemonSetSpec) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} selector := labels.Set(spec.Selector).AsSelector() @@ -152,10 +152,10 @@ func ValidateDaemonSpec(spec *expapi.DaemonSpec) errs.ValidationErrorList { return allErrs } -// ValidateDaemonName can be used to check whether the given daemon name is valid. +// ValidateDaemonSetName can be used to check whether the given daemon set name is valid. // Prefix indicates this name will be used as part of generation, in which case // trailing dashes are allowed. -func ValidateDaemonName(name string, prefix bool) (bool, string) { +func ValidateDaemonSetName(name string, prefix bool) (bool, string) { return apivalidation.NameIsDNSSubdomain(name, prefix) } diff --git a/pkg/expapi/validation/validation_test.go b/pkg/expapi/validation/validation_test.go index 14ca8974201..91c562feb71 100644 --- a/pkg/expapi/validation/validation_test.go +++ b/pkg/expapi/validation/validation_test.go @@ -130,7 +130,7 @@ func TestValidateHorizontalPodAutoscaler(t *testing.T) { } } -func TestValidateDaemonUpdate(t *testing.T) { +func TestValidateDaemonSetUpdate(t *testing.T) { validSelector := map[string]string{"a": "b"} validSelector2 := map[string]string{"c": "d"} invalidSelector := map[string]string{"NoUppercaseOrSpecialCharsLike=Equals": "b"} @@ -211,54 +211,54 @@ func TestValidateDaemonUpdate(t *testing.T) { }, } - type dcUpdateTest struct { - old expapi.Daemon - update expapi.Daemon + type dsUpdateTest struct { + old expapi.DaemonSet + update expapi.DaemonSet } - successCases := []dcUpdateTest{ + successCases := []dsUpdateTest{ { - old: expapi.Daemon{ + old: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, - update: expapi.Daemon{ + update: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, }, { - old: expapi.Daemon{ + old: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, - update: expapi.Daemon{ + update: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector2, Template: &validPodTemplateAbc2.Template, }, }, }, { - old: expapi.Daemon{ + old: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, - update: expapi.Daemon{ + update: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateNodeSelector.Template, }, @@ -268,86 +268,86 @@ func TestValidateDaemonUpdate(t *testing.T) { for _, successCase := range successCases { successCase.old.ObjectMeta.ResourceVersion = "1" successCase.update.ObjectMeta.ResourceVersion = "1" - if errs := ValidateDaemonUpdate(&successCase.old, &successCase.update); len(errs) != 0 { + if errs := ValidateDaemonSetUpdate(&successCase.old, &successCase.update); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } - errorCases := map[string]dcUpdateTest{ + errorCases := map[string]dsUpdateTest{ "change daemon name": { - old: expapi.Daemon{ + old: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, - update: expapi.Daemon{ + update: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, }, "invalid selector": { - old: expapi.Daemon{ + old: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, - update: expapi.Daemon{ + update: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: invalidSelector, Template: &validPodTemplateAbc.Template, }, }, }, "invalid pod": { - old: expapi.Daemon{ + old: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, - update: expapi.Daemon{ + update: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &invalidPodTemplate.Template, }, }, }, "change container image": { - old: expapi.Daemon{ + old: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, - update: expapi.Daemon{ + update: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateDef.Template, }, }, }, "read-write volume": { - old: expapi.Daemon{ + old: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplateAbc.Template, }, }, - update: expapi.Daemon{ + update: expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &readWriteVolumePodTemplate.Template, }, @@ -355,13 +355,13 @@ func TestValidateDaemonUpdate(t *testing.T) { }, } for testName, errorCase := range errorCases { - if errs := ValidateDaemonUpdate(&errorCase.old, &errorCase.update); len(errs) == 0 { + if errs := ValidateDaemonSetUpdate(&errorCase.old, &errorCase.update); len(errs) == 0 { t.Errorf("expected failure: %s", testName) } } } -func TestValidateDaemon(t *testing.T) { +func TestValidateDaemonSet(t *testing.T) { validSelector := map[string]string{"a": "b"} validPodTemplate := api.PodTemplate{ Template: api.PodTemplateSpec{ @@ -387,59 +387,59 @@ func TestValidateDaemon(t *testing.T) { }, }, } - successCases := []expapi.Daemon{ + successCases := []expapi.DaemonSet{ { ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplate.Template, }, }, { ObjectMeta: api.ObjectMeta{Name: "abc-123", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplate.Template, }, }, } for _, successCase := range successCases { - if errs := ValidateDaemon(&successCase); len(errs) != 0 { + if errs := ValidateDaemonSet(&successCase); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } - errorCases := map[string]expapi.Daemon{ + errorCases := map[string]expapi.DaemonSet{ "zero-length ID": { ObjectMeta: api.ObjectMeta{Name: "", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplate.Template, }, }, "missing-namespace": { ObjectMeta: api.ObjectMeta{Name: "abc-123"}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplate.Template, }, }, "empty selector": { ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Template: &validPodTemplate.Template, }, }, "selector_doesnt_match": { ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: map[string]string{"foo": "bar"}, Template: &validPodTemplate.Template, }, }, "invalid manifest": { ObjectMeta: api.ObjectMeta{Name: "abc", Namespace: api.NamespaceDefault}, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, }, }, @@ -451,7 +451,7 @@ func TestValidateDaemon(t *testing.T) { "NoUppercaseOrSpecialCharsLike=Equals": "bar", }, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplate.Template, }, @@ -464,7 +464,7 @@ func TestValidateDaemon(t *testing.T) { "NoUppercaseOrSpecialCharsLike=Equals": "bar", }, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Template: &invalidPodTemplate.Template, }, }, @@ -476,7 +476,7 @@ func TestValidateDaemon(t *testing.T) { "NoUppercaseOrSpecialCharsLike=Equals": "bar", }, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &validPodTemplate.Template, }, @@ -486,7 +486,7 @@ func TestValidateDaemon(t *testing.T) { Name: "abc-123", Namespace: api.NamespaceDefault, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &api.PodTemplateSpec{ Spec: api.PodSpec{ @@ -505,7 +505,7 @@ func TestValidateDaemon(t *testing.T) { Name: "abc-123", Namespace: api.NamespaceDefault, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: validSelector, Template: &api.PodTemplateSpec{ Spec: api.PodSpec{ @@ -521,7 +521,7 @@ func TestValidateDaemon(t *testing.T) { }, } for k, v := range errorCases { - errs := ValidateDaemon(&v) + errs := ValidateDaemonSet(&v) if len(errs) == 0 { t.Errorf("expected failure for %s", k) } diff --git a/pkg/master/master.go b/pkg/master/master.go index 0c6569a7e3a..6e2995c6949 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -80,7 +80,7 @@ import ( "k8s.io/kubernetes/pkg/ui" "k8s.io/kubernetes/pkg/util" - daemonetcd "k8s.io/kubernetes/pkg/registry/daemon/etcd" + daemonetcd "k8s.io/kubernetes/pkg/registry/daemonset/etcd" horizontalpodautoscaleretcd "k8s.io/kubernetes/pkg/registry/horizontalpodautoscaler/etcd" "github.com/emicklei/go-restful" @@ -824,7 +824,7 @@ func (m *Master) expapi(c *Config) *apiserver.APIGroupVersion { controllerStorage := expcontrolleretcd.NewStorage(c.ExpDatabaseStorage) autoscalerStorage := horizontalpodautoscaleretcd.NewREST(c.ExpDatabaseStorage) thirdPartyResourceStorage := thirdpartyresourceetcd.NewREST(c.ExpDatabaseStorage) - daemonStorage := daemonetcd.NewREST(c.ExpDatabaseStorage) + daemonSetStorage := daemonetcd.NewREST(c.ExpDatabaseStorage) deploymentStorage := deploymentetcd.NewREST(c.ExpDatabaseStorage) storage := map[string]rest.Storage{ @@ -832,7 +832,7 @@ func (m *Master) expapi(c *Config) *apiserver.APIGroupVersion { strings.ToLower("replicationControllers/scale"): controllerStorage.Scale, strings.ToLower("horizontalpodautoscalers"): autoscalerStorage, strings.ToLower("thirdpartyresources"): thirdPartyResourceStorage, - strings.ToLower("daemons"): daemonStorage, + strings.ToLower("daemonsets"): daemonSetStorage, strings.ToLower("deployments"): deploymentStorage, } diff --git a/pkg/registry/daemon/doc.go b/pkg/registry/daemonset/doc.go similarity index 80% rename from pkg/registry/daemon/doc.go rename to pkg/registry/daemonset/doc.go index a2452004386..435b0fe615b 100644 --- a/pkg/registry/daemon/doc.go +++ b/pkg/registry/daemonset/doc.go @@ -14,6 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package daemon provides Registry interface and its RESTStorage -// implementation for storing Daemon api objects. -package daemon +// Package daemonset provides Registry interface and its RESTStorage +// implementation for storing DaemonSet api objects. +package daemonset diff --git a/pkg/registry/daemon/etcd/etcd.go b/pkg/registry/daemonset/etcd/etcd.go similarity index 72% rename from pkg/registry/daemon/etcd/etcd.go rename to pkg/registry/daemonset/etcd/etcd.go index 1248480b498..f68b20345a3 100644 --- a/pkg/registry/daemon/etcd/etcd.go +++ b/pkg/registry/daemonset/etcd/etcd.go @@ -21,29 +21,28 @@ import ( "k8s.io/kubernetes/pkg/expapi" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/registry/daemon" + "k8s.io/kubernetes/pkg/registry/daemonset" "k8s.io/kubernetes/pkg/registry/generic" etcdgeneric "k8s.io/kubernetes/pkg/registry/generic/etcd" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/storage" ) -// rest implements a RESTStorage for daemons against etcd +// rest implements a RESTStorage for DaemonSets against etcd type REST struct { *etcdgeneric.Etcd } -// daemonPrefix is the location for daemons in etcd, only exposed -// for testing -var daemonPrefix = "/daemons" +// daemonPrefix is the location for daemons in etcd +var daemonPrefix = "/daemonsets" -// NewREST returns a RESTStorage object that will work against daemons. +// NewREST returns a RESTStorage object that will work against DaemonSets. func NewREST(s storage.Interface) *REST { store := &etcdgeneric.Etcd{ - NewFunc: func() runtime.Object { return &expapi.Daemon{} }, + NewFunc: func() runtime.Object { return &expapi.DaemonSet{} }, // NewListFunc returns an object capable of storing results of an etcd list. - NewListFunc: func() runtime.Object { return &expapi.DaemonList{} }, + NewListFunc: func() runtime.Object { return &expapi.DaemonSetList{} }, // Produces a path that etcd understands, to the root of the resource // by combining the namespace in the context with the given prefix KeyRootFunc: func(ctx api.Context) string { @@ -54,21 +53,21 @@ func NewREST(s storage.Interface) *REST { KeyFunc: func(ctx api.Context, name string) (string, error) { return etcdgeneric.NamespaceKeyFunc(ctx, daemonPrefix, name) }, - // Retrieve the name field of a daemon + // Retrieve the name field of a daemon set ObjectNameFunc: func(obj runtime.Object) (string, error) { - return obj.(*expapi.Daemon).Name, nil + return obj.(*expapi.DaemonSet).Name, nil }, // Used to match objects based on labels/fields for list and watch PredicateFunc: func(label labels.Selector, field fields.Selector) generic.Matcher { - return daemon.MatchDaemon(label, field) + return daemonset.MatchDaemonSet(label, field) }, - EndpointName: "daemons", + EndpointName: "daemonsets", - // Used to validate daemon creation - CreateStrategy: daemon.Strategy, + // Used to validate daemon set creation + CreateStrategy: daemonset.Strategy, - // Used to validate daemon updates - UpdateStrategy: daemon.Strategy, + // Used to validate daemon set updates + UpdateStrategy: daemonset.Strategy, Storage: s, } diff --git a/pkg/registry/daemon/etcd/etcd_test.go b/pkg/registry/daemonset/etcd/etcd_test.go similarity index 84% rename from pkg/registry/daemon/etcd/etcd_test.go rename to pkg/registry/daemonset/etcd/etcd_test.go index 5d2510a7bf6..341ddd01830 100755 --- a/pkg/registry/daemon/etcd/etcd_test.go +++ b/pkg/registry/daemonset/etcd/etcd_test.go @@ -33,13 +33,13 @@ func newStorage(t *testing.T) (*REST, *tools.FakeEtcdClient) { return NewREST(etcdStorage), fakeClient } -func validNewDaemon() *expapi.Daemon { - return &expapi.Daemon{ +func newValidDaemonSet() *expapi.DaemonSet { + return &expapi.DaemonSet{ ObjectMeta: api.ObjectMeta{ Name: "foo", Namespace: api.NamespaceDefault, }, - Spec: expapi.DaemonSpec{ + Spec: expapi.DaemonSetSpec{ Selector: map[string]string{"a": "b"}, Template: &api.PodTemplateSpec{ ObjectMeta: api.ObjectMeta{ @@ -61,21 +61,21 @@ func validNewDaemon() *expapi.Daemon { } } -var validDaemon = validNewDaemon() +var validDaemonSet = newValidDaemonSet() func TestCreate(t *testing.T) { storage, fakeClient := newStorage(t) test := registrytest.New(t, fakeClient, storage.Etcd) - daemon := validNewDaemon() - daemon.ObjectMeta = api.ObjectMeta{} + ds := newValidDaemonSet() + ds.ObjectMeta = api.ObjectMeta{} test.TestCreate( // valid - daemon, + ds, // invalid (invalid selector) - &expapi.Daemon{ - Spec: expapi.DaemonSpec{ + &expapi.DaemonSet{ + Spec: expapi.DaemonSetSpec{ Selector: map[string]string{}, - Template: validDaemon.Spec.Template, + Template: validDaemonSet.Spec.Template, }, }, ) @@ -86,31 +86,31 @@ func TestUpdate(t *testing.T) { test := registrytest.New(t, fakeClient, storage.Etcd) test.TestUpdate( // valid - validNewDaemon(), + newValidDaemonSet(), // updateFunc func(obj runtime.Object) runtime.Object { - object := obj.(*expapi.Daemon) + object := obj.(*expapi.DaemonSet) object.Spec.Template.Spec.NodeSelector = map[string]string{"c": "d"} return object }, // invalid updateFunc func(obj runtime.Object) runtime.Object { - object := obj.(*expapi.Daemon) + object := obj.(*expapi.DaemonSet) object.UID = "newUID" return object }, func(obj runtime.Object) runtime.Object { - object := obj.(*expapi.Daemon) + object := obj.(*expapi.DaemonSet) object.Name = "" return object }, func(obj runtime.Object) runtime.Object { - object := obj.(*expapi.Daemon) + object := obj.(*expapi.DaemonSet) object.Spec.Template.Spec.RestartPolicy = api.RestartPolicyOnFailure return object }, func(obj runtime.Object) runtime.Object { - object := obj.(*expapi.Daemon) + object := obj.(*expapi.DaemonSet) object.Spec.Selector = map[string]string{} return object }, @@ -120,26 +120,26 @@ func TestUpdate(t *testing.T) { func TestDelete(t *testing.T) { storage, fakeClient := newStorage(t) test := registrytest.New(t, fakeClient, storage.Etcd) - test.TestDelete(validNewDaemon()) + test.TestDelete(newValidDaemonSet()) } func TestGet(t *testing.T) { storage, fakeClient := newStorage(t) test := registrytest.New(t, fakeClient, storage.Etcd) - test.TestGet(validNewDaemon()) + test.TestGet(newValidDaemonSet()) } func TestList(t *testing.T) { storage, fakeClient := newStorage(t) test := registrytest.New(t, fakeClient, storage.Etcd) - test.TestList(validNewDaemon()) + test.TestList(newValidDaemonSet()) } func TestWatch(t *testing.T) { storage, fakeClient := newStorage(t) test := registrytest.New(t, fakeClient, storage.Etcd) test.TestWatch( - validDaemon, + validDaemonSet, // matching labels []labels.Set{ {"a": "b"}, diff --git a/pkg/registry/daemon/strategy.go b/pkg/registry/daemonset/strategy.go similarity index 53% rename from pkg/registry/daemon/strategy.go rename to pkg/registry/daemonset/strategy.go index 2a9e7cb3796..9861ea464c1 100644 --- a/pkg/registry/daemon/strategy.go +++ b/pkg/registry/daemonset/strategy.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package daemon +package daemonset import ( "fmt" @@ -30,32 +30,32 @@ import ( "k8s.io/kubernetes/pkg/util/fielderrors" ) -// daemonStrategy implements verification logic for daemons. -type daemonStrategy struct { +// daemonSetStrategy implements verification logic for daemon sets. +type daemonSetStrategy struct { runtime.ObjectTyper api.NameGenerator } -// Strategy is the default logic that applies when creating and updating Daemon objects. -var Strategy = daemonStrategy{api.Scheme, api.SimpleNameGenerator} +// Strategy is the default logic that applies when creating and updating DaemonSet objects. +var Strategy = daemonSetStrategy{api.Scheme, api.SimpleNameGenerator} -// NamespaceScoped returns true because all Daemons need to be within a namespace. -func (daemonStrategy) NamespaceScoped() bool { +// NamespaceScoped returns true because all DaemonSets need to be within a namespace. +func (daemonSetStrategy) NamespaceScoped() bool { return true } -// PrepareForCreate clears the status of a daemon before creation. -func (daemonStrategy) PrepareForCreate(obj runtime.Object) { - daemon := obj.(*expapi.Daemon) - daemon.Status = expapi.DaemonStatus{} +// PrepareForCreate clears the status of a daemon set before creation. +func (daemonSetStrategy) PrepareForCreate(obj runtime.Object) { + daemonSet := obj.(*expapi.DaemonSet) + daemonSet.Status = expapi.DaemonSetStatus{} - daemon.Generation = 1 + daemonSet.Generation = 1 } // PrepareForUpdate clears fields that are not allowed to be set by end users on update. -func (daemonStrategy) PrepareForUpdate(obj, old runtime.Object) { - newDaemon := obj.(*expapi.Daemon) - oldDaemon := old.(*expapi.Daemon) +func (daemonSetStrategy) PrepareForUpdate(obj, old runtime.Object) { + newDaemonSet := obj.(*expapi.DaemonSet) + oldDaemonSet := old.(*expapi.DaemonSet) // Any changes to the spec increment the generation number, any changes to the // status should reflect the generation number of the corresponding object. We push @@ -64,59 +64,59 @@ func (daemonStrategy) PrepareForUpdate(obj, old runtime.Object) { // we can at first -- since obj contains spec -- but in the future we will probably make // status its own object, and even if we don't, writes may be the result of a // read-update-write loop, so the contents of spec may not actually be the spec that - // the controller has *seen*. + // the manager has *seen*. // // TODO: Any changes to a part of the object that represents desired state (labels, // annotations etc) should also increment the generation. - if !reflect.DeepEqual(oldDaemon.Spec, newDaemon.Spec) { - newDaemon.Generation = oldDaemon.Generation + 1 + if !reflect.DeepEqual(oldDaemonSet.Spec, newDaemonSet.Spec) { + newDaemonSet.Generation = oldDaemonSet.Generation + 1 } } -// Validate validates a new daemon. -func (daemonStrategy) Validate(ctx api.Context, obj runtime.Object) fielderrors.ValidationErrorList { - daemon := obj.(*expapi.Daemon) - return validation.ValidateDaemon(daemon) +// Validate validates a new daemon set. +func (daemonSetStrategy) Validate(ctx api.Context, obj runtime.Object) fielderrors.ValidationErrorList { + daemonSet := obj.(*expapi.DaemonSet) + return validation.ValidateDaemonSet(daemonSet) } -// AllowCreateOnUpdate is false for daemon; this means a POST is +// AllowCreateOnUpdate is false for daemon set; this means a POST is // needed to create one -func (daemonStrategy) AllowCreateOnUpdate() bool { +func (daemonSetStrategy) AllowCreateOnUpdate() bool { return false } // ValidateUpdate is the default update validation for an end user. -func (daemonStrategy) ValidateUpdate(ctx api.Context, obj, old runtime.Object) fielderrors.ValidationErrorList { - validationErrorList := validation.ValidateDaemon(obj.(*expapi.Daemon)) - updateErrorList := validation.ValidateDaemonUpdate(old.(*expapi.Daemon), obj.(*expapi.Daemon)) +func (daemonSetStrategy) ValidateUpdate(ctx api.Context, obj, old runtime.Object) fielderrors.ValidationErrorList { + validationErrorList := validation.ValidateDaemonSet(obj.(*expapi.DaemonSet)) + updateErrorList := validation.ValidateDaemonSetUpdate(old.(*expapi.DaemonSet), obj.(*expapi.DaemonSet)) return append(validationErrorList, updateErrorList...) } -// AllowUnconditionalUpdate is the default update policy for daemon objects. -func (daemonStrategy) AllowUnconditionalUpdate() bool { +// AllowUnconditionalUpdate is the default update policy for daemon set objects. +func (daemonSetStrategy) AllowUnconditionalUpdate() bool { return true } -// DaemonToSelectableFields returns a field set that represents the object. -func DaemonToSelectableFields(daemon *expapi.Daemon) fields.Set { +// DaemonSetToSelectableFields returns a field set that represents the object. +func DaemonSetToSelectableFields(daemon *expapi.DaemonSet) fields.Set { return fields.Set{ "metadata.name": daemon.Name, } } -// MatchDaemon is the filter used by the generic etcd backend to route +// MatchSetDaemon is the filter used by the generic etcd backend to route // watch events from etcd to clients of the apiserver only interested in specific // labels/fields. -func MatchDaemon(label labels.Selector, field fields.Selector) generic.Matcher { +func MatchDaemonSet(label labels.Selector, field fields.Selector) generic.Matcher { return &generic.SelectionPredicate{ Label: label, Field: field, GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, error) { - daemon, ok := obj.(*expapi.Daemon) + ds, ok := obj.(*expapi.DaemonSet) if !ok { - return nil, nil, fmt.Errorf("given object is not a daemon.") + return nil, nil, fmt.Errorf("given object is not a ds.") } - return labels.Set(daemon.ObjectMeta.Labels), DaemonToSelectableFields(daemon), nil + return labels.Set(ds.ObjectMeta.Labels), DaemonSetToSelectableFields(ds), nil }, } } From 662e41cdcc715773a39aba1244e4081b304e94a3 Mon Sep 17 00:00:00 2001 From: Quinton Hoole Date: Thu, 10 Sep 2015 09:08:19 -0700 Subject: [PATCH 42/46] Banish services up and down e2e test in parallel to flaky. --- hack/jenkins/e2e.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 56f8b8e6728..bec74401388 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -138,6 +138,7 @@ GCE_PARALLEL_FLAKY_TESTS=( "Services.*identically\snamed" "Services.*release.*load\sbalancer" "Services.*endpoint" + "Services.*up\sand\sdown" ) # Tests that should not run on soak cluster. From 746dcb807548271bd06cd4805771e037ac8d9d6d Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Thu, 10 Sep 2015 18:26:04 +0000 Subject: [PATCH 43/46] Make fluentd-es output its warning logs and fluentd-gcp mount /var/lib/docker read-only. --- cluster/saltbase/salt/fluentd-es/fluentd-es.yaml | 2 +- cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml | 1 + docs/getting-started-guides/logging.md | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml index caf90526c26..c2105b29476 100644 --- a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml +++ b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml @@ -11,7 +11,7 @@ spec: limits: cpu: 100m args: - - -qq + - -q volumeMounts: - name: varlog mountPath: /var/log diff --git a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml index 6eae4c8f699..a7749b75801 100644 --- a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml +++ b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml @@ -19,6 +19,7 @@ spec: mountPath: /varlog - name: containers mountPath: /var/lib/docker/containers + readOnly: true terminationGracePeriodSeconds: 30 volumes: - name: varlog diff --git a/docs/getting-started-guides/logging.md b/docs/getting-started-guides/logging.md index de308b9e84c..82ab815eec3 100644 --- a/docs/getting-started-guides/logging.md +++ b/docs/getting-started-guides/logging.md @@ -182,6 +182,7 @@ spec: mountPath: /varlog - name: containers mountPath: /var/lib/docker/containers + readOnly: true terminationGracePeriodSeconds: 30 volumes: - name: varlog From 08442974bb4d30d99807ec17b519e891cd917594 Mon Sep 17 00:00:00 2001 From: Jeff Lowdermilk Date: Thu, 10 Sep 2015 11:46:37 -0700 Subject: [PATCH 44/46] Revert "Turning on pod autoscaler on GCE." --- cluster/gce/config-default.sh | 10 +--------- cluster/gce/config-test.sh | 11 +---------- cluster/gce/configure-vm.sh | 6 ------ cluster/gce/coreos/helper.sh | 2 -- cluster/gce/debian/helper.sh | 2 -- cluster/gce/util.sh | 13 ------------- .../kube-controller-manager.manifest | 6 +----- hack/verify-flags/exceptions.txt | 3 ++- 8 files changed, 5 insertions(+), 48 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 6aae9232ebb..c0a18753c08 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -44,8 +44,6 @@ MINION_TAG="${INSTANCE_PREFIX}-minion" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}" MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}" -RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" -ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}" # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 @@ -89,6 +87,7 @@ CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" # Optional: Create autoscaler for cluster's nodes. +# NOT WORKING YET! ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}" if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" @@ -96,13 +95,6 @@ if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" fi -# Optional: Enable feature for autoscaling number of pods -# Experimental feature, not ready for production use. -ENABLE_HORIZONTAL_POD_AUTOSCALER="${KUBE_ENABLE_HORIZONTAL_POD_AUTOSCALER:-false}" -if [[ "${ENABLE_HORIZONTAL_POD_AUTOSCALER}" == "true" ]]; then - ENABLE_EXPERIMENTAL_API=true -fi - # Admission Controllers to invoke prior to persisting objects in cluster ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index ef6ffa6f44a..a45477a3e7d 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -45,9 +45,6 @@ MINION_TAG="${INSTANCE_PREFIX}-minion" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.245.0.0/16}" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}" -RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" -ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}" - # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET @@ -95,6 +92,7 @@ CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" # Optional: Create autoscaler for cluster's nodes. +# NOT WORKING YET! ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}" if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" @@ -102,13 +100,6 @@ if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" fi -# Optional: Enable feature for autoscaling number of pods -# Experimental feature, not ready for production use. -ENABLE_HORIZONTAL_POD_AUTOSCALER="${KUBE_ENABLE_HORIZONTAL_POD_AUTOSCALER:-false}" -if [[ "${ENABLE_HORIZONTAL_POD_AUTOSCALER}" == "true" ]]; then - ENABLE_EXPERIMENTAL_API=true -fi - ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota # Optional: if set to true kube-up will automatically check for existing resources and clean them up. diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index c5ef423800a..c199a6643c5 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -277,7 +277,6 @@ dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")' -enable_horizontal_pod_autoscaler: '$(echo "$ENABLE_HORIZONTAL_POD_AUTOSCALER" | sed -e "s/'/''/g")' EOF if [ -n "${APISERVER_TEST_ARGS:-}" ]; then @@ -569,11 +568,6 @@ EOF # CIDR range. cat <>/etc/salt/minion.d/grains.conf cbr-cidr: ${MASTER_IP_RANGE} -EOF - fi - if [[ ! -z "${RUNTIME_CONFIG:-}" ]]; then - cat <>/etc/salt/minion.d/grains.conf - runtime_config: '$(echo "$RUNTIME_CONFIG" | sed -e "s/'/''/g")' EOF fi } diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index 846bb2591b0..5ef057acf4a 100755 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -54,8 +54,6 @@ KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) -ENABLE_HORIZONTAL_POD_AUTOSCALER: $(yaml-quote ${ENABLE_HORIZONTAL_POD_AUTOSCALER}) -RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME}) KUBERNETES_CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME}) RKT_VERSION: $(yaml-quote ${RKT_VERSION}) diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 694f29ec85e..6b929c2d9ee 100755 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -51,8 +51,6 @@ KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) -ENABLE_HORIZONTAL_POD_AUTOSCALER: $(yaml-quote ${ENABLE_HORIZONTAL_POD_AUTOSCALER}) -RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-}) KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-}) KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-}) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 604c855f236..c884c9c2195 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -53,18 +53,6 @@ function join_csv { # Verify prereqs function verify-prereqs { - if [[ "${ENABLE_EXPERIMENTAL_API}" == "true" ]]; then - if [[ -z "${RUNTIME_CONFIG}" ]]; then - RUNTIME_CONFIG="experimental/v1=true" - else - # TODO: add checking if RUNTIME_CONFIG contains "experimental/v1=false" and appending "experimental/v1=true" if not. - if echo "${RUNTIME_CONFIG}" | grep -q -v "experimental/v1=true"; then - echo "Experimental API should be turned on, but is not turned on in RUNTIME_CONFIG!" - exit 1 - fi - fi - fi - local cmd for cmd in gcloud gsutil; do if ! which "${cmd}" >/dev/null; then @@ -477,7 +465,6 @@ function write-master-env { if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then KUBELET_APISERVER="${MASTER_NAME}" fi - build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" } diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index a4f82888a01..7c0e214213a 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -1,7 +1,6 @@ {% set cluster_name = "" -%} {% set cluster_cidr = "" -%} {% set allocate_node_cidrs = "" -%} -{% set enable_horizontal_pod_autoscaler = "" -%} {% if pillar['instance_prefix'] is defined -%} {% set cluster_name = "--cluster-name=" + pillar['instance_prefix'] -%} @@ -12,9 +11,6 @@ {% if pillar['allocate_node_cidrs'] is defined -%} {% set allocate_node_cidrs = "--allocate-node-cidrs=" + pillar['allocate_node_cidrs'] -%} {% endif -%} -{% if pillar['enable_horizontal_pod_autoscaler'] is defined -%} - {% set enable_horizontal_pod_autoscaler = "--enable-horizontal-pod-autoscaler=" + pillar['enable_horizontal_pod_autoscaler'] -%} -{% endif -%} {% set cloud_provider = "" -%} {% set cloud_config = "" -%} @@ -38,7 +34,7 @@ {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%} {% endif -%} -{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + enable_horizontal_pod_autoscaler + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} +{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration {% if pillar['controller_manager_test_args'] is defined -%} diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index c541cc21188..a8816eff7e5 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -1,3 +1,4 @@ +cluster/addons/cluster-monitoring/README.md:Heapster enables monitoring of Kubernetes Clusters using [cAdvisor](https://github.com/google/cadvisor). The kubelet will communicate with an instance of cAdvisor running on localhost and proxy container stats to Heapster. Kubelet will attempt to connect to cAdvisor on port 4194 by default but this port can be configured with kubelet's `--cadvisor-port` run flag. Detailed information about heapster can be found [here](https://github.com/GoogleCloudPlatform/heapster). cluster/addons/registry/images/Dockerfile:ADD run_proxy.sh /usr/bin/run_proxy cluster/addons/registry/images/Dockerfile:CMD ["/usr/bin/run_proxy"] cluster/aws/templates/salt-minion.sh:# We set the hostname_override to the full EC2 private dns name @@ -38,7 +39,7 @@ cluster/saltbase/salt/kube-addons/kube-addons.sh:# Create admission_control obje cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + etcd_servers + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + " " + basic_auth_file + " " + min_request_timeout -%} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = params + " " + cluster_name + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + " " + token_auth_file + " " + bind_address + " " + pillar['log_level'] + " " + advertise_address + " " + proxy_ssh_options -%} -cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + enable_horizontal_pod_autoscaler + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} +cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers + ":7080" -%} From fd73a600a0cc405b5c6580fe7db366e063ad03d1 Mon Sep 17 00:00:00 2001 From: Brendan Burns Date: Thu, 3 Sep 2015 20:47:10 -0700 Subject: [PATCH 45/46] Add a code of conduct. --- README.md | 4 ++ code-of-conduct.md | 59 +++++++++++++++++++ docs/user-guide/kubectl/kubectl.md | 2 +- docs/user-guide/kubectl/kubectl_annotate.md | 2 +- .../kubectl/kubectl_api-versions.md | 2 +- docs/user-guide/kubectl/kubectl_attach.md | 2 +- .../kubectl/kubectl_cluster-info.md | 2 +- docs/user-guide/kubectl/kubectl_config.md | 2 +- .../kubectl/kubectl_config_set-cluster.md | 2 +- .../kubectl/kubectl_config_set-context.md | 2 +- .../kubectl/kubectl_config_set-credentials.md | 2 +- docs/user-guide/kubectl/kubectl_config_set.md | 2 +- .../kubectl/kubectl_config_unset.md | 2 +- .../kubectl/kubectl_config_use-context.md | 2 +- .../user-guide/kubectl/kubectl_config_view.md | 2 +- docs/user-guide/kubectl/kubectl_create.md | 2 +- docs/user-guide/kubectl/kubectl_delete.md | 2 +- docs/user-guide/kubectl/kubectl_describe.md | 2 +- docs/user-guide/kubectl/kubectl_exec.md | 2 +- docs/user-guide/kubectl/kubectl_expose.md | 2 +- docs/user-guide/kubectl/kubectl_get.md | 2 +- docs/user-guide/kubectl/kubectl_label.md | 2 +- docs/user-guide/kubectl/kubectl_logs.md | 2 +- docs/user-guide/kubectl/kubectl_namespace.md | 2 +- docs/user-guide/kubectl/kubectl_patch.md | 2 +- .../kubectl/kubectl_port-forward.md | 2 +- docs/user-guide/kubectl/kubectl_proxy.md | 2 +- docs/user-guide/kubectl/kubectl_replace.md | 2 +- .../kubectl/kubectl_rolling-update.md | 2 +- docs/user-guide/kubectl/kubectl_run.md | 2 +- docs/user-guide/kubectl/kubectl_scale.md | 2 +- docs/user-guide/kubectl/kubectl_stop.md | 2 +- docs/user-guide/kubectl/kubectl_version.md | 2 +- 33 files changed, 94 insertions(+), 31 deletions(-) create mode 100644 code-of-conduct.md diff --git a/README.md b/README.md index d94a9a2e831..8604bf4c5a1 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,10 @@ Do you want to help "shape the evolution of technologies that are container pack You should consider joining the [Cloud Native Computing Foundation](https://cncf.io/about). For details about who's involved and how Kubernetes plays a role, read [their announcement](https://cncf.io/news/announcement/2015/07/new-cloud-native-computing-foundation-drive-alignment-among-container). +### Code of conduct + +Participation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md). + #### Are you ready to add to the discussion? We have presence on: diff --git a/code-of-conduct.md b/code-of-conduct.md new file mode 100644 index 00000000000..0552eb7f6b8 --- /dev/null +++ b/code-of-conduct.md @@ -0,0 +1,59 @@ +## Kubernetes Community Code of Conduct + +### Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of fostering +an open and welcoming community, we pledge to respect all people who contribute +through reporting issues, posting feature requests, updating documentation, +submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free experience for +everyone, regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, body size, race, ethnicity, age, +religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic addresses, + without explicit permission +* Other unethical or unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are not +aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers +commit themselves to fairly and consistently applying these principles to every aspect +of managing this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This code of conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by +opening an issue or contacting one or more of the project maintainers. + +This Code of Conduct is adapted from the Contributor Covenant +(http://contributor-covenant.org), version 1.2.0, available at +http://contributor-covenant.org/version/1/2/0/ + +### Kubernetes Events Code of Conduct +Kubernetes events are working conferences intended for professional networking and collaboration in the +Kubernetes community. Attendees are expected to behave according to professional standards and in accordance +with their employer's policies on appropriate workplace behavior. + +While at Kubernetes events or related social networking opportunities, attendees should not engage in +discriminatory or offensive speech or actions regarding gender, sexuality, race, or religion. Speakers should +be especially aware of these concerns. + +The Kubernetes team does not condone any statements by speakers contrary to these standards. The Kubernetes +team reserves the right to deny entrance and/or eject from an event (without refund) any individual found to +be engaging in discriminatory or offensive speech or actions. + +Please bring any concerns to to the immediate attention of Kubernetes event staff + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/code-of-conduct.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl.md b/docs/user-guide/kubectl/kubectl.md index cf8c2085cfb..a9be3c5f030 100644 --- a/docs/user-guide/kubectl/kubectl.md +++ b/docs/user-guide/kubectl/kubectl.md @@ -100,7 +100,7 @@ kubectl * [kubectl stop](kubectl_stop.md) - Deprecated: Gracefully shut down a resource by name or filename. * [kubectl version](kubectl_version.md) - Print the client and server version information. -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.476725335 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.165115265 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_annotate.md b/docs/user-guide/kubectl/kubectl_annotate.md index 21909136b72..a6b7831c3c6 100644 --- a/docs/user-guide/kubectl/kubectl_annotate.md +++ b/docs/user-guide/kubectl/kubectl_annotate.md @@ -119,7 +119,7 @@ $ kubectl annotate pods foo description- * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-02 06:24:17.720533039 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.16095949 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_annotate.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_api-versions.md b/docs/user-guide/kubectl/kubectl_api-versions.md index eaeef7866f5..15051d3f917 100644 --- a/docs/user-guide/kubectl/kubectl_api-versions.md +++ b/docs/user-guide/kubectl/kubectl_api-versions.md @@ -76,7 +76,7 @@ kubectl api-versions * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.476265479 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.164255617 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_api-versions.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_attach.md b/docs/user-guide/kubectl/kubectl_attach.md index 6155446b7d0..c47cf2035ec 100644 --- a/docs/user-guide/kubectl/kubectl_attach.md +++ b/docs/user-guide/kubectl/kubectl_attach.md @@ -98,7 +98,7 @@ $ kubectl attach 123456-7890 -c ruby-container -i -t * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.471309711 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.155651469 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_attach.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_cluster-info.md b/docs/user-guide/kubectl/kubectl_cluster-info.md index f7728387eab..465984de41b 100644 --- a/docs/user-guide/kubectl/kubectl_cluster-info.md +++ b/docs/user-guide/kubectl/kubectl_cluster-info.md @@ -76,7 +76,7 @@ kubectl cluster-info * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.476078738 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.163962347 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_cluster-info.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_config.md b/docs/user-guide/kubectl/kubectl_config.md index a16a4603b21..f0eb708b869 100644 --- a/docs/user-guide/kubectl/kubectl_config.md +++ b/docs/user-guide/kubectl/kubectl_config.md @@ -94,7 +94,7 @@ kubectl config SUBCOMMAND * [kubectl config use-context](kubectl_config_use-context.md) - Sets the current-context in a kubeconfig file * [kubectl config view](kubectl_config_view.md) - displays Merged kubeconfig settings or a specified kubeconfig file. -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.475888484 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.163685546 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_config_set-cluster.md b/docs/user-guide/kubectl/kubectl_config_set-cluster.md index a24a48be32e..414f7cc08a0 100644 --- a/docs/user-guide/kubectl/kubectl_config_set-cluster.md +++ b/docs/user-guide/kubectl/kubectl_config_set-cluster.md @@ -96,7 +96,7 @@ $ kubectl config set-cluster e2e --insecure-skip-tls-verify=true * [kubectl config](kubectl_config.md) - config modifies kubeconfig files -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.474677631 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.161700827 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_set-cluster.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_config_set-context.md b/docs/user-guide/kubectl/kubectl_config_set-context.md index 116d2802511..15643a2d8c8 100644 --- a/docs/user-guide/kubectl/kubectl_config_set-context.md +++ b/docs/user-guide/kubectl/kubectl_config_set-context.md @@ -89,7 +89,7 @@ $ kubectl config set-context gce --user=cluster-admin * [kubectl config](kubectl_config.md) - config modifies kubeconfig files -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.475093212 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.162402642 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_set-context.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_config_set-credentials.md b/docs/user-guide/kubectl/kubectl_config_set-credentials.md index dc84b808ce6..c4663ae0e34 100644 --- a/docs/user-guide/kubectl/kubectl_config_set-credentials.md +++ b/docs/user-guide/kubectl/kubectl_config_set-credentials.md @@ -109,7 +109,7 @@ $ kubectl config set-credentials cluster-admin --client-certificate=~/.kube/admi * [kubectl config](kubectl_config.md) - config modifies kubeconfig files -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.474882527 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.162045132 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_set-credentials.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_config_set.md b/docs/user-guide/kubectl/kubectl_config_set.md index ab0229e3a81..9f72013f061 100644 --- a/docs/user-guide/kubectl/kubectl_config_set.md +++ b/docs/user-guide/kubectl/kubectl_config_set.md @@ -78,7 +78,7 @@ kubectl config set PROPERTY_NAME PROPERTY_VALUE * [kubectl config](kubectl_config.md) - config modifies kubeconfig files -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.475281504 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.162716308 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_set.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_config_unset.md b/docs/user-guide/kubectl/kubectl_config_unset.md index 5f86a0f58d9..1f609d34faf 100644 --- a/docs/user-guide/kubectl/kubectl_config_unset.md +++ b/docs/user-guide/kubectl/kubectl_config_unset.md @@ -77,7 +77,7 @@ kubectl config unset PROPERTY_NAME * [kubectl config](kubectl_config.md) - config modifies kubeconfig files -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.475473658 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.163015642 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_unset.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_config_use-context.md b/docs/user-guide/kubectl/kubectl_config_use-context.md index 6a62618aa4c..d2dbdc773fb 100644 --- a/docs/user-guide/kubectl/kubectl_config_use-context.md +++ b/docs/user-guide/kubectl/kubectl_config_use-context.md @@ -76,7 +76,7 @@ kubectl config use-context CONTEXT_NAME * [kubectl config](kubectl_config.md) - config modifies kubeconfig files -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.475674294 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.163336177 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_use-context.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_config_view.md b/docs/user-guide/kubectl/kubectl_config_view.md index bbfadeb2d91..2153266a154 100644 --- a/docs/user-guide/kubectl/kubectl_config_view.md +++ b/docs/user-guide/kubectl/kubectl_config_view.md @@ -103,7 +103,7 @@ $ kubectl config view -o template --template='{{range .users}}{{ if eq .name "e2 * [kubectl config](kubectl_config.md) - config modifies kubeconfig files -###### Auto generated by spf13/cobra at 2015-08-29 13:01:26.775349034 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.161359997 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_view.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_create.md b/docs/user-guide/kubectl/kubectl_create.md index f718dc10fee..aeaf523944c 100644 --- a/docs/user-guide/kubectl/kubectl_create.md +++ b/docs/user-guide/kubectl/kubectl_create.md @@ -96,7 +96,7 @@ $ cat pod.json | kubectl create -f - * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.469492371 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.152429973 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_create.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_delete.md b/docs/user-guide/kubectl/kubectl_delete.md index 18fc8061ca9..fb8a25085e4 100644 --- a/docs/user-guide/kubectl/kubectl_delete.md +++ b/docs/user-guide/kubectl/kubectl_delete.md @@ -119,7 +119,7 @@ $ kubectl delete pods --all * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.470182255 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.153952299 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_delete.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_describe.md b/docs/user-guide/kubectl/kubectl_describe.md index 7eac8d62820..8337c2ddbd6 100644 --- a/docs/user-guide/kubectl/kubectl_describe.md +++ b/docs/user-guide/kubectl/kubectl_describe.md @@ -119,7 +119,7 @@ $ kubectl describe pods frontend * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.469291072 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.152057668 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_describe.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_exec.md b/docs/user-guide/kubectl/kubectl_exec.md index a1471e6f347..0324769af11 100644 --- a/docs/user-guide/kubectl/kubectl_exec.md +++ b/docs/user-guide/kubectl/kubectl_exec.md @@ -99,7 +99,7 @@ $ kubectl exec 123456-7890 -c ruby-container -i -t -- bash -il * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.471517301 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.156052759 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_exec.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_expose.md b/docs/user-guide/kubectl/kubectl_expose.md index 397de0db7a2..5d1e82a957c 100644 --- a/docs/user-guide/kubectl/kubectl_expose.md +++ b/docs/user-guide/kubectl/kubectl_expose.md @@ -121,7 +121,7 @@ $ kubectl expose rc streamer --port=4100 --protocol=udp --name=video-stream * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 09:05:42.928698484 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.159044239 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_expose.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_get.md b/docs/user-guide/kubectl/kubectl_get.md index ca42aba4b53..a62e562d6c6 100644 --- a/docs/user-guide/kubectl/kubectl_get.md +++ b/docs/user-guide/kubectl/kubectl_get.md @@ -132,7 +132,7 @@ $ kubectl get rc/web service/frontend pods/web-pod-13je7 * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-08-29 13:01:26.761418557 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.151532564 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_get.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_label.md b/docs/user-guide/kubectl/kubectl_label.md index 1b0ee3119bc..1a0a3fbdc6c 100644 --- a/docs/user-guide/kubectl/kubectl_label.md +++ b/docs/user-guide/kubectl/kubectl_label.md @@ -120,7 +120,7 @@ $ kubectl label pods foo bar- * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-08-29 13:01:26.773776248 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.160594172 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_label.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_logs.md b/docs/user-guide/kubectl/kubectl_logs.md index 79ac2d3b0b6..0a28240154b 100644 --- a/docs/user-guide/kubectl/kubectl_logs.md +++ b/docs/user-guide/kubectl/kubectl_logs.md @@ -98,7 +98,7 @@ $ kubectl logs -f 123456-7890 ruby-container * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.470591683 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.154570214 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_logs.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_namespace.md b/docs/user-guide/kubectl/kubectl_namespace.md index 3e686d46f00..14dbd6aee2b 100644 --- a/docs/user-guide/kubectl/kubectl_namespace.md +++ b/docs/user-guide/kubectl/kubectl_namespace.md @@ -79,7 +79,7 @@ kubectl namespace [namespace] * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.470380367 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.154262869 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_namespace.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_patch.md b/docs/user-guide/kubectl/kubectl_patch.md index aee0b3a18be..7c6b57d6c46 100644 --- a/docs/user-guide/kubectl/kubectl_patch.md +++ b/docs/user-guide/kubectl/kubectl_patch.md @@ -102,7 +102,7 @@ kubectl patch pod valid-pod -p '{"spec":{"containers":[{"name":"kubernetes-serve * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.469927571 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.153568922 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_patch.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_port-forward.md b/docs/user-guide/kubectl/kubectl_port-forward.md index ee8771f2a6a..189acda0fc5 100644 --- a/docs/user-guide/kubectl/kubectl_port-forward.md +++ b/docs/user-guide/kubectl/kubectl_port-forward.md @@ -99,7 +99,7 @@ $ kubectl port-forward mypod 0:5000 * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.471732563 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.156433376 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_port-forward.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_proxy.md b/docs/user-guide/kubectl/kubectl_proxy.md index 131d1c3060a..2be1e5f938b 100644 --- a/docs/user-guide/kubectl/kubectl_proxy.md +++ b/docs/user-guide/kubectl/kubectl_proxy.md @@ -121,7 +121,7 @@ $ kubectl proxy --api-prefix=/k8s-api * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.472010935 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.156927042 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_proxy.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_replace.md b/docs/user-guide/kubectl/kubectl_replace.md index ad65549704d..96a15b17b9e 100644 --- a/docs/user-guide/kubectl/kubectl_replace.md +++ b/docs/user-guide/kubectl/kubectl_replace.md @@ -110,7 +110,7 @@ kubectl replace --force -f ./pod.json * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.469727962 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.153166598 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_replace.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_rolling-update.md b/docs/user-guide/kubectl/kubectl_rolling-update.md index 26aea54d971..751c939335f 100644 --- a/docs/user-guide/kubectl/kubectl_rolling-update.md +++ b/docs/user-guide/kubectl/kubectl_rolling-update.md @@ -118,7 +118,7 @@ $ kubectl rolling-update frontend --image=image:v2 * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-08-29 13:01:26.768458355 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.154895732 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_rolling-update.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_run.md b/docs/user-guide/kubectl/kubectl_run.md index c664afd417c..7a8cf4da6a9 100644 --- a/docs/user-guide/kubectl/kubectl_run.md +++ b/docs/user-guide/kubectl/kubectl_run.md @@ -133,7 +133,7 @@ $ kubectl run nginx --image=nginx --command -- ... * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-07 06:40:12.142439604 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.15783835 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_run.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_scale.md b/docs/user-guide/kubectl/kubectl_scale.md index 068cbed9df0..1589bcf8157 100644 --- a/docs/user-guide/kubectl/kubectl_scale.md +++ b/docs/user-guide/kubectl/kubectl_scale.md @@ -108,7 +108,7 @@ $ kubectl scale --replicas=5 rc/foo rc/bar * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.471116954 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.155304524 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_scale.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_stop.md b/docs/user-guide/kubectl/kubectl_stop.md index 33c5fe100c9..29bf099afa8 100644 --- a/docs/user-guide/kubectl/kubectl_stop.md +++ b/docs/user-guide/kubectl/kubectl_stop.md @@ -110,7 +110,7 @@ $ kubectl stop -f path/to/resources * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.47250815 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.158360787 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_stop.md?pixel)]() diff --git a/docs/user-guide/kubectl/kubectl_version.md b/docs/user-guide/kubectl/kubectl_version.md index b5743a53c9a..1b1d2595079 100644 --- a/docs/user-guide/kubectl/kubectl_version.md +++ b/docs/user-guide/kubectl/kubectl_version.md @@ -82,7 +82,7 @@ kubectl version * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager -###### Auto generated by spf13/cobra at 2015-09-03 21:06:22.476464324 +0000 UTC +###### Auto generated by spf13/cobra at 2015-09-10 18:53:03.164581808 +0000 UTC [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_version.md?pixel)]() From 78ce5da9888c7c7dbd5a6f5e2bfb25ca3be76acb Mon Sep 17 00:00:00 2001 From: tummychow Date: Wed, 9 Sep 2015 10:45:01 -0700 Subject: [PATCH 46/46] Move util.StringSet into its own package A lot of packages use StringSet, but they don't use anything else from the util package. Moving StringSet into another package will shrink their dependency trees significantly. --- cmd/genconversion/conversion.go | 4 +- cmd/gendeepcopy/deep_copy.go | 4 +- cmd/integration/integration.go | 7 +- contrib/mesos/pkg/offers/offers.go | 18 ++-- contrib/mesos/pkg/queue/delay.go | 8 +- contrib/mesos/pkg/queue/historical.go | 8 +- contrib/mesos/pkg/scheduler/scheduler.go | 4 +- .../mesos/pkg/service/endpoints_controller.go | 5 +- hack/verify-flags/known-flags.txt | 1 + pkg/admission/handler.go | 6 +- pkg/api/helpers.go | 5 +- pkg/api/latest/latest.go | 6 +- pkg/api/mapper.go | 4 +- pkg/api/serialization_test.go | 7 +- pkg/api/validation/validation.go | 39 ++++---- pkg/api/validation/validation_test.go | 5 +- pkg/apiserver/apiserver.go | 3 +- pkg/apiserver/handlers.go | 5 +- pkg/apiserver/handlers_test.go | 4 +- pkg/client/unversioned/cache/delta_fifo.go | 4 +- .../cache/expiration_cache_fakes.go | 3 +- .../cache/expiration_cache_test.go | 7 +- pkg/client/unversioned/cache/index.go | 4 +- pkg/client/unversioned/cache/listers_test.go | 26 ++--- pkg/client/unversioned/cache/store_test.go | 16 ++-- .../unversioned/cache/thread_safe_store.go | 6 +- pkg/client/unversioned/debugging.go | 6 +- pkg/client/unversioned/flags_test.go | 4 +- pkg/client/unversioned/helper.go | 5 +- pkg/client/unversioned/request.go | 3 +- .../providers/aws/aws_loadbalancer.go | 10 +- pkg/cloudprovider/providers/aws/aws_utils.go | 8 +- pkg/cloudprovider/providers/gce/gce.go | 4 +- pkg/controller/controller_utils_test.go | 5 +- .../endpoint/endpoints_controller.go | 5 +- pkg/controller/framework/controller_test.go | 8 +- .../namespace/namespace_controller.go | 3 +- .../namespace/namespace_controller_test.go | 5 +- pkg/controller/node/nodecontroller.go | 11 ++- pkg/controller/node/rate_limited_queue.go | 5 +- .../node/rate_limited_queue_test.go | 15 +-- .../replication_controller_test.go | 3 +- .../resource_quota_controller_test.go | 6 +- .../serviceaccounts_controller.go | 8 +- .../serviceaccounts_controller_test.go | 7 +- .../serviceaccount/tokens_controller.go | 6 +- pkg/credentialprovider/keyring.go | 4 +- pkg/expapi/latest/latest.go | 6 +- pkg/expapi/validation/validation.go | 3 +- .../cmd/config/navigation_step_parser.go | 8 +- pkg/kubectl/cmd/log.go | 4 +- pkg/kubectl/describe.go | 4 +- pkg/kubectl/resource/builder.go | 4 +- pkg/kubectl/resource/result.go | 4 +- pkg/kubectl/resource_printer.go | 3 +- pkg/kubectl/resource_printer_test.go | 7 +- pkg/kubectl/rolling_updater_test.go | 3 +- pkg/kubelet/config/config.go | 12 +-- pkg/kubelet/dockertools/fake_docker_client.go | 4 +- pkg/kubelet/dockertools/manager.go | 3 +- pkg/kubelet/dockertools/manager_test.go | 7 +- pkg/kubelet/image_manager.go | 7 +- pkg/kubelet/image_manager_test.go | 4 +- pkg/kubelet/kubelet.go | 13 +-- pkg/kubelet/mirror_client_test.go | 6 +- pkg/labels/selector.go | 23 ++--- pkg/labels/selector_test.go | 94 +++++++++---------- pkg/master/master.go | 7 +- pkg/registry/generic/etcd/etcd_test.go | 11 ++- .../service/ipallocator/allocator_test.go | 6 +- .../service/portallocator/allocator_test.go | 3 +- pkg/runtime/conversion_generator.go | 6 +- pkg/runtime/deep_copy_generator.go | 6 +- pkg/storage/cacher_test.go | 3 +- pkg/storage/watch_cache_test.go | 5 +- pkg/util/bandwidth/linux.go | 4 +- pkg/util/iptables/iptables.go | 6 +- pkg/util/iptables/iptables_test.go | 30 +++--- pkg/util/proxy/transport.go | 54 +++++------ pkg/util/{ => sets}/set.go | 42 ++++----- pkg/util/{ => sets}/set_test.go | 34 +++---- pkg/volume/gce_pd/gce_util.go | 7 +- .../namespace/lifecycle/admission.go | 6 +- .../pkg/admission/serviceaccount/admission.go | 10 +- .../algorithmprovider/defaults/defaults.go | 10 +- plugin/pkg/scheduler/factory/factory.go | 7 +- plugin/pkg/scheduler/factory/plugins.go | 12 +-- plugin/pkg/scheduler/generic_scheduler.go | 6 +- .../pkg/scheduler/generic_scheduler_test.go | 12 +-- test/e2e/daemon_restart.go | 7 +- test/e2e/density.go | 3 +- test/e2e/kubelet.go | 15 +-- test/e2e/kubelet_stats.go | 3 +- test/e2e/load.go | 4 +- test/e2e/service_latency.go | 3 +- test/e2e/util.go | 13 +-- test/images/network-tester/webserver.go | 4 +- test/integration/service_account_test.go | 4 +- 98 files changed, 473 insertions(+), 429 deletions(-) rename pkg/util/{ => sets}/set.go (79%) rename pkg/util/{ => sets}/set_test.go (88%) diff --git a/cmd/genconversion/conversion.go b/cmd/genconversion/conversion.go index 69dfbd08a2b..b89ecd5b5ce 100644 --- a/cmd/genconversion/conversion.go +++ b/cmd/genconversion/conversion.go @@ -30,7 +30,7 @@ import ( _ "k8s.io/kubernetes/pkg/expapi" _ "k8s.io/kubernetes/pkg/expapi/v1" pkg_runtime "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "github.com/golang/glog" flag "github.com/spf13/pflag" @@ -84,7 +84,7 @@ func main() { glog.Errorf("error while generating conversion functions for %v: %v", knownType, err) } } - generator.RepackImports(util.NewStringSet()) + generator.RepackImports(sets.NewString()) if err := generator.WriteImports(data); err != nil { glog.Fatalf("error while writing imports: %v", err) } diff --git a/cmd/gendeepcopy/deep_copy.go b/cmd/gendeepcopy/deep_copy.go index 7c10aa9fd5c..7cfb05d9332 100644 --- a/cmd/gendeepcopy/deep_copy.go +++ b/cmd/gendeepcopy/deep_copy.go @@ -30,7 +30,7 @@ import ( _ "k8s.io/kubernetes/pkg/expapi" _ "k8s.io/kubernetes/pkg/expapi/v1" pkg_runtime "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "github.com/golang/glog" flag "github.com/spf13/pflag" @@ -80,7 +80,7 @@ func main() { } versionPath := path.Join(pkgBase, group, version) - generator := pkg_runtime.NewDeepCopyGenerator(api.Scheme.Raw(), versionPath, util.NewStringSet("k8s.io/kubernetes")) + generator := pkg_runtime.NewDeepCopyGenerator(api.Scheme.Raw(), versionPath, sets.NewString("k8s.io/kubernetes")) generator.AddImport(path.Join(pkgBase, "api")) if len(*overwrites) > 0 { diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go index fd4ecf9fb75..82d9357243b 100644 --- a/cmd/integration/integration.go +++ b/cmd/integration/integration.go @@ -55,6 +55,7 @@ import ( "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/tools/etcdtest" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/volume/empty_dir" "k8s.io/kubernetes/plugin/pkg/admission/admit" @@ -738,7 +739,7 @@ func runMasterServiceTest(client *client.Client) { glog.Fatalf("unexpected error listing services: %v", err) } var foundRW bool - found := util.StringSet{} + found := sets.String{} for i := range svcList.Items { found.Insert(svcList.Items[i].Name) if svcList.Items[i].Name == "kubernetes" { @@ -864,7 +865,7 @@ func runServiceTest(client *client.Client) { if err != nil { glog.Fatalf("Failed to list services across namespaces: %v", err) } - names := util.NewStringSet() + names := sets.NewString() for _, svc := range svcList.Items { names.Insert(fmt.Sprintf("%s/%s", svc.Namespace, svc.Name)) } @@ -1011,7 +1012,7 @@ func main() { // Check that kubelet tried to make the containers. // Using a set to list unique creation attempts. Our fake is // really stupid, so kubelet tries to create these multiple times. - createdConts := util.StringSet{} + createdConts := sets.String{} for _, p := range fakeDocker1.Created { // The last 8 characters are random, so slice them off. if n := len(p); n > 8 { diff --git a/contrib/mesos/pkg/offers/offers.go b/contrib/mesos/pkg/offers/offers.go index d91a6806f1d..962fc81663c 100644 --- a/contrib/mesos/pkg/offers/offers.go +++ b/contrib/mesos/pkg/offers/offers.go @@ -30,7 +30,7 @@ import ( "k8s.io/kubernetes/contrib/mesos/pkg/queue" "k8s.io/kubernetes/contrib/mesos/pkg/runtime" "k8s.io/kubernetes/pkg/client/unversioned/cache" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) const ( @@ -453,7 +453,7 @@ func (s *offerStorage) nextListener() *offerListener { // notify listeners if we find an acceptable offer for them. listeners // are garbage collected after a certain age (see offerListenerMaxAge). // ids lists offer IDs that are retrievable from offer storage. -func (s *offerStorage) notifyListeners(ids func() (util.StringSet, uint64)) { +func (s *offerStorage) notifyListeners(ids func() (sets.String, uint64)) { listener := s.nextListener() // blocking offerIds, version := ids() @@ -493,8 +493,8 @@ func (s *offerStorage) Init(done <-chan struct{}) { // cached offer ids for the purposes of listener notification idCache := &stringsCache{ - refill: func() util.StringSet { - result := util.NewStringSet() + refill: func() sets.String { + result := sets.NewString() for _, v := range s.offers.List() { if offer, ok := v.(Perishable); ok { result.Insert(offer.Id()) @@ -510,14 +510,14 @@ func (s *offerStorage) Init(done <-chan struct{}) { type stringsCache struct { expiresAt time.Time - cached util.StringSet + cached sets.String ttl time.Duration - refill func() util.StringSet + refill func() sets.String version uint64 } // not thread-safe -func (c *stringsCache) Strings() (util.StringSet, uint64) { +func (c *stringsCache) Strings() (sets.String, uint64) { now := time.Now() if c.expiresAt.Before(now) { old := c.cached @@ -549,8 +549,8 @@ func (self *slaveStorage) add(slaveId, offerId string) { } // delete the slave-offer mappings for slaveId, returns the IDs of the offers that were unmapped -func (self *slaveStorage) deleteSlave(slaveId string) util.StringSet { - offerIds := util.NewStringSet() +func (self *slaveStorage) deleteSlave(slaveId string) sets.String { + offerIds := sets.NewString() self.Lock() defer self.Unlock() for oid, sid := range self.index { diff --git a/contrib/mesos/pkg/queue/delay.go b/contrib/mesos/pkg/queue/delay.go index 6ec436b71e8..be240223ade 100644 --- a/contrib/mesos/pkg/queue/delay.go +++ b/contrib/mesos/pkg/queue/delay.go @@ -21,7 +21,7 @@ import ( "sync" "time" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type qitem struct { @@ -277,13 +277,13 @@ func (f *DelayFIFO) List() []UniqueID { return list } -// ContainedIDs returns a util.StringSet containing all IDs of the stored items. +// ContainedIDs returns a stringset.StringSet containing all IDs of the stored items. // This is a snapshot of a moment in time, and one should keep in mind that // other go routines can add or remove items after you call this. -func (c *DelayFIFO) ContainedIDs() util.StringSet { +func (c *DelayFIFO) ContainedIDs() sets.String { c.rlock() defer c.runlock() - set := util.StringSet{} + set := sets.String{} for id := range c.items { set.Insert(id) } diff --git a/contrib/mesos/pkg/queue/historical.go b/contrib/mesos/pkg/queue/historical.go index 6fe65f21e2b..a9021c14b4d 100644 --- a/contrib/mesos/pkg/queue/historical.go +++ b/contrib/mesos/pkg/queue/historical.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type entry struct { @@ -177,13 +177,13 @@ func (f *HistoricalFIFO) ListKeys() []string { return list } -// ContainedIDs returns a util.StringSet containing all IDs of the stored items. +// ContainedIDs returns a stringset.StringSet containing all IDs of the stored items. // This is a snapshot of a moment in time, and one should keep in mind that // other go routines can add or remove items after you call this. -func (c *HistoricalFIFO) ContainedIDs() util.StringSet { +func (c *HistoricalFIFO) ContainedIDs() sets.String { c.lock.RLock() defer c.lock.RUnlock() - set := util.StringSet{} + set := sets.String{} for id, entry := range c.items { if entry.Is(DELETE_EVENT | POP_EVENT) { continue diff --git a/contrib/mesos/pkg/scheduler/scheduler.go b/contrib/mesos/pkg/scheduler/scheduler.go index 0148e0d9368..fc722a5370c 100644 --- a/contrib/mesos/pkg/scheduler/scheduler.go +++ b/contrib/mesos/pkg/scheduler/scheduler.go @@ -47,7 +47,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/tools" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type Slave struct { @@ -711,7 +711,7 @@ func (k *KubernetesScheduler) explicitlyReconcileTasks(driver bindings.Scheduler // tell mesos to send us the latest status updates for all the non-terminal tasks that we know about statusList := []*mesos.TaskStatus{} - remaining := util.KeySet(reflect.ValueOf(taskToSlave)) + remaining := sets.KeySet(reflect.ValueOf(taskToSlave)) for taskId, slaveId := range taskToSlave { if slaveId == "" { delete(taskToSlave, taskId) diff --git a/contrib/mesos/pkg/service/endpoints_controller.go b/contrib/mesos/pkg/service/endpoints_controller.go index c38f692af9e..589aa0dfe3c 100644 --- a/contrib/mesos/pkg/service/endpoints_controller.go +++ b/contrib/mesos/pkg/service/endpoints_controller.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/workqueue" "k8s.io/kubernetes/pkg/watch" @@ -132,8 +133,8 @@ func (e *endpointController) Run(workers int, stopCh <-chan struct{}) { e.queue.ShutDown() } -func (e *endpointController) getPodServiceMemberships(pod *api.Pod) (util.StringSet, error) { - set := util.StringSet{} +func (e *endpointController) getPodServiceMemberships(pod *api.Pod) (sets.String, error) { + set := sets.String{} services, err := e.serviceStore.GetPodServices(pod) if err != nil { // don't log this error because this function makes pointless diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index cd7cfad7bac..9fe4e91dc48 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -194,6 +194,7 @@ path-override pod-cidr pod-eviction-timeout pod-infra-container-image +pod-running policy-config-file poll-interval portal-net diff --git a/pkg/admission/handler.go b/pkg/admission/handler.go index fe79285fd6a..a0d26c46971 100644 --- a/pkg/admission/handler.go +++ b/pkg/admission/handler.go @@ -17,13 +17,13 @@ limitations under the License. package admission import ( - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // Handler is a base for admission control handlers that // support a predefined set of operations type Handler struct { - operations util.StringSet + operations sets.String } // Handles returns true for methods that this handler supports @@ -34,7 +34,7 @@ func (h *Handler) Handles(operation Operation) bool { // NewHandler creates a new base handler that handles the passed // in operations func NewHandler(ops ...Operation) *Handler { - operations := util.NewStringSet() + operations := sets.NewString() for _, op := range ops { operations.Insert(string(op)) } diff --git a/pkg/api/helpers.go b/pkg/api/helpers.go index 56e9836e306..931ff3b84b1 100644 --- a/pkg/api/helpers.go +++ b/pkg/api/helpers.go @@ -28,6 +28,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "github.com/davecgh/go-spew/spew" ) @@ -77,7 +78,7 @@ var Semantic = conversion.EqualitiesOrDie( }, ) -var standardResources = util.NewStringSet( +var standardResources = sets.NewString( string(ResourceMemory), string(ResourceCPU), string(ResourcePods), @@ -111,7 +112,7 @@ func IsServiceIPRequested(service *Service) bool { return service.Spec.ClusterIP == "" } -var standardFinalizers = util.NewStringSet( +var standardFinalizers = sets.NewString( string(FinalizerKubernetes)) func IsStandardFinalizerName(str string) bool { diff --git a/pkg/api/latest/latest.go b/pkg/api/latest/latest.go index 39bd0a319a9..2e53b4f00f3 100644 --- a/pkg/api/latest/latest.go +++ b/pkg/api/latest/latest.go @@ -25,7 +25,7 @@ import ( "k8s.io/kubernetes/pkg/api/registered" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // Version is the string that represents the current external default version. @@ -79,7 +79,7 @@ func init() { // the list of kinds that are scoped at the root of the api hierarchy // if a kind is not enumerated here, it is assumed to have a namespace scope - rootScoped := util.NewStringSet( + rootScoped := sets.NewString( "Node", "Minion", "Namespace", @@ -87,7 +87,7 @@ func init() { ) // these kinds should be excluded from the list of resources - ignoredKinds := util.NewStringSet( + ignoredKinds := sets.NewString( "ListOptions", "DeleteOptions", "Status", diff --git a/pkg/api/mapper.go b/pkg/api/mapper.go index dba16d36af9..600973aa093 100644 --- a/pkg/api/mapper.go +++ b/pkg/api/mapper.go @@ -20,7 +20,7 @@ import ( "strings" "k8s.io/kubernetes/pkg/api/meta" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) var RESTMapper meta.RESTMapper @@ -34,7 +34,7 @@ func RegisterRESTMapper(m meta.RESTMapper) { } func NewDefaultRESTMapper(group string, versions []string, interfacesFunc meta.VersionInterfacesFunc, - importPathPrefix string, ignoredKinds, rootScoped util.StringSet) *meta.DefaultRESTMapper { + importPathPrefix string, ignoredKinds, rootScoped sets.String) *meta.DefaultRESTMapper { mapper := meta.NewDefaultRESTMapper(group, versions, interfacesFunc) // enumerate all supported versions, get the kinds, and register with the mapper how to address diff --git a/pkg/api/serialization_test.go b/pkg/api/serialization_test.go index 686a33f63d9..beccf9604b5 100644 --- a/pkg/api/serialization_test.go +++ b/pkg/api/serialization_test.go @@ -31,6 +31,7 @@ import ( apitesting "k8s.io/kubernetes/pkg/api/testing" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" _ "k8s.io/kubernetes/pkg/expapi" _ "k8s.io/kubernetes/pkg/expapi/v1" @@ -87,7 +88,7 @@ func roundTrip(t *testing.T, codec runtime.Codec, item runtime.Object) { // roundTripSame verifies the same source object is tested in all API versions. func roundTripSame(t *testing.T, item runtime.Object, except ...string) { - set := util.NewStringSet(except...) + set := sets.NewString(except...) seed := rand.Int63() fuzzInternalObject(t, "", item, seed) version := testapi.Default.Version() @@ -119,8 +120,8 @@ func TestList(t *testing.T) { roundTripSame(t, item) } -var nonRoundTrippableTypes = util.NewStringSet() -var nonInternalRoundTrippableTypes = util.NewStringSet("List", "ListOptions", "PodExecOptions", "PodAttachOptions") +var nonRoundTrippableTypes = sets.NewString() +var nonInternalRoundTrippableTypes = sets.NewString("List", "ListOptions", "PodExecOptions", "PodAttachOptions") var nonRoundTrippableTypesByVersion = map[string][]string{} func TestRoundTripTypes(t *testing.T) { diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index d5965bce358..1829e7a8dbf 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/util" errs "k8s.io/kubernetes/pkg/util/fielderrors" + "k8s.io/kubernetes/pkg/util/sets" "github.com/golang/glog" ) @@ -307,10 +308,10 @@ func ValidateObjectMetaUpdate(new, old *api.ObjectMeta) errs.ValidationErrorList return allErrs } -func validateVolumes(volumes []api.Volume) (util.StringSet, errs.ValidationErrorList) { +func validateVolumes(volumes []api.Volume) (sets.String, errs.ValidationErrorList) { allErrs := errs.ValidationErrorList{} - allNames := util.StringSet{} + allNames := sets.String{} for i, vol := range volumes { el := validateSource(&vol.VolumeSource).Prefix("source") if len(vol.Name) == 0 { @@ -497,7 +498,7 @@ func validateGlusterfs(glusterfs *api.GlusterfsVolumeSource) errs.ValidationErro return allErrs } -var validDownwardAPIFieldPathExpressions = util.NewStringSet("metadata.name", "metadata.namespace", "metadata.labels", "metadata.annotations") +var validDownwardAPIFieldPathExpressions = sets.NewString("metadata.name", "metadata.namespace", "metadata.labels", "metadata.annotations") func validateDownwardAPIVolumeSource(downwardAPIVolume *api.DownwardAPIVolumeSource) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} @@ -688,12 +689,12 @@ func ValidatePersistentVolumeClaimStatusUpdate(newPvc, oldPvc *api.PersistentVol return allErrs } -var supportedPortProtocols = util.NewStringSet(string(api.ProtocolTCP), string(api.ProtocolUDP)) +var supportedPortProtocols = sets.NewString(string(api.ProtocolTCP), string(api.ProtocolUDP)) func validatePorts(ports []api.ContainerPort) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} - allNames := util.StringSet{} + allNames := sets.String{} for i, port := range ports { pErrs := errs.ValidationErrorList{} if len(port.Name) > 0 { @@ -739,7 +740,7 @@ func validateEnv(vars []api.EnvVar) errs.ValidationErrorList { return allErrs } -var validFieldPathExpressionsEnv = util.NewStringSet("metadata.name", "metadata.namespace", "status.podIP") +var validFieldPathExpressionsEnv = sets.NewString("metadata.name", "metadata.namespace", "status.podIP") func validateEnvVarValueFrom(ev api.EnvVar) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} @@ -763,7 +764,7 @@ func validateEnvVarValueFrom(ev api.EnvVar) errs.ValidationErrorList { return allErrs } -func validateObjectFieldSelector(fs *api.ObjectFieldSelector, expressions *util.StringSet) errs.ValidationErrorList { +func validateObjectFieldSelector(fs *api.ObjectFieldSelector, expressions *sets.String) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} if fs.APIVersion == "" { @@ -782,7 +783,7 @@ func validateObjectFieldSelector(fs *api.ObjectFieldSelector, expressions *util. return allErrs } -func validateVolumeMounts(mounts []api.VolumeMount, volumes util.StringSet) errs.ValidationErrorList { +func validateVolumeMounts(mounts []api.VolumeMount, volumes sets.String) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} for i, mnt := range mounts { @@ -818,7 +819,7 @@ func validateProbe(probe *api.Probe) errs.ValidationErrorList { // AccumulateUniqueHostPorts extracts each HostPort of each Container, // accumulating the results and returning an error if any ports conflict. -func AccumulateUniqueHostPorts(containers []api.Container, accumulator *util.StringSet) errs.ValidationErrorList { +func AccumulateUniqueHostPorts(containers []api.Container, accumulator *sets.String) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} for ci, ctr := range containers { @@ -843,7 +844,7 @@ func AccumulateUniqueHostPorts(containers []api.Container, accumulator *util.Str // checkHostPortConflicts checks for colliding Port.HostPort values across // a slice of containers. func checkHostPortConflicts(containers []api.Container) errs.ValidationErrorList { - allPorts := util.StringSet{} + allPorts := sets.String{} return AccumulateUniqueHostPorts(containers, &allPorts) } @@ -865,7 +866,7 @@ func validateHTTPGetAction(http *api.HTTPGetAction) errs.ValidationErrorList { } else if http.Port.Kind == util.IntstrString && !util.IsValidPortName(http.Port.StrVal) { allErrors = append(allErrors, errs.NewFieldInvalid("port", http.Port.StrVal, portNameErrorMsg)) } - supportedSchemes := util.NewStringSet(string(api.URISchemeHTTP), string(api.URISchemeHTTPS)) + supportedSchemes := sets.NewString(string(api.URISchemeHTTP), string(api.URISchemeHTTPS)) if !supportedSchemes.Has(string(http.Scheme)) { allErrors = append(allErrors, errs.NewFieldInvalid("scheme", http.Scheme, fmt.Sprintf("must be one of %v", supportedSchemes.List()))) } @@ -930,14 +931,14 @@ func validatePullPolicy(ctr *api.Container) errs.ValidationErrorList { return allErrors } -func validateContainers(containers []api.Container, volumes util.StringSet) errs.ValidationErrorList { +func validateContainers(containers []api.Container, volumes sets.String) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} if len(containers) == 0 { return append(allErrs, errs.NewFieldRequired("")) } - allNames := util.StringSet{} + allNames := sets.String{} for i, ctr := range containers { cErrs := errs.ValidationErrorList{} if len(ctr.Name) == 0 { @@ -1130,8 +1131,8 @@ func ValidatePodTemplateUpdate(newPod, oldPod *api.PodTemplate) errs.ValidationE return allErrs } -var supportedSessionAffinityType = util.NewStringSet(string(api.ServiceAffinityClientIP), string(api.ServiceAffinityNone)) -var supportedServiceType = util.NewStringSet(string(api.ServiceTypeClusterIP), string(api.ServiceTypeNodePort), +var supportedSessionAffinityType = sets.NewString(string(api.ServiceAffinityClientIP), string(api.ServiceAffinityNone)) +var supportedServiceType = sets.NewString(string(api.ServiceTypeClusterIP), string(api.ServiceTypeNodePort), string(api.ServiceTypeLoadBalancer)) // ValidateService tests if required fields in the service are set. @@ -1150,7 +1151,7 @@ func ValidateService(service *api.Service) errs.ValidationErrorList { } } } - allPortNames := util.StringSet{} + allPortNames := sets.String{} for i := range service.Spec.Ports { allErrs = append(allErrs, validateServicePort(&service.Spec.Ports[i], len(service.Spec.Ports) > 1, &allPortNames).PrefixIndex(i).Prefix("spec.ports")...) } @@ -1220,7 +1221,7 @@ func ValidateService(service *api.Service) errs.ValidationErrorList { return allErrs } -func validateServicePort(sp *api.ServicePort, requireName bool, allNames *util.StringSet) errs.ValidationErrorList { +func validateServicePort(sp *api.ServicePort, requireName bool, allNames *sets.String) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} if requireName && sp.Name == "" { @@ -1441,7 +1442,7 @@ func ValidateLimitRange(limitRange *api.LimitRange) errs.ValidationErrorList { } limitTypeSet[limit.Type] = true - keys := util.StringSet{} + keys := sets.String{} min := map[string]resource.Quantity{} max := map[string]resource.Quantity{} defaults := map[string]resource.Quantity{} @@ -1884,7 +1885,7 @@ func ValidateThirdPartyResource(obj *api.ThirdPartyResource) errs.ValidationErro if len(obj.Name) == 0 { allErrs = append(allErrs, errs.NewFieldInvalid("name", obj.Name, "name must be non-empty")) } - versions := util.StringSet{} + versions := sets.String{} for ix := range obj.Versions { version := &obj.Versions[ix] if len(version.Name) == 0 { diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index 7a1a0167899..8e320114e23 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -30,6 +30,7 @@ import ( utilerrors "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/util/fielderrors" errors "k8s.io/kubernetes/pkg/util/fielderrors" + "k8s.io/kubernetes/pkg/util/sets" ) func expectPrefix(t *testing.T, prefix string, errs fielderrors.ValidationErrorList) { @@ -769,7 +770,7 @@ func TestValidateEnv(t *testing.T) { } func TestValidateVolumeMounts(t *testing.T) { - volumes := util.NewStringSet("abc", "123", "abc-123") + volumes := sets.NewString("abc", "123", "abc-123") successCase := []api.VolumeMount{ {Name: "abc", MountPath: "/foo"}, @@ -896,7 +897,7 @@ func getResourceLimits(cpu, memory string) api.ResourceList { } func TestValidateContainers(t *testing.T) { - volumes := util.StringSet{} + volumes := sets.String{} capabilities.SetForTests(capabilities.Capabilities{ AllowPrivileged: true, }) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index c87234edd3a..a15e626d09f 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -42,6 +42,7 @@ import ( "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/util/flushwriter" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/version" "github.com/emicklei/go-restful" @@ -115,7 +116,7 @@ const ( // It is expected that the provided path root prefix will serve all operations. Root MUST NOT end // in a slash. A restful WebService is created for the group and version. func (g *APIGroupVersion) InstallREST(container *restful.Container) error { - info := &APIRequestInfoResolver{util.NewStringSet(strings.TrimPrefix(g.Root, "/")), g.Mapper} + info := &APIRequestInfoResolver{sets.NewString(strings.TrimPrefix(g.Root, "/")), g.Mapper} prefix := path.Join(g.Root, g.Version) installer := &APIInstaller{ diff --git a/pkg/apiserver/handlers.go b/pkg/apiserver/handlers.go index 706e42854f4..6dfe6cb8afc 100644 --- a/pkg/apiserver/handlers.go +++ b/pkg/apiserver/handlers.go @@ -35,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/auth/authorizer" "k8s.io/kubernetes/pkg/httplog" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // specialVerbs contains just strings which are used in REST paths for special actions that don't fall under the normal @@ -351,7 +352,7 @@ type requestAttributeGetter struct { // NewAttributeGetter returns an object which implements the RequestAttributeGetter interface. func NewRequestAttributeGetter(requestContextMapper api.RequestContextMapper, restMapper meta.RESTMapper, apiRoots ...string) RequestAttributeGetter { - return &requestAttributeGetter{requestContextMapper, &APIRequestInfoResolver{util.NewStringSet(apiRoots...), restMapper}} + return &requestAttributeGetter{requestContextMapper, &APIRequestInfoResolver{sets.NewString(apiRoots...), restMapper}} } func (r *requestAttributeGetter) GetAttribs(req *http.Request) authorizer.Attributes { @@ -417,7 +418,7 @@ type APIRequestInfo struct { } type APIRequestInfoResolver struct { - APIPrefixes util.StringSet + APIPrefixes sets.String RestMapper meta.RESTMapper } diff --git a/pkg/apiserver/handlers_test.go b/pkg/apiserver/handlers_test.go index b9d948d73ba..c296a0cdb65 100644 --- a/pkg/apiserver/handlers_test.go +++ b/pkg/apiserver/handlers_test.go @@ -31,7 +31,7 @@ import ( "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/latest" "k8s.io/kubernetes/pkg/api/testapi" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type fakeRL bool @@ -246,7 +246,7 @@ func TestGetAPIRequestInfo(t *testing.T) { {"PUT", "/namespaces/other/finalize", "update", "", "other", "finalize", "", "", "", []string{"finalize"}}, } - apiRequestInfoResolver := &APIRequestInfoResolver{util.NewStringSet("api"), latest.RESTMapper} + apiRequestInfoResolver := &APIRequestInfoResolver{sets.NewString("api"), latest.RESTMapper} for _, successCase := range successCases { req, _ := http.NewRequest(successCase.method, successCase.url, nil) diff --git a/pkg/client/unversioned/cache/delta_fifo.go b/pkg/client/unversioned/cache/delta_fifo.go index 4b432208ba4..808a854556e 100644 --- a/pkg/client/unversioned/cache/delta_fifo.go +++ b/pkg/client/unversioned/cache/delta_fifo.go @@ -21,7 +21,7 @@ import ( "fmt" "sync" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "github.com/golang/glog" ) @@ -319,7 +319,7 @@ func (f *DeltaFIFO) Replace(list []interface{}, resourceVersion string) error { return nil } - keySet := make(util.StringSet, len(list)) + keySet := make(sets.String, len(list)) for _, item := range list { key, err := f.KeyOf(item) if err != nil { diff --git a/pkg/client/unversioned/cache/expiration_cache_fakes.go b/pkg/client/unversioned/cache/expiration_cache_fakes.go index 5fc380abc42..2e9a25d121a 100644 --- a/pkg/client/unversioned/cache/expiration_cache_fakes.go +++ b/pkg/client/unversioned/cache/expiration_cache_fakes.go @@ -18,6 +18,7 @@ package cache import ( "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type fakeThreadSafeMap struct { @@ -32,7 +33,7 @@ func (c *fakeThreadSafeMap) Delete(key string) { } type FakeExpirationPolicy struct { - NeverExpire util.StringSet + NeverExpire sets.String RetrieveKeyFunc KeyFunc } diff --git a/pkg/client/unversioned/cache/expiration_cache_test.go b/pkg/client/unversioned/cache/expiration_cache_test.go index 4ecccc54c2b..375ffcceaa3 100644 --- a/pkg/client/unversioned/cache/expiration_cache_test.go +++ b/pkg/client/unversioned/cache/expiration_cache_test.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func TestTTLExpirationBasic(t *testing.T) { @@ -30,7 +31,7 @@ func TestTTLExpirationBasic(t *testing.T) { ttlStore := NewFakeExpirationStore( testStoreKeyFunc, deleteChan, &FakeExpirationPolicy{ - NeverExpire: util.NewStringSet(), + NeverExpire: sets.NewString(), RetrieveKeyFunc: func(obj interface{}) (string, error) { return obj.(*timestampedEntry).obj.(testStoreObject).id, nil }, @@ -66,14 +67,14 @@ func TestTTLList(t *testing.T) { {id: "foo1", val: "bar1"}, {id: "foo2", val: "bar2"}, } - expireKeys := util.NewStringSet(testObjs[0].id, testObjs[2].id) + expireKeys := sets.NewString(testObjs[0].id, testObjs[2].id) deleteChan := make(chan string) defer close(deleteChan) ttlStore := NewFakeExpirationStore( testStoreKeyFunc, deleteChan, &FakeExpirationPolicy{ - NeverExpire: util.NewStringSet(testObjs[1].id), + NeverExpire: sets.NewString(testObjs[1].id), RetrieveKeyFunc: func(obj interface{}) (string, error) { return obj.(*timestampedEntry).obj.(testStoreObject).id, nil }, diff --git a/pkg/client/unversioned/cache/index.go b/pkg/client/unversioned/cache/index.go index 6e189c2bf60..0730ca459e4 100644 --- a/pkg/client/unversioned/cache/index.go +++ b/pkg/client/unversioned/cache/index.go @@ -20,7 +20,7 @@ import ( "fmt" "k8s.io/kubernetes/pkg/api/meta" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // Indexer is a storage interface that lets you list objects using multiple indexing functions @@ -63,7 +63,7 @@ func MetaNamespaceIndexFunc(obj interface{}) ([]string, error) { } // Index maps the indexed value to a set of keys in the store that match on that value -type Index map[string]util.StringSet +type Index map[string]sets.String // Indexers maps a name to a IndexFunc type Indexers map[string]IndexFunc diff --git a/pkg/client/unversioned/cache/listers_test.go b/pkg/client/unversioned/cache/listers_test.go index 2ddd427539b..f9505d26136 100644 --- a/pkg/client/unversioned/cache/listers_test.go +++ b/pkg/client/unversioned/cache/listers_test.go @@ -22,12 +22,12 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/expapi" "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func TestStoreToMinionLister(t *testing.T) { store := NewStore(MetaNamespaceKeyFunc) - ids := util.NewStringSet("foo", "bar", "baz") + ids := sets.NewString("foo", "bar", "baz") for id := range ids { store.Add(&api.Node{ObjectMeta: api.ObjectMeta{Name: id}}) } @@ -52,7 +52,7 @@ func TestStoreToReplicationControllerLister(t *testing.T) { testCases := []struct { inRCs []*api.ReplicationController list func() ([]api.ReplicationController, error) - outRCNames util.StringSet + outRCNames sets.String expectErr bool }{ // Basic listing with all labels and no selectors @@ -63,7 +63,7 @@ func TestStoreToReplicationControllerLister(t *testing.T) { list: func() ([]api.ReplicationController, error) { return lister.List() }, - outRCNames: util.NewStringSet("basic"), + outRCNames: sets.NewString("basic"), }, // No pod labels { @@ -81,7 +81,7 @@ func TestStoreToReplicationControllerLister(t *testing.T) { } return lister.GetPodControllers(pod) }, - outRCNames: util.NewStringSet(), + outRCNames: sets.NewString(), expectErr: true, }, // No RC selectors @@ -101,7 +101,7 @@ func TestStoreToReplicationControllerLister(t *testing.T) { } return lister.GetPodControllers(pod) }, - outRCNames: util.NewStringSet(), + outRCNames: sets.NewString(), expectErr: true, }, // Matching labels to selectors and namespace @@ -130,7 +130,7 @@ func TestStoreToReplicationControllerLister(t *testing.T) { } return lister.GetPodControllers(pod) }, - outRCNames: util.NewStringSet("bar"), + outRCNames: sets.NewString("bar"), }, } for _, c := range testCases { @@ -162,7 +162,7 @@ func TestStoreToDaemonSetLister(t *testing.T) { testCases := []struct { inDSs []*expapi.DaemonSet list func() ([]expapi.DaemonSet, error) - outDaemonSetNames util.StringSet + outDaemonSetNames sets.String expectErr bool }{ // Basic listing @@ -173,7 +173,7 @@ func TestStoreToDaemonSetLister(t *testing.T) { list: func() ([]expapi.DaemonSet, error) { return lister.List() }, - outDaemonSetNames: util.NewStringSet("basic"), + outDaemonSetNames: sets.NewString("basic"), }, // Listing multiple daemon sets { @@ -185,7 +185,7 @@ func TestStoreToDaemonSetLister(t *testing.T) { list: func() ([]expapi.DaemonSet, error) { return lister.List() }, - outDaemonSetNames: util.NewStringSet("basic", "complex", "complex2"), + outDaemonSetNames: sets.NewString("basic", "complex", "complex2"), }, // No pod labels { @@ -203,7 +203,7 @@ func TestStoreToDaemonSetLister(t *testing.T) { } return lister.GetPodDaemonSets(pod) }, - outDaemonSetNames: util.NewStringSet(), + outDaemonSetNames: sets.NewString(), expectErr: true, }, // No DS selectors @@ -223,7 +223,7 @@ func TestStoreToDaemonSetLister(t *testing.T) { } return lister.GetPodDaemonSets(pod) }, - outDaemonSetNames: util.NewStringSet(), + outDaemonSetNames: sets.NewString(), expectErr: true, }, // Matching labels to selectors and namespace @@ -252,7 +252,7 @@ func TestStoreToDaemonSetLister(t *testing.T) { } return lister.GetPodDaemonSets(pod) }, - outDaemonSetNames: util.NewStringSet("bar"), + outDaemonSetNames: sets.NewString("bar"), }, } for _, c := range testCases { diff --git a/pkg/client/unversioned/cache/store_test.go b/pkg/client/unversioned/cache/store_test.go index 2d3b153af7a..07275f493de 100644 --- a/pkg/client/unversioned/cache/store_test.go +++ b/pkg/client/unversioned/cache/store_test.go @@ -19,7 +19,7 @@ package cache import ( "testing" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // Test public interface @@ -54,7 +54,7 @@ func doTestStore(t *testing.T, store Store) { store.Add(mkObj("c", "d")) store.Add(mkObj("e", "e")) { - found := util.StringSet{} + found := sets.String{} for _, item := range store.List() { found.Insert(item.(testStoreObject).val) } @@ -73,7 +73,7 @@ func doTestStore(t *testing.T, store Store) { }, "0") { - found := util.StringSet{} + found := sets.String{} for _, item := range store.List() { found.Insert(item.(testStoreObject).val) } @@ -93,17 +93,17 @@ func doTestIndex(t *testing.T, indexer Indexer) { } // Test Index - expected := map[string]util.StringSet{} - expected["b"] = util.NewStringSet("a", "c") - expected["f"] = util.NewStringSet("e") - expected["h"] = util.NewStringSet("g") + expected := map[string]sets.String{} + expected["b"] = sets.NewString("a", "c") + expected["f"] = sets.NewString("e") + expected["h"] = sets.NewString("g") indexer.Add(mkObj("a", "b")) indexer.Add(mkObj("c", "b")) indexer.Add(mkObj("e", "f")) indexer.Add(mkObj("g", "h")) { for k, v := range expected { - found := util.StringSet{} + found := sets.String{} indexResults, err := indexer.Index("by_val", mkObj("", k)) if err != nil { t.Errorf("Unexpected error %v", err) diff --git a/pkg/client/unversioned/cache/thread_safe_store.go b/pkg/client/unversioned/cache/thread_safe_store.go index 20113937890..653b9f297b5 100644 --- a/pkg/client/unversioned/cache/thread_safe_store.go +++ b/pkg/client/unversioned/cache/thread_safe_store.go @@ -20,7 +20,7 @@ import ( "fmt" "sync" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // ThreadSafeStore is an interface that allows concurrent access to a storage backend. @@ -142,7 +142,7 @@ func (c *threadSafeMap) Index(indexName string, obj interface{}) ([]interface{}, index := c.indices[indexName] // need to de-dupe the return list. Since multiple keys are allowed, this can happen. - returnKeySet := util.StringSet{} + returnKeySet := sets.String{} for _, indexKey := range indexKeys { set := index[indexKey] for _, key := range set.List() { @@ -208,7 +208,7 @@ func (c *threadSafeMap) updateIndices(oldObj interface{}, newObj interface{}, ke for _, indexValue := range indexValues { set := index[indexValue] if set == nil { - set = util.StringSet{} + set = sets.String{} index[indexValue] = set } set.Insert(key) diff --git a/pkg/client/unversioned/debugging.go b/pkg/client/unversioned/debugging.go index ae68ed43bed..df43e8984d0 100644 --- a/pkg/client/unversioned/debugging.go +++ b/pkg/client/unversioned/debugging.go @@ -23,7 +23,7 @@ import ( "github.com/golang/glog" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // RequestInfo keeps track of information about a request/response combination @@ -75,7 +75,7 @@ func (r RequestInfo) ToCurl() string { type DebuggingRoundTripper struct { delegatedRoundTripper http.RoundTripper - Levels util.StringSet + Levels sets.String } const ( @@ -88,7 +88,7 @@ const ( ) func NewDebuggingRoundTripper(rt http.RoundTripper, levels ...string) *DebuggingRoundTripper { - return &DebuggingRoundTripper{rt, util.NewStringSet(levels...)} + return &DebuggingRoundTripper{rt, sets.NewString(levels...)} } func (rt *DebuggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { diff --git a/pkg/client/unversioned/flags_test.go b/pkg/client/unversioned/flags_test.go index ea8da3546db..ab0f94d0412 100644 --- a/pkg/client/unversioned/flags_test.go +++ b/pkg/client/unversioned/flags_test.go @@ -20,12 +20,12 @@ import ( "testing" "time" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type fakeFlagSet struct { t *testing.T - set util.StringSet + set sets.String } func (f *fakeFlagSet) StringVar(p *string, name, value, usage string) { diff --git a/pkg/client/unversioned/helper.go b/pkg/client/unversioned/helper.go index cae4fafc38e..081e5f9214a 100644 --- a/pkg/client/unversioned/helper.go +++ b/pkg/client/unversioned/helper.go @@ -35,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/api/latest" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/version" ) @@ -187,7 +188,7 @@ func NegotiateVersion(client *Client, c *Config, version string, clientRegistere return "", err } } - clientVersions := util.StringSet{} + clientVersions := sets.String{} for _, v := range clientRegisteredVersions { clientVersions.Insert(v) } @@ -195,7 +196,7 @@ func NegotiateVersion(client *Client, c *Config, version string, clientRegistere if err != nil { return "", fmt.Errorf("couldn't read version from server: %v", err) } - serverVersions := util.StringSet{} + serverVersions := sets.String{} for _, v := range apiVersions.Versions { serverVersions.Insert(v) } diff --git a/pkg/client/unversioned/request.go b/pkg/client/unversioned/request.go index b48448bc1a0..00c3afdfe6f 100644 --- a/pkg/client/unversioned/request.go +++ b/pkg/client/unversioned/request.go @@ -39,13 +39,14 @@ import ( "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/httpstream" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" watchjson "k8s.io/kubernetes/pkg/watch/json" ) // specialParams lists parameters that are handled specially and which users of Request // are therefore not allowed to set manually. -var specialParams = util.NewStringSet("timeout") +var specialParams = sets.NewString("timeout") // HTTPClient is an interface for testing a request object. type HTTPClient interface { diff --git a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go index eef723be8cf..c3469bcddb2 100644 --- a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go +++ b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go @@ -24,7 +24,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" "github.com/golang/glog" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func (s *AWSCloud) ensureLoadBalancer(region, name string, listeners []*elb.Listener, subnetIDs []string, securityGroupIDs []string) (*elb.LoadBalancerDescription, error) { @@ -61,7 +61,7 @@ func (s *AWSCloud) ensureLoadBalancer(region, name string, listeners []*elb.List } else { { // Sync subnets - expected := util.NewStringSet(subnetIDs...) + expected := sets.NewString(subnetIDs...) actual := stringSetFromPointers(loadBalancer.Subnets) additions := expected.Difference(actual) @@ -94,7 +94,7 @@ func (s *AWSCloud) ensureLoadBalancer(region, name string, listeners []*elb.List { // Sync security groups - expected := util.NewStringSet(securityGroupIDs...) + expected := sets.NewString(securityGroupIDs...) actual := stringSetFromPointers(loadBalancer.SecurityGroups) if !expected.Equal(actual) { @@ -255,12 +255,12 @@ func (s *AWSCloud) ensureLoadBalancerHealthCheck(region string, loadBalancer *el // Makes sure that exactly the specified hosts are registered as instances with the load balancer func (s *AWSCloud) ensureLoadBalancerInstances(elbClient ELB, loadBalancerName string, lbInstances []*elb.Instance, instances []*ec2.Instance) error { - expected := util.NewStringSet() + expected := sets.NewString() for _, instance := range instances { expected.Insert(orEmpty(instance.InstanceID)) } - actual := util.NewStringSet() + actual := sets.NewString() for _, lbInstance := range lbInstances { actual.Insert(orEmpty(lbInstance.InstanceID)) } diff --git a/pkg/cloudprovider/providers/aws/aws_utils.go b/pkg/cloudprovider/providers/aws/aws_utils.go index 99baeeac3ef..1704d2988da 100644 --- a/pkg/cloudprovider/providers/aws/aws_utils.go +++ b/pkg/cloudprovider/providers/aws/aws_utils.go @@ -18,10 +18,10 @@ package aws_cloud import ( "github.com/aws/aws-sdk-go/aws" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) -func stringSetToPointers(in util.StringSet) []*string { +func stringSetToPointers(in sets.String) []*string { if in == nil { return nil } @@ -32,11 +32,11 @@ func stringSetToPointers(in util.StringSet) []*string { return out } -func stringSetFromPointers(in []*string) util.StringSet { +func stringSetFromPointers(in []*string) sets.String { if in == nil { return nil } - out := util.NewStringSet() + out := sets.NewString() for i := range in { out.Insert(orEmpty(in[i])) } diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 3829956b98e..1926b2683ba 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -28,7 +28,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/cloudprovider" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" "github.com/golang/glog" @@ -464,7 +464,7 @@ func (gce *GCECloud) UpdateTCPLoadBalancer(name, region string, hosts []string) if err != nil { return err } - existing := util.NewStringSet() + existing := sets.NewString() for _, instance := range pool.Instances { existing.Insert(hostURLToComparablePath(instance)) } diff --git a/pkg/controller/controller_utils_test.go b/pkg/controller/controller_utils_test.go index ef6d69036df..215c95b10bd 100644 --- a/pkg/controller/controller_utils_test.go +++ b/pkg/controller/controller_utils_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/securitycontext" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // NewFakeControllerExpectationsLookup creates a fake store for PodExpectations. @@ -224,13 +225,13 @@ func TestActivePodFiltering(t *testing.T) { podList := newPodList(nil, 5, api.PodRunning, rc) podList.Items[0].Status.Phase = api.PodSucceeded podList.Items[1].Status.Phase = api.PodFailed - expectedNames := util.NewStringSet() + expectedNames := sets.NewString() for _, pod := range podList.Items[2:] { expectedNames.Insert(pod.Name) } got := FilterActivePods(podList.Items) - gotNames := util.NewStringSet() + gotNames := sets.NewString() for _, pod := range got { gotNames.Insert(pod.Name) } diff --git a/pkg/controller/endpoint/endpoints_controller.go b/pkg/controller/endpoint/endpoints_controller.go index b6db5593049..49a439508f6 100644 --- a/pkg/controller/endpoint/endpoints_controller.go +++ b/pkg/controller/endpoint/endpoints_controller.go @@ -33,6 +33,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/workqueue" "k8s.io/kubernetes/pkg/watch" @@ -141,8 +142,8 @@ func (e *EndpointController) Run(workers int, stopCh <-chan struct{}) { e.queue.ShutDown() } -func (e *EndpointController) getPodServiceMemberships(pod *api.Pod) (util.StringSet, error) { - set := util.StringSet{} +func (e *EndpointController) getPodServiceMemberships(pod *api.Pod) (sets.String, error) { + set := sets.String{} services, err := e.serviceStore.GetPodServices(pod) if err != nil { // don't log this error because this function makes pointless diff --git a/pkg/controller/framework/controller_test.go b/pkg/controller/framework/controller_test.go index 619b994accf..836790b3c83 100644 --- a/pkg/controller/framework/controller_test.go +++ b/pkg/controller/framework/controller_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned/cache" "k8s.io/kubernetes/pkg/controller/framework" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "github.com/google/gofuzz" ) @@ -104,7 +104,7 @@ func Example() { } // Let's wait for the controller to process the things we just added. - outputSet := util.StringSet{} + outputSet := sets.String{} for i := 0; i < len(testIDs); i++ { outputSet.Insert(<-deletionCounter) } @@ -161,7 +161,7 @@ func ExampleInformer() { } // Let's wait for the controller to process the things we just added. - outputSet := util.StringSet{} + outputSet := sets.String{} for i := 0; i < len(testIDs); i++ { outputSet.Insert(<-deletionCounter) } @@ -235,7 +235,7 @@ func TestHammerController(t *testing.T) { go func() { defer wg.Done() // Let's add a few objects to the source. - currentNames := util.StringSet{} + currentNames := sets.String{} rs := rand.NewSource(rand.Int63()) f := fuzz.New().NilChance(.5).NumElements(0, 2).RandSource(rs) r := rand.New(rs) // Mustn't use r and f concurrently! diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go index ec972ef6840..4b95ba32131 100644 --- a/pkg/controller/namespace/namespace_controller.go +++ b/pkg/controller/namespace/namespace_controller.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" "github.com/golang/glog" @@ -128,7 +129,7 @@ func finalize(kubeClient client.Interface, namespace api.Namespace) (*api.Namesp namespaceFinalize := api.Namespace{} namespaceFinalize.ObjectMeta = namespace.ObjectMeta namespaceFinalize.Spec = namespace.Spec - finalizerSet := util.NewStringSet() + finalizerSet := sets.NewString() for i := range namespace.Spec.Finalizers { if namespace.Spec.Finalizers[i] != api.FinalizerKubernetes { finalizerSet.Insert(string(namespace.Spec.Finalizers[i])) diff --git a/pkg/controller/namespace/namespace_controller_test.go b/pkg/controller/namespace/namespace_controller_test.go index dd1c78701e0..ccdd96b7b85 100644 --- a/pkg/controller/namespace/namespace_controller_test.go +++ b/pkg/controller/namespace/namespace_controller_test.go @@ -24,6 +24,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client/unversioned/testclient" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func TestFinalized(t *testing.T) { @@ -90,7 +91,7 @@ func testSyncNamespaceThatIsTerminating(t *testing.T, experimentalMode bool) { t.Errorf("Unexpected error when synching namespace %v", err) } // TODO: Reuse the constants for all these strings from testclient - expectedActionSet := util.NewStringSet( + expectedActionSet := sets.NewString( strings.Join([]string{"list", "replicationcontrollers", ""}, "-"), strings.Join([]string{"list", "services", ""}, "-"), strings.Join([]string{"list", "pods", ""}, "-"), @@ -112,7 +113,7 @@ func testSyncNamespaceThatIsTerminating(t *testing.T, experimentalMode bool) { ) } - actionSet := util.NewStringSet() + actionSet := sets.NewString() for _, action := range mockClient.Actions() { actionSet.Insert(strings.Join([]string{action.GetVerb(), action.GetResource(), action.GetSubresource()}, "-")) } diff --git a/pkg/controller/node/nodecontroller.go b/pkg/controller/node/nodecontroller.go index 2da83b9ab3d..18541f5ea82 100644 --- a/pkg/controller/node/nodecontroller.go +++ b/pkg/controller/node/nodecontroller.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) var ( @@ -58,7 +59,7 @@ type NodeController struct { cloud cloudprovider.Interface clusterCIDR *net.IPNet deletingPodsRateLimiter util.RateLimiter - knownNodeSet util.StringSet + knownNodeSet sets.String kubeClient client.Interface // Method for easy mocking in unittest. lookupIP func(host string) ([]net.IP, error) @@ -126,7 +127,7 @@ func NewNodeController( evictorLock := sync.Mutex{} return &NodeController{ cloud: cloud, - knownNodeSet: make(util.StringSet), + knownNodeSet: make(sets.String), kubeClient: kubeClient, recorder: recorder, podEvictionTimeout: podEvictionTimeout, @@ -211,8 +212,8 @@ func (nc *NodeController) Run(period time.Duration) { } // Generates num pod CIDRs that could be assigned to nodes. -func generateCIDRs(clusterCIDR *net.IPNet, num int) util.StringSet { - res := util.NewStringSet() +func generateCIDRs(clusterCIDR *net.IPNet, num int) sets.String { + res := sets.NewString() cidrIP := clusterCIDR.IP.To4() for i := 0; i < num; i++ { // TODO: Make the CIDRs configurable. @@ -256,7 +257,7 @@ func (nc *NodeController) monitorNodeStatus() error { // If there's a difference between lengths of known Nodes and observed nodes // we must have removed some Node. if len(nc.knownNodeSet) != len(nodes.Items) { - observedSet := make(util.StringSet) + observedSet := make(sets.String) for _, node := range nodes.Items { observedSet.Insert(node.Name) } diff --git a/pkg/controller/node/rate_limited_queue.go b/pkg/controller/node/rate_limited_queue.go index 2fcd0963d1b..71950f54a47 100644 --- a/pkg/controller/node/rate_limited_queue.go +++ b/pkg/controller/node/rate_limited_queue.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // TimedValue is a value that should be processed at a designated time. @@ -58,7 +59,7 @@ func (h *TimedQueue) Pop() interface{} { type UniqueQueue struct { lock sync.Mutex queue TimedQueue - set util.StringSet + set sets.String } // Adds a new value to the queue if it wasn't added before, or was explicitly removed by the @@ -143,7 +144,7 @@ func NewRateLimitedTimedQueue(limiter util.RateLimiter) *RateLimitedTimedQueue { return &RateLimitedTimedQueue{ queue: UniqueQueue{ queue: TimedQueue{}, - set: util.NewStringSet(), + set: sets.NewString(), }, limiter: limiter, } diff --git a/pkg/controller/node/rate_limited_queue_test.go b/pkg/controller/node/rate_limited_queue_test.go index 5dc8a4d81a4..762e8263822 100644 --- a/pkg/controller/node/rate_limited_queue_test.go +++ b/pkg/controller/node/rate_limited_queue_test.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func CheckQueueEq(lhs []string, rhs TimedQueue) bool { @@ -33,7 +34,7 @@ func CheckQueueEq(lhs []string, rhs TimedQueue) bool { return true } -func CheckSetEq(lhs, rhs util.StringSet) bool { +func CheckSetEq(lhs, rhs sets.String) bool { return lhs.HasAll(rhs.List()...) && rhs.HasAll(lhs.List()...) } @@ -51,7 +52,7 @@ func TestAddNode(t *testing.T) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } - setPattern := util.NewStringSet("first", "second", "third") + setPattern := sets.NewString("first", "second", "third") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -75,7 +76,7 @@ func TestDelNode(t *testing.T) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } - setPattern := util.NewStringSet("second", "third") + setPattern := sets.NewString("second", "third") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -97,7 +98,7 @@ func TestDelNode(t *testing.T) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } - setPattern = util.NewStringSet("first", "third") + setPattern = sets.NewString("first", "third") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -119,7 +120,7 @@ func TestDelNode(t *testing.T) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } - setPattern = util.NewStringSet("first", "second") + setPattern = sets.NewString("first", "second") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -135,13 +136,13 @@ func TestTry(t *testing.T) { evictor.Add("third") evictor.Remove("second") - deletedMap := util.NewStringSet() + deletedMap := sets.NewString() evictor.Try(func(value TimedValue) (bool, time.Duration) { deletedMap.Insert(value.Value) return true, 0 }) - setPattern := util.NewStringSet("first", "third") + setPattern := sets.NewString("first", "third") if len(deletedMap) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } diff --git a/pkg/controller/replication/replication_controller_test.go b/pkg/controller/replication/replication_controller_test.go index 5376a69bf09..cd58f9e68c2 100644 --- a/pkg/controller/replication/replication_controller_test.go +++ b/pkg/controller/replication/replication_controller_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/securitycontext" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/watch" ) @@ -626,7 +627,7 @@ func TestUpdatePods(t *testing.T) { // both controllers manager.updatePod(&pod1, &pod2) - expected := util.NewStringSet(testControllerSpec1.Name, testControllerSpec2.Name) + expected := sets.NewString(testControllerSpec1.Name, testControllerSpec2.Name) for _, name := range expected.List() { t.Logf("Expecting update for %+v", name) select { diff --git a/pkg/controller/resourcequota/resource_quota_controller_test.go b/pkg/controller/resourcequota/resource_quota_controller_test.go index 346d6bdf438..4cb0b1f4f73 100644 --- a/pkg/controller/resourcequota/resource_quota_controller_test.go +++ b/pkg/controller/resourcequota/resource_quota_controller_test.go @@ -23,7 +23,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/client/unversioned/testclient" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func getResourceList(cpu, memory string) api.ResourceList { @@ -103,11 +103,11 @@ func TestFilterQuotaPods(t *testing.T) { Status: api.PodStatus{Phase: api.PodFailed}, }, } - expectedResults := util.NewStringSet("pod-running", + expectedResults := sets.NewString("pod-running", "pod-pending", "pod-unknown", "pod-failed-with-restart-always", "pod-failed-with-restart-on-failure") - actualResults := util.StringSet{} + actualResults := sets.String{} result := FilterQuotaPods(pods) for i := range result { actualResults.Insert(result[i].Name) diff --git a/pkg/controller/serviceaccount/serviceaccounts_controller.go b/pkg/controller/serviceaccount/serviceaccounts_controller.go index 7da8f89e495..90c2702bd56 100644 --- a/pkg/controller/serviceaccount/serviceaccounts_controller.go +++ b/pkg/controller/serviceaccount/serviceaccounts_controller.go @@ -29,7 +29,7 @@ import ( "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" ) @@ -45,7 +45,7 @@ func nameIndexFunc(obj interface{}) ([]string, error) { // ServiceAccountsControllerOptions contains options for running a ServiceAccountsController type ServiceAccountsControllerOptions struct { // Names is the set of service account names to ensure exist in every namespace - Names util.StringSet + Names sets.String // ServiceAccountResync is the interval between full resyncs of ServiceAccounts. // If non-zero, all service accounts will be re-listed this often. @@ -59,7 +59,7 @@ type ServiceAccountsControllerOptions struct { } func DefaultServiceAccountsControllerOptions() ServiceAccountsControllerOptions { - return ServiceAccountsControllerOptions{Names: util.NewStringSet("default")} + return ServiceAccountsControllerOptions{Names: sets.NewString("default")} } // NewServiceAccountsController returns a new *ServiceAccountsController. @@ -117,7 +117,7 @@ type ServiceAccountsController struct { stopChan chan struct{} client client.Interface - names util.StringSet + names sets.String serviceAccounts cache.Indexer namespaces cache.Indexer diff --git a/pkg/controller/serviceaccount/serviceaccounts_controller_test.go b/pkg/controller/serviceaccount/serviceaccounts_controller_test.go index e1525bd8aa7..f04892b9869 100644 --- a/pkg/controller/serviceaccount/serviceaccounts_controller_test.go +++ b/pkg/controller/serviceaccount/serviceaccounts_controller_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned/testclient" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type serverResponse struct { @@ -101,7 +102,7 @@ func TestServiceAccountCreation(t *testing.T) { "new active namespace missing serviceaccounts": { ExistingServiceAccounts: []*api.ServiceAccount{}, AddedNamespace: activeNS, - ExpectCreatedServiceAccounts: util.NewStringSet(defaultName, managedName).List(), + ExpectCreatedServiceAccounts: sets.NewString(defaultName, managedName).List(), }, "new active namespace missing serviceaccount": { ExistingServiceAccounts: []*api.ServiceAccount{managedServiceAccount}, @@ -123,7 +124,7 @@ func TestServiceAccountCreation(t *testing.T) { "updated active namespace missing serviceaccounts": { ExistingServiceAccounts: []*api.ServiceAccount{}, UpdatedNamespace: activeNS, - ExpectCreatedServiceAccounts: util.NewStringSet(defaultName, managedName).List(), + ExpectCreatedServiceAccounts: sets.NewString(defaultName, managedName).List(), }, "updated active namespace missing serviceaccount": { ExistingServiceAccounts: []*api.ServiceAccount{defaultServiceAccount}, @@ -170,7 +171,7 @@ func TestServiceAccountCreation(t *testing.T) { for k, tc := range testcases { client := testclient.NewSimpleFake(defaultServiceAccount, managedServiceAccount) options := DefaultServiceAccountsControllerOptions() - options.Names = util.NewStringSet(defaultName, managedName) + options.Names = sets.NewString(defaultName, managedName) controller := NewServiceAccountsController(client, options) if tc.ExistingNamespace != nil { diff --git a/pkg/controller/serviceaccount/tokens_controller.go b/pkg/controller/serviceaccount/tokens_controller.go index 7fd568a0629..7de46edef14 100644 --- a/pkg/controller/serviceaccount/tokens_controller.go +++ b/pkg/controller/serviceaccount/tokens_controller.go @@ -31,7 +31,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/registry/secret" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" ) @@ -495,8 +495,8 @@ func serviceAccountNameAndUID(secret *api.Secret) (string, string) { return secret.Annotations[api.ServiceAccountNameKey], secret.Annotations[api.ServiceAccountUIDKey] } -func getSecretReferences(serviceAccount *api.ServiceAccount) util.StringSet { - references := util.NewStringSet() +func getSecretReferences(serviceAccount *api.ServiceAccount) sets.String { + references := sets.NewString() for _, secret := range serviceAccount.Secrets { references.Insert(secret.Name) } diff --git a/pkg/credentialprovider/keyring.go b/pkg/credentialprovider/keyring.go index 87a8c862e79..404e8ed4aae 100644 --- a/pkg/credentialprovider/keyring.go +++ b/pkg/credentialprovider/keyring.go @@ -28,7 +28,7 @@ import ( "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // DockerKeyring tracks a set of docker registry credentials, maintaining a @@ -90,7 +90,7 @@ func (dk *BasicDockerKeyring) Add(cfg DockerConfig) { } } - eliminateDupes := util.NewStringSet(dk.index...) + eliminateDupes := sets.NewString(dk.index...) dk.index = eliminateDupes.List() // Update the index used to identify which credentials to use for a given diff --git a/pkg/expapi/latest/latest.go b/pkg/expapi/latest/latest.go index 11b4ff2536c..13dbc2c70d8 100644 --- a/pkg/expapi/latest/latest.go +++ b/pkg/expapi/latest/latest.go @@ -26,7 +26,7 @@ import ( _ "k8s.io/kubernetes/pkg/expapi" "k8s.io/kubernetes/pkg/expapi/v1" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) var ( @@ -51,9 +51,9 @@ func init() { // the list of kinds that are scoped at the root of the api hierarchy // if a kind is not enumerated here, it is assumed to have a namespace scope - rootScoped := util.NewStringSet() + rootScoped := sets.NewString() - ignoredKinds := util.NewStringSet() + ignoredKinds := sets.NewString() RESTMapper = api.NewDefaultRESTMapper("experimental", Versions, InterfacesFor, importPrefix, ignoredKinds, rootScoped) api.RegisterRESTMapper(RESTMapper) diff --git a/pkg/expapi/validation/validation.go b/pkg/expapi/validation/validation.go index 82984add68a..2e1c653808d 100644 --- a/pkg/expapi/validation/validation.go +++ b/pkg/expapi/validation/validation.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/util" errs "k8s.io/kubernetes/pkg/util/fielderrors" + "k8s.io/kubernetes/pkg/util/sets" ) // ValidateHorizontalPodAutoscaler can be used to check whether the given autoscaler name is valid. @@ -79,7 +80,7 @@ func ValidateThirdPartyResource(obj *expapi.ThirdPartyResource) errs.ValidationE if len(obj.Name) == 0 { allErrs = append(allErrs, errs.NewFieldInvalid("name", obj.Name, "name must be non-empty")) } - versions := util.StringSet{} + versions := sets.String{} for ix := range obj.Versions { version := &obj.Versions[ix] if len(version.Name) == 0 { diff --git a/pkg/kubectl/cmd/config/navigation_step_parser.go b/pkg/kubectl/cmd/config/navigation_step_parser.go index 835f1998645..1d4272a813d 100644 --- a/pkg/kubectl/cmd/config/navigation_step_parser.go +++ b/pkg/kubectl/cmd/config/navigation_step_parser.go @@ -22,7 +22,7 @@ import ( "strings" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type navigationSteps struct { @@ -55,7 +55,7 @@ func newNavigationSteps(path string) (*navigationSteps, error) { if err != nil { return nil, err } - nextPart := findNameStep(individualParts[currPartIndex:], util.KeySet(reflect.ValueOf(mapValueOptions))) + nextPart := findNameStep(individualParts[currPartIndex:], sets.KeySet(reflect.ValueOf(mapValueOptions))) steps = append(steps, navigationStep{nextPart, mapValueType}) currPartIndex += len(strings.Split(nextPart, ".")) @@ -103,7 +103,7 @@ func (s *navigationSteps) moreStepsRemaining() bool { // findNameStep takes the list of parts and a set of valid tags that can be used after the name. It then walks the list of parts // until it find a valid "next" tag or until it reaches the end of the parts and then builds the name back up out of the individual parts -func findNameStep(parts []string, typeOptions util.StringSet) string { +func findNameStep(parts []string, typeOptions sets.String) string { if len(parts) == 0 { return "" } @@ -141,7 +141,7 @@ func getPotentialTypeValues(typeValue reflect.Type) (map[string]reflect.Type, er return ret, nil } -func findKnownValue(parts []string, valueOptions util.StringSet) int { +func findKnownValue(parts []string, valueOptions sets.String) int { for i := range parts { if valueOptions.Has(parts[i]) { return i diff --git a/pkg/kubectl/cmd/log.go b/pkg/kubectl/cmd/log.go index bb58e5d9b40..d5c601dd24b 100644 --- a/pkg/kubectl/cmd/log.go +++ b/pkg/kubectl/cmd/log.go @@ -26,7 +26,7 @@ import ( "github.com/spf13/cobra" "k8s.io/kubernetes/pkg/api" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - libutil "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) const ( @@ -42,7 +42,7 @@ $ kubectl logs -f 123456-7890 ruby-container` func selectContainer(pod *api.Pod, in io.Reader, out io.Writer) string { fmt.Fprintf(out, "Please select a container:\n") - options := libutil.StringSet{} + options := sets.String{} for ix := range pod.Spec.Containers { fmt.Fprintf(out, "[%d] %s\n", ix+1, pod.Spec.Containers[ix].Name) options.Insert(pod.Spec.Containers[ix].Name) diff --git a/pkg/kubectl/describe.go b/pkg/kubectl/describe.go index 4c872542ed5..63a078fbf47 100644 --- a/pkg/kubectl/describe.go +++ b/pkg/kubectl/describe.go @@ -33,7 +33,7 @@ import ( "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/types" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // Describer generates output for the named resource or an error @@ -933,7 +933,7 @@ func describeService(service *api.Service, endpoints *api.Endpoints, events *api if sp.NodePort != 0 { fmt.Fprintf(out, "NodePort:\t%s\t%d/%s\n", name, sp.NodePort, sp.Protocol) } - fmt.Fprintf(out, "Endpoints:\t%s\n", formatEndpoints(endpoints, util.NewStringSet(sp.Name))) + fmt.Fprintf(out, "Endpoints:\t%s\n", formatEndpoints(endpoints, sets.NewString(sp.Name))) } fmt.Fprintf(out, "Session Affinity:\t%s\n", service.Spec.SessionAffinity) if events != nil { diff --git a/pkg/kubectl/resource/builder.go b/pkg/kubectl/resource/builder.go index 4386ed8d723..9da9c106666 100644 --- a/pkg/kubectl/resource/builder.go +++ b/pkg/kubectl/resource/builder.go @@ -28,8 +28,8 @@ import ( "k8s.io/kubernetes/pkg/api/validation" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/errors" + "k8s.io/kubernetes/pkg/util/sets" ) var FileExtensions = []string{".json", ".stdin", ".yaml", ".yml"} @@ -685,7 +685,7 @@ func (b *Builder) Do() *Result { // strings in the original order. func SplitResourceArgument(arg string) []string { out := []string{} - set := util.NewStringSet() + set := sets.NewString() for _, s := range strings.Split(arg, ",") { if set.Has(s) { continue diff --git a/pkg/kubectl/resource/result.go b/pkg/kubectl/resource/result.go index f8a8c688e19..3ac6d720157 100644 --- a/pkg/kubectl/resource/result.go +++ b/pkg/kubectl/resource/result.go @@ -24,8 +24,8 @@ import ( "k8s.io/kubernetes/pkg/api/latest" "k8s.io/kubernetes/pkg/api/meta" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/errors" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" ) @@ -123,7 +123,7 @@ func (r *Result) Object() (runtime.Object, error) { return nil, err } - versions := util.StringSet{} + versions := sets.String{} objects := []runtime.Object{} for _, info := range infos { if info.Object != nil { diff --git a/pkg/kubectl/resource_printer.go b/pkg/kubectl/resource_printer.go index fc9ea6b58dd..c847d7e4ed8 100644 --- a/pkg/kubectl/resource_printer.go +++ b/pkg/kubectl/resource_printer.go @@ -41,6 +41,7 @@ import ( "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/jsonpath" + "k8s.io/kubernetes/pkg/util/sets" ) // GetPrinter takes a format type, an optional format argument. It will return true @@ -435,7 +436,7 @@ func (h *HumanReadablePrinter) printHeader(columnNames []string, w io.Writer) er } // Pass ports=nil for all ports. -func formatEndpoints(endpoints *api.Endpoints, ports util.StringSet) string { +func formatEndpoints(endpoints *api.Endpoints, ports sets.String) string { if len(endpoints.Subsets) == 0 { return "" } diff --git a/pkg/kubectl/resource_printer_test.go b/pkg/kubectl/resource_printer_test.go index b7d67eaffd7..bb76beb692d 100644 --- a/pkg/kubectl/resource_printer_test.go +++ b/pkg/kubectl/resource_printer_test.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/expapi" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "github.com/ghodss/yaml" ) @@ -484,9 +485,9 @@ func TestPrinters(t *testing.T) { }}}, } // map of printer name to set of objects it should fail on. - expectedErrors := map[string]util.StringSet{ - "template2": util.NewStringSet("pod", "emptyPodList", "endpoints"), - "jsonpath": util.NewStringSet("emptyPodList", "nonEmptyPodList", "endpoints"), + expectedErrors := map[string]sets.String{ + "template2": sets.NewString("pod", "emptyPodList", "endpoints"), + "jsonpath": sets.NewString("emptyPodList", "nonEmptyPodList", "endpoints"), } for pName, p := range printers { diff --git a/pkg/kubectl/rolling_updater_test.go b/pkg/kubectl/rolling_updater_test.go index 4b6b6406e93..2ca9ab6ef58 100644 --- a/pkg/kubectl/rolling_updater_test.go +++ b/pkg/kubectl/rolling_updater_test.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned/testclient" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func oldRc(replicas int, original int) *api.ReplicationController { @@ -1140,7 +1141,7 @@ func TestAddDeploymentHash(t *testing.T) { }, } - seen := util.StringSet{} + seen := sets.String{} updatedRc := false fakeClient := &client.FakeRESTClient{ Codec: codec, diff --git a/pkg/kubelet/config/config.go b/pkg/kubelet/config/config.go index 0de727354bd..5d0e5bc4722 100644 --- a/pkg/kubelet/config/config.go +++ b/pkg/kubelet/config/config.go @@ -29,10 +29,10 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubeletTypes "k8s.io/kubernetes/pkg/kubelet/types" kubeletUtil "k8s.io/kubernetes/pkg/kubelet/util" - "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/config" utilerrors "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/util/fielderrors" + "k8s.io/kubernetes/pkg/util/sets" ) // PodConfigNotificationMode describes how changes are sent to the update channel. @@ -61,7 +61,7 @@ type PodConfig struct { // contains the list of all configured sources sourcesLock sync.Mutex - sources util.StringSet + sources sets.String } // NewPodConfig creates an object that can merge many configuration sources into a stream @@ -73,7 +73,7 @@ func NewPodConfig(mode PodConfigNotificationMode, recorder record.EventRecorder) pods: storage, mux: config.NewMux(storage), updates: updates, - sources: util.StringSet{}, + sources: sets.String{}, } return podConfig } @@ -124,7 +124,7 @@ type podStorage struct { // contains the set of all sources that have sent at least one SET sourcesSeenLock sync.Mutex - sourcesSeen util.StringSet + sourcesSeen sets.String // the EventRecorder to use recorder record.EventRecorder @@ -138,7 +138,7 @@ func newPodStorage(updates chan<- kubelet.PodUpdate, mode PodConfigNotificationM pods: make(map[string]map[string]*api.Pod), mode: mode, updates: updates, - sourcesSeen: util.StringSet{}, + sourcesSeen: sets.String{}, recorder: recorder, } } @@ -306,7 +306,7 @@ func (s *podStorage) seenSources(sources ...string) bool { } func filterInvalidPods(pods []*api.Pod, source string, recorder record.EventRecorder) (filtered []*api.Pod) { - names := util.StringSet{} + names := sets.String{} for i, pod := range pods { var errlist []error if errs := validation.ValidatePod(pod); len(errs) != 0 { diff --git a/pkg/kubelet/dockertools/fake_docker_client.go b/pkg/kubelet/dockertools/fake_docker_client.go index ca93a0feb2a..d9de133117d 100644 --- a/pkg/kubelet/dockertools/fake_docker_client.go +++ b/pkg/kubelet/dockertools/fake_docker_client.go @@ -27,7 +27,7 @@ import ( docker "github.com/fsouza/go-dockerclient" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // FakeDockerClient is a simple fake docker client, so that kubelet can be run for testing without requiring a real docker setup. @@ -45,7 +45,7 @@ type FakeDockerClient struct { pulled []string Created []string Removed []string - RemovedImages util.StringSet + RemovedImages sets.String VersionInfo docker.Env Information docker.Env ExecInspect *docker.ExecInspect diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index f1b8a6da4d0..10c72fb81fc 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -51,6 +51,7 @@ import ( "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/util/procfs" + "k8s.io/kubernetes/pkg/util/sets" ) const ( @@ -402,7 +403,7 @@ func (dm *DockerManager) GetPodStatus(pod *api.Pod) (*api.PodStatus, error) { return nil, err } - containerDone := util.NewStringSet() + containerDone := sets.NewString() // Loop through list of running and exited docker containers to construct // the statuses. We assume docker returns a list of containers sorted in // reverse by time. diff --git a/pkg/kubelet/dockertools/manager_test.go b/pkg/kubelet/dockertools/manager_test.go index ef289f4e309..800233b4710 100644 --- a/pkg/kubelet/dockertools/manager_test.go +++ b/pkg/kubelet/dockertools/manager_test.go @@ -42,6 +42,7 @@ import ( "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util" uexec "k8s.io/kubernetes/pkg/util/exec" + "k8s.io/kubernetes/pkg/util/sets" ) type fakeHTTP struct { @@ -74,7 +75,7 @@ func (*fakeOptionGenerator) GenerateRunContainerOptions(pod *api.Pod, container } func newTestDockerManagerWithHTTPClient(fakeHTTPClient *fakeHTTP) (*DockerManager, *FakeDockerClient) { - fakeDocker := &FakeDockerClient{VersionInfo: docker.Env{"Version=1.1.3", "ApiVersion=1.15"}, Errors: make(map[string]error), RemovedImages: util.StringSet{}} + fakeDocker := &FakeDockerClient{VersionInfo: docker.Env{"Version=1.1.3", "ApiVersion=1.15"}, Errors: make(map[string]error), RemovedImages: sets.String{}} fakeRecorder := &record.FakeRecorder{} readinessManager := kubecontainer.NewReadinessManager() containerRefManager := kubecontainer.NewRefManager() @@ -324,14 +325,14 @@ func TestGetPods(t *testing.T) { func TestListImages(t *testing.T) { manager, fakeDocker := newTestDockerManager() dockerImages := []docker.APIImages{{ID: "1111"}, {ID: "2222"}, {ID: "3333"}} - expected := util.NewStringSet([]string{"1111", "2222", "3333"}...) + expected := sets.NewString([]string{"1111", "2222", "3333"}...) fakeDocker.Images = dockerImages actualImages, err := manager.ListImages() if err != nil { t.Fatalf("unexpected error %v", err) } - actual := util.NewStringSet() + actual := sets.NewString() for _, i := range actualImages { actual.Insert(i.ID) } diff --git a/pkg/kubelet/image_manager.go b/pkg/kubelet/image_manager.go index 42a00d9bfec..e17c647328b 100644 --- a/pkg/kubelet/image_manager.go +++ b/pkg/kubelet/image_manager.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // Manages lifecycle of all images. @@ -141,14 +142,14 @@ func (im *realImageManager) detectImages(detected time.Time) error { } // Make a set of images in use by containers. - imagesInUse := util.NewStringSet() + imagesInUse := sets.NewString() for _, container := range containers { imagesInUse.Insert(container.Image) } // Add new images and record those being used. now := time.Now() - currentImages := util.NewStringSet() + currentImages := sets.NewString() im.imageRecordsLock.Lock() defer im.imageRecordsLock.Unlock() for _, image := range images { @@ -286,7 +287,7 @@ func (ev byLastUsedAndDetected) Less(i, j int) bool { } } -func isImageUsed(image *docker.APIImages, imagesInUse util.StringSet) bool { +func isImageUsed(image *docker.APIImages, imagesInUse sets.String) bool { // Check the image ID and all the RepoTags. if _, ok := imagesInUse[image.ID]; ok { return true diff --git a/pkg/kubelet/image_manager_test.go b/pkg/kubelet/image_manager_test.go index aa0f97e3d15..1b96807d7a1 100644 --- a/pkg/kubelet/image_manager_test.go +++ b/pkg/kubelet/image_manager_test.go @@ -28,14 +28,14 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned/record" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/dockertools" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) var zero time.Time func newRealImageManager(policy ImageGCPolicy) (*realImageManager, *dockertools.FakeDockerClient, *cadvisor.Mock) { fakeDocker := &dockertools.FakeDockerClient{ - RemovedImages: util.NewStringSet(), + RemovedImages: sets.NewString(), } mockCadvisor := new(cadvisor.Mock) return &realImageManager{ diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 482564b4729..f93edf0e80e 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -64,6 +64,7 @@ import ( nodeutil "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/util/procfs" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/watch" @@ -983,7 +984,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Cont return opts, nil } -var masterServices = util.NewStringSet("kubernetes") +var masterServices = sets.NewString("kubernetes") // getServiceEnvVarMap makes a map[string]string of env vars for services a pod in namespace ns should see func (kl *Kubelet) getServiceEnvVarMap(ns string) (map[string]string, error) { @@ -1408,7 +1409,7 @@ func getDesiredVolumes(pods []*api.Pod) map[string]api.Volume { // cleanupOrphanedPodDirs removes a pod directory if the pod is not in the // desired set of pods and there is no running containers in the pod. func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*api.Pod, runningPods []*kubecontainer.Pod) error { - active := util.NewStringSet() + active := sets.NewString() for _, pod := range pods { active.Insert(string(pod.UID)) } @@ -1446,7 +1447,7 @@ func (kl *Kubelet) cleanupBandwidthLimits(allPods []*api.Pod) error { if err != nil { return err } - possibleCIDRs := util.StringSet{} + possibleCIDRs := sets.String{} for ix := range allPods { pod := allPods[ix] ingress, egress, err := extractBandwidthResources(pod) @@ -1486,7 +1487,7 @@ func (kl *Kubelet) cleanupOrphanedVolumes(pods []*api.Pod, runningPods []*kubeco desiredVolumes := getDesiredVolumes(pods) currentVolumes := kl.getPodVolumesFromDisk() - runningSet := util.StringSet{} + runningSet := sets.String{} for _, pod := range runningPods { runningSet.Insert(string(pod.ID)) } @@ -1724,7 +1725,7 @@ func (kl *Kubelet) HandlePodCleanups() error { // podKiller launches a goroutine to kill a pod received from the channel if // another goroutine isn't already in action. func (kl *Kubelet) podKiller() { - killing := util.NewStringSet() + killing := sets.NewString() resultCh := make(chan types.UID) defer close(resultCh) for { @@ -1771,7 +1772,7 @@ func (s podsByCreationTime) Less(i, j int) bool { // checkHostPortConflicts detects pods with conflicted host ports. func hasHostPortConflicts(pods []*api.Pod) bool { - ports := util.StringSet{} + ports := sets.String{} for _, pod := range pods { if errs := validation.AccumulateUniqueHostPorts(pod.Spec.Containers, &ports); len(errs) > 0 { glog.Errorf("Pod %q: HostPort is already allocated, ignoring: %v", kubecontainer.GetPodFullName(pod), errs) diff --git a/pkg/kubelet/mirror_client_test.go b/pkg/kubelet/mirror_client_test.go index 2ae9c845b28..9c372d5b33d 100644 --- a/pkg/kubelet/mirror_client_test.go +++ b/pkg/kubelet/mirror_client_test.go @@ -22,14 +22,14 @@ import ( "k8s.io/kubernetes/pkg/api" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type fakeMirrorClient struct { mirrorPodLock sync.RWMutex // Note that a real mirror manager does not store the mirror pods in // itself. This fake manager does this to track calls. - mirrorPods util.StringSet + mirrorPods sets.String createCounts map[string]int deleteCounts map[string]int } @@ -53,7 +53,7 @@ func (fmc *fakeMirrorClient) DeleteMirrorPod(podFullName string) error { func newFakeMirrorClient() *fakeMirrorClient { m := fakeMirrorClient{} - m.mirrorPods = util.NewStringSet() + m.mirrorPods = sets.NewString() m.createCounts = make(map[string]int) m.deleteCounts = make(map[string]int) return &m diff --git a/pkg/labels/selector.go b/pkg/labels/selector.go index 28bc6d838af..7d811cc4830 100644 --- a/pkg/labels/selector.go +++ b/pkg/labels/selector.go @@ -24,6 +24,7 @@ import ( "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/fielderrors" + "k8s.io/kubernetes/pkg/util/sets" ) // Selector represents a label selector. @@ -79,7 +80,7 @@ func (a ByKey) Less(i, j int) bool { return a[i].key < a[j].key } type Requirement struct { key string operator Operator - strValues util.StringSet + strValues sets.String } // NewRequirement is the constructor for a Requirement. @@ -91,7 +92,7 @@ type Requirement struct { // of characters. See validateLabelKey for more details. // // The empty string is a valid value in the input values set. -func NewRequirement(key string, op Operator, vals util.StringSet) (*Requirement, error) { +func NewRequirement(key string, op Operator, vals sets.String) (*Requirement, error) { if err := validateLabelKey(key); err != nil { return nil, err } @@ -198,7 +199,7 @@ func (lsel LabelSelector) Add(key string, operator Operator, values []string) Se for _, item := range lsel { reqs = append(reqs, item) } - if r, err := NewRequirement(key, operator, util.NewStringSet(values...)); err == nil { + if r, err := NewRequirement(key, operator, sets.NewString(values...)); err == nil { reqs = append(reqs, *r) } return LabelSelector(reqs) @@ -480,7 +481,7 @@ func (p *Parser) parseRequirement() (*Requirement, error) { if err != nil { return nil, err } - var values util.StringSet + var values sets.String switch operator { case InOperator, NotInOperator: values, err = p.parseValues() @@ -535,7 +536,7 @@ func (p *Parser) parseOperator() (op Operator, err error) { } // parseValues parses the values for set based matching (x,y,z) -func (p *Parser) parseValues() (util.StringSet, error) { +func (p *Parser) parseValues() (sets.String, error) { tok, lit := p.consume(Values) if tok != OpenParToken { return nil, fmt.Errorf("found '%s' expected: '('", lit) @@ -553,7 +554,7 @@ func (p *Parser) parseValues() (util.StringSet, error) { return s, nil case ClosedParToken: // handles "()" p.consume(Values) - return util.NewStringSet(""), nil + return sets.NewString(""), nil default: return nil, fmt.Errorf("found '%s', expected: ',', ')' or identifier", lit) } @@ -561,8 +562,8 @@ func (p *Parser) parseValues() (util.StringSet, error) { // parseIdentifiersList parses a (possibly empty) list of // of comma separated (possibly empty) identifiers -func (p *Parser) parseIdentifiersList() (util.StringSet, error) { - s := util.NewStringSet() +func (p *Parser) parseIdentifiersList() (sets.String, error) { + s := sets.NewString() for { tok, lit := p.consume(Values) switch tok { @@ -597,8 +598,8 @@ func (p *Parser) parseIdentifiersList() (util.StringSet, error) { } // parseExactValue parses the only value for exact match style -func (p *Parser) parseExactValue() (util.StringSet, error) { - s := util.NewStringSet() +func (p *Parser) parseExactValue() (sets.String, error) { + s := sets.NewString() tok, lit := p.consume(Values) if tok == IdentifierToken { s.Insert(lit) @@ -670,7 +671,7 @@ func SelectorFromSet(ls Set) Selector { } var requirements []Requirement for label, value := range ls { - if r, err := NewRequirement(label, EqualsOperator, util.NewStringSet(value)); err != nil { + if r, err := NewRequirement(label, EqualsOperator, sets.NewString(value)); err != nil { //TODO: double check errors when input comes from serialization? return LabelSelector{} } else { diff --git a/pkg/labels/selector_test.go b/pkg/labels/selector_test.go index 987bafc116f..181cb26890a 100644 --- a/pkg/labels/selector_test.go +++ b/pkg/labels/selector_test.go @@ -21,7 +21,7 @@ import ( "strings" "testing" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func TestSelectorParse(t *testing.T) { @@ -273,16 +273,16 @@ func TestRequirementConstructor(t *testing.T) { requirementConstructorTests := []struct { Key string Op Operator - Vals util.StringSet + Vals sets.String Success bool }{ {"x", InOperator, nil, false}, - {"x", NotInOperator, util.NewStringSet(), false}, - {"x", InOperator, util.NewStringSet("foo"), true}, - {"x", NotInOperator, util.NewStringSet("foo"), true}, + {"x", NotInOperator, sets.NewString(), false}, + {"x", InOperator, sets.NewString("foo"), true}, + {"x", NotInOperator, sets.NewString("foo"), true}, {"x", ExistsOperator, nil, true}, - {"1foo", InOperator, util.NewStringSet("bar"), true}, - {"1234", InOperator, util.NewStringSet("bar"), true}, + {"1foo", InOperator, sets.NewString("bar"), true}, + {"1234", InOperator, sets.NewString("bar"), true}, {strings.Repeat("a", 254), ExistsOperator, nil, false}, //breaks DNS rule that len(key) <= 253 } for _, rc := range requirementConstructorTests { @@ -302,23 +302,23 @@ func TestToString(t *testing.T) { Valid bool }{ {&LabelSelector{ - getRequirement("x", InOperator, util.NewStringSet("abc", "def"), t), - getRequirement("y", NotInOperator, util.NewStringSet("jkl"), t), + getRequirement("x", InOperator, sets.NewString("abc", "def"), t), + getRequirement("y", NotInOperator, sets.NewString("jkl"), t), getRequirement("z", ExistsOperator, nil, t)}, "x in (abc,def),y notin (jkl),z", true}, {&LabelSelector{ - getRequirement("x", InOperator, util.NewStringSet("abc", "def"), t), + getRequirement("x", InOperator, sets.NewString("abc", "def"), t), req}, // adding empty req for the trailing ',' "x in (abc,def),", false}, {&LabelSelector{ - getRequirement("x", NotInOperator, util.NewStringSet("abc"), t), - getRequirement("y", InOperator, util.NewStringSet("jkl", "mno"), t), - getRequirement("z", NotInOperator, util.NewStringSet(""), t)}, + getRequirement("x", NotInOperator, sets.NewString("abc"), t), + getRequirement("y", InOperator, sets.NewString("jkl", "mno"), t), + getRequirement("z", NotInOperator, sets.NewString(""), t)}, "x notin (abc),y in (jkl,mno),z notin ()", true}, {&LabelSelector{ - getRequirement("x", EqualsOperator, util.NewStringSet("abc"), t), - getRequirement("y", DoubleEqualsOperator, util.NewStringSet("jkl"), t), - getRequirement("z", NotEqualsOperator, util.NewStringSet("a"), t)}, + getRequirement("x", EqualsOperator, sets.NewString("abc"), t), + getRequirement("y", DoubleEqualsOperator, sets.NewString("jkl"), t), + getRequirement("z", NotEqualsOperator, sets.NewString("a"), t)}, "x=abc,y==jkl,z!=a", true}, } for _, ts := range toStringTests { @@ -341,19 +341,19 @@ func TestRequirementLabelSelectorMatching(t *testing.T) { req, }, false}, {Set{"x": "foo", "y": "baz"}, &LabelSelector{ - getRequirement("x", InOperator, util.NewStringSet("foo"), t), - getRequirement("y", NotInOperator, util.NewStringSet("alpha"), t), + getRequirement("x", InOperator, sets.NewString("foo"), t), + getRequirement("y", NotInOperator, sets.NewString("alpha"), t), }, true}, {Set{"x": "foo", "y": "baz"}, &LabelSelector{ - getRequirement("x", InOperator, util.NewStringSet("foo"), t), - getRequirement("y", InOperator, util.NewStringSet("alpha"), t), + getRequirement("x", InOperator, sets.NewString("foo"), t), + getRequirement("y", InOperator, sets.NewString("alpha"), t), }, false}, {Set{"y": ""}, &LabelSelector{ - getRequirement("x", NotInOperator, util.NewStringSet(""), t), + getRequirement("x", NotInOperator, sets.NewString(""), t), getRequirement("y", ExistsOperator, nil, t), }, true}, {Set{"y": "baz"}, &LabelSelector{ - getRequirement("x", InOperator, util.NewStringSet(""), t), + getRequirement("x", InOperator, sets.NewString(""), t), }, false}, } for _, lsm := range labelSelectorMatchingTests { @@ -378,60 +378,60 @@ func TestSetSelectorParser(t *testing.T) { getRequirement("this-is-a-dns.domain.com/key-with-dash", ExistsOperator, nil, t), }, true, true}, {"this-is-another-dns.domain.com/key-with-dash in (so,what)", LabelSelector{ - getRequirement("this-is-another-dns.domain.com/key-with-dash", InOperator, util.NewStringSet("so", "what"), t), + getRequirement("this-is-another-dns.domain.com/key-with-dash", InOperator, sets.NewString("so", "what"), t), }, true, true}, {"0.1.2.domain/99 notin (10.10.100.1, tick.tack.clock)", LabelSelector{ - getRequirement("0.1.2.domain/99", NotInOperator, util.NewStringSet("10.10.100.1", "tick.tack.clock"), t), + getRequirement("0.1.2.domain/99", NotInOperator, sets.NewString("10.10.100.1", "tick.tack.clock"), t), }, true, true}, {"foo in (abc)", LabelSelector{ - getRequirement("foo", InOperator, util.NewStringSet("abc"), t), + getRequirement("foo", InOperator, sets.NewString("abc"), t), }, true, true}, {"x notin\n (abc)", LabelSelector{ - getRequirement("x", NotInOperator, util.NewStringSet("abc"), t), + getRequirement("x", NotInOperator, sets.NewString("abc"), t), }, true, true}, {"x notin \t (abc,def)", LabelSelector{ - getRequirement("x", NotInOperator, util.NewStringSet("abc", "def"), t), + getRequirement("x", NotInOperator, sets.NewString("abc", "def"), t), }, true, true}, {"x in (abc,def)", LabelSelector{ - getRequirement("x", InOperator, util.NewStringSet("abc", "def"), t), + getRequirement("x", InOperator, sets.NewString("abc", "def"), t), }, true, true}, {"x in (abc,)", LabelSelector{ - getRequirement("x", InOperator, util.NewStringSet("abc", ""), t), + getRequirement("x", InOperator, sets.NewString("abc", ""), t), }, true, true}, {"x in ()", LabelSelector{ - getRequirement("x", InOperator, util.NewStringSet(""), t), + getRequirement("x", InOperator, sets.NewString(""), t), }, true, true}, {"x notin (abc,,def),bar,z in (),w", LabelSelector{ getRequirement("bar", ExistsOperator, nil, t), getRequirement("w", ExistsOperator, nil, t), - getRequirement("x", NotInOperator, util.NewStringSet("abc", "", "def"), t), - getRequirement("z", InOperator, util.NewStringSet(""), t), + getRequirement("x", NotInOperator, sets.NewString("abc", "", "def"), t), + getRequirement("z", InOperator, sets.NewString(""), t), }, true, true}, {"x,y in (a)", LabelSelector{ - getRequirement("y", InOperator, util.NewStringSet("a"), t), + getRequirement("y", InOperator, sets.NewString("a"), t), getRequirement("x", ExistsOperator, nil, t), }, false, true}, {"x=a", LabelSelector{ - getRequirement("x", EqualsOperator, util.NewStringSet("a"), t), + getRequirement("x", EqualsOperator, sets.NewString("a"), t), }, true, true}, {"x=a,y!=b", LabelSelector{ - getRequirement("x", EqualsOperator, util.NewStringSet("a"), t), - getRequirement("y", NotEqualsOperator, util.NewStringSet("b"), t), + getRequirement("x", EqualsOperator, sets.NewString("a"), t), + getRequirement("y", NotEqualsOperator, sets.NewString("b"), t), }, true, true}, {"x=a,y!=b,z in (h,i,j)", LabelSelector{ - getRequirement("x", EqualsOperator, util.NewStringSet("a"), t), - getRequirement("y", NotEqualsOperator, util.NewStringSet("b"), t), - getRequirement("z", InOperator, util.NewStringSet("h", "i", "j"), t), + getRequirement("x", EqualsOperator, sets.NewString("a"), t), + getRequirement("y", NotEqualsOperator, sets.NewString("b"), t), + getRequirement("z", InOperator, sets.NewString("h", "i", "j"), t), }, true, true}, {"x=a||y=b", LabelSelector{}, false, false}, {"x,,y", nil, true, false}, {",x,y", nil, true, false}, {"x nott in (y)", nil, true, false}, {"x notin ( )", LabelSelector{ - getRequirement("x", NotInOperator, util.NewStringSet(""), t), + getRequirement("x", NotInOperator, sets.NewString(""), t), }, true, true}, {"x notin (, a)", LabelSelector{ - getRequirement("x", NotInOperator, util.NewStringSet("", "a"), t), + getRequirement("x", NotInOperator, sets.NewString("", "a"), t), }, true, true}, {"a in (xyz),", nil, true, false}, {"a in (xyz)b notin ()", nil, true, false}, @@ -439,7 +439,7 @@ func TestSetSelectorParser(t *testing.T) { getRequirement("a", ExistsOperator, nil, t), }, true, true}, {"a in (x,y,notin, z,in)", LabelSelector{ - getRequirement("a", InOperator, util.NewStringSet("in", "notin", "x", "y", "z"), t), + getRequirement("a", InOperator, sets.NewString("in", "notin", "x", "y", "z"), t), }, true, true}, // operator 'in' inside list of identifiers {"a in (xyz abc)", nil, false, false}, // no comma {"a notin(", nil, true, false}, // bad formed @@ -458,7 +458,7 @@ func TestSetSelectorParser(t *testing.T) { } } -func getRequirement(key string, op Operator, vals util.StringSet, t *testing.T) Requirement { +func getRequirement(key string, op Operator, vals sets.String, t *testing.T) Requirement { req, err := NewRequirement(key, op, vals) if err != nil { t.Errorf("NewRequirement(%v, %v, %v) resulted in error:%v", key, op, vals, err) @@ -480,16 +480,16 @@ func TestAdd(t *testing.T) { "key", InOperator, []string{"value"}, - LabelSelector{Requirement{"key", InOperator, util.NewStringSet("value")}}, + LabelSelector{Requirement{"key", InOperator, sets.NewString("value")}}, }, { - LabelSelector{Requirement{"key", InOperator, util.NewStringSet("value")}}, + LabelSelector{Requirement{"key", InOperator, sets.NewString("value")}}, "key2", EqualsOperator, []string{"value2"}, LabelSelector{ - Requirement{"key", InOperator, util.NewStringSet("value")}, - Requirement{"key2", EqualsOperator, util.NewStringSet("value2")}, + Requirement{"key", InOperator, sets.NewString("value")}, + Requirement{"key2", EqualsOperator, sets.NewString("value2")}, }, }, } diff --git a/pkg/master/master.go b/pkg/master/master.go index 3cdc937b8ca..e3344954838 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -79,6 +79,7 @@ import ( "k8s.io/kubernetes/pkg/tools" "k8s.io/kubernetes/pkg/ui" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" daemonetcd "k8s.io/kubernetes/pkg/registry/daemonset/etcd" horizontalpodautoscaleretcd "k8s.io/kubernetes/pkg/registry/horizontalpodautoscaler/etcd" @@ -566,7 +567,7 @@ func (m *Master) init(c *Config) { apiserver.InstallSupport(m.muxHelper, m.rootWebService, c.EnableProfiling, healthzChecks...) apiserver.AddApiWebService(m.handlerContainer, c.APIPrefix, apiVersions) defaultVersion := m.defaultAPIGroupVersion() - requestInfoResolver := &apiserver.APIRequestInfoResolver{APIPrefixes: util.NewStringSet(strings.TrimPrefix(defaultVersion.Root, "/")), RestMapper: defaultVersion.Mapper} + requestInfoResolver := &apiserver.APIRequestInfoResolver{APIPrefixes: sets.NewString(strings.TrimPrefix(defaultVersion.Root, "/")), RestMapper: defaultVersion.Mapper} apiserver.InstallServiceErrorHandler(m.handlerContainer, requestInfoResolver, apiVersions) if m.exp { @@ -575,7 +576,7 @@ func (m *Master) init(c *Config) { glog.Fatalf("Unable to setup experimental api: %v", err) } apiserver.AddApiWebService(m.handlerContainer, c.ExpAPIPrefix, []string{expVersion.Version}) - expRequestInfoResolver := &apiserver.APIRequestInfoResolver{APIPrefixes: util.NewStringSet(strings.TrimPrefix(expVersion.Root, "/")), RestMapper: expVersion.Mapper} + expRequestInfoResolver := &apiserver.APIRequestInfoResolver{APIPrefixes: sets.NewString(strings.TrimPrefix(expVersion.Root, "/")), RestMapper: expVersion.Mapper} apiserver.InstallServiceErrorHandler(m.handlerContainer, expRequestInfoResolver, []string{expVersion.Version}) } @@ -784,7 +785,7 @@ func (m *Master) InstallThirdPartyAPI(rsrc *expapi.ThirdPartyResource) error { } thirdPartyPrefix := "/thirdparty/" + group + "/" apiserver.AddApiWebService(m.handlerContainer, thirdPartyPrefix, []string{rsrc.Versions[0].Name}) - thirdPartyRequestInfoResolver := &apiserver.APIRequestInfoResolver{APIPrefixes: util.NewStringSet(strings.TrimPrefix(group, "/")), RestMapper: thirdparty.Mapper} + thirdPartyRequestInfoResolver := &apiserver.APIRequestInfoResolver{APIPrefixes: sets.NewString(strings.TrimPrefix(group, "/")), RestMapper: thirdparty.Mapper} apiserver.InstallServiceErrorHandler(m.handlerContainer, thirdPartyRequestInfoResolver, []string{thirdparty.Version}) return nil } diff --git a/pkg/registry/generic/etcd/etcd_test.go b/pkg/registry/generic/etcd/etcd_test.go index cd389b7ef8c..8569ccc84df 100644 --- a/pkg/registry/generic/etcd/etcd_test.go +++ b/pkg/registry/generic/etcd/etcd_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/kubernetes/pkg/tools/etcdtest" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/fielderrors" + "k8s.io/kubernetes/pkg/util/sets" "github.com/coreos/go-etcd/etcd" ) @@ -93,7 +94,7 @@ func NewTestGenericEtcdRegistry(t *testing.T) (*tools.FakeEtcdClient, *Etcd) { // setMatcher is a matcher that matches any pod with id in the set. // Makes testing simpler. type setMatcher struct { - util.StringSet + sets.String } func (sm setMatcher) Matches(obj runtime.Object) (bool, error) { @@ -189,7 +190,7 @@ func TestEtcdList(t *testing.T) { R: singleElemListResp, E: nil, }, - m: setMatcher{util.NewStringSet("foo")}, + m: setMatcher{sets.NewString("foo")}, out: &api.PodList{Items: []api.Pod{*podA}}, succeed: true, }, @@ -198,7 +199,7 @@ func TestEtcdList(t *testing.T) { R: normalListResp, E: nil, }, - m: setMatcher{util.NewStringSet("foo", "makeMatchSingleReturnFalse")}, + m: setMatcher{sets.NewString("foo", "makeMatchSingleReturnFalse")}, out: &api.PodList{Items: []api.Pod{*podA}}, succeed: true, }, @@ -560,8 +561,8 @@ func TestEtcdDelete(t *testing.T) { func TestEtcdWatch(t *testing.T) { table := map[string]generic.Matcher{ - "single": setMatcher{util.NewStringSet("foo")}, - "multi": setMatcher{util.NewStringSet("foo", "bar")}, + "single": setMatcher{sets.NewString("foo")}, + "multi": setMatcher{sets.NewString("foo", "bar")}, } for name, m := range table { diff --git a/pkg/registry/service/ipallocator/allocator_test.go b/pkg/registry/service/ipallocator/allocator_test.go index 3d84053449d..c853ebab2e6 100644 --- a/pkg/registry/service/ipallocator/allocator_test.go +++ b/pkg/registry/service/ipallocator/allocator_test.go @@ -21,7 +21,7 @@ import ( "testing" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func TestAllocate(t *testing.T) { @@ -34,7 +34,7 @@ func TestAllocate(t *testing.T) { if f := r.Free(); f != 254 { t.Errorf("unexpected free %d", f) } - found := util.NewStringSet() + found := sets.NewString() count := 0 for r.Free() > 0 { ip, err := r.AllocateNext() @@ -118,7 +118,7 @@ func TestAllocateSmall(t *testing.T) { if f := r.Free(); f != 2 { t.Errorf("free: %d", f) } - found := util.NewStringSet() + found := sets.NewString() for i := 0; i < 2; i++ { ip, err := r.AllocateNext() if err != nil { diff --git a/pkg/registry/service/portallocator/allocator_test.go b/pkg/registry/service/portallocator/allocator_test.go index b7b108cefb0..dd6d5dd1701 100644 --- a/pkg/registry/service/portallocator/allocator_test.go +++ b/pkg/registry/service/portallocator/allocator_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) func TestAllocate(t *testing.T) { @@ -34,7 +35,7 @@ func TestAllocate(t *testing.T) { if f := r.Free(); f != 201 { t.Errorf("unexpected free %d", f) } - found := util.NewStringSet() + found := sets.NewString() count := 0 for r.Free() > 0 { p, err := r.AllocateNext() diff --git a/pkg/runtime/conversion_generator.go b/pkg/runtime/conversion_generator.go index 485d13d612e..979313ee53b 100644 --- a/pkg/runtime/conversion_generator.go +++ b/pkg/runtime/conversion_generator.go @@ -25,7 +25,7 @@ import ( "strings" "k8s.io/kubernetes/pkg/conversion" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) type ConversionGenerator interface { @@ -33,7 +33,7 @@ type ConversionGenerator interface { WriteConversionFunctions(w io.Writer) error RegisterConversionFunctions(w io.Writer, pkg string) error AddImport(pkg string) string - RepackImports(exclude util.StringSet) + RepackImports(exclude sets.String) WriteImports(w io.Writer) error OverwritePackage(pkg, overwrite string) AssumePrivateConversions() @@ -279,7 +279,7 @@ func (g *conversionGenerator) targetPackage(pkg string) { g.shortImports[""] = pkg } -func (g *conversionGenerator) RepackImports(exclude util.StringSet) { +func (g *conversionGenerator) RepackImports(exclude sets.String) { var packages []string for key := range g.imports { packages = append(packages, key) diff --git a/pkg/runtime/deep_copy_generator.go b/pkg/runtime/deep_copy_generator.go index 45fe3a01926..5dee4e263ab 100644 --- a/pkg/runtime/deep_copy_generator.go +++ b/pkg/runtime/deep_copy_generator.go @@ -25,7 +25,7 @@ import ( "strings" "k8s.io/kubernetes/pkg/conversion" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // TODO(wojtek-t): As suggested in #8320, we should consider the strategy @@ -69,7 +69,7 @@ type DeepCopyGenerator interface { OverwritePackage(pkg, overwrite string) } -func NewDeepCopyGenerator(scheme *conversion.Scheme, targetPkg string, include util.StringSet) DeepCopyGenerator { +func NewDeepCopyGenerator(scheme *conversion.Scheme, targetPkg string, include sets.String) DeepCopyGenerator { g := &deepCopyGenerator{ scheme: scheme, targetPkg: targetPkg, @@ -100,7 +100,7 @@ type deepCopyGenerator struct { shortImports map[string]string pkgOverwrites map[string]string replace map[pkgPathNamePair]reflect.Type - include util.StringSet + include sets.String } func (g *deepCopyGenerator) addImportByPath(pkg string) string { diff --git a/pkg/storage/cacher_test.go b/pkg/storage/cacher_test.go index b9b7895efac..b0c61e86e14 100644 --- a/pkg/storage/cacher_test.go +++ b/pkg/storage/cacher_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/tools" "k8s.io/kubernetes/pkg/tools/etcdtest" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/watch" ) @@ -160,7 +161,7 @@ func TestListFromMemory(t *testing.T) { if len(result.Items) != 2 { t.Errorf("unexpected list result: %d", len(result.Items)) } - keys := util.StringSet{} + keys := sets.String{} for _, item := range result.Items { keys.Insert(item.ObjectMeta.Name) } diff --git a/pkg/storage/watch_cache_test.go b/pkg/storage/watch_cache_test.go index f94e35d94f4..6ab467e8ba1 100644 --- a/pkg/storage/watch_cache_test.go +++ b/pkg/storage/watch_cache_test.go @@ -24,6 +24,7 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned/cache" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" ) @@ -76,7 +77,7 @@ func TestWatchCacheBasic(t *testing.T) { store.Add(makeTestPod("pod2", 5)) store.Add(makeTestPod("pod3", 6)) { - podNames := util.StringSet{} + podNames := sets.String{} for _, item := range store.List() { podNames.Insert(item.(*api.Pod).ObjectMeta.Name) } @@ -94,7 +95,7 @@ func TestWatchCacheBasic(t *testing.T) { makeTestPod("pod5", 8), }, "8") { - podNames := util.StringSet{} + podNames := sets.String{} for _, item := range store.List() { podNames.Insert(item.(*api.Pod).ObjectMeta.Name) } diff --git a/pkg/util/bandwidth/linux.go b/pkg/util/bandwidth/linux.go index e1ebd687b1a..7989394ec94 100644 --- a/pkg/util/bandwidth/linux.go +++ b/pkg/util/bandwidth/linux.go @@ -25,8 +25,8 @@ import ( "strings" "k8s.io/kubernetes/pkg/api/resource" - "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/exec" + "k8s.io/kubernetes/pkg/util/sets" "github.com/golang/glog" ) @@ -65,7 +65,7 @@ func (t *tcShaper) nextClassID() (int, error) { } scanner := bufio.NewScanner(bytes.NewBuffer(data)) - classes := util.StringSet{} + classes := sets.String{} for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) // skip empty lines diff --git a/pkg/util/iptables/iptables.go b/pkg/util/iptables/iptables.go index 5de379a9fe5..ab0f0e71ff5 100644 --- a/pkg/util/iptables/iptables.go +++ b/pkg/util/iptables/iptables.go @@ -26,8 +26,8 @@ import ( "github.com/coreos/go-semver/semver" "github.com/golang/glog" - "k8s.io/kubernetes/pkg/util" utilexec "k8s.io/kubernetes/pkg/util/exec" + "k8s.io/kubernetes/pkg/util/sets" ) type RulePosition string @@ -352,7 +352,7 @@ func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...st tmpField := strings.Trim(args[i], "\"") argsCopy = append(argsCopy, strings.Fields(tmpField)...) } - argset := util.NewStringSet(argsCopy...) + argset := sets.NewString(argsCopy...) for _, line := range strings.Split(string(out), "\n") { var fields = strings.Fields(line) @@ -370,7 +370,7 @@ func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...st } // TODO: This misses reorderings e.g. "-x foo ! -y bar" will match "! -x foo -y bar" - if util.NewStringSet(fields...).IsSuperset(argset) { + if sets.NewString(fields...).IsSuperset(argset) { return true, nil } glog.V(5).Infof("DBG: fields is not a superset of args: fields=%v args=%v", fields, args) diff --git a/pkg/util/iptables/iptables_test.go b/pkg/util/iptables/iptables_test.go index b515df2614d..c8f7c1a100e 100644 --- a/pkg/util/iptables/iptables_test.go +++ b/pkg/util/iptables/iptables_test.go @@ -20,8 +20,8 @@ import ( "strings" "testing" - "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/exec" + "k8s.io/kubernetes/pkg/util/sets" ) func getIptablesCommand(protocol Protocol) string { @@ -68,7 +68,7 @@ func testEnsureChain(t *testing.T, protocol Protocol) { t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } cmd := getIptablesCommand(protocol) - if !util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAll(cmd, "-t", "nat", "-N", "FOOBAR") { + if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll(cmd, "-t", "nat", "-N", "FOOBAR") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } // Exists. @@ -121,7 +121,7 @@ func TestFlushChain(t *testing.T) { if fcmd.CombinedOutputCalls != 2 { t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-F", "FOOBAR") { + if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-F", "FOOBAR") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } // Failure. @@ -158,7 +158,7 @@ func TestDeleteChain(t *testing.T) { if fcmd.CombinedOutputCalls != 2 { t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-X", "FOOBAR") { + if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-X", "FOOBAR") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } // Failure. @@ -196,7 +196,7 @@ func TestEnsureRuleAlreadyExists(t *testing.T) { if fcmd.CombinedOutputCalls != 2 { t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-C", "OUTPUT", "abc", "123") { + if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-C", "OUTPUT", "abc", "123") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } } @@ -232,7 +232,7 @@ func TestEnsureRuleNew(t *testing.T) { if fcmd.CombinedOutputCalls != 3 { t.Errorf("expected 3 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[2]...).HasAll("iptables", "-t", "nat", "-A", "OUTPUT", "abc", "123") { + if !sets.NewString(fcmd.CombinedOutputLog[2]...).HasAll("iptables", "-t", "nat", "-A", "OUTPUT", "abc", "123") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2]) } } @@ -319,7 +319,7 @@ func TestDeleteRuleAlreadyExists(t *testing.T) { if fcmd.CombinedOutputCalls != 2 { t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-C", "OUTPUT", "abc", "123") { + if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-C", "OUTPUT", "abc", "123") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } } @@ -352,7 +352,7 @@ func TestDeleteRuleNew(t *testing.T) { if fcmd.CombinedOutputCalls != 3 { t.Errorf("expected 3 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[2]...).HasAll("iptables", "-t", "nat", "-D", "OUTPUT", "abc", "123") { + if !sets.NewString(fcmd.CombinedOutputLog[2]...).HasAll("iptables", "-t", "nat", "-D", "OUTPUT", "abc", "123") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2]) } } @@ -484,7 +484,7 @@ COMMIT if fcmd.CombinedOutputCalls != 1 { t.Errorf("expected 1 CombinedOutput() call, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[0]...).HasAll("iptables-save", "-t", "nat") { + if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll("iptables-save", "-t", "nat") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0]) } } @@ -522,7 +522,7 @@ COMMIT if fcmd.CombinedOutputCalls != 1 { t.Errorf("expected 1 CombinedOutput() call, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[0]...).HasAll("iptables-save", "-t", "nat") { + if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll("iptables-save", "-t", "nat") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0]) } } @@ -573,7 +573,7 @@ func TestWaitFlagUnavailable(t *testing.T) { if fcmd.CombinedOutputCalls != 2 { t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAny("-w", "-w2") { + if sets.NewString(fcmd.CombinedOutputLog[1]...).HasAny("-w", "-w2") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } } @@ -601,10 +601,10 @@ func TestWaitFlagOld(t *testing.T) { if fcmd.CombinedOutputCalls != 2 { t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-w") { + if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-w") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } - if util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAny("-w2") { + if sets.NewString(fcmd.CombinedOutputLog[1]...).HasAny("-w2") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } } @@ -632,10 +632,10 @@ func TestWaitFlagNew(t *testing.T) { if fcmd.CombinedOutputCalls != 2 { t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls) } - if !util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-w2") { + if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-w2") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } - if util.NewStringSet(fcmd.CombinedOutputLog[1]...).HasAny("-w") { + if sets.NewString(fcmd.CombinedOutputLog[1]...).HasAny("-w") { t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1]) } } diff --git a/pkg/util/proxy/transport.go b/pkg/util/proxy/transport.go index b4f5b6b49d2..ca88147752d 100644 --- a/pkg/util/proxy/transport.go +++ b/pkg/util/proxy/transport.go @@ -31,38 +31,38 @@ import ( "golang.org/x/net/html" "golang.org/x/net/html/atom" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) // atomsToAttrs states which attributes of which tags require URL substitution. // Sources: http://www.w3.org/TR/REC-html40/index/attributes.html // http://www.w3.org/html/wg/drafts/html/master/index.html#attributes-1 -var atomsToAttrs = map[atom.Atom]util.StringSet{ - atom.A: util.NewStringSet("href"), - atom.Applet: util.NewStringSet("codebase"), - atom.Area: util.NewStringSet("href"), - atom.Audio: util.NewStringSet("src"), - atom.Base: util.NewStringSet("href"), - atom.Blockquote: util.NewStringSet("cite"), - atom.Body: util.NewStringSet("background"), - atom.Button: util.NewStringSet("formaction"), - atom.Command: util.NewStringSet("icon"), - atom.Del: util.NewStringSet("cite"), - atom.Embed: util.NewStringSet("src"), - atom.Form: util.NewStringSet("action"), - atom.Frame: util.NewStringSet("longdesc", "src"), - atom.Head: util.NewStringSet("profile"), - atom.Html: util.NewStringSet("manifest"), - atom.Iframe: util.NewStringSet("longdesc", "src"), - atom.Img: util.NewStringSet("longdesc", "src", "usemap"), - atom.Input: util.NewStringSet("src", "usemap", "formaction"), - atom.Ins: util.NewStringSet("cite"), - atom.Link: util.NewStringSet("href"), - atom.Object: util.NewStringSet("classid", "codebase", "data", "usemap"), - atom.Q: util.NewStringSet("cite"), - atom.Script: util.NewStringSet("src"), - atom.Source: util.NewStringSet("src"), - atom.Video: util.NewStringSet("poster", "src"), +var atomsToAttrs = map[atom.Atom]sets.String{ + atom.A: sets.NewString("href"), + atom.Applet: sets.NewString("codebase"), + atom.Area: sets.NewString("href"), + atom.Audio: sets.NewString("src"), + atom.Base: sets.NewString("href"), + atom.Blockquote: sets.NewString("cite"), + atom.Body: sets.NewString("background"), + atom.Button: sets.NewString("formaction"), + atom.Command: sets.NewString("icon"), + atom.Del: sets.NewString("cite"), + atom.Embed: sets.NewString("src"), + atom.Form: sets.NewString("action"), + atom.Frame: sets.NewString("longdesc", "src"), + atom.Head: sets.NewString("profile"), + atom.Html: sets.NewString("manifest"), + atom.Iframe: sets.NewString("longdesc", "src"), + atom.Img: sets.NewString("longdesc", "src", "usemap"), + atom.Input: sets.NewString("src", "usemap", "formaction"), + atom.Ins: sets.NewString("cite"), + atom.Link: sets.NewString("href"), + atom.Object: sets.NewString("classid", "codebase", "data", "usemap"), + atom.Q: sets.NewString("cite"), + atom.Script: sets.NewString("src"), + atom.Source: sets.NewString("src"), + atom.Video: sets.NewString("poster", "src"), // TODO: css URLs hidden in style elements. } diff --git a/pkg/util/set.go b/pkg/util/sets/set.go similarity index 79% rename from pkg/util/set.go rename to pkg/util/sets/set.go index 431312c6d51..a0b8900b361 100644 --- a/pkg/util/set.go +++ b/pkg/util/sets/set.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package sets import ( "reflect" @@ -26,19 +26,19 @@ import ( type Empty struct{} // StringSet is a set of strings, implemented via map[string]struct{} for minimal memory consumption. -type StringSet map[string]Empty +type String map[string]Empty -// NewStringSet creates a StringSet from a list of values. -func NewStringSet(items ...string) StringSet { - ss := StringSet{} +// New creates a StringSet from a list of values. +func NewString(items ...string) String { + ss := String{} ss.Insert(items...) return ss } // KeySet creates a StringSet from a keys of a map[string](? extends interface{}). Since you can't describe that map type in the Go type system // the reflected value is required. -func KeySet(theMap reflect.Value) StringSet { - ret := StringSet{} +func KeySet(theMap reflect.Value) String { + ret := String{} for _, keyValue := range theMap.MapKeys() { ret.Insert(keyValue.String()) @@ -48,27 +48,27 @@ func KeySet(theMap reflect.Value) StringSet { } // Insert adds items to the set. -func (s StringSet) Insert(items ...string) { +func (s String) Insert(items ...string) { for _, item := range items { s[item] = Empty{} } } // Delete removes all items from the set. -func (s StringSet) Delete(items ...string) { +func (s String) Delete(items ...string) { for _, item := range items { delete(s, item) } } // Has returns true iff item is contained in the set. -func (s StringSet) Has(item string) bool { +func (s String) Has(item string) bool { _, contained := s[item] return contained } // HasAll returns true iff all items are contained in the set. -func (s StringSet) HasAll(items ...string) bool { +func (s String) HasAll(items ...string) bool { for _, item := range items { if !s.Has(item) { return false @@ -78,7 +78,7 @@ func (s StringSet) HasAll(items ...string) bool { } // HasAny returns true if any items are contained in the set. -func (s StringSet) HasAny(items ...string) bool { +func (s String) HasAny(items ...string) bool { for _, item := range items { if s.Has(item) { return true @@ -93,8 +93,8 @@ func (s StringSet) HasAny(items ...string) bool { // s2 = {1, 2, 4, 5} // s1.Difference(s2) = {3} // s2.Difference(s1) = {4, 5} -func (s StringSet) Difference(s2 StringSet) StringSet { - result := NewStringSet() +func (s String) Difference(s2 String) String { + result := NewString() for key := range s { if !s2.Has(key) { result.Insert(key) @@ -110,8 +110,8 @@ func (s StringSet) Difference(s2 StringSet) StringSet { // s2 = {3, 4} // s1.Union(s2) = {1, 2, 3, 4} // s2.Union(s1) = {1, 2, 3, 4} -func (s1 StringSet) Union(s2 StringSet) StringSet { - result := NewStringSet() +func (s1 String) Union(s2 String) String { + result := NewString() for key := range s1 { result.Insert(key) } @@ -122,7 +122,7 @@ func (s1 StringSet) Union(s2 StringSet) StringSet { } // IsSuperset returns true iff s1 is a superset of s2. -func (s1 StringSet) IsSuperset(s2 StringSet) bool { +func (s1 String) IsSuperset(s2 String) bool { for item := range s2 { if !s1.Has(item) { return false @@ -134,7 +134,7 @@ func (s1 StringSet) IsSuperset(s2 StringSet) bool { // Equal returns true iff s1 is equal (as a set) to s2. // Two sets are equal if their membership is identical. // (In practice, this means same elements, order doesn't matter) -func (s1 StringSet) Equal(s2 StringSet) bool { +func (s1 String) Equal(s2 String) bool { if len(s1) != len(s2) { return false } @@ -147,7 +147,7 @@ func (s1 StringSet) Equal(s2 StringSet) bool { } // List returns the contents as a sorted string slice. -func (s StringSet) List() []string { +func (s String) List() []string { res := make([]string, 0, len(s)) for key := range s { res = append(res, key) @@ -157,7 +157,7 @@ func (s StringSet) List() []string { } // Returns a single element from the set. -func (s StringSet) PopAny() (string, bool) { +func (s String) PopAny() (string, bool) { for key := range s { s.Delete(key) return key, true @@ -166,6 +166,6 @@ func (s StringSet) PopAny() (string, bool) { } // Len returns the size of the set. -func (s StringSet) Len() int { +func (s String) Len() int { return len(s) } diff --git a/pkg/util/set_test.go b/pkg/util/sets/set_test.go similarity index 88% rename from pkg/util/set_test.go rename to pkg/util/sets/set_test.go index 936891feebf..dd401cfd197 100644 --- a/pkg/util/set_test.go +++ b/pkg/util/sets/set_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package sets import ( "reflect" @@ -22,8 +22,8 @@ import ( ) func TestStringSet(t *testing.T) { - s := StringSet{} - s2 := StringSet{} + s := String{} + s2 := String{} if len(s) != 0 { t.Errorf("Expected len=0: %d", len(s)) } @@ -60,7 +60,7 @@ func TestStringSet(t *testing.T) { } func TestStringSetDeleteMultiples(t *testing.T) { - s := StringSet{} + s := String{} s.Insert("a", "b", "c") if len(s) != 3 { t.Errorf("Expected len=3: %d", len(s)) @@ -83,7 +83,7 @@ func TestStringSetDeleteMultiples(t *testing.T) { } func TestNewStringSet(t *testing.T) { - s := NewStringSet("a", "b", "c") + s := NewString("a", "b", "c") if len(s) != 3 { t.Errorf("Expected len=3: %d", len(s)) } @@ -93,15 +93,15 @@ func TestNewStringSet(t *testing.T) { } func TestStringSetList(t *testing.T) { - s := NewStringSet("z", "y", "x", "a") + s := NewString("z", "y", "x", "a") if !reflect.DeepEqual(s.List(), []string{"a", "x", "y", "z"}) { t.Errorf("List gave unexpected result: %#v", s.List()) } } func TestStringSetDifference(t *testing.T) { - a := NewStringSet("1", "2", "3") - b := NewStringSet("1", "2", "4", "5") + a := NewString("1", "2", "3") + b := NewString("1", "2", "4", "5") c := a.Difference(b) d := b.Difference(a) if len(c) != 1 { @@ -119,7 +119,7 @@ func TestStringSetDifference(t *testing.T) { } func TestStringSetHasAny(t *testing.T) { - a := NewStringSet("1", "2", "3") + a := NewString("1", "2", "3") if !a.HasAny("1", "4") { t.Errorf("expected true, got false") @@ -132,37 +132,37 @@ func TestStringSetHasAny(t *testing.T) { func TestStringSetEquals(t *testing.T) { // Simple case (order doesn't matter) - a := NewStringSet("1", "2") - b := NewStringSet("2", "1") + a := NewString("1", "2") + b := NewString("2", "1") if !a.Equal(b) { t.Errorf("Expected to be equal: %v vs %v", a, b) } // It is a set; duplicates are ignored - b = NewStringSet("2", "2", "1") + b = NewString("2", "2", "1") if !a.Equal(b) { t.Errorf("Expected to be equal: %v vs %v", a, b) } // Edge cases around empty sets / empty strings - a = NewStringSet() - b = NewStringSet() + a = NewString() + b = NewString() if !a.Equal(b) { t.Errorf("Expected to be equal: %v vs %v", a, b) } - b = NewStringSet("1", "2", "3") + b = NewString("1", "2", "3") if a.Equal(b) { t.Errorf("Expected to be not-equal: %v vs %v", a, b) } - b = NewStringSet("1", "2", "") + b = NewString("1", "2", "") if a.Equal(b) { t.Errorf("Expected to be not-equal: %v vs %v", a, b) } // Check for equality after mutation - a = NewStringSet() + a = NewString() a.Insert("1") if a.Equal(b) { t.Errorf("Expected to be not-equal: %v vs %v", a, b) diff --git a/pkg/volume/gce_pd/gce_util.go b/pkg/volume/gce_pd/gce_util.go index 261e1a04ca9..47ddeb5f518 100644 --- a/pkg/volume/gce_pd/gce_util.go +++ b/pkg/volume/gce_pd/gce_util.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/operationmanager" + "k8s.io/kubernetes/pkg/util/sets" ) const ( @@ -62,7 +63,7 @@ func (diskUtil *GCEDiskUtil) AttachAndMountDisk(b *gcePersistentDiskBuilder, glo if err != nil { glog.Errorf("Error filepath.Glob(\"%s\"): %v\r\n", diskSDPattern, err) } - sdBeforeSet := util.NewStringSet(sdBefore...) + sdBeforeSet := sets.NewString(sdBefore...) devicePath, err := attachDiskAndVerify(b, sdBeforeSet) if err != nil { @@ -120,7 +121,7 @@ func (util *GCEDiskUtil) DetachDisk(c *gcePersistentDiskCleaner) error { } // Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails. -func attachDiskAndVerify(b *gcePersistentDiskBuilder, sdBeforeSet util.StringSet) (string, error) { +func attachDiskAndVerify(b *gcePersistentDiskBuilder, sdBeforeSet sets.String) (string, error) { devicePaths := getDiskByIdPaths(b.gcePersistentDisk) var gce cloudprovider.Interface for numRetries := 0; numRetries < maxRetries; numRetries++ { @@ -287,7 +288,7 @@ func pathExists(path string) (bool, error) { // Calls "udevadm trigger --action=change" for newly created "/dev/sd*" drives (exist only in after set). // This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed. -func udevadmChangeToNewDrives(sdBeforeSet util.StringSet) error { +func udevadmChangeToNewDrives(sdBeforeSet sets.String) error { sdAfter, err := filepath.Glob(diskSDPattern) if err != nil { return fmt.Errorf("Error filepath.Glob(\"%s\"): %v\r\n", diskSDPattern, err) diff --git a/plugin/pkg/admission/namespace/lifecycle/admission.go b/plugin/pkg/admission/namespace/lifecycle/admission.go index df9eae8eedd..ac644cf1a9a 100644 --- a/plugin/pkg/admission/namespace/lifecycle/admission.go +++ b/plugin/pkg/admission/namespace/lifecycle/admission.go @@ -30,7 +30,7 @@ import ( "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" ) @@ -46,7 +46,7 @@ type lifecycle struct { *admission.Handler client client.Interface store cache.Store - immortalNamespaces util.StringSet + immortalNamespaces sets.String } func (l *lifecycle) Admit(a admission.Attributes) (err error) { @@ -120,6 +120,6 @@ func NewLifecycle(c client.Interface) admission.Interface { Handler: admission.NewHandler(admission.Create, admission.Update, admission.Delete), client: c, store: store, - immortalNamespaces: util.NewStringSet(api.NamespaceDefault), + immortalNamespaces: sets.NewString(api.NamespaceDefault), } } diff --git a/plugin/pkg/admission/serviceaccount/admission.go b/plugin/pkg/admission/serviceaccount/admission.go index 6b5f8aaa643..8590b33c328 100644 --- a/plugin/pkg/admission/serviceaccount/admission.go +++ b/plugin/pkg/admission/serviceaccount/admission.go @@ -31,7 +31,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" ) @@ -249,7 +249,7 @@ func (s *serviceAccount) getReferencedServiceAccountToken(serviceAccount *api.Se return "", err } - references := util.NewStringSet() + references := sets.NewString() for _, secret := range serviceAccount.Secrets { references.Insert(secret.Name) } @@ -293,7 +293,7 @@ func (s *serviceAccount) getServiceAccountTokens(serviceAccount *api.ServiceAcco func (s *serviceAccount) limitSecretReferences(serviceAccount *api.ServiceAccount, pod *api.Pod) error { // Ensure all secrets the pod references are allowed by the service account - mountableSecrets := util.NewStringSet() + mountableSecrets := sets.NewString() for _, s := range serviceAccount.Secrets { mountableSecrets.Insert(s.Name) } @@ -309,7 +309,7 @@ func (s *serviceAccount) limitSecretReferences(serviceAccount *api.ServiceAccoun } // limit pull secret references as well - pullSecrets := util.NewStringSet() + pullSecrets := sets.NewString() for _, s := range serviceAccount.ImagePullSecrets { pullSecrets.Insert(s.Name) } @@ -340,7 +340,7 @@ func (s *serviceAccount) mountServiceAccountToken(serviceAccount *api.ServiceAcc // Find the volume and volume name for the ServiceAccountTokenSecret if it already exists tokenVolumeName := "" hasTokenVolume := false - allVolumeNames := util.NewStringSet() + allVolumeNames := sets.NewString() for _, volume := range pod.Spec.Volumes { allVolumeNames.Insert(volume.Name) if volume.Secret != nil && volume.Secret.SecretName == serviceAccountToken { diff --git a/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go b/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go index 753827f6f58..c8b801fd52f 100644 --- a/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go +++ b/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go @@ -18,7 +18,7 @@ limitations under the License. package defaults import ( - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/plugin/pkg/scheduler" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates" @@ -48,8 +48,8 @@ func init() { ) } -func defaultPredicates() util.StringSet { - return util.NewStringSet( +func defaultPredicates() sets.String { + return sets.NewString( // Fit is defined based on the absence of port conflicts. factory.RegisterFitPredicate("PodFitsPorts", predicates.PodFitsPorts), // Fit is determined by resource availability. @@ -73,8 +73,8 @@ func defaultPredicates() util.StringSet { ) } -func defaultPriorities() util.StringSet { - return util.NewStringSet( +func defaultPriorities() sets.String { + return sets.NewString( // Prioritize nodes by least requested utilization. factory.RegisterPriorityFunction("LeastRequestedPriority", priorities.LeastRequestedPriority, 1), // Prioritizes nodes to help achieve balanced resource usage diff --git a/plugin/pkg/scheduler/factory/factory.go b/plugin/pkg/scheduler/factory/factory.go index fbe4467fc59..2b7b79293eb 100644 --- a/plugin/pkg/scheduler/factory/factory.go +++ b/plugin/pkg/scheduler/factory/factory.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/controller/framework" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/plugin/pkg/scheduler" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api" @@ -137,13 +138,13 @@ func (f *ConfigFactory) CreateFromConfig(policy schedulerapi.Policy) (*scheduler return nil, err } - predicateKeys := util.NewStringSet() + predicateKeys := sets.NewString() for _, predicate := range policy.Predicates { glog.V(2).Infof("Registering predicate: %s", predicate.Name) predicateKeys.Insert(RegisterCustomFitPredicate(predicate)) } - priorityKeys := util.NewStringSet() + priorityKeys := sets.NewString() for _, priority := range policy.Priorities { glog.V(2).Infof("Registering priority: %s", priority.Name) priorityKeys.Insert(RegisterCustomPriorityFunction(priority)) @@ -153,7 +154,7 @@ func (f *ConfigFactory) CreateFromConfig(policy schedulerapi.Policy) (*scheduler } // Creates a scheduler from a set of registered fit predicate keys and priority keys. -func (f *ConfigFactory) CreateFromKeys(predicateKeys, priorityKeys util.StringSet) (*scheduler.Config, error) { +func (f *ConfigFactory) CreateFromKeys(predicateKeys, priorityKeys sets.String) (*scheduler.Config, error) { glog.V(2).Infof("creating scheduler with fit predicates '%v' and priority functions '%v", predicateKeys, priorityKeys) pluginArgs := PluginFactoryArgs{ PodLister: f.PodLister, diff --git a/plugin/pkg/scheduler/factory/plugins.go b/plugin/pkg/scheduler/factory/plugins.go index 99953f7e50d..6251c86098c 100644 --- a/plugin/pkg/scheduler/factory/plugins.go +++ b/plugin/pkg/scheduler/factory/plugins.go @@ -22,7 +22,7 @@ import ( "strings" "sync" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities" @@ -66,8 +66,8 @@ const ( ) type AlgorithmProviderConfig struct { - FitPredicateKeys util.StringSet - PriorityFunctionKeys util.StringSet + FitPredicateKeys sets.String + PriorityFunctionKeys sets.String } // RegisterFitPredicate registers a fit predicate with the algorithm @@ -209,7 +209,7 @@ func IsPriorityFunctionRegistered(name string) bool { // Registers a new algorithm provider with the algorithm registry. This should // be called from the init function in a provider plugin. -func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys util.StringSet) string { +func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys sets.String) string { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() validateAlgorithmNameOrDie(name) @@ -234,7 +234,7 @@ func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) { return &provider, nil } -func getFitPredicateFunctions(names util.StringSet, args PluginFactoryArgs) (map[string]algorithm.FitPredicate, error) { +func getFitPredicateFunctions(names sets.String, args PluginFactoryArgs) (map[string]algorithm.FitPredicate, error) { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() @@ -249,7 +249,7 @@ func getFitPredicateFunctions(names util.StringSet, args PluginFactoryArgs) (map return predicates, nil } -func getPriorityFunctionConfigs(names util.StringSet, args PluginFactoryArgs) ([]algorithm.PriorityConfig, error) { +func getPriorityFunctionConfigs(names sets.String, args PluginFactoryArgs) ([]algorithm.PriorityConfig, error) { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() diff --git a/plugin/pkg/scheduler/generic_scheduler.go b/plugin/pkg/scheduler/generic_scheduler.go index 07a2e855162..e0e7b06a15c 100644 --- a/plugin/pkg/scheduler/generic_scheduler.go +++ b/plugin/pkg/scheduler/generic_scheduler.go @@ -25,12 +25,12 @@ import ( "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates" ) -type FailedPredicateMap map[string]util.StringSet +type FailedPredicateMap map[string]sets.String type FitError struct { Pod *api.Pod @@ -124,7 +124,7 @@ func findNodesThatFit(pod *api.Pod, podLister algorithm.PodLister, predicateFunc if !fit { fits = false if _, found := failedPredicateMap[node.Name]; !found { - failedPredicateMap[node.Name] = util.StringSet{} + failedPredicateMap[node.Name] = sets.String{} } if predicates.FailedResourceType != "" { failedPredicateMap[node.Name].Insert(predicates.FailedResourceType) diff --git a/plugin/pkg/scheduler/generic_scheduler_test.go b/plugin/pkg/scheduler/generic_scheduler_test.go index 37b95483454..4f1a1b416c4 100644 --- a/plugin/pkg/scheduler/generic_scheduler_test.go +++ b/plugin/pkg/scheduler/generic_scheduler_test.go @@ -24,7 +24,7 @@ import ( "testing" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" ) @@ -101,7 +101,7 @@ func TestSelectHost(t *testing.T) { scheduler := genericScheduler{random: rand.New(rand.NewSource(0))} tests := []struct { list algorithm.HostPriorityList - possibleHosts util.StringSet + possibleHosts sets.String expectsErr bool }{ { @@ -109,7 +109,7 @@ func TestSelectHost(t *testing.T) { {Host: "machine1.1", Score: 1}, {Host: "machine2.1", Score: 2}, }, - possibleHosts: util.NewStringSet("machine2.1"), + possibleHosts: sets.NewString("machine2.1"), expectsErr: false, }, // equal scores @@ -120,7 +120,7 @@ func TestSelectHost(t *testing.T) { {Host: "machine1.3", Score: 2}, {Host: "machine2.1", Score: 2}, }, - possibleHosts: util.NewStringSet("machine1.2", "machine1.3", "machine2.1"), + possibleHosts: sets.NewString("machine1.2", "machine1.3", "machine2.1"), expectsErr: false, }, // out of order scores @@ -132,13 +132,13 @@ func TestSelectHost(t *testing.T) { {Host: "machine3.1", Score: 1}, {Host: "machine1.3", Score: 3}, }, - possibleHosts: util.NewStringSet("machine1.1", "machine1.2", "machine1.3"), + possibleHosts: sets.NewString("machine1.1", "machine1.2", "machine1.3"), expectsErr: false, }, // empty priorityList { list: []algorithm.HostPriority{}, - possibleHosts: util.NewStringSet(), + possibleHosts: sets.NewString(), expectsErr: true, }, } diff --git a/test/e2e/daemon_restart.go b/test/e2e/daemon_restart.go index 57f8b2d0fa0..4d3e971b8cf 100644 --- a/test/e2e/daemon_restart.go +++ b/test/e2e/daemon_restart.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/watch" @@ -140,7 +141,7 @@ func getContainerRestarts(c *client.Client, ns string, labelSelector labels.Sele pods, err := c.Pods(ns).List(labelSelector, fields.Everything()) expectNoError(err) failedContainers := 0 - containerRestartNodes := util.NewStringSet() + containerRestartNodes := sets.NewString() for _, p := range pods.Items { for _, v := range FailedContainers(&p) { failedContainers = failedContainers + v.restarts @@ -224,8 +225,8 @@ var _ = Describe("DaemonRestart", func() { // Only check the keys, the pods can be different if the kubelet updated it. // TODO: Can it really? - existingKeys := util.NewStringSet() - newKeys := util.NewStringSet() + existingKeys := sets.NewString() + newKeys := sets.NewString() for _, k := range existingPods.ListKeys() { existingKeys.Insert(k) } diff --git a/test/e2e/density.go b/test/e2e/density.go index 00c7ad44ad2..cb310aaa117 100644 --- a/test/e2e/density.go +++ b/test/e2e/density.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" . "github.com/onsi/ginkgo" @@ -148,7 +149,7 @@ var _ = Describe("Density", func() { expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after")) // Verify latency metrics - highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events")) + highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, sets.NewString("events")) expectNoError(err) Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests") }) diff --git a/test/e2e/kubelet.go b/test/e2e/kubelet.go index 0cee3811392..6741e7bf63a 100644 --- a/test/e2e/kubelet.go +++ b/test/e2e/kubelet.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" . "github.com/onsi/ginkgo" @@ -41,8 +42,8 @@ const ( // getPodMatches returns a set of pod names on the given node that matches the // podNamePrefix and namespace. -func getPodMatches(c *client.Client, nodeName string, podNamePrefix string, namespace string) util.StringSet { - matches := util.NewStringSet() +func getPodMatches(c *client.Client, nodeName string, podNamePrefix string, namespace string) sets.String { + matches := sets.NewString() Logf("Checking pods on node %v via /runningpods endpoint", nodeName) runningPods, err := GetKubeletPods(c, nodeName) if err != nil { @@ -65,9 +66,9 @@ func getPodMatches(c *client.Client, nodeName string, podNamePrefix string, name // information; they are reconstructed by examining the container runtime. In // the scope of this test, we do not expect pod naming conflicts so // podNamePrefix should be sufficient to identify the pods. -func waitTillNPodsRunningOnNodes(c *client.Client, nodeNames util.StringSet, podNamePrefix string, namespace string, targetNumPods int, timeout time.Duration) error { +func waitTillNPodsRunningOnNodes(c *client.Client, nodeNames sets.String, podNamePrefix string, namespace string, targetNumPods int, timeout time.Duration) error { return wait.Poll(pollInterval, timeout, func() (bool, error) { - matchCh := make(chan util.StringSet, len(nodeNames)) + matchCh := make(chan sets.String, len(nodeNames)) for _, item := range nodeNames.List() { // Launch a goroutine per node to check the pods running on the nodes. nodeName := item @@ -76,7 +77,7 @@ func waitTillNPodsRunningOnNodes(c *client.Client, nodeNames util.StringSet, pod }() } - seen := util.NewStringSet() + seen := sets.NewString() for i := 0; i < len(nodeNames.List()); i++ { seen = seen.Union(<-matchCh) } @@ -90,7 +91,7 @@ func waitTillNPodsRunningOnNodes(c *client.Client, nodeNames util.StringSet, pod var _ = Describe("kubelet", func() { var numNodes int - var nodeNames util.StringSet + var nodeNames sets.String framework := NewFramework("kubelet") var resourceMonitor *resourceMonitor @@ -98,7 +99,7 @@ var _ = Describe("kubelet", func() { nodes, err := framework.Client.Nodes().List(labels.Everything(), fields.Everything()) expectNoError(err) numNodes = len(nodes.Items) - nodeNames = util.NewStringSet() + nodeNames = sets.NewString() for _, node := range nodes.Items { nodeNames.Insert(node.Name) } diff --git a/test/e2e/kubelet_stats.go b/test/e2e/kubelet_stats.go index 5a543cef505..98fca776661 100644 --- a/test/e2e/kubelet_stats.go +++ b/test/e2e/kubelet_stats.go @@ -37,6 +37,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "github.com/prometheus/client_golang/extraction" "github.com/prometheus/client_golang/model" @@ -66,7 +67,7 @@ func (a KubeletMetricByLatency) Less(i, j int) bool { return a[i].Latency > a[j] type kubeletMetricIngester []KubeletMetric func (k *kubeletMetricIngester) Ingest(samples model.Samples) error { - acceptedMethods := util.NewStringSet( + acceptedMethods := sets.NewString( metrics.PodWorkerLatencyKey, metrics.PodWorkerStartLatencyKey, metrics.SyncPodsLatencyKey, diff --git a/test/e2e/load.go b/test/e2e/load.go index 41dddbc51c8..f7137376724 100644 --- a/test/e2e/load.go +++ b/test/e2e/load.go @@ -26,7 +26,7 @@ import ( client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -86,7 +86,7 @@ var _ = Describe("Load capacity", func() { } // Verify latency metrics - highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events")) + highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, sets.NewString("events")) expectNoError(err, "Too many instances metrics above the threshold") Expect(highLatencyRequests).NotTo(BeNumerically(">", 0)) }) diff --git a/test/e2e/service_latency.go b/test/e2e/service_latency.go index bcbb24096e5..91171378f5d 100644 --- a/test/e2e/service_latency.go +++ b/test/e2e/service_latency.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/watch" . "github.com/onsi/ginkgo" @@ -70,7 +71,7 @@ var _ = Describe("Service endpoints latency", func() { f.Client.RESTClient.Throttle = util.NewFakeRateLimiter() defer func() { f.Client.RESTClient.Throttle = oldThrottle }() - failing := util.NewStringSet() + failing := sets.NewString() d, err := runServiceLatencies(f, parallelTrials, totalTrials) if err != nil { failing.Insert(fmt.Sprintf("Not all RC/pod/service trials succeeded: %v", err)) diff --git a/test/e2e/util.go b/test/e2e/util.go index d148d2d97a8..63233a25e5b 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -45,6 +45,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/watch" @@ -1096,7 +1097,7 @@ type podInfo struct { type PodDiff map[string]*podInfo // Print formats and prints the give PodDiff. -func (p PodDiff) Print(ignorePhases util.StringSet) { +func (p PodDiff) Print(ignorePhases sets.String) { for name, info := range p { if ignorePhases.Has(info.phase) { continue @@ -1258,7 +1259,7 @@ func RunRC(config RCConfig) error { unknown := 0 inactive := 0 failedContainers := 0 - containerRestartNodes := util.NewStringSet() + containerRestartNodes := sets.NewString() pods := podStore.List() created := []*api.Pod{} @@ -1312,7 +1313,7 @@ func RunRC(config RCConfig) error { // - diagnose by comparing the previous "2 Pod states" lines for inactive pods errorStr := fmt.Sprintf("Number of reported pods changed: %d vs %d", len(pods), len(oldPods)) Logf("%v, pods that changed since the last iteration:", errorStr) - Diff(oldPods, pods).Print(util.NewStringSet()) + Diff(oldPods, pods).Print(sets.NewString()) return fmt.Errorf(errorStr) } @@ -1342,7 +1343,7 @@ func RunRC(config RCConfig) error { } func dumpPodDebugInfo(c *client.Client, pods []*api.Pod) { - badNodes := util.NewStringSet() + badNodes := sets.NewString() for _, p := range pods { if p.Status.Phase != api.PodRunning { if p.Spec.NodeName != "" { @@ -1851,8 +1852,8 @@ func ReadLatencyMetrics(c *client.Client) ([]LatencyMetric, error) { // Prints summary metrics for request types with latency above threshold // and returns number of such request types. -func HighLatencyRequests(c *client.Client, threshold time.Duration, ignoredResources util.StringSet) (int, error) { - ignoredVerbs := util.NewStringSet("WATCHLIST", "PROXY") +func HighLatencyRequests(c *client.Client, threshold time.Duration, ignoredResources sets.String) (int, error) { + ignoredVerbs := sets.NewString("WATCHLIST", "PROXY") metrics, err := ReadLatencyMetrics(c) if err != nil { diff --git a/test/images/network-tester/webserver.go b/test/images/network-tester/webserver.go index 7ffba00f80f..5b31a72a884 100644 --- a/test/images/network-tester/webserver.go +++ b/test/images/network-tester/webserver.go @@ -45,7 +45,7 @@ import ( "time" client "k8s.io/kubernetes/pkg/client/unversioned" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" ) var ( @@ -235,7 +235,7 @@ func contactOthers(state *State) { time.Sleep(time.Duration(1+rand.Intn(10)) * time.Second) } - eps := util.StringSet{} + eps := sets.String{} for _, ss := range endpoints.Subsets { for _, a := range ss.Addresses { for _, p := range ss.Ports { diff --git a/test/integration/service_account_test.go b/test/integration/service_account_test.go index 7218b547f8a..fe3ae4fd2db 100644 --- a/test/integration/service_account_test.go +++ b/test/integration/service_account_test.go @@ -45,7 +45,7 @@ import ( "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/tools/etcdtest" - "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" serviceaccountadmission "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount" "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union" @@ -170,7 +170,7 @@ func TestServiceAccountTokenAutoCreate(t *testing.T) { } // Wait for tokens to be deleted - tokensToCleanup := util.NewStringSet(token1Name, token2Name, token3Name) + tokensToCleanup := sets.NewString(token1Name, token2Name, token3Name) err = wait.Poll(time.Second, 10*time.Second, func() (bool, error) { // Get all secrets in the namespace secrets, err := c.Secrets(ns).List(labels.Everything(), fields.Everything())