From 06b1a9636dbc1d30b0d5f7c6a1f59194800bf3e1 Mon Sep 17 00:00:00 2001 From: deads2k Date: Wed, 28 Sep 2016 09:37:24 -0400 Subject: [PATCH] promote contrib/mesos to incubator --- build/common.sh | 4 - cluster/mesos/docker/util.sh | 1 - contrib/mesos/OWNERS | 2 - contrib/mesos/README.md | 35 - contrib/mesos/ci/build-release.sh | 38 - contrib/mesos/ci/build.sh | 36 - contrib/mesos/ci/run-with-cluster.sh | 87 -- contrib/mesos/ci/run.sh | 56 - contrib/mesos/ci/test-conformance.sh | 44 - contrib/mesos/ci/test-e2e.sh | 34 - contrib/mesos/ci/test-integration.sh | 34 - contrib/mesos/ci/test-smoke.sh | 34 - contrib/mesos/ci/test-unit.sh | 34 - .../mesos/cmd/k8sm-controller-manager/doc.go | 23 - .../mesos/cmd/k8sm-controller-manager/main.go | 52 - contrib/mesos/cmd/k8sm-executor/doc.go | 18 - contrib/mesos/cmd/k8sm-executor/main.go | 46 - contrib/mesos/cmd/k8sm-scheduler/doc.go | 18 - contrib/mesos/cmd/k8sm-scheduler/main.go | 45 - contrib/mesos/cmd/km/doc.go | 24 - contrib/mesos/cmd/km/hyperkube.go | 202 --- contrib/mesos/cmd/km/hyperkube_test.go | 144 --- .../mesos/cmd/km/k8sm-controllermanager.go | 39 - contrib/mesos/cmd/km/k8sm-executor.go | 41 - contrib/mesos/cmd/km/k8sm-minion.go | 39 - contrib/mesos/cmd/km/k8sm-scheduler.go | 40 - contrib/mesos/cmd/km/km.go | 41 - contrib/mesos/cmd/km/kube-apiserver.go | 40 - contrib/mesos/cmd/km/kube-proxy.go | 52 - contrib/mesos/cmd/km/server.go | 82 -- contrib/mesos/docs/architecture.gliffy | 1 - contrib/mesos/docs/architecture.md | 67 - contrib/mesos/docs/architecture.png | Bin 69252 -> 0 bytes contrib/mesos/docs/architecture.svg | 1 - contrib/mesos/docs/discovery.md | 91 -- contrib/mesos/docs/ha.md | 526 -------- contrib/mesos/docs/issues.md | 232 ---- contrib/mesos/docs/logos/k8s-256x256.png | Bin 17168 -> 0 bytes contrib/mesos/docs/logos/k8s-48x48.png | Bin 2562 -> 0 bytes contrib/mesos/docs/logos/k8s-96x96.png | Bin 5248 -> 0 bytes contrib/mesos/docs/networking.gliffy | 1 - contrib/mesos/docs/networking.png | Bin 46650 -> 0 bytes contrib/mesos/docs/networking.svg | 1 - contrib/mesos/docs/scheduler.md | 178 --- contrib/mesos/docs/scheduler.monopic | Bin 10504 -> 0 bytes contrib/mesos/docs/scheduler.png | Bin 222525 -> 0 bytes contrib/mesos/pkg/assert/assert.go | 43 - contrib/mesos/pkg/assert/doc.go | 19 - contrib/mesos/pkg/backoff/backoff.go | 96 -- contrib/mesos/pkg/backoff/doc.go | 19 - .../controllermanager/controllermanager.go | 371 ------ contrib/mesos/pkg/controllermanager/doc.go | 20 - contrib/mesos/pkg/election/doc.go | 18 - contrib/mesos/pkg/election/etcd_master.go | 198 --- .../mesos/pkg/election/etcd_master_test.go | 78 -- contrib/mesos/pkg/election/fake.go | 53 - contrib/mesos/pkg/election/master.go | 121 -- contrib/mesos/pkg/election/master_test.go | 106 -- contrib/mesos/pkg/executor/apis.go | 45 - contrib/mesos/pkg/executor/config/config.go | 29 - contrib/mesos/pkg/executor/config/doc.go | 18 - contrib/mesos/pkg/executor/doc.go | 21 - contrib/mesos/pkg/executor/executor.go | 755 ------------ contrib/mesos/pkg/executor/executor_test.go | 636 ---------- contrib/mesos/pkg/executor/messages/doc.go | 18 - .../mesos/pkg/executor/messages/messages.go | 36 - contrib/mesos/pkg/executor/mock_test.go | 90 -- contrib/mesos/pkg/executor/node.go | 67 - contrib/mesos/pkg/executor/registry.go | 340 ----- .../mesos/pkg/executor/service/cadvisor.go | 51 - contrib/mesos/pkg/executor/service/doc.go | 18 - contrib/mesos/pkg/executor/service/kubelet.go | 84 -- .../executor/service/podsource/podsource.go | 200 --- contrib/mesos/pkg/executor/service/service.go | 321 ----- contrib/mesos/pkg/executor/suicide.go | 65 - contrib/mesos/pkg/executor/suicide_test.go | 197 --- contrib/mesos/pkg/executor/watcher.go | 150 --- contrib/mesos/pkg/flagutil/cadvisor.go | 47 - contrib/mesos/pkg/flagutil/cadvisor_linux.go | 24 - contrib/mesos/pkg/hyperkube/doc.go | 21 - contrib/mesos/pkg/hyperkube/hyperkube.go | 26 - contrib/mesos/pkg/hyperkube/types.go | 54 - contrib/mesos/pkg/minion/config/config.go | 33 - contrib/mesos/pkg/minion/config/doc.go | 18 - contrib/mesos/pkg/minion/doc.go | 18 - contrib/mesos/pkg/minion/mountns_darwin.go | 25 - contrib/mesos/pkg/minion/mountns_linux.go | 55 - contrib/mesos/pkg/minion/server.go | 381 ------ contrib/mesos/pkg/minion/tasks/doc.go | 20 - contrib/mesos/pkg/minion/tasks/events.go | 98 -- contrib/mesos/pkg/minion/tasks/task.go | 431 ------- contrib/mesos/pkg/minion/tasks/task_linux.go | 28 - contrib/mesos/pkg/minion/tasks/task_other.go | 38 - contrib/mesos/pkg/minion/tasks/task_test.go | 311 ----- contrib/mesos/pkg/minion/tasks/timer.go | 52 - contrib/mesos/pkg/node/doc.go | 18 - contrib/mesos/pkg/node/node.go | 226 ---- contrib/mesos/pkg/node/registrator.go | 151 --- contrib/mesos/pkg/node/registrator_test.go | 160 --- contrib/mesos/pkg/node/statusupdater.go | 190 --- contrib/mesos/pkg/node/statusupdater_test.go | 77 -- contrib/mesos/pkg/offers/doc.go | 18 - contrib/mesos/pkg/offers/metrics/doc.go | 19 - contrib/mesos/pkg/offers/metrics/metrics.go | 89 -- contrib/mesos/pkg/offers/offers.go | 570 --------- contrib/mesos/pkg/offers/offers_test.go | 391 ------ contrib/mesos/pkg/podutil/doc.go | 19 - contrib/mesos/pkg/podutil/filters.go | 132 -- contrib/mesos/pkg/podutil/gzip.go | 83 -- contrib/mesos/pkg/podutil/gzip_test.go | 69 -- contrib/mesos/pkg/podutil/io.go | 124 -- contrib/mesos/pkg/proc/adapter.go | 89 -- contrib/mesos/pkg/proc/doc.go | 19 - contrib/mesos/pkg/proc/errors.go | 85 -- contrib/mesos/pkg/proc/once.go | 101 -- contrib/mesos/pkg/proc/proc.go | 196 --- contrib/mesos/pkg/proc/proc_test.go | 397 ------ contrib/mesos/pkg/proc/state.go | 55 - contrib/mesos/pkg/proc/types.go | 89 -- contrib/mesos/pkg/profile/doc.go | 18 - contrib/mesos/pkg/profile/profile.go | 27 - contrib/mesos/pkg/queue/delay.go | 409 ------ contrib/mesos/pkg/queue/delay_test.go | 409 ------ contrib/mesos/pkg/queue/doc.go | 19 - contrib/mesos/pkg/queue/historical.go | 406 ------ contrib/mesos/pkg/queue/historical_test.go | 217 ---- contrib/mesos/pkg/queue/interface.go | 106 -- contrib/mesos/pkg/queue/policy.go | 70 -- contrib/mesos/pkg/queue/priority.go | 56 - contrib/mesos/pkg/redirfd/doc.go | 19 - contrib/mesos/pkg/redirfd/file_descriptor.go | 41 - .../mesos/pkg/redirfd/file_descriptor_test.go | 54 - contrib/mesos/pkg/redirfd/redirfd_unix.go | 208 ---- contrib/mesos/pkg/redirfd/redirfd_windows.go | 39 - contrib/mesos/pkg/runtime/doc.go | 19 - contrib/mesos/pkg/runtime/latch.go | 35 - contrib/mesos/pkg/runtime/latch_test.go | 61 - contrib/mesos/pkg/runtime/metrics.go | 47 - contrib/mesos/pkg/runtime/util.go | 122 -- contrib/mesos/pkg/runtime/util_test.go | 76 -- .../components/algorithm/algorithm.go | 209 ---- .../pkg/scheduler/components/algorithm/doc.go | 18 - .../components/algorithm/podschedulers/doc.go | 19 - .../algorithm/podschedulers/fcfs.go | 101 -- .../algorithm/podschedulers/types.go | 41 - .../pkg/scheduler/components/binder/binder.go | 162 --- .../pkg/scheduler/components/binder/doc.go | 19 - .../components/controller/controller.go | 108 -- .../scheduler/components/controller/doc.go | 20 - .../scheduler/components/deleter/deleter.go | 125 -- .../components/deleter/deleter_test.go | 178 --- .../pkg/scheduler/components/deleter/doc.go | 19 - contrib/mesos/pkg/scheduler/components/doc.go | 20 - .../scheduler/components/errorhandler/doc.go | 19 - .../components/errorhandler/errorhandler.go | 97 -- .../pkg/scheduler/components/framework/doc.go | 18 - .../components/framework/driver_mock.go | 165 --- .../components/framework/framework.go | 825 ------------- .../components/framework/framework_test.go | 336 ----- .../framework/frameworkid/etcd/etcd.go | 61 - .../framework/frameworkid/frameworkid.go | 58 - .../components/framework/frameworkid/zk/zk.go | 157 --- .../components/framework/slaveregistry.go | 61 - .../framework/slaveregistry_test.go | 90 -- .../scheduler/components/podreconciler/doc.go | 19 - .../components/podreconciler/podreconciler.go | 121 -- .../scheduler/components/podstoreadapter.go | 63 - .../pkg/scheduler/components/scheduler.go | 152 --- .../components/tasksreconciler/doc.go | 18 - .../tasksreconciler/tasksreconciler.go | 235 ---- contrib/mesos/pkg/scheduler/config/config.go | 109 -- .../mesos/pkg/scheduler/config/config_test.go | 112 -- contrib/mesos/pkg/scheduler/config/doc.go | 18 - .../pkg/scheduler/constraint/constraint.go | 106 -- .../scheduler/constraint/constraint_test.go | 79 -- contrib/mesos/pkg/scheduler/constraint/doc.go | 21 - contrib/mesos/pkg/scheduler/doc.go | 73 -- contrib/mesos/pkg/scheduler/errors/doc.go | 18 - contrib/mesos/pkg/scheduler/errors/errors.go | 28 - .../mesos/pkg/scheduler/executorinfo/codec.go | 92 -- .../pkg/scheduler/executorinfo/codec_test.go | 69 -- .../mesos/pkg/scheduler/executorinfo/doc.go | 19 - .../mesos/pkg/scheduler/executorinfo/id.go | 103 -- .../pkg/scheduler/executorinfo/lru_cache.go | 95 -- .../scheduler/executorinfo/lru_cache_test.go | 55 - .../pkg/scheduler/executorinfo/registry.go | 178 --- .../scheduler/executorinfo/registry_test.go | 194 --- contrib/mesos/pkg/scheduler/ha/doc.go | 18 - contrib/mesos/pkg/scheduler/ha/election.go | 73 -- contrib/mesos/pkg/scheduler/ha/ha.go | 285 ----- .../mesos/pkg/scheduler/integration/doc.go | 18 - .../scheduler/integration/integration_test.go | 872 ------------- .../mesos/pkg/scheduler/meta/annotations.go | 41 - contrib/mesos/pkg/scheduler/meta/doc.go | 22 - contrib/mesos/pkg/scheduler/meta/store.go | 23 - contrib/mesos/pkg/scheduler/metrics/doc.go | 18 - .../mesos/pkg/scheduler/metrics/metrics.go | 102 -- contrib/mesos/pkg/scheduler/podtask/debug.go | 54 - contrib/mesos/pkg/scheduler/podtask/doc.go | 18 - .../pkg/scheduler/podtask/hostport/mapper.go | 211 ---- .../scheduler/podtask/hostport/mapper_test.go | 207 ---- contrib/mesos/pkg/scheduler/podtask/leaky.go | 29 - .../mesos/pkg/scheduler/podtask/pod_task.go | 381 ------ .../pkg/scheduler/podtask/pod_task_test.go | 421 ------- .../pkg/scheduler/podtask/procurement.go | 309 ----- .../pkg/scheduler/podtask/procurement_test.go | 223 ---- .../mesos/pkg/scheduler/podtask/registry.go | 340 ----- .../pkg/scheduler/podtask/registry_test.go | 336 ----- contrib/mesos/pkg/scheduler/podtask/roles.go | 83 -- .../mesos/pkg/scheduler/podtask/roles_test.go | 66 - contrib/mesos/pkg/scheduler/queuer/doc.go | 19 - contrib/mesos/pkg/scheduler/queuer/pod.go | 112 -- contrib/mesos/pkg/scheduler/queuer/queuer.go | 209 ---- contrib/mesos/pkg/scheduler/resources/doc.go | 18 - .../mesos/pkg/scheduler/resources/resource.go | 138 --- .../pkg/scheduler/resources/resource_test.go | 111 -- .../pkg/scheduler/resources/resources.go | 205 --- .../mesos/pkg/scheduler/resources/types.go | 67 - contrib/mesos/pkg/scheduler/scheduler.go | 37 - contrib/mesos/pkg/scheduler/scheduler_mock.go | 74 -- .../pkg/scheduler/service/compat_testing.go | 32 - .../pkg/scheduler/service/compat_unix.go | 38 - .../pkg/scheduler/service/compat_windows.go | 51 - contrib/mesos/pkg/scheduler/service/doc.go | 18 - .../mesos/pkg/scheduler/service/publish.go | 124 -- .../mesos/pkg/scheduler/service/service.go | 1095 ----------------- .../pkg/scheduler/service/service_test.go | 118 -- .../mesos/pkg/scheduler/service/validation.go | 49 - .../pkg/scheduler/service/validation_test.go | 161 --- contrib/mesos/pkg/service/doc.go | 20 - .../mesos/pkg/service/endpoints_controller.go | 472 ------- .../pkg/service/endpoints_controller_test.go | 63 - contrib/mesos/target.sh | 42 - hack/lib/golang.sh | 17 - hack/make-rules/test.sh | 1 - 235 files changed, 26615 deletions(-) delete mode 100644 contrib/mesos/OWNERS delete mode 100644 contrib/mesos/README.md delete mode 100755 contrib/mesos/ci/build-release.sh delete mode 100755 contrib/mesos/ci/build.sh delete mode 100755 contrib/mesos/ci/run-with-cluster.sh delete mode 100755 contrib/mesos/ci/run.sh delete mode 100755 contrib/mesos/ci/test-conformance.sh delete mode 100755 contrib/mesos/ci/test-e2e.sh delete mode 100755 contrib/mesos/ci/test-integration.sh delete mode 100755 contrib/mesos/ci/test-smoke.sh delete mode 100755 contrib/mesos/ci/test-unit.sh delete mode 100644 contrib/mesos/cmd/k8sm-controller-manager/doc.go delete mode 100644 contrib/mesos/cmd/k8sm-controller-manager/main.go delete mode 100644 contrib/mesos/cmd/k8sm-executor/doc.go delete mode 100644 contrib/mesos/cmd/k8sm-executor/main.go delete mode 100644 contrib/mesos/cmd/k8sm-scheduler/doc.go delete mode 100644 contrib/mesos/cmd/k8sm-scheduler/main.go delete mode 100644 contrib/mesos/cmd/km/doc.go delete mode 100644 contrib/mesos/cmd/km/hyperkube.go delete mode 100644 contrib/mesos/cmd/km/hyperkube_test.go delete mode 100644 contrib/mesos/cmd/km/k8sm-controllermanager.go delete mode 100644 contrib/mesos/cmd/km/k8sm-executor.go delete mode 100644 contrib/mesos/cmd/km/k8sm-minion.go delete mode 100644 contrib/mesos/cmd/km/k8sm-scheduler.go delete mode 100644 contrib/mesos/cmd/km/km.go delete mode 100644 contrib/mesos/cmd/km/kube-apiserver.go delete mode 100644 contrib/mesos/cmd/km/kube-proxy.go delete mode 100644 contrib/mesos/cmd/km/server.go delete mode 100644 contrib/mesos/docs/architecture.gliffy delete mode 100644 contrib/mesos/docs/architecture.md delete mode 100644 contrib/mesos/docs/architecture.png delete mode 100644 contrib/mesos/docs/architecture.svg delete mode 100644 contrib/mesos/docs/discovery.md delete mode 100644 contrib/mesos/docs/ha.md delete mode 100644 contrib/mesos/docs/issues.md delete mode 100644 contrib/mesos/docs/logos/k8s-256x256.png delete mode 100644 contrib/mesos/docs/logos/k8s-48x48.png delete mode 100644 contrib/mesos/docs/logos/k8s-96x96.png delete mode 100644 contrib/mesos/docs/networking.gliffy delete mode 100644 contrib/mesos/docs/networking.png delete mode 100644 contrib/mesos/docs/networking.svg delete mode 100644 contrib/mesos/docs/scheduler.md delete mode 100644 contrib/mesos/docs/scheduler.monopic delete mode 100644 contrib/mesos/docs/scheduler.png delete mode 100644 contrib/mesos/pkg/assert/assert.go delete mode 100644 contrib/mesos/pkg/assert/doc.go delete mode 100644 contrib/mesos/pkg/backoff/backoff.go delete mode 100644 contrib/mesos/pkg/backoff/doc.go delete mode 100644 contrib/mesos/pkg/controllermanager/controllermanager.go delete mode 100644 contrib/mesos/pkg/controllermanager/doc.go delete mode 100644 contrib/mesos/pkg/election/doc.go delete mode 100644 contrib/mesos/pkg/election/etcd_master.go delete mode 100644 contrib/mesos/pkg/election/etcd_master_test.go delete mode 100644 contrib/mesos/pkg/election/fake.go delete mode 100644 contrib/mesos/pkg/election/master.go delete mode 100644 contrib/mesos/pkg/election/master_test.go delete mode 100644 contrib/mesos/pkg/executor/apis.go delete mode 100644 contrib/mesos/pkg/executor/config/config.go delete mode 100644 contrib/mesos/pkg/executor/config/doc.go delete mode 100644 contrib/mesos/pkg/executor/doc.go delete mode 100644 contrib/mesos/pkg/executor/executor.go delete mode 100644 contrib/mesos/pkg/executor/executor_test.go delete mode 100644 contrib/mesos/pkg/executor/messages/doc.go delete mode 100644 contrib/mesos/pkg/executor/messages/messages.go delete mode 100644 contrib/mesos/pkg/executor/mock_test.go delete mode 100644 contrib/mesos/pkg/executor/node.go delete mode 100644 contrib/mesos/pkg/executor/registry.go delete mode 100644 contrib/mesos/pkg/executor/service/cadvisor.go delete mode 100644 contrib/mesos/pkg/executor/service/doc.go delete mode 100644 contrib/mesos/pkg/executor/service/kubelet.go delete mode 100644 contrib/mesos/pkg/executor/service/podsource/podsource.go delete mode 100644 contrib/mesos/pkg/executor/service/service.go delete mode 100644 contrib/mesos/pkg/executor/suicide.go delete mode 100644 contrib/mesos/pkg/executor/suicide_test.go delete mode 100644 contrib/mesos/pkg/executor/watcher.go delete mode 100644 contrib/mesos/pkg/flagutil/cadvisor.go delete mode 100644 contrib/mesos/pkg/flagutil/cadvisor_linux.go delete mode 100644 contrib/mesos/pkg/hyperkube/doc.go delete mode 100644 contrib/mesos/pkg/hyperkube/hyperkube.go delete mode 100644 contrib/mesos/pkg/hyperkube/types.go delete mode 100644 contrib/mesos/pkg/minion/config/config.go delete mode 100644 contrib/mesos/pkg/minion/config/doc.go delete mode 100644 contrib/mesos/pkg/minion/doc.go delete mode 100644 contrib/mesos/pkg/minion/mountns_darwin.go delete mode 100644 contrib/mesos/pkg/minion/mountns_linux.go delete mode 100644 contrib/mesos/pkg/minion/server.go delete mode 100644 contrib/mesos/pkg/minion/tasks/doc.go delete mode 100644 contrib/mesos/pkg/minion/tasks/events.go delete mode 100644 contrib/mesos/pkg/minion/tasks/task.go delete mode 100644 contrib/mesos/pkg/minion/tasks/task_linux.go delete mode 100644 contrib/mesos/pkg/minion/tasks/task_other.go delete mode 100644 contrib/mesos/pkg/minion/tasks/task_test.go delete mode 100644 contrib/mesos/pkg/minion/tasks/timer.go delete mode 100644 contrib/mesos/pkg/node/doc.go delete mode 100644 contrib/mesos/pkg/node/node.go delete mode 100644 contrib/mesos/pkg/node/registrator.go delete mode 100644 contrib/mesos/pkg/node/registrator_test.go delete mode 100644 contrib/mesos/pkg/node/statusupdater.go delete mode 100644 contrib/mesos/pkg/node/statusupdater_test.go delete mode 100644 contrib/mesos/pkg/offers/doc.go delete mode 100644 contrib/mesos/pkg/offers/metrics/doc.go delete mode 100644 contrib/mesos/pkg/offers/metrics/metrics.go delete mode 100644 contrib/mesos/pkg/offers/offers.go delete mode 100644 contrib/mesos/pkg/offers/offers_test.go delete mode 100644 contrib/mesos/pkg/podutil/doc.go delete mode 100644 contrib/mesos/pkg/podutil/filters.go delete mode 100644 contrib/mesos/pkg/podutil/gzip.go delete mode 100644 contrib/mesos/pkg/podutil/gzip_test.go delete mode 100644 contrib/mesos/pkg/podutil/io.go delete mode 100644 contrib/mesos/pkg/proc/adapter.go delete mode 100644 contrib/mesos/pkg/proc/doc.go delete mode 100644 contrib/mesos/pkg/proc/errors.go delete mode 100644 contrib/mesos/pkg/proc/once.go delete mode 100644 contrib/mesos/pkg/proc/proc.go delete mode 100644 contrib/mesos/pkg/proc/proc_test.go delete mode 100644 contrib/mesos/pkg/proc/state.go delete mode 100644 contrib/mesos/pkg/proc/types.go delete mode 100644 contrib/mesos/pkg/profile/doc.go delete mode 100644 contrib/mesos/pkg/profile/profile.go delete mode 100644 contrib/mesos/pkg/queue/delay.go delete mode 100644 contrib/mesos/pkg/queue/delay_test.go delete mode 100644 contrib/mesos/pkg/queue/doc.go delete mode 100644 contrib/mesos/pkg/queue/historical.go delete mode 100644 contrib/mesos/pkg/queue/historical_test.go delete mode 100644 contrib/mesos/pkg/queue/interface.go delete mode 100644 contrib/mesos/pkg/queue/policy.go delete mode 100644 contrib/mesos/pkg/queue/priority.go delete mode 100644 contrib/mesos/pkg/redirfd/doc.go delete mode 100644 contrib/mesos/pkg/redirfd/file_descriptor.go delete mode 100644 contrib/mesos/pkg/redirfd/file_descriptor_test.go delete mode 100644 contrib/mesos/pkg/redirfd/redirfd_unix.go delete mode 100644 contrib/mesos/pkg/redirfd/redirfd_windows.go delete mode 100644 contrib/mesos/pkg/runtime/doc.go delete mode 100644 contrib/mesos/pkg/runtime/latch.go delete mode 100644 contrib/mesos/pkg/runtime/latch_test.go delete mode 100644 contrib/mesos/pkg/runtime/metrics.go delete mode 100644 contrib/mesos/pkg/runtime/util.go delete mode 100644 contrib/mesos/pkg/runtime/util_test.go delete mode 100644 contrib/mesos/pkg/scheduler/components/algorithm/algorithm.go delete mode 100644 contrib/mesos/pkg/scheduler/components/algorithm/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/fcfs.go delete mode 100644 contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/types.go delete mode 100644 contrib/mesos/pkg/scheduler/components/binder/binder.go delete mode 100644 contrib/mesos/pkg/scheduler/components/binder/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/controller/controller.go delete mode 100644 contrib/mesos/pkg/scheduler/components/controller/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/deleter/deleter.go delete mode 100644 contrib/mesos/pkg/scheduler/components/deleter/deleter_test.go delete mode 100644 contrib/mesos/pkg/scheduler/components/deleter/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/errorhandler/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/errorhandler/errorhandler.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/driver_mock.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/framework.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/framework_test.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/frameworkid/etcd/etcd.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/frameworkid/frameworkid.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/frameworkid/zk/zk.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/slaveregistry.go delete mode 100644 contrib/mesos/pkg/scheduler/components/framework/slaveregistry_test.go delete mode 100644 contrib/mesos/pkg/scheduler/components/podreconciler/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/podreconciler/podreconciler.go delete mode 100644 contrib/mesos/pkg/scheduler/components/podstoreadapter.go delete mode 100644 contrib/mesos/pkg/scheduler/components/scheduler.go delete mode 100644 contrib/mesos/pkg/scheduler/components/tasksreconciler/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/components/tasksreconciler/tasksreconciler.go delete mode 100644 contrib/mesos/pkg/scheduler/config/config.go delete mode 100644 contrib/mesos/pkg/scheduler/config/config_test.go delete mode 100644 contrib/mesos/pkg/scheduler/config/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/constraint/constraint.go delete mode 100644 contrib/mesos/pkg/scheduler/constraint/constraint_test.go delete mode 100644 contrib/mesos/pkg/scheduler/constraint/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/errors/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/errors/errors.go delete mode 100644 contrib/mesos/pkg/scheduler/executorinfo/codec.go delete mode 100644 contrib/mesos/pkg/scheduler/executorinfo/codec_test.go delete mode 100644 contrib/mesos/pkg/scheduler/executorinfo/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/executorinfo/id.go delete mode 100644 contrib/mesos/pkg/scheduler/executorinfo/lru_cache.go delete mode 100644 contrib/mesos/pkg/scheduler/executorinfo/lru_cache_test.go delete mode 100644 contrib/mesos/pkg/scheduler/executorinfo/registry.go delete mode 100644 contrib/mesos/pkg/scheduler/executorinfo/registry_test.go delete mode 100644 contrib/mesos/pkg/scheduler/ha/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/ha/election.go delete mode 100644 contrib/mesos/pkg/scheduler/ha/ha.go delete mode 100644 contrib/mesos/pkg/scheduler/integration/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/integration/integration_test.go delete mode 100644 contrib/mesos/pkg/scheduler/meta/annotations.go delete mode 100644 contrib/mesos/pkg/scheduler/meta/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/meta/store.go delete mode 100644 contrib/mesos/pkg/scheduler/metrics/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/metrics/metrics.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/debug.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/hostport/mapper.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/hostport/mapper_test.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/leaky.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/pod_task.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/pod_task_test.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/procurement.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/procurement_test.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/registry.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/registry_test.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/roles.go delete mode 100644 contrib/mesos/pkg/scheduler/podtask/roles_test.go delete mode 100644 contrib/mesos/pkg/scheduler/queuer/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/queuer/pod.go delete mode 100644 contrib/mesos/pkg/scheduler/queuer/queuer.go delete mode 100644 contrib/mesos/pkg/scheduler/resources/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/resources/resource.go delete mode 100644 contrib/mesos/pkg/scheduler/resources/resource_test.go delete mode 100644 contrib/mesos/pkg/scheduler/resources/resources.go delete mode 100644 contrib/mesos/pkg/scheduler/resources/types.go delete mode 100644 contrib/mesos/pkg/scheduler/scheduler.go delete mode 100644 contrib/mesos/pkg/scheduler/scheduler_mock.go delete mode 100644 contrib/mesos/pkg/scheduler/service/compat_testing.go delete mode 100644 contrib/mesos/pkg/scheduler/service/compat_unix.go delete mode 100644 contrib/mesos/pkg/scheduler/service/compat_windows.go delete mode 100644 contrib/mesos/pkg/scheduler/service/doc.go delete mode 100644 contrib/mesos/pkg/scheduler/service/publish.go delete mode 100644 contrib/mesos/pkg/scheduler/service/service.go delete mode 100644 contrib/mesos/pkg/scheduler/service/service_test.go delete mode 100644 contrib/mesos/pkg/scheduler/service/validation.go delete mode 100644 contrib/mesos/pkg/scheduler/service/validation_test.go delete mode 100644 contrib/mesos/pkg/service/doc.go delete mode 100644 contrib/mesos/pkg/service/endpoints_controller.go delete mode 100644 contrib/mesos/pkg/service/endpoints_controller_test.go delete mode 100644 contrib/mesos/target.sh diff --git a/build/common.sh b/build/common.sh index 7081ef2235e..633cb4c4ba4 100755 --- a/build/common.sh +++ b/build/common.sh @@ -566,10 +566,6 @@ function kube::build::run_build_command() { "${DOCKER_MOUNT_ARGS[@]}" ) - if [ -n "${KUBERNETES_CONTRIB:-}" ]; then - docker_run_opts+=(-e "KUBERNETES_CONTRIB=${KUBERNETES_CONTRIB}") - fi - docker_run_opts+=( --env "KUBE_FASTBUILD=${KUBE_FASTBUILD:-false}" --env "KUBE_BUILDER_OS=${OSTYPE:-notdetected}" diff --git a/cluster/mesos/docker/util.sh b/cluster/mesos/docker/util.sh index c470492f68b..709b581a005 100644 --- a/cluster/mesos/docker/util.sh +++ b/cluster/mesos/docker/util.sh @@ -168,7 +168,6 @@ function prepare-e2e { # Execute prior to running tests to build a release if required for env function test-build-release { # Make a release - export KUBERNETES_CONTRIB=mesos export KUBE_RELEASE_RUN_TESTS=N "${KUBE_ROOT}/build/release.sh" } diff --git a/contrib/mesos/OWNERS b/contrib/mesos/OWNERS deleted file mode 100644 index ce4549bec81..00000000000 --- a/contrib/mesos/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -assignees: - - k82cn diff --git a/contrib/mesos/README.md b/contrib/mesos/README.md deleted file mode 100644 index 5474febb71b..00000000000 --- a/contrib/mesos/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Kubernetes-Mesos - -Kubernetes-Mesos modifies Kubernetes to act as an [Apache Mesos](http://mesos.apache.org/) framework. - -## Features On Mesos - -Kubernetes gains the following benefits when installed on Mesos: - -- **Node-Level Auto-Scaling** - Kubernetes minion nodes are created automatically, up to the size of the provisioned Mesos cluster. -- **Resource Sharing** - Co-location of Kubernetes with other popular next-generation services on the same cluster (e.g. [Hadoop](https://github.com/mesos/hadoop), [Spark](http://spark.apache.org/), and [Chronos](https://mesos.github.io/chronos/), [Cassandra](http://mesosphere.github.io/cassandra-mesos/), etc.). Resources are allocated to the frameworks based on fairness and can be claimed or passed on depending on framework load. -- **Independence from special Network Infrastructure** - Mesos can (but of course doesn't have to) run on networks which cannot assign a routable IP to every container. The Kubernetes on Mesos endpoint controller is specially modified to allow pods to communicate with services in such an environment. - -For more information about how Kubernetes-Mesos is different from Kubernetes, see [Architecture](./docs/architecture.md). - - -## Release Status - -Kubernetes-Mesos is alpha quality, still under active development, and not yet recommended for production systems. - -For more information about development progress, see the [known issues](./docs/issues.md) or the [kubernetes-mesos repository](https://github.com/mesosphere/kubernetes-mesos) where backlog issues are tracked. - -## Usage - -This project combines concepts and technologies from two already-complex projects: Mesos and Kubernetes. It may help to familiarize yourself with the basics of each project before reading on: - -* [Mesos Documentation](http://mesos.apache.org/documentation/latest) -* [Kubernetes Documentation](../../README.md) - -To get up and running with Kubernetes-Mesos, follow: - -- the [Getting started guide](../../docs/getting-started-guides/mesos.md) to launch a Kubernetes-Mesos cluster, -- the [Kubernetes-Mesos Scheduler Guide](./docs/scheduler.md) for topics concerning the custom scheduler used in this distribution. - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/mesos/README.md?pixel)]() diff --git a/contrib/mesos/ci/build-release.sh b/contrib/mesos/ci/build-release.sh deleted file mode 100755 index e12cb7e0563..00000000000 --- a/contrib/mesos/ci/build-release.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Cleans output files/images and builds a full release from scratch -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/build-release.sh - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -"${KUBE_ROOT}/contrib/mesos/ci/run.sh" make clean - -export KUBERNETES_CONTRIB=mesos -export KUBE_RELEASE_RUN_TESTS="${KUBE_RELEASE_RUN_TESTS:-N}" -export KUBE_SKIP_CONFIRMATIONS=Y - -"${KUBE_ROOT}/build/release.sh" diff --git a/contrib/mesos/ci/build.sh b/contrib/mesos/ci/build.sh deleted file mode 100755 index 399ba03acfc..00000000000 --- a/contrib/mesos/ci/build.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Cleans output files/images and builds linux binaries from scratch -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/build.sh - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -TEST_ARGS="$@" - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -export KUBERNETES_CONTRIB=mesos - -"${KUBE_ROOT}/contrib/mesos/ci/run.sh" make clean all ${TEST_ARGS} diff --git a/contrib/mesos/ci/run-with-cluster.sh b/contrib/mesos/ci/run-with-cluster.sh deleted file mode 100755 index ebeb1cff824..00000000000 --- a/contrib/mesos/ci/run-with-cluster.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Deploys a test cluster, runs the specified command, and destroys the test cluster. -# Runs all commands inside the mesosphere/kubernetes-mesos-test docker image (built on demand). -# Uses the mesos/docker cluster provider. -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/run-with-cluster.sh ./cluster/test-smoke.sh -v=2 - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -RUN_CMD="$@" -[ -z "${RUN_CMD:-}" ] && echo "No command supplied" && exit 1 - -KUBERNETES_PROVIDER="mesos/docker" - -MESOS_DOCKER_WORK_DIR="${MESOS_DOCKER_WORK_DIR:-${HOME}/tmp/kubernetes}" - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -# Clean (test artifacts) -echo "Cleaning work dir" -echo "${MESOS_DOCKER_WORK_DIR}" -rm -rf "${MESOS_DOCKER_WORK_DIR}" -mkdir -p "${MESOS_DOCKER_WORK_DIR}" - -echo "Detecting docker client" -# Mount docker client binary to avoid client/compose/daemon version conflicts -if [ -n "${DOCKER_MACHINE_NAME:-}" ] && which docker-machine; then - # On a Mac with docker-machine, use the binary in the VM, not the host binary - DOCKER_BIN_PATH="$(docker-machine ssh "${DOCKER_MACHINE_NAME}" which docker)" -else - DOCKER_BIN_PATH="$(which docker)" -fi -echo "${DOCKER_BIN_PATH}" - -# Clean (k8s output & images), Build, Kube-Up, Test, Kube-Down -cd "${KUBE_ROOT}" -docker run \ - --rm \ - -v "${KUBE_ROOT}:/go/src/github.com/GoogleCloudPlatform/kubernetes" \ - -v "/var/run/docker.sock:/var/run/docker.sock" \ - -v "${DOCKER_BIN_PATH}:/usr/bin/docker" \ - -v "${MESOS_DOCKER_WORK_DIR}/auth:${MESOS_DOCKER_WORK_DIR}/auth" \ - -v "${MESOS_DOCKER_WORK_DIR}/log:${MESOS_DOCKER_WORK_DIR}/log" \ - -v "${MESOS_DOCKER_WORK_DIR}/mesosslave1/mesos:${MESOS_DOCKER_WORK_DIR}/mesosslave1/mesos" \ - -v "${MESOS_DOCKER_WORK_DIR}/mesosslave2/mesos:${MESOS_DOCKER_WORK_DIR}/mesosslave2/mesos" \ - -v "${MESOS_DOCKER_WORK_DIR}/overlay:${MESOS_DOCKER_WORK_DIR}/overlay" \ - -v "${MESOS_DOCKER_WORK_DIR}/reports:${MESOS_DOCKER_WORK_DIR}/reports" \ - $(test -d /teamcity/system/git && echo "-v /teamcity/system/git:/teamcity/system/git" || true) \ - -e "MESOS_DOCKER_WORK_DIR=${MESOS_DOCKER_WORK_DIR}" \ - -e "MESOS_DOCKER_IMAGE_DIR=/var/tmp/kubernetes" \ - -e "MESOS_DOCKER_OVERLAY_DIR=${MESOS_DOCKER_WORK_DIR}/overlay" \ - -e "KUBERNETES_CONTRIB=mesos" \ - -e "KUBERNETES_PROVIDER=mesos/docker" \ - -e "USER=root" \ - -e "E2E_REPORT_DIR=${MESOS_DOCKER_WORK_DIR}/reports" \ - -t $(tty &>/dev/null && echo "-i") \ - mesosphere/kubernetes-mesos-test \ - -ceux "\ - make clean all && \ - trap 'timeout 5m ./cluster/kube-down.sh' EXIT && \ - ./cluster/kube-down.sh && \ - ./cluster/kube-up.sh && \ - trap \"test \\\$? != 0 && export MESOS_DOCKER_DUMP_LOGS=true; cd \${PWD} && timeout 5m ./cluster/kube-down.sh\" EXIT && \ - ${RUN_CMD} - " diff --git a/contrib/mesos/ci/run.sh b/contrib/mesos/ci/run.sh deleted file mode 100755 index c92ee3b67df..00000000000 --- a/contrib/mesos/ci/run.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Runs the specified command in the test container (mesosphere/kubernetes-mesos-test). -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/run.sh make test - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -RUN_CMD="$@" -[ -z "${RUN_CMD:-}" ] && echo "No command supplied" && exit 1 - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -echo "Detecting docker client" -# Mount docker client binary to avoid client/compose/daemon version conflicts -if [ -n "${DOCKER_MACHINE_NAME:-}" ] && which docker-machine; then - # On a Mac with docker-machine, use the binary in the VM, not the host binary - DOCKER_BIN_PATH="$(docker-machine ssh "${DOCKER_MACHINE_NAME}" which docker)" -else - DOCKER_BIN_PATH="$(which docker)" -fi -echo "${DOCKER_BIN_PATH}" - -# Clean (k8s output & images) & Build -cd "${KUBE_ROOT}" -exec docker run \ - --rm \ - -v "${KUBE_ROOT}:/go/src/github.com/GoogleCloudPlatform/kubernetes" \ - -v "/var/run/docker.sock:/var/run/docker.sock" \ - -v "${DOCKER_BIN_PATH}:/usr/bin/docker" \ - -e "KUBERNETES_CONTRIB=mesos" \ - -e "USER=root" \ - -t $(tty &>/dev/null && echo "-i") \ - mesosphere/kubernetes-mesos-test \ - -ceux "${RUN_CMD}" diff --git a/contrib/mesos/ci/test-conformance.sh b/contrib/mesos/ci/test-conformance.sh deleted file mode 100755 index cef300375f8..00000000000 --- a/contrib/mesos/ci/test-conformance.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Deploys a test cluster, runs the conformance tests, and destroys the test cluster. -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/test-conformance.sh -v=2 - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -TEST_ARGS="$@" - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -TEST_CMD="KUBERNETES_CONFORMANCE_TEST=y KUBECONFIG=~/.kube/config go run hack/e2e.go --test --test_args=\"--ginkgo.focus=\\[Conformance\\]\"" -if [ -n "${CONFORMANCE_BRANCH}" ]; then - # create a CONFORMANCE_BRANCH clone in a subdirectory - TEST_CMD=" -git fetch https://github.com/kubernetes/kubernetes --tags -q ${CONFORMANCE_BRANCH} && -git branch -f ${CONFORMANCE_BRANCH} FETCH_HEAD && -git clone -s -b ${CONFORMANCE_BRANCH} . conformance && -cd conformance && make all && ${TEST_CMD}" -fi - -"${KUBE_ROOT}/contrib/mesos/ci/run-with-cluster.sh" ${TEST_CMD} ${TEST_ARGS} diff --git a/contrib/mesos/ci/test-e2e.sh b/contrib/mesos/ci/test-e2e.sh deleted file mode 100755 index 6345b379a6b..00000000000 --- a/contrib/mesos/ci/test-e2e.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Deploys a test cluster, runs the e2e tests, and destroys the test cluster. -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/test-e2e.sh -v=2 - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -TEST_ARGS="$@" - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -"${KUBE_ROOT}/contrib/mesos/ci/run-with-cluster.sh" ./cluster/test-e2e.sh ${TEST_ARGS} \ No newline at end of file diff --git a/contrib/mesos/ci/test-integration.sh b/contrib/mesos/ci/test-integration.sh deleted file mode 100755 index d44865bf405..00000000000 --- a/contrib/mesos/ci/test-integration.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Cleans & runs the integration tests in the test container (mesosphere/kubernetes-mesos-test). -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/test-integration.sh - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -TEST_ARGS="$@" - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -"${KUBE_ROOT}/contrib/mesos/ci/run.sh" make clean test-integration ${TEST_ARGS} diff --git a/contrib/mesos/ci/test-smoke.sh b/contrib/mesos/ci/test-smoke.sh deleted file mode 100755 index 6c18b36de96..00000000000 --- a/contrib/mesos/ci/test-smoke.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Deploys a test cluster, runs the smoke tests, and destroys the test cluster. -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/test-smoke.sh -v=2 - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -TEST_ARGS="$@" - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -"${KUBE_ROOT}/contrib/mesos/ci/run-with-cluster.sh" ./cluster/test-smoke.sh ${TEST_ARGS} \ No newline at end of file diff --git a/contrib/mesos/ci/test-unit.sh b/contrib/mesos/ci/test-unit.sh deleted file mode 100755 index 52670850ef3..00000000000 --- a/contrib/mesos/ci/test-unit.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Cleans & runs the unit tests in the test container (mesosphere/kubernetes-mesos-test). -# -# Prerequisite: -# ./cluster/mesos/docker/test/build.sh -# -# Example Usage: -# ./contrib/mesos/ci/test-unit.sh - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -TEST_ARGS="$@" - -KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) - -"${KUBE_ROOT}/contrib/mesos/ci/run.sh" make clean test ${TEST_ARGS} diff --git a/contrib/mesos/cmd/k8sm-controller-manager/doc.go b/contrib/mesos/cmd/k8sm-controller-manager/doc.go deleted file mode 100644 index 9a63f702c14..00000000000 --- a/contrib/mesos/cmd/k8sm-controller-manager/doc.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This package main implements the executable Kubernetes Mesos controller manager. -// -// It is mainly a clone of the upstream cmd/hyperkube module right now because -// the upstream hyperkube module is not reusable. -// -// TODO(jdef,sttts): refactor upstream cmd/kube-controller-manager to be reusable with the necessary mesos changes -package main diff --git a/contrib/mesos/cmd/k8sm-controller-manager/main.go b/contrib/mesos/cmd/k8sm-controller-manager/main.go deleted file mode 100644 index a04f57d4af1..00000000000 --- a/contrib/mesos/cmd/k8sm-controller-manager/main.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - - "k8s.io/kubernetes/pkg/healthz" - "k8s.io/kubernetes/pkg/util/flag" - "k8s.io/kubernetes/pkg/util/logs" - "k8s.io/kubernetes/pkg/version/verflag" - - "k8s.io/kubernetes/contrib/mesos/pkg/controllermanager" - - "github.com/spf13/pflag" -) - -func init() { - healthz.DefaultHealthz() -} - -func main() { - - s := controllermanager.NewCMServer() - s.AddFlags(pflag.CommandLine) - - flag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - verflag.PrintAndExitIfRequested() - - if err := s.Run(pflag.CommandLine.Args()); err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(1) - } -} diff --git a/contrib/mesos/cmd/k8sm-executor/doc.go b/contrib/mesos/cmd/k8sm-executor/doc.go deleted file mode 100644 index 02f3f517dcc..00000000000 --- a/contrib/mesos/cmd/k8sm-executor/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This package main implements the executable Kubernetes Mesos executor. -package main diff --git a/contrib/mesos/cmd/k8sm-executor/main.go b/contrib/mesos/cmd/k8sm-executor/main.go deleted file mode 100644 index ed2b78c8c24..00000000000 --- a/contrib/mesos/cmd/k8sm-executor/main.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - - "github.com/spf13/pflag" - "k8s.io/kubernetes/contrib/mesos/pkg/executor/service" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" - "k8s.io/kubernetes/pkg/util/flag" - "k8s.io/kubernetes/pkg/util/logs" - "k8s.io/kubernetes/pkg/version/verflag" -) - -func main() { - - s := service.NewKubeletExecutorServer() - s.AddFlags(pflag.CommandLine) - - flag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - verflag.PrintAndExitIfRequested() - - if err := s.Run(hyperkube.Nil(), pflag.CommandLine.Args()); err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(1) - } -} diff --git a/contrib/mesos/cmd/k8sm-scheduler/doc.go b/contrib/mesos/cmd/k8sm-scheduler/doc.go deleted file mode 100644 index 6e5aa514a7b..00000000000 --- a/contrib/mesos/cmd/k8sm-scheduler/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This package main implements the executable Kubernetes Mesos scheduler. -package main diff --git a/contrib/mesos/cmd/k8sm-scheduler/main.go b/contrib/mesos/cmd/k8sm-scheduler/main.go deleted file mode 100644 index 28802288505..00000000000 --- a/contrib/mesos/cmd/k8sm-scheduler/main.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - - "github.com/spf13/pflag" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/service" - "k8s.io/kubernetes/pkg/util/flag" - "k8s.io/kubernetes/pkg/util/logs" - "k8s.io/kubernetes/pkg/version/verflag" -) - -func main() { - s := service.NewSchedulerServer() - s.AddStandaloneFlags(pflag.CommandLine) - - flag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - verflag.PrintAndExitIfRequested() - - if err := s.Run(hyperkube.Nil(), pflag.CommandLine.Args()); err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(1) - } -} diff --git a/contrib/mesos/cmd/km/doc.go b/contrib/mesos/cmd/km/doc.go deleted file mode 100644 index ed0803a9128..00000000000 --- a/contrib/mesos/cmd/km/doc.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This package main morphs all binaries under cmd/ and several other stock -// Kubernetes binaries into a single executable. -// -// It is mainly a clone of the upstream cmd/hyperkube module right now because -// the upstream hyperkube module is not reusable. -// -// TODO(jdef,sttts): refactor upstream cmd/hyperkube to be reusable with the necessary mesos changes -package main // import "k8s.io/kubernetes/contrib/mesos/cmd/km" diff --git a/contrib/mesos/cmd/km/hyperkube.go b/contrib/mesos/cmd/km/hyperkube.go deleted file mode 100644 index e2b311b17b1..00000000000 --- a/contrib/mesos/cmd/km/hyperkube.go +++ /dev/null @@ -1,202 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// clone of the upstream cmd/hypercube/hyperkube.go -package main - -import ( - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "os" - "path" - - "k8s.io/kubernetes/pkg/util" - "k8s.io/kubernetes/pkg/util/logs" - "k8s.io/kubernetes/pkg/version/verflag" - - "github.com/spf13/pflag" -) - -// HyperKube represents a single binary that can morph/manage into multiple -// servers. -type HyperKube struct { - Name string // The executable name, used for help and soft-link invocation - Long string // A long description of the binary. It will be world wrapped before output. - - servers []Server - baseFlags *pflag.FlagSet - out io.Writer - helpFlagVal bool -} - -// AddServer adds a server to the HyperKube object. -func (hk *HyperKube) AddServer(s *Server) { - hk.servers = append(hk.servers, *s) - hk.servers[len(hk.servers)-1].hk = hk -} - -// FindServer will find a specific server named name. -func (hk *HyperKube) FindServer(name string) (*Server, error) { - for _, s := range hk.servers { - if s.Name() == name { - return &s, nil - } - } - return nil, fmt.Errorf("Server not found: %s", name) -} - -// Servers returns a list of all of the registred servers -func (hk *HyperKube) Servers() []Server { - return hk.servers -} - -// Flags returns a flagset for "global" flags. -func (hk *HyperKube) Flags() *pflag.FlagSet { - if hk.baseFlags == nil { - hk.baseFlags = pflag.NewFlagSet(hk.Name, pflag.ContinueOnError) - hk.baseFlags.SetOutput(ioutil.Discard) - hk.baseFlags.BoolVarP(&hk.helpFlagVal, "help", "h", false, "help for "+hk.Name) - - // These will add all of the "global" flags (defined with both the - // flag and pflag packages) to the new flag set we have. - hk.baseFlags.AddGoFlagSet(flag.CommandLine) - hk.baseFlags.AddFlagSet(pflag.CommandLine) - - } - return hk.baseFlags -} - -// Out returns the io.Writer that is used for all usage/error information -func (hk *HyperKube) Out() io.Writer { - if hk.out == nil { - hk.out = os.Stderr - } - return hk.out -} - -// SetOut sets the output writer for all usage/error information -func (hk *HyperKube) SetOut(w io.Writer) { - hk.out = w -} - -// Print is a convenience method to Print to the defined output -func (hk *HyperKube) Print(i ...interface{}) { - fmt.Fprint(hk.Out(), i...) -} - -// Println is a convenience method to Println to the defined output -func (hk *HyperKube) Println(i ...interface{}) { - fmt.Fprintln(hk.Out(), i...) -} - -// Printf is a convenience method to Printf to the defined output -func (hk *HyperKube) Printf(format string, i ...interface{}) { - fmt.Fprintf(hk.Out(), format, i...) -} - -// Run the server. This will pick the appropriate server and run it. -func (hk *HyperKube) Run(args []string) error { - // If we are called directly, parse all flags up to the first real - // argument. That should be the server to run. - baseCommand := path.Base(args[0]) - serverName := baseCommand - if serverName == hk.Name { - args = args[1:] - - baseFlags := hk.Flags() - baseFlags.SetInterspersed(false) // Only parse flags up to the next real command - err := baseFlags.Parse(args) - if err != nil || hk.helpFlagVal { - if err != nil { - hk.Println("Error:", err) - } - hk.Usage() - return err - } - - verflag.PrintAndExitIfRequested() - - args = baseFlags.Args() - if len(args) > 0 && len(args[0]) > 0 { - serverName = args[0] - baseCommand = baseCommand + " " + serverName - args = args[1:] - } else { - err = errors.New("no server specified") - hk.Printf("Error: %v\n\n", err) - hk.Usage() - return err - } - } - - s, err := hk.FindServer(serverName) - if err != nil { - hk.Printf("Error: %v\n\n", err) - hk.Usage() - return err - } - - s.Flags().AddFlagSet(hk.Flags()) - err = s.Flags().Parse(args) - if err != nil || hk.helpFlagVal { - if err != nil { - hk.Printf("Error: %v\n\n", err) - } - s.Usage() - return err - } - - verflag.PrintAndExitIfRequested() - - logs.InitLogs() - defer logs.FlushLogs() - - err = s.Run(s, s.Flags().Args()) - if err != nil { - hk.Println("Error:", err) - } - - return err -} - -// RunToExit will run the hyperkube and then call os.Exit with an appropriate exit code. -func (hk *HyperKube) RunToExit(args []string) { - err := hk.Run(args) - if err != nil { - fmt.Fprint(os.Stderr, err.Error()) - os.Exit(1) - } - os.Exit(0) -} - -// Usage will write out a summary for all servers that this binary supports. -func (hk *HyperKube) Usage() { - tt := `{{if .Long}}{{.Long | trim | wrap ""}} -{{end}}Usage - - {{.Name}} [flags] - -Servers -{{range .Servers}} - {{.Name}} -{{.Long | trim | wrap " "}}{{end}} -Call '{{.Name}} --help' for help on a specific server. -` - util.ExecuteTemplate(hk.Out(), tt, hk) -} diff --git a/contrib/mesos/cmd/km/hyperkube_test.go b/contrib/mesos/cmd/km/hyperkube_test.go deleted file mode 100644 index 8035b93d9f7..00000000000 --- a/contrib/mesos/cmd/km/hyperkube_test.go +++ /dev/null @@ -1,144 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// clone of the upstream cmd/hypercube/hyperkube_test.go -package main - -import ( - "bytes" - "errors" - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -type result struct { - err error - output string -} - -func testServer(n string) *Server { - return &Server{ - SimpleUsage: n, - Long: fmt.Sprintf("A simple server named %s", n), - Run: func(s *Server, args []string) error { - s.hk.Printf("%s Run\n", s.Name()) - return nil - }, - } -} -func testServerError(n string) *Server { - return &Server{ - SimpleUsage: n, - Long: fmt.Sprintf("A simple server named %s that returns an error", n), - Run: func(s *Server, args []string) error { - s.hk.Printf("%s Run\n", s.Name()) - return errors.New("server returning error") - }, - } -} - -func runFull(t *testing.T, args string) *result { - buf := new(bytes.Buffer) - hk := HyperKube{ - Name: "hyperkube", - Long: "hyperkube is an all-in-one server binary.", - } - hk.SetOut(buf) - - hk.AddServer(testServer("test1")) - hk.AddServer(testServer("test2")) - hk.AddServer(testServer("test3")) - hk.AddServer(testServerError("test-error")) - - a := strings.Split(args, " ") - t.Logf("Running full with args: %q", a) - err := hk.Run(a) - - r := &result{err, buf.String()} - t.Logf("Result err: %v, output: %q", r.err, r.output) - - return r -} - -func TestRun(t *testing.T) { - x := runFull(t, "hyperkube test1") - assert.Contains(t, x.output, "test1 Run") - assert.NoError(t, x.err) -} - -func TestLinkRun(t *testing.T) { - x := runFull(t, "test1") - assert.Contains(t, x.output, "test1 Run") - assert.NoError(t, x.err) -} - -func TestTopNoArgs(t *testing.T) { - x := runFull(t, "hyperkube") - assert.EqualError(t, x.err, "no server specified") -} - -func TestBadServer(t *testing.T) { - x := runFull(t, "hyperkube bad-server") - assert.EqualError(t, x.err, "Server not found: bad-server") - assert.Contains(t, x.output, "Usage") -} - -func TestTopHelp(t *testing.T) { - x := runFull(t, "hyperkube --help") - assert.NoError(t, x.err) - assert.Contains(t, x.output, "all-in-one") - assert.Contains(t, x.output, "A simple server named test1") -} - -func TestTopFlags(t *testing.T) { - x := runFull(t, "hyperkube --help test1") - assert.NoError(t, x.err) - assert.Contains(t, x.output, "all-in-one") - assert.Contains(t, x.output, "A simple server named test1") - assert.NotContains(t, x.output, "test1 Run") -} - -func TestTopFlagsBad(t *testing.T) { - x := runFull(t, "hyperkube --bad-flag") - assert.EqualError(t, x.err, "unknown flag: --bad-flag") - assert.Contains(t, x.output, "all-in-one") - assert.Contains(t, x.output, "A simple server named test1") -} - -func TestServerHelp(t *testing.T) { - x := runFull(t, "hyperkube test1 --help") - assert.NoError(t, x.err) - assert.Contains(t, x.output, "A simple server named test1") - assert.Contains(t, x.output, "-h, --help help for hyperkube") - assert.NotContains(t, x.output, "test1 Run") -} - -func TestServerFlagsBad(t *testing.T) { - x := runFull(t, "hyperkube test1 --bad-flag") - assert.EqualError(t, x.err, "unknown flag: --bad-flag") - assert.Contains(t, x.output, "A simple server named test1") - assert.Contains(t, x.output, "-h, --help help for hyperkube") - assert.NotContains(t, x.output, "test1 Run") -} - -func TestServerError(t *testing.T) { - x := runFull(t, "hyperkube test-error") - assert.Contains(t, x.output, "test-error Run") - assert.EqualError(t, x.err, "server returning error") -} diff --git a/contrib/mesos/cmd/km/k8sm-controllermanager.go b/contrib/mesos/cmd/km/k8sm-controllermanager.go deleted file mode 100644 index 2b7fb207fa3..00000000000 --- a/contrib/mesos/cmd/km/k8sm-controllermanager.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// clone of the upstream cmd/hypercube/kube-controllermanager.go -package main - -import ( - "k8s.io/kubernetes/contrib/mesos/pkg/controllermanager" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" -) - -// NewHyperkubeServer creates a new hyperkube Server object that includes the -// description and flags. -func NewControllerManager() *Server { - s := controllermanager.NewCMServer() - - hks := Server{ - SimpleUsage: hyperkube.CommandControllerManager, - Long: "A server that runs a set of active components. This includes replication controllers, service endpoints and nodes.", - Run: func(_ *Server, args []string) error { - return s.Run(args) - }, - } - s.AddFlags(hks.Flags()) - return &hks -} diff --git a/contrib/mesos/cmd/km/k8sm-executor.go b/contrib/mesos/cmd/km/k8sm-executor.go deleted file mode 100644 index 69a6a52c19d..00000000000 --- a/contrib/mesos/cmd/km/k8sm-executor.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "k8s.io/kubernetes/contrib/mesos/pkg/executor/service" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" -) - -// NewHyperkubeServer creates a new hyperkube Server object that includes the -// description and flags. -func NewKubeletExecutor() *Server { - s := service.NewKubeletExecutorServer() - hks := Server{ - SimpleUsage: hyperkube.CommandExecutor, - Long: `The kubelet-executor binary is responsible for maintaining a set of containers -on a particular node. It syncs data from a specialized Mesos source that tracks -task launches and kills. It then queries Docker to see what is currently -running. It synchronizes the configuration data, with the running set of -containers by starting or stopping Docker containers.`, - Run: func(hks *Server, args []string) error { - return s.Run(hks, args) - }, - } - s.AddFlags(hks.Flags()) - return &hks -} diff --git a/contrib/mesos/cmd/km/k8sm-minion.go b/contrib/mesos/cmd/km/k8sm-minion.go deleted file mode 100644 index d921469e0fc..00000000000 --- a/contrib/mesos/cmd/km/k8sm-minion.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" - "k8s.io/kubernetes/contrib/mesos/pkg/minion" -) - -// NewMinion creates a new hyperkube Server object that includes the -// description and flags. -func NewMinion() *Server { - s := minion.NewMinionServer() - hks := Server{ - SimpleUsage: hyperkube.CommandMinion, - Long: `Implements a Kubernetes minion. This will launch the proxy and executor.`, - Run: func(hks *Server, args []string) error { - return s.Run(hks, args) - }, - } - s.AddMinionFlags(hks.Flags()) - s.AddExecutorFlags(hks.Flags()) - - return &hks -} diff --git a/contrib/mesos/cmd/km/k8sm-scheduler.go b/contrib/mesos/cmd/km/k8sm-scheduler.go deleted file mode 100644 index 7b0100fc079..00000000000 --- a/contrib/mesos/cmd/km/k8sm-scheduler.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// clone of the upstream cmd/hypercube/k8sm-scheduler.go -package main - -import ( - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/service" -) - -// NewScheduler creates a new hyperkube Server object that includes the -// description and flags. -func NewScheduler() *Server { - s := service.NewSchedulerServer() - - hks := Server{ - SimpleUsage: hyperkube.CommandScheduler, - Long: `Implements the Kubernetes-Mesos scheduler. This will launch Mesos tasks which -results in pods assigned to kubelets based on capacity and constraints.`, - Run: func(hks *Server, args []string) error { - return s.Run(hks, args) - }, - } - s.AddHyperkubeFlags(hks.Flags()) - return &hks -} diff --git a/contrib/mesos/cmd/km/km.go b/contrib/mesos/cmd/km/km.go deleted file mode 100644 index 412cba7b7c6..00000000000 --- a/contrib/mesos/cmd/km/km.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// clone of the upstream cmd/hypercube/main.go -package main - -import ( - "os" - - _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration - _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration -) - -func main() { - hk := HyperKube{ - Name: "km", - Long: "This is an all-in-one binary that can run any of the various Kubernetes-Mesos servers.", - } - - hk.AddServer(NewKubeAPIServer()) - hk.AddServer(NewControllerManager()) - hk.AddServer(NewScheduler()) - hk.AddServer(NewKubeletExecutor()) - hk.AddServer(NewKubeProxy()) - hk.AddServer(NewMinion()) - - hk.RunToExit(os.Args) -} diff --git a/contrib/mesos/cmd/km/kube-apiserver.go b/contrib/mesos/cmd/km/kube-apiserver.go deleted file mode 100644 index 59c1ba81fc5..00000000000 --- a/contrib/mesos/cmd/km/kube-apiserver.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// clone of the upstream cmd/hypercube/kube-apiserver.go -package main - -import ( - "k8s.io/kubernetes/cmd/kube-apiserver/app" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" -) - -// NewKubeAPIServer creates a new hyperkube Server object that includes the -// description and flags. -func NewKubeAPIServer() *Server { - s := options.NewAPIServer() - - hks := Server{ - SimpleUsage: hyperkube.CommandApiserver, - Long: "The main API entrypoint and interface to the storage system. The API server is also the focal point for all authorization decisions.", - Run: func(_ *Server, _ []string) error { - return app.Run(s) - }, - } - s.AddFlags(hks.Flags()) - return &hks -} diff --git a/contrib/mesos/cmd/km/kube-proxy.go b/contrib/mesos/cmd/km/kube-proxy.go deleted file mode 100644 index 7550d989c5f..00000000000 --- a/contrib/mesos/cmd/km/kube-proxy.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// clone of the upstream cmd/hypercube/kube-proxy.go -package main - -import ( - "k8s.io/kubernetes/cmd/kube-proxy/app" - "k8s.io/kubernetes/cmd/kube-proxy/app/options" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" -) - -// NewKubeProxy creates a new hyperkube Server object that includes the -// description and flags. - -func NewKubeProxy() *Server { - config := options.NewProxyConfig() - - hks := Server{ - SimpleUsage: hyperkube.CommandProxy, - Long: `The Kubernetes proxy server is responsible for taking traffic directed at - services and forwarding it to the appropriate pods. It generally runs on - nodes next to the Kubelet and proxies traffic from local pods to remote pods. - It is also used when handling incoming external traffic.`, - } - - config.AddFlags(hks.Flags()) - - hks.Run = func(_ *Server, _ []string) error { - s, err := app.NewProxyServerDefault(config) - if err != nil { - return err - } - - return s.Run() - } - - return &hks -} diff --git a/contrib/mesos/cmd/km/server.go b/contrib/mesos/cmd/km/server.go deleted file mode 100644 index e4dddae0506..00000000000 --- a/contrib/mesos/cmd/km/server.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// clone of the upstream cmd/hypercube/server.go -package main - -import ( - "io/ioutil" - "strings" - - "k8s.io/kubernetes/pkg/util" - - "github.com/spf13/pflag" -) - -type serverRunFunc func(s *Server, args []string) error - -// Server describes a server that this binary can morph into. -type Server struct { - SimpleUsage string // One line description of the server. - Long string // Longer free form description of the server - Run serverRunFunc // Run the server. This is not expected to return. - - flags *pflag.FlagSet // Flags for the command (and all dependents) - name string - hk *HyperKube -} - -// Usage returns the full usage string including all of the flags. -func (s *Server) Usage() error { - tt := `{{if .Long}}{{.Long | trim | wrap ""}} -{{end}}Usage: - {{.SimpleUsage}} [flags] - -Available Flags: -{{.Flags.FlagUsages}}` - - return util.ExecuteTemplate(s.hk.Out(), tt, s) -} - -// Name returns the name of the command as derived from the usage line. -func (s *Server) Name() string { - if s.name != "" { - return s.name - } - name := s.SimpleUsage - i := strings.Index(name, " ") - if i >= 0 { - name = name[:i] - } - return name -} - -// Flags returns a flagset for this server -func (s *Server) Flags() *pflag.FlagSet { - if s.flags == nil { - s.flags = pflag.NewFlagSet(s.Name(), pflag.ContinueOnError) - s.flags.SetOutput(ioutil.Discard) - } - return s.flags -} - -func (s *Server) FindServer(name string) bool { - if s == nil { - return false - } - _, err := s.hk.FindServer(name) - return err == nil -} diff --git a/contrib/mesos/docs/architecture.gliffy b/contrib/mesos/docs/architecture.gliffy deleted file mode 100644 index 097ac0eb11e..00000000000 --- a/contrib/mesos/docs/architecture.gliffy +++ /dev/null @@ -1 +0,0 @@ -{"contentType":"application/gliffy+json","version":"1.3","stage":{"background":"#FFFFFF","width":760,"height":510,"nodeIndex":227,"autoFit":true,"exportBorder":false,"gridOn":true,"snapToGrid":true,"drawingGuidesOn":true,"pageBreaksOn":false,"printGridOn":false,"printPaper":"LETTER","printShrinkToFit":false,"printPortrait":true,"maxWidth":5000,"maxHeight":5000,"themeData":null,"viewportType":"default","fitBB":{"min":{"x":2.5,"y":10},"max":{"x":760,"y":510}},"printModel":{"pageSize":"LETTER","portrait":true,"fitToOnePage":false,"displayPageBreaks":false},"objects":[{"x":290.0,"y":60.0,"rotation":0.0,"id":35,"width":470.0,"height":300.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":4,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#6d9eeb","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":310.0,"y":150.0,"rotation":0.0,"id":13,"width":130.0,"height":140.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":5,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#6d9eeb","fillColor":"#ffffff","gradient":false,"dashStyle":"8,8","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":30.0,"y":200.0,"rotation":0.0,"id":0,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":6,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#1155cc","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":2,"width":86.0,"height":28.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

controller-

manager

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":160.0,"y":200.0,"rotation":0.0,"id":3,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":9,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#1155cc","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":4,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

api-server

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":160.0,"y":80.0,"rotation":0.0,"id":5,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":12,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":3.0,"strokeColor":"#6d9eeb","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":6,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

scheduler

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":330.0,"y":170.0,"rotation":0.0,"id":8,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":15,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":3.0,"strokeColor":"#6d9eeb","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":9,"width":86.0,"height":28.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

kubelet-

executor

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":330.0,"y":230.0,"rotation":0.0,"id":10,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":18,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#1155cc","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":11,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

kube-proxy

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":330.0,"y":80.0,"rotation":0.0,"id":33,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":21,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#6d9eeb","fillColor":"#6d9eeb","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":34,"width":86.0,"height":28.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

mesos-

slave

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":460.0,"y":80.0,"rotation":0.0,"id":37,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":24,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#b7b7b7","fillColor":"#b7b7b7","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":38,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

docker

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":460.0,"y":300.0,"rotation":0.0,"id":53,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":27,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#b7b7b7","fillColor":"#b7b7b7","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":54,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

iptables

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":580.0,"y":120.0,"rotation":0.0,"id":78,"width":160.0,"height":180.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":30,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#1155cc","fillColor":"#ffffff","gradient":false,"dashStyle":"8,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":375.0,"y":122.0,"rotation":0.0,"id":84,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":31,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":33,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":13,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[0.0,-2.0],[0.0,8.0],[0.0,18.0],[0.0,28.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":423.0,"y":190.0,"rotation":0.0,"id":86,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":32,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":8,"py":0.29289321881345237,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":37,"py":0.9999999999999998,"px":0.29289321881345254}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-3.0,-8.284271247461902],[63.36038969321072,-8.284271247461902],[63.36038969321072,-70.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":571.0,"y":100.0,"rotation":0.0,"id":88,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":33,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":37,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":201,"py":0.5,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-66.0,20.0],[-66.0,68.0],[112.2142857142859,68.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":374.0,"y":272.0,"rotation":0.0,"id":90,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":34,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":10,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":53,"py":0.5,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[1.0,-2.0],[1.0,48.0],[86.0,48.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":525.0,"y":121.0,"rotation":0.0,"id":91,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":35,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":37,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":204,"py":0.5000000000000003,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-20.0,-1.0],[-20.0,75.00000000000003],[72.50000000000023,75.00000000000003]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":522.0,"y":113.0,"rotation":0.0,"id":92,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":36,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":37,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":202,"py":0.5,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-17.0,7.0],[-17.0,111.0],[161.2142857142859,111.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":524.0,"y":114.0,"rotation":0.0,"id":93,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":37,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":37,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":203,"py":0.5000000000000003,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-19.0,6.0],[-19.0,138.00000000000003],[73.50000000000023,138.00000000000003]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":30.0,"y":140.0,"rotation":0.0,"id":98,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":38,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#6d9eeb","fillColor":"#6d9eeb","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":99,"width":86.0,"height":28.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

mesos-

master

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":30.0,"y":80.0,"rotation":0.0,"id":100,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":41,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#b7b7b7","fillColor":"#b7b7b7","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":101,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

zookeeper

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":160.0,"y":140.0,"rotation":0.0,"id":102,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":44,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#b7b7b7","fillColor":"#b7b7b7","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":103,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

etcd

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":117.0,"y":101.0,"rotation":0.0,"id":106,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":47,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":98,"py":0.29289321881345237,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":5,"py":0.7071067811865475,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[3.0,50.7157287525381],[23.0,50.7157287525381],[23.0,7.284271247461902],[43.0,7.284271247461902]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":77.0,"y":83.0,"rotation":0.0,"id":107,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":48,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":98,"py":0.5,"px":0.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":33,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-47.0,77.0],[-70.0,77.0],[-70.0,-43.0],[298.0,-43.0],[298.0,-3.0]],"lockSegments":{"1":true},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":209.0,"y":118.0,"rotation":0.0,"id":108,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":49,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":5,"py":0.5,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":3,"py":0.29289321881345237,"px":1.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[41.0,-18.0],[60.0,-18.0],[60.0,93.7157287525381],[41.0,93.7157287525381]],"lockSegments":{"1":true},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":207.0,"y":202.0,"rotation":0.0,"id":109,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":50,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":3,"py":0.7071067811865475,"px":0.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":0,"py":0.7071067811865475,"px":0.9999999999999998}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-47.0,26.284271247461902],[-60.33333333333334,26.284271247461902],[-73.66666666666669,26.284271247461902],[-87.00000000000001,26.284271247461902]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":120.0,"y":220.0,"rotation":0.0,"id":110,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":51,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":102,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":3,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[85.0,-40.0],[85.0,-33.33333333333334],[85.0,-26.666666666666657],[85.0,-20.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":80.0,"y":120.0,"rotation":0.0,"id":111,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":52,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":98,"py":0.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":100,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-5.0,20.0],[-5.0,13.333333333333343],[-5.0,6.666666666666671],[-5.0,0.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":73.0,"y":240.0,"rotation":0.0,"id":114,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":53,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":3,"py":0.7071067811865475,"px":0.9999999999999998}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":10,"py":0.5,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[177.0,-11.715728752538098],[207.0,-11.715728752538098],[207.0,10.0],[257.0,10.0]],"lockSegments":{"1":true},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":490.0,"y":400.0,"rotation":0.0,"id":116,"width":130.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":54,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.5999999999999996,"y":0.0,"rotation":0.0,"id":117,"width":124.79999999999998,"height":12.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

kubelet-managed pod

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":706.0,"y":42.0,"rotation":0.0,"id":118,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":57,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":116,"py":0.5,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":78,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":-11.770164604422094,"endArrowRotation":-89.58463943745998,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-86.0,378.0],[-46.0,378.0],[-46.0,258.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":120.0,"y":340.0,"rotation":0.0,"id":120,"width":130.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":58,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.5999999999999996,"y":0.0,"rotation":0.0,"id":121,"width":124.79999999999998,"height":24.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

slave-managed executor container

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":437.0,"y":218.0,"rotation":0.0,"id":122,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":61,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":13,"py":0.8857142857142857,"px":0.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":120,"py":0.5,"px":1.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":171.89845704086358,"endArrowRotation":171.89845704160854,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-127.0,56.0],[-157.0,56.0],[-157.0,142.0],[-187.0,142.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":127.0,"y":111.0,"rotation":0.0,"id":129,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":62,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":98,"py":0.7071067811865475,"px":0.9999999999999998}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":0,"py":0.29289321881345237,"px":1.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-7.000000000000014,57.2842712474619],[6.0,57.2842712474619],[6.0,100.7157287525381],[-7.0,100.7157287525381]],"lockSegments":{"1":true},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":217.0,"y":212.0,"rotation":0.0,"id":132,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":63,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":141,"py":0.5,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":3,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-42.0,68.0],[-12.0,68.0],[-12.0,28.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":33.5,"y":330.0,"rotation":0.0,"id":133,"width":31.5,"height":50.0,"uid":"com.gliffy.shape.uml.uml_v1.default.actor","order":64,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.actor.uml_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":137,"width":33.0,"height":12.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"both","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

admin

","tid":null,"valign":"middle","vposition":"below","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":35.0,"y":325.0,"rotation":0.0,"id":135,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":67,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":133,"py":0.5,"px":1.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":-2.95764250679468,"endArrowRotation":-87.48214222550786,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[30.0,30.0],[86.0,30.0],[86.0,-29.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":532.0,"y":123.0,"rotation":0.0,"id":138,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":68,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":8,"py":0.7071067811865475,"px":0.9999999999999998}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":78,"py":0.8722222222222222,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-112.0,75.2842712474619],[-50.0,75.2842712474619],[-50.0,154.0],[48.0,154.0]],"lockSegments":{"1":true},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":700.0,"y":320.0,"rotation":0.0,"id":139,"width":50.0,"height":28.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":69,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

SLAVE

\n

HOST

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":105.0,"y":260.0,"rotation":0.0,"id":141,"width":70.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.ellipse","order":70,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":2.0,"strokeColor":"#1155cc","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.4000000000000001,"y":0.0,"rotation":0.0,"id":142,"width":67.2,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

kubectl

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":80.0,"y":400.0,"rotation":0.0,"id":179,"width":192.5,"height":70.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":97,"lockAspectRatio":false,"lockShape":false,"children":[{"x":62.5,"y":40.0,"rotation":0.0,"id":147,"width":130.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":80,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

comm / data transfer

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":24.5,"y":48.0,"rotation":0.0,"id":148,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":78,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[0.0,-2.0],[17.0,-2.0],[17.0,-2.0],[34.0,-2.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":62.5,"y":20.0,"rotation":0.0,"id":144,"width":130.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":76,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

life cycle management

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":24.5,"y":28.0,"rotation":0.0,"id":143,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":74,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[0.0,-2.0],[17.0,-2.0],[17.0,-2.0],[34.0,-2.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":0.0,"y":0.0,"rotation":0.0,"id":152,"width":192.5,"height":70.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":3,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":290.0,"y":400.0,"rotation":0.0,"id":180,"width":180.0,"height":110.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":98,"lockAspectRatio":false,"lockShape":false,"children":[{"x":37.5,"y":79.0,"rotation":0.0,"id":175,"width":130.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":96,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

user spec'd container

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":20.0,"y":79.0,"rotation":0.0,"id":176,"width":11.0,"height":11.0,"uid":"com.gliffy.shape.basic.basic_v1.default.circle","order":94,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":1.0,"strokeColor":"#38761d","fillColor":"#38761d","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":37.5,"y":59.0,"rotation":0.0,"id":168,"width":130.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":92,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

core service

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":20.0,"y":59.0,"rotation":0.0,"id":169,"width":11.0,"height":11.0,"uid":"com.gliffy.shape.basic.basic_v1.default.circle","order":90,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":1.0,"strokeColor":"#b7b7b7","fillColor":"#b7b7b7","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":37.5,"y":39.0,"rotation":0.0,"id":163,"width":130.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":88,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

mesos component

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":20.0,"y":39.0,"rotation":0.0,"id":164,"width":11.0,"height":11.0,"uid":"com.gliffy.shape.basic.basic_v1.default.circle","order":86,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":1.0,"strokeColor":"#6d9eeb","fillColor":"#6d9eeb","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":37.5,"y":19.0,"rotation":0.0,"id":158,"width":130.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":84,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

kubernetes component

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":20.0,"y":19.0,"rotation":0.0,"id":156,"width":11.0,"height":11.0,"uid":"com.gliffy.shape.basic.basic_v1.default.circle","order":82,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":1.0,"strokeColor":"#1155cc","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":null},{"x":0.0,"y":0.0,"rotation":0.0,"id":178,"width":180.0,"height":110.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":1,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":490.0,"y":440.0,"rotation":0.0,"id":192,"width":130.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":99,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.5999999999999996,"y":0.0,"rotation":0.0,"id":193,"width":124.79999999999998,"height":24.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

kubelet-managed container, via Docker

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":873.0,"y":419.0,"rotation":0.0,"id":194,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":102,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":202,"py":0.5,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":192,"py":0.5,"px":1.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":90.16611201730352,"endArrowRotation":171.03524989098037,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-189.7857142857141,-195.0],[-189.7857142857141,41.0],[-253.0,41.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":547.5,"y":10.0,"rotation":0.0,"id":196,"width":152.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":103,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":3.0399999999999996,"y":0.0,"rotation":0.0,"id":197,"width":145.92000000000002,"height":24.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

establishes pod network, ipc namespaces

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":716.0,"y":52.0,"rotation":0.0,"id":198,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":106,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":196,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":201,"py":0.5,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":82.84229462672464,"endArrowRotation":89.45311382172923,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-92.5,-2.0],[-92.5,25.0],[-32.78571428571411,25.0],[-32.78571428571411,116.0]],"lockSegments":{"1":true},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":597.5000000000002,"y":140.0,"rotation":0.0,"id":216,"width":120.0,"height":140.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":127,"lockAspectRatio":false,"lockShape":false,"children":[{"x":0.0,"y":28.0,"rotation":0.0,"id":204,"width":68.57142857142858,"height":56.0,"uid":"com.gliffy.shape.basic.basic_v1.default.hexagon","order":125,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.hexagon.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#38761d","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.7142857142857144,"y":0.0,"rotation":0.0,"id":210,"width":65.14285714285711,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

logger

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":null},{"x":0.0,"y":84.0,"rotation":0.0,"id":203,"width":68.57142857142858,"height":56.0,"uid":"com.gliffy.shape.basic.basic_v1.default.hexagon","order":120,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.hexagon.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#38761d","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.7142857142857144,"y":0.0,"rotation":0.0,"id":212,"width":65.14285714285711,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

cache

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":null},{"x":51.42857142857133,"y":56.0,"rotation":0.0,"id":202,"width":68.57142857142858,"height":56.0,"uid":"com.gliffy.shape.basic.basic_v1.default.hexagon","order":115,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.hexagon.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#38761d","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.7142857142857144,"y":0.0,"rotation":0.0,"id":211,"width":65.14285714285711,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

webapp

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":null},{"x":51.42857142857133,"y":0.0,"rotation":0.0,"id":201,"width":68.57142857142858,"height":56.0,"uid":"com.gliffy.shape.basic.basic_v1.default.hexagon","order":110,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.hexagon.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.7142857142857144,"y":0.0,"rotation":0.0,"id":209,"width":65.14285714285711,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

pause

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":null}],"hidden":false,"layerId":null}],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":219.0,"y":128.0,"rotation":0.0,"id":220,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":128,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":8,"py":0.5,"px":0.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":3,"py":0.5,"px":1.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[111.0,62.0],[61.0,62.0],[61.0,92.0],[31.0,92.0]],"lockSegments":{"1":true},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":137.0,"y":121.0,"rotation":0.0,"id":221,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":129,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":98,"py":0.5,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":3,"py":0.2928932188134525,"px":1.1102230246251563E-16}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-17.0,39.0],[11.0,39.0],[11.0,90.7157287525381],[23.0,90.7157287525381]],"lockSegments":{"1":true},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":130.0,"y":230.0,"rotation":0.0,"id":222,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":130,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":5,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":102,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[75.0,-110.0],[75.0,-103.33333333333333],[75.0,-96.66666666666666],[75.0,-90.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":130.0,"y":230.0,"rotation":0.0,"id":223,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":131,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":33,"py":0.9999999999999998,"px":0.29289321881345254}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":8,"py":0.0,"px":0.2928932188134524}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[226.36038969321072,-110.0],[226.36038969321072,-93.33333333333334],[226.36038969321072,-76.66666666666666],[226.36038969321072,-60.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"},{"x":127.0,"y":111.0,"rotation":0.0,"id":225,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":132,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":100,"py":0.29289321881345237,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":5,"py":0.2928932188134525,"px":1.1102230246251563E-16}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-7.0,-19.284271247461902],[6.333333333333343,-19.284271247461902],[19.666666666666657,-19.284271247461902],[33.0,-19.284271247461902]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"N4HAvHaEUGxu"}],"layers":[{"guid":"N4HAvHaEUGxu","order":0,"name":"Layer 0","active":true,"locked":false,"visible":true,"nodeIndex":133}],"shapeStyles":{"com.gliffy.shape.basic.basic_v1.default":{"fill":"#38761d","stroke":"#6d9eeb","strokeWidth":3,"dashStyle":"8.0,2.0"},"com.gliffy.shape.uml.uml_v1.default":{"fill":"#FFFFFF","stroke":"#000000","strokeWidth":1}},"lineStyles":{"global":{"stroke":"#000000","strokeWidth":1,"dashStyle":"1.0,1.0","endArrow":1,"orthoMode":2}},"textStyles":{"global":{"bold":true,"size":"12px","color":"#ffffff"}}},"metadata":{"title":"untitled","revision":0,"exportBorder":false,"loadPosition":"default","libraries":["com.gliffy.libraries.basic.basic_v1.default","com.gliffy.libraries.flowchart.flowchart_v1.default","com.gliffy.libraries.swimlanes.swimlanes_v1.default","com.gliffy.libraries.uml.uml_v1.default","com.gliffy.libraries.erd.erd_v1.default","com.gliffy.libraries.ui.ui_v2.forms_components","com.gliffy.libraries.network.network_v3.home","com.gliffy.libraries.images"],"autosaveDisabled":false,"lastSerialized":1432239993668},"embeddedResources":{"index":0,"resources":[]}} \ No newline at end of file diff --git a/contrib/mesos/docs/architecture.md b/contrib/mesos/docs/architecture.md deleted file mode 100644 index 431f2c0ed0a..00000000000 --- a/contrib/mesos/docs/architecture.md +++ /dev/null @@ -1,67 +0,0 @@ -# Kubernetes-Mesos Architecture - -An [Apache Mesos][1] cluster consists of one or more masters, and one or more slaves. -Kubernetes-Mesos (k8sm) operates as a Mesos framework that runs on the cluster. -As a framework, k8sm provides scheduler and executor components, both of which are hybrids of Kubernetes and Mesos: -the scheduler component integrates the Kubernetes scheduling API and the Mesos scheduler runtime, whereas; -the executor component integrates Kubernetes kubelet services and the Mesos executor runtime. - -Multiple Mesos masters are typically configured to coordinate leadership election via Zookeeper. -Future releases of Mesos may implement leader election protocols [differently][2]. -Kubernetes maintains its internal registry (pods, replication controllers, bindings, nodes, services) in etcd. -Users typically interact with Kubernetes using the `kubectl` command to manage Kubernetes primitives. - -When a pod is created in Kubernetes, the k8sm scheduler creates an associated Mesos task and queues it for scheduling. -Upon pairing the pod/task with an acceptable resource offer, the scheduler binds the pod/task to the offer's slave. -As a result of binding the pod/task is launched and delivered to an executor (an executor is created by the Mesos slave if one is not already running). -The executor launches the pod/task, which registers the bound pod with the kubelet engine and the kubelet begins to manage the lifecycle of the pod instance. - -![Architecture Diagram](architecture.png) - -## Scheduling - -The scheduling of a pod on Kubernetes on Mesos is essentially a two-phase process: - -1. A new pod is noticed by the k8sm-scheduler and possibly matched with a - Mesos offer. Then: - - - The offer is *accepted*, - - the pod is *annotated* with a number of annotation, especially `k8s.mesosphere.io/bindingHost` - - the pod is *launched* on a Mesos slave. - - The existence of the `bindingHost` annotation tells the k8sm-scheduler that this pod has been launched. If it is not set, the pod is considered *new*. - -2. The Mesos slave receives the task launch event and starts (if not running yet) the k8sm-executor (possibly via the km hyperkube binary). Then: - - - The k8sm-executor *binds* the tasks to the node via the apiserver, which means that the `NodeName` field is set by the apiserver. - - The k8sm-executor sends the pod to the kubelet which is part of the k8sm-executor process. - - The kubelet launches the containers using Docker. - -## Networking - -Kubernetes-Mesos uses "normal" Docker IPv4, host-private networking, rather than Kubernetes' SDN-based networking that assigns an IP per pod. This is mostly transparent to the user, especially when using the service abstraction to access pods. For details on some issues it creates, see [issues][3]. - -![Network Diagram](networking.png) - -## Resource Accounting - -Mesos is designed to handle resource accounting and enforcement across the cluster. Part of that enforcement involves "growing" and "shrinking" the pool of resources allocated for executor containers. - -The implementation of the k8sm-executor launches pods as Docker containers (just like the upstream kubelet). The containers are resource limited (cpu and memory) with the means of `docker run` by the kubelet code. Moreover, all containers launched by the kubelet code are children of the k8sm-executor cgroup. This parent cgroup is assigned to the k8sm-executor by the Mesos slave. - -To actually enforce the defined resource limit for the k8sm-executor and its pods, enable the cpu and memory isolator in your Mesos slaves. - -The described resource allocation also applies to static pods which are run on every Mesos slave which runs a k8sm-executor. - -Kubernetes allows to define pods without resource limits for cpu and/or memory. The upstream kubelet will then run the containers without resource bounds. Because Mesos enforces resource accounting, it assign default container cpu and memory limits for those pods. By default these are 0.25 cpu shares and 64 MB of memory. These values can be customized via the `--default-container-cpu-limit` and `--default-container-mem-limit` of the k8sm-scheduler. - -Note that currently static pods without cpu and memory limit are not allowed and will make the k8sm-scheduler refuse to start (compare the [k8sm issues](issues.md)). - -[1]: http://mesos.apache.org/ -[2]: https://issues.apache.org/jira/browse/MESOS-1806 -[3]: issues.md#service-endpoints - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/mesos/docs/README.md?pixel)]() - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/mesos/docs/architecture.md?pixel)]() diff --git a/contrib/mesos/docs/architecture.png b/contrib/mesos/docs/architecture.png deleted file mode 100644 index 1de30f242f5a3f792a7d577f00aa1b955f62096e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69252 zcmd43Wmr{R)HX^>DGeJC1Zf1>gp_njcO%l>4bm+o-Hn7I-6<(Zw{&+SNXMD@)cc<6 zT-W*W{rP^0d#}CLoMVo>$34~ud0Fwts6?nRFfflL-iRo|z`&!!z`$W6BY{u!#@|rF zz>vX6hzKgX>g=VXrG-!2ck@?rIBV?nYBo;B#7h;wPoSj#Ek(Ee{AWlgtw;#A1lD>? z)HAxkUMk1zclVMpA_v4&TIva+&Tl76n)&X3o*!9O&K~r$vTpYCuwHxWxS93Ww+y)q za%FC&9dhBzjIkYcASA)y=)y_+pg>`%TOa;I>+2JU2tSMXzdzN5%fr%bYprSfe_b!| z89^OdX7|sJ|Gkr+uhoCI{QJ{xlp|FC)qYN-|9xv(R&a~B;%N_G#ESw9zoL$!I}s zer<@ZPvigHt!*-l+?uvn4+?~=)!7(E(M(VsS_ktn_rImQ#2Kcs=gD>;q@t0BgRNmg zF>dn{gcFhV`|lnpFmhXIV%=DeF?Fw5V1DRe;-jA-AD{oX4oNtkY$b`rG2`nPU3dlv zrNA2He_}5{ISLpwb9_1i1MUtVwTGzJruz}od+1+D7gDz3aK}4xS%HVMxBlPta34Av zNBl6v6fw~kS6lZwGw1H6?-yUo^aX|(GsHSM;l>ZeovjP8x>M4lA$`={lurIf+kA@< z=J>KL=IWTw_Ge8hVfRpw-WL~`{<2`wD%e?jlOs086+?=KWMWp&0scAvwK4av#C&rQ zwDHj;BqX+GYnjdt=EHVNzmXm`pOX^3G7ktuBUqVGdnamcPQS9U0(Cji?oVcYRHRml zPxVGluG^xtIS7OQ2Ge9HL)6i+3VE^8L~c01`)qF-!tcfN{Q2`T!ynjlj$<+_i)QPB z_l=);W@>Fd`uM;!T^}?;xa>b6zh7#FRW4F}92plsI+E$<2d7*p4_o@}u7Os`(2x?J zNhiF&ZpaB;^X)6YC3IYdL5x(Q`R;%a>OEt zlGv??#6FTTwrEuvKen!4Pr&ZG9F=0itGGTH;gtu5Qm44f>Oqv%dNJ`@g+yuCshs zg5OH8QmK0ssJh1ekHH@4z`c~)GF{=eT@sjkf7q=4&8s7d)zJy5j z`6G0El8A*mZpkYkVN>-bG7)UQR-9b1Su(TdH|)VA7fWE&3d;1pBXpUwdvte_c^~4P z$?x5wdo;kYm<8_cda)eLtlJ9HbTarF^2!{yFk@0qSC@=hI>~3bBX~9VnOAF_-D(*) zN5wv~W_uz1PZmE*T$g+w1-!{-Gwx3!|E1OF%*c0ps^oFh_9!pmBP3dYf?TP^!>ui$ zv+J$Pe4x|rBnm9evfXqGWn9~CtZv)VZFVTJ`;t|?y1d(Yj9jNS=kqR^7cMDHJAQ(hX=8?s z#sev2unI9_mCZb-KcIQLrW#T>=#$TbMtR&%3o#KU{d^HRx5o+sA!M+E%aYjWllS>Q zORMbh&j(Crs_4E)lDz1IwGw~+9fw91SF71H&o5)NSB;19N?PAnm5}dxk5waRzryne zeF#TEhhJN&*2liVzx`qBHA)>%c5d;Tfm9BH>jnO+EsEoW>sgy`qPvsD%=(?k$2BRc znB0zAWFC0E@h^f6x+75*1GH?K@x}_|#M8N*0&KJJ;C}@czHMYT9!#UF!C=&^LVlrE zf~(~miWW5bi`I-q8plDrPg5JJp12(=$Jh3e|BkliXjrwNAj4hq!Zb?YYNx0)Dk_T1 z1A)}@RN?rLt(>Wa>c@z^W@T;#Rw;pwtDvPtPO)eeNjO4B9Gw!>beK%x25eAsVXaYc zHkGoN1(@T)J&r>&5(7lrq=)VwbJT?!#Ibb~a=AFtE4Rr=^Dr<4tZY`oIkg9CV*;}#rn66EylSmxgE$sFl#t_mLIN3 zUGdAV;*m^6f2}#W;^*suU@9bQ^jzE$n_;IirEWJW-6*4LNtewV5+~1|$X4*)QUL&$H_#=3~ zPr_ET^NZ?eF(o$k#{YKg43pCdMwQL*Kx3OZ+D> zMdYIZSyVCEPq=WF@35u)Ybhmsdj7}vvN%g9>0?`*Ik4b+B(iC=cE2`K^1Fj6t%X|H-^F}akWWsQ}|e{&>kM}t6eF4`|HJfsM{|?y8{2;I1mW!OSW98o~_YZyVdV_3trO8TiDePfkEo$ z=;$aWCMMaz`$DzSj5>MiHb2C|ci&oif+O+rY#pPYzP+pQx^=gcnE4(-+;#hi^5vU4 z5M(5^1;mZAx#N_bo5QILh02u*WIuQ_?Ps%XiDby;{k>?)2 ze$ZSTcRw4ccQC0kpFkbryJldwobmzYzWno@H{)nZmz?G(|CnD>Cj=n>Wu z4!w{bz7w7A&z~k;&oF^DA`>5FM{eqiB@nroE+Y_zkfow9cj1T+vj+# z$Eyi&P%7O~#RF-cA1=Pr|8taQ#(y8B-5Y3B$KzYh$&$BPomq4Q7Vw)x{r>yQnH(ZF z@0S++cNXCvZ=4#(JfmL9Ly+dpKH0d2G}Z56TTPFjQI%&%#MLKX~E0qN_Qu7TZZOV=0#IgL+<%cNpTF>G?X3+-5r#4 z4!MTr`n_NMX!$ivAAT#R#Jz13@Wjf8>R=AVLv5^I<#I^~rG z{`IRj`n^BuB7mcv3u=%y?mVN)p58}A>h)hx9r*%x7#%N=_LqO7rOvWZ&<{54%Hnd< z0KS|ijIuxp`=J(c)>DE;Ij}zFWWc42`rCn9lL_3ilN3+oNfFoyAo5GMq5%gzh(`1y zEUX7S;ux5MeL{Bk)hHHOQ~}vkVuPJC@_v&^h5a z4t~c2HJC#nX?L}_&wd{%IsD?RwqDjXzH2M9K(zm#Xd*2lz^@;FU^smE9jb6#M=PDh z3F|4;ZJ6XE31?~Ya+pyCdgQYTrxMxwkcu3U!E!%Iy&InAIt_kCt!od*6YWsun=g|Z zRB1A_0^+p`)HwOcp!0^%Ng_SOwbjm-Ufk%vLfhv#To^uMJE#@PY?p;Zxoj(>vuI(@ z^mN1i_>&6=EO(V(ZQPZ5uDw%|51uc``EdIZ7&~?*ijvWAY|+4_wrIwK0S{4Zi4U$R zZ`|`M7Y+3i3S26?72GB5NLRxR6xIb(_ZgCl{07gT;>nt_RZDUnIki$Bj+6|PCLub_ z?s|(Yo}@kLl?-7BQ^DB`HQYaw{|$^sns6X7ee>o`>+x!LbIlWN+r#IreCP`kI@+2K zana{3++N3DE>K4%D=Q0Tf2O(vJrq}r~BP*`fxfjOOskQ zcwXov+>S>(?zYR#{xH{2SMk?&bbmBE2&D|eUr+gI6pAaOubPe^P~H<`LY_4=SAi?+NLj zThV5xmPbM^+?wizVNX?AW6U#}hpW3Y`p|mydzTqhzr*J6{L((ig%Gt%QluJ2a27n0 z0;1g;BdEz_4rh8$K?+-}Bp+fF5F;c*kTk89f>FS=)4{&`p+FBlV#|+|yHyEM_oR73 zq=%8U5vZ5M;D|Y_J3(sw5F`JrCjRf$%*<%Nq?Q-^vl?XX66yz2%!-1!WTvF1t}xY2 zwLMIr+!%N;9@|F%tNBG&iQ(^&pY|N*2nmitJW|Bbkqv~F<;Pv?EcDN_^tFwiA&EwG z{dhQ-@COO154I48dO|#Et?G@XTSDuMO_F@fe$~> z+9~;xw%3sr%23dU(B$7*sE`ZN5+6FHd{=Sb`Z&eEej;d&W2X;<&iO|$ri7e44-0So zi7cJLL->FkBsPq&Q+f41fyG;;U+j=~-V3(qMUa*NVl?gzZ77%D>sS(-O2fl}GW+tr zKg$A{*F#0`ezs?}5@SDN__s3jIgs?%8}N0ps)leO-Jo$uDivx zP}d_}p5Nn2pgO4uqqBEp#raEVaH} z^P#+aa+L}EY(>bCh<|)k!5>SI9<9?xrNgIn36!`qOAgz;;oPFui&uWy+D4|(cJw+ z+0n9Or0Uu07vf`QRGVbmhEE2KamnajC0dR1v3H6hnXA0GA}IUeWE<60RDWD& zlrP5k;!0AH?!5`rz$&&`G^wmL;NHtQD}DFfH87&}!)qcmH` zkCy1bHdd+fqiv$9?~z88Rw;&9K|Z<8kLYejHO3`L51VJTYw}0M%Ceb!j0av@ zdDcnuo$-2tVV0a)CM5tMG!$O;`N1QnxOXf^S2|Nwss*CG-A$QiFBV_A?k53ek)T28d zjx1O&@U6Y(Taj6avkU7`Sj{~|GAA7XX(f!h39DB^ESfJ`_A%>H#vHy1-&{W z?~d!%H5(H#XJ2$PZsU5e3A)qGHNSVUoUTBe_@Z($qM1^v2~O$bC3M){T2T@EF(w24 z+KeXoP+cA8z}{@EDe-8|;Y(au+{pC7aj%*8`1NjkwM$<$5Rfj+I9`1YO$zZAX=hER zxA^@WUWSWYX{4O1ucXSgq0p$DEb;3rxk@yxFI*qB>}MD0UzHNo#wi?)1iX^*|Mrc4 z>y*lC?jd^YU_C?99KF4}K6J`N+@Y1`inf!hr2iob&1=pkdF{;hO6#_mQGu+y%xsfT zDN#+rf=;r^Gt4~YB~kwklA35YH4dGXXi6E2%mu?RLZTYJ;V7O>ID*nIv!uzQFJ|8N zJk~#~&J|UOn)K%sW!oRIW`(U74`ndc&>ZIiGq#(c;6n%bf~LNHPD_LC46Kl6Ed79B zNYKTFD|__phbION)76ouXU$Z7e~u{ zn!+c_Qh%fn)Y&P*NJSNh=&w18O=4qd`E?SwAEhU)lPzf8`G%%i=jpGExF*jjjhmt_5^5#>Y6Uns=tS3`_Wy)lGfr92{5(TSq zLs7YvbbX&|lE?GCtdvE6nP6%UG%!BwIQC0*C992ij6_WjgMy_LOHqWPgxj3r*xCJL z6xS&{f*f#Q<^1tF+N@U?|OLqM`@+~=(+!qO^{8-9iyH0-`SH#UWG?0%RUeJuX?`-*NP_SQUI4YXJ5 zXTX+F>2g99iK9(<&e{r+DOpMys=O>sh1DZh* z#pZ2MiLrr~{^sPL6cJ{eQ+)=WeM1P#e8zp65!#RDZ7`ei8JXc7Y%M=KF>{2qptUo# zm*#KO7-x^5xjm^63%6(1`>Z5)zU#I-N7j}a%vdHdIN4^1Ax$+@*4vl#?YTP?{2avm zB}pNR-kRhsjW3sBkKYRDx}_d#=We=MKJ=GPx*!Y?k8+j!&tq-=k48Y1a{~;n<*fq7 z#9*=CGJxG5-ERLZ&AgyE(Q6}%&EKy|c=%1LhhK^F&7uU4z-_BSonDPt;P9I^al)h>R*Y+*W zYzZQH*WX#}Y0|pLFVJb>^>a+<(^Tjt5pTCaCFrsxGkpgDcv!J!mFxo`|2>@0Ka%A2 zm6y@Hws0*fQoNHvTULzqNR?S2Il#HUzTA3&4#44{p{DjF(%0aZL?#n{zLFdT_wmH? znI6}22nY!sO}?*6y=1Xpz?>!{bVUWyl(}|3u5&!VMH=K~XlH;ZK2pgg&e?pykLyL? zJqt5#<;9V+LlqML$UPyv9*bb`X&a8sEv!0&`h_%afNF%`x9vvD?+y_etK}L{&cS{H zgEHNtE>%X&0uUya0WQ`tQ*Gh-t-`t_TMMxxnnK+6bW6d`*c#L%d8+Y<{>o|2III@k zeSGN7sj`wV_~<2ijs?1|#>a#T>Td<&PTdY-M`(Nc=oIN@D%eaHu%1p|drwt)q+ueR z#udpU?!T82y`99+UucD_ZR&iF68-FxuXlj+YU!CesXxKfNYs%Pq^XGh z^c4&7B^up~hdHKqqu)C_bpbeCsBzLOh3&P5XRAxLeNYUJ zCtmao%pALGLO&IolF#rq&B_+M4zu^gPt%^9ljmmn<&dAn07=U}lY3f;f`kqG&4Uk~ zuGaG!t%*9*3v11jN|bMQ zcQQ0IgwlRMn4^bKCS1RhNVW=WBYv$UKGXe`gb58vTyx~I3@pw>oOys6^2?sdV0TBY z3l%OsMwy}n510~lCwzKL)nzV$9rdM}`VB)!h?So+FLkbFv8$cr=!P8i(s?GIZ=^D$RFh2 z%|t=l>POHZzUo`ET!_2!(drQ^)Yy*YNrAEd!o?;5jPN3tpW4;9h_N$lMurOl|BX9V zo1jDm?twBR*iP^Eo({%QZ&HYbW1izT{hrh+l+T9S7|Q$zatZWYu?QhAua-Bl<4_b>t;KCK zzLS2|90pdCL0o-VNKR1COxwa~O0~l`LUhaG`QpO@eiS_`&|Ow5u;U7T$?tRruwtIqq1%yoNIxaF zw95M$bdy+&P`xgeAB8{VQf%5vVKtG>)-8MW-4kq0y~U#*a-Wl%+odj?qc;OQzF%AP z{}#SOxBvi|I(a?=&l@SaHcW*LJQ}nS1K#OVmAmcPo(6+@nLtzh;o>)b*$f`Kg{?8+ z*sG8bWO95<&jPuF!=@8iyYlI;8kO<|vLw-Bpq=$ld`%Rog$-fwU1V{-3hneOK0IYp zE7lAI4R!^5O|Gmb^KY7fw2?tuI{KFq^pCDTRUd{#>-nmmP7r#O^OEr2XOL6Ro$h(; zB8`7w2)4LBY^x8ydCv2jhSL;cX!>VOp!Vgr|f=-nWSES$TW9(g+h= z0PA~lDt?AyMx`PKAQcw2BEV$n`DDgl44{4Y(=BqZ_mis-gGBR za+hRiCEG1Hh=`(QGc$Z5E50?X8vKNbXaaC62t!{2z?Gv}pkPQ*BzMP#)0T&xc+hn~r@w-`>%2 zJox$$SwR593NObq^VB$qdiyj#AEe>xO|Ip@s%7n#+XEqd9(AroSKoT)XO5_Zk%5yg)h3wU% zt^42DuHD9{J+sU!6@qGO=uP?k3|_R_g5#fA5f}1RCHhesq`I)>lC9q*2o%QMzw9o64^(@Fxl%x#kp@h7=#>@kO0po1idiyj$im%)SC3G~Jj zZ3A-<+jJxcu}3VGR@pBvAftVKHg$o%dbxf1@&t`49 zK@-e`BZH5V(}rHuAfrO8TIBC&3Fw$v8eolEMI-c`~M?G6}|m|mzqOe)IF=bWbETw!^2O;osgF4}+fr4UzDwWSwhhJZdYU1zC)|T43Qs+rM>Fe}M z8}1=YN$V*T&Uf&q8yIcLWit!ILBV%d65;nteCI_70(vF!M?`-Eeq3&2Obe!f&KMK;n$!J|Xh)Or;A&lK6JYTIpy2BI;?J;11j?Lgx8q?@nXGP_TDqVF#l_(^osLTB zTZo+>%nJ_oSB)NBZf2r+b7~npO`kajl${XUy!4p09!R%eo5G9bqY})EUwur0-i0m2 zbMaNZ?%EGEMA`hr@t3du%~ib9Blt|IrKQj*i?E72-prV_6hv41K;HwEsS5q)gOoHv zW2l&JR#J1LCn^Er6T#KGZ}^h~K&2Nl{>@qFMGt`sYt`W^(YdvoAD>>^GBYB=8y^sO zDbUg{>wJQ+T?z^_N$lN}{aJ+S7H~F0S8xP5({!)_% zxSp2p)6&1Q6Dz8mQX>&Jx?jQlmDkxm&V8?6sarOrF_}Pl%@x0oS%Com8>LDzAtNKQ z+Ty3A|CbztSXE988wpL82CA1gp195Lt_i0TN6U?`;6uLrBzh}WLi-$|+U$%WOR^9c z{O$sEI(f7FEaDNC(_3PmiyM=v<8@2Rx4eY&G#V$2jMx&G(lJaL>c=KS7=+h(+}GoMeFPvBqEt zCFo``Vmq=yc`1nutz;eyW+}g5`3GK57ln0eo{0)3>_Uy&Ipq%&8Tt`j=TF|)Q#8%MOWiJK`7Iz#z8d9+B zrKK)DI^S#+`wMmeVgfvvaESljV;X_%^CW=CN>BBoIxG7rR|g{#Wyi8BpjZl#2&og-62gpacX~sNYGX`>~K@{)rF;Wvgb_BADi^TV; z6#AVt=jV9sJ_`E^efqi#C6%A(-Qhjd^UBc! zG?R@8`~p&nAZk*0=-9nl=GaDU!Gcu$;$y1Q$1!AC>m2~H42jL{p-YerN`dP6U1;A_ zG42kTmCJzwyNWp-nMn!C8QnZj3-Hj!WNysXSU*7~>SFAtGcEZ1rOAF#l2{O^g*R5) z(Osc&o$DDVsn_ufZKX=A4U7hR`^1-!n&Q+*P{iYUp{?x-%tq6{#qs|iRU#&)`T{Kd zj$ehYeD`FkqAHiQ8%}VAsw($#%s~Ua*S(6y>XNyqN_dfu8-{-oS)ZmyJPFgj(cw&W z5vQODu-8(CBElsC`k8& z%o(7vAo_=1YXDT9<%hc>beK*emERp;Ki+{|Lw+Gs)TTcTSiVezGChYpxv6`qr7G>0 zzn**fRv1&6PdOy$7ao1?P@EN9ckrs_s`gH1-DR|d=~-q$=eOu@9g`EKp%ZA_R3Bv3 zc7{z-FflUR)uFDTP4&Oxd99O+oVAIx5KM#N=2d08(N0#M{?(D;{E5GS8CF751L_h} zfXr6)FhGlGhkN1X)dcn1rUB2Ta_wmtjS_Si75g^YZMPEiNSK9cgx@Pdd%gxIkB!0% zz^V5lYF2GG7fJ4y8cw9vj`lt`VYP7NTL)-1 zSDl_Z65)z$%HtD{J$Ayk>}YSV3J5^>9RA~2+*LVp4m*j6k?90MT^|XOQUrO z#sXqUDqk3Vwy)-9YA(XhLa6ut<#mU2w>m&BEA2v8YmLHf@n7} z2}yBRcXz5i%p{Gza^j&&#z)kM(Zq{&+V z=0u0sn9?&kKv2CeG0gxm6k<$BcnQ($iXU^YMSEq$4|)qi@E0^@V;{Awvs;t1KvO{- zFl~w0Ok3H~$Uv$IfK*{g$1)z#{)EmJdiXnKb?D7RMu#N?1g zOZDjOx7SM7=0kYVM<$X%HBO5fX6ugiwtO)Hsozzq8%FRDe>SEBnYwVG*?q9nIMthS zrq$9qTNTAeNA<>O(A{!!C`^zqO_O5(@+FDK`tx%w%C}Sxr#6`UhH4m~iRch0%mdqU z+_6tm9|WUQ>gc;+EfEbH?B7AXeSKr5A2oF{ru^DxW7f3FVm!0PjjN>Jh!>95P^x9F znz_wXIc;{XkpXH`?kXH??=DZM|F!lP!ug9BwfN@tk~*^iqAD; zDt;sET=G4q&8hN64RvAZn?T@(os{4zfsEphy1b_GG+%7EhoTrD-N%3tMJKWN6UV7Y zSN<_89i4=fq~s_fywT>Uh?tXGGp*;2BjQ89gYo4{S$eg%w8P9_SV(uRBz_dIR6lc0 zkJi*7G&-GR5^^xOEgDsx23XabP3KjI{*OXGqO-U(+)+ z*y?n%)N(b$5LwY?R$i^!_5H44O7Qy`p*4zP>v#XW%xsL>4whlxa;e+yAyJ?|qbDe3 zVLU#VtH)83lg=sq9Vv{l^9!Ndu3rZ)S44*rA7qQZJ}^IPGI43L^()@Esb8~fx!t+( z2)}lwM zJR6-Ka7ceF&a$e27R4HA(8^=MzNRXUt1oi?wY9O`y>c!uhnWj1YRR|p6qz-~$6iXX zxuSoT>o3^6NpI7uzzWUIgdm%~*y!o)7L;p^$x|E{Eoaw6;qJMLH;YEgQ z_8TcFVPj+JmpV;7aP3bFcT9zag|V=){f>@odw3py{`6Hv8o&62D=wwTYARjh0!b|} zqCREgc(ImjeMv5)#2d$)-E`O7zISB||D)`H5SamSOofn+3~)^O4gGO^=^2i+uO68Z zB62#j4;jIdkJT9FQ-x8X1SEZdMK0Y6c8?Z>?wY-coK!oji;s@>1yoivJbGn;`35Hv zM%m~iz@2Ju@w^_cvojc*OW8fJ>%;QNNTIcFp+bxDITAxQV}1gw_t;Yhy2YLqMkc;j z^Qr8?uU$dwPy<%4sbRT!%@#+VVn6BWf7$xs)MKpa$9#=7QB7%Tr0QES^6Wm-F*caD zz4(fQAJbAtIN6a)GJ=#fyOb@ZK=Us-U+*bxNIxD=vKf~=;uCWgxCcW1<;kW(y}eP| zYnMU8yww#b>?{JH8GHc5eF9$h(<3)!9Y#PkqBGB2a}}251auVf;3p&=3$9XMvO<64 z)cR7s&f+(OHyU}Oe2T^ZDY-#4oJk0E{#CR1r8Xgv`t1Gr0bZjs)4xgnTs`fa)?#!hu)jz0%L@z=G}4j`nb zkv1g0i2imS|2|`?+~CmzYH0V&zm&gXU+xd9ro}o?oBuOH@0EyjR}CAu<+zxOl0J)- zl(DAu7e13*AHWM*l{fll$F|Guh@G9C$ox;=c^l4VhT2stx3$8{DxAb!o(*X|e%EC< zGOHf6B~29I3E>%of<0w?4m@0U z?_;c8dYEcybl&$UwQqS2@n|qgD@^OJ^_DfX38kf{4$Ci0f)8AZr+%bMh~DeBj7pna z)8*Heb({=^0CH8n0!T`N8k@C$WInV@wicQjo3n13@%~Cj%04i|K0=QB{UIsT$_%)i zM=9fMf<`k5{Z-QNA!tC$z;YwQfsj=X(76D^e!2T-uX3n?RjYIe^jiUE328uRW0v1v zBjsfhP_u1%vjPDl&pe4P5oLgKSUKmFLVFrkg+*VxQ7G9ku!8j%6xk)KU}f#U0WH5I z9=D1w4ARjhmTh1DaDM#r+u@(+|l(5 z+4m36lXobSf05#{@r6<;KKg;*8slZ$Z;rlkSs;^z(K~l8w>$~2uyWnHcIAAla@{DK zf&a(YmiO)_wB|!CQr=5S^=B+xc5crQsZIEiJAOAcR`Yo|NoH=AiQzZq|H4jj&_Ea zQY^0?YYJ0vrHn2h=WUG@e4I*H>-%VVt@MXfCWxlYtiX~~ZRW=eKgJf=jhge9j0}9F zW0>z%WkxY&as53ZX+G5Cp?fH)0nhEQVH6b=!MuXcYWK%$HjL@B0)1^jX1s9h<5Z}x zxHC?U`kchTJ%URb(f&f_*KyI-)LNpVXDEwR7^45xnPNQrHMH=##p}1zIP62??dSFb zgM(u0z43OJzl;`((o`b;SO}n_dCVZx+gG6cr%2QL2a%`1#q?LrDYy&@KYmc8K^0&a zu7giUNqs}{UaX9NRs!(5Uvrkub@RVFJc$8jrkjwDmJfVEgX9dIyv92-MP@9((r`AL`~`K;f;o~d`jJuQmXF_(Y+-51z;3e;4rXXBjS=z2CA;d(iJk?0eJV4X znCAZ}mWwl0!ROPsb{JfcEKR-DJe%b^4wB-Aa3P_@5!nKY*ZSIqS*q1J&sNM4vh9k= z5FTm7**SDmQZg~g<#M2ynVGFrBve(g8U2o)$Ip;v;^EOod-6og)s^#$T4^L05~G^? z%Xzz1F!o2)zV<3IE-YY5@O`l5FAR`4yPq($gdfuw%BOo*4PRM`gY!&j57z6*khWUA zW6&NP={Hms)m*(e)QYmVMa$W(6&*mbif1b;`jCK8VOp#6EL>JbIL=m?S);1NK~XVA z%x!IbosRBXkw-I?mww?!$Zt zX*P`+a17B@|Kih_s^~4(5}>N#u=x-v^58qr0p0;T6?3jiFpYKTC&y|*YD`U)scMVF z-{wNuBuRS%0sZEEvb*p4O((Je2-*dQE&IzTe?79nfz%usXf;tu+81)T zt0cqi+wYk(SNoehS8zp&A>Ynci6i7sQpo6ow?+Kb(2!ivIsYf~X82>-xWMF72{!b7 zWsMyS*SKM2&Z& zqs+@C|7;>d8XxV~p3EmhMxq0AqMZ48>4h^&yjam00D{FC_C70H%eQl}56mG4DQ;{v zuf$)`q){)Gkst4yO6A0eT3-hrdWMaI|$aqOK`^CdbHiROzVX-N~ClEx^c>>F3#r;F{+O z?rNU+(zPO6=C{(A@1RtfL7+`9Sq-1;1agKS44I9i`hf5o&F;bnNQud3(Y$&m(u~rv zMUeOT#oV7|-rBV*8^}pllUbddih(wu4gCn&?45FW5$W-Ng$f#~_iTgdKZQ!%GX}=^ zK1@{iY9&R*DnP7Jx;4dP|4%L+(`m;m z<>LtnKS>1METeKL5Z7Sv_?tu3cmJ*n>jslyUaT1lPXp;!=TUzm;>Wp;GRG59tM`iR zXh?zl6mOwbHn?ATljPghpMe&>#YnY<8agpMdZqC|jN-4q_&r_9{F-!ltZL75P9h^B zCcrr47FbGB$d|3D{t1NdlHee|b6&dBKKKZ-v@zKi$o{hpU zBOBWhRl5fMGOYK1ujK-&{`&QaGvzD(M{l7$_FmM{4*^df5IC|Va>GsjV)-_wr=TlKq*3G#A500W$7x4Turt^Y=!yyyZL-2hJ@(La$w2zzg-%}sI9yDvt zzuo@jnp?rH`eZmRqPte4uZI5tAbp&tCSpWt#A}u8g?^<%rbVAfQFX{&;q~`S;W-S)(^X%jMmq@F;-~Pyr6AyPi6fbAmxu9L$3? zrbQNspW;75dhZ$Dycj^UqE(9esf*95vUo3^fB9~N-6p@5U=incZAn@G&Hlv1`m?|f zIKcQ;BLIrVS%fWwpV7-<{eyWwok=cLhe4iX%R0#OWJP8ZW?xZCJ7Q0@Ez4eFc0qTE z{%cbjL;o{nH1hc8!!@%AmUkCa>NhvT+2_iJIv2#VlhUHPm#B5O<C315&hI%~jgn|)r z2nW|U8@_3N`!wRP$46pC7eL<+%%iQEB{0zvk}7VN^Dxf(I;DExhyn6xr7ATEy)7%} z9U#z7?;7mgQuAOf&IB)yb#T;CLP#N%gBt^<#$Wl-f?hn()2-FM!}f~{G-WS(vq@QS z?D(jKAd={Pe40d~x|GM>umE`AKmo)8?t* zW$5yoI&XCP%y1Euf4Bfaq92A-EZBvHBvQ-Frq%bZov{vF^+~I4cswdyX=ivYo@*|! zqDzU@b_Ra?m~(EyPjRitJt^%CqwrfZx333p>T26!>bOgKZpH=a<`!P0X~Tt=Q6#k3 zwPbQA`eJrc2FHfTY0-ZLr)89HD2)rJBN&4~-*-VV+BNDHjJwH&0M|KQVVpTtqNF)<{ekLbe}n+2&jLcsr62b(yU-riatESiKUypv__D17R%9^8Jo zVngiHFmNcQ+>B8EOJFF5nPa6rnZk9Ym>(t#5iRp`@bDMrEe_Ab0ayBIe7RW#&~QX? ze2!gAeY*Qo&P+iAO51f^+G&K0ghzEY`61mBpT!emm^eT@cB0w{oaK;u>YrG#9IYLw zt6p|j#eQ~cpi|wDHP%4Ru70iOPSLzadTjFG8Ls$K32H|saouyLYHi+qR^5_qJL+Fy z&Dm@4q1`^2qsdi>N>ut3S!HB4+(f}yaeiNBcF9SOzD1{&jXHCl5_(nqth33Nxo4<3 zoIFesR;FW)#=Mokj0cH{)0j0w2cK8CSz3`?i^|yM+V_jceqP_hpkn=EA13<3N(WhD z+IvN%^7%eqDh`vE^hnYUlU1jR!2XZ%AM(YFO10oqZt=}gQ@L$|+{aC?+YO4)K?W*% zZ=wWMXHY{>b9Zl04p#d=qZS<;fLg2ZLnesw3)%pGJA#*CmtedOF z&S2^zqn!S)=vQ2i&&`dI>>_f#bzP}b-;jkv@o4MWqG>?NPXsp+wufr+HdCf8%SIxb z^}>iGPa?tqbfcaxB@UTnt@>~kI?6D9O+5~=4Qg?jVgZmdf#Q?9fJb7bqPq2{>Q_R< z0ixft&n>h9U9((k&ocp-uWfm=#8tJ!f{OB13>f%iCL17n9Z)}6TU*PLN__cz^)WGf z-8UGjHCkb)t3L*SoJ!VIei!K@`pQat!TO0xawUzk z0{))4842Tzy?Mdlg)h>{F4#yKt5B@k#7Sj#LW6U3Etlz|YCu%8A6YX`-Of+eW6VQ{ zP9out=uc5~N8$_!0a^P>ku%%)oF-DT5=@wUZ-@ZtRlAazE+x_926(F0sfKwlC9jrULyomy)WzKbPzyHiVm)jYgo0gOST zDkvy$dR|p=M$o@{g@?g=iV0YbI}R_<7nVA*a&jHJhqSw#nOx#hPoPW>S^on2pZff) zH%bU9t>vVc+OQHZ6oP2}A<0~D=A(`F$6;OmT5Fj<)v=&3m}^`4S^Hf6(b|qVJb3$q zGbRJX@+1Qc%RG#RhZDU*y}Z0kVm1h>D=X2ewSJ!SlE^f0xdVg0^VfRc1K&qPT>OEo zBg7`1%Klg>U%Cyv^T7vj0z(-k95(uox!SVy6<;(O#HMF7dKH6A>E-CgVDLkl{&!8N zWsj%7e6h$iGcZJQ6iS>tCD@5f-x(y;N0BL0Cy4jIy_Q;PV8;ITbp_E zc>>V_?s}ukBE#N?eH_~v&P^JOmiBFRy*qn}I+pHlkDaqJ}YI|nxvgcHMG&>L8=hR3Vly4#lNMi`El#Y$}O<%@jQGnlvaQ`>g%19jW@t*MITxuTk;>@+bl;NFGagTykL6=I17Yf2c%pC}8(rmgxK)Sn2=`LyM&J9R6f^@fZH+T8G&wGF8 zo_h{|z-HF0nOWcOr>4jCB)03BLOAH)b$X`h4|%YP`#@uG(gYa+I{t5#0RIM1Jcu%M z8*s_2_bLYD0P7X*4lt@zyBui;1qFrSC0KB61H7CV@E5O^0ALQMK#B>fNXYSWD>|~J zw&m^P#rP{6oHd&Zxkyb7FHiyOW>AVBG77LVst7Uexgppe2bpWW0X<8E{Oh+Sx%p0; z&~W+29njkZPjRdk264{{`>69)plhN)M?yZNdCP!)fjooT_^$DY8#Z0U2bEAn#CcDM zAeHF3uYpSqKpWxgd~F(<3Rs^^aJhXSsa0PVP%9`6mVQoj{Ko(6#IgLd%JipXhTQw0 z?*x?R3rS6Yt{x?}G?wqFjDT?lumS$J()=F{0d>|Ji#Af|lX^56ilbqov2J8+~!3{{|j)aQF%M(MN(1yXZR`^*oMd+;Z8q=k1a zpEspHyP=A_&{KML??^<|mnE}(I?lWeZP@Pm`Hl`$Ova;ZU!X_vy4*QY?4M7LmrJnN zG-sO-C7jQ*|K5Z-e}F{U#xS3^N{10I?lDDR1mwep(@a|Qgwqs#xyKQC27enmcIPQU z4ad0g(Q9n#gBEVL)4TEit5CenktF8tQ3{!^sh+zzqFy@d6|}f>zco8ksdGP-s%!Gx zOFTzF$W{7XZQ=C*gMfh8TB!Uv)NrD)G)@iN9?f$O96RE2`XJ(eq?lW;ob2V98a$gD zk#>fZe)?JdAX#`D%?29Szndyl5NDY{V~j)svXUx_z(Et}6!!dOm|32^kZK%|A+%(B z^fP-I49D+`-da8DH4|;D7A3x*t!cinNT$%=_cP+dO=zbV@C`4YQS(=o`r_5*WpyUS z@$+s?(u8^buNqk10c<=EseFZW@W(eNv0rqi*&WwEY;Em_3i7FVG9&MmRinMFaAF}? z%+&&a>+aXnNw^Gu{GXEb`}F-@e9E^)VIQVui^MKs@IyTBm#3>fkrE!fCvIu?1X(!b z_2a=K10fx0zxHL{Aq4vP(}auXcTed115T9d^Q{x_Y@n!1M573qoBOnGEiDcK^uWQ< z%FLLT0y_lAh6fSMZ?UpZ2)8FGWB`Dn9w2&J%mWO|zb`b=^`DL<7iF@2Wdm8=EW)SV zlloLLzRw<(2xPb+WCW~Z=^eDAK2Tz7@2pP#RLOQdB58{`q4jY#~%Sc+}mSd z?6PBlT;h4<2?3}w^SLaidiMvLBXE81hMNO~r(Z6exNUJUF0mR8Xav?zkTdR@q;lU_ zP*6l@KJ?}2e{`=V$>~%mM6Ih}m2>5;WYNI2((J5R+|-p8mVW!buasgsuBDAH04{=h zq)8V>Pn1^;dnN#y`fffVKksFeAf&7e1wC}Oe>;lv((TTZ_NC<&(0$7Ms}p&7IH4>c zeMH5M(vR$NB&fO!=0=H(k~sO@5OJL*@f z+ykPK=lTt*MNxVA@f9JAdqO|@V`n|4-GN=Bdr zgVSej8j5{lL^jOxmZqC+1=b#>drQEfnXex$+6y=|M^BCNTY9sDB4E{w&Xy9s9rqZr zg33AQ&qT{cXjO;qFYbrF!a-9)t=%yGh7i@a9=8l9c(6(R;V+CFeF#U$@gtRecq5BegaZm((d^*AgVXjyFP72MUk=*UB>JlQ3PqyEpW{_www7{U;El1E~Gva@AKb$7W4UWzq z@ktol55qGLFU$W_L6pBYKwQ5MlSVr*DnR(T&?ou1Fyn{R3(1OYUaJdZ+$Yf{n(o%= zQB&R@j3Rf;O22j2J+Ox+s!D~-{7Mpj;9~k#WC6xawziW{PnU(qO0Lb9TTK@{e4U{C|;)8duw~Ct89XOjF^E}rm@Usu|Y{e zSy|KN046<*k`gHwh$UmCqJqYeHKnKjGe6Gce_YM2436$(SWI>aE&7$sH$GVrPQBe( zQ`N{~ylZI{vfb9C;uS?l#}rpI0b5B2<>=-ZP}TlNkn@$9WaOi3zUhaf9>Y3PQ`QWd zA~oc@aG(3B?MUqttrsz?`_F1ubg`@K;;$z@H_`DZPnX^-Z9B|IJW%rv^yNeEHX3|S zJte^`%U&NAv%Udl$MtbhB8-ZSAv)v_iv=Z*i`%&7S0OQ!%Iq5IHE*6UtVjEghZz`g10?nyXm_YomrxxR-X@9C- zDV;mlezgOOqJDSJB@xj3va)~B<)}1K&Jk~0`BrE6nu4Mdc#hW%fm2L-#cw#_=Af0dt%?B1esRmfGLOdRVN$8T1P!tf zHK@js^9WFoD@XPZp0)>&hRs4m+LTILO9e_i}QdS#1YlNH(z z)wNqq#FRB+BBG+7KYylP-_ZR7OVpoqXEPEn%zB$E4NvK?b}DU_YA^3a1e9?+B{j~r z7C3%=oBG^8;LY7|Fctu9G6f(Y;t?&BkShUA%gS;P!f)*e>Ft&8}_Pk!caSt>-#>Rg0Zz4yZdfMr)hun0S48HZ7y|mmtF+2T)h+*>h zUwDO{uvqq=S={>!71jAl=;N7tD(K|hUDn7@$v@X{IOR6!_t1Nkh(lVinrg8|PczE@OGItiNp5o8cV&@V6NDx{XhtlLMQ{zb@)-yZoM~dUn0;p(Xn) zFQOKgqXHi;4EG_JO`JUtos64@(ZqaY0*_bk>yPXrIY zT#zrrrtPgfO+n#gNOPha!_38fgsVGd8{+54_me7QORe^61H_P}ix|k{N5!Q6(nTwR(f(nUFKHz5%wbkr74RHn+)-a7KWbK~_;w z(#uPbgvYM@pz&Cxg%$@uwEnC3;>8@yg=+!<>a884_y|eYWW!>;c2e&5wqX=E29nu4 z&yt_nAx~N_FJ}~_{l%eHYnRP}f{zOnydL~Ax{jQ=nw%|ls1m{F&z~B($Y_YjrQT<0 z&??caJLBA&$kYGcupA`BFl%Hk6+sOI0`ZJ@Mp8E6MXC*k>XR?2M#0ei zY%&>60$AlA@0-viwTkmulK?c7a*=R}3S@P;5L{_s`^Jv0{pfD>FFzM&2+80 z^`^}2QIRs(ePna*cO^x{VmeK>t7WE?hr6#gQ%txEmk_}j<0zd%k|DrZin9#(0Hhy1 z6P*whXywT)sbTNM%wgM?AJodbmDc#%FVS?9 zyLU2^#;HTAJGV z&iPocEe8~1&s}JJY1a2wULUAgsQc63)KCy&^o0*)RV60W;-Hu8s@4f}ylisCj1_)p zE86Z^h{Wz>Tp8%czP3Z&V|+5I^LLeTvtds7M@({#7I^ktpcwR#or(o-ZAg}GK2$6k zTLP-yzl}9c(N!rEg&tBGdx$`0dwSE)+MAR=kk$H6=Mr((Y_7|O1J|t`G>4|C3*I|( z=FqH`3fBO!J!VFs`%?5o6{UrtI)CaMSWj|X0o5QLd( zS6pV7okH-(Q`L)grJi2ZkHt1I=6bU)xGdo^@9MAfeJ_UimvLT~KY`oh^<83b*#J%i zcwFBH(kVH(A8lfq)p|b%t=>fL0P|lX?(W_V@V}|DOV!;IR=1V6JM|$(bZf`SSP>i9 z+AE~~u!K}_LqJKRj^JToILUDO`+Ajzt?JI{vs#AY@XxEjYY5OLtSt;U=GeZUZQV%JJAt7g#;!|s2KEhq5?&9c*RhIxL3ozUZ-^|Qx8#$1 z9%@Yvc`Ghp_;FVGK7h25LI|vI`1m{7Lwd8svvku)L->cMS0%^Jh6?Fl%g;Of6vRL( zMe4XiR2{hh=ptFBLcSc<1cmT}h@?h%gV%ps_Xu?)N9XUtzcF0@uBQ1-%6cK-d;BDq zic(@E{)xUTFx~fi*jrX0&Ygy^527T~mB_Ql)KHd!8OO{FHYD!L(g{|6yL%r+>GG(a`hQE(8GqRrwO3Sns_x67>N^RiV8cQ{b{> ziMkn-a+M^F+cp9)+2$GjjM%_c^pR)Cc@OTYx$FA9utyi0?g7Wz!#x1&v0rPDFWwxH z^!bG1t(*B1r1@t$vFcQaM8?zF{J-mFv(%8< z$GO%Ph>-4;0`g%1#aVYl{d7U5QzdnK5L-TwXJO2VvJGrZK zgBg=fRgEFos+x4n67I))l+3Xd8V^KRO{rWv16rVCFY6omhe1Kt1j&A!kd!fiI9%42 zUb+Q~RfN|V#*P2h1{b#|Y5SCTQ~&H1ATsk6mV3CBscu|7t-ZQ3dDSqQq8+lC!nl@J zt;6N$x8k^#=K@G-NK0D*e)J@$ul3AZ2Tzi;!Q35 zEhFH1obgFL2NhiiQ1k#)E5N_J$~Nsp{g6nWsD8~Tr@z{S%M66wQ;(F%X`v(Cx7z}l zU>R#DJ@&} zHYS+1l}fPjo$6P=|9Q$Nx8x*Gqe7^`FeFZ9OB}?JR|R;AjJR^^crr@qZx1hR9*cx7 z20vXMxhbtrCiEE(p$`U#kt+z%>4D>>FMy=5pigc9Tl$Ifo1gYZP{nC4nK@Zz1)@Zb zFHTnTE~2y58Ra93&#lKe|Hz{awc~ls2byyF?>C;oAM9T|U2br~Dk`%}CSR zdPXemyh60QfBB=kXSOfrZI$QwP+A3Ky0p__8m}sZ{DjN+v_M)kw!N|Zk_ip_!G$y9 znViq7fH@u@Ml#eLmo-=jR(g0#l@*nBTu#Sv_MMhGne8q7j`4hag?T}a45`+Xq3BQMDZ(I z)0(U0um}CV9$Si@-q=v8uD7BEeD*yby>6cbRaTns_&%L>?tiM^_BPoy6);V4=IQz0 zx+%eyAyL3Qp-}*E*xz;ZD7$cv06(u9;cz@D{n|*-#JVDHz;9d?DzxD|!+(PusO-LA z{eSAZygy11H`m*GH?LWPrn8m^>Lp+OM=}8{y?_e0ao5Q1J)q2;e<7Ee3b!dIHo#uQ z))xn)jJ9YY8*t*CpFM5^@&`tLOU|#{AL7%q39(I4eLg?vXZ&#38~q091Uyel zPM3c$*2L#BLikvPpz=y(;bEN2%S38!xly3@!KK*=OUA+cc*M)eb;au>$zP+^Yu*&L za|G>YdLGnXOybtE&)tDp1w(u6+;r>NaBZ8VJLgX=MNy`|QLwytZK}G~)2g{apw5lG zXf_=y2+}oW=ECZh0WxR7FV~BemMEwuGb?3k0>Qk4>B`#zZtDmKrxaNIn!`m&IY(Ls zxe_RhOnwtqs_sges;qBdHRup@K3*bDV$n+YgMUs9LD5lz7hEFE)$K&bNG=|&r($6u zCQ-sL_NCBPAy`7=72P*_U$tq$5~WWJeKyc+bs8>}aK|<;2JcY|`XoK0tt=2CK8pTc z-R&&JTA}0Emc{Kl!mOWTo#c`UHmffa+`{b@2DEsJ9N+Fizf|nH8+vJ}w?sh_s-QrMpII z5uS3O((+bQzKaC!-5>6?If!a993@;*$G<|!I>Te4!yxP|h(?jnUm&pf!ABiORF3Bk ziD~(zgUZ+&IQVeVi48=J2SpOS#f40WuRS;Lj|9uM3VLS6OHnTTehZh*pb>P5;djvU zmV<|qrDY+vH7@ciQp6Q6GVv{zQrR0Sj0H!BpxB1rrb!6SB13GUyBc9Z>&I6j>nqWC z6AdoHU!`80slo@>sD^(**L1udp*&AJEPSt!j-Jm1BIHn=y+ua(^cAk_9W*ii_&91R ze+^*=?w}YP9*#^`n7a8ID^ZfSok9Rz86)?}a({K)CoTS4D!P^h!@^rM8=Ap_!ROsz zJ@w(u<}s$ycXhLBoP}T_h-h*_oFAS_S7DslMJ@qx85T?3o{l%^c+zKN_J%;+gs5K* zXdc|SygIgKL$94*9Wz<=%;Td+E$kSLxP8N+KMF)4*L7sJ^2KAJ4p~2}IZ%Q25DR){ zhZ2=kwqBKARl?Z*A%OK1@VgVpPc6eG)H`Bhy9sur%gEWtiNwhs?LIZsA#-q^fazms z;++@k?Hf_#n;%`fy$Rm#SX1vt9l~xKSSqypQ8Bndc@Wb~odJHVcYT$y^3cp)YRP3g zu@Q0x-_n8CMxP%GK0W9s+;VV69rOrXwzst2H*-$e;6*x;c$i4SgBaGbpU(dXU%P$x z9Y$1zv8Dew+|Tk(^y?>q)WZSw9#Bj(tS$Hu6MsbV$4gJ1oHt~b*;A^Aze>4YiP2{ zN~}azj=@5=g0jxL36>lWNa0Vk{PY49KxR5fe5SI6XOp8iClJzaq3O zjP7ZX^xe641=;6OvM`}uBeoB2kS~lhVfp&3!>bRsfbV{y3v_Q#ZKniF2-7YvtEOM6 z8=JhjwAP4<#$3*+8cUXFhgt1lTOwf10Gnuoo`bmS6&t~iUhLD|T@pzp_d%<#y&_}9 z`W>Oay7x*Miio)DV>1@;AYfAUm>^gI(}Hy~#gt!fo9Z{r9j{sBj2FiJ8or3Y$t4)R zH0zztJU7QBWuciawHpquv8vRGvd=lX5g7V$5@LAFUJ;wgk*s*&a#qg2I_KLC9F8|% zzlgKM!ShW5hRC?0_ShaM=Onen+`>+&x+quYs(niI?VRLhC zQ3gp=0bpJMD;VrsAIwN`A0mOINsEnzH(?60fnN>ot9;p+6WTM8VpvN1o^AR0qD(&w zm5}v;az?sVOmocfT&rrQyGs6`g596U!rV^bgp0LM0z%zk&CJuPx7kJx^pNE7Kf~M{ z?|#bY9?>s^NVX<4A#)HFkKZQ-A1f`*!)~|sRD+%V=*cR54Z=erb|Po|JX5S`@ukO8 z_H=jfD+9|M=tLSviX;qtx`c{9_-=oLTS}iOjbhgapmGQ_fxt(c2#WS+jHGS$FF-@rJ>#?Aj>tje)<6t|{ z@=y)EEU5mX#)7lKnrWlNf)5g6pWd1Iani&(aOyLObta{dO2p-#tIOaKR3FGIxDce~x?X>w#dOP&QGnMbmh9!kfq=bc+$iSsuSaBiCd)cyzw%(SYZcXwf452{ zSzG-~*!E!vIl+Lod+>Fr;eKrGy5%|E#8v8g@td-er)%Nc{+uYX{@*2i1dI=g4Yw5E znA|ngqZ9obpTp-~SauTX49)B6EL<_-5lvTR+Ux7FBfqw|$*Z-P3I}1b0;fdC&@c7a zOF!g@%3-iNw&Ux>Tw2k=ayNUMwr$pjyKsCTkwouUtZYx(W$%=NJ%K`m$Tsap#c(gx z>6V|wKN1B=3)yYa(xcezg>b65&wUnwD{YbvzPVC$8}jD0VcTd2!TQdCJpa_Lb5wba z20zWppWM~`>ZKR_w&1HVzI9SHKx~O9YRp5ZTtB4qB6;Xb2xxKx0-#?&wQ=rMR5vtP z1{lO@m!`2x9~CB(lVmU?0NvvpqS!s!zQ1sg;=>ZY5V&G@WHzMb?b*tl+wjA_c!Kkq z5xrc~my=ZIbI0<$6MSqITRrJOl4I4MQ;47+71rBd7iPzj8NhuhNIX0|4Q&>tyLw?@ zP|TYX_;ii&L$HU)^N+2@oZQiaD^Fq8LSWD}Jl8%KuVrs2bgs?JB{cNMwj_g~ICUll zlsyNIW%Wl6<(wX-2=Rz^VI6F;!YphQ`>a=12X;wH@?)*B&3)Sn0nSSCm!dgvZTO=) z*jB%PjKXnF?vh*#{b35xE_0(ebsuhholMWV@Q~sisyg+qAWq_jl8bd(Dm-Ezg_nO$ zZdvEsXt$GU5w&V+-aK0gxc&8vd{>uwlxdU0I^50c6u+Gc-?r?)uyiRF3>pndS%H{d zTwpD}Rnt`yeuHJJBd!fz>kO?H7E7TaeY@B#xGo>5>Pj#LUx>n7XL;6T#cYeY+N9eC_6AWQTYD;evQpH!AEx#{L4h!v@A`h)kCjv zU#~OHn0T&v&Y@ym^DD5t-LSNSz;)usE<@RDNvv9?N9cF|w#%J8)WbzRR^%=i>T>Yd(?#%y$gQ3A_%L5XY3mdbZk> zY+@k^7ir;dYoi>efp2(+0%g~tyGuLGj<;+jE_aQrBrv1E@)ks96$hCZDt=JqYKUIc zO2$OrEqZFD>$56I2n7%V6+)79` zBC6X}3KAn*1+5@A=%T=#QV3l#)(d)x}|A# z?uW4MhlJREJ7>|9-CgifLhq6=t^%G%BTbk6N(&UYfMW6U@a7a3k>K9u;v2QMI0>5| z@F}q=E<`0y%;*nE9&&Ru3_QnVi(fT*NP?UMw)j6wq>gla$r-@6@^ErEF>NLKz=zD`eY&`=1ZpcbgOl^VP306&UqW|tt z1n}}Jf}w+A@Y5dOLaXUJWqKEX34w|HlGA@KW1Mv5BbF3 zAFviLY{A-VVjk*|@N+o?KC)d`-Ixd}-qPb99=_}U+K!ziA6RoqhJU7X+T{(SfKThkI%2Iv@+| z`_&snJmE`(7q0lYG&X2N{7TiKFSgK za~t_`n-0#m22yPl$DpDV z2z9`IQR{@9F+!nGKQ`$-{-nj#nnIMo41oQ%6(7G(kC>agys`4Y9LaL#dxsbf?|Ftp zmTi}~<2D@+)`im_N*bTXI??tc(SIM8Ttl@Tg8K@o=c-@Y%B;EVQfEQCg_z7N@gd|< zn%us|?$xJdrrkKb>^ilcj>i$33+nE;;l}j-^zHwxd6ZZFw`&yX`8C_f^D96gRyAj* zc!EWIO=yi;P(o$(>=zvkZD3P zrvI*XcnC2vkray9I0GEn_VAF#2@difmbLP7{A;jE9!BPjKR6wI}%;m zmk}s|ch2IP%E0&6!_0IBc48qp3sdLrL*MoQhlZ`*gg&zZqUgn}$g79!nHJhy=Ggys zkHnQeT=gOn9=8^rdS+yz$$5v+ftKenMu&No`32kfZ#n7TT->h(17C$@+z%Y{8!$LIRGXG#%b6b@3njnEIT}#M#V5dcKKjRt)51A6b4e+Y z8<$OPVX-kG;NgE=MOgVU{v?sI`;^I1Ljh!V$%480C}`Nt^8tr25#b?VC8(AMFLyNR z54hfwG37R%0o|;R=fME$Dj>Mk$Z-9KMLW>7CoT91H*iUZbZ;kiV6YwqR^-;yhx;v= zZAGAjd0fJ|D0{=l1PKRcq4$g?_{=OJOG$KQB++8*P+Em-82<3s`2}Ccl0(-MXT*vu zAh9F<h$!&yfiS4-v7@A&L()nQ=0KO6iyaB(cN%1$EI5A1&ewx! z*9uI=U@TVd8H`?PwY%32i~avkO-WoT-F~f)X|5-pYaA_VhX5%$N8^fVHQK#yg4s0_ zCe1=(P|%k18;yZ=u5QC(n9~xVZ4ZYowxQLu?rZKymCZ zc=Zbe7tq5FQ|Rx-#Q4ctUy(q+XxFvA<=q@kspFH2GVxTS_bS_)3K&_gD=aV~qd(yM zbxyh$;A2P9?0)Nd_#xv77Uu+ut?H46g@;8em9VLLFaF)gz(b0yoy5WRm@3x_SVcrfToc#%Dgh@pKR`+hivwCH?xj(JU^3-%hR1D2suD3Hy`K41#J`8iVvrb0uB zK3L2`^4$a33%1^hwCfwoS?5;jE0|3Jz6(&F>eOD8lQK34z5w6`0- z#SIhj`|qC$4=bb|atx5T>9ssW!_|E^-Y-wYU&Ey=BXlub`g2_GR-%owH3N4`h$oB= zzWl6Fgpw@AT+R1O8m6hS0N#nEXX&zl!;qjG?l*iXr++&R4gyyzdt!ugDeMs`(5Yvv z_lH4ZSae(@QWXst&J`GMpqVI@4J1W~@*j(a8$zd891Q+o6HkKpE%Y)89iGhmhYcOIsHVOI9z4gJ<;1pIdWt`$dv2|A+FY6QBMBiPVPa`;wplC?UK6x zrS>c8kpRb*>0vcxh8FHA8#asEcz?mwhpGoR($+=^xd>ewU$4(x0L)-abfc%I2XAK6 zfzEs_1eQ-?EG6tLBe#zEEM)V0c>SG-ny}0b^5ciJq{N%PKAkMa6g3@jQ3|*nY7E&8 z_Mg?cgD98L>B74)Ff^DlmCw{lD}s_-p{hw{zg{Rkhshs}Rr+E!{{$iPmxY7ZnC#C) z-l(6KGyNLsO0NP+HC)KrbEamts17|w${?CC!fEu=3~+I>=esHkJlM)-qOQTon8M%D z;R=Y^>&cjC+%kV)kV-d4*K16>Q}CPJX!uqHV?Lu_Abm4EmUVqmf+Qh3eP1L~W4%~G zzhk9|^*y7v@U~RD@}~KxZQXw@;{6^aCBTml1-=DMzr{H`J{YhT;#h)Bf%0b5QAt$| z=5h!kP`E&_GPx8T{jG(~syun1Y1=h4+f3hYjUv#n|Ln^|o|#O_vNc~h-z}fb<0i9( zN+F{cp9M#h_B3XihCMvRFAL-!@9)-QC!VVILQCS(vf$o9J~h~5VOR+hZs0x#fBy3z z=B$hVjaeV|QD6iv+p!+1Z;YHwGY%#4H0ugvm~^mNVU_hb!{8>-)mt_mjyo>MrPec!f|;XiR=J*vLKTefR0|mi$DOw%8uX86G#gsAqre zz=7kQukFZ}i`ld?0lyi5@4B)LZmSl%?M2*RS&Q_{Ni2_@Tdii`SCG9{&d+bJ#iog( zfGO<(d`{~?r5nYWQwUvMr2wvrEgSfv0>&UD8h_|A+X_laT=#j@8w5y$_E0Apz>qxu zpCp~aF${8c}=4ud7Yv@+?|bJ?M+pQ zj4TKfk{*&8dNjT@+#W|a99~b<3W7Y}e5Im0Li~&HtiC-7++U3e;-c||$p&bq)ARQ# z6ZJS=S7)vc>(>Tg$1lIjUC@0|{C-`2Oj-HzSwkQ-XzMbS$}j2C&CJ$0)^I$ZW$?91VWkftZV2lQay9; z$yv7TZti%f_}SDMAYCkSDekDXw5GB!$EC#Jg}1M#Mm1A6TG~Hk_qx&iM&j5w!Ikp~}#lGtd;M<3m`| ztN4xkqT-j9mRQ~s)-kwU9zdf8(?z)F2rRXK|5T}0#N4YbpgT-+#=p*b#zNw`2_^KO zfxNFIILtXwZ==R%{)n44T0Ycr?|seH^yLNVQfrF(&sywN^&^Qh;zsrpy6vus2hPCH zK`3#|Nc&(p?uk;xx0xUAIoNZShil1Nxi~k6ujZ6Ze^-8phYU2wg^oaPl3?#Pi;tFhpSyc`JqRLM~CYG z^-~!az9h_nAaSJdN3uz4`#ZvawE%RlAYeR5m)$8ih^SWS(`&>RVWnW(uT0gV)9wO! z!14)xA|p&3(bc>x+>qpRt6s=YcEqr3^wCA!Tk66T)*^t7xlV0VFczazNsE}q*;ZGT zsX&&RM0R)5jVLl9W6{&B@aTH*8`sGKPSE>CTZ9oMP~D{DxgTTPzABGjaO=C1l9Z*H zX(O~7AfQ_oxRQK!$=3^8&yU5<6g*2;VpDzYXD8SC-X`c^`=BeQa-qk(U9-D~fqkO( ztbYXXo`hcHM78p8rfQ@%lUWj_g6q<@Bj@?n4A)HEE-zu7Gad+o5Hb*AsS6>e=P?e< zJ0zr8o?p2VfmUe6#*E9s(IFt)Tz|xhVR&SpLjt(AA_2yxk4{b)1A;cX%Sg-uk4bpc z8ISVq@XIs7@+98DK^wtjIgeeDhDESV^&J|7kpK1ZsWId=CNF_#D0u$0 z^o|BvsK=`+T1b`tuiMy+o7eg@(#2yryX&n53)`#Qyn9+%H_9Aifsty$x6i%wpUy74 zZi~f4K$9Ntns}SVFH(gkPM$u1tL8z3yEu5R$VnLUXy$h0fyz8sLp9)$#V1BQp8Db4 z(~G+k$aTzT2g*}i{IT7>rOcI*DTI(Z5H?%Pk0{!|K)DuZ=QwSE1p`)((6s(*1va1e zcl(Mrdwh6Q=`kX`;PUIpR+tX=a{P^i#;QdZ=g7X)q6D&+@Ln-iC5)ft zgxM#=kn+JPrEx+m-_D%6rhi!VmpE%#`$Ub$%(Q_c>+Oa`-$6lR^Q1%!P!knb}(JxyP{*MF_b&kv(N;hVrxwu@P9>veZ-3t20hXidg_$5VSFv8D1-_u>%^5Y)=ls2oE=#E=7 zzvpzQeSs~hG3SE7j`L;mT}8{S?owx!xhH@@2@xC&-8p5MHjuwuZR?E0Vz=S(W$HWu zDyqbFXoB{$O9?l=u6M}6wDw*09NUOwa=hU+B>T z&qLFyStq)|xt4WRS0_u&95fifwYZSfPv7vZ`Q|Lg_vMUQE~iub9avak^T-Rj!mUwD zy~CKa&?mpV)?j^;$h?mbT}BYpU{3(;#p!maC2Xherqup}@zq|eJ8nCo?y`S=>UxtU zy5vA!wo7g>U)FN;&iB1s{;adwHpm?hY0`g;nnvxS1 zVBDq!g?^nkwx1d%=j8Nyy5E zt$dCIkRKN-NyIv)OMPeN;WWRwA*0h)v!DUN-?R)P69=b91rbXXn6r7Be*v@KqMqGR zENi~S`!2gn8YchJd0f@X6!J&Av$gv|%FGjd!*H~!YT@9m0K3+%*HeSm#9W_5xCv_r?>9%+pqc#Po`#<4Lo8$keBwnUb#&Q>z|!LLNZUvc!|?jqnIgJN zN@{AjI2HtCXZG&X>E`9BjLgmEgHHb^=F;!p?(_e);l=#SfqAI=#iajsKrFL}q)7ik zJbJsqMp8j_o3iCf02sUIkC$!UWlxmVRX6Ioc2?`Ge_tjW)>R1)D)3{_1R@h96U05B}#EYxAN^GxL0Bj}b3@~>4TW{mj@ zbU;iH`CYSCVX}mLhm>wG_ZIh<9B~{xq+5QOLTL7%H$Ay!NPvUzEvDVEJ$H?0kXR_c zod79wv_5feJZ*Hrq^!mn9sB*FX>Q27hG7{Gh4!?TGF;;+__HC1oDhh|j`VQ87TIpK zqrwTGj9l8bbD346<8xWeph@L<8};K=lQxlle!U7ZZZwmilS2n}txrwDpvLktI-z)w;WB6+~PE*lD|$Gd{z zE_dy*jwc7jB(ig{Rv-gclLr6+B+y)mQ0Er{1FSSM3R*6CCAe z-Z>cO=FUkrE`&;QAq$uz11w49y;eH;KaFkvnpekt>Q_z$gx$#OM2HYTRAMkp)&o?f zb5!!HkT0Ex3f8_)Q&!ju3$PO}0<}WM&&P5aWW#B>5NXG#`?wg{fnwkf;wI$^Ly6ib4Kl4#y%L3|D$nLsoE^P7*bG z?LVe2e;8CrgsnazX^_FN>L}`K6O|_o3|jwu_?h~X72~{jg4J~9In?=8Le0}r{es(W zcbE&EpZWJ!c%cQweEa^?O^y}ighJp;%3e?7P1aZxc#3w;j}ZgtNSI`Se1C}Vh1ks$ zsZC#VkaIP%!$Srf0dI+WnDgbqTMgegKd*H8yplh*J}#zp=2yx;B{x(L@u{<^D=i3K z`w3pNB(rnw;51`o?#ezM7$a^bbE9jtFfDAQX*<@tLToWu`(_; z^9l*eKo(aF1x3n_yz6m&`#-q|~ct?_}Fpnw-d5!z~HAWFt_gL?}mxy&UeRxE%bwoougH zi_5V92%O+neyY@(DY^WqZDzq}GnWYR=gDi?eg#Md`>;n`ZHzXT9HI*8mA zM=;{sENQz3*!X-WwxWB;ZxaiZ7Zy<$EmbJm%7x|c`X(|r`O9MFrUk)J<>0{5VgNTn zm>tWuu-W5PS2L29jvYvX-115cfRp42M64Z%5|}Fsdn#OId6hx1cL;Q-QY-qeV*<{& z>eUjo6<{u->|F~51YKi`G2Y|HQQmN@ulFGDS2{=_4aC=`Vh>214_4WCm&RIG5 zudX6yQ-G|M^eb|tl{Yw$wdi7Z7L30|NgR%QasSEcUykVeyT?rVV-(fu2ADv06BC--d84)Xj+$qd*7>Ai8N-F?v{GrWstn1Is1#(Q@lc25JQDUL<#e1 zVg`lGsQ*-00z}ejn@v(SHiEG4lE4G@Lx~N^iv%a1v8FiO#(Mcp&T7ELdX-xi0z5RAz1&~DT$!^fMT-kC^pV{9S187L; z7o~+mU(wP-EhU0k#2WDFgm8iwN-+j(1Vp8S(#;SWI1HSgj_b*4N@V*!POsPdbG<*;@Av)ddtI)s{@^(t&&T6FZnxY0e&4D4{qy_O zX%PV%ac<=37*)~e0i)@~bE4R)QYNhD|0*-IGbUeuB6szO%cX7pBrVDGcg^_$LwG@Z zM_xIC96^X35}lwlrpd6c=X}Ir?{LGduvu(X3Erk7)AJJ7Xytecy48AYJpEGYOa*e4 zMK*FnVman^YhZWJfa|#5(lYr80@w2SE?aeg>((TSqYFU7 z)FQYPcztQ#JR*c6F;qDUrHI<&c>0V29b!A&;$sISAjRJYX3WpgArI}S^8=#?*O3*9 zAW9^_DIB}<(PpSG$?w~&lDDgiLMM;9!F;D#=$tQbD8>Mc)ZHmF87aiBC`#O4SrWpE zlM!23$mSoIZk4I(;nAOL@JC6xO!ox6GkYd?e!Y=@Eubcvti&?{iU9ZQn)GaJJhWH9R73+q`t|d2+^j<;U~VRAE9^ zhcqnqN;2R~{&)kW5?H{|RmW0n+Z|UBaZysDqkB|)B_CmCu@_~dB5As5gB=UK!trX? zk4Szw^?S$r$ce-EJ{oUlz$oJW!ToM8x);}#Z`4|81^R4p$A@QCQA^Y!c-3_SaAt+d zfbSd`%BFJNTqrh5_!?sCFdeg><8H7FqJJqw{u9K9pVFc1wvHF<>lpW^5z_Kr_xszA9O3~&+L^I zz!pPO@EOI=+wJEccvp(EFeTj8uT%XM8M?!IrJSw)V?1PA9G0lIa|< zD;R5PXyt-4@Qf5Xpmy$V5pnsPnLF9vQd#PoC7#T>*qc@%0sUYy1SG0C^$@aO4_Tka zSw?+%QZU`%+M5C%0>w3ot7M`T`2}4KxtQBX$<$) zzjN;{c=7;jY+k93CI(fxK$^E{=b{zrIB8Eh#}cMDq^olG!QpcJrL=yBeUQ+e9bI z-#}Yia&5WAA=%|QnK$+dpfBoPkPh-`Ap&Mpr5G=-Ny)aLN+q!Om6H9@+nKDFutl=O zupGj&Z+z+kub)M30a2|%B5J#wP0u`X)Qpzx^|jJpv8wR99g1Rd^BNKMa9s<~9+|ZV z44ldF!~R@31nu^kU^%w4d-Jw)JrMuIr#So|%cO9S!CO$@xdQaGyAf4Z1@dFIP>wgz5-jKXDecTCP9bUOK9j=yjDOhB zfrZp+1#^~hD*~OfQ#IRz-ULDN*35_U9V)xvqy>2c1eUc0>nX4!Z2B^bbz8xuh zb%`^qu+$&l1)@p2j~`f3Z#;xRUQR{rD+IL(J!iITx}ZDLHEGrDPV0|Xw6qrtSqf)H z4p>HabXD4h46tpT{!q-R4;7U9B8~ce-v%}Gf)16{BXD?2qLDCgpbKby(w2)0|3`ES z^b?CTE8$pj3qz$7EvWSBT5cfLs@pbD0mk(XCnPY9UhE1+En>nvom37B48XZ$?I7rj zP8JOxyA;l-XzRax)79~T(RTCh9SY>Hmu($aG{>ISMot7SvIlriB7fq-og=tVeS;}x z6xA*R2y}i}NbRDz4Uzq-L6`Hf<9~QpU!UOgQ?~`>qprb=M~9tQw^IDQeLO!3N=sA3 z5)FrTRUoM{Sp(CXW$iA2phIy{)FfxC{*tE9vz;R;WMrBCB7yO)pO=YLODTiL>317- zrtyUIJfW)YJNqBHZFvobwpacTK8BbxtVJ!~;QFyvQbatw;5Jikq_aI)_GB<}w3UZB z`uY#+aknfHIlZuq9rQD%XDc5T@<%B5mnO9VB(a+@DL0O9U|@s#6tsd7mL|V3M*g?r z-KKM_1F!MNa)g48c@8v*Av$p?O2MU@~7DfS*DwUL`j# zGKrR83l-1xc2OQQ^eNyI!`VE)R4YZ_mjd!PhUfI(_80Qe|0v#LiN8%P40##3K^xYX=Z6SIKMGi@9|=6N$Zo5W8J-fP*qga}dWmsEHF`S16(u z$O0;``g)aqDWDIs2BHzvLR(nbCa1L#o2L%%6fgL3*#dLuhJ(c%hOKg@T8T4xSJT%0 zGoNK(zt~HO{K*=uycSJ29(t(6Yib{)+wMAxAm2s z$2Brq6Sm>O#|Sp9ziYQYXb^z1&4LZ#9qn8Ki1U8O|EN zts#IsD5y}0Bf98y-{g+Bu-qq?O15rWG2aA6jv&Qh9P?klf|VX7Ib!7Tg<#leA?Ww* zn4vR4W_Ss`-B>%$Uy3KE7d%xj!iDAvq~;@u&JI9Yh|a*Z(g>of{99l>RazP^|3Oyi z4JNE}#KN0JTd-oF&(ImkE+aXjcfc%(*FqzjNt4Ui&dFJ={rC?O)Kr@ObB5j^Dt8y zyU1$Dy00F`m*M~iVrUGmCrFq}*l1tXGqlo*-hNJm@K}%g7vaPU&8kBNl#AItAOZ;I zH~@3E{#<^rod3o_tYUEs8Tqjv1t1G{o*$%w^tCjj3vzfb?lavJD|$G7gNzO{$43dw z>ag1yf2p3T@ts6^n>C*wB%eb?f?94f91N`NwqId?^Gwd}LJRCxG5#TgYWt*qG&&sp z;;f|oC0DoYSDV16w^9!xlhb4KMePha(bEY8IQ=|H9yf63roTgu{Exi(R*6GuoOyO{b{PgIeMyr>|&sO-tRIG78~d4a6J^YiAe#k#AmqEVbMW9 zOO~!2A4zm7F(gXYpOIk$$mwcugHcCT&lLX3Qe)K$uc;Fq^nu}-} z*LaZUTlItOY#Z0mXEf$q3`Qcb@5}rU&t-o5r?jx(rh}Q=U=7CfGh!mIAc^6GoJU0n+I%jJuxI#zq7Pd;GkKC5Q98e$%8g zEELmi6f+yyEl*;m27GeDYXpV~sGk)C6Ahd-)fe8o0*rT$dcyp#7uLO4*xuQBqhT0O zWRE<*)wh1N zf#e*XKRoa)3_L3L;;qh3p#SfUe6Q}M_D3(ZpYqG~m21=fMN7eFdpeXOk+Z9+M8d`1 zNWU&7;m$@%-D8`VvN37i+u!e#IZryE4Ps{(E-g-1v5b#$TS4~s$_44?*E!pZt)E#l zfh0N}ky9=nGJl{@y{-)3?-U18{qz*;i$0#CjHf;R+Vui$J@VsbjGdsD_QyYF~N6FM;?~>OAjBG(W1cpE3C`RHSBCh=nVf8MW}Ich=OH=RK1B zyH)Io?_hw?qqv2PK#qJq9o2KlkWs5OfJffxlxSkW5l2BY?D5cOi=OwJw=46+05@Zl zw{%Fv9yvO9{FL8_5MUAb1qB5qf;Hvyzy3uKj4IN8bgd{kbyc~T9(?XGiH|C_Owo-o z;34~U99J(C*{fv4$lr&^X&HLgxQRi4;yA6VZV|T z<y}qv)vwIo7Eo%YytP0|Y230VA?vXOQ0t)U z`*U4mstmd&8lJVN8(+*=a(U$up&ReCflQfE|LfXMh3apd#K%GZK}JtJ7-t@2O@3cg z>K=rw-Yrzb4%|m$uZ_J}ZtAO#GHyG(-FpcPJ|Q*V8Tn6^82?B~^sEjYfrVU~3o8#{ z)11Qwwb8;?kTRclBK}I4*XUoXy_yJY<|aOHNRadA36A8|Sa!Cldt?53X-6*> z9gU1*|J7BF*js)0@Bxd9_}jN{S^C5R-;H{G(qPdvd@BC6X(mf9Zun~Ed14rgAwrB1 z%(pE#CCxtBZ}<%y0$N+Owj8Fq-Q$yXw8bPv{f;Pwd;6D-7$+BpQt6NHgzMT;@8_3; z996E{^On0s8S9KdqUmihIDdJ~CtD;3>MikD!Of@V8FDBh)Ngf}Afh zGogKnba7ym#LGlG*OfMciug@73OpV6$B`GWWmV72Buz(@tw$y~#^@awjy{%WI4{x> z-=(W4@5e07SUD<0+O!sFHd9W+X8bSOnIQNQyeRI_^F9ydaa0F$#GzFiagP`bdDU^y7;`_QBzk}24mN-IyoCGfuyi(Z0cDxPq!^D!xx2p zuW!BeNV#tbnKxXq^mc;Pih#r+FVv?`GSbDd`talMGaDjqMn*go=;oQoTz>1hOdkwV zGS@>CDkmpbduat`giF6EZ0Q(p@dlFSqe5=`ik9`k2Z}%lj<~-d<((u|XIW|ks-ncSG0R({>0BOj zb&u;cUd4rTetX@7_nKY2i5+MfyO^DgM+hVO84LFYmW^m!;S=_4VolARvx5)_6$1ky z=yMS}oGl`NoNOrv+U(FVBZqO2m=n7>dAxVKQgNLtkySUNOOnq_W0R9U2Y`{EsI1{C3}Jy+xzR;HrF#EZZ_fF{mlt1 z85RJfPtzsa@%j;k+*Hwp^*Uf!5AXdU-$0+I-sf%*si4NV-p}u$N3RUU{zvcED>x>u z!n3_%tUk9LPX4PEL5F^uL!>hItJDW{4Tp=`L!U|e#KuWY4Y73>&K1`bM7k~uSBUv5 zo!$9iH9=1*R7%*ycVrc^Q;*Z?Z*}sAK7X(Kjg{QW%2%;1nLV%gkQ~qp!=L<1tC+7F z8t=qv*KThxFlS$ntEfl|y;c|rG(|med$cjiC%Y@}7{_dXo$@w?Ua1*}&!(HWU5xV( zZ}{qDEPpYm^1WPs{*2VW^XgLe_wFnem@7`i8h$Q%1-fH?Nn+M2+yy6PWs@8}mGWT)uU9`e_)fTIiWFpnhhLstLnpol0F;HB)yRYh z1>Sp<$Ncla!%klAwzf~RrrqfOXS`Iw8U+P)dEHI#IN9Y+agigX#BaQ~ASKa#8B~0t zF$i1hKOgk!k?oIz&W}ZX$l$X6f{Zsa@InqWX48+uHfvdT@yJ++;af+G0n0V#^LH~b zpLyEd?t^E&=q%)i*4gERY^7;Rx5ZP%yf8y}ss((L0kjl;8z6Nh>#p(S32&&v(C(o6Hm2)$BO+ zvMuh$UAQj=%gsj@#_Av}f5hH@x;sp`I})aO{m^UVZj{2oRxDCAG64=@@V7jsF=8T}Q1H^Hu2zB^6h?u`BCwTsvp_(}N( z7`fet=oVBo;$9oE<$W3Mq(#mA=WJ{0@P*bFJ=9hEdq*uJ>mOdv1T}S5lKRL`2#*yw zBOny*^`uQwu7#$lK5T!=tZqZ?;+KucAUHU)^H;lZT!vR!R)Folvk=uJ!g}B1YKbJe zdT#&)3TeBhri?6|gH*5gA;66}%kL+CuKThgM~4uSDFCB{tKSCBH+nVp)sN;;d39?e zyGo|GHwS-uYnAVXQiCsN|K+tr`7eHgq?hO2_~luCk)Y_{WuTpgsaz4$dnOmESkVej`t#w~ z{|Hb2uMpOa*Uql+%}s2oj>l2erCgxImgiY}F3i)(()rwNmNWqcaXdJcCoaOb?6?)A z4gK>1wMm=prZ z2f5rxqNDtOl~sRuD64&IFk&oV0Ch|z&;F~6XiMRWM|Ge$>iz3j3Y^YZH8XN?C*`~* z3Bn(8+IBB+kA!jL^P8fdKgYXebvt5kXHT&=nvQrqw_UpjP|WP?(##1o4-vVDoEw$p zLsPwOadDzVdKxi`FYO(h7e@r`U)M=Q@Wn=2X1vvN7LvCHURh^s!1(mV4QJA#l9Fzq zl<;)amL84%TL1})?V1Vm9zMt)dgC;?zqKmy+SbsKv(3bAuFkdkQkh<&!x1O(0sWi! zkF!u-{v2q@UB7^UIISlE(_h8t{XmKde1;FeTU`b5U zlp&#^v+JIJgM`0fo4`2gs|i(6icY z;nP9o?>A4@s7eir36ogl=UyR|mUMlhm29vi>aR@R;FU4oBr3b+v@tA6GIk&YF~2=C?LJ zhlE7vjp{?3tGM-dTWxV&)0^*7tG7OrojxOHgHA>&0*MS7gj8|mMiu1_ic#ua7+rr= zhE7IdA03~BNE#jo5r_DP>$NyII9*FiEMUmH>w|7vdg#cE07xXs+_xWt>-4hUsj`$l zJ{Y_C(qp!ERYpJZ9tSbMDb#`)vNSgCt}m77ijvmOP@iz&%J-i7R6x`x_$38zrc@uN zF{z;F8DVrBqRjA z(i2C^5P8zFUKy?mA8GVAUNrPyjd(KnO|PT*TJAhNo;T+a_sBf!{Udl|1a0iIg!tr- zZ8z@Pcv1Hhc8I^EC}YH=2!3PJGU|WEZeA%S(h6Z&nR~WlviLEhhg!9ELaHXkWE$Zl zu6_9(zxu7u&?%qk)*Fi6v9{GT2N(uCl&`g8?k%gdZf6j-YJpZ5S8@P|*In;3C-m)K z5Y)X-N$+b%$jSBJ-{-UCZ+c)Z%|YByCHic$5sc#i7(1H_6One;{3xm{oC!TvL9n~p zxNCJ0&@|>tHRMZ|F}x$JchyXNVWQzLI+Tn=TM>Hh&_x1B@a3X~+E#On9O&*NDVW6~%Fr4OGcattJ zpff9Lz~K)i*qYf~ECR2U>3>DC+jNAaWoGt1gl}dI9|K7$HA6a7Kh@x`)`%&c_%`L| zFK??F&x`6F!{n*RXZZetPHIJRKU z;0VdL%@^UZ0N1QmVeIHr62PHRMp?qZ03B48kOex9H4HF7!sFP0k?xHQJ^k(ajoU1Z ztX-8-H(HiRgNx*FXXcumUe9NaO3kb|#?Fqt6!?`SJ}C)$Z&#DIlpM05XuH%Fl$MhN^Vu8)#)cGSbEG6n zY>!fxECLhQrEfs}mE$ZF?l`B%0-RIJ?cwd!8}jqnQC)*Se*8eVPH^3S(zl7%j~9BB z`vZ=5pLs5gje}UCF6ULrM4hXl)WbEH&&`ZZ6ugJcSr-^}TFWMHrMrK)vdIH+BNakV z74s!L3j_9+fTbv91S}@2hp(Elr463A^KI^d&Yrew>2?XCB*3Fx<8X4^x%psKb>mXb zt{ZxvQ{&a0CU3jH#MR)4KvXKg$OS>b z40H$=?Oho5Cs^TS=)p(Hh>pQ#8qlg}D=7WEEe-|J;-c~K@n^qoZOS8hKjx(4+1S{W z`F@*FW$0ZQvHI2Y;45kvpv@N-&4Vo0@4>`e#<4&yG@|EZO~JL=tSw3`Q6dlDO1iJ5 z3G43eE(HAqy$`pIB!ZSadFX!i9{55H>#-m5c~MDVq$k& z0`^z{cQEdza}U`yJgif0Ue8Z~S7Yk)RKb7O6qK28B+=i9&0hd8>-*Ls8Ncmb%zagn z?C;)+y$LFidH_+l_-%+9zK!{$>R3J#4q3u=Zr}KUzx5e2^1RRoz#7fT&b!W{_a%VByt{`{&0ZXSeN!+ABETWdy5~cSPr%j2vd{1cz#a z=a{5@@~s(FEE(&lC*G;>#oBZ_2_dk`(TV&IwU@)MT>g`TP5hBm-57F&BqXqzMnMj&Ac*Bg(1w60Ljjs-;BGdCA?ur;TvCPd&d<-Q zxVd4-ySU4Ih2;Kz4W7;Tt+ zlf)DVdyhq~fzOx&`kRQ9a6IZkukq;RdFzx%5m*joi;#it1RQam?`r|c1pKGd6mE{+ z3b;<1h>gO(iwS2?0XE!FfMrYckds=Yq-;FbN3dPc4jqx=1Oy`B`?ars^tl0+ zeTn;8HrK(br<+&ck_+JEes50=-_?J6VH19Vn^S%W#&KJ&t&6h*h5KAWKEkF=aAzd< z#{S`qO+H&p9wnrovpVb|IBOyR@UE)xq_|I8S>PB)fi@_pSQ5rB7Xo*6RU%-~uE%_y zpdARGeO6^E--74`-#b|<;`%!l{?psvpTnYqXVm>T8O2=4ZAl$|3lu5COd*ku!&%sQ zPeJ4la#jdYL^jKJ0D=KgRQKVL5s|JMja32_T9Vt{iQ=CU$Hh*P%t6>e z>O5vAs71{@fx~3Kem$dozwi|kHV5&L=FIZ1YoPM`ytJMw*YxpS5FVlTJUw{t*{oS zj|@#=7c!vtC02)Ew@w`}!B36o46VcdEC!5-dyYPnMt}R3dq|&=0vyzhnRW*rtESY9KJ-@a}w2405&CaVBglmy9{G?lVN9~p|6#`C1lS>X{DPH`0P`SnSIRn(oD1#5k5fBFa~R$6_K>doi?`GvFo=e%zlDDKQe-wqST^MSoSWk@p0g7>A>4YMKlut9EmxR z6?Kd=;)WD;#T-YUE**-W1`2W?Y?H_@lY1cqkjv%9vLrw)T|?*TMqlkb z@#JMug%hjsr%t~XyBFlUVkG0K5^4?v^ZYT$SjRqq5hM0i0Wl=9RPsAEAfQly()>+e zW@#yJ)+OEhMzdG8%RqbT{4H8k9Y*%sg)f`z;FR>xMNR6#D%7(1VT>8+_RPDng^KFJ zM+Py3G^X|pTvXSaVPs^^#Ul03 zPzdrpnK(>z(R1!f+zX#S=4FTKe={F@BLim~v2gjo{fOLp5Ta(OV=YU)e0QTHc+vzY!cR|&#t5*B zVL49_yv+GR(*VGbuSmf67#WSncVNL?^hJ=fda*X%>!Pd4F@`@OwH2gb`SHoEaF{S6 z3(7?^O9T$22Q+I;;Vg-p_83$im{CRnIeV6+Vm^YG8#a6P50Ikex%MTX`Qf@LN_V!_ zQCgc@9~toghR$avrtEnubP9k7J)0N5rP&LqfjifxxKXJ#cL z%7S?WjN$VK0HXG|%^$-N%yb(9{U1WWX=ET*{49j$HyHHQFZnSnn!z}!zmN>v{FHq4 z4Ag>l1SVw0{t-bxmpzs!(CCufhx;bEHVf%1obRg~e{Wod+jTLHHhL|O{wV8O{)y)Y zf-;mSLZ4_=Uxk{W=U-0&KB^(q*;|55r9 zoxCbXSg?ZFY6fwVT<^LTJ?5TT@crxgfDEs-xaPwTXlKX>b}FIsPO(xnUW9^=H16z| z(*@VCZM8zTsq)9F!*|y5{$^~}ms%iF0#SQq3c?l*+X%?^`Z!FY9kO#d zWn!b$6@{CuGEZo(_d?-$I**vAK09<08IhqA0#1rscT~SUBMccEtNl0OV)K`#x8t>H zsaxCG0q#|BdZFlGbRurE-ReJDfOyG#2REa(+-C_7Fp-@(vmFNC3OQNN-pwGl!hQW& z)F(~B-fqFbw1cLExhJCV=H+qEujrQ)T)sGTNgibOvlGQ|$fWWzXI zJ~6ROSTyB$=8PkO9nluRC*72%F$XM?%U=&dan2;*pT6Dxf|V_`QVBsNVM z=N9ClXuq(jXLRAVtrvx?dyqdQurkEQW~zS8kGq8Z(vQ6H6&0B(geVdTm-skOGxTPv05!^|LxtfjJst9+}%&fs+`g%-rQz&`CLL`8&Xz= zE26*%c+zJwenWTl7v?uJe~F!4tu%eoW7>?wrP?2PE=z|H#u)}tEIS)pgkvTqq*)uz zCIb;#Z>=t%IFj!NZ#28Xdm=79+k+1$_-I8a%oWy~4d`laaHTG)$WE3UJo`B@gsZ4-&&&*_prqIiGp8b&B0#)W93yHIO znBe+Tboh;>ZhNXyug}!BATi9TGwi}t@y15~$1+=M3rje0NJDY=*+XiJ?%0i2q|jT> zq`H2YeEt43mKFdjFGCIGl@<9QTX{&yoK8$(<~oko^n2^F5jo?V8f|HUw#G~VhpZJg z%~ck=zmU`3+bg+VLMu1F*cL%FgUUWD1-*Ys*Xq#anAWG@E} z5VLjjN6(8$i(IMuIU??13lULWyCr4{rNE1R`6_{58Ef$g2&dO_@Mw}pT4JL>Y1$d0 zncZxr#Wj`IW_*?LV0)?}U+yu$ajJ_Xw)`s_UZoYc*U@1!cSY!RERalJD0dbhj=&-uB9V$ZU& zGQk(}d36ai0{!%Alp3&ol|qAJZSQ@wl2e9w9k9C8eC2P{p@>HYmX=Ilf}9e_i6~(5 ziAX5+=jX3m5gm5L9-3~GZjKJh0ePPD4N7GO#hKzGFC{vN`=#{SjtxE4Ofu+wZ$9+) zwjFfY$CYpj?vCmNQ{CQ=?F-TYu2({Yxn}NTM zggH}{Wpl?H`i`Ki zgbjK(S`o|^d(Zp&X|jfr7h%Ymb+I-L%WBimxCR`foCXoIGqeyBt$PtK^)G+2uMIlb znCUqPXDU$ykQXB5WN_OM3QYcE2L!w5_4Q&+ z#Q(Z63j7)nQIqX=-vTkq6kr&?V4HiYvD!5SAM#JWvw2}gh43tIZr&||i0AWZ;| zgg0zm0=C1Q1o+f7D26~{hM^r}U5pfbx6%f2x*Nae^y1lMmDg->5=%LaI6cOmhx;TP z{$#;xqkC&yY3`xjcz@9zv}QO?nx?N4w*6HAFiKju9D4VLzXcDm;@z{g40~Q_F)7IgVulosD%e%3>PpyC;8| z`aA+uO+{NA`x%FMYYe-PQ1T))-lhcrGDdqs)6;Kz#r)@0um&aatepXaH@OBpn>q=S zfdaE{6k)u|Sj$No>gwIV&{5Y9m;D0vYxIYYVm32>2|L^0?7mK1v`jp2F^v~ZNAez0 zSR4%=PY)*d=qSlCVw`l^Kqd?rpb61Cp?^0840|26-zJG29hEKyL{nNPilu!57;i{v zR~?aI$Fc)#k ziyEC<8**f_7GRc|w{FD}Q1RsO^N%&s{blUb>$`VG(|K)wpMx~ZZn9M0Y4>>x2lSZ3 z`{$~f6MIh%qalX%%KUl9;@{`%A$M795w|tg7{R;eMlTnLAfjXe5|px6zd|re$wBON zM?kke#tgb25u=JtA7m{6pG=Wt-#D}vkSZK=19lH|hUi}7#vo2|yp)PC^1nxr4twM> zT3FS^RiYk!!x6wm`%91Za{uQ+*%T`g&!+(b`95r}(TfZu39(gHt%O@2WmM;elxFYN zd1B)ePYk@(3^e3U1(3;CQDIPx`_H8rvD+4p!31|RNyhvrlW!BnX66QM0}%tAYLh-bJ*^$qV`65WRjSj?y-p5vYdP8Up_~s?x+^~aJaX5Sudu3C*brd> zj`xgd4}u|E(57H-;|8AD-@4$4?1@+e#8%DNCkQ1aW%snun^WLg?epo)&T?yo=Ve{9nEl_WJDzeBo9)V#GAw?@6f?B{Aoz#RptIFLf9C7gucHCVzS~XNOIwM6fBt!OG>RivZhvv*Wv{GE zQfN@bsYl+#p_xv3XHHH3t{@xRD~_lL%Scx5VnTklLv8(Xv2>n7F(ue511KLyA#Xd; z%UH#^tCP|8{Hu)J*$ki2AJVPMK`bn6xj|#HFLD=i_i}@r^CZ250Vzrey@?4*xBr08 zZrsSWx(S450YOT-s;Wpr4!|V6B}C=P_?5R9QP?;EUv?h8;O?1MRycH(zvOs}HndC97DEMa|-PCk8Xma`c&Dw`_mG(~o9b07UKf~9CvxM>i zH?#cH>)d*$nHT?2b$&($2RX}+Uz;PjhTdH)mmS_KQRcIXVL~?_e3Un)OJmhYwXYvc z5g=~r?*P<3zf;$Vgf?eK4j(+JP?M2M5yFMGftZ@1_+?X+r}C>?QnU4e=b3U(SdWS7 z66=a^Z?$?1IVYx)=UIl?9^%??1%6HSR5f+>;9={6^XY z_-{4Y6tJl#W~?9SA-dETAWJTsb^-jGFF7k~4Rt?Za%bz?B zZvWZO$S=IoLw57-dy+11Ezz&;gwBiSi@`vQVHu1GD0@%`A2Q1keb|e zf$te$17Q)IXJhr_x83bCwr(e|NCghnPQ2s#w`;cjP1;?BomMVgkFkE_Y%R67+In$D z?2fhSFyrfx)wBYCKqe(j_Xf}P1T=_W8{o(JwMr!MHH%-sk4=#1 zt2OR<c9h z!_blVcOxWkk=6zW1190Alw!!^uU>C`(Zrs|sg=g$1TgrYfQK-=@SE9VfA`kyCbc?fV3;tAA2LcxNtVeb)eEw-?Keq@{>d5!h3BTao&=S_IF z(F@RpT(=RY@ZB|rZC`L>bV z0x;dXNnhC(Jul=?QTWx+j#z3eMWuEdD9Ldw4 z6{M4a7FWCP$A&;`G?}{k_3KnN^^iRubpsklIJ9Regvl=9Ng_bvf;8M+<;eonnrdU( zFPjtr!K1MK#fV8##;zMFdkzR$WJ@lkGL6QE)qWkO8~86CK+**mK>!;tNPbPE`QU|B zDpWH9^Yd024L}Jajb{!cQ~7z~?mSX=ia-%UuwG5uV<;#f2Vwdvo{A|OkqGUP7>BLPzka#K?)hGbsh`D{|b6ClqP;i+h31d9|R6W z3uuuh$H0TDZvwSM8PUSCCz4U@JUr?Epn4i6AU5h5>!ND5KnXnR58a~rg3VFd?U++( z$WF}!dUv@*FTV k$ioEwS)d(9iILtY*jUS4Xedwq)Ky{`y13G;)+wEX`Gt$Y!m9 zeE)@GOy1Qnct+cKo%NxofYK07U>Z#ukf8B+YIi*jD`genj!nZ_C+M!|rx86QU8XUL z6GN$wWgt@r08!UM*hZx^^m-2EoI5`&?cp(ga)%bu1qMwn&+U4c?gTB(Jq&p%9bRKM zjKS=S0r{$}dGEK8&bf=|N_L5KXqK_o-z|{D8BsY1G#J?+{84HA)OmTTV;z3sHw(s; z-M0|46~?w8!rQgbC<+7EDs?f~nC0IQn5;V69A<&P3}CY|HlO%tVEQY_WL9Z0(!rA- zt$R?Dw%hpm2sK-%>AOx4xl3`yGpF4fkB3< zni^kZBx6=hJ8V9fjr!Ac;0D-_miOr%ianh^Xsch(8Ie8vRBfk1FYcE4uRE${Yrwfc zy#5f$O3G-i%L`XNj_xbRn1Bd9n5_o>>K~wNDxd%QuH#f2z@e4?j}d115uUg`K$kTI zx?-rxP)3=ApMJY04Kl0xviUEqoEj;bKyz~ssJi&W?hR$~X)B@=0(F1av_!XGx+DTi zr|&5{l*1Zn08>`4o&?5a63M7ERvqb+LlGS@g{#}o&Pz^*g&p<%=9ggP5vYiLzPsFM zQ*K;Y-k_)?){^h-Eeh4(}uvp6b&*C>Gx^Kp>}!31|*rG`-`gqx5 z_fR)d)CaWUygE)w!UkNrd?5|g#7%4%VZ?TcjR}EC7H0#thVS+N{bzTIG%5U;WL7l~ zOMqtQ!vJge+msK^;GO3Q@Sa_z5_NlTv>SK-pZ31OtEw&PmpGK7lqd)iQqrY_q=D$MfV6ZAi1Oy9vOi`=z8vn%E49}P_g|hm2;BlguxX^2P% zIgUBhNac5X{Co)q)%Yg%*8nYde;ugsD?gv_8ywe7mF>S*3$kodRZ}5l6!6}6ai4z~ z;$l`OzLHz966KFeuuyUh_ZK5t3^PH`<#p`OS4LtHGPANOfl6~4j`?6L40{|H8)Duj zd$f{zUNn`&K8C+Zy-D=Xnq)>B;*@19_o+)ggL8~pD+GYE7Em#zLzWplm`1iqq`?>8 z6|ep~b%5P7LM|7x_L&=K-bU$tnSuxQ+Fxy{B*RmZGbQvgZqoFh|0YS zf5#E{d_JW=@Vd?4$9;K$DE+)Fyo9{85rih$u%LD$q&1G?ADT=@|9KD$yM#+4rRu}r zdIu!)(rYI{%l61Z%07oM?-my~}hsmPOZ^|=HyxT77l8lCb07pjO6IPIB@j--q|W> z5~H8{+dt(K)X)D}Q8eVcurK>}7-_ZbWj%iF&85384lIyFdlTv@@rmT8zj@ON_5W)K z-w217shgL)pa=O$6ri07t7BDo?f{LNuZ(`sDzVkK9?XBKl5|-cbWObA%Z@@UWf&}f zQflhVaMR_XqPGS(i{mvlOJh|dk~y;Ke?<~xP8hIId<21dKtty;sy8W*g2v?9F-i{e zhZ1MkNwAfPf??$B{0Qi&sGy+n3X8&LbU3UZ?w~jj(bPEg;k}C|0$(oo zD}Lkty-NKKiP}@7jH~x%X%d9(lNm`;F5QW3Xl%^Pnf~kIes`~>$vq)Y>^lC{PcI1c zQG`1cUjrflp3l;I0LuByegfU|7=3XKc=@hiui-ySY>||2Kj1x5M8PY(a!TR_haUT- zqv`=RNQVa0Hj88pb#P>xL-US8PzbS5<+t*gH-*?Txq=AJhe^rO%dI@qZ3j6&55%rmXOzNq(XLIg0iUNVZIaM2FV|VVg?f z&Q1N<3$?AN{A-nvJ#Bzcoo3cDA(v>Didp3zj6(ZL7ukna0t?=k-~7v%Zk{-8TB z{Boy1$56Q5D@QXe_>nq@yUR_4tebt|ZVsBB|9g?QQPeu_%ylMR59y|Q3c@NDIMo)a zxmZ2PQ_=ZW6+L(UT$(Sj7MPA12qlr~Qa?ZOjVyl7i;IxlS3rCK^|rl{zHn;n#tEm$ z{h69r7Qc#oA-mBFEXuJL+nX?f1X&eDa7vrUtDfE9h4CJ4%=Z0jyiq8JM?H|$Sirss z-v2&m7Lh4d4AhT##;voP^zi?avdzrQ`aq*y=knadkvyN%|6*n6qbE=Lx2Y%QnsYnU zYi^OB8Sa^YIW3*H*IOdLq6fk+Nn?3H)gbp|Ph`2hQE7i_F9;WTA{;^V_ z1w!1t&o#d+-W^`F<3WbxCCWFCYMv}%mByeKXVW%W$tKFs`eK{0@7w9e(SF8+@VxK@=}iT3XwTRVpcz-}AW=g$O>gck zv3f-?4QcThRc5T9(S^dujV?F&UtyCzU3v-rVkk_XBpiy>gcn`goBb8#@N_ssGtC?{ zj=>|r7?16R7Lb23z=3(GQS!-t1Tr}T|1mjd)ex}-ln`|6dikkeOQxo#WN24#57g{o>^^#tajtf}>m%Oj=9!7GhH@V0y!(o(6JIZp}ahCMP2^S#Z8RDg_^K zahk17rC@kG@*E~1=S4*43t7LZ07bO5j<7#cB7pFV5nQnqc7`j&+4@CtA0y~_0fH0S z7QMJ4g*-5QXLL7FWaC{`*z5xx@^h-^YaKDn`4_VuW39-#E#{O?gW06dqUlAQo7&%X z&zQLt9Stdrd`RIxCfzcj7S_>gFje_fNauh!zwqve5WCiEg;;OPD|g6{kJA0uM?-5L zk}*v~Y=}LQjaBA7*jf3)2?3O%L5UBK1X(F}TmECEsH4=9#n~ajj0PnSwWL>>y2ZA+ z&s@^|*RrAHy+u~)5|(-crpUAj5&mD3u0wssYrQ_MvUc{4PqfAhn}M*)&rSL$gN&*? zT7cP{MiN``?N8U=IJMpuzrQl-G}}%MIZB*yN?S;RGO~<}eZ6;G%UdRy(Z}F6ujlt> zbP(xT++1rUG2ZZCqIwra+THbA3veUz{NQubxoJ)c`vsd|)h1^^1-M0$GOoW>TkyfF z{Pz96L7Br#2BS&9n%2Ij?TJUjCY(pv&W0Gjy~Cx3^9D`oR?SSG}M?s-4zuTTYVzvBw}(xDcD+(EGq@f1?4mo>2XF;U>Rrz z`}Z`UTO7Afk60U&+Hr%PG4?5w{JZhAlkxEbdB)=wzl81%HzTDN_eUfUL|YAhaGVN8 zG{ZcL#oj;eSwq#8DB-CCGR!w#E|$-pKj#p2o{d3HerOh+5eb3yDxLS1CWc7XZJ^rw z$0_9*!_NzE=lr=3*a+la8FlVvQVl9_)QURVj55gIN^F~k50I84&1;_n|u)>^H8Xp>km7` z6UszZv^KuAm7af9^guo>Dzx-E#p>fwC7+j=FJ$$`+^>8-%Hqh8=eL$&NhP~Y61i{p z>!t$+_3qIMsvZ)1X7yz81#a=NWaEl^s&G2gab1 zh1YD(w9Vah?AtWy$HN}*zxkD~DvI#{3r?LU_RsvUSY6UfL>0=`wg!CNc5Jd&G4|zQ zw+~-Ctf8;VjE$qv;=S)tiJrEScs#dt!`)bNu6lZV>*lYqWK?!9!304A4r?2yPPW1s z;&@xLSK*;Nti|CiWoGBYS_C?KLi}ojwszO2I8vOQKK1mi-g=)ZNUwYN(cEoozNfd) zLYajp(c9&Hk`kv>P$L*Ma!~js!e5-bB;Ms3NSYOr`1>EkFEOP+`NowUdB%Q_Q6Ld7 zP*`Yrs7=uG1Y=;%9q=X*$8r?VsUZj8J%TJ^;>%etv&uH3N~YQR^W1H@M&s;hZzS2qgly9k@e8cohrQPOUZ=P4n9oaj0R~<6RXlaGA zYwO3or5~r%lh{&Vgy^ybpx2Hu-fQ4-b@4u)q!y4R)%B}{rKLR*0jbj)wLwe?z3nwX zTAmNbRiiOD2~xh*wuAYM-Vd%Ues}BhnrtB2){!#XV5Z7fwFc5w>)j1@7TWwq0*W5M zw93P3x?3amtBJQ3!=qE;R->RmnfH$pgV=`T z>%P2kg%Se7=?N4UvW^9xk-8jJHu<)uvEYPaR-l<@Z(qb1=cZ?zOU0#@p1iP=bd7X& zmCx##XQI_5HSJm7_iqBW($op}z z781@j+TgIx5aP?#U|u+Gk)gQK)X#sMcq&f^2XT0Y#81Mp#-aVekqGa-lOqN-@`sp` z^KS#0nX>&Gy8~sr7tzW7V_-Rt*74!=Vl%`L+%jIcqD6%n3SYgEXo-3m zc0Fv1x#X^;2WcXS_?q8-xcM<7WaW$uEb)?5_Jlot-1ja0*Q6T_H4V*TZ)6s?V&~0b zd@p-e&MxuIu*!ptwt$-Tl6k1Pbd_feqG_6(tK8qbC5ot*@=f{lwS|{cHI<8#%Hv$w z`$O{pLo&Y|?SPjza$k3(ak$+y;S+Q_#d$njVsgGd_h@1Dw+5CLgXSRWG`B&9a49Li zTex$b(H&Z;(9-fwMb3DNnJ;*`li#aRA(rW5xgrTala@M4Gbz2+K6atI0{bblV;ER* zDGa>uk@7Zc*mk6SH~-7=bsmsX`}34=+1@=GS`!S7%hS~QVmFce5ax_-OEkLsd4rpv zzQ01P`vBz9&!7nASnmd{s4Mw{td=MBXY7-|S=UiyL|+!)o3WdmUp;zkFP_R%)aJszas@-QY+5u)**EqEnrHO&nL{O#CF~2A@wR76*t*+~ssU zO+1DQS;!PVaNU+Tm9eT?)bv2Q|8W-!wl0d!bBTWsVi=Owv!yaXp_5wV*g;3dXWf`0 z&6dq$?)m`ph54fTP^k>B# z^Jvt3w#zm3iE1v0^3;-kli?5$Pi?R}`sY+R>U8 z85JmuNjGqZW0?)4m%5nbvdZHrOxc5?Y$yq)+hnCvlP+gbIpxP^w?3(_dm0B^eUA!! zn>rFUI*K>4a8UleT%-vvPm@2bgNT9WSRBse1kRtEbvsRLfLkpbO*y>>a_Adtf&t#J zE^;1O!yaD#+94>R>W1U06~b1NebK+L5G^gWSM!aH;*75^JiD0@vZrJwPj~gToD6!s z-P58Cg`3Z;Uuk+phnwc=b1lerUE`M>@l1J$Z3qeG(Nyf6Y&p{xOxM>S2ksvFh*T0= z8nnXJ;q+2QcNGmvY!g0^2)2|{v$@7kzw5Hh;i1xa`M{8ejq8wI@}ObQkUW{yWHUq0uc`9#3i2gl|51tEA;N)_h4N_Ca|{yNl1v1`fc<-Ac+%H{s?KqK;^UhHN9a9vlcF4?TKj`x=GapmL3l`F*Jny<2?Y0>jgCe#nCeXFz}a=4jq+t*#d zMPA!l2NeSfyG!NeUlqXbLDen2;HXY?2#?2v(0M38|1{d$y1NHhNN!A&^1J%iIZd2mU+r$z zwkjO;YBj1dmw}DBWd@(p=~_t^eNx}R6X`i_c__|jo(G+@cv=f^vc=M10icG@xeU++ zXh!7NVnJr+w)8q!RWptQtIv~T-I2Tesm9lrTG%d;nVFeNy48&W4yTa_W)`mTd#br# zR9q{TFWkT3R@e95S^0#WxmHN2HFip}yS}B2)-1N_?~E>2xY)!f3mOAAI9xiCCQp(8xcCGuA{R80 z{IMjMAnsmRxz(%uzv|D|8`{}=nlnhLjue5w;W4FnZG?iMTlyrO_=UX6jYkWm%^?RQ zlDnyvK30cNi@s|GAW<7i!_-zt4C~W7;La+xe?B;46baR6t316o^J>-iWKg*&ShmMA zLLr&)rV5a*HW++56xmQ4vqCWz4Rt|iJCj`O1d@t zb`n+teQss4+y1I-#5z%?b(GdYa>leIx3m0*EH_Y)f#_YgwgObU0_l>NXp5{wWeMPd=0D#NP8_Y z9D^%gDlSpqo*|iC?L_CkP&X!Rw>8Astp=#WD>SyffFE5H}2s8yZT z@W-{qEZe7S$3>+M9<2G1^r-O{Q@7nwO)O@8=9awaT+Eto(DixfeHDkDHfF8o(PF9E zEfp6|9E~f4!LRPiv;7>uKFmoReh*~WF-W!S=2j+hNMm}kZF@I?$%Oh4DtskkmFE`~ ztJ3RHv=rXO)+#Dt;5|@snrUVEU0{BF`E2|HzsBt72WK0LT2bqvhRxBv@NrO5-5Ls< zsDiSz|M1Ju)h`-Npjy0g=-h<0b8ANUM@3)ONn5QqFiVQfw+CcUh_=JXXW{`C1(666 zSeA}kk`P0anOI=tGlV^M@24KF9*d)2&8;L%2^7bn5!cu zQ7M{yw`UyLs1l{*#>APX8m`}onjfvHOp3I~WU?xV)W)mtOj9I!Txt|eLl=!DQSYhCJkgRFO86GBv7h~2!8?)htlgw!LVfqy=V20G{HW9 zLj{GxQ)BFDT^$wwEv`T)mS%*AfW8VF;M}qq-kKKkiMXzjpAP0{Ya5z6&tDb!(PuK8uXR4`E-F@`&|tANwFP$s8!;C&&;KOrj7|#k zmd^5OiDG#s;>av7dOup$YGh2U+2!x&pGux$uU1(w)BR+mhs>IX=_5sS{UkkioSY0Lek}rU&RdJ>3~3I@7#2%TkSs`JYP5Avwzr6hZJIz66bg|GFF>(#3*9n!>O| zfED$;jxAeY;yW%rv4(Q?L&I7lCog^(E-;O&s>V9FWSVVE5-N4XH@SxnxEDqjklcRO zq?N~snSgCzJ2FC)(8;rW-I4P+{tff>ouG!K5Q^x$QI$|alH-pXy0ok&W=jFbx+G`m+kIbOZHsBAPxdExk_h_AW3 ztql$YE@vASgiq=^8eRKr9n}~Z2^a5_d;jCigU*E_w}%vvFA=V@nr`Dt*V;`5@c!D5 z`b>kFh!DHmzJgONz&}E7zif8DHY=~1BgOaEhf~}JpGHjE&%8Rq$188uTyW8O(Wod5 za6MX{!M`1~B*Y_f(CB}baI|TBuxZ8SILpRlGTF_o5I@(3N>JC!Z*Ith+|~WwOiU}H zv56yP&!ghvnipOnW`A?0>73X@f;x+bBDqu{L=*A0wpV^a0es7-Im5LpB1!UinK)U< zxm2FP@CI3&WH{g4&QE{+evgLAMJKLNapELA-X&6IQrZA)Os+`P9RN-_q>VZC&PgHS zmi|V6!ht47jMUGq=-Ec^lVS$no6BgfR#inejfmOPazEpjLXn*8b_0cV|Fwr@&7`J^ zZC2{v5A^l~{l#>WJr>wqi^PNIDTv$v#m5+Go!x`1=zkJzgLiNG)rc&g9J%g*;)@c) z_&u(22zj_!b#7%Pbls&OWs~ttIIAP={pA%kI^)&Kp4JFl9tY{*sThXr#IjBO#bo;E z@@NxFM}=nK*fM1qd7he|$39}nWZJG7Rh$4Mu5FUJmJ)vln|uvgY^%&l*uajUG zVJ8f@(q@(8>65*Wkru1k8NNdfasZv6cm)Pdc)*1+9hqTY@bQ!SEgT_@Mbk6*Gn^uh zXAp+W3%og*I@B9;UA)F#>kmEwhIdZl$y6qpt{;Eb-YZ1n-2yCBW1j@SLCyX9{wYuh zFD3~a>}ie*kGWm_@+4qcXD}8;XCQg!%O-5ukHX_#Owim(x3rtdX9|G-(&vx2n&ggF z*=g7fN7UJ9l=)xeWd2Cm#2P@_RS4qg>MUYMk4O#;Zq-`E5#oO|bhEuCqrrz0>)1m- zrZ+X~sQ$g-_Rp~|cs@|u9!-KV(COu+1g;Oz0U}CjY6IEP0}@he!resv!=yxr*aME; ztm7;I*;})0#Vbf7gp$OTTkr=p4=%g=u+Hw;U)i_E$F3m47o-|&nMHIf5c?NCv8faz zP5FjO?Nes5{Mw{J!1U%9LgmBCp8*J^X4PRQQg|Wf&2ni!g)*qqh(b^Ya**G$+59eD_4N5b zG_(2<hCd)`OF8|oLDL=tm!ZPVR4(aCm z&r+N%adVc9l5LTG{YS+#nUD2fTs9i=wKm=w=E?J{O+w7v%^*Js6>+YDC~Q~CdQQUQ zhLQUWCBlK+$k4drP^b*5m}%hN7v=~fv*A7A?TMn-kx5XCz+`d~CO1e<)rXKMLt83y z@L?nBif1HyNffq|TM6KDQ%Vr9xS_u!e^2`~_VdoJTAx|KIJ5lZ6W@nV@j6OM9>`2E zg+KiCE>O67s5~0Kb`MQ+<{i*`B3D{gT6%bPRqTceuboT3Y&*#34pv|9T!Z}g1zlKr z4AVzra{Bj}Kz7e;RB^Av%uB|=z(7ewMa2n3Kd(kF$rf35eCo&m20R+<8}FQl&&gke zV&q0uW+GHp*#_^GfNkR?BO|lfh%jSMOG`@!0addEQ0MXXLk;4ii3CryhG zT2Jr?qd*+DAu9!|N+f75E<6$mwSodhz^aqh6{8Qv z&aaO{E3;7RtBR!cR=Pf-BW;5`yIvD+gSRT~eK>W9ly+~r?+w;1_dxALdEgB%Yf|CG z7kK2Ea~VJN#l}>J?H7iZoWD-JgKPrMs<90_A3L$-VB-QH+PYL>g3WZv;O+DFn$_iEFcBm63ZP{x$y zdP!%&A6EVtE3ROBb=>$Kw1gQ8Nqlkf(j{%xVnz|(Tjepy@@>JTb-Gz2ZsZN&PSii0 z*r*u%KzJ+p{@BB=M9%fRM$)@sFACOwK$Xq8{<<`r@!+XR|6OUM%w*F47jO9vjcD7? z<2$PxM#(ah8Q!Dwm}e{vz^^g+EGb1~4s%C()*Cs~tk;zSM;BfXuWD9ohh*n_2mxE zV3Q>_WpKD&TyX}My3zZg50~EE7e{!VpAI#I^}t3&24xiBmJZ+Rp!3aFv{gTwC?-Ekss{ zNWX+^2OS7D|IWidP2)xvK(mtf%U83qvsRNqzs2qS9`}p}qv(cpKZa(!@RO)aS)}$y zyegT`nj~}k=r*z-F1w(QU5o*O$MCHRmM$#jto1JlC8G*rje)WR0l=qNPFuWBSQ#{=a>=tn(Mj#t%vxZL(@n&JeTDm#>K^zlGU1?p5AzdQ-^CLkVJBR zT`!b&P6P|<#G<0?l^fbic=Dv12uV((h`L}F5K+1dZ4aZtrLNrnPP_kJDrD;xCFBdc90opMzsaI`Acxl9C)P)^{ThhxIUjstT1RbWt zA1S=BW*5sS8Vr2CZq6!%?@wr8Jwz<+S-6Dx=M?(n+|)?rYCb$BoBWzSvRHLHJT zn22Pd_=QJA4+s_X&!a6ce#d@7J{}5cZ6!eD?#bHN*i0X?@+ks8(*l}-*~Yv2Hm;{P zWNZJvH3F1C2_vIq6WLOP8{NYP;6rPP=QsPb^EzxK4p>dNt;Z>1SF&f~oU=)nE_*@8 zgW`i9spweUDCaKVPt>(IOOL$+9KI7W`3k zg~wokc+$;59tU6<2z0J+i0i%Oo+ly-;=)$Tmnr!cSU9=VrEjleTVG#t*%liTJ@WjM zaDpus<4G|3N3zrs=`;&@Yv?5Y6ut1`*|YQhk%0G;_I7l&M*CeNbO^9`axeAFM^dz} zP%u*txT^FJ0M~UmAURUHnS27pY5qd;0%zwh+L(@3rKi#gu%`&W&05VU)}BR)U<&2L zx{sR?=1bAkKh**0TlQ5*X)oWsqOQPTPUH|kS*S`MLuuuEvenWeU4h{>fGt5Tn?)PU z=o2`JX#KQv^AGqzj`rlM0w>6M2LO{qCzYhKF_RX;wT}(GA!&V7Bt>9RRH(&F+6y6G zrhHG|oje~OTaZmJ%hxC>-}>;~THiWbz1DS>oWU=RuQ7p#DtCXa35sTGdU}xAqS}|b zFdrXngCaDa0OrKAn4*f=Ka~|dfLffIS!Kx`sS@8?&6+!3cun>i?&R{cVd&oGDFr*D z9BtV=wvcy!A+qfYU2^*hhKS&J;y7Hz_%=T;JPLMHdp!C?;^EGdyb|B!7(ou!JkGcF zqvcL}-=3d4A5wjh@h$ta!uTF=k|;nH#+uiTURCbL{pn@wwJl`NH_!r=7Clj zgq`;+aujNHP+`gFoZ$}_P3k^homP^m9JwnG?NHR`$4nzH`j%;T+pBXJRhH38`HbcT zAa5T+?2?At*O(G7l}pilrMP?KtfYg5=P_Tv1^38^TyQ(#r@`cXPhF+y!xBbo(a>eZ z2yJgIxssIGdv~`Wk~2! zxhH3}attmFy}uJ@3kMA3oNvA8glywMdT_P3w|8ajTH8m^S3X;Y7)_M?z~dROvC*+J zJX}CA$AA#%f*}H_0)x;D^nA=0XlzLhwk8nd%Y(%4!nH@IG)Do~U_!h9*tr`g1ih0| zSYm@+l+MY8t#n31ToikEd2ceLxJdI87Zqr58gT+y71l7!BXP^I~863LjXR z35-D$O_201Z+{`I9*Uh~1w9p1fPV)E+);wn3YX6Fa$nVln|}x);KqqN-|D z+P;C%Sz$Xa5JziQL*o+t{=0y(KzYRu!kH*|Uui2mhIA0W6+(+T7RY4C#E;kE{W30h zVV8)AeUhF3sqf5_3wK<6h3eRbk3{eiu$8vD&Gca!t`bs7U~I&4hjMB<4`y{xwT^_r zg^e7n9m~J&#_9m4nS9YYs`z$~3}3=W*>kQf$r2Sv8dG%TpceWHeOk>ms^WlFXq^qJ zZKqpr0?3$i5v@H5T2k%9&mMAsB2EJk$x)LPWM$@xT~h;|1*l!v`d%Bg$^8vpu6cgK z9ZBRduFeL-)XQW@0Fd0INUT-L>S z7qPL#6aaHUy(*RYPkXl4U1Xp$4zm`tT z>04yJm0I9l%uL!IwsJ?-+!~@h^`)5Ggrib`jL#$Bz2#hQP(fC3YyV>0D8AraOM>vr z&(T-Yzg<w>q2VRNo44q|NC=&LEemlnruKZc*nDiML@qk0+~(O@m^@%l)iF3uPszVWaU z8haIbY}-&@cF8yZeag3Xz}hn>FF;cOUNCaSfK=ULIO??puHm!%YZ<_>zT*n&1@DzW ze_KzorlzJ+<*5rA(r1NiFM!-V3Y-`0U;QX48{A^J^ac8Cf{M`1$J(TQ8(`o0=aQtT zBVua6R&e1rY%uDg?o0%f+H;Lu{&6pNzH;xT_L|4!%&yKQ;})UfCv#@ny4iX!Cr{>= z`zi9WP7V4V0@I5$Kbiq8)D?9JZqlW0@AzS;9&5sh+%@Q0{+ZQzi)r% zK^7po)ylmos*3TmGNU2a5g?WKziT*n46guu>O$JQw(6~A-Yxok<_|1V#fqtaigvqB zZi3fA<^>ks)!U^~chqrW8IA%CWgz+IgF+F}Ukl@8_aDg1zA9|VY*E~EKBMr<>b(`y zn6>Dj8QLWkJE4)ad;8ezTv12B2UmSzaza&@tCvG7*_Vj)9{D9uWcQ23vDl6ts_tg) zPLU^{KCN@nx1e~iRsFTcaIiV0v}Iv|(z(D6k`n#=_ZJosO+Thq5`q?~0|Z;VN}_o` ze+afGf)uaA^v5zluno0LrC=rC;m?2nNJnGg;ffLb_Q(9~pu@xc@h>mkQF5}1Dyy3i6O26y zM8EwQ{*jRSkqW4GJo?MokNi*g@mqyJP}jpj9ST Xb~;zL)gmDQ7W`7YrY4&!gAVvV2qKTR diff --git a/contrib/mesos/docs/architecture.svg b/contrib/mesos/docs/architecture.svg deleted file mode 100644 index 2d74a42cd5d..00000000000 --- a/contrib/mesos/docs/architecture.svg +++ /dev/null @@ -1 +0,0 @@ -controller-managerapi-serverschedulerkubelet-executorkube-proxymesos-slavedockeriptablesmesos-masterzookeeperetcdkubelet-managedpodslave-managedexecutorcontaineradminSLAVEHOSTkubectllifecyclemanagementcomm/datatransferkubernetescomponentmesoscomponentcoreserviceuserspec'dcontainerkubelet-managedcontainer,viaDockerestablishespodnetwork,ipcnamespacespausewebappcachelogger \ No newline at end of file diff --git a/contrib/mesos/docs/discovery.md b/contrib/mesos/docs/discovery.md deleted file mode 100644 index fb9e2d66a63..00000000000 --- a/contrib/mesos/docs/discovery.md +++ /dev/null @@ -1,91 +0,0 @@ -# Discovery - -## DNS - -### kube-dns - -[**kube-dns**](https://github.com/kubernetes/kubernetes/blob/release-1.1/docs/admin/dns.md) is a Kubernetes add-on that works out of the box with Kubernetes-Mesos. -For details on usage see the implementation in the `cluster/mesos/docker` source tree. -kube-dns provides records both for services and pods. - -### mesos-dns - -**NOTE:** There is still no support for publishing Kubernetes *services* in mesos-dns. - -**mesos-dns** communicates with the leading Mesos master to build a DNS record set that reflects the tasks running in a Mesos cluster as documented here: http://mesosphere.github.io/mesos-dns/docs/naming.html. -As of Kubernetes-Mesos [release v0.7.2](https://github.com/mesosphere/kubernetes/releases/tag/v0.7.2-v1.1.5) there is experimental support in the scheduler to populate a task's *discovery-info* field in order to generate alternative/more friendly record names in mesos-dns, for *pods* only. - -To enable this feature, set `--mesos-generate-task-discovery=true` when launching the scheduler. - -The following discovery-info fields may be set using labels (without a namespace prefix) or else `k8s.mesosphere.io/discovery-XXX` annotations: - -* `visibility`: may be `framework`, `external`, or `cluster` (defaults to `cluster`) -* `environment` -* `location` -* `name` (this alters record set generation in *mesos-dns*) -* `version` - -In the case where both a label as well as an annotation are supplied the value of the annotation is observed. -The interpretation of value of the `name` label (and `discovery-name` annotation) is a special case: the generated Mesos `discovery-info.name` value will be `${name}.${pod-namespace}.pod`; all other discovery-info values are passed through without modification. - -#### Example 1: Use a `name` label on a pod template -```yaml -apiVersion: v1 -kind: ReplicationController -metadata: - name: frontend -spec: - replicas: 3 - template: - metadata: - labels: - app: guestbook - tier: frontend - name: custom-name - spec: - containers: - - name: php-redis - image: gcr.io/google_samples/gb-frontend:v3 - resources: - requests: - cpu: 100m - memory: 100Mi - env: - - name: GET_HOSTS_FROM - value: dns - ports: - - containerPort: 80 -``` - -#### Example 2: Use a `discovery-name` annotation on a pod template -```yaml -apiVersion: v1 -kind: ReplicationController -metadata: - name: frontend -spec: - replicas: 3 - template: - metadata: - labels: - app: guestbook - tier: frontend - annotations: - k8s.mesosphere.io/discovery-name: custom-name - spec: - containers: - - name: php-redis - image: gcr.io/google_samples/gb-frontend:v3 - resources: - requests: - cpu: 100m - memory: 100Mi - env: - - name: GET_HOSTS_FROM - value: dns - ports: - - containerPort: 80 -``` - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/mesos/docs/discovery.md?pixel)]() diff --git a/contrib/mesos/docs/ha.md b/contrib/mesos/docs/ha.md deleted file mode 100644 index 9ddbce3a8da..00000000000 --- a/contrib/mesos/docs/ha.md +++ /dev/null @@ -1,526 +0,0 @@ -# High Availability - -Kubernetes on Mesos will eventually support two HA modes: - -* [Hot-standby](#hot-standby) (*work-in-progress*) -* [Cold-standby](#cold-standby) - -Hot-standby mode is currently still work-in-progress as controller manager is not -yet HA-aware (the work is being tracked [here][2]). Nevertheless, we will -describe how hot-standby mode is intended to work. It is recommended to use -cold-standby mode for HA for the time being until this work is done. In hot-standby -mode all master components (apiserver, controller manager, and scheduler) -actively run on every master node. Additional logic is added to the controller -manager and scheduler to coordinate their access to the etcd backend to deal -with concurrency issues when modifying cluster state. As apiserver does not -modify cluster state, multiple of these can run concurrently without -coordination. When the leader (i.e., the node whose scheduler is active) -crashes, other master nodes will detect the failure after some time and then -elect a new leader. - -In cold-standby mode, similar to hot-standby mode apiserver will actively run -on every master node. However, only one scheduler and controller manager will -run at any instance in time. This is coordinated by a small external program -called `podmaster` that uses etcd to perform leadership selection, and only on -the leader node will the `podmaster` start the scheduler and controller -manager. Cold-standby mode is how Kubernetes supports HA, and more information -can be found [here][1]. - -## Hot-standby - -### Scheduler - -The implementation of the scheduler HA feature includes: - -- Checkpointing by default (`--checkpoint`) -- Large failover-timeout by default (`--failover-timeout`) -- Hot-failover w/ multiple scheduler instances (`--ha`) -- Best effort task reconciliation on failover - -#### Multiple Instances - -Multiple scheduler instances may be run to support a warm-standby scenario in which one scheduler fails and another takes over immediately. -But at any moment in time only one scheduler is actually registered with the leading Mesos master. -Scheduler leader election is implemented using etcd so it is important to have an HA etcd configuration established for reliable scheduler HA. - -It is currently recommended that no more than 2 scheduler instances be running at the same time. -Running more than 2 schedulers at once may work but has not been extensively tested. -YMMV. - -#### Failover - -Scheduler failover may be triggered by either the following events: - -- loss of leadership when running in HA mode (`--ha`). -- the leading scheduler process receives a USR1 signal. - -It is currently possible signal failover to a single, non-HA scheduler process. -In this case, if there are problems launching a replacement scheduler process then the cluster may be without a scheduler until another is manually started. - -#### How To - -##### Command Line Arguments - -- `--ha` is required to enable scheduler HA and multi-scheduler leader election. -- `--km-path` or else (`--executor-path` and `--proxy-path`) should reference non-local-file URI's and must be identical across schedulers. - -If you have HDFS installed on your slaves then you can specify HDFS URI locations for the binaries: - -```shell -$ hdfs dfs -put -f bin/km hdfs:///km -$ ./bin/km scheduler ... --mesos-master=zk://zk1:2181,zk2:2181/mesos --ha --km-path=hdfs:///km -``` - -**IMPORTANT:** some command line parameters specified for the scheduler process are passed to the Kubelet-executor and so are subject to compatibility tests: - -- a Mesos master will not recognize differently configured executors as being compatible, and so... -- a scheduler will refuse to accept any offer for slave resources if there are incompatible executors running on the slave. - -Within the scheduler, compatibility is largely determined by comparing executor configuration hashes: - a hash is calculated from a subset of the executor-related command line parameters provided to the scheduler process. -The command line parameters that affect the hash calculation are listed below. - -- `--allow-privileged` -- `--api-servers` -- `--auth-path` -- `--cluster-*` -- `--executor-*` -- `--kubelet-*` -- `--km-path` -- `--mesos-cgroup-prefix` -- `--mesos-launch-grace-period` -- `--minion-*` -- `--profiling` -- `--proxy-*` -- `--static-pods-config` - -## Cold-standby - -Setting up Kubernetes on Mesos in cold-standby mode is similar to Kubernetes in -standalone mode described in [Kubernetes HA][1]. However, special attention is -needed when setting up K8sm scheduler so that when the currently active -scheduler crashes/dies, a new one can be instantiated and take over the work. -More precisely, the new scheduler needs to be compatible with the executors -that were started previously by the dead scheduler. - -### Environment Variables - -We will set up K8sm master on 2 nodes in HA mode. The same steps can be -extended to set up more master nodes to deal with more concurrent failures. We -will define a few environment variables first to describe the testbed -environment. - -``` -MESOS_IP=192.168.0.1 -MESOS_PORT=5050 - -ETCD_IP=192.168.0.2 -ETCD_PORT=4001 - -K8S_1_IP=192.168.0.3 -K8S_2_IP=192.168.0.4 -K8S_APISERVER_PORT=8080 -K8S_SCHEDULER_PORT=10251 - -NGINX_IP=192.168.0.5 -NGINX_APISERVER_PORT=80 -NGINX_SCHEDULER_PORT=81 -``` - -Other than the 2 K8sm master nodes (`192.168.0.3` and `192.168.0.4`), we also -define a Mesos master at `192.168.0.1`, an etcd server at `192.168.0.2`, and an -Nginx server that load balances between the 2 K8sm master nodes. - -### K8sm Container Image - -We use podmaster to coordinate leadership selection amongst K8sm masters. -However, podmaster needs to run in a container (preferably in a pod), and on -the leader node, its podmaster will instantiate scheduler and controller -manager also in their separate pods. The podmaster image is pre-built and can -be obtained from `gcr.io/google_containers/podmaster`. An official image that -contains the `km` binary to start apiserver, scheduler, and controller -manager is not yet available. But it can be built fairly easily. - -```shell -$ cat <Dockerfile -FROM ubuntu -MAINTAINER Hai Huang -RUN mkdir -p /opt/kubernetes -COPY kubernetes/_output/dockerized/bin/linux/amd64/ /opt/kubernetes -ENTRYPOINT ["/opt/kubernetes/km"] -EOF -$ cat <build.sh -#!/bin/bash -K8SM_IMAGE_NAME=haih/k8sm -git clone https://github.com/mesosphere/kubernetes -cd kubernetes -git checkout release-v0.7-v1.1 -KUBERNETES_CONTRIB=mesos build/run.sh make -cd .. -sudo docker build -t $K8SM_IMAGE_NAME --no-cache . -EOF -$ chmod 755 build.sh -$ ./build.sh -``` - -Make sure Docker engine is running locally as we will compile Kubernetes using -a Docker image. One can also change the image name and which Kubernetes release -to compile by modifying the script. After the script has finished running, -there should be a local Docker image called `haih/k8sm` (use `docker images` to -check). - -Optionally, we can also push the image to Docker Hub (i.e., `docker push -$K8SM_IMAGE_NAME`) so we do not have to compile the image on every K8sm master -node. - -**IMPORTANT:** Mesosphere team is currently maintaining the stable K8sm release in -a separate [fork][3]. At the time of this writing, the latest stable release is -`release-v0.7-v1.1`. - - -### Configure ETCD - -We assume there's an etcd server on `$ETCD_IP`. Ideally this should be a -cluster of etcd servers running in HA mode backed up by redundant persistent -storage. For testing purposes, on the etcd server one can spin up an etcd -instance in a Docker container. - -```shell -$ docker run -d --hostname $(uname -n) --name etcd \ - -p ${ETCD_PORT}:${ETCD_PORT} \ - quay.io/coreos/etcd:v2.0.12 \ - --listen-client-urls http://0.0.0.0:${ETCD_PORT} \ - --advertise-client-urls http://${ETCD_IP}:${ETCD_PORT} -``` - -### Configure Podmaster - -Since we plan to run all K8sm components and podmaster in pods, we can use -`kubelet` to bootstrap these pods by specifying a manifests directory. - -```shell -$ mkdir -p /etc/kubernetes/manifests/ -$ mkdir -p /srv/kubernetes/manifests/ -``` - -Once the kubelet has started, it will check the manifests directory periodically -to see if it needs to start or stop pods. Pods can be started by putting their -specification yaml files into the manifests directory, and subsequently they -can be stopped by removing these yaml files. - -```shell -$ cat < /etc/kubernetes/manifests/podmaster.yaml -apiVersion: v1 -kind: Pod -metadata: - name: kube-podmaster - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: scheduler-elector - image: gcr.io/google_containers/podmaster:1.1 - command: - - /podmaster - - --etcd-servers=http://${ETCD_IP}:${ETCD_PORT} - - --key=scheduler - - --whoami=${MY_IP} - - --source-file=/src/manifests/scheduler.yaml - - --dest-file=/dst/manifests/scheduler.yaml - volumeMounts: - - mountPath: /src/manifests - name: manifest-src - readOnly: true - - mountPath: /dst/manifests - name: manifest-dst - - name: controller-manager-elector - image: gcr.io/google_containers/podmaster:1.1 - command: - - /podmaster - - --etcd-servers=http://${ETCD_IP}:${ETCD_PORT} - - --key=controller - - --whoami=${MY_IP} - - --source-file=/src/manifests/controller-mgr.yaml - - --dest-file=/dst/manifests/controller-mgr.yaml - terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /src/manifests - name: manifest-src - readOnly: true - - mountPath: /dst/manifests - name: manifest-dst - volumes: - - hostPath: - path: /srv/kubernetes/manifests - name: manifest-src - - hostPath: - path: /etc/kubernetes/manifests - name: manifest-dst -EOF -``` - -One must change `$MY_IP` to either `$K8S_1_IP` or `K8S_2_IP` depending -on which master node you are currently setting up the podmaster. Podmasters -will compete with each other for leadership, and the winner will copy scheduler -and controller manager's pod specification yaml files from -`/srv/kubernetes/manifests/` to `/etc/kubernetes/manifests/`. When the kubelet -detects these new yaml files, it will start the corresponding pods. - -### Configure Scheduler - -The scheduler pod specification will be put into `/srv/kubernetes/manifests/`. - -```shell -$ cat < /srv/kubernetes/manifests/scheduler.yaml -apiVersion: v1 -kind: Pod -metadata: - name: kube-scheduler - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-scheduler - image: haih/k8sm:latest - imagePullPolicy: IfNotPresent - command: - - /opt/kubernetes/km - - scheduler - - --address=${MY_IP} - - --advertised-address=${NGINX_IP}:${NGINX_SCHEDULER_PORT} - - --mesos-master=${MESOS_IP}:${MESOS_PORT} - - --etcd-servers=http://${ETCD_IP}:${ETCD_PORT} - - --api-servers=${NGINX_IP}:${NGINX_APISERVER_PORT} - - --v=10 -EOF -``` - -Again, one must change `$MY_IP` to either `$K8S_1_IP` or `K8S_2_IP` depending -on which master node is currently being working on. Even though we have not set -up Nginx yet, we can still specify `--api-servers` and `--advertised-address` -using Nginx's address and ports (make sure Nginx is already running before -turning on the scheduler). Having `--api-servers` point to Nginx allows -executors to maintain connectivity to one of the apiservers even when one or -more apiservers is down as Nginx can automatically re-route requests to a -working apiserver. - -It is critically important to point `--advertised-address` to Nginx so all the -schedulers would be assigned the same executor ID. Otherwise, if we assign -`--advertised-address=${K8S_1_IP}` on the first K8s master and -`--advertised-address=${K8S_2_IP}` on the second K8s master, they would -generate different executor IDs. During a fail-over, the new scheduler would -not be able to use the executor started by the failed scheduler. If so, one -could get this error message in the scheduler log: - -> Declining incompatible offer... - -### Configure Controller Manager - -The controller manager pod specification will also be put into `/srv/kubernetes/manifests/`. - -```shell -$ cat < /srv/kubernetes/manifests/controller-mgr.yaml -apiVersion: v1 -kind: Pod -metadata: - name: kube-controller-manager - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-controller-manager - image: haih/k8sm:latest - imagePullPolicy: IfNotPresent - command: - - /opt/kubernetes/km - - controller-manager - - --master=http://${NGINX_IP}:${NGINX_APISERVER_PORT} - - --cloud-provider=mesos - - --cloud-config=/etc/kubernetes/mesos-cloud.conf - volumeMounts: - - mountPath: /etc/kubernetes - name: kubernetes-config - readOnly: true - volumes: - - hostPath: - path: /etc/kubernetes - name: kubernetes-config -EOF -``` - -Controller manager also needs a mesos configuration file as one of its -parameters, and this configuration file is written to -`/etc/kubernetes/mesos-cloud.conf`. - -```shell -$ cat </etc/kubernetes/mesos-cloud.conf -[mesos-cloud] - mesos-master = ${MESOS_IP}:${MESOS_PORT} -EOF -``` - -### Configure Apiserver - -Apiserver runs on every master node, so its specification file is put into -`/etc/kubernetes/manifests/`. - -```shell -cat < /etc/kubernetes/manifests/apiserver.yaml -apiVersion: v1 -kind: Pod -metadata: - name: kube-apiserver - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-apiserver - image: haih/k8sm:latest - imagePullPolicy: IfNotPresent - command: - - /opt/kubernetes/km - - apiserver - - --insecure-bind-address=0.0.0.0 - - --etcd-servers=http://${ETCD_IP}:${ETCD_PORT} - - --allow-privileged=true - - --service-cluster-ip-range=10.10.10.0/24 - - --insecure-port=${K8S_APISERVER_PORT} - - --cloud-provider=mesos - - --cloud-config=/etc/kubernetes/mesos-cloud.conf - - --advertise-address=${MY_IP} - ports: - - containerPort: ${K8S_APISERVER_PORT} - hostPort: ${K8S_APISERVER_PORT} - name: local - volumeMounts: - - mountPath: /etc/kubernetes - name: kubernetes-config - readOnly: true - volumes: - - hostPath: - path: /etc/kubernetes - name: kubernetes-config -EOF -``` - -Again, one must change `$MY_IP` to either `$K8S_1_IP` or `K8S_2_IP` -depending on which master node is currently being working on. - -To summarize our current setup: we have apiserver and podmaster's pod -specification files put into `/etc/kubernetes/manifests/` so they run on every -master node. Scheduler and controller manager's pod specification files are -put into `/srv/kubernetes/manifests/`, and they will be copied into -`/etc/kubernetes/manifests/` by their podmaster if and only if their podmaster was -elected the leader. - -### Configure Nginx - -Nginx needs to be configured to load balance for both the apiserver and scheduler. -For testing purpose, one can start Nginx in a Docker container. - -```shell -cat <nginx.conf -events { - worker_connections 4096; ## Default: 1024 -} - -http { - upstream apiservers { - server ${K8S_1_IP}:${K8S_APISERVER_PORT}; - server ${K8S_2_IP}:${K8S_APISERVER_PORT}; - } - - upstream schedulers { - server ${K8S_1_IP}:${K8S_SCHEDULER_PORT}; - server ${K8S_2_IP}:${K8S_SCHEDULER_PORT}; - } - - server { - listen ${NGINX_APISERVER_PORT}; - location / { - proxy_pass http://apiservers; - proxy_next_upstream error timeout invalid_header http_500; - proxy_connect_timeout 2; - proxy_buffering off; - proxy_read_timeout 12h; - proxy_send_timeout 12h; - } - } - - server { - listen ${NGINX_SCHEDULER_PORT}; - location / { - proxy_pass http://schedulers; - proxy_next_upstream error timeout invalid_header http_500; - proxy_connect_timeout 2; - proxy_buffering off; - proxy_read_timeout 12h; - proxy_send_timeout 12h; - } - } -} -EOF -$ docker run \ - -p $NGINX_APISERVER_PORT:$NGINX_APISERVER_PORT \ - -p $NGINX_SCHEDULER_PORT:$NGINX_SCHEDULER_PORT \ - --name nginx \ - -v `pwd`/nginx.conf:/etc/nginx/nginx.conf:ro \ - -d nginx:latest -``` - -For the sake of clarity, configuring Nginx to support HTTP over TLS/SPDY is -outside of our scope. However, one should keep in mind that without TLS/SPDY -properly configured, some `kubectl` commands might not work properly. This -problem is documented [here][4]. - -### Start Kubelet - -To start everything up, we need to start the kubelet on K8s master nodes so -they can start apiserver and podmaster. On the leader node, podmaster will -subsequently start the scheduler and controller manager. - -```shell -$ mkdir -p /var/log/kubernetes -$ kubelet \ - --api_servers=http://127.0.0.1:${K8S_APISERVER_PORT} \ - --register-node=false \ - --allow-privileged=true \ - --config=/etc/kubernetes/manifests \ - 1>/var/log/kubernetes/kubelet.log 2>&1 & -``` - -### Verification - -On each of the K8s master nodes, one can run `docker ps` to verify that there -is an apiserver pod and a podmaster pod running, and on one of the K8s master -nodes, there is a controller manager and a scheduler pod running. - -One should also verify if we can create user pods in the K8sm cluster - -```shell -$ export KUBERNETES_MASTER=http://${NGINX_IP}:${NGINX_APISERVER_PORT} -$ kubectl create -f -$ kubectl get pods -``` - -The pod should be shown in a `Running` state after some short amount of time. - -### Tuning - -During a fail-over, cold-standby mode takes some time before a new scheduler -can be started to take over the work from the failed one. However, one can -tune various parameters to shorten this time. - -Podmaster has `--sleep` and `--ttl-secs` parameters that can be tuned, and both -allow for faster failure detection. However, it is probably not a good idea to -set `--ttl-secs` too small to minimize false positives. - -Kubelet has `--file-check-frequency` parameter that controls how frequently it -checks the manifests directory. It is set to 20 seconds by default. - -[1]: http://kubernetes.io/v1.0/docs/admin/high-availability.html -[2]: https://github.com/mesosphere/kubernetes-mesos/issues/457 -[3]: https://github.com/mesosphere/kubernetes -[4]: https://github.com/kubernetes/kubernetes/blob/master/contrib/mesos/docs/issues.md#kubectl - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/mesos/docs/ha.md?pixel)]() diff --git a/contrib/mesos/docs/issues.md b/contrib/mesos/docs/issues.md deleted file mode 100644 index a5da5d9e9ed..00000000000 --- a/contrib/mesos/docs/issues.md +++ /dev/null @@ -1,232 +0,0 @@ -## Known Issues - -This page identifies significant known issues with the Kubernetes-Mesos distribution. - -* [General Known Issues](#general-known-issues) -* [DCOS Package Known Issues](#dcos-package-known-issues), in addendum to the above. - -## General Known Issues - -These known issues apply to all builds of Kubernetes-Mesos. - -### Upgrades - -Upgrading your Kubernetes-Mesos cluster is currently unsupported. -One known problem exists with respect to expressing executor (kubelet and kube-proxy) process configuration via command line flags. -It is **strongly** recommended that all of the Kubernetes-Mesos executors are destroyed before upgrading the Kubernetes-Mesos scheduler component: -- destroy all daemon controllers running in the cluster, across all namespaces -- destroy all replication controllers running in the cluster, across all namespaces -- destroy all pods running in the cluster, across all namespaces -- invoke the "kamikaze" debug endpoint on the scheduler (e.g. `curl http://10.2.0.5:10251/debug/actions/kamikaze`) to terminate all executors - -Not following the above steps prior to upgrading the scheduler can result in a cluster wherein pods will never again be scheduled upon one or more nodes. -This issue is being tracked here: https://github.com/mesosphere/kubernetes-mesos/issues/572. - -### Netfilter Connection Tracking - -The scheduler offers flags to tweak connection tracking for kube-proxy instances that are launched on slave nodes: - -- conntrack-max (do **NOT** set this to a non-zero value if the Mesos slave process is running in a non-root network namespace) -- conntrack-tcp-timeout-established - -By default both of these are set to 0 when running Kubernetes-Mesos. -Setting either of these flags to non-zero values may impact connection tracking for the entire slave. - -### Port Specifications - -In order for pods (replicated, or otherwise) to be scheduled on the cluster, it is strongly recommended that: -* `pod.spec.containers[x].ports[y].hostPort` be left unspecified (or zero), or else; -* `pod.spec.containers[x].ports[y].hostPort` exists in the range of `ports` resources declared on Mesos slaves - - double-check the resource declarations for your Mesos slaves, the default for `ports` is typically `[31000-32000]` - -Mesos slave host `ports` are resources that are managed by the Mesos resource/offers ecosystem; slave host ports are consumed by launched tasks. -Kubernetes pod container specifications identify two types of ports, "container ports" and "host ports": -- container ports are allocated from the network namespace of the pod, which is independent from that of the host, whereas; -- host ports are allocated from the network namespace of the host. - -**Notable on Kubernetes-Mesos** -- Mesos slaves must be configured to offer host `ports` resources in order for pods to use them. Most Mesos package distributions, by default, configure a `ports` resource range for each slave. -- The scheduler recognizes the declared *host ports* of each container in a pod/task and for each such host port, attempts to allocate it from the offered port resources listed in Mesos offers. -- If no host port is declared for a given port spec, then the scheduler may map that port spec's container port to any host port from the offered ports ranges. -- Any *host ports* explicitly declared in the pod container specification must fall within that range of `ports` offered by slaves in the cluster. - Ports declared outside that range (other than zero) will never match resource offers received by the scheduler, and so pod specifications that declare such ports will never be executed as tasks on the cluster. -- A missing pod container host port declaration or a host port set to zero will, by default, result in the allocation of a host port from a resource offer. -- If a pod is the target of a Kubernetes service selector then the related target container ports must be declared in the pod spec. -- In vanilla Kubernetes, host ports with the value zero are ignored. - To obtain the same behavior with the Kubernetes-Mesos scheduler pods must be assigned a label of `k8s.mesosphere.io/portMapping` with the value `fixed` - (see [#527](https://github.com/mesosphere/kubernetes-mesos/issues/527)). - -### Pods - -#### Pod Updates - -Once a task has been launched for a given pod, Kubernetes-Mesos is blind to any updates applied to the pod state (other than for forced, or graceful deletion). - -#### Pod Placement - -The initial plan was to implement pod placement (aka scheduling "constraints") using rules similar to those found in Marathon. -Upon further consideration it has been decided that a greater alignment between the stock Kubernetes scheduler and Kubernetes-Mesos scheduler would benefit both projects, as well as end-users. -Currently there is limited support for pod placement using the Kubernetes-Mesos [scheduler](scheduler.md). -This issue is being tracked here: https://github.com/mesosphere/kubernetes-mesos/issues/338 - -**Note:** An upcoming changeset will update the scheduler with initial support for multiple Mesos roles -(see [#482](https://github.com/mesosphere/kubernetes-mesos/issues/482)). - -#### Static Pods - -Static pods are supported by the scheduler. -The path to a directory containing pod definitions can be set via the `--static-pods-config` flag. -Static pods are subject to the following restrictions: - -- Static pods *are read only once* by the scheduler on startup. - Only newly started executor will get the latest static pod specs from the defined static pod directory. - -#### Orphan Pods - -The default `executor_shutdown_grace_period` of a Mesos slave is 3 seconds. -When the executor is shut down it forcefully terminates the Docker containers that it manages. -However, if terminating the Docker containers takes longer than the `executor_shutdown_grace_period` then some containers may not get a termination signal at all. -A consequence of this is that some pod containers, previously managed by the framework's executor, will remain running on the slave indefinitely. - -There are two work-arounds to this problem: -* Restart the framework and it should terminate the orphaned tasks. -* Adjust the value of `executor_shutdown_grace_period` to something greater than 3 seconds. - -### Services - -#### Port Specifications - -In order for Endpoints (therefore, Services) to be fully operational, it is strongly recommended that: -- service ports explicitly define a `name` -- service ports explicitly define a `targetPort` - -For example: -```yaml -apiVersion: v1 -kind: Service -metadata: - name: redis-master - labels: - app: redis - role: master - tier: backend -spec: - ports: - # the port that this service should serve on - - port: 6379 - targetPort: 6379 - name: k8sm-works-best-with-a-name-here - selector: - app: redis - role: master - tier: backend -``` - -#### Endpoints - -At the time of this writing both Kubernetes and Mesos are using IPv4 addressing, albeit under different assumptions. -Mesos clusters configured with Docker typically use default Docker networking, which is host-private. -Kubernetes clusters assume a custom Docker networking configuration that assigns a cluster-routable IPv4 address to each pod, meaning that a process running anywhere on a Kubernetes cluster can reach a pod running on the same cluster by using the pod's Docker-assigned IPv4 address. - -Kubernetes service endpoints terminate, by default, at a backing pod's IPv4 address using the container-port selected for in the service specification (PodIP:ContainerPort). -This is problematic when default Docker networking has been configured, such as in the case of typical Mesos clusters, because a pod's host-private IPv4 address is not intended to be reachable outside of its host. - -The Kubernetes-Mesos project has implemented a work-around: - service endpoints are terminated at HostIP:HostPort, where the HostIP is the IP address of the Mesos slave and the HostPort is the host port declared in the pod container port specification. -Host ports that are not defined, or else defined as zero, will automatically be assigned a (host) port resource from a resource offer. - -To disable the work-around and revert to vanilla Kubernetes service endpoint termination: - -- execute the k8sm scheduler with `-host-port-endpoints=false` -- execute the k8sm controller-manager with `-host-port-endpoints=false` - -Then the usual Kubernetes network assumptions must be fulfilled for Kubernetes to work with Mesos, i.e. each container must get a cluster-wide routable IP (compare [Kubernetes Networking documentation](../../../docs/design/networking.md#container-to-container)). - -This workaround may be mitigated down the road by: -- Future support for IPv6 addressing in Docker and Kubernetes -- Native IP-per-container support via Mesos with a custom Kubernetes network plugin - -### Scheduling - -Statements in this section regarding the "scheduler" pertain specifically to the Kubernetes-Mesos scheduler, unless otherwise noted. - -Some factors that influence when pods are scheduled by k8s-mesos: -- availability of a resource offer that "fits" the pod (mesos master/slave); -- scheduler *backoff* (to avoid busy-looping) during pod scheduling (k8s-mesos scheduler) - -The scheduler attempts to mitigate the second item by cancelling the backoff period if an offer arrives that fits a pod-in-waiting. -However, there is nothing that the scheduler can do if there are no resources available in the cluster. - -That said, the current scheduling algorithm is naive: it makes **no attempts to pack multiple pods into a single offer**. -This means that each pod launch requires an independent offer. -In a small cluster resource offers do not arrive very frequently. -In a large cluster with a "decent" amount of free resources the arrival rate of offers is expected to be much higher. - -The slave on each host announces offers to Mesos periodically. -In a single node cluster only a single slave process is advertising resources to the master. -The master will pass those along to the scheduler, at some interval and level of 'fairness' determined by mesos. -That scheduler will pair each resource offer with a pod that needs to be placed in the cluster. -Once paired, a task is launched to instantiate the pod. -The used resources will be marked as consumed, the remaining resources are "returned" to the cluster and the scheduler will wait for the next resource offer from the master... and the cycle repeats itself. -This likely limits the scheduling throughput observable in a single-node cluster. - -The team plans to conduct benchmarks on the scheduling algorithm to establish some baselines, and is definitely thinking about ways to increase scheduling throughput- including scheduling multiple pods per offer. - -#### Runtime Configuration - -- mesos: `--offer_timeout` : Duration of time before an offer is rescinded from a framework. - This helps fairness when running frameworks that hold on to offers, or frameworks that accidentally drop offers. - ([via](http://mesos.apache.org/documentation/latest/configuration/)) -- k8s-mesos `--scheduler-config` : An ini-style configuration file with low-level scheduler settings. - See `offer-ttl`, `initial-pod-backoff`, and `max-pod-backoff`. - ([via](https://github.com/kubernetes/kubernetes/blob/master/contrib/mesos/pkg/scheduler/config/config.go)) - -What is not configurable, but perhaps should be, are the mesos "filters" that the scheduler includes when declining offers that are not matched to pods within the configured `offer-ttl` (see https://github.com/apache/mesos/blob/0.25.0/include/mesos/mesos.proto#L1165): the current `refuse_seconds` value is hard-coded to 5s. -That parameter should probably be exposed via the scheduler fine tuning mechanism. - -#### Backoff - -If no matching resource offer can be found for a pod then that pod is put into a backoff queue. -Once the backoff period expires the pod is re-added to the scheduling queue. -The backoff period may be truncated by the arrival of an offer with matching resources. -This is an event-based design and there is no polling. - -#### Debugging - -Good insight may be achieved when all of the relevant logs are collected into a single tool (Splunk, or an ELK stack) in a manner such that it is trivial to search for something along the lines of a task-id or pod-id during cluster debugging sessions. - -The scheduler also offers `/debug` API endpoints that may be useful: -- on-demand explicit reconciliation: /debug/actions/requestExplicit -- on-demand implicit reconciliation: /debug/actions/requestImplicit -- kamikaze (terminate all "empty" executors that aren't running pods): /debug/actions/kamikaze -- pods to be scheduled: /debug/scheduler/podqueue -- pod registry changes waiting to be processed: /debug/scheduler/podstore -- schedulers internal task registry state: /debug/registry/tasks -- scheduler metrics are available at /metrics - -## DCOS Package Known Issues - -All of the issues in the above section also apply to the Kubernetes-Mesos DCOS package builds. -The issues listed in this section apply specifically to the Kubernetes-Mesos DCOS package available from https://github.com/mesosphere/multiverse. - -### Etcd - -The default configuration of the DCOS Kubernetes package launches an internal etcd process **which only persists the cluster state in the sandbox of the current container instance**. While this is simpler for the first steps with Kubernetes-Mesos, it means that any cluster state is lost when the Kubernetes-Mesos Docker container is restarted. - -Hence, for any kind of production-like deployment it is highly recommended to install the etcd DCOS package alongside Kubernetes-Mesos and -configure the later to use the etcd cluster. Further instructions -can be found at https://docs.mesosphere.com/services/kubernetes/#install. - -This situation will eventually go away as soon as DCOS supports package dependencies and/or interactive package configuration. - -### Kubectl - -The following `kubectl` and `dcos kubectl` commands are not yet supported: - -- exec (see [#356](https://github.com/mesosphere/kubernetes-mesos/issues/356)) -- logs (see [#587](https://github.com/mesosphere/kubernetes-mesos/issues/587)) -- port-forward -- proxy - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/mesos/docs/issues.md?pixel)]() diff --git a/contrib/mesos/docs/logos/k8s-256x256.png b/contrib/mesos/docs/logos/k8s-256x256.png deleted file mode 100644 index ecbb2a0e49e58f49e5cb7bd0e9060ac83e1114f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17168 zcmXtA1yCGIwB21?f@^ShOK^90cS#5m++72~-JRg>1a~LM!wK&0?)=HG|BKq%t(}^g zo$lNBo_o$|!ju)IJ|KQX1c5*wWTeGaK_D>j5e$Nd1>W=>OU!^b*e`NY;-L3`f4LpS z3BWf9_R=~|z!?kwUSLt$40qs5IA<9J3Al9_WL#X@pt(vU5Qr2cBQB!uzI2-9l4>w_ zdoT2?pTn0vCQsdh$wf-`37rwz&BC5RQ=b}(p*>i23Bo~W3t{?iYhW|P3sHoEOHq`c zjEmD&Ym(>rbz44={gacy#Jxu#e_G3ZX5;wDRgjzn8)ucBF0rUSpgn9e3O2m?s zl-}Y(zCnIMFe#BiqrCURM4{NE=3pyqi|p-NXob==Xk+p}Y_?@xU5Wb0gov}CiPaa` z&#VOQT3QW$6Mi*DA#@T6AbHdEFeBI%q|AkuiB$tD1o42bCF_;9aUbHj#4;HW^(TYN z;wLF`9+GAhWsIBJ@$Rf{2El6cD7tU|&Rq-h16}ADe^O2?sQwG_F6cYo^AfO9D9r%w z-BH{DV}K$a>IVracsS098$QMs@6)fA`mEJQ3BoKDxGVHqbh^?+1s4|5@B?3^s{X5OtPSiSGyM5O>>oqAP!huz@p&KfwT4DigI-@k3QsQUM4 zt^u`;Zvl`ZXL?`D=#DkV4HN{Dv1`xYspU2p{$V7F&)+vtHfVF{hm72aF5+XiVw{R^ zExEMTDg9s6pGtitj_S1G9pLOWh{H({SIXdt|8wc>v?R7XLref&?e^zDXCmptYhUXV zj(1aZ`*|BqiaNQaLdVwQak*?u)9vPWui^CwU<&18KVa5k9HpuM5x?>z+0)3;a=NHehB zwKqtF8|k8&$pV5&SuQ-v_cQ(-P)q8Ba6_8$@l0eX=^mh)l|tSp?mfpC#1lZ%F?4DL z51r?OLSS=+eL@^ehGzprX9v>ZhXn z5C3~*?4mdONI|cCC}K5o{D5!WUH<04Q&RtX3`)bx>J|0KJRu07(v_}lS}1B%w;Ffz znp=z%*S|}9%leJ;B9*zo+f&&$(#2KuGIcu%*1P?Z0kNe&B?K5jg+L7yiPvH1`MU*p z`4~=;|GVf3&R46~e3%!^R0~;(;*0dE1GRmj#Ra?YUP}L*th`GRja@9V%CKi-mXe4V}{d=25Ib^iEgeX;2JF&@p& zw==kgUPT_`flU)qeaP#VCbT+ZiXja9b!?7c6x)CwlxcIe{4OC87Chu@py)CX&BC`= z70K%23AI`BH^+&l8Lr<(&@Klh0rcs(e;tmZ#;veL%57`q$o?9iDpSy7p%VVwyE9Eo z5H+=`dtwU3oeeQ28HiWIfgI$OF!U*|4C^y!RnY*ki${UB`=6Y%<%CYhI47D3VAmgJ z8w@a2Ior34@y^}6WBzyzK=ps*XnZRsd+?v+qs*sA`)rnW-|3R}#fJVViWP6h z@+_-gWX8mpg`xVj)3Hg2AWU}9r~^I211T1{!MGq_=LY8OK(YNf-(i}e;p;Z!qRnI;On*hQa<8m zg04;-&G9s^W(ep$KbD6`LyZR56=3;gDIY79D(b4W!40HzukCpjOT7PGsyjX*HV{>7 zO9LlLu9S!xscMfUWBe{{x$-hssPs!NeX*njnPi@|iE`7A%XZ&j-s%8DdB`IM+OE1{ zpdJ0u4ejJH6QL}gce>?|8cfv_Dg(mzXZZg8bzu)0EZqa{rrJv{B2m%6&~U5TATp65 z1w#Ml2I_Xz(jphGzQ6fc;~269Z(9RMT_Nu~u%zI_3uEt1Il!~CPt{OQX;x|Y$}VAL ze^L>8#YY`<$LHQ0bYwvJ)r-bGge7Sz%0Y|Y;$zUv(3J;8_G5q}I1$`vC9GPkPNdLn z@-Uh~I*6G8i^WCr=OdE1#>mRXRYqpQ5GeXeEN)jt5H=N#Cr+9K2>)lEqQNLJ1c)j< z*5YHcXl%M)vz7MaGHuz8XRe6Myk+nq3dTPnpJ7pq#eeI7aE#kqT=7Lgyz*UTG#cO9 zgjQoJ92}2z3ZeU`VNB5m7iMi6Oh16(v1nLl^_!4cU#SwY;9+miJm>F;eQYdRWk6}- zAkqX;kW48|>@dYRhg_v!^>OZ@WKj;t6Ne598_(}aOS?^DOK(dPk8e?gmHZI?kiGFR zh#W%#9PtGjV|+O<@k=gqB1SGV13YL)W-p!U-Bim(Mja+ajMXzXN(Jg9lK3RouP{lq zsXGteTGL(|MpJHq<9CJhrqMz&rbbkIfU)+KPsd=3beNt+N=X+}P)>a_7Yt}I4I3;6ig{urjsIoi+Z%5M#{vVJEQux+CsHK-iSaZ% zoR|7C-qgCHj>#Tj8@fiF_m9!K59pnb%wrYb&^fuvn@_?mi6~J-au(ELFQJvyj=Mk) zO)Z9(B6E}y%)#}jbUTqhRuZvjwHed=>b|#Lp5!ngURU2QYqtvqj(-_M#(-pPb{=N$ zrV&GNFY1>rDtalTB&KLD<}VFe!nN|atKV@*t>IAe=M9dab12SRYkKS@xO$h9S1jk{ z$;p?K?g@iJKnPU+L04Bnx?N&F(o!4WSL;uEsh5ezZx26o26)ITbX#@DMOJWBeD%aG)HM1`b&k9mE>lptxmbk%XU?^q&tr_29pL zKd+9a-O`f6*&Aoh?&9qwdbeD&MK?`{`PvAd=^9Wa*GpegHkP=bS>6|L-H~lv)C$rN zK6jRto}={f)aH2)6-6OBEvTCfER)f+(@d)+?T^TD`R~7?;yL7~5)7r5q=b8xjRvSy zeC+?^UiK35G7h^eXss!_mrz|R0J6@gYsoXZe zesS3l22kX;tzN{i0kXptc}*Bwp=#$8Y#V_HCjR$#dTvi|kvOLz25z6sx8)fO}8*R50s&76%JW%ChhkJZ|ABqz% zhUa$Wi4Q3g1NRDkmFyJFg!9S<<){wqXZ&3TF-fAcZ$Sc7@o z3YG{^LuK43C7|bJz{FG<+b7f=wuEzKsbwy!`%^VaCpuZH#-;v^5DYKv1y=rAE~(~q zpV)%;%-78$_|Lc@K#Kb&mNnYfxEo3E{e0fs z_mfcnxDq?t*w?QmLxTYnL??5TiXOFr66kaUBxkPhReUW$0#AQiPD_iLAwlUVxP&VG z3dIAQ!EfU_V2BBk-jhKKX22o780f1N*BQ)ovTdpK##xFU9!Je{&&M{6i~d{eSw%bJ z7Nb+mY=3C=56t{^v87$KihrMNZ9l=EFW;i$R7BVGl6;e^kIwU-`{E2{W=7*aS$(%o z5i_obPxdAGcG(KgGJYbA{$Eq?=rEX5@~){K7_5A_Wrv~--rL1X=BFgR=EpGV`31|> ziEVfZN?=m(>aYCxg@*mi8q5Q)piXjx{|NX3Q8NigJY)fryvFU1GKa1C=Ha5`d4KS> z-4NnD(bX1!{U)7qUZC;etg`@4DpEn=?2XSUu>C~`&gypwbT1G$=qJsEZVmw|%j>Jl zA=Sx0Ti&Jy-`P5LfBS{n6pkQrh@I$ro09!dQ%+v_INNd(0E<%{VSK*z1e|Q8M87xv z^^4QUS~~W|T$H&HPlUVWIT-LM~&$b{o=^Y|Bmqu_w}U`ht|=> zZ+WTmg!R~lU@CNCzszpR-OV+2+je=h)XjExl}F@AH{MOh@I>`t^1XaxgSjeZmA?5g zGCnm?2rZfa)d8-F|F1l^Q(}&yK}Ku|YPiyM^$j1oJPfRPxO0$ybnMQl4)`UC-s`4O zZAZ2B_BkZV;Y%ZYr!J0&Y#5qk;3#L|e=hb*n`?hOKUs=jCBBc_LHKVZ2~b~+7>t-c zP_V{pwSFwNZ^N44UtG2M-0@4{YT^Vm#e7gljx1qiG3Q~Y7#p(>M5;t#Sx8d$z(@<2 z$GIWlNt0-Ol!P(B20N)WNExflOSfe^@Kf}9@$UAFV;bWIiURbadd75D*H`LCh zpq&fpsz`U8yryrTRLaV|E>gAMTu&oSlKp{&d5Y41n?5%D9zKtE?>Vmqsn74#Nq?f5 zWSg(#5S+9m-EUSo1r4zBy3zdt#(vMVw9zt#3Qe;Fq4Q_7Q1(km;jn${f;`>GR}{O? zSW*R%ORo23OfrV?s1#R8Z11Ll-!5%4Og@_X9u3-~1q2GzLO*6%JtC@ z^9@g&PtX?{eX~YfA&y@a)N!?PZ2G=(C;!<=QygBxw3pL3hZtw=Tj7da6`dN}XIi@4x6zf}M%C0OL|J!>|6+ILMI5v1)eZ9vcKbV9MtmbS-PS~h zOg{#RSh9$5fs5gDjHK_iI|aefjM`SCZN#V@vE5EGO@=G}NX*byDQ#F1NID$Vx(QL! zR+Z5GrWI+HT7co5qDU(I&xC7D0y`>pb!BM{_NQK>q2z^dtYR3ZItHw6K#@Jl~kKecP9J zh!sY4*KePzmpOFb$9vvn8)-qXl==vh66fW^vyG6)PwSs?!5M-OZb3NH&~6L)%cd0F z#aQ9>oBad$!y|g6rM`%1g?atmoJJMyxsaBYf!BW}meg61hDnk2>Kv}kLJPiAr zgsg1qwKi5h&(OM|^=265AdDF^O56`_W+v!d6MWbmbp}0&?skp1AiuLR1(l)~2PCZou7puz zixebMHf!YC3IU+!a0Hi@a$@(qo|4bXL|$W@K9Q0pJhB`eSUSQCJh-37{=;OXG+`x@ zlu6k@fYyx0p)Exs6VzX%8=E`DO9C$zvXIJnDT_6LC?oy^0&k+(W} zhb$95R}iHN8J6+6gBn+OB030xQEH6x0OP0Da`R?Ge1Ui8IC=BQBDvv{4GaHna$>A7 zzesnus@yb!@XY4U>ni_i1|o{gG^2Qvn*I4V0o7?sZiQ6R!iIu)(5Q?P+(vI7 zK}ZLWB}keG49`u5TgdHNlXWRZME6()Y1^oZ7;U}|Bu>O-kg0_&h^)3Ktq~@}7xA}O z|JSZa@e%kIA?_0%6}3~+@Uf@J&^;GNf=pqZL5hQ6?6fON2cfl!hOLG10%bOA z*wm+pC~d<1=#P$k7$ne>zc~X5nsNtYeGVWQ(K0CqP4!xT`4hPmJ99>7~XH z?xw7Y{pSI09;$yW4vE;$n|HYa9^aY6LQB8sh-mtq!*zAm+|iGI-Lj%GnOc^?2%}K& zKdJVKHHW*Z%8j?|rF6fYIKtcVR|ZAfPq20Smezq#MyN^;F%)^b6+9fw9)3cVSG2oY zQJE9(41N!LEr=HmVCamnD-Uiix|Owremd)@faA*66|H6X%)F(Ps@G(Zy!=WR)mpHFI9okooOq`$KSApl+J*Q(~4i<_5H#6vuk;`2`1yS z;Hbp?T)n50;}e3uqceKUTPsaI?B6yl(#SGaOVbx{)2|MjgSHXacnk>H$YTiZ6@X16 z4{|coLl!zuHhxMGh~U+RwJ`LyziFH(I8<|Y(k|3NdHpB>==2@=H$U+teP8R71r;L} zr0dhCgS(?o5nqsL1fK0&Dwa|+vmt$DQ8>FP#Ci?O-+7djki71q-gU2cpb%ZHz*w!& z?~gVx3z^!4Ggx_58@2b#kE;pe%R{*VQ^+?Y=MCtgYx(p>?JkGr%W9ixVfRteIR9qr8xZ zK4&~e%IDXa3%5_W7wv&3r3G-V6zAgzXUg--@YS6f=A)gp61N5@!7;c&2k^Tn2-fK? z=LrU{f0-Ga#%H6%VyDxPAga(OUBwi8q;q0+4QHI6stzxni_8JN`n zWlCnj*3iZb4SkfjixEC!b+&9rVQ?{_vap`=mwLQCw@*_ziTZEG0&??GZag$1@DF=! zta4o9_*0WO3CLDI7d_luS+#E7b9b6Ss(bQpU1;{*hP5!mK=E74z&ue6j9jL3-^t>% z^{e73J^!j=Y)@jPn)tp?>ljwR(A6b!9Dil^V}|-+Iihw%8=ci5l_G@~hK@sxD_TYp znUn@zIE6KLI=q;lQ)#a-Ex;CgSR2dz^bcr%RV^lL^=q)v?pW7s?ai}A3Js+WQ;od> z``*U!n-`DjS`|}itU)!>L44?a=YQ28juz4kX?4e!&e~gLC#o)QvDx};RC z$jf5!`$5u+u(=yBJ%{_WXWFzz7<{Cux1{)O>#rK|Ifm^_l z>$G0Z9PdAaLu*Rp!7NbB< zqm8z8wS5!h!eEOe3-tQY!B(!mNu|g3({C^UbrciSG33}a8h<;L2)j@aq!3lx^6{0v zLRw~m9hpe$Sjf3*;aaAGxx2PS4D_i8Hc3Tc8b`i^r?XTh{}+eD+m*(er>2c6odW|_ zi|b2qf8|90%Ww{cc1N@0*F^Q>c;={zZtef_rO8ryYuX|!IRJwPi?th$g%!-4<WJeTdCn=` za(OwJ%IPd8yk7@Lur(2RtYx{IiU32GV#G@Rb+n3oFmWFXs`;-3^*M^0_PBZ%NYafM zmS^K#Ovl7}%D`YWF){3DI+UxKFmRg&$Nn~PYl?+m>i@j}&4PAL34?Q}T>bj(1@IOY zb1e^VPtCTljEF0aW(ZRAc&|wL>GCqGK5A{mGp#LIfrN4cOW)|bTxj;{k_f+pnW?aN zCC0ikDt=G3vLAu-%1o=@T%*ez9EL4u`pRt_y3Wgupcj`ahFosVRE($6|50Gv-qt7E z9fX`cM!p(@j(&*%kMn3@7zq7LmT(tg#P~Vyj?DATLVu+9+!b(M5-M8`?4m($kI_2wh^y{BWAnlkE4ey7nQ~Qv zN4Wlo0z{Ara0do1$x^$yTD_hx=xifKg9jsej<))wpzR>_3gHGhXi{jJy>#eV_f~rs zC%7XFb&b3u$@|T1YOI>cY?N>VNRyh3O#FyHJ;NC{f-poNobiTM*49Idq0(?g?i9L&)l_hmhiPO|{T=fC_#QG*EL&zlzMra7C@iGpqcV#*se_Wjzp zQqF@!f<}Z}KRn3x7*S`4r!7%OPyc%Id2vNx=y4jUC-7iAy*)}M)3}6zB-dNjf6;OA zg5DoMF^wrXyoj-jQ>t$OF2-cdHn88i7wJj)R>E*kNTc#UN zs~G-;E$)sJ9j*R%+6cmI5R>Is*;V)rBEk6qI=JRE?y-OYQAw0!ZtKVdsdcLey7!(M zYs&2q-cNfzC!9T@Z?2LzO-@%}7*dlKq0wU_7U8DS%wAwnKP7zrjeh3;p!fB=ql}zMIOLFHvN!Y0<4Q z>OEhkP+-;;4Umny2@S3HATcscsCRMc+w-HX=v< zx}dXHEDhFtC%`2!5cg7!8G)48O`(bI(8CeFI6)ppl1oZg7f&Cz{)&U6Y;@&iXUk{f zT}Hb;bn(>(HNDM7IEbAZkb@#1QnTD?mre9VC;OwLZog?Iy{ffu+{HR1C(63wP;1KZ z{3{@y`vp%(eMW?PwQ+WMi+@J97ZU636@nHzkuNS^NIz8aDz?SleMFQiV_+oI+1{F! zv4y>DsFJXpw6f{jHB}plLDAP)VG4iOflCSgXBj>v z-`N0-fc0#Z(aYy%S|)YJ$D3n}c{`1;GK}X*TWm+@4!-x3L=eib#Jb<4FDDA|?pFUR z;;@@GRtaOR{izI9rjImPAq>#Lb_WFzC^49=puyQq92virNmq3klCh$tHhqQ+ml6Qo zu_G{HsQOb-P7&wb#AB6jZU`cAa-03H2828lBOY!GwY?-!JTc^A4rmugzu#3Hry#L- z8m`YMZWrCVMOPUIf{28H=C&2a?u}U%k--x9;bOkFWKgVC^Ig?J)t^+-3g5)Ys}cvl z3@fg$eMiS?frdd5@dyjvf&@=NodOJn`3q}EOM*gw6hM7;8seB;F2TDneT=f|lul2L>24A2x{Wd*sNPgJI9|}2&-1_EyU^Q> zBTl1F)6pyA?bWj7RY#51ojCKr-v0)adTe`44XR2J8PWKoeAqv)$)Z_50h!Qc-YmXkt#E5F$%wy9gWeZPRcZpJ&T_!PKo zso2LY3>sY{Mgh)juM!bx-)J*3*cpX*CD;vAk2IU#>xd%>(pxm6s`_55o2~#FZQ1F% zkX<$0WmUR#tvn~ogqY#^d=~)u1W8mli4%e4+DPuI2pJmx)k{0K`1Q@>L^X9GT#QAc z=iL+qKInHvJ34w~xh#2M**Fi(-{Hg)&4@bJr^9!Zcw(M_y zYWY9A3@P$t8M6eCuq96ZvrfdwbnVISVm23lAEO-jhm!$Z=*lau;Tj)QB&R-(x1pkA zR#+!)VsA&#F^UBU*M}Nrg>FhPN~rn8BFv;^Jq=uX9=4K_@_}my*Pi^#2)RY`e%u$# zZTML6Cu)~v+Y!w>sd;DRU2e8+#rfJ#a;BIWGSM6@0nU&xuK)PX>`WK=(L(#f^Vy-XGK-%6xWRN(SNmeG zT#ie@@^l#JhqqyjUq)}tkOh9?@A1#gqv_bAB+;@^5yiCjN3TZS@f%7o%$$@I%(LIQmbgZGcoo zJCNKX4*Ko@$Rfv(*Xi+Fp$*KoFaKzG`Mf5x%sjoK5|{moLIm6rfF#!A8%#;=H7)t zJDk-FR50m1OGyD|I+SGK(h_{Dr-4OvOJwST04OKsreSA*F@dh_N;Po;I(Y(`NFk(h zoRXkkarHoA{h0I9zF z%o_z$-us{_rQN0@F|(Q5k(JqO1P*2vu@atc>J^ocx$nqMxN~zlFRu(D|Flb;StLh| z(v%QJ>x_u9vfRV!KN0Cuk(#{MsM zbVh2+Epjo*f!TcB?EWMiMSO;0%<7Hb<_rp;@Q8?*V`>C#!1NAQQAq51laW!*N)HP}DUyI+B2B?{>LFLzq;t zg17sRi8=2@YkixUG7_3x|LyDwjp(A-a#O-6ldvrJA!MtU3LCVuoU5(xc1!RooYaF+{Flna@t zJI|N*s*eimJJENPN6&}*+vbforcd}~BHUhnTo&V024RP_JjmhD7lRBrxbx+g=UHnB zoJQL)p59r^Cntx{n7v@_X6nF!FWP9bTqEv1>F^(Hc(d_5uiH)n)o5Bgp0@82gB4v6 z;G!J2T+M8^pMY!(Xpz<>T59{mGTgVqT+w(Q4o(T9)3;N$0@V_1Z(};>o5$a7k*j|@ zENbJzMLBI35}mQJb<4tMfm%4|JMF@~AKU-? z;JY=~k{LGD#nH)1-%9(Hgw!acKmrDFD3T>K&HnA$(!YfItEXZ4Kn?qXt%k?TD`)m= zL00wv&fWS|0Dgi1~K%#od?}O@%6s7QVLf z{TVfkER0v@_w8klhug`ASXJ3PytjybbS zj^gu>xn>Oa_vX@IM^@xU2CmZ6I@`#dDBJT%^zt>po9N%R-+d!>g>ZL3r-;dT`Zmqc zH#34|RLfx91lM~jl{ltmtL#!v%M|Ff6shjs-@i9z32K)Bs1=wbgV`Ad+N8~=C)3S?=57P` zIsDTy9x^Xs>GhpR*Yg#i=$tIcpw|l%cV_mx>gq&P}3fqRcO!7`HX`SwkQiFeyxjZfR?Ot{Z9^Z*_fCobE$CcY#fOc|45HeE>YudlZd(x zv;CC0o~7~8$@3tZ^BJ9knO<(2@AX*EdF`R~`TX_>7px1pq&r{iah3ewOj!Yz1TdW5 zWcWG#P0}Qvc889FM~CQ75h%k{uF9BB1>_j8!XqQPM1a(~4IpKGM1s}SX}JpOAAn-! zrGM+s`8Q2;i;a$Ii3^?Z{jaaUPfkTEmzHflNhOE9jTXRe<;cAri;RvWa!(g!yaUYA z!sd@zlgF0O4j!Fy-v$P0Nd_!oGTXZ{N@=hLBq%{ZBEw%~==7WdjFRpA^%OlAZY=7NYz9Ru_wkiJR z*Sf6T>qfR@d4t96`y?v;_zA0Q>pzgd36jmUq4gwdMuryAJOoO{;(zS?m<9W4W-DvX ziEG!t9AGF4=K2GQO?ZYlARA(vpoBHso;3)TzlVT+zf#qFvO^HY!#3g)AYL!hf?jdg zMZo;Y^MxF<9K*}znbVt_rT<7_>OnUa;i58})m*~cW;i-S8BMKUpiK5V>~hOvSx$iWwWf{zMtR z8EE8&A}(ZMtkzW!Yu zv!h2ZC@+WI)!sAwZ$z)Vil<`=^=DaAUR$?PP;-y89KcHGB^j{QD8a`tA`wh6u)leL zaB^5O>X8i8VTNeRHoxtBfz=owe(icr(!{9>qilDqW1_94C9gu}k01x~ghlZ=$s8ph zU~P8f(MZ5tuFna)`SMMUTTI>u=2!w&$f~SC57*m_Xx4-Pmq&UYfKXfXvU$)PG-9(+ z*_gN<<-X4CJ*BAkjXrWj@*0}@>w1clui^R$Vi4}X&~=hqMRnH=C@{PwDi3_f-(9;pr*!*x`=j2H|JtRBQJHgsAQ ziq=!k9+CV^((H+Q1Jk^nefS5+Hhi_s;Z<7Ib9kJLt0o zBb+$CjM1FFJpZ`Mg2lgvUU+KH_H(%nX=@AQ`htwT!e_J^;ALp}Yq8RPZ>N7uRVBy) zV=!D;`>MD2l&)i6)5U#n+jS77ZYWXTX;Is}H|OxRMdZwztac$sC$uSl<8=P*cmfqz zFR&XYiX;>)DhH|$$4G#2(o#y2e{zh9Y_C$*$7o?PB}k`%^Yhb4rvTsZxDw}X+Z7Ac zpT809#2l8;|H(iiV7@pC$}Qyqsqgz{R5g^PUoudp6#rrTJD0e-3LA@lUB z<_s7R6DiGeyT+>R0bT+W=fbi%j=h?ooIZT>jf_10 z`}LcScN(NX{X64VW`e9ea9+0uU_3IA57DVu*LCi;>#93m(*9&!B2917gedsVX^otU zm%VhkSI$|w^6JP?mfd;Zqzlc!S@y5uTd-C$xWCPyoINNZ!C!Uso^t~JQSC8t;$s%( zu4++WZ;hJ2#tsj$NAdV#Wa+1_0G7@V=#j$>oWIq1{}gvUa3)Tl`#f#C0)iR7hTN(B z{MctPkBR-;75sIn{;>3N&O3==9bpZ@Y6))owDzV1p)5T~4~OrHeaZe4<`I!(r3_qU zfQ?^uA&-o04}n&+fAvjvCTg!B-7K>qsSxKMZ%{Kjcp$<_eY#uFpOt?9A&APY!y4Kc ziO^&H@7L(Wt!|H*r=V$B&_D0ab^zD4k3>O`;WwOitvgfu^V#T2l>g;56%=^o79<`# zby7L>)H|2aBxL){wZ!+EtuDKeSoq$5$$dpDWsU-q^p#a2SJl-4l<8f?Z#9+gkycKH z&E>YF1x}l<9~jrsINF~4f10M+rJK9FNQwvkb5ehJfTC7*8OfPE?_t|Ez@PGWl&c?{ zH;Zb_;P}m64o!|0ejvTN2*!aS%BD1AsDmwGxTat74h`0$cRdPHdMa z$T-D!4Qrg;BKfoH1we$wt1U?Y30NfDiU6dZktVxcph$(46kWM(%^8ar>x$l6=H=8B z%?m-N!X&00&+)JrCj4t(07rx3;zq2#gp~BvR6D7k$FE>}^>l$3zitd!{N1*`d~;z#vlYVL zHVQ7vCy&!U=toV(j6oaMBLbbHB~m*1Uwpw-0@KZOc$@+CHxe><{dDAArH}t^;=IA* z^y(`tQ5p;-#j7B40zGuD=Ksfb=|!lI1lV5nJo0o=AiKPOn~CIYQuPb)((icv7&XXi z7G^8Hdz%&O0uH?hq^-v~w_=OFKo*Uesb?GU?k}lbBLPIn_TkgGiucc|_4E@K8D8+1 zPl*zReRfOsf9T1pA35{(beeNHa(`HY{MkyMA?r6s%JZ1Q#SE-a+(PSGjqTFE>~N)g zB{W8Wku^E$`@pD8G}7v}I$d$OfUDPh@K(j8wpyPv9tmu%^zffN)=3DjmeuKPojmp^ zE1EZgdf_!%a1r<8`|<2tdAU!p4!t;AQfnJA`0VuKro6#XJt_jI#qOTECCp7`vqT;S z5rajNQU5r3pzsIIp<}_U02=y!2zLdN%_QIAJPt68lUHbE?1A3poJ1Vu%C1ZYGeGNK6b-}@(Wxl(bx4`e9Jf<>WZ zZFKRTyVodhH~Mm&uD7pElgBNrg@cl1>-OhM<5ZmNLK_n)8X6oUV2Vs&hsQp%pY3Kk zZHkck`fdP@@T9xMje1@yWIQJlWDyUl=9G?9Y=T1 zIel3KBn6f3lt*C*#`e?s?Yh>P>(S()U+LG!pR~z%oSjt!IDusW)KQl^0L=-o4a*GA z`X~W?W-qn0l8*5vI(S_KK>t}niS~03C5lPmqrz!CfU}A?KS-c!W7lYl%TMBIPz@>4 zZdBu*@ydB$F46950nR^!^o2;PY;brqSEE?Zc%Sp?>RSVbFbzT`TzgzO!!mmOiv9Ny+h?$ME33Gk z@MHvoeFWrMD@5mP-KD@!n1ul_%ZZa_&b)S>1KTm~3$MYS0=}?7C+{weN@|RxC9D#D z8g0?T-RwdVL&)Gs{qUd5LwD0NVTuv)mf5=iQOnZZvjC-^;=Ncw7Kpd_gTeG6(k?PU z=R2N`479zr{LY=|nU* zU^{_xsovQV6Fmdxp3&ByKo0cZ&cB%R^7=(j5e8arj#`8IybYBkXNOFY8=s0i)}Vb; zkc!?Bo|{wK-4;fDbKWZ`SZ3Dq%XWUW81R9VR=WHz#hiMvW4lML+q2I>m@X#i=u17RH7}Rd`Vy4R8LSi=eEf;(E}9zy=B={{hP$a__2p`L6B?ws7F%R|)O-J4-+OH2>eG8A;>A}= z|7Wzl7F(QL%9Ik_d`s-H-G>X*1*sc6PQaV<-5oGk&b0w&6&_P~tlrLlNB6!qSkNo- z8m&}M(W^h@_|c8u>4!2a4>`a=a5sbyPke;LvKA;UaP7z6sw@RQ3d|Dcc1mn7-x@eU zVS#^?Rs6+)dq|o*9$D1E$M<;ymRG5%N>OfOgj0Dz5TF&p|5SijAjx*>ejJ@!cuoMa z9a#biD-yD@2YBPpO)*yYQ)`uuM?lR4ff3BuFWfA54>PHhThy?E*4zK&W3M7 z#vYgst~I&9xN(~zB4ox>ORT2`2hV;0!72I9hY1fpK4Q;)EAeR(Y6BSNB_^FX_C|be zzc(?TJYci!Aa*KhX4c_ki(~$1M!jp+%GKyTQ9Ju*|5Y>Bt`*o6gNqG&jo0| ziR8UVnh6|bY5?lZTgUp$uay2XB)I`F)d-a^V`TDiAf6Ry0Z%sGJ5Vz3%ot|YzJD%W zJL$)Guy#l-K?Isj-zzr&yB|xXKUf&D7D-al1tDf`UEif8aX3yEDb$_25HQAty5n=R zCFFe0lZ91ZO06>je+v!-l$n2mu~oepO6TH-UKN>AJ(Fp88nGwsN82BxV*_LW6gV73 zwNC*uDO-6c*D|NHLgjiCn~!_7ysu)16MaqM;BC^}mx;Dx`rjPFb9~=YsOlj(08TUi zmQhj9jBbAC|JG!!5ADrv=k zphDxA8xT*Q3XMC#5!7{-A1%Nx*>tv6E zR4ico9K66ioB4bm${dyiu6^uJ@wev%36mNl(#%+&nbvOt4ai2`i2Al?N|-X@XxwxA zx3l}BNW1C+>66EgG#BwLN(z(+Zos6)I2{H%R*^QwM%Vkd>=|;z8S@`sd-~>Bu_QS` zU!=LE!V*GNaNJjfL2qJ)wNv66G1BO%0#VCbmmA_AR?%(d%$&1diINT-Ye0E{V4u+9 zr_dkfKgoUi$-M zckPz)0&coF8r?kpf1?-rfrqxY?xgiILd{nFnT;J4@(uuebMQJEeb%p%T*c7#2HNSRg(k z#Z#_O2K0!9n!5M+@gb_^{rV$4>j2mI_M{eg=hj^ocrI4^`cgQLh|fSVNqJ-hISDeY zAhlbDlU0)j!1bzjrk@A04r%-Y){Ywn#_e=b?cIL>E8hRCXJ`4Y&vxoZwlMI7FOYkj z_JB87%CdiVz>lRX|14wOpzM8s%Z6`G=gmZm>lgU$JpNquF9Eba{jT7_j|X}WG%hnK zVp`wuXZQJN+tVVgo(!>vj>P8&g!syE{W;pho1%NqrC@Tv(@i`1!)NP0_GRk0sI;T# zFoQ@T*9Ln7mKU5B9<@BKOj6JOG|zWtWs+#Nn$9$zX+3Zoxmd6X`Y{ol!XN%K?qBr( W%AT)x1Au2&FnGH9xvXZs?w>th_x_^ zMSOt@h8e5d7Q{}i)oHcRR;we*fXJf*j;wp=j2nly3Ee$uGN`roli=!&M~WDiVprG&Jp$ zWUHr#evqd$OJAq)Lfkw@P?G8}^&sDlq5bVwT=eVn~bo|JcR0ec@!9P09isHi7aI6Gs9H zf{>N{O#8fYJQsv+&$i*DN{PR(sAT(^YQ!N@FQ~Dydu{bF<>90=&4bH-%l3z^3IDc&>4xs2WzYz+0=T0Qmf5jIQ2zE(hL`6+&*0 zUrNBomW!AgW)!jdmQoZI4mHQ9nUFd`??1bFX|Su^acTu=(YFP`mS$NmV4 z3MEAXw>3CfFw>D@Eq{!2#+%WA#p~MH-W0)-o4q4|V~`i5ApxB;bxJ0Iw|!xb1r`2$ zZ$*akb5{pBa5Re9BqUSjSf?rZ`&g%#kOBPz8jTx6{OjWZ*59Rj?_61zHRs`0jt7;I zlm{>;{?wrms?AqXiUOQ$A5Mr@)!K(`T2_)0HP&IuWGAb@m9@5EF%#%gvj*-^jszxU zi-9|h)Hj}MoWp|iwgv~AS5{%O2n@z+dAb1!4`_hXF0%EBDsHMP*0V|i>Vg0Z8=RI zHB~t6`nEAeqkdTnQAIcV*6%q`RYQ#T^Nb*Nd;DcUTfS8{UEyHq^?JHovY~@*O%ckR zCSHB861!DMW`tl!rQBso{;vV;-SMPXG$7O)XU-!5y2BcC>l`deq~R5R>f(*}`i&Sd z#^;qoqEYr$<8bj5J;tLkjm;l~5rB>!l^?7KaI!5G8x?8F51vvwqz*SLT>W5xp0G}2 zFKq2edh+t~ti+9#9plm=RG)8lOwpKFp(m}j?l@*sO2zJe{WL>;nLB@YD9T5NqZ!|t zJ5+91)k-+3r4@bxZQW{;G-HY(3%;Fhz|BTHP?})JzDOKFOa<~_9dxucmYo0mmR{zq z3G$QGK{oB`OMX8VRIy03eXqP|0#2dMgw_%IuBf6WW`{rjB6m+8TOQH_PYPmSkri^9|Y=w|1?BOvG*Zjf&bj(PyW?eTvT z_$c4(zKF)|wbhiBh(w|qz2P`6hnNAS^zrWg2#bE(jvxqZSX{=1HP#%d8M^x7IPC&f zvp`Fy%9Z!FP-Hb^$P@{fmaC@kAJ9rIZR+c14>z zRK93YI1*6!^hAt<$739BiSgXxa;BYa$?28F>pD2QT4KsXGc~fAGpi)ds*;e)rL1c@ zd*j@_zJmitqFFCE5^%+^(~pCDvV{tliT0j2dk#i;c}E{Jzh!5Bygfz2ZD z#7|3;-)1grp);*iwnzdy8>^8_5RPip-r-L&wW4atL&*W5b}`W!%v!iv%|b?2D7J|- zZVV-{>%FrKMNOT4!}L^v)^s?^)_q~#*xS!1$5ZLGewsZ=T8awK+*yW{KxyN~5DuG| zQ2|@c!bk<)b}_xK$lF8`;IBLT>FCxqm_FIU&t^Fph-nA{+;g2PId}OBomeCRi&@~l zKcp@<%WiZL*E9xV8n-q$nK4<9m-cRzO}qPzRHNy`Bl^7ZQJ}$izX#^738qTeYn&{< zX$+T6O{K{l`v*7{OqCW#0t$OR8Aws+*KI7j(ZyY}QA)Ypn8l<>lGB5}yL+8kI7k!o*+h ztHhp`=M`1YBwP1I=!@u3bJ#`dr`cFI%bC%o8;EJlTNC7Pi&DtRH~GABZO-1P+v9Ho z$_i=XzH)zM#ud}ZIMl3g-OAQN+l~RBSFXy}nVVW@x2da7Kk23~qWRm-C-jWe(y2n> z-MX5*ojIRZZgqS7KL%bgBC*0{;-VTWukPvRSI_I2XYzP6Ri!2zb|G0u1v*teZ;la( zYBVk!OT%GXvvlpH~hQ(q3N++%qzK;MIpJIiphI`ws*Nbg70kk;^1rd#I8_ z%?j7AY&DX79`$+U6^7gE2~YeIXd2O<6D7{5lsM6*78Hn|o`})frE+$)#Ka0y-sh)W z%0vE#xe=d|js;X=ipG;0LlhVAfZ?zUJpM|ENL0((rWsOK)_tibCb!350R(c*6$D1z zI?uCK1Im0}xpTx`RlZGpUO5QN9_cOM%e*6uFlOdc-jD$aBVGr7&sR5|@pE5F<4l`k_4!$PIIJ^l##o7L$UeLk=3C}hEC9(pQ)L#ILcj~V6j*8lbR Y528qoZf8$3GXMYp07*qoM6N<$g2ZRvm;e9( diff --git a/contrib/mesos/docs/logos/k8s-96x96.png b/contrib/mesos/docs/logos/k8s-96x96.png deleted file mode 100644 index f68194f5374069050bb282468740d9307b6044b8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5248 zcmV-`6o2c9P)2|U}*SEm8% zT~5h=3IJ^wZvy5552ktPOyEhEQwp8}AamwF3rq(L8QvlU{NCl1W}gBeRi@8&TTyrw zC<+gG3<5R-CEd*Y8~`o?_Pd;t(piIwoyh!) zfF@v9H)K9QUtk~5V0XAje`_Sb?r`4>{2J(Wk_D+1z@09q^xR1afZgF<0NkP{73IMyqT?pKeLAoF#l4aQVKq0?@|El4z~?h z0o>G?$~hQP_+(`ttmeq{o4qn+bC2k)iqz=0z#^AZYRU8l#ta#62Hph5b*3kE;`mB}_xB#fLJKS?M1;Flb z&y414x&;XW)Ha2Whp!li0gMJP8@pU^CwC|ivpd}ViDU>n zo#`IGLis2w(=Nt0goqcF z4h2FA!4MckTBGkYNCGM|wE@?rGm_OT@bbK3_WV&|?v;6nDMkxQRE^wtSuP)}=*>-+ z<#veqZE0Yskp zf{mg5A}-1G!?l8G%3^=8)OTzw-Y11y`-lXsSu($i)@Nx^QfTUH6gTqFsGyOTfr?-c z@m&>MwWON&w)@(>@BFKJ@Wd^Jao~CV3UA(D!hq6P6Iodo=EEJnv~5T-UM2&eEE5BK z{;h|3W>a&_|IZm{X3e4!a;zf$pn@$+TNW;6)6YeG!BzlulKHpYFPi22*U4pMge5!Qa}QDy$I6{Gf9Pn(!9 zac40JJh`EXOXgK@C_&<`E%Oxx?tZ$CS&!Aw?ClWq0WL{P1=$_$3($aUR*|RPYG(fG zx^}<2`ftrlUslzwXCesda+{YWCN++N05wfAmoBK_<6Z4{u3xvJnvL7MnfU#xCerS3 zpOKCPOw!8rGl=lfj^q!#b(f!q|I&!n)c(T?0_wT{6ay@aNnqw&VzjSYzyx!DaDr6zMP8*7%k8~0Ez;q6&d;Hq23AAAb`azphIf! zqI3WlrPZkZpu$~0%5UE{&lP!RNhv@6j~u)KB?*>-DxP8@$%s;uS7y!?d8}Df+-_dT zD8lq{IfS$=GmJ_J0Q)Sr4Jg!V)IpMoNn^57R#$iZXFgBPDN4x9NG?ZEw+$-LC<^q* z5&7%l5`Hq#mh!ykT#=VlN;04nZGezGbWS&7o$-Bj*Kc!@K&fn~g?~KKhjRv+B|l$K|AzH)@aU4)^3#RM2#-2F$h!{NTL!)&J2> zKQ}&dl>44f-aPO>w-s^gl)M@e=!Sx%HsQucYPfEB zHAiaW_05fwatVYrwFcLv0ziMQM*B&*y2wBN*G6))L|)(G;rbOvle(TiooM6TrKObg zG+?s|tXW)=&^W)ou!L-@z{oSqeEhpUlqX4Zgk_ktu##^?O&AWorpQ zTx_LQ4=B}Qq(D&N;t}zy|6p~P&-Vqem;?j?cI*%EgGH6B|8+@1FUFQ7^2WXGceEGf zi+r%WcS>_BpI1qX@3?+>_cL{sIB85aYd`j;UE4@NrAZ|q+81xsVk89^a(cYpUH)o) zTu!~RF3hBb6}0#i^seky)P*UZR}sVfV=8RrU*eMn4z?NNpZhEw)-R^0RU#6-de%vvG zGzFknboYN-BP>`!(F}kbYrKQ?$sT`7`4xi**RQB%(}U{%Kq#y*;qD58QC?+R1U~w0 z?}T%?^T}FFDHu^f5cu#*KUZE5>-iMsi6{zM<%W(e^k5qmq-f^w$3eON}- zgF43033o>dBgfJYzJS8Sc@?S#ToB;gvn(kERDuAz4+Y}>pJf)(tpg`bX8{+g*??J^ z0iY$?A%zyN%$UIz9-CFjg6Zn{Bj>s%ntk!ytZ9;2yVaw5%_F}oVAbqG zE*xgX<5v>aj|JdhRajLbHKvR7 z=bmY1_T_mrd1Tgn?BRni{d~DUK$az*+p-S9iQVBYaXF>RV*m`)diAS``O=>ka9UBv zLQ0cSU}!%xJN5@!^}0cXk!M&k+9Q^mEpqL68`qAHq)qCYW!1=82Px5siVw4mh)LXw+LB*n^x9fH48jCvp}=CtJRkM!AOljV@bFi_@@vRhXBx_E_+2e zKEd~wmy(y=rMs9kr`jf&i|;xTOC?Ea!Lb*Xl|k+ohw6O1*y=;M0*xRc?P5+ z1HF41$j=ce?qTR~;_B1Ae*W<PQe{`YJF>usht}2R5X7MIfZGbVdP}jn3*2lbgH> zI}Zfdxj)EPM}mA)72;rZh^l%SgGlScj2K|%nLCOy8sk6itf0v&qbLxT6+)p%>9cPy z1AU84oYBX~5Xr>Qe&*Iq-@RUb~+&%fh9 zkgFc3A`ntA83jxRA*SahyNws}*;kn#eppo>CdVp(00vROY>Y_7-oqjG9uD!he@3J^ z7y?;Qcx^!m!~3gi;=S%5r}r`@Y*U$9md)eq8k2Tgs5wJM;NozjCIyBDKDKq@pcb8;?I+Y~g0^)*piU`TnQH(Z)|$Mm)1b9jA=C#gX<8zaCJ9hGv;SSYhm7D~=!LGQ6Le>^3d*_04Igt;Y#=hr1d`OU8Q_7Oe) z$bc4y%KXp14)E2HpsLz28zc3Dx1%+L3{ATt0^2hUw%Q9$tY(2#>l;-+ zTQ;KrUoiQKdGzMQ-7&9yGrcGRPXTp$jbJhgI3BN2{p_}>`JCA&p2@NdSC7wCwGs8r zGN0~^tt*H^1nA}jrN-}%R^JITu++%hsp?tRIe)0tdz_djm>X6eO=|c>0d97xOTpY}d3gN_6GmpS?$;#=-O=)S z6}qIKPoP;Yr?me>BOV)?IozMfOzNvt5 zBjO9`+xG<+F(7f-`}!3}*|9&MC-dXBDygm~yTg5u$PRBo4@vm$*%lt3m0kmO+3iQ- zc3LK)+$@pJ%hGG`yZwn;-u>Ly(XB|yXqU?=#o5|K!(D(*N7RS{Y~AT+#_x`%e9bE# zw$SL2351lS>Oqzx$-?S}Fn|5KC*}Lke5{6d^ku$}1iMa??UI}Tysf7t>X9Sz^1S%X z(23~1FMMp??&IHwgLwQ(YuVFih$Qg_NoK~4u<*U%srB(Rw8%_fQcXo&SdWTVOxP2( zv8e+P1>qK8rJgej%kcZzg{=ki&Z1ohBHLO;{p&_Qb~&Y&lSR_B1HuQK(8r<|jUAG{ zUC5MXuc~3o&Q5v9&!1dQDd~YHso*~j@aR3$F=v@OZS|jCWbBM#D%xXyN&vW=QkY2F zimv{dx6R$%H6dR8sHMaA-SDZGx;C|ku>Nn6@enx}`DQ(vjfG7y_f%~h;%*M9Y@H^ zj-Og69PxZYvJ%(pDGFdP2$&40rV%`Tg)dh1Y5i>RbMI^zYJuS$nQb7X@QX`hD`M;Z-lB(EXeEmY*0s_5*O%M0IP|q&rtc8QI1Y3dpi9nX?;zk_EtuaS8_>oc zgy_H-b0;AU*Xl-^3BJoI{gX6K+C7QsQkPTuy4wK|1>ptY>2ERfm${tMTbc%ldd$-9 zaQ~YGCq$jZ^tsC^jnw+4dcI4t0m%E7Fu%&>lyrFmte(!_uy4`oAE?)jbpycVlnwzu z{}wR+haGuUVOL53E~oS~@bF2@{CiwZX`P+}bV>+hceqy)dAIP%MEALz(mi?)(rGcM z-QhkvDh-2(yyQ96v6bkL-XsH6z#ib2Q5AZMPSGjSmHrQV?-3ztp0Gv$0000kubelet-managed pod,

\n

network namespace

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":12.5,"y":10.0,"rotation":0.0,"id":35,"width":400.0,"height":330.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":2,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#6d9eeb","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[]},{"x":202.0,"y":104.0,"rotation":0.0,"id":266,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":44,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":263,"py":0.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":37,"py":0.9999999999999998,"px":0.29289321881345254}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-10.63961030678928,33.50100681951386],[-10.63961030678928,12.000671213009241],[-10.63961030678928,-9.49966439349538],[-10.63961030678928,-31.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":543.0,"y":176.0,"rotation":0.0,"id":273,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":53,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":269,"py":0.5,"px":0.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":37,"py":1.0,"px":0.7071067811865476}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-204.75,-58.0],[-314.3603896932107,-58.0],[-314.3603896932107,-103.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":304.7857142857142,"y":33.0,"rotation":0.0,"id":242,"width":80.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":31,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ff9900","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.777777777777778,"y":0.0,"rotation":0.0,"id":243,"width":76.44444444444446,"height":36.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

<bridge>

172.17.42.1

docker0

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":165.0,"y":33.0,"rotation":0.0,"id":37,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":7,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#b7b7b7","fillColor":"#b7b7b7","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":38,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

docker

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":50.0,"y":33.0,"rotation":0.0,"id":33,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":5,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#6d9eeb","fillColor":"#6d9eeb","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":34,"width":86.0,"height":28.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

mesos-

slave

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":294.5,"y":90.5,"rotation":0.0,"id":284,"width":154.25,"height":230.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":57,"lockAspectRatio":false,"lockShape":false,"children":[{"x":15.71428571428578,"y":104.0,"rotation":0.0,"id":202,"width":68.57142857142858,"height":56.0,"uid":"com.gliffy.shape.basic.basic_v1.default.hexagon","order":51,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.hexagon.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.7142857142857144,"y":0.0,"rotation":0.0,"id":211,"width":65.14285714285711,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":5,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

pause

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":15.71428571428578,"y":48.0,"rotation":0.0,"id":201,"width":68.57142857142858,"height":56.0,"uid":"com.gliffy.shape.basic.basic_v1.default.hexagon","order":49,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.hexagon.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#38761d","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.7142857142857144,"y":0.0,"rotation":0.0,"id":209,"width":65.14285714285711,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":5,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

webapp

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":-188.0,"y":-11.0,"rotation":0.0,"id":272,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":48,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":269,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":201,"py":0.5,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[237.75,44.5],[237.75,65.75],[238.00000000000006,65.75],[238.00000000000006,87.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":43.75,"y":21.0,"rotation":0.0,"id":271,"width":110.5,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":47,"lockAspectRatio":false,"lockShape":false,"children":[{"x":0.0,"y":0.5,"rotation":0.0,"id":269,"width":12.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.circle","order":46,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":2.0,"strokeColor":"#38761d","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[]},{"x":15.5,"y":0.0,"rotation":0.0,"id":270,"width":95.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":45,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

:8080

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[]}]},{"x":10.0,"y":174.0,"rotation":0.0,"id":220,"width":80.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":29,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ff9900","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.777777777777778,"y":0.0,"rotation":0.0,"id":221,"width":76.44444444444446,"height":36.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

<veth>

172.17.42.x

eth0

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":0.0,"y":0.0,"rotation":0.0,"id":78,"width":100.0,"height":230.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":11,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#1155cc","fillColor":"#ffffff","gradient":false,"dashStyle":"8,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[]}]},{"x":10.0,"y":353.0,"rotation":0.0,"id":286,"width":152.0,"height":150.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":59,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":3.0400000000000014,"y":0.0,"rotation":0.0,"id":287,"width":145.92000000000013,"height":144.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

kubernetes-managed service portal

 

:80 is the declared service port

 

10.10.10.2 allocated from portal_net range,

not assigned to any nic

 

${name}_SERVICE_HOST

${name}_SERVICE_PORT

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":202.0,"y":94.0,"rotation":0.0,"id":282,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":56,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":10,"py":0.29289321881345237,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":263,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-62.0,90.7157287525381],[-11.0,90.7157287525381],[-11.0,55.5]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":185.0,"y":137.0,"rotation":0.0,"id":265,"width":110.5,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":43,"lockAspectRatio":false,"lockShape":false,"children":[{"x":0.0,"y":0.5,"rotation":0.0,"id":263,"width":12.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.circle","order":42,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":2.0,"strokeColor":"#6d9eeb","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[]},{"x":15.5,"y":0.0,"rotation":0.0,"id":264,"width":95.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":41,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

<host>:31000

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[]}]},{"x":288.0,"y":438.0,"rotation":0.0,"id":261,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":40,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":258,"py":0.5,"px":0.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":10,"py":0.7071067811865475,"px":0.9999999999999998}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-102.9994829041413,-236.7157287525381],[-117.99965526942754,-236.7157287525381],[-132.99982763471377,-236.7157287525381],[-148.0,-236.7157287525381]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":192.0,"y":84.0,"rotation":0.0,"id":260,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":39,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":53,"py":0.29289321881345237,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":258,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-52.0,190.7157287525381],[-1.0,190.7157287525381],[-1.0,123.5]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":185.0,"y":195.0,"rotation":0.0,"id":257,"width":110.5,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":38,"lockAspectRatio":false,"lockShape":false,"children":[{"x":0.0,"y":0.5,"rotation":0.0,"id":258,"width":12.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.circle","order":37,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":2.0,"strokeColor":"#1155cc","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[]},{"x":15.5,"y":0.0,"rotation":0.0,"id":259,"width":95.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":36,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

<host>:49153

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[]}]},{"x":50.0,"y":173.0,"rotation":0.0,"id":10,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":3,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#1155cc","fillColor":"#1155cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":11,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

proxy

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":456.0,"y":-5.0,"rotation":0.0,"id":288,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":61,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":286,"py":0.0,"px":0.7071067811865476}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":252,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":-82.17649372239768,"endArrowRotation":-82.17649372213663,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-338.51976925964476,358.0],[-338.51976925964476,329.75],[-266.0,329.75],[-266.0,301.5]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":15.0,"y":320.5,"rotation":0.0,"id":139,"width":80.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":15,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

SLAVE HOST

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[]},{"x":140.0,"y":275.5,"rotation":0.0,"id":279,"width":44.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":55,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":5,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

nat

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[]},{"x":184.0,"y":284.0,"rotation":0.0,"id":278,"width":110.5,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":54,"lockAspectRatio":false,"lockShape":false,"children":[{"x":0.0,"y":0.5,"rotation":0.0,"id":252,"width":12.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.circle","order":34,"lockAspectRatio":true,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.ellipse.basic_v1","strokeWidth":2.0,"strokeColor":"#999999","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[]},{"x":15.5,"y":-7.0,"rotation":0.0,"id":251,"width":95.0,"height":28.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":33,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

10.10.10.2:80

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[]}]},{"x":182.0,"y":84.0,"rotation":0.0,"id":254,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":35,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":252,"py":0.5,"px":0.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":53,"py":0.7071067811865475,"px":0.9999999999999998}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[2.0,206.5],[-12.668996335391626,206.5],[-27.33799267078325,206.5],[-42.00698900617488,206.5]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":50.0,"y":263.0,"rotation":0.0,"id":53,"width":90.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.round_rectangle","order":9,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.round_rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#b7b7b7","fillColor":"#b7b7b7","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":54,"width":86.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

iptables

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":15.0,"y":223.0,"rotation":0.0,"id":295,"width":130.0,"height":27.5,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":62,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.5999999999999996,"y":0.0,"rotation":0.0,"id":296,"width":124.79999999999998,"height":12.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

ephemeral proxy port

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":466.0,"y":5.0,"rotation":0.0,"id":297,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":64,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":295,"py":0.5,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":258,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":-2.226088371282604,"endArrowRotation":-80.888785880177,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-321.0,231.75],[-275.0,231.75],[-275.0,202.5]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":15.0,"y":92.0,"rotation":0.0,"id":298,"width":147.0,"height":54.5,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":65,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2.0,"strokeColor":"#ffffff","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.9399999999999995,"y":0.0,"rotation":0.0,"id":299,"width":141.11999999999998,"height":48.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

mesos-managed host port resource, declared in pod container ports spec

"hostPort"

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":466.0,"y":5.0,"rotation":0.0,"id":300,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":67,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":298,"py":0.5,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":263,"py":0.5,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":21.082066264888986,"endArrowRotation":21.082066264590114,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-304.0,114.25],[-292.5,114.25],[-292.5,138.5],[-281.0,138.5]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":210.0,"y":160.0,"rotation":0.0,"id":302,"width":74.35714285714289,"height":22.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":68,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":0.0,"strokeColor":"#ffffff","fillColor":"none","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.4871428571428578,"y":0.0,"rotation":0.0,"id":303,"width":71.38285714285716,"height":24.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

container port

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[]}]},{"x":466.0,"y":5.0,"rotation":0.0,"id":304,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":70,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":302,"py":0.5,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":269,"py":0.5,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":1.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":-6.405570867964701,"endArrowRotation":-6.405570867687305,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-181.6428571428571,166.0],[-154.69642857142856,166.0],[-154.69642857142856,113.0],[-127.75,113.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":220.0,"y":408.0,"rotation":0.0,"id":306,"width":192.5,"height":70.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":306,"lockAspectRatio":false,"lockShape":false,"children":[{"x":62.5,"y":40.0,"rotation":0.0,"id":147,"width":130.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":19,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

forwarded connection

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[]},{"x":24.5,"y":48.0,"rotation":0.0,"id":148,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":18,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[0.0,-2.0],[17.0,-2.0],[17.0,-2.0],[34.0,-2.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":62.5,"y":20.0,"rotation":0.0,"id":144,"width":130.0,"height":12.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":17,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

socket listener

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[]},{"x":24.5,"y":28.0,"rotation":0.0,"id":143,"width":100.0,"height":100.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":16,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":0,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[0.0,-2.0],[17.0,-2.0],[17.0,-2.0],[34.0,-2.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[]},{"x":0.0,"y":0.0,"rotation":0.0,"id":152,"width":192.5,"height":70.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":1,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[]}]}],"shapeStyles":{"com.gliffy.shape.basic.basic_v1.default":{"fill":"none","stroke":"#999999","strokeWidth":0,"dashStyle":"8.0,2.0","gradient":false},"com.gliffy.shape.uml.uml_v1.default":{"fill":"#FFFFFF","stroke":"#000000","strokeWidth":1}},"lineStyles":{"global":{"stroke":"#000000","strokeWidth":2,"dashStyle":"1.0,1.0","endArrow":1,"orthoMode":1}},"textStyles":{"global":{"size":"11px","color":"#000000","bold":false}}},"metadata":{"title":"untitled","revision":0,"exportBorder":false,"loadPosition":"default","libraries":["com.gliffy.libraries.basic.basic_v1.default","com.gliffy.libraries.flowchart.flowchart_v1.default","com.gliffy.libraries.swimlanes.swimlanes_v1.default","com.gliffy.libraries.uml.uml_v1.default","com.gliffy.libraries.erd.erd_v1.default","com.gliffy.libraries.ui.ui_v2.forms_components","com.gliffy.libraries.network.network_v3.home","com.gliffy.libraries.images"],"autosaveDisabled":false},"embeddedResources":{"index":0,"resources":[]}} \ No newline at end of file diff --git a/contrib/mesos/docs/networking.png b/contrib/mesos/docs/networking.png deleted file mode 100644 index c9c4b3a794ee5bb352f5eb1d829142b80a7ad70f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 46650 zcmaI7Wk6I>*Y_<5N=TPl zE{#G$7OuE_LczLFKxY_#s;Q-iWr>G}d(G{cNLwcQOpVX2vX+jHAXj`i4)qTtHu$I) zz@K;GKe}M$2(r~7SRWDI!pQ_;#UZglPo@6n!$$;Msvod&Or`3G|Nj3+@CN_?{#+^$ zqD7fzfcJk|gblg`{`adN!e8K2NU$|C0?q!XK}hn}bFcq8&&7n(fVG(QquKu+B}NZD zaMAi_5gR-d>9dVAwk>xA9l9-<2{>9BC!=-n`8Q(Uus(vL+2LFIh=>D`FW1*D37;)S zqDKJZ1Y8$FIiO)vu3H`dTK^k>kOo3yi&6e@9gyLK`mbpi+4C2_^%41Oa$hAqoAf!C z@&{~?$ab8LoOk$N7kt2Y2Vq1pXBl^}e-FknKZ4D{X!Gl9VgKn(7HBvf)Z=92lJjrE zkci-wDwQh5Gkq7?{*7$~7&a4*E#Ii^;eTdYTrv=n4B@s)j3u^so@5FVU>rc3{azz} z;C~OSfJx_ozOp}c>iK7B2=%`#fVmTht`&(S$UBJtdk>|yeAIh=IYU{qPvRO0`)2H)TpqZCN)%rrB7CSBT1W&(HZjVzQ48&8@3~*akWl)OF=Foi=J%QZg$sy znC`oRG|>_+GQZ>XiZmPi%`8eca}WoWdnr?ddE*cWwo zM-d{hon$Q{^YXWt!@x>JflT0IGJMny%Yu+V8d|O2x6{C5ZjL=%{(i_N@J8+4^&KTB z;0s$Hn|Kt+JWcVOi{DOBss#B$g)Gy59!G>0JLc#{|i zcNX*O|Al;7YLRxNl8m?+HZ6J_{PteKs`{a~Uk>qv{ktBa#UTuMaoE92zQ+wbrVm#%Bh77|P;zu`+GT zE0FQ&JdV_zVuli-wY`UEWsKDdb0K&7flxzsvLM{BaJSo*iun}D#&pm_Q{f&=kCG!^ zyTlB;^L<~CCbxLCQl&r1(3fTT^?{W4nJKKsBAwAi0j0EBRn)A&r^gA$tk3U`zahM^Q52wX zb_zT5b5*B9waqwWW?4lklhAOz zt4Qau9Lb>liEFg&&~Zb@?eHfn0+#_>If+6eEyy_F!A;(^Lc1zqEQCKh41-+dsN=5M z_fV-anMtP@FgtW}G@D7DK&m`5h)JtlYVI(N(-M8D-A4!e+vEO0<1*bqt<#!}u5p=q zQ7NzqzLT9y$m7r{#ch1`IpmlB?zTJ4J~h*S)*WwcYT0RZR~-U=-U$b6J8egak&51- zV!4;PsqghSlZCQ=V#BXM$5320*EP`8Yc7UDJa41e{i-%AgX7p%xE=Of}arrBwHAc+aQ71v|pw zQnS42yjcdT(q-S z<9Su6y-&s^24geD3ejIaEM6mT53i7(?`3tk9?p@S)ZYYhtb4``^1hWT5^^Kh8* zKG}~&b6KwyqFs&~TkTcdZpdoH(bp28)7FxE)k#J-`_%U$gvamJHG{)^(A1}rZG@`* zVpjkBbcl88bYn;un@Oq9cD6DP*dj5xCp^}ZG#MZ!%~CPjU)9M7$ehqK&NL5`l&?Zw z9F*$#eX^boQ~A=7>tsxqu6c$f$`yatQXn%9ththD*lmBZzU>iw=~V6t+^)diVNhAT z)>OkZ%Qv&OI!TRYFqxdzJgp~7;?knK@yRHmE)##^uCtI~oo{y*(z52ysJpwe>V=`7 zOJv7xH)oJqD_m`D58f0^Zzn3d-rdq>h_0gzCbiaT5c0}>#a!q9t z#)7zgyTLi_J#CV1X?JJ&PI=RJ`s0uJZA(OBJ8}NV8O%aGb(`;U8s0*xI-{g{DaQ8< z28twHhYyQM%*yySJZ7y6&VMe%3|Z*3-9WEe>F8$v#{8J6&?!m>Ew`eTasfUtIFnnO zHE#xT@c`K}L-y#5wMkAVTWmc8jj-W068eS2?BAx(4&Q&key1Kp!%!yCBc9Z#Nh&_7 z8^V8$Ur!!DME}!Tb0{jv6-T`l+P{=CWj;YXZGQgUwz!rZnh`eKfS3k)Dar-BVnt}g zJmcM=s5{eT8tOh(hOImg$H7RwZwCN(#aovSY7d!x164WD4SE#zNt;KJ2;tddG!!}G zC0+a;$TRY!dAQW@7RoxmxNtfYTqxWmyC%3*%1GJY0yb1}@&4Wi#xI{;WdffF) z$)5GZ$-ec#G#j~#>oz!2Yw!EC-|kjm=M*WX-bQI$?9ROZwc@fSf^P=Cz-n@Q*9Y5e zR0IBvt;Y%TR!wu^_2>OyQ+SLrnpP3w=uN~dk8jkq*YP0diFkzfwHI9>r;07v=gw3i zwLbS8#fVo>!4q*TWfbdXu#J`^^;Uh+6YJBE*Kb7g-)SI2$)B)ty9h6x_*mGB*Lkrb zw{ilAlsk`^w@!pxeJ5|o+DHXlb48!-69IQ5Y1$W+M_}Z+!cI6cyHg(QnBW{UZdPPi z^Wq#!v}V^m{%7@QO~+-wAwQjVKrYdyTi8qJS7D!&{QUgFUvyLpf73#%p%TH;A(_Cr zFepvXPd#5}P2Dba@}{5CF$w<=y{dYD45>V9t>{m?&(y&} zUQza*kWdNE2f3Iuz(KK4*GQ~O`l+2_+E>6}}pu-*H4 zFf(R2999BmC^97q`{wavFQHBrgPV%QUR1Nx{n?l(P4#Kw8I}vYX!$(1K};Ixp+TJ! zdW$xHT*XHcj+X()1y5rCLYK;@LZy1)xobZV(7H_@?3W&rO8g8RwG%>KEhe?J8JLDR%J%PMG83m=mf$f%>o%TugaHO zgtjx~(#Y6C=hzrh5z(;nH5*srkpx=BQj3v{)0eMRq`iZ*84tg^?RD8UsI-5|QuW(R zw=8v7?of?uPj+7#rKHpDYzIHTkUYLj=&!-OgpyL@U6g)>w-%C3v3LjuWfrGfa#Z+Csme6e_7@8w0=)PQ)^FsW{B42B?&+YH zw*u0AI+mn9s4bXPp%HK8wSvo7|=-9_sVXxNihrK4GNd?5I zs+qCtw3Zf;MqZ1*&cXqY`2*q4#6~Vd^ZeG?mYE6IFAMhZ@%7J@Fv7Ufug7o*F05R$ z%9%WBhl|bablL?17bwS+VL$HMy)V3}jzf~&g-3~JLBbO2wy)^4jpHMGH@ZKA+@kRs z!vc>T@d&WP@#yrdn8!gT4bnyigkgMDY`?SlU1SbJcTbEj%5rh`nw-|wolvX_!{>q# zI5{p2#YZWuw{Vq8=;>zrA@#uiJ0Ia)ma7#O>4@`xxX@`HixInRf)<0I;4p_yrsx^1 z>`}b8eo7{)SG@$r=go$(inurxvf;iXKVZ8|&PGX*JM#O|@o@v#=&%SjXAql-MKI?? zsaiok1aDNHZAilQvv%ZzPMR|@w+)SHQ;_De&ylcv)>5lSiDiyoipbObS$|gQxu4#C z8Q5uhPs)dlsV5H^Y78v_+{9X4quiTnIn29_&ZkE~Qhw(Um|wj*o54gFX6UF{WM?bm zk$tudo#|GmzWp!ia{QAsP(ak1eLTCsG{es)9+e+!apcCvU^ZmUVoRTb`KtVG&*SeZ zk*d*4XsoJ{++%Dz`eMmh+k@iEHOr{DRrua^NxeDxgU6&j;45R))%1-BZf2zk0S#Z} z1BE%CQdb%BhS+Emb0%cwUMQZ{m*y2U8rEILHwlVi3ivH*&tT5a0*a8fQJQVrY<9_d zpYkcCRx-5G;q)JXAP zHe!XdB@-}rY#lKuB!)3m&5|vN z`4-PYq<1Im?RXccMMK&jFn9Xv?0?GH2zUw}fJEkxi%BB$?+mgaGnU(SI&qL0cS&r% z&ww)traBER`$S4k8~1Ha=}RGSHIY(h9_NDgrM1Z z<=n`-0m5C{T2-nPk+f_LQmSkqPWXZReB4#0gWgN*9X1eIf`vN*qLIw!GFh@kXf8Jo ze=gX@oo8BET$WKBdMHb?Hj+ptioBCDU*4CMu4th4`2TAXPke88me3NMy;$|mu>BZg z=FW<+2#+8?);jS5$hPi#Fui{AUur=KP&pk>>vRV)Go5kWy7p{STM_m>l5Zp%Fvgy3t@m zH~W#Yro#{jXW684{o?Q?P&=L;?_!s@f<4>ApQ2t+@mYjg$y3Y56aqN{W}Q6dQeQMN zgHE*p#}62Asyxj(WnJw3lI_nV<8!3Q6Uj=Hv64cdECWIF+w5OoPzEB!m{7h&?gmTv z_}}i7Aj`r&pcC+zjbtRpkO|w^ORz!pdd8HkXF(6I>^-%Z-9xy)@Zsc8bVw3UzKgy# zekv5pHob^Fr_^BLxh7q;tD@_^y0K?|i0R87B2re_$SwTMW#b5Q7Ij?Vqhg{YA{eN2 zTI>6c%7(b6^qp+|CF~1mEPNmomGF<}8d`>EKu1)0XeTubTQ}Km@!y)xN4VwMP=D-JF;_mJfatGZ*FrC(|GTLaOj5gFw!6Zhiq3XNG^tYMe!vM$Ep?W zLN^PNnRPkp*dI6JudB!eUh2s=X~OCBt_X->zLj{p3(MfK*K2WH3G#il+VjY?`fAPL znjnJ)J}MCNq7a7(tPuPLNS5ChxKdI&BJDb>j7vGG1skD%ry$23Ai0oimG)BzcN;^B z=5crb@EM&bnQZ6E>*KE%lG>Y5YQMix%D)SOT^b_GkDF~_Bm zp%M{Qa6)$rQELxw&bBt9rNZmXr7GF*dRCA;&UPo-FlSE}n_W0i2)NX@)(3ZxS2#^> z5#tLC@BOcjO>8k)1GEYXP)RjS5@ftGb-PE2#Q~)G;X4ysCdQvxi^|SH*zU;wC3~z& zCE5_(l)|RUU%G!m0P_IM{vh&3P$eSUu-i}znrtS)rR*&G4AM2V- z=Z>$3<7?#@vc-A&Z^+^OUao*3mx2cD z=qE{p#tW~MEQ>ohjt)ihm+RcL& zY36@nQQ;4fif`$~=jjw;VRLt|5g8zFqU9s}EOqj-)(!{^i_WmaF+9G5F>p8a5%ypI z2GZ(eAWEe+l{C$nNv8n9BxSF&&1Ag;Ag`jlQgotxIq>$b8HjM=rH~Wcq$``$kVB$) zLT-$(U0Ymtd~+xl^10?`m&FN(Y&C78$I{ZV0s$n%r5#>@K_vI0vuC1jIM>DK7ZTr^ zZj~IkN@fD)-+wWlCz)|Q0lAoPJVjy<6I%{VRkhMiU2z4veZODOLYb@mTX6E!=e|DN0j^IfB8? zS{zy56OJAAocmr{jCh4Ys!3fw5J9(YPvP^;ey4vl97z|Z4lO3S7)KzXD4{icLpO^_ ztDP3oU)3gYj44Cl3LbCm(W|_h`HjRHXEQYTlR+)y+h@dgT{Lf|7O62-WP5HvYjBCQ zVuspFN=m%C5CWmkCdIhLe~-H{^yW@=$P<{zmO7BLBKed)Uo0&k`r8-Xw?3fQO9QWP zU#S;9<#U~=#Ah{3Bw|#=0@x4y$;&Ep$RL7AGZoNSg=W~bOoqDJL!#hUCh~p=BIX;EYYs7pJaI6|CQDS_QKZtD8 ztSVQy7@Apau)a}4=hf)yOI_@tg#Q@(oY0f0^zwlmcuuho<5J?J2l8CC_*X};7I85) zp!g^Y-uMhg=>E^w9ltIw@_r5^g(P2ryllca;hzf(D?**k4m`Blf(gQ^_%dR6{~nfa zV*fG)aIqY?t>0$qXE&HZ{Fwx4{ghf&*pCESnCY;BnneY={p+l1jH#|{(aQkqJ&1Y!b_~aFKN8)J@qevwyIL?Q?&83^)3ESY1Z{88q3;(mRA*YZzmm9FL(mm-X(V#&F z)-}fStZ4g_cay*gCcHg@hTRlKZY-GsMUm)@8nJ7SOmtwmdk7sk{WR+1GOZKO zh^TXIgg~a|(cG2GoB`Ojc@m*kR{$aDAB%wv`czgz=TCY+r}D%sZp*s*CdWg#5yCGH zms*oV{JeMPpw!A4T#%0YQ$@YTPow~pj_HdclyvBP-~k9hIB+^Ew7MVU-`|`K`Nqe` zbBz7?{Bq}fXDW$FhlPmS23M=}!{^8AH4K~CO2TJCzbW|$kBIVi(ZA`bz;kE|p$&m=qN+;ujD1F{GLZ+xia!e3elqxC3DlxIl z*Z{0Km@UYwRjvu?iy>76j+Y_=W8ZiH-72wm-gDG{TFS5iQhLP{=Dc`n*~xOPiuE82 zZ2!yOmezmTytrJqKgros`+Ldg>4H-k>z*m_=#< zL&;3^$ZgV*_~#eDj5QiREg`uH&=36S3<%gA&kKgV3K4PHRGLZ)iID#tL(}I|iD6mD zh{g~l&ys7X;Z~gn)TP4lBCo6T8{bh;1+Bs$XLC=%R@k%(G*87PFbIy~2znvE_qkQM zZf!9QIwt$BLOc~>HlI^a0-dq~5c=j->eea*@eIatcuV+vUDpNbK{VV}eE>?2TQ5!x zHi$cVe7HLeh6~^w%%PFT7Vx=bUReePBtNob7?4P-NLR15-sye8NUQpFaf6J>4Bny? z)pOO@n2p_uOCT3D#KqK|W;XY;$ynHXTe-`9?QQv@!Jr>7A_-VI>*{Dm!s8QAK*%@k zkHHG3wE%!B?gIdw)z?DP-|8`)k_&rlV#jhq^D8Uv+4o#&U!HCUeECUmFkNk^|A^+i z1KU-S`Fa{0IF=*I3VCa>QD;3h=pA(%iGhg8e!3jQiK)SY<}xpSkK(&KE5H7W?$k$4w5=WxGM zlCy3^^|c3rgf0v)*us~u(dE9+163+K7K5~0@%5jH{cN4O;F&8YlLW-FPk-9d!_bK7 zM!KuHvfe0_t*W{ibwDm?oD1j)zQ=Am8dng2ah>1R)wz8 z-?y(^c)3gk>HnPvB3bO^_|Af3ogWgEpZ#GN5;SabU6X-{w%dNblC3c#LH6ezVzozkU+{Bf+_84{ONt}xl~0eO6yP;-Tw&fTp#;9A443%O z=fmQk*Q9>-nhJ79(2OIN)Q<>cRKPh5y#8HjtmXKp9|y={?8}KTW|4oIFqo@LJ*CeKJjd zpTlGUuBU{CDZJ6sVhyW7*sDd&!BQ0=Hhu4MH4oOGg=8+1zBcQ+WX)x23MA2@h)DF06Jt#9{rG+rbdc%BNKHK_eLD^G}5^EC>&kaYAy zh*C{sGjTo7Ps{HeE@2e;Hqktqc0d^jFG1CD^~a~q_r{s95^hU1dYvs}>DkO(b{pQ} zMfSBfK9Q-sC!rS&LcIc)H_?}-;)2*rErMVelJ`OVFSGHuxeQHfxzBw^V~KaAThl47 zyOW|}%ANW}ob{e%F1gaR(_}amUf1yXPzh${ZX~N0BX}9KpH#YzWS~~@>nNz3+_RNnFI;uFEJE+5Cy}r&noj%|5PsuEr;#OsHB`l;Rz4}{|A$Zj5 zM;AkFlG(ObXoa3B^XlAWl~AHL!FV6DAb4s49}{5wN(I+#GBA9Nx_<`Av?GPbemuW^ z!4|O>iSaVzSkl*IYh49h=a{JXjJv$Q)Q}XzVGX+?oOS;Ir0d;^N6^3or2^wgXoEZP zHvi7a0fssEYV_H=9duO73Zgp{6ZP(?AKmsCY$Dz?5bNhuFSjTZ=#3PE`ImBpWohcK zmIHGBK}~d%y*K#LB>c_;@83$r-mQEgH}jkXTqp_!`Yg5ATG3zd9+B2T#IxS5jQwU$!=enwk6>ocf~29NhWueRgvbeZ*RrLJa!oi3p}rV8!cRP^sZ zOf_v1vR{g*y4HGMP&|lPrZm!LDIe^blP||s{w95aNk6} z^#8U@w2)99D)gc0Gh1d%r3#_CMR8enirRN6hBujC>-~kI4W@Gjr8|^@Gh;lMsKQ-) z!q7M@_(e{k_Fw7) ze=mUzV#+cr!)|4|sewoC3FO4i0R;1m@LR!rP9Ej{*%=}TY`;y^*G`=fetpT>PK_sd zw>4O4k2^+i(Hik&J6&zm*yse&f)QANwXP)c>8z^W8prJhA)>BJqoqZzglMAoZdOe;*Vg78SgdGHO$ln*(>*YtHdVk~rR@+iP~-Vy8y_ z$p38rF78vYQhMPOO%}S}djNPjZMThLegLC|`UtekjP^%0`Jq&4bS~emfTBIpPGa>q}M&3KL zWgE`77@uo8ZWX+vOo0kGBQwpTQl41lw&-vdg@#qN+#Js96@g{RsLn47M`gSAw3xOv zyqt<)G?}MdED27j`ObjOIs+g^m4A$}rE@}~y%dXAa2v}H0cCFqiIjfBb6bTW-&iif z5Qp%YO#vE~9D-tS9DIcs5bVBZ%U^J%Jkt#RW6CYY#{;)ECY`J*SBCK(Dcr3w{$0P$VCz+Sno0S(AoRVIKO4xVJNZF6rS!h@ZUw~GGj zCO)tA7_*q2MyLmdl1VbrD~kha9}nsou8T_VCqM6$boeYD_JdX3011b>DWccvHmNWD zw=I+dTxWjUN=GZ0Y--JhP^DtZ7+yc!BT`!5pjkIknz$Hm@>=zxOf->vV(B1mIjkJ} zW$SE_f+WY3=rv8CT&>wp3G1oiWOPW2?yQDz%NaJjxYACOX8IX+%<)RKMI3!NHx(6? zcP!qixY3sI>|~LmA_mxasCfz$uoMtIC`DKF{U%X(5#f>&5Jv^}*|=8Ed^x!+ll!)C zCBKdN9$`$*s=7ClKxg(PowoUh+X|{7q6FR`@w?cId~#+?)u3C20S}_F8A{W2IO`E13h8Y;6-rF7(dGieU7ihb7gQ8#gR-bOA%L7#6yw;)&R^ zKuE?pEn2_Q`{c(&SeZu2U%ULce-eQaud;~Q) z7UPgh05YHNntn~p-YH_=tWC5~y@n5078yha{&??I1)d8TY^If;d<3ImLNoqr0i_W>iYnhAUVUPx71ut9WFYZs9!!a2Q=wqt}hBk-68 z{lHshNv|(j5_;?w!2y*<-^qOcMSd^n=g#5I>ThpS z`HO9Y08s_dBKS3N@1z@#15xkT1rHfiMRU{%BHX*kO{HD%w(V>9%o{rPJ?E6Wk{tg8 zqu{ugDmsqW64Uhm?XgrXfC9I#CML?|5nu);xUzq9bTzzGE=wfpKkb4#JF8fZg zcF9CJK6m;YD%%EzC!4+&MXxg)?VFcIFF-^+`C#JT@W0m5X+ysr&P4lBGu{Z+a0<{fzF z6-Ve0$$zrg7cim$LTF1YdtFwYZ`3S*=J^mNO$g(-gP> zb8eRCcQ2AGPRz2}!@(};0)#q__d1B;P z;iNJH5ZutHAu0fi&<4ODr`o8PR4m3SsVvz-B}AO~9IH&N5r|PPZKKdQtO1EmW#FwQY~^SQ$tpYM5NG|EKquw3VKt869IZ1&cG^4#0vqW(5y{%8qybNf1H=xe?O>MtLjA`>L1e#zp+ zV$k&M7rGMfkyV}dld@j>5#7oY?-N^E!#!JwLfd>a-c{Ar=&P$U9xdppBKBV3_(OB;bSc#CI+WWVq=B9!E=VjP6!xZGd3-V_^a-^z&3<)xVoWR>P%t!t6e ziotMDG?MKxVo7ZAEB1M#?r{$D_CJefD;!JtvzG+pY; z;eCr$wj270wJzXype5k7(?PS8lAP#6AE?2yRQTzO6*%DWh8~bzLU=wczI#uE6cgC) zwm1|SqDc6xUu)} zG?45LUkUv-w`z>u8T_&v1L46q4$h9tAem~$`G14Bvv^lN@_!FPs`5{tj(fGD8n3ce zt=E|aVN$2v@9!)_q_`osMHI6kC#2Ig{x*{KxO(=}S&^=L)iOsqs^ZO@X@XbVtl-}F zB0uKdn542Af90{Cr-2|EK>;0EGTHR?LagZHyEWNhvG}Y#QZ6e@05V1bGOu^pYkvzb zZ?oaU2+$ z@ElVgr_}3iP7vWSGY7{h#k%Ps-}b0PcR-=!P&hB#5Dx)a=ARZd*ZR_SNri8r)c4QX- zSE~n0uQTw$0?k&s4kAF}F%-AJIuH2rm2bp^W91=bkTJiY@JhRJ`obQ=dviOSYMmz` zIeDbdsr&VK^QUFOrt%uQxR=_h>O<9U^Ba=Wo+Bnb-L&qfe~Ib3YSe(Lhx|>=MLJXKV{` z<$peVRJcy)wM_$I;c)mX238!~xWFxom%(CJI|S<;WxPa8m%2W~O%zwAuX&x@c^m(u zJx*|aQ*PWTQg1a86oLQ#Hl%M`DV&cC5H-D*|3L8`lO>#hi2Kt{mp^?t0kYXK;MJvZ zB$U9R{P=p?09Ptn!-o%k8=4!^hoYOOMcu)Xm9U7#RHxya#%EhOetiu6yT@rPqYL=` zo|oh=;}(W!FGPlq>eW!|s1qYj0`Kimc+w!Ret<-dHa1{eb}NyA z%lI4t2hgvA9xC3yR=Y2Nv#y0!dub6>gLMF}7;FXof=(JiHUUWNCij2WmH|xcG(4P4 zn}ZID&!8@qz)+_pwRD;!woTz~bqj|#1Za4X^-`+k*L9Z*gI)=A zR;#~JLnG~SV6?=xU8^f)Kl^^;%zGa4_PAdAlze_g>jM1|tWyYU-4_uh|5UABfJ3$r z&d#0pnp`0MJ-rkYyF%&vt+}5`l&u!p5b>aOU(wApmyRi(@{HmaGkc+U`(85A(pXey z)8$%JKvg{a!$3x{GK4dHNA-q)ozMEiwRtM+Oa-Ci3 z+XZV@0?1AsKzLpa(^P_iXs?}KyW%W(h!dV2>v$-i=hFrrAlQSWAVP64*4YACA|06^ z#C>p!nMvRNi}$8GsD$wmJ=$d;j*`O$w@xgJKXf)ycm%krGZl_Sjg-NlKK503JQC+vBArt8`|bMp1R5oLOlK{2pvv_Jr{aJ< zH~r_JR}TT4RUUP!K?)p2oHK;8yqQx(sKont+K^BP9vO~CVD|5TV%Jw>M*LGiEjsvM zlh(CVxXoEXO>ptcqAJx=T9T?=>(a2wv`Peyf1!N!Kj2ob4a8jNg#gnIi^6+jNwu7A7$E7}hEa^@7EaQBg z>p#D*^`OwjJ&yvn_~|UYY009vX{k6G`1@x0lWE0?S{`K=;hSQIG$@)nml_wLuCkz~ zbHD(W?V(cq{B+EaYx8@Y9IOwFqj=A;%)Cv!iOGGiSd+od2Dz)9TE&=@)8oC@?nFVq zDIB}Ox!kPi)-TBXaj2gp@G?ERRxjyP7;lnx+Eg+ACzx#jX@QVrC>p3fYDsqM8pQ2! zjtQ-4>!Za0n9#{eAcxQ;duCJ#eOe%!7b{x1h zqPMkCDDdCk;sUDi7m(Tz1b91?h9kC{mbkUmpd9~Oc_VLfYH3-}cORgyf>Yx-{~OBB zW;gHRBV2khX#n#wgQ`Z&SjZEA+ja9EbH23Ni^`dz0sey-+!aho#WJrUh_bzR*wC}jR7(Zx^G&cV58pA;y#Uf+Yx;cf(RAh(_ptNTN1AusxlLhM)(p);osjRuFb_Qxa zMdx*L1ITLYwsHcDjQnFdkAByFp`xa?o-C9nVvz<41qG(Zf4ef+f>-^D0XS(3fX2|5 zSLIZOEw1u+>xrr)08F7ac=46r#RB^(*`~<>kEo)_*U$yH{dKyr14wYD>#Q|iA9^Bs z<&6Nxlm?KykN^_1RD!(aHm@mWuU1-qPSl&tk<5*alcoV-nTT1>mmA;Ac)@PYX7F6a z%r8bk8H5-fijsEfz%{p?K$<4h2C*N^&2Lc*tMs`0BUw92+yAFXaSQNIQlM%#1t?%w zOZDo@!TOg#XtF9x-=~qBhQ|EW*?tQb7 zO1j!+=}J~05LW{nym|mQSSQ;AE-#V-H>f5aawq?`8`^_CK;%$m(Y_i zRPH!`DADs{2$;>s{e|_#oqS0qhjm-o@=;iRPvj=@(_)8=hIVv*aq|4s?*LVeREYA< zV)9hjn2MpmA>L;D9>$anZjG(HtvUDtWw)OLxX%HEQ*7|nm>*;+ z^dKD5F9wHR#cGw`u-O^$Pfg5FDyyP1a4)MRh0h7x67VE)Nx0#2pi~KzGmC*Zj6e6{ zPm@!y*tb!ys9j8gP(k1s-uu_{V)_HDX5)Ztesjp~n^L80EEz2y^U82}B#r!xUEBlz z((E{yxMxs!nPs-PCS?G(fTxY9+uL7(8SU2{_qyi`cIEml3>Vom@vFTMS@M~uEE!f( zsVVK)7`j>2&OLFD!=>UQUFwH90A+99tZ_`?iGVD$*Gsq=BpkB=Y`z&=Wq~g zCU$;Wg7L`lxva)1c9%Ovk$*>7INBfVdq3N4*DPPP7YRe#=-;!p;F!BPcH zIHK{7Ni;CK9cT6Vbyc+uksGrGrn@JVfNT+pK}`sq+`8;-^RiKk>w(bB+YkHf{M4(o zx!f-?6y7)9e`(GiYR65^Sw*tH1i1C~YQ2l5io5F-jU}4H~%|6oIRkn zv-(agKV-Jq#j^Xik~kH5UbZzG7V1>xEWo=6QLj>s{hgTg8}^YCYgD{}8g(2(HlQ;B z-l_ysAOs<*R>wiu3uMsU&$OYEn82BTAEr|ynHEUO));Z8To1@jQY=QZc0kR}MT(rY z5NneR5LP+nUDXM$`@66tdo`16{UKe-CQLL zHoN>)+!sE!H1GfL#AayP$eygp2+dwbFZ`mN*qt}u`T9G=BPH0M)RiVfC}&J7MPN1p z8LP%z>s z2eET&tEprJnBq5fA3z=H2n7b2{emWo(r6I4h7RJ9Tiol$(=5tFRD{tMny~D5k*Ip32SR6&>nCG$^9B%L`y6a zn+2bbrq+!2qsc}Jk7=B{H>XU}EIvf5ZtrmYF6(}BR&=RxnS|8aEC*e4INIOtO&`_iSyhh0BA#j%=@Y0{rN<{l1Pg|C+*-Uhy%ey^=1ea( z44_QE=e&M?gn@C`{wV2<_8{?5=0<@`;mTRKwXH0kZw1cubJzxp#5mX4FX#>B?B^B$ z?|2f2xkBx?>vg=g!Yg-$xAAEm3v8h>?;nzO+r~rYwf86=Q^JjQ6OFAZ56ZkY(4EOI zr+e49?w5AX7rNZ0?N0?Ki1z5uTI6*_D+NPRK0U?UjvXou<-4mzz{(2Kn>GKS&NfuhwJqdm>!}>ucl{b()-tI%)=wvf5(|~IT!RSH z#d;pz@}6k)_y}!x+uB4NPXX#ZAs$But&LWzgYn(1SHe4Ji{=T(hj>OE+@a|=?L_42 zB;r||v%bpRP=$oOnan2z<(to!fqhRxQA9d+vh*w@(JZgQ+}@kkUzHB(l!mg|CoOTl z8HvgooratzjsfC`|9C3k^4}Y((e?)0U!#84Ze&BM0W{$2izCI`^Q2E(5k&`l- z9BH3(h)!P=cHs?jNcPj&169PTln?CAd`_32`q!XH=GCg{J7Joc)JZ&1Nn2gI(PIEM~ z5z5baZ;Dt!rwv2-I%}N{#I;*_z`Zzs@E(6p0!TE|M}hcZjnKrRd93iGzlv0+kV(IRBl7*VIyhN3CI}=FVbu}7wk20_AHhAY&Xs-T9;&I zJdepLx+)t5*~Zn0(I1)g1o0kUs94_l@`ieL8in|)=%k59W={xPf~;$RAW}rFb@dYq@!35ewdrWEpHW~-E+< z&F=*(d>IjP_iuNyz@z(Ug3_@64OaXS6_uVY!v%(~s`Xpfbf&XBO;b2Jzjl1cNsbje=x!#GaFHD<=3E9%Q*;QqZ85WXPYJG@Qgv+(_hYN;D6UOBOD!$XxA z)QlC!SSbVIz)s-SKI(Pn-TMcM2+}Peh?I0oNw<`M(nCmhr*wx59ny_} z#L(R#T>{cdNJ@7|-aWqG-(7dze_Tt2nKOI#+2?sa^`H|tXb zhtss=5R`e^8%<;dy&-lA;%Y<~aK54Lpjn|eJ<>;W;fa5j{0c&9T|3jfcBrV`Qgtrr zOTHT$Z6_?loc86j3fSJSW%M{jzkLgY{EQaJ8`X0T01eTV*9Qsu84aFE>!37Kv~6TG zs>k|X`#ke*7Ukh8viPN_V>wXk-xjx5$J|~QHvK@_&=xJXM;0+h6WT?m5w?p<-ko;z z6nncVG8z@fK+zqMnZIVojEvQ~>*JClZJ9QWx9hDrJdxJEH%$9)>JKk3Z9hH}teveD=!eUl5f3)dY>9kXco8&)qJ)rJmp~|qZ}dY02Eo%T zle0JICGV>8R4O`X1%whtQ~nHTNwmVf3&aR?eHV>`-91)V9T3gy&35}gGxzrSjD+Gu zMo+XWcMpNqDKUR&@E#TDogal6ujkz_z|JQhG0?;hQQl+;z^sJ!niKyNKeP)0{}6H= zM0fKd?#-F^>hu&$i|g;>>h72iexLplE6I;*@P|A>1i~41FXpFbL?+A!fe(G88fACAu_~QPdLxAN~mA02- z3Z(2eG4orls=gsZ9LU$8H?ThjE%rOsg7*t1I!>%QSJ@#L}Z` zLHwrI^+@N`L{b43&?{J9S7=4`8k#<(jgz%d!9$`vTHnLeO=MLPwCB|=2Sf`kC!fCF z-<)LdXz0I>*f33j82?2&a=r|bf=$pFwmjLs)Qt6E|McUe$%sw^I3vgEu8zrE%*qtq^=sd>rIJ`_ zUCXVUL*F*pHo6yIye@ebg!24b`B(lbc`;gcH`czn^;MbDY??jfZqc-3I=4|7Iv-PN zc7FGTGj7mAd0A-6XZ00n$6khvmby8>2cj5ew<7(Sum(}oXgzd0T-B|)^O)utWkmz` z-W}FvzyYDkHuV>x>RaJ(ilxny-ZgOhWK>zsomxzobR9 zOpx^MuuS2s>p4mBkGU@{nz7pqaSst8=0%5Ret&?2&_d!X+1Wu@$M#<#xnlM6Gqv+C zq87VdyJrg+#TG_eWNHryLA6F(gyB{xsSDHI;aRqI5o{k?4ijNaT(7E1q!M;H#X74) z*I!mwf54N35L5-*8sigH`Sair@U7&dW2$T6!j$ik(RNM0`8lgL_ z12?yc5avr0pPwBpV)NPZV*J6Ij1ovgxBcH2bG!F12ewL|YV9jlAu{4pjAmA9W|W;5W{9R+7!wbkKCOxej zQErIwx#6=^h!27ydme#8eW4%&wJ(_fUwh@8dE48+$c#JRffuu16PZ!x8r8G#uTnVi zEY&H6h);UB+Z=N#8M{IkPu5#Tx$7VN^p-t=KPDy#2eQIuu%q)CnaAD+hG=+e#=3K; z6cOwi9gA0A{!v8xARb^HwWa#xv9#R}gh$VDnQ1VR7#A$`kQB**GN8~PD7TK8qf5Z< zi#6;pieKV*mCwQ?-DWcQ=SAsDHt6}c*Idf80we`c;>|_Fvg!$Q=XcQTgfpkAr`)CH zY&);lkZ6@~KMAxYi62xQ+}=o^S-|dDCy9|6kdL+*F_&Pvmpd#VAGKR?in+wSU_n!d?80sgi+-y+9U!mQ+l40U!R2 zQ-p*q8QXH7vD*$N4L^F_*7vzZ8v1$RtepQb>CbaI*eq1Zg_ig=qQkZmuA6=T7E4NtrzVwSipKc+{ znxiv2e#OTjYpSvzn?uX|9_}YD$9A0HT6uwwd$NFYEV#KmR9)+fvIHi@r>_sb#Xvz4 zj%os(qKRn6jY_c!tv2w96EsqO&-S~Q%M@_U1MSxkCwuD=q7MLO;AVv%e0lJ%;cFj- zWyq@WTWh|nikyU!rB5V_&Q_~I91Pl}CC8P_eph_&ei0c*O(T|+*Q#JllMt2Z@YV^X z)jZ58T2V!>-LhVo2Bk_sA-yC*C5wme0-M}sivmLz7;3fM&DXw}8mf`Ie@P3R)9+a* zQD-K7ZI@x$-R`Mu9PW2i>5{|hs!ts7^PICc%Q?zOA8Pt0(UWt0}@mA6Anco zXg7q8bDEoF4Qun!2paT4^(%b{Z46%wFi7Gbkh8L=UvKdW_^z4 z8nCL&uJ?>rPx)SECZf(i?WQxOd7^X zqWIZ`qt(&2U%K7y<}RWAnrJ zahDgdht~#D$7N#W$xboa2Rvi@ptJ07Fd0Y(K90RuAS518%^?M%>EOa&@ zLooq2#Q>r+7r=z7r<Uy4UJ) zx~}d{InShCB9w%KbO$;GdBvwrl-s1%L!TM9m;D|JKz5EYSylf^IMN3^W>gYWWt=b) zTVNzkJ8}~i=j!|PCY{$dUl$}Q9a&}pZ;Jvi8&uK;#bTp*zu`skj?cVsa}JS(3|(}q zi4&b6HI%q_m+Se*yH(VrJ}dHrarRt&7jk)Fxo7j?b_dUQ69=w!IVXq21%SFUd7*?k z3eT-Wf}j^i_G~Ou@H3aW(lyAa+6I{|@d&xJX2VHQfXkIg|dr-1q)l?L2T-E_D39^LM4%?sC^*@}0p+O;{L78c3X60r6GG&uF1M_{X%8KodD z`$!?kzDx2-qtv5`JqO?Y8!S?82RSDW5=rI^K{vKUwh!8R91jcD6~c7P8J{W-|IV9d z3scSJN`9iD0;sr5Qji&@R&EGjFSnk6OezURmHgOhiQyRnZViLStN>G9n8e(Ft5QIf zYN2ydGRHaIGZU~39SYWEs|mM7SG3VY8lf5KvGVPKV{t^gZ$Td$mgahp;erU_n9%os zL1U9*ypRPS%G15XB`7sh!gG5A*UwyzR>aeUy?BA!D?Q6|hf%=i+R!03<@`Tlbzov1 zb61Gb_+WJBdk%>?K<5OQ!(L#^0l{Jph<o33qoMC<60>9KVCJmHQxCC#s2TP5r7P@-yeca$zHeI?79m`<-Hrb`RWjja$UJA z)K2n$CL9rFvhp0cFM51$3?B&pg8Oy%M-1%qCG#2@7;5iXA!4i zoTx~MVv$x`*JRc=6Q4{G(2!zq&7TOb;&BBDMiDMG@Qnqw$*l+V@tX|bhKGg4E;aH! zMlT~ClLE)dSCA8j%dMIZxYwcOTrD^S%UK1Wu|(j-Qv^XcUw~Ul9>=UU#w35c+Zzi% zbdZ#MoYE`B6UermiI>?b#`vsPVa~FnmS_9@ z?H+ZLadLO%b-?-poWwu=KW%bEV$Up#Kvj5=@z?>nyg4P8Kh$sXw~F-=s{eUTR^1Dz5pLuk!^^ zssd-7G;H-Ymu`##-N+R+O5H8#^ni9#cZn*;n0YM7YE{T$fK}p80ff?|)j+c<@aL55 zyVZ;wHv@JFKO2APd78Hn%@)`7rDY9^_Acr6UCIAG!%L0aINAQ7($|MI zGdrJX+0`yJqjEa;c^`}H$fo;c$zv_;g98%l&DwDc0bdioD&Y#B3HH(W~uUk#O_%gMXhzyi8!iKm;*> z_@!*TbloYv<{Me1(`e7rhu21|ex^+p)9*P<*?SnOR^FL4g2pTivQKKVyd!(XEGr6n z`8c!)XPES2Xv2|!W5(*-{Lhm}YJ##GG(_rNt?#gHg7^H6w%SFW51yx_D=^xz51fGa z&@ea!%W5pkS~u-Cwv4CQy^jM$eQT!t>E#d5%fM+_po@*->EY3+A&5WiXG3zxB(W>@ zSc`gXq}eZ--vFx1uLshCEc3zx?bB3~L1z)3c{et7O7N0$sK?5c&B^2WOaF22#uBWC z8i#5MP>!BF>)FAbXyC>RVcx6fF^bL!BP)cW8V!xr!#6h2GToH%vcz#BIoOgKZLAWJ z@FFt@scXFfzbJ$FCR>Z}5~jk(&qDGDTc*%ek~$H2Tk}8b?GO=K`G7C#N}%JfR!$J> z2wn&ahuUP2Ya4w`@skapjlboA8whS&e<_Mm4Dc!2R4li0M{0krI87kFDp{@i{V2_W4Zdp#%^oZeMJl7*u zje;yBQF#e3392EQ#x5Do|ioO=z6S*RRVIv>I1iNyd3J7Jshy}PbroWsw%XN1X0 z#!GQnk0O^X7QHO2oBm(l?BpYN_3GR{BLqF%*wzs=O1s&!vbu<3DED~8Ju*GHMWLzN)5I#u z@xhMUpQ4ig1UbrVj|ng4*^(bl?AKV^tH&mrT`5HVt2sHV(z3gByDv~zs*m6p;f+)( z#8y;SgdjkPdX?y<2mhz5S?(&t7n!&%-lws@dPbFM#bOr)zdaNdJkBauZNgVge7F;5 z#P2Ua_Bn}doK@F*_a_?S=B*xET@aJAC3sK(TImTKWe;4&=pAe!*`PG@T#e};s=Mqs zszP|{HlHmj9!3bbjk+weQQp3Z^?gHxGy{o}e#PtKcqNq3l*5YWPtS3o=~(re-s@$K zm}Gpu4NrHxQtW*Cz7(IDfKV}NW}9~mfCzS9ArSPqjTe*j()=~Es9a>NI-t-APti{|Vn+7dTq}Jq_M9SJ`||}wiyW$jA+46Hi4jO5@`V6o5(gz2RR-x` zsg3!u#8i>;OR^T?k3K4uvCK_G*gZ_E=BjD08G*^*e}TRA$v7T z1-G^ze;iC8;|au;fR1rr@MDk?pqCY4bLqY0thKYjM$2ad>(mq9U8bB(WW_3yl@uSm zm{n_uTF)a8gFDP^8Z+&d&wcc#JxtX1s+dS|c#)sCIl94DildTr6Nrf%e?RLc&l!lO>D$hXz}GhUe6<6qKNc zD)y*4=2W?z1#Nr(JGOH3(__m&Em{S1=(ZO>$(xE?OgKSHhVY@oHM>5NK@8>HQeJ%K z#l$Cjv%GRw&UlCVJmW39AFVj2<7vX0Cc^DB>Uv$_|E#6c$(AFgd!e1c{aOrM10M+@ zHje#4x2QX--HS}7{r6z=dsQHWLWq)-`;x}hn6U#B!3o__i9u^OzFNW}pfu~%&$T!@k zO|gr3{$Jop%-Q2=`)sd=zuHmxNx}T5j(>k%TbPabo_2yA-K0q>Z~UXs%yU1XP|HOn z@!1ss2;i-ku-8RCfX`LHg$FU?Hk9K(VvbAJ9i-YN zh!)fSec|Q3WUn?aKJEH@gsD7&pwM6#`|VhDaWy3fVCXCH9CP*w$8B^!rnX&f&*Zh4 z6NC2`z}!B>5n@IFk`}0m8gIelni!uMI`EW_W~9yp91P0V0BGxb)Fi27>>7qkPuiV3 zM>DTTp5B?KhS_t~=DdZ?s@wQJ%@eK?1tr~OW6X{cDxF4wWPU9tu+-pHtsY8PPj_!m zI`gsYld8X;?Mbw{`Bx@C2Qd#XIa>yeT|+=pD(?PL+fOT=q#+6s?-bBHzX2MG{A$zT zNg&nRxGNq^px+W!dpi&+4NNFa;`X%Lx6Kb}8l{H;m9Nm)chWVP2~yRH(IEZr;LoWi zv5=~FGeA89lm^7W;F1lx9I5XjH?Y=#^_uS@UjQWqSJSLh>i|M3oS2;)H17c-S1XV4 zvVVvJtR=ZnO!IslfEJTq-@OBt;m_y307sd$cKOxl=wyEYvVD13KJotkAYe7c;6Yvu z0>Zg%7r5a$PP)HPu*qaWdlbvbXFZP719j4@PG(S+xcJsI1}c8ZCmfxqWQa!HEu+?f72}XXXOFw|pRjS-Ji%h|V3Cq41G{n<>`rJ09+0K;)oX$@|S{Y6UN)^j#Pehk*!t?<8f1G&bgR*TI%({gOvtQPW9{~{-7C^e>6t@pF8m@oF#p7uP>;R-=qK0$wE?Xp1tQvqE zdEj&4qi(=Ee;|NqVMRGXVx(6{;UL9}CBC>CC%?GSwS*m*xRqC?&3yYmEkF%`N7U-9 z)aPYzg|9n|deMtt@mfuVfkbiALST&mLFClupwtvd5VZZK9(*=(PliF)z4)56MvGq$`cA%qH$Tj@YG-&R zZ+ekn5r9tub}Cue+}j`zReFv#OY=a42_vd@u#f~&g#-aDo-V3oT$U+}D5(b3;8bFm z200T$2p(-z8gA8+W>Vd(d&liCDGA0ZzsM z79O5aGHl&mxngXOo%Xy|eQyxl3`4Py#p$(ZNE;&xU%l;5>AA1LmZbUUCqdpE82v)` zYh`<=U9*jj#uA+{rGqEnY*qPc<3=h{*Xr)B*6Mi(kt8 zOSdkO{#_RRQLr)bfX-_<6k|gysdO83Y^0&~x?Fk5lf3gkVG8X?+n0I19j3}8fLM+F zzm<%7!t+@%{yzSN*Qmn*VlfB@S*CTlxBB^GnAi%aBy2^El6Yv?y`DKmD+D+|ocfd! zkptjvkYP7qvGa?2-kM;@+kgCY?{Bj__$_=&-4r~I0666(l@PCMUe=TYecNZ-F)PsQ z^Krin-ST#)vZu!i5S;_0PI1XF*W|ZgqJ#e;-g^M}{4uZns9SoNy-;yev5=%YEJgQ& z=zMgofvdEfWUhub#0O(64`m z_UM?0hZ&QPt)fK9 zk_TAHyw!cb^h%uwiB97%jE5nJY>XN0lGOpVl9A5=l5Q)P7-~}gVeF6;EZr$r(0{Ix z;71_#>f(hIc!KzNJ=2Z{X1IS5B4w%HCpi%f;VUQIO%x&9{ZK?BEX+J!M$2mkk57Ng!tGu+^v3aKz4;A!R2iG+O` z`oI57>0tULEZx}-$1?VRejATh01PqQkvE?9mxc8ut~6Y|?8Naem&DYlZezqOcZ#aB z(7+W^QVu>-zIOSXcJ@EuLjhKG{vw>`4&;~Iz1NwEwORg#i}xPBzj-jj&Ou?Y07|ky zpy+Tp-BfIL+CYbPIMy<}W_JdeOSPavR?U-wG5}osY)bWA+|_eqB5DkS#;{^}NpxbN zz(vr9pT5LRic1DOMm{{K7CLq7?G8f<;{}dtI1fXo83A!}JcJ*4KZV&cg$VG~fw{<*4+T?XSzo`{qBDgCvNE@aDBUZ9p5tp3?h zbDy-ggO_(I2xU9C$4M4ap)VZx52fVb=PJJ#7HHz@paqR71?HYNF5CbN+L2JMMEX8jw>+Om#u z?R3<#X-oYD2YlY~=>pt#1pTR%WL@|ieXI|x{k zaX5`&-+KInkX8FLJZBBm%sC)g?Iu@7Nqvj}TVc++tVYtgf!%f}a77w^uWLLg{lTqh z)2_3knt#j^329Wxm!%?>!)2krp<1a6bup-xwoXfbW$w8+uk~PmsNGqXHglmVTac$K zskt2;{rPF}d-;u|x3?uhG%8YmhpFD=3E(^NvnA$Hzj2jUq6%R{NqoOGoBJHf#-LLB zaOXSr%lu%P%zCuxA_M%fDVSw0>UCJxN4EC-u(9>@rtfw8E05}UM%2@A=5pND*sgh7 zVC8rEY1@E?O1$}7PUj$PB6b`UMs#YN$L6oAi&+6Y$e1QhBPnIBmR{fOE1bV2ZR z#l$GUyIKG0umrM(-`cWz0E}RLBUwM`hmxS8+{NTuc30MhuaQY4ZRck~#h!7;y}>;@ zXw#zCf)D${b-AK2d$K7=o11s@J4yu_RNyZ3HGTzYu=`-~@JYkJ*8vO%DR~q5{IwFX ztS8b}LBv46v2zK#eS+)Oj%NF15&jkb$Ll+#6-mGuuZ_*W9vdj_cRKpo>GrsP7$p8y zkeULImIXME6p{4LiqC&-*DBTuuu_weE&(Ja+a6m_Q7uu(T)6W4j-fs&@Xy45Ty3!|A4 zL$Qe_CZ86bAv;XUu*}*G`9=Hc3JY_j#X&wWVG`)>XdiSNvOa+sIzOj9EP)Yp5(u|k z^{Ow90T-B?AK{}FTL*>QCGl8#{995Q4SZ{96o+VaX`|69Y%DG3f0s>VGKvwy_kTZ7 znmhrfa_EeTn2(XM$Iy;iC-Pg*u(*D%{7SiVs(5r8C=2t=8?461b7+fgp?N}WBi2dl zn>9vA|0sRz?GuM?bcK}Gz~2Gr9x8DhP@Z!qx06BoVGwbmYHkvpA>rSXmV4kL2YZO4 zJ8v?wf`QG4`5^+yd?bYm-YSFWm0_@Z%d@;u9I z3e3M}Iqg;T6RpFuu;L%)>wnjUj&L<7f`j&quOn}J2^!jQSxvlS_ zB!Bbyf##Eewgn6|v5LEBbufd}m_=>Q*I6fHKI0HO37TUDAIcm@VH!b zBjoae>9u;Vh>2j2d=p3N$3+zUMoc5d@)HM1#afmP+Qo*cH+g@O+)-qVr;=x0F3jHk zjjxxi_pJ@!iIGo_Be}*HrbH@C1p6pZRSlwC(6f>f&@+A4J)B|F! z)TSlG9r#eAE6t%jRu#s;`RD{9|-fIRxHEN z?0f)^i(< zR4jT#cN8Ca98K~1oTzC_cU8vdJevD^RdXjx97G(q#{9d6QzuRB0B0No&Yu4LxqMHT z!Z%^laLR zzDaqwrI`$@QBUcNEcPszMVLYBX3!HZtIWawvMcg?W$1QHP)?T1zx#upUoqh5RRw$F z3Ze(FAh-%?gE>Ya=!7mdAo?>VTJci5P#txwY}JN2u#|FsE38-#hF0lIg+2us+|Osn zoc>&!?uM?3|BeI|HBt?V7n%@{5h)|rM=5f~y(?hiLk-(FpH!ACjE}n=cpZ3 ziStmxo7xoFumht+hbYd3+w2Tb7po?x%>Ny3 z$f&W950Mh|QG`Jx+6=A!-}$%5c-W}|g#5q&31Q6WHj>swBT*FlcuIY56fRCgmzRbe zcxVLQnELf72Jklx6*z(wTW$_ocw$=#m2gdt!FV9_+Bp!-y8E2vLN`bOe5^`-Fi(m* z9!r++^(Ox)PdbI|?Rr|a?@f9G zAMk^8@Im}adpy}DZ(nvI2c02WU|%yxTH!zLA^1R0TC7lta7D1vi+_&?C9O#Jd}Zcn zfu$x2V19|0`s zLC)0#)@}UKWP#fu43&^#vIuG-4%{N)0EcDY8{5VHP_1VenT)@Ci@-ic zzA?}cIm6S-Gf?lm=eX!P?hauM?0Fi>dBhpYJcf%x9Kqi$EU^R-G(vph-6O>es>`1b&6q}3RCVYAqYf*2eUT%YZ8G2c9TGNWTZSo}51$|gF2 zajsaw>(LAsG{oo~@hhn6GzJ!F8uJV>q(g}Izo#lQbovKEttqzGQK-w$jg6)vTxLDJ zBVHWVrd?}k2Fg_ltV7P!%3r@{3~}%GE{M9Rva{$rWJ!@3NAkChd!f%A;)~AS ziGzc%kn8s`CX`1YE?JChd?(it_}G~(lh+_mYmxSL1(5eXNzZg2>?tk+b8EcOH`r%-N>@NHzag#jHq6 zO5K2m<=6fhmhY~D0Sm*GLNjH=ED1BIB_mQXZeO1}D4ZKb6azh+K)oR{u*tKq9~p!u z?J_m>dog$OvW;k=3DqJ{W2x+_hbKKobOr{ zwildwfixQIPV3%!$vxK;WGdOrbYQDH`%WT_i|w+_AcSM*%ck^JYKCHbS)|#kprRc< zlnOs0uNzyEz<2~A{VhC|7+FjW-aT9T<52YNxC~Tujj@d!&@7M5RKyBr8mV3|v%=lp z%eLnEZJAIx4~>1#%9~*Kbysj#IrqZr7a_CgjK4dc3uo}77!Z#=a0 zDzyq>AEO9m=oOYm3>hAK4e?<=`N2aL4{I~_YN`4f&J{lnIqo*_s5JUMZJqw(8MK&( z22_sHi^uLg;1SM>`rqp-ag;@dpT=iAZ^A( zH}rWDFkeq(`j;9R9zFcVx?u=dxH2q&ocUr-vFI5CUO9LpC;=ndobq#_@cc&aFiFky1!~WfcPt~(rvR`{VzS=t;L)=m))wm{5A|=wAjnBwbrTSL7&6p9vd%^Wyox zUHe|mqg5#8&I8Fp?u<-{s|L&ntR-@6Fsf1(-Re}hg z9iP-Z?7l0Jin+~C3M0d!C?q6Cy-SQ^l+-{$gXjs_Qp=stc1*$(L##4X^}y+GxZ#?0 zn#E{(YjmXQ)N}C0gf5sT*cFl8It&(yz^>G9huc8xH3yWBF`f}?z3cIB*(Oje9nd8` zP$jiNnp_}*f$vayEzAGvdOJMpd%W1zTXA_KtdG)6Rh#LhCnv_c(xDJmS%-n_s=%NZ zl)i}vo>cl=Z10F}n^LCs;^|rY%uN)ifmL)cVOk2C+IBJ@DlJ@XS1e!qzSAlI0XyvT zlPz}0n-$wAuCDQGLo_k++w5ok3+zHs8zXHT?`B3B=*qrs$xTN@{?}Z9ix4T^++a_m znK3ZW!~ge}1Z!!cD$2B=H&}(*NY-M{cq2bJ}4_JbY#-^*Hcyz z+0GGp3eaaHpDf>=RK81kNMGMsTd5=@9K2H}c*@s+Rzs%Bhv%RO!+o%S_#I^T^K>x@ z2O~q*5wFi}+2<=>v3wFs6NSH0PnvtGz(sZ@oUQ4>q3ot++le zY~XJa(Nu`yY2enV-2^xJ!0YLZsm z53xDYYb=1A7u$y^6?#2L2Ei19np!6{mFh)#XNU(n138^x{54E9s4AnpbCs? ziSi71GStbsalPk$NT%giHNae-{HIk8@|uu@JJ@w3ueH=Fu0 z3kb1Lp&|)c$|mtQ_T+j@{?5F;c_2lSFL&VWo?zl*r$RtuqeY<7)QmX!GD0);ji{KmIM%SrZz3*G*y`VWXp^kkgdqQu^oJlMCH%(3~K(&OZq< zQhqtb8hrfk1lSxx@SxLP>|3Av(&oI~F}~^G7|vHVNarbvh>7Jl)*%`0I+Sw_Ep-$o zQ5)edd=h3WbfAu%k-%Pc?}#bdw4$}BG%b|~!ZMfSqMDxBpceHx{}c-TwM0>-g<4%G zUc-;CHj31`hi671Qga4R5y_$FQU7y?kFksys821a`m=UB`iSV}9C*1-RRZ2y-)qIc zD09Rwk-alyj_r{*8okXik0KFAm`KF+#9c~P0H3TPL|3dDY&C0ZicdAUsj)jpqT*cV z3WXmhZg+6N1CWP6GRxKE*(0g?u8lR4)AkXE8>a$+4x$s?u9rXek4pb_);2MwGZYAj zM^E}*p^`ZpptB?TvabKOHLRsM{qF}XqP*pI(F5Sq|4t-1Co%WX%l1r9tf{{1VK|kV zz1s>;;Le0!Us-C`%pY-YB&CjxB%Ltn0X)j8(CVOH;JN50t3=U&wsDmI9_$r3Ap?qm zHq&6nU#M3FR3R;i6LDjw|DM=90^ic(3nRpQJ8`vud3z~n@F@vWIE?B|VF4BK6gvi@ zVwd+C>fiTY-Su!gX@9u=a{L6);IU!jk#t(5+sW8!8s9PXw?^kn1>DG9{gLmaPy^t_ zerBS351l~f=BxyHCebGym5%EmQf8hRhZq<6E%X3Dw-3W}cgF+|T@tfXy9507F+BLR z349p;w~E&_RGX}GRmPu2vV?WYn|Df5UgEWF-qgj)RBZhIJXi6r#{I_M&6voHq*0;A z3Pjg!8Q>{ie!`{NhhNK}HX21sxuhS0EBk=d zP22R}&xBf=sFG^_!&%0ef2Z+SZ!(E;g%CUf-<(4|o2SZOkSYdNJZkArOjS>#J$#onL!#tlRj{)C>7c$#jseg8=Md(eb64?22){N4ub*e%eVO{vt3(e)wvJ)#)A#F0goF1X zs*_--9UXl3D+uwDEs#%KPce=@ZlWWSLArf|uvK(=a*_|`8Qc?*bxq2`-cZ7U-Cf^V zprL03eW&6l0NzjS55Lzu)RliQ_KGABi18|b*ZNVm!dn<*J|4Db4J2A(zo$sKU~g^8)*QdZqJeEoNcg^KAr=eodi z5+A|VEUO2Ay6vU5miDUA41O-!Dy4K@MX@AcW@xklV}OUN*b1LC0q=&cC^k)@P34qq)L_I%@WW+8%;mK2QTGjCY7=jn)QT$gQ;P>G^gDbx zoz`Wy7h4*HKzJKaeo%O<$H~-YbO7LqRN;k*8vs9tf!Cu9gQM2aJFCQ|p(mU;a8}>% zgFZT2>yd=d2=(nSTZ_WwrAi%_CiEOcLBzq?d8?_S+?16NOQ0RO<}}&&%JcL82~QmZ z{QU{Ibr}r791#5*TX+!n82{Fvx05918B}3gIv}2LT1768@hcUV6zYE$?%*m3GtYEQ zV$rUZ-I*?>2f=#ktAZdB7DJYu@UJ*?Z+9C|k55bG4nj5|DItdFJ*Lx?A0mPsS)Xy| zsF}9VY9QIjtdplym#yqyJ$LgUcb!r`0aU$w$;VCfs#RJ`Fo$3v=BPcygN&3e>?x^5 z&=V}A&7Z)hLcEggMEUA5UKko8yuu#JJqU&5`uHE^9CvYtA#r*i_YxNXSBl)AJwTN_ z>xf6fI4D;6Z|~t#GC^NIQ!O_6Pm!?vaF-o_8Gu600Oo^?AF>!J+JsrgOc!?zL~#DL zMfY&vLdU-k#k|6%8&cbkE(x)Ag+qTq1h20WvNgIk4U z7j|LOOi1fBF54<3!k)FQ0AZI89G6J2OdMAbZ-=7Ye4YCt)eKt74KUwLqlS6}%Gd7> z%tu7jGiulp#TO9p#$`{j<;PXxj@&?O^|Uy;Np?x5GK~gb$TUn#6*M*^9)EH_P37x@ zAdk;$fh1uFcn;G3;3QMzf8B~(s{X82k!*=wwJ_2uZ7|Fgs2nF7?DePJfuV^F7+^jd zehb}RViJ)FT*Hthr8CB5yVzFZ-*~DICER)mD0~<@67XzIR_ptmtKZ$ES(nFwX|#r- zF#Mk+v_>JJ=>06XjCMg)wP&9n|9flMa0DnAXzmo=NHz18tw{XAw3-*J%qtU}6bQ{+`-K~I23iFi---^wf6 zg%L`jL;4^hR&f8CQ-jcYa_fN0-#o@4`LFr6crt^o`Pcmv8vaLoO)WO`2;e%`owrO5J>SgK#&C(Tj%W6+Dizysu?&4P^;19aLy5?NtSMpg;|a(6#meL0)Gk)O0kP5r8u z2C*rmvykVol=13nVbrPA@QuajUo82px*OGPQMS#oiO)%WX~N5y2y#d>3P z5m?lS%Hk`#eyn5kGd@nbWuiuzT+mnuKRR^;0vn^4OVM+?MyhJtoE0g`9n^7Vz8&F| ztlKQ^nX{4=)p;?5c8H394atBy>waoxEu?(t_^p0qKs$X5F9Y1OL`ca2{E~=D&%g^CUoO zpuPyMAjM5H6x?%-HA(Pqe2-czil^nuy!0cuyjYx~CE%=*2K?oqG4G?_!$welVx*!f zPjP0q699jFWv`UqWa+Rr4!rrdEPGs+mv6jX1rKBi{alx#X6(C2dAzkoB**gtSs|_7L3nG2F1e=7%4>sPl5&JDp_DxH37t= z(<+pH=v6=2OTYWmY#?IONhHD)L@V#HAz`@0C(OrGgj zCTSdof$;JqTJV66^|X4JjcaJ{^Fv8#c|2eR{m$lMhp&(a$KPTLkZf zsEBnP0y5l|JmfzY5sPX_kX%u4CIN~MNOAT4*n8tBzQ~6wCI5mIP5cGh$}r*G5M7Uz zGyy1H;Ig}6Kw$4>6WKZ@aCwU(f)HW`N-4HsRBxt2UtcU|=86N)W%bf7>-C1X7HY-=z`%>8(N(!? zXL?MMd$;XVijQ@{OwPpMecO71%ySr{s2-_7eV=_wVhpHfd&g+}fCcn(SR6$f*GU}2 zxTB}AEyr{I%VqrRc8DOc*B~>U9S4I%s%6aM*G?fk@5XSMiJbP{c1vND)qxm7PLC54 z{?iez4X~C;AW0A7g7K9Y*vAEC%i#Fe(DT?Jj2>Ts%p$vloRcgCj8z6NMdIp{@8=>{{d2cXixhsGk=Llq+=7Wd1eC2p>mANkH z8v5_dRDA~=zhS-p)6O`u$$|1%Kj9W^oaO^bH!$7((ia8unHY$FGWXtGm-iAsuMYDX z+zo{|cg>H3&k`zyH>BW2))>bO^Kc4>{Pw}HwFT0Ek~>5Frwfb&Pn7t^yA29^vKYxC zz%=yGIZ%)P1;ff*V{oJSKp8O6Cb9RTc^abl{aX|2rj~_Mp%6Hd3(pP~bvpX*ImrCW z+pn#@_k{asrNrk(1*s*oXyvgJa$pEr!U}I28D2=C?H%O>CdUJEWRyYl;!_+2n1Hqf z8XRP^vr265AwoVxOZK9fjMJYac+r6x^Gajk(6P`XpGuX_nJ87X!92fnKas0^RfKtg z%;;8b>lp?8_1{HTgY?;`M*+qrfb4gDaW+WxYTps}3Kgp|5S)_;$57J=hlRkPr?oZ; zV(yH_tFQ1L;h^TXUuZ-9BSxkf?{4|Ba*g}zYP*VEKVGxN>sl_3ut6I_L0mJ zN8(GN1=RjOy?uE+R&BI)#>n)T74eu0Wgas;h7_S>7Efl13>i{}%u~ozhEOT9@SqHt zXF?=0&+}B2p^&q-_dUPwobNmToPWOezdZZB?|a{S?|ZFvt!rJ2Q8xw?(-Y>HM&a7= z5OojdeGORXjM7`}oZCmrRC2ly>v8>v_lvgyF6=GHbE# z;FYL>>taR>V80oV|CQkm(Q@QcE4%`@kl}|yeRo`;M7zYUk3(*ss?-x4{(((Y?Rg;c zgzPQFB<)CrZRv90RC9JCrjir;^i?nSLx+a`DZUqs)-C2FtWjBhbFF zAQjLSo*1lO+)0jxb83LfB+!}lm%Y|ZwcNqQwb+T9Wzx@Q{hC+WG_Ia&qTBzbDQmIz zAl2=QT8Yc7D#c|I$;X5^KZ6aZU4PeRQ~PIy-#s?efUOaoQ43$6bSr-dSHiaMc1p6&Kss-kCJT50pn>a*6F#?Rb3R4t~4||KG!!7+5)4dAsmR{5m zXu}I1m|yx4*9{ZjleO?vX*{ffSa0PoWziUv*U+OFb}plKP@B)u*xP)VX_}8Qd=W(U7$FNk2ATU{Ecjw@f>nDXX@)kTmHH;KV^!BU%UDaDebb* zUqF$t#LsXV7y821d1*9VW1g2LpRwF4#tKIJ6f0}wU?KD3q)>5&o)sLCcJA7$Z=>d= z&^ThT52+SMRqmTtUil=MQ!Acw37j@N@uf^mF;z}ENu{EFRHwwSSzo@$akqoxdS#R* z3#wT_VvJ<&AVB?7WIkTf&)IrXBAWbs-1LR`^E*YOp3OPX`>jY2pu(jqR|i!>a+%DT zaEt`q9#C1W-k7yTE3WTVFsJLd8~!9zZU;9yOEl4Ixxu1{rX$N^w*Ba~$%Cs>x=1%$ z|Lm>24F~QINtRy1KLek$Z3rM*;qsHyUCSN`r|-Gu%vv8jM^Ztx$Q@Ied!S;mJ>&7R z!gI)y?V}tPb)s27=(oM- zM?i^EVGFQii%vE)z%-81KM>_iwXirCQ@J89>(ou*rD19KJSMK=)G(ATykfkEmXNYY zg>lqM&uq_VoR6+Ht2MXJg+c_?MHOCKwID9BmXjWNY+9kqGMBTMB5I!o{-8Sa z<_?QZ%;?qp+tFU`a8zuirbbO-r5K!j0PxkX`y^xP;?7eTqgwCEK9|q(6E0XK}>@Q`6_ap3#3h&#;+iQ6;|gZHIQB z?qls#j`+wROsnG1B5Sz#n%rEeH<$iPJZl#oLVTke!oJjTz}i(*b+kM}vaFBt=>xhf z`H76tg`q+7CmWVrM0KFHBWlw%jpa9uXb&y~wMhnHO7c-*_R{)SN} zAr?~(wWh}7Fk?sM2PcfD)4*RtBdJ$G=l0iq3lk~tMpE8(FhX3Q>rOk?>9_gW#b>LwqdM>Xk+oq6?^F9nQuDVX(jatW z4t1d8-;mh^hW(uhut#n}XVFS{JGTp)Z$CcN1M!C3G92Q?6Di_i54c{seNx}8)IC9{ z=a~2Cw;a4D5G8S4K|5Q#Cz*O1uN4R4YwFOglaP^e(qr`Aqa$;$kVK5%0pm7xN~#ZT zs%U9QFAza+ozpUIISv!mMou|QqBfCJojm-&?N-AgNDO1SK8s5;_12i{=MKQg3c2C$#Kd%a?<^G$RCIAR{Ey_ziyoPnr^l*5QUO-TfeF@~i@cZ1PknXiIdk-XS8P z66WVBTp)V|x(`qK8HuUs$0!P7s1FL9}Rg4{MhwM?-Y^q0?@G&{QCsIv^n5LW^|6=EU$3$p)xNQc%0Upe=(<)Fiy&*lN>%G z_9qjzzXZVF7KTHA-_osQcriX#Xq@{HkE}07*g|eTL1*syEl2GrchD!_0^(Y}+OMn^ zj$aOMN(q~@(VSU(^Ayx0w4l{sEIiRp_$~`l*fc&3m6l}(Qpp>VZsfd)9{EgePfe&% z50tvRv4@Lkd}LisKNmj>>h?fm-9oRI006|cv3h9`lQX?C&RRA~(q@))EbwaU{bLqD ztZ3v8Z1|hNQ)7piK_?*CgY>5qR$#69?ti}XSt4mHuX?G_E_Qv0gU;YIFZ#ZZB0cr) z@taVAMoHm}pNQ-*gMv_Dke&4IlxH`R(OzhU@SQXaPvrpw#FuDR!3odZ&vbSSAV8%? zV7~4Uo@7ljd2P)@6S}RC-ZpWbB>-UD46mG_rZQf;KD55wJA~Nvaii*5cZ>6s%Oi%D zoC+g&TR|*#Piqc^Za3+Y--HxOxa`4?ro8u~;RDEf@$vjv(9>KEUW?#1iLRX?-jLjJ zt>ZUB^dfDM-Y#FAQN9oj->Uxg)2)xI@`yVz4xJY^6q&Bxci-m9E)$3oGuOEXSLE z93(mh6}gJKJY(usMAOA>GCVE-JL1CpNFlk=rYb;POv}%q+3@wrH4+pRlffr;29dC@ zS_N90NtUGPPgH-lG!b3xiYo;;*7bw^aIqDP&nA910h_-I+EJ$*tBn9toNxAY$s>aDcIO3*Y9l=>xO&OO;V5Xb3Q-?t!LE3$QO>z zoBI$5^HtoyGx!9szpH}9`?kB$!jY|6@rKfTGm%gL+Ec?e`@waZ>ch`)r{cYmU(U#L zdT?~3$`3Ww$c8FDI{$o+_=(^(DoJ>r&bw5ZocOUS@a#Gq-}K4@X1+~&l|h*>to|`z z(f7rgPs0OD^?G@a7atFPq#!Y5U?v4mDt3wIlp0}1FidECc=^uZWr$=1(*@w9@%M(G zS%_UA_i$!$!ge$qgMsZJz<=JN3aKr7uBvb)8h<+&D@y;dwlOlxpC`Q6IN@!{GK=<4 zLT>)|3A3N!7!(Phi1>QXPtzl4-}3oRHA0Rcm`^F~vbN6^nD5eCff#sq;lW^1VVKs+ z-C++76bf$3&(9hci-84%J7&bFpp9VxZw78B4Bm#3{?2rKffu+PM(8w0*}`A&cK<@9 z(j3IZv8P3Cu41w2gFD@llUE`3nQx*}NPyD~Vo9Aj=$15bJw!4nsMBI>3$dP{Qrd0u*N zh5Ho)IEBe91;*iGgh+}i-Yfdc9Ba2FK=Hn5yw$(|Hv-1Yb?+e&Le~H^g zSh)+d)`HNXxUh2J%(LIM`y27{K7Z({+YK8f?RqgUmGL`RUAu(0pccaj5=xv7<^kA{= zSO^a09*n&7y;|p6H0W`xOH%!ab9~7kde*}b54v^NXqA{dWT)8krh8`S78q&+h+p`V zzO(UP9)Hzy?jFs#>#4StV~hHuR|`(BSnn%AAs(stWmeBA|1CgmJJ>^6GK}-jThAJ4 zFvU=|F9DM0P6t87jENca-`0O4+4byW3sqK+j6{tkKHK1bath1#^bgF512>2^7$j9(t zc{H{-CY!@)iJqtcNI4JYkO6gZq!>C!l1II=)1wu`1(+}h7{<8Yz$AdsnsWj{b4a34 zwJfke^;Dx-a%aa@t4}=ltZb=9DCXPNgJM(~6}&x~@eVtSn(i!ay^12x6v32n!e`gC zVW$%^vQZ-3J*5Q?jEM=+IGYnzy5<~h;I^p)9{Gi1@Kt*7;HjG}87?3=v=NG85JxeJ z$yTHN>M_mX@w>gsR0ERsW#7Tip|e{-a~vS$+ACzia+!RL56zWAz4iG_tIF0!-d_XR z{rnWX&I7m~1nk|vQxJZ5;vqX@`el)hC*C=C&C#vMDA{9mzTee0Byj)+Y?{B7lK)?>bL2;7T2 zv>B^SJde+b6Z9==DvG*XpPXLW8htM#CxKw4!KLu!=J>cVf#fDE3m7NADin19Odfc^53Q7w#hj!n^Q;=ey$c% zwIXT9U6ZNiAfHoqNcBtJOTbjR(X#a~?YuP@g` z9{=^0KK#|M_o?;MiIp=#W*BuH=NWwSxWwiapEyFzf(h=<1KNMRzP#t8tCS>6zNFYi z_+SOD3e{!E_RIRYri&Y_UC=7yNwz9gaLQ{^DRBI0TgxkQn=@S8tV@`LmR2kuaHyMr z`SUPMp~K){H!0=5IFszV0|?8|(-n+bA}OTK(N2IhE} z#fYW}jC;^O+j?H_a;OUtrjjCoj_pVGuJG@F^o4V}e-1 z_lv`hXp99!)c7j(58$_2S6R^URt7T~8#D9_W9_`q|eu z{|HQHXKaiFzT-^9pPKrrT+wynO~pynkB^f+w2?&A{^T$?yyNBt__cF*{6vbwEW1p+x~{9lV7X z^BmXU4G*&5PQ-^H$2;44w|EZ%a`hr}ubp9A>wf`lOM(98mM^RZVtNN=I{TW_eP5!2 z?+(oBv*!CYv(-+AFpGU11M8o&V1~+9i~b*%7}`Q5?CY{lUpBBg+h*T|l@xaEBlj}t z)7=cLp~*kLVV7r^qc7(;k0Uw3NQfR3(wHW3i}q});Qz*TC8L!!!;vBwS*jaBa61hw zRRE7I6DyTxHLi(k&C^TG2O&MdfwwB#X6V)L1R71184rlCs}nRDP{ z>0L;?^O#UX`a@&H5pZz{N4b%XP&I#wxr%&d#UD$Ioc{20oomn`5`R#Q@IH(GIe}Ki z2avC%_?2$KRgIa^(ocOB!u=Km97R0V40Ay>=C$Y$oK;^x2II~K^0Q38dssyY|_M&JL#nPMk5%OU5Nk$ zSaklc!s**`!Th2t4wxCkZ1*4Ql#jYNd&#qKg?;K0tem6azPUIes! z9ztQ@GqY0A}swRUIP&~3cke}F^P@paZ z@-a)po?`>Dklam%WKUAhQ!lwT$GmURrQ7AC$fC4Nkwq!G40ap)zfB!En>rhP9ttYfxt_mV$=L^q?#X##d}$1y{h2O9zi9)p%)YR6w!#l&0L zzF)Vk0%s$Cgs;cWGXD+OXSL6EFSbTU4MEi?>Sl<@`gtSfw4SGysh-lvGQy#@b;4=C=#4PoK+iUOc`1x*NBi2^=+RAcRb0tw9M~a)-dzDE^fGHG&94PxFz3R`to`~?BYHR~<1nfvC_B7Yzfca)r z`gTpgY)x(YH#o!jqNJ(neBf~-oyRb!s~vwD8k@?B5>(3<4>-X?(LGT%dqR|_s|d>f z6A-w7~cMN{OXZ>A4&Riowl{yWZGJ6bw2R2m#_3Ys=$Sd@fDLl$zs9$mnD zDjE?>{R{F(oXR2OeZS{EMCFEoJClpuPkxl|fZTQ#Qf`}UW5n~K?#@&6K*qC*Mf@$( z3g_1{r#1d{|MuClKEK=2+;pwkcv3N3>Qc%$Pgpc+SU*(s=jG9Fw(;^5Y1ga`?HZuA z1xJXz|0J)Sw)5=E+D@vkzyMSCa_i_B+qQ=F-_Ej>@fWZGo8$}2;A6S}ZJZ{_GUh6` zeJ{MLSkPw zoYAG!){<>3%VYFor@eq#%y|~YsXhNUBIsd=ml#%jbUq3UYwTZQXHt4i%O2Lk=N9!A zqf+k&ktd-t=dgSe_-Afx1U)1|w;So~jmtE=F4}S6pQ-;9RGKnWY=YPPxyiOI}EIIsr-+&s8{`61u1}5x2`#XA!zH6h6!vwaIW{^%8i)(|) zxSX_i9M8aacQaM;7YTz63pBg(j<0CbwP{RVrP$w=Tcg3V&+HTfzySXR znXuB6flEsH93ncw={hfTZ`P1%$2K_tAcZpmwE}Wu;G;B+DYXSlq5 z7UugNQ-Xo{02LwlU!fw4zL_rmq)cWEZ0dwk8_L5M==hXjF9`7eK9m=5l+V1B3D#z18pCJIety0 ze?^2NJ~&NYM7X}PK<`D65X+jZ5UC3*JD;(Smbp)h*~JvxD%S!alLG-S^9!rgpIpm6 zJiXdXqW2Y4ysL@G7}gZbIKC1`s1rs6VE)P^CLRNPbf0+rsx$oNwuj$D%c%eTyIaT4 zj>N*@^mVZSk7PCz;dko(0gitrWOKGa0fP747DR9faNoCO+?LQF%N!3x-wyktnyh=m zci4A619_ew^3k#2GIRpZD?wLbe@zss2y9QSfTrKaukUUdbPPP&0nj{_T{)y3a1BBc zjdWR~h0iYq(TYVQ*F`MSpfbPKBocBd-)#?}yJRL51n}`}d-3C%2%h=w za{rZZJgXWI(dT}LpcRNUMcQ%xo5*1YjhTLLv@n-skTc?^l40JcM(wP!l{d?PaKiFv zd674;?t!PufKb>S#wxVc6V9X-=Mx?KuR$NPz;77Y%C~zppq{=C`8IUTK|H2dPIul(XVWU^9>Ky zhP1mI7=(G{?IejnucYA*hd(1;Orjo@RV9ct%r}w=(#0@g34uUq6t+es_v;4tBlT% z-4zm@73F_z@}&6+RJ^AMbv1T$^?Lq=PNq7if86ww(@#>MwW*7RE6M6OQ z`Rn^EXC|rx^KFMyzd9&E@b*NkVU6Kd{GEcOry$|Skasf3xYS1TH((F%9d-8A3X)Sx z3Z4MqTFe>aVsQv7O_(cgKY~~DewQ%lGg=gFf!h5u5WV?;!f{<35uy(Cdo7$nZg-yn zXUt{Ei&}L;^@Cq>R$L(o3*!HRKmZZnN#rhnhG#s7@cW@W|EgMnXBN`$yY6Ve?pO`u z>n4Ddd8#Dc`(^!FI#+5+0ElyrpA+ZagABN?3)Pr^kuDLf-<<)7|b8}DTZK4V>4=vTZ6Ma9GR zkR07JeuF9w{c4plw|>p4aU$0V=H3LC^MYXP+v0L^rI2LitVIN=SZ!VOrQyFy zm*tp1!H=KXUgfr&Au=7zY4m?@Fp+TmKK&WC_Fm(P^||D|^25iT`RPCptP(u+e^M{o zJe>McV?srAX>mOF1@bfl16~d`oGArSsvkvcOLIvyN zgUtXwv;s@$RImG=~P3pUN_=M0zbSFQ6aKY8Lk5=L(5lJCjFLsCKCA_#Y!4#^`>-D-Lljr?eP$ zhw^FlZUiRJ(;(M2`eF3Lo^1w{vI#P^J#ii*^*;|%P6D|WM7&SD94M@e(Tzfx=MfhD zuqxWVxzn>@IEs<{3J;23Nqy#~pDr$dHhh3}H>o*g{Z7{(gvmjRZ^W+ON=A95dJ2kc zK0$Uc4;P`+a_U1@O;$zQ^M}dlZRRNx3BmA@IG|(bEiNmc7TkNfL8)AA)*j2JUz~|c znPCNDwy=*8FHCE$g62o2osM9P^iPK0$LZGSV7@c0%bLkZ&Vcye`3$Pf$g5#J>&u1r`*ezp`Xu3#|xXtMUhKWCtg~ zHVBg&BD2EZ_kuC{;RCtUk&J8CzlmUr6{Je&kj?coAXK_ifMu>ffQydzyAg42G-?!$ zj&pD-=q+#vI46T0kc=E$69joG`=m#KCC3+sVdlWcQLQB#K7ovOykY;KD(uL`!1qV8 zE#8bJc>fHK;sGbce1)P@>7Nu|EaXVe{Jv;K@b6?Ah$r@y8QD+dd2&ESg7}tcJMq7h zC4v8gKp|xRtVM+t)-v}ohT?c-D1RX&f28AdgW;dGgvo-B)c$~-`FLfp`A7mWYvPr_ nziW92Yl;1?B=P_0#{aVQM1I5Mproxymesos-slavedockeriptableskubelet-managedpod,networknamespaceSLAVEHOSTsocketlistenerforwardedconnection<veth>172.17.42.xeth0<bridge>172.17.42.1docker010.10.10.2:80<host>:49153<host>:31000:8080webapppausenatkubernetes-managedserviceportal:80isthedeclaredserviceport10.10.10.2allocatedfromportal_netrange,notassignedtoanynic${name}_SERVICE_HOST${name}_SERVICE_PORTephemeralproxyportmesos-managedhostportresource,declaredinpodcontainerportsspec"hostPort"containerport \ No newline at end of file diff --git a/contrib/mesos/docs/scheduler.md b/contrib/mesos/docs/scheduler.md deleted file mode 100644 index c6101f8be7d..00000000000 --- a/contrib/mesos/docs/scheduler.md +++ /dev/null @@ -1,178 +0,0 @@ -# Kubernetes-Mesos Scheduler - -Kubernetes on Mesos does not use the upstream scheduler binary, but replaces it -with its own Mesos framework scheduler. The following gives an overview of -the differences. - -## Labels and Mesos Agent Attributes - -The scheduler of Kubernetes-Mesos takes [labels][1] into account: it matches -specified labels in pod specs with defined labels of nodes. - -In addition to user defined labels, [attributes of Mesos agents][2] are converted -into node labels by the scheduler, following the pattern - -```yaml -k8s.mesosphere.io/attribute-: value -``` - -As an example, a Mesos agent attribute of `generation:2015` will result in the node label - -```yaml -k8s.mesosphere.io/attribute-generation: 2015 -``` - -and can be used to schedule pods onto nodes which are of generation 2015. - -**Note:** Node labels prefixed by `k8s.mesosphere.io` are managed by -Kubernetes-Mesos and should not be modified manually by the user or admin. For -example, the Kubernetes-Mesos executor manages `k8s.mesosphere.io/attribute` -labels and will auto-detect and update modified attributes when the mesos-slave -is restarted. - -## Resource Roles - -A Mesos cluster can be statically partitioned using [resources roles][2]. Each -resource is assigned such a role (`*` is the default role, if none is explicitly -assigned in the mesos-slave command line). The Mesos master will send offers to -frameworks for `*` resources and – optionally – one additional role that a -framework is assigned to. Right now only one such additional role for a framework is -supported. - -### Configuring Roles for the Scheduler - -Every Mesos framework scheduler can choose among offered `*` resources and -optionally one additional role. The Kubernetes-Mesos scheduler supports this by setting -the framework roles in the scheduler command line, e.g. - -```bash -$ km scheduler ... --mesos-framework-roles="*,role1" ... -``` - -This permits the Kubernetes-Mesos scheduler to accept offered resources for the `*` and `role1` roles. -By default pods may be assigned any combination of resources for the roles accepted by the scheduler. -This default role assignment behavior may be overridden using the `--mesos-default-pod-roles` flag or -else by annotating the pod (as described later). - -One can configure default pod roles, e.g. - -```bash -$ km scheduler ... --mesos-default-pod-roles="role1" ... -``` - -This will tell the Kubernetes-Mesos scheduler to default to `role1` resource offers. -The configured default pod roles must be a subset of the configured framework roles. - -The order of configured default pod roles is relevant, -`--mesos-default-pod-roles=role1,*` will first try to consume `role1` resources -from an offer and, once depleted, fall back to `*` resources. - -The configuration `--mesos-default-pod-roles=*,role1` has the reverse behavior. -It first tries to consume `*` resources from an offer and, once depleted, falls -back to `role1` resources. - -Due to restrictions of Mesos, currently only one additional role next to `*` can be configured -for both framework and default pod roles. - -### Specifying Roles for Pods - -By default a pod is scheduled using resources as specified using the -`--mesos-default-pod-roles` configuration. - -A pod can override of this default behaviour using a `k8s.mesosphere.io/roles` -annotation: - -```yaml -k8s.mesosphere.io/roles: "*,role1" -``` - -The format is a comma separated list of allowed resource roles. The scheduler -will try to schedule the pod with `*` resources first, using `role1` -resources if the former are not available or are depleted. - -**Note:** An empty list will mean that no resource roles are allowed which is -equivalent to a pod which is unschedulable. - -For example: - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: backend - annotations: - k8s.mesosphere.io/roles: "*,public" - namespace: prod -spec: - ... -``` - -This `*/public` pod will be scheduled using resources from both roles, -preferably using `*` resources, followed by `public`. If none -of those roles provides enough resources, the scheduling fails. - -**Note:** The scheduler will also allow to mix different roles in the following -sense: if a node provides `cpu` resources for the `*` role, but `mem` resources -only for the `public` role, the above pod will be scheduled using `cpu(*)` and -`mem(public)` resources. - -**Note:** The scheduler might also mix within one resource type, i.e. it will -use as many `cpu`s of the `*` role as possible. If a pod requires even more -`cpu` resources (defined using the `pod.spec.resources.limits` property) for successful -scheduling, the scheduler will add resources from the `public` -role until the pod resource requirements are satisfied. E.g. a -pod might be scheduled with 0.5 `cpu(*)`, 1.5 `cpu(public)` -resources plus e.g. 2 GB `mem(public)` resources. - -## Tuning - -The scheduler configuration can be fine-tuned using an ini-style configuration file. -The filename is passed via `--scheduler-config` to the `km scheduler` command. - -Be warned though that some them are pretty low-level and one has to know the inner -workings of k8sm to find sensible values. Moreover, these settings may change or -even disappear from version to version without further notice. - -The following settings are the default: - -``` -[scheduler] -; duration an offer is viable, prior to being expired -offer-ttl = 5s - -; duration an expired offer lingers in history -offer-linger-ttl = 2m - -; duration between offer listener notifications -listener-delay = 1s - -; size of the pod updates channel -updates-backlog = 2048 - -; interval we update the frameworkId stored in etcd -framework-id-refresh-interval = 30s - -; wait this amount of time after initial registration before attempting -; implicit reconciliation -initial-implicit-reconciliation-delay = 15s - -; interval in between internal task status checks/updates -explicit-reconciliation-max-backoff = 2m - -; waiting period after attempting to cancel an ongoing reconciliation -explicit-reconciliation-abort-timeout = 30s - -initial-pod-backoff = 1s -max-pod-backoff = 60s -http-handler-timeout = 10s -http-bind-interval = 5s -``` - -## Low-Level Scheduler Architecture - -![Scheduler Structure](scheduler.png) - -[1]: ../../../docs/user-guide/labels.md -[2]: http://mesos.apache.org/documentation/attributes-resources/ - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/mesos/docs/scheduler.md?pixel)]() diff --git a/contrib/mesos/docs/scheduler.monopic b/contrib/mesos/docs/scheduler.monopic deleted file mode 100644 index f29ac1cea55076e0e31855e91431ee11e94f57af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10504 zcmV+jDfia@O;1iwP)S1pABzY8000000t4-R?Q-Nsa_y@y_-k|n9HTO;va0msM(ld; zt{q-mvg{3q<4Q5nB70_NQ6#sUqg{DJ5q=s!n4iS>AQAxCMWL#Y=xsu~5hL&m1yloB zStn1P^Oy6xZ~tR^`LI9#{4e$41v+0J-dx{q-(79?AKz_nH{afDug*Vzzq#3Oi}Rbi z%kPF?cE4Ev)&2SBWpVzg`S-5+_ow3ghGL?4v*KefB5mH`Vy-v*j<0P zu7*Cpa0Q%Kf4TY4Tv_1)iBzkQ}%^@IDn{q@83-R-;G_I))G|G$|CU7G4g?Gf>j zjnyP+%|7rJrbGdz2jndzDdoTO#hwtlK ztZwkh*fbtz|LNQIULSw!n|^v8uK)b~=6*1yR^>TQ^xgaS`|U&Zld`D4+ssfqUy6U{ zx;{}g13#XBuKAnQX}34k-<|hYIle+)U)SZs$KBnh4L!$8#Bj=f%tCZ$7K`-bbUA zk0%F1^~w1MK5)M3ZD~Gi_4s!RaP)V|9s0v&KAX4fVf*KYcaNW5`%yOEY3{iBY=3$@ zu?CfNLArXV56_a;Hg}e+CYie>B{Yp_USB}7#Rb|xNV6ss(u`5(-HQZ#gy;RsB^T0$ zBkI_YW(-XUX_yt#j4?4G4V6g?X{vMu=`N(9r=LAd{f58mKGEuAet2dWeAw@AHXrtV zARw>PF42smE~UjGQ(AuW?(4a(Cck{EpZxD<;!*e4*H?c&|NKG?E*NwdxJw^9g*0&n z6_zqV2}mf~wdZ|Z(au!SE||cnP_#qiA0rM@$wx{s#tlbQu`vz_*|TS+VYF=?4f)5AQj_LzkM9FzWvSZhfg<~-MjV|nuqxP zt_~!d+slu4yW!pX`hG_zsy~(bL{Gxm^I3Wl!0MZ*e)=z$AGcSZZnnE$-F&Fx%)`g; zzrMXV>%RNu{_g6tvy11gtHkCQikARfQ@7+j~#(Et3`fAbYz-tBJd z&rywnT=V`seu~Yewu|ajj^X%kU%J|F_k$<(X(T=r3JA9HG#}Ztw@Vqa~VWn%6tT-q)?_1b@GF+ znx!(BhGaj2NUw8mQihNi3?ebQ1DA~^KC*ly2`&#ubn!?`xd0NQqrheDQkRF)7McS) z8L@bPCiKdpG1-B=g^3RZc7WV?04=yJ15fF7K^5r2P(_!BqS1V0&ikiF6133caiTyA z;}!rdjEw>^=;DW3>Y5fFj9+YA6hZW>^bvIM*Sqb%7srHhfg<%m#Vl><$*%h@2WTb{_Ift^i6Eo zdlJna-@h)lD%-}qgAwd-%<4|K-`;eO!+6dH;q1kj)qJa4XWK%91VoBrr3M!Yv~{^Z zH>*hmcEMPwYP{4w6cS+xzyfU-!7P+%XVFkpdp@QPgu;%>-&m_!(7?a-GAku1^y?Gu&y4H4&A1> zl-zvxBevtjZN6cN&9{nU3`BEr)Z~4n5uzT}uanT;mmo0`cv~Rj1t>)^wI(t5w7yV^ z(?*Yz`335)kzYV=W`4oGu9F{+HBdb}d87fAMjGgDdmPIV+YK+?G@- zk5d&J{}0zUH~+fXfA{9$v*RuH3UgQc3!S#yk8gk4qHmgw*b_(z z7bF600U>mUM}rY=3(qHm1U@_|pF{mCVP@}$ESk)1hg{eJVQ8szwa z%VYbQ?Q_r`IX>$4EyVj4vT7ZHi<SyQXQ+b*oM}pWl_p|sY@cM`W=<2{fkv7Y**oj;6h+FcATk^=P1b!zw=T1DeEuzuf@xA7b(3%g%B{SCg zt_?k*LE$j}*B6G7&06guHvceF*YZReADdb}aZsq9YmPP%`z1Z-oupK?le&)Gi=9YS zM9`l$G7aVzi07{X;xSDxU%x1VT^gMR#-D-P^ne=!!SRt_3p%Z4Toc<+{JPK(P{jrK z5F8Se$05$Nh$KOfXXC_xAVdp*AVhJ(U*1V^WYd`DhVZ22U22t~c~`%j1+P0(s$h$5 zWKEVAZ2794^PBCNn!eom5d(_~f{e9p90jK=c8-@b#~lKm`TDHbVmvIc*r~sQ#m?&) zHCfCP>dJhrP#9|D(U2_?p%BENU?4NYmXqwtN-9Aa3&bu*MPw2^GNm28WN~FMqK_}G zs3nRkWmIv+q*A>GJ8Wfy-g#=vqlX%Jnksl++fi~0q$>;~9oCp&XH+(sv+Z9xT?xHr z2IkDMezU#VK5TcjDqMT`#ceAKcSN+o9Z4+jrMZ}h#<$X;9FMgg+};A}cq{^0#0VZ!t2|$ z;7hxfF!E`bKFNaJM3Qa3&&G+vKw$|CG^))Pzz7A%V7Hehll!vkRPnBHtyp=nS~0nm zy`&I^7Fe?%6(2F55>XUdP~&l;poNBkDh(~5j(J_{PFtD&ab={GH`ax*nIOidxeZxc z;3LV$e>`HW`A3 z+g}*8N6@b@@D#IKnnBCbMT&)l6iXK=!%`MTt%N~qW^D;et;m2Kf=MQ%sR;QpUK(_; z6pOk0B?a-`bd#OcTq!0jZl$C;ZBQQ5=DVDbEu~bb(1sv1wD}O43aOFQq~YAkpbBZs zVt&~XPFuE$%-oGLU%Y?6-R<8TvxHaXF8zrmyuqzc>;^ZZ{u!d;0`!vY;B4kfY^_S{ zBS2ixBeN1iD>8F-&!G1+V1*>5r)uc5oBoWOJ9ho*?BiyCwmGYQce~r(UT)9sZqI&w zwz<7JyWHH|?9YCERAK@gC8|xDl!;tXH;Q+vXlfdZ^+h-)gF4plajf6tSii@uB-$C5 z9qmT`14KdVx$GoPr^+GMRe9Z)y%FUKFMeg5&N5hd0rmspUL@pRBm^s2co~dnqYEz$ z_gK2{0xpiz>7+cxj5_HAK-1iW#QVFeKipq!9=7|hZ{Pg!)ArN$?cZJRAFB7ETIzoL z_rLw(h}#;V!W;$BoHQxZ5&%sjiXy4U@Um12pji;0r2@1^#7CBoR>D!#ww!ar1HoDr z*$G#B%9$hQKW_JTReWH}@2_uvv%CJGx{70Nm`v`5 zNu|Wg7p7ujQK)rxWs$(@$y;)tq}c98vB8dF88XGfXQ~xGTLF=~kx`qiA{3;j8C&K8 z=|&~$o2pd*TQ#J$=|rx1wMQFx)<`jpOsdag!|dkk)A35-lPg*z<((94x1^|c%fqH@ z2pKx%{n9MeYicQt?u_B8Yd)17Txws2Er|0FBc&AzK*^Fny(cQH%%0f?xv0c$?Zn>c z#MBECb!#7w7)u>r!mXX++}cUZpfgzH*2d>k$3u1KnpB?4v#?rkqg2mw~ZkpBHZ=|^oNK?s2E0Js0 zG$%Jtr`G2cs-c!g&|?!kW#`sJ8T!7gy=bKLhXHYhJW-Y9X-gxhxnuTk>w*4sxBKqv z+xGw0-{0RfFUEgrP*MGC`K&H10H!T$3p@8SaD=K_$fWOcw5&os=`k(*$ z@9EGRjQQBr6Wb6F7sK8Zv#}Mku@$qifsr{_3mY47MiB+?PCLxt+>EEct!{C5cXQJU zk9_s%_RVpXUNf~z!pGrhm>4ozu`13IO>v=?BylS4G*#T&s@RrHu`QWmTQbFx zTMF$;Abj*Qh7q^2&QcZBK>*&D0PjnH_a)HoON3yp;fN|0)`DgyAq;B)L6q^i2?0dL z*lbNir=(g#G?;Xrb5G3!E7;8w>io1o*~A2^r0PMKT9zA$+JO!ge_DO6sdo=>aEEHs zswUVyQrgBkNU*_tZR6x301Wa#gDcy%1{+i~x4UX?chyGH z!elsNw1tT_3s{(F6J=py6P?ukY1?cmn7ilX|Lt~nyT9AqUOl3Z&DXW^b8)ua?T!h@ z`dm2H6&I9}Fa7+KOs(dO+fAzLfkmr1+y|$)D@}19oMKNd1#6MS#yKUi!6qspJ2hwt zHBti9HG5vj?buj2pvF!4YTQn}&J;lfQUnSE+rYp!@{#5vBMNNec$_F;yMcAvD+PRLpXP5b2~VD@UZC*Gu)Wq#%iLRMe_`EM`z#A$;KK)+UZm-fmH*R2I?|tAyD?TE1}BGjr(nG zsy}RA%71y-Z63BCeypLc-(Ej_xvSz#^^I1HnD>=tg>j($c%jALN`>d=zL;$`x6reWyT$0rdYNitVe_y=ISxw9u!t< z5o1z6FUADZJ2_xwYjm*Q#S12gF3iuyiK}f*T9|caZL5rH+VIH+FT}Z`9<&sRf&f54 zua|6%=9~zB()LBBzRJX9ChH^(a>pY^ zMi%Q9Fvzto${?4lln-7|h=^2npBdqV25i8T9-!>pAdw8M~?^-f*8^f%zYw1@JT}vZhmvZT* zrM39A*jfErG^RD~)D@t zmFC+88fmPJY^0HL7I&dcT;=;Q5hT_lAXtycS7Ir$%u1*A%Y?)5Fgx>lCOz+0}vm_76L zwP-)e({Z9Od%Z+IijL|>(QC@$Tri8HzDIzY1i1gelB6X4a6}iM#VsI7Zz)QWUXzt- z_EY4g^rZ-eAZA3Jwq%xKDrJ^T2CIvelqqNmLM8^%gTwwkQu-7iSDOO|VtuYaQm1a` zV?~I+3nTu0ry94aaZg|^Y$A#H!+4x1#2*&eE7Tj^m@a}>C=FT#!?q33D z)e`cZw*D8l+g5v|dK9+1{n^{MZ;zVPCfG`)$_K>L7bXkgmmyVD^x5{2wYbSi_cE0@x0dvEb!WS?HxHZrcV{=7 zPt}M`>V9!n|K!cL)j!v_AC8!-S1B&#rs_#;PSa;*V(MOwJZ>(GFWi!;d&zB1CHGI4 z+&^7%|8%()JDYM=JDXS9sGLwxWsx*3>Rj`5oVbGO63#U`%DF~46f$InG?n36`BmnE zT$&J;1rroZ&*p->!cBp~qE(1HxRcf4oh-8uO)rJVb#5c6h%_8KnkHe*0*XkrW_MHr z5=u27rOD)(pplV5pWnCyX^cjjSXTx=&>U&w)r!iIK`@F~)|6P+lvvob?p+fsM-E1G z@#V+`Hum+a6GL63*VouaZF|YVrz%K@HLRy$K? zxvmk<7m!aU)$|tXwF;C>x`sjxqgbYDL{(q(zU;LXt5nOpS{V}|+|z2A&0<;N+Oo!V zZ2>8m6%NLZ&kA9QtPrBILL2m~vY1`pu&?x)4en$x_{_3qnq|op%lhTkh)O98#*UX# zFiS`&7!}uJmAXvTbPLPePdWE6srn0~)nCvh%CG6+>Sk7^s-S}_kY+BNW@egZE}RzS z!VO1^uCVJRxNtg-3uok1Kq%c%s841!iWL~OO20rBk9Vgn#SA!iA+cfl9j{Dia|Iq$ zn!rPM+tX@*fa4mVD_U=wiABJR>?$Cz+C8xLJTMCcn3DncGgUyEKd}ZeT8m zXoNJHf>a~z^GhKmNOc?sxP*Q}90T22Ow9}Q&~v5T#k}bF zIO9`>38uPnqKJE{@NX9-Kha6~;-GtEf&Z=y!%Fdj~)|}|uQuM*+;brUF z{r09qj)Nw;FFpZ7JwhTogYlq#g$>y5B?s$Z1<7z`V%YrGK<{@eN^}^F7)x}3C5R3X zhdIYv-AvZci01%I-f)zOQI)Xh_pl_yr3EkM->OsT85Rd-hwij{=b_QZ_ys6m94J>q zpu9;hmj%ju*Y@7Fz4vYBQY5q@>uUhYVMd@_OL>tfr&GyvWr6I~t-(oorBH@f3SXaf zWk#ysTasHngF_tJ5QdC48$&j!kn%KPMN;R0TKTczN3TWb7 z8Q-TW3dL0TG@A6%J6|A4>6>FIrnjo_q_ZG9yN6;m{@v~6_09Ip!x5W{l|GU$#FN{f zmK52iHj41bid?FP*s_tLWg|t)Mv9h=6zy7!6zv2f#R%u+L^BCWtq0n&pgtFXMCb_2 zpq7AAO`~RI&0RWJK|`m`dxmLugn~%aaKvar1vVz_!blW49o2GU%0x;phBDpeqsS@? zY4s{m*Ji7H5OpoXsB5>97^V5h@R4M%$#CrGUK67h=ru9Q#xZ?UHJh~`>@C3Bpb
  • _ECw{_vj^HkMZ3LLgl-Ve^6Hf-9|P3GJ%{Oe}_-TtdCZ9b;;x^K+Omq$>a z&%i`1we8pIVlwLDLM^##0Is0{SG0g@Xu#^#&{mUP|E>X=w#nI`RH@Pu`tlVjdj|9E zptgIfJJ)H`^|i6?NVM;0Fk*b)Q8}pwtn&Jf%BUJJwKd>_l(2kjV@w3s6{y1WXlxlY zee)(a%ZE}3iMFZKkH(v0!2XE_K^084qL4GUH%iXsB07CklgS0IP&N)~(v}GkgDFS_ zWLZMXvV@jp32iG&C`APvjTlP>Y=i`*seq-6vpNuJhOTwrV_ya{)k!I})CWTy6x2bC zro`;%t)#qltqv+)XRnJC-#;`tIC>?ux7Kjl4(hYl%r^Je`|a+Byv=W>le#}r!EAW% z6AjgsIzI%qFJ`H(%9U8EE11(#-Nb8)Pv#$#L6ot~M;S{kwlaboC&QMB1QrX00w&lv zchbV@pkU@;{pHOyC?N7`~- zdu1$XB8gvL1P!NMe10Kv+m6C&cDbzsSst9LIiyl;_?(4_-rpXpwJZJaI*H?c& z|NLSI?tuQOH^B;Q4=|g@UX%A_5IGNWh+Jw57?}e4=C@K#SHrAK?DPc-YTqq2#05*s zCD8H&Wu#)3~ZW?u-1wDVHgUQDlH`PgQ*>G&J@Hs zg4H3;h0RytTC6P*0#Puou|2KVpR+ncL5pPl`m&HeS;FYm5eT<(wc&l^%aNU<5$=97hi1~1NS3Cn6*H5E`*un3Hjo@#rU{G+eyVf?&gs)u^5EOQ5$%jfRY>C zOKx;8{aPSQHD@Q+D{STP#RUYxHysAwkij?2M@AHK8;%$Yxiz%~X~?au3ryaNkPAw* zW@q)-@;6SFWr2R~!rlJI+sn8A_^JBph&={)B$~zO=BF)X=tt!^S1h@XwI6?b#IL>Q zK;sP`t)vu(-p#m<5t<-doATM(p^h@lKe;K>oG76b0)9-qxbS3{*2u{%>%>l3WrPHN zzzOYR6|D_Q4%>%vEG3SmHpz%?K zI9;Y`;;7Pyg-MVq<%1VWnryJ6v8@p2u`!&M(sCrR_=hHqk7UJ!A*B>C?aHi2ASmXI z3~OV_L_j5JI5HkF8jFM_u*fLG5wc=l+h|=-Mbzf2h^8$gphEzN!tAo7mTD$cN{$ky zy-H+j*cog$6?!Qa9m;L1p#4aG@C6|!Fh`x+R zMqmkKB+f@Xh$&mlW(K1aI!LK4AGJCrH>DUO6!F18GPt6l_h!c+IZHi#4oQmUm$+fp>5-|< z_ITAeZJI*UUp~7Y;+rFq@Q$Sf(ckz)enLxdbXi?QtJvGV*JwZB%(m|R1*(>)tDHvfjJROD{)Op`A)^E=W-i}E^ z4#N>+A%_M=k%kY1Vs-MZkOTSHyxCW(m zby5JbmLd+SGQJAxr-J#kGE622@oKhw)(rM)hF&#;y;>CP9giK2&cG7rOcdCwNDA=u zQZmYk#9=MQOLbPRG9`saqY+~>_zeHd9!}DyZLiUwxO2iuU({`#N5zv=c_ct-aO)GB zH(Od?x8e{jZK%N*K5*3lJhyRrZsYWBEr3!vC!l0~qEwZbNT<<4$t64~7ffF5RH`tL zFAQ4_DkVp{&{AAVjV`DXmCaL9UQ0bgSc>WDlAOSkO~qxMEtDQOBcJqoPj>G0{vWPy zZt6G zpFP=mgg+4NP(YrkntiTtJlMG8i$Q+%N3T(ieN(Y zu9`lUo4G(Km&={D8K!nP^XmPpKW(lb-hB4;?fu==+24Nkhu@rSf7sS}>hX+0j<$V& zVJ@Z_g=nXr8HFslu4}gv-%IVxtVSW$&0%b;6N`gNCpI)twVd)6n3P;1#- z$0P1n-I3O~15+>+s)pg(X8gh?Sil8M;9@FpF%>-fLAPba73d7M?ME~1SxuR^XI0fA zG$5Lka*fu6X@(J$*k@=Gd|&)z9$>1)t{q6-Y6C$Eoy}c6mC+e?Ui})Bs_9#u1=&kj z+t6~WiE{7MrrLKXtp|R`Bvo2LsZIA_NwC$L9>t<1w^zja zGQ^5Dq)I;G9s&g`DG4?r<8y7Vw4+o+#}?Qx=+21+N=mH^jq)lqnA#Us4#gQ0YCw~T zAavre*xYXLjSKW?O?O zqSB8TZ?3HYmY{h>Au5mx-Z#GTJxUrZFw{JW2zgD63xweaqT;>04s@YEns#c2YX7q~x>|Pz7nfKHJtF)rsTjbL+D0 zFF#g)bu=)><_Bv%Jy>51oZ}wvdm?Z?7GGX%$>PbH*KC8spmm7G#20n$`TO{*dht1zUp=$$KU|C+AMXG;ftwhXHh8CE4StV(2B zRU#<^#L?KX+NF(-khB4!_fZCjE>!+%B?~hJcZD!GG9ad7V?y=`HH#^?hcBVJF?o~a z))gRBFhBrt2bg6hj8&prt0&{cZLM&Ltrf%>@kzb8<;s?^V2c3qb{rW1Kvvm7E4hpf zM~oJNMYq5*hBj&$1MPxFij?o}7)q9nHu)}%sT*rs7~MJw; zITZy_Sd2np4-HM8;!fP;vr&PPNQNC}`O9j~LP+ zhLfl*kR6OThht1P+RmItV{6#+V_A$tH?NVLbjkUDQ7rfL4R5gP}4=BK7&T zJ16z>1Eu*lwEO85p)(LhXSf8;Zyu1OUgvPcXwVjD0aIF_IK9qr^~PlDjJ4aF6(KY_ zjL?*flCZ~Si=&C*GUa%jC|suDmWbmr7=BhBC(~;fA`jv?ARbUXbdwbjN>DJ-f!0gSrz?= z6y0fQXaVgiLn@9{5j641x3DH7W`~2uWC*qJB5#7gqsa#z0YDDhSR2VIxJyUgGb8WP zk?}DkNuv62#Au>A(gH+vq&T8_hL-zK*24#D-1+_*R*YJygP3+(e9F6SMN)-txiF)= zlt7D5jo&x~XDJBIQq(3jP~sN)#QbS%FDmbT=@lLIib7gre_X35lp~ErQ;UT~Xe?f2 zcZ;^4dwayZ#gh4i+)D~2tfN~L$~i?Au?(UsDjOHV)M`+)0@P3K#neyoU(SDMzVp9& KTU^@JF9HB-|D(zP diff --git a/contrib/mesos/docs/scheduler.png b/contrib/mesos/docs/scheduler.png deleted file mode 100644 index c4570de6977c434fe2bf2dbe5f98e2abf65f8a21..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 222525 zcmeFZWmFy6);0+}(q_yE|RTj413!tdAfdAh6j_ zLw}#6DvwL8T`UF0CJXISuemPR?V-CkyZV}4-XMaDN??y~P`mdXIwTtOqDXnf8PxqT z7N74(1Mkk@pn;LUm)^Z`({R4Wk=HxY++>GlYdQo%SbGjUbN$zB>n^jvYwMn(iPUV~fOCDVCu&yG{Z08^$&$Eik!Q#)OKYzm zI2+4J>9L4>?_*Zh(kcrP&xN$gmOqrVXXlsKhY@=32e4h!Y|(~Hu;BYo=WeI6PS^W?k*?9i=%?`Q zO{g~9!M9TVeW`pzr0f!H)6wyReBU^_XSQb`TdAK|wRea{6#Vbk z(H6{#{o&p-p_HRwhUw;*pkwcMa$kL_OC9TK?@o9f9MtuHXdJJjG79GXidKX5994!k zZG2f8sAY)E=S{;7#z%%pfVAj8p~ED87i-T1O5_&CEqxb{Q-f@D{Q{$cNW)J8tWctZWs-whe?)Y5`@H7sCZ}&-E5ok`b<~kFp!3yQH8LoMOsZW2hIfflWb&;HkH^8c7qe@CU zItclW5i}?jWu_RsWUYY>alUpFeUr}A0*|aoCz|mD)7j2N^=i&2%BVh zL_)%fQ^a8|$O!Z5F^nBM1~w(cAwM9NUZD>`4#(Wuh*+~WTqNme{FK>|*K|B$FXQ27 z#0{VF)Xm{EzU*7`(_!@V>i+wV#?EkXG2ef3=nW$S77}C$k;TAB`2Wd)K+qXTL<9NH zzQiC6X_qs|GGs;;|4)SU1#grKF#bpD@&zx10;NMqCBqyuiUWa)`3DDjSQX4axgIK} zlZ6ndjA@d*R4F6^(0_1jd`3+AC)WdSo4dWzb4T9 znazpmK!`p^TnU`l~AaoJcqJg~3Q-lE6P5YkTLg6TPlBZEGTCPwE=k z#4f}XaWXp)CX0RPAI;hg-0dsV5~?wD({~G1VjDeC*HP0M9Tq0xNL9KtRs&sjM~pm}leXV>_qM=PzkvIFO&eaAIBV4i|Vm~Cf|6(W$z>!iiRouH*OhI2s9iLY33k zi2&~xiM($m{=2Oq2RhARmS95kKf?kfAzx@=B=ESdf99n8&Bz;{W1S`lheiKx&Hvps zpooBSLV*qG{p|w(>q&xDD7zf7=6!_SzghfW)$zv}KX62#oZx;&41cqU|6NLwn6qE| zyXqKn_1E3{ziEp9D-e^#zVb6+Ac?W~HMDqRL)Kj1t_gh@yZGC+AqnwgcDUF-XA$JW z1pQ^_2o`@yu^52K3$;LUp^M*&esPH1fCLz0@C~jza%r@ zU_wC>qXcA@G#>lBXmWl3;kIG` zkvk3}3MTL`qWoVmc`6Ht-T7W<@_(zUKdkL9AH-mO$q6GG^tZ_H-|f#_7U-2zyx|lC zf2Yv@p@=`(B1sTXPLd#ELBhX?@_!4Zes#al!#rCUCFY;$1xfH1di2ER{7H`ghhO=Z zexXO*1kuO;mMFzkfY;O*57pF8DGLn)q5m(cdJo17X zw$M|=%H_Z10n-o$Y${STxu@q`x0ZGekOD7o`O>p8h9ae~fA2B@Eyq)>z4YHw;NVx3q(V6W~EuMj6N1}0@s z@9bfVXhcMWz(^42{oWoo+V~NOw=aW)-u`1jF<4+H9r;1bhT^oKNyjr+CRV@wt!)HY zNMM1^eTamRCW)~CEAuHP8wW6D0)gDf9ZunuQc^Fc7O9?<74VqpSZIZn%7Bz&0e;2j zVM-vL{kkKqNNX+y|GNuxqL6mJj-ZS!m}k)>9h>Wbqpr7l49_~go2Ci0wNPhNN3p=>jLSkEASU1IKS3sS%`p_QQYiNoE%EZ zKb}xW27JQI;H%#nU8^GxbORk_|8vHmIoADM*Y(`fVDqI43i?>|2}4hIm6oaNdxc*D zL=f}0rQ55E_{EF)-V%n|VX{C;@%!G|g7JGwff0hc=88cgto?mq>8lOao&$Pwd#i6Z zXvz*L`RchkLF{v>K%;%sS^6@$jnm|y)u{P#V9OJy31zjMl<*)0yb#9E9_CK_v+X%4 zOyL*QAr#IcsV=?G|0@C08~&9TQYIHjA^f&9{(b}u{LbokT=MSo+KrO0nl0BL&F`n& zMVJgP6y$!lWxwLjdU7|Za2vii4V8%6{@dmm4D{ZQiJmlSRUq&$0MCy*INh?;Kft%_ zm44_8>L+Xq1(dFAbqH9h%aw_C+(TE*r+%d|KuHzu-0w1kwi(#!Yshz{+O6fh49fkM&vcRI*F1?AdO z%}kA75%#IhgZpC8=dAf7&6sW!dP`({oOp8ZuR`+Q2Y(t8M#l|)?u3%x$HPhs?7O#t zK9PFTzAVD_)+VsmR}$<^gNfV!h0dLf6x+KH}Y7gq3d zO|~s`Ze^i?AQ2=;q`_o?0F_Ob@%nEi5(hcfUjT+`#<*RJ&9vW3zJC@z<&4M&F(KhZ zM*PC0I{_QgepYnte{V}=3Y2h z`0A)djnp04@P}_r$MR1cvI*8`>YP=ze(OfySFo+AGE?TvkReZ6@ieqzj2s>h675R` zSE;bCQX{<5rg=$P9|D#+w6w!k`rLV&Xbm;J1FFO6Ygp+V)(K{4Ret8o_l(&L-R#UV4!EG@G zvn#Urc7t+;6(U>7#s{)i3O7}Kd|>Ic>wLoX4sT=2h*_vEeGa@E4q99hYbas*hmM4z z3d)~)R|FaS{q%P5vPjlq1U--d_sf~&A9=*|@WXrYp$ZJ1ZW!yHLy@q;!!}8wo)?_I zMYqO_;gJ%K@^xxvsUvjTwZLIoX}@U^4IKa~oE6UzY} z&o4Xcb|!i#E*$gv%9mQKaAO23I8T&V*&a~1$$p%!_;v{fPw|Or9j*M1g((WS**NC_ zJ@l-XMa&2j`p&{~O9q;Dh&6x|^e66~&Th|@dLzvCyB#!|Y+2mzMyNjj*1g^^62>!b zRw7@y!t_!EpVi7&w7R_06%dt*v`W(D`C#<)6<7F0wO0Up)yhxVsSF4y1^oR*;zlCp z(YJz99BOC>u(`D^oz_?U%FP)|Q@hP_?~ttE#w)vIN7gx>W;p8b(MQ#|bbBX!A57Er z2eDo{-MqY%mj`;oF^9~EVdkM1#@Mho9N`}qKTBx>5TW(pSisl`8xwWG{how0B#810 z?h}*2p$;83E#c8has~`2bQ&PWt&zmpSN0#PJQKxr8IpVtg20gk_-BTQb7kNn)?oG@ zJtp5qL|9s9{5J3VT#YwBdcG{TRJj=^Fwq^o#TI_T@w_9s!hwScpK#wx_I@X;62>Sf zu*2woKV?31`lQQ2^j_3)Z&ctlY@q7;Skg|CXeu9`c>HEDcDk8YGC-$>V@(Oe?aA(> z58Vz>;Or`tG@+gn{H%#)u(&G2{XE!1k%m1p8Rq!C_>BA5%)4^v zGTfZ|V_PFc6tRLCqoVc)@lI+Z?%C!1;KbNXWa(W1pHza<0J$X3iaPzVU{OcSl5q(( zu8BE}EUWb&piM@Ap!sI}l|K#u2y0O|gKm7z>`|S4g&SUdW%O$8!L{HkeiZ6>6>oev zg6aTVNf2n2%d(zSx=?trT(1wS$hxY5rdz&h#9wBZXVZjv)beUbt7Xd~=j5rJk#iou zTS!INr2dg1?#DWS;GQ8oMme;a+X*DZ>+Uu6R`J|3ojz=}<6s&Q$Cp z!BKCGEkLSS_6@RT06(3{}NzWYR8w zR4f3Dl@>e1im{)-o^K*B+%AaxhDc+=zpR_rlYCY`c9FwzQ;aC60){>@f?M#J%)$yJ z8pvgwME|X)ucCgEahUC1v^H19_Kq;{Uz3K}tFOE9;I@1%8Sm;ota1)p+UHKco1kQ; z>KjHrbWr69N!xzcg}v)jvaPtS3|XINQ^}=wTv8F-G8Q@ah&4z=UGWPj|Al!B#!!HZ z$IWKFBdyYE5*NBNb!r_bN!4lUgG-1=zqckAz48_D1HdWis=wX#PkbL}+0LcF4`+}s zdg1R9>tW5fWehB=L#;5LF*LN_z0BFS=f_(l_iDHAm|K@8tr%=lukvOiT%f%?+xXwB`qbh z$9A3s`2iji`cE&YCFw_V$Ongq{9a9JpOYd+Y#z6gro(u{Md=rDY(I`U9;H&LdZb1cueHyj#74EiIp{Fj~xAP*+W$Ps|KkT_9{VjbL;5~3B%!_AK-j$Ocb z^FsR<>$B`1K%JTk61Zl$HF6Apg1VgQYE&RuF{fb@+)tAyq(`E(JrmmALt4~iyvuG4 z%dNkzk{+!%(|&>Z@t7?lK;tK3AHM*=F*|!^WlJ6uZL8GqhuvMaJJeYCm`;IoiTFq4 z7eTtn8W_34D7L!giP3n27_Z;zxY1~YIz@ja50f+*5>F@P%_h>JY-pMEes;lFs%HW` zwAXWVbF^>LYW&u(>XhllF}rX=MVC7#at&0mi?O?9%k z*8EvD!*@9nvoR^Cw4*mDta0$0s7M%dwV34^Jy8?j&1qbVV3cWzaw_TpOC`#FP9m3O zyb8?`GW(fJMIUF2kTLrisqTe&_TP|oHnoHqV^w*{{qm70ME6rg62RxgP&duq%5z_N zFuwbsMiy1K*++g#<#cU^@Gw@_sxPurntAF})`}<%WY=H&XwDe&;Yq{WUwN2W-|_H` zyNHgim~%3EwV+E@2$eNUVC|OMD%9MD3?uROt-5HUJ;V7X)vL|O=Xt+R6SvFQBkBm* zY(=?bi-t};>Nfoi&R>v>i$qD4E!{(2fM8%Do&6Ocpk1$o@<0KutPrWTHJDE zaCq=q=odvlrQNUM7krSgvrBW-HA!+IW0}zduZd49lyNSTf6oXBrlDY7Us-RB&-~$O zK9Z2TOnyulf%3RZWQP@RJw5aIdVgxWQL_^G={}F1ke(i%IT>H5Dt$X}->8at&uI>Fw`&8?s~2LkHLP=+*<=o7R5dcM#0`9Oj@GX{H&evI2ry#YoIz!z)iMVzNp1 z$?tCZh)tg#X`I_oCIGT#S*--76M^iLMSP@GdYR!(wH%;H8F7Ph|2s8v1jUqp|6YbK zx}! zSIJgrqW+8576}?W?i4=7CdP*^+ovvlCt6zZ_MwrLCpn~K4pU*j@0s2aGKQ$BW@*N# z1xMoS6{j!&KM{^)Hgw|P)Ld}=Eb++P%rX3x>LNA8Ii&B$3!jL1EP7>qm`w6b)S@Gb zE!=76@QU{rzrbsqT`71@taDPFhk4VBmy%DnijX-OjLII)HKHYU{dnKSY1 zBTb6BU(Jz5na(EvWVf)Ik`_f~_0XRaQgQG`a}seqWXuz;4z94Iq(Nj3m$Q^$_Ssd0 z>H#UAy)uQdK6!wVUQP(hboCjR0a2ZCUXxz;4=tCp!G&PepS_;&hKWbdk_~{2?B&o^ zeI>J49dL)<3KeytU3Q^PnpLL-U6OAFDp92HLfdxWYl#6uls0`MXwf%l!tB>rXB1@V zy?v@d%U4bQG-L-(z|#FUs8)>Z1PDJrt!7?wDaF!Y8AxZNl__(^ z8hDv>8ko0ozALqGmvi)Hyc<+stZ9&F5F3DSnIJ5q{5Zq`3;XAZmOX-4=W!8-LD75) zq>=H+J0ep~3sp=P?#L)_U-brb!vlJYm=~y?pEMQ=H>OdhOpT*YrJBn6x!F_-QJK24 zGGT*hj^q|4spnMO`Y935lHrfZzfIA=;+HbEXOly2SgDJ`x36Y9?>7XC+TK^*m39w! zJ~u89vvZfZGqZCX+n?@4fqVJzqF#nBt^nrCQ)<7_{t>S zbiGf(*iaG&%~OLt@DZtndrP9CvLqXwMvnd7vE#^)bDy4aTpgt^Fw7CRc!7uSTPhN@ z@vT?VAyHP^_P8{7f?hi&LX_%ocd)^05}mKd^hoW>qV25^O<{2u3!G|eAaoYp;c6F* z(9kheBfw_H`3i$As%x||q=d zbX9B}gw%@ixG?XbLb7IJu=+abf=NEMo~<{-=BG4ZswwAA^85TC#Q>bHuD+Pkp?NsQ z_5j&!6OkUNK#+HZKC|l^hqR$h=PeA%>Ki114AG_2D;8hJ9HA91;I+#=!)KJxx$F&$ z55Eb=?(n;o-?OjyBRQ|UqtcmVa(%A%0M+gYngDt`HAifiFj1_IbE(4-EGNJ!^k(e6 zK-X%xx4oD^tWE287juidw(?ut>0QT9gn~@_mlQ zaQW5rh_U?sZo9E8gLjAOtFzoDhsZ9(-RZ!DzS+FP_UA{-7tqyQ5i<*hVbdd$uf`{b zwbZ6xZ`EPkhhAM3{Sy4aF4e6MZD00q0bO0Zt*s4U*!L1)4Nx6-&j^8}cd69X7k`A? ze{E8y*oI-h=2`)(UmfQbTREk1tKD!hQGWo2EHWeSHZD|%Ao^BzDq(*D;$YY6G23u; zeV(|~$uYgIEjCrc7a{}EvNL|C*hxb_WHFL^?N!y=wT(7AkyQvwSAx4>-YS(eepFQt zh?Ui0%xwYcmp6OHCW}U3(M;&D_D9+wr3>~Bpj_I@)Uyb3KPkxJB=+WlGIm=*kacG| z$@%Sz!5*y+Vx7ymuyW>s5+`2Qs&L;{K4Q@U^lT2*#TFt(+r4A6tPJGaRk-2C_NOO3 zIj;_PnS06rqn)r7KIEdY2O99V;t6W(4@bRFRIyL4CIWEnmzMSCjdy+!CNgHCVDX>z zYy~~UcrZGtC#W93p)?%MK^yI^QH(UA^PS@`4IiKOI)0gQV6xxc`>YAZ2Q8@rt>-^j76FL&l%XlFD4D!w@GLyl4M`ck`TJMpVLB4(Plt{9^?me zOfDBmrEwWa>f^AxQrH+@c0rP*Q9+9Z;p*_NycF&mO|KHVo3FcE=9t6IfWbZD36hqi zCW@nQ;DK_ZLea#bW63&ECf6PC5fsrk51BJ&wVG4>p0z_0_NB!=?dcfY=IafH( z)syzT;eCoXnsKaSz0`oe58rZ8Tw3fKr&5WFuj7j#u$vO5S~;}{Hu9>#B@4dA?I%o) z)3r|ZH&H4$vT&Z?EDiw6Jnip_hQs+!W{`JSf6 z!4``5;-rfYk0Z)2htRce{b!7-rxwE%tL20H%%H*f+ah)>ZV=DqW_?o0w{n0|K&jaM z9?vK(tr$Tc_=@I5uix|)*aIQXlXq$f{BZk2m8ZL$Tw(64@exOnMIYK1PZr*paIXr~ zBZKctSM*piP#SlMw!@(o14jex(Pue&(#Kq@A${^PM_N}Gs>VKeWjZe?GP+p3jcV5C z*v$dxE=Rkbovsq0csYSLi}B1dUSuN{!t{%HwqaV{-qYF(?i-NJPvDZAHk5C0XC5l; zj$1M>M+7{QY5zN;1K;|^9kAcbGer`E1Klnr;+7`iOu(dqk|OfUFqp~ut_<0@ zt_OBd^Ws~x=B)f`P|Z*AC(JOvmvaMKUZV>%}azZrMVu5jV|PKAHHP*;c{eu8^lxxURj@=BaxZ#kx0 z`O+gr<2(dSHB+=1RN=2HsH2kwNT+;MftDkscQCJP#%HcI8J@VhDEKMd-4)oiSuMeKU7bUfqBgN+;?(DkGxQUiV*v%Gx@HRXA@GM= zejTY_KC)SHBfgv8<*-%b-QWOiF4p>!MJyw*OqXBRB3m}!&MR9CmoIRow(Liy_Qx+$ z;NC~6@Nd!3F`jMz5Vff8S3UXlsew{P%BP?yttZU;@}%#jVM0(v155k4ODw1U<;T7bczd#B-q!vL`?ly#&SJPCW;>>5!l-30^dtM`}|gH za$Dfb2ynpha8JBsL0lBLoHN0xP#OOq@5w2*?)-6Vw`*v8h5J;`xKG$G#1^mxrDSO< zmLTeiI;M`tjy|ARBJ2b;36L!|B~@`D!V?$M z6fDJXed+KyMdf&e%H$i>Wj|o&;jG!zVG{jWWwQk|%WDROB<+o@Be8s(7eZ)09AgLD z=L-1P#O}?juMzC5MCam+(ajjqdfn|rh-MdGY0OC8^{1<=)%{qBaly;+NGz>ci!6Ps zS+m26xQh7mW?dpTcF|V={PiJ+NPez1LPc5A+ReIb9Yn<$iS$)Xhxww5W<9EUI#1A{ zh^88eQO8n_<|BT~N@2MP3bEep0U*%bwuWOm_Fr^c1Ur{Gw5b8A4UHFO@{u;>)=9N` zq((9D1$6{KO;V-OsB8p%+Wfr^e0;P{Y%={~t7%4CjPV>Cb3eo!0X`e=@%+Ab{0@){ zM;#=h2{({mR>IrcOFKs~G4K&z^1nU|My^SxIW;A62dJq{Gb6o_vbJh3Bs8$tg=xDT z3$xfzWH>#<6Z&=sTvH5vOAlc1%GDZ&Ng4^tLbRgA5?n8^986=3Lfpxz>U1RvDF#G&CN0$8y{G~cn04>zl6Qf?MvZcHS-jT7}^uQ zO!vK6jB|{;Nxi+?bKP*%T+=_~9zukPp5SlC*zQk5E!_1>q9g}S%N%wh7$SiUz+^jl8AQ{)TK{%g_4QQA_isfu@(h=asep>4HRvteXB{2gr zIz-^!OJ%~49%Js7?K88*gsXk)8`E0rmazQ6KJ=GOxAm)U>xTiBlZJpZ6m61RNxDc( zF}<6|kQn!*TX%}tRicM@`s@9Hj4K>$!#w^CqV@-fxi5jM|5-N7&UhO2>(@mzkF&mv zF{eKovKdof!F_ z(1?Tq0y2a?DkQoeoG$z3Hn!MV`u0Y;{@K9&kFNBu4|-_`fCUzDQxWp((~9-?JLjHl z?0M4yo51K5C(1vGtUZG~-Neg^I!I<|9$M(t< z(fgHT*Js8!3rE2~kF=qy4xEOO^}JH9)JMS!8G(VeI)U}m_X@N5DH(@0SXpmF@8;2iCdn5ejM{Jy^?;QRuEBX3Il!xJl)LohK( zL_kBAdJlROI`j<>kNlc2L!4|0=Eiw5^H{y{i#X%nzK~2;V*`9v!~zK(*8158vgU?$bvKn97Q!gL}Nh#EfjVcPZ=Z;V|jtwuq(H)FSIZ@nI@gdl%@9B=Xvo(!l8ALkd1qQMW^8 zmK(cQep=egs+09Vp>xxf{)PfyP@j~Iw*aWasq}%2wNQM*r@3Hpuy4Zvz;R%+uUCWL({*4&)PC1sA{V7RR3iWBRVIJX zFCROaeyR5G9wmiJ6kmpSJ%GJWbj52Cx{6e4fGVLCaHx7!ElpuAYW-pST%&?pY~fs! z_PDBU*brtyKH75Mh2Ih}B06tPeQI71U@mMI8|Ug-FBu8qeYLqtl`ryW!B3 zhrNrv-{F>q$F{j~22a_Yy(e!rM!e5Q7dp)-4~dV;PbhmLX(Gnu_^E}VgACOPSD0>i z6z)y)!99xP`EtY52j%x7k$oGYhPrrT!s%)js~PJksugWRxPR@8K0! z2zs_Vu+D9U1xp<(UZ9;1@^upgbA}>m>97>1!yj)0D7N0M1vi>^Sy*W+S`9Vqaf~f7 zu9LPd+&DfjCi}OH3|BLS(FL2m)8Sq~vU^gEwiYlKs7NYvVt1R!JGn#ca`DXE!qp8e z$6R%iHGV&wY;sHZ4oXHjHnMHLI8CCEIuCMZWNMEheQ5XTU&MhUC9Y~ue>DmNGi*`2 zA2!e>`KV-Dfm?+}^eBjvLUD>{>=oHSaza?BxbplMZy-wTmV> znxX#pxU2<9DLNYm&v(YgZ}c|pm>lZ!0iAKNfs6Ks@oKmA4NtqBL>S^fOUOGA{98Ez zR~7HS31?7E)gX2N)|#QvlY0>k5}e`^u9Ξ8c{r-)F^s;~YK$j&X1)Bm@MpEIGqT zs89b@(oCiE*W9#Lo^xa&Aa8W7rDX|ltHgJDfmfG~$&V4HoOYC2a0Ry#7h6(Y1S3hSpU(HZM--m{pmrCuAQ z200PGl~KjHIO;n`;oReQwdCcaTtubUrKP4u*-vXXQH>(na;?P^iS}%R66A}fir${-@!IcGD;np#633WhrsHH0+Oz0et$kmuKr`4z+K98fOml=Q0EvFD>~B0UuJOZL&G z|HU4OaR=LeM}A3KEk?3n5y}H^T>qWc2L%7^yy0^BjhIbhMjV?o|C+g85*wXMPDeeA zKlCa++(}#m4FUCMIuyfiWa)tL1B1WYMfWn~>-1G2;gxk%2Wb^*uE1<~oSP zV2>&vubogE>BnM=n)&FA5*CG2x_VEMln-g&S1fDDn$}B%!SjLUr8Dl!8K~906=kFE zvhqC%GiQ2uSQY$>nV8aiCHfZG%U*pdyP>rkCfZQN`l;DxyGKdr?Z+=UJ+_&cNJx6r zyf8VF1++-9OF^d2JOMd(B8^mOLO6It_1kyWc5Ks0&LS@Qc_ ztBgipvGv%iu6MUxqH2)m)CA)9fmf=mY;$KW46Zexs8`ESs$R${!s+!0T=5bvtYLiE1`8KVg)ll+fLV`F zab5T4?NkkXM*8wyo@J{#K|ISE8;c$w`>#1HMubs1alI)6A6h1_YMn>hZF%Mr z@~^+P&LpC%3i8|?%Zxm1-bWceonD<;-Lti>w1H~g+KwQ_qXx+tP?^5R|2Iu~kOVAr z{+bu(rRc+~Fwrh7uGvb7=bSC4glhS~x^kA#^(DH(Y>t--%Cx$YqtGHC-kN%3@|G~80gCfy82>r z?c6fVO1vc;oNP4qZeq3}&L3j$RDbcN3?BBx!fzD#WRyWCzY#MiYSSp)ZM>654?W<9 zi1;!5#f(6_eS*d-$p}M{1kii$f($}dF=kk7`3q$@A1Lgh$P8!!yCXhm;a!qMS>K+| zc6l+gprbrjkIp$_`$@nJob?`9pFX(o`6;$yRvRFhg&_U;>8ZV>~C$>CHMYDLm z`c0;9YWdJV`YeV7C4XXgetqifMg=@JxaeOGcMpc%-foI1;=+a5n(T4nVBERJ(6^^P z8116bs7{!}+@G~MVSRaeMUQgQg7fs}^yxEPc(^5PBwaE0WV0`vh|2gAAIXi`6YtES zqQ6;mUf)>$07m0tx%u>sGtA0fPqeaTY2^j%7w2e5Q%0mua{37m{%y}3AqHN1z&ose zU*{`7m9u;3v*YhvD*`4#Tl}at02o}y3|#aDlc_%5m92V+u?yGb1u=Y{nT-s1hmICL zVzkpsf_mR^m74^^mM;Cm;woei@ zJ^2k1L-ZC}fF*zV zCGI}|vj*2yG>*Q{i1z&4zY(dFQ8G)fP3JkToVk94$X0MzCd&dtLK0qt;JJiH)@d?& zi%>wvB%}ZSO1Cm#G8C(BaM$Ce*_-~y3CrpeeGtm!Z!g>w68beR5qD9q-ac=r<0P!z ziYL1wssV>ZP#WTIj?fa-hNaH7458EH1)C|KG)5>pqwq63C(+z4#l@D*E-d%^5p?M! z=FUZYmLacH&rx&gw>u}A<(l()Wd<)FE^iHVZy8s#AHY@6dOawbc;u~aiG$rQpP8R7 zKR;;>j*RrsN=j7U4oNpwH^5f5?Fx7C99Na7vD*?H^;9S6ZVlV9Wke_ExK?<7lD%n) ze#Rb$iP(ZO4dG)IM|;{Wj*?AeP7LCBEKllxfl4{AE}rQB&~kAyF|Nnr0m^2r?Y{|3 z(tqRAZIv(oPN>UIX-Th4UkGjB1Lm^v^z3;LyW1CHj*5=HtB82 zol)cSwrpkcA0{EMWUZ{_WxRYD=a;3XO&^8GhHz~8NTwfKl9oD17_%7`oNU%j4JH*X zA1E7K;T>Q3-g&tdL)^i^bz-N?4R@bBIKF#%+I(w8kSaE$w3%g{o?9LRpV1VLOSu)2 zv)c91NkUJSdBu5xYxQjiS|4~ipXegydV`uxS3T0DrW`MNuDpFl7TcOaE-rz8EUTTz zQ@1E8s17r;tPCN$jS6PhSB@389i;Bp9P2s`md|?;zQRVZF+mZImaHkn!8<@s> za4X>X?JbW4!e4KBaMr-NJPKcFJq;$Zb*Q219f^J{`f-{YO5f4( zrsO5WC-HonIS=EF3gfbo6ZJ!SWia46vs6nax8|5GFPGA!K zmaj_`IH+V(w$j0|k_6v2@q8kg9WpAGkY8hCFbbBdU)ef{C0e7FE|}O)%ww)AM$e&O zm8$3zyB%4NuX5IP^A@;{ge`h1H)YDDlzwPW;w+5waXlbJ%)y}#6>0lu7@VdjJjSx{ z=?;6h8bKx;?n%wZ17SQr=Px$KQ}(Y(moWVZIN!)5r8b;YL8!) zx9?f-;3i-vuZWW)+k`jTsB+!HYsD5X+B-{>F@5ZQVb)aPvEB_0$Y0dJJE!=f@(N3B zX@9dHOy4~F_&s0z4V!ns$g!1(RK}(F(fTtFi+$=O1omXg3P8w9WshARkN2ER!rd~J z>23eY1%r%VP)O#AgO%jR#quV#w~JT7Co<^Fec1aLEo{x(uB6wKkWcd^bD?Pmj#Smj z%w1zq8B8{ow`|il>JB4H1UIfvgw1y-yPIE3(k`Uo4~C;2WM?@1AEyg^=8cP1>&T1XBG=Qh z5O}{_e=*?(b569GZcpdcX)y`3KX4tiaSgpTC4Un-u|I%nxvrZ#_Lri}x)bx_!w~FH zzOkRNH+#^Zc9~a&v-FCY3nwHFR2oWc>}IwXv#%4yM^22JcOoZfZ%M5*-Jl5Ok2;zE z{eJaGpr1O|H`%Nj1seibHp~z{XoXi7@f`HX>x3g`;>G&d zC=DYvm*NSHU4n8-?V92gWfgz)BonEjth9#R$u6pW7o2Xn+sc0SQ^oVW509D8EsIPx z+$*k!J-)~bc~Xhkp5_57$_yWjQYB14=T?N(3AeO|_m5s?#A<@|LM-hyJhK$Q24uW9t7riS!;VidXcydC5;HW;;))?aL-;I#x60o$Znty$Zi2H zx0lw3wmF=;kx5y&UR1cYUtRY+td5vTXOAAPPQNc>TzVbl=dc)~JxFYEusY;#g*t6@`eil}+Jb7KPA!s~JE&K{ zv)!7W;cnIPce$#)$K4!zceX zVbjC?+4hPZ8dr65_S2!_UL$SU@4T%36!XEdUvl%x1&%#tPWvli&Iefb71xbO-@sv) z-9HnhmOK*lp^AEtxHU}4ginCkD6cuP)PG|xJKpQNL~DX4TJtEQn{oNBKv9%Xp`?9T zCyGojm=iuk7ITAY7li7)+duv3QPzx0dw4Y{QC(Gb_E8uU{<+CQ7L!5y^ruX)T@ZJR ztJ-px%?c-}N?1wh)+W5Rd4(OrIuw#|V)59`VxnSM<*n|e>$56sOS3VzfoMmKB}QW^Ga6J6MasV(nr;}}%}iw>jIGZmj6HK2qn1v6Mp z9_8%O4^^kJ!jO`ILm%CAT^k1W6FMcBkiC0iTP_{AcX~5&=xuOLpIFr6Ayh- zPSGjHEFE@W%L9PT;d=?6Y5CjmV2F>a>jEE*4C-|qYe1csMl}CN>D%N%WZ~Wg)zwpq z?#Sj^9*4Ls4+9GFVY;UcTMHSC>bdCF!n1^UnaJ65GW_cX~7tck{^d$X}UEIU_>-xAD>?|d(llq~?-!vXw7* znyFBH#(fz24%^d17=%1o-^q6mjAw@}a|ena)s!3up@{lFVen3rK1#-RS$I{I^T8fs zPTn{sxas+$KakBiphaig9hH+=n;bGrp9Q2jof?~^Mi=87Tx5^GXiH}=SihCoXRD$2 zqv@2Lsy!cu*Yif))D54{WcW=B_PZxcFZ;jD*Ss%=MJ@6jEb2>SKVF^zERU(}DglP}&I*spr;;Cc%lj*J@-@aUJjLQ7b28Mh(XWLNC%;}#K{jIj$Q4?QB37TE z@8eS$A@^cw`--;;m*YYQKKJRPD4pk1Xrcx|H;c;y+000hdL4cTh&6rm!tDytxt=I& zyL`zyXW}oOEJHy2LHfcpdUOV}oSas~2D$&hCN5U*#!rNx-fVgoMit6|Frf#Nx>t{) z*gH+|MZho4F=qU`Xr5MZURwq_%639sIy?$RXB(}~?79d9?_Tz%9c|Gp;51V!?K zmEIB+kq#;#QiC8My?01NL^_c!T|hv(^iHIP9(spBLT@3o5JJco@8><|JMZ%^$;@Q- zp0%#MSG(2-)aqY1tuY|7if&TtZQ3&l?XJ4iL6pt(iz8JYcV|N4Xqxcm$&scI^{rF+ zz#FG7g-w*E3kw^W8|B61UL^B_3A$03T7A}6@Ik&1#x|B}!dZ+?grd1&irXbXL!gz$H*E#(Ew?&`mL z;{Z_2t5;!ecx#TJU}?7C%87g zYO!jd-o0Jo=f-q|wp&n2>(=`hXS9(@JDpTS@y0MOCVQCVg)3y|Q1O#upmnCUfs(jX z@_-~@ASJRLKjNWWVsf@r7Du5sbbZGV^dU-Re(*bttvG3W{U2U_EYzT#tI~Y=+j>=k zugIps>@cy%QQG(huUEn2E`E=`n^vENpAQGG*PNhXp6>R`L-Equ6L87QGXAiQklT-S zOgQi6Av~%YU#|rzbym5sLessrOZm+HMVMYn37PQR1Y?B|k_Gp=eJmW|C>>DA@dKM@ zu>o=Z@}kY#%A_>Y>#@c|&EZ%q5bfYUeCAGFTAmT+CH_`6xB_0zFX#=q(kLLiB5ZaG z+xzX0=Ls&^N;eKX>J-2Vfx#m(Zxo~s{Dl*}=w|0`^)IRrnjddD6m}g08P2BCT1Z&T zK{9>1&rW1WLP;@aaJH>r9euS$?Bn;C7TD+`x~7}|MSLj0Jf@-9lx0oY)g z)eeV@Vl3XU((>4bF&!G7duJ}Mtv?iH$+4XLZ)J1EkYV-h_GCX3ryg{@bLmf=@PVZC zMGcvQa>Nor-*c*$oxW+*C!q%y$Ck=rQwN%YtjoRWHbnM z@>pK$E3&Wf3%R09$@KoUYxbU1u9{1Z##q4KL-lc7KYBwR_oTLD`XS%Lxr&k3!K;>p z*T3vi?$zU*>#lgty)gGY#XD!VmnDw!blr&o)=4u5$z0!h*75jNw~xqrR%yWR{;o2h zC5x{#)eRp6`X>CACl@4#K5{`*2BrZFSBQmfAlA!g!~Ler^se${^$j1<6zBN=K9}HV z`5^lwn8NtX=+=Yvw*lHlp$}-8t*erBEZA)dC zW_A1Q)e+8du4NlLs=#`sB9EfxzVNo`pnZfEN3}%AC)&F*LTMw-1~8xNJ)IXC5$~VK zk=-?0^*{eMPY}HF<^GrLDP)I@`dgHajhribP`X=_j{_0{4WMb7+~9ZrM`-u8^#(7#%;9|_fF2ANxE1|+@Q z+38v=kb3X?Z2D^Vm-jC`%quv3iAu6OQ+C2P1A#ziS`J!hhra}^W!BQOqJM)X$pIBI z%LBco@7l%{6L)eJjFE1X7 z*LI}!Se9UKW>jiQ(1^TF4b`DnOQZk1dWPOgPfPiym%Si{#iy*0HbaJtTq>hBXqI7nn#`5fnm9dPiSpW zUPcEJO!h^6uWMc(DH(x3OH(`i>3hoWaRYW1J})gm;M;p#60e<+o-)p%ysMqMLQ|Tr z5VX(Bei#8L$TymMAn}C~o*QMP$Jss%W`xkYtGFxiJzEPnVBFM<{4RD(r~jDn$er2# zOak_H^-CS4{b5oO|A(v3|x;q$xiXW`}LV9}F`7SLO z-IeMsk&~2+TIOD5SIaC6zWOc(J)0%E>=-Xi5eRAgY~RW?V8FE3J2EpF?B^=7VcJrs zm&Ts>aUoD_e?%n+21ST=f)@%dgz~S^)Oou}xt$za1b>Qk%Uq76YM#?$Ns6Vk{$JU}p}S_|6l4o*>I&n)rN# z@T0`^*`zV2mYRp4Vc)Hzl!c2gyKYItVAc90)vOCA!8xFDh_JR!p(fHHz{pOptA2Z+ ze9rawv&28{s33~gPw9W2w`r$?MU25J@u9vAjnW?rouui0@V|t&_t(T3E-w4Lq)~tP z8a(D0koN7Ia^1F!KgT6yKepw}yh-s^Jbx`2D{qgwNDsO{Yvst*pqUB?zumMhbrZC6 zpM#NxEw9ImzwPVt+VIt=$cfNBkE|Pud+PPpCw4z@Kl^56%J(YOcNz?$?SGWF3q?D% za$bk;M(n&&WE86Xq-olf3DCvJ_4l})Tm--xRev_OQXv-(t_>jc$77ig<%1oJj-pyj z*6A-@S>QGC@_g?;(%w~ZqV>vRYny6X{KJ}wTt24be?66R<|bz0+9 zH+7dD3_M#+GrLG9R3%<0guUeh-|L@V-2;x_Gg9(zc_xI&*2#RylaQ(E=nP$6Hve8W zYSn1lH0-mpK;yyQ0VK9iYPs=COC+=yzqQ0LQrH<~wZ(%>!wubZIg&XNxGQqJl~&bP zTkHu)i;goCbB<;piT8T;#+SckU8BLDs@Q6ZqxgNJ{x|<5L0vBejg`ifJ5f=}pPW2& zKRTcORoPd4t;GNwb(fej@cyc%@PX6wi+RH!Fnqot&*zMUT_UE8O5FSHeLUt9@RPQy z?~&egArTn=W5?Ln^|*3p&M;ywLAp!o^1#0omeS-ZUyb#psOZzX>s{o^n(JM+aRz=kG+-jA@EooXbgVGB|c=WRPkFUdb!F2WkTVY3JLQINQc9G}49Zb;p$iShO zwE(Ojf1=w>{~qWyW?}5N=mhL60n#RPSp=&7y3qUh?lii_WbWITc1z9uh9TSki-SL4 z)mfak?&QU0gtlO`fCfmEEziP=OqjjUfIWj933e}cI=Zi;xv@7Ts+(L<$Htc_vlVk7 zEw83-@H8((1vzgWy|~@}&NI#$J@;Y-wBy$eQ%?pp{U(aK5Lb)+Ko2$Ri2UxijK)|{)-_6o}bepsMxIgmx zXR|>P>#Df7r4y_F*-qbPm%Yk0x=lfQWnzqew`;vK$~pSFVn4}}`S!0Xs=+Ek(y|Zl zz|}M$+tQdnG}I@{t1S%E^#)sc65+bfz#nB8KIDAk_v1CLZqzJ^b-ZgIHH&p7d0njn z#i3}4*;|go>;MZEtsQ)F*ncdmF`luwt54#dp8E;#%) zk<`70{))(ASfw>h zsDCi#fcBKjt9r6!qrG@gxmFGn`?5Rto#nF zm$6*|>Qf)c)qC;TDu1JFsy8o9S)zELrCn>TicU_7*`=nC2bbWB99A)zAP=NnY~Rwm z4Sx8LrBZB{4%OFIW`sbu_gv9PwO{#-x|;-W1zo5VeaN>2r(I})(iK*i2c!PMZ_day zID?z^GXXE4DGN(iC$6>ZJ5z*=P*;JCAl>=qJq==y2|bd3?X}ShWLXQ1#YtDsfcL$| zD@M;p%c840_`*VfHDWBNe(>KAT<>T280T#@e>Z2kM|)W=t?)8{hEw05($rA>{`qo* zi$%Ww^FTi9W{Vq=TlFJLPokYG_tmWEf(zJ_zj|4=zDIf7aIO^!MugPMu0&q_`i0jq zT`qb6%+D*1l6#MHHC5uEFTfZeHUYa9f;kQX{I7O6_LbfF zQnx66>Zq#z%PM+r{u$9p!m7T~V|vyRHGeta*wyi~=q%Q}F-l)a772bSnZ{x*(s6RI z6yGQUEy`+{xj1dfA39E|SO?`CJd0eo%QVQkb2fJE8TZW(%t4)-dDr^);>Q-DG}Va_ z4-|H`>@6Q9{=dk zohOqqpCK1TrFP#%SaG=qOdsaao6QdBedk}S{#ibHiTTUDFe#_FS6P~^&y4Pm`dTsb z<1l{?E_bj_eYrulj0(lF;7y+QczEuO9t(6_r9%el`~4#Gx2T{kOhdl{Uv~uitWlDQ zT3kL6XTjfu?VncA7AA?-8NZ+B^=3V8B5nRSRJP4{`Sxy=^UWf!J=J`n%eJX zX?<54sj29-5DBc93Oz^N4gM!@s4Ixz8)?vET<{rc+8;30>`Jft>`G`ld zM%q@gRZH`)Z>}n_vvK?&=H#LFGs<)jov~0}F*}kt>7^i4UQkM$?fZ?Z8ht{10n>@r zJx}n1j&rzXtEyK$xWeV`+(t*C-A{K%N#=z3WG8lUBCL2PjfUc_)OAef#iu{LciNhY z7>r#?q-BC6|A%z+LfH||L`Czb%VtADhiU%lFaq5UzZplN=l2rl^-dW@m~?6$P25LG5M%4Q!FZa4@Z@a zn}4iOb8=lF;I~GSGM+Z8ZN`JCmP6+A9-!?_ZPEc|snupxo_{aO2fY7C!Pz6bN#oqp zJ9qo&@y?u41OvlCgd%oOXZ>!KT0NUWeT^9{*Xj1Jwqzyk%ExlwFcw2FpMy40rlt!` zU3HDSs+^b60G}2!?0QE%Ce^R`^fv~!tefQaUAjVESVSgPcnD z@I&oiZvX<8Yq>6~*6H+gXQZ4xwip$fEsHQiOdojs39Y~HYfCy-1Eoz{e?xr(SJpQR zkZ@!3>;IwcnA@@?%lftcW4zE8S>sjD7;qDj)k?pCjv_Q~Yy-kE&Z*(&=SpFLg<*gJ zz8zA?$>+y=*lL2p(23b|XPhjX>^{BI_uW@a#Eyw5s*0%fHzrBPbnkhuWw%6i-n7i) zG(75pk2C!<$HpKdeP?IFPmCR&L|6snrF;~3ZofOzYKyL$z$a%fT25D2w-2cN8S?tO z2?+?OzJ@%ylTiw`KSy5(lVj9U>kIVhwO~ir-Zl_MZ_T&U1ax7^j*eNbDSZ9O@R7+H~SK?=N_di3_K5z`1)%fFMwB`C>9)L5 zV9Y5^vOQuDU2=qr`tstVJegR;wrMMsKe)Uf=?#6^6n=1M6bjn8O>@YdCz~FLgsi?O zEi_i-wsBOs@tvG@A7c(x>M6nnA9P&2O7FlqIm6BJwM1vPkD zFh}0ZSL+JOl`Na4jYJD-29WoP&Y!aj;ij|;&l*Pql`FKiFfKNMPqkIPn@PX4H>Q~# zZ+f%i)vxjNU|AdX?oyeMvnZtCL00tUA*Cz>?kz#Ja2NKaXVCaxt4#c|I_yu8sFvRo z)JNzO6$BKoqcuL|HqP2Tt`1h-fkz*ob}6LHXC@I8?XvjZz|>uZh+kt0x|rd45Dq7= zw{>Ubd@YY&g#Y=O9v1!xCi9FZqM#F4ISSpm(d%9y{((*O08s+)W8vFNPQzEtiRpL- z1esKfbB-tKJTAML@hq-gcbn4W0h*$TIC(XSATop_+XHP+$_9~N3V<72=P=^M`FGL_ z$D=1Ka$;8g&3E~{c!-R0P7+Be-)lNuR%GRfY(|bM9|fxKWvPNl;VcXFJuh&R>HBG# zV|E|--lrG61O#+poPOPoM>+s>kK-=~XPlWEK;-cwvz zH4j8T(w2DGR3-t5>-|6Ex!jhEA)~XM-W1*Hdj{H)zX}4v4i8uL9jM3aGM?L8x%B%v zc_jpnRwZ==uF~7}nY#4I&nJ4USPj04+H!ujDS8{E5M(#f@u(@i%*_F^QTt8hf4FyT zxfj7hg4+xAXJXq_1#Mrh-tpRYX?$f0VOBIy**{8c|Gl27eCe&Q-X7cZscMBA-SL$A z#SDU?$z8` z^l`UAy;E_sd-1hslCdmp-v-}N+1vsTb!|L(M8r4GHqbxk*G1iy^^93lVQwnp`@7rE zMBWMc85~&EHD9$D==&UynI!z@g_REBt(}uuf|NeB7Ab!AA`)QzKkVI|JHKcDx+Ymc ziSjWh)Xjy{k$-x^Qx|aebmse#*k5eH2OqW%z4g^#cmZ5J|F+XM`lh!!?`sYrVh<^+g>WgZxRvHAsf#!9s`1^| zPp4IxpM9{LiA+d%mz*wB;_ZTx+`ff&S55!aljF(dD#{eLqMfi4a+)BSkiKxO!3_{@ zq|zD1CjXm_D> z>$eGY+T*&jlCLMX54;Ktgxz(gg7tUB8$^glI_h!E{Y2h|zGm~jRg2o~NycW3^TA=J z1A68KrICs6OZ5ZQ4S`i1d&>~V8IxNNn7TaWZ=q){`quqIN)rv-DCUu8uXfkpl={G) z&_Z%CtoC!7yv|i3!malg73UuJUTBDZpS>lXpWIoq)!2moa4=&rd!soJQ-w|T1|=`9 zY2Z!5aD@yx?d&gW893D3pTyndP8_6c`IZO#zz!_#Z~S>wd$QZ#4|OSDB)99&-Ie3= z2c%)@BxU#Zu_b)vhc%<^+uw#-7aiO1<-KwbaTzn81D7f(?WudcIP9O{Q#WZe%A>aX zk!XjDS0=Az1BIiDfqJbkcc5AZlVH)wxI=4kyl|iOJ$^o5xduVB*rd*Os#G#@#!9I4 z7t9(S?u858S{6pe=q}aVpo~2P?x}z!tf;Qg`y1 z4dWIIrn{HI29Cc?@Y>gJJ(q?lV9+lP4YzvlY&~e1P2)@+McaO`&&M_jqTJZ| zbeYhn8w2q)mOQc$(JUE=lj8pKohIA-n{N8y)&3PTt}bP0r>1`CI0NJnT^A#etVWnkH2fn2*WPQ%{NXU^#lnRzL zl&+d~IffV7x^Vm=;Nu*LEo}{peG;9JKcCFf7^BI>CLhQJ-XGmoGvx>I!O#)!s@z-^ zjL~{UX>dr2ZpK#i-GxYI_*L4q`x4MmPBMUwElx3h{xDaqP<1U z6kWN@bO0d17#k)Y^i1Eq?yBcAj7x`AAr-byCfp0qU1*XbDreR8XA^2k%gcv>pIGepl9AWq3<&YWv%~ zDMs`U-Z@F|YfieCE@XX@voZ^s+P9KU zGo%5q5!qlHQV|dISFmrS-T(xNJ4f?{7}45KP7b|IulX67)RpSF%|~rqV>j8+Pa$mD zz&9!Rh$b%}VAVNpvFJAEWh)U-egLCps%HuPBSL#+z7|tgC4*2rgw9sV%zx(a6qQS( zsd-l|XS(Lt@|ZtdQ;kcPXjK+kB5K&7B~YS&*_3Daygs32Kc#!SXk`+zh3wpOj_K6p z2SvdAeR(Da_n%Gn&|F7<0D+ERgm^3=Lr(4ItJ(py`HOtly1ptj6Xjo3*z~z`kLT&9 zr#81f0616e(m^c6lVy=!&749etRH-V3$dk^n~l@1&YM+X*M`i+r(#1&L`^IBCizKx zpjH1|O(!*DqY7)sz6|5BnDkbWeox<~v9!!>?fJ_8k!5b(!?%39)yq)AEi22}SjVVE)n_|7c!upS(P@9PlBh zr=*ek{?i`}m)Fe{SwM>f11wrP6Y9^Q3g*%e*o{Iq)-FS-jbf=>@7#ZCZXKj-ZvpRs z!edgPf_tX5*h!zY6k}GI(Fa#=kdJ@*=h9_UvEfe74^zeu?_au1$+r}^^?5|&JNfs& zCw!B?M5pX#V0HLyc_o~?SIIJ>yrO+frEzy+ceP(FF#2M@92MigSJbYa`(cCj7G@cKeJ|Z0i@2r7t82n;uTWs_F4A&Xl(fU)*%1h=jjig zB|igQ@_?l-p*tO(pCDc@IlA8#^I^RT_Bm?m(u)bO{|J5~PhzhKBrQ%i;hvg6^y8uF zDOqeaQj<<%Ap4p%kymgOEFwj&qY?=1APx`EDkK!*94d%-*!>eIi(f!$lsqOkDTI;bV3NjHY(roUM73# z1q(_9N|$>msV$K}Nl{4Q_(SMwRFmoJ=v&V|{S)-^n%v1*7yj3XpodAsALDYE-n^U} zpOp>V$%sD9X`V4nP73nmCHcg&%Xl{L?PNH|Wp~<-^}zbY49cqCSKp8?Cpy|Wu}*(X z|M*Yu19w^%mDq-j8iAcHoQbLCW=68vbUr<=(x<13SEJ}MUR!Qui*G#>{g0vqbzd`q zNjRV0G2NG&Oj>kBh8f56siw-8kN~EehS_s~r$^vD^4~zERqRjlq>d^)+mOtz%Fb zX*aIxkbOVl)+P1|@IsL`qU?My{%Jw2{jPhapn}bammOiCPiD$%_l&o=+2#Lg^4|}` zYUBZUy>aZ(;JB(u->W3j%l#i!9!|V$3FnBIV6CO^_gV@Wske13K+wxFm1oM(mP22f zcy!IaONm<;NzRI}FKZ}$m#-4{WMLX>wbhqj^CP5hwhXSF6#-!eurdil{7Xz?N9(Oc z5>I4>UI$Y|wfG5YGs>Nz!v9$(9c{gvAkz=2rp#-N+}=%|%*1tyE9aQnSkQ(252*e{ z-M@o2e@TXy^-4wjzeiR;OXKdK$RL<6xjPzu(X;m;5*;`d@CXbhrn)@)edjnp?qVe- zT#bvh7MK#O#uYv?Z60tCVUK!MSx?w!t*{)6uyyI5B=yVu#@E*S-bYOVNbjNm0lJ#D zeq-(9NEgvoVoiA`2zG+5-;da~oTJ*-2+fSN)VbXmFGgd4;$UTs%wsZRG=M>L&)d6e z5_gZ$wKm|Zm9^Wkr{okEdmV4IeFr({*TSBbiZFv$Bf`#On##MU00ep0!$1oAbDQ$` zl4ZL%+YWW|yuX%AXyokfHk!S3;Q#-p{CCA6&wYdM5|YwW*HMu2-><5)rz6MEZG?V;5aNbkl$_@IJVg40>C=Ewn9NW+=XRZ&#eF9KBZ&3a+z`>-pN+8m1JAx{_@2C^vA@Mf{~4 zjlA&)VEut{)ytVlFGYJNHnO(CIU0GT9Mr6>Psmmd+?mS2EkjTK;Q9Z|^FK@MRXO7& z$BYcc2JZhnh&93AK5BUTV5$mF!pXGRX`Scchw5OO?oj0Mri|2+hThQt%JfJjh{WSd zTxTcU$EGJfNmknsUN6V*kLHPq>*AIGT%uS~`w2YI*beik;)Ad3>`zn7JcGQqQ}l|^ z?x_8J@NTLGYDgutkM-O!@{h^vG0F$05-+F8$JdNOrSS6()I8k%m_=SCR8aVGK9CHP zSsJdpw+fAIvB?a_L_^`N#{J%hDbuC|$fQe6#QbhHDUXo%{P$qMJ+bpSoJ$n6mCax9Q29D)ubkap zK~!Z(YiYs;9Yn#OGkviMirA!aNijY;yw_NPe^HD=XJvjg3C0BqvF$iygA7Q1y`ft_sntWES4h2R;#>o%FOOzo;} zJxclRg3;Q3vR=(SXXDhpaG0xb$^I}DNMFp-iu*9>l2R;d{71G1;hi|$8&8jH zYT?moljAqz9RP~-G{g}ic0vglw%VTzpF?_^IuHg#T-%1@u~R_ia7oyH`02)3kxFp7 zJcG{dbKho#W}Y*!yL$=9Ffb)Z$Pbli#mY&>jjFHZf%4U zMsBR7@BfA|J^*l*sDd6(iOj`x({%R&kxc$DSDCLMovr0QnoeipVod$<>^&5-JS`M( z`ZNG9gy=k=VzuybdGr>9D?A-I5vj1%YMEi8V6`PMj0<>&_r^v)8sm>UYseHEAvwCA?$z9#7 zn3#B-nC`7@j0Uv~J?LyK*L7i+RkvMw#;ou0$0@6DTS%JSRM+r2YFZb0~)`J}cq6;LZV0r2;jFV)kT5|$Vlij#wI$xgMVOW=zEZW;j|rhW87u`7+%Bv7}@ zsIk<-Y2nWdeNX}YaH?v#i%(vL2(;B0EXfEV0K19)Fcu&)SkVkZ%sHLTAQ|s8AI}Ov zOT?jrj0iRnu^opd0A zox(x1(&O(Y>Lwnm2k(d{LTDmcs+{;~6^n(wzC;UwPr(1RagxntgIiJlp3Q%j76;L4 zXjTl;{U`cRFu0VCSrPo>9VU&2&Dr#)#g@?}^a~q3$AU>ESx_&xc6VsOn+$d~ojt-U zl7e_ZK!5?V^P;`s;}+~W69xzg?}GE%$4HAga(d1=rGlg3Zn*>9 ztD9rJi>a{R^2vB?lFcE00<#$AUacr}4j>GT%EfKJ^wCp3GRHof=xC6yNF$ zd##ZMYpYh$^<|wM0(n-O#Q_%w+;c*KtA8kU%fP8xy|&NW$}t92~)+` zybIjR%}vgV_uG3uXFcY(0sfC_`KzAZDi8J!l<`vZw6n-NAQhf=2$hK~P;uozrh;Ui z_{f|jT}g~-a@$0yhoPQVHa1pUZI!jJ9dOeo>CB}(&f3*3*Q;GAXvTuF#E1^CO2@Au zMFEV^+24}X9DvO6j-VAV4;}dq=VPY_BZUZWS%#)KzlR0mFnh>BF`>=p?9WQB;a=kf z9d^9Xy(SzQAP*qwV17Bw`7UkBfX`iaW>Nz(<_GLYc(GM^w_A@!Ay9$B0p_t`OM)Po zH;y=eqe&b*i3*~WXDW?}wn|r0L9eDBe(XNEJL~BO5u17$zuU;{U^u^j#9cx9!;S6X z_Ro;S#>Q2aRm$WVfo77qU0dLR@^QHzqv^khP*srJ|8rFxnOP5ZJT{en^3xzpATLAlU?*1Qd}Xn>_WaKR zHkrQ&@uMGQKm)u7r6=sPO=k?8WcVDzFVwTN&W^Zsy0r|nTB?Q`d`_Ty5p8e6N}Fit z%PFrz$$766jHs#ZcvF*!EIWkkp!~mPS5O&k4?#28tOs__rR7zBjg>G`i|5|Q2$+$( zL97v{{n)iAx-l(A3ypt)JyPd?zB)33U0z1?8E_-T_WsfL<_aIA z$Ul#344D4;ek;T6Zg_!r;+zrbbXl;fWLLRKbSlu1tQV>WW2b$lW-NA7@s~6-Y~VBr z-3jDn;A}Xc8nGoGQEf9nIEyTHjQ3yD+xiW`nD3U|n_{6!t|2AHb8^fZ-^+@#`4No) z9;o|Hbs|R$x^sxzlaI~=j`TQJh^#T|MUTEAR_N`>d z)N=f4VKtaMDk@j9=orGH^`!;wvCo5g`;VGRwBEL3iFDqa(HBI|q2zdY48INPU|~jT zN3-dInFF#KU8Jc;F}UnjSF_FbHb|K6{;(z4(dH<2>B@Nh6!JY5u6_AX@CRv7sJy4? z71;|rkXW@*JBa5l=eYB|IZd_U_nzRJr2V?gr>hjT$JD=Dg0FU?x-3=D8x@wW4z&;! zX!mfqEzow*EJ?|`i(x&Bvp-o3x(WDY_{*nHmuL$0~fzpcuitng;-U zEUs(1XHzDy@yOZ1FyN#6fq5)yn}Ed?XhimAkuEZ9)P;x@?)~|_?7e{x0B0%eBP)Rd zMePM5)(9IThbm|LXMUDqaAbnhVa0h|R;pre5%(F+DKdIuqI#?GbV^<$hvq_=t-R5_ z8`??=5Fod=U&NRgGiY@7Y1MURj_$HjD;^WSm$b(EI4!k}Ry3h>r%H=%)d_0wLP!$0Haip(IN zogtsoe9uI7OECbgcff_%xW6)w#e=PK@iU|`cHqhwi&kP(SIk{zyxIBCR~kLOj{`|{ zR82H+oR^+X?RSwEUx{;BQBBV2Y#=@h8P;-=l5APq>HgMXzmNcgrkjTJ0AHJZwAyYA z6lEWU1U5Tw;XIR@L4!&{k!1MWt!nU_Bko@t42~NQ%cA$2k@W9tyAZxPb z&Dx^{47A;JkCoLmPVF^PU@$D@t`2cR7dz2}Nt<4PNjn(JEg&9h%L}hF(R@G&|)Md+q@4RA-|Qk{x3Pld8<}3_r9n)q$oOkVkK(~D0ng2xivp#%%Y(V+fY1R zF^_VXo9<*1U(a7Xit{AZS2|YKq{m=@6HA&2N(eY9-o&n?r0`c*q-dajhxetz4?nDz z9nPT#Ced$v?Y#<0n$PB+&g(W#&a@m(rMaZA%&6W5$jL98-L$#B29Tu5x04HVsN zl*{Z+avuZCN$d^Pz@+m5xZKKPSha^V98u!5HbmiWo3)_0m%q6dIA=0D(+IlAirdbh zh~G>OMOCb8ch*RkguC-+M#`%fjAuxbsty7MZjG6Y3aPlQ?HqZ%U6g)e638gzJa{18 zP&HSEuGsd+MEk3+-mAxy74>EjUH4BrT$Kjd*D8N<-~LbV>i=%!-zfeDWM63*T%gvb zYYtQ29q3AOK4WkPuMMPCpiH;XN5I*{Mwl7)i$=2Hjbx3BUU|V3HCUReulKI0G9$>t zb5wNP->_icn~dA|Gf?(p-f!P{bErQLI(7(08qn0V?7Q? z)xEvW)Jh#*k2&*4ORM0z&Z;pBXD6Mkhd_Om~ zPb06UbsJ@qulZSE&$g)L<5Ya@aZ+X7w%=nZSj&|;8{_T;{1#?#>S@P=)Gdf@1z**0 z-yz+Ym(E;LOQ097UY@F`N$P{GjS!I8n~%<@j>nyf^AxS*ilnORrpfgCxiZQr4R+BF z%oX08fI5OtgpHZJf#NJM&m(zkMjI^JS&^uz@aP+T?Wr6y5kmx_LaGQBlsw(y|ECwg z?oodjE^dJBlKXm&ZWk<}EAQHetIo?b+{Nk^}o{ZcKSTZJ?2yz69d@U^=Q$uKUpkuV>LfScx_x|$3os-Nxp6-{>fNhNR9 zlV!T~&f09~Jwf=mK)qqj>aT$&)L=K)K0jwTSz_&+wV<$PJ+X7-;%;x#pMwE8>I`Mj z@Q_+rkiN&<*gB-@P+rK?350>wPSlQ13L(-GF0*zHX;I5Aq!o*n>_yvClAM~4Pfk0; zvciB(n3#HeYx|d23Uv8(l4Hrj_cz1i_2f5S{_oB8Kybgd+1jM_rbpVJ0dG2iWGOrs zZd+-U;jaPy$R{E29>%Ou4t=zw398ye|a%V8*4?x+VaBslGxs#S`_xJu=m#O1pjsA4Sf<>*w@? z%M?a1K{5f&CR;^UwuBe!aIdhgfcPezE>J)okKF3ey@}&}PH1}O?KkgOWbZ;uz?ln` zR*z+^td*NFxq)#uBMIR3DM6K8;sCI`u}bUYs%mZ0`<|dFfCzv# zMdNS@1B2W7JH&E^nZ8*-llQ!1h#i0TW1LrQK1MocKcTtZu>nl-2C{<$wDcB;tW)_4 zagKYNgIX0HRM=4=-7*>$sOLzfBz7BS*ea^F=PA`;>B0h?i2d)_MFHjahohU4JP<{$ zM+EXVcIS%m9#jC?l;n-qASXm_U$(cHYgY566C0WMDk4zpz_|$c;Tl9aun8ppf zTGa@1w=A&SnA7RB0fNl}R{B-!M@@14Sy5DZB6W`S+^vCj7L`3ckvd&T5F@{8Rjshm zK0H%5K|PXY@!TKVIo!GObxcX8Q)FsBmNBPZ#2%2lu&JMD-j&a4X>fD%0!Y-J$TVc6 zaSBJ`N8R&oN*18QfzxcbguV0%d>!zOJ5!A3lyliQ(v)-h4OYS1?_ujSDX6bAZNV7Q zV8^B0Nq`*JX{5-|PeBf7P}7^?DNp$`=a?`HWS;i1@Qkp6*IGG+x@X3G1Prn;0Mq1P zJv)4;(sCTD)0Jw7E|*(c!lCfl1cf9klth!HQ+@s#hcQE=Y=J>|AwHz5 ze-S|YaX<5P3PjGHWB}O7w;L-^i3=ra1x9x&)tx&y5Jq!dcRGs`i6Psc#UNOrykG)q zYka*%E4p9qzxPG|AXl-=R~b#rk4}TIA0FABBG-s;CI!S3az%hgP+Ln{B(WZXh~1QD za?y|ljnsJu$cS*-K%xA|(1h+ErRdG@97>6 z1K@Bbi}}GxWU;+-B`7btYC>gFRGLZDo+R{BR-wL5!s!IcsjZkjvF5O4)UjK&4PP)y zar1X9TeL@SRM#|AS3Ng0^Q}EOv*{=+uo1S%KZ^58ns4|yonMfp%erIW%vP4}+;3@( zceYCptXUtRV$Dy-)B&CV#&~<9Zm`!@vCesh()Lr6rD8|US4y(I^)q=2*}z`^aY{*c zMMWOMI^F-~LjR3dF}^kDX}t(ZOY3X@lO)zT(w(I@tT#{1_w)zlwWNeTJq-Rhawa7k!Qr$z8b z6s|tadpA5|buK}ZlA!`Tcj5D4_7bZFA|BCwQA$_0^sYAG;F-Yc!^*X)h0Vl6MKHYDr@OnK_JBm6p=8Qq>>=n*0=beQO zHf2RgbF@n4lK_6U^DY=^bngOPLosz#Ag)aPOyiBy9}wVVy6q=BgF$U~Eu_Gbsm`_CCiG^T0V>7cxu~H-{rI)vT%=dV zA~^(+W;+#`j0h-5aceG4*2Ccysu4Wvl#lkTw=q_$;V=z#{ezR$Rh6)bz4E$A&H55z z`L`2Pm(}!2tl`99j#+oUgL8ghQm_}g-dNOFnv|g-FI4}x2`OT1fU;$Je4C^v4 z(O-IA5LzAbPY|A7rWTVl?K1b22i6k}cSg!&sfno0bC+j^#g}JAuJ$u^ZOnAJ_Rfig zTWJ(PDqd$aRE+VXD8yPgY~r1dME&3l71paLpAGanJSq5b?${=sSU0OHSXjK!{H42*J{HmM6R-*R>x%#WSYHb;7|J?YKSE9nORd>xvxe z`slXzuF2X;w($&ti0%w$(ti2I_pzJ;TfYHJ=&eW^yQgsBVRC4^d2eBb=fPPnnlgj-|kXXyQXbVcEL>CEkHqXr=j|B*7AvY@ZC z#{;h=stpJw9SIe%sc=U#X`4-m``|03-Kt{v!scfyx+$?S$4Upou>X&}w~nfE>;A=c z0|Fu~sf0+kf^;cTZbIqq?(P(kZUhnOZt3ps?(PORu<5uD=JA~K{_giT#{J`tao;f* zXYAwN&x*O``ph{$bFTH+&M5QrVG&6%J#}V5WWjG*!*1DJR?H5Sa<b=Fh-{8RjAgoihrF(0=XVKSpGjQCj9>PI z+HWp{A$APG&6Nt|x8fX(m=P;UMsrLoSz3~G3Up^GeYNFssVR83=2G!CotNP~Eei7) z&gL#VPA>5%d`UZk34$+ML`cwGs_aT#!h~-Ve-nu*M^EWiv{rTIR#7q26iBd>QH@}2UNy>dFzB=TB%WZJrzo;A(>C@Q z`oiBD2YRV|HE*IKgYWg|yKl&$Ixo6U8Qy4QpJU!-m9GGYk+z)pqy?u?p#*#T){bMt zI+RNDMeS9!96+%6 z*e4w^`B=dO++*e;gk0utj4Lgaj5iy?^yv4KWGtF;r=G&&d?5f$+HgjsK2I-3B00;@+AGHw=eT&76-ek+&TT0YwoPn`NLfp zR{PV)4OB|2L!I*IBe!oVRZ_{MX@3+CO~ESHSt|unOFZYR8btxtr(YGQ>v5_{9i0H3 z@#~ERPurBaUyaQ`v#oP+08N0VY2LW;9)wu(wP%Hz9Sn7wgM?VGjlVZXYzE3x+n3ea zpYn0Ho$(3Nx@nLDXmE?RCBp6Y<)dqx4A@hw{_A}uJIh)|5V;?1K$Z00g1cY3_m>YQ zH+}CcWBfgY_@_c4&;|erbfhJEneRjTGvxU5_k2@uqW29rD`5WzBS3nrp48r04*u{D zt@}q0{k8zpFey&|{PzfY?glgi#UAh$#yJz_LH{i3_?~Yl0R%GaDF36>;YM~AAT543 zLe@Du31>(_Vo+DiJs_?1j}CzZWbIAKj+{#ekU0NtO!)c`16s7y$}8kIJ!1BPAK77T zRA496|IN~TpNxJgAhO)y`#3)zzU6_96oAZ)&59cF*oConK95A_-rut#KYmN>2|SC} z5Bq%sGu@?C;94@}1O(b&UcU@P6B^SpDYOd=w?VmR^3^F=1wNj9-gnFC_xk-p>45Do zRu6qyHj=Sl{IcFg7>q0MA_4FATC#5OLol%>I_&qkB$R;Q>zwVT*{yZNQvMV5YQ8X- zd#S(E#zVwuDzB6|KUw7)!HTm>ZK=oQJXE>=9@h!tTfTJQrs16F^AeXLi!lqkLRv$b z?kKP8hJpQ{ge~JMPufoa%Im7xqbysVC0OeMZMapK8}yu0k6~`mb&RUwm=i2=q)fvD z4jo2`o6_E*vowuyARi29Nfj2sI^}!%7l-gB{rmWULJAf^6o;)ay>8qL$%S=iZlvtm zbn~cKE4vJ_t*DZ_yELa}xvIu^+@;*m9FKta3wsS0x>6gE_a)?T)((b{>1>Y>4Njrr z&>O6U@$7Mu!^?mxaLiS9U4$mtfO?XPs8RY&lOff@5}Yy23SKX2J|tLW?{X9o&N8-q zZO;4@1?`*>5l@ z*r|Nhk`#dKLf7lI*8}y#0y5qC(=yuI%}fyQ^I9F}%)zJPH&V9DY|K#HwuyRin%x#p zjqeX@Ni|2>eOByff(wmMu{L#l1LC?pzBbw4_2Xp7!! zzba1H>_O>Kr27`y*dr+==lpo0G{=aswJxBjk9M2EV*=R9(KGGnu-Gr_UxvbIL{WZI z&~-GAf|E(QwYDB69&}BkdqKPYP%tNHS0PXo_+5h;v+Mi$%bK0aNO|u~a^eP?De-PW zfHKja-vd#QGeWfq4+$2Y_$gjO+g2U#N7O@;% zWFn#T_Se4xA1HxP8%Q8U{lXAFA%vS%lp5E@4&wE%%4InVj41z(&>pil zZiL;IYR9j!bBL*L01PQakAqHbpa0*w1`2FRflR%MbhO{!io*Vc@$WMFC<3f|wDtCH z^!(TP+z7!s6u_*TW(QTiy96FUk!(iQ-8AO!i^VtehxZ`njrD7I30AP3@y8FL4ian9 zexHBmiPxV4LU#Nzh{a9>`c|MDM-*6PoaKZt?Vk}?zq|WZ z<(-g+e?jT`YQWSz-D}1m4iWnndE9XZ5KPyfkMcXhBp3m0Sc}O}BYcO9{v}Y}mSgp8 zN}gX3;(>w=*bIK^7c2;(-=dAhw66S0>rMKtAHxVnET>;DxGSgpvq?DRF&mQ+YTVyV zC?kHP49ngOQ}_$6{bini1suK`uf1>z@dsl6Le_uz(Eb6i9~D{j(ZHdpKTzmjn$`v) z7t*TO-_F4w$obdjJ^njbwEs%>)_>Ii;kO#>ka-XhUYZt^3)Maw2Diw{{(T?wjoppl z-D3wlsJvV)xZl3tw`~9ZYw-Sf_;-J-qOBLI4Z-^KeZOX4usN-2TbPs^#t8gu(&GXV zXbKN-?XIXLf`0j_fVWtmL{IIXgtkuYF|t>|LDlExYp3d{A@>2S@qNR4tit7OPZ3tx zL0ib=)P|kaf3s*Df`DH}AcghxH#R1u1J2#X#-`kfIQBOZ<%k|2QN(-Gs(<)7CTQ{n zSMSF|FAhk4;sbChpwDSv#m!RU7vLl)!E!mKDH%$#{|oB;9khx7SMVuP$jpBPpWl#a zdV{+Ew>68c3M4*`CU||cw2OtneMrVlrVWlt*x#OwG@)+rlucN*2~ z7NRZLOw6i%n-?(6A3RGa_vKQDyVUVFN@d2Sal|fLuZd|UXH4bn=%In3bH7<2*g#7( z|2UaBype@|V`=yOjCXcx>;MENW+>QyoWs@GdcS|+v6C`RiZZK#-o}nMQk0{X)uioG z*iqOGt-@5R&j6hioIDaYV(6*8j;Y2lGVM4Q+ z7?>IOwtmEJpf#+*&&q7}0NtzsN)P72cRO5%0HsFYw3Y2{;GJkKRikYiS_~<>@z@E) z&&#dx=#>r6CI*=x7X-)64ewHcMC%PDv%yED6#Rpw87O;Z>t?+lbjypsr06~){RXA7 zsZKJ?%Kk!8$ue!nx8~feLiw+n^dMMr^gJm5Xxr zXeZh0l4o~r3ZPPr4gibD$AEyF@01w%;hT3k zN%ghVmYf`*3W2Q_0`F0zxB9V*NpB6;V;(p|SqW@yts+qjDg>a&^5X1uz7!SjZIB5D z&@iInQ&@yz74_b|>&ZAR#G6C4tZ)HiH41yL#BMP3fCAuiFDCq=zZ(PqE%@$u6xeSg zL5bL<2o~xi&zHvKq1atOzaLR=FAX|p_PaBO=1}U<7Vdp&a?r()cH|RP{h~;;9&A) z!v%nIv9XALLG+Ig>WOgJbk=uLewze$J`we8?TfO|+VwrPS;mg9;Kxs_ElFAOlt#^Q z;crBsaeGO!N&258&9cA^Yz{{_|6+lx>*@z>EcTZAtlii@gxvDDK){XKmy)x5^_NL} z8;Jhv`eZ%)cb31s?!OQ8*Av8hfV^(<67j!;{vSW|CjdS+@(cEWn_oBz&3g;gDFHb; zGvsG7>p2UbUNtxT5UB?WnmuHIbE*2E*I(&~VCP4qQ;1|w6g49*3@A>qRK5L-xQW4y zc?Gl!z*nS_oPe9h`y;-EH})6n6=`n;M$LOt7xlks8em@{$0}jPjBxnDGE}N07LK2c855sItYB@iMVeF6ja$tl3&K~FUtGV zwfJr)Yp8&qnvGUT{_lVK-&X(6O2G0FaHN$g28_K%Qup*b9B`mz38uDYi{oLGq#|RheW<9 zsjm43dv)_lvA!x%3M$iuFq9@TB6^qWliqSp?(9;)bpZiEx1u}c-5)qj{me6I8-7gn$ooe!Ve#oA(zn2Krf zeNOOGUH$vl*zjfd-dRx9h;)~i0RIv|xDu<}3FL%0>r?RKAichbR~pKj?>D9IfSB$|pPdH?O^Dne1d!ACju5{`_vgO- zu__*)0ayzQ_X`vIS3&;l^_xEZKhmsz{CE0m{KjT{&>uAyBU@DeAss$=g!0`D_g5is zAXr5bbuO^pau|@V2kuy)0Gw(zCn`*U5Wku=wb+1W-C(=oLFYH8!3NHCf7PhPC3$U= zFs$;iQTcHDuc}t-FYa4$D0vSR9`hwo-Ck1Qz=^WPQ9?wMX9K*4#tCpVUlrWpv8_Ki zU+yOU8~G$>dNY~~nlESEayZ8|2ZKmkgyE$?qs8UMKLc6oLZDIcJOT%xQ9Pj0O6>n> zqs~yB7MCbB8-p3GL0fI)DC{VcpSmKBZ9X&9QS#Uf7J>OOF!`~?|Iy_C1^|ER_2Fkg zPTX=UcYj*C|E#cwe~9$It^WUzK+(;SdJfHw#R-($iTG#AVx+Qvm3rsiVj9Ubr-0HJ0=N{@_6iY))sF1=I?F;y?4(vrf#-mikUDs?mfKlDE@j^w^KiEGa0);*@6E zK~?ijQcT8FE30aXw<^==aO5zeWfEq_;)`_KZv$>tjQbUjSuqLu>&ND*8gvytMrR3Loz14U$vn4#@L{5()FQiZLFrIm*98#j(j}9O<@3u9bFiZEY!!7=%-ry_Ouzn3ZO& z$vT?I+X~rqR#0X0uvK%TR~AVXb$7qYL%#7xeMxpKB0ISdq{Z($!7o8fw$>iOy#Wk% zHi#c}24ipLDcYr25Q9P!G^@z+htHUcnW5}$efly~H@{Djf|eX8?~qA&>dJC@m9NCz zxw3e3bP^MedZ$Qij5}7p-`^q2nJ8h_so?BGMH}ql@y#(z6-JLTI{8iv@_uR$COsvr zaC<<%U*8AP>97OKCt=SX+WtUrz=%* zd+~MM&L58REH^Rj?#E}!Qod}zF1sS>-`xr^m0<81zgczZ1YjSj;#MrDrY(ODnyusA zw?cNarN}l#0f&#HNOd{uHW15;V+Z4#&dYlP!cD;Qh_60XgnS*a>iHHmRY*VlW^;j^ z|B?QC^JbM_=m54Lo3E;^sG7~P11tvB<#D0UnRSxl%S1d30{WQEG$K=x2l~JsRok>e zD=##2+MX?;aoyVVUq$l=m_MMgZw8mO7BIN!d!IGH!2L8Y4StqBB|HMQg15DkBXS}q za1%ftn*OnUBn~sZLuTe|cY2@Kh?Ch}%4a}~W==53X_K3RMDs;oBRq=3V;e*>I)}y? z<_44kq3M^@?~g=bC0Ac$fn)qQjLOq6EEs=y&cp%$j69l&Q@vXt223XN+hiF3zb0dM zUqRk2^M4rzaKLA7pE1lRsNHaVeb&MvAz{J`%APftaNEPcv{@6V}4k78R%&-RH=jf-uyoh z3)TVdeO{bt)RUjkp|i54zTcwzy}Q6rt8C)q!x&TuRd6!S>bjEq!s$iBYG_Ctr zIRA%}~*x>!+1mNQFd>(V=?V zzYCNP{_cr#ISfrIvV!EE311}a&DtDR-}qN^u=}7d#S2w_#ysYFIZB_REIWXrq*%(E z08$Q&&HAC{4|>4k{tsoQYGwfl$yCq|Vc&jZI~l{X5v0?@3Uh|x{llkO$G~l3-xqrH z?q}Ax>P-QM%?j_uJosyJ`r4DX2Hw@sQ6r1fR->i8ItNmE;R#+kJ_Aqlji<_+RXLaA zYg%o{M&U56i$-OiiC9rnh&T&uj2;rr$bF`KdbEgX(YAK|;yh<@u=l3qPS1*JyKY-6 z-oMIB|3ds(YadrU(X5{yp*`arBol3ZH{-h9&a6?4H+41I$a0vM#UIW##5jv}JVcF(U zHb_ctL+u`41ulkHWw$+PkHZU=LEV^D9=xHYGmCQtu><0JQ4L%=*y3mFIC@eiu4dam z_9M38Mg@RdQxBi`fVY521S(-qs-(wW51zes9?c(8%?IAA<2oh01+JM@9TW{htF3Gu9>|aOy@WqDrFX510%}9? zy@1=^n2Eno;*oCEVlybP6y%#3ZC~jx1fhiIGwIC+VC^XqrC$%^0c7=x)1d@#9)eW4 z@?O*#ebhy`owRKv@b&;#xzJpTwS0+Lhz3MJt}QQiQV)(np}#ll8F|!JB`r5*-<*ZS zLs1kJ9tV4VaI3_ccP4VGDQAr1e1y$HntOz-((Y65tGKwP-YG-nr}SbbAL#akq%c1g z($`QCr;bcl;qulnh#O2+n2k~CWX%p2tqjdUk2ix}EACEQg=oI1@dcN(E?c&hFYEtY z!foD_)SwNam3t$noKenzE@xOl4!Kov99P%5SXAb)L9iSm>a9Y3(xy8_-a(P5|j;bKdc z$01P%@=Q8eJf)Ub2kX&>0Hdr(|F!hJCYM!dV$;Ky2?U^(D!<`087J1zNtaOsZJ>6L z+-Y1>cxmn7Z8(a49uM~8<0Bm#Q!2cu)l~}S+n^!XfcPO{W8(f2(3#ViZ0=k1b#{;G zK%jH1sf<1^r@ZW}*amPnSxl z(|B9D&LNUL3x9?YzX@A85D!kTj72lMT0Ti9WCNc412 z{j2>bL_ASui&2{8&pfUkOSrhPV0;ue2<=-!B7uw`+DkIKZAyY{&;IUTaoGU}z|&bH zMy0C}Cl=PkZ$28I%Wc%H3^OeWZzNikV;V?%?+hePB4_LC=T5?Z9Rot~i~6<3+}GV| z4>!5pY{=Wbq5H&5!J6%5`%D0+Yq8bIuG{`y?7v&Welb7+fIu!{-FT3VOz+m!UTq?EDI(~otu+_RA;D3{{2wo9a*6><8 zoib~HV9DY32j>pL`a~4K-$ab!Vaj{+-Q<8^%JT$t4?GILZ^Ss;EI^D0Hm~3UR?Rw= zZ`6X(s5}owni#~6@x7m}H&Wrib+#1!J4@iQsQfMb6i`n#U)iy@DZ^Yf3rZ+vdo-Ww znG;YMT(lXb0<^NMnlROe3J74Dl@D$m-$Q`ugZAlWmtWP*;e=?EqWmT}ab$u(qM8!6 zSx7Ckz|g>3Pxc6*Sbl!G-52+wVXNTetzGk(P4=fC?yZ}758>Sh);JQ;+No~mXul-q zkq_VsromOZIBy*7KT}x%9TtBdkD3Ilsuu;L05QWQH^b@YMx$?0_o~8=@i>?gCQRm1BD3d@YI=4mUz8M@BDg5wYGX5POR4$eg3L)du$CvJgC@_Hbfd$P zl>X;k0OKwvf#U@R0_THqWf9_S@zYP9h-fz*QCRVcq|_f?^3Gt4O~276c3KSFn;|?< z0rTRXyi~RtQ#U~_q1u^z$*FI$>{M{7LiBN+MkuVd(4y8z!&FDD^h&MOqo=H;2L`?F z=(J7qa3LlQRmKXeiIUjYQb2KMu_MZx#sU8y7kpX?Yb(Cqam4pUhDDG@0z zTj`}a&3IE`TvT5nx0LBoQ*fLe+zQU8;1w)iPXY=>W*|sJSMnEz*Fu^mN^bWDRtw7* zD{_{A_=T>!G6SJlRvqZ5<6Wy}M;2;Jy6N=r<1xOK{KMR` z=PrvNSMk^C7mFoChBHnTH;W(XD!nt*ZFzMzmknu-x?)zPzi(+QaS{L4- zcKRu^CbJDlOV}E=zaQnymZOtoX+wHJZh zF(X6mk}9>&ti==An!i2%;7JH@a$$bee)}MPJwZe&8FB%mw3yd;uVi>^)qc6XF8Bd! z>CsY-m*H3}Pi=~ylh=!t&rB!sXE0iTLT+0W z;RHFuh%7&3TQmhNYP-NW5jc28L{0U=%}q9=*EMf;=E-R+=Ou9V^7j|u~+)EU5gcK zO*w;d{k!mM*HorcMUy(hw=d+`cvGZDT~O#e+6v^#lSlRX7DHX20$L|;sl`6r!jEC~ zW#zs?hB*(;Le=Ml^k<1P+n_GYrwy~(D&7@$Lz2w{x74x0b(oXR%Yigsyk}`A=Rnst ztcCxm>SO2j%sn|45^WONOHgiPcsnn4T=ok*`U+C+n1jV8$7@Kq{m*+B>q!Ng0^Q^8hgrJ88%;uhbcjk-J^bezPy?tgqMV5D(Aj zUOLWHooK{5gKPuyTDf*cfBXkgSuE9oY!`X6kE9rercF!nXTkP!d$X;564#GyeG15R zUR#xZ{S+B+Csp?PeeWS@!H%=Wtv(-Ll1x)?a8E7ik}?rnUEN4H{l05q9);caMHH@w z@z@rdexeHF_F*EQ4` z)D`*hk)g%sBFch8nDLeuXhQJ^QYGrD$n!e?+5w$Y51xyYkjqQ_eI}z=@ z)V|jEe9i!+tjq^{*(RCHMV?4joL^zA1rs%9c^|cA5IU|SI-6FbLMK$@Nh57}x}IHv z-^gK53xP^uU+kE$mFJ4r9Yv|}dQt4Yo(n_e4Z@-4dTYWS=M`GumUhzTnRR{t6QS`r zISOy-Jx&g+63&9P^D6U3bx+fKc2ChlwxZUGR65jeF@*EoM(CbzW+m*W@MSJVoo{E^ zvC7JRQ)h~KRh@}SklyLPs+t{frdN0+z@mElE}B4?`(sezk@$iV#wC+bC~iguqW1P) z*5pt;mJJKGas?8DZL{19>`%q9gl$J9zRyWhQ43x6q}Wc&@)t@!cazgxoFoi6lE%Dg zQM)GZzpfbmrj05Jvwu-j6oxEI_~^j>D1M7VaxO3<5rT}H}60C7z|rD zZ78-dW8g8sC}#6!MsWMyZKaV94Y_9GzGi-Lj<@yZ&Ik@|MDoI za{36{K5WS#m|k&p_FbaEPt|#|lFv18?$NgD*}1~>D2?Ke2zO3pcIla~*O!?)&-aX% zC1sCs1;naVJ}4viD&+jwa2oFB`FPHSkR;3g064RowfUx*kZJ2o!h6B|?INZ5BZ zkEN{kl=y|R>@%JbW9lZ0_K*Tk@B@&Cx{nCnI=>craU5(y^8T-@nG|o4uYD! zy2B@7h*{;)_1=~|@l$pAt=E1*%L|V+eZkFoLO!$&6m|NSc?-iM8W+n%YONblY7H%{ zt-~!WS>_Vv+$=oB=tf4EtYp|^X@VB}=X3nW2bVqw5&mZRFQB<){${#&ARB$I_$6R! z+BcH%!s8xu%59tPDPa3(arTBA>Eu$%eJouH5B3nJLZMnj+ zd(X7Fk^@FA82Tu!`sBUi91_90{la~cR}-rDAjnXsFF@}PuC?hTbp&Ki@6#Uxf<@uX zcUycM4VNio^Pr4`lHAV+t`41t)o)}=a!djIes$@ce2b7pJVa8>_e0E)dX66c$kwrc z(Iwl3=Rw9??VungdmIZQJJ*V!YLaT_NmTg@(`7PI%6D^O!fS+xCqDJi<>MaesQjif zC#C*VK4+3Cs9i2MXLrD2x%}u!wBGq^ECltZ7|(4kqafPh>s!MqyIaD?ePI{r>S}L1 zcjk6me8r&}xgGCc?zW#s$va2e4$(!uE#AYp+=qSL!Cm2;5FgdtqhCR04yBEIL$Q^d z2xD?a8v1U$^OK|TTqK+LWS4*Z*IC~1JU7!F@Z0b{n6p&p!8BS#5Y;(L=1^E}Ss_IG z+;ebWGA3@{Z@XMfqxG@Wr_cq^rNSLPo=PrHHwT$mQeqy8S$zN!A5iUv6HbR|(2B0| z2qCnuog{3X3r;{}2|jO&$8F7xxV08tTBs;ha(wj{D)Sp7KO3;xy*ru*h6D(VGNbpg z5S8XOnW=5H2|Ve&F6eW49?&xtume7VUzvn2J1(0&@3QCp^6^XwDQ#7x&_r zsjn?AbeI={yghT`dI6O?PM|qL%OZs5JW8%@eV^{lr7eB4(SSc6bf2^dk{#qX=#=<| z4@5Dk@nZvu+Zt~p540CktuLz=KTkfy8Oq-G+{nY7KUP3<6xps73g8%c+;9f|s2Mw`#Cjq-E3$gY+X$eD0W3{fUJ zQB#WKo-Vr?yY0o|xb*r>J>+1^+%TB4QYsE2aPuOr<(gFlF**zp#X=hm8uxFSI&A6@??TXA8Im1 z+pIql_^}uU?BqGkXHH~n801M!LQ~Yn_Nz~D9!-=~zbp0nT7|eVO(O$RbTL<&1GlKQ z9pPP66Zmf=qSW*-mWd2K<~Ep$7>xs$mTRBsec~>?GGa+O=G`&=x~w)`8q=srMSp5K zr^T+dSJrcC9OJv$QGBWyB+P3@E?L;7SUWXZ7FI%?ST1$`z7pT&dVG*GQn9#jx6j*R90{hQ(#5a%CCJ^waPWJ z8UL79*iE-pkA2i0yNcH;ZOyv4>dk{GBfrTx8wE7dSV}L*^GrwW@_fr@v!7m4Jn5Co zftry{ryTKPxv>(Cw1y7g265n#k99_4p0~mm>pfSDF1k`w0MFuNpU&sG^gr#Zor^iU z8s({utt&eWN8Tw`v7b89up|ajwI<5AJp<@Hv{zy7ulTadT4sYfbU)7Rgs(K(lZKdp z_OJq_vC9z7+??;|^)_FNyI$Yg%$7}*sus&K4n(%ic9vdjm^HPwwn?t4Le>Nc8)s(; zA3*z4%!>TxJp`@e37h#OstAd^kLFX~3@PPFt$lsBpcL4nQiF?+&!?oN%qvse(JdPwm$SZi#<7xNXID89Q@k=@Xc%qG^NGU}b)5h9DO$9J@L4;*OPa`tmho zI+lOV16lsDte=lbAVImUCa2-uvX4H`MWU4NtxV>f7|qR=3?uVbe>6{rK1bxTKuP3$ zp&1x>f^BTrWRPl%8i%bTC7SJHg16sTPz@SOE>tp?s5W2dVZ9cMXv7%r%qox^!Mj|0 z@>FYh;<1F5WYCe$vIB(S>HfX<%QJiU+E!7p&f@bsY8Nlc7*FC|CFbDs5_BdUYs3&s zteYo~XY;VRyLX`vEH?c*V8F+H!`j)0%h1@&b_K|hv@_;bSu_i9ZEV&|#Ij__kxG)9 zY0}n3KR*xUi}khGWZm{^^pCNQN*}Nq)Awu+R&{wU-$h+T8^&#&Q5dMrT8d-7tV;=< zH?zP*;B^C~pPiX+4reacg;Q9VGPkthD32+9@Yu2^+^RBi$ckG~y;zU%6`7;@yd9|B zE_!MaBi|_A#$SD8$0V;&BkHEBvv|yeb<+OIh__az<`6WYnqH&C!-!1xKNHNPITPa+rIx7*|* zR5~aptjJ6j;!?wVS#=chvPM%!#pkGD=h6iGtOFm90-9L;aK!M0kN?4)JB|H@1(uZs zH8|;2pGMFa7E1<z&zUC!ovDkcK% z_nA1uPhpb?JeN9jC4z+*0?BG7L!4)zZx(a?)WP}L<7ZRYRI{zZ;>Ii-7``vtcGAfF zvG|K+)LulEspM@GeS+z+Jrfgb%g1{r0T-f zo1|%~qNSd#CHVk33H?*kg*fWB+4jE9CGiEkMwCm5>+TI=_A8m0`4f5!YDi?GuKVlo zA0nntaPs%}_v|op%H=9yi_hjuYoq*Hf_9ir9IPJj5N?c4*`)@LX{^5sj2AzQdXbwR z{v`laKRf#oKC?>rPQ1Yp@i;y~bX<_QxVRuiexT4}?j3Rh+^*+(Aw48h2a3CFpWzv6(HB=O|LM zL~&w&gC%b4Y*##P!Ic9!obADZAIWT+$@F2L_IcH+W!gxc@?2e27kp7)WTBN?vi8b>~e^Iwf(DtRDu@j9-@V%>{5&P^+N~6=)5m1R7(MM@s&8_VZ zhAmD+nwT)MXULbD6Q#}rPWz{|l$Ty8jF2Y;55?%LmbrYHq_A?^%8ECp)C-z|=l^VL z6Ya`P1j-#a!AvR8_peNhynQlvn$2!{Is!{Q*D4^)7k{}t|A;H^ZmH(k)9cT*r&$m-sDXS zq*`S&=ca57web;C1f%KL=X-W`*%ToAlOa1TT|*Ei=H{Fz*{)f?tZKQBTaUVCMh%RT zEd7(ax2Pl4PY0#5ZAjs5teKhK%8ihbWSKhDB~vOUOl)n=F5AExIYuO=<)MHzv+cSy z3>T^MvXphH_Zm4&9B{jitps49J9F(V@l70fr5Ipryr(i!y(gTfNL=TKceoPh4={{3 z8Y(3IIRi{`y! zF#^$$?hq1_pXNZc_GcC$9eGQCmNMmPu3YbSdI|1{2+oPnwdp!-p*US})r&mxfI8B7 zCeT9LgD(zKy2i8Bcn>z`Je5GGzWEQcmH2UZ%+}x4*g$d|AHB=xDwRdTI#(%lWSSgy zeSV$kFKiv9eXjR@`P@!xqR}fOE9=FGSQ+t`D+`sj$lXWAna_?;04JuxRw~H9fu~@r zx<+QPVDY)2G4{gEl7?DM~7t3QSMl_VPOwJhLXs%AWOXN~br% zxmP#4NA!6Nequj*&r!R6$xv>T$LW#y<@`u|O-}yyY5gUQrmsw%d@}E2hohGGDI{~3 z(T}J-_LeGvkG$?)bo9o)Us|4dQHp9IHa$1jc@!Qy<_(*?RW#%aqqBGP0Nss4(s!a(p_Au!LhCA`EIE2yMtVT0 z^gh_zJ$doeZsl#WmW);c-w-!r%ZnC2CoQm(g7=XwB&hW5mlOF;m=UBvk?5nQla(n= z#4pU@5Ak_y%BaLqP-sk>_GnPG4k_5#LlVi@ZSdNz&+Wb*?((0dYZrqj^Pdr6CkbxN zzq9l4=H%?}pZ0?qfih}b;ADsC`nVR7-FOLE$IG@lYz)^#!WHM$sOV>HY-NbjW*Y=# zwDp^6cKL)OtUQIDsycZ zP;fdf{8u*9r7X8J>QB(CJe2x!emC8eCv!(QBPNZ$>!k1dE@2t!obkESRqv~WH(yQ% zLG9F657}&UpH(-1^g4%MgElywc1v)F(Xr3&Fuk%}g8#Hc&v%UX!aM@9@mNC6_0W+u47P?v!XKT5fgaD?jt?#lf@OK8^k6$mwwdj9AF)PvF)FraR^`M9o~(x#Xkyqq@7= z_e1a32Ua(ZB`aKr$5mR>~^m!N}tY0cL0_?&ho^(_?q@A>TBC7>GV+_bI0Nk6RzXc z6I$#c3G%s0cRf2U_e(!&@6n~qz5sQ#K_zkbV%~5=)*7of+tx#~)Ljw=6_2NCX`Wiu zh^%=EJUMef!ib~9^=N)&o`mgJxo%A39&euZ(et(Vrkd@xRYJfT@gX=4{i#P7@qEfG z@RB4_RhF$POouz%QnO1&H6;AoS@s)Oc_Nph-iLb82&b2hWQKD2{+A|G zC8$=I$!w6={Kdo?Na)zioFlqh?I`2>PkaMls=;?4Twn_!5%@Hw2_DJfHm&le5EYdsDW!TI9Cm=&7t z!U}zqCkbOXq9@qJ0}-*HhD4A`P)prI47r6EQqM*HI?`lnr?ttnlKY7ov~q_tWn%ON zIX7pI*aqyBd9GR2a3>NZqFE}``aoE8aT&F%;6(OGC`oCn_Yoc5>9AzW!>fhoz!iR4 zN~pD^P}G8A$ruDCuQe8F4WmUoJIkjZO2&s+0~uo7kn;tu@OU?Nni9q7%yAQ|u0T<` z!6~XPf}$7(yF7vCS7T*gR8O5>+0(??n_>l-WP&5Z{gWeL&@43K%2S5rxGp*yL^Vxj zn~nH2Cq9q(OTOyc^m;pzj2Eksaix0sNKw|L%TZGb*G)={Sw0wbD$v4VH8U+fQF$v( z2OJlt{+^(3e}Bb}X}qC}^mP*K3yF(4d+qS>QkzHE`DlXFn9P!=m@duRw$tkgQ&>W) z#Ofn~8LEYpt7B2LnQgUhKJ&3&i#=4K^;gjIuPqyoGlnWDATO7$$u9iQZJFeh1K!IY z&ukfpH627tuhf73@{)&?;yG`8x2KR7DWj80wo5E7;i*xMH8ZXnE^jD-IG{)<+g6)u z-GUr}LUr2YZCYyR=8|SR_iQm3VxTj=T%K|CZD30#Q?35Ckvz|DQc@5RFWxaDjZ}m&!I4)lB%4L^@}2jSNXLN> z!yqQxyXjx|5p8UZ_;Dsn^O;03&^vkq669izLnU9q8Or9hq|r@Td>u#Z77t3uR&Zve zEG59o?$a-$VnOuL5r|>UZK+f-(MN2~w=+eauUKHVFb)glz;YFsex2&K;RQ}gqori+ zS@`6D>2h>=A??S;rj{oW>2smk{M_E``CZjuUDs&!kBddpv$Wl=+ZA1U1xnLRZKYUX z=!IlZQ7@B8MQ>qsz!h27{vzyYJghL^dv_g~FVeo0E~pR#JRPK9&FR78k4-U?Q=*BA zq}cM|d+{w^zSoA=;=FkoO=FIqMw*h@z9ws2h7~^E2m36kPp7if*DI`X(VhOKO-_`ZE#50Bu%ngPHQ29&@l~oz zKNacTa!&kb+V?`Fe4v}Z*9f4Ac*?MKr;VfP*=xUAwenbja;Ml6-Qq%Md_B8VpZ-Z| z-hE5~YK!YT&<3Gvly?j%aXyTC4wM);az58Y7xFdKM`(B6OKusDh#{Y^vPt%O`3PXN zZldbW>M0nS9;WW4t>Y?^8R~UX8+}bz8$QHFwH$PorG%7PB7c$~*G)05F=BW>1Z2&v zE~~a@C9*79ziy>Gpw^NkEce-CF<`g3YAk>9MpVTa4v{9DzJo|Q$GzOm$0YO;-h9)~ zMUsiBk-Sf7VQXMy>`FwheBnwbj;_-YB!d(tM;Tx4ei_W2;&0%#&*Yo$=&N*Md`hOb z!lAVP`7Fw~V+dm54STavoit{zQ@7ht;l-pZ)9*l3R(780WaluSa86t)aAitgN99f3 zG075MONK{wfdSZs!kLJru+FFOYh+OCkiul*Qxmy|nazBpA*`SeG-DgJ>27vR;z5%$ zAG_x}KWGz0+wq3JY&N*2r3%aTG@e-Nu9KIs!tUcgSI6pLA`MXj30y4lmu@Px1%DnfHH8hPrzU}mliS~f;(fvD)BRE&)tfor(<#A)@@DoR){M?;`_C^+? zbu{*wV1g&u3^MN}9yS$dwbo&->r1J1Zo6T7%1H)8I7;>~w@Mn8XY2`yuY5AhZ;vL7gQuaBiQyIL|wCp+<8>lqfi=D3|0D1Y>!;BZ_; z*qzMU&=;#Au zrQ!h>oe~;zIzPk_%-`JT6}z_-4bzted3%zvW5+Vwbw-MnkhA*yp3So?9F&DTf;Q-d zNb&bJ&iPe@tv zCl?gwy3wh>M#q{MGQOe6Vx2!Zr7?8v*4W%+jGHn_@%!BXF7mV@I@~P!}&}R->Gf1u|L_h z-JNFdnv1q-IzytY7-{5z8jskW4;R^tHkAja+@09m8;lpxt>^MRYV~*Xkm<_$a+>*= z%}V1wCR36G!>r-wa?9I7w(k>AU0Cy;+eD-Ld7r~;Jw-XznNr$tj7WuyYUa3ECF>Q) zxez@*{7P~7K_ft+fFvW=9;e}&i_@?=qIhq(wlXkyGdqH^Po&n(^gh$#X?w+*?IO?0 z!B?u%x5cQ0du9g%65<~I=_21U>cYfsut^4`!cP3s`WVP1$Z3dJf#QObS_=_hHm*K3 zMbU2DEA<*|9T-HyxA{mI;8*dbz~01jOw>SJaEc#dBwLf7ys*kA=rGEhJP<}@0?XIsj}Nj_<=<${utZXL&r$L#-$y|;{t>-qXU z6GDO{NYEs>d(dEkhT!h*!QF#va1Sm48h3YhcXtc!-dNKxP4bif+~>J-@0*#mX3ebC ztJk3qed<)zu3dF@eZIRj2vITRg`-A;^vnQ(T!`cWuLJAUw0b>o8Z#ZJ%?!}muZ(R+ zq=i0zKr>%Cq!R@@r}SD|m;)!|b+)A#2<(Nm*~Y3GM!z>1%vNXLo_x+ZH!z)V)P2Y8oppvV%$n#@*cBVJliku79lwjB+8ekY zXLgG~8qq5nvH0%1UGq_*q_uEMdp-Vvw!H(Kssspbeq2`=ZLiTepDXa7&ss*OYHc?@ znJYr#3IfCJ-4T9nA=Qq^>QSeQ2`+s=dVkFP+28ymxckU~)CP)PmbE&D&+jj_1U?M4 ztJFkY7X#$ZTU3VO9FO?9qISUQY!Lm#JBEDj4Mah>>BGsRDWE&)0_W$PINbsCi(s$G zt8|6*o8=a{VeI`Q9vD?ES%IaJq#9lZ-|ZU9H(}Cxt3HF0%HHP%o3)#C&!^C*Ma9j? zTT)aMQ(0beV!3!H=^Mn}erCsXvRZ2z1pSscWBnCc{RofQLjpdC3ltu&u zG%U12X7>iqQ7>P3Gc2=qE^G@`jDhHj`Lo4wMDp$)GrEZ+fnxa z#kM10#S(ctM%BS`dq)L}wr}{kEpi-~@N?6v;by}vyiF`PY-B4ub3h0Ma=QvjAt`sk zdKq|5?tCbeC!_iclH)DnHlk0)o%1fLp|wyWc2d4XyQGiV^43IocPDJ9URwippv<9h zqLp=tf{f~&z+kGh+RDvPD^MoZcAiIGw||O<102;B^kV0Mj3ej5>HNqCH(sCHSpa4a zp7ieJ9^Z2_YoFn|Xo+UU&Axwk&!p?sgyH<2t9ibyCAWzrsD<7GG%TX;IU6;9X@o_} zm5#1*HgA zKv7dFfmuX4I;w%!-A5`zcGt7Trl=>Htw}MzQbqOF@i2u$A*gS%a;t7@MqNhr7gvjW zK|M#|7SWh*`kqf-ai4cGM|tS^E_+$o8@tKx!ozaxom2p;47UYCTP4a*Vg29V2)hLw&c4{7IFxuUeQKv z1v1{}VpT5Y%N?sPKR^rT(oHFg8^J9`t_S&4oK5US9jmQ46^PwvYH`jqZEM5amek&o zCn|$uSWhPsc8wLky4a#APCM%4`;2%)_l#(zoju9NEyZjjtLgBi`J}C2vULY{d}7#z z2AI5??&4_=j!8M3!G_jb=kVl$t$>-a;I9hg%_1W}Z1B;Oo{I!S&BRgb57%o#~**>9PHJ{N;tn`@i6OOj%h zVk`tq^uwBRLURyW5s1BxTRoyFUP`O+Apa5HJK7CjrYnkMD&L#~{PIf-#fF(Q1vL0g z`+;0dJFz3bobBu)l4@#ZVMqoqofOxq8}@2x;I33_axI4%x~R(p!JH7{-A)tRB33!8fi|G zDJ1kDmS)FbB-1B=$l(vZ6ugb*rJ8aS6wW`Ry``loekf^52~`z=exXV~|0d*Mt$+Y6 zBx)55mw2UAgO91(q}bEK7l%(#VMH!;cC}QTrz+=-*eRaB!`YHuvWwf6Qk(6pB!hV5 zgO2)!|1YZt5HeU%1m6&zBqz!!`Dpe^>BLu#LE}^{Br^53ukY?^Zbm+S5cun#|9Z^& z^RV^z`z@*U*_=|*!gz#?-fLF3D=k`VdheD6=Vy%(Jv{eeIk{(4i_*AT^5O%3a3v^3 zVa?}-E|@H%`|dBQ;tyN#KY90!IL7oOM@?tdSa{SK0T@_D_3;1my$INqpC2FSM;}=n zoKGn__r9Nfcv}76#CdJQjW%9F$Aa7IDE~-a1k_X4W>BL$nM_w6@zd#5NXY8Zg++VT zR}P}J>$^2au&o?r0wqEWrMF!ENp^-&5Y~$Vx-W!fAZLMBE@R^>{xflgENq|B^XoBx zd5b4-m*Wj00W%{-Qe~kFTU@kLS1|M%xQ5I=k-Q@aS(>t&tDca;1Q=#Jj#(d=r?`0< zC4#Ys^+Gm5Oqg;EY90o{5IT%UX{|@CRVs#jk|>OzXxqs{_sJEeL3)Oqpm^H}s2%!; zMSF%WsmwE&Y`Il&!BH4}B0;P`UeztMsbVc`FwGA{3~?AzLin3z_tQpq+AdNVTG)va zN;ybmM9<}YGs=<*?VTLTQ)~6nJXhtR#Gx#Ta~Vth;@*PoNEU~K50 zvmJb2$)x`j9qaZy>pAHU!M}=z{&aBkpANLORyT zPwqV~ zj;o7DDGIdlT4Jbx~O@ zJ8a=Dk#`}8*n?H0(!y{ZaV0gm8%eQNw?D&eq$T0EyJbeA)vCc2X9`TMKgBJ0p{{4@ zsF~D5QbPd#*&aKnmmu)Iz|2%;kH}HnU%dTXn&wNsXRff+*Xe>@z2^;rsTLK(l~ZTV zez5m^I{O0#O=mM!GFnP^TYS55FeSU2df!%=i>v#@=T?!VZ?wg++kHNBL`@aXT2Z>D7=4i-v{MrSd=>HR= zu~|>wJ-rB(kRl)ZcfPO{!aB&|yUm4XL9v6Ni}`4Trt$i> zg$}||(Qm4LEgUQo9b`wL(|pgYr(wSx^6X4E0-uH#j2wZqdsZGN0%Ha1VPiTSvf;<} zso9O$_8h;dkyYDm4PB4l?w87Z-Jhk)Yk2QidiTPusl-2efNLx7;z8~!=obZ?}oW~AVd1xm(+^9@v4{~#!GksBavN_U9`io}D+XtM? zm#=haI}gARMjn>b=n2vr&A8}#wt_)Thvhjb_6VX z4f474B;dm8?Lxe&iT{&l^v)n$(`OZwr8HI0y({+Y319vknoA3o#CvgLrJlTsyQ-Dm zyk{v&ZA4Kp9m5=y+HlHe&J=>?b{*}R*Hdx(*6QHEWI-d2LvctW^a_JdI@1jdL3FcJ z{yavdgt_O2!E<-lGZpQ2h`y$Z*2m;+lFVI&*%c6w9y@h0v`vXo9@2Hh5#*>>+DHR|0T3RaIN5#D}eyo!-K(JEh%So~#}Q?6RV-oCRu7{6^MjjF2<$06!V;9z5*v6@^6k z-X(K+$IjQq4z}0TVtdr*J&hzFN#cdK2eHt%ee9d7^^xl9u`Z%#1L+5Zt45B;jb6f^1EI9g_i9j8=ZHF=7+S} zwRpYZcz!l!P+{9OJ0=2^)^VF&x5{)2=;RE39g*zmv_;h&^v#>U-yeS{qrVWv@U zxu^j+4wigcd{=r{8JhhqnE7&9tCE5_PNVVkV?y}8+Hzof$*_V_nW^w6x}D4WwS2c3 zzm3^;Q>y}lCtZ{M1Gw>a@Y7xPeR--?l-Q)_z_jB`=B-kLNvWS#0%b9`#9OoVV1PWy z2b^I~B;zG}3M@6^(Rg%QsP`T_D@-vbKuoAq2A8cch>}h%`KPW-?jG$4{d{K@Y!#hA zmlm8&o^-8O-_Ll5&_2yDPA%n3&pE15{j7ANH>|KLIv#D|-clRE;H4tjb2LhBm~0MN zft@9<+{gVir<-?=pB>h}X@uAZsc+xZJpS~g0Axg27V<($NOlhnI$MNICtnWrS6?>d z6PKp?J3Z&A`NVxjg>B{Mq)M9uTh|qcp(On&fa#vL+n-_`rI8qnf<524{-)rfq7}jS ziBZQtT%>T*u}AY+rfX;pz;}UhpK*N&X^2lzwK(^W7r^bA-oa3!>zmiv8`{Em@1|g} zU+e94_-e5%##IM_C&pW76M46aVB~ln#t(bdp0$Vx-aL1nk!_?}2}oNVHY(AVja!eO z0xY+a^rEKVfB6Z}&O&!>@gxfA*p`>Gg=D+knT^p8=fa4p+kHukxzDwCHv*@IWOTBK zNAu=2qYHPD-jfIqdG=cCk2WN`NmY@Gh=Sug)OQX6xu-5zxv!vYA4`v|br@Lsjt0Z{ zGj}*TxJFMCqBd2!=zFAFBw7!A4ejtnTPD}x!8%8Wz^#UrbB`fa0LLKQ9m40N#@4GB z7Q3moH>ikUore8u_12DeuM$ZoHD;a0lVyF+A~x4$0VPb}AcqKbjEpN5PU zihuVz^@EFBgsb3aT&pG7bv=ef_ADicK4~HWpuTVG)P|JKDFIFSB=W7P(70>=;)lJV z30A={O7z8L{eCz34PokU-J0Y#_1^um%T#GDAvaLbBJ^&0i&|(-dCaV3OH*z;i607* z$aH~t*whdNcv|Cw7L^zBn0+oN9$yK!fVq6d(N|M3V+my~>F?4%m}uwvGU+va#{Cvl7QhF*u4*15SQS#vhiCA%&wp=&ujZF8%5U%(eH&K$O` zKHW?xJA55nV6m7Sq6`n&G6Xu;peTAjBPV&QFzfkVn$@Cv{{#Pqg@VkXs|j0d)~=%@ zulN>2KRrS=jcFNVv6+4RmbZ!Co_o5{_#E`yPI>&A4~|od-7j!`66=NH^tFz=F$wEK zzZz}1*~)U9Su=h=h(vFGM@>5DA!^5#7>EMhY5v9ReJj;g{D|SPr@yoMRp0jccWrZC zO(=*zwYA|HqFkcUo6;WEzxr|XJCB{>s^sm!)iSJaz(qXo06^B1)J}PHW1gVt`@!qD zuNO6NjLygG;eRv*Qyc$i3Iqw7A?S@D;y;uWT|w}Ng;?n918#SQ=z-;j;K!5Xbn ztdIH|CQ6m=VmJALbnknDG*)-IEQ+Xs9a}8~>uq0m zHZCeQC`)=N^uemBmC_|E)Ve!9kF-KOjLa^GpRbzz1AzBv$AyLcw-gO}f)dW1oM?;a zovoj(5AO}lwfEM2zC|519=K!`>^7{plvzQ8>>lS4^c$ChrEBXrdZ^r_<%+*j*Idlq zj7er@k|0Zp$z10zLe~sw)w%qRjZ%dL-bBy-pCENsC#?t{Jg}^`-sd)GDh3BU+TUs& z!Y_nLh9OmK0)l?zT)N@k1=Ea9zi#;LF(0aGV_jfqbod= zM+vv3-6bmxacu7`W}Py|7w{22Xd)#Y%GkHf5jq{3myN~Q%JWe;P^tK4N@LT=xE6m8WP(k$v#b1pB=?JlT# z-5fj(kffh|m|Tl5x@OAo%)Gj$z3%gOQggMumE0J`*etmt+U7cU=4Se-4*goap8KdB z^7|fFzN5FZ9TafY<@q{Q`KowEOrYr(tQ)#aXIXdR1%gcHEbpKe`@&Nx=9(j^y#5|E z(K@2XAa_-K!RO1nSqveE>4t zU@Jpt(TV6*Ex^t>WlaP$!{Bm(pSe@nGF8=uY@D{k#|Y6Jon+xiAPQ8|P*&rOM$ysH zHyb^Cz%Lu@IjB^NE#lK8-2!Z&nM(KQ@fxq~$wSYd4HA&je(#~tH=kC0w|4_M7MJFX z!VB@uJibHDF}$3TU`5*AdZRsFO29&R06xq{>z)*?p3gvvg=17JYDzUduNoBr)Mq+Y zY@7`ZIk0lk~B!8=%(drMnv$to=)VMK%(;PoH|a6d$voo z-XI~R70ZIz!>PHpMln1blr%q(86?K0?XX1OoQnUi|baleRUTmxPYzbmPsQ^cjv^Cf#pxY7a zH_yFGjT~^?vGQYJ_@bb1UoX1gEE7!~0D-uWMH{nB@w+7PLy7M0V* z1r=*Y&vTjEX>(+gZR^>4XOSWaOtm0`xS(H{dEl%Pf%95!gGEeH?oVjC#mJ6F!C|Ri zhAx zCA*S0n0BPL#oBi@-CxA*VXy_U#&vXY_#VUW8L-Sv`A-H*<8XzQurf9?*CZ-4MAFW( zg8F26zpvL_FGRtrO* znjhwf+tD)&=ljSQ@j6RZ_zuwfBWL5cG2{JZ%swW^4rE7@enaSws=gi>_dN#wS8Son z5dbd(Ho7i<3+OKTy+L`LwPlUS7Gw^jct%}a;a9sJfGu~4w@x(m2=%|_+~*7rR7OBU z^JkL9N4p%{UVT5_^|EmBa9tWdqX-*)92k@>7JAK6^;L)P%`gphF3zH>O+%q7(iyQ9 z&Zq6XS(ifcL4w~Ql~>;OlcK@jgw%8e--74ezl*bsS-QWTGCeS_5_iYpIx70y87csq zl*#WOlrkBK46!h|p#3Gj9ClY7x5DV-#Q0g>bC7E}bhe9D%7E-g=ucVc@&&PpqS3R- zpjtzmLnSrF9IZnQ7R`%XBfAKjkz6J-K-!K)1LoKDB(2HS0eM4jBNa~p=I~ZQzTXaZ zuy`H;sQW0?0o_z1%ZoeO#AGNzs@gYbDepYg?nR<8n;ZJU`Yzk1JRS(2faJ`NccxfQ zXxNLh&n<-;4ygF5@Xqg1d%He7wt7QROwHuMDXRr##lsKS-@iplDsi@A`y^}Yp-clb0mCK*qTSOefmxbS z&BMYaq=hLcC^DmGdom+auLRzsyElfXZ_cVVO~FCJTJ@Xb zMW;l?JzP$W|o-^g3R_LLY>IH4+U@-ZUS)ik$spvEp;|vj7YWcj z&qH_+1#ptoGVp#)%HqUlsFQyK>RtLczY~tc2QuvL6#%m*gHDWZQ^9AIx9dDx_r6r> zw$|);ctMv!i%HXrLs^`dp*8lG0`y@}>7RXCT)IY2uV`a*{|9`z1>%k6#B>rAVa3`-VT}OFXW4OKZJnjGJ;lVOhv3 zQ#0PZn7zvG)0M;rZ&5uvV?p= z8hxPP%)P^MzNbHhyiPP^lHPE1d1LhX4~ee6f$3QM{YHV`{4=eqFRp^2t z$sv}uEtlp!(Fdl`9BeNi^mn8>ZE}_cZ{}WEB);0*arH@5SO*ak$*4S<_{sQ_)f(HS ze^q&|_o{hyx%d^z+ArzVu*7@>35h&U=GeN{FRm{hi61^iA5`kWhy^K;2XZb1_zRaWAb%5gIrg~O(U zu9t#u-jnyVLius(Pfz^#Oy2XjqLonAYL0!LZ*POvqx>A?;{{p^Gp(EDd0VGiaR#wI z3MeXK7FagC^ae#_YEQRjyTYrRhT-ARgT7+?S~lH6s?j^iQG1w5CZ05)Q04oUc$j2Sy0t;Kwg^PYX8xKtN!vqhe?A=tVElBTV22{5Z=a;v7&mx6HAvE@eW7mU?RjL7Fv0=J=X+-$!I)1|^Zb7Di zL$?0OuAOOT(XObZ%}w|B%d#ZO-H_$cm2DL9u2b`yULRDw+%8>=V9*ApNC`QB!`2a^ zDiqhREu~wsZVv9lD}QcXBs@KEMj>=*XrEKX3(J>ol(&6wC*+YgKXZ7YwOY+7&k!)% zy#qUv(kAA<-h(SN+U!+qhL3r7xiQZBT4@x&%l1TR|8B}-6U&{aM!x0^0m^gX?hF5c zm%0`lKk3HoQ6$P`%GX!s*+p86di5JqdeD+w{gQqO2D1#}^f@x_Flc|?QH!MNles+JhGBYA^ zN^?nf(7C|rpT~;uDZ6K zty%BrN48-KXb?&c32d!BE3qxjG6a`;GQUT`fpY0|!6)Po^n9mZ%irm>PQX0$T-j+% z9OZoU^-vc6<&X;xK}&Ke>t#ypa@82dZB zRRL~eEE(DpBB!-e2r;2hn>1E_kU)R3$!HyLN&2RhHv&doOGvGdjV@Ts1Re?4piW)# z$NC01B6I!x;2!0)<*qpzx;f_h`C3QPVhista3hXhAa2HE{TD1gzLvFrkkg&5LZF_eRJNftr0_LRG!(G8XPzPYd+;vx)Lj zk}8si%RFfAjFo&a%gnUlPJI#+U6x@d>lw@?5oIYS&t-}y@8w!9I&h8^Q@Ql0vC)N8 zwj`d6=5VG(B9IM^bNVzbsGl18OPrwqx?L*6UPg zu!-<_7;kW|(t-h8tG|^E2YlG#5nyD8R5zHR(jQK~Jkhg|;EV9hjtbTSy)JGCkDyu^ zmx)Oj22{`O?#>EmocMS%l?4gyfP(FcwwOVP#;XCy*8&XH7TF0tl&!JN(g4exgxjqs z(VtVZ2A56ER}N+ml3_$ElHznUHU?O&lc~+hWFxi_urfN2Cd1xT0$pqc)x*2vSjH5w z+EDgyqr8RS3u%UisO`|5BHQO|qEfyhw=}hENKz;*o70g3`sLCM*@;F|70t}mN!-Hf zES=OG8)!=&$PiQuzVgxf(+IEvn?_T?y^Mrk1qpk)3m#M?aP3 z9a(Oc7^?=t?Wj3}!`7(1x7Hnz7;%v>r`Rz@952y9qtPJF=`YhnQHa#KL2fGs+jtNc z$s78I?3iP_P*x+)Ro7Oj$$pCm=dj+Z^s)`{*gVbVk5{!8vw$JV(Dg3ImSAYk2w^El z&jZ2x+XcG2W=Ym#yQNG1rZ}^c4Rp2Jk>OJ4t~i^=CXgp4>TxI?*k{`c-L_%4)8-61 zMh_BI7TZ%z?%pvi5BP3SJvTocm>ChAPaJTSc9LK11Z0JeRM2RyJ+VB91ly)$S>LOM z>wyK9@-OCOs>R%=r6t7)jO2P6H+FK&l`%|six+Cuu7ae4y%K>R>T9KQPI4C7|12eW z31`Ul;iDgHyTx!rgluTt^O7|sxe~+z>-MfbK-L8>s1I z3;bG#Y}R{U4-~7aJMeFli5$rf?x<2|oadu3aCl5pmsKoG2NOr{(@w}hnuBmi%nG=e zSK%?{Jk@NZzurrN%**fzY8pAr6Y_9dd1~HD7m0}v>8PPn!84M+f>Pb>e^`Vc;|kze))E4ei=u63eMtT z8;Zl%XH84ZVDu~2O98o$K@z~!sc8k!TylZ{pes5}n<;bCCJ2pE@X;hhO=3p=MAMw! zm6J+)bwhW)#kFI!Z=OV^@9uaWalG+I((DOT$N(0X+I;`D4~*S-R=M6h%D9l+>AvTY zk`ujnfAIn8dxHDuW9bUobGY%~wL=K)MM6yiw7dI;;<8*hWchS|UVs^kL!E9w8YN*B z%c0WmHbgHgop=mkjTaoFQ8+Hq=b`HkWt)dy`gIlX7hFV|(eG&LW$EvwSv(Mcw@w!8 z`f*pLYyhfM8(d6z9AfsR6(0~AtiVA3xwm=EiBLD*CUDZl>;|_+_B%7!>lXpO@72Ek zzv=@VucQt;oPsjN*E$%ei}4m00{XdEs`}@!Pg~sj5pAxO@d{^In`uG)J{5L#CCFtU zASc`)_GU?#YLZ5w+>=@tBsqJ^&{Gs`Uo1Pt+-ap%se!`*4)*w1h(ZfeNIBH^2|?#r z=`=?10!KVzZ08e0>FN-l`t$;1=2ijVd7Hw#oLefTN93buS{dy2M_{+*k(N&PM3$(e zntNZ8|1ae(-Y@%871X2clGVvgwr+&1iPCd#_j(ONGJ$QQ;OYU-&z7vkdCa1aC!R!h{*bM3 zY-d=a(FT2{ooWSWq*g2$n=oW~Zh>bfh%3<}kN6$f4L`*)EjdS=x2eugNl*8nbUog$ z?DWN&(Qlxu9vhu`&??z(0!#B=mQq5x-Uy9e|7sgSavJSWFEfQGb&^IBCM|Zp`v4SJ zEq9-*s=%#35V+nB3sRZGFAB{+Iv>Jy+? zH+;X~N~{xpUC~L_D;B&74LyMNeMb5$o1BOeA%&cSLrEI$N5ISX`8dbgaU3vERU80b zfsTQT7!`)%qt2rq%3jnW@IF+XgtvabeQp772o5LC$woM`x$7Z_Z7$RPpzwJ8j- z-bJH==c96G)XNuylQ^jGHx<`5kbWvGgKZQfc9|`iJGtl5N6-ucT$nMGzn-8IDOHtv3#+3l1o$notJJn-T0fZa6_sJ@*3*gGYGP+oNW#uoQ z@RRVXud6k1vStrTmHj2=xHH`6q&x=)Ay^3YXKIf45+1iTI` zX6zzx-)kJve#hD+-uzP&EI0TI^C^sBEtJ;vKgHxTeq-{9T}y9ra&o#>-)tL}>;Y91 zl5gn@FU{n2c{q}W#+!kU%%9YFW_a#dK1y_<<8j-;I$BKIO90O^rhkrQu#Uiao~qro z^%VZQu(M1)pyR? z6~9jRcs-_EA@J#(W<<|c$fLCRO+w3wfv}Qvro^p-0S*3VHz6x!DZsK<^S)bzKM$BC z?w#*caFBzV^L(=*jTCY(MaU2o?&mcQdw5n*eHQTsJ<7|x>gRxw(XJy>qU?F&r#UQ3 zxnlM|Jp2LwOF(6%q@?`i_#alkKs%umngxlg_J~`>oxNN)98IN0qwqVa$L3`y!T_Hq z74RX+Lo~yQ0TDr;^v8my?ys;q^)OGU3Sjr>vg=gTT&tQyZ$Jx1WY)3fM5~OsWEB1n z6p#Eb6faQ5myq)KAlD&g@`&~^44i!aavDLeTgz`a#7^`%De@+G7OkTK%kSD#m4A;Z zolWchb&;Bq->@`Zd*bWNgoL+x1M5fW2<3OfktsyG{HtMRGMa`GOP%iFjPC=$6Abyre9_CR;fr92=fi#YgMS%IB>Im+W$fYi&Ot+QGB#&^=JrjD3p`p zg+cT-PNJJwK~?gJvhc2$#eg1&IeoNs%S-XiRG{bO9d zpjDqr%lUGaj};O{_CjpmS1ICXp*V_()UTr36qpkq`WRZ3d+jaeYSAw({XqMC*2Z00 zAV`S2>ZC`%?j*%a>6mt_({(~Hk`Kzj=Xtt#s+|zLKPiQY15>{=*TtpNcjd-(OLUTh zkSRWY?<};82gYu%oI17}(pY>|!WX5nl}uVl;k5lG~f%;}b6+we$6- z1lOwEF&-^lpHr#Ho9P8@hFbo(2N8aU-dl}oVf>??`1>MPM&7&%5_qBjqZXbC1OE)4 zeVw?Ki1GInJx;?OgR$;Z!6M}~C{$D5+!s!+>R)xhll~xqA#2G9G!BWH>C+;#K!R3#g* z$wHEv`XyO2mI_vt7;+O7cL#-mq;PM+H;eoIL{*u-sglLTck0IrioR;7iClpTUDu?S z;@KT!tu8R%B5>QYR~#V2H}A@H&>Ok~_Ml)$=%U}L>}4H2y z&e$GyZ4_-aGgO1=eXW33D7UKyW!p8?3TYOmXDKed&OzyKK#fv?tSz0;XvI|bfG8BV z-yhu^sCIJ?IK^jj834uG7wt>oL#ls>wCZn<>Y+j5MW4r_#ToBoLkkGKq|!4B?%eV5 zcPI2}Csxfj%#!{ZPw2#2Z}Y7fq3q)Xx{43Z5WhSzpL6{-Lsu?=MMc`QI8kJl1Mez_ra{~ zIjjOb+I$eX1C@p)cd3rQZM6hwFv1V@fI)pbK0;N-5P@#FB8|k{Y5oB#uZCkL8BK(Z z*)-kcxT%dox*k^_bvg(FG-3vL{B5g+dCTA5GS(EL2MSF2L1SXVnv(mF$g{-sU77C0 z1uiR$&`+ZjEe&{!L!-|$=W{U)pBUvIydUASetLDhSKr&MYh>jHpy4&JpyA=X4+@uF z7|xOcrp94IW#)u68l!hDoyTpjL z=hGUbbd~b2{KfR*zkS2u#&rx%@>059YGI_nDb4%(F6O6#+rn7(?%Z79DZSpfs4S0d zeXMWOyxF*Tq{wBI4il+IZ^>3%P{-nH?BdWI3*|Rf3)Q`GnNh~dF*U}+l5JNL0!l(kDwPZ;W*q6N zdapFa{afdW(zfspTpgV5IAgzG#=9*5Mby5cjQfX+Xa$GIduV!KT(TZ3Q9l&jfRe+t za0o)QWa7;Pu9_%AQsm9PpCi4R}Bs# ziRZ=ZS&dVjmCKK!-eE&y1@u8@$4e+jJ@!%6(uI|W@kS$RPyC;VB=66;xDXCXiAmnH zC$2sIomAaYG5DNq{B8h4Dy%sO7$K2fGh>*O!7*GRHbv`*I)Y0d(W_OzsWzP$B-_O8 zNMjnOkk(O#Hdr5Sl~3abYrSJ}6`K1m%)US}Gi_m8{CSeXTLUfca(8lRUkUAe0PCxB zN`p|3TrJjs-*rc~MWZcb*(80F`1&Y#0&9Av2T5ZeH&g=5+e)Jxevc;J0*Lu4I(!~e zNHu>B-E*ul;Q&yj=WbM0V#_@BfaOPUlH7e5=FJU0Z= zyn7=7L&~5HZ=E5MziF?EU61n~LWNZA7qCAm{ z#GO-rlzw8sXB5FP9{3aAxlrK=mP4%9L1A@WGB_WPJY^nNH1Ma=;D2vL(tE4wNW`*} zT-8TDjqP@XF?vj{*Ll$6->(=hZ?pqxT6HjHRqgb~-S=#GNPdHwUcV6x64bURMPbOc zdBmd_W66IfIVAMvLKFEcyBWXZ`9agpen;T> zy0wKhY|oE?m*LX+{%n6MnN$vUK1QC7g~w^-33QK_*GphWs3R*oWo7wRwejm9_NFW4 z3OH~3Q=c*5`Zwv_tx=Pwj)gG9+!`Zj%ON@Nn}Oe|vSzSZeS7NqgtfQO{HS;ptKykq zd#h%?*rGI})+7EUO<~_Pe>V|^SsEP18+0q(`YV^4REKo!`U*9%LOfX-)kj z81SHruNoB4NxdbD1_`?EIzFyDf`e;xWXqvxe1FRZf-7V&`KYca5Wh6<>c@$>(_?an zeDukev4kJ<-1euDCxiC8hZO;oUl)jDMnu^3ZF8N>R(k^UKR&L&SxR7s&OWIF+G=6- zA!+7ZtKYM}lFGoq-fLMwpgbc6V}S@v3)$N5a^2J!OjM+xus;AQ!b`?LBo~7r6Z_K; zsu|s0rqNGb-~X2sL9Av?7TSc+e#|Pk^I`PivEPAnGh#>j&G1wFDU5=X5t=)L*AjZk zmAmB@XjGLfX#wTl4Y%rsA@{ z=$K;k0zD9W19HXqwM|9;71(48=@`C)bHHy)91oJ4Oul=XtN(`=?-vB9)B18I&;Iu* zuM_w{1Y!!5lP=eSxL@hbV4JOFj@3jFpC{A$^Gc(HA8HAy3~<66dnWfyc0HZESx_(jqw}lA20C)rB9H{AS)DLT@oyN ze@bSMTeuZH`b(nxPr+WW=R(kKI`>cd{?FaGK|4;sEwPv-_@CQjL9p%+G1umb zQwCJKLJu{MetIi0lD|JZMeyd3LOm0ent1c%{(irB(AIq4@@!LE{3A2^`wUtimUo3Y zS(p7k<%{S5P!oREg`#`9KSMbJ?Ds9rdw%w_fb$H+v21CVbs`&UJb6GQP^-s!yc-%suGdq2Bl2%^%@Z@&-I6-EK?G+Z*n1wM1l%#QpgG^SXR1j$dN@}+)$a?< zuna;wpZzxmyygg#Ou2rIo8$h^NmXKN%wFKbsFSf(ZsFxat0zJCxRg<}SSW(0J(%vM zUZAP%oPtBP3m)p+{uW<3lQ4jPM&4h!0xpH&*{h_03=?C)t@l@+Jp*L8Xm)|&?f=&K z?;ddz@p9}23@If2BkTHs19PIj=B9`$>2tl}gsqmOUbY5#T_ykCvsC->6V|e9K+xkF zV1@2+uO@Vl`9G|2^E>bDdc=0^YNaWoR=*#SMD*YN`FCN^;@za6-S3v#&hmHaClNw* z*toTJO<`2XMXq(l1}fiYWDc>q`xn<=M_Ybmmc{XES0(>~{#yLs-A$Jc#Jvqpk@=I{)ZriTwM{2?Jn5(S;W3J4BF0 znN9w!dY=e%q%aZdKKu8hf0_llG$=#;rvg8CS>Y+sg=(mPS?~E1(1j|##F+fYNf(&P zwwMp!(FGauEyY$f9mvxT+=F(+oYtgT1&{K*2?oYz!C$L?JBtIRFE>e-@|EUl3 zpo~IBHbfI_dQ^Hw`)}X;M>WJ?G-OH1K4|^tm;N`7J>NoW=eb2u{2%x5-`wf{cqQ)` z*a<0&+pSw*4>9tCKDz--)9;bzL7sNoe(~{X!se;WvLKZ?{AGyRN6*n(U9OC2WPgUwOKn`#3By67Q*}`sF|L3F|RJxkY3KtZbZ3jVMBIj)tXx*$wWqiJho(4<=VKM6}|}%QzMJ^tB<||J4hoB&KBnoai>9sjh0{Z zaE=%opW^$$8R57?myWivnB~{aoxK<5SuKH5MO7juxW=#QBVG?9?P7gK4kfWvwGI35 z6OP7Xtz42fd^Jpt`DzP*ik?5bq%} zSWd-oQYm>fE#XhKzsw$b}P+o+g%VP2kW4%R2U4jfJan@Z_6x z=`4L?Gm&HdIP!dTH*;gEgZzcdXBnPG?o9TLaqXq$V=XJ5#$ulkO^yR6C$&a)_Zhk) zo!VrF4UT1G%YGE;+A54Kfix%BCIG#2vq$%&e7;HgYl(W9O>LfsZ)NwH=zdyA0|8#w zutR5T9XEqw8w((NPAw&B^|qin5bYZER<*H{Vf^Yrs#dACgDd~BTqa(HS%{GX_fgI^ z$RW+c!BxnnC$qJ#ak+WL!$BPmyU}*$?6k_k=B`M6q1}?>sK>4WZz+l()zXA&Ac*%2 z-pO6-+yxSxn=3cThn--XE;0=f|GxD&q*;sY!c9p4lZvi$Z%`}xyFD7KKQ>K4+e1qd zF;l?FuokcBcn@@EP|LRY@zVlPV~vyH?4+fsmKEH-QLPb%5!YQvv9?+ zJQQg&?A+0YY`nv`NlUKW#df;eUwd(J8OiV%5ni)9WTmsEPA=4+=u(oGQtkJgZPY=nx++G5R%~#USQuAOYbW1mb zbVzrD2uKS^w{#BOrBYHuBi%XlFvP&X_psgjocBBDyl21v%#V0x)-!jkb*<}K>wX9( zEPS=IwUsH(FowbNV;WzcqM$iacL*ktYN|-wLAeAR$?4vwP|-@jXFZjZQ&nPpz-HSz zo-&S(g7#SS4hq^FGuwI;)qXiip46RyG>k`}x2r;D7DqziCm>|ttDjVR;!;RY?-=ET zuCqeen~eTg*X##cpHWYsAKnu@47!7g{g4fZ|C53S#RD{~!Xp3ljs{N+vp*5)#`8b_EEh1Uq8>EH`VmWz%Jk)+X`ak13~*NvUUq!( zI{}txN%sp-Rya^G_*gnx*7#l#Q!ViWZg7g57+q2{LKk=-^`Cy_EgRrxUMA&dJ^Q~o z8c|!okHzYORHQmqQE5EDi}nk0q9`qNqV1;!sVdZHk9+0)J=i!|-%$ID?xQCqI@-x& zV>Iaddwkzxmc9sA_IJ>YC{s*-ayxd9MYHVDj&CeW?V0Ewpzae8Q4E(X-jI^}I-U7Q;g5*4vFb)9KXJm?v}=@7bIOs>|Zn zes$JLRJutwa6hs;85+i#Z(vu7-aF^qn`x0Dt4GpiE4obBdC>+^C2 zeN;9~V3c>7PvYG0npr50T7zCJk@k#7E0O&%WXiRVV_|}GF99A_X6?18P(nt%$#E1R z&Vij4MVMMo7|nT<3$o_WM9zf!Fp|3T<@%XXixi9tA%!kP?_RVM1l6SY0l8pcJ^d*4$H&akw3v;F`XT*(sE>f0}SO{nTMFcPG{PL_TvU zgD>|UZ0_4U;;7@}1;drMJ}ZL2TFGw9)JHW{5&f8dEGNOfGb26P;^3$C(NQpDJ2@nU zdehY_Tew%2Vb(&!i;F6tiv~*w*`vh{p|QjFZfL zc(e2NobbfG`Zu6_zeC7m8iHAptsNmkG}*M%C;jD{Q@Rzl`Q-R?LDXRX_o*G{r4y-Q z{dUpW2z{dtTitmik=ra6-8>0hzljuzHI& zayc-b8H)LP4BtyB_!@C~BdsJMyh%2hd1Db0B}j}c-6-l)Ax^|#V{OdMrl&>qcxsr@ zOD(KTZK&$+V2u}L@t1G#hoXw{LinOt{^+vi1-g93t(p?_r8j#Zi)Y)Q z)C6naya|Q9`{UJ|Es4@Up6`WfzcAX+tFjN8EX80RV1slC*YEoBw@ANmu6uvv$Q^XI zDnn6rj6M9{1t%W38dRWx;<`J?}K3~2>2>eD5q|qf}BG(wF4L;(R--d5E$_b*|$K< z;YV4GTRMr(NOZN9x|w%Cv71NK3R~~;9hUGSoM#Jc!<%^~FjuspOCRK}_?q&@?l+ez z#|{lC*S-?;N~yI_tR``*wcP9ToLKMy@0XP*d#I4Db{=B2k_#W`-ei6C^3y`E6V~*) z`EZU&6MlleYQP&338!0kXy=QBc88d~ZKo(oF>b4H-ygZ8Bq`CyC8!^-XHtJ-%g}uG znw#WhE+c*~Q}qQ?oNGm)Z1;tB!MhBX8^M@RIwZdva%NX~60(0_J_+Z55$~XK-DSsTDqfB5jyixNYedqONvsDU0o}EO*~@0qvp`0f;cyP;?}F;lyV4&TDOsOhY)4!^GcR`FwAhAE+*xg*p`gT7qfmUh^+u!|kbf(I%g6cX6*G+2%nb~SF(tqzQ`Z9r^Mv4}7a zS<~*p$hYBi=9Leu(`};dpRTNL8st>lo@H?3urz34%DdHxTb~+?E?|b&(s^4amLmm> z36##qowh$sPS7$DvhzITfRq}%E9XjP0iQ#SSMiSMj0~`FB70h5S9qk0z?wdD>aZY? zuiujFGutZ`df0xGn5v?d8Sa?SY#JmuUygT~A6C$M7D7*rDAtK*r+oy=qN*y^3wy3baVOP2C@3iaDo&Ms#}{q zo1to`L6cxiAL0Iq1Ba2*cXPL&CRfqQu^dK7Eae_@9-P(gDAH`+IO8Hi66Gh z0 zWD*?UoIqcC%J0&cBV67`RrZWUhWx6i>@f&6H*e|Te|pRgi|uORQV29{1##jiQrjA^ zw0YQnvs<*(Fqj0EWa6qD{>$|G&U2qKw%ajfig0D~l3j7?yNK|38N<*8T}e&MM|k0^ z&gZQPNgVJfaVjO@x@IQRjmF4LL-Iq@O5AOeAn%bBJ(rhULVYa*4cs*Oo=cS^a5duad|M17L97H|g@{iS_TwZmu+St zvGWyN!wifXUYNK8-1BTI0f_M~eysWT4Ddb;$X)p?SSzH)A;jAtl@~}Y#fK7KG2GuS z;dN7zdjQ@K%{8I(vCbCX(M?WHksWIs z5f2>dDAXdHrfpa4$&%ug{Guo6Q-8S%PN^jZ`i@Y!)70vZvzrFp>vPqyc9QURnxT^? z7%>Vc2p3Vnw+{+S=wLJ`06=rtXdti%u{RFPbR{-4i1H+r)m_rC8;Ig4Z#MFX-Cp1x z*4%-KXL%&k3C{l%(t6d+ygC^;pTf|n_l$!Tn773G7hO_!P*@*SyzP`LlhF@TGt^p~ ztx?kj3Fe32oiZ)2P9oZ!l4JGeeLtE-lgX%b_;T!E&jKeCIx$lW=Xbr<;k*7A3_gQ; z>6%-1h5I}>+HEa`3zZzk;@A|O6|FkAOa72ih`=tsnJe+MMKTE?MlTv3QC$77`v@X6 z^}d>I;v*_v?tqf(wB#|eFtN700FbnbwF)7Cs(?JiLTYf{K@y%L;>S?N;B50MD}3XA zQH{}KjDYt*v^Eus;d*>ncL(;del~H&0|?g8LDxJWm-x*&besU#K6&BOV~nRQ9lzAu zv3dO@;Nb*?&XY{c`b}ziPJ|_mj4g|jW1NUHm|X0Ra@#I(x)m%4a<;|TU-IF|e2_uC z>qFO!qgG1dQ#sjI%1;g;(eT4>s_~5D9>vVwwVJaO+NG}~watk|@|-rTj$|`W&xjJG zR^6`XOQy3j^t-XlP@@-wiZgXdBC5BtZL*!N7OQWw6dq=Lu{!}==t=j-J|0U~oVen# zGxnEL<)cA{9*7&g8KKj60Lk#@itdWH^b9QI!o=W z+)l;)r9_2+uir^96Z4)BVvCixc%Z7AMrYnr?;vC6OQYTkmZVBp&lKgg-rUO~wT*=r z_<29WmCQm|6EwfjcE%NJ)?01OtIn7vCyy^#k8*-!2f*^s)7ES}n`Pap&fJ;NI$o9i zX;~II`M22mK&FGuMU0$rAx}i5{wx7x1b57i1_FN|UZ((yFul~eKNEVTR^m43FG&5F z8z719m>vztX+!y5sHve2;00`$WzZKZ9t8Ap*b>)MY;~)l{FzCK<96y@eoieEtj;Fp#Uwg3Cw+U3g@f+8Ml>=k9$H*nqMmVz2z{VWNi>W_Z6EfA21r1&EKm98(lw~wf%oqXZfuj16Jc{=tzsPTyk`oEN^hrGl%`plU$lXdehwlx)# zz4BixevUxru)B|dtMw*(DaMmbJr?bF5HQfg3|ms$i*4LgD_TFAv1rfmS|M&N3#-9o zca}hOGfJJ9lr?_j-77>Bf2Q~h=Fr*l^nTJIl@sJ6ntLeV2}~Bi$#nTv-2*}evXRyp zT9_sa5Tg3Y*zD>*jnFHibSpd5HP7pefp+_z-~ie~{jK0&8hHo|*!+3)o4@f;I{Yt<)V2a7A4eyx+7Q@%DO4NSFxDhR1aQGHT7|;urQ{3b$M;|B(5Y%&{X5 z20eMh0wLq74K~RR<~`B&C`&HvC!)_vPz^HgSBJ$*8rdGn=P)F^-fg4QrtOwNQ^9C3 zC2KSlZ0O>Ya9fJx72^KuEC4Rl1XRT{>-q>cxl%4g{=H<$7>t$IC!W`TO@NO}OmLIDK~37oExrDLvl&sQZZVRK!KE*)JVLoGs^^HdZXUr$K&Wzc<3Jno`ju{cHk40ct z_!(1X`k|!%wF;ZU>(_!T9dDjQ^Ri7ifkki0t)N?SOF%8uG4+Cv9AdzpepaKY2<~xI z5*P-zE)W(`qF&%0UJGuOmU;>9HU07tsY<0h3RCV7TJ0rY{U(&r^=r1MS|4TevR0tG z8Mq|ji@Pm>J*1&lU}3Mjda8KHYle}o@-e+hKr^Y%c%2)#t4t-w;UzLH8cK~|=A^Dm+B0{vGj&mZF#mg*L0tcKhN z4s;#{jVe<;Lggehl!{3vjYa({1^w*^|0IR}^7jAub?G4rL0H4rWcnBXx9i!Qfa?e5 zK0p1xt!WfZ3}6r74~jqklf?Oh$@$-30+N9}W3myP``_>SAOCH5lhuizm;3qOzSlpl z_#NKfHlXGpJ@?NtN2bFbs^@C=WiyHao=M2Fs!ni_2n`l-2ie7`AR{sN5?DWOb)QRLPks(RvXZnq>Zc6eKwIMXfU(b%0;o(XaKchof7tepsZte^kuPdaX;bWq@9}874M+2w_YA> zU)`oQHDSXH?3-8pLs->Q(YSY=7&CVY+jVx{wuM-e@dsL!=Fo{CClkdtz{eDzZb#}i z!#>WtyOzA?6=^<-?N81ycNDBoJMu)%jbah32eKPGsxtTN}kwS4ze%-;c!n6ie zGE0wGlAGM$Q`E3$qHpSx9)&qkq_wplzIx&Dh=Y|T6>TWjHMuRbaR-(hc@gCCvJsNI z&-(M7G2}6;2DRs|Zz354Pq}y?ij3@F0Pm;5>a}mWUcoQUeK05>YF5jHF3M&AZw$Gx zEM9rnL(dE8HREw^j!>IhM%&p;GA>Zjo4(L;5JVa_QG#Q2M({gxT zsE4;h3qBDD@6i(a`YGA=i(Za^O7*u`lbg{HV+E962F+8IV#-wuPt9p>5&7$f=Q!Mo zBx1bl(Un`qj*hu@qrbi}n+Xc1nRR{?mF zwIjmw^iYqN(=><4Kbes!j%M{pGcF~qD;1y(!I(!_39Z_ZjRGd_dP%&>)12W!T<_Y` zJnzoaed)Z)GuED}^Hiuz)n~8m<&EM`_uhT?4nUvzc0n^YCdJ~SG@Y2*#46F<>AVWL zy3||Y+)JE4b()F0UfDl&YHF8H1pRor7$8YYioJsNOS#isb2&itH6|BV5fJqCp7eAJ zPkvZzQ)MWbjQJ@;9@@Nvo1u^o8_C zAZjD8adkmGQ8r(Iq+^c%Vk>h1uSTrOAD*0iqNF(($g@$Ml@#eN@d_s)a(i$?Xy-vz z-nn3J%NN0W<{a0DHK!&9vaS!(UiNwXOrdg2I3mpcBtU;Jk$4c#L5H}d;>d$4sVVR% zi7s}wny5QaS6FGybdJ_7bS|TkeSx{hW_R;=E`cxBb9?u&?8k-BGb4_96j*reTVcHq zFu4Xk)dPmnTyjw4b^pz%hunFy?;FmI9Aug=WlsAGuBM|Zq8XtQB-4gZ6IF5?=NRC$ zO-P=xP=B^VSzLt_yIAYDg)gUE^%rYepS4Y#_FZy&jfe|c^EID})e0TfS)Ia9)=tx9l6~ zL%$053$jbVynF;3@_bdflq(Hnxorm_>x#{XUjA4P^${ZAM9xuX)0so>`f<40T7<{& zc&k_~`{LVt5iFe; zG#;Pw4|W|x9Id?0zSy{;lmIAh^GJJqyG6fe@(RS&ElPEbPl{RNpfTatpGCE zYQA$P3P?>lm2El1{?()Z|NcIU4{gOQ)Q~r3#lq#XHn;s(JvA+AGVtCPs_ftYOmqHe z>|cHoB|#n0ry^-~E3A7FNsso@2n8YcgjZPIP0N4#*E{>Fzq74{)@RjTn$tYvd=orl zRKx*K3%G+cz2`12@$4=D1Wf!T0Q&Mj$ECl%MXz|49_d-F8E!y@1hc5$%bksa?_csPoS%-*}2tZ%@14v75U0V8?J+TSqfeB>3 z^HuDr))xBu{BsPzeNeNYQv3VgN8cR^q{!{Hj<*PDIgaxPc)&*Su=F)OHtMUVhAo?q z6LX*R#`yon#1r+piuG&e^=X~I2QLda0Ngi|ND-KAd#!>N_gxRu4sYhERxKlt0 z{3iS0h@F}B5)zf)+3=K|g3~I8_g>Y)#S$+AGc%S`lw6*hmBZefoJm1IYfM5`rc)0TR$aI*4=$mq~s&Z?{ir3K~8c%W6+k$^>t{7 ze+ZD9u4H%TGW^XJY*1N3Se8XmP zw{;z~H|6cpbzXgqyPT%~=lj4wcYI(+{EGwo_h0#_+0v*|!@(}Wt}Sfk^_dsavziii z?A4xwk~WL5Gy~)}-6ki+Ff=n~{+pXR;rRZDVthOgZ>A0O5R7MUPf#y1;bCnL?)-O; zBl=7V6>rLUx4FmJW>?kf*SSCw;*6aCq)!_Vil5K7s(@}*HEayjDFYpf|EXP5QG<%US`|$qcr1QUmX2O=^lKHeNI*akz4vZNKT+27}fK_%#qIRxI{K}#dK{3h_gK^8+?K%*_*{_j|Dv6w&ggGse zm#s7KGfz8o1s;sA!Noy?R~gxr!n=- ztTx?Uv&=e{Fd4wY66#%ZlC(qx^_u#ZgCFzV_ZzcHy4Tm#3O*U@k+fYE^USY?CvfAo zXbS0fze%hU*0<0nctz?}k=Mp>YI^vqJx}wJ^~7SWgP*31m3V)u-UkqSK9S&YFIldw zTMOh-)|4xTnfG^DvWUCt7)NZ8vAI_&sc>j``i$ zycHpxOt{k%KC+1{>FQppKHd!tnx;*D+Q#PtuWf#vd>F!dkwC}t<|_P~({UFQuBC_a;JcI_7XIe&j4DvZ$P=ca%cPjr>zKOtpDc^_WW^Y0z zZracLm5O|IH4VoeTr|5{R8-LW05*6bTlN1ST03%Vq?NuQ&G(Q>)!$M&VH;<-Z2^-Dcn zPLblNi9vRu)n@4#?`QJ+dE-tkDx-$9f%=)`I|5;A5eXmvE+~+pEM}K2OHeIyFzc1w zv@o`e^(;?0TXN}4;w<=bUVSeRxi)i;kv<=|!cQHiSP;rNTbm8rN3xerHDiZ}?M;jU z@6Eu%rfgrrr6L0wMK3pc}B9;(FcUcuyq;hj#-RvYkeX06pkicda2*#Z6a zGk}L57P8a-RhIgvhVyU1vJp?(V6-ypbWyS@oF$ZKe3s6Odb2=q%SZL*tO98F6iBt` z^>CYi(io|5@}6I6y;#4J_Pn~*C6o5vedjBI8*R`uo{m8kjMZSwPp4KpW;fzZYEOyg zT)N^tnRngYary(IKg0qiI7fPRx3Viq@)v$-cJ3@EE>bq&Rv*)n0j+E5!qXzWV+aYk z@j{4t7i(;&!72dyk8kN4@A^9zf0Mr~`2(!g8u*@+$aSQ|kQ3K|!K$-inRHOsdPh-j z*oS5pFXtecu#S!Gn!71a)Ks*~BAW#!oW@ueUmLr{WMpo)jtmxuYX=&Zsy+~2XfkgDPfsRK&t!YH)1UX3sD-O=g27n~Rp~>B2e9tHGz{Dz)ZDhs0A5i}*Dcj?l(K z65w`U9EJ+6%hml2_Ib9H)zLALD%03{6W-R_xV1oh>v@-G*}HShMJZ`)8rZWL3BT8W zJepu=a_Icn{T-9gQz>K2kUnVZ#?An1VU5<4YCZ>dzMiW>Ols*#n}p(4(9z9kZxq!v zsEzPdYDplw?gL0CP1C0rmf4Uk{B%X9)kNt^uLU-O8F2%^Bb9(uIMM+0i%JL-MhEad z?)sbb?infSB@UdyH?o7f^>jYt?I>0W{1uFXr8{}_c=o&Y{?sCzhUtQ6f2==XUI;b% z*wMHs_@Z$9t~JRUR?leYoD0l*tMR619w^~_1QG_r?rtnRJ37jJmo9|g1dllJ-%rDH zYTY%s_-?IpbqN~&Y0W-mge@f%r_tQ0r*KxGhXckjpPVht2ruemROmD{UzAu~mE04H zh7pQ{F>W2s+R@K>I&791FnJz^FOO-$9h@cJzIodWmy-@o1hCxUZ8hRK?N|T5nHQ7S zS?1ww7c0=^p!2JcEOg^GGRf5!&#X+Q^5j7sEsGx+hkC7L~o7hV2H2Wf4qo zEvykZVU3s(f5@+@+Y>WA?PcMC+4y*O!EP2u1WuJw@{-Fx+zc-9m_cgU3J{kqc=%ls zGIDMLU3rVc83~u&!aWMf2*g(7-sj^H0vf;8_~OWr>NI)&?i;g!)0Edxr>oC9y z^iTdKoq6?S%gR?8zuL@lvb6DdkNrTmN^TKhW#jD|UoyThO(Z-;y|~gSnWoK<-mE1s zXc3u?*eL$BPNSkdcn`k&aFI6OEqZ4&;od5ecP3Xo&(zFc zAT7$fWBwsSuWs;j9Mq^Pr=)G{9(EmvZAH2ap#3F8$A2V#IP#|w4atyu*40#&&fnBO zx7fub(K@4)rfjZS=u+Be8T?2n(bBj_A>G1GhK+|{qVxWjRzwi)OXhTP#lZ#|Wuxy8 zE)H=4Uq|;@5V{U#T8sF%X!0-H>=o|$%G8+!v`wF+!CN(t29HEJ$f|hN4NYdC$#u|G zpJ?%GfE!pAAD@JnL`7mkNJmR}Vk%-?XUtQaTK779ft2Gb@oTwF4NKo=iqQQ|rVrqL zYmFkf=+Y*b%N?0-iydhlrQDtU#F+PU@y=Vl{-lvO?h;<;xk*kr&ZDhv@z9uqzhvwO z{VEZBrY`&(uZpNgw%DcCXS_{D&qH6u+iMwA>s&@DSxv!F^jq2A4S%sPX;H)6*UCDs zQsYT({#aEu-tAN1;siCFA18~Kk0|-tq!Sz9%l_g}XLj-^v)^WXBqMypQL;7G z&HiUa2l2gn)%Hi#&-j4V``TX@vWAk*^mn)M8)pZ21MKb#%AK{)|31O~c&Mwf4r576 z#^Zb&KyA|P>|;u7-NkrX*m1e(N5Q1MsB}sGGW(Y9*~GY){LE0bgA8YroTQ#7T!!n9 zos@`i*tQZI{59#Xl_MI1`3pI>11QUbCiQ8yKOumWyU7vo%B?sL_8FUeXL9C}qh&p! zzgX7ciD*{CEwOeQYa5!#Ag2HLFt(%J_X=w0B1xB46#s|Sh)_6EPRk}gCB3W z6FFGSnP~}rxJ4C~n7`8Rj}7_zTRu2!izUDd2ov}BD3T^gv7$1>U|ZO}htUAAT$>sN ztJCqdfALLD$)Z2sl+CtGm8mnjS7k2c+Lh4+LFKjyvK0BSA2DX7VuMb zC)H`7-aaw^w%(os*RaK_kiAHD1mn+9!7}=5R6_n3aM9-&d`Mw^43{Ad;$O;tk$eor z!sPNt)o1hZ|4Q<`)vi}8>sHFgXuE~nTI7+`QO%IfYB48)^j2C3A+ax^_TNcgtULi#w6Dv zx8T-Ip2p$~Sb1FLw($fS8$%P@S7tlLtk)lb-A>=ZZ>Jo&u^Gh~xf1#R(|AGulg7hx zGm0cdUMMp25E!l1HW8F@9xuo+gW8yZLO1ph8x!p6w<}-bP=k0R(@F(Ap8rR>{;ypD zltE=(H@&Kn&TU&q3l}$=Gnj%SH`|H|PR}~?)D~Se2z4|&)BA_Q$u@Z%6nYH!^Q{Sj z%Ff5BCcx((5+(-f8&U#Qb9{x`EE+u{b7zMZ)SzK~x$?C4SfS@J2gT5HZGHCgd$#@t zE(aApd(4a5(a}1K<*9^q zjM24`tV-wV9`k-`ZmT>(6Er^Sx+0^!F?8`$F5(G!YbNYKm0E6+PHNLXMUVfGCa~ZU zwVjLEV0y!B1zm+No4=V~n}e2C&aUZgfoxwaaD28T9@DlubL%O)Q^sQNeb1uCR}Igo z)Y*4PXW@Nvv+BAT4C1Flnn7_4oV<;nWDqk@*5aM)jU>)xu^;Ww!`G`pNVb&&87&*tv;8N&QQy8f^y7|ZK5IQiY*WOo{yQ4oRvRE;=SrI4uKfhS zwHib+X#bTkcy@`3w}VvNa>cUZ%dMR&e;3^O!xLr~!Hk6(-7VHM)2NQ=gs{oJdqIqe zGa!MX5~$9SjBM@tF;}HXOCT@rpD(?L8#Hrg+lgq0>(!)VHC8#L1I8D zC^8}<67@rLf_dAQcGe3_+#bd34GUKZdV^JJoJ}BGe@1BzgOOWJ1^C=|Ug{8bA$wJp z?WR;XVzM)xPia8L16tpGrH)MqfL z+Zg}?;6IBhg8X=ahsuj-2A@P~dY%9ex_hdqOM-g4;y9lgQlve9HUdOUC{SyuO-J(@ z?IA$_8&oKWeFLccfH!zsetYqU%o5(7D={ft;k?B#@O3S}fv@P;u7)pZlvxD#0P(vYp7Z1v`v=V$jOk=+S_GO#bz}qC7t20va zg9oO=E=&Gxg{;!X-MqWqkQRJWtjwUvMx}rD3E?CS#W55oSe4lY@?3pm7>q7kkcW}} zv(?D=huy+FesR;PEwE+p2_b&v6i4={^7{c zS+pl&wyou4eyq3y*5MX$CcC_; zvMQSuRJa+)B{u?|4*cKv-WgJ<-}nSNKjhksS;#1Rl+(QJS1T$XW6<`-!yfGuPZa^3XrCFZVZDrLuCxp0y%Z z7lN()+{LYZh`R=!M-PC$wOhghW{ARlPXZ{u%en)@B}WNZd1BZ?sHCf^3M?eUVAS@D ztsie6VwJdnOi4nq&Y-hxT{B~;k0;NM+v}lO-Zlj%fs+$q`ct1DbL1pWL0l}p_1PRb zczR9Lig*_9!P>4^o)UY278fSTrU$81IBwqA%;GMT)7U#6JXFwOdP@A85aw)~RP-hS zxsc|w*21^nK&!U8h}U=>_Hq+6%OY7z98#k?a{9Fflrxhww?@C*u!7E$d*Q1$CvrBR zN~&Ye-;6Ym4sT8-8s)W_3;BDE*-*EMCR&ex(b_4Hn)-l^$p;7A8KGF2mDdn-eeNNe zf)paQYnSf5$9O(+J|`Q38JMtEo4a?FGuYs(IaF(r)d zBmU%=kLHF?FLs%qR%*BV&-Y&1S?D@dJB(*B-erX@T_?r7bIx4IU`fATduS{#ewl1Z z;ay&u;NX;N=(Bjejuje2|3h?}`uargeFdJlh0!~h0> z^MjZ=1hsn0#HJqum6jWMEAQQ`^wUry&TfCWZ6`EgU@P_W<=l|_Wtjt-3J3V-ozxlU zVhHO^GDR)dN2;@3Z*eargH)cp8q`ObQ@q9mp*7AK!+CN({3d1uin*m+W#f*7g;&Bp z1wFr7JI-G@eAQKeFPD;>7H@WjE?s0~o?sG@VI)KhSG_O$VmgeZ%n&dc4o#tlBENL-K16*_Z8W(+vfCY^xvngwD3l z0xzR}S%xId1%|*C_&xKL%$3ME68Jt+-gkrc+;A%(nVm_2|bzubYv zs*bJ&ie94=9bDMmmA%C;4X-<{Wb3lug&Lg^S1?>IH4ZHe-3_o-NcY+K^?u(`x!`Vr zh{_%Uea5U*V7mQOV1~a@iL-V2$GYF{r(N+%Xs6c<;R12iX@`i#LK!}yZ~1WlsYZ~? ziC!MiPfs}Hphig+iz)#VYBPXNjw@*Y!GU{2DU1G~z=@ET7V}l=h@Eb_lejZuLJ(SP zb?Rcp!Yfjal0_F;^6nwljU3zQhcZ2mpgmN3<^?kh&={`mTJh%mVL8Fn@o-i-)A-w({z?T%JE)JM~gQSSbVA<7mt=-KUMCvbYN-Gl<)eZ(vJ zXV2X|R_%s2)=ajV#s-N-$4NXQtBduTAB_-;*FpCrbBaDj^>Md57PBtpW!OZ{)VRd) z*v#a@OUx}_BQ>co^~Zmei99O%fWq!%siyk@vBc(e)ew!C2c0x zi}mjptX|jY*SR;Qe1G*v_up}qK-6w?r^$Xt_B=wHaL}!gvq`%1kqQrpyGY{-?T6Y9 z4EjSij0ikE@$KX4v?U56V{By&$f+Dqg;@6{qpq7*hvu8$);^P3p;!OfV;4Jz!?^U3OegE^w<LLsW6K>zpbS^+869feZS}=7^s5jO zy2#fe+t$D%z`b2z)udqXJYgv)D@sE|uI?sg-J_(Ua`RJ&`Fp>l{f4%^_a@$vdOI0a z5N;Px*hlgm>r9UK#opVQClEECA3K7`plhQER+aSXSy0mbZSx|;mkEz%WQ6+(XUX-K z9Wod$aDw$>1wZ6!W&irSyPMl3`?ZBgiAUq}AMM67$mXx}?lB^}Chkk{D~`2ZvBT`2 zVr_>>Fc(E^1?>;nKNUIOFC7fQ;c@T;k${sOx@J~|^qDW(=&X_eJU^zpf3qdNX7_2& zhc^zqxhF5*SU_4&?Gx9n5^5+k1DPokyLW*uAIze#Z-Y_qF?x2t?I%TvYxv889bMYy zXo~dR{r!yH{oORd87H&j`RLLa#Kzil&dgx31fPDgn{EY9T8L!qr^$ln8wdR{8xb$% zn)ZZ!VXZ%Nyu=nCpl?ofwo#;-(e6DaWi7&ygg&_uQakakU`jBiXRE{QQRXt}W~u8V zeiBV4KEo*C6^ts`pRA3fnwH1DzzOPcT{T&hKJi9Kv6~)hpS7NhQRPnWA0denoo@># zlhECP*abs%^Q%<$a$}y_RZ4M7ib#0(e%DxiLABu9;*7P25DCZiC-)&-3WS`?tmZDi`)@=}##9Sj zD*ESqvZdO({Jy&N2&l!akSZ>6-lS&CX(dWFZdfd|N>&)G(s?(h=1RwaS@$RR@?QaMX{n_xLDGpo1og`WQ74Lt+kv#epn_&{d!6!2nP@`GR{QVBV47o#29I zM3macJ>iMNP(4_aT<+HaxhG*A0aLT-h%RoowV)||DtyxxDSFiV26k>cgi~`dga&S* zYG3Z6ap2RBA^TC!s2+HGEPzY!!Y&2$jjW7A={S6ZX26>oOq13RnH1R_;b67Jw;>66 z4U3`4po7XxC}MvitWo4<$w1UKwD zIBa>G)<29KpyGhaimAd;Xt@Xs8iu`&)9&eNb^=E$d2lzBzI@6acayl)mbdY>5XXhl z7Ue_rp5ZA}UY7E(>Ht??oTHhm{7LQzBR{#;K-|xqO)TT=!O4aop30HE309sUVv6WC z$}-hC8$YuomXRmDKzAf@Fde|?-e%1i_-tt~KrJfGIr~_7zKZpjp7oh)INV_Jdkpd~j*O^fO)hf^>8co%fqe&e?{_b7tWN&~ z*Id=vE1yCS5cgKYx#tQJ?>2j(QLL|rv6%iA5I4P|p?crVIFKUyij8kg&W}CPDWxY^ zLtIY3Oq){-r#rpTM!Pw%5cowdb*j%L%%Zw})fH{Tz=G9|l(Zl`XsDiKB@jCAIt(g7 z@U3TV?M*`Xl?2Se?eIA1^huGHUPau_*UuOMh4AM?>d-%kFL$wLSKwbp3~>#q(gQR= z$J8AZ9N<6|PoPf%7nOz3(7@5-u8KDr$7915^WSx?z#&fzn(xOKhVN1GAJ*F`XrJjU zRzLI$0yD#fUDEcaEcm)Rx3UnB*5b`c$Y>60dc9(g7S`tm4{7HBQO7S$SV}=>6v}Mv z2brY9g)xxEgS}xffx+3l7d_!=KiS%)1$(gjPP_zWel)_ZY$iO(1^fno2~Zs-4IjdN zee9r2W$u-%WM=tFKM)J~c{7L=c73u2ilLrf%L)){>*2ip3YN}sW2Z({qFq_#aL+EQ zfTrz82{OBjD&Xo?!}LyGoIIgi*=IRkqV%Os%YJu!Uhm?-_JZTkGlYOtK(WBH1vZmu zaq;z`jc2(S7(_O_v{gtOp)MCV#Zqh>q7+l8z^JZ{n;HhJYB#cJmyxttxM)LDNG4)c zTHw)Xd$B@)s`2xD!3<72gL6+X#mvcvO8U)|a(&Wa|B=K2k7tiNY?1i`f_vdPh=TI; z?BVV*JMY@*?S4yhB~dgZprQ~r@U|}Mc6N?#!kCfs;-7C7>lUgG!L9g&TnY^L+eIy4 zBct#*NzwmEIQ=moyW9yd!q+OKbK^m0JXl;Tf>kb_)5fZmzuhdVJDfJK@iM907)Z&} z?pE4PYt(5OEm)eDlT>}rYnaC4#v?61a}2-Y-r2#9ANTzqF3p^*84L?ig-YL7SU~0EwXv9$5hmrv`U#nXkgF_wQbYb#ecTQqQSJA zoQq^PSu~t9-QA+|%})+Cd&<6In58N=C!?3?<(&?7OV4|UZw4d6Vipx)u5`au)6|DD zrrf0C)r>8kMgLSyQ8H_t@MjS*x_nWrq*DVs}J_m7Vxbcex1G)ryL(VhET$^$3lg zzn_E@KYD3~TauOS6lk|iV?2OH!OYxtK1&d!2=hZD!vjtXG9EYicuRpVVFzrr8(Hrz zu&)a#%_~#N-pWNL48UneMMTR0l{W=Yd5r|LK4=f604k5q=C#bNTl$V$I9;Kh4Td$3 zANlELRM}6I=yIRzzuece|3jxl=zX6)lv|a%6j&*jzX2nm8N0*H1=td<`52OR z79%67LLBYYQF0uoL6KKRIkQ3;aN5KQ=R$)taJ-b!sI0O2-cg~Q^_ zyA4Hnd(02IyH{vArO&4gkZ)q-f=;0@gSeasP4tNL z&q4iJ{UK*Mj`;T$38H(eacR0LW|_Lb_*U0FC`bI3I=Bm*PlNp|4jq`B7ac8Mo)88+ zv9ARpz{H>wFiroDy|<2va%=yGkANZ|sE8oa-CY6_BHfbGNH<9L5F*{(NH@|wAV_x& z-OT_aF(CcB(c?Lu=lOomZ>@K&_pkT;XVx(HF!#Q%z4w)$>$CsxoG)ME4}_HF<8bgr}i;bMtGQnwy-D$jBTov8Kx23BMNgRGyfim7B2t zxcWG2p0!vZJEI!F2Dg33MsIRc7BwE}9*&?b+#hL+!M&FmXD$PM9BOgKSmVWXw|YU92k00 zjpO+|fA6Mh5$AO&i5k<^){+;0O5N*;qdq0_XVjOxZlsFSl zhfrH-vzR8~_1L4(ba@Kwjci7gz_^&A2F}Ze$T+#_`iUj{D!IL9XrV-naJxbRq4i3( zu-$s`SkW6)E;k{>_5GaG^J$uaZ@ivbmL?Ifg)VrW9Arwu=f!;kWdIzZwoc9hjP_>* z`G2auco(C^C>6oG-#?uZfVp0DV>4Lq$CjCvK?F`R%tEZ|K@hl)BO%7Tt`Nf_B%a=y zBFcdT6CfW$z^2xzQOlFFW?aHlsL}&8ka&6Z3;4fdLc0qEhnD$vxKkps%d}woI$Q%K zYM3jx=!M%MA|vrH3lbUtC2LNk1%aC*-C}V1$!tKX3s&RVbZEQ_N}#vkGFrmwa?|V8 znr|Z1KP*tIG?KGUVj$uBW=fmC`>owd%*K&_&THn=&lfI}RxcZewTFgE(GD^Z;2PSI zH@hefgl4L<3tBT1qnUA9mOI=RrJdo{sxtk1CYj|ggl;9){qeSGVw&3SQ}&lIQom8m z7({QRpB*A>Sx-=KY>!8+JRMip^PiIk15{G|2h1zObpz&xJ@~h`2B3{iWos!34>ZYb zpq;V0rv6=;N+v-;w@Ml?oQkMZS54+z3o(9)E18o=$(cKvl8PDG?%*AjVwCG_TtD3( zUsJ;bxhh_?;gw(^!8B-SE`a(@`tnzpLUv$2^0?=&N6Y`nM$tWZpD^rF?fm5bLjo_J3K? zw-Xl!ARa0xBzXQF(Z9VD^ah%jvs>{3R{p)R{KFf)eVNAmQ$#C7NTn4{}?^S!tFuLbdXyEXR3*aQj@z>Qqm)6fe zoyGnv?6lBD(*k9kAQ95U0DuADH+9g&&{|^LG!Guf`2Ewr=HekrjCuRfJnlqZ`Gw0w zxhy#u8_;z1_f7sB%>Oz7$&6x0-v+#~5{2lrMfqea-L`+qSA+yWlV+M2(FOw6uu_Vk z`vd(WY76ht>E%NfLe$1j)C9oNT*ADwpsG14k2UuAPNy$fzk!ceFjDX6f^v4bo4;9~ zrV*Ak5uh;e+$XxYNn zm}Pnc!~TpCQ-E=auhgf;xzTOkPss*sgwJ3`)`oLQzjA~&EEFs0!IOSKzCx@s+^?0c z)A#EpJ-=}xS(bB`FOBY%<^x8Ka1W^xsF}e&?V-}vB(wP=5vM1Aa%4RRr3XaPe!Onm z3{1H9kO*70QA_|v86U;=_oQ*-{7E9TbW0m3{YNzTgkP-Jn5zZssPj3+yF&ene)?K0 zZnLDVGmVTet^Dd;gZ??%m0<%vrd#T@5GZi(;TXY~&KA=cJaP(pjB_q^n_Wu)t|XR* zf7Q-w83uyucKa@E7!%ymm<&XPTR+aXzKEX$G~2dez^mnOIb(|^`3i`Rqzb{-4*aTD zIY0spG;_skWFXZ2trdD^*!{}p=ej|aY~4QH_9?u4fYu&H+DWBN2$0U{EbTLKn!!rf zlddF=((p#AIy?DQ*7+QsUzRk-O=gj-h6mvM#YgolzJjHBx=D=Bdxhg?>+UPG+?4k@ zlAf=)qh%0`ZN7BucnqrhUHeGI@t=MJw{40Ce^IJ}iG?6{BCdWBYCTtC>3@Rec`+!y zo(C)g30|-|9*PE!b>P8iFnzO}m_ju_^+(4g7+;U2Tq70G$}PQ#SqIkbc?s=K`P)8q zC$<*13Pq>n$hHAbps}RaMLA&&jVjLy2(#8P;!U+D<4M(NSSQ6tu)^HBS)gQC&N)mwrMrq6S8bivX0RfBW$7%ZCi|rJ)zhGfZhs^(R0VoRsN;=3WGopj~+x1yGa?$+hn5S4F%-%ChDMrEIJFb`Ji2r1l@SZ~hfM%%|&$;6f(L_=JkRuW+=J&~P5Wxchp2#=sv)-wLLqH>7 zfZ!d#b29n*>AglP78cOh3HT>yfPZ3eCzAXnS*!CmX+?#{NVb$eJe8SteeOOVpf@E$ z5Gmza6Ud&UW~q9Ey~TL_q)= zV#9tk!~zM;Rp36(P2L6Cq6i`WA$9jd1S3&na7>#EI1xIimX0?O$~INq`2wE*FfGm! zKeohltaAY?z`x6c@`GbI72p_-C!4V45fGylp}hZszV(Ievy>mQ3r7R6iU;>{U2%cl zU)B}UTkIY%HEGy3=XTzcM@RO?pKQ|}*FiwqM02zgsSn3Z-z6FsiikX-7WWQZzH_!% z|7mn~UrB4P&4*s!;uG+2VFR;!T|gtU8=8XWzpac`MTD#d$FstZN|}W_X0e;_+2S8W zO`(rX7c!p@42=r;Et|)~2a#Len3|f_t#{^(@>WlM*W)akVEF9G?t0)>Sh$EuuYB4_ z=2}51g&tAqx{=$mR$PZm{?hd-sr24Gp}M|7fW^r#cC6Na-WeAwG(!#RnF}mCj%g8F z`XHZWgA@BQp-*o;n4ZRpPtgEf*5QBNw?7xZw-5G@=GFa?6*uhdmUHAIjqx`vggOQ> zh6j8*O%cr%O=HzxzOy>k??9pL0-Xn|UF<1+GqRdvMNIp{J2_I-?3b0EUBNscOZNEs zLE^nvR}OY20NSh%1&1D!r7$V$nicUYy>N3WOrTcV4qIuPP#=FzKV+bFXr`rsS!Tv@ zRVJD~-i}sY`;=&6{#ow|-Qsy9s)J)UjZh@{pim^KsY&6#f?Ml^TBi)NXf=noPwBcn zLP-r-xNf?I;(p@MD66a33l^kd`_uKaG6vr>9b;5_Cf((mJzFl9)6;wh)U~UH7F!aU zMVO5ojLu)WANbEe@FWd4D_eNS*6@SVgRfWKE;Fz;d~X&QRlIN`Ro%gR?&IOO?lVie zS&XsUSt%K8CyN@h-B;o-gzLzcL!@O?5DmhnB)6VE*T6W`r4y7x@iSr%jP zbZ8T{dUS>HZi6p9JSB7;@;E+?xvw!LUAdgJedooN#!zt4 zDaD!WqSs^%QwycRWMpOwaa^(s&PSn&sQA>UnlT){Y zjJm7F#|o=vnN!btahToS#B8WTD3F9Nk; zxZ|lJc*;@nxCJpU!nz7q9EWF>K1N zYn(0NQ)bCSI{0M~&oKMd>XNC&dc} z!?x-NS?`S+qB&y6)#p za@5)Q)mQR)HDi8o2{tYDRrxMiO20*C(-9loMdY}B=;PgeVdy}P_n^q z-|GU_@S}*-;f+opsc+Ln1~0)P)PpZT$MjrHEJYVrIdln-1gY z7|F1m#LfdK+n#BY4PRa7H5!?S3NLiXF9mO#$h2bSjS`6Qg-&;xM`|L4igYOoA6D-n&ysX~I1x&71x=$W&BIO_(nD@$`I9tUGA zT6wxO%ew9GtSaTij;J-OROh++9Mun*Z(cpU*5|Oyb2vNX=7>ukPB6HjAF%M} z-=*MXZA*P1gFoTvex+d&zehbB#1ak8rh)Mtw_%_NAiOgEHpzDljkB{zYP_W#=1UZ!}2bq{w0CQ&D(msI8*;K{CcbTSC&LBqSKIRx|=;0&}YHD?V=ft zT@6{X+A{}3_oz=5Dxjxq4UesFE<-u#-hc3zO0V?iGbNC2eSxH6%~g=UC6y($j6@kw zr&Wqp##PkX*ehNdrgIe%R0(k{`x3NKIWyg)ed3?hY^yc7kCr}ISOcXj=O)U=EIPPpc^=Seqr_^E+hk>5M`yCwoPWN?gqX!jmLpra6tN*{Xw~XcP97!t()Ib z*lzJI*7yh)lO_6TOqdJ^jlJJtIAg*&dVe-or=9)%9LEfwd3H#*)zPGa<>HQM*8t9@ z179xx;vF%?K9h)mR7wU#?!M;8*N{kOouOTX&AubO`Q|T43SaG~F+L|THK#wkGLYVE zrydjgSDj+zEz%1f=-n+I4ib4vTT(FDFmcw~yoWI#8K2b}{_bj`u56pZysLb9p$;6H zcF|#Yz0!n7Lw9U=G`;@riFgE<*#fv5TpyV}2zP(B8{Ge0>4O}`6K)Jv%WFls zh|U+5VdKdwZm?%|v=>rE11xM5ANfbA(!SrN%iVTrsm+z5R?ln1@i)>a&g|;%c5)WI zK7AKIpD;fe%szeX7bHF8nKJ3{p)-cxS@i>(xBsd;9~!-$w#CD(-ei@pkftrzCDhjDB-KZj%C%tTTy z_lCP$*ig4&;ZO!Q3g|)*Dy^ARR~Q%sbRaJ=N&FhuY+VMCgde=6iSs;xs9lwueLs)S zePGV{%uOZX(W9JmF$p8yOR~t-rNFhfCe_m}qZFqnK3>=#trxdO$|w6O<^a@JEvM+HAbECbE4&i+=ET|y9t2JtXlV9^l2g@8`W4!Y`RC*O22UxZF1>IY4EC)8BM-{QC3wP49l z-;?sKKy?cFq38|FISoAahSJCv$H^FtpK7<`4pepn$3I7`9#pe^j$rpFZ%y*N4HNO$zrTTU%`m3pT9@#iTS6Lh$K%0Zp1fL$+teM$o2K8t% zfmdGMU>s|X7AWQwr#ouB4>xT!S*fd8+0)UCm7yJ5D~oq73Wvk#7hH?o5w*_u3-78| zo5h|QfAQJddEn9WKvg93Xxj4Gcim$SdXq#Z-aA8fT3&*Y_540nME~ZkB;4_iIkw9h%r7OL6okNwuQabWudJ|U zF4y%w6^3q>)ZjUS%`I4Wfa9n;p?Jvw=EK0aNEGzm?n zJE0!Xw|IB~Z0le4 z>9~}Zgl8syp&B_W_B;WfCEajXv?J0{&bl2A2?3kYXgOn;{NHr6?#cZ+cpF4{=6aV! zYiB=}z?fxjGUxS0dS#MYHq%{CR2W}UuE64$Nh)l(p%|S9GOfQ(6xUcQ#blwA(Pk`W z?4BT!-mZB=24A>V2ee`iNhCql(lYj%d>=q_qrIUkEmIZkwYneFVOs#D;S9N ztZ8R-IY=y~MK4hS=X;!-xK@S2(-KJDw{>2)aUzX=QjGRz#XKkhLgGrn9G{jLPM8@(g*<5XQaDtUgu3|GnIzyFYVU8tS}8;4WqU?eWwJc%iZxdTY7KJQi42%I$EL_=rXOyb z)H}?nmAI)G(9oItsT4a);~D6DBK6aoi)BNvcSh!xs)8r; z%R4YWz3(Ly7;@0EG^Vp0shvdn7@v?4=3ExvI)ig>KVff4ignE(rSbguvU_uJ*8*(VPbQndhaBo~Go= z3`{yyl(TlfWqrl*1?8#EP4;?+n*&^0QTzVJA|0i%SazX?tiip4rC7@JN7;**<=Xzi za?U_|s15ts{Hw7#4PS*Y^nO$xOBgv%dckSb(P#H^h4q>1YKumSmsE+ZK97&YEBHqU zkfMZ^6pmEZVJ5Lz2R>mi#)K1oiUzPL?9o%;oG*kqF*YyRSipx6#6ZQrelJ+>ZQeRXkC-~sutf;lrIv}eHgVgJXeiTEEv)af zqJGUi{e{hEYT75we}2dFv=I0EgZV8kMzvaV7c70ENjU3V&RzkXq0CgqD0Y09MIV~k zU<~Z*bPJ8O7}j0c4cEZT?_T859(-ujw27Sa8=k%2=H=EcVTCSIgT?!)SI;B!hZK!D zL_F~uTa7v*^SC2%qieEiou!spBypNMXhr`bK?)=kM`KnZquYW2cjmSq^0!La ze{5y1R(EXi9+ip6)j~;9U#CIv3QWdjmT=uKnW7`z2OSBME&}MA-P@bR)S#&z2>8RB zl&wuR=ih$t_IAvscPcTX)Yv;Oo_#A0V(%lOgSd;ygmygFtkVn=D^Zg+a!ED?0377$ ztwa<$@mr4(dPu&#+7xi*wOv-#LfMwjgi49$ri}Kwa%*6Fm7Yf#O~543!=7|ZQGj#}cWQy380)@Z znU}Qp2;6>JO&sy|&VJTcS4BMF(IR#cF10029_NC}@7oA`%@k2b+Q|E(?8%Yc+&26Y zOZ@mMZ5KW;R1~01l`7b4Te4Wgx7{G(4>XPv74Q24`#0i#u|tkgar`mW<@K6r$)aJc8hFtaVFm9_+5@HUc5)NLDh1le(dX)>e`Fa#78^P4{fU)ia&V5h} z^ltn&gyrx~x8@H_p;?50J5z`8E57_+57s3M0o3C= z`4>O%%>Sup|B_h*K0;WRV}Y5Q_V>Y;gxB9bi1py4-N(^vL;E1m3-^(7_WmZ;uF0f@ zYu%^TdC0c>rBmQsY&CrY;5z+H-v9mcc5iiUOfb08a({Tz!kEvVhX5GYCn$h_9rdn^PxSx&#MYi2P_31Zm`FX)fH1*HA z-e)wBz_exV;gtgOl~{eZCXdCLIc^8P!OWUq$uLpctw>qR8^&1aS(UgfRf`}-?l#s3pih;+0)WV7TbtM;7w$^{hvR+DYamM`lCve0&6W-Y=Szv)qh?8gt(Q->EJ1@wGQvFSXB-3W2?1YWv1d*MqpRHra}OgeUjXOH z^8=%*!+ew=fDn8HDKdj;Tha~6iswDKI*$Sz0G}7Wu}j>(-<5935Xc3+CeOY{8n<7h zAJnF${Mvi1#W>w>)<+~cUXBv0&Oe+0=e_MntOSp+Kk{~=94?CEyKwhl6evWm)a8tH z;rzoM=by??Hx9MVt#m8(A*gJ8T&c+KwZe{Z6|2)(olZpn*CM@PLX8pnF?=iQnq)ra zeGw~8o{R<`)-6Tb2Owa_q;2E>rHz<75T@}Q6V*hX_k#$|-;Jp~g=w*MaL5BRchaI6 zrxwA5T42nXLc-^#p`|_JueJZv%ic(8W8vgN7+Loc8IAc$gE)zrtAj zn5>K_s{bh9jaQDb9ugV%uaQ2g6qF2etd4c7bl7#1G``Sd^nL*B97}{|*X;r`o$VaU znS}OQr`*sapa0O78!4>=XL5yARhqG}>%FxapxHK*7K6vu{Eb2Nhwj^cCAXU$+rs(P zz6ASu4ZB-soCFrDpPJhsf6+KaEi^KShntsYja7hPBM)xlOWR90!dDB@NL{seF4WBZ zEcGrpiEi(~tTP5Fvjk9zNhBwp{xtw^6C_*0`K&`K4Cv=X<+TE$RMiXmi;sfr9>nMf zJ=DWa*A&VL*89$c42!eWZNl!@m%)D-nx7`ZON=pyX07uH&`r+Z<)K3}d{Eh3A>u&^t$JQ)xW#zV+=#9ht}5;p zgr)Lp&!^3q*vRxU&v+@d4mmSb!$j0ds=5|`w~1@XV0CS7ib=Q8(HsR)2yKa-#dxPb zgqm%Qm$ba`0dzD_#o@9q13QU0vJ_|F0bQE^kqF-`{XD%6`#^?E1MGlGIRf|mO>qxs z=V|l7GLb=7LN~d#?I~9k*rvX$Sp24QhjlJXe~3VFNcUq<4S(_YQQ3~*Sg>o}4bWGo zQCJ2$^ERs~dGmS8>w2=Rgr8eoiSQBminWS*4aBjT;4QF%m_bUXcSk-I?qX=)%%H4~ zePd0O`m|sp-b;hwVX2ZIpNz2JCNEEl*;MW8SW8BNsVE#9BO^}a207ECAcqZe6-YZ$ zpfgb^YBi@){`Fd8A#G3j&^^A!vXo)SEJ2pbl`-#{1&22k9LK(#_FJ&VL0z50=$l+K z(DaF@f1#886z{_3Oq-ZYhWJ)Y8|6&{Rp%O0VT;?GU}T-9m3#hcUfO=4sf{;pu*W%v z!ixHvI<@V0I=$h`EiYG_@{1_7ocD%TfQg#_zT~i%|E>R0J^QpFUG;V94J~tMEKf>_ z*^T`t+;*X&L)>QqPVs4u05W0e6X%{?LA$1Y*!kGGgz~|&p@pLX^_m%1qTXESmb`gt zGug!A-u?SnxTDENyC*KPn^KcNzj*>(kArl00(+X+x`aQ(t@b;z_txEO>htXEzag** z#exePd`2pdhuZsT4_60!u*CqtVt`_+$9C7uyXM8=D%f&__0U%}>giwFvYt(INxcD*&R-Z;lg0CVDLpdo`CN8J z^9wEMNHy>EwO@#j9)W!`0Plem>Xrn4tLs$w3a@s5@@ya6sz7QX2kF3q>LBN~6Opgv zo5qa2sO5l!)woEbVRCpjyK)kXleaW!=kZNW;pWkn;Pw0o4AI!yAFoC95vJ9U-8o!G zGpk@zEAVCVUZ1fYZ?KYy7~iHt{9x#~IL$Ja?~6~?bx4?M$$`tp`u1u{@<7OgwCN|BGmGcuu~VD$2S);hM*}JE zPUyn5A>J}sWymgf&5yTo2VD1jhcpWI@|-mlVie!5q#NpY_~#T_aWDaC)fr6%RwGK3 z#@vxjtaWVbs4|E4bKmy0 z2SPA@5@~GUZ>4!=J#ejNWhM(QGnRBWwtaZ1DJ4KNPUg=ys!zlX0#AP+;&479bcJVf zo*65!n9k!IyZbWfLaGY9r#fN6S4SLE0-T|5XM@klFgpR2k7b|@Q1XH&3Nap#e>`S zI$^Bp>gx+szS?9z5F*gyTZG8S2O~N|m%wX(Xg>Tp$D`LU?s__pwd<<4_icu;oL0Hq z2amLr;J`*xGhB4ca&z%MC0Y3olB8YMf_dq+IQYGj`Z~>0h*pbyYxUvhKanMN1GmVM zyE~9+NW02Iv#LH+%mV`PvDa=VQ_Xl6DW2huN=XjJ{aIP(R1KKSk+QXcj1{ZFj{NSh zsV4ka#Hc_TAybePbvhR(Hrx&y|0Q#cqvL5_aBz#&b^KyZ{>G3xkoiPnaseXHNxd#b( zq837cZA%@!?vO=p@o>C{wH)nS9n;`!_A1#`gGJo5X(5-6E8XE;`YAu9;zq5Dp_7?L z#cR#;C9RRz#LowKqt)TNl=W4d=8^SI?xzu({OT1Zkt+P7x)e4$zTutVKFZK_OGjo?RO*uPd zS~%AOD_=JZt&X>vWYv*RVSY(V_Uml&rTe-YNc3VOZ3I{3{1`$*r)O9tv0TfuFnr^P zoWgp*aFSWk;a7V%B|(>MS_`2;-~|?z`V*VZuaDqjNoW*W^+t)Qd&)yw&KDx+S|A-U z^F3XHg7Y$v5%y_nm{*w51-*7wgX^Mu zVu>8Syy{xZL~f2Ns)(*9>Vqw!6MTOo(sEE z7(8HuG~%sysp`Y;@deb@FAIvUxY%tEG8tV#=M#DZ^p72iSTkjDQ*7qkZ*d%SF4^|i zay!;z10>PuQ+Lp;wNKQUgonpPRJ_ngKB?(8W$-86EQ4Q9 z1+s~}1}9s? z3Zbc3#7F(%cILtU`Nd|57n6StwjnZBCnZ8TkeI?u#`){SQg50k)btl%;L#Afx^yP( zdTDcrkv7U(ZB7NY9-FI8%^(_*N*fUBJh1<}YeqK=ML13QD{pdlLYc`>`TmF|Qm%4q zi4I}y20gW<;_5yogWSV}l6OlUsWnDa0S&l=ldEWto^k2i+T#J0{51 zUB-Tm*10WE=_Z74?WGIz|^wRkC+QbIoK zjzuhQz#qor@#lfB-qR!zYQ;q3=SEMtFS}o9;T|D>+clvq5hUhD`UpSznCGF@Q#KLZ z@gb1Bt8NtS@9^r3^pU5^W1}aDOsjGNRAjmfHaoRPAhZ5aocR`K&$fq!dvkRA%|kOy zg^My+#Ja|;Ln9kfw;&5;0Az7|H%6E9hf#S7oB&;#fg&=xACWa4Kc`%)1K>|9XWS4G zy{*AAvB4biJ#Iu7ad*qh0xwZ=SaZI%p;gc7d%kz4`i*X)X(e6dWYhQYFyc_`LRN* zYDoQ)>E@~qN_4{O`@&3u8REw(e>;o0nz{pFFgMkbnCJH2Hs=+!`R|f?dCBS4U2NLE zwWgjJ7ciyVU;E3kx!ojoQo6gvq=0^{6^(!l#QDt8cTl4zVdfwUx#dUmatYm+G;k_?;<^UIuosUv;gEUox`xnx753Lyh{h-62 zcZ+_|mBkv%WxqO069vse3@wMt2MN#k%YH+CFd-c~m}x4ES;J$uXYybIZjyW+9=6Hp zVE$TVkYH@u!1A5?d8;EfUCuJ->Z$wY1@g5F;S{F0Qeo2+?kubLGMC@ic^uW)+{6vfFgR+R z&dM&9z4;wd1wO&7-e!~$?mRGaTIi%IJ_UK4dAfr&5Gk+Da898As7L>k^?LN!NVUQRWd-eq(QyY()wB3e8GHg z!o&^b9jWJY&fhe4!fS*tl|ewrjQ^Y`iKiU_bTYtqjjdGH3>=bFGFo^yxN7vxo&e#1 zsN~6O9EC9IQ_v|} z4rBuLUJti{DK()nG{+tL=JuVBis5t7IZDw^e_}pZE3rxlHuO#%Xfnli9~@O0NPNU__}Cw$N4x*BN(~nmcf&YR0jEU1nnOZT41T2^)6mTm|EcWmJv`|) z!<2WC&E!FQkI(1SYhRykmJLUVfo`x4%HGZW(3Hrxsy=L zG5I}%(I#vg7R>bg9gDRJL>A2ZHM4@_OxDf(cV50HLL?~}qJLwa?6GRU6*xgx)t#-Q zh3ydmSwZkGu8nAh$L~+K#%JsX`)$?%rZX@(EaWf6KYkwzR#=B9NswZ)B=uGx>0!303urMVxne@)c>ygdASx4b` zInIwUFuH=O>+DjZ>D-R2yT}$f`E@&W3w-mWyqe@o*#<^l-!~9-?(?uyFj6|Vo?YWk z#PP;h%gpJiMn&?Rh|qG7HK-9+2z8S3$17ES3xqu`Wk4&5-p2is<!M3TPr>BNe#ojR&m&}3w$d$7gEJ*F zB<3CZe3h_TVsQJQC(BdD?P>F>+Ud)XMRu5)Tx(arS>Wd5N-6cUu#gaHcgyV=M}x1I zA(j%ZgU^)-zFl65do)ivG=sV|S=)D7-WeJ8^r{$PtuE1UVB^TS$}M$ucXJLIjfV>Z zd75|I+#RkP-vi8Ce;`5Xg%$1fyXWL;85LdC*TNULIib}HdD7?I>8`_aq zYTko%oECq`aXXGJkKPehMh$#)A1lDU`d+TyPr8yHas2bU3dg$ZwIz2Qe{lhfDfm)h zd8A{cxhf2OX76NoRXlFO+_Ib`bmNT7l4DKoZhG#{^LJ)5A_ZZ4=2%+U)-hC@gn1r; zX1jN=d@~_ozn4-CoL5q>A~<#UHalZe`nk*RhaEhq#}3e}WsTU>#N4W?k+d}HGxIa-qn^ps^gwrOZEi@02ZU9mdR^yq z2#ZY}Q{(LEdQ4@G)KsrhTyvZ5p$Y?TT#DuSCyr$C97oIU z#Ps{ez`;(x>BygHQr%9Z1s;Xmr+wI_?I(Hgwm?N5x09F?YbMAJYF?SlhSZ0LAL-ZG z=4K0w(-+F}PA!J_!r9+imW^;t3gf1->Nn@C)RO`UO*_ME6RJNIHhH~Z5V)pKT7LRl zz}igS=8ThxHOtLd`kJroX|6^FtT=6E^FmwnY(%Sfxh#DfJJ0>U$MuKzeT2dGi?x^d zwxc8YPEbg=+VkyB}H`5 z8cN1?6^zL6Nv#2X3?O1ZuX&oE58Cb}`c?*#UpT}KOZoI}H2rbfoY6O;#R!A5LP~r(u!4F^vfnfUc#yVdEXvVzPioxugR+)v!K8 zL^*e79)Ziy5&DrazrICbA*Cx#U7Cm{YJVc1)Ad_Feg3?D;Gu_ zX%PN z*qV$Np2SJ-o$K-2oWrG44-#ZW%g#2YE&KfS;)#O*&Zb%4$y|Q)(By}v!7hQrern7X z0`A8ND&ECRo;@1Z0G8ya3Zjn#ph$0F?^XPdHW ziK3m2N~A#x(bi)BJlT`Uxvb4XsC&=;McmGg#<42i<_6J3k7ugP%eZVoT_vxJcjG<` zK#7d06)NnE6GrOQx^@@yN6m_oiM>&()y^^eVaaXMl@cgex?lPr1%3S~LSizg-NNAw zfIpEn=W|>REFCJ1H@<~;Tu>*9+EHFa<*ryNyKeXq6^zb4h%bDYme_NzL)ocf@!>I3 zX&)&`qU&t>(GaC=72F!GQ&{7d3T7h-O*Ejm9b1(D?vXSwv^4Y=^CH9;fEZT<${Bh@F`(Ng@A9XGNF#jp7!s+Yk(?JXK zd;;%!lAnb%83v$`Hf?9qoLGNHhVJjg@XzfmF6r$JCUkIeDyp?6rn%rclSMD8y$EV( zD9z_WC4EW7B#UxiI{pt>iflan`d_(#Xsqsq&wc&0qq^77;fKkCh0d@fi8ygbq$rI) zt7iN)%6#P4Qxf|JxlX@E`6~{bLWq}b%7})Y(hcxG+Heu9-6~Jjz&dlIu)3OWSDP6j zS)WWF>nLnv&XHajwh0d-;K@y67BQHs5IXbIQF$e6lu%i@HS8r7UQ*Z9q(L*0-dI}L z*0IWNmBiOokf;d6{YYl`e3ey`yR)B;>SqvcD=y z{c!@tpH(ys$=NCvB zJnu=PS4Pn^Vs#+2-0I^X_vB87^G<&rr8mpy6#+`lo~Bva{8bwDtJqjHPI_U+Y7@c| zYcd2S#(J*I>Qd6HTuGq}0CEyQ3}rtP<@ps7^w35eZ4t&M3qF2Ao0;cnuf3;uCG*k1 zFH7f0HUoAkRrq<~k=fZo=lhmd>@b_HKilyCmbV1sXXf^OU9e=IDwNWVYw0Xn&er~@ z?trMEAuC%`#8u2`9<%g=SKIEuEAuRneEUhc_)xADZ?{lxTyK){XsF2fTJ5HN-Nb3` z$D(Q>x_LB*tM85jSFqrYW!xUChOjPH&#{Sny#N>UZGM2nx$rNlk4(&81+<}J(g_Ei zo4wWNPNH4;s1yKPj&eAUjN>U-HgQ^*bCWe$ydsqO>OWbDh-vS15nkoHXr$;6ODMiq zt~?7Yq_oZ)_fwe(@l(h0;N(tv?x5?ICeU?rq?%WCbmG8j7cD+W;JQAFZ8?;jIu=xS zRgz{x<~S56nX+?q5VojP>wEYp0R;#UjD%8oer*KrXGpeO`wGCi$ySz~AlM#1n7_-i zMoELv!?d*T-tm7Zd+V^Ox^8_~5Tq0dk&y02=?0bV?oztDTM6mz?gr^DB{wDA-3^

    gUqn{{!`HP)PCj<`qhBnZilvMjkUke?@Q5ZrqezhiulAcg6)^iFF& zuwz~;3i;S>j!-!*f9L56KL4?h8a;FUtcXqx*QY0wXD~n59dqPeMA|mLxq&a-o`duN zvnWds3%@tTyECqvOn7=CmlqtYFXZf;B>ZmHY>HII_l|J!u5@;;i2t`z-0LAzWfj}; z#;Jw^G4G!DkHwxmlroWHNZd@%09y97+o7WmqnA3Ao*MqQg5kjGfG2;Ga3&xP9rKMS__w2P6a2?>wT!13(H+Z&5}N+7FE5?YSXpZuRlKA-&;T!{}NMr@9;a~ zxQMGIik;t$5}4BDH*>s8{UW?i{whL4Z*pS`r)gDtC*24EkI9p8kPbhh>FUT%0q1sg zdP-esWB$x!=1=S>_da%HecaGf(K4BTDv7@Jabak5kCtm!DbKZ4JVK4uwt~4!+FgYw zj}jMEY1sNyS$EdFZ_a3ZXfSPz-I;92x-dCowdQy9YpuX4Ed-HGT=n^==4)4kE6AkZ zF||$Sjf{-%c|NWja+7OAo-~dSIb>{ZmQGI?J;b5(f`C-3ay@bb*n56LQpPtw!i|dO zMcywYyxL8K|&zW$zboMZ1Zwn|65(;;Zyyvw6;j3@9};0oq? zJJTi>A9-F{P(0&i=?3j$yqa#yoa)qZ4Ia+ncx%!FKkN$ZR6f(ku6r}>>~urnizOzQ zG-@oL;4VK93Dujt#9fjE#I=Cx;FG>jJY#f-kp)W@vAKaBSh{m@etpBB8G8*xRN{}F z8Fj*OvaZ>rsvO6XcgBJPn5x0gBv%SH zsNSojn7R{l#~lU?TUVr+5WFsc+NuAjz~tZYmmj@gGtmqrBi}X!R{U^^sC6Y zn}smB{BK<9$3h&xRYl$LPY@?`(LqE)&QgJWKC5w3b?>^Q`q;&;qe8+mUb6#Nk-y_R zt+BhLylo$^Cl7ta7NOy*M!FV(FFf+c{wW+4Y*K#P0+-^Zfc@+|!(O4nPe6=(;r0{A zi$b5dm3O@0hJ_4k8+UXm&J?lozba=up4hAosa`b7^tq#Q`uIfF9V4y=$E|9AazRb( zr=uJi3S}HGw%yp)tRY!3*w}j(=O_5l2?)(}U&1sVsR#NP25JUc3+7eXt%;_@wwo8n zvw7|2F>+3dOdnh90Yhu!RRlis?c@*9M=hsd===)0jCB7Fy6kb!VpO5fiHtwaiqaE7 z9uUJZ-%Pp8G}(Ve^{c3tB>!(wFH_vvM?0l;GAIAB^66fakNoSgbdb9Uuy-LHWsTex zVM&)x*~ldQW*&BGd(D*E3z=T`p>!Y)MM2+l=%5Be)W`R6joi=aYU!lHxrG6;ToO?I zlPshaz`)<%mKiKvs77wQK)HmzHyj!TM{a&!AL!MsdB?#j@j>p<4~^Z;I-1-LN$&Ha zn>H8JgiTa(^H`bcB$*08wIM%_BOW}?aUEhxSx)-SuvijorN{-CE-cx-4 z(YUQ53-^cQt#Mq*@m;)!SWIUsmjdJd_4k=D2GBF;C%adpH?FM&kZ?U1^r&F{ z8PstFl2>^d&b}M$EN>JnzR4lDXG|*Ber91(!%{^DSzEF(aceInsmMY;l+L;-3+Aut zcJ`4f3up;M0GMw1J@4atClmwszW1%X&2hHT>DhUjje%uz$-ZipPtdDr zs5#Xr)_myB@xkOa;KnB8G?=S53HlOVNii_XBLQNp^@<9G+lGRz_touSwv(Ce(BJ|o4--Ka;sYa{LsD|)* zu}Rm~Wks!y_nTd+t>`n96{_VlyWkPGV8@TWJzw$?46`3CPH{F=epp zVS8)tY8ilJ!FvUX^yRJAA=c<9x}mMDuSyDQ>OR+0;f(T6cbn>2#r&yw=Zb^jx9;y8 zIoMf$a?mt!NxHhLy>j6;l`@yxIl8foL4FJ;OoWh0TXEA%OR4DU*XUGC=h|lZ)&D3_ z^EB&+p9;}<)C9xh3*T$WpZUEdij!EzTyt&ApZ>eG489x?ofi}kupvHnfIQP*RMpjX z+MhT_pDOFtVAk#3ghPy_GYAUW{*u_)+ve2hdiKJ#TW{;JGv~AFCaY`~x&%Y1^hBnr z?a{|K_+j9V2&eoV9wWy(oSmgHgje)(FV8P!!x~B18M2a58x$lo_bGn& z<^tF^_A{in?!00c+>66rN~;BjsCX7=m`A-nv^2C6nT^n>6)T6O>-F?=(dC7y@alLfHJ>o@m#gua zg)*|vb$?m+VKNYZGi|xmaFGxH`LS5BW>8I=Xd59JXV_MXblb`4C8;eV19|;P&+?e_ zmn>7G8Fds+JW$IeJC10m(2HgCly$EpG6+Zp(gQ&T1qQUgk`ZvJ@1iU(9glF`R#J<5 z{mvOPwzrVnE%{;4jPs!y&XJ5GmLz!!Q&>26dy+K^$O7O{{pQ0TvMnDBh1YBl# zE=1BX$X?F8p`;(O%0g{#9-}eQ#Opnv0c)@hgk;#~tLHDi++<(-+nVvJlbsrSXAt0< zuJkE&&}j^GRa!ssj6(xLWqQ|n5AnV)k}Uu?rgoflN`1~y_VRD*6e_MIeO)*nf2I;t zJa~%ojc;uVcIdxSOtatP&j}lGJU3Mk5b!xy&K%*VlH0vB^JmZBA5S0QAFF-pN<$p` zZSVbFUGo8qHg>U6P9ceCh2$D{oEXJw`|}B+#6zP)2I9qMwV$R662~A9KUKv zRlf2f^h$1}lc>~ArLhlysUNpyWv!Y-4m-_D<6K5I6OSh7w_kIR7BXoJ9Dhn;SKt2% zvfm1GE{ohUHO{AxE|QB={^VdNwEHVYLNsb@KK@C*pu(t?u*gR4GDNZhdH84hCY$st z){qvlJF`I+Jn{g9ipiupC^j+`2qnc!B2&@}OOy>jdnS*|sjoYCW)Ib>PgUo%((pdvQ%|CY%9Ii!^yFleC;;O3nS5&>s?1_Xi>-?Rm zAqK)o^l%~z7N*fX)jFAs-eh&PtQ&jDf{x%;Sn3}p=xm_MqgFaEZ1Ld>eU0(%i=ohE z8C}<+o%EN%$_M=Z^9=m$@|`Dpn=jqZ-$o^1N;jSJT5t?oRc~H1h~Mzy)T)h@v7Q@o zItrLZH9-i6Fxn-j$&?aNY*Qr^PUOBRkSTt@^FN4xbl59Ipwn4m8P-Jv^w#fK9MfThSdheAL+YMLF? zLeCdmO6L_L3x3PhPaz ztrZS5Y7;{V-2tGsU&R?T7vD#}`9=t#8+T9ZNx=!*FH1W z6!H!+Cj5)cZU<2!hY3X6l^q^=M^{cyH~zpK}H*hXI}%^G8?xT2+1H=@DQII z46#9F^LdQ>QGCcm0>ERUKd@|ekFYK5Kqi(1J5Ue-8;HDM>-JQ51O|wD#`}B05-a?| zjN$JvW75nezVVn)328V;xR(puBClCTyvh{x`i=y^hH@1qmnIIO)qi15PkOT(5$}5hz^=w>R0ugp|EIK{;U=px)gJ zp7>Y6iuWnY=62z^lApAC(-n;+F0plcNN@G`#0NPi4B$3{Zwbt`JarF~_m} zO6)N*EYWYcxU(n3==~|DiaBZE6QJ<(!NC0IFVDOe3pnF>Bbn3>H|iyDb$Bti^p6o29>Yv@vY|=X-@zJQD@aa%ZON5xGt> zj)bQr*`r+`UDq{byifWidd`?BY0m3}LjG&}v?!rvvZ&o1`GI(joUU#VNHpL|P`wCE zrM6Pp^@hjCmn#`rYU>>Wfp*%%nJ`iH)|ohdmZX)&lkTNMakL zawSDT?)57IjLInQ`WcYaE9654_xy0rHhNN87Ua3$<3yC}zDOjS<0AE~w})m41b%lF{3;-~Zpu8ut-%k0vCKod#0( zsHbwn8+w1li^*G_N@nk%y63E|YmTu=zOoeu(T(-1M@`Aa;>bvrNH8{Mp4kQ{P}^>A zq{Xb#cL&(t@Ln>zsqkFfXjD2PP>gv-v!m(5*!`fTjK1LX;bK3Xd2{kqFV|rEy9n29 zJ|bgduKh*H=2T_Y#YHk+uHJNf=&BBY7xNXc-{?vqh=&S|2k1>=4tzgPPd1G#Pb zZ;4Ym?M^f&?O5C^1fxGGTMvo{-w|pk#(i!+Zn@b-wH^WC`Z77_6<$Lz4q7s%M8L7e z*^yDMb#StM+@w>|uM~S)g`l-p8{AoaRAX@^`pVByiI-e!C`YEG(tr+!`T4}1+^>#z znylWbn%f7iouurBmXAUuk7&NOg?3ro?(j7tpC%X(X*}^pL?thqNO_Mnl;8_$&}^Up zuw0W=AB{2Xks z>(B4xzJdwu9p9#sk|VTi)m2=cPW=^Szp9|{y)AlDou$^b0mtf?uwYz%XXc(%rJ?ac zM%95dDmjV)jzhF5)LIi@67*ib$%MJTD7<{04Smjqn@oKG0=pL&8;lCuTj6mz6jJ%! z$v(?){yV|^Y3$~h^j*Q?Jq-B!SFgNh$ei2s{Bip2Yb?eko()a+HYg?~Aay@pfK8Hp zbWF+ko8UV#MM^C=~f7BXJ0{wX}(E zY~@wXKkM^L=SS)$2{TNmg?M8&5n(P$I7zh6G;46ZLLv^=)F>HFW=^)uANppYP$IvZ zmpquAnSC)Gt2vcxy`GpMdJ`+RDxPDTT?FL9Ff>kby^dG399i$iSy-Z>ZdMZT}QDrr5~CM#sN^%by2f|pKE{q&Q-|PRk@iQZe!u&1gq=5E-IxAK^D_7 zV>~Z7y~CXNosPEaZI1aRO{|)R)lc!&ljR93o4l8upxLjzm%1r+wlI}kHhM_I4N3B= zl)a+E;^KPQ%Q$+5F;`C47S5&gQ3X>M?7IwanYhNygvP)O5k&)3;hGse>nYO3_OX8& zG@kX+FUgGiniUOH=|2&cmiXYeQ>&riqLja-Ff7pd#FUl-F4j?}tbavN>+}xUy6pV& z^1<4SKp~%yA@U`Y;=4Tp31MMC!{ z;<@<}<42UJ`R18ko$o?u(CZV#XB4)ZIH11dI=Vns`$!{W{ib8Tl_sNTZA3Nk%nI@6 zmxOTN7(48{21n?zh(G3t^vLyFISXDK6tNWI_%`^<)SFPztTee8b-6KB9oc#$i*;wRJ|^Fc6@xC zvM<+qY;$&g#+Ea%Ihe>cOrCN(vP;j#O^G^{j$K}`()3byHKNswM6{$EPcZ5n_YhK` zjE$*C3{p7Js&ql+L_W&J!qZCVIx4xVTgJg@(P65JCtm`^uS2EwfYwl+`Hb>b0wXE`N>y!<v|G8{6&GVULhN9|u%z)(K1~Kkv=3A{`Mt6;K zu86b)mXCADrf&SiEk9L3BmNsiUzhg0;E1;*UL{+_n5)cPlt+i9&S)~II;S0RlAB*C zpTotmb-Noh+)PEjri_t*0u_H7M(70UNS}2N#>`UUJts%yk91^{rzq$pPU)h8x}s{~ zL%tp3BDwGXj|azCbe+#gh05;}>3CePF++K&&>2&CjQp72HnnA?7F{=}E?#JSo<}B} zC!96ZoBAQL@JG}xNO*#+^4^3VK}WHTIo*?%sYMxN9 zf_4V^cKHV9CzPi?A(_SB2MJH|7mIG!8$p&QA>8b54C>)U>9Bf6Y=$DSX|}qKek!qs z08@AIqeP@q*Qd{DP7(8$dnxw)V^}C zbLBRALG$f)nM|nqzilNXDQHUc?j|Xr(EqOh6JLS01F!p_dBJ6Vh^Ga;p zcBEBJ+o{%#;E7&eXe(0?CkBak*D{NKq%H@wvb?Q8M_KVi8YzY&D8~hEor(%j7FClA zF-}jj!$Q>@vXqxMNd#Je@md(j4k2L?Y=7&jTN)w7x0?z2!gltfW?=?utMON67$6zY z-3Xuix>GAS+U#l2iJ|(b9Q?Qbi+vGGdAfxhWi?H>nxp$Iq!q!6&6FSj)A3sVB*z@` zB)oxF*wKVPY+-*uFuiNP`KL*QKRD9dO}vS=6RfsC++4wz4dxn5KiZLyWSNd5z1~Rf zm4Ip>0Jz9}gedF5@vuVZRV8$v&yzrN{7JzUb1J}W|02=T!xoYOT)t<{pJBWcDH!uw zyp3J;<9~H%@%#uEsi>YI&?u{cfh#Vd=c3d!UL^4%FF*X}i|4Bqpb#P6ByE(LTG3!N z5ybq)_(ZYujM?J~1FEn@c2{K%8ZhV0R1bY@Ma~a&d8>JNrik~dItLZeR!$8J6q8~+ zL@8W$IWh5Ml`bWk(LkYig8(pqLF~n$1GFsX{W0T9YaMo&=QdMK(MVqJyjvouLjEeR zwf1d*IQ3>RGhLYOdy3dhrz1klTP08zV{*H9Z46VLjwo8>yL1 zG*i|G9~Ol)0-muTw}+8>w`!STLxo`#tyst$amRGNlZk3Auh)wAaRQ zG-xoU6$czd7Sfe9D2=Ij6BHa=a1!poJPqq03S5P`d`XczPxzN^*Fl$3K%1N$#9+(e z_(3Ix*4^Je_z|wRi_WRKAk4nYCrb>KBAW5=LF3(Da~RLwZ4C@6R}CID2L00Ge<;ftZ$tN9a(>cD0j8Qa5%iN zT)rWqn#RYXAL2k}*)El$RoxSl8AE9a!eMK?1)Qh6@3_%wHm0yu#M`VXnB--$o8;Vr z$iTT@cKw=|6E%pH8s-XmDqo}@df%2v4TDxt*kt1}=^burY^t7Qz5!tb@Y3~9yk$l@ z@Zi0WU-o<_4Rz1f?&lo@Z=IBGLdCI?2q}Z4#UCd?@=U?-_UgCE0K(Bx`Zvs^ zqfB_w=km2YZbhnP5{N!3DMd;(Q^t!@G1BawRi9Vpq4}Ul zdP>_L1tV`iGUsD4lNY~2p_d~mIP;ElhpR_newKxf7*jb{ppz6HO`k%xo{?`?YO*hX zv&tEPigXao3zFX{`X-I_urUx^p7f5XyQ+k{XZRC=q&JiesVC z29-RP@NP)d3XVo#*p~&A8dZzrKuy(F*Pp@3z!WBp)WXwY`rJZF6t^7{cLCbC5dTnE z3={{$f$GswzA2qyfA;qu*dC&5590WbWs_Z$midO{d?_}Yd~~by5q<|hQK#iZdHec zKf5(*i$(wAto86Eil-e; zQkmfuLxLjMP;T#&%XYsq{dS$CKf5?zRav}{1YdPUc_Q>86cjbmg45SWVdO{$HIJ^n zI;mop2&CRY)L)|44^t5o(FiXaA~5`#hfIQ+n!+MEz=wQf=f#(r3+e87*(Y=r!R)G` zZ|}t4PGkioLfm32N9Yhv!<#C@U2cj`a20CezJ(4~vO7OgoB@~6!Bb8`oYJvEi?bPp z)9Mf#?O7xNnLY>@w6wFQxg#-v7A3_?dPw~!Bo#P%4+N-< z^*%)uRQ}#+WE9VIc)9TzY`@#+kU6;#n~JVo2}w~PXSM;@q?N3xV_9WN-R?S5@T~r= z@DyP?qR6Fer|6S~=$Pz(~vS?Bg%1PjU#P$2~~r; zQ%*_(HABhEvp5z{G!M(F*|K-jzIU9c`Ph3EY2e!x>|Mjws%3+f7-zXTLAS;jue_ydcg_eZcfNHhbN0==C)tOf0Sk!3o=u=;72h?{vf-wAKo{bjq? zxPR^*Z`mVM{%|gbhqj47RZ{n?Rr!-B3FohJBATjF54_<5fi8|FnEoAuXFo;qsHxfC z{X4?+@O4VWfDIOa?|0W(=-1sm@Zwc&G#OPuGn}U&Qc@Y%w&nHzU&!-l0?0d7pLmD?{!~bHTO~F0i*%E++v8YOg@HHI(7GB4(Jt`YnLI@* z@QckMCcfXD2eN*DtX3_s7&RHmyL*A!vI-gXtgJ{fiLxw9RwB5*fm3H*&(qSe*jk&h zk2j^zEVqo!HT=IE=KF8!B5mOdGhxSm_y-rt9|zt=hw}W%+VaRL%#UT;ae1y2O|S!> z%$6^ge`7?J4TSnHL-WsS{&$O|52C!eCw}3hCy2kyRuW&vqcy00btE&O=iQj;3yc_a zySxHF#{IAXnvd~cf_@S|9NnKIy+7d<5@5xKSXQ`udJj@s(noPnk|>Ah>fLRoG(7yR zoQIDLj8@T3Uz%90Kr7GViI?lIuj>Ric1K{{)A4H)7jPiv&{bt(O3&IIRF;H^@9ys~ z)W_?FVfwXCSCO^&Av%8puctfxCV9b{U1FPFrSYUd=KS=6pv-z0Ck3m%#^>QgG) zG`p9;s2$(B`2^IfFI~Hw#;@ci-!Yfm#NXsdBxD1XOL;>DQvavj8j~kW3X|1!H4(3~ zb2>7>t#3CGvlYMLB$$WJpX`hCr0yBsBovi#ElZ!_{v2+`jl8*O4TSDmrH_7-X$wO- z&`54P;z*(HehXXRrz9vU_D>F9Ao6@SkFK8%I%yg_S`@oUYCWRGgx|P4{UrJX;(E3M zUF{h9`0Qmt<0pLv8(NaRf|iS1?O1A-eB10<_!KZVUvwzQPSwQ<47EX(fObz{$qjw;I)4pE+;siK|deNC;Z5(r7Ff! zE@jjWsElBq^Y7c81R_p;jWi@wu4Vn-=FG720-6Ld7pm+*M%KL5>dvOMl@CiAa`wt2 zG|MedUq2k8%XHPZVG65ieI}!N3?=-(DJI@jsXL>NQpn+c8(Qdo8}S=}I}>xwOM9L* z6UmU#QkpZxe?5q4sM^mM7AS@NZ`<&Gee63u0we!K`60 z{S0V_SzPl9il~MW;#6DoNgY6sp8th9UpN4&`+)Y&?%^Nw5hA;DSWH9o-qgHLX0vOb zoFuVtN5*g!ZmQKLN3q|&5tmU^NYwfAF!%P~E4MfS+oxnM5(ukAc+inIFR+ApGx~+( z5cy9J4h6W#QCES}Y60!((n`r#%8yZn&P#0PlJhTYE& z5a4WUp=#&`NZKs_e2hf~z(g}j#w(NPzb#TWGW{Kjz2lK~p?BJ+#DbK72=&Q)|WR*b`O(VKfzv3ye|ahbShZ8>7?(5lbmo%LJd=cAL`$%v?2z z=}%VA?`;okZ|WA*#%-Q;XQ->wQ?PXDD~{=NL}^3APq^|ZY1zRA^GZYS?AZsXU=RvE z@=S;QA^E~PN(h;k7kQy6QSbf&XuOC;lfl)X#?XD&vJ08xJT9`_;Rbj7w5PQ%U`7KQ z{1;!Q{XezK|L|;r2)IwqiNetL!sGv&Eb==`Nkju$=YOI0U&JGQ z6-@0-))4|$$0mBsMyPpvtZkKEr;>lvJ2J7Y4c1J?iyS?Xcl23cgyN(H4EjU(YzH5x zohDo%x*{idteo9bu^uiCU{%i-5Eof50CdEIH~!Hz9~Ru*!++460XTYZC_Bz!Yt&QA z1om<-#vU#OCBC?Al#ffV$t=979cz+mCt{Aq*+xw<)nSqi~2^Vqg&%{ zZKr3l+(f_Dj#1(84g%KBUP~UoTRR_EJ0G?L`3~rHBb4{Ld-#vFrGX2Mfwt@d#fdMa z+${4%1xzDUz}ZyBJs-jL9AX~U0_QwCpTYq5_u5Jn-XXx+q-)7{4XAk~J=}yPy8L3GDWW)qpjvCm8-D&>yUpV}%V70{4T?vG5%QA|yXjY6|&ZM*j5$9wAU57dZug ze-An++Q#F;iv8K!&O(nkjvU@CDWM(vkx}8mS^+qR(RnYno~$7a}cX0|IAa% zKsA5ok=bYAR86fy?e5|jYXL`JL##z?^0J2nd%y2FH|=!HdNV!ZIP$pZyWv2Smj$lb z*%^)fQ5tWiDkpEI_AI@RD-=-NXk8MkrWsT@tYzx^qX*isgl76}XGxP2YLeJy>a7Yn zoHCey9@|Z_m*KyO)j^E0QJF&cmKDZl$J$4dh*z+$nK)Q{a$VWDXvLAIIRuWnP%AwB zm^~+Z)M4xnyW-Tm6umub!o5ir*w}p1DgTz@pK873EAJN=z&n(|ugM|FRQr*}JMcB_ z>e(3-;o`RsCpS!XNnhGoEcJNLuxZq`TjC~awZC&I4~%}xC8gLCP}yAM?4uokKBkCH696>%k1Yj7o< zTKL%CUNZi9_#&&VDzQer(9X1CPTPE9;vkB*`0TpfD8+A_cd2T2b5IiFk|V5y77;#x zHw>!xCaFqjl0fB;+~;jUflzoMJvRyFZ4rfBMUx*~VyqS!AF9ht7Z-1&VY%{s-gc_1 zDW54{v7x@ls42f7Q0**ngt9?{wTQgeo(K>7{XpVLg}abll_VN;7Vz zi>CvaCDTvWjqBlZgo$}B$Go>&Q}z|%bI(?&d==f0yH4ENx0hpy_B5>+nR`0lT=-!x zVL!x8?j*u@$Y?PZ0u(5uuWBtjVkb&9T&)e@NJIIsHgk$~G-NV@1zt}E*ENN;MRqGK z-SDhf5A|L4=+xF{^_^%#3`%d=B&w%ndxD3jE2AZok-xt8*d|b}mAjU)qV>N8nKYvF z=w~kQL1gSJCMrFlI^yH`kVR@q;Ng+XTR6{D^RY-nZRf%1OC&tN{mvi?3taX+q}cd= zs+6gG!frA$cTu);j1nY^kd|XPPsdeuN~F+*cBHQb>Ir>a$(v@69`QaB=6fqwKf4US z8COJcZDHqZRBPc>KL+k*fV6|P;3B4q$=4L==Red$Q`>JJAf#ZG#veGNlIX3Aa!gP8 zDicxjPuU3ATR`|gR9sd9iXL7L1nPtd53WKqiE?s_rR)Y;h3f#)3$*uL+8yqfAgJ8! zF_Sd5mNbFA)2|mKclz~RiweXAoH3W*uffyzV)qIkP#ruY^tycbzP#XSb0u+N7<6Cc zA7~5gL%JVwB%$xb_`sBpUUg1oEg3Tu(n0#8>u_}AD0%{|B0%?7Cb#`Zh++;M&~#_p zu_%P0zPwk{EHs6r;lAiO_^D(qwwXEKxSIjB4M?g=n!f)dfeo}RA)*eAJ`{M6j_`70 zJj9nAbt_d-C%@DUk`O?r2-;$0${#w%{?By21n8{Zju|4*;!PD%aJ5hHJYoY3xFJBI z4v9}JYeLkQzzmgF6;19Mn(p20Spzw5^U9q?1FJ;E%k!a#tn{B@Nsk!7B*3H_r`3I6 zoqwL+{dWI-h5{8!9`4af`X93XA1dXqa=H7ppbID%hgv&|gaYU}){|+W8((*(kGa@- z-!VnJB9JZoJnXm<=^b$}pRJ?3EZek#n5yid3ftAm{M6=qo1Ku)i9D214_oMhkZem^W?k!{g|K7seXKK&4bg>{lj^#gW!usCN zf*z6<`L#1w?ERON?*h+btixPmU}H^F^uE`6d7E6wwV2d1zv|EF4W?o~S+AkAwDB>Z z3ocGh*9~@}!=gbZ3YGQPrXxcp`=dzjKxnmC{jxwn$K@NF1xFqMNTg0~!=L?=f~E~h z4CiT7rNyghe7mNoGCmCDvUpWDwBUXme~vb#sV4W-k)!D0jEC-Q~Nd*1M;}OU5Z5F zkIW{oyfeboA7=uVCn9QWr06;=C^k}9+_Xr}@oP+CgVo#qW+sBwQ6?FdW>CfL%BFr+ zwxuiaRm&utvbz4YSeW0D;7V~lv%t_qNoHw2tYPevudE9z8zkpP6BTN*t{J@v5tCL| zLu}{YQ|7|JU@In=VZv3sP;U@lyh-1v32Q)ra>a(hIDZDoM}Q=JI?sc*DLM3=owa1YCTn6bG!>8og#nkK&J7g|^HhUW)d z^qG8POfK?OH1QD(Q(;5{ptSiA;kA%c+4S6>xrMV|<*YgrQTo$kO0F#$XDazAK4^!i zx!RU@y99#Hs`<$L7^`q-LBwGemIYW%#E)rS>54aZeV1$?N4->7-!}NEJ2-}@&E%tJQ>ICoZIC!#uM6xs<54Ag^?h{6_MOJ;Smc{Y zp5v0p`l-*|okzK%8H)zvZl>ei;sJ^uAXZg)z}lEM5nax!C17{zt4L~d5sgKIs~J)7 zH_`Q##(lKaKvm+ePJZ1@XqEstEp=tti-ozTdBxV0V9M1o6;Qsf+J}YI&j1O!41X_q z``L!aT5L`Rp;D_mqior@sEoWMi=dZ7eeLy`%D3=8 zb{i285G#+CUPI2I#YVed?!U_7;v`BhmEq12dqFYU1}%t5RlqosiwQDc>dd`raBa+V zGfQa{3j$Xc#DG2?1vu)2aCZ)Z;VZomD9eg@!)W3Uhk1gKm$IGF5qj?0a+ zEF0;UiGGl2!;A~NwUs){q)|cdB_; zrFr(;dEe{lt_soOxow!Bc8r>vZ9;EAS}%Dg55@MG5l8%L+k4SWie?E3gvgUgwc9u* z5kx{MM$eBy+6p?cm+0=n`+C#Cho?&`0Q-orFF(Yk938u zJR5pl0+Ew)EiyzL*0H7+mPSy!?qe+TQQXatSipst)`)M(CxT+p{ zqQxR|U|i6t95p~@Fyz0is})Jt`aG~dTZ9qT}sj} z@x^q6W|sOgT0k>+=De8tsbo`k25pdchWcf#dMDPaS8qIxOdFo=!y$H+(7EtOkLOnp zF29|`OCUz!PE%10;=1}4@7sMd9N*>#?dbyL&l9{F5wNpvahqN^x&{m}-6;Mv>j5E~BLs?=<8upVmS%}2Pn~)MBA$q7iT->-N^TsPf?8DXBo$BJkA@ke=$6{B z4$Gr*+ST6ljGm{YnKb?xx;W_kc36_NIk@XT`YGt3c5N4_SHZFZ_606d7qG|ABZv5q z!JC9VI)`hsWvnDX``P)Z)~+o~)zuWIaosrlP?R6f=t{8BbqNBxGImR0z#4wp4ew~w zY`rlO$QU#gIL8P%m`_D5)+4OI#vqmxh8yHzIFDs3t!?+f651c^_;f(as*GoHpm5Vh z5RdkG${05%^Gh)+-wgHj>9(uk;%ChCk}1VvPuZ*lqRkDyM1n!)Zj@VsWc*Z>=XMGe z)W~ljDIiaSwJH48&%!U!+q-?2o!yFvuBjt-`P)p-iZ=Q6L#m-j2XKOmpeSfu!E~2n z{PWFB$0Aa3u+tdID5g@$SlD6NwkMeIJgOCNV4Ab}m~}v|$?1L9?eZD61GcG1pzHZR zZ3c`NUF)(^pQWwX0f1ax>k9PcYRmmfCioVt>$v+IeYE{){%Zf4ZW(wns;q{)3E#b! z@G7C}`j&dVlQQ!>?4@CB%*3@ADDN>(cnqC*w_t)IXvoLwK&0mJ0CuRT?C4O#X}OFW z7<#gC9R}=MTD?+Ky$1TMdaJ=cM$>=8K+2%FG7)l_x!);B{IjlWrWSP8XuKDx>lsxB zW(d&L@JV@`M~fF@-Ki0(E{P>3Nnx||*+KUt5N_4hhnSEvo0Dnc5W#Ar7O(4O-Iw@h zv`iKBsEqY8WbNg0A2<(&hvXy5+Cx{!xFKkN4mhBU@ouwQ{ zPEY)q-U0lHk)}wTyA`4&y&zKz6jlPWvS_Az9&zygykmBCZWMDWXUTe{6AVhZp`60T zl73x$Iq|-%!O)Gd;rmUFx-V6m;O>;vtrdUp@r{&QiQ2$XfsR_ z=cymP5F>3^CvzpmNyLp4NQK~eu_l@A z%bN-bJ6_|Eel<3dfvW@ExcnOVF*7AvZL;S=`?A+ogt@)(>QY4Zb8gixABLTDuS985 zkq06<6KbEwHQ>XX+Vu<%U`H5b%?45I7V7~DC{Mgw)zoM%@>s+QC+Z_MgXZ&@=FEy7 zub5<<`Da1eaQGm-n_Ut%wVzS1_Ed+h`*2I(8%~GuI{lds_(X?@Z+&d1GsWY`QEr6e zZ}6_vlI2MK+ek$VmO-kZ8cl-zuGUrq#_M|fo<^`GT(ddcZ9;<6ygsuUb6JT%~hUS zPA1yl304&YiPr6sYT~V2s4H^VoC<5V`M{-K7I~X(8r9EyNW&6O$&ZQlU5wOQA2|Q`-}`R zA8xhB{c3n059LzaRJHmRn*()Au2X)up>3Uae6Yguh)>eG!R^Ta>7-cVEFK4GA6HtB zuwap6+KGPqX34T+VD5z;H(Aht)kjbrN&g)WWq~e`rZPNql=6 zj!g>&tz9Z4VSnv`ft<2UaGjBx(>LBf&tiHZ<~+=ajAbx5HvsdFEPPK>LdUfP^OYvM z?0j>=Q}p6;$O}gXfK9xf)4uTk;^7T>ep3IER7mb#O?y0p+?Sr4IlS|0#9$dkRL61I zT&}#J2B20qZg8bZKk?zt8e^^g@ZPw9Fa6TPZ#04)1+v?EuL`PSKhb(uBNW@AFzN21 zB!NALDk@o5-7_FNjF7_VBm;?uNNy~}hu3GyW#z(3-nzI<|06?;;|pi@==xJhfuSBG zez*E(PLu^$*zWo2=lq)+8I{Jo=gc-WCS*G9nla=N+L406Wfb~+#)bmHyD9`{`=Q{O9s2kx&Fw9f| zA^Og8-QbMNEOV{q9myKEp<+)HY1SKzU3OzMwkq|6P0h8Qt!txZtQkVV0rBh?>gUhS zTnBP5Rdl~Ut~^}M+FM^{>u!`+r=wEf<0x-UL#_oUqzHDzb_DybHgOaWK2MQq8X=&? z=5@cK%DinLlzj3M6r5&YhoxG-V&JgD!A)+u#mhC})@*}Y8yPvD-ADHY{6S`Au&A&G z?Nz6Kzwo**p2EPhEoi#xZo>BZL% z^zgz!EFBJ_oMH6nIr^=S&7S#l(wAE7HryCb)yGs#-zhJ?Q)MFlN~S9x<`u>6E<6#4icUxLgnUHV*tEnegC+ z=-xmS;aW|{AImkh7ZiOPny_SiV`%G4ZF6n}sqhW6v;jK3Knr^}&^@4`)YLtARCmdC zbS5vx?A<$x1pSK<^J%Kht#xV=-k}}`$M4yBG~}BjUg#>y!&CC#*qjp1y#rLqS&&Mr zFy{8ylg$w;M{?dj-m1P1T#@h5-?PikLUpBeW1X@sbI4YdFd1KL=s42Nb;kr+podCL z`9lhK29te$@ngTE>CiMYMN)z*MDfs=>+5=8S$srYoq)AyF4GT;+S7UoHN+&GuCM7r z@6MkFxx=wB47qD?4Wey_s9ee@^vR4t`ni|a5Ji417G-40N{$Q(7&%A^i+Uey$Jc|g zH|=CqbQ2k>pOK)_tdsftY0d}686^wJUcqykWT|-z&;>cFh`L6gdsZEiB*X3z_|V}R zx-PsVs)uZ;0L++!jX+Ytpt=_`L|Id_tVD2v)Ms?XzIH=W1pYWKy>|&$9k@^0S&9*K zeu@%AnB0VOR#a+IzDVVH)zR#r+Hd0uQ+Z~PN0m-uDAA@sB<`G&VxzGYaP=K+YTy;l zy!iZnhd)L&6MHdVMrz6z=`v3bQ7G`Xy}As>|6}j3!>Zc4H(*$i6cK6Zl5S9t29@sa zkd*Fj3+e8X?(RmB?(UH8&JCOWE>t{6&U2pc`Tl#a>;3)1wb^U!wPuVl=9puQ`<^Ag z3;D93@+cR&*7?zKyk09~i{nswH#yemE$OzE@n zR*_1F%1IT0*1AYg7X}|R9MH`<*`O93S+ktDIIXDBEV&<>to9^$FE8yVh_=%PbsQd; z*bVq^;Lav3e>-|P)Y+#jM4&6@lxG);;~wa0`T?ZiupGRjLy^rddR7AtgGfkbj)+t| zkZ9K`#kZefnzVCQSQ)lAEAhy#5o4_oXf)>-z9MChDw_ZDfp{4=o4OM?7n+kasMx)7 z&z(nCkQraPwW8$wRK&0<`X#u;fw8VY&X2a%BvCxb@V@-n)#XK>2s^0W{@EA7^UF;0 zsE+-$$M^)9LKN2k4l{xLv8A9?DyLlIPzvwUEKY}Zq*8+pFlVZX5{$EC`$`5o)!k_@ zb5W+$D4nPZVY+qVaL<_F*B4wp<$b*MC%(Qo!G-QlR;YV*8RdhivxZ3cP7<=YQk>OK z$7Gn=ieGH-D}28aFd=~9snvI7V-GaqfGLZ5gD#0bR}OWsg?l%hj&Q`hi5VT#nF}r^ zg(O<;VoqUC2s>I>thu>2e5YMoMPrV4aYarN=MCWlXyb2IDdGZ~71Niu% z7FU;=3i7R2^Q^NQc3`W&*K96vEYEFisHMA6;@Bt%41O(#AD7(!E?KTkRgFkxML1xA%(ep2w`VJE2v%IMz6rPA0Y9IAULh z(>ZW_upJ)2x!}P*U15~H6s<}kdw%yE*Nj=3Kt$Q0GcEo?f~#<}#pJbkfZ5LzYMN)q7WbmdZ+WB=MjHT4Gw3tZ-|eqyvj?*3coNhpwYFl z^?w*NG}em+MAe{ycFfv|;H9hG~Z*A5!SyNx+;_>AaEde6W36Vavy{ zFShtxUuIV-f1q-S%J)!geOTaE-6ADEofGc6v9|f8nIC(uh_db6=QZL)Vq~Lnsn+lj zqE8Q~h|c;Ft5$!%j)wVoh<_qk9q!AH{y`88Lsvx$)tq7_kJW06aBXj?Wc*}7&nbff z5;9e;nW?|}y=@l{syM8|&`vRij-d^%!ioyMQ6HC&APq#&ITsD}cG(B9X2P6TF4Mh{ zL7Dgr%YnY%zUeO7EcEW#JJvl6n*LtQ;@+jy`m zQT5?aV#efZR-OH;1gp5%km0eHh3yd8))ps@<~p^4)L{~R7qc1yoe*mXYAU5(^Rva; zb1T|Txi>CW8)urr%V?tsP!Y2E^$vrHa`$Bu_sk`+YeY{d>ypMw66ekcG-c9?PKS=r zVxRZ@O}~idHGveFmBmbm4YVi; zZeYDZg6?$6VJ)0zGt|*4?8R=~LA!sDmlL(!M!s~pXEiZ5MruaqbM_8Z`AjR!9YmaO z+L@Oa2yR=pjKVRutmit}S}z+4bqMd8#%(5~|42A#(eX+P;}p`NjEg{9&H24ArbXh} z?)bgat~QcG{*{g~O}tMqCXgkirSQ>5x-Nc2A>WZ3+2xpV>4mX#>l92)ku$CR$7Z!dlj zho$K~!zxx#w5Z0xO)9rib$KX4d6bhknjrq1REUE!G{?|zvv+GzWN^aVDY+~$oIOLu z#{BD$<3&834R6W{i5vv);)$7;y2${AbL(M@Lg(}s@nwmCo>$zjQs=M8A_lunlmd1$ z+KP+Bleb&G0RELPg?}!L&hEHoV_xmXQbZS)?lKEAZ7Y;Y{p*^fF=icY*UY5;e#bwu zeo5CDxl9bHViZOSM*2psEcfOreLiPahKO?ogt^up5N`Xv6bp4{S7=z0t484RIn3me z=Y(cUBQC}z_}r8ta!;8!QMH05-F`qvrQE!9Yt}(Jqpi1#?`vPuB}tasWokmw-gKQ1 zhaz2M`jm4x#+{Uli!rv&Qc{(#lyh8-W+62%y5>Krch|LxT{}kS?vKMCn;rx>;!=I> z!yzNrH+G>tp-|4wQ=_ABE)(GdTaS5gf4*!I4k3KU(B;J96O9yA{=956kIM%xVYVP* zb4DB@+n6Vik{gfQ5#(;mcfw1kA(>XJn@T&V0wxOiHg4|)zFhj`3hq_0UQKWiyy}T9 z|I+l0z9pkTb7RvuomyX5jN#pfMJ)obVI7-jp9qf&J>CRymwm>A&Myfi2PWRkSHmJa z`!&aShlvK0hK~LU9;ya=Z`CDnbq$w}l}6I10JK%xTS}**xDG>t0UU0!Hb=b%f--*~ zjwcfBVZVB2Zh(haSSH^OE?8`HIL!A9HbqJ728t)XDD% zLga;Lew9jkLKwEbKF$^`na~h#Yk657B>+}h2S18mp^M7Aou$93afA=1$@o@@X#M=HjF_=dBQ_C9$wMrw=EG zL%W?OorXL7`gZAiydk`}s@f!t;?#yJ%vZ}(9*P}qG^!~oxRWIBGFeUsSUS^VLDAKg z!jTM9*faQ7f)-_G1UpG-!~=I6Sqxg!;}U0Sz+Gh&-r#~6R$GkaF#TBw#l zDXS2%TCWyJ%^cH*SG?l(ym3Fv(^k6HA) zE+CVYTSD{Fx&MG5A}NWBj6y0=9CA7+!tNk&q3jXpx`yb$Fo8)5b54R>c^_#=smX1Eq6k9m;`3JTS)h5c6j?y}~Qg)4< z>V~<8YrQGZpf`jama}n6`?w;tp_zfuip9zb(qClx8fy-X-IUuIJnN zS)3H;8wt4GOb_Ptne9#0Cjte+Dwte$)a~YiU9Gq2+3miEs@K@^(^Hzgs5KsfqT=lS zIOIp;Mz}o*-mdAtN|)W4`+VGJjA*nK?u}D&B(#o16~qm5a@hHxV;FCl&4@WDk1SE= z%b8&sBoEnCR>>AcW$tnCtS)Ksm9~Mkyr3jOXklSAS&}nIUWYzn+ooi3)Dy(Q-(!7r z-oq?3F;%v0v@Yv?wbNwHZc8kd8IGEYr5Strbc|<_=HOG=Tq%!zBRavH1I&((MC_I^ zSgJD?PLZp7nI+WPG>`h~xj_TLg5Y%HH@a@KY9m($TRb$TDa}{wRx=KB#svb8I`N?n zcQlFVj1@1&7{Vz=PWKkE0z0Cx3VbjYiZKZN%%XIfecfl38-RiuC#yfIl^kV>z5z9^ zcvO$3hfrlX5~M#Hj!R!1AhTgkq5)yL@3tjilPwOEuI6ts4NEq-xGy_bg*q;nmpH2^ zg6%bpYmAnG;2X^FnajtBT~-S21$|*eEOrYH+cFxL*d7FZKWHk-ddHPA=in9YOxMj^xzZ-iKv5M&<869-|H* zIvu|fK4a2%gG4&PKtW>61 zRyEMu{g>HlFvqpzASXI-@!1(_{VY`S>zPe*_k6=mu?1aqqHT~Lp`*CH@3=U`!!gKF2mT41Gy09RgC=c<|_S+SP1JSRc&6?%^u6 zHPzYa4Bn^=!cAY@v%-Gk=e}K)Ljrc0K*9y-d3ZA`5Uryijg94e%#}n^BDiREuyX>b zR;gkGJQg3&sKUsm9(GQox}}J-ig-}Ym0c~k5sFS|2|FXmC}d5IAY9n8_P9iL2`QF| z%G2Rk9c7}WOqp@Nvp$Dtq(jVf=`5ZpQ1KGVlWisv1iCHeU3}NT_IR6Nz6^U@=e`d1 zsVMQ6yo=`0V-))3ur0mx8Bu@_wo>mvt{UCi`j>iUbf*DoTpo?wZTlvh>h2NuL$&ks z4B$+*lBz``qao`Whk&}^y*Qrbk#h~tXB=apnEGdx#Y)GT5zX9YB4+$)XmlYfEW)Syd?G(pF5ATfehfrq7XUWzw*@ z#?5-gQcARuAfP;bvq*iWlD=e8j4NR;51VU67G@le#e){zs8_2lKiiyH2jfRsw#`n@JJ;~)v|gMf?4;qM)p(m?5H-kPmKC)i z*nRfwOp-UGdi41pg|H1MmV|(_@0l2S1)Cb7GC&#-YZ>cbD>+aMG>i4wjYZ_+TcFVPE|k84 ztXkCh)YybhqMTahp~Sc;ao%%9^22c3RTUHkrUk}r6>E0Ah9@3FUP{xmvWd-;CPi5k6b!My(+I@;tg)#W{z}lDbPU#w=$ZcC-u)<5x7qzazrw&{5hAPWbm?2J5Xxl8)-lJne zU958jD6_A4%!Yy}0O#~om4^>!>?|-O=I(eyrv7DEr~bRD>y9T0d=q%tCX`=rh0yx1 zjZ$c865~qC#El{PQ6)uoT_u&xN(CI4L8P6@r;=|vo|Y$qV@m4zaV z@>wI(a|vR-0$^sG`xurIatqr7cle)ad%IdOm7*UgEr|mdoSHje*8v~y2v4t*uWq@S z#))5vf*XOVB`XDvuUH9*r-5vC3$3?6XeIXawm5aApVUuaxbvFf8 zfA_Y2edi^MSeH4BP2>jss2|I5aoBw&cIr@$HbDpkH(0?C-VK>v?6 z&oE;g0zBoW>_*-$GZz^K9UUKcG+m`ypKO%n4Mnm!1khvCMOAIHBQ~xJ&Ks}75HRUO8uQGLf zfF~z(+2-r7OLlrs<2mKq)2CqYrV3VvyEc%H%~#1FWT=*=we%ry9UwIT9GY}&Q85e7 z7RWYYxH7#rT+6$eAj=6M4e@z@mkHt8})AP}W9c#kD=jYqSl-o0;V7eP`hpD=T)XJtz z53i+d*2B8H3DZe)-#=_Ki;wRu)=N)&jRy(~wB>iUYlffAM;pS1zc&icQm<(0nAR1&5VcTSwDYqpQU}+GiSAuP$_iriy0Odn{RNfH$P7e>uUv z&5}q_pWl6+rZH47q$XZfKg@mh-c(H!i!-_|_EQCCT~7NoGE#GjIBfM{M$*QaidEWyG*! zpR7?^dzII4g;Oi}WExKDJ*+D8dD+Ka@o!T2xI0WwcZ|;MC#$|52WK}NpK|73mDxvt z;%SluH@RmICJw^x>|TJu4oDybUXAVW5h^GHh~#GYWf<~}fbmIbD>`N#cq0-3rL~M+ zL-jymn{Fe(TeLL4BH*-Yel&d7X`=}{+tCALVJ;4o(A)4#6QySMZ4mT@!+$^0XReW# z?4VJzZ`XdVY?EtC7vO=$T1TT>wfBV<`Rn@Pk~IQh<99S%z2;C=Fud)FHNMKYlqENM z`dqpOOCXuOMm6ZmHygCd1k6BtiCRgsB%j68rFI);m+!IPv89+ByN88s^l*V<311XIv`kbSLHUe(*asL&*dfJ zWWH^x5_XKWtV;HLfCed|w~L#-M9}jB!356-<}L$bztnXNGA>j)#{6zs`Z0ywuK}eD zY;y5Mqbt6~M_pk?C(3s5KHIi{7Y*_xX*%X(`JvU6ue7a~TJXYNeJ$RN(BQ?PdV>9x zLmhawx6-4DmrZv8K7l_Fi`Ukx;Mydbb;Q z5!Kxcg%rTB01yiZ6W3AhMODuEqnlW_`lTPb$CQEMK3~;oO@uRM-Whu-Nh0|}%ovIG zcGL=mdI^%4AAOS3U;tjQnHvm6ByD>a2K6~t&jmTO4>>L+hz)_@>7(UpP z{wPK-x#rEVvte?LbJ*ysRFL{&2M|oyN{{~l;!At-DcpUHu%UQ&hhkdQW(oK11@s!l z5BiBqaz4mHB~E7D&js`BC9wKmWf_XDzwp|KjU%mYSWacDW<1r40i_}lJ;IuI{-joP z1MXjeu1aED6{Li|d;)M-WKL$}!H?7ML39e)vVSdWp4@;86t~65|20|v?UPq1Fq3IZ z3Vg8t?XJJcyUX<|8pM+Qr_b=OYd;wP^APUH^?dL@-hXmEi3EnSzX8Ahy;-94z@qy9 zA$%nuDksmw?*HjK{hPxjZ4H>T@Vfi$|8~P~-{j}yXa@m7-7PC}jQ_68H&+4E2vm$a zVFdq8I&QB0Co|AQ05oTOQJnf;h2@V60CqN-QuctF=-wY+|7&0VPw@ZKpx=D=8(8f5 z+KWvR%!+DorbPO&8TIoK)Uek2rYGKWAz)9&ZpSZqt#AzRZx%bLcEakV6TJ?NF$7{ujfqqQL+)+e$7y3T1 zU{J{i@BI>306GX{l6CTfDgK*U+{_T*YrtPC)Gg5Mzsma^D9P)M9h%7pu3I~re>1fL zj{sYYBJJ~IiT&%f|Mtl;@_tj00HC`S#P|Nu^at9flut^tzL(7>6X{ko!DFSyaD$+oQe8 zJG|VByP(XY+velBFo5IZ;nAk#rMnj-cJJ`jgBUC3e^}JBYwX5Vfrsc{fd8N7(lZsB zdQ+*BPpm=3IHFBybk&7<^*>DjCD4Op>|b*M7$=P7leCvNwr}3rtFulC!w7qv`Z$7~ zfqvpYP*#{?QkJrF|K=S^y8nQuUZWzAR+t+6uX{T6X;=IYQ@`-krOcl0Vg?LQ6EMHg z+J!8MHAb#kE_(TN~dCcKuO$ z1QaNm0i7xih<*Qa!uB$oj)on}=6`VF;7%8x!=7tEW{>W%pFENzg#XE2!vXBvAU%o` z1xX4gK$P*J7cY2|e|?v*+c9#e;8YyjEN==uhLR+ufO@czd?YJTiso(33t9Rs< zP}4R9mBk64$?y)5-gzvh!S|Eyq}XzJU5UQ;d*0cC-f)!@QcpgBDAqS;jlCy-u)K_Z z-L*}9SEO3h1EqHr3G)43@x2LPt|Ic2{0jNxc}^cLyT7~ed7N1CJ;V8Q8BDaOrDLQ2 z*Gb<$^r^|BZJ@sl@jlf`MeVVo%v}M_JT#!^O?!rPVaaEQ|LrMz&jtO}L_ zJKGi`E+)B0l>9}@`lAt_&!Bdx+)>m1T<7(GeOITeKPeFTl-4$$)V^bwRFF>{8sD`v z@+vfF|Nh-mc*d*h%z}2RKa^tr8PFrgd_k%|P$k#$4V(g?gdsvHwP8$x;5~lhj{jXF z+pm3GCnBqVIxgXYL{17QifB|4biveryzjcvjv~PCG;<_6|40J>L+ZH-y^!l$_P{g( z#N3Aw4D}xru;TvSDu!CM$b)~lC;oqj2MfNl=K05afq0ZJVEknH9RG;m{+LTiw5(8t zxtWzR@esnl_wlD)1nBNAkNjzJ1H}KwXj|(wkapVt(QPaBpN+UD&2_Y`)0lL~{n3vI za0?zl5JTv@SQIGa)TcDBNYQW(&1C%7@d`)?U z{a)GskEQkpqE|Mh13L(#o1!K8%2jp%{l}3t0DXYjbo_in8>&>H&x5)p?{0ybi3yz4 zPlWpdu!*!ntoXG8HE}}e&r;nd`{(xK6u!GX0ugH|A0ec=aT_^V?;mbT(zhd`xt)dkWfWE6}wVZ^5h2ED)1B9ie8EyrRgjQ@&ISZXLOflWi

    |7fCt>b<=1QnIB2*ez~}ogR9Y zOptt}A(wz8Z>=KE*r^{hNN;PN!)h`Xe;LD=3R524EwnZ1>bPximM0oJw^$*t;mo5e zlp~X>rL>MnO}|ZwrGEM7M5jg@`nF+cdI1cA@+XnsSd8DUj&^ZVIL_R7c(*cMdHO(r z3QnB4-ZI7{bg-b2nHtI1`#$GWiS^lB@iYY9w=T<9$c;2#irof3C*W|)EfG)%Q6?Zie+$C@)p7Kf+FU`xT{OdCIW&6q1gjSd;Z2I` z6ZMj1TAwFo4$Yhnx+*&(+g(H4C$6ShuR~<~=n*A)YJ~@dXDTW6=yTENzohBLvzlfA zyx7w*VzS%56A`ahjF|O&%sp$P4QC_GV}$CnzWS2Yb2R3YQ8g08oe%~9%$ldB9>reJ zfoQF5QOO1FX391<4KLP~HjdC#bP%u1w=rjRp znOYW{3IBm;#smg7TPxqvOaMk4RT3$HB?cj349JEm{L@D9te zv13v{5&A^E`~|=>a<|r8T*aA|RaA5jRtbP=D}~3tmzJ0rfu4kkF>q5oOA;3`eIpEVFh)t|3F`XZ zG4BqjUz>5rH=&~nb|XnO_VifDk$mHGw>-~#=5w1FH*hlaa&)}WT(WuSgGWt zt6)D&BUkI+M>$v2{toGz1>X^}BRH9bu#*aWtq&zFvh>G8Bw2(XQ+0mLMIu>56H+5x zNjvt%P6rXW#t3?Bkw2x9!9V_mTD5M$beXWf7>bkGIc!Ml$1+-crKbV?&$QuMQ4stX z*2psi_nS=ASgq64gGqYUxNBYPPq%hd!0e%l7pt}1C97Pk=NC@B1ltpbTQ+4Zq5V}F z0Ry{c64q??YZp`0&F`eT>zw<_-dy?igs=-8Yc1|pnZf!#23M{4M7!s`xlCY!^A&F( z%qVGujJp2V2!LL$W8^<%|{h*u9lmIdQq*X!4L+k--D(o9w> z+H2rIZ{g~^kF!wPPP0rLO!7T^Ah&e-kuZUm6q%Gl)-P(dzX?Zo7GDJk99os}61$ts zEE?Q0E8@v8&*yr>iLLmO zy~D3@+%JnY@u%6cB6I?6tlQ%sI1bP5$ml9Crl{p_Zy!AVezlrx&+T4m@e!`pTg)!t zV%UA5tPU%{nZTI!;~i)M38^aEGK!o;oBoos@|%&5^;{Dw(q-; zmpD=u`w9_}QL~Lk%1PDa@mNYonu9)B6WIGbOMHQxq%iJ#0uKkIt93& zX7#?`H`Pl2sH>Fk=J4B`UZV8t5L0Rf{MxMYxV%!|2WwcF_lCTyHaLfMy7N$6 z!|7RT~f2hj)&z_AI{{DQ`d`3XpK_p??hc zLPPql#&so0zLuP8vn>pnE|4;9S3K9aC+3y0YVB|d({mQl7<)p9bo%CsNhy<0Yr!*y zH$!VS_aB;79a+#NU*%df=B+BiRfO+cDfQH976_ffb(7pj%-72bH@dQ>bFJw_{Krfh>#=}Yok9=LWSOtg$%B&X*BK$6x2W8Her%-$0xb@mic;tHT#sb`gpF<=pic8%llnySrhRQDTm`Q zA67}Q`#UPY26?b<&3I2x;oFDCDz;fF1HjB&dgi-?%;q4bSyHi<*7=Agq7W3Mqhp+V zT%ch?F1W+UH+Gh)A<1j;{<-HXwD_}#7P)PMCwx9m)U@!(=tcPgkiO%E(#;p|UQy#r z+8L0;`kdY0oi6xbvqH(st95R(+NJ*hm8s1u>wCX|8EtuQ8FN-()LXU#tMy{72 zrf;i*Y|)u;x!u^zw)~9`>^_%6Hjc~Kg`f1FsDrQ2iS?XiN(;0g!VdIm*;^EAl zzDKs5VT=5Vhr8cV2nxVhlqnW$lzXK70HXETx6S{e*|q|kzmeH|0rkRigl^6RkpI%yqYvwczINk~u;{g80F&}o$K+-wg1$R>|bE!Oxl}q z*3?eU&(df~-*;2`iDI>!%r#;71z}rtGN{B_^NMFbQ~lq9yDgvMNx(Q}^o7O)!|D~>l)|8to$Ow1&4acd zk>_#I-%mkZl`HPnIHFz49WF8qwVuM)tpE(DOVUKH@D{@fz%7ir&(n!N7+1~-#!@-l zHiEwnnxM3I1cF7i_rkO$%FF{-&dCY)kzRdr(1v&0(nwSLN-A!8Py#FQJmuLybk_{; zI6lr3*`euz#;hw1x^_g(Gs6C^%>xB9&Gw)U%0hY0z`6b~!?{7V*`$qtz(r$YM8T8x z`lY_3UQ0hwoe3xv>{RIu`^%ehB2=IeI@}3b|MHS1_tICTrxVQ z;zqT{@x~%M{kb_ni_>ER0tH}3z{Hu+bw@8ROzzW#5j6EfUKc)#YO>ZGc+gF$qsKks z{waU`$qRoO$>K%5g*2_tLX~50R?$8>PiZDx^g7QRFfhcFj#rf@r#zSBczZi19qs1j zsAbXZ-jj7r4Lpbl9M(tcC;f}FQZE;@@WEsj@ zwY3DbUp#2BzOwek_Mrv?GMDx72mKWZ9WHKAfAps(3=325QR-cK<8zuda`xUoAjR20 zUg~$+@&{E!2tAapvM1ZgfBlwBsIC<^4b}JQ=dqyZwt}2@#<<-V;z zB&n>az%2pCAHFVUO!KLbti1fjVdVa!2`9EVCWm!64L6kRr@)H&_EfFAOU^Md7;2`p zhU8J@;iukL5~2g(=67g#lsdDc`!mR?8@MV+*p;tk-rQ-))HRP7JZz34=Pn|qDOC2j zJnod{f7x7Ixrt!*07;xu5j?Q>DSU&Ba|9m&Uo_?WW{tLg@jgyz$XVhb+!X@dR}}Jw zGVNpXk+^}W#Bnn!7AfR{L!=`nf`syecsA~P#3+!US5FzFPI2!MUn1|?*JDQPv=u%v z=lh-p;zyg&B8lBGO`qrCRzMS;ln1|As}PWK1B>+@kYe19At0b?e-?iGndlS-2}=5?crek{EzH5olbrG zw>5E0+@(f26Q@r{c$wm+v!4ZV=N`Iwg}`%DsFmrxWx>*4oxyBxu9fp>SdA_?IW}r? zJUK=>`}k3r07kX$D~7U#i@b-!WoWgT^o&r_*b9d+GV`MpE45f8v2vrn*9p7pNXwfh zbF-^jPf3;5G&k2`)@Wb6lTXWAD6ets$K(G@PM6JX%G=)ZSV0#%Xt9rptDn5abAQyY zyhF}VaI$a)GK?=Oz2QC?eNM8MWnVpFUB@Y(6!L|~E{Q$IaC9ck`y>%5eV;Cz``|tM zn9q{fX*~-!aQ0)8lT$9i-i)SUUR(QTSXfy9)z5;kh3m}@_d+hB7d##XU2zOD`Cjk!j;z`qvK-tr%iV(yB!!Y@*({FsdlzHdGHf+n`ca zE}=MW^nAfh;9QQ4+m@=_^Xfq}w^)yb;NUD3HQERGuJkH2?8kGmX}RQpARH~cQy2w&TT&0RP7m@>BQ604&ij&iS$QO#9l9tlNLUu1A+u6F=hAOv%56WN7bk~V zHr{^o)MkBnMM3Oo+uW`oVQ)+>8O`*;oQ5HzYcsLO($^7VL?p-cVKf>{<3JP-fn()& z=1?v1GZ)X6S66ZS*afi>%V+eqrms9#^!k`8mp749U^B(b3CT+v$BP9K#sz%fX_QwK zEwN6`vLQw_L5t>#z`{m@eMc zFtvEt!}wZ4b!5E21dcVQ31>~KcVu!^GP|DXdHWN*mhJEJKZET5JMQ{XJ3(((Uhk-D6_JNdP`^FeB{)%t=;`}VuE>9jOkv=Di%Tn zsxo|J1R(~2dJpx;Xn#{6c!LXzKv+(_S^~Ulo5hncbkB!o`kC_vO%n17x-*?ioxrZG zvbdWX^@eR$Ga61fs;{X^doN)ooxyay)_Llf}dKQm82uQ!1vZE>R; z^%eLMK+nBzsbcv->2^pxRFUWaHzF;c9OJfx-|<>UKRXEHEMKp7K31>E*vdnhk!}$6 zB{hWhp)a&O#!SF1Q0(JW(i2BW#sz~r8EV834_ZOa#8SDH`27JJjidUDC$#5+!&}t? z)JN$P6{I!rQ^$1+5t9QtTxOMmu$YC@X^YK4xtXgkjU*!l&Gk&&_o2>nt3dGtbNJZ2 zMUlCkbaWh*%e&ypa_qI!9vZ8cU{V!oYWjz1;g!;|IA=2z=?*(iavz9NA5VwKjGUgVb+(ZTVi~Nr7Javwa$R0--E9sP zZMHqaZ8bCWpW1jlP~r+fiXYajebHUyGyIIT`p~^-5Z*z>N7tP+S9do zCpCOs4iq<|t;+r<2W0=qtFTZ!f~a}M(1gX`b{_bE&DqC-IYwjY&kLbGJ(rJi%oP)T z#d_3+Xrs?0SgjHL6k@ad)*3h@D>SZP4X6q~_dCL>a_*0Qu$9#zwm@}b8XrqV7BrW$ zdR<^`)7-;31ggA0ZP%V&%gfo}Mj0;=pFX_qE#@6pHdL1{`Ld@u*4*bgAN1{92_xBE zSYL6UpnrIdz;X(@ul`&*f4MxtdhF?&ndZ~%Z;xLcE>^6TyLo6wf7(bKQNq+49TOw1 zH!OFU3p&a*u`o6aYumbX=B{RzAx$21@JQ9a<&gTIwCAy$e~Xs=M}DeKN%&)HhU3b>|h?$^yc`AmpTys^=*1}uO*E3 zGMn?IZ3N?weBn(lh8_#Z>&~falO?pqG`Tm8Y3fFR=Jk%3=wA(f6~$&_y%Z0B_XNvj z8KJiOJa0qwbiFv4YsQ0(t)pn{$uRYNj=R9@0lgh~YZbKDe0tnbj$6aDYOhe$n8QoS z&{4%t!V^~~U}9%EM+cIS4CDbP?Z}_!!1>20l0z`2-p^HzK77qya3|5qXdG}^W^AMdn9O-N?~V2U5UL9~&VRFCc5B_Q+O3+>xrS;+f+)x5Dq1vc#_ zr38%!oLX-qoYG~}{-Wdwind4bF1-4d2D|`K42(zz6vHZXm;J#K>Pu zAb^~BOc~1KCL%T{P6QlY?BTalcX9I61iR-~zQ;=sOZez|_@lP2hAm95hoA8Oujk%g zYi|eqe`W-{n6B4ma%9|psNYS-;E(5c9iad&6zGVI?w@I?+vf7e4_nc-KetD7ncmL0 z{KuyQJ9{BdFOv14s50n(q)Pv2$IaEpa=`t%2StA!_46+e{`qTRLO_|*q_hSBp6>rV z9ybsEc~x5+xZl5{`Onj4{`D1qP3r&j^gI7L_`<|q@muIQvN$)hMAEmJ=mW>D%i)Y;W%dV1t@e}`Ib&f?j)dC-{P&!^2?bZZLW zSOB%ONglg(jvWZUi(OjL+j~0@O$)PR7B@maRV}(b^!a3}LYx58T=L29k3Y}1>8oEl zm)d8#HZLF0Ef+m!x?Ul=eUxvyauXzelF#?06yx0=5^H>7)Bzwxf&KUCiG4r8sFOT@K{K>Am;kYH z%BjvbY_~^6Op1M_p7MM0Zom6i4v~);;QwEHhj0k%eCg~^JHQ}(g z-NGA5xb?6Ts0Zl94#eoUM*Ls?73RFwF^#A}qW?|Dnlu3wKB50h4sX5v*1tv+O?)9` zo3c1J*8kS^+y9OuGD?4&{UH6nHHK*W+AP~^h`#&ZH2AIy(An|YN4`yYh;Sq8vj9z{R&nxFG7bq!+1k6!{HZJ5CF zfcXG3Owtp~vR?E4KCvg?TbSn$Ip7RGf3o3}C1Fg3YP?z^^|*RR%A)!p=C_-|^y~RL zIO2Sw2t|!)@(wa8Kxc_wLw#vPW;-Jwd%c4Go-YI5(zPQzQL@Lr#3RF?QN{#l0ngpQ zcfEAjlnc{=;qXAFXaH;HcokATWf3!A-TwPvuTPji*CDJK%~3fqPcT_I$Zzsm?tD<^ zcVDKYQ2{63qRpR$I=Ll^(iuR+(`Qo(;E5Q~pzz=la+#&SRpV2kM9X+xrf^=K&pDlL zv#37&tMvoFTAwo2Z)KwY{Chd*5=3U-3R(_${{lgn<|O3EHq8wLJ9wkCzZA#Q2AJ6Yd;DLe2!H?d ze}nV)(fB{f`G28Wu~6^LP5{xFUFTF6!7gH(8}m=3N_57)WcrrJ`$GD%2K3 z_K4Q-H6&0t{Ei19Dwza3yj~)(59r=Jls9I&Fr+@lv;%_CF^}e-uc=3Pk@Rj0`!xCA zzXXK^U7jNccE4fL`+e}%H+W#2Q5FfXm_LV)@!qw?O#RDZrZ7xz_Caha=a^HMgV}G)+#|Z+wLW4WdWM?%yv)cu z7h=Z8SmoDHDO9qiIqz2@au1(h~!_TQ~(x~F9rJSBAC*slb`4i`gwz% zW>TiC-vqB*c4Za~6nyvlJ8k@{IV!zj*D+~kIc;er1`Qd2s6bvluTF*A*FUQ=Q;2Xb zc!iEu=Jn3eQG3&f7^vl_N<+IW_MbIhx794T21@--sOy2d#tEtbO;m|u&mx#(oAfs4lI2BVy~c~L)AZ@ zR*fpd`c!Skm7mi$Rg%PT6g!o*Qc-Ouf|mjtXNx{8m46}`6NrF*8I*{RyvXdkCK1m*Ui zMPoE(JAYW2b`t1QpoeeKzw705pHW&bMfTfEA`zJ0#!;N;{jXVChd#*V`ub;>ED^b5 zCSARTwnrBh>!mgZDR)af;IX593{(#A2ukvb{v+J*m+@wy?a%Dp#o?VVxGE zlbw23<`l)FKq)w%>^kIjTZ@?Yo$szYXh@wEBJY4UJ!IUL$zf&mzcnz{2V4#x83>sJ z9=5UCF#{aMv0o$b3YZ%bNRWsA_w102wihe2(Hrs$_Sxj0qGgmGqtlzA|Mtf0I?} zt;JeVzBYb7n`I`aVK>?kGFxgqQ7Yu38kp0U^`bAP+1a3(LjeTRoq;uTGXOZ*@zv~e zo>6e$n4N!mSEcm*7jpIE*jWP2UlbM2u{cNES2psL0`BX5ZqlwyoCDQKo}3lTcDL&C zjaj^kRFgjWR8iCXdO+#RS2g7qL=8>KQGtt@I>5%{Ns{Mke8J;#>KMHqSbF@CkG0bW z-_kb2GxJ4UaBFSNr%Iyp8$6AbWW=i~YvgKsoosbaRhI=V@YhC5cgQA<+?x>1PMk-K z!k5jr`cEqrTGMKst%H1NE64UTH@D=c3Zx$vBeKmZ$sHS<`x?_aKOW2B<- zASrI*SWJiin=MY#O8sDA0gX~gTc_o1j^uYeb_Q|YZRWMPg5Hw0>$Vp>*hwQ$K^gd< z*HTAkEt=baTw0Ah&1qH|MF`sHexJCVJ8Vg7{A#ZWn_D0IiL6J_ODu*&MVkN`+tH^)l`f^IU#6SxFuH++GbavDu=W;uWWce}irYlJu#oYsLrnwB zbe~0`>c?%)90Q3Idl;0)fqI6#jxCDv^$6f6OUGm%KuMi-lPaEEUru4;nTE8jC6QpH#N#3VR;Me-h$ffSud?zVV5TyXR z@iuoywjLF0k(0t3$itGF(EMHA@$IyZFY)3smhl=K@TSz%m>YNJ1f)6OY0w72C{Z^Rszfr}c4Lb(PQW#>(Gt>ZPCytBgw1`}mN zF~%aMt)_A;nH{TVC@KH@cH2++DIQmz+Ry={Xlmgg&CgT6izn-4>lW5|(aGmKd1x2G z;cuUU11H>a;l4i-3w=c%l&IfXG{=tK$fM`=mH|>gO zaNJ*|17Dh9Gd^R!8FkZW{#J;EeUtDTS)`BDQTy7WtRL!nMyZeM{=|By@YSFq69jJ6SDa>83q_x0KA`97SzWQ8YI9UWVkdmO7iJ zY_z>s^;+3$gJg&1SNjs9P+!vnH0wQ`r=v?d(}H8E-O{FQe3LoJwk_NS^KWUhtETb) zBaZsQIy4Iwq57x49;au@EIs!($vrDXls0A9h}&+PXsB-Mt51atJD=^o#ZQV|kVJ)N zBv#qeZSB55!D&8fWh@s7dpr0Z@Urc~OeA?v)x`NDMx6$TXtZ*~xC-Me#Z6_B7Rakr0ps5jv&&a2aTZI}O#_COLH)x<(&81MSSO zz9GC00Wb3%oX^J$A`}6LQX_3pnP-gT4ScaO26vs@8Zg_G47M0*i^dyM4Ku zVmtRzD^zhUjsZpct2K+W=8_Nl+F~!0PRVWkrQ+bquDhl}p1|(H=9(T<4+DLcG!Gl&E22gaf5xFqfHuXjLik@~Rnp^vqc3 zC_^|2`SwAY(dX}vyVD$QLUo{xeFZxlf_|QCD1?MeckK{MupLheX2lR@gwPWZT#LZD z#1>FIKg0V=Nb|R3Cm1IEe?1QzzF_vh?m-%q)m@Wze8d z-=a4ON4^yzlY#<$3S`C4Kk2}zH+mG+8H#^uD&NG49TC{ea#<*8?{J~uo|B*F^C`xsO znDdl?CEBmnLZ2Fbij7>C4?Gt$(6@HKT4^<&_%O&pPwepg*Y`}&_e0*>%fO^rP@xG< zAk!CG;)u9Q@&(5#Z7)_!f5@1L*6gJ(od^fhP{vc?&42UcsW%@~8=F1JemAf(`xyOc zLB8cwX(K8PBk+pUG@$I=>^Np{yDLl@0VB*jR9e~q2hWf?ISX7_ARBce9@z?CiHBTl zRyoqmJK_#prfm4Qy6~47s?pEWh4}lAGb1rWB`~bWJH@eJ&*p0>a*QtQP2eQ9-yDEs zG*RbGS7_HhDM_fHd}l%c?c4s@6H>7kMD=$4cb2n(?Oe3T2RD@s8^HocsMQr-T-IsU zCa`~C;;QFRV4k*1@n3%8W4j9EFrAW2p+XKwjsRHkJ#!N&8QO0@AI6<@VqZ0Yd51Vy z=xggMF}>kUyxd}J0v|LJio`wbW}!MS{k4D*3d&9#2y1-$3e)Dg2deheZ{^?{&x{6p zOmK3Uq156jEKi|lMHnl;VzaVgKkFUa*L+n+lfw`TNy2;R3j%3z!WzyM6m8%U}N0%}{<= zMZ0A#3kY12hj?M!haBa9*~Nc0R=q@ikwDIxaZc{WBRfH7A2YF2A&Hf{E^Ny|m9pEb zA(9=6IE)1BhdU19g@M(F-E4+UOmjB@FTjYnbrkB^9jR?4N9CWrv5SG@`_=BBmh&AQe z-r39AM^aW;Sb^#!dbKE&EX#1$D&%8RaQzrk_VHlE==|2Z4UB@B2i1#oN7xLKgC;Zzl!6fU&1 zLY(BKG{Fm1t}Rdr*!Dk3fQ@dNSnz&b!v2uC$sOJhUdOSfbt7pmc_%Y81SP09pz0j% zxOaG%nRP(P8PFZsRsOd_WdAcC_jm~bG}Ya6{04YlI3C>eu5BdL!eHNkU&~Yn^(bW= zdtU$xu5&vS5wC3@_12e%nKdwU8ANp<-!P$F6`f=WSSS~8 zjInJuTh>I0X%#UY9h*!qgW|OwM zl?Edr8wBXC$8EK5=A+F-t(&j;DXf_{jgls)DHedkLknB$#MJ~y6uiCTY*0)sZ>>*a z;|a@3-85SEd~KBPYC<)e@z0_w!8mYpwbtU_KP#u7frzJTWa0rbjxp>k#ot#AurGtv zy<`bByCn!J3WSR4cFzUL-&w1s)X4SIFXoSqhH-FX{&%WIhzA)@aAv_vHmj+}GVAEe z*yAojM$%exrpJD}$xc3AS=*ct&REdQp0UABv|4Rny-hW>^P^{o($PZAMU~?*{Ik%> zUWIi~k*{xeg_qJ|+Im-|{8sBZ>s7Xr9LFmi{R$~c^|z(|Y_0CtczZMnR&aTE%aS*_ zZ(sENGuHiQt>0heDv{MG&jV>whl%=-{j#5|OtDIiqAJ7u1qJ#IDgF)M_?d|&xIsab z!Di-{|Jnro11$datK>7N|IFTmr|}ai{;A2>bmMfO zUP2*w*Aa3U^t$;rfdZGnYNP>Npw=R?mOnDAytMAWUmJoY*EAYawOr8>U zOxSnrjr*x#W9>T#tF}j+^E2Oe5hzO~t4zZ|@<`M`2Jsac^$(py8%bIdBrZE2J@(hU z`VC^NI7)mwt*OcRF?zWo`k#3&DGSu*Z*JzKfXC46ta3j)d7V`|1Y-i>DxU`b%K`b_J4Z0uaG=+G_M}({C6LNiIx7{~Nmb7rTB39nE8@!+&PY|FaPzW;a0P0AtPm<|O~OFZ%?l4Uv$4_b;8@ z@7>(rzYMfKr*f>>iw)gx0=Ksh+Baqd=GRCisQq?{sv14fBn!NgEsV#DmJyt`zQaMJ zX1m+$NxppP2iCpK>2ee6RF>qFO`f^$d}fp`6C5pd(PX9 z?nN=`%&X*|^d5qii09^7s4^#h?7;?9x8w5~0VgG#;#P!~9{}e(^(&umsG89ay%9Cv z#@kg`ypOkk482lZ-e?Txthq|=1b>pgPKot|=o}yFTFlR3?#x@R4m<*%%1k3U$ufN? zI;tMFrj1u5TTCqTPM>S@3_E%7W;yrPetHZqS#-|S-vIXOyEp4@TE`ILHIL*Oub2z# z;vh!Ycq5m`q`JG7Qh#!}Pnj@T&{l?I)NqYSi&@?JFZWND=zR!W2)9<7x~e+Hmr!uE zR$EMWKhNqy^-<>`uKK%-yNpNYs7-d~@q-Jxp&f*^Y2z(7h2PWs7l*WK=OKQajbcUJ zmvqD}X&;WI`~VQ`p%jFfQ~I8uK|a}$gHw;wBhO-|bKLXzit~s~>-&!zy?JljVo(f9 zI}%K+g!9A4P8hiw4W076cQt;q*iZ^(;f_LAaZMNv+SyQo#jmw0;Oz6)kp3%ayZsA0 zxPXrW3LVe=9CR^MNH9lQrot7bz zq@yO=zVAl`#>SCKai&;U5xFTqBFP>8>uR&2oJT@EL(u-k;~37nH_6onm0Y>cz?vMr zH4QS|D`>>Cm2@T^jR+{`MAOUF%@}SZOb_LTNewPS>kk5E@-d!ehpZr$tYBsazHNW` zj=Y*>kAVo`3P@G$@&Py3OAy)Ax=F;XmQD&6SC@CyOQ6P{AJM<-6JT$8{(oTR?yB+V8yj-4qr7r1+__KR#1bQQ6A|Rc9nltyTYMq|#2=zASkD2>NHhy=dd&!N0@_=G2Ef zastP7JKYi*91&t&7!rUsUV2;C&EPgTi2Yn-L5D0q>tu*Cw86~vy*;%Fwfth+G7|RA)rhlamofC zHT7RZ78&u6z|Ofd++-{F+h$$zHn3w#COOvn$Q9cQOpO?R93Eak+iuU~bQ7NRLjl8u zyK<0rFOPu^9`Szg5&s%IG@AHTW8qCqzEPVfqr=9=rL|QBC5it?lx-zWaN_crIQPoU zpZqOmF`>fr0Z2Gehn^v~wd)R_&;DXd>}r z9)JD#bPe+etn4(ct>G~J8scAP>pahT&o_oO(lxv(0h=^1RMEcL`rA7yVOn?^O(V`P zfuOogtApOtx)+JVYq@Ps_z8w(trvqlLy5ZCO`#`Wn?fA{%Z@`X!^Y`wGGcn)U`@{I z6j-O}9R8^SZ1)p#t_>bE>O17kk=Gv7mFG6Yko;R%^;5^fYdAe{n4bFBU)H$xirBT8 zty+K{S<;z=lis?2MS8gHd!D;NJmrg(!u(G>yfp^5Lm5msPlWWgp*cZf<1_C6uXIKz zroAhA598S{s!u{kgo+js77qUOo69^5t;!7`$0-`ej-5B~3oXnzO?!!HZfx}6-^O?^ z2h}@o3nP#qmDc!MaRWXjl&)J5z0;fLhq84hr^88AJ|wtSG-a}|->9H8MXAN4Ka}No zEkOWPNw<$WyptFCN5+#N>gPv73|Vq<+KiQT6O8*t->)`m9RgU!r9B<_f?iN<7fRR_ zdb)jCRAS+Eme71aU=EhN5`9zC4GAQ<`Vt!)&nD<06d!&k1Uk4X^&Y*e5V&;(k}O@T z6)x91gzC1TZtsrX6~x8sjGF7Jv(W4mPAJJun|rz0Yqk5mp`1MZzPw!?X$Dm2$i$8? z--*81tgWZMmp}i-GF~(Y9mt&-n5KuLB50nSP!>++%GRVc6vh_e4oYlLdXR9A%$IQ9 zLYz`vAu$Z!SA(x1HT~0L!R+qqqvF+-)lA;}c@97VLA7~$rNTVx@HK?+Q{ZICovV~E zH*$^sxW-u;cX1r<9AM6pik@a{%wvCi8d$1&^=|V@pTbRt&p{Yo+T>GsL?vs$LqpF4%AazTI{)17_ZWz3?u(UitIQ4p?# z!A}eB-CT`_f1i(g1{@>RRllj(O-Tc0-#le%J7#&}&vvpF)=@G6ui;K}ab0W#8VI4q{@YELK?P z>Zyi%4{%Sqg1Mr%F$2K%S>4SP@hwZE=zF=G%qF!c!!Tbtc1TO3PfjN6NTSspG(deZ z7=(4pO^kq6i%=Up>%D6xma#B&@#TgwQY1%r9ismBHao-WNj{%&r-)GQ$z#kZfu0+1 ze%1kd?lxvY%6CgiFCu9N0h6ba2lhhMX#dI1{3Mazd&@@}eqZ+!??!|!KH-fp+LV_Y z2+!RZn@fNRx~UrlKP=-E+8TG$`Tc?U?gwIlp?z zl3Px9zLpWsezmOGmeNKPT5=&l>o>BD4PC91Wo$ocG|8Oc+7A-U)=L~ z+>;??E&SwR$(L>^WtVi=@v#LXTzfHMY$MY}IwyQKOtwL)2zjwY!N8!gEw{olB1-hU zGk{{87r{lVsl$WHh7Y%`NH9l%{RC`QX_O>DI6@?b(2mcH^vxhcpbUFW|l! z{D0pIuFp0JYc}DkC+AJr>WVlTsIt^~L6dVkH|EqQSWMzw9g8DMMIRh3R8*Gp*=ez-_cub*DbdlwjnrAZC>{Jl3&9r;Xjjne)m?1<701NV8r*%3lV zzD@~E5|&nl+u3FdI+Q0;@YmFGdV{T!#su>7N0wov2+d=e`J_U~EKGF8c!>N1QvlWV6;|sk(X+4Q>=vrf6eOzCG zY;2JhkJ%cfBEdebC1@a}ELPxPW0t#Yu5ZPfs@Ddvf*h~zed@N`5f_|@HE`(|bFx)( zFD<-vIzcC%3ZObJL)K@?m*I|RR4k) z&_8Yy%BqdJ^3-w|PQ7Gtw=107guJK+GTgi==N~Pr2HisZkQqh~Vk@TZH{IQHf|+#{ zZRKqu2UvzBRnUu>JyL9pv^;Xb`AQCZzZJLk_rpaLz`Y4WG+iQg6GOH#edSORD_hbt zSH7%0;yow~hEQ^Gog!6W2;}_$YBETshzfi z*i&n*|HFBTz&hXQxonSm&wP!frR?}_$LA0gXs;rk)bfRG3g*9Iy z^JnWNbi+7P9Aj>@jwj$m0qi$)}tj@`aToc3zk_##3_N=#Y;x$X{EmNE+bz0EXimkT+D}E4F zsnA7nDWb?WUnXlyRwOkYU0q+5vZ0`vyBMG8V> zIE`t2ZZTB|IfsCzN;coGgtAxO^D_4#^-s=?n!&qRoD{xHGMqT0SWDzLEsZO-8}Nxw z>FC9Oa-L#1P$mSYvf97mv>I~Nh=uTSOyw8qmlkJYm+&Iz5N8hY^e>Wzw_ z&Fr%CSUvvG1gYT$#sS~{EoI$Dq@~tvhDDNrj@yl<&Diwg= zUdftht|wZh#GrvfvK&LEbw`ibWHzpjQ@75!X9c~^98O!h*%~jm+sVD4z;;xt*PDUA zkAv+=@6@UjJ4=A#<^0-N(j$@3nnsfdf$ukGkrz>DH9T)(=fgKs!&Tt|Lf&&+;rwD< zFapb7LbQ!?}ofWoCHvDIvp?s$m{>*%Em?WQ*i%Af_TMKz#QMGr$2X>X@6 z)C_lUr}TRwtkeg8vr=fWH~iUo!^)LLs_FtZ|4EiQ|5_XY`gEeH8_=;4Glgf&ey>WH zHD3A~A444=iy>_WH;GuWyTK&=oPoVAZRAx|xEuE<*ax~>OZ6mAopwAnR#!4I1t%1_ z^@mfGi&fL{@-o&M>CwT1nrsK!xE=h5=9K&2Xx*gjrTKk>rhH$vF0eS-VRLHQWwN@- zOEe|co}>5h z@a6S68VT|9lS(;OvN?R=@}bpX>Tnz?0OrY7-iGgR_c`QJH%6bV7CcSe-{`APER8;3 z$J}Fy6#zuVt)*kt2qjgtrH_BXFfMLva zY@8xM)Q1foiM3+q|CB)gG$zEyMjHk7FNSw1GbEgmmH6dq1{_0~dd`T*LHG;n>&>n- zlad$5M&Bp8`Ys59oD-!;qc|#H7ThT+mSX-pmHszGWH1TJFqwn_|J7;eFaGW?K(b%w zJ^6P3@$K*7?h#j?g;LH&%9M|RzfqQUzE?;&}>lIbt7(L{M=I?Zz(!L@(Ig~neg*NU|`>t@^6VmH^zZ(41u~!^-PLJ1qm5diq-sAge z%zvoc|5g&x4*p~oIrvr%&_>zV3^)+w8*d3PCxt-_r7X|Slnp~gbw6HmTN5?@(c`zB z4(YJL$^GakMMus7+RS(Y3rfl~gDO)fUqXxN2D62{M~QED51{4fiF6;Ef5WtXcn<{v zIzJhS2xZ>zk8b36P_8EK^Gn}A5kWqvT(V})gwO4V&wWw%AWbu`&gby&ACLF5%#h5t zy&4e(nvWd!aK@zSW=FxfjYm-?{=-}U{nv3?rpzTE34S~qo2`Bjl4Bm{g~$en-C30P z(QR2wYN|C3H=VJM>KiFsv*Hl2?D)$tV#ChR!^7Lz2_p0uJ_!IQf59-a<N7LC$ET}H_}I(LzGv=X&@b{=Z_P#T5 zrq8+nCFaO9EsUR1#X3Vo{GZ?n>j~L%s8}`~lpQBTAo0JoMtQ3LIhZ-t#L&=LRv~>x zl-n>|IW??N;U7)#|F$^Z!`U?7DNWjxp(4!CP1XRXmaXXI1`fUt(Gv8d%699I-x{Ge zcAcTBs6cEN-HQOt=561F#eR-U#NFunHc?_nnK8ugdBKwZ1nEOzGD^YT6-AXHq(vgF znr59golH%|%ZS<>>aC`kv{spawXXQRrl5n${vhNmn~%Sr#6!=**zb9|Z(rjEQ>DzN zQZ|ypTejKI50_C-_aZ{5Z=0T{oJT(GXYGhgQad|A-WKMZ!#U`^L!N^^lnUQi%GyU1 zL9Vac=B0<85YHPD-ox`}RsQYk|FW6(r?7>myUPcvi5nI>z?0?igGS&XNWNmH>`wG7 zHaW%ys|8*96&-#w8aj7T>^NXp$}x(D^+JwPF)^vzVY6QuW-bgW-#JC!`Z$`hAjxK) zeNn9Qy+?_lUp*f+d8+VkP`{>mBzIiLaC1HLIeiOQz-w>IB9;-+CkB6WZ;J5lwo$g( zTMF0PzKh5xu`CDM^0YTA!pzuQUm4Zisd|IdNxjbEYx6_mxtri-(l@bkkXUDdV|*_# z$Ypb7%O%Gxc5bopO>CDlJ>JA@)}0bPW6N_NWqka4nzM;|1Nw0DlM7VrL%F(*h#uHN(jH;`0o=9qT(R#3r z3nATEivVwZE%C{@4x=^IYwz|gyf{@_L)u%_~0a+h3u1n$Ski_1>=9 z6?`FDY>lML?>IT)+uTSoxTpaT-Y*0@vCO>mG@?EFU2@7WX z+3@e^WTrCXyH3tdy?|oat!*VNOhP7?UXV614^M@i0|tTcv}X(-t>+!x@G z9BT7c^6~Ia)(%t(423+354ZTDy?#z{<=r$c=$<*Gi z$fBO=X0xz6cjbJolQE^_qby@J>G{_cx#geoA&y8YFSd%2JS|k;ZR*g&yD=Jjalk-qE?yEJ?4Ui=&<`mW$*W~w)D`}ZC#DB3`Xs#?C zueO9mGMN~A^0VpiBlF|^DIV7Er*aJH$3fb=HxX$GJ~4~#hx;(BpHl!Ts$2v|6k>xQhi$u(9DcwcLZ ziwcHzk2Q+#cwMO2tD*SvHCxropgwN5w8VysZ^MJBJ}$RLx1Ir5-)`M*y@94C`wKUN zj!wWdTz~d;%O|A{$rg5-BBbt3b1^$OX0t)MMP-^IIqz0`itmommfU(EE~k_HR3LS< zqJ*4Jy$0!W+nF11BrkMQt~`K1yoGt>6+E| zCmQn0kL^neW&3T$%t$02&NkydJLDLJhU#gP1tOQdP zY=xg|^8GB%VZ9?T_uJykOL$`xZEVLH$vQuL?nyrlbBf5MaDm;jDDjXuJ=+UxylKWx zRc0QSi#gniE_N(QhW@F*2o>fgNl?YIUbZ*3matH6_gB1R5tDCPMy5^6*9<^+kT%VU zi+wVitkESg?fS!txNlfo&sUpBUcRhX1U$ki1%#*97)G3V6-Qn)2%CK;4bG!U!qeX6 zvz>&DBUET433MsDAW~ka=9`u(iLN|{qY`?EB<~CJH2uNPOEL&-Wfa3}?* z^JE*A>p2}S_-uolV%4{H&{BJrf)m&3qQ+EQt5@xt&-&{4$b-1g$s2SwV&9^u5ntv{ z^Lqsr9C4k8;2&>@OPb@;6y6ejAzxA&?|t^%EnMv`G&?kuHyI{H!HDB5krU2%oypIh$$hg`bzi&d*Qin6_+06!%v z35Fq;&#AEar@(4PxdmXZN}(>O!cmcDOg1cUgii%ST(I+vpmrIw!!xJT*QW?U`M&iM zzn-9w6p~Tbs*!}06kKr(ONDm3{DLI(Ut}t`KP~Kn9h{_OSb}=dGwlMgRuy zI16kX&8BHL6R~5?dbG9RIq^lCe@hB=Z$-VCB3+xT8WRl(bPU^V3}w9;(I#^3nPZ5p ze=H{nt@$HFacypunZq5*do@F_Fm0)4^MzrO41+s9^gBy2^+3E@4!Bpl88dXW@bbtZ zoY(xO0eywJpw+VKpaGCFo>Dkdxbc2E~Oa(satEx=o;0 z=CNPXYudy$7__@DA(!UDAuK9!E+YPESht)@Ow+8J`QSqE+>26g3fl@pCqFeUxA>je zk8%Q7b;TH@xo+Kp02xZk{Wyf1jkK(Z&Gz|J0}V{DLJ|Z~0B<&2<>P>>uE74~(qR4) zV`~hOkxodH7d#I5^Y?_yKE(*f`rY_L#}oV<_2neH7?U9c_+WZ#JN+aed^`sZBR=ABKOpbojl*Kwvo(5C@} z5k4Ri627^qS6G8E6E7izkhr6r?F2uWD}K&e6%-ocu{14kVWQ|$$~AwHjLsjDx?|$4 zPY7!5Y5uiVD{un4uiVyBk@-UsD-bW<`a0Y5w=X*(gAl(KtB=@kBTB_uepkZdJL4%- zv%#z~Jk^UXcx)W)q3|@Ur~(7(XQw10LJDl~o7o~;jNm#|FzwG?2BY_=q(nzm73+Ql zz7&{PCkH%cPwxFZB%0wl_eC{C)X`F5b|$ojR)&c|=iSSfqK#!ca13fhH>NA^>}=m5 ziL8}SzbfiBP$1)ndxxrXSf&^>j}()If7d|-dCRz8K^azI7l5S{S1;ZleACrn3VJ8( zC>AM&m&1qSb)kd7d6^*+*Z4w8U^{Hs*tB&RlgpGD!>O0`Go7l3i2Bmq3ac`Rl7%eUf?EMI8uvMN`OYhd&zGvy)rrp(5!Xd(7RpsY(%x>5 z_oxpW3~tZSc9qql27WqKzZi_b;j614$vZVkqO3XA4Dr`o)Y@@&dCyGhspbRG<`Lc+ z*)OfP!r9Iy%PVhRynbz_E)LM{PW?a<4_CMFzFtcL2P>@q^@j0rVDJ;O%oWVn3<`)* ze(W;fQ9v$UQ}!GAsYO)i*)TXW!=#-&S;bF2(B-^X9TUt_)5?8n>Q~>~x~4^GBJNj> zGjf-SL%Ge5W>$@luryPI0zGMA@3UOt9pV+dn{Ut~gsG65@hg5^Yz>km?qp5-5)Y3+ za!K4(5A+5E?NeZzFEVG#PGeo_NiE+wkc8+q9=+WDxH)D>m})qF@|!^iX2tB)tJExY zBq98UN<7X$f80BCVkc~W5d!qob}wjW>bE;7CPS)W)(CNHdMZJ8R`mgY0?&(r_Eits z(TRE@^<{aq@|YxPX-~g}`8Q0*2r@}GvA#OUTO=U$yAN-0T?_1DVV+8``wh}p-Tz(_Un*TG(6PV!Sj-^)9AC}(qX6ZN?tca@%vp^xHJs+T=HsXD90fG zdNSGAf#0sKE}xlcQzW$2%R>)TA8F`|P4~uq-o&uRsh-MPZBfiht-@w6+I7a|r6bL% z8&@>9=EJ?mC!1MwSxGNYoe!jf2xVC7f0Pi!eM}5mI09+tVoi$Z9(kQ{+ zyDjJs50RuG(-xRI4P3Z`YPolwjqu{T7#8} zjx}uuBwUb$pi2!NMn5c_)K~^xpe6#lYB46csja9|9%_?_>XZeRY8jEJ3W`|NN5vN( zGk!(=d#~F|U(!B2Mpht*Vbs>P7z&pAI8QC)_Y||7kHTbXW)&O&2Q~iu>a06;QLU<= z{MK<(+&Dw<$%fQmISyvKBN0&m;PKTJ6^prc3KSb_E*p^6 zf1OlV37(-2orrh4^}U!gX72H*CQa-QeSVJ47}RMBe{&6@hG(dau4ik7dwFb|1u@-k zpj>yIrFvR7>vXH4Y~8cGOW3*7WLaTeck)@IDZ%TTX(UKsO`T<^X7gNbh_0Sl61Fw& z#fA|xboB3^JRzsdegwO+BcK$HdV6g=sjNOqRsQgz%mr=XNd!=1X1L!_YB71@PA(|3 zfD}1Itx?UoX>{ayE+RAkdymkU2Dunj5=C&RrRX~8-#wPk^Tl)ZfiZ(N&7+)|vY2+K zUya>#0R9gZ?wC}>QgJemHU|Tfw^z0C@Gj_(rvH^&4Z*S5UL|=`92Y+avR|agbUNxf zl*-7yFl|t`_%&r*Fjel}0x9nbzgTEdag94F5GTww1xA!bZNlBS zL-1b2hxk38eWMr205|J$&{6S>n-c^+vOoKfzS%&hVP6(AYV3*8y%==b9c>w_Pt}vPW_yjdQ6Tiwp`TURt zY5J?DX;{oF8-kd(au;D>+(52YGbHcMew1(&O@*yy`dfsuSYOL zX{{pdVov-@N9fa>~L$Gqk`K;~Rdo&IjX78oR{7159O-7eN~Msg)UEeHl7 z?z|rXaad)oJM9&<)D5{AA07JOV`^&n%sNaMT>QL%Gg(4|?K$mzatTd> z?2tt4yah(s5r-_%6)+B7op}xDYcSkOQ_!`6-hFhwdJQvT=E3lNKEW*J&9oCfH8DTK ztSF(LWsEo5LKSlERbuJRi;(^KLWRanE0!3WFGnU<+hM2?bc{l|I~wv74mdbjjl`b< z9DBLNR-fm6^&F=9jzttzMmbAR?y2w)2?tlnvpDHgsHX$ss+Y(7;*@546Hcw*4(rA)RuNv0I5ssr_jy#kpea|K)jDr`W z*%sK;P*0-Xk8|O!@5V1CC=f1oh1b!dezj~>&{2zEqcpXMJbO6|Ii~b@N%Bd2@7vOc zGY6j73&Y56u#M#8tCRza582z1HpY&)rNqU@&6H_0BK5%1dt zJC=N7wIP<4$62M5A8CDbF(B%@8z{=4ip{jb^@ao^7{}Z$^1lATRYfv-*x~(UlyOyB zx)Lw0$`-bxZN7~s<2Kq4znSP+&|x7XpXeX*-v+xk5P$Kn_jE$y=ElcMetNZ3{p8~= zp7Cz+T2K5lmM=4#R1Qcs;@*5T_)hE;zxG6ka%5j4gg+Od2?tQMJELO+cid0TVC945 zG6=ixqkyz4goTD80h+djv-klkYttrB9N$UK1Gr53_KT{`;M{6T5a`A^PQ9Lim7%mx zivT-bveP(W@CvRsWPm-28U-C~bzAvkuM1N2$+s*r#_1i(ZW^Gvox%o5cV>Oqa4#ZJ z*6`J44-?I{n*3Y5cuPzDrz~6Va!+scLN&?7)YJ^MEGXol18Noe`0#`1G2Y@OcF!j@ zM8m_TR*^iRVIG38{p#bBtY`|Q|3vEFi&35o9tK{SPnXmmkoe;FKECRC`^ib`Zk3%j z1ut;gD2Bk4tgIaInJ!Wi9Bxj&KD|-G-yp!h z4D7c_L0^EWa33)$RNhkjcEEr6=YP3;^5`Mj)1Zem(b+<5umW3mYHBQA2WTSX+_|s* z*zljX3Lzo(f|0M?TFKxvSWjun7lrE#B%m9o3b;#eoW=7DQviP=0UXsQPZ&?qiDeTa zs?6nZX6h$Oe#<^b4YZJ>f&`N@HBp05mQ7Dt2x1zLbX$p?>1>zz>A@nhOKwL-sk0ukajy(=-dHdJOx8+qDNU9fztPrA@HyjecP55?dhm( zPr@Gq`@0!PAp0GCeWumQ18>w=#Ovby@YwNDXy_;9)9yF$6Gv4D@_L2N*fl3>%vN;A z2h7e}cyWx&+ejV~?@JJ$aNutbLo18^8q)jd+ciNiTy82(|I7Uj8JXDMXEE*yT~!*t zrGS(yAHiG)bVGWMW>`bj($pgwkdf^?j`r#y&KJvbX=z7@8%>#0CX+>&NhA<(2pQU z|FB1Wx0lX0rxuRinRjh5;^X zgQslgYO%2SNlUH|qonh-l{Vc*y$JCW=F2%c=X>#_NW`KYW5Qh7+D?HUes_OBji2+7 z00&*SiEsdu|KcC~-XewSZYr8I3mGHaZs_r=#>N;doQwAk1JcFlFnw?9<(P^MTAC@LD$J;h*LE zGX;51|KxbOsa(kf_HVNSbqU=Fhl3Y&8Jv^|W9;ea8^kSK!Gaz`AH&3iN6Jw?r9;;{ ztTi4^2-NU|0C2-CSB+76V8>Vze|)LJZ3VT({Uqzo>;AH%*Rl2{cN0(0+5mWJkuPt( zg2>0TO^ZD)R&^F0NlVw?;>4(=XF^r|kKEU&sYA2wHW|zBHl{~BFy4|$Jy7DV<=3V> zrdEFK#wd+?v6iC?>=c=LZnad2mX(KQco(92Iw$Ve1^3o@8j0@~z}|n{60S|;&vm^` z%XO-OzWclgF2qsZZrq@_z9$rc)cr|!v(-Uy%$XmjuK!IbD-K_4qr@yp|LeYw772A4 zqMn3SdGHFg-9C%2B_s6zxXqt4wTZDJgu!nSD*P8i_EUb+%g{ddj$}8^BnG`iD%Ont zuyF)aL*%2>(}%SlnboiZ_imW%BMe%~ew>(VI{ZtxrMw5o<)67LKAhmC=Mj?f)MiQF zx{}_&3NC6OH@R%>?#uB2KcXE_rW;xx);-zvyt`) zPQ?pbZgxXESyj~3s&+E3t*#81&C*Y6T4Ba+R;Z6c!dOxY7?s9S33xtkz*+L|P^rw! zEx{-`rXl`6#@;%r%5G~PRuGi#P7#pq?(Pr}>28p2Y3T-O>5%U3E=lR`l5RH5??#W$ zBj-Ke7~da`0h{d|YsQ-MTGyOYE8hpCd3n-jbRfNHGmms_M|8ke=c`N_o21AMO2Vsm~o?Y$fyHv)q#m6B%rIr;|_G`fy0u% zz#h;(|3d8!dbyyavrnnFX^rK$@YBywf1SoOY+~JflkP=ovq;bq#_@?Pl`+yAijQyw zBl`6rTT{7)bdeQ9(Pp!6fWskHRcy8691wN;AupHrgJMHJJ8R$$y)Oza3z@f&=~vF< z95YEe%M*_Kg?pC53Aow-aYd&?;g{|=DW-Hmpabn)ESRzZBCtSW8qHf-sdQ8>ejM1) znVuW+G&y!a*Y+%tzHakqr~pDX<9>yNCJ{?2D!P{wy@z&u*s2*l*0_kGmUnevD|%&+As?75wo4{ zn=Q+TZ5_$c#LDpVPgUR@VSMYfqUU}} zE_H_uumPh61hrn0;36<=cO4}p$YnfmZ_p}D3}B<=12gkt_VY`huZ6B%_QT0jq2Rk5 z(S&e{&@m9*h2mIw7$jFd;JR-mZ4$S3j84)siV4BTCHrT%p4%C(I@+=yBwyGPXj`en zlkwt2#NjM^icUAv&TLZM`r(p?Y^l#TEwXIvzLF{7aXg zL-;MSd-`2~fd2TWuI(Bv_v{wb8oo=(dmUV%cRWa0a3ML=Yj)(HezK33Pzo4VNDd5; zGPtjhnWnl#?WbjgD)QQ9-wgVvbPd_353eKd*>^f9G2<@1b)|o=hMcEW5cHT=6QTl* zH_Frx)wpu5s*l1LK?!rz^7N7kZUuv-F>(1k8x-gg=?~v8w zSHoyeMCA!Kf)QU;;x$qNtpG$yfBc)6gO{v_?TZfu4=7ubHME^CCof71zB|q8XJ_PS zfD1n}ae&$--S^Zr3pHhl{7y@op1i-ThgRTj_z1?HeII=A(=K^a-B7JjI4jG^O1N+e zQ=@fbyY2w}t~Hep82YQixIV+!C}^1PC-|ayB*<~#XK{axwe>eshPJGPMm&cG+i=@# zWRD8B_p8-l9yjBGhT76&&P>VV-83ep{`QXGg%XG`AGZ*6lSh`UB0uVjS69<;;;Km~ zevu%0j!t?UbL!4Gm8v57%mwytEr*TzdjpLijYx~y+|N_q2&wRqc`BoOqPb8zicG7Zebk)eUnvWl>2>DovF7BvF~m9WB_IsP^j z7`a2E85E}y2aKeS?Di%<)L+uBNpfM|+22a^?TM`%r2Xu^|GXCv03op#6#IA=rB#^Z z5&i|8>qa?lJM*o8N!SV+dr`%#y-L10-6?bbyAEFZsPR_! zxY6NGFufH)PMf>zi2++Y#a;5+45O4rubacB56V6YXYW%ZTDM%tMC_ubc@lVioL5kJ zy}!%ffcYcMJp{on{R4KPw(0D$(I%{Yh%z;f)x))f<0^MmX>!cr_tnIuXv_$teo?=4 z#Bd&ZMnnhQQ4hbRMm6}U&x@z`{e_B)D`S>*hOo8=Y90?nMTGMr|0?5SBuu#ZsHC>K z-sV?6e|iBJ{v9HF=_5rL#=S$Ea$!N5prOeQ0rDS%uV2s}^VT%vUbQm(QAAkfg5YBT z5X6*bk2b5ja-%p@o%!8D-^oWROlsKb;?mZ@;`Jncv$@dT z-O(LD+pvpewmon~Evam^XN^BzW*mY*SQ#>0Eyvrh3U946e$61v(;rJMxOuBPuG+P) zpA`Pp{L(@*?dJQso}OSRWv-j!%j=Cu>SF5W_c-jKZuqi!KOdfh6jSs9+lsk1$#wtmKy(!>)UVhxT=#pyHQ2hWF>32w0p#&hQq zlw(ifWeDwF;RjM!-=rki5NSi*6=@iq-V3eCk9ug?IR))x+AMuFhdl}V9tb_Zup4Kw zZaR(_ug~oQn72k738D_Y)GPMEn zX<&8#2qBz&9Apg}n=5K`p z!bZ_$v+oM;W22v3`>)&lj~@w86jt+Um(^-GNx(h%0GLU| zQR8RdY$LkYk0-VA8lH#V*J|n)Qj^B8B(8fN-v;CF`Vfu^9tGIWA zZS4c)p0}ScVNRXiSvIN^(^SZ!0Nt{fOpHG;W`nunbw#u4pcpm0{Ww;t4djmT0A2!v zLM3MP3G(?Wu!9k9KwR?Vwgj@j6KcWJlPAmugK(s={|5ABw+JxdkxKf#IuT-Ex3`j6 zliDhVYJ7*g{FaR~#k8lg2*8cbjEyE4svmDO$ttG++{m=QR2{fc?cXIP)h^=rRyIEx z^=>kTckh&62}0xVha(`Yj!w|)Z=$A=Ev3zQ-0nR`jnCIO=H$ii=+5{Vj*8V_2iqTK zX}@7Dwl0U{&1DBJ#J+mji^zkM+}W6v7!kL-G)iyxoWSjxgn^Oq8?qEw0&Au+Y!=kG z#(8(J4YOV>?v4F`aRS{xcJ{a4dHV6C*;-?_)dl{orGbQWVG1SpV3r>DBo@k*=xUQO z!~*GtEwEvXxs7flpM=D}-V~=dGqD;cK!)M9jv$DxJd8FgA*(oQpT;u3N-0-rBr8g} zF<_3J3c~)XYP~<;ki3D}Mr*p@EhJ^HK@SiF3yXh2O*WJ)DqfFZQ#ZT?>;k)l=6zH` z`%LBsJB|9!dpL)pgWU{E<puhZi@so)^wy##PB=|5|+zVEoHcU$I2I7$vRE0uWA$M=dtC}(Aq z%wJh8^kjVZLZaLH!hR|Z(kR|9=Yj0qJAC$VFS#mF0ID8D*{+8{3UanOEK%u#~YyS#UB8b6M z*yZ^JYkQN36~FrGVjm^v?7X|OoT8VFxiM=GEtAHSFTRw2$VEmEO(-*vjwn_`o!an%xQ!!{SL>h*-!rU2}z3<@98B0xX zGz(AqT$Wqyr`0Qr_F9KQN)VWY%V_jiAN*zcr1V!ljhnj=t|u5XRAlvS=)Z2XX1G#r zDMD_jR@GU$1CzNMwuvS=J7a~%T|5@ejSYD>78vD1`T5N2WAxM^hR2G%r?~Txw9^P8 z7ygZXt_GFcwS9bYNZS)&;XnHsUl5Fs%>`Q*)RU0b4%rG8VR_-Fo=Vy< z!rNf8YWk=bvXEi(%0>!qCHFkl%7h7K^DTwkmmHoDe9?y$kIwJuQ1d#Ap3Efqmqd#z zaqxP^L@zf7dHFW;SW9d6(Oj86p(&Nnvy>J~K&N_U z10AbVWB|@``<-I_st!*~CL0{1eyKGDqkTYjT(b_5)^HdTW!JsjihcVe+86I%AWlhm z0evfNIlD{QIvfju=ZrAqp5ko}$SLK~c+YO*Fx`$eUi4Aaq*G}v$Q-nSTQ);(Eqv(Y z)<3%3Ap(gEJ{XPmBYO)$0tl?&jAk3mjTg=`{s!IalmoX_^5{=zP{S`IS>AV0K?a?* zLeAy!M;DQw!A3kYJ5Z7Dw%)^n!8bt~OJyU7F|RH6$V|N_cMWR_1M%EJpI~CY3b}IR z4Y4=4I=FsD$?6eoO~4AoA_Dz->+EN@K{qx9t(~T^Dot~W zqZxcE-qcsRg3Gk99Nfb73W7bQjoQ>)UjQXe61jU8_N9It4DdfsF-udvd_=IzHV;I1r3eFZt$ONPu{) z7m-SF8{y%eGp?DVq&2#F3#O?xuT$8>lM7y`O&*q+sW|%ihohZe|@iRq>-@5n3 zx(>#S?##_2;{9ccro%Tr3IuI&ZP^rvo-ys#KRWpHJaG4wi z{=D5<^=3{kmmRs)YpHpQbFVuldDsK;HmwH6|09c?{mc(_@|UG)nFTN`WeewyxiC0~ zbiJHasf_5l2@UeKPRG38Y|LIJ#}q+iUT%S_S!sFe>zQzG4w(nJKG#N+h|8xmy8 z`gC{M2x<6DV5Beljw4M!1GMYuyCw5?4Dcr$h*$*snj<@w=u-s*WPemsOv9_LRC(Vi zqxJ>9BQ$+%+U*G*83v;|+m^y_CiNOjQ>b^?6navP`Zcl3t zd?DpoQsnCRuw#P}5z_0OM^-R&ru^r>Si#ZeJTN+&TPVC)aG$vUVg*mdQP#yo$EIiO z0He_}=!rz3NYHKOE41w%&iD%#H&P%vezT!3>lA}H%Uu&uE1b!>x+VfS;~3w)uZBDs zG(mS$4TLOEoG&xv#@DJKUEX(URIq_JNx-zXtV+G2W$V;kZgf)!*hV)ia?%)B|8bj1 zaE)cmh|;~SM^)n4pd_>Ue|dXYLvq~2Da{ELoDUlAOa zDI&4nw4`|^VS&iv?C{bS#{po|_daJl=`YP5ct}E6*?60SeNdq(ORuT%)wj$2030^V zWy_3Dv)D{16cma*%Ys_Z(64E<{8+&8vc=bDuN)CEdh%Zuv zl)Y(v#0BudILd?y`1{ywn4n*&2wy$)Q7xf{TFQAg-z>}#n~Dc8^wu`|+N}G@N#qY? zEA>{Ofd~?>xv`vu+bZ4wqBjavHz&p?!zh3S8&C$mue!a9MG}V1Ii_qPg|&KdW`i*EU7Uk; z-S60DRN1o!g6kl%!DAJ-1!#8ZZpglkoP0>hB{O*-o^HOu7x^OzGw9zih!E`sz^YBH zz{QW-lO~N)P~{r0I>DL82Uk>v*HqHx8VFMJx$5I{3t-D$PeI z*}MF~E&c4KFqIuywKq@vA*xSbmJN|n8b^y&CXQUnY`BKb-PXl&bdQ=9=pCgE z5{cJqO7IuEh(#z(TK>v#kePhQ&{8sD1QZT9s#$s%_RO&sS^)@!`#aLFsTosP6f)|> z94p3XsA_&I#w^rPt!EFnYBJAAm{w)q&-rc})BYdAsW9PDRSRH`3&Y1T|5sWK%y$I0 z+*5=M9Itd(c{}0ZD;_@qR# z-bkDcJxBTvCP~!~kcaq(BEgE?HEr^@dahh`1j6^tWdU;Bq5d+<%@3Dqh4zctJ%1>y z;?Mj6>d<#~s$OSB>E<{o&t7#Wu!IYge>^W(YU-h=c}?pFw8dlGM|H>SC(q8f59v{>XE< zC6~x2bl5u+VzSYQmKl4E{q+jd`S|IwyUW z9qdFU7=1%Iwu|`bmZ(0{oia+wfX?f#Lz2iSGsz8%TXdCN=?#vr?kCqRE#V1=+rei3 z#^wg}#H_j0MuP{epxaP+x|;Md-59HSH0k666}rL+(n3x4Ac_{oNd2`T`7Ky#-LwGn zXK`J`v?3*&(UO}WmEmQdV_Cy%F<1+;^_s%;1I%PBrYMa{q^6=0J%ggtF72)p0yVno z7J-2-8%N>4xS31t zJchL4L0uCmD&@Lh+Nrt3V*A>8embQ;FY_x5*rsk>0~@)|yT2N`HfibI7QirnZUH9< zlu`K@y!(6OBTnIyK{xcXenF9fiIz5Sdu;C%j(~Z4-k-0Q>hz6~9m?4+{Juho%YkDk z`Mn|e@ifwRi3?<9iUp|YU!2YT&hDPOr6Cbc5vCfE&&aqJ93ufuY0d=FiiZ! zvqBB#)JDiKa_^9CrE5~*$^1vI8jZV}kM2UwYcMI2%B8hbBp7Bd&^}3(%JRJ8+m`WGyo#kIbhTG>MV6x1)Yb{Ul z&k*2sjExb!Gh};9R8z%ty%(KSO|C3}jCR3&e{ObO;7OqQpU7!c5lCo9t2R_D|Ix09 zb+D6gk>r}5(n9sH7)qt<`-K{09F8F}=DUL^&gc!li5k5jfE&YNL=!|k>g#GyG9@5_ zlww>VhGSleI^2`QZcEoPoiTvAf+nbP-rQWghX})H{|;!Tgd{}r)+Z0c!QSsQn5XLAGiotFr4uUuz^^?qtQRZ-1Uv2fe%RWFjq>DHFE zR*TDW1jXgRYOl#2ius1?_=HNtuBKGL!^EAe`8HKjdKUECuj|-zr6)Zl6csycAkd); zE;~E>swEA>7L9=S_(yIykdDQsx``Hib4$q*milR=+~r#~;_$;VX6>sCFjQ2k_6*_h z6)DfKLe><+*Gw$sT5la&CXMAS{tH6?&uPXHK)^wwMJ)acCMEXg`{4N4np5`hn3;)&n|*^+Yt|yOl=0kpWWMGH)Vf7k z9=`a%d?1*tK6)YTBxo<2I)Gc3^Zj{cDh!jRB;)If|FAr{uE7W)g$G}ZDvx5x9&qkW zF=mrbl^ErBll-H$f1e6He|Y41c05=d09d`Dw86KtU*4Nd)K=Rq28gcFC|TsQ;Jykx_eBFB1BpN z6Cby8A`f<5UrNVj(G_)l!&Gw~;mycTh>3l2i_En~-~OLjWc`>T!gtUonJsyYsfR+2Scjb$D>-pNPv)-+*{O9{tId6G zn`vhk!7FI*vNx>XkdaCfzeBbiml&B$;~+Td*-aHG6Yvj)^)D^`lY;SiWj%7pvP@~v zWB&~re^DIxL|*S8;Pbn^M;?>ue5dMra6qx2JTKv;{Hj!|MWP*N;#*2GD7R=^G;dThl*+$85;5{w_{%q|diuXLjBoq}+dE>@XsH0O5wy!YjMN5X+|(e8 z`bQq)pF==|2g-92&5Mxi$>+&_c3l%R}FaJDctM;YK8*$^Ub#0x&pJJ)jo( zs5I3f&YJPhx`WL&;#Fp$OFgp9$0}A)1L|6JZSlS)emzgWSW5Cj8yp2#el2+2KS+tfsLpG53twUZfFo1RX%!(L{q4fM0bib;~ZWvn) zwwG!$cUH&-p-b~0LMXzo)e-v$T%Xk8rqu5DTJQjcuf%S%Uzz^`h5xt2{(K5dSOAuH zkn>r=fAMOb3i!|0A3x>0eg)7cuPd3&B>t7!6EE<8d;{OtN5$7df>pQz~Hm;Dz% z`1PsR89;4|JTozy{QrIS1dO`q6+6AIxc zpM0CJ3c_hOVcbG%^8XgN{9ZG1V(nHxBhHR|S%a+#vK6+-(gDo)C#xj(c-Skxe_^oy z2Vr^opkzQIG}8_Dvk?5itq+5Qe}x zotxJ{;j6LrT`~bak#JwW&N~itU)`^OsF9)d{kaOn47T7gE22t+s2HAxh+de``ki#< zr2nE9z!PeKRl9MP>1mgV&I98!dWyak95qC_#XGKXg}ZqHqdC?5f#D1`cnj1Y_67WE zV@lpM%l2xJSZf|eg3RhOrd(97Q9jLvrAR;?}LOCTOykid^xI-2@73iD!4Y=iQ4G z-&$mP^8SaMG`M5|eB7qRuOefPr7$v9>_z0sR1jbv<-k`~B?45`uE)jw7+Mn}C=l8; z+O#}ul}VMic^YQ_uao>A)TNh;5=_iAqLZwyhjsMy#}GD_v~9fvOeP1QIPx`oTx!dU4 zNBx1^JHSxOnsc<}{h=v&?sL<<{OiSb-bR%?kw2WG1ag2=gw~V5754rJIj<#`ht+l* zV*RP)5gEPM$5xRl#vhO5($s!$^i}Pnv;J@9B<*uL0{*cjLPZEj_>UsiRcxB36>Gh* zU)~py?#5&R9!-4KipoLa71>hG)l~%YK2{;}8hDAWMcH8?5p(iZnaA1yU!?Z7)x#b% zS!yB|0!ss2+dMhq%jX-mUrRu2#*8q(4b9WTtK@2_i~FBD{`OqRNtJ;6X#)v`#yEshQfdY zGW1PO_+R_BsJ5oRB;%mC1-FPA84e=h0wZ6K4JwN&L^tdxBL^Yx8xTo>yqR+WfE00~iPec)cuR zQuHrRFZ#s6{=L?~q6pCdC0da<3=sbx5B&7~eu+B*^-)5ogPj7|nXPZ{9(M8_0II>Z zvfE_BKlgos77-qI?c9hu9`yY2j#*L9cnL=Hs#K=lBQ5W;D5`!rPvf(C-n{6IL;vEp z@PK#$cy$B=AXdNwYk63;RVq9t$BV@J4YK^Yh?l2pBUAUw64@AA+}o32z{X(15j(=kLzpJ9%|YPBPBW2SCHd4w4sf#-PCncGG(XUnsz*RF4; za4boUXa-~XGEzJes1g3;6#r37;_%i&O$l*)Qh>Dbop`yG*8QDl5w~Yv9fr|PKXUdF zfeNd(v-d&;SM~>W&I-J`6t%iXx0^5Xri52 zD~pPt%t|s^4!4jy5{@vbf3zp2l}^)b(|@{hE_)We&(Wson5$g;Iwsvw)A#&tIC7u4 zmxFRkWWMUwr8Xyvj^9K5OYN4{mu7|1e{}knDew(EhHjgk1AryIEd@^3>+~-A(uS@e z#}sl4Z&~M^-uze?g`bLu>{5aH)Gn%Va!HlSU5b@$N~sOZ28^V@T#cf(P50#SO0Oxk z8T+85Zo7Look1J(vlF=j6-|uXIaw`C=2|B^Bbp@0vMulap8aeHhOD+^QmH-%pyQ&K zEqrbsc|#W3jq~OvxP-Chk9{IWw$6Zitk8>f>a&2avKel)J3k+rz@gR75U)KQTWSpR zgB<2(!JnX;SiUTA_dCfj#)bWqgK@5aHI!bALeNg)U9JkXS ziRU>fH5m^&;Hs|;*=x_(qjJ6!4(V<{g-TlTd0#yOB}HOIufhb)2C?XD68=J`?O>-f zp^i*Z?d`WEUjK%&56ohnqtih}2(B-+9(EXAtyUk>92^^rC*=YXV~&wB_#{Aah;W!8 zzt;|y8J674;5 zLw3}OcvvTmp}J&T0*OIfly9B&Aiz;4l=6ynJc`m7!fAT8xk->tl`b2-aU=l^C)NNvb+OTixz4lG9k<}T2?Am60BYk;`&)frM${#JV^s4La zv$iZyck>AF7D`$-&k^_`^V{83*6eM05sqV|AIduxScS~SF=T6Q||33o>R3m;n8 z(iKJutyr3&k=)-7Jc0u)>ukV`}|;pDTCbl5JfhfNp_8Tjy@o>r1`U8$$9OpRof~q z>UUbE-Ltce#< z2s(AF6NBAo?cen1x92W|eLl%KveRJhgNC!&X&(M`dMZtxMI8onm}W4%v|8#!Po-S? z41$uNBaGVYXer0VN~V$N!2QxXdc&-;Y4O@QCjVi1shwM@)8~3eQPV}SAhK2pF7xIb z-mcLd)?sUI1dK388*5UPMY|)l@N-LXa@2OK?*VJ+maldRuOd6r=B`Fi{tUeAqHp53 zctKp(O{d&0@5-n#`he{FjLtP?>OQMW0&5#%FvoaSZRzg=+EDh$*RkglmZr4?w09@O zq~+*3r?K{!-1e9^T9p}7G})h3)jbUyi(1!YgQrB?J*Px|fQ7FuEjcD%IW9O9h%k!@ z1%A~Cuvc6>Jnhp2f!zdSh!GbA&TA3nVXVmmWl3w`PMbG#KN6$Bw^glw42aN%tqyBe z=w#A=0sTmup#x$F#C*Qmp6{cemV#9rC-mk{r0rtz$tD+T+5@lCo7z)Leih%#=KO1! zV%8M-a5fJaQ5&w3X!Z~Y`i9N;SH-STNIhP24+m^v%1R9_9#WTydNQfOo5!v{4i#5Y zJme;LsfK%pF?<}kctiA7xkmgmb~jx>YIV?k&IsBgO|se=CQL0Ay$MJE1{sfL9gx_{ zL;&PW0?^ZQ6ggcuZI$1uXnjsJ5-reml2;}?Ha9i*Ycre1R>QvI>r5PT#o^@8<|$z9 z!(dO1e$xFUk0+|9QcN&lK9-4n!_5+!l4`+lLtFCg>hdg}LxA|)HXFsly_l^{w1(i+$@)gIiGrsC6Z6XhPPZaMMFKim5TyC5RH7G)bU6Ej5?NCQ zd-7A2B%VgLQ$1hLClXT)bzgsM9G%NHW!;|=t`PAUQR7V|3O$!rl90)+r)eoU7gHDE zt%mprliFNF5jDBnQ#H7C{&qrYcvnNH$u)!7n6vW2xyeHDFtyp7iDQAZY52;%M~De?Abb^0tFyN+N=+N`AyET{E;b*n;CSF-4Sbq-_w z+Sj3R9VbBs)=fRtIx!xGQ2560V`X;D;n|D)+zs#b@^p6%Fl*)8>!j# zbM_+ElnO;V{F?fV9k$}&btj&A8RD9_%_}eQhLbzN(^Jiy;Kf75G56-4mz;hGON`~G z5Gyq|8{4E;H>G)s%;L$<>p1EU<_KD}afM$HtY^@@`7cqLv> z;E;&PopcgDuF?fRF?o-@InzYzqznE9N-`qGj0S?*zT5N^FHw>3HB$%%RaeA;&2t55 z#xT%~ush4RI9e5Rp~zTSdO`?M9)aEZ)__XK#*aXrGlwejeJ_Apa=tt+#ol#G#QSr< z{dFRoh6W20WvDj4s1n9U-G<%6cNi>-D|cpN$;-LCX7!#wKS zyV*Vsp8+a+$xXHb#D$)ZmEdRfu<0fvs$#q^e7@y%Y&bV6BvC}+!C2^i@ z0#t&{icA7t^*Y7C_fsRjr&~8aNLw(oxSMI+n$8u<<rC%@Ul}IxbNau?m4t@P=ywi27(HPCbJVjEbNWN zTOa3DC|a(nv^zGr*_biAw_e`Bj25)~+|dsa@?v}SSPwWk`J8n$MNihmF zNx;1TMIRFAYI2tN53Ok|EEOcp_&eWsDe7{kL?+Qj#J8J*Tt9LVyDgPZtzJS*I1}c< zbHCL3f?V7d`x5$_0ngK9Qk*hEb4^!S4VpOI^FDGSYAhKYM2mHI4=s4Ud&AP;LIcmv z8~-(@q&B{@L|p2?j>{dbBnW0)3Y6yo<$QHH@bL{ZOA8|$Ta?lX3`1sb)uQ2s(jGJ3 zBBQ=~f<@F01Lt3RsDg*^zk-F^yfzUpfy*27j!IT zjz&ZLNWt9H67mX!c=yas^#q5;Vt;ly+!tY6~&0ZciymkKIXQf`8 z6u)f*v?Rn=Rr<7{f<|Y4F}rokoRD(M#qdgRV@;4*Ld*2*5+cs_17uYB@ESAa=#&<3 zNxQT=7qtvt{b}AF><`)VhBKDov;KO)ttRN@B>HT}`T|h-mV=@e%+`4h=)Bm%KYMMb zJVeGZ*In*l0e%MrmFlKz@hwdIa6!IE@A^LP3tm-$Q3<`Y<5?XqUNoVeQZ{dM;XUHJ z3!kdAyIqMfI|Gd|JOBMoyELk4trBch3lh`M{uj+Ve?+!`&;H?8rSmcY|RUcM2r zH%HmlxE^)<(7Nq<2#dbfJ+DMWvRbUc4@#nqFxmW;q*j#LbEbZltII2| zZLvAF85gT^I;_F?MzWM6ZcVE_B{(0)-s=iQ6d&Kie_a)GH~;dT?DeWlr)Q(O;_G=F z?)%i5Wt#W_LuD%TV^@q#VK<4$guT@%_-l0@Zcxg5Wf=Ff4n;-?3`u8$#1V=?%`w|O zT&@N{SGuiNKJs1OF_a%h#&XfsO9n|HvxTkEx z-k(YuN3v8+@N(H6S`BOKN{0tCMgk29vz#>s%%>G~&ZOB}V9mxs&LC){>s{HyU5y2J zX{UU3`(fwf-S(c{xkd>lwSrH(IU1UhYA^E}5BXDm)o=QHI?QXUF`i5)NdhJCRq2=w(uUHr6 z%J>3p;P_l2GHjCyuP<2ARpC|96sJ9p4cBZ{4}l0n;wAi#F;8LJEuyz?U$UQKv{yP% ze#*j>_8-dZ%jLNk-_77jK{#}Yv0M@4en74>a@vA%mBb!BZq#9~o~#?*K4ZFlLz7E7 zj=P7%m?H7r;-(FH3AXm1)bh8^a6CVj_I=!~l7aT%I>kokZd*3z^Cha3T4h32o0?SW z_3~8HT}wu_2V93}PKBR2zbrkxwLWKb&cQ2yY3!powBepR`fjAFA2hWbaI}Nr>^_u% zd8qzVjzOQQoqN@YeSftu=(QzvMa=$3uN-1(6u4ClLXgM8Oznsd|fI9o{yEB#NX;5mz3 zbC#)mQ)R3=yVmZpwFFgWmOYn&l(||lZ!L8_5+VcIk2>tChT5js(m{$bBk0Y%w^gP{9j9iniQ$#bZR(3PT^8eE$)#UTQrlMStY^h z+C8~DN_rKA3IU(&Mky?hN(4X})7n^{_mNAvnwbSfI{#zZ^DiilR8j<6cq{%Ce zh0`P&HI^2xOuc->cr-(WLW%~nCk(?d&5R^aji^Jrw4+lupVcf*%>7%tf_E3ic3Bms zCFV)pJ`8*e?)vh)yPUbWh5CG0&r}~#RP$DntGxeiX8fqea&=fqk=)Jaj-=i^!u768 zZ zcTHjsk*-epagrf={E}Jb{P9E5q%#)*xnCXDme$Q{^njJC%+!=F$*O{~xhusY1L~Bk zHA$yMy$`~K@^MGaO2YDWDktxQaIcRW)7N#F5~}N)Z z+)t&T_NSmHq0S4}Vn!0F@5+dP6~lwAt_FdogXnAq`_9~8upUEsdlzs=>m2qku} zvH|)w#=j*d%1tf4{X9uj{~T6iZx6pC>MqB6_8zCvr4fhv!}eZiJnG~hid*v_>VnM+ zY3iYU#Oi)^DrF*y6|U#oCA<_ygcxg(ht@bNn{gUJ-eY5InB#*Uo+FeU&75!$?uO+7 zrS8Q^e#iZ(Vl`e2%$fC6K6O0p_K~K<8Yf-br@9>J?@o?mUU^8q*E(FpBOYH3h*}x# zWJ&Knn$v1R38-~PYV>RBNACV4V)2^7!J*Y*5I)%Uh3CGTI(CUb;LgjCJz5>lYZs4G z8KUpk{r~odT+EHZmj3VlkRe+TL`9=rE&Y$D7c*E+A0n~$oW^r~HUKlDn*$qRlL@oM zTfO>aW<-%mIPZWeBW%p1IHjwd$=Ay32idXid%`i%~%Z` zPKtzh2YbqJ&l@=A=_>JDHPFhyFr`%oZ`0maR`gZF$izWkiG1~ck?=>@^^2PJBrO3c zD}kh}*BLryqbuA|{axo(Ket8d9MKGo8CkfbGh;T9D1GTyl8D zqw8qMCfB0fD{uIIVK{^+a%yC-w+?d?jvK1GG_G;bsEm$3aVrWr;n zyhf0WjsW(L5!*dfkxp|jF26zb2C8JC^RO-TR4k!Y#vc{{MUX_VDE@)h#St8L!gR<>$kbPMD*pxf!dK*pD?uk&&T zUy+$nB#T36OQ&K{f~9c9(5?G1SQrIRZjs4E`|P94Dg&`HYqj-CBgc~Yi^LS0a&#$G0n2utA|viyFYFDQK@8q@$O@;UaKqyPw@4?Tw0OpDCW`>OpW3u)1Pc{gqXim`*;wjaBDR)dneXM=0| z9JTHxW!q^NcTRDdpX_!lUo_Ad){oy8Lr%oTl@B= z4I%8OU!1p=n04i0>D5=Bt8(_gCddOL*WX%cYzAcz0|Uocy_#-bu7{eAlQf;U9>Kj} zX4?6%d+Ea$^bK??PPQTJ7EPCtr4BMb^YFCff>a%MoV-17vQk)oi0bygsqoORZ=A;V zp9LLkU`|c1A(vppO#K`tFo7kYTQFx^blXUrZJZ&`8Os|SvfRYVYFk#-{@~uDe%OEMKUfeFfAQeeu z1I@)lae4mI$7+&&EHeFzLkwb#sXmu8C)=Z+#pa0}H4SdJk;5d(5HE@T8t%+@0fR{P z<<3nA4k=zmdz3P~NAK2&dAljU{}3{&w&{V?(kZn^IfcC-I_&3Mwk6k%=}+%rIlP+> z3d=mz_gr6_?^`P?;G?NkQ7KIMKo&Dbu3}E&aWtXE`s)(!BZP`hq1Frzv@mFB>YE4mTS6DtU;O0daS@0U5#6A77ZK&|#`~&A zfZtTRVPWoZ5io>pxOxA5M2EZP@ajh!oGQC?rHr+>y^*mv=ua;Ip=>m92tFa=_%Y#| zJMCeT%g{XSsM*PnV`VGar2$K}skYLW;I~7Yw7E?A8y9`SeMG)7_5S!UgCm4jCH{x0 zSdmk;@I#HYlAvl5c*fEtp6{HuOO@1T>~5zkCFFI1e*5*&JF<@U{z8zI&7U&EU^SqV zm!eM8u_lA(Fmmr_@)CmOmxDRS51bu_<63wWd;D9zRb;kNTCGojtLo@>#Tb4hYtVG` zttN2YD{0@F`FTWduCkyT({7_y9hSP%hmDfc*$pK#_zjtdqVZ@~>0Y%U?t^iE)XOuV z#aHO3&@F~H*Oq6aDtOZyu&ExVnwdKpRt$PxrzG-kMrHVY1KgE@K6km zit8Ru5(CmQ+%`c?n!SP0Axeb#R`1bp<)}giBkJFb%9vR=I3WIHNY)I|@|xE(FctFU z*`;;HZm84>7pv}TI0sJfp6l?`gPgX3e(s;PkYV=MkPB^l3&nSBm2fw1O5lm}&38lE z?hD>+)Lw&n-p$!qeUY%eyVc=bi0WUrx6qZ?QVj zUz`YO4>;z2$9HEB40nds~*{q-U8FNsQA-CyaO3mjD-W-uZSq2W(K|)em8YGqO4oPWgX)h5_I;6Y1yHO;h zySw3%?tTv{-he*8=U(gk*8BeR@CWOhbDcA@XV>h#XYZM|NA_l4mD8E=$}(ohv!}5h zykpOt)hkIe7^B{2KQ;8DqzX&;Je<|U?E;paC^Xh+nTLKsgXrQ~lkl`=ljLFRmZw3D zf`(FV4MJ*P_xDfrbp5#J(vC3=6;9fgpVT(ckv>~xk2_VIAVq5F)qd!ZAz4_JKN+R& za0{!KQIAe_&I3!A?vg~aPvGeqmXxP&m*vzH$ptUCZ>%%!@IB3)Ltnw?jI%x3_s}1K zOEP@bKYhmKrU#^ZH;W&W(w^FKT0aSkwme74@QM*{D2SSB94pi-nyTspWd_r7#`Z3>fJbGRHGA!xy?(T%+y|vA#wG<-dnc@n zMK-P0hh&DT54cNO-Z;-VhNoK=oH)`+pg0rz+d|3|dNSgTBr9h=q*@;lb-mv#2Em>i zq^)9jR&6UIFgK}~FVv8m8rU%b9?%R=ug!^U2iqmKp=uztI1Vczo1qR#%Vw&1wAI=m z8ct<*Z-8vnzMrWDZynQuB073{V})LNDDbNf3nN zq&9cnxt)Pcxz_(SAeJuigH4Vnk`Wwgu+=fC_0~5St8;Z1U-4)g0=d(j*nO2Eof4X$ z3VQkkB%keyFPPe-=o4`(87)xpM(7QxWsi5a+sc^y=t$LG`vvF#mWjZXc6 zmKd-0+Dv06>QD}$o$SicJ7%1BDvL8<5-HD;oY6E~Mc!K=RpcrA&fu7Ym>-9t z9n$!N{7AN<{r4V9mO;)AdP{72v!WY@q@FD2n-pzjVE1>JP`!{H2PS3N>F3#~=~V?S z2XOdRkJ1^-Z6#MupCE)LSoHS|xbp;|0oZ+P<6&#Xzb-M66p zGgtv9(>5caj%&+L3uJ@kM7(1))xW7b$&gsqh$^)?KR)gZ9WD*Py&vWPvM{l?-yg%^ zg9FZ^#0TkUkfo*;Da{9#FD?()#Kg)m$cRUIwq~8@yePLhgDRi3I}=uTBK$SYT4>wV z6918wR}lY|GvWxJP(nZo=rssw{X7Vio^oSPWVEP4m#1R^(Zq2BpPD7%p7TJZgbBl zu2frjt;BSlbh2nyX1`cQcCX1*`r%LZ+QYZmiQyy@(4nEvqxdF%)@e2omEC} z00%PKAuq9)2|2-Nd)nB5qpNWiz7J5*CFgryJ9%!d=$4CunDyk4!+BN+r-=-XcQXFq zd3;0~b=K8`40&o`0i%zW^EvyL_*8X#EX)Yw!BJk3rQBXo?spW!)!|%^rq5l-V}j;5<%!Bt{^2(Gn}Tg4jRl?cx58WC_5^I;X70YtRRk6cc5R!*}pTT&m9` zr4#&4D^+A&&Z`^mhJ-tS!@$fn5<~BJYT#d2FB*0B&|F{&q>9o!mU*fg6kh||R#>;r zeMBgEdwq^W;Zey!0?A08wZnIL;M_PuK1hLixr|H=jDv5I0jRFOki-m?<8)w37_2`g zs1*pD|Cj~UR-4*Xu)4miRP6b68Fgx%0-?P{nX{sLKubuE*?e5DFte3;G*An&Z);N*KZUj-hoL~^ zjIzEYk%20Z2BufaO1BpeFB$}#bPy5r&@v+{3lv2LH&)sbwe46G4q8gc)fO+_)B)Xc zk6>Ttm83ggeU{||&*=>+r@z~!Tomq`ET8IKoO|G^gj|3tli zCL~Eixw7So56@>f)KFB_AcKp8cYm_qEy!&ONygk{nWtAC{qv(GVb}hc%m`Bz6~&^_ zAXgW4=D-R|?mfe4AB4m`vC?O3MtG{PpLbdVhZ0rVXVVQ|(;t>ZJaNrYd4Yfp_`A2u zt2#$mQE!KL4HZXy^Pah%TcW*oTPIvM+4Y8iHi4t-^U?Dr*m@6VO2ixGuX3MoA;2^S zcSzW;KKw;j&>@QA9+}w6D~;?xzoyAnEyojLxe58yZtj)5vDmbMq{5KT&={HEHf4c` zWyj5RjVcSNggIlOt7Kw?2goWYTJY@3!%Kb-(*U)|Meg)vV6ANY6E`VSL{KyNiVV((Bg`nnG`Rkl78B{$=|XlE7qu z`<*ev`#&{!z5O9%Ldmbb*arWSyKzwoB_F}C`%ql9-~NxDY@lFK^AiQJ|3bw}Y+n&p zLKtXR)Nh}~6I=cpR(=lREgpc&?`Z*dKzCuQSa!B~9=qlu>Blj8UTO0NeKk#?+xDwH z-b^}}#(%^60RspN05Tln!#HeP=+CL4$kw99kf;1A5Sj{_S|?hB>7%B1lKB2_`rwq(3zCTp*7Wja*0tk)-g2u4jxmrakIwAHZmumbk*~t}B0E@~i;U;f4(?YE0 z#hlE{vZ+bMdp=TGa30LgtsP4hGHMMH{rvJO*?$X5k_6E8j#41XRooK3<%x&;ByXg< z`*v?HrDucGS^?!zpeJV2Rk~bC2hf5_XHsWi)D3Xp#p6_wPTS)@DUVw%naK)83BS<8 z{hRdqa{P~pgA^E+!!!In=ePOQB`avij((0^Zp6W8l$#nUMzU(n zC_ZX8a#enn&rQSo>h^M$?(5GaI61`Ub_d^J&IKh*H1%M>4D=4AdZTZuPd=X~jpKy0 zM#;_PPb_~=(L>pc;Z6xT-+}L>L{{bC=GWw>TJS=cHRVq&_;3THIu2kKr|-n$+59z9 ze;~L0v_WcjZ;vFa?bj>=w@)_0%Vvc2YnD-16sbQi5Rp*A_%=X=-dwUdl7I4%C+-7k zgj~{o{(Jj&WZ)RIleKq9eYh~y^!7w`j=rCNr~00L@15HkF5%5msG?_iG&+`^$%ZP%EcR7m;-8p8+zoNd6N z;ZEtCTw~HZLGZN#(YT!gl!cvJPycICU=!nwfmmXUeFa?qG&SqmyT!1|pW)u5@@9E>@6lU9~@ct5ZS}K&Yi> zW^Q~90TqZ$KA4hfOJ+&fbRF1BhL?gchhrmYE1;Qh-`K|v1Lss8F0d8h)V zw_xT-2ak%M=<-*BP{+_2i0t_(Cc=*ZToYlH5I^Yzs!^6)dG1vOZL1=CUSJ2FA857- z(-tAA$w2QR0^q>etoOQ(T#1!8h(G3)u;^Q}Yu`_VeRyc#fF`BQ8U};UyjWf$Kn~N?Sx2sN@}K?$ ztvTveE&{#2*z=z#79|7BS}V5ie46;s7cLh0X$ZS%Op~%CBhahlQ-FQldIyV80Z(d- z3udWyY>nGcBMeft)NOdZ2>T~qDS@FYrsv)JaR+L<;7&Ws7dN=`I-UNR9To)QUa7xG zmi&WY&2e=}9We*-cU&l)L|wXSRW{!;_W@D-Ny{_2)206HqoF1pTyw7|KjsYj6R!S7 z8>;={)?ettBbCzcLA^QnK$ItteBx0E1ordo3Vk^0+7N%?V8U4FvRzTGeHxRS!a*ol zH~qSK-XeX!eD-K>k8O63LH%@Ro|8_9M0wY6X~@=k$}QCTNX;OC>l3bz{mDmplZ@Qw zpMl59ZFt|;ZBE7oG?wC z==ag+MadqO)nI6MGXEZ;uZB?a7pveuIf{kU6WnOzF$;}~K0W%pCfMNaD5}mD-`@|O_Y_4}US@j7 zgB$H?Z?vD$Tn*A)Z-#TyUAEs1cks4=L{jjwuT<`{JHcC3FC81hIWsA(dVPoLFAB$w zrY>85tIE{?|EI|PqnxE6(TYdP@*n4=n`tU4Oq@I?#H{bGQQFT|#k5}PYeD10L4Gvz zYEAY$P9#Uh$c*6}&|IllINp5PGQJW;>D%{(Y7-jIus0bw!l!xR*R;gtyT9LEW!lStb$L&2f~jbg@!0;Wl?KyN(m6BdUsz_ zFyP&i9(H&yB*~^(#DbMW8pnk|BttfkC@IrrxZQ%ZB&!2wl@~_WM`7jmzGdf%PaxS( zuYWnm)RpP#bjIULnysOh4Cg0zroLO-g+4OfRgW}AlB+M9m>5rBR=#FrcyHB3r54Ml zt^UCIg~03a-91{S8K1F54H^YG-S6#&Ns~3Ff=fK9XDc}IgaKV48AsGBi0xYRuV+jI z(8wI--3k^4RS}-7bre52{OJbz??P6^bRN+py!(M_XrelABIurOMh08xqZo%5mF}Se zOxNtgH`)p5@ri*@^1>KqGn4NSXjFw{7I~)Ckp`^_GHv1C0R|2^EQGrL{0vyQ5xk=s zb*54+YcM-2Hd08ub23<^WZlRPrB96z*5$Dt$*x*aThXSPe=wf9Kf#i~`{gr%ZU)_X z)Ur``PXRhpv8~}LwufQpc+My|1Sg(o)6dU&5z*9GF;_yNaV=N2zP>Tr(wh)t+19h> z(7Sp|OGMMj6bjSU(tIrXOqG1Bmf-cYQ~H7BTkyk)ueHz^stJ>}v%6KhSf78HLJPcg zO}8O{_?%|m=rV)*`}f?Gwr9FuoRUPohcw@jFk%2ACsyr>CY_<%4a1ya!JP=QfIHPg z2CREBu7cAXu{yIF4bMv)4G3nfGX~#}4$j`KDWO{>rNKqiK)_3-fuWD8;EJl@Emdli z9jUXMe|>-(!hLpvSHrjX1N~{a zQ2mmiNlTI^*6K>JFo*Os^T%Z($61{ks;@tCz+iS}b;1BwRpM>)cP3oIGegzqewzQl zRNA=j_6BfqnBl38gGjB5a^=4@&91vW;W`VTFw5*+BCu9mljy_mQEHuf=hbhmmrZ0| zqu4rEC9+&L@oG4c3%rmI2jhBHK9YswLchsYLU1{7)9p`TiSx;-)?P*d? zU!$9y$gZHPcA9n{5;jR<>N|+q0<(1V^_uPS(q6@muBbQ`vSw9D>-TOi9Ll2h51nCa zVzxn2}WO%Nq$Xe35r2JeQoB2KZdVZQTfCFQZsO&LH*okK``PNm!t0f~Z!himz(EsI_hMO7 z$XIB)Zg<{mSi)ooH!fEhjpa-zBv$Y)=XvLMnZcp#DnEf8R`DpR#KuXctg3`*)XMlN zCe@5YX=gd9V`bToERCp^TFu<5G%8>r>yH^?OO?7|CH|SmK|? zYD)p`@10Cf-Y>;+zdd4p4!@e$Wqe>Pn^A{6i=oCgRH+1`@(vEzQA;(a%bV~_tlbY> zC`V@|A)6y-D-nd-Xfmzy-tDQv*nLslLl+spn*0(93lYs>c{}s+yuVm1%^DZ8)?`hx z?#)^2J+yaa5fuJB$w#w+f;>m1rGv<#CkuHIjpG)zy}35}Pg~R6?&ufdpPyt)jp~eH zF6{ZVTksqE+g#-?zBd6*^g3@1X;P=FFGi?{v7cKU^1e~k;HR>7s`G8&rqcw?fg^Y&v8b%K)c+uwjSFa@Zx3CrZvEb#Ml zrOG@skTLpHMf<`;yy*=Rxm`G{5BJziehD4uDR6+~4zK3ukj@wl^9B7bJRKX4p1(Rz zmyPl4U4-ANo%q^+jhE2%hx_zt4{)D4HPcZ6V`KYa-7Xiw$}(lpnMf=N>qGy?z9OAkM#`;0 zlb3t!**COCvpOi@dp0)HKe8Ey1hEdV)xm4g2DEa4Qo za>2=PmV2sfxfM<)rO!C);&zGqA?UMinbK6HDk`VU2myR=Vu;w%h?aEN@ z*o&j>+(7ms-D5Vnj-dY0^D?nal{=4wN^@$8Eycm+%l+$_`}zcFjCWF#W(3EJk1UTw z)$Z}a7qTI79Ib_B7S~TkN2l%%tR1~CKdsn*m~Cn)u-;k6dS8vu2o^P`Zf?mA&uQ6i z7oRiR_MW>yqr~p2EjXsrQQW0qk1JbQJs2OiQ^l~t#K-$3Z?vmIu|vNj=O`xHM#auX zPygqNDCQOE*qDkZG0BUv4>4{k;roEfffU#(H8a5aMM)jEx-{pW60!W)S_us%kF$er zYh`wcAUo7hS(wbyES(NlM(1NMf{g5JS~`zJrBm)Rb|#uwGViZUt>v7pFWSSOWo0^7 z&l(vT9>rE{3NGsm@K;9$NZ!Ki{99P$K2W9&DFC0>Yw&y8mX1?(IVqNBI_)%4 z#+3SgoQjk|yYak=PpsW2VNb1>Vq6;Ce8Z8tVzFjw=E1g%ghzx5jw|9Im*n8*3C-yo z*NXEJBa3nk>1ey9F$GG$7vux0;E4V>4R8()acA8DDUC*{3~e@oDM8I~guF4s5#jAc z9!D`?i5Uc^%9_55%uCdBG4vReABh`6z+Mf>8Iev zET47S@8O8_Yu$fdZNG9{X>H}8rU(9nksO?1z7H`=R@^A+ood4&MEv%x6R0qE2TFf~ zu3(AFT8Vv7d|WHVe4KQ3Y&*&KbV|%gOB@n*@thjM;wnm&f4;VbW-5}*mWeV!u; z0`;n14x*0s{@GLuXAYfqwG}pcXYkAFGYRhg^w^q|QWqyW#&3LXYo?019?aZH#lRFLjlIKcg@x-A&{!w9ZLDHu0(g$;#4?p~L4?fdS zk>7s0Hr@QUITAd(vkx>r3TZD}L**P7gzWXs_jio;J&rc&1?*oBPG&fPm6ejBGK^Yw z8IP8~BJDPsoyR?32;DRv-&m$K;5&U_DH7o_3ikL$$!C8?pwav0o3zPr4euI*kku4> zHQOqE3cZqicIA47kcQotJT>}k$%xTsHn2L<@g#F)WPhJc>+XP;H$Z5rz-`TjAtfv6sA zZ2t3fqkNC$&^z=Mm;Sz{AYGzhv}r-$0e=RPoZg9JRK-B8Zv2TA*VB3i{Mvz%$tMV!^7)1UO8HC z+E2w}JHr#&humg+Qi%rc67Ug@x5Xr)5j*xAubckHd-aT>qrl9J!f zp`qto!{k{y%;nNBzRegTkf^&JF^uG0IXo)VcfI}+g65~<#VaI*og zLVVM+-j$25%+>fP172_T{sO)lXN3$?fX3PYRcmG#MX%~s*~Al5HWWw9ZIQ1WIZH~5 z@T*v`rZ$YW87^I~Ceyi!-8GL!))bmPiqI{4_jfZ`Of1N1yprHEDiuHMMU3f*UmNHe zP>d8=Q3M>!#^G+1jqluj$Q`vNr+jSYk5dRdlUFv78{Z`=c}Uz_+1WguK9d=gIeM;o zj@2}hky@K)VY0-X6~pIY{d9&Sot%jxXs?Ot=(9$smSsl}g!SsZy$^-eDo#zu-Ze{M z^e-H$zugBJBa`<>XL|dW)g_7`nGB~g5GQ_9=-7LTEXCh*L631uLBGk-2Q+!7lC>St z?AdF2#GlM7guWgaFgXUlk74OyDH%479lq1ZAoFhW4_)Pt0O^K?!Dn>276D-d5_ReA zr!reGdn$22Q>I$eN>^wLrTnrDQ>#OMdW?WQv$=nxG#hXA6y3+>m`0;G#*A2%Mc=Pe z!`MZL?@bDf5O^AjAr6SwsLJ!Du74Q{>s+-t9#fz_R&Cn%7}cvv+1risim{zvS?+ip z9pRv*FIfJ4be*v_dt$Bfe(zClv2?wdL!t_M^QB2`qQtkKR8wv5V~#=S-wPR}A04gI7x0qFzgP4tH*h9NuvS z4yhSD%L5?`%!0A1aL3>0x7OIw2<|)WKR%{LICo1^b?MFw6mPMFEsIs?9K zg6?tW5}FzoUVqx=EVV|w^+FNRY(9e~Cfu{h5}WE%5uZ2OaIJE3O4yZj_T#6Noafm- z72J#_LVQi;(RO-4bQ*5`+9C7ziVO^t!YL>yWXlcfPnG0ldk2GE( ztXeNk!>k1kqA+e&dfT^JYXq?}G>;yM^@VN=^$ns&h$OwN-80VcF%s<#w9n4ecu;&= zYF3M^Y`cg%W{1q zMjfpz-({ptoZcc`=F|Hc(2HmICTenM)rmzfT0{BwoOVnH?l;4L5=g*1v^n+h;W2IK z<$a^YLZeap85LTf7*6wvG2~5$y;=UP*}!q|c%p&>lKaY7d@hrqZ}$9A%i1aOYIvtW zd31(%%_Ds?VF zF4&u(pK3}Kr*uy0<_C+xqPp|wj+5f;mIEao)~7qEsa7XnDOYe_yWvb0oH5%;s12+Y zBY&1Pz$6PK2-S0`W>;R9ZzE6Z8C?%JZ)88~wsa;aG}F*I#p)ztQoP|$03lW5iXUoa^k1mPYB@6(Rn`X8 zUHo4R+H)SJV(jcxL`WvTt9?N<_ys=z=d{(Cp!SnXNwDMxFJuYr4vB6>Gr^t%zvwl( zj8X}H{x(tDr8=A=1wUC`^atkO5BL-AKlnE58;;y4&q)m5+Lf^C{2@~#$TV9V*B#aN zDbD?75s*dyKrX_G@#JX{sgRJ>4*R)O*dyI4U1z0AN zjGb?5m8?dxkzjSEP$~YAb^$U*YhKtlDCqaL|BWpy6wI@3hH1ipg|*qEg%X!slKBOg z6`;N*|Ji6lf-q5ryn1G;xbJc%pV^AoAnlI*`E%>t2$=nEF%a#oaGG5;k+l$eV+Q(&9+-M7CqVxjxSzz4mRjsZFF|Y5?-?5 zG8gjTeyN%Q_CYf1-4w2lSte&9_2XalVQmxV=GT8q=JW={ZkSSdVGx<1UD{fRZ1 zG^~~0X>~QR%kONB`V^ni&c63CXE8?_!F%Ep_oa(p;^VgOaQz$f+g~Vti`}IfL@lVZ zb{l&$@*|V-LM7^yz7^u;#6BS{k3dX!SgZh6FcKFg5LQ@bxgIbl$ueaQ81@cC+b<+1 zM!eSOinp0{CIdC`o))&Agmn|Sx}O?O7RiHcY%}Y*={z=<#uvK0>U)#u^w#p^)W5?x zCtDvaZ0>2s3zjw45FmMkxYjl%4oE9KM(iY?IqG+Ngc7pSA$Jx@jOkZ{tmYeusUy8! zgBCH;pp>~I>ise`<}*dg?s?6iH&P19;;UMM2aDhEq3(Y`CGWwr^|+&1oogf&grsW| zm1kQmVaUN)gB0~NvAa3tFf>en;Ta0!V{TSVmV%AD-%M|MOrJWAR^0$3;q*z_R{}e& z=Umj?NsGqE%jL4Ly#xyTx?-x2x;5K`qGNcgr_M`i&!yDyH@o#-2W6j|4D`w?p??gS z*d~k5#@LnWgXs-`=}aIXDtw6rg2dQ}BIxN7PB3pv>^+Tt%R8Pd@ku~a*D5N{p;}=j z$NJvq7f87M;gra`Ea)CC7Ubq}&0qQ7+)0Zs3{8AsKjBQxF)H7O=!MqEfPl*UTZ(?; z;xn)aEWMF>>(;L){?(Ad6;3{^!A*Kj$8(30a@E*eB@!ahYQ3`KWz_8avycJ%-fYwBYKS=8FQtq`@UiqVv-pX zj#f{sA8rluC1Yu?)lc>ecTY_*Y^^|K!?JWFJkuS-iKqMA^ttJbf@94Wk)SuEpBf!5 zi-j)cBA)g~xDZ|iSO)*H=A!KPOPrNxHimQ8;0NiHbfZb7A%H3N-cRT@xo;Erave6QWP`=pFZ1#LH7TU96@U$s$K40ubYWhqFw=r3(Rtd_aBwRkhrJaP7HHSO|b5?|ew zcsJt{it$h!rFA$(kb+gOiXlSeFBF_>fB8B>szl6raa2@74MAAMyFRN^2fuZljR z5Kr3(-5tiX2Xp5%95Hgk$)wR^s6Qh|JtJ9!U_d3{UW5R*XfV2}=X&}Ov7!)REwugq zT35Aiaz%VwnT?m1HD56wdqo>W3h(LUiPl(~v`y2;=GLe3Ow(mL+xE0-XZ}Yce7t_Q z9p=j7pFVM>skRj|?(PN)wC*;arRR^}+L|d&--THeAj?zN<|wpzc{Mrmr);2>0BXc- z{JaSN7vUB};vUJqZrDL12htJ{QujRc3;kQsB2p&f*Je|#(wGjWRqURdiwa)UcojY) zXoub__bysUEUuIYJ%1Ur`yE4XJfl^c>l4-Bz~UXGjcv$m&AfRqd#Xu zloW<(V<2r9s-JN!7I5L*leIE(~JF4COWV^J{csCdQqKd1!capd@sqrp}Ml~&$ z+W0`I)!G}|PwC-yGK=PFP6nA2?f0f98|W8})5(32S{}o#QssVF<#xHVx1yx_G6XZ{ z{rofQF;)~i#mo9Ox>crkhD1mVA*=hY-bnTBD0OAJ62^^m^Ld^?DZ|*=)MQ3s@#6*M zNJA6AeNzo}NoXY5)c#Oy{^hJpXWgE*-#t~McT;?OSUbZcI~s4=yzJ2eR+EeD_#Qr< zDa)cCeJ^zExqSp*r%T9es$JGO%dAq;oz_dObUB$3zJ05q3F>^q^V@|*tpN7FRGAy< zrqth$yC}zU!VSWoEg=X^E+7)~MKdyQe-NzVq=%_sIcr1lBfP)O^eem{@u;^^H{`3k zXedTj;9|=M8t115Z6Q)g`Ed1%DGd~~Vs~CiD+mhi#7|30F{XcXV@a#X=$V;q`H*sa zJQM9j58oR$n(etVTJ)&A<3sbKegT|a(c%w`xMsBkJ(6$Wm2^&bW1gwNZ+4xma_0)> zs`kriSJ5yu67Pr6RZ3a=WCY@C(6fsVl}}VRDmr~Oxf|QKq3XzI-D84-$C{f-A=5qg zp2B-EuSLgwUr8O;W`w1cXq(OMOuf`@+=LPFx~?GfgQnludw~z`X8di7%K*h98A>r@ zqR8EH6w-FO!J8;!6_4x}akF$Qsx(nrG7V*MTn0lQjVL5#M9jZ=8IyJvo9T~ZJkUf-qbn7Sn6b) z>SU=xZ1I;QvK}%ZHRCUNp??zh2bp=~FVc|u$NA>}f>Ko{QUhA@;F>Uq5gzwyhAW05Tw?!6#mm%l5jGy_| z*6TVsC2#GC-|`edF6?qO?#56l$>XumInzlJ*vj{Nwg7Ef0r^(F%XvuG>2m94QWb~_ zHLb={H%>mb;_!$n3s3IZEszyZ3Y03tux&?e^h!=2C`YcakXk@U<%hbT2dohrdHGzc z>bkJu+QNCdqvk~$F9*Pb2zRlXAvt=nn&Dw6CWp*Z=X+}q*ktqja~Y*l-bx<=d;ksX z0F33m3Vo>8fsA_sq%=PEc1{VtZ0}Oubll$B?gUQQO1yg6AeNCq)$YkA1$W6{|Eyic z`Cl}UG?9{hqXB5DC+USv_SPrwRP^4s4}#^EcVqin99DLRpZO@ zIYj`pcC?fppX*wM`OBk1GNHc!3c`X0&WRHwLj0$SzT3>yTWUCv)Ls?bp|Z*M^oUZ`~qfd(O`*`$HxygaJOq zU7^ajR^|eW>iyPM@p!~qw2;G z2+m_aKEttciX65e1^1m3?wXX%z~qqvC)Y83-=#WRd+U+;kPtcWTIE`|Ld;Og5y?EV z0h#sQWk}f)d{Ab3uoGGo@BFb`cV!=lOgjLPX>So*jywUdA@$KDLl%aEGf`=P_GwBk z)_g-v^>lw%{`yKC0xfu_vWLitob@s&&LAh>7FfO4pz{~^h0uQ1UnA3F(6erke6{djkO1A}XC z!f2$sM~jMSX*>XHc#tcvPmJW`Z`q1+mYKOWG4Zu9e z#P8>W@u)%eX?7n;5sjsd)r8JsZ;uT0pfzuHxt>?95c@rOCtNeH8qax~4!S}WducVW zuuy}0XdDwh@FQ2F`f**BTqNDGdOEAo~aBsoJ?19b|C~sJZBD_5fq@OvI9{KJsOWJxMb7}3854xY#udrGv zdLY?)_yIvrw&4}2BGu7KJ~Y;$OkRoay!g>iAh{s@+)cAJ@S`wD%BDSId|)4#{|L&G z3|<6fDF%NgzK#}zo{Q#d`v`9M|2(13oYk=`yxJgJDMNu zYA{O^5$eT*HoYe>0s*@A&|xnhRmaGbqoo^~TfebRq5N82t4|^PRa8znd0s@zOi_Dn z=N!M6fJgy_3WK^=1OO7?E~S0#=7nSu-=T+;^b>w zv~ZTDM<;4<`|Zv`!OTTWnXCD(((5k$k)2!_#h`RP!QG&9L*s4kh?<;EcTxs8(~Dd- zm{6VffnyqKdO38geoPv`_AjbVEAZkX z_vK=P*)hPO)$vFPiF=~8K+PRNC5^92jpo<^ak7y^H1N7d?lDE++A%pAh3_?; zIgLe|zaT2Kefl7ZkEUGptk9^!0o>+vOqAJjxlDT{FT(#h!~|z+60`1A@0e!~mXR^j zQffwsqJV7S1#1B#>K#FTv7U2vRj z98c$j!A}%JKfsqw-buVNs)w1_CK>Dou{xe7SIcGo^i!csBvVlRS`y$C}F`@E&Uq=%E>h}vx^S)+by?` z8d|GA2dbRA;-aJBb_!RMLtSOLpHMWLfWQ`&M}3#h&(EiMkDqXz`Y;(UyePN}e{h@s z4sDtBR(Lej$D|c?sf3wk#zg(%eVoX8l|AmwRP$16xb?QFXJYjVI-(R>9CvowYaLZP zk)ROPPg;~4T(*;eJ(Y~%@t<@h)D$*m5yIgI1;N?mVf_gV`K^o9>bM|->mu$i0T}>6 zlxawn65O2H*deEKG|_`ozsTgc=k3dJoP<6$)fFRo-#v=czKF&rCrI6n1An$D^Z9!J z#;VEKgi8=hRi`$U3S-HMDH9V+Dbmi~j4Ds=ZWRd;cGBHYA zqoQ(Mv!}D}dwdm+Rz}jL(a&wfW8xQU5T+#T$!omzUMSiIT$5y~$eaN}$3PJqoeYoF;NpuYlb%Bb47%4eAyn#G!@*)EK9 zI`ZcUpZ&#Xw-5tN3{^1Q%JpI^)OT+;A6MJW-iPGx>7Q;;whF2zV(^GiX769G$O@9+ z&YGu~#r7er25z=?8RWyL)u>4c<>|TgiiFDb@Yq?=PVZ}+p z+9(amj=fsJp`^}~U&W7&OTFIz2}d0V+#_dOofr!K{F^_f70Fk|=0HP&l;AG1y=*ET zOX~%*f4nNKKW%5w80ypjL;0{-->-U%Yh3YP4McjpL1Ro@&lnRwXA2 zA0A%#Cu`_BrM<7N4;4g(4z%aR?r}*}o&~6kbyY0#9~%*rKYI)Z6mki?1v{w^^(wxy z!`>#M1h!>+e{0s1^wVP?x%nD+pgEvS_xXe;@h^-6&hbTk%&YN>EeM*jlYrFwDU2!m zMUD&GoSqU$!&NeSann=KBn32z-UBV}KW&D;A>NcE&w)dsNoWuSHMs!qfAKFhz#lNkJtu)} zt9}6%_xg8n0EBAg9>`RwSiaWIL=wSF6|>l^0iL|y2jCSb4vWhg=h(tD7D zAe@Ygu^Q7^dgT_ij&Et$?5?Q(7@pg(F?1RD9l@M2F! zg+(v!$%fd`oK+vDkDhuCA@EPsF{ZYqer3;}vkrGK-p#hIW7@(h@*YlPTjyRk#hx2fD->Fd2(#~|^1WU)vEkpXfSN=kga-h&scx@Mp>5s+x zzw?JN5RlaROVsW!O#>+BS-Xf|IyedX{AnQkwS5%;4?|kS$p4ai94@#=JyuHlPn6z7 z#zl8^Apmoh3<(31|G(jfxtI&l8r1K@{41<~^qB|jhVy}%`N37!>i_8Jf2jY@MED=- z|1MA$?BxId5!Zv>J@~Qsum@P!s*z#@vre^=tEk0m{Lt5CPQI zmUs%a2>l<~FF*mhJfJ?`t=eP%&gQsOpQf;~ljZc1-4f%DPNuS9$~WRiC+Dl<@xC%r z_wP&o@b~wB@Zhd$KF3$(2cppLWCPL2<)7iXXU*ZI5t7H;Y9*eMtl~j~%4al! zeF?JZ93N$f&b6@Yf@ikQ$RxrP(a|3{eJsDUhpH|NBv0u*!sagE&r!?|JUgMaQZ)%x|F~FBqgx_ zFG>_B36i_UrFzLRxX8r><)#E5p+Xze?a<&0)qqe*8@bg z*8kc)k2nNxK$PVm(+%zTC#e3CJd%(8C777=#wvq1`sT5Ooa71(MT!lU2znlFtT?vP zn!(|;AJ4Ij$igvAcU?L2yyYG7kexzH*4% zK;0brMlVD8VlFVZUPVopf8EP)>JXKF1`V2)+pGcEZ5%g|K8FsVnA2taI*LIlP>thR zREv=zR?_H&roA4Ve~_Fo5BPGE=_bvvA7GkQ$?4Q^P>+|z_XIz?0qA{LLPGKfn!+z0 zT}OC06!R#BR-|ys`g>s8#A}U$&Y++KjtX~3uaAu<4g+*RjN03uJ#{hM)p7vz!DVOcXJ`Q6WdytqrVfpW^m-ntmSe|!u0 OdoCc(pYu%f-Twn{j6zcY diff --git a/contrib/mesos/pkg/assert/assert.go b/contrib/mesos/pkg/assert/assert.go deleted file mode 100644 index 47497d35c32..00000000000 --- a/contrib/mesos/pkg/assert/assert.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package assert - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -// EventuallyTrue asserts that the given predicate becomes true within the given timeout. It -// checks the predicate regularly each 100ms. -func EventuallyTrue(t *testing.T, timeout time.Duration, fn func() bool, msgAndArgs ...interface{}) bool { - start := time.Now() - for { - if fn() { - return true - } - if time.Now().Sub(start) > timeout { - if len(msgAndArgs) > 0 { - return assert.Fail(t, msgAndArgs[0].(string), msgAndArgs[1:]...) - } else { - return assert.Fail(t, "predicate fn has not been true after %v", timeout.String()) - } - } - time.Sleep(100 * time.Millisecond) - } -} diff --git a/contrib/mesos/pkg/assert/doc.go b/contrib/mesos/pkg/assert/doc.go deleted file mode 100644 index 43fd0506b98..00000000000 --- a/contrib/mesos/pkg/assert/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package assert is an utility package containing reusable testing functionality -// extending github.com/stretchr/testify/assert -package assert // import "k8s.io/kubernetes/contrib/mesos/pkg/assert" diff --git a/contrib/mesos/pkg/backoff/backoff.go b/contrib/mesos/pkg/backoff/backoff.go deleted file mode 100644 index eb5b239ee2b..00000000000 --- a/contrib/mesos/pkg/backoff/backoff.go +++ /dev/null @@ -1,96 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package backoff - -import ( - "math/rand" - "sync" - "time" - - log "github.com/golang/glog" -) - -type clock interface { - Now() time.Time -} - -type realClock struct{} - -func (realClock) Now() time.Time { - return time.Now() -} - -type backoffEntry struct { - backoff time.Duration - lastUpdate time.Time -} - -type Backoff struct { - perItemBackoff map[string]*backoffEntry - lock sync.Mutex - clock clock - defaultDuration time.Duration - maxDuration time.Duration -} - -func New(initial, max time.Duration) *Backoff { - return &Backoff{ - perItemBackoff: map[string]*backoffEntry{}, - clock: realClock{}, - defaultDuration: initial, - maxDuration: max, - } -} - -func (p *Backoff) getEntry(id string) *backoffEntry { - p.lock.Lock() - defer p.lock.Unlock() - entry, ok := p.perItemBackoff[id] - if !ok { - entry = &backoffEntry{backoff: p.defaultDuration} - p.perItemBackoff[id] = entry - } - entry.lastUpdate = p.clock.Now() - return entry -} - -func (p *Backoff) Get(id string) time.Duration { - entry := p.getEntry(id) - duration := entry.backoff - entry.backoff *= 2 - if entry.backoff > p.maxDuration { - entry.backoff = p.maxDuration - } - //TODO(jdef) parameterize use of jitter? - // add jitter, get better backoff distribution - duration = time.Duration(rand.Int63n(int64(duration))) - log.V(3).Infof("Backing off %v for pod %s", duration, id) - return duration -} - -// Garbage collect records that have aged past maxDuration. Backoff users are expected -// to invoke this periodically. -func (p *Backoff) GC() { - p.lock.Lock() - defer p.lock.Unlock() - now := p.clock.Now() - for id, entry := range p.perItemBackoff { - if now.Sub(entry.lastUpdate) > p.maxDuration { - delete(p.perItemBackoff, id) - } - } -} diff --git a/contrib/mesos/pkg/backoff/doc.go b/contrib/mesos/pkg/backoff/doc.go deleted file mode 100644 index ff1a37dd836..00000000000 --- a/contrib/mesos/pkg/backoff/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package backoff provides backoff functionality with a simple API. -// Originally copied from Kubernetes: plugin/pkg/scheduler/factory/factory.go -package backoff // import "k8s.io/kubernetes/contrib/mesos/pkg/backoff" diff --git a/contrib/mesos/pkg/controllermanager/controllermanager.go b/contrib/mesos/pkg/controllermanager/controllermanager.go deleted file mode 100644 index 9d50c52a4da..00000000000 --- a/contrib/mesos/pkg/controllermanager/controllermanager.go +++ /dev/null @@ -1,371 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllermanager - -import ( - "fmt" - "io/ioutil" - "math/rand" - "net" - "net/http" - "strconv" - "time" - - kubecontrollermanager "k8s.io/kubernetes/cmd/kube-controller-manager/app" - "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" - "k8s.io/kubernetes/contrib/mesos/pkg/node" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/unversioned" - "k8s.io/kubernetes/pkg/apimachinery/registered" - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - "k8s.io/kubernetes/pkg/client/restclient" - "k8s.io/kubernetes/pkg/client/typed/dynamic" - client "k8s.io/kubernetes/pkg/client/unversioned" - "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" - clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - "k8s.io/kubernetes/pkg/cloudprovider" - "k8s.io/kubernetes/pkg/cloudprovider/providers/mesos" - "k8s.io/kubernetes/pkg/controller" - "k8s.io/kubernetes/pkg/controller/daemon" - "k8s.io/kubernetes/pkg/controller/deployment" - endpointcontroller "k8s.io/kubernetes/pkg/controller/endpoint" - "k8s.io/kubernetes/pkg/controller/informers" - "k8s.io/kubernetes/pkg/controller/job" - namespacecontroller "k8s.io/kubernetes/pkg/controller/namespace" - nodecontroller "k8s.io/kubernetes/pkg/controller/node" - "k8s.io/kubernetes/pkg/controller/podautoscaler" - "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" - "k8s.io/kubernetes/pkg/controller/podgc" - replicaset "k8s.io/kubernetes/pkg/controller/replicaset" - replicationcontroller "k8s.io/kubernetes/pkg/controller/replication" - resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota" - routecontroller "k8s.io/kubernetes/pkg/controller/route" - servicecontroller "k8s.io/kubernetes/pkg/controller/service" - serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" - persistentvolumecontroller "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" - "k8s.io/kubernetes/pkg/healthz" - quotainstall "k8s.io/kubernetes/pkg/quota/install" - "k8s.io/kubernetes/pkg/serviceaccount" - certutil "k8s.io/kubernetes/pkg/util/cert" - "k8s.io/kubernetes/pkg/util/wait" - - "k8s.io/kubernetes/contrib/mesos/pkg/profile" - kmendpoint "k8s.io/kubernetes/contrib/mesos/pkg/service" - - "github.com/golang/glog" - "github.com/prometheus/client_golang/prometheus" - "github.com/spf13/pflag" -) - -const ( - // Jitter used when starting controller managers - ControllerStartJitter = 1.0 -) - -// CMServer is the main context object for the controller manager. -type CMServer struct { - *options.CMServer - UseHostPortEndpoints bool -} - -// NewCMServer creates a new CMServer with a default config. -func NewCMServer() *CMServer { - s := &CMServer{ - CMServer: options.NewCMServer(), - } - s.CloudProvider = mesos.ProviderName - s.UseHostPortEndpoints = true - return s -} - -// AddFlags adds flags for a specific CMServer to the specified FlagSet -func (s *CMServer) AddFlags(fs *pflag.FlagSet) { - s.CMServer.AddFlags(fs) - fs.BoolVar(&s.UseHostPortEndpoints, "host-port-endpoints", s.UseHostPortEndpoints, "Map service endpoints to hostIP:hostPort instead of podIP:containerPort. Default true.") -} - -func (s *CMServer) resyncPeriod() time.Duration { - factor := rand.Float64() + 1 - return time.Duration(float64(time.Hour) * 12.0 * factor) -} - -func (s *CMServer) Run(_ []string) error { - if s.Kubeconfig == "" && s.Master == "" { - glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") - } - - // This creates a client, first loading any specified kubeconfig - // file, and then overriding the Master flag, if non-empty. - kubeconfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.Kubeconfig}, - &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: s.Master}}).ClientConfig() - if err != nil { - return err - } - - kubeconfig.QPS = 20.0 - kubeconfig.Burst = 30 - - kubeClient, err := client.New(kubeconfig) - if err != nil { - glog.Fatalf("Invalid API configuration: %v", err) - } - - go func() { - mux := http.NewServeMux() - healthz.InstallHandler(mux) - if s.EnableProfiling { - profile.InstallHandler(mux) - } - mux.Handle("/metrics", prometheus.Handler()) - server := &http.Server{ - Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))), - Handler: mux, - } - glog.Fatal(server.ListenAndServe()) - }() - - endpoints := s.createEndpointController(clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "endpoint-controller"))) - go endpoints.Run(int(s.ConcurrentEndpointSyncs), wait.NeverStop) - - go replicationcontroller.NewReplicationManagerFromClient(clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "replication-controller")), s.resyncPeriod, replicationcontroller.BurstReplicas, int(s.LookupCacheSizeForRC)). - Run(int(s.ConcurrentRCSyncs), wait.NeverStop) - - go podgc.NewFromClient(clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "pod-garbage-collector")), int(s.TerminatedPodGCThreshold)). - Run(wait.NeverStop) - - //TODO(jdef) should eventually support more cloud providers here - if s.CloudProvider != mesos.ProviderName { - glog.Fatalf("Only provider %v is supported, you specified %v", mesos.ProviderName, s.CloudProvider) - } - cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) - if err != nil { - glog.Fatalf("Cloud provider could not be initialized: %v", err) - } - _, clusterCIDR, _ := net.ParseCIDR(s.ClusterCIDR) - _, serviceCIDR, _ := net.ParseCIDR(s.ServiceCIDR) - nodeController, err := nodecontroller.NewNodeControllerFromClient(cloud, clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "node-controller")), - s.PodEvictionTimeout.Duration, s.NodeEvictionRate, s.SecondaryNodeEvictionRate, s.LargeClusterSizeThreshold, s.UnhealthyZoneThreshold, - s.NodeMonitorGracePeriod.Duration, s.NodeStartupGracePeriod.Duration, s.NodeMonitorPeriod.Duration, clusterCIDR, serviceCIDR, int(s.NodeCIDRMaskSize), s.AllocateNodeCIDRs) - if err != nil { - glog.Fatalf("Failed to initialize nodecontroller: %v", err) - } - nodeController.Run() - - nodeStatusUpdaterController := node.NewStatusUpdater(clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "node-status-controller")), s.NodeMonitorPeriod.Duration, time.Now) - if err := nodeStatusUpdaterController.Run(wait.NeverStop); err != nil { - glog.Fatalf("Failed to start node status update controller: %v", err) - } - - serviceController, err := servicecontroller.New(cloud, clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "service-controller")), s.ClusterName) - if err != nil { - glog.Errorf("Failed to start service controller: %v", err) - } else { - serviceController.Run(int(s.ConcurrentServiceSyncs)) - } - - if s.AllocateNodeCIDRs && s.ConfigureCloudRoutes { - if cloud == nil { - glog.Warning("configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.") - } else if routes, ok := cloud.Routes(); !ok { - glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") - } else { - routeController := routecontroller.New(routes, clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "route-controller")), s.ClusterName, clusterCIDR) - routeController.Run(s.RouteReconciliationPeriod.Duration) - time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter)) - } - } else { - glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", s.AllocateNodeCIDRs, s.ConfigureCloudRoutes) - } - - resourceQuotaControllerClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "resource-quota-controller")) - resourceQuotaRegistry := quotainstall.NewRegistry(resourceQuotaControllerClient) - groupKindsToReplenish := []unversioned.GroupKind{ - api.Kind("Pod"), - api.Kind("Service"), - api.Kind("ReplicationController"), - api.Kind("PersistentVolumeClaim"), - api.Kind("Secret"), - } - resourceQuotaControllerOptions := &resourcequotacontroller.ResourceQuotaControllerOptions{ - KubeClient: resourceQuotaControllerClient, - ResyncPeriod: controller.StaticResyncPeriodFunc(s.ResourceQuotaSyncPeriod.Duration), - Registry: resourceQuotaRegistry, - GroupKindsToReplenish: groupKindsToReplenish, - ReplenishmentResyncPeriod: s.resyncPeriod, - ControllerFactory: resourcequotacontroller.NewReplenishmentControllerFactoryFromClient(resourceQuotaControllerClient), - } - go resourcequotacontroller.NewResourceQuotaController(resourceQuotaControllerOptions).Run(int(s.ConcurrentResourceQuotaSyncs), wait.NeverStop) - - // If apiserver is not running we should wait for some time and fail only then. This is particularly - // important when we start apiserver and controller manager at the same time. - var versionStrings []string - err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - if versionStrings, err = restclient.ServerAPIVersions(kubeconfig); err == nil { - return true, nil - } - glog.Errorf("Failed to get api versions from server: %v", err) - return false, nil - }) - if err != nil { - glog.Fatalf("Failed to get api versions from server: %v", err) - } - versions := &unversioned.APIVersions{Versions: versionStrings} - - resourceMap, err := kubeClient.Discovery().ServerResources() - if err != nil { - glog.Fatalf("Failed to get supported resources from server: %v", err) - } - - // Find the list of namespaced resources via discovery that the namespace controller must manage - namespaceKubeClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "namespace-controller")) - namespaceClientPool := dynamic.NewClientPool(restclient.AddUserAgent(kubeconfig, "namespace-controller"), registered.RESTMapper(), dynamic.LegacyAPIPathResolverFunc) - groupVersionResources, err := namespaceKubeClient.Discovery().ServerPreferredNamespacedResources() - if err != nil { - glog.Fatalf("Failed to get supported resources from server: %v", err) - } - namespaceController := namespacecontroller.NewNamespaceController(namespaceKubeClient, namespaceClientPool, groupVersionResources, s.NamespaceSyncPeriod.Duration, api.FinalizerKubernetes) - go namespaceController.Run(int(s.ConcurrentNamespaceSyncs), wait.NeverStop) - - groupVersion := "extensions/v1beta1" - resources, found := resourceMap[groupVersion] - // TODO(k8s): this needs to be dynamic so users don't have to restart their controller manager if they change the apiserver - if containsVersion(versions, groupVersion) && found { - glog.Infof("Starting %s apis", groupVersion) - if containsResource(resources, "horizontalpodautoscalers") { - glog.Infof("Starting horizontal pod controller.") - hpaClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "horizontal-pod-autoscaler")) - metricsClient := metrics.NewHeapsterMetricsClient( - hpaClient, - metrics.DefaultHeapsterNamespace, - metrics.DefaultHeapsterScheme, - metrics.DefaultHeapsterService, - metrics.DefaultHeapsterPort, - ) - go podautoscaler.NewHorizontalController(hpaClient.Core(), hpaClient.Extensions(), hpaClient, metricsClient, s.HorizontalPodAutoscalerSyncPeriod.Duration). - Run(wait.NeverStop) - } - - if containsResource(resources, "daemonsets") { - glog.Infof("Starting daemon set controller") - informerFactory := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "daemon-set-controller")), s.resyncPeriod()) - - go daemon.NewDaemonSetsController(informerFactory.DaemonSets(), informerFactory.Pods(), informerFactory.Nodes(), clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "daemon-set-controller")), int(s.LookupCacheSizeForDaemonSet)). - Run(int(s.ConcurrentDaemonSetSyncs), wait.NeverStop) - informerFactory.Start(wait.NeverStop) - } - - if containsResource(resources, "jobs") { - glog.Infof("Starting job controller") - go job.NewJobControllerFromClient(clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "job-controller")), s.resyncPeriod). - Run(int(s.ConcurrentJobSyncs), wait.NeverStop) - } - - if containsResource(resources, "deployments") { - glog.Infof("Starting deployment controller") - go deployment.NewDeploymentController(clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "deployment-controller")), s.resyncPeriod). - Run(int(s.ConcurrentDeploymentSyncs), wait.NeverStop) - } - - if containsResource(resources, "replicasets") { - glog.Infof("Starting ReplicaSet controller") - go replicaset.NewReplicaSetControllerFromClient(clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "replicaset-controller")), s.resyncPeriod, replicaset.BurstReplicas, int(s.LookupCacheSizeForRS)). - Run(int(s.ConcurrentRSSyncs), wait.NeverStop) - } - } - - alphaProvisioner, err := kubecontrollermanager.NewAlphaVolumeProvisioner(cloud, s.VolumeConfiguration) - if err != nil { - glog.Fatalf("An backward-compatible provisioner could not be created: %v, but one was expected. Provisioning will not work. This functionality is considered an early Alpha version.", err) - } - params := persistentvolumecontroller.ControllerParameters{ - KubeClient: clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "persistent-volume-binder")), - SyncPeriod: s.PVClaimBinderSyncPeriod.Duration, - AlphaProvisioner: alphaProvisioner, - VolumePlugins: kubecontrollermanager.ProbeControllerVolumePlugins(cloud, s.VolumeConfiguration), - Cloud: cloud, - ClusterName: s.ClusterName, - EnableDynamicProvisioning: s.VolumeConfiguration.EnableDynamicProvisioning, - } - volumeController := persistentvolumecontroller.NewController(params) - volumeController.Run(wait.NeverStop) - - var rootCA []byte - - if s.RootCAFile != "" { - rootCA, err = ioutil.ReadFile(s.RootCAFile) - if err != nil { - return fmt.Errorf("error reading root-ca-file at %s: %v", s.RootCAFile, err) - } - if _, err := certutil.ParseCertsPEM(rootCA); err != nil { - return fmt.Errorf("error parsing root-ca-file at %s: %v", s.RootCAFile, err) - } - } else { - rootCA = kubeconfig.CAData - } - - if len(s.ServiceAccountKeyFile) > 0 { - privateKey, err := serviceaccount.ReadPrivateKey(s.ServiceAccountKeyFile) - if err != nil { - glog.Errorf("Error reading key for service account token controller: %v", err) - } else { - go serviceaccountcontroller.NewTokensController( - clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "tokens-controller")), - serviceaccountcontroller.TokensControllerOptions{ - TokenGenerator: serviceaccount.JWTTokenGenerator(privateKey), - RootCA: rootCA, - }, - ).Run(int(s.ConcurrentSATokenSyncs), wait.NeverStop) - } - } - - serviceaccountcontroller.NewServiceAccountsController( - clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "service-account-controller")), - serviceaccountcontroller.DefaultServiceAccountsControllerOptions(), - ).Run() - - select {} -} - -func (s *CMServer) createEndpointController(client *clientset.Clientset) kmendpoint.EndpointController { - if s.UseHostPortEndpoints { - glog.V(2).Infof("Creating hostIP:hostPort endpoint controller") - return kmendpoint.NewEndpointController(client) - } - glog.V(2).Infof("Creating podIP:containerPort endpoint controller") - stockEndpointController := endpointcontroller.NewEndpointControllerFromClient(client, s.resyncPeriod) - return stockEndpointController -} - -func containsVersion(versions *unversioned.APIVersions, version string) bool { - for ix := range versions.Versions { - if versions.Versions[ix] == version { - return true - } - } - return false -} - -func containsResource(resources *unversioned.APIResourceList, resourceName string) bool { - for ix := range resources.APIResources { - resource := resources.APIResources[ix] - if resource.Name == resourceName { - return true - } - } - return false -} diff --git a/contrib/mesos/pkg/controllermanager/doc.go b/contrib/mesos/pkg/controllermanager/doc.go deleted file mode 100644 index 2a048806c58..00000000000 --- a/contrib/mesos/pkg/controllermanager/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controllermanager is largely a clone of the upstream implementation, -// with additional functionality to select between stock or a customized -// endpoints controller. -package controllermanager // import "k8s.io/kubernetes/contrib/mesos/pkg/controllermanager" diff --git a/contrib/mesos/pkg/election/doc.go b/contrib/mesos/pkg/election/doc.go deleted file mode 100644 index 807b9afcd61..00000000000 --- a/contrib/mesos/pkg/election/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package election provides interfaces used for master election. -package election // import "k8s.io/kubernetes/contrib/mesos/pkg/election" diff --git a/contrib/mesos/pkg/election/etcd_master.go b/contrib/mesos/pkg/election/etcd_master.go deleted file mode 100644 index 59d9faf9790..00000000000 --- a/contrib/mesos/pkg/election/etcd_master.go +++ /dev/null @@ -1,198 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package election - -import ( - "fmt" - "time" - - etcd "github.com/coreos/etcd/client" - "github.com/golang/glog" - "golang.org/x/net/context" - - "k8s.io/kubernetes/pkg/api/unversioned" - etcdutil "k8s.io/kubernetes/pkg/storage/etcd/util" - "k8s.io/kubernetes/pkg/util/wait" - "k8s.io/kubernetes/pkg/watch" -) - -// Master is used to announce the current elected master. -type Master string - -// IsAnAPIObject is used solely so we can work with the watch package. -// TODO(k8s): Either fix watch so this isn't necessary, or make this a real API Object. -// TODO(k8s): when it becomes clear how this package will be used, move these declarations to -// to the proper place. -func (obj Master) GetObjectKind() unversioned.ObjectKind { return unversioned.EmptyObjectKind } - -// NewEtcdMasterElector returns an implementation of election.MasterElector backed by etcd. -func NewEtcdMasterElector(h etcd.Client) MasterElector { - return &etcdMasterElector{etcd: etcd.NewKeysAPI(h)} -} - -type empty struct{} - -// internal implementation struct -type etcdMasterElector struct { - etcd etcd.KeysAPI - done chan empty - events chan watch.Event -} - -// Elect implements the election.MasterElector interface. -func (e *etcdMasterElector) Elect(path, id string) watch.Interface { - e.done = make(chan empty) - e.events = make(chan watch.Event) - go wait.Until(func() { e.run(path, id) }, time.Second*5, wait.NeverStop) - return e -} - -func (e *etcdMasterElector) run(path, id string) { - masters := make(chan string) - errors := make(chan error) - go e.master(path, id, 30, masters, errors, e.done) // TODO(jdef) extract constant - for { - select { - case m := <-masters: - e.events <- watch.Event{ - Type: watch.Modified, - Object: Master(m), - } - case e := <-errors: - glog.Errorf("Error in election: %v", e) - } - } -} - -// ResultChan implements the watch.Interface interface. -func (e *etcdMasterElector) ResultChan() <-chan watch.Event { - return e.events -} - -// extendMaster attempts to extend ownership of a master lock for TTL seconds. -// returns "", nil if extension failed -// returns id, nil if extension succeeded -// returns "", err if an error occurred -func (e *etcdMasterElector) extendMaster(path, id string, ttl uint64, res *etcd.Response) (string, error) { - // If it matches the passed in id, extend the lease by writing a new entry. - // Uses compare and swap, so that if we TTL out in the meantime, the write will fail. - // We don't handle the TTL delete w/o a write case here, it's handled in the next loop - // iteration. - opts := etcd.SetOptions{ - TTL: time.Duration(ttl) * time.Second, - PrevValue: "", - PrevIndex: res.Node.ModifiedIndex, - } - _, err := e.etcd.Set(context.TODO(), path, id, &opts) - if err != nil && !etcdutil.IsEtcdTestFailed(err) { - return "", err - } - if err != nil && etcdutil.IsEtcdTestFailed(err) { - return "", nil - } - return id, nil -} - -// becomeMaster attempts to become the master for this lock. -// returns "", nil if the attempt failed -// returns id, nil if the attempt succeeded -// returns "", err if an error occurred -func (e *etcdMasterElector) becomeMaster(path, id string, ttl uint64) (string, error) { - opts := etcd.SetOptions{ - TTL: time.Duration(ttl) * time.Second, - PrevExist: etcd.PrevNoExist, - } - - _, err := e.etcd.Set(context.TODO(), path, id, &opts) - if err != nil && !etcdutil.IsEtcdNodeExist(err) { - // unexpected error - return "", err - } - if err != nil && etcdutil.IsEtcdNodeExist(err) { - return "", nil - } - return id, nil -} - -// handleMaster performs one loop of master locking. -// on success it returns , nil -// on error it returns "", err -// in situations where you should try again due to concurrent state changes (e.g. another actor simultaneously acquiring the lock) -// it returns "", nil -func (e *etcdMasterElector) handleMaster(path, id string, ttl uint64) (string, error) { - res, err := e.etcd.Get(context.TODO(), path, nil) - - // Unexpected error, bail out - if err != nil && !etcdutil.IsEtcdNotFound(err) { - return "", err - } - - // There is no master, try to become the master. - if err != nil && etcdutil.IsEtcdNotFound(err) { - return e.becomeMaster(path, id, ttl) - } - - // This should never happen. - if res.Node == nil { - return "", fmt.Errorf("unexpected response: %#v", res) - } - - // We're not the master, just return the current value - if res.Node.Value != id { - return res.Node.Value, nil - } - - // We are the master, try to extend out lease - return e.extendMaster(path, id, ttl, res) -} - -// master provices a distributed master election lock, maintains lock until failure, or someone sends something in the done channel. -// The basic algorithm is: -// while !done -// Get the current master -// If there is no current master -// Try to become the master -// Otherwise -// If we are the master, extend the lease -// If the master is different than the last time through the loop, report the master -// Sleep 80% of TTL -func (e *etcdMasterElector) master(path, id string, ttl uint64, masters chan<- string, errors chan<- error, done <-chan empty) { - lastMaster := "" - for { - master, err := e.handleMaster(path, id, ttl) - if err != nil { - errors <- err - } else if len(master) == 0 { - continue - } else if master != lastMaster { - lastMaster = master - masters <- master - } - // TODO(k8s): Add Watch here, skip the polling for faster reactions - // If done is closed, break out. - select { - case <-done: - return - case <-time.After(time.Duration((ttl*8)/10) * time.Second): - } - } -} - -// ResultChan implements the watch.Interface interface -func (e *etcdMasterElector) Stop() { - close(e.done) -} diff --git a/contrib/mesos/pkg/election/etcd_master_test.go b/contrib/mesos/pkg/election/etcd_master_test.go deleted file mode 100644 index 1b06d9a413e..00000000000 --- a/contrib/mesos/pkg/election/etcd_master_test.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package election - -import ( - "testing" - - etcd "github.com/coreos/etcd/client" - "golang.org/x/net/context" - - etcdtesting "k8s.io/kubernetes/pkg/storage/etcd/testing" - "k8s.io/kubernetes/pkg/watch" -) - -func TestEtcdMasterOther(t *testing.T) { - server := etcdtesting.NewEtcdTestClientServer(t) - defer server.Terminate(t) - - path := "foo" - keysAPI := etcd.NewKeysAPI(server.Client) - if _, err := keysAPI.Set(context.TODO(), path, "baz", nil); err != nil { - t.Errorf("unexpected error: %v", err) - } - master := NewEtcdMasterElector(server.Client) - w := master.Elect(path, "bar") - result := <-w.ResultChan() - if result.Type != watch.Modified || result.Object.(Master) != "baz" { - t.Errorf("unexpected event: %#v", result) - } - w.Stop() -} - -func TestEtcdMasterNoOther(t *testing.T) { - server := etcdtesting.NewEtcdTestClientServer(t) - defer server.Terminate(t) - - path := "foo" - master := NewEtcdMasterElector(server.Client) - w := master.Elect(path, "bar") - result := <-w.ResultChan() - if result.Type != watch.Modified || result.Object.(Master) != "bar" { - t.Errorf("unexpected event: %#v", result) - } - w.Stop() -} - -func TestEtcdMasterNoOtherThenConflict(t *testing.T) { - server := etcdtesting.NewEtcdTestClientServer(t) - defer server.Terminate(t) - - path := "foo" - master := NewEtcdMasterElector(server.Client) - leader := NewEtcdMasterElector(server.Client) - - w_ldr := leader.Elect(path, "baz") - result := <-w_ldr.ResultChan() - w := master.Elect(path, "bar") - result = <-w.ResultChan() - if result.Type != watch.Modified || result.Object.(Master) != "baz" { - t.Errorf("unexpected event: %#v", result) - } - w.Stop() - w_ldr.Stop() -} diff --git a/contrib/mesos/pkg/election/fake.go b/contrib/mesos/pkg/election/fake.go deleted file mode 100644 index ba7eb62c62e..00000000000 --- a/contrib/mesos/pkg/election/fake.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package election - -import ( - "sync" - - "k8s.io/kubernetes/pkg/watch" -) - -// Fake allows for testing of anything consuming a MasterElector. -type Fake struct { - mux *watch.Broadcaster - currentMaster Master - lock sync.Mutex // Protect access of currentMaster -} - -// NewFake makes a new fake MasterElector. -func NewFake() *Fake { - // 0 means block for clients. - return &Fake{mux: watch.NewBroadcaster(0, watch.WaitIfChannelFull)} -} - -func (f *Fake) ChangeMaster(newMaster Master) { - f.lock.Lock() - defer f.lock.Unlock() - f.mux.Action(watch.Modified, newMaster) - f.currentMaster = newMaster -} - -func (f *Fake) Elect(path, id string) watch.Interface { - f.lock.Lock() - defer f.lock.Unlock() - w := f.mux.Watch() - if f.currentMaster != "" { - f.mux.Action(watch.Modified, f.currentMaster) - } - return w -} diff --git a/contrib/mesos/pkg/election/master.go b/contrib/mesos/pkg/election/master.go deleted file mode 100644 index 74a2df12932..00000000000 --- a/contrib/mesos/pkg/election/master.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package election - -import ( - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/pkg/watch" - - "github.com/golang/glog" -) - -// MasterElector is an interface for services that can elect masters. -// Important Note: MasterElectors are not inter-operable, all participants in the election need to be -// using the same underlying implementation of this interface for correct behavior. -type MasterElector interface { - // Elect makes the caller represented by 'id' enter into a master election for the - // distributed lock defined by 'path' - // The returned watch.Interface provides a stream of Master objects which - // contain the current master. - // Calling Stop on the returned interface relinquishes ownership (if currently possesed) - // and removes the caller from the election - Elect(path, id string) watch.Interface -} - -// Service represents anything that can start and stop on demand. -type Service interface { - Validate(desired, current Master) - Start() - Stop() -} - -type notifier struct { - masters chan Master // elected masters arrive here, should be buffered to better deal with rapidly flapping masters - - // for comparison, to see if we are master. - id Master - - service Service -} - -// Notify runs Elect() on m, and calls Start()/Stop() on s when the -// elected master starts/stops matching 'id'. Never returns. -func Notify(m MasterElector, path, id string, s Service, abort <-chan struct{}) { - n := ¬ifier{id: Master(id), service: s, masters: make(chan Master, 1)} - finished := runtime.After(func() { - runtime.Until(func() { - for { - w := m.Elect(path, id) - for { - select { - case <-abort: - return - case event, open := <-w.ResultChan(): - if !open { - break - } - if event.Type != watch.Modified { - continue - } - electedMaster, ok := event.Object.(Master) - if !ok { - glog.Errorf("Unexpected object from election channel: %v", event.Object) - break - } - - sendElected: - for { - select { - case <-abort: - return - case n.masters <- electedMaster: - break sendElected - default: // ring full, discard old value and add the new - select { - case <-abort: - return - case <-n.masters: - default: // ring was cleared for us?! - } - } - } - } - } - } - }, 0, abort) - }) - runtime.Until(func() { n.serviceLoop(finished) }, 0, abort) -} - -// serviceLoop waits for changes, and calls Start()/Stop() as needed. -func (n *notifier) serviceLoop(abort <-chan struct{}) { - var current Master - for { - select { - case <-abort: - return - case desired := <-n.masters: - if current != n.id && desired == n.id { - n.service.Validate(desired, current) - n.service.Start() - } else if current == n.id && desired != n.id { - n.service.Stop() - } - current = desired - } - } -} diff --git a/contrib/mesos/pkg/election/master_test.go b/contrib/mesos/pkg/election/master_test.go deleted file mode 100644 index 5b65256026c..00000000000 --- a/contrib/mesos/pkg/election/master_test.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package election - -import ( - "testing" - "time" - - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" -) - -type slowService struct { - t *testing.T - on bool - // We explicitly have no lock to prove that - // Start and Stop are not called concurrently. - changes chan<- bool - done <-chan struct{} -} - -func (s *slowService) Validate(d, c Master) { - // noop -} - -func (s *slowService) Start() { - select { - case <-s.done: - return // avoid writing to closed changes chan - default: - } - if s.on { - s.t.Errorf("started already on service") - } - time.Sleep(2 * time.Millisecond) - s.on = true - s.changes <- true -} - -func (s *slowService) Stop() { - select { - case <-s.done: - return // avoid writing to closed changes chan - default: - } - if !s.on { - s.t.Errorf("stopped already off service") - } - time.Sleep(2 * time.Millisecond) - s.on = false - s.changes <- false -} - -func Test(t *testing.T) { - m := NewFake() - changes := make(chan bool, 1500) - done := make(chan struct{}) - s := &slowService{t: t, changes: changes, done: done} - - // change master to "notme" such that the initial m.Elect call inside Notify - // will trigger an obversable event. We will wait for it to make sure the - // Notify loop will see those master changes triggered by the go routine below. - m.ChangeMaster(Master("me")) - temporaryWatch := m.mux.Watch() - ch := temporaryWatch.ResultChan() - - notifyDone := runtime.After(func() { Notify(m, "", "me", s, done) }) - - // wait for the event triggered by the initial m.Elect of Notify. Then drain - // the channel to not block anything. - <-ch - temporaryWatch.Stop() - for i := 0; i < len(ch); i += 1 { // go 1.3 and 1.4 compatible loop - <-ch - } - - go func() { - defer close(done) - for i := 0; i < 500; i++ { - for _, key := range []string{"me", "notme", "alsonotme"} { - m.ChangeMaster(Master(key)) - } - } - }() - - <-notifyDone - close(changes) - - changesNum := len(changes) - if changesNum > 1000 || changesNum == 0 { - t.Errorf("unexpected number of changes: %v", changesNum) - } -} diff --git a/contrib/mesos/pkg/executor/apis.go b/contrib/mesos/pkg/executor/apis.go deleted file mode 100644 index aa657aa0df5..00000000000 --- a/contrib/mesos/pkg/executor/apis.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import ( - "k8s.io/kubernetes/contrib/mesos/pkg/node" - "k8s.io/kubernetes/pkg/api" - unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" -) - -type kubeAPI interface { - killPod(ns, name string) error -} - -type nodeAPI interface { - createOrUpdate(hostname string, slaveAttrLabels, annotations map[string]string) (*api.Node, error) -} - -// clientAPIWrapper implements kubeAPI and node API, which serve to isolate external dependencies -// such that they're easier to mock in unit test. -type clientAPIWrapper struct { - client unversionedcore.CoreInterface -} - -func (cw *clientAPIWrapper) killPod(ns, name string) error { - return cw.client.Pods(ns).Delete(name, api.NewDeleteOptions(0)) -} - -func (cw *clientAPIWrapper) createOrUpdate(hostname string, slaveAttrLabels, annotations map[string]string) (*api.Node, error) { - return node.CreateOrUpdate(cw.client, hostname, slaveAttrLabels, annotations) -} diff --git a/contrib/mesos/pkg/executor/config/config.go b/contrib/mesos/pkg/executor/config/config.go deleted file mode 100644 index 1489d2106db..00000000000 --- a/contrib/mesos/pkg/executor/config/config.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "time" -) - -// default values to use when constructing mesos ExecutorInfo messages -const ( - DefaultInfoID = "k8sm-executor" - DefaultInfoSource = "kubernetes" - DefaultSuicideTimeout = 20 * time.Minute - DefaultLaunchGracePeriod = 5 * time.Minute -) diff --git a/contrib/mesos/pkg/executor/config/doc.go b/contrib/mesos/pkg/executor/config/doc.go deleted file mode 100644 index 27994a4b206..00000000000 --- a/contrib/mesos/pkg/executor/config/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package config contains executor configuration constants. -package config // import "k8s.io/kubernetes/contrib/mesos/pkg/executor/config" diff --git a/contrib/mesos/pkg/executor/doc.go b/contrib/mesos/pkg/executor/doc.go deleted file mode 100644 index c57252de9ec..00000000000 --- a/contrib/mesos/pkg/executor/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -Package executor includes a mesos executor, which contains -a kubelet as its member to manage containers. -*/ -package executor // import "k8s.io/kubernetes/contrib/mesos/pkg/executor" diff --git a/contrib/mesos/pkg/executor/executor.go b/contrib/mesos/pkg/executor/executor.go deleted file mode 100644 index 6bfea1cc56b..00000000000 --- a/contrib/mesos/pkg/executor/executor.go +++ /dev/null @@ -1,755 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import ( - "bytes" - "encoding/json" - "fmt" - "math" - "strings" - "sync" - "sync/atomic" - "time" - - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - - dockertypes "github.com/docker/engine-api/types" - "github.com/gogo/protobuf/proto" - log "github.com/golang/glog" - bindings "github.com/mesos/mesos-go/executor" - mesos "github.com/mesos/mesos-go/mesosproto" - mutil "github.com/mesos/mesos-go/mesosutil" - "k8s.io/kubernetes/contrib/mesos/pkg/executor/messages" - "k8s.io/kubernetes/contrib/mesos/pkg/node" - "k8s.io/kubernetes/contrib/mesos/pkg/podutil" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/executorinfo" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/pkg/api" - apierrors "k8s.io/kubernetes/pkg/api/errors" - "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/kubelet/dockertools" - kruntime "k8s.io/kubernetes/pkg/runtime" - utilruntime "k8s.io/kubernetes/pkg/util/runtime" -) - -type stateType int32 - -const ( - disconnectedState stateType = iota - connectedState - suicidalState - terminalState -) - -func (s *stateType) get() stateType { - return stateType(atomic.LoadInt32((*int32)(s))) -} - -func (s *stateType) transition(from, to stateType) bool { - return atomic.CompareAndSwapInt32((*int32)(s), int32(from), int32(to)) -} - -func (s *stateType) transitionTo(to stateType, unless ...stateType) bool { - if len(unless) == 0 { - atomic.StoreInt32((*int32)(s), int32(to)) - return true - } - for { - state := s.get() - for _, x := range unless { - if state == x { - return false - } - } - if s.transition(state, to) { - return true - } - } -} - -// KubernetesExecutor is an mesos executor that runs pods -// in a minion machine. -type Executor struct { - state stateType - lock sync.Mutex - terminate chan struct{} // signals that the executor is shutting down - outgoing chan func() (mesos.Status, error) // outgoing queue to the mesos driver - dockerClient dockertools.DockerInterface - suicideWatch suicideWatcher - suicideTimeout time.Duration - shutdownAlert func() // invoked just prior to executor shutdown - kubeletFinished <-chan struct{} // signals that kubelet Run() died - exitFunc func(int) - staticPodsConfigPath string - staticPodsFilters podutil.Filters - launchGracePeriod time.Duration - nodeInfos chan<- NodeInfo - initCompleted chan struct{} // closes upon completion of Init() - registry Registry - watcher *watcher - kubeAPI kubeAPI - nodeAPI nodeAPI -} - -type Config struct { - APIClient *clientset.Clientset - Docker dockertools.DockerInterface - ShutdownAlert func() - SuicideTimeout time.Duration - KubeletFinished <-chan struct{} // signals that kubelet Run() died - ExitFunc func(int) - LaunchGracePeriod time.Duration - NodeInfos chan<- NodeInfo - Registry Registry - Options []Option // functional options -} - -// Option is a functional option type for Executor -type Option func(*Executor) - -func (k *Executor) isConnected() bool { - return connectedState == (&k.state).get() -} - -// New creates a new kubernetes executor. -func New(config Config) *Executor { - launchGracePeriod := config.LaunchGracePeriod - if launchGracePeriod == 0 { - // this is the equivalent of saying "the timer never expires" and simplies nil - // timer checks elsewhere in the code. it's a little hacky but less code to - // maintain that alternative approaches. - launchGracePeriod = time.Duration(math.MaxInt64) - } - k := &Executor{ - state: disconnectedState, - terminate: make(chan struct{}), - outgoing: make(chan func() (mesos.Status, error), 1024), - dockerClient: config.Docker, - suicideTimeout: config.SuicideTimeout, - kubeletFinished: config.KubeletFinished, - suicideWatch: &suicideTimer{}, - shutdownAlert: config.ShutdownAlert, - exitFunc: config.ExitFunc, - launchGracePeriod: launchGracePeriod, - nodeInfos: config.NodeInfos, - initCompleted: make(chan struct{}), - registry: config.Registry, - } - if config.APIClient != nil { - k.kubeAPI = &clientAPIWrapper{config.APIClient.Core()} - k.nodeAPI = &clientAPIWrapper{config.APIClient.Core()} - } - - // apply functional options - for _, opt := range config.Options { - opt(k) - } - - runtime.On(k.initCompleted, k.runSendLoop) - - k.watcher = newWatcher(k.registry.watch()) - runtime.On(k.initCompleted, k.watcher.run) - - return k -} - -// StaticPods creates a static pods Option for an Executor -func StaticPods(configPath string, f podutil.Filters) Option { - return func(k *Executor) { - k.staticPodsFilters = f - k.staticPodsConfigPath = configPath - } -} - -// Done returns a chan that closes when the executor is shutting down -func (k *Executor) Done() <-chan struct{} { - return k.terminate -} - -func (k *Executor) Init(driver bindings.ExecutorDriver) { - defer close(k.initCompleted) - - k.killKubeletContainers() - k.resetSuicideWatch(driver) - - k.watcher.addFilter(func(podEvent *PodEvent) bool { - switch podEvent.eventType { - case PodEventIncompatibleUpdate: - log.Warningf("killing %s because of an incompatible update", podEvent.FormatShort()) - k.killPodTask(driver, podEvent.taskID) - // halt processing of this event; when the pod is deleted we'll receive another - // event for that. - return false - - case PodEventDeleted: - // an active pod-task was deleted, alert mesos: - // send back a TASK_KILLED status, we completed the pod-task lifecycle normally. - k.resetSuicideWatch(driver) - k.sendStatus(driver, newStatus(mutil.NewTaskID(podEvent.taskID), mesos.TaskState_TASK_KILLED, "pod-deleted")) - } - return true - }) - - //TODO(jdef) monitor kubeletFinished and shutdown if it happens -} - -func (k *Executor) isDone() bool { - select { - case <-k.terminate: - return true - default: - return false - } -} - -// Registered is called when the executor is successfully registered with the slave. -func (k *Executor) Registered( - driver bindings.ExecutorDriver, - executorInfo *mesos.ExecutorInfo, - frameworkInfo *mesos.FrameworkInfo, - slaveInfo *mesos.SlaveInfo, -) { - if k.isDone() { - return - } - - log.Infof( - "Executor %v of framework %v registered with slave %v\n", - executorInfo, frameworkInfo, slaveInfo, - ) - - if !(&k.state).transition(disconnectedState, connectedState) { - log.Errorf("failed to register/transition to a connected state") - } - - k.initializeStaticPodsSource(executorInfo) - - annotations, err := annotationsFor(executorInfo) - if err != nil { - log.Errorf( - "cannot get node annotations from executor info %v error %v", - executorInfo, err, - ) - } - - if slaveInfo != nil { - _, err := k.nodeAPI.createOrUpdate( - slaveInfo.GetHostname(), - node.SlaveAttributesToLabels(slaveInfo.Attributes), - annotations, - ) - - if err != nil { - log.Errorf("cannot update node labels: %v", err) - } - } - - k.lock.Lock() - defer k.lock.Unlock() - - if slaveInfo != nil && k.nodeInfos != nil { - k.nodeInfos <- nodeInfo(slaveInfo, executorInfo) // leave it behind the upper lock to avoid panics - } -} - -// Reregistered is called when the executor is successfully re-registered with the slave. -// This can happen when the slave fails over. -func (k *Executor) Reregistered(driver bindings.ExecutorDriver, slaveInfo *mesos.SlaveInfo) { - if k.isDone() { - return - } - log.Infof("Reregistered with slave %v\n", slaveInfo) - if !(&k.state).transition(disconnectedState, connectedState) { - log.Errorf("failed to reregister/transition to a connected state") - } - - if slaveInfo != nil { - _, err := k.nodeAPI.createOrUpdate( - slaveInfo.GetHostname(), - node.SlaveAttributesToLabels(slaveInfo.Attributes), - nil, // don't change annotations - ) - - if err != nil { - log.Errorf("cannot update node labels: %v", err) - } - } - - if slaveInfo != nil && k.nodeInfos != nil { - // make sure nodeInfos is not nil and send new NodeInfo - k.lock.Lock() - defer k.lock.Unlock() - if k.isDone() { - return - } - k.nodeInfos <- nodeInfo(slaveInfo, nil) - } -} - -// initializeStaticPodsSource unzips the data slice into the static-pods directory -func (k *Executor) initializeStaticPodsSource(executorInfo *mesos.ExecutorInfo) { - if data := executorInfo.GetData(); len(data) > 0 && k.staticPodsConfigPath != "" { - log.V(2).Infof("extracting static pods config to %s", k.staticPodsConfigPath) - err := podutil.WriteToDir( - k.staticPodsFilters.Do(podutil.Gunzip(executorInfo.Data)), - k.staticPodsConfigPath, - ) - if err != nil { - log.Errorf("failed to initialize static pod configuration: %v", err) - } - } -} - -// Disconnected is called when the executor is disconnected from the slave. -func (k *Executor) Disconnected(driver bindings.ExecutorDriver) { - if k.isDone() { - return - } - log.Infof("Slave is disconnected\n") - if !(&k.state).transition(connectedState, disconnectedState) { - log.Errorf("failed to disconnect/transition to a disconnected state") - } -} - -// LaunchTask is called when the executor receives a request to launch a task. -// The happens when the k8sm scheduler has decided to schedule the pod -// (which corresponds to a Mesos Task) onto the node where this executor -// is running, but the binding is not recorded in the Kubernetes store yet. -// This function is invoked to tell the executor to record the binding in the -// Kubernetes store and start the pod via the Kubelet. -func (k *Executor) LaunchTask(driver bindings.ExecutorDriver, taskInfo *mesos.TaskInfo) { - if k.isDone() { - return - } - - log.Infof("Launch task %v\n", taskInfo) - - taskID := taskInfo.GetTaskId().GetValue() - if p := k.registry.pod(taskID); p != nil { - log.Warningf("task %v already launched", taskID) - // Not to send back TASK_RUNNING or TASK_FAILED here, because - // may be duplicated messages - return - } - - if !k.isConnected() { - log.Errorf("Ignore launch task because the executor is disconnected\n") - k.sendStatus(driver, newStatus(taskInfo.GetTaskId(), mesos.TaskState_TASK_FAILED, - messages.ExecutorUnregistered)) - return - } - - obj, err := kruntime.Decode(api.Codecs.UniversalDecoder(), taskInfo.GetData()) - if err != nil { - log.Errorf("failed to extract yaml data from the taskInfo.data %v", err) - k.sendStatus(driver, newStatus(taskInfo.GetTaskId(), mesos.TaskState_TASK_FAILED, - messages.UnmarshalTaskDataFailure)) - return - } - pod, ok := obj.(*api.Pod) - if !ok { - log.Errorf("expected *api.Pod instead of %T: %+v", pod, pod) - k.sendStatus(driver, newStatus(taskInfo.GetTaskId(), mesos.TaskState_TASK_FAILED, - messages.UnmarshalTaskDataFailure)) - return - } - - k.resetSuicideWatch(driver) - - // run the next step aync because it calls out to apiserver and we don't want to block here - go k.bindAndWatchTask(driver, taskInfo, time.NewTimer(k.launchGracePeriod), pod) -} - -// determine whether we need to start a suicide countdown. if so, then start -// a timer that, upon expiration, causes this executor to commit suicide. -// this implementation runs asynchronously. callers that wish to wait for the -// reset to complete may wait for the returned signal chan to close. -func (k *Executor) resetSuicideWatch(driver bindings.ExecutorDriver) <-chan struct{} { - ch := make(chan struct{}) - go func() { - defer close(ch) - k.lock.Lock() - defer k.lock.Unlock() - - if k.suicideTimeout < 1 { - return - } - - if k.suicideWatch != nil { - if !k.registry.empty() { - k.suicideWatch.Stop() - return - } - if k.suicideWatch.Reset(k.suicideTimeout) { - // valid timer, reset was successful - return - } - } - - //TODO(jdef) reduce verbosity here once we're convinced that suicide watch is working properly - log.Infof("resetting suicide watch timer for %v", k.suicideTimeout) - - k.suicideWatch = k.suicideWatch.Next(k.suicideTimeout, driver, jumper(k.attemptSuicide)) - }() - return ch -} - -func (k *Executor) attemptSuicide(driver bindings.ExecutorDriver, abort <-chan struct{}) { - k.lock.Lock() - defer k.lock.Unlock() - - // this attempt may have been queued and since been aborted - select { - case <-abort: - //TODO(jdef) reduce verbosity once suicide watch is working properly - log.Infof("aborting suicide attempt since watch was cancelled") - return - default: // continue - } - - // fail-safe, will abort kamikaze attempts if there are tasks - if !k.registry.empty() { - log.Errorf("suicide attempt failed, there are still running tasks") - return - } - - log.Infoln("Attempting suicide") - if (&k.state).transitionTo(suicidalState, suicidalState, terminalState) { - //TODO(jdef) let the scheduler know? - //TODO(jdef) is suicide more graceful than slave-demanded shutdown? - k.doShutdown(driver) - } -} - -func podStatusData(pod *api.Pod, status api.PodStatus) ([]byte, string, error) { - podFullName := container.GetPodFullName(pod) - data, err := json.Marshal(api.PodStatusResult{ - ObjectMeta: api.ObjectMeta{ - Name: podFullName, - SelfLink: "/podstatusresult", - }, - Status: status, - }) - return data, podFullName, err -} - -// async continuation of LaunchTask -func (k *Executor) bindAndWatchTask(driver bindings.ExecutorDriver, task *mesos.TaskInfo, launchTimer *time.Timer, pod *api.Pod) { - success := false - defer func() { - if !success { - k.killPodTask(driver, task.TaskId.GetValue()) - k.resetSuicideWatch(driver) - } - }() - - // allow a recently failed-over scheduler the chance to recover the task/pod binding: - // it may have failed and recovered before the apiserver is able to report the updated - // binding information. replays of this status event will signal to the scheduler that - // the apiserver should be up-to-date. - startingData, _, err := podStatusData(pod, api.PodStatus{}) - if err != nil { - log.Errorf("failed to generate pod-task starting data for task %v pod %v/%v: %v", - task.TaskId.GetValue(), pod.Namespace, pod.Name, err) - k.sendStatus(driver, newStatus(task.TaskId, mesos.TaskState_TASK_FAILED, err.Error())) - return - } - - err = k.registry.bind(task.TaskId.GetValue(), pod) - if err != nil { - log.Errorf("failed to bind task %v pod %v/%v: %v", - task.TaskId.GetValue(), pod.Namespace, pod.Name, err) - k.sendStatus(driver, newStatus(task.TaskId, mesos.TaskState_TASK_FAILED, err.Error())) - return - } - - // send TASK_STARTING - k.sendStatus(driver, &mesos.TaskStatus{ - TaskId: task.TaskId, - State: mesos.TaskState_TASK_STARTING.Enum(), - Message: proto.String(messages.CreateBindingSuccess), - Data: startingData, - }) - - // within the launch timeout window we should see a pod-task update via the registry. - // if we see a Running update then we need to generate a TASK_RUNNING status update for mesos. - handlerFinished := false - handler := &watchHandler{ - expiration: watchExpiration{ - timeout: launchTimer.C, - onEvent: func(taskID string) { - if !handlerFinished { - // launch timeout expired - k.killPodTask(driver, task.TaskId.GetValue()) - } - }, - }, - onEvent: func(podEvent *PodEvent) (bool, error) { - switch podEvent.eventType { - case PodEventUpdated: - log.V(2).Infof("Found status: '%v' for %s", podEvent.pod.Status, podEvent.FormatShort()) - - if podEvent.pod.Status.Phase != api.PodRunning { - // still waiting for pod to transition to a running state, so - // we're not done monitoring yet; check back later.. - break - } - - data, podFullName, err := podStatusData(podEvent.pod, podEvent.pod.Status) - if err != nil { - return false, fmt.Errorf("failed to marshal pod status result: %v", err) - } - - defer k.sendStatus(driver, &mesos.TaskStatus{ - TaskId: task.TaskId, - State: mesos.TaskState_TASK_RUNNING.Enum(), - Message: proto.String("pod-running:" + podFullName), - Data: data, - }) - fallthrough - - case PodEventDeleted: - // we're done monitoring because pod has been deleted - handlerFinished = true - launchTimer.Stop() - } - return handlerFinished, nil - }, - } - k.watcher.forTask(task.TaskId.GetValue(), handler) - success = true -} - -// KillTask is called when the executor receives a request to kill a task. -func (k *Executor) KillTask(driver bindings.ExecutorDriver, taskId *mesos.TaskID) { - k.killPodTask(driver, taskId.GetValue()) -} - -// deletes the pod and task associated with the task identified by taskID and sends a task -// status update to mesos. also attempts to reset the suicide watch. -func (k *Executor) killPodTask(driver bindings.ExecutorDriver, taskID string) { - pod := k.registry.pod(taskID) - if pod == nil { - log.V(1).Infof("Failed to remove task, unknown task %v\n", taskID) - k.sendStatus(driver, newStatus(&mesos.TaskID{Value: &taskID}, mesos.TaskState_TASK_LOST, "kill-pod-task")) - return - } - - // force-delete the pod from the API server - // TODO(jdef) possibly re-use eviction code from stock k8s once it lands? - err := k.kubeAPI.killPod(pod.Namespace, pod.Name) - if err != nil { - log.V(1).Infof("failed to delete task %v pod %v/%v from apiserver: %+v", taskID, pod.Namespace, pod.Name, err) - if apierrors.IsNotFound(err) { - k.sendStatus(driver, newStatus(&mesos.TaskID{Value: &taskID}, mesos.TaskState_TASK_LOST, "kill-pod-task")) - } - } -} - -// FrameworkMessage is called when the framework sends some message to the executor -func (k *Executor) FrameworkMessage(driver bindings.ExecutorDriver, message string) { - if k.isDone() { - return - } - if !k.isConnected() { - log.Warningf("Ignore framework message because the executor is disconnected\n") - return - } - - log.Infof("Receives message from framework %v\n", message) - //TODO(jdef) master reported a lost task, reconcile this! @see framework.go:handleTaskLost - if strings.HasPrefix(message, messages.TaskLost+":") { - taskId := message[len(messages.TaskLost)+1:] - if taskId != "" { - // TODO(jdef) would it make more sense to check the status of the task and - // just replay the last non-terminal message that we sent if the task is - // still active? - - // clean up pod state - k.sendStatus(driver, newStatus(&mesos.TaskID{Value: &taskId}, mesos.TaskState_TASK_LOST, messages.TaskLostAck)) - k.killPodTask(driver, taskId) - } - return - } - - switch message { - case messages.Kamikaze: - k.attemptSuicide(driver, nil) - } -} - -// Shutdown is called when the executor receives a shutdown request. -func (k *Executor) Shutdown(driver bindings.ExecutorDriver) { - k.lock.Lock() - defer k.lock.Unlock() - k.doShutdown(driver) -} - -// assumes that caller has obtained state lock -func (k *Executor) doShutdown(driver bindings.ExecutorDriver) { - defer func() { - log.Errorf("exiting with unclean shutdown: %v", recover()) - if k.exitFunc != nil { - k.exitFunc(1) - } - }() - - (&k.state).transitionTo(terminalState) - - // signal to all listeners that this KubeletExecutor is done! - close(k.terminate) - close(k.nodeInfos) - - if k.shutdownAlert != nil { - func() { - utilruntime.HandleCrash() - k.shutdownAlert() - }() - } - - log.Infoln("Stopping executor driver") - _, err := driver.Stop() - if err != nil { - log.Warningf("failed to stop executor driver: %v", err) - } - - log.Infoln("Shutdown the executor") - - // according to docs, mesos will generate TASK_LOST updates for us - // if needed, so don't take extra time to do that here. - k.registry.shutdown() - - select { - // the main Run() func may still be running... wait for it to finish: it will - // clear the pod configuration cleanly, telling k8s "there are no pods" and - // clean up resources (pods, volumes, etc). - case <-k.kubeletFinished: - - //TODO(jdef) attempt to wait for events to propagate to API server? - - // TODO(jdef) extract constant, should be smaller than whatever the - // slave graceful shutdown timeout period is. - case <-time.After(15 * time.Second): - log.Errorf("timed out waiting for kubelet Run() to die") - } - log.Infoln("exiting") - if k.exitFunc != nil { - k.exitFunc(0) - } -} - -// Destroy existing k8s containers -func (k *Executor) killKubeletContainers() { - if containers, err := dockertools.GetKubeletDockerContainers(k.dockerClient, true); err == nil { - opts := dockertypes.ContainerRemoveOptions{ - RemoveVolumes: true, - Force: true, - } - for _, container := range containers { - log.V(2).Infof("Removing container: %v", container.ID) - if err := k.dockerClient.RemoveContainer(container.ID, opts); err != nil { - log.Warning(err) - } - } - } else { - log.Warningf("Failed to list kubelet docker containers: %v", err) - } -} - -// Error is called when some error happens. -func (k *Executor) Error(driver bindings.ExecutorDriver, message string) { - log.Errorln(message) -} - -func newStatus(taskId *mesos.TaskID, state mesos.TaskState, message string) *mesos.TaskStatus { - return &mesos.TaskStatus{ - TaskId: taskId, - State: &state, - Message: proto.String(message), - } -} - -func (k *Executor) sendStatus(driver bindings.ExecutorDriver, status *mesos.TaskStatus) { - select { - case <-k.terminate: - default: - k.outgoing <- func() (mesos.Status, error) { return driver.SendStatusUpdate(status) } - } -} - -func (k *Executor) sendFrameworkMessage(driver bindings.ExecutorDriver, msg string) { - select { - case <-k.terminate: - default: - k.outgoing <- func() (mesos.Status, error) { return driver.SendFrameworkMessage(msg) } - } -} - -func (k *Executor) runSendLoop() { - defer log.V(1).Info("sender loop exiting") - for { - select { - case <-k.terminate: - return - default: - if !k.isConnected() { - select { - case <-k.terminate: - case <-time.After(1 * time.Second): - } - continue - } - sender, ok := <-k.outgoing - if !ok { - // programming error - panic("someone closed the outgoing channel") - } - if status, err := sender(); err == nil { - continue - } else { - log.Error(err) - if status == mesos.Status_DRIVER_ABORTED { - return - } - } - // attempt to re-queue the sender - select { - case <-k.terminate: - case k.outgoing <- sender: - } - } - } -} - -func annotationsFor(ei *mesos.ExecutorInfo) (annotations map[string]string, err error) { - annotations = map[string]string{} - if ei == nil { - return - } - - var buf bytes.Buffer - if err = executorinfo.EncodeResources(&buf, ei.GetResources()); err != nil { - return - } - - annotations[meta.ExecutorIdKey] = ei.GetExecutorId().GetValue() - annotations[meta.ExecutorResourcesKey] = buf.String() - - return -} diff --git a/contrib/mesos/pkg/executor/executor_test.go b/contrib/mesos/pkg/executor/executor_test.go deleted file mode 100644 index 6425068100d..00000000000 --- a/contrib/mesos/pkg/executor/executor_test.go +++ /dev/null @@ -1,636 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "reflect" - "sync" - "sync/atomic" - "testing" - "time" - - assertext "k8s.io/kubernetes/contrib/mesos/pkg/assert" - "k8s.io/kubernetes/contrib/mesos/pkg/executor/messages" - "k8s.io/kubernetes/contrib/mesos/pkg/podutil" - kmruntime "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask/hostport" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/testapi" - "k8s.io/kubernetes/pkg/api/unversioned" - "k8s.io/kubernetes/pkg/client/cache" - "k8s.io/kubernetes/pkg/kubelet/dockertools" - "k8s.io/kubernetes/pkg/runtime" - utiltesting "k8s.io/kubernetes/pkg/util/testing" - "k8s.io/kubernetes/pkg/util/wait" - "k8s.io/kubernetes/pkg/watch" - - "github.com/mesos/mesos-go/mesosproto" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -// TestExecutorRegister ensures that the executor thinks it is connected -// after Register is called. -func TestExecutorRegister(t *testing.T) { - mockDriver := &MockExecutorDriver{} - executor := NewTestKubernetesExecutor() - - executor.Init(mockDriver) - executor.Registered(mockDriver, nil, nil, nil) - - assert.Equal(t, true, executor.isConnected(), "executor should be connected") - mockDriver.AssertExpectations(t) -} - -// TestExecutorDisconnect ensures that the executor thinks that it is not -// connected after a call to Disconnected has occurred. -func TestExecutorDisconnect(t *testing.T) { - mockDriver := &MockExecutorDriver{} - executor := NewTestKubernetesExecutor() - - executor.Init(mockDriver) - executor.Registered(mockDriver, nil, nil, nil) - executor.Disconnected(mockDriver) - - assert.Equal(t, false, executor.isConnected(), - "executor should not be connected after Disconnected") - mockDriver.AssertExpectations(t) -} - -// TestExecutorReregister ensures that the executor thinks it is connected -// after a connection problem happens, followed by a call to Reregistered. -func TestExecutorReregister(t *testing.T) { - mockDriver := &MockExecutorDriver{} - executor := NewTestKubernetesExecutor() - - executor.Init(mockDriver) - executor.Registered(mockDriver, nil, nil, nil) - executor.Disconnected(mockDriver) - executor.Reregistered(mockDriver, nil) - - assert.Equal(t, true, executor.isConnected(), "executor should be connected") - mockDriver.AssertExpectations(t) -} - -type fakeRegistry struct { - sync.Mutex - boundTasks map[string]*api.Pod - updates chan *PodEvent -} - -func newFakeRegistry() *fakeRegistry { - return &fakeRegistry{boundTasks: map[string]*api.Pod{}, updates: make(chan *PodEvent, 100)} -} - -func (r *fakeRegistry) empty() bool { - r.Lock() - defer r.Unlock() - return len(r.boundTasks) == 0 -} - -func (r *fakeRegistry) pod(taskID string) *api.Pod { - r.Lock() - defer r.Unlock() - return r.boundTasks[taskID] -} - -func (r *fakeRegistry) watch() <-chan *PodEvent { return r.updates } - -func (r *fakeRegistry) shutdown() { - r.Lock() - defer r.Unlock() - r.boundTasks = map[string]*api.Pod{} -} - -func (r *fakeRegistry) bind(taskID string, pod *api.Pod) error { - r.Lock() - defer r.Unlock() - pod.Annotations = map[string]string{ - "k8s.mesosphere.io/taskId": taskID, - } - r.boundTasks[taskID] = pod - - // the normal registry sends a bind.. - r.updates <- &PodEvent{pod: pod, taskID: taskID, eventType: PodEventBound} - return nil -} - -func (r *fakeRegistry) Update(pod *api.Pod) (*PodEvent, error) { - r.Lock() - defer r.Unlock() - taskID, err := taskIDFor(pod) - if err != nil { - return nil, err - } - if _, ok := r.boundTasks[taskID]; !ok { - return nil, errUnknownTask - } - rp := &PodEvent{pod: pod, taskID: taskID, eventType: PodEventUpdated} - r.updates <- rp - return rp, nil -} - -func (r *fakeRegistry) Remove(taskID string) error { - r.Lock() - defer r.Unlock() - pod, ok := r.boundTasks[taskID] - if !ok { - return errUnknownTask - } - delete(r.boundTasks, taskID) - r.updates <- &PodEvent{pod: pod, taskID: taskID, eventType: PodEventDeleted} - return nil -} - -// phaseChange simulates a pod source update; normally this update is generated from a watch -func (r *fakeRegistry) phaseChange(pod *api.Pod, phase api.PodPhase) error { - clone, err := api.Scheme.DeepCopy(pod) - if err != nil { - return err - } - - phasedPod := clone.(*api.Pod) - phasedPod.Status.Phase = phase - _, err = r.Update(phasedPod) - return err -} - -// TestExecutorLaunchAndKillTask ensures that the executor is able to launch tasks and generates -// appropriate status messages for mesos. It then kills the task and validates that appropriate -// actions are taken by the executor. -func TestExecutorLaunchAndKillTask(t *testing.T) { - var ( - mockDriver = &MockExecutorDriver{} - registry = newFakeRegistry() - executor = New(Config{ - Docker: dockertools.ConnectToDockerOrDie("fake://", 0), - NodeInfos: make(chan NodeInfo, 1), - Registry: registry, - }) - mockKubeAPI = &mockKubeAPI{} - pod = NewTestPod(1) - executorinfo = &mesosproto.ExecutorInfo{} - ) - executor.kubeAPI = mockKubeAPI - executor.Init(mockDriver) - executor.Registered(mockDriver, nil, nil, nil) - - podTask, err := podtask.New( - api.NewDefaultContext(), - podtask.Config{ - Prototype: executorinfo, - HostPortStrategy: hostport.StrategyWildcard, - }, - pod, - ) - assert.Equal(t, nil, err, "must be able to create a task from a pod") - - pod.Annotations = map[string]string{ - "k8s.mesosphere.io/taskId": podTask.ID, - } - - podTask.Spec = &podtask.Spec{Executor: executorinfo} - taskInfo, err := podTask.BuildTaskInfo() - assert.Equal(t, nil, err, "must be able to build task info") - - data, err := runtime.Encode(testapi.Default.Codec(), pod) - assert.Equal(t, nil, err, "must be able to encode a pod's spec data") - - taskInfo.Data = data - var statusUpdateCalls sync.WaitGroup - statusUpdateCalls.Add(1) - statusUpdateDone := func(_ mock.Arguments) { statusUpdateCalls.Done() } - - mockDriver.On( - "SendStatusUpdate", - mesosproto.TaskState_TASK_STARTING, - ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(statusUpdateDone).Once() - - statusUpdateCalls.Add(1) - mockDriver.On( - "SendStatusUpdate", - mesosproto.TaskState_TASK_RUNNING, - ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(statusUpdateDone).Once() - - executor.LaunchTask(mockDriver, taskInfo) - - assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { - executor.lock.Lock() - defer executor.lock.Unlock() - return !registry.empty() - }, "executor must be able to create a task and a pod") - - // simulate a pod source update; normally this update is generated when binding a pod - err = registry.phaseChange(pod, api.PodPending) - assert.NoError(t, err) - - // simulate a pod source update; normally this update is generated by the kubelet once the pod is healthy - err = registry.phaseChange(pod, api.PodRunning) - assert.NoError(t, err) - - // Allow some time for asynchronous requests to the driver. - finished := kmruntime.After(statusUpdateCalls.Wait) - select { - case <-finished: - case <-time.After(wait.ForeverTestTimeout): - t.Fatalf("timed out waiting for status update calls to finish") - } - - statusUpdateCalls.Add(1) - mockDriver.On( - "SendStatusUpdate", - mesosproto.TaskState_TASK_KILLED, - ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(statusUpdateDone).Once() - - // simulate what happens when the apiserver is told to delete a pod - mockKubeAPI.On("killPod", pod.Namespace, pod.Name).Return(nil).Run(func(_ mock.Arguments) { - registry.Remove(podTask.ID) - }) - - executor.KillTask(mockDriver, taskInfo.TaskId) - assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { - executor.lock.Lock() - defer executor.lock.Unlock() - return registry.empty() - }, "executor must be able to kill a created task and pod") - - // Allow some time for asynchronous requests to the driver. - finished = kmruntime.After(statusUpdateCalls.Wait) - select { - case <-finished: - case <-time.After(wait.ForeverTestTimeout): - t.Fatalf("timed out waiting for status update calls to finish") - } - - mockDriver.AssertExpectations(t) - mockKubeAPI.AssertExpectations(t) -} - -// TestExecutorStaticPods test that the ExecutorInfo.data is parsed -// as a zip archive with pod definitions. -func TestExecutorInitializeStaticPodsSource(t *testing.T) { - // create some zip with static pod definition - givenPodsDir, err := utiltesting.MkTmpdir("executor-givenpods") - assert.NoError(t, err) - defer os.RemoveAll(givenPodsDir) - - var wg sync.WaitGroup - reportErrors := func(errCh <-chan error) { - wg.Add(1) - go func() { - defer wg.Done() - for err := range errCh { - t.Error(err) - } - }() - } - - createStaticPodFile := func(fileName, name string) { - spod := `{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "name": "%v", - "namespace": "staticpods", - "labels": { "name": "foo", "cluster": "bar" } - }, - "spec": { - "containers": [{ - "name": "%v", - "image": "library/nginx", - "ports": [{ "containerPort": 80, "name": "http" }] - }] - } - }` - destfile := filepath.Join(givenPodsDir, fileName) - err = os.MkdirAll(filepath.Dir(destfile), 0770) - assert.NoError(t, err) - err = ioutil.WriteFile(destfile, []byte(fmt.Sprintf(spod, name, name)), 0660) - assert.NoError(t, err) - } - - createStaticPodFile("spod.json", "spod-01") - createStaticPodFile("spod2.json", "spod-02") - createStaticPodFile("dir/spod.json", "spod-03") // same file name as first one to check for overwriting - staticpods, errs := podutil.ReadFromDir(givenPodsDir) - reportErrors(errs) - - gzipped, err := podutil.Gzip(staticpods) - assert.NoError(t, err) - - expectedStaticPodsNum := 2 // subdirectories are ignored by FileSource, hence only 2 - - // temporary directory which is normally located in the executor sandbox - staticPodsConfigPath, err := utiltesting.MkTmpdir("executor-k8sm-archive") - assert.NoError(t, err) - defer os.RemoveAll(staticPodsConfigPath) - - executor := &Executor{ - staticPodsConfigPath: staticPodsConfigPath, - } - - // extract the pods into staticPodsConfigPath - executor.initializeStaticPodsSource(&mesosproto.ExecutorInfo{Data: gzipped}) - - actualpods, errs := podutil.ReadFromDir(staticPodsConfigPath) - reportErrors(errs) - - list := podutil.List(actualpods) - assert.NotNil(t, list) - assert.Equal(t, expectedStaticPodsNum, len(list.Items)) - - var ( - expectedNames = map[string]struct{}{ - "spod-01": {}, - "spod-02": {}, - } - actualNames = map[string]struct{}{} - ) - for _, pod := range list.Items { - actualNames[pod.Name] = struct{}{} - } - assert.True(t, reflect.DeepEqual(expectedNames, actualNames), "expected %v instead of %v", expectedNames, actualNames) - - wg.Wait() -} - -// TestExecutorFrameworkMessage ensures that the executor is able to -// handle messages from the framework, specifically about lost tasks -// and Kamikaze. When a task is lost, the executor needs to clean up -// its state. When a Kamikaze message is received, the executor should -// attempt suicide. -func TestExecutorFrameworkMessage(t *testing.T) { - // TODO(jdef): Fix the unexpected call in the mocking system. - t.Skip("This test started failing when panic catching was disabled.") - var ( - mockDriver = &MockExecutorDriver{} - kubeletFinished = make(chan struct{}) - registry = newFakeRegistry() - executor = New(Config{ - Docker: dockertools.ConnectToDockerOrDie("fake://", 0), - NodeInfos: make(chan NodeInfo, 1), - ShutdownAlert: func() { - close(kubeletFinished) - }, - KubeletFinished: kubeletFinished, - Registry: registry, - }) - pod = NewTestPod(1) - mockKubeAPI = &mockKubeAPI{} - ) - - executor.kubeAPI = mockKubeAPI - executor.Init(mockDriver) - executor.Registered(mockDriver, nil, nil, nil) - executor.FrameworkMessage(mockDriver, "test framework message") - - // set up a pod to then lose - executorinfo := &mesosproto.ExecutorInfo{} - podTask, _ := podtask.New( - api.NewDefaultContext(), - podtask.Config{ - ID: "foo", - Prototype: executorinfo, - HostPortStrategy: hostport.StrategyWildcard, - }, - pod, - ) - pod.Annotations = map[string]string{ - "k8s.mesosphere.io/taskId": podTask.ID, - } - podTask.Spec = &podtask.Spec{ - Executor: executorinfo, - } - - taskInfo, err := podTask.BuildTaskInfo() - assert.Equal(t, nil, err, "must be able to build task info") - - data, _ := runtime.Encode(testapi.Default.Codec(), pod) - taskInfo.Data = data - - mockDriver.On( - "SendStatusUpdate", - mesosproto.TaskState_TASK_STARTING, - ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Once() - - called := make(chan struct{}) - mockDriver.On( - "SendStatusUpdate", - mesosproto.TaskState_TASK_RUNNING, - ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(func(_ mock.Arguments) { close(called) }).Once() - - executor.LaunchTask(mockDriver, taskInfo) - - // must wait for this otherwise phase changes may not apply - assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { - executor.lock.Lock() - defer executor.lock.Unlock() - return !registry.empty() - }, "executor must be able to create a task and a pod") - - err = registry.phaseChange(pod, api.PodPending) - assert.NoError(t, err) - err = registry.phaseChange(pod, api.PodRunning) - assert.NoError(t, err) - - // waiting until the pod is really running b/c otherwise a TASK_FAILED could be - // triggered by the asynchronously running executor methods when removing the task - // from k.tasks through the "task-lost:foo" message below. - select { - case <-called: - case <-time.After(wait.ForeverTestTimeout): - t.Fatalf("timed out waiting for SendStatusUpdate for the running task") - } - - // send task-lost message for it - called = make(chan struct{}) - mockDriver.On( - "SendStatusUpdate", - mesosproto.TaskState_TASK_LOST, - ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(func(_ mock.Arguments) { close(called) }).Once() - - // simulate what happens when the apiserver is told to delete a pod - mockKubeAPI.On("killPod", pod.Namespace, pod.Name).Return(nil).Run(func(_ mock.Arguments) { - registry.Remove(podTask.ID) - }) - - executor.FrameworkMessage(mockDriver, "task-lost:foo") - - assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { - executor.lock.Lock() - defer executor.lock.Unlock() - return registry.empty() - }, "executor must be able to kill a created task and pod") - - select { - case <-called: - case <-time.After(wait.ForeverTestTimeout): - t.Fatalf("timed out waiting for SendStatusUpdate") - } - - mockDriver.On("Stop").Return(mesosproto.Status_DRIVER_STOPPED, nil).Once() - - executor.FrameworkMessage(mockDriver, messages.Kamikaze) - assert.Equal(t, true, executor.isDone(), - "executor should have shut down after receiving a Kamikaze message") - - mockDriver.AssertExpectations(t) - mockKubeAPI.AssertExpectations(t) -} - -// Create a pod with a given index, requiring one port -func NewTestPod(i int) *api.Pod { - name := fmt.Sprintf("pod%d", i) - return &api.Pod{ - TypeMeta: unversioned.TypeMeta{APIVersion: testapi.Default.GroupVersion().String()}, - ObjectMeta: api.ObjectMeta{ - Name: name, - Namespace: api.NamespaceDefault, - SelfLink: testapi.Default.SelfLink("pods", string(i)), - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: "foo", - Ports: []api.ContainerPort{ - { - ContainerPort: int32(8000 + i), - Protocol: api.ProtocolTCP, - }, - }, - }, - }, - }, - Status: api.PodStatus{ - Conditions: []api.PodCondition{ - { - Type: api.PodReady, - Status: api.ConditionTrue, - }, - }, - }, - } -} - -// Create mock of pods ListWatch, usually listening on the apiserver pods watch endpoint -type MockPodsListWatch struct { - ListWatch cache.ListWatch - fakeWatcher *watch.FakeWatcher - list api.PodList -} - -// A apiserver mock which partially mocks the pods API -type TestServer struct { - server *httptest.Server - Stats map[string]uint - lock sync.Mutex -} - -func NewTestServer(t *testing.T, namespace string) *TestServer { - ts := TestServer{ - Stats: map[string]uint{}, - } - mux := http.NewServeMux() - - mux.HandleFunc(testapi.Default.ResourcePath("bindings", namespace, ""), func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - }) - - ts.server = httptest.NewServer(mux) - return &ts -} - -func NewMockPodsListWatch(initialPodList api.PodList) *MockPodsListWatch { - lw := MockPodsListWatch{ - fakeWatcher: watch.NewFake(), - list: initialPodList, - } - lw.ListWatch = cache.ListWatch{ - WatchFunc: func(options api.ListOptions) (watch.Interface, error) { - return lw.fakeWatcher, nil - }, - ListFunc: func(options api.ListOptions) (runtime.Object, error) { - return &lw.list, nil - }, - } - return &lw -} - -// TestExecutorShutdown ensures that the executor properly shuts down -// when Shutdown is called. -func TestExecutorShutdown(t *testing.T) { - var ( - mockDriver = &MockExecutorDriver{} - kubeletFinished = make(chan struct{}) - exitCalled = int32(0) - executor = New(Config{ - Docker: dockertools.ConnectToDockerOrDie("fake://", 0), - NodeInfos: make(chan NodeInfo, 1), - ShutdownAlert: func() { - close(kubeletFinished) - }, - KubeletFinished: kubeletFinished, - ExitFunc: func(_ int) { - atomic.AddInt32(&exitCalled, 1) - }, - Registry: newFakeRegistry(), - }) - ) - - executor.Init(mockDriver) - executor.Registered(mockDriver, nil, nil, nil) - mockDriver.On("Stop").Return(mesosproto.Status_DRIVER_STOPPED, nil).Once() - executor.Shutdown(mockDriver) - - assert.Equal(t, false, executor.isConnected(), - "executor should not be connected after Shutdown") - assert.Equal(t, true, executor.isDone(), - "executor should be in Done state after Shutdown") - assert.Equal(t, true, atomic.LoadInt32(&exitCalled) > 0, - "the executor should call its ExitFunc when it is ready to close down") - mockDriver.AssertExpectations(t) -} - -func TestExecutorsendFrameworkMessage(t *testing.T) { - mockDriver := &MockExecutorDriver{} - executor := NewTestKubernetesExecutor() - - executor.Init(mockDriver) - executor.Registered(mockDriver, nil, nil, nil) - - called := make(chan struct{}) - mockDriver.On( - "SendFrameworkMessage", - "foo bar baz", - ).Return(mesosproto.Status_DRIVER_RUNNING, nil).Run(func(_ mock.Arguments) { close(called) }).Once() - executor.sendFrameworkMessage(mockDriver, "foo bar baz") - - // guard against data race in mock driver between AssertExpectations and Called - select { - case <-called: // expected - case <-time.After(wait.ForeverTestTimeout): - t.Fatalf("expected call to SendFrameworkMessage") - } - mockDriver.AssertExpectations(t) -} diff --git a/contrib/mesos/pkg/executor/messages/doc.go b/contrib/mesos/pkg/executor/messages/doc.go deleted file mode 100644 index 4e3991f09cc..00000000000 --- a/contrib/mesos/pkg/executor/messages/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package messages exposes executor event/message names as constants. -package messages // import "k8s.io/kubernetes/contrib/mesos/pkg/executor/messages" diff --git a/contrib/mesos/pkg/executor/messages/messages.go b/contrib/mesos/pkg/executor/messages/messages.go deleted file mode 100644 index 1fd48e57980..00000000000 --- a/contrib/mesos/pkg/executor/messages/messages.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package messages - -// messages that ship with TaskStatus objects - -const ( - ContainersDisappeared = "containers-disappeared" - CreateBindingFailure = "create-binding-failure" - CreateBindingSuccess = "create-binding-success" - ExecutorUnregistered = "executor-unregistered" - ExecutorShutdown = "executor-shutdown" - LaunchTaskFailed = "launch-task-failed" - KubeletPodLaunchFailed = "kubelet-pod-launch-failed" - TaskKilled = "task-killed" - TaskLost = "task-lost" - UnmarshalTaskDataFailure = "unmarshal-task-data-failure" - TaskLostAck = "task-lost-ack" // executor acknowledgment of forwarded TASK_LOST framework message - Kamikaze = "kamikaze" - WrongSlaveFailure = "pod-for-wrong-slave-failure" - AnnotationUpdateFailure = "annotation-update-failure" -) diff --git a/contrib/mesos/pkg/executor/mock_test.go b/contrib/mesos/pkg/executor/mock_test.go deleted file mode 100644 index 7462d638dda..00000000000 --- a/contrib/mesos/pkg/executor/mock_test.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import ( - "testing" - - "github.com/mesos/mesos-go/mesosproto" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "k8s.io/kubernetes/pkg/kubelet/dockertools" -) - -type mockKubeAPI struct { - mock.Mock -} - -func (m *mockKubeAPI) killPod(ns, name string) error { - args := m.Called(ns, name) - return args.Error(0) -} - -type MockExecutorDriver struct { - mock.Mock -} - -func (m *MockExecutorDriver) Start() (mesosproto.Status, error) { - args := m.Called() - return args.Get(0).(mesosproto.Status), args.Error(1) -} - -func (m *MockExecutorDriver) Stop() (mesosproto.Status, error) { - args := m.Called() - return args.Get(0).(mesosproto.Status), args.Error(1) -} - -func (m *MockExecutorDriver) Abort() (mesosproto.Status, error) { - args := m.Called() - return args.Get(0).(mesosproto.Status), args.Error(1) -} - -func (m *MockExecutorDriver) Join() (mesosproto.Status, error) { - args := m.Called() - return args.Get(0).(mesosproto.Status), args.Error(1) -} - -func (m *MockExecutorDriver) Run() (mesosproto.Status, error) { - args := m.Called() - return args.Get(0).(mesosproto.Status), args.Error(1) -} - -func (m *MockExecutorDriver) SendStatusUpdate(taskStatus *mesosproto.TaskStatus) (mesosproto.Status, error) { - args := m.Called(*taskStatus.State) - return args.Get(0).(mesosproto.Status), args.Error(1) -} - -func (m *MockExecutorDriver) SendFrameworkMessage(msg string) (mesosproto.Status, error) { - args := m.Called(msg) - return args.Get(0).(mesosproto.Status), args.Error(1) -} - -func NewTestKubernetesExecutor() *Executor { - return New(Config{ - Docker: dockertools.ConnectToDockerOrDie("fake://", 0), - Registry: newFakeRegistry(), - }) -} - -func TestExecutorNew(t *testing.T) { - mockDriver := &MockExecutorDriver{} - executor := NewTestKubernetesExecutor() - executor.Init(mockDriver) - - assert.Equal(t, executor.isDone(), false, "executor should not be in Done state on initialization") - assert.Equal(t, executor.isConnected(), false, "executor should not be connected on initialization") -} diff --git a/contrib/mesos/pkg/executor/node.go b/contrib/mesos/pkg/executor/node.go deleted file mode 100644 index d2a26d3f75b..00000000000 --- a/contrib/mesos/pkg/executor/node.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import mesos "github.com/mesos/mesos-go/mesosproto" - -type NodeInfo struct { - Cores int - Mem uint64 // in bytes -} - -func nodeInfo(si *mesos.SlaveInfo, ei *mesos.ExecutorInfo) NodeInfo { - var executorCPU, executorMem float64 - - // get executor resources - if ei != nil { - for _, r := range ei.GetResources() { - if r == nil || r.GetType() != mesos.Value_SCALAR { - continue - } - switch r.GetName() { - case "cpus": - executorCPU += r.GetScalar().GetValue() - case "mem": - executorMem += r.GetScalar().GetValue() - } - } - } - - // get resource capacity of the node - ni := NodeInfo{} - for _, r := range si.GetResources() { - if r == nil || r.GetType() != mesos.Value_SCALAR { - continue - } - - switch r.GetName() { - case "cpus": - // We intentionally take the floor of executorCPU because cores are integers - // and we would loose a complete cpu here if the value is <1. - // TODO(sttts): switch to float64 when "Machine Allocables" are implemented - ni.Cores += int(r.GetScalar().GetValue()) - case "mem": - ni.Mem += uint64(r.GetScalar().GetValue()) * 1024 * 1024 - } - } - - // TODO(sttts): subtract executorCPU/Mem from static pod resources before subtracting them from the capacity - ni.Cores -= int(executorCPU) - ni.Mem -= uint64(executorMem) * 1024 * 1024 - - return ni -} diff --git a/contrib/mesos/pkg/executor/registry.go b/contrib/mesos/pkg/executor/registry.go deleted file mode 100644 index 16bdce3f39d..00000000000 --- a/contrib/mesos/pkg/executor/registry.go +++ /dev/null @@ -1,340 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import ( - "encoding/json" - "errors" - "sync" - - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - - "k8s.io/kubernetes/contrib/mesos/pkg/executor/messages" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/pkg/api" - - log "github.com/golang/glog" -) - -type ( - podEventType int - - PodEvent struct { - pod *api.Pod - taskID string - eventType podEventType - } - - // Registry is a state store for pod task metadata. Clients are expected to watch() the - // event stream to observe changes over time. - Registry interface { - // Update modifies the registry's iternal representation of the pod; it may also - // modify the pod argument itself. An update may fail because either a pod isn't - // labeled with a task ID, the task ID is unknown, or the nature of the update may - // be incompatible with what's supported in kubernetes-mesos. - Update(pod *api.Pod) (*PodEvent, error) - - // Remove the task from this registry, returns an error if the taskID is unknown. - Remove(taskID string) error - - // bind associates a taskID with a pod, triggers the binding API on the k8s apiserver - // and stores the resulting pod-task metadata. - bind(taskID string, pod *api.Pod) error - - // watch returns the event stream of the registry. clients are expected to read this - // stream otherwise the event buffer will fill up and registry ops will block. - watch() <-chan *PodEvent - - // return true if there are no tasks registered - empty() bool - - // return the api.Pod registered to the given taskID or else nil - pod(taskID string) *api.Pod - - // shutdown any related async processing and clear the internal state of the registry - shutdown() - } - - registryImpl struct { - client *clientset.Clientset - updates chan *PodEvent - m sync.RWMutex - boundTasks map[string]*api.Pod - } -) - -var ( - errCreateBindingFailed = errors.New(messages.CreateBindingFailure) - errAnnotationUpdateFailure = errors.New(messages.AnnotationUpdateFailure) - errUnknownTask = errors.New("unknown task ID") - errUnsupportedUpdate = errors.New("pod update allowed by k8s is incompatible with this version of k8s-mesos") -) - -const ( - PodEventBound podEventType = iota - PodEventUpdated - PodEventDeleted - PodEventIncompatibleUpdate - - updatesBacklogSize = 200 -) - -func IsUnsupportedUpdate(err error) bool { - return err == errUnsupportedUpdate -} - -func (rp *PodEvent) Task() string { - return rp.taskID -} - -func (rp *PodEvent) Pod() *api.Pod { - return rp.pod -} - -func (rp *PodEvent) FormatShort() string { - return "task '" + rp.taskID + "' pod '" + rp.pod.Namespace + "/" + rp.pod.Name + "'" -} - -func NewRegistry(client *clientset.Clientset) Registry { - r := ®istryImpl{ - client: client, - updates: make(chan *PodEvent, updatesBacklogSize), - boundTasks: make(map[string]*api.Pod), - } - return r -} - -func (r *registryImpl) watch() <-chan *PodEvent { - return r.updates -} - -func taskIDFor(pod *api.Pod) (taskID string, err error) { - taskID = pod.Annotations[meta.TaskIdKey] - if taskID == "" { - err = errUnknownTask - } - return -} - -func (r *registryImpl) shutdown() { - //TODO(jdef) flesh this out - r.m.Lock() - defer r.m.Unlock() - r.boundTasks = map[string]*api.Pod{} -} - -func (r *registryImpl) empty() bool { - r.m.RLock() - defer r.m.RUnlock() - return len(r.boundTasks) == 0 -} - -func (r *registryImpl) pod(taskID string) *api.Pod { - r.m.RLock() - defer r.m.RUnlock() - return r.boundTasks[taskID] -} - -func (r *registryImpl) Remove(taskID string) error { - r.m.Lock() - defer r.m.Unlock() - pod, ok := r.boundTasks[taskID] - if !ok { - return errUnknownTask - } - - delete(r.boundTasks, taskID) - - r.updates <- &PodEvent{ - pod: pod, - taskID: taskID, - eventType: PodEventDeleted, - } - - log.V(1).Infof("unbound task %v from pod %v/%v", taskID, pod.Namespace, pod.Name) - return nil -} - -func (r *registryImpl) Update(pod *api.Pod) (*PodEvent, error) { - // Don't do anything for pods without task anotation which means: - // - "pre-scheduled" pods which have a NodeName set to this node without being scheduled already. - // - static/mirror pods: they'll never have a TaskID annotation, and we don't expect them to ever change. - // - all other pods that haven't passed through the launch-task-binding phase, which would set annotations. - taskID, err := taskIDFor(pod) - if err != nil { - // There also could be a race between the overall launch-task process and this update, but here we - // will never be able to process such a stale update because the "update pod" that we're receiving - // in this func won't yet have a task ID annotation. It follows that we can safely drop such a stale - // update on the floor because we'll get another update later that, in addition to the changes that - // we're dropping now, will also include the changes from the binding process. - log.V(5).Infof("ignoring pod update for %s/%s because %s annotation is missing", pod.Namespace, pod.Name, meta.TaskIdKey) - return nil, err - } - - // be a good citizen: copy the arg before making any changes to it - clone, err := api.Scheme.DeepCopy(pod) - if err != nil { - return nil, err - } - pod = clone.(*api.Pod) - - r.m.Lock() - defer r.m.Unlock() - oldPod, ok := r.boundTasks[taskID] - if !ok { - return nil, errUnknownTask - } - - registeredPod := &PodEvent{ - pod: pod, - taskID: taskID, - eventType: PodEventUpdated, - } - - // TODO(jdef) would be nice to only execute this logic based on the presence of - // some particular annotation: - // - preserve the original container port spec since the k8sm scheduler - // has likely changed it. - if !copyPorts(pod, oldPod) { - // TODO(jdef) the state of "pod" is possibly inconsistent at this point. - // we don't care for the moment - we might later. - registeredPod.eventType = PodEventIncompatibleUpdate - r.updates <- registeredPod - log.Warningf("pod containers changed in an incompatible way; aborting update") - return registeredPod, errUnsupportedUpdate - } - - // update our internal copy and broadcast the change - r.boundTasks[taskID] = pod - r.updates <- registeredPod - - log.V(1).Infof("updated task %v pod %v/%v", taskID, pod.Namespace, pod.Name) - return registeredPod, nil -} - -// copyPorts copies the container pod specs from src to dest and returns -// true if all ports (in both dest and src) are accounted for, otherwise -// false. if returning false then it's possible that only a partial copy -// has been performed. -func copyPorts(dest, src *api.Pod) bool { - containers := src.Spec.Containers - ctPorts := make(map[string][]api.ContainerPort, len(containers)) - for i := range containers { - ctPorts[containers[i].Name] = containers[i].Ports - } - containers = dest.Spec.Containers - for i := range containers { - name := containers[i].Name - if ports, found := ctPorts[name]; found { - containers[i].Ports = ports - delete(ctPorts, name) - } else { - // old pod spec is missing this container?! - return false - } - } - if len(ctPorts) > 0 { - // new pod spec has containers that aren't in the old pod spec - return false - } - return true -} - -func (r *registryImpl) bind(taskID string, pod *api.Pod) error { - // validate taskID matches that of the annotation - annotatedTaskID, err := taskIDFor(pod) - if err != nil { - log.Warning("failed to bind: missing task ID annotation for pod ", pod.Namespace+"/"+pod.Name) - return errCreateBindingFailed - } - if annotatedTaskID != taskID { - log.Warningf("failed to bind: expected task-id %v instead of %v for pod %v/%v", taskID, annotatedTaskID, pod.Namespace, pod.Name) - return errCreateBindingFailed - } - - // record this as a bound task for now so that we can avoid racing with the mesos pod source, who is - // watching the apiserver for pod updates and will verify pod-task validity with us upon receiving such - boundSuccessfully := false - defer func() { - if !boundSuccessfully { - r.m.Lock() - defer r.m.Unlock() - delete(r.boundTasks, taskID) - } - }() - func() { - r.m.Lock() - defer r.m.Unlock() - r.boundTasks[taskID] = pod - }() - - if pod.Spec.NodeName == "" { - //HACK(jdef): cloned binding construction from k8s plugin/pkg/scheduler/framework.go - binding := &api.Binding{ - ObjectMeta: api.ObjectMeta{ - Namespace: pod.Namespace, - Name: pod.Name, - Annotations: make(map[string]string), - }, - Target: api.ObjectReference{ - Kind: "Node", - Name: pod.Annotations[meta.BindingHostKey], - }, - } - - // forward the annotations that the scheduler wants to apply - for k, v := range pod.Annotations { - binding.Annotations[k] = v - } - - // create binding on apiserver - log.Infof("Binding task %v pod '%v/%v' to '%v' with annotations %+v...", - taskID, pod.Namespace, pod.Name, binding.Target.Name, binding.Annotations) - ctx := api.WithNamespace(api.NewContext(), binding.Namespace) - err := r.client.CoreClient.Post().Namespace(api.NamespaceValue(ctx)).Resource("bindings").Body(binding).Do().Error() - if err != nil { - log.Warningf("failed to bind task %v pod %v/%v: %v", taskID, pod.Namespace, pod.Name, err) - return errCreateBindingFailed - } - } else { - // post annotations update to apiserver - patch := struct { - Metadata struct { - Annotations map[string]string `json:"annotations"` - } `json:"metadata"` - }{} - patch.Metadata.Annotations = pod.Annotations - patchJson, _ := json.Marshal(patch) - log.V(4).Infof("Patching annotations %v of task %v pod %v/%v: %v", pod.Annotations, taskID, pod.Namespace, pod.Name, string(patchJson)) - err := r.client.CoreClient.Patch(api.MergePatchType).RequestURI(pod.SelfLink).Body(patchJson).Do().Error() - if err != nil { - log.Errorf("Error updating annotations of ready-to-launch task %v pod %v/%v: %v", taskID, pod.Namespace, pod.Name, err) - return errAnnotationUpdateFailure - } - } - - boundSuccessfully = true - - r.updates <- &PodEvent{ - pod: pod, - taskID: taskID, - eventType: PodEventBound, - } - - log.V(1).Infof("bound task %v to pod %v/%v", taskID, pod.Namespace, pod.Name) - return nil -} diff --git a/contrib/mesos/pkg/executor/service/cadvisor.go b/contrib/mesos/pkg/executor/service/cadvisor.go deleted file mode 100644 index 5487e461d91..00000000000 --- a/contrib/mesos/pkg/executor/service/cadvisor.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "k8s.io/kubernetes/pkg/kubelet/cadvisor" - - cadvisorapi "github.com/google/cadvisor/info/v1" -) - -type MesosCadvisor struct { - cadvisor.Interface - cores int - mem uint64 -} - -func NewMesosCadvisor(cores int, mem uint64, port uint, runtime string) (*MesosCadvisor, error) { - c, err := cadvisor.New(port, runtime) - if err != nil { - return nil, err - } - return &MesosCadvisor{c, cores, mem}, nil -} - -func (mc *MesosCadvisor) MachineInfo() (*cadvisorapi.MachineInfo, error) { - mi, err := mc.Interface.MachineInfo() - if err != nil { - return nil, err - } - - // set Mesos provided values - mesosMi := *mi - mesosMi.NumCores = mc.cores - mesosMi.MemoryCapacity = mc.mem - - return &mesosMi, nil -} diff --git a/contrib/mesos/pkg/executor/service/doc.go b/contrib/mesos/pkg/executor/service/doc.go deleted file mode 100644 index f85f03406e2..00000000000 --- a/contrib/mesos/pkg/executor/service/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package service contains the cmd/k8sm-executor glue code. -package service // import "k8s.io/kubernetes/contrib/mesos/pkg/executor/service" diff --git a/contrib/mesos/pkg/executor/service/kubelet.go b/contrib/mesos/pkg/executor/service/kubelet.go deleted file mode 100644 index dbdd48d3922..00000000000 --- a/contrib/mesos/pkg/executor/service/kubelet.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - log "github.com/golang/glog" - "k8s.io/kubernetes/pkg/kubelet" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - "k8s.io/kubernetes/pkg/util/runtime" - "k8s.io/kubernetes/pkg/util/wait" -) - -// executorKubelet decorates the kubelet with a Run function that notifies the -// executor by closing kubeletDone before entering blocking state. -type executorKubelet struct { - *kubelet.Kubelet - kubeletDone chan<- struct{} // closed once kubelet.Run() returns - executorDone <-chan struct{} // closed when executor terminates -} - -// Run runs the main kubelet loop, closing the kubeletFinished chan when the -// loop exits. Like the upstream Run, it will never return. -func (kl *executorKubelet) Run(mergedUpdates <-chan kubetypes.PodUpdate) { - defer func() { - // When this Run function is called, we close it here. - // Otherwise, KubeletExecutorServer.runKubelet will. - close(kl.kubeletDone) - runtime.HandleCrash() - log.Infoln("kubelet run terminated") //TODO(jdef) turn down verbosity - // important: never return! this is in our contract - select {} - }() - - // push merged updates into another, closable update channel which is closed - // when the executor shuts down. - closableUpdates := make(chan kubetypes.PodUpdate) - go func() { - // closing closableUpdates will cause our patched kubelet's syncLoop() to exit - defer close(closableUpdates) - pipeLoop: - for { - select { - case <-kl.executorDone: - break pipeLoop - default: - select { - case u := <-mergedUpdates: - select { - case closableUpdates <- u: // noop - case <-kl.executorDone: - break pipeLoop - } - case <-kl.executorDone: - break pipeLoop - } - } - } - }() - - // we expect that Run() will complete after closableUpdates is closed and the - // kubelet's syncLoop() has finished processing its backlog, which hopefully - // will not take very long. Peeking into the future (current k8s master) it - // seems that the backlog has grown from 1 to 50 -- this may negatively impact - // us going forward, time will tell. - wait.Until(func() { kl.Kubelet.Run(closableUpdates) }, 0, kl.executorDone) - - //TODO(jdef) revisit this if/when executor failover lands - // Force kubelet to delete all pods. - kl.HandlePodRemoves(kl.GetPods()) -} diff --git a/contrib/mesos/pkg/executor/service/podsource/podsource.go b/contrib/mesos/pkg/executor/service/podsource/podsource.go deleted file mode 100644 index 8a2d93f72ee..00000000000 --- a/contrib/mesos/pkg/executor/service/podsource/podsource.go +++ /dev/null @@ -1,200 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podsource - -import ( - "k8s.io/kubernetes/contrib/mesos/pkg/executor" - "k8s.io/kubernetes/contrib/mesos/pkg/podutil" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/client/cache" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - - log "github.com/golang/glog" -) - -type ( - filterType int - - podName struct { - namespace, name string - } - - // Filter is invoked for each snapshot of pod state that passes through this source - Filter interface { - // Before is invoked before any pods are evaluated - Before(podCount int) - // Accept returns true if this pod should be accepted by the source; a value - // of false results in the pod appearing to have been removed from apiserver. - // If true, the caller should use the output pod value for the remainder of - // the processing task. If false then the output pod value may be nil. - Accept(*api.Pod) (*api.Pod, bool) - // After is invoked after all pods have been evaluated - After() - } - - // FilterFunc is a simplified Filter implementation that only implements Filter.Accept, its - // Before and After implementations are noop. - FilterFunc func(*api.Pod) (*api.Pod, bool) - - Source struct { - stop <-chan struct{} - out chan<- interface{} // never close this because pkg/util/config.mux doesn't handle that very well - filters []Filter // additional filters to apply to pod objects - } - - Option func(*Source) -) - -const ( - // if we don't use this source then the kubelet will do funny, mirror things. we alias - // this here for convenience. see the docs for Source for additional explanation. - // @see ConfigSourceAnnotationKey - MesosSource = kubetypes.ApiserverSource -) - -func (f FilterFunc) Before(_ int) {} -func (f FilterFunc) After() {} -func (f FilterFunc) Accept(pod *api.Pod) (*api.Pod, bool) { return f(pod) } - -// Mesos spawns a new pod source that watches API server for changes and collaborates with -// executor.Registry to generate api.Pod objects in a fashion that's very Mesos-aware. -func Mesos( - stop <-chan struct{}, - out chan<- interface{}, - podWatch *cache.ListWatch, - registry executor.Registry, - options ...Option, -) { - source := &Source{ - stop: stop, - out: out, - filters: []Filter{ - FilterFunc(filterMirrorPod), - ®isteredPodFilter{registry: registry}, - }, - } - // note: any filters added by options should be applied after the defaults - for _, opt := range options { - opt(source) - } - // reflect changes from the watch into a chan, filtered to include only mirror pods - // (have an ConfigMirrorAnnotationKey attr) - cache.NewReflector( - podWatch, - &api.Pod{}, - cache.NewUndeltaStore(source.send, cache.MetaNamespaceKeyFunc), - 0, - ).RunUntil(stop) -} - -func filterMirrorPod(p *api.Pod) (*api.Pod, bool) { - _, ok := (*p).Annotations[kubetypes.ConfigMirrorAnnotationKey] - return p, ok -} - -type registeredPodFilter struct { - priorPodNames, podNames map[podName]string // maps a podName to a taskID - registry executor.Registry -} - -func (rpf *registeredPodFilter) Before(podCount int) { - rpf.priorPodNames = rpf.podNames - rpf.podNames = make(map[podName]string, podCount) -} - -func (rpf *registeredPodFilter) After() { - // detect when pods are deleted and notify the registry - for k, taskID := range rpf.priorPodNames { - if _, found := rpf.podNames[k]; !found { - rpf.registry.Remove(taskID) - } - } -} - -func (rpf *registeredPodFilter) Accept(p *api.Pod) (*api.Pod, bool) { - rpod, err := rpf.registry.Update(p) - if err == nil { - // pod is bound to a task, and the update is compatible - // so we'll allow it through - p = rpod.Pod() // use the (possibly) updated pod spec! - rpf.podNames[podName{p.Namespace, p.Name}] = rpod.Task() - return p, true - } - if rpod != nil { - // we were able to ID the pod but the update still failed... - log.Warningf("failed to update registry for task %v pod %v/%v: %v", - rpod.Task(), p.Namespace, p.Name, err) - } - return nil, false -} - -// send is an update callback invoked by NewUndeltaStore; it applies all of source.filters -// to the incoming pod snapshot and forwards a PodUpdate that contains a snapshot of all -// the pods that were accepted by the filters. -func (source *Source) send(objs []interface{}) { - var ( - podCount = len(objs) - pods = make([]*api.Pod, 0, podCount) - ) - - for _, f := range source.filters { - f.Before(podCount) - } -foreachPod: - for _, o := range objs { - p := o.(*api.Pod) - for _, f := range source.filters { - if p, ok := f.Accept(p); ok { - pods = append(pods, p) - continue foreachPod - } - } - // unrecognized pod - log.V(2).Infof("skipping pod %v/%v", p.Namespace, p.Name) - } - // TODO(jdef) should these be applied in reverse order instead? - for _, f := range source.filters { - f.After() - } - - u := kubetypes.PodUpdate{ - Op: kubetypes.SET, - Pods: pods, - Source: MesosSource, - } - select { - case <-source.stop: - case source.out <- u: - log.V(2).Infof("sent %d pod updates", len(pods)) - } -} - -func ContainerEnvOverlay(env []api.EnvVar) Option { - return func(s *Source) { - // prepend this filter so that it impacts *all* pods running on the slave - s.filters = append([]Filter{filterContainerEnvOverlay(env)}, s.filters...) - } -} - -func filterContainerEnvOverlay(env []api.EnvVar) FilterFunc { - f := podutil.Environment(env) - return func(pod *api.Pod) (*api.Pod, bool) { - f(pod) - // we should't vote, let someone else decide whether the pod gets accepted - return pod, false - } -} diff --git a/contrib/mesos/pkg/executor/service/service.go b/contrib/mesos/pkg/executor/service/service.go deleted file mode 100644 index 322e6346e96..00000000000 --- a/contrib/mesos/pkg/executor/service/service.go +++ /dev/null @@ -1,321 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "fmt" - "net" - "os" - "path/filepath" - "time" - - log "github.com/golang/glog" - bindings "github.com/mesos/mesos-go/executor" - "github.com/spf13/pflag" - kubeletapp "k8s.io/kubernetes/cmd/kubelet/app" - "k8s.io/kubernetes/cmd/kubelet/app/options" - "k8s.io/kubernetes/contrib/mesos/pkg/executor" - "k8s.io/kubernetes/contrib/mesos/pkg/executor/config" - "k8s.io/kubernetes/contrib/mesos/pkg/executor/service/podsource" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" - "k8s.io/kubernetes/contrib/mesos/pkg/podutil" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/apis/componentconfig" - "k8s.io/kubernetes/pkg/client/cache" - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - "k8s.io/kubernetes/pkg/fields" - "k8s.io/kubernetes/pkg/kubelet" - "k8s.io/kubernetes/pkg/kubelet/cm" - kconfig "k8s.io/kubernetes/pkg/kubelet/config" - "k8s.io/kubernetes/pkg/kubelet/dockertools" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - "k8s.io/kubernetes/pkg/types" -) - -// TODO(jdef): passing the value of envContainerID to all docker containers instantiated -// through the kubelet is part of a strategy to enable orphan container GC; this can all -// be ripped out once we have a kubelet runtime that leverages Mesos native containerization. - -// envContainerID is the name of the environment variable that contains the -// Mesos-assigned container ID of the Executor. -const envContainerID = "MESOS_EXECUTOR_CONTAINER_UUID" - -type KubeletExecutorServer struct { - *options.KubeletServer - SuicideTimeout time.Duration - LaunchGracePeriod time.Duration - - containerID string -} - -func NewKubeletExecutorServer() *KubeletExecutorServer { - k := &KubeletExecutorServer{ - KubeletServer: options.NewKubeletServer(), - SuicideTimeout: config.DefaultSuicideTimeout, - LaunchGracePeriod: config.DefaultLaunchGracePeriod, - } - if pwd, err := os.Getwd(); err != nil { - log.Warningf("failed to determine current directory: %v", err) - } else { - k.RootDirectory = pwd // mesos sandbox dir - } - k.Address = defaultBindingAddress() - - return k -} - -func (s *KubeletExecutorServer) AddFlags(fs *pflag.FlagSet) { - s.KubeletServer.AddFlags(fs) - fs.DurationVar(&s.SuicideTimeout, "suicide-timeout", s.SuicideTimeout, "Self-terminate after this period of inactivity. Zero disables suicide watch.") - fs.DurationVar(&s.LaunchGracePeriod, "mesos-launch-grace-period", s.LaunchGracePeriod, "Launch grace period after which launching tasks will be cancelled. Zero disables launch cancellation.") -} - -func (s *KubeletExecutorServer) runExecutor( - nodeInfos chan<- executor.NodeInfo, - kubeletFinished <-chan struct{}, - staticPodsConfigPath string, - apiclient *clientset.Clientset, - registry executor.Registry, -) (<-chan struct{}, error) { - staticPodFilters := podutil.Filters{ - // annotate the pod with BindingHostKey so that the scheduler will ignore the pod - // once it appears in the pod registry. the stock kubelet sets the pod host in order - // to accomplish the same; we do this because the k8sm scheduler works differently. - podutil.Annotator(map[string]string{ - meta.BindingHostKey: s.HostnameOverride, - }), - } - if s.containerID != "" { - // tag all pod containers with the containerID so that they can be properly GC'd by Mesos - staticPodFilters = append(staticPodFilters, podutil.Environment([]api.EnvVar{ - {Name: envContainerID, Value: s.containerID}, - })) - } - exec := executor.New(executor.Config{ - Registry: registry, - APIClient: apiclient, - Docker: dockertools.ConnectToDockerOrDie(s.DockerEndpoint, 0), - SuicideTimeout: s.SuicideTimeout, - KubeletFinished: kubeletFinished, - ExitFunc: os.Exit, - NodeInfos: nodeInfos, - Options: []executor.Option{ - executor.StaticPods(staticPodsConfigPath, staticPodFilters), - }, - }) - - // initialize driver and initialize the executor with it - dconfig := bindings.DriverConfig{ - Executor: exec, - HostnameOverride: s.HostnameOverride, - BindingAddress: net.ParseIP(s.Address), - } - driver, err := bindings.NewMesosExecutorDriver(dconfig) - if err != nil { - return nil, fmt.Errorf("failed to create executor driver: %v", err) - } - log.V(2).Infof("Initialize executor driver...") - exec.Init(driver) - - // start the driver - go func() { - if _, err := driver.Run(); err != nil { - log.Fatalf("executor driver failed: %v", err) - } - log.Info("executor Run completed") - }() - - return exec.Done(), nil -} - -func (s *KubeletExecutorServer) runKubelet( - nodeInfos <-chan executor.NodeInfo, - kubeletDone chan<- struct{}, - staticPodsConfigPath string, - apiclient *clientset.Clientset, - podLW *cache.ListWatch, - registry executor.Registry, - executorDone <-chan struct{}, -) (err error) { - defer func() { - if err != nil { - // close the channel here. When Run returns without error, the executorKubelet is - // responsible to do this. If it returns with an error, we are responsible here. - close(kubeletDone) - } - }() - - kubeDeps, err := kubeletapp.UnsecuredKubeletDeps(s.KubeletServer) - if err != nil { - return err - } - - // apply Mesos specific settings - kubeDeps.Builder = func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, standaloneMode bool) (kubelet.KubeletBootstrap, error) { - k, err := kubeletapp.CreateAndInitKubelet(kubeCfg, kubeDeps, standaloneMode) - if err != nil { - return k, err - } - - // decorate kubelet such that it shuts down when the executor is - decorated := &executorKubelet{ - Kubelet: k.(*kubelet.Kubelet), - kubeletDone: kubeletDone, - executorDone: executorDone, - } - - return decorated, nil - } - s.RuntimeCgroups = "" // don't move the docker daemon into a cgroup - kubeDeps.KubeClient = apiclient - - // taken from KubeletServer#Run(*KubeletConfig) - eventClientConfig, err := kubeletapp.CreateAPIServerClientConfig(s.KubeletServer) - if err != nil { - return err - } - - // make a separate client for events - eventClientConfig.QPS = float32(s.EventRecordQPS) - eventClientConfig.Burst = int(s.EventBurst) - kubeDeps.EventClient, err = clientset.NewForConfig(eventClientConfig) - if err != nil { - return err - } - - kubeDeps.PodConfig = kconfig.NewPodConfig(kconfig.PodConfigNotificationIncremental, kubeDeps.Recorder) // override the default pod source - - s.SystemCgroups = "" // don't take control over other system processes. - - if kubeDeps.Cloud != nil { - // fail early and hard because having the cloud provider loaded would go unnoticed, - // but break bigger cluster because accessing the state.json from every slave kills the master. - panic("cloud provider must not be set") - } - - // create custom cAdvisor interface which return the resource values that Mesos reports - ni := <-nodeInfos - cAdvisorInterface, err := NewMesosCadvisor(ni.Cores, ni.Mem, uint(s.CAdvisorPort), s.ContainerRuntime) - if err != nil { - return err - } - - kubeDeps.CAdvisorInterface = cAdvisorInterface - kubeDeps.ContainerManager, err = cm.NewContainerManager(kubeDeps.Mounter, cAdvisorInterface, cm.NodeConfig{ - RuntimeCgroupsName: s.RuntimeCgroups, - SystemCgroupsName: s.SystemCgroups, - KubeletCgroupsName: s.KubeletCgroups, - ContainerRuntime: s.ContainerRuntime, - }) - if err != nil { - return err - } - - go func() { - for ni := range nodeInfos { - // TODO(sttts): implement with MachineAllocable mechanism when https://github.com/kubernetes/kubernetes/issues/13984 is finished - log.V(3).Infof("ignoring updated node resources: %v", ni) - } - }() - - // create main pod source, it will stop generating events once executorDone is closed - var containerOptions []podsource.Option - if s.containerID != "" { - // tag all pod containers with the containerID so that they can be properly GC'd by Mesos - containerOptions = append(containerOptions, podsource.ContainerEnvOverlay([]api.EnvVar{ - {Name: envContainerID, Value: s.containerID}, - })) - kubeDeps.ContainerRuntimeOptions = append(kubeDeps.ContainerRuntimeOptions, - dockertools.PodInfraContainerEnv(map[string]string{ - envContainerID: s.containerID, - })) - } - - podsource.Mesos(executorDone, kubeDeps.PodConfig.Channel(podsource.MesosSource), podLW, registry, containerOptions...) - - // create static-pods directory file source - log.V(2).Infof("initializing static pods source factory, configured at path %q", staticPodsConfigPath) - fileSourceUpdates := kubeDeps.PodConfig.Channel(kubetypes.FileSource) - kconfig.NewSourceFile(staticPodsConfigPath, s.HostnameOverride, s.FileCheckFrequency.Duration, fileSourceUpdates) - - // run the kubelet - // NOTE: because kubeDeps != nil holds, the upstream Run function will not - // initialize the cloud provider. We explicitly wouldn't want - // that because then every kubelet instance would query the master - // state.json which does not scale. - s.KubeletServer.LockFilePath = "" // disable lock file - err = kubeletapp.Run(s.KubeletServer, kubeDeps) - return -} - -// Run runs the specified KubeletExecutorServer. -func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error { - // create shared channels - kubeletFinished := make(chan struct{}) - nodeInfos := make(chan executor.NodeInfo, 1) - - // create static pods directory - staticPodsConfigPath := filepath.Join(s.RootDirectory, "static-pods") - err := os.Mkdir(staticPodsConfigPath, 0750) - if err != nil { - return err - } - - // we're expecting that either Mesos or the minion process will set this for us - s.containerID = os.Getenv(envContainerID) - if s.containerID == "" { - log.Warningf("missing expected environment variable %q", envContainerID) - } - - // create apiserver client - var apiclient *clientset.Clientset - clientConfig, err := kubeletapp.CreateAPIServerClientConfig(s.KubeletServer) - if err == nil { - apiclient, err = clientset.NewForConfig(clientConfig) - } - if err != nil { - // required for k8sm since we need to send api.Binding information back to the apiserver - return fmt.Errorf("cannot create API client: %v", err) - } - - var ( - pw = cache.NewListWatchFromClient(apiclient.CoreClient, "pods", api.NamespaceAll, - fields.OneTermEqualSelector(api.PodHostField, s.HostnameOverride), - ) - reg = executor.NewRegistry(apiclient) - ) - - // start executor - var executorDone <-chan struct{} - executorDone, err = s.runExecutor(nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, reg) - if err != nil { - return err - } - - // start kubelet, blocking - return s.runKubelet(nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, pw, reg, executorDone) -} - -func defaultBindingAddress() string { - libProcessIP := os.Getenv("LIBPROCESS_IP") - if libProcessIP == "" { - return "0.0.0.0" - } else { - return libProcessIP - } -} diff --git a/contrib/mesos/pkg/executor/suicide.go b/contrib/mesos/pkg/executor/suicide.go deleted file mode 100644 index febb6e67e6f..00000000000 --- a/contrib/mesos/pkg/executor/suicide.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import ( - "time" - - log "github.com/golang/glog" - bindings "github.com/mesos/mesos-go/executor" -) - -// func that attempts suicide -type jumper func(bindings.ExecutorDriver, <-chan struct{}) - -type suicideWatcher interface { - Next(time.Duration, bindings.ExecutorDriver, jumper) suicideWatcher - Reset(time.Duration) bool - Stop() bool -} - -// TODO(jdef) add metrics for this? -type suicideTimer struct { - timer *time.Timer -} - -func (w *suicideTimer) Next(d time.Duration, driver bindings.ExecutorDriver, f jumper) suicideWatcher { - return &suicideTimer{ - timer: time.AfterFunc(d, func() { - log.Warningf("Suicide timeout (%v) expired", d) - f(driver, nil) - }), - } -} - -func (w *suicideTimer) Stop() (result bool) { - if w != nil && w.timer != nil { - log.Infoln("stopping suicide watch") //TODO(jdef) debug - result = w.timer.Stop() - } - return -} - -// return true if the timer was successfully reset -func (w *suicideTimer) Reset(d time.Duration) bool { - if w != nil && w.timer != nil { - log.Infoln("resetting suicide watch") //TODO(jdef) debug - w.timer.Reset(d) - return true - } - return false -} diff --git a/contrib/mesos/pkg/executor/suicide_test.go b/contrib/mesos/pkg/executor/suicide_test.go deleted file mode 100644 index f080a562d08..00000000000 --- a/contrib/mesos/pkg/executor/suicide_test.go +++ /dev/null @@ -1,197 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import ( - "sync/atomic" - "testing" - "time" - - "github.com/golang/glog" - bindings "github.com/mesos/mesos-go/executor" - "k8s.io/kubernetes/pkg/api" -) - -type suicideTracker struct { - suicideWatcher - stops uint32 - resets uint32 - timers uint32 - jumps *uint32 -} - -func (t *suicideTracker) Reset(d time.Duration) bool { - defer func() { t.resets++ }() - return t.suicideWatcher.Reset(d) -} - -func (t *suicideTracker) Stop() bool { - defer func() { t.stops++ }() - return t.suicideWatcher.Stop() -} - -func (t *suicideTracker) Next(d time.Duration, driver bindings.ExecutorDriver, f jumper) suicideWatcher { - tracker := &suicideTracker{ - stops: t.stops, - resets: t.resets, - jumps: t.jumps, - timers: t.timers + 1, - } - jumper := tracker.makeJumper(f) - tracker.suicideWatcher = t.suicideWatcher.Next(d, driver, jumper) - return tracker -} - -func (t *suicideTracker) makeJumper(_ jumper) jumper { - return jumper(func(driver bindings.ExecutorDriver, cancel <-chan struct{}) { - glog.Warningln("Jumping?!") - if t.jumps != nil { - atomic.AddUint32(t.jumps, 1) - } - }) -} - -func TestSuicide_zeroTimeout(t *testing.T) { - defer glog.Flush() - - k := NewTestKubernetesExecutor() - tracker := &suicideTracker{suicideWatcher: k.suicideWatch} - k.suicideWatch = tracker - - ch := k.resetSuicideWatch(nil) - - select { - case <-ch: - case <-time.After(2 * time.Second): - t.Fatalf("timeout waiting for reset of suicide watch") - } - if tracker.stops != 0 { - t.Fatalf("expected no stops since suicideWatchTimeout was never set") - } - if tracker.resets != 0 { - t.Fatalf("expected no resets since suicideWatchTimeout was never set") - } - if tracker.timers != 0 { - t.Fatalf("expected no timers since suicideWatchTimeout was never set") - } -} - -func TestSuicide_WithTasks(t *testing.T) { - defer glog.Flush() - - k := NewTestKubernetesExecutor() - k.suicideTimeout = 50 * time.Millisecond - - jumps := uint32(0) - tracker := &suicideTracker{suicideWatcher: k.suicideWatch, jumps: &jumps} - k.suicideWatch = tracker - - k.registry.bind("foo", &api.Pod{}) // prevent suicide attempts from succeeding - - // call reset with a nil timer - glog.Infoln("Resetting suicide watch with 1 task") - select { - case <-k.resetSuicideWatch(nil): - tracker = k.suicideWatch.(*suicideTracker) - if tracker.stops != 1 { - t.Fatalf("expected suicide attempt to Stop() since there are registered tasks") - } - if tracker.resets != 0 { - t.Fatalf("expected no resets since") - } - if tracker.timers != 0 { - t.Fatalf("expected no timers since") - } - case <-time.After(1 * time.Second): - t.Fatalf("initial suicide watch setup failed") - } - - k.registry.Remove("foo") // zero remaining tasks - k.suicideTimeout = 1500 * time.Millisecond - suicideStart := time.Now() - - // reset the suicide watch, which should actually start a timer now - glog.Infoln("Resetting suicide watch with 0 tasks") - select { - case <-k.resetSuicideWatch(nil): - tracker = k.suicideWatch.(*suicideTracker) - if tracker.stops != 1 { - t.Fatalf("did not expect suicide attempt to Stop() since there are no registered tasks") - } - if tracker.resets != 1 { - t.Fatalf("expected 1 resets instead of %d", tracker.resets) - } - if tracker.timers != 1 { - t.Fatalf("expected 1 timers instead of %d", tracker.timers) - } - case <-time.After(1 * time.Second): - t.Fatalf("2nd suicide watch setup failed") - } - - k.lock.Lock() - k.registry.bind("foo", &api.Pod{}) // prevent suicide attempts from succeeding - k.lock.Unlock() - - // reset the suicide watch, which should stop the existing timer - glog.Infoln("Resetting suicide watch with 1 task") - select { - case <-k.resetSuicideWatch(nil): - tracker = k.suicideWatch.(*suicideTracker) - if tracker.stops != 2 { - t.Fatalf("expected 2 stops instead of %d since there are registered tasks", tracker.stops) - } - if tracker.resets != 1 { - t.Fatalf("expected 1 resets instead of %d", tracker.resets) - } - if tracker.timers != 1 { - t.Fatalf("expected 1 timers instead of %d", tracker.timers) - } - case <-time.After(1 * time.Second): - t.Fatalf("3rd suicide watch setup failed") - } - - k.lock.Lock() - k.registry.Remove("foo") // allow suicide attempts to schedule - k.lock.Unlock() - - // reset the suicide watch, which should reset a stopped timer - glog.Infoln("Resetting suicide watch with 0 tasks") - select { - case <-k.resetSuicideWatch(nil): - tracker = k.suicideWatch.(*suicideTracker) - if tracker.stops != 2 { - t.Fatalf("expected 2 stops instead of %d since there are no registered tasks", tracker.stops) - } - if tracker.resets != 2 { - t.Fatalf("expected 2 resets instead of %d", tracker.resets) - } - if tracker.timers != 1 { - t.Fatalf("expected 1 timers instead of %d", tracker.timers) - } - case <-time.After(1 * time.Second): - t.Fatalf("4th suicide watch setup failed") - } - - sinceWatch := time.Since(suicideStart) - time.Sleep(3*time.Second - sinceWatch) // give the first timer to misfire (it shouldn't since Stop() was called) - - if j := atomic.LoadUint32(&jumps); j != 1 { - t.Fatalf("expected 1 jumps instead of %d since stop was called", j) - } else { - glog.Infoln("Jumps verified") // glog so we get a timestamp - } -} diff --git a/contrib/mesos/pkg/executor/watcher.go b/contrib/mesos/pkg/executor/watcher.go deleted file mode 100644 index 2116c4a53ae..00000000000 --- a/contrib/mesos/pkg/executor/watcher.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executor - -import ( - "sync" - "time" - - log "github.com/golang/glog" -) - -type ( - // filter registration events, return false to abort further processing of the event - watchFilter func(pod *PodEvent) (accept bool) - - watchExpiration struct { - // timeout closes when the handler has expired; it delivers at most one Time. - timeout <-chan time.Time - - // onEvent is an optional callback that is invoked if/when the expired chan - // closes - onEvent func(taskID string) - } - - watchHandler struct { - // prevent callbacks from being invoked simultaneously - sync.Mutex - - // handle registration events, return true to indicate the handler should be - // de-registered upon completion. If pod is nil then the associated handler - // has expired. - onEvent func(pod *PodEvent) (done bool, err error) - - // expiration is an optional configuration that indicates when a handler should - // be considered to have expired, and what action to take upon such - expiration watchExpiration - } - - // watcher observes PodEvent events and conditionally executes handlers that - // have been associated with the taskID of the PodEvent. - watcher struct { - updates <-chan *PodEvent - rw sync.RWMutex - handlers map[string]*watchHandler - filters []watchFilter - runOnce chan struct{} - } -) - -func newWatcher(updates <-chan *PodEvent) *watcher { - return &watcher{ - updates: updates, - handlers: make(map[string]*watchHandler), - runOnce: make(chan struct{}), - } -} - -func (pw *watcher) run() { - select { - case <-pw.runOnce: - log.Error("run() has already been invoked for this pod-watcher") - return - default: - close(pw.runOnce) - } -updateLoop: - for u := range pw.updates { - log.V(2).Info("filtering " + u.FormatShort()) - for _, f := range pw.filters { - if !f(u) { - continue updateLoop - } - } - log.V(1).Info("handling " + u.FormatShort()) - h, ok := func() (h *watchHandler, ok bool) { - pw.rw.RLock() - defer pw.rw.RUnlock() - h, ok = pw.handlers[u.taskID] - return - }() - if ok { - log.V(1).Info("executing action for " + u.FormatShort()) - done, err := func() (bool, error) { - h.Lock() - defer h.Unlock() - return h.onEvent(u) - }() - if err != nil { - log.Error(err) - } - if done { - // de-register handler upon successful completion of action - log.V(1).Info("de-registering handler for " + u.FormatShort()) - func() { - pw.rw.Lock() - delete(pw.handlers, u.taskID) - pw.rw.Unlock() - }() - } - } - } -} - -func (pw *watcher) addFilter(f watchFilter) { - select { - case <-pw.runOnce: - log.Errorf("failed to add filter because pod-watcher is already running") - default: - pw.filters = append(pw.filters, f) - } -} - -// forTask associates a handler `h` with the given taskID. -func (pw *watcher) forTask(taskID string, h *watchHandler) { - pw.rw.Lock() - pw.handlers[taskID] = h - pw.rw.Unlock() - - if exp := h.expiration; exp.timeout != nil { - go func() { - <-exp.timeout - log.V(1).Infof("expiring handler for task %v", taskID) - - // de-register handler upon expiration - pw.rw.Lock() - delete(pw.handlers, taskID) - pw.rw.Unlock() - - if exp.onEvent != nil { - h.Lock() - defer h.Unlock() - exp.onEvent(taskID) - } - }() - } -} diff --git a/contrib/mesos/pkg/flagutil/cadvisor.go b/contrib/mesos/pkg/flagutil/cadvisor.go deleted file mode 100644 index 423e4e401d3..00000000000 --- a/contrib/mesos/pkg/flagutil/cadvisor.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package flagutil - -import ( - "flag" - - // kubelet attempts to customize default values for some cadvisor flags, so - // make sure that we pick these up. - _ "k8s.io/kubernetes/pkg/kubelet/cadvisor" -) - -// FlagFunc retrieves a specific flag instance; returns nil if the flag is not configured. -type FlagFunc func() *flag.Flag - -// NameValue returns the name and value of a flag, if it exists, otherwise empty strings. -func (ff FlagFunc) NameValue() (name, value string) { - if f := ff(); f != nil { - name, value = f.Name, f.Value.String() - } - return -} - -func flagFunc(name string) FlagFunc { return func() *flag.Flag { return flag.Lookup(name) } } - -// Cadvisor fields return the configured values of cadvisor global flags -var Cadvisor = struct { - HousekeepingInterval FlagFunc - GlobalHousekeepingInterval FlagFunc -}{ - flagFunc("housekeeping_interval"), - flagFunc("global_housekeeping_interval"), -} diff --git a/contrib/mesos/pkg/flagutil/cadvisor_linux.go b/contrib/mesos/pkg/flagutil/cadvisor_linux.go deleted file mode 100644 index bf9d8ebe300..00000000000 --- a/contrib/mesos/pkg/flagutil/cadvisor_linux.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package flagutil - -import ( - // TODO(jdef) kill this once cadvisor flags are no longer configured by - // global variables. Importing it this way guarantees that the global flag - // variables are initialized. - _ "github.com/google/cadvisor/manager" -) diff --git a/contrib/mesos/pkg/hyperkube/doc.go b/contrib/mesos/pkg/hyperkube/doc.go deleted file mode 100644 index 307b4387b4f..00000000000 --- a/contrib/mesos/pkg/hyperkube/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package hyperkube facilitates the combination of multiple -// kubernetes-mesos components into a single binary form, providing a -// simple mechanism for intra-component discovery as per the original -// Kubernetes hyperkube package. -package hyperkube // import "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" diff --git a/contrib/mesos/pkg/hyperkube/hyperkube.go b/contrib/mesos/pkg/hyperkube/hyperkube.go deleted file mode 100644 index 72791525ad3..00000000000 --- a/contrib/mesos/pkg/hyperkube/hyperkube.go +++ /dev/null @@ -1,26 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hyperkube - -const ( - CommandApiserver = "apiserver" - CommandControllerManager = "controller-manager" - CommandExecutor = "executor" - CommandMinion = "minion" - CommandProxy = "proxy" - CommandScheduler = "scheduler" -) diff --git a/contrib/mesos/pkg/hyperkube/types.go b/contrib/mesos/pkg/hyperkube/types.go deleted file mode 100644 index 637f18b834e..00000000000 --- a/contrib/mesos/pkg/hyperkube/types.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hyperkube - -import ( - "github.com/spf13/pflag" -) - -var ( - nilKube = &nilKubeType{} -) - -type Interface interface { - // FindServer will find a specific server named name. - FindServer(name string) bool - - // The executable name, used for help and soft-link invocation - Name() string - - // Flags returns a flagset for "global" flags. - Flags() *pflag.FlagSet -} - -type nilKubeType struct{} - -func (n *nilKubeType) FindServer(_ string) bool { - return false -} - -func (n *nilKubeType) Name() string { - return "" -} - -func (n *nilKubeType) Flags() *pflag.FlagSet { - return nil -} - -func Nil() Interface { - return nilKube -} diff --git a/contrib/mesos/pkg/minion/config/config.go b/contrib/mesos/pkg/minion/config/config.go deleted file mode 100644 index a4e99ae4c6d..00000000000 --- a/contrib/mesos/pkg/minion/config/config.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "k8s.io/kubernetes/pkg/api/resource" -) - -const ( - DefaultLogMaxBackups = 5 // how many backup to keep - DefaultLogMaxAgeInDays = 7 // after how many days to rotate at most - - DefaultCgroupPrefix = "mesos" -) - -// DefaultLogMaxSize returns the maximal log file size before rotation -func DefaultLogMaxSize() resource.Quantity { - return *resource.NewQuantity(10*1024*1024, resource.BinarySI) -} diff --git a/contrib/mesos/pkg/minion/config/doc.go b/contrib/mesos/pkg/minion/config/doc.go deleted file mode 100644 index d16decd6835..00000000000 --- a/contrib/mesos/pkg/minion/config/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package config contains minion configuration constants. -package config // import "k8s.io/kubernetes/contrib/mesos/pkg/minion/config" diff --git a/contrib/mesos/pkg/minion/doc.go b/contrib/mesos/pkg/minion/doc.go deleted file mode 100644 index 162694eab29..00000000000 --- a/contrib/mesos/pkg/minion/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package minion contains the executor and proxy bootstrap code for a Mesos slave -package minion // import "k8s.io/kubernetes/contrib/mesos/pkg/minion" diff --git a/contrib/mesos/pkg/minion/mountns_darwin.go b/contrib/mesos/pkg/minion/mountns_darwin.go deleted file mode 100644 index 27ec29c4fc8..00000000000 --- a/contrib/mesos/pkg/minion/mountns_darwin.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package minion - -import ( - log "github.com/golang/glog" -) - -func enterPrivateMountNamespace() { - log.Info("Skipping mount namespace, only available on Linux") -} diff --git a/contrib/mesos/pkg/minion/mountns_linux.go b/contrib/mesos/pkg/minion/mountns_linux.go deleted file mode 100644 index 0f1804c3776..00000000000 --- a/contrib/mesos/pkg/minion/mountns_linux.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package minion - -import ( - "syscall" - - log "github.com/golang/glog" -) - -// enterPrivateMountNamespace does just that: the current mount ns is unshared (isolated) -// and then made a slave to the root mount / of the parent mount ns (mount events from / -// or its children that happen in the parent NS propagate to us). -// -// this is not yet compatible with volume plugins as implemented by the kubelet, which -// depends on using host-volume args to 'docker run' to attach plugin volumes to CT's -// at runtime. as such, docker needs to be able to see the volumes mounted by k8s plugins, -// which is impossible if k8s volume plugins are running in an isolated mount ns. -// -// an alternative approach would be to always run the kubelet in the host's mount-ns and -// rely upon mesos to forcibly umount bindings in the task sandbox before rmdir'ing it: -// https://issues.apache.org/jira/browse/MESOS-349. -// -// use at your own risk. -func enterPrivateMountNamespace() { - log.Warningln("EXPERIMENTAL FEATURE: entering private mount ns") - - // enter a new mount NS, useful for isolating changes to the mount table - // that are made by the kubelet for storage volumes. - err := syscall.Unshare(syscall.CLONE_NEWNS) - if err != nil { - log.Fatalf("failed to enter private mount NS: %v", err) - } - - // make the rootfs / rslave to the parent mount NS so that we - // pick up on any changes made there - err = syscall.Mount("", "/", "dontcare", syscall.MS_REC|syscall.MS_SLAVE, "") - if err != nil { - log.Fatalf("failed to mark / rslave: %v", err) - } -} diff --git a/contrib/mesos/pkg/minion/server.go b/contrib/mesos/pkg/minion/server.go deleted file mode 100644 index 591e8463fe3..00000000000 --- a/contrib/mesos/pkg/minion/server.go +++ /dev/null @@ -1,381 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package minion - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "os/signal" - "path" - "strconv" - "strings" - "syscall" - - log "github.com/golang/glog" - "github.com/kardianos/osext" - "github.com/spf13/pflag" - "gopkg.in/natefinch/lumberjack.v2" - kubeletapp "k8s.io/kubernetes/cmd/kubelet/app" - exservice "k8s.io/kubernetes/contrib/mesos/pkg/executor/service" - "k8s.io/kubernetes/contrib/mesos/pkg/flagutil" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" - "k8s.io/kubernetes/contrib/mesos/pkg/minion/config" - "k8s.io/kubernetes/contrib/mesos/pkg/minion/tasks" - "k8s.io/kubernetes/pkg/api/resource" - "k8s.io/kubernetes/pkg/client/restclient" -) - -const ( - proxyLogFilename = "proxy.log" - executorLogFilename = "executor.log" -) - -type MinionServer struct { - // embed the executor server to be able to use its flags - // TODO(sttts): get rid of this mixing of the minion and the executor server with a multiflags implementation for km - KubeletExecutorServer *exservice.KubeletExecutorServer - - privateMountNS bool - hks hyperkube.Interface - clientConfig *restclient.Config - kmBinary string - tasks []*tasks.Task - - pathOverride string // the PATH environment for the sub-processes - cgroupPrefix string // e.g. mesos - cgroupRoot string // the cgroupRoot that we pass to the kubelet-executor, depends on containPodResources - mesosCgroup string // discovered mesos cgroup root, e.g. /mesos/{container-id} - containPodResources bool - - logMaxSize resource.Quantity - logMaxBackups int - logMaxAgeInDays int - logVerbosity int32 // see glog.Level - - runProxy bool - proxyKubeconfig string - proxyLogV int - proxyBindall bool - proxyMode string - conntrackMax int - conntrackTCPTimeoutEstablished int -} - -// NewMinionServer creates the MinionServer struct with default values to be used by hyperkube -func NewMinionServer() *MinionServer { - s := &MinionServer{ - KubeletExecutorServer: exservice.NewKubeletExecutorServer(), - privateMountNS: false, // disabled until Docker supports customization of the parent mount namespace - cgroupPrefix: config.DefaultCgroupPrefix, - containPodResources: true, - logMaxSize: config.DefaultLogMaxSize(), - logMaxBackups: config.DefaultLogMaxBackups, - logMaxAgeInDays: config.DefaultLogMaxAgeInDays, - runProxy: true, - proxyMode: "userspace", // upstream default is "iptables" post-v1.1 - } - - // cache this for later use - binary, err := osext.Executable() - if err != nil { - log.Fatalf("failed to determine currently running executable: %v", err) - } - s.kmBinary = binary - - return s -} - -// filterArgsByFlagSet returns a list of args which are parsed by the given flag set -// and another list with those which do not match -func filterArgsByFlagSet(args []string, flags *pflag.FlagSet) ([]string, []string) { - matched := []string{} - notMatched := []string{} - for _, arg := range args { - err := flags.Parse([]string{arg}) - if err != nil { - notMatched = append(notMatched, arg) - } else { - matched = append(matched, arg) - } - } - return matched, notMatched -} - -func findMesosCgroup(prefix string) (cgroupPath string, containerID string) { - // derive our cgroup from MESOS_DIRECTORY environment - mesosDir := os.Getenv("MESOS_DIRECTORY") - if mesosDir == "" { - log.V(2).Infof("cannot derive executor's cgroup because MESOS_DIRECTORY is empty") - return - } - - containerID = path.Base(mesosDir) - if containerID == "" { - log.V(2).Infof("cannot derive executor's cgroup from MESOS_DIRECTORY=%q", mesosDir) - return - } - - cgroupPath = path.Join("/", prefix, containerID) - return -} - -func (ms *MinionServer) launchProxyServer() { - bindAddress := "0.0.0.0" - if !ms.proxyBindall { - bindAddress = ms.KubeletExecutorServer.Address - } - args := []string{ - fmt.Sprintf("--bind-address=%s", bindAddress), - fmt.Sprintf("--v=%d", ms.proxyLogV), - "--logtostderr=true", - // TODO(jdef) resource-container is going away completely at some point, but - // we need to override it here to disable the current default behavior - "--resource-container=", // disable this; mesos slave doesn't like sub-containers yet - "--proxy-mode=" + ms.proxyMode, - "--conntrack-max=" + strconv.Itoa(ms.conntrackMax), - "--conntrack-tcp-timeout-established=" + strconv.Itoa(ms.conntrackTCPTimeoutEstablished), - } - if ms.proxyKubeconfig != "" { - args = append(args, fmt.Sprintf("--kubeconfig=%s", ms.proxyKubeconfig)) - } - if ms.clientConfig.Host != "" { - args = append(args, fmt.Sprintf("--master=%s", ms.clientConfig.Host)) - } - if ms.KubeletExecutorServer.HostnameOverride != "" { - args = append(args, fmt.Sprintf("--hostname-override=%s", ms.KubeletExecutorServer.HostnameOverride)) - } - - ms.launchHyperkubeServer(hyperkube.CommandProxy, args, proxyLogFilename) -} - -// launchExecutorServer returns a chan that closes upon kubelet-executor death. since the kubelet- -// executor doesn't support failover right now, the right thing to do is to fail completely since all -// pods will be lost upon restart and we want mesos to recover the resources from them. -func (ms *MinionServer) launchExecutorServer(containerID string) <-chan struct{} { - allArgs := os.Args[2:] - - // filter out minion flags, leaving those for the executor - executorFlags := pflag.NewFlagSet("executor", pflag.ContinueOnError) - executorFlags.SetOutput(ioutil.Discard) - ms.AddExecutorFlags(executorFlags) - executorArgs, _ := filterArgsByFlagSet(allArgs, executorFlags) - - // disable resource-container; mesos slave doesn't like sub-containers yet - executorArgs = append(executorArgs, "--kubelet-cgroups=") - - appendOptional := func(name, value string) { - if value != "" { - executorArgs = append(executorArgs, "--"+name+"="+value) - } - } - appendOptional("cgroup-root", ms.cgroupRoot) - - // forward global cadvisor flag values to the executor - // TODO(jdef) remove this code once cadvisor global flags have been cleaned up - appendOptional(flagutil.Cadvisor.HousekeepingInterval.NameValue()) - appendOptional(flagutil.Cadvisor.GlobalHousekeepingInterval.NameValue()) - - // forward containerID so that the executor may pass it along to containers that it launches - var ctidOpt tasks.Option - ctidOpt = func(t *tasks.Task) tasks.Option { - oldenv := t.Env[:] - t.Env = append(t.Env, "MESOS_EXECUTOR_CONTAINER_UUID="+containerID) - return func(t2 *tasks.Task) tasks.Option { - t2.Env = oldenv - return ctidOpt - } - } - - // run executor and quit minion server when this exits cleanly - execDied := make(chan struct{}) - ms.launchHyperkubeServer(hyperkube.CommandExecutor, executorArgs, executorLogFilename, tasks.NoRespawn(execDied), ctidOpt) - return execDied -} - -func (ms *MinionServer) launchHyperkubeServer(server string, args []string, logFileName string, options ...tasks.Option) { - log.V(2).Infof("Spawning hyperkube %v with args '%+v'", server, args) - - kmArgs := append([]string{server}, args...) - maxSize := ms.logMaxSize.Value() - if maxSize > 0 { - // convert to MB - maxSize = maxSize / 1024 / 1024 - if maxSize == 0 { - log.Warning("maximal log file size is rounded to 1 MB") - maxSize = 1 - } - } - - writerFunc := func() io.WriteCloser { - return &lumberjack.Logger{ - Filename: logFileName, - MaxSize: int(maxSize), - MaxBackups: ms.logMaxBackups, - MaxAge: ms.logMaxAgeInDays, - } - } - - // prepend env, allow later options to customize further - options = append([]tasks.Option{tasks.Environment(os.Environ()), ms.applyPathOverride()}, options...) - - t := tasks.New(server, ms.kmBinary, kmArgs, writerFunc, options...) - go t.Start() - ms.tasks = append(ms.tasks, t) -} - -// applyPathOverride overrides PATH and also adds $SANDBOX/bin (needed for locating bundled binary deps -// as well as external deps like iptables) -func (ms *MinionServer) applyPathOverride() tasks.Option { - return func(t *tasks.Task) tasks.Option { - kmEnv := make([]string, 0, len(t.Env)) - for _, e := range t.Env { - if !strings.HasPrefix(e, "PATH=") { - kmEnv = append(kmEnv, e) - } else { - if ms.pathOverride != "" { - e = "PATH=" + ms.pathOverride - } - pwd, err := os.Getwd() - if err != nil { - panic(fmt.Errorf("Cannot get current directory: %v", err)) - } - kmEnv = append(kmEnv, fmt.Sprintf("%s:%s", e, path.Join(pwd, "bin"))) - } - } - oldenv := t.Env - t.Env = kmEnv - return tasks.Environment(oldenv) - } -} - -// runs the main kubelet loop, closing the kubeletFinished chan when the loop exits. -// never returns. -func (ms *MinionServer) Run(hks hyperkube.Interface, _ []string) error { - if ms.privateMountNS { - // only the Linux version will do anything - enterPrivateMountNamespace() - } - - // create apiserver client - clientConfig, err := kubeletapp.CreateAPIServerClientConfig(ms.KubeletExecutorServer.KubeletServer) - if err != nil { - // required for k8sm since we need to send api.Binding information - // back to the apiserver - log.Fatalf("No API client: %v", err) - } - ms.clientConfig = clientConfig - - // derive the executor cgroup and use it as: - // - pod container cgroup root (e.g. docker cgroup-parent, optionally; see comments below) - // - parent of kubelet container - // - parent of kube-proxy container - containerID := "" - ms.mesosCgroup, containerID = findMesosCgroup(ms.cgroupPrefix) - log.Infof("discovered mesos cgroup at %q", ms.mesosCgroup) - - // hack alert, this helps to work around systemd+docker+mesos integration problems - // when docker's cgroup-parent flag is used (!containPodResources = don't use the docker flag) - if ms.containPodResources { - ms.cgroupRoot = ms.mesosCgroup - } - - cgroupLogger := log.Infof - if ms.cgroupRoot == "" { - cgroupLogger = log.Warningf - } - - cgroupLogger("using cgroup-root %q", ms.cgroupRoot) - - // run subprocesses until ms.done is closed on return of this function - if ms.runProxy { - ms.launchProxyServer() - } - - // abort closes when the kubelet-executor dies - abort := ms.launchExecutorServer(containerID) - shouldQuit := termSignalListener(abort) - te := tasks.MergeOutput(ms.tasks, shouldQuit) - - // TODO(jdef) do something fun here, such as reporting task completion to the apiserver - - <-te.Close().Done() // we don't listen for any specific events yet; wait for all tasks to finish - return nil -} - -// termSignalListener returns a signal chan that closes when either (a) the process receives a termination -// signal: SIGTERM, SIGINT, or SIGHUP; or (b) the abort chan closes. -func termSignalListener(abort <-chan struct{}) <-chan struct{} { - shouldQuit := make(chan struct{}) - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh) - - go func() { - defer close(shouldQuit) - for { - select { - case <-abort: - log.Infof("executor died, aborting") - return - case s, ok := <-sigCh: - if !ok { - return - } - switch s { - case os.Interrupt, os.Signal(syscall.SIGTERM), os.Signal(syscall.SIGINT), os.Signal(syscall.SIGHUP): - log.Infof("received signal %q, aborting", s) - return - case os.Signal(syscall.SIGCHLD): // who cares? - default: - log.Errorf("unexpected signal: %T %#v", s, s) - } - - } - } - }() - return shouldQuit -} - -func (ms *MinionServer) AddExecutorFlags(fs *pflag.FlagSet) { - ms.KubeletExecutorServer.AddFlags(fs) - - // hack to forward log verbosity flag to the executor - fs.Int32Var(&ms.logVerbosity, "v", ms.logVerbosity, "log level for V logs") -} - -func (ms *MinionServer) AddMinionFlags(fs *pflag.FlagSet) { - // general minion flags - fs.StringVar(&ms.cgroupPrefix, "mesos-cgroup-prefix", ms.cgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos") - fs.BoolVar(&ms.privateMountNS, "private-mountns", ms.privateMountNS, "Enter a private mount NS before spawning procs (linux only). Experimental, not yet compatible with k8s volumes.") - fs.StringVar(&ms.pathOverride, "path-override", ms.pathOverride, "Override the PATH in the environment of the sub-processes.") - fs.BoolVar(&ms.containPodResources, "contain-pod-resources", ms.containPodResources, "Allocate pod CPU and memory resources from offers and reparent pod containers into mesos cgroups; disable if you're having strange mesos/docker/systemd interactions.") - - // log file flags - fs.Var(resource.NewQuantityFlagValue(&ms.logMaxSize), "max-log-size", "Maximum log file size for the executor and proxy before rotation") - fs.IntVar(&ms.logMaxAgeInDays, "max-log-age", ms.logMaxAgeInDays, "Maximum log file age of the executor and proxy in days") - fs.IntVar(&ms.logMaxBackups, "max-log-backups", ms.logMaxBackups, "Maximum log file backups of the executor and proxy to keep after rotation") - - // proxy flags - fs.BoolVar(&ms.runProxy, "run-proxy", ms.runProxy, "Maintain a running kube-proxy instance as a child proc of this kubelet-executor.") - fs.StringVar(&ms.proxyKubeconfig, "proxy-kubeconfig", ms.proxyKubeconfig, "Path to kubeconfig file used by the child kube-proxy.") - fs.IntVar(&ms.proxyLogV, "proxy-logv", ms.proxyLogV, "Log verbosity of the child kube-proxy.") - fs.BoolVar(&ms.proxyBindall, "proxy-bindall", ms.proxyBindall, "When true will cause kube-proxy to bind to 0.0.0.0.") - fs.StringVar(&ms.proxyMode, "proxy-mode", ms.proxyMode, "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.") - fs.IntVar(&ms.conntrackMax, "conntrack-max", ms.conntrackMax, "Maximum number of NAT connections to track on agent nodes (0 to leave as-is)") - fs.IntVar(&ms.conntrackTCPTimeoutEstablished, "conntrack-tcp-timeout-established", ms.conntrackTCPTimeoutEstablished, "Idle timeout for established TCP connections on agent nodes (0 to leave as-is)") -} diff --git a/contrib/mesos/pkg/minion/tasks/doc.go b/contrib/mesos/pkg/minion/tasks/doc.go deleted file mode 100644 index 6577105a05c..00000000000 --- a/contrib/mesos/pkg/minion/tasks/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package tasks provides an API for supervising system processes as Task's. -// It provides stronger guarantees with respect to process lifecycle than a -// standalone kubelet running static pods. -package tasks // import "k8s.io/kubernetes/contrib/mesos/pkg/minion/tasks" diff --git a/contrib/mesos/pkg/minion/tasks/events.go b/contrib/mesos/pkg/minion/tasks/events.go deleted file mode 100644 index 3aa6095f11e..00000000000 --- a/contrib/mesos/pkg/minion/tasks/events.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -type Events interface { - // Close stops delivery of events in the completion and errors channels; callers must close this when they intend to no longer read from completion() or errors() - Close() Events - - // Completion reports Completion events as they happen - Completion() <-chan *Completion - - // Done returns a signal chan that closes when all tasks have completed and there are no more events to deliver - Done() <-chan struct{} -} - -type eventsImpl struct { - tc chan *Completion - stopForwarding chan struct{} - done <-chan struct{} -} - -func newEventsImpl(tcin <-chan *Completion, done <-chan struct{}) *eventsImpl { - ei := &eventsImpl{ - tc: make(chan *Completion), - stopForwarding: make(chan struct{}), - done: done, - } - go func() { - defer close(ei.tc) - forwardCompletionUntil(tcin, ei.tc, ei.stopForwarding, done, nil) - }() - return ei -} - -func (e *eventsImpl) Close() Events { close(e.stopForwarding); return e } -func (e *eventsImpl) Completion() <-chan *Completion { return e.tc } -func (e *eventsImpl) Done() <-chan struct{} { return e.done } - -// forwardCompletionUntil is a generic pipe that forwards objects between channels. -// if discard is closed, objects are silently dropped. -// if tap != nil then it's invoked for each object as it's read from tin, but before it's written to tch. -// returns when either reading from tin completes (no more objects, and is closed), or else -// abort is closed, which ever happens first. -func forwardCompletionUntil(tin <-chan *Completion, tch chan<- *Completion, discard <-chan struct{}, abort <-chan struct{}, tap func(*Completion, bool)) { - var tc *Completion - var ok bool -forwardLoop: - for { - select { - case tc, ok = <-tin: - if !ok { - return - } - if tap != nil { - tap(tc, false) - } - select { - case <-abort: - break forwardLoop - case <-discard: - case tch <- tc: - } - case <-abort: - // best effort - select { - case tc, ok = <-tin: - if ok { - if tap != nil { - tap(tc, true) - } - break forwardLoop - } - default: - } - return - } - } - // best effort - select { - case tch <- tc: - case <-discard: - default: - } -} diff --git a/contrib/mesos/pkg/minion/tasks/task.go b/contrib/mesos/pkg/minion/tasks/task.go deleted file mode 100644 index 7a4bcddc8d1..00000000000 --- a/contrib/mesos/pkg/minion/tasks/task.go +++ /dev/null @@ -1,431 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "fmt" - "io" - "io/ioutil" - "os/exec" - "sync" - "sync/atomic" - "syscall" - "time" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" -) - -const ( - defaultTaskRestartDelay = 5 * time.Second - - // TODO(jdef) there's no easy way for us to discover the grace period that we actually - // have, from mesos: it's simply a missing core feature. there's a MESOS-xyz ticket for - // this somewhere. if it was discoverable then we could come up with a better strategy. - // there are some comments in the executor regarding this as well (because there we're - // concerned about cleaning up pods within the grace period). we could pick a some - // higher (arbitrary) value but without knowing when the slave will forcibly kill us - // it seems a somewhat futile exercise. - defaultKillGracePeriod = 5 * time.Second -) - -// Completion represents the termination of a Task process. Each process execution should -// yield (barring drops because of an abort signal) exactly one Completion. -type Completion struct { - name string // name of the task - code int // exit code that the task process completed with - err error // process management errors are reported here -} - -// systemProcess is a useful abstraction for testing -type systemProcess interface { - // Wait works like exec.Cmd.Wait() - Wait() error - - // Kill returns the pid of the process that was killed - Kill(force bool) (int, error) -} - -type cmdProcess struct { - delegate *exec.Cmd -} - -func (cp *cmdProcess) Wait() error { - return cp.delegate.Wait() -} - -func (cp *cmdProcess) Kill(force bool) (int, error) { - // kill the entire process group, not just the one process - pid := cp.delegate.Process.Pid - processGroup := 0 - pid - - // we send a SIGTERM here for a graceful stop. users of this package should - // wait for tasks to complete normally. as a fallback/safeguard, child procs - // are spawned in notStartedTask to receive a SIGKILL when this process dies. - sig := syscall.SIGTERM - if force { - sig = syscall.SIGKILL - } - rc := syscall.Kill(processGroup, sig) - return pid, rc -} - -// task is a specification for running a system process; it provides hooks for customizing -// logging and restart handling as well as provides event channels for communicating process -// termination and errors related to process management. -type Task struct { - Env []string // optional: process environment override - Finished func(restarting bool) bool // callback invoked when a task process has completed; if `restarting` then it will be restarted if it returns true - RestartDelay time.Duration // interval between repeated task restarts - - name string // required: unique name for this task - bin string // required: path to executable - args []string // optional: process arguments - createLogger func() io.WriteCloser // factory func that builds a log writer - cmd systemProcess // process that we started - completedCh chan *Completion // reports exit codes encountered when task processes exit, or errors during process management - shouldQuit chan struct{} // shouldQuit is closed to indicate that the task should stop its running process, if any - done chan struct{} // done closes when all processes related to the task have terminated - initialState taskStateFn // prepare and start a new live process, defaults to notStartedTask; should be set by run() - runLatch int32 // guard against multiple Task.run calls - killFunc func(bool) (int, error) -} - -// New builds a newly initialized task object but does not start any processes for it. callers -// are expected to invoke task.run(...) on their own. -func New(name, bin string, args []string, cl func() io.WriteCloser, options ...Option) *Task { - t := &Task{ - name: name, - bin: bin, - args: args, - createLogger: cl, - completedCh: make(chan *Completion), - shouldQuit: make(chan struct{}), - done: make(chan struct{}), - RestartDelay: defaultTaskRestartDelay, - Finished: func(restarting bool) bool { return restarting }, - } - t.killFunc = func(force bool) (int, error) { return t.cmd.Kill(force) } - for _, opt := range options { - opt(t) - } - return t -} - -// Start spawns a goroutine to execute the Task. Panics if invoked more than once. -func (t *Task) Start() { - go t.run(notStartedTask) -} - -// run executes the state machine responsible for starting, monitoring, and possibly restarting -// a system process for the task. The initialState func is the entry point of the state machine. -// Upon returning the done and completedCh chans are all closed. -func (t *Task) run(initialState taskStateFn) { - if !atomic.CompareAndSwapInt32(&t.runLatch, 0, 1) { - panic("Task.run() may only be invoked once") - } - t.initialState = initialState - - defer close(t.done) - defer close(t.completedCh) - - state := initialState - for state != nil { - next := state(t) - state = next - } -} - -func (t *Task) tryComplete(tc *Completion) { - select { - case <-t.shouldQuit: - // best effort - select { - case t.completedCh <- tc: - default: - } - case t.completedCh <- tc: - } -} - -// tryError is a convenience func that invokes tryComplete with a completion error -func (t *Task) tryError(err error) { - t.tryComplete(&Completion{err: err}) -} - -type taskStateFn func(*Task) taskStateFn - -func taskShouldRestart(t *Task) taskStateFn { - // make our best effort to stop here if signalled (shouldQuit). not doing so here - // could add cost later (a process might be launched). - - // sleep for a bit; then return t.initialState - tm := time.NewTimer(t.RestartDelay) - defer tm.Stop() - select { - case <-tm.C: - select { - case <-t.shouldQuit: - default: - if t.Finished(true) { - select { - case <-t.shouldQuit: - // the world has changed, die - return nil - default: - } - return t.initialState - } - // finish call decided not to respawn, so die - return nil - } - case <-t.shouldQuit: - } - - // we're quitting, tell the Finished callback and then die - t.Finished(false) - return nil -} - -func (t *Task) initLogging(r io.Reader) { - writer := t.createLogger() - go func() { - defer writer.Close() - _, err := io.Copy(writer, r) - if err != nil && err != io.EOF { - // using tryComplete is racy because the state machine closes completedCh and - // so we don't want to attempt to write to a closed/closing chan. so - // just log this for now. - log.Errorf("logger for task %q crashed: %v", t.bin, err) - } - }() -} - -// notStartedTask spawns the given task and transitions to a startedTask state -func notStartedTask(t *Task) taskStateFn { - log.Infof("starting task process %q with args '%+v'", t.bin, t.args) - - // create command - cmd := exec.Command(t.bin, t.args...) - stdout, err := cmd.StdoutPipe() - if err != nil { - t.tryError(fmt.Errorf("error getting stdout of %v: %v", t.name, err)) - return taskShouldRestart - } - go func() { - defer stdout.Close() - io.Copy(ioutil.Discard, stdout) // TODO(jdef) we might want to save this at some point - }() - stderrLogs, err := cmd.StderrPipe() - if err != nil { - t.tryError(fmt.Errorf("error getting stderr of %v: %v", t.name, err)) - return taskShouldRestart - } - - t.initLogging(stderrLogs) - if len(t.Env) > 0 { - cmd.Env = t.Env - } - cmd.SysProcAttr = sysProcAttr() - - // last min check for shouldQuit here - select { - case <-t.shouldQuit: - t.tryError(fmt.Errorf("task execution canceled, aborting process launch")) - return taskShouldRestart - default: - } - - if err := cmd.Start(); err != nil { - t.tryError(fmt.Errorf("failed to start task process %q: %v", t.bin, err)) - return taskShouldRestart - } - log.Infoln("task started", t.name) - t.cmd = &cmdProcess{delegate: cmd} - return taskRunning -} - -type exitError interface { - error - - // see os.ProcessState.Sys: returned value can be converted to something like syscall.WaitStatus - Sys() interface{} -} - -func taskRunning(t *Task) taskStateFn { - // listen for normal process completion in a goroutine; don't block because we need to listen for shouldQuit - waitCh := make(chan *Completion, 1) - go func() { - wr := &Completion{name: t.name} - defer func() { - waitCh <- wr - close(waitCh) - }() - - if err := t.cmd.Wait(); err != nil { - if exitError, ok := err.(exitError); ok { - if waitStatus, ok := exitError.Sys().(syscall.WaitStatus); ok { - wr.code = waitStatus.ExitStatus() - return - } - } - wr.err = fmt.Errorf("task wait ended strangely for %q: %v", t.bin, err) - } - }() - - select { - case <-t.shouldQuit: - t.tryComplete(t.awaitDeath(&realTimer{}, defaultKillGracePeriod, waitCh)) - case wr := <-waitCh: - t.tryComplete(wr) - } - return taskShouldRestart -} - -// awaitDeath waits for the process to complete, or else for a "quit" signal on the task- -// at which point we'll attempt to kill manually. -func (t *Task) awaitDeath(timer timer, gracePeriod time.Duration, waitCh <-chan *Completion) *Completion { - defer timer.discard() - - select { - case wr := <-waitCh: - // got a signal to quit, but we're already finished - return wr - default: - } - - forceKill := false - wr := &Completion{name: t.name, err: fmt.Errorf("failed to kill process: %q", t.bin)} - - // the loop is here in case we receive a shouldQuit signal; we need to kill the task. - // in this case, first send a SIGTERM (force=false) to the task and then wait for it - // to die (within the gracePeriod). if it doesn't die, then we loop around only this - // time we'll send a SIGKILL (force=true) and wait for a reduced gracePeriod. There - // does exist a slim chance that the underlying wait4() syscall won't complete before - // this process dies, in which case a zombie will rise. Starting the mesos slave with - // pid namespace isolation should mitigate this. -waitLoop: - for i := 0; i < 2; i++ { - log.Infof("killing %s (force=%t) : %s", t.name, forceKill, t.bin) - pid, err := t.killFunc(forceKill) - if err != nil { - log.Warningf("failed to kill process: %q pid %d: %v", t.bin, pid, err) - break waitLoop - } - - // Wait for the kill to be processed, and child proc resources cleaned up; try to avoid zombies! - timer.set(gracePeriod) - select { - case wr = <-waitCh: - break waitLoop - case <-timer.await(): - // want a timeout, but a shorter one than we used initially. - // using /= 2 is deterministic and yields the desirable effect. - gracePeriod /= 2 - forceKill = true - continue waitLoop - } - } - return wr -} - -// forwardUntil forwards task process completion status and errors to the given output -// chans until either the task terminates or abort is closed. -func (t *Task) forwardUntil(tch chan<- *Completion, abort <-chan struct{}) { - // merge task completion and error until we're told to die, then - // tell the task to stop - defer close(t.shouldQuit) - forwardCompletionUntil(t.completedCh, tch, nil, abort, nil) -} - -// MergeOutput waits for the given tasks to complete. meanwhile it logs each time a task -// process completes or generates an error. when shouldQuit closes, tasks are canceled and this -// func eventually returns once all ongoing event handlers have completed running. -func MergeOutput(tasks []*Task, shouldQuit <-chan struct{}) Events { - tc := make(chan *Completion) - - var waitForTasks sync.WaitGroup - waitForTasks.Add(len(tasks)) - - for _, t := range tasks { - t := t - // translate task dead signal into Done - go func() { - <-t.done - waitForTasks.Done() - }() - // fan-in task completion and error events to tc, ec - go t.forwardUntil(tc, shouldQuit) - } - - tclistener := make(chan *Completion) - done := runtime.After(func() { - completionFinished := runtime.After(func() { - defer close(tclistener) - forwardCompletionUntil(tc, tclistener, nil, shouldQuit, func(tt *Completion, shutdown bool) { - prefix := "" - if shutdown { - prefix = "(shutdown) " - } - log.Infof(prefix+"task %q exited with status %d", tt.name, tt.code) - }) - }) - waitForTasks.Wait() - close(tc) - <-completionFinished - }) - ei := newEventsImpl(tclistener, done) - return ei -} - -// Option is a functional option type for a Task that returns an "undo" Option after upon modifying the Task -type Option func(*Task) Option - -// NoRespawn configures the Task lifecycle such that it will not respawn upon termination -func NoRespawn(listener chan<- struct{}) Option { - return func(t *Task) Option { - finished, restartDelay := t.Finished, t.RestartDelay - - t.Finished = func(_ bool) bool { - // this func implements the task.finished spec, so when the task exits - // we return false to indicate that it should not be restarted. we also - // close execDied to signal interested listeners. - if listener != nil { - close(listener) - listener = nil - } - return false - } - - // since we only expect to die once, and there is no restart; don't delay any longer than needed - t.RestartDelay = 0 - - return func(t2 *Task) Option { - t2.Finished, t2.RestartDelay = finished, restartDelay - return NoRespawn(listener) - } - } -} - -// Environment customizes the process runtime environment for a Task -func Environment(env []string) Option { - return func(t *Task) Option { - oldenv := t.Env - t.Env = env[:] - return Environment(oldenv) - } -} diff --git a/contrib/mesos/pkg/minion/tasks/task_linux.go b/contrib/mesos/pkg/minion/tasks/task_linux.go deleted file mode 100644 index 715cf47ec42..00000000000 --- a/contrib/mesos/pkg/minion/tasks/task_linux.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "syscall" -) - -func sysProcAttr() *syscall.SysProcAttr { - return &syscall.SysProcAttr{ - Setpgid: true, - Pdeathsig: syscall.SIGKILL, // see cmdProcess.Kill - } -} diff --git a/contrib/mesos/pkg/minion/tasks/task_other.go b/contrib/mesos/pkg/minion/tasks/task_other.go deleted file mode 100644 index 24a0f946c08..00000000000 --- a/contrib/mesos/pkg/minion/tasks/task_other.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build !linux - -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "syscall" -) - -func sysProcAttr() *syscall.SysProcAttr { - // TODO(jdef) - // Consequence of not having Pdeathdig is that on non-Linux systems, - // if SIGTERM doesn't stop child procs then they may "leak" and be - // reparented 'up the chain' somewhere when the minion process - // terminates. For example, such child procs end up living indefinitely - // as children of the mesos slave process (I think the slave could handle - // this case, but currently doesn't do it very well). Pdeathsig on Linux - // was a fallback/failsafe mechanism implemented to guard against this. I - // don't know if OS X has any syscalls that do something similar. - return &syscall.SysProcAttr{ - Setpgid: true, - } -} diff --git a/contrib/mesos/pkg/minion/tasks/task_test.go b/contrib/mesos/pkg/minion/tasks/task_test.go deleted file mode 100644 index 8b62bb5dbea..00000000000 --- a/contrib/mesos/pkg/minion/tasks/task_test.go +++ /dev/null @@ -1,311 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "bytes" - "errors" - "fmt" - "io" - "sync" - "syscall" - "testing" - "time" - - log "github.com/golang/glog" - "github.com/stretchr/testify/assert" -) - -type badWriteCloser struct { - err error -} - -func (b *badWriteCloser) Write(_ []byte) (int, error) { return 0, b.err } -func (b *badWriteCloser) Close() error { return b.err } - -type discardCloser int - -func (d discardCloser) Write(b []byte) (int, error) { return len(b), nil } -func (d discardCloser) Close() error { return nil } - -var devNull = func() io.WriteCloser { return discardCloser(0) } - -type fakeExitError uint32 - -func (f fakeExitError) Sys() interface{} { return syscall.WaitStatus(f << 8) } -func (f fakeExitError) Error() string { return fmt.Sprintf("fake-exit-error: %d", f) } - -type fakeProcess struct { - done chan struct{} - pid int - err error -} - -func (f *fakeProcess) Wait() error { - <-f.done - return f.err -} -func (f *fakeProcess) Kill(_ bool) (int, error) { - close(f.done) - return f.pid, f.err -} -func (f *fakeProcess) exit(code int) { - f.err = fakeExitError(code) - close(f.done) -} - -func newFakeProcess() *fakeProcess { - return &fakeProcess{ - done: make(chan struct{}), - } -} - -func TestBadLogger(t *testing.T) { - err := errors.New("qux") - fp := newFakeProcess() - tt := New("foo", "bar", nil, func() io.WriteCloser { - defer func() { - fp.pid = 123 // sanity check - fp.Kill(false) // this causes Wait() to return - }() - return &badWriteCloser{err} - }) - - tt.RestartDelay = 0 // don't slow the test down for no good reason - - finishCalled := make(chan struct{}) - tt.Finished = func(ok bool) bool { - log.Infof("tt.Finished: ok %t", ok) - if ok { - close(finishCalled) - } - return false // never respawn, this causes t.done to close - } - - // abuse eventsImpl: we're not going to listen on the task completion or event chans, - // and we don't want to block the state machine, so discard all task events as they happen - ei := newEventsImpl(tt.completedCh, tt.done) - ei.Close() - - go tt.run(func(_ *Task) taskStateFn { - log.Infof("tt initialized") - tt.initLogging(bytes.NewBuffer(([]byte)("unlogged bytes"))) - tt.cmd = fp - return taskRunning - }) - - // if the logger fails the task will be killed - // badWriteLogger generates an error immediately and results in a task kill - <-finishCalled - <-tt.done - - // this should never data race since the state machine is dead at this point - if fp.pid != 123 { - t.Fatalf("incorrect pid, expected 123 not %d", fp.pid) - } - - // TODO(jdef) would be nice to check for a specific error that indicates the logger died -} - -func TestMergeOutput(t *testing.T) { - var tasksStarted, tasksDone sync.WaitGroup - tasksDone.Add(2) - tasksStarted.Add(2) - - t1 := New("foo", "", nil, devNull) - t1exited := make(chan struct{}) - t1.RestartDelay = 0 // don't slow the test down for no good reason - t1.Finished = func(ok bool) bool { - // we expect each of these cases to happen exactly once - if !ok { - tasksDone.Done() - } else { - close(t1exited) - } - return ok - } - go t1.run(func(t *Task) taskStateFn { - defer tasksStarted.Done() - t.initLogging(bytes.NewBuffer([]byte{})) - t.cmd = newFakeProcess() - return taskRunning - }) - - t2 := New("bar", "", nil, devNull) - t2exited := make(chan struct{}) - t2.RestartDelay = 0 // don't slow the test down for no good reason - t2.Finished = func(ok bool) bool { - // we expect each of these cases to happen exactly once - if !ok { - tasksDone.Done() - } else { - close(t2exited) - } - return ok - } - go t2.run(func(t *Task) taskStateFn { - defer tasksStarted.Done() - t.initLogging(bytes.NewBuffer([]byte{})) - t.cmd = newFakeProcess() - return taskRunning - }) - - shouldQuit := make(chan struct{}) - te := MergeOutput([]*Task{t1, t2}, shouldQuit) - - tasksStarted.Wait() - tasksStarted.Add(2) // recycle the barrier - - // kill each task once, let it restart; make sure that we get the completion status? - t1.cmd.(*fakeProcess).exit(1) - t2.cmd.(*fakeProcess).exit(2) - - codes := map[int]struct{}{} - for i := 0; i < 2; i++ { - switch tc := <-te.Completion(); tc.code { - case 1, 2: - codes[tc.code] = struct{}{} - default: - if tc.err != nil { - t.Errorf("unexpected task completion error: %v", tc.err) - } else { - t.Errorf("unexpected task completion code: %d", tc.code) - } - } - } - - te.Close() // we're not going to read any other completion or error events - - if len(codes) != 2 { - t.Fatalf("expected each task process to exit once") - } - - // each task invokes Finished() once - <-t1exited - <-t2exited - - log.Infoln("each task process has completed one round") - tasksStarted.Wait() // tasks will auto-restart their exited procs - - // assert that the tasks are not dead; TODO(jdef) not sure that these checks are useful - select { - case <-t1.done: - t.Fatalf("t1 is unexpectedly dead") - default: - } - select { - case <-t2.done: - t.Fatalf("t2 is unexpectedly dead") - default: - } - - log.Infoln("firing quit signal") - close(shouldQuit) // fire shouldQuit, and everything should terminate gracefully - - log.Infoln("waiting for tasks to die") - tasksDone.Wait() // our tasks should die - - log.Infoln("waiting for merge to complete") - <-te.Done() // wait for the merge to complete -} - -type fakeTimer struct { - ch chan time.Time -} - -func (t *fakeTimer) set(d time.Duration) {} -func (t *fakeTimer) discard() {} -func (t *fakeTimer) await() <-chan time.Time { return t.ch } -func (t *fakeTimer) expire() { t.ch = make(chan time.Time); close(t.ch) } -func (t *fakeTimer) reset() { t.ch = nil } - -func TestAfterDeath(t *testing.T) { - // test kill escalation since that's not covered by other unit tests - t1 := New("foo", "", nil, devNull) - kills := 0 - waitCh := make(chan *Completion, 1) - timer := &fakeTimer{} - timer.expire() - t1.killFunc = func(force bool) (int, error) { - // > 0 is intentional, multiple calls to close() should panic - if kills > 0 { - assert.True(t, force) - timer.reset() // don't want to race w/ waitCh - waitCh <- &Completion{name: t1.name, code: 123} - close(waitCh) - } else { - assert.False(t, force) - } - kills++ - return 0, nil - } - wr := t1.awaitDeath(timer, 0, waitCh) - assert.Equal(t, "foo", wr.name) - assert.Equal(t, 123, wr.code) - assert.NoError(t, wr.err) - - // test tie between shouldQuit and waitCh - waitCh = make(chan *Completion, 1) - waitCh <- &Completion{name: t1.name, code: 456} - close(waitCh) - t1.killFunc = func(force bool) (int, error) { - t.Fatalf("should not attempt to kill a task that has already reported completion") - return 0, nil - } - - timer.reset() // don't race w/ waitCh - wr = t1.awaitDeath(timer, 0, waitCh) - assert.Equal(t, 456, wr.code) - assert.NoError(t, wr.err) - - // test delayed killFunc failure - kills = 0 - killFailed := errors.New("for some reason kill failed") - t1.killFunc = func(force bool) (int, error) { - // > 0 is intentional, multiple calls to close() should panic - if kills > 0 { - assert.True(t, force) - return -1, killFailed - } else { - assert.False(t, force) - } - kills++ - return 0, nil - } - timer.expire() - wr = t1.awaitDeath(timer, 0, nil) - assert.Equal(t, "foo", wr.name) - assert.Error(t, wr.err) - - // test initial killFunc failure - kills = 0 - t1.killFunc = func(force bool) (int, error) { - // > 0 is intentional, multiple calls to close() should panic - if kills > 0 { - assert.True(t, force) - t.Fatalf("killFunc should only be invoked once, not again after is has already failed") - } else { - assert.False(t, force) - } - kills++ - return 0, killFailed - } - timer.expire() - wr = t1.awaitDeath(timer, 0, nil) - assert.Equal(t, "foo", wr.name) - assert.Error(t, wr.err) -} diff --git a/contrib/mesos/pkg/minion/tasks/timer.go b/contrib/mesos/pkg/minion/tasks/timer.go deleted file mode 100644 index efa7ff1ae13..00000000000 --- a/contrib/mesos/pkg/minion/tasks/timer.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "time" -) - -type timer interface { - set(time.Duration) - discard() - await() <-chan time.Time -} - -type realTimer struct { - *time.Timer -} - -func (t *realTimer) set(d time.Duration) { - if t.Timer == nil { - t.Timer = time.NewTimer(d) - } else { - t.Reset(d) - } -} - -func (t *realTimer) await() <-chan time.Time { - if t.Timer == nil { - return nil - } - return t.C -} - -func (t *realTimer) discard() { - if t.Timer != nil { - t.Stop() - } -} diff --git a/contrib/mesos/pkg/node/doc.go b/contrib/mesos/pkg/node/doc.go deleted file mode 100644 index 9d57176ee67..00000000000 --- a/contrib/mesos/pkg/node/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package node provides utilities to create and update nodes -package node // import "k8s.io/kubernetes/contrib/mesos/pkg/node" diff --git a/contrib/mesos/pkg/node/node.go b/contrib/mesos/pkg/node/node.go deleted file mode 100644 index ce41434b31a..00000000000 --- a/contrib/mesos/pkg/node/node.go +++ /dev/null @@ -1,226 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - "reflect" - "strconv" - "strings" - "time" - - unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" - - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/errors" - "k8s.io/kubernetes/pkg/api/unversioned" - "k8s.io/kubernetes/pkg/util/validation" -) - -const ( - labelPrefix = "k8s.mesosphere.io/attribute-" - clientRetryCount = 5 - clientRetryInterval = time.Second -) - -// Create creates a new node api object with the given hostname, -// slave attribute labels and annotations -func Create( - client unversionedcore.NodesGetter, - hostName string, - slaveAttrLabels, - annotations map[string]string, -) (*api.Node, error) { - n := api.Node{ - ObjectMeta: api.ObjectMeta{ - Name: hostName, - }, - Spec: api.NodeSpec{ - ExternalID: hostName, - }, - Status: api.NodeStatus{ - Phase: api.NodePending, - // WORKAROUND(sttts): make sure that the Ready condition is the - // first one. The kube-ui v3 depends on this assumption. - // TODO(sttts): remove this workaround when kube-ui v4 is used or we - // merge this with the statusupdate in the controller manager. - Conditions: []api.NodeCondition{ - { - Type: api.NodeReady, - Status: api.ConditionTrue, - Reason: slaveReadyReason, - Message: slaveReadyMessage, - LastHeartbeatTime: unversioned.Now(), - }, - }, - }, - } - - n.Labels = mergeMaps( - map[string]string{"kubernetes.io/hostname": hostName}, - slaveAttrLabels, - ) - - n.Annotations = annotations - - // try to create - return client.Nodes().Create(&n) -} - -// Update updates an existing node api object -// by looking up the given hostname. -// The updated node merges the given slave attribute labels -// and annotations with the found api object. -func Update( - client unversionedcore.NodesGetter, - hostname string, - slaveAttrLabels, - annotations map[string]string, -) (n *api.Node, err error) { - for i := 0; i < clientRetryCount; i++ { - n, err = client.Nodes().Get(hostname) - if err != nil { - return nil, fmt.Errorf("error getting node %q: %v", hostname, err) - } - if n == nil { - return nil, fmt.Errorf("no node instance returned for %q", hostname) - } - - // update labels derived from Mesos slave attributes, keep all other labels - n.Labels = mergeMaps( - filterMap(n.Labels, IsNotSlaveAttributeLabel), - slaveAttrLabels, - ) - n.Annotations = mergeMaps(n.Annotations, annotations) - - n, err = client.Nodes().Update(n) - if err == nil && !errors.IsConflict(err) { - return n, nil - } - - log.Infof("retry %d/%d: error updating node %v err %v", i, clientRetryCount, n, err) - time.Sleep(time.Duration(i) * clientRetryInterval) - } - - return nil, err -} - -// CreateOrUpdate creates a node api object or updates an existing one -func CreateOrUpdate( - client unversionedcore.NodesGetter, - hostname string, - slaveAttrLabels, - annotations map[string]string, -) (*api.Node, error) { - n, err := Create(client, hostname, slaveAttrLabels, annotations) - if err == nil { - return n, nil - } - - if !errors.IsAlreadyExists(err) { - return nil, fmt.Errorf("unable to register %q with the apiserver: %v", hostname, err) - } - - // fall back to update an old node with new labels - return Update(client, hostname, slaveAttrLabels, annotations) -} - -// IsNotSlaveAttributeLabel returns true iff the given label is not derived from a slave attribute -func IsNotSlaveAttributeLabel(key, value string) bool { - return !IsSlaveAttributeLabel(key, value) -} - -// IsSlaveAttributeLabel returns true iff the given label is derived from a slave attribute -func IsSlaveAttributeLabel(key, value string) bool { - return strings.HasPrefix(key, labelPrefix) -} - -// IsUpToDate returns true iff the node's slave labels match the given attributes labels -func IsUpToDate(n *api.Node, labels map[string]string) bool { - slaveLabels := map[string]string{} - for k, v := range n.Labels { - if IsSlaveAttributeLabel(k, "") { - slaveLabels[k] = v - } - } - return reflect.DeepEqual(slaveLabels, labels) -} - -// SlaveAttributesToLabels converts slave attributes into string key/value labels -func SlaveAttributesToLabels(attrs []*mesos.Attribute) map[string]string { - l := map[string]string{} - for _, a := range attrs { - if a == nil { - continue - } - - var v string - k := labelPrefix + a.GetName() - - switch a.GetType() { - case mesos.Value_TEXT: - v = a.GetText().GetValue() - case mesos.Value_SCALAR: - v = strconv.FormatFloat(a.GetScalar().GetValue(), 'G', -1, 64) - } - - if errs := validation.IsQualifiedName(k); len(errs) != 0 { - log.V(3).Infof("ignoring invalid node label %q: %v", k, errs) - continue - } - - if errs := validation.IsValidLabelValue(v); len(errs) != 0 { - log.V(3).Infof("ignoring invalid node %s=%q: %v", k, v, errs) - continue - } - - l[k] = v - } - return l -} - -// filterMap filters the given map and returns a new map -// containing all original elements matching the given key-value predicate. -func filterMap(m map[string]string, predicate func(string, string) bool) map[string]string { - result := make(map[string]string, len(m)) - for k, v := range m { - if predicate(k, v) { - result[k] = v - } - } - return result -} - -// mergeMaps merges all given maps into a single map. -// There is no advanced key conflict resolution. -// The last key from the given maps wins. -func mergeMaps(ms ...map[string]string) map[string]string { - var l int - for _, m := range ms { - l += len(m) - } - - result := make(map[string]string, l) - for _, m := range ms { - for k, v := range m { - result[k] = v - } - } - return result -} diff --git a/contrib/mesos/pkg/node/registrator.go b/contrib/mesos/pkg/node/registrator.go deleted file mode 100644 index 6dd912586c2..00000000000 --- a/contrib/mesos/pkg/node/registrator.go +++ /dev/null @@ -1,151 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - "time" - - unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/errors" -) - -type Registrator interface { - // Register checks whether the node is registered with the given labels. If it - // is not, it is created or updated on the apiserver. If an the node was up-to-date, - // false is returned. - Register(hostName string, labels map[string]string) (bool, error) - - // Start the registration loop and return immediately. - Run(terminate <-chan struct{}) error -} - -type registration struct { - hostName string - labels map[string]string -} - -func (r *registration) Copy() queue.Copyable { - return ®istration{ - hostName: r.hostName, - labels: r.labels, // labels are never changed, no need to clone - } -} - -func (r *registration) GetUID() string { - return r.hostName -} - -func (r *registration) Value() queue.UniqueCopyable { - return r -} - -type LookupFunc func(hostName string) *api.Node - -type clientRegistrator struct { - lookupNode LookupFunc - client unversionedcore.NodesGetter - queue *queue.HistoricalFIFO -} - -func NewRegistrator(client unversionedcore.NodesGetter, lookupNode LookupFunc) *clientRegistrator { - return &clientRegistrator{ - lookupNode: lookupNode, - client: client, - queue: queue.NewHistorical(nil), - } -} - -func (r *clientRegistrator) Run(terminate <-chan struct{}) error { - loop := func() { - RegistrationLoop: - for { - obj := r.queue.Pop(terminate) - log.V(3).Infof("registration event observed") - if obj == nil { - break RegistrationLoop - } - select { - case <-terminate: - break RegistrationLoop - default: - } - - rg := obj.(*registration) - n, needsUpdate := r.updateNecessary(rg.hostName, rg.labels) - if !needsUpdate { - log.V(2).Infof("no update needed, skipping for %s: %v", rg.hostName, rg.labels) - continue - } - - if n == nil { - log.V(2).Infof("creating node %s with labels %v", rg.hostName, rg.labels) - _, err := CreateOrUpdate(r.client, rg.hostName, rg.labels, nil) - if err != nil { - log.Errorf("error creating the node %s: %v", rg.hostName, rg.labels) - } - } else { - log.V(2).Infof("updating node %s with labels %v", rg.hostName, rg.labels) - _, err := Update(r.client, rg.hostName, rg.labels, nil) - if err != nil && errors.IsNotFound(err) { - // last chance when our store was out of date - _, err = Create(r.client, rg.hostName, rg.labels, nil) - } - if err != nil { - log.Errorf("error updating the node %s: %v", rg.hostName, rg.labels) - } - } - } - } - go runtime.Until(loop, time.Second, terminate) - - return nil -} - -func (r *clientRegistrator) Register(hostName string, labels map[string]string) (bool, error) { - _, needsUpdate := r.updateNecessary(hostName, labels) - - if needsUpdate { - log.V(5).Infof("queuing registration for node %s with labels %v", hostName, labels) - err := r.queue.Update(®istration{ - hostName: hostName, - labels: labels, - }) - if err != nil { - return false, fmt.Errorf("cannot register node %s: %v", hostName, err) - } - return true, nil - } - - return false, nil -} - -// updateNecessary retrieves the node with the given hostname and checks whether the given -// labels would mean any update to the node. The unmodified node is returned, plus -// true iff an update is necessary. -func (r *clientRegistrator) updateNecessary(hostName string, labels map[string]string) (*api.Node, bool) { - if r.lookupNode == nil { - return nil, true - } - n := r.lookupNode(hostName) - return n, n == nil || !IsUpToDate(n, labels) -} diff --git a/contrib/mesos/pkg/node/registrator_test.go b/contrib/mesos/pkg/node/registrator_test.go deleted file mode 100644 index f2242c2694d..00000000000 --- a/contrib/mesos/pkg/node/registrator_test.go +++ /dev/null @@ -1,160 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "testing" - - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/errors" - "k8s.io/kubernetes/pkg/api/unversioned" - unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned/fake" - "k8s.io/kubernetes/pkg/client/testing/core" - "k8s.io/kubernetes/pkg/runtime" -) - -type fakeNodes struct { - *fake.FakeNodes -} - -func (f *fakeNodes) Nodes() unversionedcore.NodeInterface { - return f -} - -func calledOnce(h bool, ret runtime.Object, err error) (<-chan struct{}, func(core.Action) (bool, runtime.Object, error)) { - ch := make(chan struct{}) - return ch, func(_ core.Action) (bool, runtime.Object, error) { - select { - case <-ch: - panic("called more than once") - default: - close(ch) - } - return h, ret, err - } -} - -func TestRegister_withUnknownNode(t *testing.T) { - fc := &core.Fake{} - nodes := &fakeNodes{&fake.FakeNodes{Fake: &fake.FakeCore{Fake: fc}}} - createCalled, createOnce := calledOnce(true, nil, nil) - fc.AddReactor("create", "nodes", createOnce) - - lookup := func(hostName string) *api.Node { - select { - case <-createCalled: - return &api.Node{ObjectMeta: api.ObjectMeta{Name: "foo"}} - default: - return nil - } - } - - r := NewRegistrator(nodes, lookup) - ch := make(chan struct{}) - defer close(ch) - r.Run(ch) - - t.Logf("registering node foo") - ok, err := r.Register("foo", nil) - if !ok { - t.Fatalf("registration failed without error") - } else if err != nil { - t.Fatalf("registration failed with error %v", err) - } - - // wait for node creation - t.Logf("awaiting node creation") - <-createCalled -} - -func TestRegister_withKnownNode(t *testing.T) { - fc := &core.Fake{} - nodes := &fakeNodes{&fake.FakeNodes{Fake: &fake.FakeCore{Fake: fc}}} - updateCalled, updateOnce := calledOnce(true, nil, nil) - fc.AddReactor("update", "nodes", updateOnce) - - lookup := func(hostName string) *api.Node { - select { - case <-updateCalled: - return &api.Node{ObjectMeta: api.ObjectMeta{Name: "foo"}} - default: - // this node needs an update because it has labels: the updated version doesn't - return &api.Node{ObjectMeta: api.ObjectMeta{Name: "foo", Labels: map[string]string{"a": "b"}}} - } - } - - r := NewRegistrator(nodes, lookup) - ch := make(chan struct{}) - defer close(ch) - r.Run(ch) - - t.Logf("registering node foo") - ok, err := r.Register("foo", nil) - if !ok { - t.Fatalf("registration failed without error") - } else if err != nil { - t.Fatalf("registration failed with error %v", err) - } - - // wait for node update - t.Logf("awaiting node update") - <-updateCalled -} - -func TestRegister_withSemiKnownNode(t *testing.T) { - // semi-known because the lookup func doesn't see the a very newly created node - // but our apiserver "create" call returns an already-exists error. in this case - // CreateOrUpdate should proceed to attempt an update. - - fc := &core.Fake{} - nodes := &fakeNodes{&fake.FakeNodes{Fake: &fake.FakeCore{Fake: fc}}} - - createCalled, createOnce := calledOnce(true, nil, errors.NewAlreadyExists(unversioned.GroupResource{Group: "", Resource: ""}, "nodes")) - fc.AddReactor("create", "nodes", createOnce) - - updateCalled, updateOnce := calledOnce(true, nil, nil) - fc.AddReactor("update", "nodes", updateOnce) - - lookup := func(hostName string) *api.Node { - select { - case <-updateCalled: - return &api.Node{ObjectMeta: api.ObjectMeta{Name: "foo"}} - default: - // this makes the node semi-known: apiserver knows it but the store/cache doesn't - return nil - } - } - - r := NewRegistrator(nodes, lookup) - ch := make(chan struct{}) - defer close(ch) - r.Run(ch) - - t.Logf("registering node foo") - ok, err := r.Register("foo", nil) - if !ok { - t.Fatalf("registration failed without error") - } else if err != nil { - t.Fatalf("registration failed with error %v", err) - } - - // wait for node update - t.Logf("awaiting node update") - <-createCalled - <-updateCalled -} diff --git a/contrib/mesos/pkg/node/statusupdater.go b/contrib/mesos/pkg/node/statusupdater.go deleted file mode 100644 index 277a9d27e82..00000000000 --- a/contrib/mesos/pkg/node/statusupdater.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - "time" - - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - - log "github.com/golang/glog" - "k8s.io/kubernetes/cmd/kubelet/app/options" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/errors" - "k8s.io/kubernetes/pkg/api/unversioned" - "k8s.io/kubernetes/pkg/client/cache" - "k8s.io/kubernetes/pkg/cloudprovider/providers/mesos" - "k8s.io/kubernetes/pkg/fields" - "k8s.io/kubernetes/pkg/util/sets" -) - -const ( - nodeStatusUpdateRetry = 5 - slaveReadyReason = "SlaveReady" - slaveReadyMessage = "mesos reports ready status" -) - -type StatusUpdater struct { - client *clientset.Clientset - relistPeriod time.Duration - heartBeatPeriod time.Duration - nowFunc func() time.Time -} - -func NewStatusUpdater(client *clientset.Clientset, relistPeriod time.Duration, nowFunc func() time.Time) *StatusUpdater { - kubecfg := options.NewKubeletServer() // only create to get the config, this is without side-effects - return &StatusUpdater{ - client: client, - relistPeriod: relistPeriod, - heartBeatPeriod: kubecfg.NodeStatusUpdateFrequency.Duration, - nowFunc: nowFunc, - } -} - -func (u *StatusUpdater) Run(terminate <-chan struct{}) error { - nodeStore := cache.NewStore(cache.MetaNamespaceKeyFunc) - nodeLW := cache.NewListWatchFromClient(u.client.CoreClient, "nodes", api.NamespaceAll, fields.Everything()) - cache.NewReflector(nodeLW, &api.Node{}, nodeStore, u.relistPeriod).Run() - - monitor := func() { - // build up a set of listed slave nodes without a kubelet - slaves, err := mesos.CloudProvider.ListWithoutKubelet() - if err != nil { - log.Errorf("Error listing slaves without kubelet: %v", err) - return - } - slavesWithoutKubelet := sets.NewString(slaves...) - - // update status for nodes which do not have a kubelet running and - // which are still existing as slave. This status update must be done - // before the node controller counts down the NodeMonitorGracePeriod - nodes := nodeStore.List() - - for _, n := range nodes { - node := n.(*api.Node) - if !slavesWithoutKubelet.Has(node.Spec.ExternalID) { - // let the kubelet do its job updating the status, or the - // node controller will remove this node if the node does not even - // exist anymore - continue - } - - err := u.updateStatus(node) - if err != nil { - log.Errorf("Error updating node status: %v", err) - } - } - } - - go runtime.Until(monitor, u.heartBeatPeriod, terminate) - return nil -} - -func (u *StatusUpdater) updateStatus(n *api.Node) error { - for i := 0; i < nodeStatusUpdateRetry; i++ { - if err := u.tryUpdateStatus(n); err != nil && !errors.IsConflict(err) { - log.Errorf("Error updating node status, will retry: %v", err) - } else { - return nil - } - } - return fmt.Errorf("Update node status exceeds retry count") -} - -// nodeWithUpdatedStatus clones the given node and updates the NodeReady condition. -// The updated node is return and a boolean indicating whether the node was changed -// at all. -func (u *StatusUpdater) nodeWithUpdatedStatus(n *api.Node) (*api.Node, bool, error) { - readyCondition := getCondition(&n.Status, api.NodeReady) - currentTime := unversioned.NewTime(u.nowFunc()) - - // avoid flapping by waiting at least twice the kubetlet update frequency, i.e. - // give the kubelet the chance twice to update the heartbeat. This is necessary - // because we only poll the Mesos master state.json once in a while and we - // know that that the information from there can easily be outdated. - gracePeriod := u.heartBeatPeriod * 2 - if readyCondition != nil && !currentTime.After(readyCondition.LastHeartbeatTime.Add(gracePeriod)) { - return n, false, nil - } - - clone, err := api.Scheme.DeepCopy(n) - if err != nil { - return nil, false, err - } - n = clone.(*api.Node) - - newNodeReadyCondition := api.NodeCondition{ - Type: api.NodeReady, - Status: api.ConditionTrue, - Reason: slaveReadyReason, - Message: slaveReadyMessage, - LastHeartbeatTime: currentTime, - } - - found := false - for i := range n.Status.Conditions { - c := &n.Status.Conditions[i] - if c.Type == api.NodeReady { - if c.Status == newNodeReadyCondition.Status { - newNodeReadyCondition.LastTransitionTime = c.LastTransitionTime - } else { - newNodeReadyCondition.LastTransitionTime = currentTime - } - n.Status.Conditions[i] = newNodeReadyCondition - found = true - break - } - } - - if !found { - newNodeReadyCondition.LastTransitionTime = currentTime - n.Status.Conditions = append(n.Status.Conditions, newNodeReadyCondition) - } - - return n, true, nil -} - -// tryUpdateStatus updates the status of the given node and tries to persist that -// on the apiserver -func (u *StatusUpdater) tryUpdateStatus(n *api.Node) error { - n, updated, err := u.nodeWithUpdatedStatus(n) - if err != nil { - return err - } - if !updated { - return nil - } - - _, err = u.client.Nodes().UpdateStatus(n) - return err -} - -// getCondition returns a condition object for the specific condition -// type, nil if the condition is not set. -func getCondition(status *api.NodeStatus, conditionType api.NodeConditionType) *api.NodeCondition { - if status == nil { - return nil - } - for i := range status.Conditions { - if status.Conditions[i].Type == conditionType { - return &status.Conditions[i] - } - } - return nil -} diff --git a/contrib/mesos/pkg/node/statusupdater_test.go b/contrib/mesos/pkg/node/statusupdater_test.go deleted file mode 100644 index a160c8ff9a6..00000000000 --- a/contrib/mesos/pkg/node/statusupdater_test.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - cmoptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" - kubeletoptions "k8s.io/kubernetes/cmd/kubelet/app/options" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/unversioned" -) - -func Test_nodeWithUpdatedStatus(t *testing.T) { - now := time.Now() - testNode := func(d time.Duration, s api.ConditionStatus, r string) *api.Node { - return &api.Node{ - Status: api.NodeStatus{ - Conditions: []api.NodeCondition{{ - Type: api.NodeOutOfDisk, - }, { - Type: api.NodeReady, - Status: s, - Reason: r, - Message: "some message we don't care about here", - LastTransitionTime: unversioned.Time{Time: now.Add(-time.Minute)}, - LastHeartbeatTime: unversioned.Time{Time: now.Add(d)}, - }}, - }, - } - } - - cm := cmoptions.NewCMServer() - kubecfg := kubeletoptions.NewKubeletServer() - assert.True(t, kubecfg.NodeStatusUpdateFrequency.Duration*3 < cm.NodeMonitorGracePeriod.Duration) // sanity check for defaults - - n := testNode(0, api.ConditionTrue, "KubeletReady") - su := NewStatusUpdater(nil, cm.NodeMonitorPeriod.Duration, func() time.Time { return now }) - _, updated, err := su.nodeWithUpdatedStatus(n) - assert.NoError(t, err) - assert.False(t, updated, "no update expected b/c kubelet updated heartbeat just now") - - n = testNode(-cm.NodeMonitorGracePeriod.Duration, api.ConditionTrue, "KubeletReady") - n2, updated, err := su.nodeWithUpdatedStatus(n) - assert.NoError(t, err) - assert.True(t, updated, "update expected b/c kubelet's update is older than DefaultNodeMonitorGracePeriod") - assert.Equal(t, getCondition(&n2.Status, api.NodeReady).Reason, slaveReadyReason) - assert.Equal(t, getCondition(&n2.Status, api.NodeReady).Message, slaveReadyMessage) - - n = testNode(-kubecfg.NodeStatusUpdateFrequency.Duration, api.ConditionTrue, "KubeletReady") - n2, updated, err = su.nodeWithUpdatedStatus(n) - assert.NoError(t, err) - assert.False(t, updated, "no update expected b/c kubelet's update was missed only once") - - n = testNode(-kubecfg.NodeStatusUpdateFrequency.Duration*3, api.ConditionTrue, "KubeletReady") - n2, updated, err = su.nodeWithUpdatedStatus(n) - assert.NoError(t, err) - assert.True(t, updated, "update expected b/c kubelet's update is older than 3*DefaultNodeStatusUpdateFrequency") - assert.Equal(t, getCondition(&n2.Status, api.NodeReady).Reason, slaveReadyReason) - assert.Equal(t, getCondition(&n2.Status, api.NodeReady).Message, slaveReadyMessage) -} diff --git a/contrib/mesos/pkg/offers/doc.go b/contrib/mesos/pkg/offers/doc.go deleted file mode 100644 index f48d2c8cd67..00000000000 --- a/contrib/mesos/pkg/offers/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package offers contains code that manages Mesos offers. -package offers // import "k8s.io/kubernetes/contrib/mesos/pkg/offers" diff --git a/contrib/mesos/pkg/offers/metrics/doc.go b/contrib/mesos/pkg/offers/metrics/doc.go deleted file mode 100644 index ff2ea8da4e3..00000000000 --- a/contrib/mesos/pkg/offers/metrics/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package metrics defines and exposes instrumentation metrics related to -// Mesos offers. -package metrics // import "k8s.io/kubernetes/contrib/mesos/pkg/offers/metrics" diff --git a/contrib/mesos/pkg/offers/metrics/metrics.go b/contrib/mesos/pkg/offers/metrics/metrics.go deleted file mode 100644 index 7af5089b959..00000000000 --- a/contrib/mesos/pkg/offers/metrics/metrics.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" -) - -const ( - offerSubsystem = "mesos_offers" -) - -type OfferDeclinedReason string - -const ( - OfferExpired = OfferDeclinedReason("expired") - OfferRescinded = OfferDeclinedReason("rescinded") - OfferCompat = OfferDeclinedReason("compat") -) - -var ( - OffersReceived = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: offerSubsystem, - Name: "received", - Help: "Counter of offers received from Mesos broken out by slave host.", - }, - []string{"hostname"}, - ) - - OffersDeclined = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: offerSubsystem, - Name: "declined", - Help: "Counter of offers declined by the framework broken out by slave host.", - }, - []string{"hostname", "reason"}, - ) - - OffersAcquired = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: offerSubsystem, - Name: "acquired", - Help: "Counter of offers acquired for task launch broken out by slave host.", - }, - []string{"hostname"}, - ) - - OffersReleased = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: offerSubsystem, - Name: "released", - Help: "Counter of previously-acquired offers later released, broken out by slave host.", - }, - []string{"hostname"}, - ) -) - -var registerMetrics sync.Once - -func Register() { - registerMetrics.Do(func() { - prometheus.MustRegister(OffersReceived) - prometheus.MustRegister(OffersDeclined) - prometheus.MustRegister(OffersAcquired) - prometheus.MustRegister(OffersReleased) - }) -} - -func InMicroseconds(d time.Duration) float64 { - return float64(d.Nanoseconds() / time.Microsecond.Nanoseconds()) -} diff --git a/contrib/mesos/pkg/offers/offers.go b/contrib/mesos/pkg/offers/offers.go deleted file mode 100644 index b8af015bf0b..00000000000 --- a/contrib/mesos/pkg/offers/offers.go +++ /dev/null @@ -1,570 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package offers - -import ( - "fmt" - "reflect" - "sync" - "sync/atomic" - "time" - - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/contrib/mesos/pkg/offers/metrics" - "k8s.io/kubernetes/contrib/mesos/pkg/proc" - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/pkg/client/cache" - "k8s.io/kubernetes/pkg/util/sets" -) - -const ( - offerListenerMaxAge = 12 // max number of times we'll attempt to fit an offer to a listener before requiring them to re-register themselves - offerIdCacheTTL = 1 * time.Second // determines expiration of cached offer ids, used in listener notification - deferredDeclineTtlFactor = 2 // this factor, multiplied by the offer ttl, determines how long to wait before attempting to decline previously claimed offers that were subsequently deleted, then released. see offerStorage.Delete - notifyListenersDelay = 0 // delay between offer listener notification attempts -) - -type Filter func(*mesos.Offer) bool - -type Registry interface { - // Initialize the instance, spawning necessary housekeeping go routines. - Init(<-chan struct{}) - - // Add offers to this registry, rejecting those that are deemed incompatible. - Add([]*mesos.Offer) - - // Listen for arriving offers that are acceptable to the filter, sending - // a signal on (by closing) the returned channel. A listener will only - // ever be notified once, if at all. - Listen(id string, f Filter) <-chan struct{} - - // invoked when offers are rescinded or expired - Delete(string, metrics.OfferDeclinedReason) - - // when true, returns the offer that's registered for the given ID - Get(offerId string) (Perishable, bool) - - // iterate through non-expired offers in this registry - Walk(Walker) error - - // invalidate one or all (when offerId="") offers; offers are not declined, - // but are simply flagged as expired in the offer history - Invalidate(offerId string) - - // invalidate all offers associated with the slave identified by slaveId. - InvalidateForSlave(slaveId string) -} - -// callback that is invoked during a walk through a series of live offers, -// returning with stop=true (or err != nil) if the walk should stop prematurely. -type Walker func(offer Perishable) (stop bool, err error) - -type RegistryConfig struct { - DeclineOffer func(offerId string) <-chan error // tell Mesos that we're declining the offer - Compat func(*mesos.Offer) bool // returns true if offer is compatible; incompatible offers are declined - TTL time.Duration // determines a perishable offer's expiration deadline: now+ttl - LingerTTL time.Duration // if zero, offers will not linger in the FIFO past their expiration deadline - ListenerDelay time.Duration // specifies the sleep time between offer listener notifications -} - -type offerStorage struct { - RegistryConfig - offers *cache.FIFO // collection of Perishable, both live and expired - listeners *queue.DelayFIFO // collection of *offerListener - delayed *queue.DelayQueue // deadline-oriented offer-event queue - slaves *slaveStorage // slave to offer mappings -} - -type liveOffer struct { - *mesos.Offer - expiration time.Time - acquired int32 // 1 = acquired, 0 = free -} - -type expiredOffer struct { - offerSpec - deadline time.Time -} - -// subset of mesos.OfferInfo useful for recordkeeping -type offerSpec struct { - id string - hostname string -} - -// offers that may perish (all of them?) implement this interface. -// callers may expect to access these funcs concurrently so implementations -// must provide their own form of synchronization around mutable state. -type Perishable interface { - // returns true if this offer has expired - HasExpired() bool - // if not yet expired, return mesos offer details; otherwise nil - Details() *mesos.Offer - // mark this offer as acquired, returning true if it was previously unacquired. thread-safe. - Acquire() bool - // mark this offer as un-acquired. thread-safe. - Release() - // expire or delete this offer from storage - age(s *offerStorage) - // return a unique identifier for this offer - Id() string - // return the slave host for this offer - Host() string - addTo(*queue.DelayQueue) -} - -func (e *expiredOffer) addTo(q *queue.DelayQueue) { - q.Add(e) -} - -func (e *expiredOffer) Id() string { - return e.id -} - -func (e *expiredOffer) Host() string { - return e.hostname -} - -func (e *expiredOffer) HasExpired() bool { - return true -} - -func (e *expiredOffer) Details() *mesos.Offer { - return nil -} - -func (e *expiredOffer) Acquire() bool { - return false -} - -func (e *expiredOffer) Release() {} - -func (e *expiredOffer) age(s *offerStorage) { - log.V(3).Infof("Delete lingering offer: %v", e.id) - s.offers.Delete(e) - s.slaves.deleteOffer(e.id) -} - -// return the time left to linger -func (e *expiredOffer) GetDelay() time.Duration { - return e.deadline.Sub(time.Now()) -} - -func (to *liveOffer) HasExpired() bool { - return time.Now().After(to.expiration) -} - -func (to *liveOffer) Details() *mesos.Offer { - return to.Offer -} - -func (to *liveOffer) Acquire() (acquired bool) { - if acquired = atomic.CompareAndSwapInt32(&to.acquired, 0, 1); acquired { - metrics.OffersAcquired.WithLabelValues(to.Host()).Inc() - } - return -} - -func (to *liveOffer) Release() { - if released := atomic.CompareAndSwapInt32(&to.acquired, 1, 0); released { - metrics.OffersReleased.WithLabelValues(to.Host()).Inc() - } -} - -func (to *liveOffer) age(s *offerStorage) { - s.Delete(to.Id(), metrics.OfferExpired) -} - -func (to *liveOffer) Id() string { - return to.Offer.Id.GetValue() -} - -func (to *liveOffer) Host() string { - return to.Offer.GetHostname() -} - -func (to *liveOffer) addTo(q *queue.DelayQueue) { - q.Add(to) -} - -// return the time remaining before the offer expires -func (to *liveOffer) GetDelay() time.Duration { - return to.expiration.Sub(time.Now()) -} - -func CreateRegistry(c RegistryConfig) Registry { - metrics.Register() - return &offerStorage{ - RegistryConfig: c, - offers: cache.NewFIFO(cache.KeyFunc(func(v interface{}) (string, error) { - if perishable, ok := v.(Perishable); !ok { - return "", fmt.Errorf("expected perishable offer, not '%+v'", v) - } else { - return perishable.Id(), nil - } - })), - listeners: queue.NewDelayFIFO(), - delayed: queue.NewDelayQueue(), - slaves: newSlaveStorage(), - } -} - -func (s *offerStorage) declineOffer(offerId, hostname string, reason metrics.OfferDeclinedReason) { - //TODO(jdef) might be nice to spec an abort chan here - runtime.Signal(proc.OnError(s.DeclineOffer(offerId), func(err error) { - log.Warningf("decline failed for offer id %v: %v", offerId, err) - }, nil)).Then(func() { - metrics.OffersDeclined.WithLabelValues(hostname, string(reason)).Inc() - }) -} - -func (s *offerStorage) Add(offers []*mesos.Offer) { - now := time.Now() - for _, offer := range offers { - if !s.Compat(offer) { - //TODO(jdef) would be nice to batch these up - offerId := offer.Id.GetValue() - log.V(3).Infof("Declining incompatible offer %v", offerId) - s.declineOffer(offerId, offer.GetHostname(), metrics.OfferCompat) - continue - } - timed := &liveOffer{ - Offer: offer, - expiration: now.Add(s.TTL), - acquired: 0, - } - log.V(3).Infof("Receiving offer %v", timed.Id()) - s.offers.Add(timed) - s.delayed.Add(timed) - s.slaves.add(offer.SlaveId.GetValue(), timed.Id()) - metrics.OffersReceived.WithLabelValues(timed.Host()).Inc() - } -} - -// delete an offer from storage, implicitly expires the offer -func (s *offerStorage) Delete(offerId string, reason metrics.OfferDeclinedReason) { - if offer, ok := s.Get(offerId); ok { - log.V(3).Infof("Deleting offer %v", offerId) - // attempt to block others from consuming the offer. if it's already been - // claimed and is not yet lingering then don't decline it - just mark it as - // expired in the history: allow a prior claimant to attempt to launch with it - notYetClaimed := offer.Acquire() - if offer.Details() != nil { - if notYetClaimed { - log.V(3).Infof("Declining offer %v", offerId) - s.declineOffer(offerId, offer.Host(), reason) - } else { - // some pod has acquired this and may attempt to launch a task with it - // failed schedule/launch attempts are required to Release() any claims on the offer - - // TODO(jdef): not sure what a good value is here. the goal is to provide a - // launchTasks (driver) operation enough time to complete so that we don't end - // up declining an offer that we're actually attempting to use. - time.AfterFunc(deferredDeclineTtlFactor*s.TTL, func() { - // at this point the offer is in one of five states: - // a) permanently deleted: expired due to timeout - // b) permanently deleted: expired due to having been rescinded - // c) lingering: expired due to timeout - // d) lingering: expired due to having been rescinded - // e) claimed: task launched and it using resources from this offer - // we want to **avoid** declining an offer that's claimed: attempt to acquire - if offer.Acquire() { - // previously claimed offer was released, perhaps due to a launch - // failure, so we should attempt to decline - log.V(3).Infof("attempting to decline (previously claimed) offer %v", offerId) - s.declineOffer(offerId, offer.Host(), reason) - } - }) - } - } - s.expireOffer(offer) - } // else, ignore offers not in the history -} - -func (s *offerStorage) InvalidateForSlave(slaveId string) { - offerIds := s.slaves.deleteSlave(slaveId) - for oid := range offerIds { - s.invalidateOne(oid) - } -} - -// if offerId == "" then expire all known, live offers, otherwise only the offer indicated -func (s *offerStorage) Invalidate(offerId string) { - if offerId != "" { - s.invalidateOne(offerId) - return - } - obj := s.offers.List() - for _, o := range obj { - offer, ok := o.(Perishable) - if !ok { - log.Errorf("Expected perishable offer, not %v", o) - continue - } - offer.Acquire() // attempt to block others from using it - s.expireOffer(offer) - // don't decline, we already know that it's an invalid offer - } -} - -func (s *offerStorage) invalidateOne(offerId string) { - if offer, ok := s.Get(offerId); ok { - offer.Acquire() // attempt to block others from using it - s.expireOffer(offer) - // don't decline, we already know that it's an invalid offer - } -} - -// Walk the collection of offers. The walk stops either as indicated by the -// Walker or when the end of the offer list is reached. Expired offers are -// never passed to a Walker. -func (s *offerStorage) Walk(w Walker) error { - for _, v := range s.offers.List() { - offer, ok := v.(Perishable) - if !ok { - // offer disappeared... - continue - } - if offer.HasExpired() { - // never pass expired offers to walkers - continue - } - if stop, err := w(offer); err != nil { - return err - } else if stop { - return nil - } - } - return nil -} - -func Expired(offerId, hostname string, ttl time.Duration) *expiredOffer { - return &expiredOffer{offerSpec{id: offerId, hostname: hostname}, time.Now().Add(ttl)} -} - -func (s *offerStorage) expireOffer(offer Perishable) { - // the offer may or may not be expired due to TTL so check for details - // since that's a more reliable determinant of lingering status - if details := offer.Details(); details != nil { - // recently expired, should linger - offerId := details.Id.GetValue() - log.V(3).Infof("Expiring offer %v", offerId) - if s.LingerTTL > 0 { - log.V(3).Infof("offer will linger: %v", offerId) - expired := Expired(offerId, offer.Host(), s.LingerTTL) - s.offers.Update(expired) - s.delayed.Add(expired) - } else { - log.V(3).Infof("Permanently deleting offer %v", offerId) - s.offers.Delete(offerId) - s.slaves.deleteOffer(offerId) - } - } // else, it's still lingering... -} - -func (s *offerStorage) Get(id string) (Perishable, bool) { - if obj, ok, _ := s.offers.GetByKey(id); !ok { - return nil, false - } else { - to, ok := obj.(Perishable) - if !ok { - log.Errorf("invalid offer object in fifo '%v'", obj) - } - return to, ok - } -} - -type offerListener struct { - id string - accepts Filter - notify chan<- struct{} - age int - deadline time.Time - sawVersion uint64 -} - -func (l *offerListener) GetUID() string { - return l.id -} - -func (l *offerListener) Deadline() (time.Time, bool) { - return l.deadline, true -} - -// register a listener for new offers, whom we'll notify upon receiving such. -// notification is delivered in the form of closing the channel, nothing is ever sent. -func (s *offerStorage) Listen(id string, f Filter) <-chan struct{} { - if f == nil { - return nil - } - ch := make(chan struct{}) - listen := &offerListener{ - id: id, - accepts: f, - notify: ch, - deadline: time.Now().Add(s.ListenerDelay), - } - log.V(3).Infof("Registering offer listener %s", listen.id) - s.listeners.Offer(listen, queue.ReplaceExisting) - return ch -} - -func (s *offerStorage) ageOffers() { - offer, ok := s.delayed.Pop().(Perishable) - if !ok { - log.Errorf("Expected Perishable, not %v", offer) - return - } - if details := offer.Details(); details != nil && !offer.HasExpired() { - // live offer has not expired yet: timed out early - // FWIW: early timeouts are more frequent when GOMAXPROCS is > 1 - offer.addTo(s.delayed) - } else { - offer.age(s) - } -} - -func (s *offerStorage) nextListener() *offerListener { - obj := s.listeners.Pop(queue.WithoutCancel()) - if listen, ok := obj.(*offerListener); !ok { - //programming error - panic(fmt.Sprintf("unexpected listener object %v", obj)) - } else { - return listen - } -} - -// notify listeners if we find an acceptable offer for them. listeners -// are garbage collected after a certain age (see offerListenerMaxAge). -// ids lists offer IDs that are retrievable from offer storage. -func (s *offerStorage) notifyListeners(ids func() (sets.String, uint64)) { - listener := s.nextListener() // blocking - - offerIds, version := ids() - if listener.sawVersion == version { - // no changes to offer list, avoid growing older - just wait for new offers to arrive - listener.deadline = time.Now().Add(s.ListenerDelay) - s.listeners.Offer(listener, queue.KeepExisting) - return - } - listener.sawVersion = version - - // notify if we find an acceptable offer - for id := range offerIds { - if offer, ok := s.Get(id); !ok || offer.HasExpired() { - continue - } else if listener.accepts(offer.Details()) { - log.V(3).Infof("Notifying offer listener %s", listener.id) - close(listener.notify) - return - } - } - - // no interesting offers found, re-queue the listener - listener.age++ - if listener.age < offerListenerMaxAge { - listener.deadline = time.Now().Add(s.ListenerDelay) - s.listeners.Offer(listener, queue.KeepExisting) - } else { - // garbage collection is as simple as not re-adding the listener to the queue - log.V(3).Infof("garbage collecting offer listener %s", listener.id) - } -} - -func (s *offerStorage) Init(done <-chan struct{}) { - // zero delay, reap offers as soon as they expire - go runtime.Until(s.ageOffers, 0, done) - - // cached offer ids for the purposes of listener notification - idCache := &stringsCache{ - refill: func() sets.String { - result := sets.NewString() - for _, v := range s.offers.List() { - if offer, ok := v.(Perishable); ok { - result.Insert(offer.Id()) - } - } - return result - }, - ttl: offerIdCacheTTL, - } - - go runtime.Until(func() { s.notifyListeners(idCache.Strings) }, notifyListenersDelay, done) -} - -type stringsCache struct { - expiresAt time.Time - cached sets.String - ttl time.Duration - refill func() sets.String - version uint64 -} - -// not thread-safe -func (c *stringsCache) Strings() (sets.String, uint64) { - now := time.Now() - if c.expiresAt.Before(now) { - old := c.cached - c.cached = c.refill() - c.expiresAt = now.Add(c.ttl) - if !reflect.DeepEqual(old, c.cached) { - c.version++ - } - } - return c.cached, c.version -} - -type slaveStorage struct { - sync.Mutex - index map[string]string // map offerId to slaveId -} - -func newSlaveStorage() *slaveStorage { - return &slaveStorage{ - index: make(map[string]string), - } -} - -// create a mapping between a slave and an offer -func (self *slaveStorage) add(slaveId, offerId string) { - self.Lock() - defer self.Unlock() - self.index[offerId] = slaveId -} - -// delete the slave-offer mappings for slaveId, returns the IDs of the offers that were unmapped -func (self *slaveStorage) deleteSlave(slaveId string) sets.String { - offerIds := sets.NewString() - self.Lock() - defer self.Unlock() - for oid, sid := range self.index { - if sid == slaveId { - offerIds.Insert(oid) - delete(self.index, oid) - } - } - return offerIds -} - -// delete the slave-offer mappings for offerId -func (self *slaveStorage) deleteOffer(offerId string) { - self.Lock() - defer self.Unlock() - delete(self.index, offerId) -} diff --git a/contrib/mesos/pkg/offers/offers_test.go b/contrib/mesos/pkg/offers/offers_test.go deleted file mode 100644 index 1094fb63365..00000000000 --- a/contrib/mesos/pkg/offers/offers_test.go +++ /dev/null @@ -1,391 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package offers - -import ( - "errors" - "sync/atomic" - "testing" - "time" - - mesos "github.com/mesos/mesos-go/mesosproto" - util "github.com/mesos/mesos-go/mesosutil" - "k8s.io/kubernetes/contrib/mesos/pkg/proc" -) - -func TestExpiredOffer(t *testing.T) { - t.Parallel() - - ttl := 2 * time.Second - o := Expired("test", "testhost", ttl) - - if o.Id() != "test" { - t.Error("expiredOffer does not return its Id") - } - if o.Host() != "testhost" { - t.Error("expiredOffer does not return its hostname") - } - if o.HasExpired() != true { - t.Error("expiredOffer is not expired") - } - if o.Details() != nil { - t.Error("expiredOffer does not return nil Details") - } - if o.Acquire() != false { - t.Error("expiredOffer must not be able to be acquired") - } - if delay := o.GetDelay(); !(0 < delay && delay <= ttl) { - t.Error("expiredOffer does not return a valid deadline") - } -} // TestExpiredOffer - -func TestTimedOffer(t *testing.T) { - t.Parallel() - - ttl := 2 * time.Second - now := time.Now() - o := &liveOffer{nil, now.Add(ttl), 0} - - if o.HasExpired() { - t.Errorf("offer ttl was %v and should not have expired yet", ttl) - } - if !o.Acquire() { - t.Fatal("1st acquisition of offer failed") - } - o.Release() - if !o.Acquire() { - t.Fatal("2nd acquisition of offer failed") - } - if o.Acquire() { - t.Fatal("3rd acquisition of offer passed but prior claim was not released") - } - o.Release() - if !o.Acquire() { - t.Fatal("4th acquisition of offer failed") - } - o.Release() - time.Sleep(ttl) - if !o.HasExpired() { - t.Fatal("offer not expired after ttl passed") - } - if !o.Acquire() { - t.Fatal("5th acquisition of offer failed; should not be tied to expiration") - } - if o.Acquire() { - t.Fatal("6th acquisition of offer succeeded; should already be acquired") - } -} // TestTimedOffer - -func TestOfferStorage(t *testing.T) { - ttl := time.Second / 4 - var declinedNum int32 - getDeclinedNum := func() int32 { return atomic.LoadInt32(&declinedNum) } - config := RegistryConfig{ - DeclineOffer: func(offerId string) <-chan error { - atomic.AddInt32(&declinedNum, 1) - return proc.ErrorChan(nil) - }, - Compat: func(o *mesos.Offer) bool { - return o.Hostname == nil || *o.Hostname != "incompatiblehost" - }, - TTL: ttl, - LingerTTL: 2 * ttl, - } - storage := CreateRegistry(config) - - done := make(chan struct{}) - storage.Init(done) - - // Add offer - id := util.NewOfferID("foo") - o := &mesos.Offer{Id: id} - storage.Add([]*mesos.Offer{o}) - - // Added offer should be in the storage - if obj, ok := storage.Get(id.GetValue()); obj == nil || !ok { - t.Error("offer not added") - } - if obj, _ := storage.Get(id.GetValue()); obj.Details() != o { - t.Error("added offer differs from returned offer") - } - - // Not-added offer is not in storage - if obj, ok := storage.Get("bar"); obj != nil || ok { - t.Error("offer bar should not exist in storage") - } - - // Deleted offer lingers in storage, is acquired and declined - offer, _ := storage.Get(id.GetValue()) - declinedNumBefore := getDeclinedNum() - storage.Delete(id.GetValue(), "deleted for test") - if obj, _ := storage.Get(id.GetValue()); obj == nil { - t.Error("deleted offer is not lingering") - } - if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() { - t.Error("deleted offer is no expired") - } - if ok := offer.Acquire(); ok { - t.Error("deleted offer can be acquired") - } - if getDeclinedNum() <= declinedNumBefore { - t.Error("deleted offer was not declined") - } - - // Acquired offer is only declined after 2*ttl - id = util.NewOfferID("foo2") - o = &mesos.Offer{Id: id} - storage.Add([]*mesos.Offer{o}) - offer, _ = storage.Get(id.GetValue()) - declinedNumBefore = getDeclinedNum() - offer.Acquire() - storage.Delete(id.GetValue(), "deleted for test") - if getDeclinedNum() > declinedNumBefore { - t.Error("acquired offer is declined") - } - - offer.Release() - time.Sleep(3 * ttl) - if getDeclinedNum() <= declinedNumBefore { - t.Error("released offer is not declined after 2*ttl") - } - - // Added offer should be expired after ttl, but lingering - id = util.NewOfferID("foo3") - o = &mesos.Offer{Id: id} - storage.Add([]*mesos.Offer{o}) - - time.Sleep(2 * ttl) - obj, ok := storage.Get(id.GetValue()) - if obj == nil || !ok { - t.Error("offer not lingering after ttl") - } - if !obj.HasExpired() { - t.Error("offer is not expired after ttl") - } - - // Should be deleted when waiting longer than LingerTTL - time.Sleep(2 * ttl) - if obj, ok := storage.Get(id.GetValue()); obj != nil || ok { - t.Error("offer not deleted after LingerTTL") - } - - // Incompatible offer is declined - id = util.NewOfferID("foo4") - incompatibleHostname := "incompatiblehost" - o = &mesos.Offer{Id: id, Hostname: &incompatibleHostname} - declinedNumBefore = getDeclinedNum() - storage.Add([]*mesos.Offer{o}) - if obj, ok := storage.Get(id.GetValue()); obj != nil || ok { - t.Error("incompatible offer not rejected") - } - if getDeclinedNum() <= declinedNumBefore { - t.Error("incompatible offer is not declined") - } - - // Invalidated offer are not declined, but expired - id = util.NewOfferID("foo5") - o = &mesos.Offer{Id: id} - storage.Add([]*mesos.Offer{o}) - offer, _ = storage.Get(id.GetValue()) - declinedNumBefore = getDeclinedNum() - storage.Invalidate(id.GetValue()) - if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() { - t.Error("invalidated offer is not expired") - } - if getDeclinedNum() > declinedNumBefore { - t.Error("invalidated offer is declined") - } - if ok := offer.Acquire(); ok { - t.Error("invalidated offer can be acquired") - } - - // Invalidate "" will invalidate all offers - id = util.NewOfferID("foo6") - o = &mesos.Offer{Id: id} - storage.Add([]*mesos.Offer{o}) - id2 := util.NewOfferID("foo7") - o2 := &mesos.Offer{Id: id2} - storage.Add([]*mesos.Offer{o2}) - storage.Invalidate("") - if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() { - t.Error("invalidated offer is not expired") - } - if obj2, _ := storage.Get(id2.GetValue()); !obj2.HasExpired() { - t.Error("invalidated offer is not expired") - } - - // InvalidateForSlave invalides all offers for that slave, but only those - id = util.NewOfferID("foo8") - slaveId := util.NewSlaveID("test-slave") - o = &mesos.Offer{Id: id, SlaveId: slaveId} - storage.Add([]*mesos.Offer{o}) - id2 = util.NewOfferID("foo9") - o2 = &mesos.Offer{Id: id2} - storage.Add([]*mesos.Offer{o2}) - storage.InvalidateForSlave(slaveId.GetValue()) - if obj, _ := storage.Get(id.GetValue()); !obj.HasExpired() { - t.Error("invalidated offer for test-slave is not expired") - } - if obj2, _ := storage.Get(id2.GetValue()); obj2.HasExpired() { - t.Error("invalidated offer another slave is expired") - } - - close(done) -} // TestOfferStorage - -func TestListen(t *testing.T) { - ttl := time.Second / 4 - config := RegistryConfig{ - DeclineOffer: func(offerId string) <-chan error { - return proc.ErrorChan(nil) - }, - Compat: func(o *mesos.Offer) bool { - return true - }, - TTL: ttl, - ListenerDelay: ttl / 2, - } - storage := CreateRegistry(config) - - done := make(chan struct{}) - storage.Init(done) - - // Create two listeners with a hostname filter - hostname1 := "hostname1" - hostname2 := "hostname2" - listener1 := storage.Listen("listener1", func(offer *mesos.Offer) bool { - return offer.GetHostname() == hostname1 - }) - listener2 := storage.Listen("listener2", func(offer *mesos.Offer) bool { - return offer.GetHostname() == hostname2 - }) - - // Add hostname1 offer - id := util.NewOfferID("foo") - o := &mesos.Offer{Id: id, Hostname: &hostname1} - storage.Add([]*mesos.Offer{o}) - - // listener1 is notified by closing channel - select { - case _, more := <-listener1: - if more { - t.Error("listener1 is not closed") - } - } - - // listener2 is not notified within ttl - select { - case <-listener2: - t.Error("listener2 is notified") - case <-time.After(ttl): - } - - close(done) -} // TestListen - -func TestWalk(t *testing.T) { - t.Parallel() - config := RegistryConfig{ - DeclineOffer: func(offerId string) <-chan error { - return proc.ErrorChan(nil) - }, - TTL: 0 * time.Second, - LingerTTL: 0 * time.Second, - ListenerDelay: 0 * time.Second, - } - storage := CreateRegistry(config) - acceptedOfferId := "" - walked := 0 - walker1 := func(p Perishable) (bool, error) { - walked++ - if p.Acquire() { - acceptedOfferId = p.Details().Id.GetValue() - return true, nil - } - return false, nil - } - // sanity check - err := storage.Walk(walker1) - if err != nil { - t.Fatalf("received impossible error %v", err) - } - if walked != 0 { - t.Fatal("walked empty storage") - } - if acceptedOfferId != "" { - t.Fatal("somehow found an offer when registry was empty") - } - impl, ok := storage.(*offerStorage) - if !ok { - t.Fatal("unexpected offer storage impl") - } - // single offer - ttl := 2 * time.Second - now := time.Now() - o := &liveOffer{&mesos.Offer{Id: util.NewOfferID("foo")}, now.Add(ttl), 0} - - impl.offers.Add(o) - err = storage.Walk(walker1) - if err != nil { - t.Fatalf("received impossible error %v", err) - } - if walked != 1 { - t.Fatalf("walk count %d", walked) - } - if acceptedOfferId != "foo" { - t.Fatalf("found offer %v", acceptedOfferId) - } - - acceptedOfferId = "" - err = storage.Walk(walker1) - if err != nil { - t.Fatalf("received impossible error %v", err) - } - if walked != 2 { - t.Fatalf("walk count %d", walked) - } - if acceptedOfferId != "" { - t.Fatalf("found offer %v", acceptedOfferId) - } - - walker2 := func(p Perishable) (bool, error) { - walked++ - return true, nil - } - err = storage.Walk(walker2) - if err != nil { - t.Fatalf("received impossible error %v", err) - } - if walked != 3 { - t.Fatalf("walk count %d", walked) - } - if acceptedOfferId != "" { - t.Fatalf("found offer %v", acceptedOfferId) - } - - walker3 := func(p Perishable) (bool, error) { - walked++ - return true, errors.New("baz") - } - err = storage.Walk(walker3) - if err == nil { - t.Fatal("expected error") - } - if walked != 4 { - t.Fatalf("walk count %d", walked) - } -} diff --git a/contrib/mesos/pkg/podutil/doc.go b/contrib/mesos/pkg/podutil/doc.go deleted file mode 100644 index 76d9e9417a4..00000000000 --- a/contrib/mesos/pkg/podutil/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// podutil contains utilities for reading, writing and filtering streams -// and lists of api.Pod objects. -package podutil // import "k8s.io/kubernetes/contrib/mesos/pkg/podutil" diff --git a/contrib/mesos/pkg/podutil/filters.go b/contrib/mesos/pkg/podutil/filters.go deleted file mode 100644 index 3e0c2e7f247..00000000000 --- a/contrib/mesos/pkg/podutil/filters.go +++ /dev/null @@ -1,132 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podutil - -import ( - log "github.com/golang/glog" - "k8s.io/kubernetes/pkg/api" -) - -type defaultFunc func(pod *api.Pod) error - -// return true if the pod passes the filter -type FilterFunc func(pod *api.Pod) (bool, error) - -type Filters []FilterFunc - -// Annotate safely copies annotation metadata from kv to meta.Annotations. -func Annotate(meta *api.ObjectMeta, kv map[string]string) { - //TODO(jdef) this func probably belong in an "apiutil" package, but we don't - //have much to put there right now so it can just live here. - if meta.Annotations == nil { - meta.Annotations = make(map[string]string) - } - for k, v := range kv { - meta.Annotations[k] = v - } -} - -// Annotator returns a filter that copies annotations from map m into a pod -func Annotator(m map[string]string) FilterFunc { - return FilterFunc(func(pod *api.Pod) (bool, error) { - Annotate(&pod.ObjectMeta, m) - return true, nil - }) -} - -// Environment returns a filter that writes environment variables into pod containers -func Environment(env []api.EnvVar) FilterFunc { - // index the envvar names - var ( - envcount = len(env) - m = make(map[string]int, envcount) - ) - for j := range env { - m[env[j].Name] = j - } - return func(pod *api.Pod) (bool, error) { - for i := range pod.Spec.Containers { - ct := &pod.Spec.Containers[i] - dup := make(map[string]struct{}, envcount) - // overwrite dups (and remember them for later) - for j := range ct.Env { - name := ct.Env[j].Name - if k, ok := m[name]; ok { - ct.Env[j] = env[k] - dup[name] = struct{}{} - } - } - // append non-dups into ct.Env - for name, k := range m { - if _, ok := dup[name]; !ok { - ct.Env = append(ct.Env, env[k]) - } - } - } - return true, nil - } -} - -// Stream returns a chan of pods that yields each pod from the given list. -// No pods are yielded if err is non-nil. -func Stream(list *api.PodList, err error) <-chan *api.Pod { - out := make(chan *api.Pod) - go func() { - defer close(out) - if err != nil { - log.Errorf("failed to obtain pod list: %v", err) - return - } - for _, pod := range list.Items { - pod := pod - out <- &pod - } - }() - return out -} - -func (filters Filters) Do(in <-chan *api.Pod) (out <-chan *api.Pod) { - out = in - for _, f := range filters { - out = f.Do(out) - } - return -} - -func (filter FilterFunc) Do(in <-chan *api.Pod) <-chan *api.Pod { - out := make(chan *api.Pod) - go func() { - defer close(out) - for pod := range in { - if ok, err := filter(pod); err != nil { - log.Errorf("pod failed selection: %v", err) - } else if ok { - out <- pod - } - } - }() - return out -} - -// List reads every pod from the pods chan and returns them all in an api.PodList -func List(pods <-chan *api.Pod) *api.PodList { - list := &api.PodList{} - for p := range pods { - list.Items = append(list.Items, *p) - } - return list -} diff --git a/contrib/mesos/pkg/podutil/gzip.go b/contrib/mesos/pkg/podutil/gzip.go deleted file mode 100644 index e9ae7162255..00000000000 --- a/contrib/mesos/pkg/podutil/gzip.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podutil - -import ( - "bytes" - "compress/gzip" - "fmt" - "io/ioutil" - - "k8s.io/kubernetes/pkg/api" - _ "k8s.io/kubernetes/pkg/api/install" - "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/runtime" -) - -func Gzip(pods <-chan *api.Pod) ([]byte, error) { - return gzipList(List(pods)) -} - -func gzipList(list *api.PodList) ([]byte, error) { - raw, err := runtime.Encode(api.Codecs.LegacyCodec(v1.SchemeGroupVersion), list) - if err != nil { - return nil, err - } - - zipped := &bytes.Buffer{} - zw := gzip.NewWriter(zipped) - _, err = bytes.NewBuffer(raw).WriteTo(zw) - if err != nil { - return nil, err - } - - err = zw.Close() - if err != nil { - return nil, err - } - - return zipped.Bytes(), nil -} - -func Gunzip(gzipped []byte) <-chan *api.Pod { - return Stream(gunzipList(gzipped)) -} - -func gunzipList(gzipped []byte) (*api.PodList, error) { - zr, err := gzip.NewReader(bytes.NewReader(gzipped)) - if err != nil { - return nil, err - } - defer zr.Close() - - raw, err := ioutil.ReadAll(zr) - if err != nil { - return nil, err - } - - obj, err := runtime.Decode(api.Codecs.UniversalDecoder(), raw) - if err != nil { - return nil, err - } - - podlist, ok := obj.(*api.PodList) - if !ok { - return nil, fmt.Errorf("expected *api.PodList instead of %T", obj) - } - - return podlist, nil -} diff --git a/contrib/mesos/pkg/podutil/gzip_test.go b/contrib/mesos/pkg/podutil/gzip_test.go deleted file mode 100644 index dcfe64d6eee..00000000000 --- a/contrib/mesos/pkg/podutil/gzip_test.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podutil - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/v1" -) - -func TestGzipList(t *testing.T) { - // pod spec defaults are written during deserialization, this is what we - // expect them to be - period := int64(v1.DefaultTerminationGracePeriodSeconds) - defaultSpec := api.PodSpec{ - DNSPolicy: api.DNSClusterFirst, - RestartPolicy: api.RestartPolicyAlways, - TerminationGracePeriodSeconds: &period, - SecurityContext: new(api.PodSecurityContext), - } - list := &api.PodList{ - Items: []api.Pod{ - { - ObjectMeta: api.ObjectMeta{ - Name: "foo", - Namespace: "bar", - }, - }, - { - ObjectMeta: api.ObjectMeta{ - Name: "qax", - Namespace: "lkj", - }, - }, - }, - } - - amap := map[string]string{ - "crazy": "horse", - } - annotator := Annotator(amap) - raw, err := Gzip(annotator.Do(Stream(list, nil))) - assert.NoError(t, err) - - list2, err := gunzipList(raw) - assert.NoError(t, err) - - list.Items[0].Spec = defaultSpec - list.Items[0].Annotations = amap - list.Items[1].Spec = defaultSpec - list.Items[1].Annotations = amap - assert.True(t, api.Semantic.DeepEqual(*list, *list2), "expected %+v instead of %+v", *list, *list2) -} diff --git a/contrib/mesos/pkg/podutil/io.go b/contrib/mesos/pkg/podutil/io.go deleted file mode 100644 index cdf274525f1..00000000000 --- a/contrib/mesos/pkg/podutil/io.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podutil - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/api/validation" - "k8s.io/kubernetes/pkg/runtime" - utilyaml "k8s.io/kubernetes/pkg/util/yaml" -) - -func WriteToDir(pods <-chan *api.Pod, destDir string) error { - err := os.MkdirAll(destDir, 0660) - if err != nil { - return err - } - for p := range pods { - filename, ok := p.Annotations[meta.StaticPodFilenameKey] - if !ok { - log.Warningf("skipping static pod %s/%s that had no filename", p.Namespace, p.Name) - continue - } - raw, err := runtime.Encode(api.Codecs.LegacyCodec(v1.SchemeGroupVersion), p) - if err != nil { - log.Errorf("failed to encode static pod as v1 object: %v", err) - continue - } - destfile := filepath.Join(destDir, filename) - err = ioutil.WriteFile(destfile, raw, 0660) - if err != nil { - log.Errorf("failed to write static pod file %q: %v", destfile, err) - } - log.V(1).Infof("wrote static pod %s/%s to %s", p.Namespace, p.Name, destfile) - } - return nil -} - -func ReadFromDir(dirpath string) (<-chan *api.Pod, <-chan error) { - pods := make(chan *api.Pod) - errors := make(chan error) - go func() { - defer close(pods) - defer close(errors) - files, err := ioutil.ReadDir(dirpath) - if err != nil { - errors <- fmt.Errorf("error scanning static pods directory: %q: %v", dirpath, err) - return - } - for _, f := range files { - if f.IsDir() || f.Size() == 0 { - continue - } - filename := filepath.Join(dirpath, f.Name()) - log.V(1).Infof("reading static pod conf from file %q", filename) - - data, err := ioutil.ReadFile(filename) - if err != nil { - errors <- fmt.Errorf("failed to read static pod file: %q: %v", filename, err) - continue - } - - parsed, pod, err := tryDecodeSinglePod(data) - if !parsed { - if err != nil { - errors <- fmt.Errorf("error parsing static pod file %q: %v", filename, err) - } - continue - } - if err != nil { - errors <- fmt.Errorf("error validating static pod file %q: %v", filename, err) - continue - } - Annotate(&pod.ObjectMeta, map[string]string{meta.StaticPodFilenameKey: f.Name()}) - pods <- pod - } - }() - return pods, errors -} - -// tryDecodeSinglePod was copied from pkg/kubelet/config/common.go v1.0.5 -func tryDecodeSinglePod(data []byte) (parsed bool, pod *api.Pod, err error) { - // JSON is valid YAML, so this should work for everything. - json, err := utilyaml.ToJSON(data) - if err != nil { - return false, nil, err - } - obj, err := runtime.Decode(api.Codecs.UniversalDecoder(), json) - if err != nil { - return false, pod, err - } - // Check whether the object could be converted to single pod. - if _, ok := obj.(*api.Pod); !ok { - err = fmt.Errorf("invalid pod: %+v", obj) - return false, pod, err - } - newPod := obj.(*api.Pod) - if errs := validation.ValidatePod(newPod); len(errs) > 0 { - err = fmt.Errorf("invalid pod: %v", errs) - return true, pod, err - } - return true, newPod, nil -} diff --git a/contrib/mesos/pkg/proc/adapter.go b/contrib/mesos/pkg/proc/adapter.go deleted file mode 100644 index 36f98b00086..00000000000 --- a/contrib/mesos/pkg/proc/adapter.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proc - -import ( - "fmt" -) - -type processAdapter struct { - Process - delegate Doer -} - -// reportAnyError waits for an error to arrive from source, or for the process to end; -// errors are reported through errOnce. returns true if an error is reported through -// errOnce, otherwise false. -func (p *processAdapter) reportAnyError(source <-chan error, errOnce ErrorOnce) bool { - select { - case err, ok := <-source: - if ok && err != nil { - // failed to schedule/execute the action - errOnce.Report(err) - return true - } - // action was scheduled/executed just fine. - case <-p.Done(): - // double-check that there's no errror waiting for us in source - select { - case err, ok := <-source: - if ok { - // parent failed to schedule/execute the action - errOnce.Report(err) - return true - } - default: - } - errOnce.Report(errProcessTerminated) - return true - } - return false -} - -func (p *processAdapter) Do(a Action) <-chan error { - errCh := NewErrorOnce(p.Done()) - go func() { - ch := NewErrorOnce(p.Done()) - errOuter := p.Process.Do(func() { - errInner := p.delegate.Do(a) - ch.forward(errInner) - }) - // order is important here: check errOuter before ch - if p.reportAnyError(errOuter, errCh) { - return - } - if !p.reportAnyError(ch.Err(), errCh) { - errCh.Report(nil) - } - }() - return errCh.Err() -} - -// DoWith returns a process that, within its execution context, delegates to the specified Doer. -// Expect a panic if either the given Process or Doer are nil. -func DoWith(other Process, d Doer) Process { - if other == nil { - panic(fmt.Sprintf("cannot DoWith a nil process")) - } - if d == nil { - panic(fmt.Sprintf("cannot DoWith a nil doer")) - } - return &processAdapter{ - Process: other, - delegate: d, - } -} diff --git a/contrib/mesos/pkg/proc/doc.go b/contrib/mesos/pkg/proc/doc.go deleted file mode 100644 index 75a93280091..00000000000 --- a/contrib/mesos/pkg/proc/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package proc provides opinionated utilities for processing background -// operations and future errors, somewhat inspired by libprocess. -package proc // import "k8s.io/kubernetes/contrib/mesos/pkg/proc" diff --git a/contrib/mesos/pkg/proc/errors.go b/contrib/mesos/pkg/proc/errors.go deleted file mode 100644 index fdc0bd46422..00000000000 --- a/contrib/mesos/pkg/proc/errors.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proc - -import ( - "errors" - "fmt" - - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" -) - -var ( - errProcessTerminated = errors.New("cannot execute action because process has terminated") - errIllegalState = errors.New("illegal state, cannot execute action") - - closedErrChan <-chan error // singleton chan that's always closed -) - -func init() { - ch := make(chan error) - close(ch) - closedErrChan = ch -} - -func IsProcessTerminated(err error) bool { - return err == errProcessTerminated -} - -func IsIllegalState(err error) bool { - return err == errIllegalState -} - -// OnError spawns a goroutine that waits for an error. if a non-nil error is read from -// the channel then the handler func is invoked, otherwise (nil error or closed chan) -// the handler is skipped. if a nil handler is specified then it's not invoked. -// the signal chan that's returned closes once the error process logic (and handler, -// if any) has completed. -func OnError(ch <-chan error, f func(error), abort <-chan struct{}) <-chan struct{} { - return runtime.After(func() { - if ch == nil { - return - } - select { - case err, ok := <-ch: - if ok && err != nil && f != nil { - f(err) - } - case <-abort: - if f != nil { - f(errProcessTerminated) - } - } - }) -} - -// ErrorChanf is a convenience func that returns a chan that yields an error -// generated from the given msg format and args. -func ErrorChanf(msg string, args ...interface{}) <-chan error { - return ErrorChan(fmt.Errorf(msg, args...)) -} - -// ErrorChan is a convenience func that returns a chan that yields the given error. -// If err is nil then a closed chan is returned. -func ErrorChan(err error) <-chan error { - if err == nil { - return closedErrChan - } - ch := make(chan error, 1) - ch <- err - return ch -} diff --git a/contrib/mesos/pkg/proc/once.go b/contrib/mesos/pkg/proc/once.go deleted file mode 100644 index 37865fe910f..00000000000 --- a/contrib/mesos/pkg/proc/once.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proc - -import ( - "fmt" - "sync" - "time" -) - -type errorOnce struct { - once sync.Once - err chan error - abort <-chan struct{} -} - -// NewErrorOnce creates an ErrorOnce that aborts blocking func calls once -// the given abort chan has closed. -func NewErrorOnce(abort <-chan struct{}) ErrorOnce { - return &errorOnce{ - err: make(chan error, 1), - abort: abort, - } -} - -func (b *errorOnce) Err() <-chan error { - return b.err -} - -func (b *errorOnce) Reportf(msg string, args ...interface{}) { - b.Report(fmt.Errorf(msg, args...)) -} - -func (b *errorOnce) Report(err error) { - b.once.Do(func() { - select { - case b.err <- err: - default: - } - }) -} - -func (b *errorOnce) Send(errIn <-chan error) ErrorOnce { - if errIn == nil { - // don't execute this in a goroutine; save resources AND the caller - // likely wants this executed ASAP because some of some operation - // ordering semantics. forward() will not block here on a nil input - // so this is safe to do. - b.forward(nil) - } else { - go b.forward(errIn) - } - return b -} - -func (b *errorOnce) forward(errIn <-chan error) { - if errIn == nil { - // important: nil never blocks; Report(nil) is guaranteed to be a - // non-blocking operation. - b.Report(nil) - return - } - select { - case err, _ := <-errIn: - b.Report(err) - case <-b.abort: - // double-check that errIn was blocked: don't falsely return - // errProcessTerminated if errIn was really ready - select { - case err, _ := <-errIn: - b.Report(err) - default: - b.Report(errProcessTerminated) - } - } -} - -func (b *errorOnce) WaitFor(d time.Duration) (error, bool) { - t := time.NewTimer(d) - select { - case err, _ := <-b.err: - t.Stop() - return err, true - case <-t.C: - return nil, false - } -} diff --git a/contrib/mesos/pkg/proc/proc.go b/contrib/mesos/pkg/proc/proc.go deleted file mode 100644 index f90d9cef1b6..00000000000 --- a/contrib/mesos/pkg/proc/proc.go +++ /dev/null @@ -1,196 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proc - -import ( - "sync" - - "k8s.io/kubernetes/pkg/util/runtime" -) - -const ( - // how many actions we can store in the backlog - defaultActionQueueDepth = 1024 -) - -type Config struct { - // determines the size of the deferred action backlog - actionQueueDepth uint32 -} - -var ( - // default process configuration, used in the creation of all new processes - defaultConfig = Config{ - actionQueueDepth: defaultActionQueueDepth, - } -) - -type scheduledAction struct { - action Action - errCh chan error -} - -type processState struct { - actions chan *scheduledAction // scheduled action backlog - running chan struct{} // closes upon start of action backlog processing - terminated chan struct{} // closes upon termination of run() - doer Doer // delegate that schedules actions - guardDoer sync.RWMutex // protect doer - end chan struct{} // closes upon invocation of End() - closeEnd sync.Once // guard: only close end chan once - nextAction func() (*scheduledAction, bool) // return false if actions queue is closed -} - -func New() Process { - return newConfigured(defaultConfig) -} - -func newConfigured(c Config) Process { - ps := &processState{ - actions: make(chan *scheduledAction, c.actionQueueDepth), - running: make(chan struct{}), - terminated: make(chan struct{}), - end: make(chan struct{}), - } - ps.doer = DoerFunc(ps.defaultDoer) - go ps.run() - return ps -} - -type stateFn func(*processState, *scheduledAction) stateFn - -func stateRun(ps *processState, a *scheduledAction) stateFn { - // it's only possible to ever receive this once because we transition - // to state "shutdown", permanently - if a == nil { - ps.shutdown() - return stateShutdown - } - - close(a.errCh) // signal that action was scheduled - func() { - // we don't trust clients of this package - defer func() { - if x := recover(); x != nil { - // HandleCrash has already logged this, so - // nothing to do. - } - }() - defer runtime.HandleCrash() - a.action() - }() - return stateRun -} - -func (ps *processState) shutdown() { - // all future attemps to schedule actions must fail immediately - ps.guardDoer.Lock() - ps.doer = DoerFunc(func(_ Action) <-chan error { - return ErrorChan(errProcessTerminated) - }) - ps.guardDoer.Unlock() - - // no more actions may be scheduled - close(ps.actions) - - // no need to check ps.end anymore - ps.nextAction = func() (a *scheduledAction, ok bool) { - a, ok = <-ps.actions - return - } -} - -// stateShutdown doesn't run any actions because the process is shutting down. -// instead it clears the action backlog. newly scheduled actions are rejected. -func stateShutdown(ps *processState, a *scheduledAction) stateFn { - if a != nil { - a.errCh <- errProcessTerminated - } - return stateShutdown -} - -func (ps *processState) run() { - defer close(ps.terminated) - close(ps.running) - - // main state machine loop: process actions as they come, - // updating the state func along the way. - f := stateRun - ps.nextAction = func() (a *scheduledAction, ok bool) { - // if we successfully read from ps.end, we don't know if the - // actions queue is closed. assume it's not: the state machine - // shouldn't terminate yet. - // also, give preference to ps.end: we want to avoid processing - // actions if both ps.actions and ps.end are ready - select { - case <-ps.end: - ok = true - default: - select { - case <-ps.end: - ok = true - case a, ok = <-ps.actions: - } - } - return - } - for { - a, ok := ps.nextAction() - if !ok { - return - } - g := f(ps, a) - if g == nil { - panic("state machine stateFn is not allowed to be nil") - } - f = g - } -} - -func (ps *processState) Running() <-chan struct{} { - return ps.running -} - -func (ps *processState) Done() <-chan struct{} { - return ps.terminated -} - -func (ps *processState) End() <-chan struct{} { - ps.closeEnd.Do(func() { - close(ps.end) - }) - return ps.terminated -} - -func (ps *processState) Do(a Action) <-chan error { - ps.guardDoer.RLock() - defer ps.guardDoer.RUnlock() - return ps.doer.Do(a) -} - -func (ps *processState) defaultDoer(a Action) <-chan error { - ch := make(chan error, 1) - ps.actions <- &scheduledAction{ - action: a, - errCh: ch, - } - return ch -} - -func (ps *processState) OnError(ch <-chan error, f func(error)) <-chan struct{} { - return OnError(ch, f, ps.terminated) -} diff --git a/contrib/mesos/pkg/proc/proc_test.go b/contrib/mesos/pkg/proc/proc_test.go deleted file mode 100644 index e4d8581356b..00000000000 --- a/contrib/mesos/pkg/proc/proc_test.go +++ /dev/null @@ -1,397 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proc - -import ( - "fmt" - "sync" - "testing" - "time" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" -) - -func failOnError(t *testing.T, errOnce ErrorOnce) { - err, _ := <-errOnce.Err() - if err != nil { - t.Errorf("unexpected action scheduling error: %v", err) - } -} - -func TestProc_manyEndings(t *testing.T) { - p := New() - const COUNT = 20 - var wg sync.WaitGroup - wg.Add(COUNT) - for i := 0; i < COUNT; i++ { - runtime.On(p.End(), wg.Done) - } - wg.Wait() - <-p.Done() -} - -func TestProc_singleAction(t *testing.T) { - p := New() - scheduled := make(chan struct{}) - called := make(chan struct{}) - - errOnce := NewErrorOnce(p.Done()) - go func() { - log.Infof("do'ing deferred action") - defer close(scheduled) - errCh := p.Do(func() { - defer close(called) - log.Infof("deferred action invoked") - }) - errOnce.Send(errCh) - }() - - failOnError(t, errOnce) - - <-scheduled - <-called - p.End() - <-p.Done() -} - -func TestProc_singleActionThatPanics(t *testing.T) { - p := New() - scheduled := make(chan struct{}) - called := make(chan struct{}) - - errOnce := NewErrorOnce(p.Done()) - go func() { - log.Infof("do'ing deferred action") - defer close(scheduled) - errCh := p.Do(func() { - defer close(called) - panic("panicing here") - }) - errOnce.Send(errCh) - }() - - failOnError(t, errOnce) - - <-scheduled - <-called - p.End() - <-p.Done() -} - -func TestProc_singleActionEndsProcess(t *testing.T) { - p := New() - called := make(chan struct{}) - - errOnce := NewErrorOnce(p.Done()) - go func() { - log.Infof("do'ing deferred action") - errCh := p.Do(func() { - defer close(called) - log.Infof("deferred action invoked") - p.End() - }) - errOnce.Send(errCh) - }() - - <-called - - failOnError(t, errOnce) - - <-p.Done() -} - -func TestProc_multiAction(t *testing.T) { - p := New() - const COUNT = 10 - var called sync.WaitGroup - called.Add(COUNT * 2) - - // test FIFO property - next := 0 - for i := 0; i < COUNT; i++ { - log.Infof("do'ing deferred action %d", i) - idx := i - errOnce := NewErrorOnce(p.Done()) - errCh := p.Do(func() { - defer called.Done() - log.Infof("deferred action invoked") - if next != idx { - t.Errorf("expected index %d instead of %d", idx, next) - } - next++ - }) - errOnce.Send(errCh) - go func() { - defer called.Done() - failOnError(t, errOnce) - }() - } - - called.Wait() - p.End() - <-p.Done() -} - -func TestProc_goodLifecycle(t *testing.T) { - p := New() - p.End() - <-p.Done() -} - -func TestProc_doWithDeadProc(t *testing.T) { - p := New() - p.End() - <-p.Done() - - errUnexpected := fmt.Errorf("unexpected execution of delegated action") - decorated := DoWith(p, DoerFunc(func(_ Action) <-chan error { - return ErrorChan(errUnexpected) - })) - - decorated.Do(func() {}) - <-decorated.Done() -} - -func TestProc_doWith(t *testing.T) { - p := New() - - delegated := false - decorated := DoWith(p, DoerFunc(func(a Action) <-chan error { - delegated = true - a() - return nil - })) - - executed := make(chan struct{}) - err := decorated.Do(func() { - defer close(executed) - if !delegated { - t.Errorf("expected delegated execution") - } - }) - if err == nil { - t.Fatalf("expected !nil error chan") - } - - <-executed - <-decorated.OnError(err, func(e error) { - t.Errorf("unexpected error: %v", err) - }) - decorated.End() - <-p.Done() -} - -func TestProc_doWithNestedTwice(t *testing.T) { - p := New() - - delegated := false - decorated := DoWith(p, DoerFunc(func(a Action) <-chan error { - a() - return nil - })) - - decorated2 := DoWith(decorated, DoerFunc(func(a Action) <-chan error { - delegated = true - a() - return nil - })) - - executed := make(chan struct{}) - err := decorated2.Do(func() { - defer close(executed) - if !delegated { - t.Errorf("expected delegated execution") - } - }) - if err == nil { - t.Fatalf("expected !nil error chan") - } - - <-executed - - <-decorated2.OnError(err, func(e error) { - t.Errorf("unexpected error: %v", err) - }) - - decorated2.End() - <-p.Done() -} - -func TestProc_doWithNestedErrorPropagation(t *testing.T) { - p := New() - - delegated := false - decorated := DoWith(p, DoerFunc(func(a Action) <-chan error { - a() - return nil - })) - - expectedErr := fmt.Errorf("expecting this") - errOnce := NewErrorOnce(p.Done()) - decorated2 := DoWith(decorated, DoerFunc(func(a Action) <-chan error { - delegated = true - a() - errOnce.Reportf("unexpected error in decorator2") - return ErrorChanf("another unexpected error in decorator2") - })) - - executed := make(chan struct{}) - errCh := decorated2.Do(func() { - defer close(executed) - if !delegated { - t.Errorf("expected delegated execution") - } - errOnce.Report(expectedErr) - }) - if errCh == nil { - t.Fatalf("expected !nil error chan") - } - errOnce.Send(errCh) - - foundError := false - <-executed - - <-decorated2.OnError(errOnce.Err(), func(e error) { - if e != expectedErr { - t.Errorf("unexpected error: %v", e) - } else { - foundError = true - } - }) - - // this has been flaky in the past. recent changes to error handling in - // processAdapter.Do should have fixed it. - if !foundError { - t.Fatalf("expected a propagated error") - } - - decorated2.End() - <-p.Done() -} - -func runDelegationTest(t *testing.T, p Process, name string, errOnce ErrorOnce) { - t.Logf("starting test case " + name + " at " + time.Now().String()) - defer func() { - t.Logf("runDelegationTest finished at " + time.Now().String()) - }() - var decorated Process - decorated = p - - const DEPTH = 100 - var wg sync.WaitGroup - wg.Add(DEPTH) - y := 0 - - for x := 1; x <= DEPTH; x++ { - x := x - nextp := DoWith(decorated, DoerFunc(func(a Action) <-chan error { - if x == 1 { - t.Logf("delegate chain invoked for " + name + " at " + time.Now().String()) - } - y++ - if y != x { - return ErrorChanf("out of order delegated execution for " + name) - } - defer wg.Done() - a() - return nil - })) - decorated = nextp - } - - executed := make(chan struct{}) - errCh := decorated.Do(func() { - defer close(executed) - if y != DEPTH { - errOnce.Reportf("expected delegated execution for " + name) - } - t.Logf("executing deferred action: " + name + " at " + time.Now().String()) - errOnce.Send(nil) // we completed without error, let the listener know - }) - if errCh == nil { - t.Fatalf("expected !nil error chan") - } - - // forward any scheduling errors to the listener; NOTHING else should attempt to read - // from errCh after this point - errOnce.Send(errCh) - - <-executed - t.Logf("runDelegationTest received executed signal at " + time.Now().String()) - - wg.Wait() - t.Logf("runDelegationTest nested decorators finished at " + time.Now().String()) -} - -func TestProc_doWithNestedX(t *testing.T) { - t.Logf("starting test case at " + time.Now().String()) - p := New() - errOnce := NewErrorOnce(p.Done()) - runDelegationTest(t, p, "nested", errOnce) - <-p.End() - - err, _ := <-errOnce.Err() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - <-p.Done() -} - -// intended to be run with -race -func TestProc_doWithNestedXConcurrent(t *testing.T) { - config := defaultConfig - config.actionQueueDepth = 4000 - p := newConfigured(config) - - var wg sync.WaitGroup - const CONC = 20 - wg.Add(CONC) - - for i := 0; i < CONC; i++ { - i := i - errOnce := NewErrorOnce(p.Done()) - runtime.After(func() { runDelegationTest(t, p, fmt.Sprintf("nested%d", i), errOnce) }).Then(wg.Done) - go func() { - err, _ := <-errOnce.Err() - if err != nil { - t.Errorf("delegate %d: unexpected error: %v", i, err) - } - }() - } - wg.Wait() - <-p.End() - <-p.Done() -} - -func TestProcWithExceededActionQueueDepth(t *testing.T) { - config := defaultConfig - config.actionQueueDepth = 0 - p := newConfigured(config) - - errOnce := NewErrorOnce(p.Done()) - runDelegationTest(t, p, "nested", errOnce) - <-p.End() - - err, _ := <-errOnce.Err() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - <-p.Done() -} diff --git a/contrib/mesos/pkg/proc/state.go b/contrib/mesos/pkg/proc/state.go deleted file mode 100644 index a316c550214..00000000000 --- a/contrib/mesos/pkg/proc/state.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proc - -import ( - "sync/atomic" -) - -type stateType int32 - -const ( - stateNew stateType = iota - stateRunning - stateTerminal -) - -func (s *stateType) get() stateType { - return stateType(atomic.LoadInt32((*int32)(s))) -} - -func (s *stateType) transition(from, to stateType) bool { - return atomic.CompareAndSwapInt32((*int32)(s), int32(from), int32(to)) -} - -func (s *stateType) transitionTo(to stateType, unless ...stateType) bool { - if len(unless) == 0 { - atomic.StoreInt32((*int32)(s), int32(to)) - return true - } - for { - state := s.get() - for _, x := range unless { - if state == x { - return false - } - } - if s.transition(state, to) { - return true - } - } -} diff --git a/contrib/mesos/pkg/proc/types.go b/contrib/mesos/pkg/proc/types.go deleted file mode 100644 index 21ec4ff08ac..00000000000 --- a/contrib/mesos/pkg/proc/types.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proc - -import ( - "time" -) - -// Action is something that executes in the context of a process -type Action func() - -type Context interface { - // end (terminate) the execution context - End() <-chan struct{} - - // return a signal chan that will close upon the termination of this process - Done() <-chan struct{} -} - -type Doer interface { - // execute some action in some context. actions are to be executed in a - // concurrency-safe manner: no two actions should execute at the same time. - // errors are generated if the action cannot be executed (not by the execution - // of the action) and should be testable with the error API of this package, - // for example, IsProcessTerminated. - Do(Action) <-chan error -} - -// DoerFunc is an adapter func for Doer interface -type DoerFunc func(Action) <-chan error - -// invoke the f on action a. returns an illegal state error if f is nil. -func (f DoerFunc) Do(a Action) <-chan error { - if f != nil { - return f(a) - } - return ErrorChan(errIllegalState) -} - -type Process interface { - Context - Doer - - // see top level OnError func. this implementation will terminate upon the arrival of - // an error (and subsequently invoke the error handler, if given) or else the termination - // of the process (testable via IsProcessTerminated). - OnError(<-chan error, func(error)) <-chan struct{} - - // return a signal chan that will close once the process is ready to run actions - Running() <-chan struct{} -} - -// ErrorOnce an error promise. If we ever start building out support for other promise types it will probably -// make sense to group them in some sort of "promises" package. -type ErrorOnce interface { - // Err returns a chan that only ever sends one error, either obtained via Report() or Forward(). - Err() <-chan error - - // Report reports the given error via Err(), but only if no other errors have been reported or forwarded. - Report(error) - - // Report reports an error via Err(), but only if no other errors have been reported or forwarded, using - // fmt.Errorf to generate the error. - Reportf(string, ...interface{}) - - // forward waits for an error on the incoming chan, the result of which is later obtained via Err() (if no - // other errors have been reported or forwarded). - forward(<-chan error) - - // Send is non-blocking; it spins up a goroutine that reports an error (if any) that occurs on the error chan. - Send(<-chan error) ErrorOnce - - // WaitFor returns true if an error is received within the specified duration, otherwise false - WaitFor(time.Duration) (error, bool) -} diff --git a/contrib/mesos/pkg/profile/doc.go b/contrib/mesos/pkg/profile/doc.go deleted file mode 100644 index 77d1eaa5671..00000000000 --- a/contrib/mesos/pkg/profile/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package profile contains reusable code for profiling Go programs with pprof. -package profile // import "k8s.io/kubernetes/contrib/mesos/pkg/profile" diff --git a/contrib/mesos/pkg/profile/profile.go b/contrib/mesos/pkg/profile/profile.go deleted file mode 100644 index 08646656064..00000000000 --- a/contrib/mesos/pkg/profile/profile.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package profile - -import "net/http" -import "net/http/pprof" - -func InstallHandler(m *http.ServeMux) { - // register similar endpoints as net/http/pprof.init() does - m.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) - m.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) - m.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) -} diff --git a/contrib/mesos/pkg/queue/delay.go b/contrib/mesos/pkg/queue/delay.go deleted file mode 100644 index 7adbc37a1e2..00000000000 --- a/contrib/mesos/pkg/queue/delay.go +++ /dev/null @@ -1,409 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queue - -import ( - "container/heap" - "runtime" - "sync" - "time" - - "k8s.io/kubernetes/pkg/util/sets" -) - -type qitem struct { - value interface{} - priority Priority - index int - readd func(item *qitem) // re-add the value of the item to the queue -} - -// A priorityQueue implements heap.Interface and holds qitems. -type priorityQueue []*qitem - -func (pq priorityQueue) Len() int { return len(pq) } - -func (pq priorityQueue) Less(i, j int) bool { - return pq[i].priority.ts.Before(pq[j].priority.ts) -} - -func (pq priorityQueue) Swap(i, j int) { - pq[i], pq[j] = pq[j], pq[i] - pq[i].index = i - pq[j].index = j -} - -func (pq *priorityQueue) Push(x interface{}) { - n := len(*pq) - item := x.(*qitem) - item.index = n - *pq = append(*pq, item) -} - -func (pq *priorityQueue) Pop() interface{} { - old := *pq - n := len(old) - item := old[n-1] - item.index = -1 // for safety - *pq = old[0 : n-1] - return item -} - -// concurrency-safe, deadline-oriented queue that returns items after their -// delay period has expired. -type DelayQueue struct { - queue priorityQueue - lock sync.RWMutex - cond sync.Cond -} - -func NewDelayQueue() *DelayQueue { - q := &DelayQueue{} - q.cond.L = &q.lock - return q -} - -func (q *DelayQueue) Add(d Delayed) { - deadline := extractFromDelayed(d) - - q.lock.Lock() - defer q.lock.Unlock() - - // readd using the original deadline computed from the original delay - var readd func(*qitem) - readd = func(qp *qitem) { - q.lock.Lock() - defer q.lock.Unlock() - heap.Push(&q.queue, &qitem{ - value: d, - priority: deadline, - readd: readd, - }) - q.cond.Broadcast() - } - heap.Push(&q.queue, &qitem{ - value: d, - priority: deadline, - readd: readd, - }) - q.cond.Broadcast() -} - -// If there's a deadline reported by d.Deadline() then `d` is added to the -// queue and this func returns true. -func (q *DelayQueue) Offer(d Deadlined) bool { - deadline, ok := extractFromDeadlined(d) - if ok { - q.lock.Lock() - defer q.lock.Unlock() - heap.Push(&q.queue, &qitem{ - value: d, - priority: deadline, - readd: func(qp *qitem) { - q.Offer(qp.value.(Deadlined)) - }, - }) - q.cond.Broadcast() - } - return ok -} - -// wait for the delay of the next item in the queue to expire, blocking if -// there are no items in the queue. does not guarantee first-come-first-serve -// ordering with respect to clients. -func (q *DelayQueue) Pop() interface{} { - // doesn't implement cancellation, will always return a non-nil value - return q.pop(func() *qitem { - q.lock.Lock() - defer q.lock.Unlock() - for q.queue.Len() == 0 { - q.cond.Wait() - } - x := heap.Pop(&q.queue) - item := x.(*qitem) - return item - }, nil) -} - -func finishWaiting(cond *sync.Cond, waitFinished <-chan struct{}) { - runtime.Gosched() - select { - // avoid creating a timer if we can help it... - case <-waitFinished: - return - default: - const spinTimeout = 100 * time.Millisecond - t := time.NewTimer(spinTimeout) - defer t.Stop() - for { - runtime.Gosched() - cond.Broadcast() - select { - case <-waitFinished: - return - case <-t.C: - t.Reset(spinTimeout) - } - } - } -} - -// returns a non-nil value from the queue, or else nil if/when cancelled; if cancel -// is nil then cancellation is disabled and this func must return a non-nil value. -func (q *DelayQueue) pop(next func() *qitem, cancel <-chan struct{}) interface{} { - var ch chan struct{} - for { - item := next() - if item == nil { - // cancelled - return nil - } - x := item.value - waitingPeriod := item.priority.ts.Sub(time.Now()) - if waitingPeriod >= 0 { - // listen for calls to Add() while we're waiting for the deadline - if ch == nil { - ch = make(chan struct{}, 1) - } - go func() { - q.lock.Lock() - defer q.lock.Unlock() - q.cond.Wait() - ch <- struct{}{} - }() - select { - case <-cancel: - item.readd(item) - finishWaiting(&q.cond, ch) - return nil - case <-ch: - // we may no longer have the earliest deadline, re-try - item.readd(item) - continue - case <-time.After(waitingPeriod): - // noop - case <-item.priority.notify: - // noop - } - } - return x - } -} - -// If multiple adds/updates of a single item happen while an item is in the -// queue before it has been processed, it will only be processed once, and -// when it is processed, the most recent version will be processed. Items are -// popped in order of their priority, currently controlled by a delay or -// deadline assigned to each item in the queue. -type DelayFIFO struct { - // internal deadline-based priority queue - delegate *DelayQueue - // We depend on the property that items in the set are in the queue and vice versa. - items map[string]*qitem - deadlinePolicy DeadlinePolicy -} - -func (q *DelayFIFO) lock() { - q.delegate.lock.Lock() -} - -func (q *DelayFIFO) unlock() { - q.delegate.lock.Unlock() -} - -func (q *DelayFIFO) rlock() { - q.delegate.lock.RLock() -} - -func (q *DelayFIFO) runlock() { - q.delegate.lock.RUnlock() -} - -func (q *DelayFIFO) queue() *priorityQueue { - return &q.delegate.queue -} - -func (q *DelayFIFO) cond() *sync.Cond { - return &q.delegate.cond -} - -// Add inserts an item, and puts it in the queue. The item is only enqueued -// if it doesn't already exist in the set. -func (q *DelayFIFO) Add(d UniqueDelayed, rp ReplacementPolicy) { - deadline := extractFromDelayed(d) - id := d.GetUID() - var adder func(*qitem) - adder = func(*qitem) { - q.add(id, deadline, d, KeepExisting, adder) - } - q.add(id, deadline, d, rp, adder) -} - -func (q *DelayFIFO) Offer(d UniqueDeadlined, rp ReplacementPolicy) bool { - if deadline, ok := extractFromDeadlined(d); ok { - id := d.GetUID() - q.add(id, deadline, d, rp, func(qp *qitem) { q.Offer(qp.value.(UniqueDeadlined), KeepExisting) }) - return true - } - return false -} - -func (q *DelayFIFO) add(id string, deadline Priority, value interface{}, rp ReplacementPolicy, adder func(*qitem)) { - q.lock() - defer q.unlock() - if item, exists := q.items[id]; !exists { - item = &qitem{ - value: value, - priority: deadline, - readd: adder, - } - heap.Push(q.queue(), item) - q.items[id] = item - } else { - // this is an update of an existing item - item.value = rp.replacementValue(item.value, value) - item.priority = q.deadlinePolicy.nextDeadline(item.priority, deadline) - heap.Fix(q.queue(), item.index) - } - q.cond().Broadcast() -} - -// Delete removes an item. It doesn't add it to the queue, because -// this implementation assumes the consumer only cares about the objects, -// not their priority order. -func (f *DelayFIFO) Delete(id string) { - f.lock() - defer f.unlock() - delete(f.items, id) -} - -// List returns a list of all the items. -func (f *DelayFIFO) List() []UniqueID { - f.rlock() - defer f.runlock() - list := make([]UniqueID, 0, len(f.items)) - for _, item := range f.items { - list = append(list, item.value.(UniqueDelayed)) - } - return list -} - -// ContainedIDs returns a stringset.StringSet containing all IDs of the stored items. -// This is a snapshot of a moment in time, and one should keep in mind that -// other go routines can add or remove items after you call this. -func (c *DelayFIFO) ContainedIDs() sets.String { - c.rlock() - defer c.runlock() - set := sets.String{} - for id := range c.items { - set.Insert(id) - } - return set -} - -// Get returns the requested item, or sets exists=false. -func (f *DelayFIFO) Get(id string) (UniqueID, bool) { - f.rlock() - defer f.runlock() - if item, exists := f.items[id]; exists { - return item.value.(UniqueID), true - } - return nil, false -} - -// Variant of DelayQueue.Pop() for UniqueDelayed items -func (q *DelayFIFO) Await(timeout time.Duration) UniqueID { - var ( - cancel = make(chan struct{}) - ch = make(chan interface{}, 1) - t = time.NewTimer(timeout) - ) - defer t.Stop() - - go func() { ch <- q.pop(cancel) }() - - var x interface{} - select { - case <-t.C: - close(cancel) - x = <-ch - case x = <-ch: - // noop - } - if x != nil { - return x.(UniqueID) - } - return nil -} - -// Pop blocks until either there is an item available to dequeue or else the specified -// cancel chan is closed. Callers that have no interest in providing a cancel chan -// should specify nil, or else WithoutCancel() (for readability). -func (q *DelayFIFO) Pop(cancel <-chan struct{}) UniqueID { - x := q.pop(cancel) - if x == nil { - return nil - } - return x.(UniqueID) -} - -// variant of DelayQueue.Pop that implements optional cancellation -func (q *DelayFIFO) pop(cancel <-chan struct{}) interface{} { - next := func() *qitem { - q.lock() - defer q.unlock() - for { - for q.queue().Len() == 0 { - signal := make(chan struct{}) - go func() { - defer close(signal) - q.cond().Wait() - }() - select { - case <-cancel: - // we may not have the lock yet, so - // broadcast to abort Wait, then - // return after lock re-acquisition - finishWaiting(q.cond(), signal) - return nil - case <-signal: - // we have the lock, re-check - // the queue for data... - } - } - x := heap.Pop(q.queue()) - item := x.(*qitem) - unique := item.value.(UniqueID) - uid := unique.GetUID() - if _, ok := q.items[uid]; !ok { - // item was deleted, keep looking - continue - } - delete(q.items, uid) - return item - } - } - return q.delegate.pop(next, cancel) -} - -func NewDelayFIFO() *DelayFIFO { - f := &DelayFIFO{ - delegate: NewDelayQueue(), - items: map[string]*qitem{}, - } - return f -} diff --git a/contrib/mesos/pkg/queue/delay_test.go b/contrib/mesos/pkg/queue/delay_test.go deleted file mode 100644 index 6ecbdd1ace4..00000000000 --- a/contrib/mesos/pkg/queue/delay_test.go +++ /dev/null @@ -1,409 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queue - -import ( - "sync/atomic" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -const ( - tolerance = 100 * time.Millisecond // go time delays aren't perfect, this is our tolerance for errors WRT expected timeouts -) - -func timedPriority(t time.Time) Priority { - return Priority{ts: t} -} - -func TestPQ(t *testing.T) { - t.Parallel() - - var pq priorityQueue - if pq.Len() != 0 { - t.Fatalf("pq should be empty") - } - - now := timedPriority(time.Now()) - now2 := timedPriority(now.ts.Add(2 * time.Second)) - pq.Push(&qitem{priority: now2}) - if pq.Len() != 1 { - t.Fatalf("pq.len should be 1") - } - x := pq.Pop() - if x == nil { - t.Fatalf("x is nil") - } - if pq.Len() != 0 { - t.Fatalf("pq should be empty") - } - item := x.(*qitem) - if !item.priority.Equal(now2) { - t.Fatalf("item.priority != now2") - } - - pq.Push(&qitem{priority: now2}) - pq.Push(&qitem{priority: now2}) - pq.Push(&qitem{priority: now2}) - pq.Push(&qitem{priority: now2}) - pq.Push(&qitem{priority: now2}) - pq.Pop() - pq.Pop() - pq.Pop() - pq.Pop() - pq.Pop() - if pq.Len() != 0 { - t.Fatalf("pq should be empty") - } - now4 := timedPriority(now.ts.Add(4 * time.Second)) - now6 := timedPriority(now.ts.Add(4 * time.Second)) - pq.Push(&qitem{priority: now2}) - pq.Push(&qitem{priority: now4}) - pq.Push(&qitem{priority: now6}) - pq.Swap(0, 2) - if !pq[0].priority.Equal(now6) || !pq[2].priority.Equal(now2) { - t.Fatalf("swap failed") - } - if pq.Less(1, 2) { - t.Fatalf("now4 < now2") - } -} - -func TestPopEmptyPQ(t *testing.T) { - t.Parallel() - defer func() { - if r := recover(); r == nil { - t.Fatalf("Expected panic from popping an empty PQ") - } - }() - var pq priorityQueue - pq.Pop() -} - -type testjob struct { - d time.Duration - t time.Time - deadline *time.Time - uid string - instance int -} - -func (j *testjob) GetDelay() time.Duration { - return j.d -} - -func (j testjob) GetUID() string { - return j.uid -} - -func (td *testjob) Deadline() (deadline time.Time, ok bool) { - if td.deadline != nil { - return *td.deadline, true - } else { - return time.Now(), false - } -} - -func TestDQ_sanity_check(t *testing.T) { - t.Parallel() - - dq := NewDelayQueue() - delay := 2 * time.Second - - before := time.Now() - dq.Add(&testjob{d: delay}) - x := dq.Pop() - - now := time.Now() - waitPeriod := now.Sub(before) - - if waitPeriod+tolerance < delay { - t.Fatalf("delay too short: %v, expected: %v", waitPeriod, delay) - } - if x == nil { - t.Fatalf("x is nil") - } - item := x.(*testjob) - if item.d != delay { - t.Fatalf("d != delay") - } -} - -func TestDQ_Offer(t *testing.T) { - t.Parallel() - assert := assert.New(t) - - dq := NewDelayQueue() - delay := time.Second - - added := dq.Offer(&testjob{}) - if added { - t.Fatalf("DelayQueue should not add offered job without deadline") - } - - deadline := time.Now().Add(delay) - added = dq.Offer(&testjob{deadline: &deadline}) - if !added { - t.Fatalf("DelayQueue should add offered job with deadline") - } - - before := time.Now() - x := dq.Pop() - - now := time.Now() - waitPeriod := now.Sub(before) - - if waitPeriod+tolerance < delay { - t.Fatalf("delay too short: %v, expected: %v", waitPeriod, delay) - } - assert.NotNil(x) - assert.Equal(x.(*testjob).deadline, &deadline) -} - -func TestDQ_ordered_add_pop(t *testing.T) { - t.Parallel() - - dq := NewDelayQueue() - - before := time.Now() - dq.Add(&testjob{d: 2 * time.Second}) - dq.Add(&testjob{d: 1 * time.Second}) - dq.Add(&testjob{d: 3 * time.Second}) - - var finished [3]*testjob - idx := int32(-1) - ch := make(chan bool, 3) - //TODO: replace with `for range finished` once Go 1.3 support is dropped - for n := 0; n < len(finished); n++ { - go func() { - var ok bool - x := dq.Pop() - i := atomic.AddInt32(&idx, 1) - if finished[i], ok = x.(*testjob); !ok { - t.Fatalf("expected a *testjob, not %v", x) - } - finished[i].t = time.Now() - ch <- true - }() - } - <-ch - <-ch - <-ch - - after := time.Now() - totalDelay := after.Sub(before) - if totalDelay+tolerance < (3 * time.Second) { - t.Fatalf("totalDelay < 3s: %v", totalDelay) - } - for i, v := range finished { - if v == nil { - t.Fatalf("task %d was nil", i) - } - expected := time.Duration(i+1) * time.Second - if v.d != expected { - t.Fatalf("task %d had delay-priority %v, expected %v", i, v.d, expected) - } - actualDelay := v.t.Sub(before) - if actualDelay+tolerance < v.d { - t.Fatalf("task %d had actual-delay %v < expected delay %v", i, actualDelay, v.d) - } - } -} - -func TestDQ_always_pop_earliest_deadline(t *testing.T) { - t.Skip("disabled due to flakiness; see #11857") - t.Parallel() - - // add a testjob with delay of 2s - // spawn a func f1 that attempts to Pop() and wait for f1 to begin - // add a testjob with a delay of 1s - // check that the func f1 actually popped the 1s task (not the 2s task) - - dq := NewDelayQueue() - dq.Add(&testjob{d: 2 * time.Second}) - ch := make(chan *testjob) - started := make(chan bool) - - go func() { - started <- true - x := dq.Pop() - job := x.(*testjob) - job.t = time.Now() - ch <- job - }() - - <-started - time.Sleep(500 * time.Millisecond) // give plently of time for Pop() to enter - expected := 1 * time.Second - dq.Add(&testjob{d: expected}) - job := <-ch - - if expected != job.d { - t.Fatalf("Expected delay-prority of %v got instead got %v", expected, job.d) - } - - job = dq.Pop().(*testjob) - expected = 2 * time.Second - if expected != job.d { - t.Fatalf("Expected delay-prority of %v got instead got %v", expected, job.d) - } -} - -func TestDQ_always_pop_earliest_deadline_multi(t *testing.T) { - t.Skip("disabled due to flakiness; see #11821") - t.Parallel() - - dq := NewDelayQueue() - dq.Add(&testjob{d: 2 * time.Second}) - - ch := make(chan *testjob) - multi := 10 - started := make(chan bool, multi) - - go func() { - started <- true - for i := 0; i < multi; i++ { - x := dq.Pop() - job := x.(*testjob) - job.t = time.Now() - ch <- job - } - }() - - <-started - time.Sleep(500 * time.Millisecond) // give plently of time for Pop() to enter - expected := 1 * time.Second - - for i := 0; i < multi; i++ { - dq.Add(&testjob{d: expected}) - } - for i := 0; i < multi; i++ { - job := <-ch - if expected != job.d { - t.Fatalf("Expected delay-prority of %v got instead got %v", expected, job.d) - } - } - - job := dq.Pop().(*testjob) - expected = 2 * time.Second - if expected != job.d { - t.Fatalf("Expected delay-prority of %v got instead got %v", expected, job.d) - } -} - -func TestDQ_negative_delay(t *testing.T) { - t.Parallel() - - dq := NewDelayQueue() - delay := -2 * time.Second - dq.Add(&testjob{d: delay}) - - before := time.Now() - x := dq.Pop() - - now := time.Now() - waitPeriod := now.Sub(before) - - if waitPeriod > tolerance { - t.Fatalf("delay too long: %v, expected something less than: %v", waitPeriod, tolerance) - } - if x == nil { - t.Fatalf("x is nil") - } - item := x.(*testjob) - if item.d != delay { - t.Fatalf("d != delay") - } -} - -func TestDFIFO_sanity_check(t *testing.T) { - t.Parallel() - assert := assert.New(t) - - df := NewDelayFIFO() - delay := 2 * time.Second - df.Add(&testjob{d: delay, uid: "a", instance: 1}, ReplaceExisting) - assert.True(df.ContainedIDs().Has("a")) - - // re-add by ReplaceExisting - df.Add(&testjob{d: delay, uid: "a", instance: 2}, ReplaceExisting) - assert.True(df.ContainedIDs().Has("a")) - - a, ok := df.Get("a") - assert.True(ok) - assert.Equal(a.(*testjob).instance, 2) - - // re-add by KeepExisting - df.Add(&testjob{d: delay, uid: "a", instance: 3}, KeepExisting) - assert.True(df.ContainedIDs().Has("a")) - - a, ok = df.Get("a") - assert.True(ok) - assert.Equal(a.(*testjob).instance, 2) - - // pop last - before := time.Now() - x := df.Pop(WithoutCancel()) - assert.Equal(a.(*testjob).instance, 2) - - now := time.Now() - waitPeriod := now.Sub(before) - - if waitPeriod+tolerance < delay { - t.Fatalf("delay too short: %v, expected: %v", waitPeriod, delay) - } - if x == nil { - t.Fatalf("x is nil") - } - item := x.(*testjob) - if item.d != delay { - t.Fatalf("d != delay") - } -} - -func TestDFIFO_Offer(t *testing.T) { - t.Parallel() - assert := assert.New(t) - - dq := NewDelayFIFO() - delay := time.Second - - added := dq.Offer(&testjob{instance: 1}, ReplaceExisting) - if added { - t.Fatalf("DelayFIFO should not add offered job without deadline") - } - - deadline := time.Now().Add(delay) - added = dq.Offer(&testjob{deadline: &deadline, instance: 2}, ReplaceExisting) - if !added { - t.Fatalf("DelayFIFO should add offered job with deadline") - } - - before := time.Now() - x := dq.Pop(WithoutCancel()) - - now := time.Now() - waitPeriod := now.Sub(before) - - if waitPeriod+tolerance < delay { - t.Fatalf("delay too short: %v, expected: %v", waitPeriod, delay) - } - assert.NotNil(x) - assert.Equal(x.(*testjob).instance, 2) -} diff --git a/contrib/mesos/pkg/queue/doc.go b/contrib/mesos/pkg/queue/doc.go deleted file mode 100644 index 5494b799ecb..00000000000 --- a/contrib/mesos/pkg/queue/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package queue provides several queue implementations, originally -// inspired by Kubernetes pkg/client/cache/fifo. -package queue // import "k8s.io/kubernetes/contrib/mesos/pkg/queue" diff --git a/contrib/mesos/pkg/queue/historical.go b/contrib/mesos/pkg/queue/historical.go deleted file mode 100644 index f378aa2d9b1..00000000000 --- a/contrib/mesos/pkg/queue/historical.go +++ /dev/null @@ -1,406 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queue - -import ( - "fmt" - "sync" - "time" - - "k8s.io/kubernetes/pkg/util/sets" -) - -type entry struct { - value UniqueCopyable - event EventType -} - -type deletedEntry struct { - *entry - expiration time.Time -} - -func (e *entry) Value() UniqueCopyable { - return e.value -} - -func (e *entry) Copy() Copyable { - if e == nil { - return nil - } - return &entry{e.value.Copy().(UniqueCopyable), e.event} -} - -func (e *entry) Is(types EventType) bool { - return types&e.event != 0 -} - -func (e *deletedEntry) Copy() Copyable { - if e == nil { - return nil - } - return &deletedEntry{e.entry.Copy().(*entry), e.expiration} -} - -// deliver a message -type pigeon func(msg Entry) - -func dead(msg Entry) { - // intentionally blank -} - -// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for -// FIFO order processing. If multiple adds/updates of a single item happen while -// an item is in the queue before it has been processed, it will only be -// processed once, and when it is processed, the most recent version will be -// processed. This can't be done with a channel. -type HistoricalFIFO struct { - lock sync.RWMutex - cond sync.Cond - items map[string]Entry // We depend on the property that items in the queue are in the set. - queue []string - carrier pigeon // may be dead, but never nil - gcc int - lingerTTL time.Duration -} - -// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object -func checkType(obj interface{}) UniqueCopyable { - if v, ok := obj.(UniqueCopyable); !ok { - panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj)) - } else { - return v - } -} - -// Add inserts an item, and puts it in the queue. The item is only enqueued -// if it doesn't already exist in the set. -func (f *HistoricalFIFO) Add(v interface{}) error { - obj := checkType(v) - notifications := []Entry(nil) - defer func() { - for _, e := range notifications { - f.carrier(e) - } - }() - - f.lock.Lock() - defer f.lock.Unlock() - - id := obj.GetUID() - if entry, exists := f.items[id]; !exists { - f.queue = append(f.queue, id) - } else { - if entry.Is(DELETE_EVENT | POP_EVENT) { - f.queue = append(f.queue, id) - } - } - notifications = f.merge(id, obj) - f.cond.Broadcast() - return nil -} - -// Update is the same as Add in this implementation. -func (f *HistoricalFIFO) Update(obj interface{}) error { - return f.Add(obj) -} - -// Delete removes an item. It doesn't add it to the queue, because -// this implementation assumes the consumer only cares about the objects, -// not the order in which they were created/added. -func (f *HistoricalFIFO) Delete(v interface{}) error { - obj := checkType(v) - deleteEvent := (Entry)(nil) - defer func() { - f.carrier(deleteEvent) - }() - - f.lock.Lock() - defer f.lock.Unlock() - id := obj.GetUID() - item, exists := f.items[id] - if exists && !item.Is(DELETE_EVENT) { - e := item.(*entry) - e.event = DELETE_EVENT - deleteEvent = &deletedEntry{e, time.Now().Add(f.lingerTTL)} - f.items[id] = deleteEvent - } - return nil -} - -// List returns a list of all the items. -func (f *HistoricalFIFO) List() []interface{} { - f.lock.RLock() - defer f.lock.RUnlock() - - // TODO(jdef): slightly overallocates b/c of deleted items - list := make([]interface{}, 0, len(f.queue)) - - for _, entry := range f.items { - if entry.Is(DELETE_EVENT | POP_EVENT) { - continue - } - list = append(list, entry.Value().Copy()) - } - return list -} - -// List returns a list of all the items. -func (f *HistoricalFIFO) ListKeys() []string { - f.lock.RLock() - defer f.lock.RUnlock() - - // TODO(jdef): slightly overallocates b/c of deleted items - list := make([]string, 0, len(f.queue)) - - for key, entry := range f.items { - if entry.Is(DELETE_EVENT | POP_EVENT) { - continue - } - list = append(list, key) - } - return list -} - -// ContainedIDs returns a stringset.StringSet containing all IDs of the stored items. -// This is a snapshot of a moment in time, and one should keep in mind that -// other go routines can add or remove items after you call this. -func (c *HistoricalFIFO) ContainedIDs() sets.String { - c.lock.RLock() - defer c.lock.RUnlock() - set := sets.String{} - for id, entry := range c.items { - if entry.Is(DELETE_EVENT | POP_EVENT) { - continue - } - set.Insert(id) - } - return set -} - -// Get returns the requested item, or sets exists=false. -func (f *HistoricalFIFO) Get(v interface{}) (interface{}, bool, error) { - obj := checkType(v) - return f.GetByKey(obj.GetUID()) -} - -// Get returns the requested item, or sets exists=false. -func (f *HistoricalFIFO) GetByKey(id string) (interface{}, bool, error) { - f.lock.RLock() - defer f.lock.RUnlock() - entry, exists := f.items[id] - if exists && !entry.Is(DELETE_EVENT|POP_EVENT) { - return entry.Value().Copy(), true, nil - } - return nil, false, nil -} - -// Get returns the requested item, or sets exists=false. -func (f *HistoricalFIFO) Poll(id string, t EventType) bool { - f.lock.RLock() - defer f.lock.RUnlock() - entry, exists := f.items[id] - return exists && entry.Is(t) -} - -// Variant of DelayQueue.Pop() for UniqueDelayed items -func (q *HistoricalFIFO) Await(timeout time.Duration) interface{} { - var ( - cancel = make(chan struct{}) - ch = make(chan interface{}, 1) - t = time.NewTimer(timeout) - ) - defer t.Stop() - - go func() { ch <- q.Pop(cancel) }() - - select { - case <-t.C: - close(cancel) - return <-ch - case x := <-ch: - return x - } -} - -// Pop blocks until either there is an item available to dequeue or else the specified -// cancel chan is closed. Callers that have no interest in providing a cancel chan -// should specify nil, or else WithoutCancel() (for readability). -func (f *HistoricalFIFO) Pop(cancel <-chan struct{}) interface{} { - popEvent := (Entry)(nil) - defer func() { - f.carrier(popEvent) - }() - - f.lock.Lock() - defer f.lock.Unlock() - for { - for len(f.queue) == 0 { - signal := make(chan struct{}) - go func() { - defer close(signal) - f.cond.Wait() - }() - select { - case <-cancel: - // we may not have the lock yet, so - // broadcast to abort Wait, then - // return after lock re-acquisition - f.cond.Broadcast() - <-signal - return nil - case <-signal: - // we have the lock, re-check - // the queue for data... - } - } - id := f.queue[0] - f.queue = f.queue[1:] - item, ok := f.items[id] - if !ok || item.Is(DELETE_EVENT|POP_EVENT) { - // Item may have been deleted subsequently. - continue - } - value := item.Value() - popEvent = &entry{value, POP_EVENT} - f.items[id] = popEvent - return value.Copy() - } -} - -func (f *HistoricalFIFO) Replace(objs []interface{}, resourceVersion string) error { - notifications := make([]Entry, 0, len(objs)) - defer func() { - for _, e := range notifications { - f.carrier(e) - } - }() - - idToObj := make(map[string]interface{}) - for _, v := range objs { - obj := checkType(v) - idToObj[obj.GetUID()] = v - } - - f.lock.Lock() - defer f.lock.Unlock() - - f.queue = f.queue[:0] - now := time.Now() - for id, v := range f.items { - if _, exists := idToObj[id]; !exists && !v.Is(DELETE_EVENT) { - // a non-deleted entry in the items list that doesn't show up in the - // new list: mark it as deleted - ent := v.(*entry) - ent.event = DELETE_EVENT - e := &deletedEntry{ent, now.Add(f.lingerTTL)} - f.items[id] = e - notifications = append(notifications, e) - } - } - for id, v := range idToObj { - obj := checkType(v) - f.queue = append(f.queue, id) - n := f.merge(id, obj) - notifications = append(notifications, n...) - } - if len(f.queue) > 0 { - f.cond.Broadcast() - } - return nil -} - -// garbage collect DELETEd items whose TTL has expired; the IDs of such items are removed -// from the queue. This impl assumes that caller has acquired state lock. -func (f *HistoricalFIFO) gc() { - now := time.Now() - deleted := make(map[string]struct{}) - for id, v := range f.items { - if v.Is(DELETE_EVENT) { - ent := v.(*deletedEntry) - if ent.expiration.Before(now) { - delete(f.items, id) - deleted[id] = struct{}{} - } - } - } - // remove deleted items from the queue, will likely (slightly) overallocate here - queue := make([]string, 0, len(f.queue)) - for _, id := range f.queue { - if _, exists := deleted[id]; !exists { - queue = append(queue, id) - } - } - f.queue = queue -} - -// Assumes that the caller has acquired the state lock. -func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable) (notifications []Entry) { - item, exists := f.items[id] - if !exists || item.Is(POP_EVENT|DELETE_EVENT) { - // no prior history for this UID, or else it was popped/removed by the client. - e := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT} - f.items[id] = e - notifications = append(notifications, e) - } else if item.Value().GetUID() != obj.GetUID() { - // sanity check, please - panic(fmt.Sprintf("historical UID %q != current UID %v", item.Value().GetUID(), obj.GetUID())) - } else { - // exists && !(popped | deleted). so either the prior event was an add or an - // update. reflect.DeepEqual is expensive. it won't help us determine if - // we missed a hidden delete along the way. - e := &entry{obj.Copy().(UniqueCopyable), UPDATE_EVENT} - f.items[id] = e - notifications = append(notifications, e) - // else objects are the same, no work to do. - } - // check for garbage collection - f.gcc++ - if f.gcc%256 == 0 { //TODO(jdef): extract constant - f.gcc = 0 - f.gc() - } - return -} - -// Resync will touch all objects to put them into the processing queue -func (f *HistoricalFIFO) Resync() error { - // Nothing to do - return nil -} - -// NewHistorical returns a Store which can be used to queue up items to -// process. If a non-nil Mux is provided, then modifications to the -// the FIFO are delivered on a channel specific to this fifo. -func NewHistorical(ch chan<- Entry) *HistoricalFIFO { - carrier := dead - if ch != nil { - carrier = func(msg Entry) { - if msg != nil { - ch <- msg.Copy().(Entry) - } - } - } - f := &HistoricalFIFO{ - items: map[string]Entry{}, - queue: []string{}, - carrier: carrier, - lingerTTL: 5 * time.Minute, // TODO(jdef): extract constant - } - f.cond.L = &f.lock - return f -} diff --git a/contrib/mesos/pkg/queue/historical_test.go b/contrib/mesos/pkg/queue/historical_test.go deleted file mode 100644 index 181e6815563..00000000000 --- a/contrib/mesos/pkg/queue/historical_test.go +++ /dev/null @@ -1,217 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queue - -import ( - "fmt" - "testing" - "time" -) - -type _int int -type _uint uint - -func (i _int) Copy() Copyable { - return i -} - -func (i _int) GetUID() string { - return fmt.Sprintf("INT%d", int(i)) -} - -func (i _uint) Copy() Copyable { - return i -} - -func (i _uint) GetUID() string { - return fmt.Sprintf("UINT%d", uint64(i)) -} - -type testObj struct { - id string - value int -} - -func (i *testObj) Copy() Copyable { - if i == nil { - return nil - } else { - return &testObj{i.id, i.value} - } -} - -func (i *testObj) GetUID() string { - return i.id -} - -func TestFIFO_basic(t *testing.T) { - f := NewHistorical(nil) - const amount = 500 - go func() { - for i := 0; i < amount; i++ { - f.Add(_int(i + 1)) - } - }() - go func() { - for u := uint(0); u < amount; u++ { - f.Add(_uint(u + 1)) - } - }() - - lastInt := _int(0) - lastUint := _uint(0) - for i := 0; i < amount*2; i++ { - switch obj := f.Pop(WithoutCancel()).(type) { - case _int: - if obj <= lastInt { - t.Errorf("got %v (int) out of order, last was %v", obj, lastInt) - } - lastInt = obj - case _uint: - if obj <= lastUint { - t.Errorf("got %v (uint) out of order, last was %v", obj, lastUint) - } else { - lastUint = obj - } - default: - t.Fatalf("unexpected type %#v", obj) - } - } -} - -func TestFIFO_addUpdate(t *testing.T) { - f := NewHistorical(nil) - f.Add(&testObj{"foo", 10}) - f.Update(&testObj{"foo", 15}) - got := make(chan *testObj, 2) - go func() { - for { - got <- f.Pop(WithoutCancel()).(*testObj) - } - }() - - first := <-got - if e, a := 15, first.value; e != a { - t.Errorf("Didn't get updated value (%v), got %v", e, a) - } - select { - case unexpected := <-got: - t.Errorf("Got second value %v", unexpected) - case <-time.After(50 * time.Millisecond): - } - _, exists, _ := f.GetByKey("foo") - if exists { - t.Errorf("item did not get removed") - } -} - -func TestFIFO_addDeleteAdd(t *testing.T) { - f := NewHistorical(nil) - testobj := &testObj{"foo", 10} - f.Add(testobj) - f.Delete(testobj) - f.Add(testobj) - - _, exists, _ := f.GetByKey("foo") - if !exists { - t.Errorf("item did not get readded") - } -} - -func TestFIFO_addPopAdd(t *testing.T) { - f := NewHistorical(nil) - testobj := &testObj{"foo", 10} - f.Add(testobj) - f.Pop(nil) - f.Add(testobj) - - _, exists, _ := f.GetByKey("foo") - if !exists { - t.Errorf("item did not get readded") - } -} - -func TestFIFO_addReplace(t *testing.T) { - f := NewHistorical(nil) - f.Add(&testObj{"foo", 10}) - f.Replace([]interface{}{&testObj{"foo", 15}}, "0") - got := make(chan *testObj, 2) - go func() { - for { - got <- f.Pop(WithoutCancel()).(*testObj) - } - }() - - first := <-got - if e, a := 15, first.value; e != a { - t.Errorf("Didn't get updated value (%v), got %v", e, a) - } - select { - case unexpected := <-got: - t.Errorf("Got second value %v", unexpected) - case <-time.After(50 * time.Millisecond): - } - _, exists, _ := f.GetByKey("foo") - if exists { - t.Errorf("item did not get removed") - } -} - -func TestFIFO_detectLineJumpers(t *testing.T) { - f := NewHistorical(nil) - - f.Add(&testObj{"foo", 10}) - f.Add(&testObj{"bar", 1}) - f.Add(&testObj{"foo", 11}) - f.Add(&testObj{"foo", 13}) - f.Add(&testObj{"zab", 30}) - - err := error(nil) - done := make(chan struct{}) - go func() { - defer close(done) - if e, a := 13, f.Pop(WithoutCancel()).(*testObj).value; a != e { - err = fmt.Errorf("expected %d, got %d", e, a) - return - } - - f.Add(&testObj{"foo", 14}) // ensure foo doesn't jump back in line - - if e, a := 1, f.Pop(WithoutCancel()).(*testObj).value; a != e { - err = fmt.Errorf("expected %d, got %d", e, a) - return - } - - if e, a := 30, f.Pop(WithoutCancel()).(*testObj).value; a != e { - err = fmt.Errorf("expected %d, got %d", e, a) - return - } - - if e, a := 14, f.Pop(WithoutCancel()).(*testObj).value; a != e { - err = fmt.Errorf("expected %d, got %d", e, a) - return - } - }() - select { - case <-done: - if err != nil { - t.Fatal(err) - } - case <-time.After(1 * time.Second): - t.Fatal("Deadlocked unit test") - } -} diff --git a/contrib/mesos/pkg/queue/interface.go b/contrib/mesos/pkg/queue/interface.go deleted file mode 100644 index 27b317e8241..00000000000 --- a/contrib/mesos/pkg/queue/interface.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queue - -import ( - "time" - - "k8s.io/kubernetes/pkg/client/cache" -) - -type EventType int - -const ( - ADD_EVENT EventType = 1 << iota - UPDATE_EVENT - DELETE_EVENT - POP_EVENT -) - -type Entry interface { - Copyable - Value() UniqueCopyable - // types is a logically OR'd combination of EventType, e.g. ADD_EVENT|UPDATE_EVENT - Is(types EventType) bool -} - -type Copyable interface { - // return an independent copy (deep clone) of the current object - Copy() Copyable -} - -type UniqueID interface { - GetUID() string -} - -type UniqueCopyable interface { - Copyable - UniqueID -} - -type FIFO interface { - cache.Store - - // Pop waits until an item is ready and returns it. If multiple items are - // ready, they are returned in the order in which they were added/updated. - // The item is removed from the queue (and the store) before it is returned, - // so if you don't successfully process it, you need to add it back with Add(). - Pop(cancel <-chan struct{}) interface{} - - // Await attempts to Pop within the given interval; upon success the non-nil - // item is returned, otherwise nil - Await(timeout time.Duration) interface{} - - // Is there an entry for the id that matches the event mask? - Poll(id string, types EventType) bool -} - -type Delayed interface { - // return the remaining delay; a non-positive value indicates no delay - GetDelay() time.Duration -} - -type Deadlined interface { - // when ok, returns the time when this object should be activated/executed/evaluated - Deadline() (deadline time.Time, ok bool) -} - -// No objects are ever expected to be sent over this channel. References to BreakChan -// instances may be nil (always blocking). Signalling over this channel is performed by -// closing the channel. As such there can only ever be a single signal sent over the -// lifetime of the channel. -type BreakChan <-chan struct{} - -// an optional interface to be implemented by Delayed objects; returning a nil -// channel from Breaker() results in waiting the full delay duration -type Breakout interface { - // return a channel that signals early departure from a blocking delay - Breaker() BreakChan -} - -type UniqueDelayed interface { - UniqueID - Delayed -} - -type UniqueDeadlined interface { - UniqueID - Deadlined -} - -// WithoutCancel returns a chan that may never be closed and always blocks -func WithoutCancel() <-chan struct{} { return nil } diff --git a/contrib/mesos/pkg/queue/policy.go b/contrib/mesos/pkg/queue/policy.go deleted file mode 100644 index 7930666cab8..00000000000 --- a/contrib/mesos/pkg/queue/policy.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queue - -// Decide whether a pre-existing deadline for an item in a delay-queue should be -// updated if an attempt is made to offer/add a new deadline for said item. Whether -// the deadline changes or not has zero impact on the data blob associated with the -// entry in the queue. -type DeadlinePolicy int - -const ( - PreferLatest DeadlinePolicy = iota - PreferEarliest -) - -// Decide whether a pre-existing data blob in a delay-queue should be replaced if an -// an attempt is made to add/offer a new data blob in its place. Whether the data is -// replaced has no bearing on the deadline (priority) of the item in the queue. -type ReplacementPolicy int - -const ( - KeepExisting ReplacementPolicy = iota - ReplaceExisting -) - -func (rp ReplacementPolicy) replacementValue(original, replacement interface{}) (result interface{}) { - switch rp { - case KeepExisting: - result = original - case ReplaceExisting: - fallthrough - default: - result = replacement - } - return -} - -func (dp DeadlinePolicy) nextDeadline(a, b Priority) (result Priority) { - switch dp { - case PreferEarliest: - if a.ts.Before(b.ts) { - result = a - } else { - result = b - } - case PreferLatest: - fallthrough - default: - if a.ts.After(b.ts) { - result = a - } else { - result = b - } - } - return -} diff --git a/contrib/mesos/pkg/queue/priority.go b/contrib/mesos/pkg/queue/priority.go deleted file mode 100644 index 5d291234f2a..00000000000 --- a/contrib/mesos/pkg/queue/priority.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queue - -import ( - "time" -) - -type Priority struct { - ts time.Time // timestamp - notify BreakChan // notification channel -} - -func (p Priority) Equal(other Priority) bool { - return p.ts.Equal(other.ts) && p.notify == other.notify -} - -func extractFromDelayed(d Delayed) Priority { - deadline := time.Now().Add(d.GetDelay()) - breaker := BreakChan(nil) - if breakout, good := d.(Breakout); good { - breaker = breakout.Breaker() - } - return Priority{ - ts: deadline, - notify: breaker, - } -} - -func extractFromDeadlined(d Deadlined) (Priority, bool) { - if ts, ok := d.Deadline(); ok { - breaker := BreakChan(nil) - if breakout, good := d.(Breakout); good { - breaker = breakout.Breaker() - } - return Priority{ - ts: ts, - notify: breaker, - }, true - } - return Priority{}, false -} diff --git a/contrib/mesos/pkg/redirfd/doc.go b/contrib/mesos/pkg/redirfd/doc.go deleted file mode 100644 index 784782e5f56..00000000000 --- a/contrib/mesos/pkg/redirfd/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Some file descriptor manipulation funcs (Unix-Only), inspired by -// https://github.com/skarnet/execline/blob/master/src/execline/redirfd.c -package redirfd // import "k8s.io/kubernetes/contrib/mesos/pkg/redirfd" diff --git a/contrib/mesos/pkg/redirfd/file_descriptor.go b/contrib/mesos/pkg/redirfd/file_descriptor.go deleted file mode 100644 index ef7e9553694..00000000000 --- a/contrib/mesos/pkg/redirfd/file_descriptor.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package redirfd - -import ( - "fmt" - "strconv" -) - -// FileDescriptor mirrors unix-specific indexes for cross-platform use -type FileDescriptor int - -const ( - InvalidFD FileDescriptor = -1 - Stdin FileDescriptor = 0 - Stdout FileDescriptor = 1 - Stderr FileDescriptor = 2 -) - -// ParseFileDescriptor parses a string formatted file descriptor -func ParseFileDescriptor(fdstr string) (FileDescriptor, error) { - fdint, err := strconv.Atoi(fdstr) - if err != nil { - return InvalidFD, fmt.Errorf("file descriptor must be an integer: %q", fdstr) - } - return FileDescriptor(fdint), nil -} diff --git a/contrib/mesos/pkg/redirfd/file_descriptor_test.go b/contrib/mesos/pkg/redirfd/file_descriptor_test.go deleted file mode 100644 index 754202b05ed..00000000000 --- a/contrib/mesos/pkg/redirfd/file_descriptor_test.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package redirfd - -import ( - "testing" - - . "github.com/onsi/gomega" -) - -func TestParseFileDescriptor(t *testing.T) { - RegisterTestingT(t) - - valid := map[string]FileDescriptor{ - "-1": InvalidFD, - "0": Stdin, - "1": Stdout, - "2": Stderr, - "3": FileDescriptor(3), - } - - for input, expected := range valid { - fd, err := ParseFileDescriptor(input) - Expect(err).ToNot(HaveOccurred(), "Input: '%s'", input) - Expect(fd).To(Equal(expected), "Input: '%s'", input) - } - - invalid := []string{ - "a", - " 1", - "blue", - "stderr", - "STDERR", - } - - for _, input := range invalid { - _, err := ParseFileDescriptor(input) - Expect(err).To(HaveOccurred(), "Input: '%s'", input) - } -} diff --git a/contrib/mesos/pkg/redirfd/redirfd_unix.go b/contrib/mesos/pkg/redirfd/redirfd_unix.go deleted file mode 100644 index c39bb1ad6c4..00000000000 --- a/contrib/mesos/pkg/redirfd/redirfd_unix.go +++ /dev/null @@ -1,208 +0,0 @@ -// +build !windows - -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package redirfd - -import ( - "fmt" - "os" - "syscall" -) - -type RedirectMode int - -const ( - Read RedirectMode = iota // open file for reading - Write // open file for writing, truncating if it exists - Update // open file for read & write - Append // open file for append, create if it does not exist - AppendExisting // open file for append, do not create if it does not already exist - WriteNew // open file for writing, creating it, failing if it already exists -) - -// see https://github.com/skarnet/execline/blob/master/src/execline/redirfd.c -func (mode RedirectMode) Redirect(nonblock, changemode bool, fd FileDescriptor, name string) (*os.File, error) { - flags := 0 - what := -1 - - switch mode { - case Read: - what = syscall.O_RDONLY - flags &= ^(syscall.O_APPEND | syscall.O_CREAT | syscall.O_TRUNC | syscall.O_EXCL) - case Write: - what = syscall.O_WRONLY - flags |= syscall.O_CREAT | syscall.O_TRUNC - flags &= ^(syscall.O_APPEND | syscall.O_EXCL) - case Update: - what = syscall.O_RDWR - flags &= ^(syscall.O_APPEND | syscall.O_CREAT | syscall.O_TRUNC | syscall.O_EXCL) - case Append: - what = syscall.O_WRONLY - flags |= syscall.O_CREAT | syscall.O_APPEND - flags &= ^(syscall.O_TRUNC | syscall.O_EXCL) - case AppendExisting: - what = syscall.O_WRONLY - flags |= syscall.O_APPEND - flags &= ^(syscall.O_CREAT | syscall.O_TRUNC | syscall.O_EXCL) - case WriteNew: - what = syscall.O_WRONLY - flags |= syscall.O_CREAT | syscall.O_EXCL - flags &= ^(syscall.O_APPEND | syscall.O_TRUNC) - default: - return nil, fmt.Errorf("unexpected mode %d", mode) - } - if nonblock { - flags |= syscall.O_NONBLOCK - } - flags |= what - - fd2, e := open(name, flags, 0666) - if (what == syscall.O_WRONLY) && (e == syscall.ENXIO) { - // Opens file in read-only, non-blocking mode. Returns a valid fd number if it succeeds, or -1 (and sets errno) if it fails. - fdr, e2 := open(name, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) - if e2 != nil { - return nil, &os.PathError{Op: "open_read", Path: name, Err: e2} - } - fd2, e = open(name, flags, 0666) - fd_close(fdr) - } - if e != nil { - return nil, &os.PathError{Op: "open", Path: name, Err: e} - } - if e = fd_move(fd, fd2); e != nil { - return nil, &os.PathError{Op: "fd_move", Path: name, Err: e} - } - if changemode { - if nonblock { - e = ndelay_off(fd) - } else { - e = ndelay_on(fd) - } - if e != nil { - return nil, &os.PathError{Op: "ndelay", Path: name, Err: e} - } - } - return os.NewFile(uintptr(fd2), name), nil -} - -// proxy to return a FileDescriptor -func open(path string, openmode int, perm uint32) (FileDescriptor, error) { - fdint, err := syscall.Open(path, openmode, perm) - return FileDescriptor(fdint), err -} - -// see https://github.com/skarnet/skalibs/blob/master/src/libstddjb/fd_move.c -func fd_move(to, from FileDescriptor) (err error) { - if to == from { - return - } - for { - _, _, e1 := syscall.RawSyscall(syscall.SYS_DUP2, uintptr(from), uintptr(to), 0) - if e1 != syscall.EINTR { - if e1 != 0 { - err = e1 - } - break - } - } - if err != nil { - err = fd_close(from) - } - return - /* - do - r = dup2(from, to) ; - while ((r == -1) && (errno == EINTR)) ; - return (r == -1) ? -1 : fd_close(from) ; - */ -} - -// see https://github.com/skarnet/skalibs/blob/master/src/libstddjb/fd_close.c -func fd_close(fd FileDescriptor) (err error) { - i := 0 - var e error - for { - if e = syscall.Close(int(fd)); e != nil { - return nil - } - i++ - if e != syscall.EINTR { - break - } - } - if e == syscall.EBADF && i > 1 { - return nil - } - return e -} - -/* -int fd_close (int fd) -{ - register unsigned int i = 0 ; -doit: - if (!close(fd)) return 0 ; - i++ ; - if (errno == EINTR) goto doit ; - return ((errno == EBADF) && (i > 1)) ? 0 : -1 ; -} -*/ - -// see https://github.com/skarnet/skalibs/blob/master/src/libstddjb/ndelay_on.c -func ndelay_on(fd FileDescriptor) error { - // 32-bit will likely break because it needs SYS_FCNTL64 - got, _, e := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), uintptr(syscall.F_GETFL), 0) - if e != 0 { - return e - } - _, _, e = syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), uintptr(syscall.F_SETFL), uintptr(got|syscall.O_NONBLOCK)) - if e != 0 { - return e - } - return nil -} - -/* -int ndelay_on (int fd) -{ - register int got = fcntl(fd, F_GETFL) ; - return (got == -1) ? -1 : fcntl(fd, F_SETFL, got | O_NONBLOCK) ; -} -*/ - -// see https://github.com/skarnet/skalibs/blob/master/src/libstddjb/ndelay_off.c -func ndelay_off(fd FileDescriptor) error { - // 32-bit will likely break because it needs SYS_FCNTL64 - got, _, e := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), uintptr(syscall.F_GETFL), 0) - if e != 0 { - return e - } - _, _, e = syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), uintptr(syscall.F_SETFL), uintptr(int(got) & ^syscall.O_NONBLOCK)) - if e != 0 { - return e - } - return nil -} - -/* -int ndelay_off (int fd) -{ - register int got = fcntl(fd, F_GETFL) ; - return (got == -1) ? -1 : fcntl(fd, F_SETFL, got & ^O_NONBLOCK) ; -} -*/ diff --git a/contrib/mesos/pkg/redirfd/redirfd_windows.go b/contrib/mesos/pkg/redirfd/redirfd_windows.go deleted file mode 100644 index 8c6bb53fbaa..00000000000 --- a/contrib/mesos/pkg/redirfd/redirfd_windows.go +++ /dev/null @@ -1,39 +0,0 @@ -// +build windows - -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package redirfd - -import ( - "fmt" - "os" -) - -type RedirectMode int - -const ( - Read RedirectMode = iota // open file for reading - Write // open file for writing, truncating if it exists - Update // open file for read & write - Append // open file for append, create if it does not exist - AppendExisting // open file for append, do not create if it does not already exist - WriteNew // open file for writing, creating it, failing if it already exists -) - -func (mode RedirectMode) Redirect(nonblock, changemode bool, fd FileDescriptor, name string) (*os.File, error) { - return nil, fmt.Errorf("Redirect(%s, %s, %d, \"%s\") not supported on windows", nonblock, changemode, fd, name) -} diff --git a/contrib/mesos/pkg/runtime/doc.go b/contrib/mesos/pkg/runtime/doc.go deleted file mode 100644 index e65767f347a..00000000000 --- a/contrib/mesos/pkg/runtime/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package runtime provides utilities for semaphores (chan struct{}), -// a simple Latch implementation, and metrics for reporting handled panics. -package runtime // import "k8s.io/kubernetes/contrib/mesos/pkg/runtime" diff --git a/contrib/mesos/pkg/runtime/latch.go b/contrib/mesos/pkg/runtime/latch.go deleted file mode 100644 index 08a1d56836a..00000000000 --- a/contrib/mesos/pkg/runtime/latch.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package runtime - -import ( - "sync/atomic" -) - -type Latch struct { - int32 -} - -// return true if this latch was successfully acquired. concurrency safe. will only return true -// upon the first invocation, all subsequent invocations will return false. always returns false -// when self is nil. -func (self *Latch) Acquire() bool { - if self == nil { - return false - } - return atomic.CompareAndSwapInt32(&self.int32, 0, 1) -} diff --git a/contrib/mesos/pkg/runtime/latch_test.go b/contrib/mesos/pkg/runtime/latch_test.go deleted file mode 100644 index d9f7adec0a2..00000000000 --- a/contrib/mesos/pkg/runtime/latch_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package runtime - -import ( - "sync" - "sync/atomic" - "testing" - "time" -) - -func Test_LatchAcquireBasic(t *testing.T) { - var x Latch - if !x.Acquire() { - t.Fatalf("expected first acquire to succeed") - } - if x.Acquire() { - t.Fatalf("expected second acquire to fail") - } - if x.Acquire() { - t.Fatalf("expected third acquire to fail") - } -} - -func Test_LatchAcquireConcurrent(t *testing.T) { - var x Latch - const NUM = 10 - ch := make(chan struct{}) - var success int32 - var wg sync.WaitGroup - wg.Add(NUM) - for i := 0; i < NUM; i++ { - go func() { - defer wg.Done() - <-ch - if x.Acquire() { - atomic.AddInt32(&success, 1) - } - }() - } - time.Sleep(200 * time.Millisecond) - close(ch) - wg.Wait() - if success != 1 { - t.Fatalf("expected single acquire to succeed instead of %d", success) - } -} diff --git a/contrib/mesos/pkg/runtime/metrics.go b/contrib/mesos/pkg/runtime/metrics.go deleted file mode 100644 index e0ceb02e8b6..00000000000 --- a/contrib/mesos/pkg/runtime/metrics.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package runtime - -import ( - "sync" - - "github.com/prometheus/client_golang/prometheus" - "k8s.io/kubernetes/pkg/util/runtime" -) - -const ( - runtimeSubsystem = "mesos_runtime" -) - -var ( - panicCounter = prometheus.NewCounter( - prometheus.CounterOpts{ - Subsystem: runtimeSubsystem, - Name: "panics", - Help: "Counter of panics handled by the internal crash handler.", - }, - ) -) - -var registerMetrics sync.Once - -func Register() { - registerMetrics.Do(func() { - prometheus.MustRegister(panicCounter) - runtime.PanicHandlers = append(runtime.PanicHandlers, func(interface{}) { panicCounter.Inc() }) - }) -} diff --git a/contrib/mesos/pkg/runtime/util.go b/contrib/mesos/pkg/runtime/util.go deleted file mode 100644 index b401a24490b..00000000000 --- a/contrib/mesos/pkg/runtime/util.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package runtime - -import ( - "os" - "sync" - "time" - - "k8s.io/kubernetes/pkg/util/runtime" -) - -type Signal <-chan struct{} - -// return a func that will close the signal chan. -// multiple invocations of the returned func will not generate a panic. -// two funcs from separate invocations of Closer() (on the same sig chan) will cause a panic if both invoked. -// for example: -// // good -// x := runtime.After(func() { ... }) -// f := x.Closer() -// f() -// f() -// -// // bad -// x := runtime.After(func() { ... }) -// f := x.Closer() -// g := x.Closer() -// f() -// g() // this will panic -func Closer(sig chan<- struct{}) func() { - var once sync.Once - return func() { - once.Do(func() { close(sig) }) - } -} - -// upon receiving signal sig invoke function f and immediately return a signal -// that indicates f's completion. used to chain handler funcs, for example: -// On(job.Done(), response.Send).Then(wg.Done) -func (sig Signal) Then(f func()) Signal { - if sig == nil { - return nil - } - return On(sig, f) -} - -// execute a callback function after the specified signal chan closes. -// immediately returns a signal that indicates f's completion. -func On(sig <-chan struct{}, f func()) Signal { - if sig == nil { - return nil - } - return After(func() { - <-sig - if f != nil { - f() - } - }) -} - -func OnOSSignal(sig <-chan os.Signal, f func(os.Signal)) Signal { - if sig == nil { - return nil - } - return After(func() { - if s, ok := <-sig; ok && f != nil { - f(s) - } - }) -} - -// spawn a goroutine to execute a func, immediately returns a chan that closes -// upon completion of the func. returns a nil signal chan if the given func is nil. -func After(f func()) Signal { - ch := make(chan struct{}) - go func() { - defer close(ch) - defer runtime.HandleCrash() - if f != nil { - f() - } - }() - return Signal(ch) -} - -// periodically execute the given function, stopping once stopCh is closed. -// this func blocks until stopCh is closed, it's intended to be run as a goroutine. -func Until(f func(), period time.Duration, stopCh <-chan struct{}) { - if f == nil { - return - } - for { - select { - case <-stopCh: - return - default: - } - func() { - defer runtime.HandleCrash() - f() - }() - select { - case <-stopCh: - case <-time.After(period): - } - } -} diff --git a/contrib/mesos/pkg/runtime/util_test.go b/contrib/mesos/pkg/runtime/util_test.go deleted file mode 100644 index 177accf4dcf..00000000000 --- a/contrib/mesos/pkg/runtime/util_test.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package runtime - -import ( - "testing" - "time" -) - -func TestUntil(t *testing.T) { - ch := make(chan struct{}) - close(ch) - Until(func() { - t.Fatal("should not have been invoked") - }, 0, ch) - - //-- - ch = make(chan struct{}) - called := make(chan struct{}) - After(func() { - Until(func() { - called <- struct{}{} - }, 0, ch) - }).Then(func() { close(called) }) - - <-called - close(ch) - - // wait for 'called' to be closed - for { - if _, ok := <-called; !ok { - break - } - } - - //-- - ch = make(chan struct{}) - called2 := make(chan struct{}) - running := make(chan struct{}) - After(func() { - Until(func() { - close(running) - called2 <- struct{}{} - }, 2*time.Second, ch) - }).Then(func() { close(called2) }) - - <-running - close(ch) - <-called2 // unblock the goroutine - now := time.Now() - - // wait for 'called2' to be closed - for { - if _, ok := <-called2; !ok { - break - } - } - - if time.Since(now) > 1800*time.Millisecond { - t.Fatalf("Until should not have waited the full timeout period since we closed the stop chan") - } -} diff --git a/contrib/mesos/pkg/scheduler/components/algorithm/algorithm.go b/contrib/mesos/pkg/scheduler/components/algorithm/algorithm.go deleted file mode 100644 index cb60fdd0cb6..00000000000 --- a/contrib/mesos/pkg/scheduler/components/algorithm/algorithm.go +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package algorithm - -import ( - "fmt" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/client/cache" -) - -// SchedulerAlgorithm is the interface that orchestrates the pod scheduling. -// -// Schedule implements the Scheduler interface of Kubernetes. -// It returns the selectedMachine's hostname or an error if the schedule failed. -type SchedulerAlgorithm interface { - Schedule(pod *api.Pod) (string, error) -} - -// SchedulerAlgorithm implements the algorithm.ScheduleAlgorithm interface -type schedulerAlgorithm struct { - sched scheduler.Scheduler - podUpdates queue.FIFO - podScheduler podschedulers.PodScheduler - taskConfig podtask.Config - defaultCpus resources.CPUShares - defaultMem resources.MegaBytes -} - -// New returns a new SchedulerAlgorithm -// TODO(sur): refactor params to separate config object -func New( - sched scheduler.Scheduler, - podUpdates queue.FIFO, - podScheduler podschedulers.PodScheduler, - taskConfig podtask.Config, - defaultCpus resources.CPUShares, - defaultMem resources.MegaBytes, -) SchedulerAlgorithm { - return &schedulerAlgorithm{ - sched: sched, - podUpdates: podUpdates, - podScheduler: podScheduler, - taskConfig: taskConfig, - defaultCpus: defaultCpus, - defaultMem: defaultMem, - } -} - -func (k *schedulerAlgorithm) Schedule(pod *api.Pod) (string, error) { - log.Infof("Try to schedule pod %v\n", pod.Name) - ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace) - - // default upstream scheduler passes pod.Name as binding.PodID - podKey, err := podtask.MakePodKey(ctx, pod.Name) - if err != nil { - return "", err - } - - k.sched.Lock() - defer k.sched.Unlock() - - switch task, state := k.sched.Tasks().ForPod(podKey); state { - case podtask.StateUnknown: - // There's a bit of a potential race here, a pod could have been yielded() and - // then before we get *here* it could be deleted. - // We use meta to index the pod in the store since that's what k8s reflector does. - podName, err := cache.MetaNamespaceKeyFunc(pod) - if err != nil { - log.Warningf("aborting Schedule, unable to understand pod object %+v", pod) - return "", errors.NoSuchPodErr - } - - if deleted := k.podUpdates.Poll(podName, queue.DELETE_EVENT); deleted { - // avoid scheduling a pod that's been deleted between yieldPod() and Schedule() - log.Infof("aborting Schedule, pod has been deleted %+v", pod) - return "", errors.NoSuchPodErr - } - - // write resource limits into the pod spec. - // From here on we can expect that the pod spec of a task has proper limits for CPU and memory. - k.limitPod(pod) - - podTask, err := podtask.New(ctx, k.taskConfig, pod) - if err != nil { - log.Warningf("aborting Schedule, unable to create podtask object %+v: %v", pod, err) - return "", err - } - - podTask, err = k.sched.Tasks().Register(podTask) - if err != nil { - return "", err - } - - return k.doSchedule(podTask) - - //TODO(jdef) it's possible that the pod state has diverged from what - //we knew previously, we should probably update the task.Pod state here - //before proceeding with scheduling - case podtask.StatePending: - if pod.UID != task.Pod.UID { - // we're dealing with a brand new pod spec here, so the old one must have been - // deleted -- and so our task store is out of sync w/ respect to reality - //TODO(jdef) reconcile task - return "", fmt.Errorf("task %v spec is out of sync with pod %v spec, aborting schedule", task.ID, pod.Name) - } else if task.Has(podtask.Launched) { - // task has been marked as "launched" but the pod binding creation may have failed in k8s, - // but we're going to let someone else handle it, probably the mesos task error handler - return "", fmt.Errorf("task %s has already been launched, aborting schedule", task.ID) - } else { - return k.doSchedule(task) - } - - default: - return "", fmt.Errorf("task %s is not pending, nothing to schedule", task.ID) - } -} - -// limitPod limits the given pod based on the scheduler's default limits. -func (k *schedulerAlgorithm) limitPod(pod *api.Pod) error { - cpuRequest, cpuLimit, _, err := resources.LimitPodCPU(pod, k.defaultCpus) - if err != nil { - return err - } - - memRequest, memLimit, _, err := resources.LimitPodMem(pod, k.defaultMem) - if err != nil { - return err - } - - log.V(3).Infof( - "setting pod %s/%s resources: requested cpu %.2f mem %.2f MB, limited cpu %.2f mem %.2f MB", - pod.Namespace, pod.Name, cpuRequest, memRequest, cpuLimit, memLimit, - ) - - return nil -} - -// doSchedule implements the actual scheduling of the given pod task. -// It checks whether the offer has been accepted and is still present in the offer registry. -// It delegates to the actual pod scheduler and updates the task registry. -func (k *schedulerAlgorithm) doSchedule(task *podtask.T) (string, error) { - var offer offers.Perishable - var err error - - if task.HasAcceptedOffer() { - // verify that the offer is still on the table - var ok bool - offer, ok = k.sched.Offers().Get(task.GetOfferId()) - - if !ok || offer.HasExpired() { - task.Offer.Release() - task.Reset() - if err = k.sched.Tasks().Update(task); err != nil { - return "", err - } - } - } - - var spec *podtask.Spec - if offer == nil { - offer, spec, err = k.podScheduler.SchedulePod(k.sched.Offers(), task) - } - - if err != nil { - return "", err - } - - details := offer.Details() - if details == nil { - return "", fmt.Errorf("offer already invalid/expired for task %v", task.ID) - } - - if task.Offer != nil && task.Offer != offer { - return "", fmt.Errorf("task.offer assignment must be idempotent, task %+v: offer %+v", task, offer) - } - - task.Offer = offer - task.Spec = spec - - if err := k.sched.Tasks().Update(task); err != nil { - offer.Release() - return "", err - } - - return details.GetHostname(), nil -} diff --git a/contrib/mesos/pkg/scheduler/components/algorithm/doc.go b/contrib/mesos/pkg/scheduler/components/algorithm/doc.go deleted file mode 100644 index 7f805e59f9f..00000000000 --- a/contrib/mesos/pkg/scheduler/components/algorithm/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package algorithm implements the SchedulerAlgorithm -package algorithm // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/algorithm" diff --git a/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/doc.go b/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/doc.go deleted file mode 100644 index 99f0fa5beb4..00000000000 --- a/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package podschedulers defines an interface (w/ implementations) for matching -// pods against offers. -package podschedulers // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers" diff --git a/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/fcfs.go b/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/fcfs.go deleted file mode 100644 index 7fea90a1e1f..00000000000 --- a/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/fcfs.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podschedulers - -import ( - "fmt" - - log "github.com/golang/glog" - - "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/contrib/mesos/pkg/node" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/pkg/api" -) - -type fcfsPodScheduler struct { - procurement podtask.Procurement - lookupNode node.LookupFunc -} - -func NewFCFSPodScheduler(pr podtask.Procurement, lookupNode node.LookupFunc) PodScheduler { - return &fcfsPodScheduler{pr, lookupNode} -} - -// A first-come-first-serve scheduler: acquires the first offer that can support the task -func (fps *fcfsPodScheduler) SchedulePod(r offers.Registry, task *podtask.T) (offers.Perishable, *podtask.Spec, error) { - podName := fmt.Sprintf("%s/%s", task.Pod.Namespace, task.Pod.Name) - var matchingOffer offers.Perishable - var acceptedSpec *podtask.Spec - err := r.Walk(func(p offers.Perishable) (bool, error) { - offer := p.Details() - if offer == nil { - return false, fmt.Errorf("nil offer while scheduling task %v", task.ID) - } - - // check that the node actually exists. As offers are declined if not, the - // case n==nil can only happen when the node object was deleted since the - // offer came in. - nodeName := offer.GetHostname() - n := fps.lookupNode(nodeName) - if n == nil { - log.V(3).Infof("ignoring offer for node %s because node went away", nodeName) - return false, nil - } - - ps := podtask.NewProcureState(offer) - err := fps.procurement.Procure(task, n, ps) - if err != nil { - log.V(5).Infof( - "Offer %q does not fit pod %s/%s: %v", - offer.Id, task.Pod.Namespace, task.Pod.Name, err, - ) - return false, nil // continue - } - - if !p.Acquire() { - log.V(2).Infof( - "Could not acquire offer %q for pod %s/%s", - offer.Id, task.Pod.Namespace, task.Pod.Name, - ) - return false, nil // continue - } - - matchingOffer = p - acceptedSpec, _ = ps.Result() - log.V(3).Infof("Pod %s accepted offer %v", podName, offer.Id.GetValue()) - return true, nil // stop, we found an offer - }) - if matchingOffer != nil { - if err != nil { - log.Warningf("problems walking the offer registry: %v, attempting to continue", err) - } - return matchingOffer, acceptedSpec, nil - } - if err != nil { - log.V(2).Infof("failed to find a fit for pod: %s, err = %v", podName, err) - return nil, nil, err - } - log.V(2).Infof("failed to find a fit for pod: %s", podName) - return nil, nil, errors.NoSuitableOffersErr -} - -func (fps *fcfsPodScheduler) Fit(t *podtask.T, offer *mesosproto.Offer, n *api.Node) bool { - return fps.procurement.Procure(t, n, podtask.NewProcureState(offer)) == nil -} diff --git a/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/types.go b/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/types.go deleted file mode 100644 index b03bd3cbb4a..00000000000 --- a/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers/types.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podschedulers - -import ( - "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/pkg/api" -) - -// SchedulePod is the interface which schedules pods. -// There can be different implementation for different scheduling policies. -// -// SchedulePod accepts a set of offers and a single pod task, which aligns well -// with the k8s scheduling algorithm. It returns an offer that is acceptable -// for the pod, else nil. The caller is responsible for filling in task -// state w/ relevant offer details. -// -// See the FCFSPodScheduler for example. -// -// Fit checks whether a given podtask can be scheduled for the given offer on the given node. -type PodScheduler interface { - SchedulePod(r offers.Registry, task *podtask.T) (offers.Perishable, *podtask.Spec, error) - - Fit(*podtask.T, *mesosproto.Offer, *api.Node) bool -} diff --git a/contrib/mesos/pkg/scheduler/components/binder/binder.go b/contrib/mesos/pkg/scheduler/components/binder/binder.go deleted file mode 100644 index 35dcc3d52c0..00000000000 --- a/contrib/mesos/pkg/scheduler/components/binder/binder.go +++ /dev/null @@ -1,162 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package binder - -import ( - "fmt" - "strconv" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - annotation "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/runtime" -) - -type Binder interface { - Bind(binding *api.Binding) error -} - -type binder struct { - sched scheduler.Scheduler -} - -func New(sched scheduler.Scheduler) Binder { - return &binder{ - sched: sched, - } -} - -// implements binding.Registry, launches the pod-associated-task in mesos -func (b *binder) Bind(binding *api.Binding) error { - - ctx := api.WithNamespace(api.NewContext(), binding.Namespace) - - // default upstream scheduler passes pod.Name as binding.Name - podKey, err := podtask.MakePodKey(ctx, binding.Name) - if err != nil { - return err - } - - b.sched.Lock() - defer b.sched.Unlock() - - switch task, state := b.sched.Tasks().ForPod(podKey); state { - case podtask.StatePending: - return b.bind(ctx, binding, task) - default: - // in this case it's likely that the pod has been deleted between Schedule - // and Bind calls - log.Infof("No pending task for pod %s", podKey) - return errors.NoSuchPodErr //TODO(jdef) this error is somewhat misleading since the task could be running?! - } -} - -func (b *binder) rollback(task *podtask.T, err error) error { - task.Offer.Release() - task.Reset() - if err2 := b.sched.Tasks().Update(task); err2 != nil { - log.Errorf("failed to update pod task: %v", err2) - } - return err -} - -// assumes that: caller has acquired scheduler lock and that the task is still pending -// -// bind does not actually do the binding itself, but launches the pod as a Mesos task. The -// kubernetes executor on the slave will finally do the binding. This is different from the -// upstream scheduler in the sense that the upstream scheduler does the binding and the -// kubelet will notice that and launches the pod. -func (b *binder) bind(ctx api.Context, binding *api.Binding, task *podtask.T) (err error) { - // sanity check: ensure that the task hasAcceptedOffer(), it's possible that between - // Schedule() and now that the offer for this task was rescinded or invalidated. - // ((we should never see this here)) - if !task.HasAcceptedOffer() { - return fmt.Errorf("task has not accepted a valid offer %v", task.ID) - } - - // By this time, there is a chance that the slave is disconnected. - offerId := task.GetOfferId() - if offer, ok := b.sched.Offers().Get(offerId); !ok || offer.HasExpired() { - // already rescinded or timed out or otherwise invalidated - return b.rollback(task, fmt.Errorf("failed prior to launchTask due to expired offer for task %v", task.ID)) - } - - if err = b.prepareTaskForLaunch(ctx, binding.Target.Name, task, offerId); err == nil { - log.V(2).Infof( - "launching task: %q on target %q slave %q for pod \"%v/%v\", resources %v", - task.ID, binding.Target.Name, task.Spec.SlaveID, task.Pod.Namespace, task.Pod.Name, task.Spec.Resources, - ) - - if err = b.sched.LaunchTask(task); err == nil { - b.sched.Offers().Invalidate(offerId) - task.Set(podtask.Launched) - if err = b.sched.Tasks().Update(task); err != nil { - // this should only happen if the task has been removed or has changed status, - // which SHOULD NOT HAPPEN as long as we're synchronizing correctly - log.Errorf("failed to update task w/ Launched status: %v", err) - } - return - } - } - return b.rollback(task, fmt.Errorf("Failed to launch task %v: %v", task.ID, err)) -} - -//TODO(jdef) unit test this, ensure that task's copy of api.Pod is not modified -func (b *binder) prepareTaskForLaunch(ctx api.Context, machine string, task *podtask.T, offerId string) error { - pod := task.Pod - - // we make an effort here to avoid making changes to the task's copy of the pod, since - // we want that to reflect the initial user spec, and not the modified spec that we - // build for the executor to consume. - oemCt := pod.Spec.Containers - pod.Spec.Containers = append([]api.Container{}, oemCt...) // (shallow) clone before mod - - if pod.Annotations == nil { - pod.Annotations = make(map[string]string) - } - - task.SaveRecoveryInfo(pod.Annotations) - pod.Annotations[annotation.BindingHostKey] = task.Spec.AssignedSlave - - for _, entry := range task.Spec.PortMap { - oemPorts := pod.Spec.Containers[entry.ContainerIdx].Ports - ports := append([]api.ContainerPort{}, oemPorts...) - p := &ports[entry.PortIdx] - p.HostPort = int32(entry.OfferPort) - op := strconv.FormatUint(entry.OfferPort, 10) - pod.Annotations[fmt.Sprintf(annotation.PortMappingKeyFormat, p.Protocol, p.ContainerPort)] = op - if p.Name != "" { - pod.Annotations[fmt.Sprintf(annotation.PortNameMappingKeyFormat, p.Protocol, p.Name)] = op - } - pod.Spec.Containers[entry.ContainerIdx].Ports = ports - } - - // the kubelet-executor uses this to instantiate the pod - log.V(3).Infof("prepared pod spec: %+v", pod) - - data, err := runtime.Encode(api.Codecs.LegacyCodec(v1.SchemeGroupVersion), &pod) - if err != nil { - log.V(2).Infof("Failed to marshal the pod spec: %v", err) - return err - } - task.Spec.Data = data - return nil -} diff --git a/contrib/mesos/pkg/scheduler/components/binder/doc.go b/contrib/mesos/pkg/scheduler/components/binder/doc.go deleted file mode 100644 index 57d96264548..00000000000 --- a/contrib/mesos/pkg/scheduler/components/binder/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package binder implements the Binder which launched a task and let the -// executor do the actual binding. -package binder // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/binder" diff --git a/contrib/mesos/pkg/scheduler/components/controller/controller.go b/contrib/mesos/pkg/scheduler/components/controller/controller.go deleted file mode 100644 index 0b871e55871..00000000000 --- a/contrib/mesos/pkg/scheduler/components/controller/controller.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "time" - - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/algorithm" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/binder" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/client/record" -) - -const ( - recoveryDelay = 100 * time.Millisecond // delay after scheduler plugin crashes, before we resume scheduling - - FailedScheduling = "FailedScheduling" - Scheduled = "Scheduled" -) - -type Controller interface { - Run(<-chan struct{}) -} - -type controller struct { - algorithm algorithm.SchedulerAlgorithm - binder binder.Binder - nextPod func() *api.Pod - error func(*api.Pod, error) - recorder record.EventRecorder - client *clientset.Clientset - started chan<- struct{} // startup latch -} - -func New(client *clientset.Clientset, algorithm algorithm.SchedulerAlgorithm, - recorder record.EventRecorder, nextPod func() *api.Pod, error func(pod *api.Pod, schedulingErr error), - binder binder.Binder, started chan<- struct{}) Controller { - return &controller{ - algorithm: algorithm, - binder: binder, - nextPod: nextPod, - error: error, - recorder: recorder, - client: client, - started: started, - } -} - -func (s *controller) Run(done <-chan struct{}) { - defer close(s.started) - go runtime.Until(s.scheduleOne, recoveryDelay, done) -} - -// hacked from GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/scheduler.go, -// with the Modeler stuff removed since we don't use it because we have mesos. -func (s *controller) scheduleOne() { - pod := s.nextPod() - - // pods which are pre-scheduled (i.e. NodeName is set) are deleted by the kubelet - // in upstream. Not so in Mesos because the kubelet hasn't see that pod yet. Hence, - // the scheduler has to take care of this: - if pod.Spec.NodeName != "" && pod.DeletionTimestamp != nil { - log.V(3).Infof("deleting pre-scheduled, not yet running pod: %s/%s", pod.Namespace, pod.Name) - s.client.Core().Pods(pod.Namespace).Delete(pod.Name, api.NewDeleteOptions(0)) - return - } - - log.V(3).Infof("Attempting to schedule: %+v", pod) - dest, err := s.algorithm.Schedule(pod) - if err != nil { - log.V(1).Infof("Failed to schedule: %+v", pod) - s.recorder.Eventf(pod, api.EventTypeWarning, FailedScheduling, "Error scheduling: %v", err) - s.error(pod, err) - return - } - b := &api.Binding{ - ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name}, - Target: api.ObjectReference{ - Kind: "Node", - Name: dest, - }, - } - if err := s.binder.Bind(b); err != nil { - log.V(1).Infof("Failed to bind pod: %+v", err) - s.recorder.Eventf(pod, api.EventTypeWarning, FailedScheduling, "Binding rejected: %v", err) - s.error(pod, err) - return - } - s.recorder.Eventf(pod, api.EventTypeNormal, Scheduled, "Successfully assigned %v to %v", pod.Name, dest) -} diff --git a/contrib/mesos/pkg/scheduler/components/controller/doc.go b/contrib/mesos/pkg/scheduler/components/controller/doc.go deleted file mode 100644 index 9e75d47bb35..00000000000 --- a/contrib/mesos/pkg/scheduler/components/controller/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller implements the scheduling controller which waits for pod -// events from the queuer (i.e. from the apiserver), passes them to the -// SchedulerAlgorithm and in case of success to the binder which does the launch. -package controller // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/controller" diff --git a/contrib/mesos/pkg/scheduler/components/deleter/deleter.go b/contrib/mesos/pkg/scheduler/components/deleter/deleter.go deleted file mode 100644 index e8b509cce17..00000000000 --- a/contrib/mesos/pkg/scheduler/components/deleter/deleter.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package deleter - -import ( - "time" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/queuer" - "k8s.io/kubernetes/pkg/api" -) - -type Deleter interface { - Run(updates <-chan queue.Entry, done <-chan struct{}) - DeleteOne(pod *queuer.Pod) error -} - -type deleter struct { - sched scheduler.Scheduler - qr queuer.Queuer -} - -func New(sched scheduler.Scheduler, qr queuer.Queuer) Deleter { - return &deleter{ - sched: sched, - qr: qr, - } -} - -// currently monitors for "pod deleted" events, upon which handle() -// is invoked. -func (k *deleter) Run(updates <-chan queue.Entry, done <-chan struct{}) { - go runtime.Until(func() { - for { - entry := <-updates - pod := entry.Value().(*queuer.Pod) - if entry.Is(queue.DELETE_EVENT) { - if err := k.DeleteOne(pod); err != nil { - log.Error(err) - } - } else if !entry.Is(queue.POP_EVENT) { - k.qr.UpdatesAvailable() - } - } - }, 1*time.Second, done) -} - -func (k *deleter) DeleteOne(pod *queuer.Pod) error { - ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace) - podKey, err := podtask.MakePodKey(ctx, pod.Name) - if err != nil { - return err - } - - log.V(2).Infof("pod deleted: %v", podKey) - - // order is important here: we want to make sure we have the lock before - // removing the pod from the scheduling queue. this makes the concurrent - // execution of scheduler-error-handling and delete-handling easier to - // reason about. - k.sched.Lock() - defer k.sched.Unlock() - - // prevent the scheduler from attempting to pop this; it's also possible that - // it's concurrently being scheduled (somewhere between pod scheduling and - // binding) - if so, then we'll end up removing it from taskRegistry which - // will abort Bind()ing - k.qr.Dequeue(pod.GetUID()) - - switch task, state := k.sched.Tasks().ForPod(podKey); state { - case podtask.StateUnknown: - log.V(2).Infof("Could not resolve pod '%s' to task id", podKey) - return errors.NoSuchPodErr - - // determine if the task has already been launched to mesos, if not then - // cleanup is easier (unregister) since there's no state to sync - case podtask.StatePending: - if !task.Has(podtask.Launched) { - // we've been invoked in between Schedule() and Bind() - if task.HasAcceptedOffer() { - task.Offer.Release() - task.Reset() - task.Set(podtask.Deleted) - //TODO(jdef) probably want better handling here - if err := k.sched.Tasks().Update(task); err != nil { - return err - } - } - k.sched.Tasks().Unregister(task) - return nil - } - fallthrough - - case podtask.StateRunning: - // signal to watchers that the related pod is going down - task.Set(podtask.Deleted) - if err := k.sched.Tasks().Update(task); err != nil { - log.Errorf("failed to update task w/ Deleted status: %v", err) - } - return k.sched.KillTask(task.ID) - - default: - log.Infof("cannot kill pod '%s': non-terminal task not found %v", podKey, task.ID) - return errors.NoSuchTaskErr - } -} diff --git a/contrib/mesos/pkg/scheduler/components/deleter/deleter_test.go b/contrib/mesos/pkg/scheduler/components/deleter/deleter_test.go deleted file mode 100644 index 4663adcf59c..00000000000 --- a/contrib/mesos/pkg/scheduler/components/deleter/deleter_test.go +++ /dev/null @@ -1,178 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package deleter - -import ( - "testing" - - "github.com/mesos/mesos-go/mesosproto" - "github.com/stretchr/testify/assert" - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - types "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask/hostport" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/queuer" - "k8s.io/kubernetes/pkg/api" -) - -func TestDeleteOne_NonexistentPod(t *testing.T) { - assert := assert.New(t) - obj := &types.MockScheduler{} - reg := podtask.NewInMemoryRegistry() - obj.On("Tasks").Return(reg) - - q := queue.NewDelayFIFO() - qr := queuer.New(q, nil) - assert.Equal(0, len(q.List())) - d := New(obj, qr) - pod := &queuer.Pod{Pod: &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "foo", - Namespace: api.NamespaceDefault, - }}} - err := d.DeleteOne(pod) - assert.Equal(err, errors.NoSuchPodErr) - obj.AssertExpectations(t) -} - -func TestDeleteOne_PendingPod(t *testing.T) { - assert := assert.New(t) - obj := &types.MockScheduler{} - reg := podtask.NewInMemoryRegistry() - obj.On("Tasks").Return(reg) - - pod := &queuer.Pod{Pod: &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "foo", - UID: "foo0", - Namespace: api.NamespaceDefault, - }}} - task, err := podtask.New( - api.NewDefaultContext(), - podtask.Config{ - ID: "bar", - Prototype: &mesosproto.ExecutorInfo{}, - HostPortStrategy: hostport.StrategyWildcard, - }, - pod.Pod, - ) - if err != nil { - t.Fatalf("failed to create task: %v", err) - } - - _, err = reg.Register(task) - if err != nil { - t.Fatalf("failed to register task: %v", err) - } - - // preconditions - q := queue.NewDelayFIFO() - qr := queuer.New(q, nil) - q.Add(pod, queue.ReplaceExisting) - assert.Equal(1, len(q.List())) - _, found := q.Get("default/foo") - assert.True(found) - - // exec & post conditions - d := New(obj, qr) - err = d.DeleteOne(pod) - assert.Nil(err) - _, found = q.Get("foo0") - assert.False(found) - assert.Equal(0, len(q.List())) - obj.AssertExpectations(t) -} - -func TestDeleteOne_Running(t *testing.T) { - assert := assert.New(t) - obj := &types.MockScheduler{} - reg := podtask.NewInMemoryRegistry() - obj.On("Tasks").Return(reg) - - pod := &queuer.Pod{Pod: &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "foo", - UID: "foo0", - Namespace: api.NamespaceDefault, - }}} - task, err := podtask.New( - api.NewDefaultContext(), - podtask.Config{ - ID: "bar", - Prototype: &mesosproto.ExecutorInfo{}, - HostPortStrategy: hostport.StrategyWildcard, - }, - pod.Pod, - ) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - task, err = reg.Register(task) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - task.Set(podtask.Launched) - err = reg.Update(task) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // preconditions - q := queue.NewDelayFIFO() - qr := queuer.New(q, nil) - q.Add(pod, queue.ReplaceExisting) - assert.Equal(1, len(q.List())) - _, found := q.Get("default/foo") - assert.True(found) - - obj.On("KillTask", task.ID).Return(nil) - - // exec & post conditions - d := New(obj, qr) - err = d.DeleteOne(pod) - assert.Nil(err) - _, found = q.Get("foo0") - assert.False(found) - assert.Equal(0, len(q.List())) - obj.AssertExpectations(t) -} - -func TestDeleteOne_badPodNaming(t *testing.T) { - assert := assert.New(t) - obj := &types.MockScheduler{} - pod := &queuer.Pod{Pod: &api.Pod{}} - q := queue.NewDelayFIFO() - qr := queuer.New(q, nil) - d := New(obj, qr) - - err := d.DeleteOne(pod) - assert.NotNil(err) - - pod.Pod.ObjectMeta.Name = "foo" - err = d.DeleteOne(pod) - assert.NotNil(err) - - pod.Pod.ObjectMeta.Name = "" - pod.Pod.ObjectMeta.Namespace = "bar" - err = d.DeleteOne(pod) - assert.NotNil(err) - - obj.AssertExpectations(t) -} diff --git a/contrib/mesos/pkg/scheduler/components/deleter/doc.go b/contrib/mesos/pkg/scheduler/components/deleter/doc.go deleted file mode 100644 index 045457e68c5..00000000000 --- a/contrib/mesos/pkg/scheduler/components/deleter/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package deleter implements the deleter which listens for pod DELETE events -// from the apiserver and kills tasks for deleted pods. -package deleter // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/deleter" diff --git a/contrib/mesos/pkg/scheduler/components/doc.go b/contrib/mesos/pkg/scheduler/components/doc.go deleted file mode 100644 index 38aa33f93ca..00000000000 --- a/contrib/mesos/pkg/scheduler/components/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package components implements independent aspects of the scheduler which -// do not use Framework or Scheduler internals, but rely solely on the Scheduler -// interface. -package components // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components" diff --git a/contrib/mesos/pkg/scheduler/components/errorhandler/doc.go b/contrib/mesos/pkg/scheduler/components/errorhandler/doc.go deleted file mode 100644 index 42c8c91ccda..00000000000 --- a/contrib/mesos/pkg/scheduler/components/errorhandler/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package errorhandler implements the ErrorHandler which handles scheduer error -// and possibly requeue pods for scheduling again. -package errorhandler // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/errorhandler" diff --git a/contrib/mesos/pkg/scheduler/components/errorhandler/errorhandler.go b/contrib/mesos/pkg/scheduler/components/errorhandler/errorhandler.go deleted file mode 100644 index eabd74b91cb..00000000000 --- a/contrib/mesos/pkg/scheduler/components/errorhandler/errorhandler.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package errorhandler - -import ( - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/backoff" - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/queuer" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/util/runtime" -) - -type ErrorHandler interface { - Error(pod *api.Pod, schedulingErr error) -} - -type errorHandler struct { - sched scheduler.Scheduler - backoff *backoff.Backoff - qr queuer.Queuer - newBreakChan func(podKey string) queue.BreakChan -} - -func New(sched scheduler.Scheduler, backoff *backoff.Backoff, qr queuer.Queuer, newBC func(podKey string) queue.BreakChan) ErrorHandler { - return &errorHandler{ - sched: sched, - backoff: backoff, - qr: qr, - newBreakChan: newBC, - } -} - -// implementation of scheduling plugin's Error func; see plugin/pkg/scheduler -func (k *errorHandler) Error(pod *api.Pod, schedulingErr error) { - - if schedulingErr == errors.NoSuchPodErr { - log.V(2).Infof("Not rescheduling non-existent pod %v", pod.Name) - return - } - - log.Infof("Error scheduling %v: %v; retrying", pod.Name, schedulingErr) - defer runtime.HandleCrash() - - // default upstream scheduler passes pod.Name as binding.PodID - ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace) - podKey, err := podtask.MakePodKey(ctx, pod.Name) - if err != nil { - log.Errorf("Failed to construct pod key, aborting scheduling for pod %v: %v", pod.Name, err) - return - } - - k.backoff.GC() - k.sched.Lock() - defer k.sched.Unlock() - - switch task, state := k.sched.Tasks().ForPod(podKey); state { - case podtask.StateUnknown: - // if we don't have a mapping here any more then someone deleted the pod - log.V(2).Infof("Could not resolve pod to task, aborting pod reschdule: %s", podKey) - return - - case podtask.StatePending: - if task.Has(podtask.Launched) { - log.V(2).Infof("Skipping re-scheduling for already-launched pod %v", podKey) - return - } - breakoutEarly := queue.BreakChan(nil) - if schedulingErr == errors.NoSuitableOffersErr { - log.V(3).Infof("adding backoff breakout handler for pod %v", podKey) - breakoutEarly = k.newBreakChan(podKey) - } - delay := k.backoff.Get(podKey) - log.V(3).Infof("requeuing pod %v with delay %v", podKey, delay) - k.qr.Requeue(queuer.NewPod(pod, queuer.Delay(delay), queuer.Notify(breakoutEarly))) - - default: - log.V(2).Infof("Task is no longer pending, aborting reschedule for pod %v", podKey) - } -} diff --git a/contrib/mesos/pkg/scheduler/components/framework/doc.go b/contrib/mesos/pkg/scheduler/components/framework/doc.go deleted file mode 100644 index 659534d05b7..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package framework implements the Mesos scheduler. -package framework // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework" diff --git a/contrib/mesos/pkg/scheduler/components/framework/driver_mock.go b/contrib/mesos/pkg/scheduler/components/framework/driver_mock.go deleted file mode 100644 index cf6da1a91c6..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/driver_mock.go +++ /dev/null @@ -1,165 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - mesos "github.com/mesos/mesos-go/mesosproto" - "github.com/stretchr/testify/mock" -) - -/*----------------------------------------------------------------------------- - | - | this really belongs in the mesos-go package, but that's being updated soon - | any way so just keep it here for now unless we *really* need it there. - | - \----------------------------------------------------------------------------- - -// Scheduler defines the interfaces that needed to be implemented. -type Scheduler interface { - Registered(SchedulerDriver, *FrameworkID, *MasterInfo) - Reregistered(SchedulerDriver, *MasterInfo) - Disconnected(SchedulerDriver) - ResourceOffers(SchedulerDriver, []*Offer) - OfferRescinded(SchedulerDriver, *OfferID) - StatusUpdate(SchedulerDriver, *TaskStatus) - FrameworkMessage(SchedulerDriver, *ExecutorID, *SlaveID, string) - SlaveLost(SchedulerDriver, *SlaveID) - ExecutorLost(SchedulerDriver, *ExecutorID, *SlaveID, int) - Error(SchedulerDriver, string) -} -*/ - -func status(args mock.Arguments, at int) (val mesos.Status) { - if x := args.Get(at); x != nil { - val = x.(mesos.Status) - } - return -} - -type extendedMock struct { - mock.Mock -} - -// Upon returns a chan that closes upon the execution of the most recently registered call. -func (m *extendedMock) Upon() <-chan struct{} { - // TODO(jdef) this isn't thread safe, should make it so - ch := make(chan struct{}) - call := m.ExpectedCalls[len(m.ExpectedCalls)-1] - f := call.RunFn - call.RunFn = func(args mock.Arguments) { - defer close(ch) - if f != nil { - f(args) - } - } - return ch -} - -type MockSchedulerDriver struct { - extendedMock -} - -func (m *MockSchedulerDriver) Init() error { - args := m.Called() - return args.Error(0) -} - -func (m *MockSchedulerDriver) Start() (mesos.Status, error) { - args := m.Called() - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) Stop(b bool) (mesos.Status, error) { - args := m.Called(b) - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) Abort() (mesos.Status, error) { - args := m.Called() - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) Join() (mesos.Status, error) { - args := m.Called() - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) Run() (mesos.Status, error) { - args := m.Called() - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) RequestResources(r []*mesos.Request) (mesos.Status, error) { - args := m.Called(r) - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) AcceptOffers(ids []*mesos.OfferID, ops []*mesos.Offer_Operation, f *mesos.Filters) (mesos.Status, error) { - args := m.Called(ids, ops, f) - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) ReconcileTasks(statuses []*mesos.TaskStatus) (mesos.Status, error) { - args := m.Called(statuses) - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) LaunchTasks(offerIds []*mesos.OfferID, ti []*mesos.TaskInfo, f *mesos.Filters) (mesos.Status, error) { - args := m.Called(offerIds, ti, f) - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) KillTask(tid *mesos.TaskID) (mesos.Status, error) { - args := m.Called(tid) - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) DeclineOffer(oid *mesos.OfferID, f *mesos.Filters) (mesos.Status, error) { - args := m.Called(oid, f) - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) ReviveOffers() (mesos.Status, error) { - args := m.Called() - return status(args, 0), args.Error(0) -} - -func (m *MockSchedulerDriver) SendFrameworkMessage(eid *mesos.ExecutorID, sid *mesos.SlaveID, s string) (mesos.Status, error) { - args := m.Called(eid, sid, s) - return status(args, 0), args.Error(1) -} - -func (m *MockSchedulerDriver) Destroy() { - m.Called() -} - -func (m *MockSchedulerDriver) Wait() { - m.Called() -} - -type JoinableDriver struct { - MockSchedulerDriver - joinFunc func() (mesos.Status, error) -} - -// Join invokes joinFunc if it has been set, otherwise blocks forever -func (m *JoinableDriver) Join() (mesos.Status, error) { - if m.joinFunc != nil { - return m.joinFunc() - } - select {} -} diff --git a/contrib/mesos/pkg/scheduler/components/framework/framework.go b/contrib/mesos/pkg/scheduler/components/framework/framework.go deleted file mode 100644 index 36c8e118370..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/framework.go +++ /dev/null @@ -1,825 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "fmt" - "io" - "math" - "net/http" - "sync" - "time" - - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - mutil "github.com/mesos/mesos-go/mesosutil" - bindings "github.com/mesos/mesos-go/scheduler" - "golang.org/x/net/context" - "k8s.io/kubernetes/contrib/mesos/pkg/executor/messages" - "k8s.io/kubernetes/contrib/mesos/pkg/node" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - offermetrics "k8s.io/kubernetes/contrib/mesos/pkg/offers/metrics" - "k8s.io/kubernetes/contrib/mesos/pkg/proc" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework/frameworkid" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/tasksreconciler" - schedcfg "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/config" - merrors "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/metrics" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/errors" - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - "k8s.io/kubernetes/pkg/kubelet/container" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - "k8s.io/kubernetes/pkg/util/sets" -) - -type Framework interface { - bindings.Scheduler - - Init(sched scheduler.Scheduler, electedMaster proc.Process, mux *http.ServeMux) error - Registration() <-chan struct{} - Offers() offers.Registry - LaunchTask(t *podtask.T) error - KillTask(id string) error -} - -type framework struct { - // We use a lock here to avoid races - // between invoking the mesos callback - *sync.RWMutex - - // Config related, write-once - sched scheduler.Scheduler - schedulerConfig *schedcfg.Config - client *clientset.Clientset - failoverTimeout float64 // in seconds - reconcileInterval int64 - nodeRegistrator node.Registrator - storeFrameworkId frameworkid.StoreFunc - lookupNode node.LookupFunc - executorId *mesos.ExecutorID - - // Mesos context - driver bindings.SchedulerDriver // late initialization - frameworkId *mesos.FrameworkID - masterInfo *mesos.MasterInfo - registered bool - registration chan struct{} // signal chan that closes upon first successful registration - onRegistration sync.Once - offers offers.Registry - slaveHostNames *slaveRegistry - - // via deferred init - tasksReconciler taskreconciler.TasksReconciler - mux *http.ServeMux - reconcileCooldown time.Duration - asRegisteredMaster proc.Doer - terminate <-chan struct{} // signal chan, closes when we should kill background tasks -} - -type Config struct { - SchedulerConfig schedcfg.Config - ExecutorId *mesos.ExecutorID - Client *clientset.Clientset - StoreFrameworkId frameworkid.StoreFunc - FailoverTimeout float64 - ReconcileInterval int64 - ReconcileCooldown time.Duration - LookupNode node.LookupFunc -} - -// New creates a new Framework -func New(config Config) Framework { - var k *framework - k = &framework{ - schedulerConfig: &config.SchedulerConfig, - RWMutex: new(sync.RWMutex), - client: config.Client, - failoverTimeout: config.FailoverTimeout, - reconcileInterval: config.ReconcileInterval, - nodeRegistrator: node.NewRegistrator(config.Client, config.LookupNode), - executorId: config.ExecutorId, - offers: offers.CreateRegistry(offers.RegistryConfig{ - Compat: func(o *mesos.Offer) bool { - // the node must be registered and have up-to-date labels - n := config.LookupNode(o.GetHostname()) - if n == nil || !node.IsUpToDate(n, node.SlaveAttributesToLabels(o.GetAttributes())) { - if n == nil { - log.V(1).Infof("cannot find node %v", o.GetHostname()) - } else { - log.V(1).Infof("node %v's attributes do not match: %v != %v", - o.GetHostname(), n.Labels, o.GetAttributes()) - } - return false - } - - eids := len(o.GetExecutorIds()) - switch { - case eids > 1: - // at most one executor id expected. More than one means that - // the given node is seriously in trouble. - log.V(1).Infof("at most one executor id is expected, but got %v (%v)", eids, o.GetExecutorIds()) - return false - - case eids == 1: - // the executor id must match, otherwise the running executor - // is incompatible with the current scheduler configuration. - if eid := o.GetExecutorIds()[0]; eid.GetValue() != config.ExecutorId.GetValue() { - log.V(1).Infof("executor ids do not match: %v != %v", eid.GetValue(), config.ExecutorId.GetValue()) - return false - } - } - - return true - }, - DeclineOffer: func(id string) <-chan error { - errOnce := proc.NewErrorOnce(k.terminate) - errOuter := k.asRegisteredMaster.Do(func() { - var err error - defer errOnce.Report(err) - offerId := mutil.NewOfferID(id) - filters := &mesos.Filters{} - _, err = k.driver.DeclineOffer(offerId, filters) - }) - return errOnce.Send(errOuter).Err() - }, - // remember expired offers so that we can tell if a previously scheduler offer relies on one - LingerTTL: config.SchedulerConfig.OfferLingerTTL.Duration, - TTL: config.SchedulerConfig.OfferTTL.Duration, - ListenerDelay: config.SchedulerConfig.ListenerDelay.Duration, - }), - slaveHostNames: newSlaveRegistry(), - reconcileCooldown: config.ReconcileCooldown, - registration: make(chan struct{}), - asRegisteredMaster: proc.DoerFunc(func(proc.Action) <-chan error { - return proc.ErrorChanf("cannot execute action with unregistered scheduler") - }), - storeFrameworkId: config.StoreFrameworkId, - lookupNode: config.LookupNode, - } - return k -} - -func (k *framework) Init(sched scheduler.Scheduler, electedMaster proc.Process, mux *http.ServeMux) error { - log.V(1).Infoln("initializing kubernetes mesos scheduler") - - k.sched = sched - k.mux = mux - k.asRegisteredMaster = proc.DoerFunc(func(a proc.Action) <-chan error { - if !k.registered { - return proc.ErrorChanf("failed to execute action, scheduler is disconnected") - } - return electedMaster.Do(a) - }) - k.terminate = electedMaster.Done() - k.offers.Init(k.terminate) - k.nodeRegistrator.Run(k.terminate) - return k.recoverTasks() -} - -func (k *framework) asMaster() proc.Doer { - k.RLock() - defer k.RUnlock() - return k.asRegisteredMaster -} - -// An executorRef holds a reference to an executor and the slave it is running on -type executorRef struct { - executorID *mesos.ExecutorID - slaveID *mesos.SlaveID -} - -// executorRefs returns a slice of known references to running executors known to this framework -func (k *framework) executorRefs() []executorRef { - slaves := k.slaveHostNames.SlaveIDs() - refs := make([]executorRef, 0, len(slaves)) - - for _, slaveID := range slaves { - hostname := k.slaveHostNames.HostName(slaveID) - if hostname == "" { - log.Warningf("hostname lookup for slaveID %q failed", slaveID) - continue - } - - node := k.lookupNode(hostname) - if node == nil { - log.Warningf("node lookup for slaveID %q failed", slaveID) - continue - } - - eid, ok := node.Annotations[meta.ExecutorIdKey] - if !ok { - log.Warningf("unable to find %q annotation for node %v", meta.ExecutorIdKey, node) - continue - } - - refs = append(refs, executorRef{ - executorID: mutil.NewExecutorID(eid), - slaveID: mutil.NewSlaveID(slaveID), - }) - } - - return refs -} - -func (k *framework) installDebugHandlers(mux *http.ServeMux) { - wrappedHandler := func(uri string, h http.Handler) { - mux.HandleFunc(uri, func(w http.ResponseWriter, r *http.Request) { - ch := make(chan struct{}) - closer := runtime.Closer(ch) - proc.OnError(k.asMaster().Do(func() { - defer closer() - h.ServeHTTP(w, r) - }), func(err error) { - defer closer() - log.Warningf("failed HTTP request for %s: %v", uri, err) - w.WriteHeader(http.StatusServiceUnavailable) - }, k.terminate) - select { - case <-time.After(k.schedulerConfig.HttpHandlerTimeout.Duration): - log.Warningf("timed out waiting for request to be processed") - w.WriteHeader(http.StatusServiceUnavailable) - return - case <-ch: // noop - } - }) - } - - requestReconciliation := func(uri string, requestAction func()) { - wrappedHandler(uri, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestAction() - w.WriteHeader(http.StatusNoContent) - })) - } - requestReconciliation("/debug/actions/requestExplicit", k.tasksReconciler.RequestExplicit) - requestReconciliation("/debug/actions/requestImplicit", k.tasksReconciler.RequestImplicit) - - wrappedHandler("/debug/actions/kamikaze", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - refs := k.executorRefs() - - for _, ref := range refs { - _, err := k.driver.SendFrameworkMessage( - ref.executorID, - ref.slaveID, - messages.Kamikaze, - ) - - if err != nil { - msg := fmt.Sprintf( - "error sending kamikaze message to executor %q on slave %q: %v", - ref.executorID.GetValue(), - ref.slaveID.GetValue(), - err, - ) - log.Warning(msg) - fmt.Fprintln(w, msg) - continue - } - - io.WriteString(w, fmt.Sprintf( - "kamikaze message sent to executor %q on slave %q\n", - ref.executorID.GetValue(), - ref.slaveID.GetValue(), - )) - } - - io.WriteString(w, "OK") - })) -} - -func (k *framework) Registration() <-chan struct{} { - return k.registration -} - -// Registered is called when the scheduler registered with the master successfully. -func (k *framework) Registered(drv bindings.SchedulerDriver, fid *mesos.FrameworkID, mi *mesos.MasterInfo) { - log.Infof("Scheduler registered with the master: %v with frameworkId: %v\n", mi, fid) - - k.driver = drv - k.frameworkId = fid - k.masterInfo = mi - k.registered = true - - k.onRegistration.Do(func() { k.onInitialRegistration(drv) }) - k.tasksReconciler.RequestExplicit() -} - -// Reregistered is called when the scheduler re-registered with the master successfully. -// This happens when the master fails over. -func (k *framework) Reregistered(drv bindings.SchedulerDriver, mi *mesos.MasterInfo) { - log.Infof("Scheduler reregistered with the master: %v\n", mi) - - k.driver = drv - k.masterInfo = mi - k.registered = true - - k.onRegistration.Do(func() { k.onInitialRegistration(drv) }) - k.tasksReconciler.RequestExplicit() -} - -// perform one-time initialization actions upon the first registration event received from Mesos. -func (k *framework) onInitialRegistration(driver bindings.SchedulerDriver) { - defer close(k.registration) - - if k.failoverTimeout > 0 { - refreshInterval := k.schedulerConfig.FrameworkIdRefreshInterval.Duration - if k.failoverTimeout < k.schedulerConfig.FrameworkIdRefreshInterval.Duration.Seconds() { - refreshInterval = time.Duration(math.Max(1, k.failoverTimeout/2)) * time.Second - } - - // wait until we've written the framework ID at least once before proceeding - firstStore := make(chan struct{}) - go runtime.Until(func() { - // only close firstStore once - select { - case <-firstStore: - default: - defer close(firstStore) - } - err := k.storeFrameworkId(context.TODO(), k.frameworkId.GetValue()) - if err != nil { - log.Errorf("failed to store framework ID: %v", err) - if err == frameworkid.ErrMismatch { - // we detected a framework ID in storage that doesn't match what we're trying - // to save. this is a dangerous state: - // (1) perhaps we failed to initially recover the framework ID and so mesos - // issued us a new one. now that we're trying to save it there's a mismatch. - // (2) we've somehow bungled the framework ID and we're out of alignment with - // what mesos is expecting. - // (3) multiple schedulers were launched at the same time, and both have - // registered with mesos (because when they each checked, there was no ID in - // storage, so they asked for a new one). one of them has already written the - // ID to storage -- we lose. - log.Error("aborting due to framework ID mismatch") - driver.Abort() - } - } - }, refreshInterval, k.terminate) - - // wait for the first store attempt of the framework ID - select { - case <-firstStore: - case <-k.terminate: - } - } - - r1 := k.makeTaskRegistryReconciler() - r2 := k.makePodRegistryReconciler() - - k.tasksReconciler = taskreconciler.New(k.asRegisteredMaster, taskreconciler.MakeComposite(k.terminate, r1, r2), - k.reconcileCooldown, k.schedulerConfig.ExplicitReconciliationAbortTimeout.Duration, k.terminate) - go k.tasksReconciler.Run(driver, k.terminate) - - if k.reconcileInterval > 0 { - ri := time.Duration(k.reconcileInterval) * time.Second - time.AfterFunc(k.schedulerConfig.InitialImplicitReconciliationDelay.Duration, func() { runtime.Until(k.tasksReconciler.RequestImplicit, ri, k.terminate) }) - log.Infof("will perform implicit task reconciliation at interval: %v after %v", ri, k.schedulerConfig.InitialImplicitReconciliationDelay.Duration) - } - - k.installDebugHandlers(k.mux) -} - -// Disconnected is called when the scheduler loses connection to the master. -func (k *framework) Disconnected(driver bindings.SchedulerDriver) { - log.Infof("Master disconnected!\n") - - k.registered = false - - // discard all cached offers to avoid unnecessary TASK_LOST updates - k.offers.Invalidate("") -} - -// ResourceOffers is called when the scheduler receives some offers from the master. -func (k *framework) ResourceOffers(driver bindings.SchedulerDriver, offers []*mesos.Offer) { - log.V(2).Infof("Received offers %+v", offers) - - // Record the offers in the global offer map as well as each slave's offer map. - k.offers.Add(offers) - for _, offer := range offers { - slaveId := offer.GetSlaveId().GetValue() - k.slaveHostNames.Register(slaveId, offer.GetHostname()) - - // create api object if not existing already - if k.nodeRegistrator != nil { - labels := node.SlaveAttributesToLabels(offer.GetAttributes()) - _, err := k.nodeRegistrator.Register(offer.GetHostname(), labels) - if err != nil { - log.Error(err) - } - } - } -} - -// OfferRescinded is called when the resources are recinded from the scheduler. -func (k *framework) OfferRescinded(driver bindings.SchedulerDriver, offerId *mesos.OfferID) { - log.Infof("Offer rescinded %v\n", offerId) - - oid := offerId.GetValue() - k.offers.Delete(oid, offermetrics.OfferRescinded) -} - -// StatusUpdate is called when a status update message is sent to the scheduler. -func (k *framework) StatusUpdate(driver bindings.SchedulerDriver, taskStatus *mesos.TaskStatus) { - - source, reason := "none", "none" - if taskStatus.Source != nil { - source = (*taskStatus.Source).String() - } - if taskStatus.Reason != nil { - reason = (*taskStatus.Reason).String() - } - taskState := taskStatus.GetState() - metrics.StatusUpdates.WithLabelValues(source, reason, taskState.String()).Inc() - - message := "none" - if taskStatus.Message != nil { - message = *taskStatus.Message - } - - log.Infof( - "task status update %q from %q for task %q on slave %q executor %q for reason %q with message %q", - taskState.String(), - source, - taskStatus.TaskId.GetValue(), - taskStatus.SlaveId.GetValue(), - taskStatus.ExecutorId.GetValue(), - reason, - message, - ) - - switch taskState { - case mesos.TaskState_TASK_RUNNING, mesos.TaskState_TASK_FINISHED, mesos.TaskState_TASK_STARTING, mesos.TaskState_TASK_STAGING: - if _, state := k.sched.Tasks().UpdateStatus(taskStatus); state == podtask.StateUnknown { - if taskState != mesos.TaskState_TASK_FINISHED { - //TODO(jdef) what if I receive this after a TASK_LOST or TASK_KILLED? - //I don't want to reincarnate then.. TASK_LOST is a special case because - //the master is stateless and there are scenarios where I may get TASK_LOST - //followed by TASK_RUNNING. - //TODO(jdef) consider running this asynchronously since there are API server - //calls that may be made - k.reconcileNonTerminalTask(driver, taskStatus) - } // else, we don't really care about FINISHED tasks that aren't registered - return - } - if hostName := k.slaveHostNames.HostName(taskStatus.GetSlaveId().GetValue()); hostName == "" { - // a registered task has an update reported by a slave that we don't recognize. - // this should never happen! So we don't reconcile it. - log.Errorf("Ignore status %+v because the slave does not exist", taskStatus) - return - } - case mesos.TaskState_TASK_FAILED, mesos.TaskState_TASK_ERROR: - if task, _ := k.sched.Tasks().UpdateStatus(taskStatus); task != nil { - if task.Has(podtask.Launched) && !task.Has(podtask.Bound) { - go k.sched.Reconcile(task) - return - } - } else { - // unknown task failed, not much we can do about it - return - } - // last-ditch effort to reconcile our records - fallthrough - case mesos.TaskState_TASK_LOST, mesos.TaskState_TASK_KILLED: - k.reconcileTerminalTask(driver, taskStatus) - default: - log.Errorf( - "unknown task status %q from %q for task %q on slave %q executor %q for reason %q with message %q", - taskState.String(), - source, - taskStatus.TaskId.GetValue(), - taskStatus.SlaveId.GetValue(), - taskStatus.ExecutorId.GetValue(), - reason, - message, - ) - } -} - -func (k *framework) reconcileTerminalTask(driver bindings.SchedulerDriver, taskStatus *mesos.TaskStatus) { - task, state := k.sched.Tasks().UpdateStatus(taskStatus) - - if (state == podtask.StateRunning || state == podtask.StatePending) && - ((taskStatus.GetSource() == mesos.TaskStatus_SOURCE_MASTER && taskStatus.GetReason() == mesos.TaskStatus_REASON_RECONCILIATION) || - (taskStatus.GetSource() == mesos.TaskStatus_SOURCE_SLAVE && taskStatus.GetReason() == mesos.TaskStatus_REASON_EXECUTOR_TERMINATED) || - (taskStatus.GetSource() == mesos.TaskStatus_SOURCE_SLAVE && taskStatus.GetReason() == mesos.TaskStatus_REASON_EXECUTOR_UNREGISTERED) || - (taskStatus.GetSource() == mesos.TaskStatus_SOURCE_EXECUTOR && taskStatus.GetMessage() == messages.ContainersDisappeared) || - (taskStatus.GetSource() == mesos.TaskStatus_SOURCE_EXECUTOR && taskStatus.GetMessage() == messages.KubeletPodLaunchFailed) || - (taskStatus.GetSource() == mesos.TaskStatus_SOURCE_EXECUTOR && taskStatus.GetMessage() == messages.TaskKilled && !task.Has(podtask.Deleted))) { - //-- - // pod-task has metadata that refers to: - // (1) a task that Mesos no longer knows about, or else - // (2) a pod that the Kubelet will never report as "failed" - // (3) a pod that the kubeletExecutor reported as lost (likely due to docker daemon crash/restart) - // (4) a pod that the kubeletExecutor reported as lost because the kubelet didn't manage to launch it (in time) - // (5) a pod that the kubeletExecutor killed, but the scheduler didn't ask for that (maybe killed by the master) - // For now, destroy the pod and hope that there's a replication controller backing it up. - // TODO(jdef) for case #2 don't delete the pod, just update it's status to Failed - pod := &task.Pod - log.Warningf("deleting rogue pod %v/%v for lost task %v", pod.Namespace, pod.Name, task.ID) - if err := k.client.Core().Pods(pod.Namespace).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil && !errors.IsNotFound(err) { - log.Errorf("failed to delete pod %v/%v for terminal task %v: %v", pod.Namespace, pod.Name, task.ID, err) - } - } else if taskStatus.GetReason() == mesos.TaskStatus_REASON_EXECUTOR_TERMINATED || taskStatus.GetReason() == mesos.TaskStatus_REASON_EXECUTOR_UNREGISTERED { - // attempt to prevent dangling pods in the pod and task registries - log.V(1).Infof("request explicit reconciliation to clean up for task %v after executor reported (terminated/unregistered)", taskStatus.TaskId.GetValue()) - k.tasksReconciler.RequestExplicit() - } else if taskStatus.GetState() == mesos.TaskState_TASK_LOST && state == podtask.StateRunning && taskStatus.ExecutorId != nil && taskStatus.SlaveId != nil { - //TODO(jdef) this may not be meaningful once we have proper checkpointing and master detection - //If we're reconciling and receive this then the executor may be - //running a task that we need it to kill. It's possible that the framework - //is unrecognized by the master at this point, so KillTask is not guaranteed - //to do anything. The underlying driver transport may be able to send a - //FrameworkMessage directly to the slave to terminate the task. - log.V(2).Infof("forwarding TASK_LOST message to executor %v on slave %v", taskStatus.ExecutorId, taskStatus.SlaveId) - data := fmt.Sprintf("%s:%s", messages.TaskLost, task.ID) //TODO(jdef) use a real message type - if _, err := driver.SendFrameworkMessage(taskStatus.ExecutorId, taskStatus.SlaveId, data); err != nil { - log.Error(err.Error()) - } - } -} - -// reconcile an unknown (from the perspective of our registry) non-terminal task -func (k *framework) reconcileNonTerminalTask(driver bindings.SchedulerDriver, taskStatus *mesos.TaskStatus) { - // attempt to recover task from pod info: - // - task data may contain an api.PodStatusResult; if status.reason == REASON_RECONCILIATION then status.data == nil - // - the Name can be parsed by container.ParseFullName() to yield a pod Name and Namespace - // - pull the pod metadata down from the api server - // - perform task recovery based on pod metadata - taskId := taskStatus.TaskId.GetValue() - if taskStatus.GetReason() == mesos.TaskStatus_REASON_RECONCILIATION && taskStatus.GetSource() == mesos.TaskStatus_SOURCE_MASTER { - // there will be no data in the task status that we can use to determine the associated pod - switch taskStatus.GetState() { - case mesos.TaskState_TASK_STAGING: - // there is still hope for this task, don't kill it just yet - //TODO(jdef) there should probably be a limit for how long we tolerate tasks stuck in this state - return - default: - // for TASK_{STARTING,RUNNING} we should have already attempted to recoverTasks() for. - // if the scheduler failed over before the executor fired TASK_STARTING, then we should *not* - // be processing this reconciliation update before we process the one from the executor. - // point: we don't know what this task is (perhaps there was unrecoverable metadata in the pod), - // so it gets killed. - log.Errorf("killing non-terminal, unrecoverable task %v", taskId) - } - } else if podStatus, err := podtask.ParsePodStatusResult(taskStatus); err != nil { - // possible rogue pod exists at this point because we can't identify it; should kill the task - log.Errorf("possible rogue pod; illegal task status data for task %v, expected an api.PodStatusResult: %v", taskId, err) - } else if name, namespace, err := container.ParsePodFullName(podStatus.Name); err != nil { - // possible rogue pod exists at this point because we can't identify it; should kill the task - log.Errorf("possible rogue pod; illegal api.PodStatusResult, unable to parse full pod name from: '%v' for task %v: %v", - podStatus.Name, taskId, err) - } else if pod, err := k.client.Core().Pods(namespace).Get(name); err == nil { - if t, ok, err := podtask.RecoverFrom(*pod); ok { - log.Infof("recovered task %v from metadata in pod %v/%v", taskId, namespace, name) - _, err := k.sched.Tasks().Register(t) - if err != nil { - // someone beat us to it?! - log.Warningf("failed to register recovered task: %v", err) - return - } else { - k.sched.Tasks().UpdateStatus(taskStatus) - } - return - } else if err != nil { - //should kill the pod and the task - log.Errorf("killing pod, failed to recover task from pod %v/%v: %v", namespace, name, err) - if err := k.client.Core().Pods(namespace).Delete(name, nil); err != nil { - log.Errorf("failed to delete pod %v/%v: %v", namespace, name, err) - } - } else { - //this is pretty unexpected: we received a TASK_{STARTING,RUNNING} message, but the apiserver's pod - //metadata is not appropriate for task reconstruction -- which should almost certainly never - //be the case unless someone swapped out the pod on us (and kept the same namespace/name) while - //we were failed over. - - //kill this task, allow the newly launched scheduler to schedule the new pod - log.Warningf("unexpected pod metadata for task %v in apiserver, assuming new unscheduled pod spec: %+v", taskId, pod) - } - } else if errors.IsNotFound(err) { - // pod lookup failed, should delete the task since the pod is no longer valid; may be redundant, that's ok - log.Infof("killing task %v since pod %v/%v no longer exists", taskId, namespace, name) - } else if errors.IsServerTimeout(err) { - log.V(2).Infof("failed to reconcile task due to API server timeout: %v", err) - return - } else { - log.Errorf("unexpected API server error, aborting reconcile for task %v: %v", taskId, err) - return - } - if _, err := driver.KillTask(taskStatus.TaskId); err != nil { - log.Errorf("failed to kill task %v: %v", taskId, err) - } -} - -// FrameworkMessage is called when the scheduler receives a message from the executor. -func (k *framework) FrameworkMessage(driver bindings.SchedulerDriver, - executorId *mesos.ExecutorID, slaveId *mesos.SlaveID, message string) { - log.Infof("Received messages from executor %v of slave %v, %v\n", executorId, slaveId, message) -} - -// SlaveLost is called when some slave is lost. -func (k *framework) SlaveLost(driver bindings.SchedulerDriver, slaveId *mesos.SlaveID) { - log.Infof("Slave %v is lost\n", slaveId) - - sid := slaveId.GetValue() - k.offers.InvalidateForSlave(sid) - - // TODO(jdef): delete slave from our internal list? probably not since we may need to reconcile - // tasks. it would be nice to somehow flag the slave as lost so that, perhaps, we can periodically - // flush lost slaves older than X, and for which no tasks or pods reference. - - // unfinished tasks/pods will be dropped. use a replication controller if you want pods to - // be restarted when slaves die. -} - -// ExecutorLost is called when some executor is lost. -func (k *framework) ExecutorLost(driver bindings.SchedulerDriver, executorId *mesos.ExecutorID, slaveId *mesos.SlaveID, status int) { - log.Infof("Executor %v of slave %v is lost, status: %v\n", executorId, slaveId, status) -} - -// Error is called when there is an unrecoverable error in the scheduler or scheduler driver. -// The driver should have been aborted before this is invoked. -func (k *framework) Error(driver bindings.SchedulerDriver, message string) { - log.Fatalf("fatal scheduler error: %v\n", message) -} - -// filter func used for explicit task reconciliation, selects only non-terminal tasks which -// have been communicated to mesos (read: launched). -func explicitTaskFilter(t *podtask.T) bool { - switch t.State { - case podtask.StateRunning: - return true - case podtask.StatePending: - return t.Has(podtask.Launched) - default: - return false - } -} - -// reconciler action factory, performs explicit task reconciliation for non-terminal -// tasks listed in the scheduler's internal taskRegistry. -func (k *framework) makeTaskRegistryReconciler() taskreconciler.Action { - return taskreconciler.Action(func(drv bindings.SchedulerDriver, cancel <-chan struct{}) <-chan error { - taskToSlave := make(map[string]string) - for _, t := range k.sched.Tasks().List(explicitTaskFilter) { - if t.Spec.SlaveID != "" { - taskToSlave[t.ID] = t.Spec.SlaveID - } - } - return proc.ErrorChan(k.explicitlyReconcileTasks(drv, taskToSlave, cancel)) - }) -} - -// reconciler action factory, performs explicit task reconciliation for non-terminal -// tasks identified by annotations in the Kubernetes pod registry. -func (k *framework) makePodRegistryReconciler() taskreconciler.Action { - return taskreconciler.Action(func(drv bindings.SchedulerDriver, cancel <-chan struct{}) <-chan error { - podList, err := k.client.Core().Pods(api.NamespaceAll).List(api.ListOptions{}) - if err != nil { - return proc.ErrorChanf("failed to reconcile pod registry: %v", err) - } - taskToSlave := make(map[string]string) - for _, pod := range podList.Items { - if len(pod.Annotations) == 0 { - continue - } - taskId, found := pod.Annotations[meta.TaskIdKey] - if !found { - continue - } - slaveId, found := pod.Annotations[meta.SlaveIdKey] - if !found { - continue - } - taskToSlave[taskId] = slaveId - } - return proc.ErrorChan(k.explicitlyReconcileTasks(drv, taskToSlave, cancel)) - }) -} - -// execute an explicit task reconciliation, as per http://mesos.apache.org/documentation/latest/reconciliation/ -func (k *framework) explicitlyReconcileTasks(driver bindings.SchedulerDriver, taskToSlave map[string]string, cancel <-chan struct{}) error { - log.Info("explicit reconcile tasks") - - // tell mesos to send us the latest status updates for all the non-terminal tasks that we know about - statusList := []*mesos.TaskStatus{} - remaining := sets.StringKeySet(taskToSlave) - for taskId, slaveId := range taskToSlave { - if slaveId == "" { - delete(taskToSlave, taskId) - continue - } - statusList = append(statusList, &mesos.TaskStatus{ - TaskId: mutil.NewTaskID(taskId), - SlaveId: mutil.NewSlaveID(slaveId), - State: mesos.TaskState_TASK_RUNNING.Enum(), // req'd field, doesn't have to reflect reality - }) - } - - select { - case <-cancel: - return merrors.ReconciliationCancelledErr - default: - if _, err := driver.ReconcileTasks(statusList); err != nil { - return err - } - } - - start := time.Now() - first := true - for backoff := 1 * time.Second; first || remaining.Len() > 0; backoff = backoff * 2 { - first = false - // nothing to do here other than wait for status updates.. - if backoff > k.schedulerConfig.ExplicitReconciliationMaxBackoff.Duration { - backoff = k.schedulerConfig.ExplicitReconciliationMaxBackoff.Duration - } - select { - case <-cancel: - return merrors.ReconciliationCancelledErr - case <-time.After(backoff): - for taskId := range remaining { - if task, _ := k.sched.Tasks().Get(taskId); task != nil && explicitTaskFilter(task) && task.UpdatedTime.Before(start) { - // keep this task in remaining list - continue - } - remaining.Delete(taskId) - } - } - } - return nil -} - -func (ks *framework) recoverTasks() error { - podList, err := ks.client.Core().Pods(api.NamespaceAll).List(api.ListOptions{}) - if err != nil { - log.V(1).Infof("failed to recover pod registry, madness may ensue: %v", err) - return err - } - recoverSlave := func(t *podtask.T) { - - slaveId := t.Spec.SlaveID - ks.slaveHostNames.Register(slaveId, t.Offer.Host()) - } - for _, pod := range podList.Items { - if _, isMirrorPod := pod.Annotations[kubetypes.ConfigMirrorAnnotationKey]; isMirrorPod { - // mirrored pods are never reconciled because the scheduler isn't responsible for - // scheduling them; they're started by the executor/kubelet upon instantiation and - // reflected in the apiserver afterward. the scheduler has no knowledge of them. - continue - } - if t, ok, err := podtask.RecoverFrom(pod); err != nil { - log.Errorf("failed to recover task from pod, will attempt to delete '%v/%v': %v", pod.Namespace, pod.Name, err) - err := ks.client.Core().Pods(pod.Namespace).Delete(pod.Name, nil) - //TODO(jdef) check for temporary or not-found errors - if err != nil { - log.Errorf("failed to delete pod '%v/%v': %v", pod.Namespace, pod.Name, err) - } - } else if ok { - ks.sched.Tasks().Register(t) - recoverSlave(t) - log.Infof("recovered task %v from pod %v/%v", t.ID, pod.Namespace, pod.Name) - } - } - return nil -} - -func (ks *framework) KillTask(id string) error { - killTaskId := mutil.NewTaskID(id) - _, err := ks.driver.KillTask(killTaskId) - return err -} - -func (ks *framework) LaunchTask(t *podtask.T) error { - taskInfo, err := t.BuildTaskInfo() - if err != nil { - return err - } - - // assume caller is holding scheduler lock - taskList := []*mesos.TaskInfo{taskInfo} - offerIds := []*mesos.OfferID{t.Offer.Details().Id} - filters := &mesos.Filters{} - _, err = ks.driver.LaunchTasks(offerIds, taskList, filters) - return err -} - -func (ks *framework) Offers() offers.Registry { - return ks.offers -} diff --git a/contrib/mesos/pkg/scheduler/components/framework/framework_test.go b/contrib/mesos/pkg/scheduler/components/framework/framework_test.go deleted file mode 100644 index 0a7ffa989ad..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/framework_test.go +++ /dev/null @@ -1,336 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "reflect" - "testing" - - mesos "github.com/mesos/mesos-go/mesosproto" - util "github.com/mesos/mesos-go/mesosutil" - "github.com/stretchr/testify/assert" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - "k8s.io/kubernetes/contrib/mesos/pkg/proc" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - schedcfg "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/config" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/client/cache" -) - -//get number of non-expired offers from offer registry -func getNumberOffers(os offers.Registry) int { - //walk offers and check it is stored in registry - walked := 0 - walker1 := func(p offers.Perishable) (bool, error) { - walked++ - return false, nil - - } - os.Walk(walker1) - return walked -} - -type mockRegistrator struct { - store cache.Store -} - -func (r *mockRegistrator) Run(terminate <-chan struct{}) error { - return nil -} - -func (r *mockRegistrator) Register(hostName string, labels map[string]string) (bool, error) { - obj, _, err := r.store.GetByKey(hostName) - if err != nil { - return false, err - } - if obj == nil { - return true, r.store.Add(&api.Node{ - ObjectMeta: api.ObjectMeta{ - Name: hostName, - Labels: labels, - }, - Spec: api.NodeSpec{ - ExternalID: hostName, - }, - Status: api.NodeStatus{ - Phase: api.NodePending, - }, - }) - } else { - n := obj.(*api.Node) - if reflect.DeepEqual(n.Labels, labels) { - return false, nil - } - n.Labels = labels - return true, r.store.Update(n) - } -} - -func mockScheduler() scheduler.Scheduler { - mockScheduler := &scheduler.MockScheduler{} - reg := podtask.NewInMemoryRegistry() - mockScheduler.On("Tasks").Return(reg) - return mockScheduler -} - -//test adding of ressource offer, should be added to offer registry and slaves -func TestResourceOffer_Add(t *testing.T) { - assert := assert.New(t) - - registrator := &mockRegistrator{cache.NewStore(cache.MetaNamespaceKeyFunc)} - testFramework := &framework{ - offers: offers.CreateRegistry(offers.RegistryConfig{ - Compat: func(o *mesos.Offer) bool { - return true - }, - DeclineOffer: func(offerId string) <-chan error { - return proc.ErrorChan(nil) - }, - // remember expired offers so that we can tell if a previously scheduler offer relies on one - LingerTTL: schedcfg.DefaultOfferLingerTTL, - TTL: schedcfg.DefaultOfferTTL, - ListenerDelay: schedcfg.DefaultListenerDelay, - }), - slaveHostNames: newSlaveRegistry(), - nodeRegistrator: registrator, - sched: mockScheduler(), - } - - hostname := "h1" - offerID1 := util.NewOfferID("test1") - offer1 := &mesos.Offer{Id: offerID1, Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)} - offers1 := []*mesos.Offer{offer1} - testFramework.ResourceOffers(nil, offers1) - assert.Equal(1, len(registrator.store.List())) - - assert.Equal(1, getNumberOffers(testFramework.offers)) - //check slave hostname - assert.Equal(1, len(testFramework.slaveHostNames.SlaveIDs())) - - //add another offer - hostname2 := "h2" - offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)} - offers2 := []*mesos.Offer{offer2} - testFramework.ResourceOffers(nil, offers2) - - //check it is stored in registry - assert.Equal(2, getNumberOffers(testFramework.offers)) - - //check slave hostnames - assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs())) -} - -//test adding of ressource offer, should be added to offer registry and slavesf -func TestResourceOffer_Add_Rescind(t *testing.T) { - assert := assert.New(t) - - testFramework := &framework{ - offers: offers.CreateRegistry(offers.RegistryConfig{ - Compat: func(o *mesos.Offer) bool { - return true - }, - DeclineOffer: func(offerId string) <-chan error { - return proc.ErrorChan(nil) - }, - // remember expired offers so that we can tell if a previously scheduler offer relies on one - LingerTTL: schedcfg.DefaultOfferLingerTTL, - TTL: schedcfg.DefaultOfferTTL, - ListenerDelay: schedcfg.DefaultListenerDelay, - }), - slaveHostNames: newSlaveRegistry(), - sched: mockScheduler(), - } - - hostname := "h1" - offerID1 := util.NewOfferID("test1") - offer1 := &mesos.Offer{Id: offerID1, Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)} - offers1 := []*mesos.Offer{offer1} - testFramework.ResourceOffers(nil, offers1) - - assert.Equal(1, getNumberOffers(testFramework.offers)) - - //check slave hostname - assert.Equal(1, len(testFramework.slaveHostNames.SlaveIDs())) - - //add another offer - hostname2 := "h2" - offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)} - offers2 := []*mesos.Offer{offer2} - testFramework.ResourceOffers(nil, offers2) - - assert.Equal(2, getNumberOffers(testFramework.offers)) - - //check slave hostnames - assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs())) - - //next whether offers can be rescinded - testFramework.OfferRescinded(nil, offerID1) - assert.Equal(1, getNumberOffers(testFramework.offers)) - - //next whether offers can be rescinded - testFramework.OfferRescinded(nil, util.NewOfferID("test2")) - //walk offers again and check it is removed from registry - assert.Equal(0, getNumberOffers(testFramework.offers)) - - //remove non existing ID - testFramework.OfferRescinded(nil, util.NewOfferID("notExist")) -} - -//test that when a slave is lost we remove all offers -func TestSlave_Lost(t *testing.T) { - assert := assert.New(t) - - // - testFramework := &framework{ - offers: offers.CreateRegistry(offers.RegistryConfig{ - Compat: func(o *mesos.Offer) bool { - return true - }, - // remember expired offers so that we can tell if a previously scheduler offer relies on one - LingerTTL: schedcfg.DefaultOfferLingerTTL, - TTL: schedcfg.DefaultOfferTTL, - ListenerDelay: schedcfg.DefaultListenerDelay, - }), - slaveHostNames: newSlaveRegistry(), - sched: mockScheduler(), - } - - hostname := "h1" - offer1 := &mesos.Offer{Id: util.NewOfferID("test1"), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)} - offers1 := []*mesos.Offer{offer1} - testFramework.ResourceOffers(nil, offers1) - offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)} - offers2 := []*mesos.Offer{offer2} - testFramework.ResourceOffers(nil, offers2) - - //add another offer from different slaveID - hostname2 := "h2" - offer3 := &mesos.Offer{Id: util.NewOfferID("test3"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)} - offers3 := []*mesos.Offer{offer3} - testFramework.ResourceOffers(nil, offers3) - - //test precondition - assert.Equal(3, getNumberOffers(testFramework.offers)) - assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs())) - - //remove first slave - testFramework.SlaveLost(nil, util.NewSlaveID(hostname)) - - //offers should be removed - assert.Equal(1, getNumberOffers(testFramework.offers)) - //slave hostnames should still be all present - assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs())) - - //remove second slave - testFramework.SlaveLost(nil, util.NewSlaveID(hostname2)) - - //offers should be removed - assert.Equal(0, getNumberOffers(testFramework.offers)) - //slave hostnames should still be all present - assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs())) - - //try to remove non existing slave - testFramework.SlaveLost(nil, util.NewSlaveID("notExist")) - -} - -//test when we loose connection to master we invalidate all cached offers -func TestDisconnect(t *testing.T) { - assert := assert.New(t) - - // - testFramework := &framework{ - offers: offers.CreateRegistry(offers.RegistryConfig{ - Compat: func(o *mesos.Offer) bool { - return true - }, - // remember expired offers so that we can tell if a previously scheduler offer relies on one - LingerTTL: schedcfg.DefaultOfferLingerTTL, - TTL: schedcfg.DefaultOfferTTL, - ListenerDelay: schedcfg.DefaultListenerDelay, - }), - slaveHostNames: newSlaveRegistry(), - sched: mockScheduler(), - } - - hostname := "h1" - offer1 := &mesos.Offer{Id: util.NewOfferID("test1"), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)} - offers1 := []*mesos.Offer{offer1} - testFramework.ResourceOffers(nil, offers1) - offer2 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname)} - offers2 := []*mesos.Offer{offer2} - testFramework.ResourceOffers(nil, offers2) - - //add another offer from different slaveID - hostname2 := "h2" - offer3 := &mesos.Offer{Id: util.NewOfferID("test2"), Hostname: &hostname2, SlaveId: util.NewSlaveID(hostname2)} - offers3 := []*mesos.Offer{offer3} - testFramework.ResourceOffers(nil, offers3) - - //disconnect - testFramework.Disconnected(nil) - - //all offers should be removed - assert.Equal(0, getNumberOffers(testFramework.offers)) - //slave hostnames should still be all present - assert.Equal(2, len(testFramework.slaveHostNames.SlaveIDs())) -} - -//test we can handle different status updates, TODO check state transitions -func TestStatus_Update(t *testing.T) { - - mockdriver := MockSchedulerDriver{} - // setup expectations - mockdriver.On("KillTask", util.NewTaskID("test-task-001")).Return(mesos.Status_DRIVER_RUNNING, nil) - - testFramework := &framework{ - offers: offers.CreateRegistry(offers.RegistryConfig{ - Compat: func(o *mesos.Offer) bool { - return true - }, - // remember expired offers so that we can tell if a previously scheduler offer relies on one - LingerTTL: schedcfg.DefaultOfferLingerTTL, - TTL: schedcfg.DefaultOfferTTL, - ListenerDelay: schedcfg.DefaultListenerDelay, - }), - slaveHostNames: newSlaveRegistry(), - driver: &mockdriver, - sched: mockScheduler(), - } - - taskStatus_task_starting := util.NewTaskStatus( - util.NewTaskID("test-task-001"), - mesos.TaskState_TASK_RUNNING, - ) - testFramework.StatusUpdate(testFramework.driver, taskStatus_task_starting) - - taskStatus_task_running := util.NewTaskStatus( - util.NewTaskID("test-task-001"), - mesos.TaskState_TASK_RUNNING, - ) - testFramework.StatusUpdate(testFramework.driver, taskStatus_task_running) - - taskStatus_task_failed := util.NewTaskStatus( - util.NewTaskID("test-task-001"), - mesos.TaskState_TASK_FAILED, - ) - testFramework.StatusUpdate(testFramework.driver, taskStatus_task_failed) - - //assert that mock was invoked - mockdriver.AssertExpectations(t) -} diff --git a/contrib/mesos/pkg/scheduler/components/framework/frameworkid/etcd/etcd.go b/contrib/mesos/pkg/scheduler/components/framework/frameworkid/etcd/etcd.go deleted file mode 100644 index 6846f294213..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/frameworkid/etcd/etcd.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package etcd - -import ( - "fmt" - "time" - - etcd "github.com/coreos/etcd/client" - "golang.org/x/net/context" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework/frameworkid" - etcdutil "k8s.io/kubernetes/pkg/storage/etcd/util" -) - -type storage struct { - frameworkid.LookupFunc - frameworkid.StoreFunc - frameworkid.RemoveFunc -} - -func Store(api etcd.KeysAPI, path string, ttl time.Duration) frameworkid.Storage { - // TODO(jdef) validate Config - return &storage{ - LookupFunc: func(ctx context.Context) (string, error) { - if response, err := api.Get(ctx, path, nil); err != nil { - if !etcdutil.IsEtcdNotFound(err) { - return "", fmt.Errorf("unexpected failure attempting to load framework ID from etcd: %v", err) - } - } else { - return response.Node.Value, nil - } - return "", nil - }, - RemoveFunc: func(ctx context.Context) (err error) { - if _, err = api.Delete(ctx, path, &etcd.DeleteOptions{Recursive: true}); err != nil { - if !etcdutil.IsEtcdNotFound(err) { - return fmt.Errorf("failed to delete framework ID from etcd: %v", err) - } - } - return - }, - StoreFunc: func(ctx context.Context, id string) (err error) { - _, err = api.Set(ctx, path, id, &etcd.SetOptions{TTL: ttl}) - return - }, - } -} diff --git a/contrib/mesos/pkg/scheduler/components/framework/frameworkid/frameworkid.go b/contrib/mesos/pkg/scheduler/components/framework/frameworkid/frameworkid.go deleted file mode 100644 index 9cccc372511..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/frameworkid/frameworkid.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package frameworkid - -import ( - "errors" - - "golang.org/x/net/context" -) - -type ( - // LookupFunc retrieves a framework ID from persistent storage - LookupFunc func(context.Context) (string, error) - - // StoreFunc stores a framework ID in persistent storage - StoreFunc func(context.Context, string) error - - // RemoveFunc removes a framework ID from persistent storage - RemoveFunc func(context.Context) error - - Getter interface { - Get(context.Context) (string, error) - } - - Setter interface { - Set(context.Context, string) error - } - - Remover interface { - Remove(context.Context) error - } - - Storage interface { - Getter - Setter - Remover - } -) - -var ErrMismatch = errors.New("framework ID mismatch") - -func (f LookupFunc) Get(c context.Context) (string, error) { return f(c) } -func (f StoreFunc) Set(c context.Context, id string) error { return f(c, id) } -func (f RemoveFunc) Remove(c context.Context) error { return f(c) } diff --git a/contrib/mesos/pkg/scheduler/components/framework/frameworkid/zk/zk.go b/contrib/mesos/pkg/scheduler/components/framework/frameworkid/zk/zk.go deleted file mode 100644 index 485f76e2ffa..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/frameworkid/zk/zk.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package zk - -import ( - "fmt" - "net/url" - "path" - "strings" - "time" - - "github.com/samuel/go-zookeeper/zk" - "golang.org/x/net/context" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework/frameworkid" -) - -const RPC_TIMEOUT = time.Second * 5 - -type storage struct { - frameworkid.LookupFunc - frameworkid.StoreFunc - frameworkid.RemoveFunc -} - -func Store(zkurl, frameworkName string) frameworkid.Storage { - // TODO(jdef) validate Config - zkServers, zkChroot, parseErr := parseZk(zkurl) - withConnection := func(ctx context.Context, f func(c *zk.Conn) error) error { - if parseErr != nil { - return parseErr - } - timeout, err := timeout(ctx) - if err != nil { - return err - } - c, _, err := zk.Connect(zkServers, timeout) - if err != nil { - return err - } - defer c.Close() - return f(c) - } - return &storage{ - LookupFunc: func(ctx context.Context) (rawData string, lookupErr error) { - lookupErr = withConnection(ctx, func(c *zk.Conn) error { - data, _, err := c.Get(path.Join(zkChroot, frameworkName)) - if err == nil { - rawData = string(data) - } else if err != zk.ErrNoNode { - return err - } - return nil - }) - return - }, - RemoveFunc: func(ctx context.Context) error { - return withConnection(ctx, func(c *zk.Conn) error { - err := c.Delete(path.Join(zkChroot, frameworkName), -1) - if err != zk.ErrNoNode { - return err - } - return nil - }) - }, - StoreFunc: func(ctx context.Context, id string) error { - return withConnection(ctx, func(c *zk.Conn) error { - // attempt to create the path - _, err := c.Create( - zkChroot, - []byte(""), - 0, - zk.WorldACL(zk.PermAll), - ) - if err != nil && err != zk.ErrNodeExists { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - // attempt to write framework ID to / - fpath := path.Join(zkChroot, frameworkName) - _, err = c.Create(fpath, []byte(id), 0, zk.WorldACL(zk.PermAll)) - if err != nil && err == zk.ErrNodeExists { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - // cross-check value - data, _, err := c.Get(fpath) - if err != nil { - return err - } - if string(data) != id { - return frameworkid.ErrMismatch - } - return nil - } - return err - }) - }, - } -} - -func parseZk(zkurl string) ([]string, string, error) { - u, err := url.Parse(zkurl) - if err != nil { - return nil, "", fmt.Errorf("bad zk url: %v", err) - } - if u.Scheme != "zk" { - return nil, "", fmt.Errorf("invalid url scheme for zk url: '%v'", u.Scheme) - } - return strings.Split(u.Host, ","), u.Path, nil -} - -func timeout(ctx context.Context) (time.Duration, error) { - deadline, ok := ctx.Deadline() - if !ok { - // no deadline set - return RPC_TIMEOUT, nil - } - if now := time.Now(); now.Before(deadline) { - d := deadline.Sub(now) - if d > RPC_TIMEOUT { - // deadline is too far out, use our built-in - return RPC_TIMEOUT, nil - } - return d, nil - } - - // deadline has expired.. - select { - case <-ctx.Done(): - return 0, ctx.Err() - default: - // this should never happen because Done() should be closed - // according to the contract of context. but we have this here - // just in case. - return 0, context.DeadlineExceeded - } -} diff --git a/contrib/mesos/pkg/scheduler/components/framework/slaveregistry.go b/contrib/mesos/pkg/scheduler/components/framework/slaveregistry.go deleted file mode 100644 index 821ca880598..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/slaveregistry.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "sync" -) - -// slaveRegistry manages node hostnames for slave ids. -type slaveRegistry struct { - lock sync.Mutex - hostNames map[string]string -} - -func newSlaveRegistry() *slaveRegistry { - return &slaveRegistry{ - hostNames: map[string]string{}, - } -} - -// Register creates a mapping between a slaveId and slave if not existing. -func (st *slaveRegistry) Register(slaveId, slaveHostname string) { - st.lock.Lock() - defer st.lock.Unlock() - _, exists := st.hostNames[slaveId] - if !exists { - st.hostNames[slaveId] = slaveHostname - } -} - -// SlaveIDs returns the keys of the registry -func (st *slaveRegistry) SlaveIDs() []string { - st.lock.Lock() - defer st.lock.Unlock() - slaveIds := make([]string, 0, len(st.hostNames)) - for slaveID := range st.hostNames { - slaveIds = append(slaveIds, slaveID) - } - return slaveIds -} - -// HostName looks up a hostname for a given slaveId -func (st *slaveRegistry) HostName(slaveId string) string { - st.lock.Lock() - defer st.lock.Unlock() - return st.hostNames[slaveId] -} diff --git a/contrib/mesos/pkg/scheduler/components/framework/slaveregistry_test.go b/contrib/mesos/pkg/scheduler/components/framework/slaveregistry_test.go deleted file mode 100644 index 457bef19bb1..00000000000 --- a/contrib/mesos/pkg/scheduler/components/framework/slaveregistry_test.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// Check that same slave is only added once. -func TestSlaveStorage_Register(t *testing.T) { - assert := assert.New(t) - - slaveStorage := newSlaveRegistry() - assert.Equal(0, len(slaveStorage.hostNames)) - - slaveId := "slave1" - slaveHostname := "slave1Hostname" - slaveStorage.Register(slaveId, slaveHostname) - assert.Equal(1, len(slaveStorage.SlaveIDs())) - - slaveStorage.Register(slaveId, slaveHostname) - assert.Equal(1, len(slaveStorage.SlaveIDs())) -} - -// Check that GetHostName returns empty hostname for nonexisting slave. -func TestSlaveStorage_HostName(t *testing.T) { - assert := assert.New(t) - - slaveStorage := newSlaveRegistry() - assert.Equal(0, len(slaveStorage.hostNames)) - - slaveId := "slave1" - slaveHostname := "slave1Hostname" - - h := slaveStorage.HostName(slaveId) - assert.Equal(h, "") - - slaveStorage.Register(slaveId, slaveHostname) - assert.Equal(1, len(slaveStorage.SlaveIDs())) - - h = slaveStorage.HostName(slaveId) - assert.Equal(h, slaveHostname) -} - -// Check that GetSlaveIds returns array with all slaveIds. -func TestSlaveStorage_SlaveIds(t *testing.T) { - assert := assert.New(t) - - slaveStorage := newSlaveRegistry() - assert.Equal(0, len(slaveStorage.hostNames)) - - slaveId := "1" - slaveHostname := "hn1" - slaveStorage.Register(slaveId, slaveHostname) - assert.Equal(1, len(slaveStorage.SlaveIDs())) - - slaveId = "2" - slaveHostname = "hn2" - slaveStorage.Register(slaveId, slaveHostname) - assert.Equal(2, len(slaveStorage.SlaveIDs())) - - slaveIds := slaveStorage.SlaveIDs() - - slaveIdsMap := make(map[string]bool, len(slaveIds)) - for _, s := range slaveIds { - slaveIdsMap[s] = true - } - - _, ok := slaveIdsMap["1"] - assert.Equal(ok, true) - - _, ok = slaveIdsMap["2"] - assert.Equal(ok, true) -} diff --git a/contrib/mesos/pkg/scheduler/components/podreconciler/doc.go b/contrib/mesos/pkg/scheduler/components/podreconciler/doc.go deleted file mode 100644 index 3afc07ef13f..00000000000 --- a/contrib/mesos/pkg/scheduler/components/podreconciler/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package podreconciler implements pod reconciliation of pods which failed -// to launch, i.e. before binding by the executor took place. -package podreconciler // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/podreconciler" diff --git a/contrib/mesos/pkg/scheduler/components/podreconciler/podreconciler.go b/contrib/mesos/pkg/scheduler/components/podreconciler/podreconciler.go deleted file mode 100644 index a01417fe734..00000000000 --- a/contrib/mesos/pkg/scheduler/components/podreconciler/podreconciler.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podreconciler - -import ( - "time" - - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/deleter" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/queuer" - "k8s.io/kubernetes/pkg/api" - apierrors "k8s.io/kubernetes/pkg/api/errors" -) - -// PodReconciler reconciles a pod with the apiserver -type PodReconciler interface { - Reconcile(t *podtask.T) -} - -type podReconciler struct { - sched scheduler.Scheduler - client *clientset.Clientset - qr queuer.Queuer - deleter deleter.Deleter -} - -func New(sched scheduler.Scheduler, client *clientset.Clientset, qr queuer.Queuer, deleter deleter.Deleter) PodReconciler { - return &podReconciler{ - sched: sched, - client: client, - qr: qr, - deleter: deleter, - } -} - -// this pod may be out of sync with respect to the API server registry: -// this pod | apiserver registry -// -------------|---------------------- -// host=.* | 404 ; pod was deleted -// host=.* | 5xx ; failed to sync, try again later? -// host="" | host="" ; perhaps no updates to process? -// host="" | host="..." ; pod has been scheduled and assigned, is there a task assigned? (check TaskIdKey in binding?) -// host="..." | host="" ; pod is no longer scheduled, does it need to be re-queued? -// host="..." | host="..." ; perhaps no updates to process? -// -// TODO(jdef) this needs an integration test -func (s *podReconciler) Reconcile(t *podtask.T) { - log.V(1).Infof("reconcile pod %v, assigned to slave %q", t.Pod.Name, t.Spec.AssignedSlave) - ctx := api.WithNamespace(api.NewDefaultContext(), t.Pod.Namespace) - pod, err := s.client.Core().Pods(api.NamespaceValue(ctx)).Get(t.Pod.Name) - if err != nil { - if apierrors.IsNotFound(err) { - // attempt to delete - if err = s.deleter.DeleteOne(&queuer.Pod{Pod: &t.Pod}); err != nil && err != errors.NoSuchPodErr && err != errors.NoSuchTaskErr { - log.Errorf("failed to delete pod: %v: %v", t.Pod.Name, err) - } - } else { - //TODO(jdef) other errors should probably trigger a retry (w/ backoff). - //For now, drop the pod on the floor - log.Warningf("aborting reconciliation for pod %v: %v", t.Pod.Name, err) - } - return - } - - log.Infof("pod %v scheduled on %q according to apiserver", pod.Name, pod.Spec.NodeName) - if t.Spec.AssignedSlave != pod.Spec.NodeName { - if pod.Spec.NodeName == "" { - // pod is unscheduled. - // it's possible that we dropped the pod in the scheduler error handler - // because of task misalignment with the pod (task.Has(podtask.Launched) == true) - - podKey, err := podtask.MakePodKey(ctx, pod.Name) - if err != nil { - log.Error(err) - return - } - - s.sched.Lock() - defer s.sched.Unlock() - - if _, state := s.sched.Tasks().ForPod(podKey); state != podtask.StateUnknown { - //TODO(jdef) reconcile the task - log.Errorf("task already registered for pod %v", pod.Name) - return - } - - now := time.Now() - log.V(3).Infof("reoffering pod %v", podKey) - s.qr.Reoffer(queuer.NewPod(pod, queuer.Deadline(now))) - } else { - // pod is scheduled. - // not sure how this happened behind our backs. attempt to reconstruct - // at least a partial podtask.T record. - //TODO(jdef) reconcile the task - log.Errorf("pod already scheduled: %v", pod.Name) - } - } else { - //TODO(jdef) for now, ignore the fact that the rest of the spec may be different - //and assume that our knowledge of the pod aligns with that of the apiserver - log.Error("pod reconciliation does not support updates; not yet implemented") - } -} diff --git a/contrib/mesos/pkg/scheduler/components/podstoreadapter.go b/contrib/mesos/pkg/scheduler/components/podstoreadapter.go deleted file mode 100644 index 58a89910643..00000000000 --- a/contrib/mesos/pkg/scheduler/components/podstoreadapter.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package components - -import ( - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/queuer" - "k8s.io/kubernetes/pkg/api" -) - -// Consumes *api.Pod, produces *Pod; the k8s reflector wants to push *api.Pod -// objects at us, but we want to store more flexible (Pod) type defined in -// this package. The adapter implementation facilitates this. It's a little -// hackish since the object type going in is different than the object type -// coming out -- you've been warned. -type podStoreAdapter struct { - queue.FIFO -} - -func (psa *podStoreAdapter) Add(obj interface{}) error { - pod := obj.(*api.Pod) - return psa.FIFO.Add(&queuer.Pod{Pod: pod}) -} - -func (psa *podStoreAdapter) Update(obj interface{}) error { - pod := obj.(*api.Pod) - return psa.FIFO.Update(&queuer.Pod{Pod: pod}) -} - -func (psa *podStoreAdapter) Delete(obj interface{}) error { - pod := obj.(*api.Pod) - return psa.FIFO.Delete(&queuer.Pod{Pod: pod}) -} - -func (psa *podStoreAdapter) Get(obj interface{}) (interface{}, bool, error) { - pod := obj.(*api.Pod) - return psa.FIFO.Get(&queuer.Pod{Pod: pod}) -} - -// Replace will delete the contents of the store, using instead the -// given map. This store implementation does NOT take ownership of the map. -func (psa *podStoreAdapter) Replace(objs []interface{}, resourceVersion string) error { - newobjs := make([]interface{}, len(objs)) - for i, v := range objs { - pod := v.(*api.Pod) - newobjs[i] = &queuer.Pod{Pod: pod} - } - return psa.FIFO.Replace(newobjs, resourceVersion) -} diff --git a/contrib/mesos/pkg/scheduler/components/scheduler.go b/contrib/mesos/pkg/scheduler/components/scheduler.go deleted file mode 100644 index 1689497abf3..00000000000 --- a/contrib/mesos/pkg/scheduler/components/scheduler.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package components - -import ( - "net/http" - "sync" - - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - - mesos "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/contrib/mesos/pkg/backoff" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/algorithm" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/binder" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/controller" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/deleter" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/errorhandler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/podreconciler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/config" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/queuer" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/client/cache" - "k8s.io/kubernetes/pkg/client/record" -) - -// sched implements the Scheduler interface. -type sched struct { - podReconciler podreconciler.PodReconciler - framework framework.Framework - controller controller.Controller - - // unsafe state, needs to be guarded, especially changes to podtask.T objects - sync.RWMutex - taskRegistry podtask.Registry -} - -func New( - c *config.Config, - fw framework.Framework, - ps podschedulers.PodScheduler, - client *clientset.Clientset, - recorder record.EventRecorder, - terminate <-chan struct{}, - mux *http.ServeMux, - lw *cache.ListWatch, - taskConfig podtask.Config, - defaultCpus resources.CPUShares, - defaultMem resources.MegaBytes, -) scheduler.Scheduler { - core := &sched{ - framework: fw, - taskRegistry: podtask.NewInMemoryRegistry(), - } - - // Watch and queue pods that need scheduling. - podUpdatesBypass := make(chan queue.Entry, c.UpdatesBacklog) - podUpdates := &podStoreAdapter{queue.NewHistorical(podUpdatesBypass)} - reflector := cache.NewReflector(lw, &api.Pod{}, podUpdates, 0) - - q := queuer.New(queue.NewDelayFIFO(), podUpdates) - - algorithm := algorithm.New(core, podUpdates, ps, taskConfig, defaultCpus, defaultMem) - - podDeleter := deleter.New(core, q) - - core.podReconciler = podreconciler.New(core, client, q, podDeleter) - - bo := backoff.New(c.InitialPodBackoff.Duration, c.MaxPodBackoff.Duration) - newBC := func(podKey string) queue.BreakChan { - return queue.BreakChan(core.Offers().Listen(podKey, func(offer *mesos.Offer) bool { - core.Lock() - defer core.Unlock() - switch task, state := core.Tasks().ForPod(podKey); state { - case podtask.StatePending: - // Assess fitness of pod with the current offer. The scheduler normally - // "backs off" when it can't find an offer that matches up with a pod. - // The backoff period for a pod can terminate sooner if an offer becomes - // available that matches up. - - // TODO(jdef) this will never match for a pod that uses a node selector, - // since we're passing a nil *api.Node here. - return !task.Has(podtask.Launched) && ps.Fit(task, offer, nil) - default: - // no point in continuing to check for matching offers - return true - } - })) - } - errorHandler := errorhandler.New(core, bo, q, newBC) - - binder := binder.New(core) - - startLatch := make(chan struct{}) - - runtime.On(startLatch, func() { - reflector.Run() // TODO(jdef) should listen for termination - podDeleter.Run(podUpdatesBypass, terminate) - q.Run(terminate) - - q.InstallDebugHandlers(mux) - podtask.InstallDebugHandlers(core.Tasks(), mux) - }) - - core.controller = controller.New(client, algorithm, recorder, q.Yield, errorHandler.Error, binder, startLatch) - return core -} - -func (c *sched) Run(done <-chan struct{}) { - c.controller.Run(done) -} - -func (c *sched) Reconcile(t *podtask.T) { - c.podReconciler.Reconcile(t) -} - -func (c *sched) Tasks() podtask.Registry { - return c.taskRegistry -} - -func (c *sched) Offers() offers.Registry { - return c.framework.Offers() -} - -func (c *sched) KillTask(id string) error { - return c.framework.KillTask(id) -} - -func (c *sched) LaunchTask(t *podtask.T) error { - return c.framework.LaunchTask(t) -} diff --git a/contrib/mesos/pkg/scheduler/components/tasksreconciler/doc.go b/contrib/mesos/pkg/scheduler/components/tasksreconciler/doc.go deleted file mode 100644 index 80992a9740a..00000000000 --- a/contrib/mesos/pkg/scheduler/components/tasksreconciler/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package taskreconciler implement Mesos task reconciliation. -package taskreconciler diff --git a/contrib/mesos/pkg/scheduler/components/tasksreconciler/tasksreconciler.go b/contrib/mesos/pkg/scheduler/components/tasksreconciler/tasksreconciler.go deleted file mode 100644 index eb3a672783d..00000000000 --- a/contrib/mesos/pkg/scheduler/components/tasksreconciler/tasksreconciler.go +++ /dev/null @@ -1,235 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package taskreconciler - -import ( - "fmt" - "time" - - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - bindings "github.com/mesos/mesos-go/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/proc" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/metrics" -) - -type Action func(driver bindings.SchedulerDriver, cancel <-chan struct{}) <-chan error - -type TasksReconciler interface { - RequestExplicit() - RequestImplicit() - Run(driver bindings.SchedulerDriver, done <-chan struct{}) -} - -type tasksReconciler struct { - proc.Doer - Action Action - explicit chan struct{} // send an empty struct to trigger explicit reconciliation - implicit chan struct{} // send an empty struct to trigger implicit reconciliation - cooldown time.Duration - explicitReconciliationAbortTimeout time.Duration -} - -func New(doer proc.Doer, action Action, - cooldown, explicitReconciliationAbortTimeout time.Duration, done <-chan struct{}) TasksReconciler { - return &tasksReconciler{ - Doer: doer, - explicit: make(chan struct{}, 1), - implicit: make(chan struct{}, 1), - cooldown: cooldown, - explicitReconciliationAbortTimeout: explicitReconciliationAbortTimeout, - Action: func(driver bindings.SchedulerDriver, cancel <-chan struct{}) <-chan error { - // trigged the reconciler action in the doer's execution context, - // but it could take a while and the scheduler needs to be able to - // process updates, the callbacks for which ALSO execute in the SAME - // deferred execution context -- so the action MUST be executed async. - errOnce := proc.NewErrorOnce(cancel) - return errOnce.Send(doer.Do(func() { - // only triggers the action if we're the currently elected, - // registered master and runs the action async. - go func() { - var err <-chan error - defer errOnce.Send(err) - err = action(driver, cancel) - }() - })).Err() - }, - } -} - -func (r *tasksReconciler) RequestExplicit() { - select { - case r.explicit <- struct{}{}: // noop - default: // request queue full; noop - } -} - -func (r *tasksReconciler) RequestImplicit() { - select { - case r.implicit <- struct{}{}: // noop - default: // request queue full; noop - } -} - -// execute task reconciliation, returns when r.done is closed. intended to run as a goroutine. -// if reconciliation is requested while another is in progress, the in-progress operation will be -// cancelled before the new reconciliation operation begins. -func (r *tasksReconciler) Run(driver bindings.SchedulerDriver, done <-chan struct{}) { - var cancel, finished chan struct{} -requestLoop: - for { - select { - case <-done: - return - default: // proceed - } - select { - case <-r.implicit: - metrics.ReconciliationRequested.WithLabelValues("implicit").Inc() - select { - case <-done: - return - case <-r.explicit: - break // give preference to a pending request for explicit - default: // continue - // don't run implicit reconciliation while explicit is ongoing - if finished != nil { - select { - case <-finished: // continue w/ implicit - default: - log.Infoln("skipping implicit reconcile because explicit reconcile is ongoing") - continue requestLoop - } - } - errOnce := proc.NewErrorOnce(done) - errCh := r.Do(func() { - var err error - defer errOnce.Report(err) - log.Infoln("implicit reconcile tasks") - metrics.ReconciliationExecuted.WithLabelValues("implicit").Inc() - if _, err = driver.ReconcileTasks([]*mesos.TaskStatus{}); err != nil { - log.V(1).Infof("failed to request implicit reconciliation from mesos: %v", err) - } - }) - proc.OnError(errOnce.Send(errCh).Err(), func(err error) { - log.Errorf("failed to run implicit reconciliation: %v", err) - }, done) - goto slowdown - } - case <-done: - return - case <-r.explicit: // continue - metrics.ReconciliationRequested.WithLabelValues("explicit").Inc() - } - - if cancel != nil { - close(cancel) - cancel = nil - - // play nice and wait for the prior operation to finish, complain - // if it doesn't - select { - case <-done: - return - case <-finished: // noop, expected - case <-time.After(r.explicitReconciliationAbortTimeout): // very unexpected - log.Error("reconciler action failed to stop upon cancellation") - } - } - // copy 'finished' to 'fin' here in case we end up with simultaneous go-routines, - // if cancellation takes too long or fails - we don't want to close the same chan - // more than once - cancel = make(chan struct{}) - finished = make(chan struct{}) - go func(fin chan struct{}) { - startedAt := time.Now() - defer func() { - metrics.ReconciliationLatency.Observe(metrics.InMicroseconds(time.Since(startedAt))) - }() - - metrics.ReconciliationExecuted.WithLabelValues("explicit").Inc() - defer close(fin) - err := <-r.Action(driver, cancel) - if err == errors.ReconciliationCancelledErr { - metrics.ReconciliationCancelled.WithLabelValues("explicit").Inc() - log.Infoln(err.Error()) - } else if err != nil { - log.Errorf("reconciler action failed: %v", err) - } - }(finished) - slowdown: - // don't allow reconciliation to run very frequently, either explicit or implicit - select { - case <-done: - return - case <-time.After(r.cooldown): // noop - } - } // for -} - -// MakeComposite invokes the given ReconcilerAction funcs in sequence, aborting the sequence if reconciliation -// is cancelled. if any other errors occur the composite reconciler will attempt to complete the -// sequence, reporting only the last generated error. -func MakeComposite(done <-chan struct{}, actions ...Action) Action { - if x := len(actions); x == 0 { - // programming error - panic("no actions specified for composite reconciler") - } else if x == 1 { - return actions[0] - } - chained := func(d bindings.SchedulerDriver, c <-chan struct{}, a, b Action) <-chan error { - ech := a(d, c) - ch := make(chan error, 1) - go func() { - select { - case <-done: - case <-c: - case e := <-ech: - if e != nil { - ch <- e - return - } - ech = b(d, c) - select { - case <-done: - case <-c: - case e := <-ech: - if e != nil { - ch <- e - return - } - close(ch) - return - } - } - ch <- fmt.Errorf("aborting composite reconciler action") - }() - return ch - } - result := func(d bindings.SchedulerDriver, c <-chan struct{}) <-chan error { - return chained(d, c, actions[0], actions[1]) - } - for i := 2; i < len(actions); i++ { - i := i - next := func(d bindings.SchedulerDriver, c <-chan struct{}) <-chan error { - return chained(d, c, Action(result), actions[i]) - } - result = next - } - return Action(result) -} diff --git a/contrib/mesos/pkg/scheduler/config/config.go b/contrib/mesos/pkg/scheduler/config/config.go deleted file mode 100644 index 9d1681a8107..00000000000 --- a/contrib/mesos/pkg/scheduler/config/config.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "io" - "time" - - "gopkg.in/gcfg.v1" -) - -const ( - DefaultOfferTTL = 5 * time.Second // duration an offer is viable, prior to being expired - DefaultOfferLingerTTL = 120 * time.Second // duration an expired offer lingers in history - DefaultListenerDelay = 1 * time.Second // duration between offer listener notifications - DefaultUpdatesBacklog = 2048 // size of the pod updates channel - DefaultFrameworkIdRefreshInterval = 30 * time.Second // interval we update the frameworkId stored in etcd - DefaultInitialImplicitReconciliationDelay = 15 * time.Second // wait this amount of time after initial registration before attempting implicit reconciliation - DefaultExplicitReconciliationMaxBackoff = 2 * time.Minute // interval in between internal task status checks/updates - DefaultExplicitReconciliationAbortTimeout = 30 * time.Second // waiting period after attempting to cancel an ongoing reconciliation - DefaultInitialPodBackoff = 1 * time.Second - DefaultMaxPodBackoff = 60 * time.Second - DefaultHttpHandlerTimeout = 10 * time.Second - DefaultHttpBindInterval = 5 * time.Second -) - -// Example scheduler configuration file: -// -// [scheduler] -// info-name = Kubernetes -// offer-ttl = 5s -// offer-linger-ttl = 2m - -type ConfigWrapper struct { - Scheduler Config -} - -type Config struct { - OfferTTL WrappedDuration `gcfg:"offer-ttl"` - OfferLingerTTL WrappedDuration `gcfg:"offer-linger-ttl"` - ListenerDelay WrappedDuration `gcfg:"listener-delay"` - UpdatesBacklog int `gcfg:"updates-backlog"` - FrameworkIdRefreshInterval WrappedDuration `gcfg:"framework-id-refresh-interval"` - InitialImplicitReconciliationDelay WrappedDuration `gcfg:"initial-implicit-reconciliation-delay"` - ExplicitReconciliationMaxBackoff WrappedDuration `gcfg:"explicit-reconciliation-max-backoff"` - ExplicitReconciliationAbortTimeout WrappedDuration `gcfg:"explicit-reconciliation-abort-timeout"` - InitialPodBackoff WrappedDuration `gcfg:"initial-pod-backoff"` - MaxPodBackoff WrappedDuration `gcfg:"max-pod-backoff"` - HttpHandlerTimeout WrappedDuration `gcfg:"http-handler-timeout"` - HttpBindInterval WrappedDuration `gcfg:"http-bind-interval"` -} - -type WrappedDuration struct { - time.Duration -} - -func (wd *WrappedDuration) UnmarshalText(data []byte) error { - d, err := time.ParseDuration(string(data)) - if err == nil { - wd.Duration = d - } - return err -} - -func (c *Config) SetDefaults() { - c.OfferTTL = WrappedDuration{DefaultOfferTTL} - c.OfferLingerTTL = WrappedDuration{DefaultOfferLingerTTL} - c.ListenerDelay = WrappedDuration{DefaultListenerDelay} - c.UpdatesBacklog = DefaultUpdatesBacklog - c.FrameworkIdRefreshInterval = WrappedDuration{DefaultFrameworkIdRefreshInterval} - c.InitialImplicitReconciliationDelay = WrappedDuration{DefaultInitialImplicitReconciliationDelay} - c.ExplicitReconciliationMaxBackoff = WrappedDuration{DefaultExplicitReconciliationMaxBackoff} - c.ExplicitReconciliationAbortTimeout = WrappedDuration{DefaultExplicitReconciliationAbortTimeout} - c.InitialPodBackoff = WrappedDuration{DefaultInitialPodBackoff} - c.MaxPodBackoff = WrappedDuration{DefaultMaxPodBackoff} - c.HttpHandlerTimeout = WrappedDuration{DefaultHttpHandlerTimeout} - c.HttpBindInterval = WrappedDuration{DefaultHttpBindInterval} -} - -func CreateDefaultConfig() *Config { - c := &Config{} - c.SetDefaults() - return c -} - -func (c *Config) Read(configReader io.Reader) error { - wrapper := &ConfigWrapper{Scheduler: *c} - if configReader != nil { - if err := gcfg.ReadInto(wrapper, configReader); err != nil { - return err - } - *c = wrapper.Scheduler - } - return nil -} diff --git a/contrib/mesos/pkg/scheduler/config/config_test.go b/contrib/mesos/pkg/scheduler/config/config_test.go deleted file mode 100644 index fe56ea1301e..00000000000 --- a/contrib/mesos/pkg/scheduler/config/config_test.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func is_default(c *Config, t *testing.T) { - assert := assert.New(t) - - assert.Equal(DefaultOfferTTL, c.OfferTTL.Duration) - assert.Equal(DefaultOfferLingerTTL, c.OfferLingerTTL.Duration) - assert.Equal(DefaultListenerDelay, c.ListenerDelay.Duration) - assert.Equal(DefaultUpdatesBacklog, c.UpdatesBacklog) - assert.Equal(DefaultFrameworkIdRefreshInterval, c.FrameworkIdRefreshInterval.Duration) - assert.Equal(DefaultInitialImplicitReconciliationDelay, c.InitialImplicitReconciliationDelay.Duration) - assert.Equal(DefaultExplicitReconciliationMaxBackoff, c.ExplicitReconciliationMaxBackoff.Duration) - assert.Equal(DefaultExplicitReconciliationAbortTimeout, c.ExplicitReconciliationAbortTimeout.Duration) - assert.Equal(DefaultInitialPodBackoff, c.InitialPodBackoff.Duration) - assert.Equal(DefaultMaxPodBackoff, c.MaxPodBackoff.Duration) - assert.Equal(DefaultHttpHandlerTimeout, c.HttpHandlerTimeout.Duration) - assert.Equal(DefaultHttpBindInterval, c.HttpBindInterval.Duration) -} - -// Check that SetDefaults sets the default values -func TestConfig_SetDefaults(t *testing.T) { - c := &Config{} - c.SetDefaults() - is_default(c, t) -} - -// Check that CreateDefaultConfig returns a default config -func TestConfig_CreateDefaultConfig(t *testing.T) { - c := CreateDefaultConfig() - is_default(c, t) -} - -// Check that a config string can be parsed -func TestConfig_Read(t *testing.T) { - assert := assert.New(t) - - c := CreateDefaultConfig() - reader := strings.NewReader(` - [scheduler] - offer-ttl=42s - offer-linger-ttl=42s - listener-delay=42s - updates-backlog=42 - framework-id-refresh-interval=42s - initial-implicit-reconciliation-delay=42s - explicit-reconciliation-max-backoff=42s - explicit-reconciliation-abort-timeout=42s - initial-pod-backoff=42s - max-pod-backoff=42s - http-handler-timeout=42s - http-bind-interval=42s - `) - err := c.Read(reader) - if err != nil { - t.Fatal("Cannot parse scheduler config: " + err.Error()) - } - - assert.Equal(42*time.Second, c.OfferTTL.Duration) - assert.Equal(42*time.Second, c.OfferLingerTTL.Duration) - assert.Equal(42*time.Second, c.ListenerDelay.Duration) - assert.Equal(42, c.UpdatesBacklog) - assert.Equal(42*time.Second, c.FrameworkIdRefreshInterval.Duration) - assert.Equal(42*time.Second, c.InitialImplicitReconciliationDelay.Duration) - assert.Equal(42*time.Second, c.ExplicitReconciliationMaxBackoff.Duration) - assert.Equal(42*time.Second, c.ExplicitReconciliationAbortTimeout.Duration) - assert.Equal(42*time.Second, c.InitialPodBackoff.Duration) - assert.Equal(42*time.Second, c.MaxPodBackoff.Duration) - assert.Equal(42*time.Second, c.HttpHandlerTimeout.Duration) - assert.Equal(42*time.Second, c.HttpBindInterval.Duration) -} - -// check that an invalid config is rejected and non of the values to overwritten -func TestConfig_ReadError(t *testing.T) { - assert := assert.New(t) - - c := CreateDefaultConfig() - reader := strings.NewReader(` - [scheduler] - offer-ttl = 42s - invalid-setting = 42s - `) - err := c.Read(reader) - if err == nil { - t.Fatal("Invalid scheduler config should lead to an error") - } - - assert.NotEqual(42*time.Second, c.OfferTTL.Duration) -} diff --git a/contrib/mesos/pkg/scheduler/config/doc.go b/contrib/mesos/pkg/scheduler/config/doc.go deleted file mode 100644 index a0b0a57cf3a..00000000000 --- a/contrib/mesos/pkg/scheduler/config/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package config provides mechanisms for low-level scheduler tuning. -package config // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/config" diff --git a/contrib/mesos/pkg/scheduler/constraint/constraint.go b/contrib/mesos/pkg/scheduler/constraint/constraint.go deleted file mode 100644 index 074979320aa..00000000000 --- a/contrib/mesos/pkg/scheduler/constraint/constraint.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constraint - -import ( - "encoding/json" - "fmt" -) - -type OperatorType int - -const ( - UniqueOperator OperatorType = iota - LikeOperator - ClusterOperator - GroupByOperator - UnlikeOperator -) - -var ( - labels = []string{ - "UNIQUE", - "LIKE", - "CLUSTER", - "GROUP_BY", - "UNLIKE", - } - - labelToType map[string]OperatorType -) - -func init() { - labelToType = make(map[string]OperatorType) - for i, s := range labels { - labelToType[s] = OperatorType(i) - } -} - -func (t OperatorType) String() string { - switch t { - case UniqueOperator, LikeOperator, ClusterOperator, GroupByOperator, UnlikeOperator: - return labels[int(t)] - default: - panic(fmt.Sprintf("unrecognized operator type: %d", int(t))) - } -} - -func parseOperatorType(s string) (OperatorType, error) { - t, found := labelToType[s] - if !found { - return UniqueOperator, fmt.Errorf("unrecognized operator %q", s) - } - return t, nil -} - -type Constraint struct { - Field string // required - Operator OperatorType // required - Value string // optional -} - -func (c *Constraint) MarshalJSON() ([]byte, error) { - var a []string - if c != nil { - if c.Value != "" { - a = append(a, c.Field, c.Operator.String(), c.Value) - } else { - a = append(a, c.Field, c.Operator.String()) - } - } - return json.Marshal(a) -} - -func (c *Constraint) UnmarshalJSON(buf []byte) (err error) { - var a []string - if err = json.Unmarshal(buf, &a); err != nil { - return err - } - switch x := len(a); { - case x < 2: - err = fmt.Errorf("not enough arguments to form constraint") - case x > 3: - err = fmt.Errorf("too many arguments to form constraint") - case x == 3: - c.Value = a[2] - fallthrough - case x == 2: - c.Field = a[0] - c.Operator, err = parseOperatorType(a[1]) - } - return err -} diff --git a/contrib/mesos/pkg/scheduler/constraint/constraint_test.go b/contrib/mesos/pkg/scheduler/constraint/constraint_test.go deleted file mode 100644 index c06f32426c2..00000000000 --- a/contrib/mesos/pkg/scheduler/constraint/constraint_test.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constraint - -import ( - "encoding/json" - "testing" -) - -func TestDeserialize(t *testing.T) { - shouldMatch := func(js string, field string, operator OperatorType, value string) (err error) { - constraint := Constraint{} - if err = json.Unmarshal(([]byte)(js), &constraint); err != nil { - return - } - if field != constraint.Field { - t.Fatalf("expected field %q instead of %q", field, constraint.Field) - } - if operator != constraint.Operator { - t.Fatalf("expected operator %v instead of %v", operator, constraint.Operator) - } - if value != constraint.Value { - t.Fatalf("expected value %q instead of %q", value, constraint.Value) - } - return - } - failOnError := func(err error) { - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - failOnError(shouldMatch(`["hostname","UNIQUE"]`, "hostname", UniqueOperator, "")) - failOnError(shouldMatch(`["rackid","GROUP_BY","1"]`, "rackid", GroupByOperator, "1")) - failOnError(shouldMatch(`["jdk","LIKE","7"]`, "jdk", LikeOperator, "7")) - failOnError(shouldMatch(`["jdk","UNLIKE","7"]`, "jdk", UnlikeOperator, "7")) - failOnError(shouldMatch(`["bob","CLUSTER","foo"]`, "bob", ClusterOperator, "foo")) - err := shouldMatch(`["bill","NOT_REALLY_AN_OPERATOR","pete"]`, "bill", ClusterOperator, "pete") - if err == nil { - t.Fatalf("expected unmarshalling error for invalid operator") - } -} - -func TestSerialize(t *testing.T) { - shouldMatch := func(expected string, constraint *Constraint) error { - data, err := json.Marshal(constraint) - if err != nil { - return err - } - js := string(data) - if js != expected { - t.Fatalf("expected json %q instead of %q", expected, js) - } - return nil - } - failOnError := func(err error) { - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - failOnError(shouldMatch(`["hostname","UNIQUE"]`, &Constraint{"hostname", UniqueOperator, ""})) - failOnError(shouldMatch(`["rackid","GROUP_BY","1"]`, &Constraint{"rackid", GroupByOperator, "1"})) - failOnError(shouldMatch(`["jdk","LIKE","7"]`, &Constraint{"jdk", LikeOperator, "7"})) - failOnError(shouldMatch(`["jdk","UNLIKE","7"]`, &Constraint{"jdk", UnlikeOperator, "7"})) - failOnError(shouldMatch(`["bob","CLUSTER","foo"]`, &Constraint{"bob", ClusterOperator, "foo"})) -} diff --git a/contrib/mesos/pkg/scheduler/constraint/doc.go b/contrib/mesos/pkg/scheduler/constraint/doc.go deleted file mode 100644 index 875a0a1d579..00000000000 --- a/contrib/mesos/pkg/scheduler/constraint/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package constraint exposes Marathon-like constraints for scheduling pods. -// Incomplete. -// TODO(jdef) We need better alignment between k8s-mesos and k8s scheduling -// constraints (read: a common constraints API). -package constraint // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/constraint" diff --git a/contrib/mesos/pkg/scheduler/doc.go b/contrib/mesos/pkg/scheduler/doc.go deleted file mode 100644 index 44ece764c38..00000000000 --- a/contrib/mesos/pkg/scheduler/doc.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package scheduler implements the Kubernetes Mesos scheduler. -package scheduler // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - -// Created from contrib/mesos/docs/scheduler.monopic: -// -// ┌───────────────────────────────────────────────────────────────────────┐ -// │ ┌───────────────────────────────────────┐ ┌─┴──────────────────────┐ ┌───────────────┐ -// ┌────────▼─────────┐ │Queuer │ Await() │ podUpdates │ │ │ -// │ podUpdatesBypass │ │- Yield() *api.Pod ├──pod CRUD ─▶ (queue.HistoricalFIFO) ◀──reflector──▶pods ListWatch ├──apiserver──▶ -// └────────▲─────────┘ │- Requeue(pod)/Dequeue(id)/Reoffer(pod)│ events │ │ │ │ -// │ └───────────────────▲───────────────────┘ └───────────┬────────────┘ └───────────────┘ -// │ │ │ -// │ │ │ -// └───────────────┐┌───────────────────▲────────────────────▲─────────────────────┐ └───────────────────────┐ -// ││ │ │ ┌────────────────────┼─────────────────┐ -// ┌───────────────────┼┼──────────────────────────────────────┐ │ ┌───────────────────┼────┼───────────┐ │ │ -// ┌───────────▼──────────┐┌───────┴┴───────┐ ┌───────────────────┐ ┌──┴─┴─┴──────┐ ┌────────┴────┴───┐ ┌────▼────────▼─────────────┐ │ -// │Binder (task launcher)││Deleter │ │PodReconciler │ │Controller │ │ ErrorHandler │ │SchedulerAlgorithm │ │ -// │- Bind(binding) ││- DeleteOne(pod)│ │- Reconcile(pod) │ │- Run() │ │- Error(pod, err)│ │- Schedule(pod) -> NodeName│ │ -// │ ││ │◀──│ │ │ │──▶│ │ │ │ │ -// │ ┌─────┐││ ┌─────┐ │ │ ┌─────┐ │ │ ┌─────┐ │ │ ┌─────┐ │ │┌─────┐ │ │ -// └───────────────┤sched├┘└────┤sched├─────┘ └──────┤sched├───▲──┘ └───┤sched├───┘ └────┤sched├──────┘ └┤sched├──────────────┬─────┘ │ -// ├-│││-┴──────┴--││-┴────────────────┴--│--┴───┼──────────┴--│--┴────────────┴-│---┴──────────┴-│││-┤ ┌────────────▼─────────▼─────────┐ -// │ │││ ││ │ │ │ │ │││ │ │ podScheduler │ -// │ ││└───────────▼┼─────────────────────▼──────┼─────────────▼─────────────────▼────────────────┘││ │ │ (e.g. fcfsPodScheduler) │ -// │ │└─────────────┼────────────────────────────┼─────────────┼──────────────────▼────────────────┘│ │ │ │ -// │ │ │ │ │ │ │ │ │ scheduleOne(pod, offers ...) │ -// │ │ │ │ │ │ │ │ │ ┌──────────────────────────┤ -// │ │ │ ╲ │ │ │ ╱ │ │ │ ▼ │ │ │ allocationStrategy │ -// │ │ │ ╲ └┐ │ ┌┘ ╱ │ │ │ │ │ │ - FitPredicate │ -// │ │ │ ╲ │ │ │ ╱ │ │ │ │ │ │ - Procurement │ -// │ │ │ ╲ └┐ │ ┌┘ ╱ │ │ │ │ └─────┴──────────────────────────┘ -// │┌▼────────────┐┌▼──────────┐┌─▼─▼─▼─▼─▼─┐┌───┴────────┐┌───▼───┐ ┌────▼───┐ │ -// ││LaunchTask(t)││KillTask(t)││sync.Mutex ││reconcile(t)││Tasks()│ │Offers()│ │ -// │└──────┬──────┘└─────┬─────┘└───────────┘└────────▲───┘└───┬───┘ └────┬───┘ │ -// │ │ │ │ │ │ │ -// │ │ └──────────────────┐ │ ┌───▼────────────┐ │ │ -// │ └──────────────────────────────┐ │ │ │podtask.Registry│ │ │ -// │ │ │ │ └────────────────┘ │ │ ┌──────────────────────┐ -// │ │ │ │ │ │ │ │ -// │Scheduler │ └──────┐ │ │ │ │ A ──────────▶ B │ -// └──────────────────────────────────────┼────────┼─┬│----┬──────────────────────┼───────────────────┘ │ │ -// ┌──────────────────────────────────────┼────────┼─┤sched├──────────────────────┼─────────────────────────┐ │ A has a reference │ -// │Framework │ │ └─────┘ ┌────▼───┐ │ │ on B and calls B │ -// │ ┌──────▼──────┐┌▼──────────┐ │Offers()│ │ │ │ -// │ │LaunchTask(t)││KillTask(t)│ └────┬───┘ │ └──────────────────────┘ -// │ └─────────┬───┘└──────┬────┘ ┌────────▼───────┐ │ -// │implements: mesos-go/scheduler.Scheduler └───────────▼ │offers.Registry │ │ -// │ │ └────────────────┘ │ -// │ ┌─────────────────┐ ┌──▼─────────────┐ │ -// └────────────────────────┤ ├───────┤ Mesos ├────────────────────────────────────┘ -// │ TasksReconciler │ │ Scheduler │ -// │ ├───────▶ Driver │ -// └─────────────────┘ └────────┬───────┘ -// │ -// │ -// ▼ diff --git a/contrib/mesos/pkg/scheduler/errors/doc.go b/contrib/mesos/pkg/scheduler/errors/doc.go deleted file mode 100644 index ebf501a9509..00000000000 --- a/contrib/mesos/pkg/scheduler/errors/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package errors contains all scheduler wide used errors -package errors // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/errors" diff --git a/contrib/mesos/pkg/scheduler/errors/errors.go b/contrib/mesos/pkg/scheduler/errors/errors.go deleted file mode 100644 index a7813839b03..00000000000 --- a/contrib/mesos/pkg/scheduler/errors/errors.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package errors - -import ( - "errors" -) - -var ( - NoSuchPodErr = errors.New("no such pod exists") - NoSuchTaskErr = errors.New("no such task exists") - ReconciliationCancelledErr = errors.New("explicit task reconciliation cancelled") - NoSuitableOffersErr = errors.New("no suitable offers for pod/task") -) diff --git a/contrib/mesos/pkg/scheduler/executorinfo/codec.go b/contrib/mesos/pkg/scheduler/executorinfo/codec.go deleted file mode 100644 index 3c21501529a..00000000000 --- a/contrib/mesos/pkg/scheduler/executorinfo/codec.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executorinfo - -import ( - "encoding/base64" - "io" - - "bufio" - - "github.com/gogo/protobuf/proto" - "github.com/mesos/mesos-go/mesosproto" -) - -var base64Codec = base64.StdEncoding - -// EncodeResources encodes the given resource slice to the given writer. -// The resource slice is encoded as a comma separated string of -// base64 encoded resource protobufs. -func EncodeResources(w io.Writer, rs []*mesosproto.Resource) error { - sep := "" - - for _, r := range rs { - _, err := io.WriteString(w, sep) - if err != nil { - return err - } - - buf, err := proto.Marshal(r) - if err != nil { - return err - } - - encoded := base64Codec.EncodeToString(buf) - _, err = io.WriteString(w, encoded) - if err != nil { - return err - } - - sep = "," - } - - return nil -} - -// DecodeResources decodes a resource slice from the given reader. -// The format is expected to be the same as in EncodeResources. -func DecodeResources(r io.Reader) (rs []*mesosproto.Resource, err error) { - delimited := bufio.NewReader(r) - rs = []*mesosproto.Resource{} - - for err != io.EOF { - var encoded string - encoded, err = delimited.ReadString(',') - - switch { - case err == io.EOF: - case err == nil: - encoded = encoded[:len(encoded)-1] - default: // err != nil && err != io.EOF - return nil, err - } - - decoded, err := base64Codec.DecodeString(encoded) - if err != nil { - return nil, err - } - - r := mesosproto.Resource{} - if err := proto.Unmarshal(decoded, &r); err != nil { - return nil, err - } - - rs = append(rs, &r) - } - - return rs, nil -} diff --git a/contrib/mesos/pkg/scheduler/executorinfo/codec_test.go b/contrib/mesos/pkg/scheduler/executorinfo/codec_test.go deleted file mode 100644 index f49715b05ab..00000000000 --- a/contrib/mesos/pkg/scheduler/executorinfo/codec_test.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executorinfo - -import ( - "bytes" - "reflect" - "testing" - - "github.com/mesos/mesos-go/mesosproto" - "github.com/mesos/mesos-go/mesosutil" -) - -func TestEncodeDecode(t *testing.T) { - want := []*mesosproto.Resource{ - scalar("cpus", 0.1, "*"), - scalar("mem", 64.0, "*"), - scalar("mem", 128.0, "public_slave"), - } - - var buf bytes.Buffer - if err := EncodeResources(&buf, want); err != nil { - t.Error(err) - } - - got, err := DecodeResources(&buf) - if err != nil { - t.Error(err) - } - - if ok := reflect.DeepEqual(want, got); !ok { - t.Errorf("want %v got %v", want, got) - } -} - -func TestEncodeDecodeNil(t *testing.T) { - var buf bytes.Buffer - if err := EncodeResources(&buf, nil); err != nil { - t.Error(err) - } - - if buf.String() != "" { - t.Errorf("expected empty string but got %q", buf.String()) - } - - if _, err := DecodeResources(&buf); err == nil { - t.Errorf("expected error but got none") - } -} - -func scalar(name string, value float64, role string) *mesosproto.Resource { - res := mesosutil.NewScalarResource(name, value) - res.Role = &role - return res -} diff --git a/contrib/mesos/pkg/scheduler/executorinfo/doc.go b/contrib/mesos/pkg/scheduler/executorinfo/doc.go deleted file mode 100644 index 0190f01ba91..00000000000 --- a/contrib/mesos/pkg/scheduler/executorinfo/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package executorinfo provides a lru-based executor info registry -// as well as some utility methods. -package executorinfo // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/executorinfo" diff --git a/contrib/mesos/pkg/scheduler/executorinfo/id.go b/contrib/mesos/pkg/scheduler/executorinfo/id.go deleted file mode 100644 index 392c11d2e6e..00000000000 --- a/contrib/mesos/pkg/scheduler/executorinfo/id.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executorinfo - -import ( - "bytes" - "fmt" - "hash/crc64" - "sort" - "strconv" - - "github.com/gogo/protobuf/proto" - "github.com/mesos/mesos-go/mesosproto" - mesos "github.com/mesos/mesos-go/mesosproto" - execcfg "k8s.io/kubernetes/contrib/mesos/pkg/executor/config" -) - -func NewID(info *mesosproto.ExecutorInfo) *mesosproto.ExecutorID { - eid := fmt.Sprintf("%x_%s", hash(info), execcfg.DefaultInfoID) - return &mesosproto.ExecutorID{Value: proto.String(eid)} -} - -// compute a hashcode for ExecutorInfo that may be used as a reasonable litmus test -// with respect to compatibility across HA schedulers. the intent is that an HA scheduler -// should fail-fast if it doesn't pass this test, rather than generating (potentially many) -// errors at run-time because a Mesos master decides that the ExecutorInfo generated by a -// secondary scheduler doesn't match that of the primary scheduler. -// -// Note: We intentionally leave out the Resources in this hash because they are -// set during procurement and should not lead to a different ExecutorId. -// This also means that the Resources do not contribute to offer -// compatibility checking. But as we persist and restore the Resources -// through node anotation we make sure that the right resources are chosen -// during task launch. -// -// see https://github.com/apache/mesos/blob/0.22.0/src/common/type_utils.cpp#L110 -func hash(info *mesos.ExecutorInfo) uint64 { - // !!! we specifically do NOT include: - // - Framework ID because it's a value that's initialized too late for us to use - // - Executor ID because it's a value that includes a copy of this hash - buf := &bytes.Buffer{} - buf.WriteString(info.GetName()) - buf.WriteString(info.GetSource()) - buf.Write(info.Data) - - if info.Command != nil { - buf.WriteString(info.Command.GetValue()) - buf.WriteString(info.Command.GetUser()) - buf.WriteString(strconv.FormatBool(info.Command.GetShell())) - if sz := len(info.Command.Arguments); sz > 0 { - x := make([]string, sz) - copy(x, info.Command.Arguments) - sort.Strings(x) - for _, item := range x { - buf.WriteString(item) - } - } - if vars := info.Command.Environment.GetVariables(); len(vars) > 0 { - names := []string{} - e := make(map[string]string) - - for _, v := range vars { - if name := v.GetName(); name != "" { - names = append(names, name) - e[name] = v.GetValue() - } - } - sort.Strings(names) - for _, n := range names { - buf.WriteString(n) - buf.WriteString("=") - buf.WriteString(e[n]) - } - } - if uris := info.Command.GetUris(); len(uris) > 0 { - su := []string{} - for _, uri := range uris { - su = append(su, fmt.Sprintf("%s%t%t", uri.GetValue(), uri.GetExecutable(), uri.GetExtract())) - } - sort.Strings(su) - for _, uri := range su { - buf.WriteString(uri) - } - } - //TODO(jdef) add support for Container - } - table := crc64.MakeTable(crc64.ECMA) - return crc64.Checksum(buf.Bytes(), table) -} diff --git a/contrib/mesos/pkg/scheduler/executorinfo/lru_cache.go b/contrib/mesos/pkg/scheduler/executorinfo/lru_cache.go deleted file mode 100644 index deb444c1890..00000000000 --- a/contrib/mesos/pkg/scheduler/executorinfo/lru_cache.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executorinfo - -import ( - "container/list" - "errors" - - "github.com/mesos/mesos-go/mesosproto" -) - -// Cache is an LRU cache for executor info objects. -// It is not safe for concurrent use. -type Cache struct { - maxEntries int - ll *list.List - cache map[string]*list.Element // by hostname -} - -type entry struct { - hostname string - info *mesosproto.ExecutorInfo -} - -// NewCache creates a new cache. -// If maxEntries is zero, an error is being returned. -func NewCache(maxEntries int) (*Cache, error) { - if maxEntries <= 0 { - return nil, errors.New("invalid maxEntries value") - } - - return &Cache{ - maxEntries: maxEntries, - ll: list.New(), // least recently used sorted linked list - cache: make(map[string]*list.Element), - }, nil -} - -// Add adds an executor info associated with the given hostname to the cache. -func (c *Cache) Add(hostname string, e *mesosproto.ExecutorInfo) { - if ee, ok := c.cache[hostname]; ok { - c.ll.MoveToFront(ee) - ee.Value.(*entry).info = e - return - } - el := c.ll.PushFront(&entry{hostname, e}) - c.cache[hostname] = el - if c.ll.Len() > c.maxEntries { - c.RemoveOldest() - } -} - -// Get looks up a hostname's executor info from the cache. -func (c *Cache) Get(hostname string) (e *mesosproto.ExecutorInfo, ok bool) { - if el, hit := c.cache[hostname]; hit { - c.ll.MoveToFront(el) - return el.Value.(*entry).info, true - } - return -} - -// Remove removes the provided hostname from the cache. -func (c *Cache) Remove(hostname string) { - if el, hit := c.cache[hostname]; hit { - c.removeElement(el) - } -} - -// RemoveOldest removes the oldest item from the cache. -func (c *Cache) RemoveOldest() { - oldest := c.ll.Back() - if oldest != nil { - c.removeElement(oldest) - } -} - -func (c *Cache) removeElement(el *list.Element) { - c.ll.Remove(el) - kv := el.Value.(*entry) - delete(c.cache, kv.hostname) -} diff --git a/contrib/mesos/pkg/scheduler/executorinfo/lru_cache_test.go b/contrib/mesos/pkg/scheduler/executorinfo/lru_cache_test.go deleted file mode 100644 index 30b050be1fc..00000000000 --- a/contrib/mesos/pkg/scheduler/executorinfo/lru_cache_test.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executorinfo - -import ( - "testing" - - "github.com/mesos/mesos-go/mesosproto" -) - -func TestLruCache(t *testing.T) { - c, err := NewCache(2) - if err != nil { - t.Fatal(err) - } - - e := &mesosproto.ExecutorInfo{} - - c.Add("foo", e) - c.Add("bar", e) - - if _, ok := c.Get("bar"); !ok { - t.Fatal(`expected "bar" but got none`) - } - - if _, ok := c.Get("foo"); !ok { - t.Fatal(`expected "foo" but got none`) - } - - c.Add("foo", e) - c.Add("baz", e) - - if _, ok := c.Get("bar"); ok { - t.Fatal(`expected none but got "bar"`) - } - - c.Remove("foo") - if _, ok := c.Get("foo"); ok { - t.Fatal(`expected none but got "foo"`) - } -} diff --git a/contrib/mesos/pkg/scheduler/executorinfo/registry.go b/contrib/mesos/pkg/scheduler/executorinfo/registry.go deleted file mode 100644 index 8a5dd3d8e71..00000000000 --- a/contrib/mesos/pkg/scheduler/executorinfo/registry.go +++ /dev/null @@ -1,178 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executorinfo - -import ( - "fmt" - "strings" - "sync" - - "github.com/gogo/protobuf/proto" - "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/contrib/mesos/pkg/node" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" -) - -// Registry is the interface that provides methods for interacting -// with a registry of ExecutorInfo objects -// -// Get looks up an ExecutorInfo object for the given hostname -// -// New returns an ExecutorInfo object based on a given hostname and resources -// -// Invalidate invalidates the given hostname from this registry. -// Note that a subsequent Get may recover the executor info. -type Registry interface { - New(hostname string, resources []*mesosproto.Resource) *mesosproto.ExecutorInfo - Get(hostname string) (*mesosproto.ExecutorInfo, error) - Invalidate(hostname string) -} - -// registry implements a map-based in-memory ExecutorInfo registry -type registry struct { - cache *Cache - mu sync.RWMutex // protects fields above - - lookupNode node.LookupFunc - prototype *mesosproto.ExecutorInfo -} - -// NewRegistry returns a new executorinfo registry. -// The given prototype is being used for properties other than resources. -func NewRegistry( - lookupNode node.LookupFunc, - prototype *mesosproto.ExecutorInfo, - cache *Cache, -) (Registry, error) { - if prototype == nil { - return nil, fmt.Errorf("no prototype given") - } - - if lookupNode == nil { - return nil, fmt.Errorf("no lookupNode given") - } - - if cache == nil { - return nil, fmt.Errorf("no cache given") - } - - return ®istry{ - cache: cache, - lookupNode: lookupNode, - prototype: prototype, - }, nil -} - -// New creates a customized ExecutorInfo for a host -// -// Note: New modifies Command.Arguments and Resources and intentionally -// does not update the executor id (although that originally depended on the -// command arguments and the resources). But as the hostname is constant for a -// given host, and the resources are compatible by the registry logic here this -// will not weaken our litmus test comparing the prototype ExecutorId with the -// id of running executors when an offer comes in. -func (r *registry) New( - hostname string, - resources []*mesosproto.Resource, -) *mesosproto.ExecutorInfo { - e := proto.Clone(r.prototype).(*mesosproto.ExecutorInfo) - e.Resources = resources - setCommandArgument(e, "--hostname-override", hostname) - - r.mu.Lock() - defer r.mu.Unlock() - - cached, ok := r.cache.Get(hostname) - if ok { - return cached - } - - r.cache.Add(hostname, e) - return e -} - -func (r *registry) Get(hostname string) (*mesosproto.ExecutorInfo, error) { - // first try to read from cached items - r.mu.RLock() - info, ok := r.cache.Get(hostname) - r.mu.RUnlock() - - if ok { - return info, nil - } - - result, err := r.resourcesFromNode(hostname) - if err != nil { - // master claims there is an executor with id, we cannot find any meta info - // => no way to recover this node - return nil, fmt.Errorf( - "failed to recover executor info for node %q, error: %v", - hostname, err, - ) - } - - return r.New(hostname, result), nil -} - -func (r *registry) Invalidate(hostname string) { - r.mu.Lock() - defer r.mu.Unlock() - - r.cache.Remove(hostname) -} - -// resourcesFromNode looks up ExecutorInfo resources for the given hostname and executorinfo ID -// or returns an error in case of failure. -func (r *registry) resourcesFromNode(hostname string) ([]*mesosproto.Resource, error) { - n := r.lookupNode(hostname) - if n == nil { - return nil, fmt.Errorf("hostname %q not found", hostname) - } - - encoded, ok := n.Annotations[meta.ExecutorResourcesKey] - if !ok { - return nil, fmt.Errorf( - "no %q annotation found in hostname %q", - meta.ExecutorResourcesKey, hostname, - ) - } - - return DecodeResources(strings.NewReader(encoded)) -} - -// setCommandArgument sets the given flag to the given value -// in the command arguments of the given executoringfo. -func setCommandArgument(ei *mesosproto.ExecutorInfo, flag, value string) { - if ei.Command == nil { - return - } - - argv := ei.Command.Arguments - overwrite := false - - for i, arg := range argv { - if strings.HasPrefix(arg, flag+"=") { - overwrite = true - argv[i] = flag + "=" + value - break - } - } - - if !overwrite { - ei.Command.Arguments = append(argv, flag+"="+value) - } -} diff --git a/contrib/mesos/pkg/scheduler/executorinfo/registry_test.go b/contrib/mesos/pkg/scheduler/executorinfo/registry_test.go deleted file mode 100644 index 0316f1a3b65..00000000000 --- a/contrib/mesos/pkg/scheduler/executorinfo/registry_test.go +++ /dev/null @@ -1,194 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package executorinfo - -import ( - "bytes" - "reflect" - "testing" - - "github.com/gogo/protobuf/proto" - "github.com/mesos/mesos-go/mesosproto" - "github.com/mesos/mesos-go/mesosutil" - "k8s.io/kubernetes/contrib/mesos/pkg/node" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/pkg/api" -) - -func TestRegistryGet(t *testing.T) { - var lookupFunc func() *api.Node - lookupNode := node.LookupFunc(func(hostname string) *api.Node { - return lookupFunc() - }) - - prototype := &mesosproto.ExecutorInfo{ - Resources: []*mesosproto.Resource{ - scalar("foo", 1.0, "role1"), - }, - } - - c, err := NewCache(1000) - if err != nil { - t.Error(err) - return - } - - r, err := NewRegistry(lookupNode, prototype, c) - if err != nil { - t.Error(err) - return - } - - var resources bytes.Buffer - EncodeResources(&resources, prototype.GetResources()) - - for i, tt := range []struct { - apiNode *api.Node - wantErr bool - }{ - { - apiNode: nil, - wantErr: true, - }, { - apiNode: &api.Node{}, - wantErr: true, - }, { - apiNode: &api.Node{ - ObjectMeta: api.ObjectMeta{ - Annotations: map[string]string{}, - }, - }, - wantErr: true, - }, { - apiNode: &api.Node{ - ObjectMeta: api.ObjectMeta{ - Annotations: map[string]string{ - meta.ExecutorResourcesKey: resources.String(), - }, - }, - }, - wantErr: false, - }, - } { - lookupFunc = func() *api.Node { return tt.apiNode } - _, err := r.Get("") - - if tt.wantErr && err == nil { - t.Errorf("test %d: want error but got none", i) - } - - if !tt.wantErr && err != nil { - t.Errorf("test %d error: %v", i, err) - } - } -} - -func TestRegistryNew(t *testing.T) { - for i, tt := range []struct { - prototype *mesosproto.ExecutorInfo - resources []*mesosproto.Resource - want *mesosproto.ExecutorInfo - }{ - { - prototype: &mesosproto.ExecutorInfo{ - ExecutorId: mesosutil.NewExecutorID("exec-id"), - }, - resources: nil, - want: &mesosproto.ExecutorInfo{ - ExecutorId: mesosutil.NewExecutorID("exec-id"), - }, - }, { - prototype: &mesosproto.ExecutorInfo{ - ExecutorId: mesosutil.NewExecutorID("exec-id"), - }, - resources: []*mesosproto.Resource{}, - want: &mesosproto.ExecutorInfo{ - ExecutorId: mesosutil.NewExecutorID("exec-id"), - Resources: []*mesosproto.Resource{}, - }, - }, { - prototype: &mesosproto.ExecutorInfo{ - ExecutorId: mesosutil.NewExecutorID("exec-id"), - Name: proto.String("foo"), - }, - - resources: []*mesosproto.Resource{ - scalar("foo", 1.0, "role1"), - scalar("bar", 2.0, "role2"), - }, - - want: &mesosproto.ExecutorInfo{ - ExecutorId: mesosutil.NewExecutorID("exec-id"), - Name: proto.String("foo"), - Resources: []*mesosproto.Resource{ - scalar("foo", 1.0, "role1"), - scalar("bar", 2.0, "role2"), - }, - }, - }, - } { - lookupNode := node.LookupFunc(func(string) *api.Node { - return nil - }) - - c, err := NewCache(1000) - if err != nil { - t.Error(err) - continue - } - - r, err := NewRegistry(lookupNode, tt.prototype, c) - if err != nil { - t.Error(err) - continue - } - - got := r.New("", tt.resources) - - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("test #%d\ngot %v\nwant %v", i, got, tt.want) - } - } -} - -func TestRegistryNewDup(t *testing.T) { - lookupNode := node.LookupFunc(func(string) *api.Node { - return nil - }) - - c, err := NewCache(1000) - if err != nil { - t.Error(err) - return - } - - r, err := NewRegistry(lookupNode, &mesosproto.ExecutorInfo{}, c) - if err != nil { - t.Error(err) - return - } - - new := r.New("", nil) - dup := r.New("", nil) - - if !reflect.DeepEqual(new, dup) { - t.Errorf( - "expected new == dup, but got new %v dup %v", - new, dup, - ) - } -} diff --git a/contrib/mesos/pkg/scheduler/ha/doc.go b/contrib/mesos/pkg/scheduler/ha/doc.go deleted file mode 100644 index 22af767b13b..00000000000 --- a/contrib/mesos/pkg/scheduler/ha/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package ha encapsulates high-availability scheduler concerns. -package ha // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/ha" diff --git a/contrib/mesos/pkg/scheduler/ha/election.go b/contrib/mesos/pkg/scheduler/ha/election.go deleted file mode 100644 index 1a044dc9c51..00000000000 --- a/contrib/mesos/pkg/scheduler/ha/election.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ha - -import ( - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/election" -) - -type roleType int - -const ( - followerRole roleType = iota - masterRole - retiredRole -) - -type candidateService struct { - sched *SchedulerProcess - newDriver DriverFactory - role roleType - valid ValidationFunc -} - -type ValidationFunc func(desiredUid, currentUid string) - -func NewCandidate(s *SchedulerProcess, f DriverFactory, v ValidationFunc) election.Service { - return &candidateService{ - sched: s, - newDriver: f, - role: followerRole, - valid: v, - } -} - -func (self *candidateService) Validate(desired, current election.Master) { - if self.valid != nil { - self.valid(string(desired), string(current)) - } -} - -func (self *candidateService) Start() { - if self.role == followerRole { - log.Info("elected as master") - self.role = masterRole - self.sched.Elect(self.newDriver) - } -} - -func (self *candidateService) Stop() { - if self.role == masterRole { - log.Info("retiring from master") - self.role = retiredRole - // order is important here, watchers of a SchedulerProcess will - // check SchedulerProcess.Failover() once Done() is closed. - close(self.sched.failover) - self.sched.End() - } -} diff --git a/contrib/mesos/pkg/scheduler/ha/ha.go b/contrib/mesos/pkg/scheduler/ha/ha.go deleted file mode 100644 index 6c7863f43e2..00000000000 --- a/contrib/mesos/pkg/scheduler/ha/ha.go +++ /dev/null @@ -1,285 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ha - -import ( - "fmt" - "sync/atomic" - - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - bindings "github.com/mesos/mesos-go/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/proc" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" -) - -type DriverFactory func() (bindings.SchedulerDriver, error) - -type stageType int32 - -const ( - initStage stageType = iota - standbyStage - masterStage - finStage -) - -func (stage *stageType) transition(from, to stageType) bool { - return atomic.CompareAndSwapInt32((*int32)(stage), int32(from), int32(to)) -} - -func (s *stageType) transitionTo(to stageType, unless ...stageType) bool { - if len(unless) == 0 { - atomic.StoreInt32((*int32)(s), int32(to)) - return true - } - for { - state := s.get() - for _, x := range unless { - if state == x { - return false - } - } - if s.transition(state, to) { - return true - } - } -} - -func (stage *stageType) get() stageType { - return stageType(atomic.LoadInt32((*int32)(stage))) -} - -// execute some action in the deferred context of the process, but only if we -// match the stage of the process at the time the action is executed. -func (stage stageType) Do(p *SchedulerProcess, a proc.Action) <-chan error { - errOnce := proc.NewErrorOnce(p.fin) - errOuter := p.Do(proc.Action(func() { - switch stage { - case standbyStage: - //await standby signal or death - select { - case <-p.standby: - case <-p.Done(): - } - case masterStage: - //await elected signal or death - select { - case <-p.elected: - case <-p.Done(): - } - case finStage: - errOnce.Reportf("scheduler process is dying, dropping action") - return - default: - } - errOnce.Report(stage.When(p, a)) - })) - return errOnce.Send(errOuter).Err() -} - -// execute some action only if we match the stage of the scheduler process -func (stage stageType) When(p *SchedulerProcess, a proc.Action) (err error) { - if stage != (&p.stage).get() { - err = fmt.Errorf("failed to execute deferred action, expected lifecycle stage %v instead of %v", stage, p.stage) - } else { - a() - } - return -} - -type SchedulerProcess struct { - proc.Process - bindings.Scheduler - stage stageType - elected chan struct{} // upon close we've been elected - failover chan struct{} // closed indicates that we should failover upon End() - standby chan struct{} - fin chan struct{} -} - -func New(framework bindings.Scheduler) *SchedulerProcess { - p := &SchedulerProcess{ - Process: proc.New(), - Scheduler: framework, - stage: initStage, - elected: make(chan struct{}), - failover: make(chan struct{}), - standby: make(chan struct{}), - fin: make(chan struct{}), - } - runtime.On(p.Running(), p.begin) - return p -} - -func (self *SchedulerProcess) begin() { - if (&self.stage).transition(initStage, standbyStage) { - close(self.standby) - log.Infoln("scheduler process entered standby stage") - } else { - log.Errorf("failed to transition from init to standby stage") - } -} - -func (self *SchedulerProcess) End() <-chan struct{} { - if (&self.stage).transitionTo(finStage, finStage) { - defer close(self.fin) - log.Infoln("scheduler process entered fin stage") - } - return self.Process.End() -} - -func (self *SchedulerProcess) Elect(newDriver DriverFactory) { - errOnce := proc.NewErrorOnce(self.fin) - proc.OnError(errOnce.Send(standbyStage.Do(self, proc.Action(func() { - if !(&self.stage).transition(standbyStage, masterStage) { - log.Errorf("failed to transition from standby to master stage, aborting") - self.End() - return - } - log.Infoln("scheduler process entered master stage") - drv, err := newDriver() - if err != nil { - log.Errorf("failed to fetch scheduler driver: %v", err) - self.End() - return - } - log.V(1).Infoln("starting driver...") - stat, err := drv.Start() - if stat == mesos.Status_DRIVER_RUNNING && err == nil { - log.Infoln("driver started successfully and is running") - close(self.elected) - go func() { - defer self.End() - _, err := drv.Join() - if err != nil { - log.Errorf("driver failed with error: %v", err) - } - errOnce.Report(err) - }() - return - } - defer self.End() - if err != nil { - log.Errorf("failed to start scheduler driver: %v", err) - } else { - log.Errorf("expected RUNNING status, not %v", stat) - } - }))).Err(), func(err error) { - defer self.End() - log.Errorf("failed to handle election event, aborting: %v", err) - }, self.fin) -} - -func (self *SchedulerProcess) Terminal() <-chan struct{} { - return self.fin -} - -func (self *SchedulerProcess) Elected() <-chan struct{} { - return self.elected -} - -func (self *SchedulerProcess) Failover() <-chan struct{} { - return self.failover -} - -type masterProcess struct { - *SchedulerProcess - doer proc.Doer -} - -func (self *masterProcess) Done() <-chan struct{} { - return self.SchedulerProcess.Terminal() -} - -func (self *masterProcess) Do(a proc.Action) <-chan error { - return self.doer.Do(a) -} - -// returns a Process instance that will only execute a proc.Action if the scheduler is the elected master -func (self *SchedulerProcess) Master() proc.Process { - return &masterProcess{ - SchedulerProcess: self, - doer: proc.DoWith(self, proc.DoerFunc(func(a proc.Action) <-chan error { - return proc.ErrorChan(masterStage.When(self, a)) - })), - } -} - -func (self *SchedulerProcess) logError(ch <-chan error) { - self.OnError(ch, func(err error) { - log.Errorf("failed to execute scheduler action: %v", err) - }) -} - -func (self *SchedulerProcess) Registered(drv bindings.SchedulerDriver, fid *mesos.FrameworkID, mi *mesos.MasterInfo) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.Registered(drv, fid, mi) - }))) -} - -func (self *SchedulerProcess) Reregistered(drv bindings.SchedulerDriver, mi *mesos.MasterInfo) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.Reregistered(drv, mi) - }))) -} - -func (self *SchedulerProcess) Disconnected(drv bindings.SchedulerDriver) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.Disconnected(drv) - }))) -} - -func (self *SchedulerProcess) ResourceOffers(drv bindings.SchedulerDriver, off []*mesos.Offer) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.ResourceOffers(drv, off) - }))) -} - -func (self *SchedulerProcess) OfferRescinded(drv bindings.SchedulerDriver, oid *mesos.OfferID) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.OfferRescinded(drv, oid) - }))) -} - -func (self *SchedulerProcess) StatusUpdate(drv bindings.SchedulerDriver, ts *mesos.TaskStatus) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.StatusUpdate(drv, ts) - }))) -} - -func (self *SchedulerProcess) FrameworkMessage(drv bindings.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, m string) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.FrameworkMessage(drv, eid, sid, m) - }))) -} - -func (self *SchedulerProcess) SlaveLost(drv bindings.SchedulerDriver, sid *mesos.SlaveID) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.SlaveLost(drv, sid) - }))) -} - -func (self *SchedulerProcess) ExecutorLost(drv bindings.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, x int) { - self.logError(self.Master().Do(proc.Action(func() { - self.Scheduler.ExecutorLost(drv, eid, sid, x) - }))) -} - -func (self *SchedulerProcess) Error(drv bindings.SchedulerDriver, msg string) { - self.Scheduler.Error(drv, msg) -} diff --git a/contrib/mesos/pkg/scheduler/integration/doc.go b/contrib/mesos/pkg/scheduler/integration/doc.go deleted file mode 100644 index 3efed18fbae..00000000000 --- a/contrib/mesos/pkg/scheduler/integration/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package integration implements integration tests. -package integration // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/integration" diff --git a/contrib/mesos/pkg/scheduler/integration/integration_test.go b/contrib/mesos/pkg/scheduler/integration/integration_test.go deleted file mode 100644 index 4e2a49c6be7..00000000000 --- a/contrib/mesos/pkg/scheduler/integration/integration_test.go +++ /dev/null @@ -1,872 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package integration - -import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "sync" - "testing" - "time" - - "github.com/gogo/protobuf/proto" - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - "github.com/mesos/mesos-go/mesosutil" - bindings "github.com/mesos/mesos-go/scheduler" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - assertext "k8s.io/kubernetes/contrib/mesos/pkg/assert" - "k8s.io/kubernetes/contrib/mesos/pkg/executor/messages" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/controller" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework" - schedcfg "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/config" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/ha" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask/hostport" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/testapi" - "k8s.io/kubernetes/pkg/api/unversioned" - "k8s.io/kubernetes/pkg/client/cache" - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - "k8s.io/kubernetes/pkg/client/restclient" - "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util/wait" - "k8s.io/kubernetes/pkg/watch" -) - -// A apiserver mock which partially mocks the pods API -type TestServer struct { - stats map[string]uint - nodes map[string]*api.Node - lock sync.Mutex // guards above fields - - server *httptest.Server - t *testing.T -} - -func (srv *TestServer) LookupNode(name string) *api.Node { - srv.lock.Lock() - defer srv.lock.Unlock() - - node, _ := api.Scheme.DeepCopy(srv.nodes[name]) - return node.(*api.Node) -} - -func (srv *TestServer) WaitForNode(name string) { - assertext.EventuallyTrue(srv.t, wait.ForeverTestTimeout, func() bool { - return srv.LookupNode(name) != nil - }) -} - -func NewTestServer(t *testing.T, namespace string, mockPodListWatch *MockPodsListWatch) *TestServer { - ts := TestServer{ - stats: map[string]uint{}, - nodes: map[string]*api.Node{}, - t: t, - } - mux := http.NewServeMux() - - podListHandler := func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - pods := mockPodListWatch.Pods() - w.Write([]byte(runtime.EncodeOrDie(testapi.Default.Codec(), &pods))) - } - mux.HandleFunc(testapi.Default.ResourcePath("pods", namespace, ""), podListHandler) - mux.HandleFunc(testapi.Default.ResourcePath("pods", "", ""), podListHandler) - - podsPrefix := testapi.Default.ResourcePath("pods", namespace, "") + "/" - mux.HandleFunc(podsPrefix, func(w http.ResponseWriter, r *http.Request) { - name := r.URL.Path[len(podsPrefix):] - - // update statistics for this pod - ts.lock.Lock() - defer ts.lock.Unlock() - ts.stats[name] = ts.stats[name] + 1 - - p := mockPodListWatch.Pod(name) - w.Header().Set("Content-Type", "application/json") - if p != nil { - w.WriteHeader(http.StatusOK) - w.Write([]byte(runtime.EncodeOrDie(testapi.Default.Codec(), p))) - return - } - w.WriteHeader(http.StatusNotFound) - }) - - mux.HandleFunc( - testapi.Default.ResourcePath("events", namespace, ""), - func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - }, - ) - - mux.HandleFunc( - testapi.Default.ResourcePath("nodes", "", ""), - func(w http.ResponseWriter, r *http.Request) { - var node api.Node - w.Header().Set("Content-Type", "application/json") - if err := json.NewDecoder(r.Body).Decode(&node); err != nil { - w.WriteHeader(http.StatusInternalServerError) - return - } - - ts.lock.Lock() - defer ts.lock.Unlock() - ts.nodes[node.Name] = &node - - if err := json.NewEncoder(w).Encode(node); err != nil { - w.WriteHeader(http.StatusInternalServerError) - return - } - w.WriteHeader(http.StatusOK) - }, - ) - - mux.HandleFunc("/", func(res http.ResponseWriter, req *http.Request) { - t.Errorf("unexpected request: %v", req.RequestURI) - res.Header().Set("Content-Type", "application/json") - res.WriteHeader(http.StatusNotFound) - }) - - ts.server = httptest.NewServer(mux) - return &ts -} - -func (ts *TestServer) Stats(name string) uint { - ts.lock.Lock() - defer ts.lock.Unlock() - - return ts.stats[name] -} - -// Create mock of pods ListWatch, usually listening on the apiserver pods watch endpoint -type MockPodsListWatch struct { - ListWatch cache.ListWatch - fakeWatcher *watch.FakeWatcher - list api.PodList - lock sync.Mutex -} - -func NewMockPodsListWatch(initialPodList api.PodList) *MockPodsListWatch { - lw := MockPodsListWatch{ - fakeWatcher: watch.NewFake(), - list: initialPodList, - } - lw.ListWatch = cache.ListWatch{ - WatchFunc: func(options api.ListOptions) (watch.Interface, error) { - return lw.fakeWatcher, nil - }, - ListFunc: func(options api.ListOptions) (runtime.Object, error) { - lw.lock.Lock() - defer lw.lock.Unlock() - - listCopy, err := api.Scheme.DeepCopy(&lw.list) - return listCopy.(*api.PodList), err - }, - } - return &lw -} - -func (lw *MockPodsListWatch) Pods() api.PodList { - lw.lock.Lock() - defer lw.lock.Unlock() - - obj, _ := api.Scheme.DeepCopy(&lw.list) - return *(obj.(*api.PodList)) -} - -func (lw *MockPodsListWatch) Pod(name string) *api.Pod { - lw.lock.Lock() - defer lw.lock.Unlock() - - for _, p := range lw.list.Items { - if p.Name == name { - clone, err := api.Scheme.DeepCopy(&p) - if err != nil { - panic(err.Error()) - } - return clone.(*api.Pod) - } - } - - return nil -} -func (lw *MockPodsListWatch) Add(pod *api.Pod, notify bool) { - clone, err := api.Scheme.DeepCopy(pod) - if err != nil { - panic(err.Error()) - } - - pod = clone.(*api.Pod) - func() { - lw.lock.Lock() - defer lw.lock.Unlock() - lw.list.Items = append(lw.list.Items, *pod) - }() - - if notify { - lw.fakeWatcher.Add(pod) - } -} -func (lw *MockPodsListWatch) Modify(pod *api.Pod, notify bool) { - clone, err := api.Scheme.DeepCopy(pod) - if err != nil { - panic("failed to clone pod object") - } - - pod = clone.(*api.Pod) - found := false - func() { - lw.lock.Lock() - defer lw.lock.Unlock() - - for i, otherPod := range lw.list.Items { - if otherPod.Name == pod.Name { - lw.list.Items[i] = *pod - found = true - return - } - } - log.Fatalf("Cannot find pod %v to modify in MockPodsListWatch", pod.Name) - }() - - if notify && found { - lw.fakeWatcher.Modify(pod) - } -} - -func (lw *MockPodsListWatch) Delete(pod *api.Pod, notify bool) { - var notifyPod *api.Pod - func() { - lw.lock.Lock() - defer lw.lock.Unlock() - - for i, otherPod := range lw.list.Items { - if otherPod.Name == pod.Name { - lw.list.Items = append(lw.list.Items[:i], lw.list.Items[i+1:]...) - notifyPod = &otherPod - return - } - } - log.Fatalf("Cannot find pod %v to delete in MockPodsListWatch", pod.Name) - }() - - if notifyPod != nil && notify { - lw.fakeWatcher.Delete(notifyPod) - } -} - -// Create a pod with a given index, requiring one port -var currentPodNum int = 0 - -func NewTestPod() (*api.Pod, int) { - currentPodNum = currentPodNum + 1 - name := fmt.Sprintf("pod%d", currentPodNum) - return &api.Pod{ - TypeMeta: unversioned.TypeMeta{APIVersion: testapi.Default.GroupVersion().String()}, - ObjectMeta: api.ObjectMeta{ - Name: name, - Namespace: api.NamespaceDefault, - SelfLink: fmt.Sprintf("http://1.2.3.4/api/v1beta1/pods/%s", name), - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Ports: []api.ContainerPort{ - { - ContainerPort: int32(8000 + currentPodNum), - Protocol: api.ProtocolTCP, - }, - }, - }, - }, - }, - Status: api.PodStatus{ - PodIP: fmt.Sprintf("1.2.3.%d", 4+currentPodNum), - Conditions: []api.PodCondition{ - { - Type: api.PodReady, - Status: api.ConditionTrue, - }, - }, - }, - }, currentPodNum -} - -// Offering some cpus and memory and the 8000-9000 port range -func NewTestOffer(id string) *mesos.Offer { - hostname := "some_hostname" - cpus := mesosutil.NewScalarResource("cpus", 3.75) - mem := mesosutil.NewScalarResource("mem", 940) - var port8000 uint64 = 8000 - var port9000 uint64 = 9000 - ports8000to9000 := mesos.Value_Range{Begin: &port8000, End: &port9000} - ports := mesosutil.NewRangesResource("ports", []*mesos.Value_Range{&ports8000to9000}) - return &mesos.Offer{ - Id: mesosutil.NewOfferID(id), - Hostname: &hostname, - SlaveId: mesosutil.NewSlaveID(hostname), - Resources: []*mesos.Resource{cpus, mem, ports}, - } -} - -// Add assertions to reason about event streams -type Event struct { - Object runtime.Object - Type string - Reason string - Message string -} - -type EventPredicate func(e Event) bool - -type EventAssertions struct { - assert.Assertions -} - -// EventObserver implements record.EventRecorder for the purposes of validation via EventAssertions. -type EventObserver struct { - fifo chan Event -} - -func NewEventObserver() *EventObserver { - return &EventObserver{ - fifo: make(chan Event, 1000), - } -} - -func (o *EventObserver) Event(object runtime.Object, eventtype, reason, message string) { - o.fifo <- Event{Object: object, Type: eventtype, Reason: reason, Message: message} -} - -func (o *EventObserver) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { - o.fifo <- Event{Object: object, Type: eventtype, Reason: reason, Message: fmt.Sprintf(messageFmt, args...)} -} -func (o *EventObserver) PastEventf(object runtime.Object, timestamp unversioned.Time, eventtype, reason, messageFmt string, args ...interface{}) { - o.fifo <- Event{Object: object, Type: eventtype, Reason: reason, Message: fmt.Sprintf(messageFmt, args...)} -} - -func (a *EventAssertions) Event(observer *EventObserver, pred EventPredicate, msgAndArgs ...interface{}) bool { - // parse msgAndArgs: first possibly a duration, otherwise a format string with further args - timeout := wait.ForeverTestTimeout - msg := "event not received" - msgArgStart := 0 - if len(msgAndArgs) > 0 { - switch msgAndArgs[0].(type) { - case time.Duration: - timeout = msgAndArgs[0].(time.Duration) - msgArgStart += 1 - } - } - if len(msgAndArgs) > msgArgStart { - msg = fmt.Sprintf(msgAndArgs[msgArgStart].(string), msgAndArgs[msgArgStart+1:]...) - } - - // watch events - result := make(chan bool) - stop := make(chan struct{}) - go func() { - for { - select { - case e, ok := <-observer.fifo: - if !ok { - result <- false - return - } else if pred(e) { - log.V(3).Infof("found asserted event for reason '%v': %v", e.Reason, e.Message) - result <- true - return - } else { - log.V(5).Infof("ignoring not-asserted event for reason '%v': %v", e.Reason, e.Message) - } - case _, ok := <-stop: - if !ok { - return - } - } - } - }() - defer close(stop) - - // wait for watch to match or timeout - select { - case matched := <-result: - return matched - case <-time.After(timeout): - return a.Fail(msg) - } -} - -func (a *EventAssertions) EventWithReason(observer *EventObserver, reason string, msgAndArgs ...interface{}) bool { - return a.Event(observer, func(e Event) bool { - return e.Reason == reason - }, msgAndArgs...) -} - -// Create mesos.TaskStatus for a given task -func newTaskStatusForTask(task *mesos.TaskInfo, state mesos.TaskState) *mesos.TaskStatus { - healthy := state == mesos.TaskState_TASK_RUNNING - ts := float64(time.Now().Nanosecond()) / 1000000000.0 - source := mesos.TaskStatus_SOURCE_EXECUTOR - return &mesos.TaskStatus{ - TaskId: task.TaskId, - State: &state, - SlaveId: task.SlaveId, - ExecutorId: task.Executor.ExecutorId, - Timestamp: &ts, - Healthy: &healthy, - Source: &source, - Data: task.Data, - } -} - -type LaunchedTask struct { - offerId mesos.OfferID - taskInfo *mesos.TaskInfo -} - -type lifecycleTest struct { - apiServer *TestServer - driver *framework.JoinableDriver - eventObs *EventObserver - podsListWatch *MockPodsListWatch - framework framework.Framework - schedulerProc *ha.SchedulerProcess - sched scheduler.Scheduler - t *testing.T -} - -type mockRegistry struct { - prototype *mesos.ExecutorInfo -} - -func (m mockRegistry) New(nodename string, rs []*mesos.Resource) *mesos.ExecutorInfo { - clone := proto.Clone(m.prototype).(*mesos.ExecutorInfo) - clone.Resources = rs - return clone -} - -func (m mockRegistry) Get(nodename string) (*mesos.ExecutorInfo, error) { - panic("N/A") -} - -func (m mockRegistry) Invalidate(hostname string) { - panic("N/A") -} - -func newLifecycleTest(t *testing.T) lifecycleTest { - assert := &EventAssertions{*assert.New(t)} - - // create a fake pod watch. We use that below to submit new pods to the scheduler - podsListWatch := NewMockPodsListWatch(api.PodList{}) - - // create fake apiserver - apiServer := NewTestServer(t, api.NamespaceDefault, podsListWatch) - - // create ExecutorInfo with some data for static pods if set - ei := mesosutil.NewExecutorInfo( - mesosutil.NewExecutorID("executor-id"), - mesosutil.NewCommandInfo("executor-cmd"), - ) - ei.Data = []byte{0, 1, 2} - - // create framework - client := clientset.NewForConfigOrDie(&restclient.Config{ - Host: apiServer.server.URL, - ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}, - }) - c := *schedcfg.CreateDefaultConfig() - fw := framework.New(framework.Config{ - ExecutorId: ei.GetExecutorId(), - Client: client, - SchedulerConfig: c, - LookupNode: apiServer.LookupNode, - }) - - // TODO(sttts): re-enable the following tests - // assert.NotNil(framework.client, "client is nil") - // assert.NotNil(framework.executor, "executor is nil") - // assert.NotNil(framework.offers, "offer registry is nil") - - // create pod scheduler - pr := podtask.NewDefaultProcurement(ei, mockRegistry{ei}) - fcfs := podschedulers.NewFCFSPodScheduler(pr, apiServer.LookupNode) - - // create scheduler process - schedulerProc := ha.New(fw) - - // create scheduler - eventObs := NewEventObserver() - scheduler := components.New( - &c, - fw, - fcfs, - client, - eventObs, - schedulerProc.Terminal(), - http.DefaultServeMux, - &podsListWatch.ListWatch, - podtask.Config{ - Prototype: ei, - FrameworkRoles: []string{"*"}, - DefaultPodRoles: []string{"*"}, - HostPortStrategy: hostport.StrategyWildcard, - }, - resources.DefaultDefaultContainerCPULimit, - resources.DefaultDefaultContainerMemLimit, - ) - assert.NotNil(scheduler) - - // create mock mesos scheduler driver - driver := &framework.JoinableDriver{} - - return lifecycleTest{ - apiServer: apiServer, - driver: driver, - eventObs: eventObs, - podsListWatch: podsListWatch, - framework: fw, - schedulerProc: schedulerProc, - sched: scheduler, - t: t, - } -} - -func (lt lifecycleTest) Start() <-chan LaunchedTask { - assert := &EventAssertions{*assert.New(lt.t)} - lt.sched.Run(lt.schedulerProc.Terminal()) - - // init framework - err := lt.framework.Init( - lt.sched, - lt.schedulerProc.Master(), - http.DefaultServeMux, - ) - assert.NoError(err) - - lt.driver.On("Start").Return(mesos.Status_DRIVER_RUNNING, nil).Once() - started := lt.driver.Upon() - - lt.driver.On("ReconcileTasks", - mock.AnythingOfType("[]*mesosproto.TaskStatus"), - ).Return(mesos.Status_DRIVER_RUNNING, nil) - - lt.driver.On("SendFrameworkMessage", - mock.AnythingOfType("*mesosproto.ExecutorID"), - mock.AnythingOfType("*mesosproto.SlaveID"), - mock.AnythingOfType("string"), - ).Return(mesos.Status_DRIVER_RUNNING, nil) - - launchedTasks := make(chan LaunchedTask, 1) - launchTasksFunc := func(args mock.Arguments) { - offerIDs := args.Get(0).([]*mesos.OfferID) - taskInfos := args.Get(1).([]*mesos.TaskInfo) - assert.Equal(1, len(offerIDs)) - assert.Equal(1, len(taskInfos)) - - launchedTasks <- LaunchedTask{ - offerId: *offerIDs[0], - taskInfo: taskInfos[0], - } - } - - lt.driver.On("LaunchTasks", - mock.AnythingOfType("[]*mesosproto.OfferID"), - mock.AnythingOfType("[]*mesosproto.TaskInfo"), - mock.AnythingOfType("*mesosproto.Filters"), - ).Return(mesos.Status_DRIVER_RUNNING, nil).Run(launchTasksFunc) - - lt.driver.On("DeclineOffer", - mock.AnythingOfType("*mesosproto.OfferID"), - mock.AnythingOfType("*mesosproto.Filters"), - ).Return(mesos.Status_DRIVER_RUNNING, nil) - - // elect master with mock driver - driverFactory := ha.DriverFactory(func() (bindings.SchedulerDriver, error) { - return lt.driver, nil - }) - lt.schedulerProc.Elect(driverFactory) - elected := lt.schedulerProc.Elected() - - // driver will be started - <-started - - // tell scheduler to be registered - lt.framework.Registered( - lt.driver, - mesosutil.NewFrameworkID("kubernetes-id"), - mesosutil.NewMasterInfo("master-id", (192<<24)+(168<<16)+(0<<8)+1, 5050), - ) - - // wait for being elected - <-elected - return launchedTasks -} - -func (lt lifecycleTest) Close() { - lt.apiServer.server.Close() -} - -func (lt lifecycleTest) End() <-chan struct{} { - return lt.schedulerProc.End() -} - -// TestScheduler_LifeCycle creates a scheduler plugin with the config returned by the scheduler, -// and plays through the whole life cycle of the plugin while creating pods, deleting -// and failing them. -func TestScheduler_LifeCycle(t *testing.T) { - assert := &EventAssertions{*assert.New(t)} - lt := newLifecycleTest(t) - defer lt.Close() - - // run plugin - launchedTasks := lt.Start() - defer lt.End() - - // fake new, unscheduled pod - pod, i := NewTestPod() - lt.podsListWatch.Add(pod, true) // notify watchers - - // wait for failedScheduling event because there is no offer - assert.EventWithReason(lt.eventObs, controller.FailedScheduling, "failedScheduling event not received") - - // add some matching offer - offers := []*mesos.Offer{NewTestOffer(fmt.Sprintf("offer%d", i))} - lt.framework.ResourceOffers(nil, offers) - - // first offer is declined because node is not available yet - lt.apiServer.WaitForNode("some_hostname") - - // add one more offer - lt.framework.ResourceOffers(nil, offers) - - // and wait for scheduled pod - assert.EventWithReason(lt.eventObs, controller.Scheduled) - select { - case launchedTask := <-launchedTasks: - // report back that the task has been staged, and then started by mesos - lt.framework.StatusUpdate( - lt.driver, - newTaskStatusForTask(launchedTask.taskInfo, mesos.TaskState_TASK_STAGING), - ) - - lt.framework.StatusUpdate( - lt.driver, - newTaskStatusForTask(launchedTask.taskInfo, mesos.TaskState_TASK_RUNNING), - ) - - // check that ExecutorInfo.data has the static pod data - assert.Len(launchedTask.taskInfo.Executor.Data, 3) - - // report back that the task has been lost - lt.driver.AssertNumberOfCalls(t, "SendFrameworkMessage", 0) - - lt.framework.StatusUpdate( - lt.driver, - newTaskStatusForTask(launchedTask.taskInfo, mesos.TaskState_TASK_LOST), - ) - - // and wait that framework message is sent to executor - lt.driver.AssertNumberOfCalls(t, "SendFrameworkMessage", 1) - - case <-time.After(wait.ForeverTestTimeout): - t.Fatalf("timed out waiting for launchTasks call") - } - - offeredNodes := make(map[string]struct{}) - - // Launch a pod and wait until the scheduler driver is called - schedulePodWithOffers := func(pod *api.Pod, offers []*mesos.Offer) (*api.Pod, *LaunchedTask, *mesos.Offer) { - // wait for failedScheduling event because there is no offer - assert.EventWithReason(lt.eventObs, controller.FailedScheduling, "failedScheduling event not received") - - // supply a matching offer - lt.framework.ResourceOffers(lt.driver, offers) - for _, offer := range offers { - if _, ok := offeredNodes[offer.GetHostname()]; !ok { - offeredNodes[offer.GetHostname()] = struct{}{} - lt.apiServer.WaitForNode(offer.GetHostname()) - - // reoffer since it must have been declined above - lt.framework.ResourceOffers(lt.driver, []*mesos.Offer{offer}) - } - } - - // and wait to get scheduled - assert.EventWithReason(lt.eventObs, controller.Scheduled) - - // wait for driver.launchTasks call - select { - case launchedTask := <-launchedTasks: - for _, offer := range offers { - if offer.Id.GetValue() == launchedTask.offerId.GetValue() { - return pod, &launchedTask, offer - } - } - t.Fatalf("unknown offer used to start a pod") - return nil, nil, nil - case <-time.After(wait.ForeverTestTimeout): - t.Fatal("timed out waiting for launchTasks") - return nil, nil, nil - } - } - - // Launch a pod and wait until the scheduler driver is called - launchPodWithOffers := func(pod *api.Pod, offers []*mesos.Offer) (*api.Pod, *LaunchedTask, *mesos.Offer) { - lt.podsListWatch.Add(pod, true) - return schedulePodWithOffers(pod, offers) - } - - // Launch a pod, wait until the scheduler driver is called and report back that it is running - startPodWithOffers := func(pod *api.Pod, offers []*mesos.Offer) (*api.Pod, *LaunchedTask, *mesos.Offer) { - // notify about pod, offer resources and wait for scheduling - pod, launchedTask, offer := launchPodWithOffers(pod, offers) - if pod != nil { - // report back status - lt.framework.StatusUpdate( - lt.driver, - newTaskStatusForTask(launchedTask.taskInfo, mesos.TaskState_TASK_STAGING), - ) - lt.framework.StatusUpdate( - lt.driver, - newTaskStatusForTask(launchedTask.taskInfo, mesos.TaskState_TASK_RUNNING), - ) - - return pod, launchedTask, offer - } - - return nil, nil, nil - } - - startTestPod := func() (*api.Pod, *LaunchedTask, *mesos.Offer) { - pod, i := NewTestPod() - offers := []*mesos.Offer{NewTestOffer(fmt.Sprintf("offer%d", i))} - return startPodWithOffers(pod, offers) - } - - // start another pod - pod, launchedTask, _ := startTestPod() - - // mock driver.KillTask, should be invoked when a pod is deleted - lt.driver.On("KillTask", - mock.AnythingOfType("*mesosproto.TaskID"), - ).Return(mesos.Status_DRIVER_RUNNING, nil).Run(func(args mock.Arguments) { - killedTaskId := *(args.Get(0).(*mesos.TaskID)) - assert.Equal(*launchedTask.taskInfo.TaskId, killedTaskId, "expected same TaskID as during launch") - }) - killTaskCalled := lt.driver.Upon() - - // stop it again via the apiserver mock - lt.podsListWatch.Delete(pod, true) // notify watchers - - // and wait for the driver killTask call with the correct TaskId - select { - case <-killTaskCalled: - // report back that the task is finished - lt.framework.StatusUpdate( - lt.driver, - newTaskStatusForTask(launchedTask.taskInfo, mesos.TaskState_TASK_FINISHED), - ) - - case <-time.After(wait.ForeverTestTimeout): - t.Fatal("timed out waiting for KillTask") - } - - // start a pod with on a given NodeName and check that it is scheduled to the right host - pod, i = NewTestPod() - pod.Spec.NodeName = "hostname1" - offers = []*mesos.Offer{} - for j := 0; j < 3; j++ { - offer := NewTestOffer(fmt.Sprintf("offer%d_%d", i, j)) - hostname := fmt.Sprintf("hostname%d", j) - offer.Hostname = &hostname - offers = append(offers, offer) - } - - _, _, usedOffer := startPodWithOffers(pod, offers) - - assert.Equal(offers[1].Id.GetValue(), usedOffer.Id.GetValue()) - assert.Equal(pod.Spec.NodeName, *usedOffer.Hostname) - - lt.framework.OfferRescinded(lt.driver, offers[0].Id) - lt.framework.OfferRescinded(lt.driver, offers[2].Id) - - // start pods: - // - which are failing while binding, - // - leading to reconciliation - // - with different states on the apiserver - - failPodFromExecutor := func(task *mesos.TaskInfo) { - beforePodLookups := lt.apiServer.Stats(pod.Name) - status := newTaskStatusForTask(task, mesos.TaskState_TASK_FAILED) - message := messages.CreateBindingFailure - status.Message = &message - lt.framework.StatusUpdate(lt.driver, status) - - // wait until pod is looked up at the apiserver - assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { - return lt.apiServer.Stats(pod.Name) == beforePodLookups+1 - }, "expect that reconcileTask will access apiserver for pod %v", pod.Name) - } - - launchTestPod := func() (*api.Pod, *LaunchedTask, *mesos.Offer) { - pod, i := NewTestPod() - offers := []*mesos.Offer{NewTestOffer(fmt.Sprintf("offer%d", i))} - return launchPodWithOffers(pod, offers) - } - - // 1. with pod deleted from the apiserver - // expected: pod is removed from internal task registry - pod, launchedTask, _ = launchTestPod() - lt.podsListWatch.Delete(pod, false) // not notifying the watchers - failPodFromExecutor(launchedTask.taskInfo) - - podKey, _ := podtask.MakePodKey(api.NewDefaultContext(), pod.Name) - assertext.EventuallyTrue(t, wait.ForeverTestTimeout, func() bool { - t, _ := lt.sched.Tasks().ForPod(podKey) - return t == nil - }) - - // 2. with pod still on the apiserver, not bound - // expected: pod is rescheduled - pod, launchedTask, _ = launchTestPod() - failPodFromExecutor(launchedTask.taskInfo) - - retryOffers := []*mesos.Offer{NewTestOffer("retry-offer")} - schedulePodWithOffers(pod, retryOffers) - - // 3. with pod still on the apiserver, bound, notified via ListWatch - // expected: nothing, pod updates not supported, compare ReconcileTask function - pod, launchedTask, usedOffer = startTestPod() - pod.Annotations = map[string]string{ - meta.BindingHostKey: *usedOffer.Hostname, - } - pod.Spec.NodeName = *usedOffer.Hostname - lt.podsListWatch.Modify(pod, true) // notifying the watchers - time.Sleep(time.Second / 2) - failPodFromExecutor(launchedTask.taskInfo) -} diff --git a/contrib/mesos/pkg/scheduler/meta/annotations.go b/contrib/mesos/pkg/scheduler/meta/annotations.go deleted file mode 100644 index 66e2e45a830..00000000000 --- a/contrib/mesos/pkg/scheduler/meta/annotations.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package meta - -// kubernetes api object annotations -const ( - // Namespace is the label and annotation namespace for mesos keys - Namespace = "k8s.mesosphere.io" - - // the BindingHostKey pod annotation marks a pod as being assigned to a Mesos - // slave. It is already or will be launched on the slave as a task. - BindingHostKey = Namespace + "/bindingHost" - - TaskIdKey = Namespace + "/taskId" - SlaveIdKey = Namespace + "/slaveId" - OfferIdKey = Namespace + "/offerId" - ExecutorIdKey = Namespace + "/executorId" - ExecutorResourcesKey = Namespace + "/executorResources" - PortMappingKey = Namespace + "/portMapping" - PortMappingKeyPrefix = Namespace + "/port_" - PortMappingKeyFormat = PortMappingKeyPrefix + "%s_%d" - PortNameMappingKeyPrefix = Namespace + "/portName_" - PortNameMappingKeyFormat = PortNameMappingKeyPrefix + "%s_%s" - ContainerPortKeyFormat = Namespace + "/containerPort_%s_%s_%d" - StaticPodFilenameKey = Namespace + "/staticPodFilename" - RolesKey = Namespace + "/roles" -) diff --git a/contrib/mesos/pkg/scheduler/meta/doc.go b/contrib/mesos/pkg/scheduler/meta/doc.go deleted file mode 100644 index f0a66ed324e..00000000000 --- a/contrib/mesos/pkg/scheduler/meta/doc.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package meta defines framework constants used as keys in k8s annotations -// that are attached to k8s pods. The scheduler uses some of these annotations -// for reconciliation upon failover. Other annotations are used as part of -// the host-to-pod port-mapping implementation understood by the k8s-mesos -// scheduler and custom endpoints-controller implementation. -package meta // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" diff --git a/contrib/mesos/pkg/scheduler/meta/store.go b/contrib/mesos/pkg/scheduler/meta/store.go deleted file mode 100644 index d2d83be0941..00000000000 --- a/contrib/mesos/pkg/scheduler/meta/store.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package meta - -const StoreChroot = "/k8sm" - -func ElectionPath(frameworkName string) string { - return StoreChroot + "/" + frameworkName + "/leader" -} diff --git a/contrib/mesos/pkg/scheduler/metrics/doc.go b/contrib/mesos/pkg/scheduler/metrics/doc.go deleted file mode 100644 index 4b07fe696ec..00000000000 --- a/contrib/mesos/pkg/scheduler/metrics/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package metrics defines and exposes instrumentation metrics of the scheduler. -package metrics // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/metrics" diff --git a/contrib/mesos/pkg/scheduler/metrics/metrics.go b/contrib/mesos/pkg/scheduler/metrics/metrics.go deleted file mode 100644 index d3f54072a81..00000000000 --- a/contrib/mesos/pkg/scheduler/metrics/metrics.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" -) - -const ( - schedulerSubsystem = "mesos_scheduler" -) - -var ( - QueueWaitTime = prometheus.NewSummary( - prometheus.SummaryOpts{ - Subsystem: schedulerSubsystem, - Name: "queue_wait_time_microseconds", - Help: "Launch queue wait time in microseconds", - }, - ) - BindLatency = prometheus.NewSummary( - prometheus.SummaryOpts{ - Subsystem: schedulerSubsystem, - Name: "bind_latency_microseconds", - Help: "Latency in microseconds between pod-task launch and pod binding.", - }, - ) - StatusUpdates = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: schedulerSubsystem, - Name: "status_updates", - Help: "Counter of TaskStatus updates, broken out by source, reason, state.", - }, - []string{"source", "reason", "state"}, - ) - ReconciliationLatency = prometheus.NewSummary( - prometheus.SummaryOpts{ - Subsystem: schedulerSubsystem, - Name: "reconciliation_latency_microseconds", - Help: "Latency in microseconds to execute explicit task reconciliation.", - }, - ) - ReconciliationRequested = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: schedulerSubsystem, - Name: "reconciliation_requested", - Help: "Counter of requested task reconciliations, broken out by kind.", - }, - []string{"kind"}, - ) - ReconciliationExecuted = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: schedulerSubsystem, - Name: "reconciliation_executed", - Help: "Counter of executed task reconciliations requests, broken out by kind.", - }, - []string{"kind"}, - ) - ReconciliationCancelled = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: schedulerSubsystem, - Name: "reconciliation_cancelled", - Help: "Counter of cancelled task reconciliations requests, broken out by kind.", - }, - []string{"kind"}, - ) -) - -var registerMetrics sync.Once - -func Register() { - registerMetrics.Do(func() { - prometheus.MustRegister(QueueWaitTime) - prometheus.MustRegister(BindLatency) - prometheus.MustRegister(StatusUpdates) - prometheus.MustRegister(ReconciliationLatency) - prometheus.MustRegister(ReconciliationRequested) - prometheus.MustRegister(ReconciliationExecuted) - prometheus.MustRegister(ReconciliationCancelled) - }) -} - -func InMicroseconds(d time.Duration) float64 { - return float64(d.Nanoseconds() / time.Microsecond.Nanoseconds()) -} diff --git a/contrib/mesos/pkg/scheduler/podtask/debug.go b/contrib/mesos/pkg/scheduler/podtask/debug.go deleted file mode 100644 index 93b9fd2ff5e..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/debug.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -import ( - "fmt" - "io" - "net/http" - - log "github.com/golang/glog" -) - -//TODO(jdef) we use a Locker to guard against concurrent task state changes, but it would be -//really, really nice to avoid doing this. Maybe someday the registry won't return data ptrs -//but plain structs instead. -func InstallDebugHandlers(reg Registry, mux *http.ServeMux) { - mux.HandleFunc("/debug/registry/tasks", func(w http.ResponseWriter, r *http.Request) { - //TODO(jdef) support filtering tasks based on status - alltasks := reg.List(nil) - io.WriteString(w, fmt.Sprintf("task_count=%d\n", len(alltasks))) - for _, task := range alltasks { - if err := func() (err error) { - podName := task.Pod.Name - podNamespace := task.Pod.Namespace - offerId := "" - if task.Offer != nil { - offerId = task.Offer.Id() - } - _, err = io.WriteString(w, fmt.Sprintf("%v\t%v/%v\t%v\t%v\n", task.ID, podNamespace, podName, task.State, offerId)) - return - }(); err != nil { - log.Warningf("aborting debug handler: %v", err) - break // stop listing on I/O errors - } - } - if flusher, ok := w.(http.Flusher); ok { - flusher.Flush() - } - }) -} diff --git a/contrib/mesos/pkg/scheduler/podtask/doc.go b/contrib/mesos/pkg/scheduler/podtask/doc.go deleted file mode 100644 index d0007da970b..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package podtask maps Kubernetes pods to Mesos tasks. -package podtask // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" diff --git a/contrib/mesos/pkg/scheduler/podtask/hostport/mapper.go b/contrib/mesos/pkg/scheduler/podtask/hostport/mapper.go deleted file mode 100644 index 5a0efbb33ed..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/hostport/mapper.go +++ /dev/null @@ -1,211 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hostport - -import ( - "fmt" - - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" -) - -// Objects implementing the Mapper interface generate port mappings -// from k8s container ports to ports offered by mesos -type Mapper interface { - // Map maps the given pod and the given mesos offer and returns a - // slice of port mappings or an error if the mapping failed - Map(pod *api.Pod, roles []string, offer *mesos.Offer) ([]Mapping, error) -} - -// MapperFunc is a function adapter to the Mapper interface -type MapperFunc func(*api.Pod, []string, *mesos.Offer) ([]Mapping, error) - -// Map calls f(t, offer) -func (f MapperFunc) Map(pod *api.Pod, roles []string, offer *mesos.Offer) ([]Mapping, error) { - return f(pod, roles, offer) -} - -// A Mapping represents the mapping between k8s container ports -// ports offered by mesos. It references the k8s' container and port -// and specifies the offered mesos port and the offered port's role -type Mapping struct { - ContainerIdx int // index of the container in the pod spec - PortIdx int // index of the port in a container's port spec - OfferPort uint64 // the port offered by mesos - Role string // the role asssociated with the offered port -} - -type PortAllocationError struct { - PodID string - Ports []uint64 -} - -func (err *PortAllocationError) Error() string { - return fmt.Sprintf("Could not schedule pod %s: %d port(s) could not be allocated", err.PodID, len(err.Ports)) -} - -type DuplicateError struct { - m1, m2 Mapping -} - -func (err *DuplicateError) Error() string { - return fmt.Sprintf( - "Host port %d is specified for container %d, pod %d and container %d, pod %d", - err.m1.OfferPort, err.m1.ContainerIdx, err.m1.PortIdx, err.m2.ContainerIdx, err.m2.PortIdx) -} - -// WildcardMapper maps k8s wildcard ports (hostPort == 0) to any available offer port -func WildcardMapper(pod *api.Pod, roles []string, offer *mesos.Offer) ([]Mapping, error) { - mapping, err := FixedMapper(pod, roles, offer) - if err != nil { - return nil, err - } - - taken := make(map[uint64]struct{}) - for _, entry := range mapping { - taken[entry.OfferPort] = struct{}{} - } - - wildports := []Mapping{} - for i, container := range pod.Spec.Containers { - for pi, port := range container.Ports { - if port.HostPort == 0 { - wildports = append(wildports, Mapping{ - ContainerIdx: i, - PortIdx: pi, - }) - } - } - } - - remaining := len(wildports) - resources.ForeachPortsRange(offer.GetResources(), roles, func(bp, ep uint64, role string) { - log.V(3).Infof("Searching for wildcard port in range {%d:%d}", bp, ep) - for i := range wildports { - if wildports[i].OfferPort != 0 { - continue - } - for port := bp; port <= ep && remaining > 0; port++ { - if _, inuse := taken[port]; inuse { - continue - } - wildports[i].OfferPort = port - wildports[i].Role = resources.CanonicalRole(role) - mapping = append(mapping, wildports[i]) - remaining-- - taken[port] = struct{}{} - break - } - } - }) - - if remaining > 0 { - err := &PortAllocationError{ - PodID: pod.Namespace + "/" + pod.Name, - } - // it doesn't make sense to include a port list here because they were all zero (wildcards) - return nil, err - } - - return mapping, nil -} - -// FixedMapper maps k8s host ports to offered ports ignoring hostPorts == 0 (remaining pod-private) -func FixedMapper(pod *api.Pod, roles []string, offer *mesos.Offer) ([]Mapping, error) { - requiredPorts := make(map[uint64]Mapping) - mapping := []Mapping{} - for i, container := range pod.Spec.Containers { - // strip all port==0 from this array; k8s already knows what to do with zero- - // ports (it does not create 'port bindings' on the minion-host); we need to - // remove the wildcards from this array since they don't consume host resources - for pi, port := range container.Ports { - if port.HostPort == 0 { - continue // ignore - } - m := Mapping{ - ContainerIdx: i, - PortIdx: pi, - OfferPort: uint64(port.HostPort), - } - if entry, inuse := requiredPorts[uint64(port.HostPort)]; inuse { - return nil, &DuplicateError{entry, m} - } - requiredPorts[uint64(port.HostPort)] = m - } - } - - resources.ForeachPortsRange(offer.GetResources(), roles, func(bp, ep uint64, role string) { - for port := range requiredPorts { - log.V(3).Infof("evaluating port range {%d:%d} %d", bp, ep, port) - if (bp <= port) && (port <= ep) { - m := requiredPorts[port] - m.Role = resources.CanonicalRole(role) - mapping = append(mapping, m) - delete(requiredPorts, port) - } - } - }) - - unsatisfiedPorts := len(requiredPorts) - if unsatisfiedPorts > 0 { - err := &PortAllocationError{ - PodID: pod.Namespace + "/" + pod.Name, - } - for p := range requiredPorts { - err.Ports = append(err.Ports, p) - } - return nil, err - } - - return mapping, nil -} - -type Strategy string - -const ( - // maps a Container.HostPort to the same exact offered host port, ignores .HostPort = 0 - StrategyFixed = Strategy("fixed") - // same as MappingFixed, except that .HostPort of 0 are mapped to any port offered - StrategyWildcard = Strategy("wildcard") -) - -var validStrategies = map[Strategy]MapperFunc{ - StrategyFixed: MapperFunc(FixedMapper), - StrategyWildcard: MapperFunc(WildcardMapper), -} - -// NewMapper returns a new mapper based on the port mapping key value -func (defaultStrategy Strategy) NewMapper(pod *api.Pod) Mapper { - strategy, ok := pod.Labels[meta.PortMappingKey] - if ok { - f, ok := validStrategies[Strategy(strategy)] - if ok { - return f - } - log.Warningf("invalid port mapping strategy %q, reverting to default %q", strategy, defaultStrategy) - } - - f, ok := validStrategies[defaultStrategy] - if ok { - return f - } - - panic("scheduler is misconfigured, unrecognized default strategy \"" + defaultStrategy + "\"") -} diff --git a/contrib/mesos/pkg/scheduler/podtask/hostport/mapper_test.go b/contrib/mesos/pkg/scheduler/podtask/hostport/mapper_test.go deleted file mode 100644 index 9d1214fba2c..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/hostport/mapper_test.go +++ /dev/null @@ -1,207 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hostport - -import ( - "testing" - - mesos "github.com/mesos/mesos-go/mesosproto" - "github.com/mesos/mesos-go/mesosutil" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" -) - -func TestDefaultHostPortMatching(t *testing.T) { - pod := &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "foo", - Namespace: "default", - }, - } - - offer := &mesos.Offer{ - Resources: []*mesos.Resource{ - resources.NewPorts("*", 1, 1), - }, - } - mapping, err := FixedMapper(pod, []string{"*"}, offer) - if err != nil { - t.Fatal(err) - } - if len(mapping) > 0 { - t.Fatalf("Found mappings for a pod without ports: %v", pod) - } - - //-- - pod.Spec = api.PodSpec{ - Containers: []api.Container{{ - Ports: []api.ContainerPort{{ - HostPort: 123, - }, { - HostPort: 123, - }}, - }}, - } - _, err = FixedMapper(pod, []string{"*"}, offer) - if err, _ := err.(*DuplicateError); err == nil { - t.Fatal("Expected duplicate port error") - } else if err.m1.OfferPort != 123 { - t.Fatal("Expected duplicate host port 123") - } -} - -func TestWildcardHostPortMatching(t *testing.T) { - pod := &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "foo", - Namespace: "default", - }, - } - - offer := &mesos.Offer{} - mapping, err := WildcardMapper(pod, []string{"*"}, offer) - if err != nil { - t.Fatal(err) - } - if len(mapping) > 0 { - t.Fatalf("Found mappings for an empty offer and a pod without ports: %v", pod) - } - - //-- - offer = &mesos.Offer{ - Resources: []*mesos.Resource{ - resources.NewPorts("*", 1, 1), - }, - } - mapping, err = WildcardMapper(pod, []string{"*"}, offer) - if err != nil { - t.Fatal(err) - } - if len(mapping) > 0 { - t.Fatalf("Found mappings for a pod without ports: %v", pod) - } - - //-- - pod.Spec = api.PodSpec{ - Containers: []api.Container{{ - Ports: []api.ContainerPort{{ - HostPort: 123, - }}, - }}, - } - mapping, err = WildcardMapper(pod, []string{"*"}, offer) - if err == nil { - t.Fatalf("expected error instead of mappings: %#v", mapping) - } else if err, _ := err.(*PortAllocationError); err == nil { - t.Fatal("Expected port allocation error") - } else if !(len(err.Ports) == 1 && err.Ports[0] == 123) { - t.Fatal("Expected port allocation error for host port 123") - } - - //-- - pod.Spec = api.PodSpec{ - Containers: []api.Container{{ - Ports: []api.ContainerPort{{ - HostPort: 0, - }, { - HostPort: 123, - }}, - }}, - } - mapping, err = WildcardMapper(pod, []string{"*"}, offer) - if err, _ := err.(*PortAllocationError); err == nil { - t.Fatal("Expected port allocation error") - } else if !(len(err.Ports) == 1 && err.Ports[0] == 123) { - t.Fatal("Expected port allocation error for host port 123") - } - - //-- - pod.Spec = api.PodSpec{ - Containers: []api.Container{{ - Ports: []api.ContainerPort{{ - HostPort: 0, - }, { - HostPort: 1, - }}, - }}, - } - mapping, err = WildcardMapper(pod, []string{"*"}, offer) - if err, _ := err.(*PortAllocationError); err == nil { - t.Fatal("Expected port allocation error") - } else if len(err.Ports) != 0 { - t.Fatal("Expected port allocation error for wildcard port") - } - - //-- - offer = &mesos.Offer{ - Resources: []*mesos.Resource{ - resources.NewPorts("*", 1, 2), - }, - } - mapping, err = WildcardMapper(pod, []string{"*"}, offer) - if err != nil { - t.Fatal(err) - } else if len(mapping) != 2 { - t.Fatal("Expected both ports allocated") - } - valid := 0 - for _, entry := range mapping { - if entry.ContainerIdx == 0 && entry.PortIdx == 0 && entry.OfferPort == 2 { - valid++ - } - if entry.ContainerIdx == 0 && entry.PortIdx == 1 && entry.OfferPort == 1 { - valid++ - } - } - if valid < 2 { - t.Fatalf("Expected 2 valid port mappings, not %d", valid) - } - - //-- port mapping in case of multiple discontinuous port ranges in mesos offer - pod.Spec = api.PodSpec{ - Containers: []api.Container{{ - Ports: []api.ContainerPort{{ - HostPort: 0, - }, { - HostPort: 0, - }}, - }}, - } - offer = &mesos.Offer{ - Resources: []*mesos.Resource{ - mesosutil.NewRangesResource("ports", []*mesos.Value_Range{mesosutil.NewValueRange(1, 1), mesosutil.NewValueRange(3, 5)}), - }, - } - mapping, err = WildcardMapper(pod, []string{"*"}, offer) - if err != nil { - t.Fatal(err) - } else if len(mapping) != 2 { - t.Fatal("Expected both ports allocated") - } - valid = 0 - for _, entry := range mapping { - if entry.ContainerIdx == 0 && entry.PortIdx == 0 && entry.OfferPort == 1 { - valid++ - } - if entry.ContainerIdx == 0 && entry.PortIdx == 1 && entry.OfferPort == 3 { - valid++ - } - } - if valid < 2 { - t.Fatalf("Expected 2 valid port mappings, not %d", valid) - } -} diff --git a/contrib/mesos/pkg/scheduler/podtask/leaky.go b/contrib/mesos/pkg/scheduler/podtask/leaky.go deleted file mode 100644 index ec973e854cf..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/leaky.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -// Concepts that have leaked to where they should not have. - -import ( - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/registry/generic/registry" -) - -// makePodKey constructs etcd paths to pod items enforcing namespace rules. -func MakePodKey(ctx api.Context, id string) (string, error) { - return registry.NamespaceKeyFunc(ctx, PodPath, id) -} diff --git a/contrib/mesos/pkg/scheduler/podtask/pod_task.go b/contrib/mesos/pkg/scheduler/podtask/pod_task.go deleted file mode 100644 index 24bb8aa73d2..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/pod_task.go +++ /dev/null @@ -1,381 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/pborman/uuid" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - mesosmeta "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/metrics" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask/hostport" - "k8s.io/kubernetes/pkg/api" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - mutil "github.com/mesos/mesos-go/mesosutil" -) - -type StateType int - -const ( - StatePending StateType = iota - StateRunning - StateFinished - StateUnknown -) - -type FlagType string - -const ( - Launched = FlagType("launched") - Bound = FlagType("bound") - Deleted = FlagType("deleted") -) - -var starRole = []string{"*"} - -// Config represents elements that are used or required in order to -// create a pod task that may be scheduled. -type Config struct { - ID string // ID is an optional, unique task ID; auto-generated if not specified - DefaultPodRoles []string // DefaultPodRoles lists preferred resource groups, prioritized in order - FrameworkRoles []string // FrameworkRoles identify resource groups from which the framework may consume - Prototype *mesos.ExecutorInfo // Prototype is required - HostPortStrategy hostport.Strategy // HostPortStrategy is used as the port mapping strategy, unless overridden by the pod - GenerateTaskDiscoveryEnabled bool - mapper hostport.Mapper // host-port mapping func, derived from pod and default strategy - podKey string // k8s key for this pod; managed internally -} - -// A struct that describes a pod task. -type T struct { - Config - Pod api.Pod - - // Stores the final procurement result, once set read-only. - // Meant to be set by algorith.SchedulerAlgorithm only. - Spec *Spec - - Offer offers.Perishable // thread-safe - State StateType - Flags map[FlagType]struct{} - CreateTime time.Time - UpdatedTime time.Time // time of the most recent StatusUpdate we've seen from the mesos master - - podStatus api.PodStatus - launchTime time.Time - bindTime time.Time -} - -type Spec struct { - SlaveID string - AssignedSlave string - Resources []*mesos.Resource - PortMap []hostport.Mapping - Data []byte - Executor *mesos.ExecutorInfo -} - -// mostly-clone this pod task. the clone will actually share the some fields: -// - executor // OK because it's read only -// - Offer // OK because it's guarantees safe concurrent access -func (t *T) Clone() *T { - if t == nil { - return nil - } - - // shallow-copy - clone := *t - - // deep copy - clone.Flags = map[FlagType]struct{}{} - for k := range t.Flags { - clone.Flags[k] = struct{}{} - } - return &clone -} - -func (t *T) HasAcceptedOffer() bool { - return t.Spec != nil -} - -func (t *T) GetOfferId() string { - if t.Offer == nil { - return "" - } - return t.Offer.Details().Id.GetValue() -} - -func generateTaskName(pod *api.Pod) string { - ns := pod.Namespace - if ns == "" { - ns = api.NamespaceDefault - } - return fmt.Sprintf("%s.%s.pod", pod.Name, ns) -} - -func generateTaskDiscovery(pod *api.Pod) *mesos.DiscoveryInfo { - di := &mesos.DiscoveryInfo{ - Visibility: mesos.DiscoveryInfo_CLUSTER.Enum(), - } - switch visibility := pod.Annotations[mesosmeta.Namespace+"/discovery-visibility"]; visibility { - case "framework": - di.Visibility = mesos.DiscoveryInfo_FRAMEWORK.Enum() - case "external": - di.Visibility = mesos.DiscoveryInfo_EXTERNAL.Enum() - case "", "cluster": - // noop, pick the default we already set - default: - // default to CLUSTER, just warn the user - log.Warningf("unsupported discovery-visibility annotation: %q", visibility) - } - // name should be {{label|annotation}:name}.{pod:namespace}.pod - nameDecorator := func(n string) *string { - ns := pod.Namespace - if ns == "" { - ns = api.NamespaceDefault - } - x := n + "." + ns + "." + "pod" - return &x - } - for _, tt := range []struct { - fieldName string - dest **string - decorator func(string) *string - }{ - {"name", &di.Name, nameDecorator}, - {"environment", &di.Environment, nil}, - {"location", &di.Location, nil}, - {"version", &di.Version, nil}, - } { - d := tt.decorator - if d == nil { - d = func(s string) *string { return &s } - } - if v, ok := pod.Labels[tt.fieldName]; ok && v != "" { - *tt.dest = d(v) - } - if v, ok := pod.Annotations[mesosmeta.Namespace+"/discovery-"+tt.fieldName]; ok && v != "" { - *tt.dest = d(v) - } - } - return di -} - -func (t *T) BuildTaskInfo() (*mesos.TaskInfo, error) { - if t.Spec == nil { - return nil, errors.New("no podtask.T.Spec given, cannot build task info") - } - - info := &mesos.TaskInfo{ - Name: proto.String(generateTaskName(&t.Pod)), - TaskId: mutil.NewTaskID(t.ID), - Executor: t.Spec.Executor, - Data: t.Spec.Data, - Resources: t.Spec.Resources, - SlaveId: mutil.NewSlaveID(t.Spec.SlaveID), - } - - if t.GenerateTaskDiscoveryEnabled { - info.Discovery = generateTaskDiscovery(&t.Pod) - } - - return info, nil -} - -// Clear offer-related details from the task, should be called if/when an offer -// has already been assigned to a task but for some reason is no longer valid. -func (t *T) Reset() { - log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.Name) - t.Offer = nil - t.Spec = nil -} - -func (t *T) Set(f FlagType) { - t.Flags[f] = struct{}{} - if Launched == f { - t.launchTime = time.Now() - queueWaitTime := t.launchTime.Sub(t.CreateTime) - metrics.QueueWaitTime.Observe(metrics.InMicroseconds(queueWaitTime)) - } -} - -func (t *T) Has(f FlagType) (exists bool) { - _, exists = t.Flags[f] - return -} - -// Roles returns the valid roles under which this pod task can be scheduled. -// If the pod has roles annotations defined they are being used -// else default pod roles are being returned. -func (t *T) Roles() (result []string) { - if r, ok := t.Pod.ObjectMeta.Annotations[mesosmeta.RolesKey]; ok { - roles := strings.Split(r, ",") - - for i, r := range roles { - roles[i] = strings.TrimSpace(r) - } - - return filterRoles( - roles, - not(emptyRole), not(seenRole()), inRoles(t.FrameworkRoles...), - ) - } - - // no roles label defined, return defaults - return t.DefaultPodRoles -} - -func New(ctx api.Context, config Config, pod *api.Pod) (*T, error) { - if config.Prototype == nil { - return nil, fmt.Errorf("illegal argument: executor-info prototype is nil") - } - - if len(config.FrameworkRoles) == 0 { - config.FrameworkRoles = starRole - } - - if len(config.DefaultPodRoles) == 0 { - config.DefaultPodRoles = starRole - } - - key, err := MakePodKey(ctx, pod.Name) - if err != nil { - return nil, err - } - config.podKey = key - - if config.ID == "" { - config.ID = "pod." + uuid.NewUUID().String() - } - - // the scheduler better get the fallback strategy right, otherwise we panic here - config.mapper = config.HostPortStrategy.NewMapper(pod) - - task := &T{ - Pod: *pod, - Config: config, - State: StatePending, - Flags: make(map[FlagType]struct{}), - } - task.CreateTime = time.Now() - - return task, nil -} - -func (t *T) SaveRecoveryInfo(dict map[string]string) { - dict[mesosmeta.TaskIdKey] = t.ID - dict[mesosmeta.SlaveIdKey] = t.Spec.SlaveID - dict[mesosmeta.OfferIdKey] = t.Offer.Details().Id.GetValue() - dict[mesosmeta.ExecutorIdKey] = t.Spec.Executor.ExecutorId.GetValue() -} - -// reconstruct a task from metadata stashed in a pod entry. there are limited pod states that -// support reconstruction. if we expect to be able to reconstruct state but encounter errors -// in the process then those errors are returned. if the pod is in a seemingly valid state but -// otherwise does not support task reconstruction return false. if we're able to reconstruct -// state then return a reconstructed task and true. -// -// at this time task reconstruction is only supported for pods that have been annotated with -// binding metadata, which implies that they've previously been associated with a task and -// that mesos knows about it. -// -// assumes that the pod data comes from the k8s registry and reflects the desired state. -// -func RecoverFrom(pod api.Pod) (*T, bool, error) { - // we only expect annotations if pod has been bound, which implies that it has already - // been scheduled and launched - if len(pod.Annotations) == 0 { - log.V(1).Infof("skipping recovery for unbound pod %v/%v", pod.Namespace, pod.Name) - return nil, false, nil - } - - // we don't track mirror pods, they're considered part of the executor - if _, isMirrorPod := pod.Annotations[kubetypes.ConfigMirrorAnnotationKey]; isMirrorPod { - log.V(1).Infof("skipping recovery for mirror pod %v/%v", pod.Namespace, pod.Name) - return nil, false, nil - } - - // only process pods that are not in a terminal state - switch pod.Status.Phase { - case api.PodPending, api.PodRunning, api.PodUnknown: // continue - default: - log.V(1).Infof("skipping recovery for terminal pod %v/%v", pod.Namespace, pod.Name) - return nil, false, nil - } - - ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace) - key, err := MakePodKey(ctx, pod.Name) - if err != nil { - return nil, false, err - } - - //TODO(jdef) recover ports (and other resource requirements?) from the pod spec as well - - now := time.Now() - t := &T{ - Config: Config{ - podKey: key, - }, - Pod: pod, - CreateTime: now, - State: StatePending, // possibly running? mesos will tell us during reconciliation - Flags: make(map[FlagType]struct{}), - launchTime: now, - bindTime: now, - Spec: &Spec{}, - } - var ( - offerId string - ) - for _, k := range []string{ - mesosmeta.BindingHostKey, - mesosmeta.TaskIdKey, - mesosmeta.SlaveIdKey, - mesosmeta.OfferIdKey, - } { - v, found := pod.Annotations[k] - if !found { - return nil, false, fmt.Errorf("incomplete metadata: missing value for pod annotation: %v", k) - } - switch k { - case mesosmeta.BindingHostKey: - t.Spec.AssignedSlave = v - case mesosmeta.SlaveIdKey: - t.Spec.SlaveID = v - case mesosmeta.OfferIdKey: - offerId = v - case mesosmeta.TaskIdKey: - t.ID = v - case mesosmeta.ExecutorIdKey: - // this is nowhere near sufficient to re-launch a task, but we really just - // want this for tracking - t.Spec.Executor = &mesos.ExecutorInfo{ExecutorId: mutil.NewExecutorID(v)} - } - } - t.Offer = offers.Expired(offerId, t.Spec.AssignedSlave, 0) - t.Flags[Launched] = struct{}{} - t.Flags[Bound] = struct{}{} - return t, true, nil -} diff --git a/contrib/mesos/pkg/scheduler/podtask/pod_task_test.go b/contrib/mesos/pkg/scheduler/podtask/pod_task_test.go deleted file mode 100644 index 56112d2990c..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/pod_task_test.go +++ /dev/null @@ -1,421 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -import ( - "reflect" - "testing" - - "github.com/gogo/protobuf/proto" - mesos "github.com/mesos/mesos-go/mesosproto" - mutil "github.com/mesos/mesos-go/mesosutil" - "github.com/stretchr/testify/assert" - "k8s.io/kubernetes/contrib/mesos/pkg/node" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask/hostport" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" -) - -const ( - t_min_cpu = 128 - t_min_mem = 128 -) - -func fakePodTask(id string, allowedRoles, defaultRoles []string) *T { - t, _ := New( - api.NewDefaultContext(), - Config{ - Prototype: &mesos.ExecutorInfo{}, - FrameworkRoles: allowedRoles, - DefaultPodRoles: defaultRoles, - HostPortStrategy: hostport.StrategyWildcard, - }, - &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: id, - Namespace: api.NamespaceDefault, - }, - }, - ) - - return t -} - -func TestRoles(t *testing.T) { - assert := assert.New(t) - - for i, tt := range []struct { - annotations map[string]string - frameworkRoles []string - want []string - }{ - { - map[string]string{}, - nil, - starRole, - }, - { - map[string]string{"other": "label"}, - nil, - starRole, - }, - { - map[string]string{meta.RolesKey: ""}, - nil, - []string{}, - }, - { - map[string]string{ - "other": "label", - meta.RolesKey: ", , ,", - }, - nil, - []string{}, - }, - { - map[string]string{meta.RolesKey: "forbiddenRole"}, - []string{"allowedRole"}, - []string{}, - }, - { - map[string]string{meta.RolesKey: "*, , *, ,slave_public,"}, - []string{"*", "slave_public"}, - []string{"*", "slave_public"}, - }, - { - map[string]string{meta.RolesKey: "role3,role2,role1"}, - []string{"role1", "role4"}, - []string{"role1"}, - }, - { - map[string]string{}, - []string{"role1"}, - []string{"*"}, - }, - } { - task := fakePodTask("test", tt.frameworkRoles, starRole) - task.Pod.ObjectMeta.Annotations = tt.annotations - assert.True(reflect.DeepEqual(task.Roles(), tt.want), "test #%d got %#v want %#v", i, task.Roles(), tt.want) - } -} - -type mockRegistry struct{} - -func (mr mockRegistry) New(nodename string, resources []*mesos.Resource) *mesos.ExecutorInfo { - return &mesos.ExecutorInfo{ - Resources: resources, - } -} - -func (mr mockRegistry) Get(nodename string) (*mesos.ExecutorInfo, error) { - panic("N/A") -} - -func (mr mockRegistry) Invalidate(hostname string) { - panic("N/A") -} - -func TestEmptyOffer(t *testing.T) { - t.Parallel() - task := fakePodTask("foo", nil, nil) - - task.Pod.Spec = api.PodSpec{ - Containers: []api.Container{{ - Name: "a", - }}, - } - - defaultProc := NewDefaultProcurement( - &mesos.ExecutorInfo{ - Resources: []*mesos.Resource{ - mutil.NewScalarResource("cpus", 1.0), - mutil.NewScalarResource("mem", 64.0), - }, - }, - mockRegistry{}, - ) - - if err := defaultProc.Procure( - task, - &api.Node{}, - NewProcureState(&mesos.Offer{}), - ); err == nil { - t.Fatalf("accepted empty offer") - } -} - -func TestNoPortsInPodOrOffer(t *testing.T) { - t.Parallel() - task := fakePodTask("foo", nil, nil) - - task.Pod.Spec = api.PodSpec{ - Containers: []api.Container{{ - Name: "a", - }}, - } - - executor := &mesos.ExecutorInfo{ - Resources: []*mesos.Resource{ - mutil.NewScalarResource("cpus", 1.0), - mutil.NewScalarResource("mem", 64.0), - }, - } - - defaultProc := NewDefaultProcurement(executor, mockRegistry{}) - - offer := &mesos.Offer{ - Resources: []*mesos.Resource{ - mutil.NewScalarResource("cpus", 0.001), - mutil.NewScalarResource("mem", 0.001), - }, - } - - if err := defaultProc.Procure( - task, - nil, - NewProcureState(offer), - ); err == nil { - t.Fatalf("accepted offer %v:", offer) - } - - offer = &mesos.Offer{ - Resources: []*mesos.Resource{ - mutil.NewScalarResource("cpus", t_min_cpu), - mutil.NewScalarResource("mem", t_min_mem), - }, - } - - if err := defaultProc.Procure( - task, - nil, - NewProcureState(offer), - ); err != nil { - t.Fatalf("did not accepted offer %v:", offer) - } -} - -func TestAcceptOfferPorts(t *testing.T) { - t.Parallel() - task := fakePodTask("foo", nil, nil) - pod := &task.Pod - - defaultProc := NewDefaultProcurement( - &mesos.ExecutorInfo{}, - mockRegistry{}, - ) - - offer := &mesos.Offer{ - Resources: []*mesos.Resource{ - mutil.NewScalarResource("cpus", t_min_cpu), - mutil.NewScalarResource("mem", t_min_mem), - resources.NewPorts("*", 1, 1), - }, - } - - if err := defaultProc.Procure( - task, - &api.Node{}, - NewProcureState(offer), - ); err != nil { - t.Fatalf("did not accepted offer %v:", offer) - } - - pod.Spec = api.PodSpec{ - Containers: []api.Container{{ - Ports: []api.ContainerPort{{ - HostPort: 123, - }}, - }}, - } - - if err := defaultProc.Procure( - task, - &api.Node{}, - NewProcureState(offer), - ); err == nil { - t.Fatalf("accepted offer %v:", offer) - } - - pod.Spec.Containers[0].Ports[0].HostPort = 1 - - if err := defaultProc.Procure( - task, - &api.Node{}, - NewProcureState(offer), - ); err != nil { - t.Fatalf("did not accepted offer %v:", offer) - } - - pod.Spec.Containers[0].Ports[0].HostPort = 0 - - if err := defaultProc.Procure( - task, - &api.Node{}, - NewProcureState(offer), - ); err != nil { - t.Fatalf("did not accepted offer %v:", offer) - } - - offer.Resources = []*mesos.Resource{ - mutil.NewScalarResource("cpus", t_min_cpu), - mutil.NewScalarResource("mem", t_min_mem), - } - - if err := defaultProc.Procure( - task, - &api.Node{}, - NewProcureState(offer), - ); err == nil { - t.Fatalf("accepted offer %v:", offer) - } - - pod.Spec.Containers[0].Ports[0].HostPort = 1 - - if err := defaultProc.Procure( - task, - &api.Node{}, - NewProcureState(offer), - ); err == nil { - t.Fatalf("accepted offer %v:", offer) - } -} - -func TestGeneratePodName(t *testing.T) { - p := &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "foo", - Namespace: "bar", - }, - } - name := generateTaskName(p) - expected := "foo.bar.pod" - if name != expected { - t.Fatalf("expected %q instead of %q", expected, name) - } - - p.Namespace = "" - name = generateTaskName(p) - expected = "foo.default.pod" - if name != expected { - t.Fatalf("expected %q instead of %q", expected, name) - } -} - -func TestNodeSelector(t *testing.T) { - t.Parallel() - - newNode := func(hostName string, l map[string]string) *api.Node { - nodeLabels := map[string]string{"kubernetes.io/hostname": hostName} - if l != nil { - for k, v := range l { - nodeLabels[k] = v - } - } - return &api.Node{ - ObjectMeta: api.ObjectMeta{ - Name: hostName, - Labels: nodeLabels, - }, - Spec: api.NodeSpec{ - ExternalID: hostName, - }, - } - } - node1 := newNode("node1", node.SlaveAttributesToLabels([]*mesos.Attribute{ - newTextAttribute("rack", "a"), - newTextAttribute("gen", "2014"), - newScalarAttribute("num", 42.0), - })) - node2 := newNode("node2", node.SlaveAttributesToLabels([]*mesos.Attribute{ - newTextAttribute("rack", "b"), - newTextAttribute("gen", "2015"), - newScalarAttribute("num", 0.0), - })) - labels3 := node.SlaveAttributesToLabels([]*mesos.Attribute{ - newTextAttribute("rack", "c"), - newTextAttribute("gen", "2015"), - newScalarAttribute("old", 42), - }) - labels3["some.other/label"] = "43" - node3 := newNode("node3", labels3) - - tests := []struct { - selector map[string]string - node *api.Node - ok bool - desc string - }{ - {map[string]string{"k8s.mesosphere.io/attribute-rack": "a"}, node1, true, "label value matches"}, - {map[string]string{"k8s.mesosphere.io/attribute-rack": "b"}, node1, false, "label value does not match"}, - {map[string]string{"k8s.mesosphere.io/attribute-rack": "a", "k8s.mesosphere.io/attribute-gen": "2014"}, node1, true, "multiple required labels match"}, - {map[string]string{"k8s.mesosphere.io/attribute-rack": "a", "k8s.mesosphere.io/attribute-gen": "2015"}, node1, false, "one label does not match"}, - {map[string]string{"k8s.mesosphere.io/attribute-rack": "a", "k8s.mesosphere.io/attribute-num": "42"}, node1, true, "scalar label matches"}, - {map[string]string{"k8s.mesosphere.io/attribute-rack": "a", "k8s.mesosphere.io/attribute-num": "43"}, node1, false, "scalar label does not match"}, - - {map[string]string{"kubernetes.io/hostname": "node1"}, node1, true, "hostname label matches"}, - {map[string]string{"kubernetes.io/hostname": "node2"}, node1, false, "hostname label does not match"}, - {map[string]string{"kubernetes.io/hostname": "node2"}, node2, true, "hostname label matches"}, - - {map[string]string{"some.other/label": "43"}, node1, false, "non-slave attribute does not match"}, - {map[string]string{"some.other/label": "43"}, node3, true, "non-slave attribute matches"}, - } - - defaultProc := NewDefaultProcurement( - &mesos.ExecutorInfo{}, - mockRegistry{}, - ) - - for _, ts := range tests { - task := fakePodTask("foo", nil, nil) - task.Pod.Spec.NodeSelector = ts.selector - offer := &mesos.Offer{ - Resources: []*mesos.Resource{ - mutil.NewScalarResource("cpus", t_min_cpu), - mutil.NewScalarResource("mem", t_min_mem), - }, - Hostname: &ts.node.Name, - } - - err := defaultProc.Procure( - task, - ts.node, - NewProcureState(offer), - ) - - ok := err == nil - if ts.ok != ok { - t.Fatalf("expected acceptance of offer for selector %v to be %v, got %v: %q", ts.selector, ts.ok, ok, ts.desc) - } - } -} - -func newTextAttribute(name string, val string) *mesos.Attribute { - return &mesos.Attribute{ - Name: proto.String(name), - Type: mesos.Value_TEXT.Enum(), - Text: &mesos.Value_Text{Value: &val}, - } -} - -func newScalarAttribute(name string, val float64) *mesos.Attribute { - return &mesos.Attribute{ - Name: proto.String(name), - Type: mesos.Value_SCALAR.Enum(), - Scalar: &mesos.Value_Scalar{Value: proto.Float64(val)}, - } -} diff --git a/contrib/mesos/pkg/scheduler/podtask/procurement.go b/contrib/mesos/pkg/scheduler/podtask/procurement.go deleted file mode 100644 index 36b14aeba3e..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/procurement.go +++ /dev/null @@ -1,309 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -import ( - "fmt" - "math" - - "github.com/gogo/protobuf/proto" - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - "github.com/mesos/mesos-go/mesosutil" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/executorinfo" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/labels" -) - -// NewDefaultProcurement returns the default procurement strategy that combines validation -// and responsible Mesos resource procurement. c and m are resource quantities written into -// k8s api.Pod.Spec's that don't declare resources (all containers in k8s-mesos require cpu -// and memory limits). -func NewDefaultProcurement(prototype *mesos.ExecutorInfo, eir executorinfo.Registry) Procurement { - return AllOrNothingProcurement([]Procurement{ - NewNodeProcurement(), - NewPodResourcesProcurement(), - NewPortsProcurement(), - NewExecutorResourceProcurer(prototype.GetResources(), eir), - }) -} - -// Procurement is the interface that implements resource procurement. -// -// Procure procurs offered resources for a given pod task T -// on a given node and stores the procurement result. -// -// Initially the procurement pipe contains an initial empty Spec -// and the the complete Mesos offer. As the procurement pipeline progresses -// the specified resources go up as they are being procured -// while the remaining Mesos offer resources go down until they are depleted. -// -// It returns an error if the procurement failed. -// -// Note that the T struct also includes a Spec field. -// This differs from the procured Spec which is meant to be filled -// by a chain of Procure invocations (procurement pipeline). -// -// In contrast T.Spec is meant not to be filled by the procurement chain -// but rather by a final scheduler instance. -// -// api.Node is an optional (possibly nil) param. -type Procurement interface { - Procure(*T, *api.Node, *ProcureState) error -} - -// ProcureState holds the current state of the procurement pipeline. -// It contains the pod launch specification and the Mesos offer -// from which resources are being procured. -type ProcureState struct { - offer *mesos.Offer // source - spec *Spec // sink -} - -// Result returns the procurement result consisting -// of the procured pod specification and the remaining -// Mesos offer. -func (ps *ProcureState) Result() (*Spec, *mesos.Offer) { - return ps.spec, ps.offer -} - -// NewProcureState returns an ProcureState containing an empty Spec -// and a deep copy of the given offer. -func NewProcureState(offer *mesos.Offer) *ProcureState { - return &ProcureState{ - spec: &Spec{}, - offer: proto.Clone(offer).(*mesos.Offer), - } -} - -// The ProcurementFunc type is an adapter to use ordinary functions as Procurement implementations. -type ProcurementFunc func(*T, *api.Node, *ProcureState) error - -func (p ProcurementFunc) Procure(t *T, n *api.Node, ps *ProcureState) error { - return p(t, n, ps) -} - -// AllOrNothingProcurement provides a convenient wrapper around multiple Procurement -// objectives: the failure of any Procurement in the set results in Procure failing. -// see AllOrNothingProcurement.Procure -type AllOrNothingProcurement []Procurement - -// Procure runs each Procurement in the receiver list. The first Procurement func that -// fails triggers T.Reset() and the error is returned, otherwise returns nil. -func (a AllOrNothingProcurement) Procure(t *T, n *api.Node, ps *ProcureState) error { - for _, p := range a { - err := p.Procure(t, n, ps) - if err != nil { - return err - } - } - return nil -} - -// NewNodeProcurement returns a Procurement that checks whether the given pod task and offer -// have valid node informations available and whether the pod spec node selector matches -// the pod labels. -// If the check is successful the slave ID and assigned slave is set in the given Spec. -func NewNodeProcurement() Procurement { - return ProcurementFunc(func(t *T, n *api.Node, ps *ProcureState) error { - // if the user has specified a target host, make sure this offer is for that host - if t.Pod.Spec.NodeName != "" && ps.offer.GetHostname() != t.Pod.Spec.NodeName { - return fmt.Errorf( - "NodeName %q does not match offer hostname %q", - t.Pod.Spec.NodeName, ps.offer.GetHostname(), - ) - } - - // check the NodeSelector - if len(t.Pod.Spec.NodeSelector) > 0 { - // *api.Node is optional for procurement - if n == nil || n.Labels == nil { - return fmt.Errorf( - "NodeSelector %v does not match empty labels of pod %s/%s", - t.Pod.Spec.NodeSelector, t.Pod.Namespace, t.Pod.Name, - ) - } - selector := labels.SelectorFromSet(t.Pod.Spec.NodeSelector) - if !selector.Matches(labels.Set(n.Labels)) { - return fmt.Errorf( - "NodeSelector %v does not match labels %v of pod %s/%s", - t.Pod.Spec.NodeSelector, t.Pod.Labels, t.Pod.Namespace, t.Pod.Name, - ) - } - } - - ps.spec.SlaveID = ps.offer.GetSlaveId().GetValue() - ps.spec.AssignedSlave = ps.offer.GetHostname() - - return nil - }) -} - -// NewPodResourcesProcurement converts k8s pod cpu and memory resource requirements into -// mesos resource allocations. -func NewPodResourcesProcurement() Procurement { - return ProcurementFunc(func(t *T, _ *api.Node, ps *ProcureState) error { - // TODO(sttts): fall back to requested resources if resource limit cannot be fulfilled by the offer - _, limits, err := api.PodRequestsAndLimits(&t.Pod) - if err != nil { - return err - } - - wantedCpus := float64(resources.NewCPUShares(limits[api.ResourceCPU])) - wantedMem := float64(resources.NewMegaBytes(limits[api.ResourceMemory])) - - log.V(4).Infof( - "trying to match offer with pod %v/%v: cpus: %.2f mem: %.2f MB", - t.Pod.Namespace, t.Pod.Name, wantedCpus, wantedMem, - ) - - podRoles := t.Roles() - procuredCpu, remaining := procureScalarResources("cpus", wantedCpus, podRoles, ps.offer.GetResources()) - if procuredCpu == nil { - return fmt.Errorf( - "not enough cpu resources for pod %s/%s: want=%v", - t.Pod.Namespace, t.Pod.Name, wantedCpus, - ) - } - - procuredMem, remaining := procureScalarResources("mem", wantedMem, podRoles, remaining) - if procuredMem == nil { - return fmt.Errorf( - "not enough mem resources for pod %s/%s: want=%v", - t.Pod.Namespace, t.Pod.Name, wantedMem, - ) - } - - ps.offer.Resources = remaining - ps.spec.Resources = append(ps.spec.Resources, append(procuredCpu, procuredMem...)...) - return nil - }) -} - -// NewPortsProcurement returns a Procurement procuring ports -func NewPortsProcurement() Procurement { - return ProcurementFunc(func(t *T, _ *api.Node, ps *ProcureState) error { - // fill in port mapping - if mapping, err := t.mapper.Map(&t.Pod, t.Roles(), ps.offer); err != nil { - return err - } else { - ports := []resources.Port{} - for _, entry := range mapping { - ports = append(ports, resources.Port{ - Port: entry.OfferPort, - Role: entry.Role, - }) - } - ps.spec.PortMap = mapping - ps.spec.Resources = append(ps.spec.Resources, resources.PortRanges(ports)...) - } - return nil - }) -} - -// NewExecutorResourceProcurer returns a Procurement procuring executor resources -// If a given offer has no executor IDs set, the given prototype executor resources are considered for procurement. -// If a given offer has one executor ID set, only pod resources are being procured. -// An offer with more than one executor ID implies an invariant violation and the first executor ID is being considered. -func NewExecutorResourceProcurer(rs []*mesos.Resource, registry executorinfo.Registry) Procurement { - return ProcurementFunc(func(t *T, _ *api.Node, ps *ProcureState) error { - eids := len(ps.offer.GetExecutorIds()) - switch { - case eids == 0: - wantedCpus := resources.Sum(resources.Filter(rs, resources.IsScalar, resources.HasName("cpus"))) - wantedMem := resources.Sum(resources.Filter(rs, resources.IsScalar, resources.HasName("mem"))) - - procuredCpu, remaining := procureScalarResources("cpus", wantedCpus, t.FrameworkRoles, ps.offer.GetResources()) - if procuredCpu == nil { - return fmt.Errorf("not enough cpu resources for executor: want=%v", wantedCpus) - } - - procuredMem, remaining := procureScalarResources("mem", wantedMem, t.FrameworkRoles, remaining) - if procuredMem == nil { - return fmt.Errorf("not enough mem resources for executor: want=%v", wantedMem) - } - - ps.offer.Resources = remaining - ps.spec.Executor = registry.New(ps.offer.GetHostname(), append(procuredCpu, procuredMem...)) - return nil - - case eids == 1: - e, err := registry.Get(ps.offer.GetHostname()) - if err != nil { - return err - } - ps.spec.Executor = e - return nil - - default: - // offers with more than 1 ExecutorId should be rejected by the - // framework long before they arrive here. - return fmt.Errorf("got offer with more than 1 executor id: %v", ps.offer.GetExecutorIds()) - } - }) -} - -// smallest number such that 1.0 + epsilon != 1.0 -// see https://github.com/golang/go/issues/966 -var epsilon = math.Nextafter(1, 2) - 1 - -// procureScalarResources procures offered resources that -// 1. Match the given name -// 2. Match the given roles -// 3. The given wanted scalar value can be fully consumed by offered resources -// Roles are being considered in the specified roles slice ordering. -func procureScalarResources( - name string, - want float64, - roles []string, - offered []*mesos.Resource, -) (procured, remaining []*mesos.Resource) { - sorted := resources.ByRoles(roles...).Sort(offered) - procured = make([]*mesos.Resource, 0, len(sorted)) - remaining = make([]*mesos.Resource, 0, len(sorted)) - - for _, r := range sorted { - if want >= epsilon && resources.MatchesAll(r, resources.HasName(name), resources.IsScalar) { - left, role := r.GetScalar().GetValue(), r.Role - consumed := math.Min(want, left) - - want -= consumed - left -= consumed - - if left >= epsilon { - r = mesosutil.NewScalarResource(name, left) - r.Role = role - remaining = append(remaining, r) - } - - consumedRes := mesosutil.NewScalarResource(name, consumed) - consumedRes.Role = role - procured = append(procured, consumedRes) - } else { - remaining = append(remaining, r) - } - } - - // demanded value (want) was not fully consumed violating invariant 3. - // thus no resources must be procured - if want >= epsilon { - return nil, offered - } - - return -} diff --git a/contrib/mesos/pkg/scheduler/podtask/procurement_test.go b/contrib/mesos/pkg/scheduler/podtask/procurement_test.go deleted file mode 100644 index da54dce6a0d..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/procurement_test.go +++ /dev/null @@ -1,223 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -import ( - "testing" - - "github.com/mesos/mesos-go/mesosproto" - "github.com/mesos/mesos-go/mesosutil" - - mesos "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask/hostport" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/resource" - "reflect" -) - -func TestNewPodResourcesProcurement(t *testing.T) { - executor := mesosutil.NewExecutorInfo( - mesosutil.NewExecutorID("executor-id"), - mesosutil.NewCommandInfo("executor-cmd"), - ) - executor.Data = []byte{0, 1, 2} - executor.Resources = []*mesosproto.Resource{ - scalar("cpus", 0.1, "*"), - scalar("mem", 64.0, "*"), - } - executor.Command = &mesosproto.CommandInfo{ - Arguments: []string{}, - } - - offer := &mesosproto.Offer{ - Resources: []*mesosproto.Resource{ - scalar("cpus", 4.0, "*"), - scalar("mem", 512.0, "*"), - }, - } - - task, _ := New( - api.NewDefaultContext(), - Config{ - Prototype: executor, - FrameworkRoles: []string{"*"}, - DefaultPodRoles: []string{"*"}, - HostPortStrategy: hostport.StrategyWildcard, - }, - &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "test", - Namespace: api.NamespaceDefault, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Resources: api.ResourceRequirements{ - Limits: api.ResourceList{ - api.ResourceCPU: *resource.NewQuantity( - 3, - resource.DecimalSI, - ), - api.ResourceMemory: *resource.NewQuantity( - 128*1024*1024, - resource.BinarySI, - ), - }, - }, - }, - }, - }, - }, - ) - - procurement := NewPodResourcesProcurement() - - ps := NewProcureState(offer) - if err := procurement.Procure(task, &api.Node{}, ps); err != nil { - t.Error(err) - } - - if len(ps.spec.Resources) == 0 { - t.Errorf("expected procured resources but got none") - } -} - -func TestProcureRoleResources(t *testing.T) { - for i, tt := range []struct { - offered []*mesos.Resource - - name string // cpu or mem - want float64 - roles []string - - consumed []*mesos.Resource - left []*mesos.Resource - }{ - { - offered: []*mesos.Resource{ - scalar("mem", 128.0, "*"), - scalar("mem", 32.0, "slave_public"), - }, - - name: "mem", - want: 128.0, - roles: []string{"slave_public", "*"}, - - consumed: []*mesos.Resource{ - scalar("mem", 32.0, "slave_public"), - scalar("mem", 96.0, "*"), - }, - left: []*mesos.Resource{ - scalar("mem", 32.0, "*"), - }, - }, - { - offered: []*mesos.Resource{ - scalar("mem", 128.0, "*"), - scalar("mem", 32.0, "slave_public"), - }, - - name: "mem", - want: 128.0, - roles: []string{"slave_public"}, - - consumed: nil, - left: []*mesos.Resource{ - scalar("mem", 128.0, "*"), - scalar("mem", 32.0, "slave_public"), - }, - }, - { - offered: []*mesos.Resource{ - scalar("cpus", 1.5, "slave_public"), - scalar("cpus", 1, "slave_public"), - scalar("mem", 128.0, "slave_public"), - scalar("mem", 64.0, "slave_public"), - scalar("mem", 128.0, "*"), - }, - - name: "mem", - want: 200.0, - roles: []string{"slave_public", "*"}, - - consumed: []*mesos.Resource{ - scalar("mem", 128.0, "slave_public"), - scalar("mem", 64.0, "slave_public"), - scalar("mem", 8.0, "*"), - }, - left: []*mesos.Resource{ - scalar("cpus", 1.5, "slave_public"), - scalar("cpus", 1, "slave_public"), - scalar("mem", 120, "*"), - }, - }, - { - offered: []*mesos.Resource{ - scalar("mem", 128.0, "*"), - }, - - name: "mem", - want: 128.0, - roles: []string{"slave_public", "*"}, - - consumed: []*mesos.Resource{ - scalar("mem", 128, "*"), - }, - left: []*mesos.Resource{}, - }, - { - offered: []*mesos.Resource{ - scalar("cpu", 32.0, "slave_public"), - }, - - name: "mem", - want: 128.0, - roles: []string{"slave_public", "*"}, - - consumed: nil, - left: []*mesos.Resource{ - scalar("cpu", 32.0, "slave_public"), - }, - }, - { - offered: nil, - - name: "mem", - want: 160.0, - roles: []string{"slave_public", "*"}, - - consumed: nil, left: nil, - }, - } { - consumed, remaining := procureScalarResources(tt.name, tt.want, tt.roles, tt.offered) - - if !reflect.DeepEqual(consumed, tt.consumed) { - t.Errorf("test #%d (consumed):\ngot %v\nwant %v", i, consumed, tt.consumed) - } - - if !reflect.DeepEqual(remaining, tt.left) { - t.Errorf("test #%d (remaining):\ngot %v\nwant %v", i, remaining, tt.left) - } - } -} - -func scalar(name string, value float64, role string) *mesos.Resource { - res := mesosutil.NewScalarResource(name, value) - res.Role = resources.StringPtrTo(role) - return res -} diff --git a/contrib/mesos/pkg/scheduler/podtask/registry.go b/contrib/mesos/pkg/scheduler/podtask/registry.go deleted file mode 100644 index d739b2b7317..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/registry.go +++ /dev/null @@ -1,340 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -import ( - "container/ring" - "encoding/json" - "fmt" - "sync" - "time" - - log "github.com/golang/glog" - mesos "github.com/mesos/mesos-go/mesosproto" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/metrics" - "k8s.io/kubernetes/pkg/api" -) - -const ( - //TODO(jdef) move this somewhere else - PodPath = "/pods" - - // length of historical record of finished tasks - defaultFinishedTasksSize = 1024 -) - -// state store for pod tasks -type Registry interface { - // register the specified task with this registry, as long as the current error - // condition is nil. if no errors occur then return a copy of the registered task. - Register(*T) (*T, error) - - // unregister the specified task from this registry - Unregister(*T) - - // update state for the registered task identified by task.ID, returning a copy of - // the updated task, if any. - Update(task *T) error - - // return the task registered for the specified task ID and its current state. - // if there is no such task then StateUnknown is returned. - Get(taskId string) (task *T, currentState StateType) - - // return the non-terminal task corresponding to the specified pod ID - ForPod(podID string) (task *T, currentState StateType) - - // update the task status given the specified mesos task status update, returning a - // copy of the updated task (if any) and its state. - UpdateStatus(status *mesos.TaskStatus) (*T, StateType) - - // return a list of task ID's that match the given filter, or all task ID's if filter == nil. - List(filter func(*T) bool) []*T -} - -type inMemoryRegistry struct { - rw sync.RWMutex - taskRegistry map[string]*T - tasksFinished *ring.Ring - podToTask map[string]string -} - -func NewInMemoryRegistry() Registry { - return &inMemoryRegistry{ - taskRegistry: make(map[string]*T), - tasksFinished: ring.New(defaultFinishedTasksSize), - podToTask: make(map[string]string), - } -} - -func (k *inMemoryRegistry) List(accepts func(t *T) bool) (tasks []*T) { - k.rw.RLock() - defer k.rw.RUnlock() - for _, task := range k.taskRegistry { - if accepts == nil || accepts(task) { - tasks = append(tasks, task.Clone()) - } - } - return -} - -func (k *inMemoryRegistry) ForPod(podID string) (task *T, currentState StateType) { - k.rw.RLock() - defer k.rw.RUnlock() - tid, ok := k.podToTask[podID] - if !ok { - return nil, StateUnknown - } - t, state := k._get(tid) - return t.Clone(), state -} - -// registers a pod task unless the spec'd error is not nil -func (k *inMemoryRegistry) Register(task *T) (*T, error) { - k.rw.Lock() - defer k.rw.Unlock() - if _, found := k.podToTask[task.podKey]; found { - return nil, fmt.Errorf("task already registered for pod key %q", task.podKey) - } - if _, found := k.taskRegistry[task.ID]; found { - return nil, fmt.Errorf("task already registered for id %q", task.ID) - } - k.podToTask[task.podKey] = task.ID - k.taskRegistry[task.ID] = task - - return task.Clone(), nil -} - -// updates internal task state. updates are limited to Spec, Flags, and Offer for -// StatePending tasks, and are limited to Flag updates (additive only) for StateRunning tasks. -func (k *inMemoryRegistry) Update(task *T) error { - if task == nil { - return nil - } - k.rw.Lock() - defer k.rw.Unlock() - switch internal, state := k._get(task.ID); state { - case StateUnknown: - return fmt.Errorf("no such task: %v", task.ID) - case StatePending: - internal.Offer = task.Offer - internal.Spec = task.Spec - internal.Flags = map[FlagType]struct{}{} - fallthrough - case StateRunning: - for k, v := range task.Flags { - internal.Flags[k] = v - } - return nil - default: - return fmt.Errorf("may not update task %v in state %v", task.ID, state) - } -} - -func (k *inMemoryRegistry) Unregister(task *T) { - k.rw.Lock() - defer k.rw.Unlock() - delete(k.podToTask, task.podKey) - delete(k.taskRegistry, task.ID) -} - -func (k *inMemoryRegistry) Get(taskId string) (*T, StateType) { - k.rw.RLock() - defer k.rw.RUnlock() - t, state := k._get(taskId) - return t.Clone(), state -} - -// assume that the caller has already locked around access to task state. -// the caller is also responsible for cloning the task object before it leaves -// the context of this registry. -func (k *inMemoryRegistry) _get(taskId string) (*T, StateType) { - if task, found := k.taskRegistry[taskId]; found { - return task, task.State - } - return nil, StateUnknown -} - -func (k *inMemoryRegistry) UpdateStatus(status *mesos.TaskStatus) (*T, StateType) { - taskId := status.GetTaskId().GetValue() - - k.rw.Lock() - defer k.rw.Unlock() - task, state := k._get(taskId) - - switch status.GetState() { - case mesos.TaskState_TASK_STAGING: - k.handleTaskStaging(task, state, status) - case mesos.TaskState_TASK_STARTING: - k.handleTaskStarting(task, state, status) - case mesos.TaskState_TASK_RUNNING: - k.handleTaskRunning(task, state, status) - case mesos.TaskState_TASK_FINISHED: - k.handleTaskFinished(task, state, status) - case mesos.TaskState_TASK_FAILED: - k.handleTaskFailed(task, state, status) - case mesos.TaskState_TASK_ERROR: - k.handleTaskError(task, state, status) - case mesos.TaskState_TASK_KILLED: - k.handleTaskKilled(task, state, status) - case mesos.TaskState_TASK_LOST: - k.handleTaskLost(task, state, status) - default: - log.Warningf("unhandled status update for task: %v", taskId) - } - return task.Clone(), state -} - -func (k *inMemoryRegistry) handleTaskStaging(task *T, state StateType, status *mesos.TaskStatus) { - if status.GetSource() != mesos.TaskStatus_SOURCE_MASTER { - log.Errorf("received STAGING for task %v with unexpected source: %v", - status.GetTaskId().GetValue(), status.GetSource()) - } -} - -func (k *inMemoryRegistry) handleTaskStarting(task *T, state StateType, status *mesos.TaskStatus) { - // we expect to receive this when a launched task is finally "bound" - // via the API server. however, there's nothing specific for us to do here. - switch state { - case StatePending: - task.UpdatedTime = time.Now() - if !task.Has(Bound) { - task.Set(Bound) - task.bindTime = task.UpdatedTime - timeToBind := task.bindTime.Sub(task.launchTime) - metrics.BindLatency.Observe(metrics.InMicroseconds(timeToBind)) - } - default: - taskId := status.GetTaskId().GetValue() - log.Warningf("Ignore status TASK_STARTING because the task %v is not pending", taskId) - } -} - -func (k *inMemoryRegistry) handleTaskRunning(task *T, state StateType, status *mesos.TaskStatus) { - taskId := status.GetTaskId().GetValue() - switch state { - case StatePending: - task.UpdatedTime = time.Now() - log.Infof("Received running status for pending task: %v", taskId) - fillRunningPodInfo(task, status) - task.State = StateRunning - case StateRunning: - task.UpdatedTime = time.Now() - log.V(2).Infof("Ignore status TASK_RUNNING because the task %v is already running", taskId) - case StateFinished: - log.Warningf("Ignore status TASK_RUNNING because the task %v is already finished", taskId) - default: - log.Warningf("Ignore status TASK_RUNNING because the task %v is discarded", taskId) - } -} - -func ParsePodStatusResult(taskStatus *mesos.TaskStatus) (result api.PodStatusResult, err error) { - if taskStatus.Data != nil { - err = json.Unmarshal(taskStatus.Data, &result) - } else { - err = fmt.Errorf("missing TaskStatus.Data") - } - return -} - -func fillRunningPodInfo(task *T, taskStatus *mesos.TaskStatus) { - if taskStatus.GetReason() == mesos.TaskStatus_REASON_RECONCILIATION && taskStatus.GetSource() == mesos.TaskStatus_SOURCE_MASTER { - // there is no data.. - return - } - //TODO(jdef) determine the usefullness of this information (if any) - if result, err := ParsePodStatusResult(taskStatus); err != nil { - log.Errorf("invalid TaskStatus.Data for task '%v': %v", task.ID, err) - } else { - task.podStatus = result.Status - log.Infof("received pod status for task %v: %+v", task.ID, result.Status) - } -} - -func (k *inMemoryRegistry) handleTaskFinished(task *T, state StateType, status *mesos.TaskStatus) { - taskId := status.GetTaskId().GetValue() - switch state { - case StatePending: - panic(fmt.Sprintf("Pending task %v finished, this couldn't happen", taskId)) - case StateRunning: - log.V(2).Infof("received finished status for running task: %v", taskId) - delete(k.podToTask, task.podKey) - task.State = StateFinished - task.UpdatedTime = time.Now() - k.tasksFinished = k.recordFinishedTask(task.ID) - case StateFinished: - log.Warningf("Ignore status TASK_FINISHED because the task %v is already finished", taskId) - default: - log.Warningf("Ignore status TASK_FINISHED because the task %v is not running", taskId) - } -} - -// record that a task has finished. -// older record are expunged one at a time once the historical ring buffer is saturated. -// assumes caller is holding state lock. -func (k *inMemoryRegistry) recordFinishedTask(taskId string) *ring.Ring { - slot := k.tasksFinished.Next() - if slot.Value != nil { - // garbage collect older finished task from the registry - gctaskId := slot.Value.(string) - if gctask, found := k.taskRegistry[gctaskId]; found && gctask.State == StateFinished { - delete(k.taskRegistry, gctaskId) - } - } - slot.Value = taskId - return slot -} - -func (k *inMemoryRegistry) handleTaskFailed(task *T, state StateType, status *mesos.TaskStatus) { - switch state { - case StatePending, StateRunning: - delete(k.taskRegistry, task.ID) - delete(k.podToTask, task.podKey) - } -} - -func (k *inMemoryRegistry) handleTaskError(task *T, state StateType, status *mesos.TaskStatus) { - switch state { - case StatePending, StateRunning: - delete(k.taskRegistry, task.ID) - delete(k.podToTask, task.podKey) - } -} - -func (k *inMemoryRegistry) handleTaskKilled(task *T, state StateType, status *mesos.TaskStatus) { - defer func() { - msg := fmt.Sprintf("task killed: %+v, task %+v", status, task) - if task != nil && task.Has(Deleted) { - // we were expecting this, nothing out of the ordinary - log.V(2).Infoln(msg) - } else { - log.Errorln(msg) - } - }() - switch state { - case StatePending, StateRunning: - delete(k.taskRegistry, task.ID) - delete(k.podToTask, task.podKey) - } -} - -func (k *inMemoryRegistry) handleTaskLost(task *T, state StateType, status *mesos.TaskStatus) { - switch state { - case StateRunning, StatePending: - delete(k.taskRegistry, task.ID) - delete(k.podToTask, task.podKey) - } -} diff --git a/contrib/mesos/pkg/scheduler/podtask/registry_test.go b/contrib/mesos/pkg/scheduler/podtask/registry_test.go deleted file mode 100644 index 84efa1505f2..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/registry_test.go +++ /dev/null @@ -1,336 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -import ( - "fmt" - "testing" - "time" - - mesos "github.com/mesos/mesos-go/mesosproto" - "github.com/mesos/mesos-go/mesosutil" - "github.com/stretchr/testify/assert" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - "k8s.io/kubernetes/contrib/mesos/pkg/proc" -) - -func TestInMemoryRegistry_RegisterGetUnregister(t *testing.T) { - assert := assert.New(t) - - registry := NewInMemoryRegistry() - - // it's empty at the beginning - tasks := registry.List(func(t *T) bool { return true }) - assert.Empty(tasks) - - // add a task - a := fakePodTask("a", nil, nil) - a_clone, err := registry.Register(a) - assert.NoError(err) - assert.Equal(a_clone.ID, a.ID) - assert.Equal(a_clone.podKey, a.podKey) - - // add another task - b := fakePodTask("b", nil, nil) - b_clone, err := registry.Register(b) - assert.NoError(err) - assert.Equal(b_clone.ID, b.ID) - assert.Equal(b_clone.podKey, b.podKey) - - // find tasks in the registry - tasks = registry.List(func(t *T) bool { return true }) - assert.Len(tasks, 2) - assertContains(t, a_clone, tasks...) - assertContains(t, b_clone, tasks...) - - tasks = registry.List(func(t *T) bool { return t.ID == a.ID }) - assert.Len(tasks, 1) - assertContains(t, a_clone, tasks...) - - task, _ := registry.ForPod(a.podKey) - assert.NotNil(task) - assert.Equal(task.ID, a.ID) - - task, _ = registry.ForPod(b.podKey) - assert.NotNil(task) - assert.Equal(task.ID, b.ID) - - task, _ = registry.ForPod("no-pod-key") - assert.Nil(task) - - task, _ = registry.Get(a.ID) - assert.NotNil(task) - assert.Equal(task.ID, a.ID) - - task, _ = registry.Get("unknown-task-id") - assert.Nil(task) - - // re-add a task - a_clone, err = registry.Register(a) - assert.Error(err) - assert.Nil(a_clone) - - // re-add a task with another podKey, but same task id - another_a := a.Clone() - another_a.podKey = "another-pod" - another_a_clone, err := registry.Register(another_a) - assert.Error(err) - assert.Nil(another_a_clone) - - // re-add a task with another task ID, but same podKey - another_b := b.Clone() - another_b.ID = "another-task-id" - another_b_clone, err := registry.Register(another_b) - assert.Error(err) - assert.Nil(another_b_clone) - - // unregister a task - registry.Unregister(b) - - tasks = registry.List(func(t *T) bool { return true }) - assert.Len(tasks, 1) - assertContains(t, a, tasks...) - - // unregister a task not registered - unregistered_task := fakePodTask("unregistered-task", nil, nil) - registry.Unregister(unregistered_task) -} - -func fakeStatusUpdate(taskId string, state mesos.TaskState) *mesos.TaskStatus { - status := mesosutil.NewTaskStatus(mesosutil.NewTaskID(taskId), state) - status.Data = []byte("{}") // empty json - masterSource := mesos.TaskStatus_SOURCE_MASTER - status.Source = &masterSource - return status -} - -func TestInMemoryRegistry_State(t *testing.T) { - assert := assert.New(t) - - registry := NewInMemoryRegistry() - - // add a task - a := fakePodTask("a", nil, nil) - a_clone, err := registry.Register(a) - assert.NoError(err) - assert.Equal(a.State, a_clone.State) - - // update the status - assert.Equal(a_clone.State, StatePending) - a_clone, state := registry.UpdateStatus(fakeStatusUpdate(a.ID, mesos.TaskState_TASK_RUNNING)) - assert.Equal(state, StatePending) // old state - assert.Equal(a_clone.State, StateRunning) // new state - - // update unknown task - unknown_clone, state := registry.UpdateStatus(fakeStatusUpdate("unknown-task-id", mesos.TaskState_TASK_RUNNING)) - assert.Nil(unknown_clone) - assert.Equal(state, StateUnknown) -} - -func TestInMemoryRegistry_Update(t *testing.T) { - assert := assert.New(t) - - // create offers registry - ttl := time.Second / 4 - config := offers.RegistryConfig{ - DeclineOffer: func(offerId string) <-chan error { - return proc.ErrorChan(nil) - }, - Compat: func(o *mesos.Offer) bool { - return true - }, - TTL: ttl, - LingerTTL: 2 * ttl, - } - storage := offers.CreateRegistry(config) - - // Add offer - offerId := mesosutil.NewOfferID("foo") - mesosOffer := &mesos.Offer{Id: offerId} - storage.Add([]*mesos.Offer{mesosOffer}) - offer, ok := storage.Get(offerId.GetValue()) - assert.True(ok) - - // create registry - registry := NewInMemoryRegistry() - a := fakePodTask("a", nil, nil) - registry.Register(a.Clone()) // here clone a because we change it below - - // state changes are ignored - a.State = StateRunning - err := registry.Update(a) - assert.NoError(err) - a_clone, _ := registry.Get(a.ID) - assert.Equal(StatePending, a_clone.State) - - // offer is updated while pending - a.Offer = offer - err = registry.Update(a) - assert.NoError(err) - a_clone, _ = registry.Get(a.ID) - assert.Equal(offer.Id(), a_clone.Offer.Id()) - - // spec is updated while pending - a.Spec = &Spec{SlaveID: "slave-1"} - err = registry.Update(a) - assert.NoError(err) - a_clone, _ = registry.Get(a.ID) - assert.Equal("slave-1", a_clone.Spec.SlaveID) - - // flags are updated while pending - a.Flags[Launched] = struct{}{} - err = registry.Update(a) - assert.NoError(err) - a_clone, _ = registry.Get(a.ID) - - _, found_launched := a_clone.Flags[Launched] - assert.True(found_launched) - - // flags are updated while running - registry.UpdateStatus(fakeStatusUpdate(a.ID, mesos.TaskState_TASK_RUNNING)) - a.Flags[Bound] = struct{}{} - err = registry.Update(a) - assert.NoError(err) - a_clone, _ = registry.Get(a.ID) - - _, found_launched = a_clone.Flags[Launched] - assert.True(found_launched) - _, found_bound := a_clone.Flags[Bound] - assert.True(found_bound) - - // spec is ignored while running - a.Spec = &Spec{SlaveID: "slave-2"} - err = registry.Update(a) - assert.NoError(err) - a_clone, _ = registry.Get(a.ID) - assert.Equal("slave-1", a_clone.Spec.SlaveID) - - // error when finished - registry.UpdateStatus(fakeStatusUpdate(a.ID, mesos.TaskState_TASK_FINISHED)) - err = registry.Update(a) - assert.Error(err) - - // update unknown task - unknown_task := fakePodTask("unknown-task", nil, nil) - err = registry.Update(unknown_task) - assert.Error(err) - - // update nil task - err = registry.Update(nil) - assert.Nil(err) -} - -type transition struct { - statusUpdate mesos.TaskState - expectedState *StateType - expectPanic bool -} - -func NewTransition(statusUpdate mesos.TaskState, expectedState StateType) transition { - return transition{statusUpdate: statusUpdate, expectedState: &expectedState, expectPanic: false} -} - -func NewTransitionToDeletedTask(statusUpdate mesos.TaskState) transition { - return transition{statusUpdate: statusUpdate, expectedState: nil, expectPanic: false} -} - -func NewTransitionWhichPanics(statusUpdate mesos.TaskState) transition { - return transition{statusUpdate: statusUpdate, expectPanic: true} -} - -func testStateTrace(t *testing.T, transitions []transition) *Registry { - assert := assert.New(t) - - registry := NewInMemoryRegistry() - a := fakePodTask("a", nil, nil) - a, _ = registry.Register(a) - - // initial pending state - assert.Equal(a.State, StatePending) - - for _, transition := range transitions { - if transition.expectPanic { - assert.Panics(func() { - registry.UpdateStatus(fakeStatusUpdate(a.ID, transition.statusUpdate)) - }) - } else { - a, _ = registry.UpdateStatus(fakeStatusUpdate(a.ID, transition.statusUpdate)) - if transition.expectedState == nil { - a, _ = registry.Get(a.ID) - assert.Nil(a, "expected task to be deleted from registry after status update to %v", transition.statusUpdate) - } else { - assert.Equal(a.State, *transition.expectedState) - } - } - } - - return ®istry -} - -func TestInMemoryRegistry_TaskLifeCycle(t *testing.T) { - testStateTrace(t, []transition{ - NewTransition(mesos.TaskState_TASK_STAGING, StatePending), - NewTransition(mesos.TaskState_TASK_STARTING, StatePending), - NewTransitionWhichPanics(mesos.TaskState_TASK_FINISHED), - NewTransition(mesos.TaskState_TASK_RUNNING, StateRunning), - NewTransition(mesos.TaskState_TASK_RUNNING, StateRunning), - NewTransition(mesos.TaskState_TASK_STARTING, StateRunning), - NewTransition(mesos.TaskState_TASK_FINISHED, StateFinished), - NewTransition(mesos.TaskState_TASK_FINISHED, StateFinished), - NewTransition(mesos.TaskState_TASK_RUNNING, StateFinished), - }) -} - -func TestInMemoryRegistry_NotFinished(t *testing.T) { - // all these behave the same - notFinishedStates := []mesos.TaskState{ - mesos.TaskState_TASK_ERROR, - mesos.TaskState_TASK_FAILED, - mesos.TaskState_TASK_KILLED, - mesos.TaskState_TASK_LOST, - } - for _, notFinishedState := range notFinishedStates { - testStateTrace(t, []transition{ - NewTransitionToDeletedTask(notFinishedState), - }) - - testStateTrace(t, []transition{ - NewTransition(mesos.TaskState_TASK_RUNNING, StateRunning), - NewTransitionToDeletedTask(notFinishedState), - }) - - testStateTrace(t, []transition{ - NewTransition(mesos.TaskState_TASK_RUNNING, StateRunning), - NewTransition(mesos.TaskState_TASK_FINISHED, StateFinished), - NewTransition(notFinishedState, StateFinished), - }) - } -} - -func assertContains(t *testing.T, want *T, ts ...*T) bool { - for _, got := range ts { - if taskEquals(want, got) { - return true - } - } - - return assert.Fail(t, fmt.Sprintf("%v does not contain %v", ts, want)) -} - -func taskEquals(t1, t2 *T) bool { - return t1.ID == t2.ID && t1.podKey == t2.podKey -} diff --git a/contrib/mesos/pkg/scheduler/podtask/roles.go b/contrib/mesos/pkg/scheduler/podtask/roles.go deleted file mode 100644 index 61b8348e3a1..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/roles.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -// rolePredicate is a predicate function on role strings -type rolePredicate func(string) bool - -// filterRoles filters the given slice of roles and returns a slice of roles -// matching all given predicates -func filterRoles(roles []string, ps ...rolePredicate) []string { - filtered := make([]string, 0, len(roles)) - -next: - for _, r := range roles { - for _, p := range ps { - if !p(r) { - continue next - } - } - - filtered = append(filtered, r) - } - - return filtered -} - -// seenRole returns a rolePredicate which returns true -// if a given role has already been seen in previous invocations. -func seenRole() rolePredicate { - seen := map[string]struct{}{} - - return func(role string) bool { - _, ok := seen[role] - - if !ok { - seen[role] = struct{}{} - } - - return ok - } -} - -// emptyRole returns true if the given role is empty -func emptyRole(name string) bool { - return name == "" -} - -// not returns a rolePredicate which returns the negation -// of the given predicate -func not(p rolePredicate) rolePredicate { - return func(r string) bool { - return !p(r) - } -} - -// inRoles returns a rolePredicate which returns true -// if the given role is present in the given roles -func inRoles(roles ...string) rolePredicate { - roleSet := make(map[string]struct{}, len(roles)) - - for _, r := range roles { - roleSet[r] = struct{}{} - } - - return func(r string) bool { - _, ok := roleSet[r] - return ok - } -} diff --git a/contrib/mesos/pkg/scheduler/podtask/roles_test.go b/contrib/mesos/pkg/scheduler/podtask/roles_test.go deleted file mode 100644 index c46f1add148..00000000000 --- a/contrib/mesos/pkg/scheduler/podtask/roles_test.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtask - -import ( - "reflect" - "testing" -) - -func TestFilterRoles(t *testing.T) { - for i, tt := range []struct { - roles, want []string - predicates []rolePredicate - }{ - { - []string{"role1", "", "role1", "role2", "role3", "role2"}, - []string{"role1", "role2", "role3"}, - []rolePredicate{not(emptyRole), not(seenRole())}, - }, - { - []string{}, - []string{}, - []rolePredicate{not(emptyRole)}, - }, - { - []string{""}, - []string{}, - []rolePredicate{not(emptyRole)}, - }, - { - nil, - []string{}, - []rolePredicate{not(emptyRole)}, - }, - { - []string{"role1", "role2"}, - []string{"role1", "role2"}, - nil, - }, - { - nil, - []string{}, - nil, - }, - } { - got := filterRoles(tt.roles, tt.predicates...) - - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("test #%d got %#v want %#v", i, got, tt.want) - } - } -} diff --git a/contrib/mesos/pkg/scheduler/queuer/doc.go b/contrib/mesos/pkg/scheduler/queuer/doc.go deleted file mode 100644 index dc386c955c3..00000000000 --- a/contrib/mesos/pkg/scheduler/queuer/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package queuer implements a Pod Queuer which stores and yields pods waiting -// being scheduled. -package queuer // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/queuer" diff --git a/contrib/mesos/pkg/scheduler/queuer/pod.go b/contrib/mesos/pkg/scheduler/queuer/pod.go deleted file mode 100644 index 664654f6d3a..00000000000 --- a/contrib/mesos/pkg/scheduler/queuer/pod.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queuer - -import ( - "fmt" - "time" - - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/client/cache" -) - -// functional Pod option -type PodOpt func(*Pod) - -// wrapper for the k8s pod type so that we can define additional methods on a "pod" -type Pod struct { - *api.Pod - deadline *time.Time - delay *time.Duration - notify queue.BreakChan -} - -func NewPod(pod *api.Pod, opt ...PodOpt) *Pod { - p := &Pod{Pod: pod} - for _, f := range opt { - f(p) - } - return p -} - -// Deadline sets the deadline for a Pod -func Deadline(deadline time.Time) PodOpt { - return func(pod *Pod) { - pod.deadline = &deadline - } -} - -// Delay sets the delay for a Pod -func Delay(delay time.Duration) PodOpt { - return func(pod *Pod) { - pod.delay = &delay - } -} - -// Notify sets the breakout notification channel for a Pod -func Notify(notify queue.BreakChan) PodOpt { - return func(pod *Pod) { - pod.notify = notify - } -} - -// implements Copyable -func (p *Pod) Copy() queue.Copyable { - if p == nil { - return nil - } - //TODO(jdef) we may need a better "deep-copy" implementation - pod := *(p.Pod) - return &Pod{Pod: &pod} -} - -// implements Unique -func (p *Pod) GetUID() string { - if id, err := cache.MetaNamespaceKeyFunc(p.Pod); err != nil { - panic(fmt.Sprintf("failed to determine pod id for '%+v'", p.Pod)) - } else { - return id - } -} - -// implements Deadlined -func (dp *Pod) Deadline() (time.Time, bool) { - if dp.deadline != nil { - return *(dp.deadline), true - } - return time.Time{}, false -} - -func (dp *Pod) GetDelay() time.Duration { - if dp.delay != nil { - return *(dp.delay) - } - return 0 -} - -func (p *Pod) Breaker() queue.BreakChan { - return p.notify -} - -func (p *Pod) String() string { - displayDeadline := "" - if deadline, ok := p.Deadline(); ok { - displayDeadline = deadline.String() - } - return fmt.Sprintf("{pod:%v, deadline:%v, delay:%v}", p.Pod.Name, displayDeadline, p.GetDelay()) -} diff --git a/contrib/mesos/pkg/scheduler/queuer/queuer.go b/contrib/mesos/pkg/scheduler/queuer/queuer.go deleted file mode 100644 index 724d2cafb80..00000000000 --- a/contrib/mesos/pkg/scheduler/queuer/queuer.go +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package queuer - -import ( - "fmt" - "io" - "net/http" - "sync" - "time" - - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/queue" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - annotation "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/client/cache" -) - -const ( - enqueuePopTimeout = 200 * time.Millisecond - enqueueWaitTimeout = 1 * time.Second - yieldPopTimeout = 200 * time.Millisecond - yieldWaitTimeout = 1 * time.Second -) - -type Queuer interface { - InstallDebugHandlers(mux *http.ServeMux) - UpdatesAvailable() - Dequeue(id string) - Requeue(pod *Pod) - Reoffer(pod *Pod) - - Yield() *api.Pod - - Run(done <-chan struct{}) -} - -type queuer struct { - lock sync.Mutex // shared by condition variables of this struct - updates queue.FIFO // queue of pod updates to be processed - queue *queue.DelayFIFO // queue of pods to be scheduled - deltaCond sync.Cond // pod changes are available for processing - unscheduledCond sync.Cond // there are unscheduled pods for processing -} - -func New(queue *queue.DelayFIFO, updates queue.FIFO) Queuer { - q := &queuer{ - queue: queue, - updates: updates, - } - q.deltaCond.L = &q.lock - q.unscheduledCond.L = &q.lock - return q -} - -func (q *queuer) InstallDebugHandlers(mux *http.ServeMux) { - mux.HandleFunc("/debug/scheduler/podqueue", func(w http.ResponseWriter, r *http.Request) { - for _, x := range q.queue.List() { - if _, err := io.WriteString(w, fmt.Sprintf("%+v\n", x)); err != nil { - break - } - } - }) - mux.HandleFunc("/debug/scheduler/podstore", func(w http.ResponseWriter, r *http.Request) { - for _, x := range q.updates.List() { - if _, err := io.WriteString(w, fmt.Sprintf("%+v\n", x)); err != nil { - break - } - } - }) -} - -// signal that there are probably pod updates waiting to be processed -func (q *queuer) UpdatesAvailable() { - q.deltaCond.Broadcast() -} - -// delete a pod from the to-be-scheduled queue -func (q *queuer) Dequeue(id string) { - q.queue.Delete(id) -} - -// re-add a pod to the to-be-scheduled queue, will not overwrite existing pod data (that -// may have already changed). -func (q *queuer) Requeue(pod *Pod) { - // use KeepExisting in case the pod has already been updated (can happen if binding fails - // due to constraint voilations); we don't want to overwrite a newer entry with stale data. - q.queue.Add(pod, queue.KeepExisting) - q.unscheduledCond.Broadcast() -} - -// same as Requeue but calls podQueue.Offer instead of podQueue.Add -func (q *queuer) Reoffer(pod *Pod) { - // use KeepExisting in case the pod has already been updated (can happen if binding fails - // due to constraint voilations); we don't want to overwrite a newer entry with stale data. - if q.queue.Offer(pod, queue.KeepExisting) { - q.unscheduledCond.Broadcast() - } -} - -// spawns a go-routine to watch for unscheduled pods and queue them up -// for scheduling. returns immediately. -func (q *queuer) Run(done <-chan struct{}) { - go runtime.Until(func() { - log.Info("Watching for newly created pods") - q.lock.Lock() - defer q.lock.Unlock() - - for { - // limit blocking here for short intervals so that scheduling - // may proceed even if there have been no recent pod changes - p := q.updates.Await(enqueuePopTimeout) - if p == nil { - signalled := runtime.After(q.deltaCond.Wait) - // we've yielded the lock - select { - case <-time.After(enqueueWaitTimeout): - q.deltaCond.Broadcast() // abort Wait() - <-signalled // wait for lock re-acquisition - log.V(4).Infoln("timed out waiting for a pod update") - case <-signalled: - // we've acquired the lock and there may be - // changes for us to process now - } - continue - } - - pod := p.(*Pod) - if recoverAssignedSlave(pod.Pod) != "" { - log.V(3).Infof("dequeuing assigned pod for scheduling: %v", pod.Pod.Name) - q.Dequeue(pod.GetUID()) - } else { - // use ReplaceExisting because we are always pushing the latest state - now := time.Now() - pod.deadline = &now - if q.queue.Offer(pod, queue.ReplaceExisting) { - q.unscheduledCond.Broadcast() - log.V(3).Infof("queued pod for scheduling: %v", pod.Pod.Name) - } else { - log.Warningf("failed to queue pod for scheduling: %v", pod.Pod.Name) - } - } - } - }, 1*time.Second, done) -} - -// implementation of scheduling plugin's NextPod func; see k8s plugin/pkg/scheduler -func (q *queuer) Yield() *api.Pod { - log.V(2).Info("attempting to yield a pod") - q.lock.Lock() - defer q.lock.Unlock() - - for { - // limit blocking here to short intervals so that we don't block the - // enqueuer Run() routine for very long - kpod := q.queue.Await(yieldPopTimeout) - if kpod == nil { - signalled := runtime.After(q.unscheduledCond.Wait) - // lock is yielded at this point and we're going to wait for either - // a timeout, or a signal that there's data - select { - case <-time.After(yieldWaitTimeout): - q.unscheduledCond.Broadcast() // abort Wait() - <-signalled // wait for the go-routine, and the lock - log.V(4).Infoln("timed out waiting for a pod to yield") - case <-signalled: - // we have acquired the lock, and there - // may be a pod for us to pop now - } - continue - } - - pod := kpod.(*Pod).Pod - if podName, err := cache.MetaNamespaceKeyFunc(pod); err != nil { - log.Warningf("yield unable to understand pod object %+v, will skip: %v", pod, err) - } else if !q.updates.Poll(podName, queue.POP_EVENT) { - log.V(1).Infof("yield popped a transitioning pod, skipping: %+v", pod) - } else if recoverAssignedSlave(pod) != "" { - // should never happen if enqueuePods is filtering properly - log.Warningf("yield popped an already-scheduled pod, skipping: %+v", pod) - } else { - return pod - } - } -} - -// recoverAssignedSlave recovers the assigned Mesos slave from a pod by searching -// the BindingHostKey. For tasks in the registry of the scheduler, the same -// value is stored in T.Spec.AssignedSlave. Before launching, the BindingHostKey -// annotation is added and the executor will eventually persist that to the -// apiserver on binding. -func recoverAssignedSlave(pod *api.Pod) string { - return pod.Annotations[annotation.BindingHostKey] -} diff --git a/contrib/mesos/pkg/scheduler/resources/doc.go b/contrib/mesos/pkg/scheduler/resources/doc.go deleted file mode 100644 index 56a97b7a558..00000000000 --- a/contrib/mesos/pkg/scheduler/resources/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package resources contains the Mesos scheduler specific resource functions -package resources // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" diff --git a/contrib/mesos/pkg/scheduler/resources/resource.go b/contrib/mesos/pkg/scheduler/resources/resource.go deleted file mode 100644 index c2d930d8cf5..00000000000 --- a/contrib/mesos/pkg/scheduler/resources/resource.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/resource" -) - -const ( - DefaultDefaultContainerCPULimit = CPUShares(0.25) // CPUs allocated for pods without CPU limit - DefaultDefaultContainerMemLimit = MegaBytes(64.0) // memory allocated for pods without memory limit - MinimumContainerCPU = CPUShares(0.01) // minimum CPUs allowed by Mesos - MinimumContainerMem = MegaBytes(32.0) // minimum memory allowed by Mesos -) - -var ( - zero = *resource.NewQuantity(0, resource.BinarySI) -) - -// podResource computes requested resources and the limit. If write is true, -// it will also write missing requests and limits into the pod. -func podResources(pod *api.Pod, resourceName api.ResourceName, def, min resource.Quantity, write bool) ( - requestSum *resource.Quantity, - limitSum *resource.Quantity, - modified bool, - err error, -) { - requestSum = (&zero).Copy() - limitSum = (&zero).Copy() - modified = false - err = nil - - for j := range pod.Spec.Containers { - container := &pod.Spec.Containers[j] - - // create maps - if container.Resources.Limits == nil { - container.Resources.Limits = api.ResourceList{} - } - if container.Resources.Requests == nil { - container.Resources.Requests = api.ResourceList{} - } - - // request and limit defined? - request, requestFound := container.Resources.Requests[resourceName] - limit, limitFound := container.Resources.Limits[resourceName] - - // fill-in missing request and/or limit - if !requestFound && !limitFound { - limit = def - request = def - modified = true - } else if requestFound && !limitFound { - limit = request - modified = true - } else if !requestFound && limitFound { - // TODO(sttts): possibly use the default here? - request = limit - modified = true - } - - // make request and limit at least as big as min - if (&request).Cmp(min) < 0 { - request = *(&min).Copy() - modified = true - } - if (&limit).Cmp(min) < 0 { - limit = *(&min).Copy() - modified = true - } - - // add up the request and limit sum for all containers - requestSum.Add(request) - limitSum.Add(limit) - - // optionally write request and limit back - if write { - container.Resources.Requests[resourceName] = request - container.Resources.Limits[resourceName] = limit - } - } - return -} - -// LimitPodCPU sets default CPU requests and limits of each container that -// does not limit its CPU resource yet. LimitPodCPU returns the new request, -// limit and whether the pod was modified. -func LimitPodCPU(pod *api.Pod, defaultLimit CPUShares) (request, limit CPUShares, modified bool, err error) { - r, l, m, err := podResources(pod, api.ResourceCPU, *defaultLimit.Quantity(), *MinimumContainerCPU.Quantity(), true) - if err != nil { - return 0.0, 0.0, false, err - } - return NewCPUShares(*r), NewCPUShares(*l), m, nil -} - -// LimitPodMem sets default memory requests and limits of each container that -// does not limit its memory resource yet. LimitPodMem returns the new request, -// limit and whether the pod was modified. -func LimitPodMem(pod *api.Pod, defaultLimit MegaBytes) (request, limit MegaBytes, modified bool, err error) { - r, l, m, err := podResources(pod, api.ResourceMemory, *defaultLimit.Quantity(), *MinimumContainerMem.Quantity(), true) - if err != nil { - return 0.0, 0.0, false, err - } - return NewMegaBytes(*r), NewMegaBytes(*l), m, nil -} - -// LimitedCPUForPod computes the limits from the spec plus the default CPU limit difference for unlimited containers -func LimitedCPUForPod(pod *api.Pod, defaultLimit CPUShares) (request, limit CPUShares, modified bool, err error) { - r, l, m, err := podResources(pod, api.ResourceCPU, *defaultLimit.Quantity(), *MinimumContainerCPU.Quantity(), false) - if err != nil { - return 0.0, 0.0, false, err - } - return NewCPUShares(*r), NewCPUShares(*l), m, nil -} - -// LimitedMemForPod computes the limits from the spec plus the default memory limit difference for unlimited containers -func LimitedMemForPod(pod *api.Pod, defaultLimit MegaBytes) (request, limit MegaBytes, modified bool, err error) { - r, l, m, err := podResources(pod, api.ResourceMemory, *defaultLimit.Quantity(), *MinimumContainerMem.Quantity(), true) - if err != nil { - return 0.0, 0.0, false, err - } - return NewMegaBytes(*r), NewMegaBytes(*l), m, nil -} diff --git a/contrib/mesos/pkg/scheduler/resources/resource_test.go b/contrib/mesos/pkg/scheduler/resources/resource_test.go deleted file mode 100644 index 3c7780907c8..00000000000 --- a/contrib/mesos/pkg/scheduler/resources/resource_test.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "math" - "testing" - - "github.com/stretchr/testify/assert" - "k8s.io/kubernetes/pkg/api" -) - -type resources struct { - cpuR, cpuL, memR, memL float64 -} - -func TestResources(tst *testing.T) { - assert := assert.New(tst) - - const ( - defCpu = float64(DefaultDefaultContainerCPULimit) - defMem = float64(DefaultDefaultContainerMemLimit) - minCpu = float64(MinimumContainerCPU) - minMem = float64(MinimumContainerMem) - ) - undef := math.NaN() - defined := func(f float64) bool { return !math.IsNaN(f) } - - for _, t := range []struct { - input, want resources - }{ - {resources{undef, 3.0, undef, 768.0}, resources{defCpu + 3.0, defCpu + 3.0, defMem + 768.0, defMem + 768.0}}, - {resources{0.0, 3.0, 0.0, 768.0}, resources{defCpu + minCpu, defCpu + 3.0, defMem + minMem, defMem + 768.0}}, - {resources{undef, undef, undef, undef}, resources{2 * defCpu, 2 * defCpu, 2 * defMem, 2 * defMem}}, - {resources{0.0, 0.0, 0.0, 0.0}, resources{minCpu + defCpu, minCpu + defCpu, minMem + defMem, minMem + defMem}}, - {resources{2.0, 3.0, undef, 768.0}, resources{defCpu + 2.0, defCpu + 3.0, defMem + 768.0, defMem + 768.0}}, - {resources{2.0, 3.0, 256.0, 768.0}, resources{defCpu + 2.0, defCpu + 3.0, defMem + 256.0, defMem + 768.0}}, - } { - pod := &api.Pod{ - Spec: api.PodSpec{ - Containers: []api.Container{{ - Name: "a", - }, { - Name: "b", - }}, - }, - } - - if defined(t.input.cpuR) || defined(t.input.memR) { - pod.Spec.Containers[0].Resources.Requests = api.ResourceList{} - if defined(t.input.cpuR) { - pod.Spec.Containers[0].Resources.Requests[api.ResourceCPU] = *CPUShares(t.input.cpuR).Quantity() - } - if defined(t.input.memR) { - pod.Spec.Containers[0].Resources.Requests[api.ResourceMemory] = *MegaBytes(t.input.memR).Quantity() - } - } - if defined(t.input.cpuL) || defined(t.input.memL) { - pod.Spec.Containers[0].Resources.Limits = api.ResourceList{} - if defined(t.input.cpuL) { - pod.Spec.Containers[0].Resources.Limits[api.ResourceCPU] = *CPUShares(t.input.cpuL).Quantity() - } - if defined(t.input.memL) { - pod.Spec.Containers[0].Resources.Limits[api.ResourceMemory] = *MegaBytes(t.input.memL).Quantity() - } - } - - tst.Logf("Testing resource computation for %v => request=%v limit=%v", t, pod.Spec.Containers[0].Resources.Requests, pod.Spec.Containers[0].Resources.Limits) - - beforeCpuR, beforeCpuL, _, err := LimitedCPUForPod(pod, DefaultDefaultContainerCPULimit) - assert.NoError(err, "CPUForPod should not return an error") - - beforeMemR, beforeMemL, _, err := LimitedMemForPod(pod, DefaultDefaultContainerMemLimit) - assert.NoError(err, "MemForPod should not return an error") - - cpuR, cpuL, _, err := LimitPodCPU(pod, DefaultDefaultContainerCPULimit) - assert.NoError(err, "LimitPodCPU should not return an error") - - memR, memL, _, err := LimitPodMem(pod, DefaultDefaultContainerMemLimit) - assert.NoError(err, "LimitPodMem should not return an error") - - tst.Logf("New resources container 0: request=%v limit=%v", pod.Spec.Containers[0].Resources.Requests, pod.Spec.Containers[0].Resources.Limits) - tst.Logf("New resources container 1: request=%v limit=%v", pod.Spec.Containers[1].Resources.Requests, pod.Spec.Containers[1].Resources.Limits) - - assert.Equal(t.want.cpuR, float64(beforeCpuR), "cpu request before modifiation is wrong") - assert.Equal(t.want.cpuL, float64(beforeCpuL), "cpu limit before modifiation is wrong") - - assert.Equal(t.want.memR, float64(beforeMemR), "mem request before modifiation is wrong") - assert.Equal(t.want.memL, float64(beforeMemL), "mem limit before modifiation is wrong") - - assert.Equal(t.want.cpuR, float64(cpuR), "cpu request is wrong") - assert.Equal(t.want.cpuL, float64(cpuL), "cpu limit is wrong") - - assert.Equal(t.want.memR, float64(memR), "mem request is wrong") - assert.Equal(t.want.memL, float64(memL), "mem limit is wrong") - } -} diff --git a/contrib/mesos/pkg/scheduler/resources/resources.go b/contrib/mesos/pkg/scheduler/resources/resources.go deleted file mode 100644 index b5da64c7376..00000000000 --- a/contrib/mesos/pkg/scheduler/resources/resources.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "github.com/gogo/protobuf/proto" - mesos "github.com/mesos/mesos-go/mesosproto" -) - -type Port struct { - Port uint64 - Role string -} - -// PortRanges creates a range resource for the spec ports. -func PortRanges(Ports []Port) []*mesos.Resource { - rolePorts := make(map[string][]uint64, len(Ports)) - - for _, p := range Ports { - rolePorts[p.Role] = append(rolePorts[p.Role], p.Port) - } - - resources := make([]*mesos.Resource, 0, len(rolePorts)) - for role, ports := range rolePorts { - resources = append( - resources, - &mesos.Resource{ - Name: proto.String("ports"), - Type: mesos.Value_RANGES.Enum(), - Ranges: NewRanges(ports), - Role: StringPtrTo(role), - }, - ) - } - - return resources -} - -// NewRanges generates port ranges from the given list of ports. (naive implementation) -func NewRanges(ports []uint64) *mesos.Value_Ranges { - r := make([]*mesos.Value_Range, 0, len(ports)) - for _, port := range ports { - x := proto.Uint64(port) - r = append(r, &mesos.Value_Range{Begin: x, End: x}) - } - return &mesos.Value_Ranges{Range: r} -} - -// ForeachPortsRange calls f for each resource that matches the given roles -// in the order of the given roles. -func ForeachPortsRange(rs []*mesos.Resource, roles []string, f func(begin, end uint64, role string)) { - rs = Filter(rs, HasName("ports")) - rs = ByRoles(roles...).Sort(rs) - - for _, resource := range rs { - for _, r := range (*resource).GetRanges().Range { - bp := r.GetBegin() - ep := r.GetEnd() - f(bp, ep, (*resource).GetRole()) - } - } -} - -// ByRolesSorter sorts resources according to the ordering of roles. -type ByRolesSorter struct { - roles []string -} - -// ByRoles returns a ByRolesSorter with the given roles. -func ByRoles(roles ...string) *ByRolesSorter { - return &ByRolesSorter{roles: roles} -} - -// sort sorts the given resources according to the order of roles in the ByRolesSorter -// and returns the sorted resources. -func (sorter *ByRolesSorter) Sort(resources []*mesos.Resource) []*mesos.Resource { - rolesMap := map[string][]*mesos.Resource{} // maps roles to resources - for _, res := range resources { - role := CanonicalRole(res.GetRole()) - rolesMap[role] = append(rolesMap[role], res) - } - - result := make([]*mesos.Resource, 0, len(resources)) - for _, role := range sorter.roles { - for _, res := range rolesMap[role] { - result = append(result, res) - } - } - - return result -} - -// ResourcePredicate is a predicate function on *mesos.Resource structs. -type ( - ResourcePredicate func(*mesos.Resource) bool - ResourcePredicates []ResourcePredicate -) - -// Filter filters the given slice of resources and returns a slice of resources -// matching all given predicates. -func Filter(res []*mesos.Resource, ps ...ResourcePredicate) []*mesos.Resource { - return ResourcePredicates(ps).Filter(res) -} - -// Filter filters the given slice of resources and returns a slice of resources -// matching all given predicates. -func (ps ResourcePredicates) Filter(res []*mesos.Resource) []*mesos.Resource { - filtered := make([]*mesos.Resource, 0, len(res)) - -next: - for _, r := range res { - for _, p := range ps { - if !p(r) { - continue next - } - } - - filtered = append(filtered, r) - } - - return filtered -} - -// MatchesAll returns true if the given resource matches all given predicates ps. -func MatchesAll(res *mesos.Resource, ps ...ResourcePredicate) bool { - return ResourcePredicates(ps).MatchesAll(res) -} - -// MatchesAll returns true if the given resource matches all given predicates ps. -func (ps ResourcePredicates) MatchesAll(res *mesos.Resource) bool { - for _, p := range ps { - if !p(res) { - return false - } - } - - return true -} - -func Sum(res []*mesos.Resource) float64 { - var sum float64 - - for _, r := range res { - sum += r.GetScalar().GetValue() - } - - return sum -} - -// IsScalar returns true if the given resource is a scalar type. -func IsScalar(r *mesos.Resource) bool { - return r.GetType() == mesos.Value_SCALAR -} - -// HasName returns a ResourcePredicate which returns true -// if the given resource has the given name. -func HasName(name string) ResourcePredicate { - return func(r *mesos.Resource) bool { - return r.GetName() == name - } -} - -// StringPtrTo returns a pointer to the given string -// or nil if it is empty string. -func StringPtrTo(s string) *string { - var protos *string - - if s != "" { - protos = &s - } - - return protos -} - -// CanonicalRole returns a "*" if the given role is empty else the role itself -func CanonicalRole(name string) string { - if name == "" { - return "*" - } - - return name -} - -func NewPorts(role string, ports ...uint64) *mesos.Resource { - return &mesos.Resource{ - Name: proto.String("ports"), - Type: mesos.Value_RANGES.Enum(), - Ranges: NewRanges(ports), - Role: StringPtrTo(role), - } -} diff --git a/contrib/mesos/pkg/scheduler/resources/types.go b/contrib/mesos/pkg/scheduler/resources/types.go deleted file mode 100644 index b0fc0439047..00000000000 --- a/contrib/mesos/pkg/scheduler/resources/types.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "fmt" - "strconv" - - "k8s.io/kubernetes/pkg/api/resource" -) - -type MegaBytes float64 -type CPUShares float64 - -func (f *CPUShares) Set(s string) error { - v, err := strconv.ParseFloat(s, 64) - *f = CPUShares(v) - return err -} - -func (f CPUShares) Type() string { - return "float64" -} - -func (f CPUShares) String() string { return fmt.Sprintf("%v", float64(f)) } - -func (f *MegaBytes) Set(s string) error { - v, err := strconv.ParseFloat(s, 64) - *f = MegaBytes(v) - return err -} - -func (f MegaBytes) Type() string { - return "float64" -} - -func (f MegaBytes) String() string { return fmt.Sprintf("%v", float64(f)) } - -func (f MegaBytes) Quantity() *resource.Quantity { - return resource.NewQuantity(int64(float64(f)*1024.0*1024.0), resource.BinarySI) -} - -func (f CPUShares) Quantity() *resource.Quantity { - return resource.NewMilliQuantity(int64(float64(f)*1000.0), resource.DecimalSI) -} - -func NewCPUShares(q resource.Quantity) CPUShares { - return CPUShares(float64(q.MilliValue()) / 1000.0) -} - -func NewMegaBytes(q resource.Quantity) MegaBytes { - return MegaBytes(float64(q.Value()) / 1024.0 / 1024.0) -} diff --git a/contrib/mesos/pkg/scheduler/scheduler.go b/contrib/mesos/pkg/scheduler/scheduler.go deleted file mode 100644 index e8d82f0b089..00000000000 --- a/contrib/mesos/pkg/scheduler/scheduler.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduler - -import ( - "sync" - - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" -) - -// Scheduler abstracts everything other components of the scheduler need -// to access from eachother -type Scheduler interface { - Tasks() podtask.Registry - sync.Locker // synchronize changes to tasks, i.e. lock, get task, change task, store task, unlock - - Offers() offers.Registry - Reconcile(t *podtask.T) - KillTask(id string) error - LaunchTask(t *podtask.T) error - Run(done <-chan struct{}) -} diff --git a/contrib/mesos/pkg/scheduler/scheduler_mock.go b/contrib/mesos/pkg/scheduler/scheduler_mock.go deleted file mode 100644 index ebff007be54..00000000000 --- a/contrib/mesos/pkg/scheduler/scheduler_mock.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheduler - -import ( - "sync" - - "github.com/stretchr/testify/mock" - "k8s.io/kubernetes/contrib/mesos/pkg/offers" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "time" -) - -// MockScheduler implements SchedulerApi -type MockScheduler struct { - sync.RWMutex - mock.Mock -} - -func (m *MockScheduler) Run(done <-chan struct{}) { - _ = m.Called() - runtime.Until(func() { - time.Sleep(time.Second) - }, time.Second, done) - return -} - -func (m *MockScheduler) Offers() (f offers.Registry) { - args := m.Called() - x := args.Get(0) - if x != nil { - f = x.(offers.Registry) - } - return -} - -func (m *MockScheduler) Tasks() (f podtask.Registry) { - args := m.Called() - x := args.Get(0) - if x != nil { - f = x.(podtask.Registry) - } - return -} - -func (m *MockScheduler) KillTask(taskId string) error { - args := m.Called(taskId) - return args.Error(0) -} - -func (m *MockScheduler) LaunchTask(task *podtask.T) error { - args := m.Called(task) - return args.Error(0) -} - -func (m *MockScheduler) Reconcile(task *podtask.T) { - _ = m.Called() - return -} diff --git a/contrib/mesos/pkg/scheduler/service/compat_testing.go b/contrib/mesos/pkg/scheduler/service/compat_testing.go deleted file mode 100644 index 005a17a7130..00000000000 --- a/contrib/mesos/pkg/scheduler/service/compat_testing.go +++ /dev/null @@ -1,32 +0,0 @@ -// +build unit_test - -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "os" - "syscall" -) - -func makeFailoverSigChan() <-chan os.Signal { - return nil -} - -func makeDisownedProcAttr() *syscall.SysProcAttr { - return nil -} diff --git a/contrib/mesos/pkg/scheduler/service/compat_unix.go b/contrib/mesos/pkg/scheduler/service/compat_unix.go deleted file mode 100644 index f14abd73eb6..00000000000 --- a/contrib/mesos/pkg/scheduler/service/compat_unix.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build darwin dragonfly freebsd linux netbsd openbsd -// +build !unit_test - -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "os" - "os/signal" - "syscall" -) - -func makeFailoverSigChan() <-chan os.Signal { - ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGUSR1) - return ch -} - -func makeDisownedProcAttr() *syscall.SysProcAttr { - return &syscall.SysProcAttr{ - Setpgid: true, // disown the spawned scheduler - } -} diff --git a/contrib/mesos/pkg/scheduler/service/compat_windows.go b/contrib/mesos/pkg/scheduler/service/compat_windows.go deleted file mode 100644 index de151c69171..00000000000 --- a/contrib/mesos/pkg/scheduler/service/compat_windows.go +++ /dev/null @@ -1,51 +0,0 @@ -// +build windows -// +build !unit_test - -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "os" - "syscall" -) - -func makeFailoverSigChan() <-chan os.Signal { - /* TODO(jdef) - from go's windows compatibility test, it looks like we need to provide a filtered - signal channel here - - c := make(chan os.Signal, 10) - signal.Notify(c) - select { - case s := <-c: - if s != os.Interrupt { - log.Fatalf("Wrong signal received: got %q, want %q\n", s, os.Interrupt) - } - case <-time.After(3 * time.Second): - log.Fatalf("Timeout waiting for Ctrl+Break\n") - } - */ - return nil -} - -func makeDisownedProcAttr() *syscall.SysProcAttr { - //TODO(jdef) test this somehow?!?! - return &syscall.SysProcAttr{ - CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP | syscall.CREATE_UNICODE_ENVIRONMENT, - } -} diff --git a/contrib/mesos/pkg/scheduler/service/doc.go b/contrib/mesos/pkg/scheduler/service/doc.go deleted file mode 100644 index 159c22edf51..00000000000 --- a/contrib/mesos/pkg/scheduler/service/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package service contains the cmd/k8sm-scheduler glue code -package service // import "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/service" diff --git a/contrib/mesos/pkg/scheduler/service/publish.go b/contrib/mesos/pkg/scheduler/service/publish.go deleted file mode 100644 index cc0a67f6a1a..00000000000 --- a/contrib/mesos/pkg/scheduler/service/publish.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "net" - "reflect" - "time" - - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/errors" - "k8s.io/kubernetes/pkg/master/ports" - - "github.com/golang/glog" -) - -const ( - SCHEDULER_SERVICE_NAME = "k8sm-scheduler" -) - -func (m *SchedulerServer) newServiceWriter(publishedAddress net.IP, stop <-chan struct{}) func() { - return func() { - for { - // Update service & endpoint records. - // TODO(k8s): when it becomes possible to change this stuff, - // stop polling and start watching. - if err := m.createSchedulerServiceIfNeeded(SCHEDULER_SERVICE_NAME, ports.SchedulerPort); err != nil { - glog.Errorf("Can't create scheduler service: %v", err) - } - - if publishedAddress == nil { - publishedAddress = net.IP(m.address) - } - if err := m.setEndpoints(SCHEDULER_SERVICE_NAME, publishedAddress, m.port); err != nil { - glog.Errorf("Can't create scheduler endpoints: %v", err) - } - - select { - case <-stop: - return - case <-time.After(10 * time.Second): - } - } - } -} - -// createSchedulerServiceIfNeeded will create the specified service if it -// doesn't already exist. -func (m *SchedulerServer) createSchedulerServiceIfNeeded(serviceName string, servicePort int) error { - ctx := api.NewDefaultContext() - if _, err := m.client.Core().Services(api.NamespaceValue(ctx)).Get(serviceName); err == nil { - // The service already exists. - return nil - } - svc := &api.Service{ - ObjectMeta: api.ObjectMeta{ - Name: serviceName, - Namespace: api.NamespaceDefault, - Labels: map[string]string{"provider": "k8sm", "component": "scheduler"}, - }, - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{{Port: int32(servicePort), Protocol: api.ProtocolTCP}}, - // maintained by this code, not by the pod selector - Selector: nil, - SessionAffinity: api.ServiceAffinityNone, - }, - } - if m.serviceAddress != nil { - svc.Spec.ClusterIP = m.serviceAddress.String() - } - _, err := m.client.Core().Services(api.NamespaceValue(ctx)).Create(svc) - if err != nil && errors.IsAlreadyExists(err) { - err = nil - } - return err -} - -// setEndpoints sets the endpoints for the given service. -// in a multi-master scenario only the master will be publishing an endpoint. -// see SchedulerServer.bootstrap. -func (m *SchedulerServer) setEndpoints(serviceName string, ip net.IP, port int) error { - // The setting we want to find. - want := []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: ip.String()}}, - Ports: []api.EndpointPort{{Port: int32(port), Protocol: api.ProtocolTCP}}, - }} - - ctx := api.NewDefaultContext() - e, err := m.client.Endpoints(api.NamespaceValue(ctx)).Get(serviceName) - createOrUpdate := m.client.Endpoints(api.NamespaceValue(ctx)).Update - if err != nil { - if errors.IsNotFound(err) { - createOrUpdate = m.client.Endpoints(api.NamespaceValue(ctx)).Create - } - e = &api.Endpoints{ - ObjectMeta: api.ObjectMeta{ - Name: serviceName, - Namespace: api.NamespaceDefault, - }, - } - } - if !reflect.DeepEqual(e.Subsets, want) { - e.Subsets = want - glog.Infof("Setting endpoints for master service %q to %#v", serviceName, e) - _, err = createOrUpdate(e) - return err - } - // We didn't make any changes, no need to actually call update. - return nil -} diff --git a/contrib/mesos/pkg/scheduler/service/service.go b/contrib/mesos/pkg/scheduler/service/service.go deleted file mode 100644 index 94f0cc44806..00000000000 --- a/contrib/mesos/pkg/scheduler/service/service.go +++ /dev/null @@ -1,1095 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "bufio" - "fmt" - "io/ioutil" - "net" - "net/http" - "net/url" - "os" - "os/exec" - "os/user" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - etcd "github.com/coreos/etcd/client" - "github.com/gogo/protobuf/proto" - log "github.com/golang/glog" - "github.com/kardianos/osext" - "github.com/mesos/mesos-go/auth" - "github.com/mesos/mesos-go/auth/sasl" - "github.com/mesos/mesos-go/auth/sasl/mech" - mesos "github.com/mesos/mesos-go/mesosproto" - mutil "github.com/mesos/mesos-go/mesosutil" - bindings "github.com/mesos/mesos-go/scheduler" - "github.com/pborman/uuid" - "github.com/prometheus/client_golang/prometheus" - "github.com/spf13/pflag" - "golang.org/x/net/context" - - "k8s.io/kubernetes/contrib/mesos/pkg/election" - execcfg "k8s.io/kubernetes/contrib/mesos/pkg/executor/config" - "k8s.io/kubernetes/contrib/mesos/pkg/flagutil" - "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" - minioncfg "k8s.io/kubernetes/contrib/mesos/pkg/minion/config" - "k8s.io/kubernetes/contrib/mesos/pkg/podutil" - "k8s.io/kubernetes/contrib/mesos/pkg/profile" - "k8s.io/kubernetes/contrib/mesos/pkg/runtime" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/algorithm/podschedulers" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework/frameworkid" - frameworkidEtcd "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework/frameworkid/etcd" - frameworkidZk "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/components/framework/frameworkid/zk" - schedcfg "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/config" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/executorinfo" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/ha" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/metrics" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/podtask/hostport" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/resource" - "k8s.io/kubernetes/pkg/client/cache" - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" - "k8s.io/kubernetes/pkg/client/record" - "k8s.io/kubernetes/pkg/client/restclient" - "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" - cloud "k8s.io/kubernetes/pkg/cloudprovider/providers/mesos" - "k8s.io/kubernetes/pkg/fields" - "k8s.io/kubernetes/pkg/healthz" - "k8s.io/kubernetes/pkg/master/ports" - "k8s.io/kubernetes/pkg/util/sets" - - // lock to this API version, compilation will fail when this becomes unsupported - _ "k8s.io/kubernetes/pkg/api/v1" -) - -const ( - defaultMesosMaster = "localhost:5050" - defaultMesosUser = "root" // should have privs to execute docker and iptables commands - defaultFrameworkRoles = "*" - defaultPodRoles = "*" - defaultReconcileInterval = 300 // 5m default task reconciliation interval - defaultReconcileCooldown = 15 * time.Second - defaultNodeRelistPeriod = 5 * time.Minute - defaultFrameworkName = "Kubernetes" - defaultExecutorCPUs = resources.CPUShares(0.25) // initial CPU allocated for executor - defaultExecutorMem = resources.MegaBytes(128.0) // initial memory allocated for executor - defaultExecutorInfoCacheSize = 10000 -) - -type SchedulerServer struct { - port int - address net.IP - enableProfiling bool - kubeconfig string - kubeAPIQPS float32 - kubeAPIBurst int - apiServerList []string - etcdServerList []string - allowPrivileged bool - executorPath string - proxyPath string - mesosMaster string - mesosUser string - frameworkRoles []string - defaultPodRoles []string - mesosAuthPrincipal string - mesosAuthSecretFile string - mesosCgroupPrefix string - mesosExecutorCPUs resources.CPUShares - mesosExecutorMem resources.MegaBytes - checkpoint bool - failoverTimeout float64 - generateTaskDiscovery bool - frameworkStoreURI string - - executorLogV int - executorBindall bool - executorSuicideTimeout time.Duration - launchGracePeriod time.Duration - kubeletEnableDebuggingHandlers bool - - runProxy bool - proxyBindall bool - proxyKubeconfig string - proxyLogV int - proxyMode string - - minionPathOverride string - minionLogMaxSize resource.Quantity - minionLogMaxBackups int - minionLogMaxAgeInDays int - - mesosAuthProvider string - driverPort uint - hostnameOverride string - reconcileInterval int64 - reconcileCooldown time.Duration - defaultContainerCPULimit resources.CPUShares - defaultContainerMemLimit resources.MegaBytes - schedulerConfigFileName string - graceful bool - frameworkName string - frameworkWebURI string - ha bool - advertisedAddress string - serviceAddress net.IP - haDomain string - kmPath string - clusterDNS net.IP - clusterDomain string - kubeletApiServerList []string - kubeletRootDirectory string - kubeletDockerEndpoint string - kubeletPodInfraContainerImage string - kubeletCadvisorPort uint - kubeletHostNetworkSources string - kubeletSyncFrequency time.Duration - kubeletNetworkPluginName string - kubeletKubeconfig string - staticPodsConfigPath string - dockerCfgPath string - containPodResources bool - nodeRelistPeriod time.Duration - sandboxOverlay string - conntrackMax int - conntrackTCPTimeoutEstablished int - useHostPortEndpoints bool - - executable string // path to the binary running this service - client *clientset.Clientset - driver bindings.SchedulerDriver - driverMutex sync.RWMutex - mux *http.ServeMux -} - -// useful for unit testing specific funcs -type schedulerProcessInterface interface { - End() <-chan struct{} - Failover() <-chan struct{} - Terminal() <-chan struct{} -} - -// NewSchedulerServer creates a new SchedulerServer with default parameters -func NewSchedulerServer() *SchedulerServer { - s := SchedulerServer{ - port: ports.SchedulerPort, - address: net.ParseIP("127.0.0.1"), - failoverTimeout: time.Duration((1 << 62) - 1).Seconds(), - frameworkStoreURI: "etcd://", - kubeAPIQPS: 50.0, - kubeAPIBurst: 100, - - runProxy: true, - executorSuicideTimeout: execcfg.DefaultSuicideTimeout, - launchGracePeriod: execcfg.DefaultLaunchGracePeriod, - defaultContainerCPULimit: resources.DefaultDefaultContainerCPULimit, - defaultContainerMemLimit: resources.DefaultDefaultContainerMemLimit, - - proxyMode: "userspace", // upstream default is "iptables" post-v1.1 - - minionLogMaxSize: minioncfg.DefaultLogMaxSize(), - minionLogMaxBackups: minioncfg.DefaultLogMaxBackups, - minionLogMaxAgeInDays: minioncfg.DefaultLogMaxAgeInDays, - - mesosAuthProvider: sasl.ProviderName, - mesosCgroupPrefix: minioncfg.DefaultCgroupPrefix, - mesosMaster: defaultMesosMaster, - mesosUser: defaultMesosUser, - mesosExecutorCPUs: defaultExecutorCPUs, - mesosExecutorMem: defaultExecutorMem, - frameworkRoles: strings.Split(defaultFrameworkRoles, ","), - defaultPodRoles: strings.Split(defaultPodRoles, ","), - reconcileInterval: defaultReconcileInterval, - reconcileCooldown: defaultReconcileCooldown, - checkpoint: true, - frameworkName: defaultFrameworkName, - ha: false, - mux: http.NewServeMux(), - kubeletCadvisorPort: 4194, // copied from github.com/GoogleCloudPlatform/kubernetes/blob/release-0.14/cmd/kubelet/app/server.go - kubeletSyncFrequency: 10 * time.Second, - kubeletEnableDebuggingHandlers: true, - containPodResources: true, - nodeRelistPeriod: defaultNodeRelistPeriod, - conntrackTCPTimeoutEstablished: 0, // non-zero values may require hand-tuning other sysctl's on the host; do so with caution - useHostPortEndpoints: true, - - // non-zero values can trigger failures when updating /sys/module/nf_conntrack/parameters/hashsize - // when kube-proxy is running in a non-root netns (init_net); setting this to a non-zero value will - // impact connection tracking for the entire host on which kube-proxy is running. xref (k8s#19182) - conntrackMax: 0, - } - // cache this for later use. also useful in case the original binary gets deleted, e.g. - // during upgrades, development deployments, etc. - if filename, err := osext.Executable(); err != nil { - log.Fatalf("failed to determine path to currently running executable: %v", err) - } else { - s.executable = filename - s.kmPath = filename - } - - return &s -} - -func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) { - fs.IntVar(&s.port, "port", s.port, "The port that the scheduler's http service runs on") - fs.IPVar(&s.address, "address", s.address, "The IP address to serve on (set to 0.0.0.0 for all interfaces)") - fs.BoolVar(&s.enableProfiling, "profiling", s.enableProfiling, "Enable profiling via web interface host:port/debug/pprof/") - fs.StringSliceVar(&s.apiServerList, "api-servers", s.apiServerList, "List of Kubernetes API servers for publishing events, and reading pods and services. (ip:port), comma separated.") - fs.StringVar(&s.kubeconfig, "kubeconfig", s.kubeconfig, "Path to kubeconfig file with authorization and master location information used by the scheduler.") - fs.Float32Var(&s.kubeAPIQPS, "kube-api-qps", s.kubeAPIQPS, "QPS to use while talking with kubernetes apiserver") - fs.IntVar(&s.kubeAPIBurst, "kube-api-burst", s.kubeAPIBurst, "Burst to use while talking with kubernetes apiserver") - fs.StringSliceVar(&s.etcdServerList, "etcd-servers", s.etcdServerList, "List of etcd servers to watch (http://ip:port), comma separated.") - fs.BoolVar(&s.allowPrivileged, "allow-privileged", s.allowPrivileged, "Enable privileged containers in the kubelet (compare the same flag in the apiserver).") - fs.StringVar(&s.clusterDomain, "cluster-domain", s.clusterDomain, "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains") - fs.IPVar(&s.clusterDNS, "cluster-dns", s.clusterDNS, "IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers") - fs.StringVar(&s.staticPodsConfigPath, "static-pods-config", s.staticPodsConfigPath, "Path for specification of static pods. Path should point to dir containing the staticPods configuration files. Defaults to none.") - - fs.StringVar(&s.mesosMaster, "mesos-master", s.mesosMaster, "Location of the Mesos master. The format is a comma-delimited list of of hosts like zk://host1:port,host2:port/mesos. If using ZooKeeper, pay particular attention to the leading zk:// and trailing /mesos! If not using ZooKeeper, standard URLs like http://localhost are also acceptable.") - fs.StringVar(&s.mesosUser, "mesos-user", s.mesosUser, "Mesos user for this framework, defaults to root.") - fs.StringSliceVar(&s.frameworkRoles, "mesos-framework-roles", s.frameworkRoles, "Mesos framework roles that the scheduler receives offers for. Currently only \"*\" and optionally one additional role are supported.") - fs.StringSliceVar(&s.defaultPodRoles, "mesos-default-pod-roles", s.defaultPodRoles, "Roles that will be used to launch pods having no "+meta.RolesKey+" label.") - fs.StringVar(&s.mesosAuthPrincipal, "mesos-authentication-principal", s.mesosAuthPrincipal, "Mesos authentication principal.") - fs.StringVar(&s.mesosAuthSecretFile, "mesos-authentication-secret-file", s.mesosAuthSecretFile, "Mesos authentication secret file.") - fs.StringVar(&s.mesosAuthProvider, "mesos-authentication-provider", s.mesosAuthProvider, fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported())) - fs.StringVar(&s.dockerCfgPath, "dockercfg-path", s.dockerCfgPath, "Path to a dockercfg file that will be used by the docker instance of the minions.") - fs.StringVar(&s.mesosCgroupPrefix, "mesos-cgroup-prefix", s.mesosCgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos") - fs.Var(&s.mesosExecutorCPUs, "mesos-executor-cpus", "Initial CPU shares to allocate for each Mesos executor container.") - fs.Var(&s.mesosExecutorMem, "mesos-executor-mem", "Initial memory (MB) to allocate for each Mesos executor container.") - fs.BoolVar(&s.checkpoint, "checkpoint", s.checkpoint, "Enable/disable checkpointing for the kubernetes-mesos framework.") - fs.Float64Var(&s.failoverTimeout, "failover-timeout", s.failoverTimeout, fmt.Sprintf("Framework failover timeout, in sec.")) - fs.BoolVar(&s.generateTaskDiscovery, "mesos-generate-task-discovery", s.generateTaskDiscovery, "Enable/disable generation of DiscoveryInfo for Mesos tasks.") - fs.UintVar(&s.driverPort, "driver-port", s.driverPort, "Port that the Mesos scheduler driver process should listen on.") - fs.StringVar(&s.hostnameOverride, "hostname-override", s.hostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") - fs.Int64Var(&s.reconcileInterval, "reconcile-interval", s.reconcileInterval, "Interval at which to execute task reconciliation, in sec. Zero disables.") - fs.DurationVar(&s.reconcileCooldown, "reconcile-cooldown", s.reconcileCooldown, "Minimum rest period between task reconciliation operations.") - fs.StringVar(&s.schedulerConfigFileName, "scheduler-config", s.schedulerConfigFileName, "An ini-style configuration file with low-level scheduler settings.") - fs.BoolVar(&s.graceful, "graceful", s.graceful, "Indicator of a graceful failover, intended for internal use only.") - fs.BoolVar(&s.ha, "ha", s.ha, "Run the scheduler in high availability mode with leader election. All peers should be configured exactly the same.") - fs.StringVar(&s.frameworkName, "framework-name", s.frameworkName, "The framework name to register with Mesos.") - fs.StringVar(&s.frameworkStoreURI, "framework-store-uri", s.frameworkStoreURI, "Where the framework should store metadata, either in Zookeeper (zk://host:port/path) or in etcd (etcd://path).") - fs.StringVar(&s.frameworkWebURI, "framework-weburi", s.frameworkWebURI, "A URI that points to a web-based interface for interacting with the framework.") - fs.StringVar(&s.advertisedAddress, "advertised-address", s.advertisedAddress, "host:port address that is advertised to clients. May be used to construct artifact download URIs.") - fs.IPVar(&s.serviceAddress, "service-address", s.serviceAddress, "The service portal IP address that the scheduler should register with (if unset, chooses randomly)") - fs.Var(&s.defaultContainerCPULimit, "default-container-cpu-limit", "Containers without a CPU resource limit are admitted this much CPU shares") - fs.Var(&s.defaultContainerMemLimit, "default-container-mem-limit", "Containers without a memory resource limit are admitted this much amount of memory in MB") - fs.BoolVar(&s.containPodResources, "contain-pod-resources", s.containPodResources, "Reparent pod containers into mesos cgroups; disable if you're having strange mesos/docker/systemd interactions.") - fs.DurationVar(&s.nodeRelistPeriod, "node-monitor-period", s.nodeRelistPeriod, "Period between relisting of all nodes from the apiserver.") - fs.BoolVar(&s.useHostPortEndpoints, "host-port-endpoints", s.useHostPortEndpoints, "Map service endpoints to hostIP:hostPort instead of podIP:containerPort. Default true.") - - fs.IntVar(&s.executorLogV, "executor-logv", s.executorLogV, "Logging verbosity of spawned minion and executor processes.") - fs.BoolVar(&s.executorBindall, "executor-bindall", s.executorBindall, "When true will set -address of the executor to 0.0.0.0.") - fs.DurationVar(&s.executorSuicideTimeout, "executor-suicide-timeout", s.executorSuicideTimeout, "Executor self-terminates after this period of inactivity. Zero disables suicide watch.") - fs.DurationVar(&s.launchGracePeriod, "mesos-launch-grace-period", s.launchGracePeriod, "Launch grace period after which launching tasks will be cancelled. Zero disables launch cancellation.") - fs.StringVar(&s.sandboxOverlay, "mesos-sandbox-overlay", s.sandboxOverlay, "Path to an archive (tar.gz, tar.bz2 or zip) extracted into the sandbox.") - - fs.BoolVar(&s.proxyBindall, "proxy-bindall", s.proxyBindall, "When true pass -proxy-bindall to the executor.") - fs.BoolVar(&s.runProxy, "run-proxy", s.runProxy, "Run the kube-proxy as a side process of the executor.") - fs.StringVar(&s.proxyKubeconfig, "proxy-kubeconfig", s.proxyKubeconfig, "Path to kubeconfig file with authorization and master location information used by the proxy.") - fs.IntVar(&s.proxyLogV, "proxy-logv", s.proxyLogV, "Logging verbosity of spawned minion proxy processes.") - fs.StringVar(&s.proxyMode, "proxy-mode", s.proxyMode, "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.") - - fs.StringVar(&s.minionPathOverride, "minion-path-override", s.minionPathOverride, "Override the PATH in the environment of the minion sub-processes.") - fs.Var(resource.NewQuantityFlagValue(&s.minionLogMaxSize), "minion-max-log-size", "Maximum log file size for the executor and proxy before rotation") - fs.IntVar(&s.minionLogMaxAgeInDays, "minion-max-log-age", s.minionLogMaxAgeInDays, "Maximum log file age of the executor and proxy in days") - fs.IntVar(&s.minionLogMaxBackups, "minion-max-log-backups", s.minionLogMaxBackups, "Maximum log file backups of the executor and proxy to keep after rotation") - - fs.StringSliceVar(&s.kubeletApiServerList, "kubelet-api-servers", s.kubeletApiServerList, "List of Kubernetes API servers kubelet will use. (ip:port), comma separated. If unspecified it defaults to the value of --api-servers.") - fs.StringVar(&s.kubeletRootDirectory, "kubelet-root-dir", s.kubeletRootDirectory, "Directory path for managing kubelet files (volume mounts,etc). Defaults to executor sandbox.") - fs.StringVar(&s.kubeletDockerEndpoint, "kubelet-docker-endpoint", s.kubeletDockerEndpoint, "If non-empty, kubelet will use this for the docker endpoint to communicate with.") - fs.StringVar(&s.kubeletPodInfraContainerImage, "kubelet-pod-infra-container-image", s.kubeletPodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.") - fs.UintVar(&s.kubeletCadvisorPort, "kubelet-cadvisor-port", s.kubeletCadvisorPort, "The port of the kubelet's local cAdvisor endpoint") - fs.StringVar(&s.kubeletHostNetworkSources, "kubelet-host-network-sources", s.kubeletHostNetworkSources, "Comma-separated list of sources from which the Kubelet allows pods to use of host network. For all sources use \"*\" [default=\"file\"]") - fs.DurationVar(&s.kubeletSyncFrequency, "kubelet-sync-frequency", s.kubeletSyncFrequency, "Max period between synchronizing running containers and config") - fs.StringVar(&s.kubeletNetworkPluginName, "kubelet-network-plugin", s.kubeletNetworkPluginName, " The name of the network plugin to be invoked for various events in kubelet/pod lifecycle") - fs.BoolVar(&s.kubeletEnableDebuggingHandlers, "kubelet-enable-debugging-handlers", s.kubeletEnableDebuggingHandlers, "Enables kubelet endpoints for log collection and local running of containers and commands") - fs.StringVar(&s.kubeletKubeconfig, "kubelet-kubeconfig", s.kubeletKubeconfig, "Path to kubeconfig file with authorization and master location information used by the kubelet.") - fs.IntVar(&s.conntrackMax, "conntrack-max", s.conntrackMax, "Maximum number of NAT connections to track on agent nodes (0 to leave as-is)") - fs.IntVar(&s.conntrackTCPTimeoutEstablished, "conntrack-tcp-timeout-established", s.conntrackTCPTimeoutEstablished, "Idle timeout for established TCP connections on agent nodes (0 to leave as-is)") - - //TODO(jdef) support this flag once we have a better handle on mesos-dns and k8s DNS integration - //fs.StringVar(&s.HADomain, "ha-domain", s.HADomain, "Domain of the HA scheduler service, only used in HA mode. If specified may be used to construct artifact download URIs.") -} - -func (s *SchedulerServer) AddStandaloneFlags(fs *pflag.FlagSet) { - s.addCoreFlags(fs) - fs.StringVar(&s.executorPath, "executor-path", s.executorPath, "Location of the kubernetes executor executable") -} - -func (s *SchedulerServer) AddHyperkubeFlags(fs *pflag.FlagSet) { - s.addCoreFlags(fs) - fs.StringVar(&s.kmPath, "km-path", s.kmPath, "Location of the km executable, may be a URI or an absolute file path; may be prefixed with 'file://' to specify the path to a pre-installed, agent-local km binary.") -} - -// returns (downloadURI, basename(path)) -func (s *SchedulerServer) serveFrameworkArtifact(path string) (string, string) { - basename := filepath.Base(path) - return s.serveFrameworkArtifactWithFilename(path, basename), basename -} - -// returns downloadURI -func (s *SchedulerServer) serveFrameworkArtifactWithFilename(path string, filename string) string { - serveFile := func(pattern string, filepath string) { - s.mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, filepath) - }) - } - - serveFile("/"+filename, path) - - hostURI := "" - if s.advertisedAddress != "" { - hostURI = fmt.Sprintf("http://%s/%s", s.advertisedAddress, filename) - } else if s.ha && s.haDomain != "" { - hostURI = fmt.Sprintf("http://%s.%s:%d/%s", SCHEDULER_SERVICE_NAME, s.haDomain, ports.SchedulerPort, filename) - } else { - hostURI = fmt.Sprintf("http://%s:%d/%s", s.address.String(), s.port, filename) - } - log.V(2).Infof("Hosting artifact '%s' at '%s'", filename, hostURI) - - return hostURI -} - -func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.ExecutorInfo, error) { - ci := &mesos.CommandInfo{ - Shell: proto.Bool(false), - } - - if s.executorPath != "" { - uri, executorCmd := s.serveFrameworkArtifact(s.executorPath) - ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)}) - ci.Value = proto.String(fmt.Sprintf("./%s", executorCmd)) - ci.Arguments = append(ci.Arguments, ci.GetValue()) - } else if !hks.FindServer(hyperkube.CommandMinion) { - return nil, fmt.Errorf("either run this scheduler via km or else --executor-path is required") - } else { - if strings.Index(s.kmPath, "://") > 0 { - if strings.HasPrefix(s.kmPath, "file://") { - // If `kmPath` started with "file://", `km` in agent local path was used. - ci.Value = proto.String(strings.TrimPrefix(s.kmPath, "file://")) - } else { - // URI could point directly to executable, e.g. hdfs:///km - // or else indirectly, e.g. http://acmestorage/tarball.tgz - // so we assume that for this case the command will always "km" - ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(s.kmPath), Executable: proto.Bool(true)}) - ci.Value = proto.String("./km") // TODO(jdef) extract constant - } - } else if s.kmPath != "" { - uri, kmCmd := s.serveFrameworkArtifact(s.kmPath) - ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)}) - ci.Value = proto.String(fmt.Sprintf("./%s", kmCmd)) - } else { - uri, kmCmd := s.serveFrameworkArtifact(s.executable) - ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)}) - ci.Value = proto.String(fmt.Sprintf("./%s", kmCmd)) - } - ci.Arguments = append(ci.Arguments, ci.GetValue(), hyperkube.CommandMinion) - - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--run-proxy=%v", s.runProxy)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-bindall=%v", s.proxyBindall)) - if s.proxyKubeconfig != "" { - //TODO(jdef) should probably support non-local files, e.g. hdfs:///some/config/file - uri, basename := s.serveFrameworkArtifact(s.proxyKubeconfig) - ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri)}) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-kubeconfig=%v", basename)) - } - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-logv=%d", s.proxyLogV)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-mode=%v", s.proxyMode)) - - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--path-override=%s", s.minionPathOverride)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--max-log-size=%v", s.minionLogMaxSize.String())) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--max-log-backups=%d", s.minionLogMaxBackups)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--max-log-age=%d", s.minionLogMaxAgeInDays)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--conntrack-max=%d", s.conntrackMax)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--conntrack-tcp-timeout-established=%d", s.conntrackTCPTimeoutEstablished)) - } - - if s.sandboxOverlay != "" { - if _, err := os.Stat(s.sandboxOverlay); os.IsNotExist(err) { - return nil, fmt.Errorf("Sandbox overlay archive not found: %s", s.sandboxOverlay) - } - uri, _ := s.serveFrameworkArtifact(s.sandboxOverlay) - ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(false), Extract: proto.Bool(true)}) - } - - if s.dockerCfgPath != "" { - uri := s.serveFrameworkArtifactWithFilename(s.dockerCfgPath, ".dockercfg") - ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(false), Extract: proto.Bool(false)}) - } - - //TODO(jdef): provide some way (env var?) for users to customize executor config - //TODO(jdef): set -address to 127.0.0.1 if `address` is 127.0.0.1 - - var apiServerArgs string - if len(s.kubeletApiServerList) > 0 { - apiServerArgs = strings.Join(s.kubeletApiServerList, ",") - } else { - apiServerArgs = strings.Join(s.apiServerList, ",") - } - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--api-servers=%s", apiServerArgs)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--v=%d", s.executorLogV)) // this also applies to the minion - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--allow-privileged=%t", s.allowPrivileged)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--suicide-timeout=%v", s.executorSuicideTimeout)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--mesos-launch-grace-period=%v", s.launchGracePeriod)) - - if s.executorBindall { - //TODO(jdef) determine whether hostname-override is really needed for bindall because - //it conflicts with kubelet node status checks/updates - //ci.Arguments = append(ci.Arguments, "--hostname-override=0.0.0.0") - ci.Arguments = append(ci.Arguments, "--address=0.0.0.0") - } - - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--mesos-cgroup-prefix=%v", s.mesosCgroupPrefix)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cadvisor-port=%v", s.kubeletCadvisorPort)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--sync-frequency=%v", s.kubeletSyncFrequency)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--contain-pod-resources=%t", s.containPodResources)) - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--enable-debugging-handlers=%t", s.kubeletEnableDebuggingHandlers)) - - if s.kubeletKubeconfig != "" { - //TODO(jdef) should probably support non-local files, e.g. hdfs:///some/config/file - if s.kubeletKubeconfig != s.proxyKubeconfig { - if filepath.Base(s.kubeletKubeconfig) == filepath.Base(s.proxyKubeconfig) { - // scheduler serves kubelet-kubeconfig and proxy-kubeconfig by their basename - // we currently don't support the case where the 2 kubeconfig files have the same - // basename but different absolute name, e.g., /kubelet/kubeconfig and /proxy/kubeconfig - return nil, fmt.Errorf("if kubelet-kubeconfig and proxy-kubeconfig are different, they must have different basenames") - } - // allows kubelet-kubeconfig and proxy-kubeconfig to point to the same file - uri, _ := s.serveFrameworkArtifact(s.kubeletKubeconfig) - ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri)}) - } - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--kubeconfig=%s", filepath.Base(s.kubeletKubeconfig))) - } - appendOptional := func(name string, value string) { - if value != "" { - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--%s=%s", name, value)) - } - } - if s.clusterDNS != nil { - appendOptional("cluster-dns", s.clusterDNS.String()) - } - appendOptional("cluster-domain", s.clusterDomain) - appendOptional("root-dir", s.kubeletRootDirectory) - appendOptional("docker-endpoint", s.kubeletDockerEndpoint) - appendOptional("pod-infra-container-image", s.kubeletPodInfraContainerImage) - appendOptional("host-network-sources", s.kubeletHostNetworkSources) - appendOptional("network-plugin", s.kubeletNetworkPluginName) - - // TODO(jdef) this code depends on poorly scoped cadvisor flags, will need refactoring soon - appendOptional(flagutil.Cadvisor.HousekeepingInterval.NameValue()) - appendOptional(flagutil.Cadvisor.GlobalHousekeepingInterval.NameValue()) - - log.V(1).Infof("prepared executor command %q with args '%+v'", ci.GetValue(), ci.Arguments) - - // Create mesos scheduler driver. - execInfo := &mesos.ExecutorInfo{ - Command: ci, - Name: proto.String(cloud.KubernetesExecutorName), - Source: proto.String(execcfg.DefaultInfoSource), - } - - // Check for staticPods - data, staticPodCPUs, staticPodMem := s.prepareStaticPods() - - // set prototype resource. During procument these act as the blue print only. - // In a final ExecutorInfo they might differ due to different procured - // resource roles. - execInfo.Resources = []*mesos.Resource{ - mutil.NewScalarResource("cpus", float64(s.mesosExecutorCPUs)+staticPodCPUs), - mutil.NewScalarResource("mem", float64(s.mesosExecutorMem)+staticPodMem), - } - - // calculate the ExecutorInfo hash to be used for validating compatibility. - // It is used to determine whether a running executor is compatible with the - // current scheduler configuration. If it is not, offers for those nodes - // are declined by our framework and the operator has to phase out those - // running executors in a cluster. - execInfo.ExecutorId = executorinfo.NewID(execInfo) - execInfo.Data = data - log.V(1).Infof("started with executor id %v", execInfo.ExecutorId.GetValue()) - - return execInfo, nil -} - -func (s *SchedulerServer) prepareStaticPods() (data []byte, staticPodCPUs, staticPodMem float64) { - // TODO(sttts): add a directory watch and tell running executors about updates - if s.staticPodsConfigPath == "" { - return - } - - entries, errCh := podutil.ReadFromDir(s.staticPodsConfigPath) - go func() { - // we just skip file system errors for now, do our best to gather - // as many static pod specs as we can. - for err := range errCh { - log.Errorln(err.Error()) - } - }() - - // validate cpu and memory limits, tracking the running totals in staticPod{CPUs,Mem} - validateResourceLimits := StaticPodValidator( - s.defaultContainerCPULimit, - s.defaultContainerMemLimit, - &staticPodCPUs, - &staticPodMem) - - zipped, err := podutil.Gzip(validateResourceLimits.Do(entries)) - if err != nil { - log.Errorf("failed to generate static pod data: %v", err) - staticPodCPUs, staticPodMem = 0, 0 - } else { - data = zipped - } - return -} - -// TODO(jdef): hacked from plugin/cmd/kube-scheduler/app/server.go -func (s *SchedulerServer) createAPIServerClientConfig() (*restclient.Config, error) { - kubeconfig, err := clientcmd.BuildConfigFromFlags(s.apiServerList[0], s.kubeconfig) - if err != nil { - return nil, err - } - - // Override kubeconfig qps/burst settings from flags - kubeconfig.QPS = s.kubeAPIQPS - kubeconfig.Burst = s.kubeAPIBurst - return kubeconfig, nil -} - -func (s *SchedulerServer) setDriver(driver bindings.SchedulerDriver) { - s.driverMutex.Lock() - defer s.driverMutex.Unlock() - s.driver = driver -} - -func (s *SchedulerServer) getDriver() (driver bindings.SchedulerDriver) { - s.driverMutex.RLock() - defer s.driverMutex.RUnlock() - return s.driver -} - -func (s *SchedulerServer) Run(hks hyperkube.Interface, _ []string) error { - if n := len(s.frameworkRoles); n == 0 || n > 2 || (n == 2 && s.frameworkRoles[0] != "*" && s.frameworkRoles[1] != "*") { - log.Fatalf(`only one custom role allowed in addition to "*"`) - } - - fwSet := sets.NewString(s.frameworkRoles...) - podSet := sets.NewString(s.defaultPodRoles...) - if !fwSet.IsSuperset(podSet) { - log.Fatalf("all default pod roles %q must be included in framework roles %q", s.defaultPodRoles, s.frameworkRoles) - } - - // get scheduler low-level config - sc := schedcfg.CreateDefaultConfig() - if s.schedulerConfigFileName != "" { - f, err := os.Open(s.schedulerConfigFileName) - if err != nil { - log.Fatalf("Cannot open scheduler config file: %v", err) - } - defer f.Close() - - err = sc.Read(bufio.NewReader(f)) - if err != nil { - log.Fatalf("Invalid scheduler config file: %v", err) - } - } - - schedulerProcess, driverFactory, etcdClient, eid := s.bootstrap(hks, sc) - - if s.enableProfiling { - profile.InstallHandler(s.mux) - } - go runtime.Until(func() { - log.V(1).Info("Starting HTTP interface") - log.Error(http.ListenAndServe(net.JoinHostPort(s.address.String(), strconv.Itoa(s.port)), s.mux)) - }, sc.HttpBindInterval.Duration, schedulerProcess.Terminal()) - - if s.ha { - validation := ha.ValidationFunc(validateLeadershipTransition) - srv := ha.NewCandidate(schedulerProcess, driverFactory, validation) - path := meta.ElectionPath(s.frameworkName) - uuid := eid.GetValue() + ":" + uuid.New() // unique for each scheduler instance - log.Infof("registering for election at %v with id %v", path, uuid) - go election.Notify( - election.NewEtcdMasterElector(etcdClient), - path, - uuid, - srv, - nil) - } else { - log.Infoln("self-electing in non-HA mode") - schedulerProcess.Elect(driverFactory) - } - return s.awaitFailover(schedulerProcess, func() error { return s.failover(s.getDriver(), hks) }) -} - -// watch the scheduler process for failover signals and properly handle such. may never return. -func (s *SchedulerServer) awaitFailover(schedulerProcess schedulerProcessInterface, handler func() error) error { - - // we only want to return the first error (if any), everyone else can block forever - errCh := make(chan error, 1) - doFailover := func() error { - // we really don't expect handler to return, if it does something went seriously wrong - err := handler() - if err != nil { - defer schedulerProcess.End() - err = fmt.Errorf("failover failed, scheduler will terminate: %v", err) - } - return err - } - - // guard for failover signal processing, first signal processor wins - failoverLatch := &runtime.Latch{} - runtime.On(schedulerProcess.Terminal(), func() { - if !failoverLatch.Acquire() { - log.V(1).Infof("scheduler process ending, already failing over") - select {} - } - var err error - defer func() { errCh <- err }() - select { - case <-schedulerProcess.Failover(): - err = doFailover() - default: - if s.ha { - err = fmt.Errorf("ha scheduler exiting instead of failing over") - } else { - log.Infof("exiting scheduler") - } - } - }) - runtime.OnOSSignal(makeFailoverSigChan(), func(_ os.Signal) { - if !failoverLatch.Acquire() { - log.V(1).Infof("scheduler process signalled, already failing over") - select {} - } - errCh <- doFailover() - }) - return <-errCh -} - -func validateLeadershipTransition(desired, current string) { - log.Infof("validating leadership transition") - // desired, current are of the format : (see Run()). - // parse them and ensure that executor ID's match, otherwise the cluster can get into - // a bad state after scheduler failover: executor ID is a config hash that must remain - // consistent across failover events. - var ( - i = strings.LastIndex(desired, ":") - j = strings.LastIndex(current, ":") - ) - - if i > -1 { - desired = desired[0:i] - } else { - log.Fatalf("desired id %q is invalid", desired) - } - if j > -1 { - current = current[0:j] - } else if current != "" { - log.Fatalf("current id %q is invalid", current) - } - - if desired != current && current != "" { - log.Fatalf("desired executor id %q != current executor id %q", desired, current) - } -} - -// hacked from https://github.com/kubernetes/kubernetes/blob/release-0.14/cmd/kube-apiserver/app/server.go -func newEtcd(etcdServerList []string) (etcd.Client, error) { - cfg := etcd.Config{ - Endpoints: etcdServerList, - } - return etcd.New(cfg) -} - -func (s *SchedulerServer) bootstrap(hks hyperkube.Interface, sc *schedcfg.Config) (*ha.SchedulerProcess, ha.DriverFactory, etcd.Client, *mesos.ExecutorID) { - s.frameworkName = strings.TrimSpace(s.frameworkName) - if s.frameworkName == "" { - log.Fatalf("framework-name must be a non-empty string") - } - s.frameworkWebURI = strings.TrimSpace(s.frameworkWebURI) - - metrics.Register() - runtime.Register() - s.mux.Handle("/metrics", prometheus.Handler()) - healthz.InstallHandler(s.mux) - - if len(s.etcdServerList) == 0 { - log.Fatalf("specify --etcd-servers must be specified") - } - - if len(s.apiServerList) < 1 { - log.Fatal("No api servers specified.") - } - - clientConfig, err := s.createAPIServerClientConfig() - if err != nil { - log.Fatalf("Unable to make apiserver client config: %v", err) - } - s.client, err = clientset.NewForConfig(clientConfig) - if err != nil { - log.Fatalf("Unable to make apiserver clientset: %v", err) - } - - if s.reconcileCooldown < defaultReconcileCooldown { - s.reconcileCooldown = defaultReconcileCooldown - log.Warningf("user-specified reconcile cooldown too small, defaulting to %v", s.reconcileCooldown) - } - - eiPrototype, err := s.prepareExecutorInfo(hks) - if err != nil { - log.Fatalf("misconfigured executor: %v", err) - } - - // TODO(jdef): remove the dependency on etcd as soon as - // (1) the generic config store is available for the FrameworkId storage - // (2) the generic master election is provided by the apiserver - // Compare docs/proposals/high-availability.md - etcdClient, err := newEtcd(s.etcdServerList) - if err != nil { - log.Fatalf("misconfigured etcd: %v", err) - } - keysAPI := etcd.NewKeysAPI(etcdClient) - - // mirror all nodes into the nodeStore - var eiRegistry executorinfo.Registry - nodesClientConfig := *clientConfig - nodesClient, err := clientset.NewForConfig(&nodesClientConfig) - if err != nil { - log.Fatalf("Cannot create client to watch nodes: %v", err) - } - nodeLW := cache.NewListWatchFromClient(nodesClient.CoreClient, "nodes", api.NamespaceAll, fields.Everything()) - nodeStore, nodeCtl := cache.NewInformer(nodeLW, &api.Node{}, s.nodeRelistPeriod, &cache.ResourceEventHandlerFuncs{ - DeleteFunc: func(obj interface{}) { - if eiRegistry != nil { - // TODO(jdef) use cache.DeletionHandlingMetaNamespaceKeyFunc at some point? - nodeName := "" - if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { - nodeName = tombstone.Key - } else if node, ok := obj.(*api.Node); ok { - nodeName = node.Name - } - if nodeName != "" { - log.V(2).Infof("deleting node %q from registry", nodeName) - eiRegistry.Invalidate(nodeName) - } - } - }, - }) - - lookupNode := func(hostName string) *api.Node { - n, _, _ := nodeStore.GetByKey(hostName) // ignore error and return nil then - if n == nil { - return nil - } - return n.(*api.Node) - } - - execInfoCache, err := executorinfo.NewCache(defaultExecutorInfoCacheSize) - if err != nil { - log.Fatalf("cannot create executorinfo cache: %v", err) - } - - eiRegistry, err = executorinfo.NewRegistry(lookupNode, eiPrototype, execInfoCache) - if err != nil { - log.Fatalf("cannot create executorinfo registry: %v", err) - } - - pr := podtask.NewDefaultProcurement(eiPrototype, eiRegistry) - fcfs := podschedulers.NewFCFSPodScheduler(pr, lookupNode) - frameworkIDStorage, err := s.frameworkIDStorage(keysAPI) - if err != nil { - log.Fatalf("cannot init framework ID storage: %v", err) - } - framework := framework.New(framework.Config{ - SchedulerConfig: *sc, - Client: s.client, - FailoverTimeout: s.failoverTimeout, - ReconcileInterval: s.reconcileInterval, - ReconcileCooldown: s.reconcileCooldown, - LookupNode: lookupNode, - StoreFrameworkId: frameworkIDStorage.Set, - ExecutorId: eiPrototype.GetExecutorId(), - }) - masterUri := s.mesosMaster - info, cred, err := s.buildFrameworkInfo() - if err != nil { - log.Fatalf("Misconfigured mesos framework: %v", err) - } - - schedulerProcess := ha.New(framework) - - // try publishing on the same IP as the slave - var publishedAddress net.IP - if libprocessIP := os.Getenv("LIBPROCESS_IP"); libprocessIP != "" { - publishedAddress = net.ParseIP(libprocessIP) - } - if publishedAddress != nil { - log.V(1).Infof("driver will publish address %v", publishedAddress) - } - - dconfig := &bindings.DriverConfig{ - Scheduler: schedulerProcess, - Framework: info, - Master: masterUri, - Credential: cred, - BindingAddress: s.address, - BindingPort: uint16(s.driverPort), - PublishedAddress: publishedAddress, - HostnameOverride: s.hostnameOverride, - WithAuthContext: func(ctx context.Context) context.Context { - ctx = auth.WithLoginProvider(ctx, s.mesosAuthProvider) - ctx = sasl.WithBindingAddress(ctx, s.address) - return ctx - }, - } - - // create event recorder sending events to the "" namespace of the apiserver - eventsClientConfig := *clientConfig - eventsClient, err := clientset.NewForConfig(&eventsClientConfig) - if err != nil { - log.Fatalf("Invalid API configuration: %v", err) - } - broadcaster := record.NewBroadcaster() - recorder := broadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName}) - broadcaster.StartLogging(log.Infof) - broadcaster.StartRecordingToSink(&unversionedcore.EventSinkImpl{Interface: eventsClient.Events("")}) - - lw := cache.NewListWatchFromClient(s.client.CoreClient, "pods", api.NamespaceAll, fields.Everything()) - - hostPortStrategy := hostport.StrategyFixed - if s.useHostPortEndpoints { - hostPortStrategy = hostport.StrategyWildcard - } - - // create scheduler core with all components arranged around it - sched := components.New( - sc, - framework, - fcfs, - s.client, - recorder, - schedulerProcess.Terminal(), - s.mux, - lw, - podtask.Config{ - DefaultPodRoles: s.defaultPodRoles, - FrameworkRoles: s.frameworkRoles, - GenerateTaskDiscoveryEnabled: s.generateTaskDiscovery, - HostPortStrategy: hostPortStrategy, - Prototype: eiPrototype, - }, - s.defaultContainerCPULimit, - s.defaultContainerMemLimit, - ) - - runtime.On(framework.Registration(), func() { sched.Run(schedulerProcess.Terminal()) }) - runtime.On(framework.Registration(), s.newServiceWriter(publishedAddress, schedulerProcess.Terminal())) - runtime.On(framework.Registration(), func() { nodeCtl.Run(schedulerProcess.Terminal()) }) - - driverFactory := ha.DriverFactory(func() (drv bindings.SchedulerDriver, err error) { - log.V(1).Infoln("performing deferred initialization") - if err = framework.Init(sched, schedulerProcess.Master(), s.mux); err != nil { - return nil, fmt.Errorf("failed to initialize pod scheduler: %v", err) - } - - log.V(1).Infoln("deferred init complete") - if s.failoverTimeout > 0 { - // defer obtaining framework ID to prevent multiple schedulers - // from overwriting each other's framework IDs - var frameworkID string - frameworkID, err = frameworkIDStorage.Get(context.TODO()) - if err != nil { - return nil, fmt.Errorf("failed to fetch framework ID from storage: %v", err) - } - if frameworkID != "" { - log.Infof("configuring FrameworkInfo with ID found in storage: %q", frameworkID) - dconfig.Framework.Id = &mesos.FrameworkID{Value: &frameworkID} - } else { - log.V(1).Infof("did not find framework ID in storage") - } - } else { - // TODO(jdef) this is a hack, really for development, to simplify clean up of old framework IDs - frameworkIDStorage.Remove(context.TODO()) - } - - log.V(1).Infoln("constructing mesos scheduler driver") - drv, err = bindings.NewMesosSchedulerDriver(*dconfig) - if err != nil { - return nil, fmt.Errorf("failed to construct scheduler driver: %v", err) - } - - log.V(1).Infoln("constructed mesos scheduler driver:", drv) - s.setDriver(drv) - return drv, nil - }) - - return schedulerProcess, driverFactory, etcdClient, eiPrototype.GetExecutorId() -} - -func (s *SchedulerServer) failover(driver bindings.SchedulerDriver, hks hyperkube.Interface) error { - if driver != nil { - stat, err := driver.Stop(true) - if stat != mesos.Status_DRIVER_STOPPED { - return fmt.Errorf("failed to stop driver for failover, received unexpected status code: %v", stat) - } else if err != nil { - return err - } - } - - // there's no guarantee that all goroutines are actually programmed intelligently with 'done' - // signals, so we'll need to restart if we want to really stop everything - - // run the same command that we were launched with - //TODO(jdef) assumption here is that the scheduler is the only service running in this process, we should probably validate that somehow - args := []string{} - flags := pflag.CommandLine - if hks != nil { - args = append(args, hks.Name()) - flags = hks.Flags() - } - flags.Visit(func(flag *pflag.Flag) { - if flag.Name != "api-servers" && flag.Name != "etcd-servers" && flag.Name != "kubelet-api-servers" { - args = append(args, fmt.Sprintf("--%s=%s", flag.Name, flag.Value.String())) - } - }) - if !s.graceful { - args = append(args, "--graceful") - } - if len(s.apiServerList) > 0 { - args = append(args, "--api-servers="+strings.Join(s.apiServerList, ",")) - } - if len(s.etcdServerList) > 0 { - args = append(args, "--etcd-servers="+strings.Join(s.etcdServerList, ",")) - } - if len(s.kubeletApiServerList) > 0 { - args = append(args, "--kubelet-api-servers="+strings.Join(s.kubeletApiServerList, ",")) - } - args = append(args, flags.Args()...) - - log.V(1).Infof("spawning scheduler for graceful failover: %s %+v", s.executable, args) - - cmd := exec.Command(s.executable, args...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.SysProcAttr = makeDisownedProcAttr() - - // TODO(jdef) pass in a pipe FD so that we can block, waiting for the child proc to be ready - //cmd.ExtraFiles = []*os.File{} - - exitcode := 0 - log.Flush() // TODO(jdef) it would be really nice to ensure that no one else in our process was still logging - if err := cmd.Start(); err != nil { - //log to stdtout here to avoid conflicts with normal stderr logging - fmt.Fprintf(os.Stdout, "failed to spawn failover process: %v\n", err) - os.Exit(1) - } - os.Exit(exitcode) - select {} // will never reach here -} - -func (s *SchedulerServer) buildFrameworkInfo() (info *mesos.FrameworkInfo, cred *mesos.Credential, err error) { - username, err := s.getUsername() - if err != nil { - return nil, nil, err - } - log.V(2).Infof("Framework configured with mesos user %v", username) - info = &mesos.FrameworkInfo{ - Name: proto.String(s.frameworkName), - User: proto.String(username), - Checkpoint: proto.Bool(s.checkpoint), - } - if s.frameworkWebURI != "" { - info.WebuiUrl = proto.String(s.frameworkWebURI) - } - if s.failoverTimeout > 0 { - info.FailoverTimeout = proto.Float64(s.failoverTimeout) - } - - // set the framework's role to the first configured non-star role. - // once Mesos supports multiple roles simply set the configured mesos roles slice. - for _, role := range s.frameworkRoles { - if role != "*" { - // mesos currently supports only one role per framework info - // The framework will be offered role's resources as well as * resources - info.Role = proto.String(role) - break - } - } - - if s.mesosAuthPrincipal != "" { - info.Principal = proto.String(s.mesosAuthPrincipal) - cred = &mesos.Credential{ - Principal: proto.String(s.mesosAuthPrincipal), - } - if s.mesosAuthSecretFile != "" { - secret, err := ioutil.ReadFile(s.mesosAuthSecretFile) - if err != nil { - return nil, nil, err - } - cred.Secret = proto.String(string(secret)) - } - } - return -} - -func (s *SchedulerServer) getUsername() (username string, err error) { - username = s.mesosUser - if username == "" { - if u, err := user.Current(); err == nil { - username = u.Username - if username == "" { - username = defaultMesosUser - } - } - } - return -} - -func (s *SchedulerServer) frameworkIDStorage(keysAPI etcd.KeysAPI) (frameworkid.Storage, error) { - u, err := url.Parse(s.frameworkStoreURI) - if err != nil { - return nil, fmt.Errorf("cannot parse framework store URI: %v", err) - } - - switch u.Scheme { - case "etcd": - idpath := meta.StoreChroot - if u.Path != "" { - idpath = path.Join("/", u.Path) - } - idpath = path.Join(idpath, s.frameworkName, "frameworkid") - return frameworkidEtcd.Store(keysAPI, idpath, time.Duration(s.failoverTimeout)*time.Second), nil - case "zk": - return frameworkidZk.Store(s.frameworkStoreURI, s.frameworkName), nil - default: - return nil, fmt.Errorf("unsupported framework storage scheme: %q", u.Scheme) - } -} diff --git a/contrib/mesos/pkg/scheduler/service/service_test.go b/contrib/mesos/pkg/scheduler/service/service_test.go deleted file mode 100644 index 78d7b8fcb59..00000000000 --- a/contrib/mesos/pkg/scheduler/service/service_test.go +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "testing" - "time" - - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - - "github.com/stretchr/testify/assert" -) - -type fakeSchedulerProcess struct { - doneFunc func() <-chan struct{} - failoverFunc func() <-chan struct{} -} - -func (self *fakeSchedulerProcess) Terminal() <-chan struct{} { - if self == nil || self.doneFunc == nil { - return nil - } - return self.doneFunc() -} - -func (self *fakeSchedulerProcess) Failover() <-chan struct{} { - if self == nil || self.failoverFunc == nil { - return nil - } - return self.failoverFunc() -} - -func (self *fakeSchedulerProcess) End() <-chan struct{} { - ch := make(chan struct{}) - close(ch) - return ch -} - -func Test_awaitFailoverDone(t *testing.T) { - done := make(chan struct{}) - p := &fakeSchedulerProcess{ - doneFunc: func() <-chan struct{} { return done }, - } - ss := &SchedulerServer{} - failoverHandlerCalled := false - failoverFailedHandler := func() error { - failoverHandlerCalled = true - return nil - } - errCh := make(chan error, 1) - go func() { - errCh <- ss.awaitFailover(p, failoverFailedHandler) - }() - close(done) - select { - case err := <-errCh: - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - case <-time.After(1 * time.Second): - t.Fatalf("timed out waiting for failover") - } - if failoverHandlerCalled { - t.Fatalf("unexpected call to failover handler") - } -} - -func Test_awaitFailoverDoneFailover(t *testing.T) { - ch := make(chan struct{}) - p := &fakeSchedulerProcess{ - doneFunc: func() <-chan struct{} { return ch }, - failoverFunc: func() <-chan struct{} { return ch }, - } - ss := &SchedulerServer{} - failoverHandlerCalled := false - failoverFailedHandler := func() error { - failoverHandlerCalled = true - return nil - } - errCh := make(chan error, 1) - go func() { - errCh <- ss.awaitFailover(p, failoverFailedHandler) - }() - close(ch) - select { - case err := <-errCh: - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - case <-time.After(1 * time.Second): - t.Fatalf("timed out waiting for failover") - } - if !failoverHandlerCalled { - t.Fatalf("expected call to failover handler") - } -} - -func Test_DefaultResourceLimits(t *testing.T) { - assert := assert.New(t) - - s := NewSchedulerServer() - assert.Equal(s.defaultContainerCPULimit, resources.DefaultDefaultContainerCPULimit) - assert.Equal(s.defaultContainerMemLimit, resources.DefaultDefaultContainerMemLimit) -} diff --git a/contrib/mesos/pkg/scheduler/service/validation.go b/contrib/mesos/pkg/scheduler/service/validation.go deleted file mode 100644 index 7f3adf5d8b7..00000000000 --- a/contrib/mesos/pkg/scheduler/service/validation.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - log "github.com/golang/glog" - "k8s.io/kubernetes/contrib/mesos/pkg/podutil" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/pkg/api" -) - -// StaticPodValidator discards a pod if we can't calculate resource limits for it. -func StaticPodValidator( - defaultContainerCPULimit resources.CPUShares, - defaultContainerMemLimit resources.MegaBytes, - accumCPU, accumMem *float64, -) podutil.FilterFunc { - return podutil.FilterFunc(func(pod *api.Pod) (bool, error) { - _, cpu, _, err := resources.LimitPodCPU(pod, defaultContainerCPULimit) - if err != nil { - return false, err - } - - _, mem, _, err := resources.LimitPodMem(pod, defaultContainerMemLimit) - if err != nil { - return false, err - } - - log.V(2).Infof("reserving %.2f cpu shares and %.2f MB of memory to static pod %s/%s", cpu, mem, pod.Namespace, pod.Name) - - *accumCPU += float64(cpu) - *accumMem += float64(mem) - return true, nil - }) -} diff --git a/contrib/mesos/pkg/scheduler/service/validation_test.go b/contrib/mesos/pkg/scheduler/service/validation_test.go deleted file mode 100644 index 43c485a92a9..00000000000 --- a/contrib/mesos/pkg/scheduler/service/validation_test.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "k8s.io/kubernetes/contrib/mesos/pkg/podutil" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resources" - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/service" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/resource" -) - -func TestStaticPodValidator(t *testing.T) { - // test within limits - tests := []struct { - // given - pods <-chan *api.Pod - // wants - podcount int - cputot float64 - memtot float64 - }{ - // test: valid, pod specifies limits for ALL containers - { - pods: pods(pod( - podName("foo", "bar"), - containers( - container(resourceLimits(10, 20)), // min is 32 - container(resourceLimits(30, 40)), - container(resourceLimits(50, 60)), - ), - )), - podcount: 1, - cputot: 90, - memtot: 132, - }, - // test: valid, multiple pods, specify limits for ALL containers - { - pods: pods( - pod( - podName("foo", "bar"), - containers( - container(resourceLimits(10, 20)), // min is 32 - container(resourceLimits(30, 40)), - container(resourceLimits(50, 60)), - ), - ), - pod( - podName("kjh", "jkk"), - containers( - container(resourceLimits(15, 25)), // min is 32 - container(resourceLimits(35, 45)), - container(resourceLimits(55, 65)), - ), - ), - ), - podcount: 2, - cputot: 195, - memtot: 274, - }, - // test: no limits on CT in first pod so it's rejected - { - pods: pods( - pod( - podName("foo", "bar"), - containers( - container(resourceLimits(10, 20)), // min is 32 - container(), // min is 0.01, 32 - container(resourceLimits(50, 60)), - ), - ), - pod( - podName("wza", "wer"), - containers( - container(resourceLimits(10, 20)), // min is 32 - container(resourceLimits(30, 40)), - container(resourceLimits(50, 60)), - ), - ), - ), - podcount: 2, - cputot: 60.01 + 90, - memtot: 124 + 132, - }, - } - for i, tc := range tests { - var cpu, mem float64 - f := service.StaticPodValidator(0, 0, &cpu, &mem) - list := podutil.List(f.Do(tc.pods)) - assert.Equal(t, tc.podcount, len(list.Items), "test case #%d: expected %d pods instead of %d", i, tc.podcount, len(list.Items)) - assert.EqualValues(t, tc.cputot, cpu, "test case #%d: expected %f total cpu instead of %f", i, tc.cputot, cpu) - assert.EqualValues(t, tc.memtot, mem, "test case #%d: expected %f total mem instead of %f", i, tc.memtot, mem) - } -} - -type podOpt func(*api.Pod) -type ctOpt func(*api.Container) - -func pods(pods ...*api.Pod) <-chan *api.Pod { - ch := make(chan *api.Pod, len(pods)) - for _, x := range pods { - ch <- x - } - close(ch) - return ch -} - -func pod(opts ...podOpt) *api.Pod { - p := &api.Pod{} - for _, x := range opts { - x(p) - } - return p -} - -func container(opts ...ctOpt) (c api.Container) { - for _, x := range opts { - x(&c) - } - return -} - -func containers(ct ...api.Container) podOpt { - return podOpt(func(p *api.Pod) { - p.Spec.Containers = ct - }) -} - -func resourceLimits(cpu resources.CPUShares, mem resources.MegaBytes) ctOpt { - return ctOpt(func(c *api.Container) { - if c.Resources.Limits == nil { - c.Resources.Limits = make(api.ResourceList) - } - c.Resources.Limits[api.ResourceCPU] = *resource.NewMilliQuantity(int64(float64(cpu)*1000.0), resource.DecimalSI) - c.Resources.Limits[api.ResourceMemory] = *resource.NewQuantity(int64(float64(mem)*1024.0*1024.0), resource.BinarySI) - }) -} - -func podName(ns, name string) podOpt { - return podOpt(func(p *api.Pod) { - p.Namespace = ns - p.Name = name - }) -} diff --git a/contrib/mesos/pkg/service/doc.go b/contrib/mesos/pkg/service/doc.go deleted file mode 100644 index 27b009bb4a9..00000000000 --- a/contrib/mesos/pkg/service/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package service is largely a clone of the stock Kubernetes endpoints -// controller, extended with some very specific functionality related -// to kubernetes-mesos specific host-pod port mapping. -package service // import "k8s.io/kubernetes/contrib/mesos/pkg/service" diff --git a/contrib/mesos/pkg/service/endpoints_controller.go b/contrib/mesos/pkg/service/endpoints_controller.go deleted file mode 100644 index f06923366a5..00000000000 --- a/contrib/mesos/pkg/service/endpoints_controller.go +++ /dev/null @@ -1,472 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "fmt" - "reflect" - "strconv" - "time" - - "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/endpoints" - "k8s.io/kubernetes/pkg/api/errors" - "k8s.io/kubernetes/pkg/client/cache" - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - kservice "k8s.io/kubernetes/pkg/controller/endpoint" - "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/runtime" - "k8s.io/kubernetes/pkg/util/intstr" - utilruntime "k8s.io/kubernetes/pkg/util/runtime" - "k8s.io/kubernetes/pkg/util/sets" - "k8s.io/kubernetes/pkg/util/wait" - "k8s.io/kubernetes/pkg/util/workqueue" - "k8s.io/kubernetes/pkg/watch" - - "github.com/golang/glog" -) - -var ( - keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc -) - -type EndpointController interface { - Run(workers int, stopCh <-chan struct{}) -} - -// NewEndpointController returns a new *EndpointController. -func NewEndpointController(client *clientset.Clientset) *endpointController { - e := &endpointController{ - client: client, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "endpoint"), - } - e.serviceStore.Indexer, e.serviceController = cache.NewIndexerInformer( - &cache.ListWatch{ - ListFunc: func(options api.ListOptions) (runtime.Object, error) { - return e.client.Core().Services(api.NamespaceAll).List(options) - }, - WatchFunc: func(options api.ListOptions) (watch.Interface, error) { - return e.client.Core().Services(api.NamespaceAll).Watch(options) - }, - }, - &api.Service{}, - kservice.FullServiceResyncPeriod, - cache.ResourceEventHandlerFuncs{ - AddFunc: e.enqueueService, - UpdateFunc: func(old, cur interface{}) { - e.enqueueService(cur) - }, - DeleteFunc: e.enqueueService, - }, - cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, - ) - - e.podStore.Indexer, e.podController = cache.NewIndexerInformer( - &cache.ListWatch{ - ListFunc: func(options api.ListOptions) (runtime.Object, error) { - return e.client.Core().Pods(api.NamespaceAll).List(options) - }, - WatchFunc: func(options api.ListOptions) (watch.Interface, error) { - return e.client.Core().Pods(api.NamespaceAll).Watch(options) - }, - }, - &api.Pod{}, - 5*time.Minute, - cache.ResourceEventHandlerFuncs{ - AddFunc: e.addPod, - UpdateFunc: e.updatePod, - DeleteFunc: e.deletePod, - }, - cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, - ) - return e -} - -// EndpointController manages selector-based service endpoints. -type endpointController struct { - client *clientset.Clientset - - serviceStore cache.StoreToServiceLister - podStore cache.StoreToPodLister - - // Services that need to be updated. A channel is inappropriate here, - // because it allows services with lots of pods to be serviced much - // more often than services with few pods; it also would cause a - // service that's inserted multiple times to be processed more than - // necessary. - queue workqueue.RateLimitingInterface - - // Since we join two objects, we'll watch both of them with - // controllers. - serviceController *cache.Controller - podController *cache.Controller -} - -// Runs e; will not return until stopCh is closed. workers determines how many -// endpoints will be handled in parallel. -func (e *endpointController) Run(workers int, stopCh <-chan struct{}) { - defer utilruntime.HandleCrash() - go e.serviceController.Run(stopCh) - go e.podController.Run(stopCh) - for i := 0; i < workers; i++ { - go wait.Until(e.worker, time.Second, stopCh) - } - go func() { - defer utilruntime.HandleCrash() - time.Sleep(5 * time.Minute) // give time for our cache to fill - e.checkLeftoverEndpoints() - }() - <-stopCh - e.queue.ShutDown() -} - -func (e *endpointController) getPodServiceMemberships(pod *api.Pod) (sets.String, error) { - set := sets.String{} - services, err := e.serviceStore.GetPodServices(pod) - if err != nil { - // don't log this error because this function makes pointless - // errors when no services match. - return set, nil - } - for i := range services { - key, err := keyFunc(&services[i]) - if err != nil { - return nil, err - } - set.Insert(key) - } - return set, nil -} - -// When a pod is added, figure out what services it will be a member of and -// enqueue them. obj must have *api.Pod type. -func (e *endpointController) addPod(obj interface{}) { - pod := obj.(*api.Pod) - services, err := e.getPodServiceMemberships(pod) - if err != nil { - utilruntime.HandleError(fmt.Errorf("Unable to get pod %v/%v's service memberships: %v", pod.Namespace, pod.Name, err)) - return - } - for key := range services { - e.queue.Add(key) - } -} - -// When a pod is updated, figure out what services it used to be a member of -// and what services it will be a member of, and enqueue the union of these. -// old and cur must be *api.Pod types. -func (e *endpointController) updatePod(old, cur interface{}) { - if api.Semantic.DeepEqual(old, cur) { - return - } - newPod := old.(*api.Pod) - services, err := e.getPodServiceMemberships(newPod) - if err != nil { - utilruntime.HandleError(fmt.Errorf("Unable to get pod %v/%v's service memberships: %v", newPod.Namespace, newPod.Name, err)) - return - } - - oldPod := cur.(*api.Pod) - // Only need to get the old services if the labels changed. - if !reflect.DeepEqual(newPod.Labels, oldPod.Labels) { - oldServices, err := e.getPodServiceMemberships(oldPod) - if err != nil { - utilruntime.HandleError(fmt.Errorf("Unable to get pod %v/%v's service memberships: %v", oldPod.Namespace, oldPod.Name, err)) - return - } - services = services.Union(oldServices) - } - for key := range services { - e.queue.Add(key) - } -} - -// When a pod is deleted, enqueue the services the pod used to be a member of. -// obj could be an *api.Pod, or a DeletionFinalStateUnknown marker item. -func (e *endpointController) deletePod(obj interface{}) { - if _, ok := obj.(*api.Pod); ok { - // Enqueue all the services that the pod used to be a member - // of. This happens to be exactly the same thing we do when a - // pod is added. - e.addPod(obj) - return - } - podKey, err := keyFunc(obj) - if err != nil { - utilruntime.HandleError(fmt.Errorf("Couldn't get key for object %+v: %v", obj, err)) - } - glog.V(4).Infof("Pod %q was deleted but we don't have a record of its final state, so it will take up to %v before it will be removed from all endpoint records.", podKey, kservice.FullServiceResyncPeriod) - - // TODO: keep a map of pods to services to handle this condition. -} - -// obj could be an *api.Service, or a DeletionFinalStateUnknown marker item. -func (e *endpointController) enqueueService(obj interface{}) { - key, err := keyFunc(obj) - if err != nil { - utilruntime.HandleError(fmt.Errorf("Couldn't get key for object %+v: %v", obj, err)) - } - - e.queue.Add(key) -} - -// worker runs a worker thread that just dequeues items, processes them, and -// marks them done. You may run as many of these in parallel as you wish; the -// workqueue guarantees that they will not end up processing the same service -// at the same time. -func (e *endpointController) worker() { - for e.processNextWorkItem() { - } -} - -func (e *endpointController) processNextWorkItem() bool { - eKey, quit := e.queue.Get() - if quit { - return false - } - // Use defer: in the unlikely event that there's a - // panic, we'd still like this to get marked done-- - // otherwise the controller will not be able to sync - // this service again until it is restarted. - defer e.queue.Done(eKey) - - err := e.syncService(eKey.(string)) - if err == nil { - e.queue.Forget(eKey) - return true - } - - utilruntime.HandleError(fmt.Errorf("Sync %v failed with %v", eKey, err)) - e.queue.AddRateLimited(eKey) - - return true -} - -// HACK(sttts): add annotations to the endpoint about the respective container ports -func (e *endpointController) syncService(key string) error { - startTime := time.Now() - defer func() { - glog.V(4).Infof("Finished syncing service %q endpoints. (%v)", key, time.Now().Sub(startTime)) - }() - obj, exists, err := e.serviceStore.Indexer.GetByKey(key) - if err != nil || !exists { - // Delete the corresponding endpoint, as the service has been deleted. - // TODO: Please note that this will delete an endpoint when a - // service is deleted. However, if we're down at the time when - // the service is deleted, we will miss that deletion, so this - // doesn't completely solve the problem. See #6877. - namespace, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - utilruntime.HandleError(fmt.Errorf("Need to delete endpoint with key %q, but couldn't understand the key: %v", key, err)) - // Don't retry, as the key isn't going to magically become understandable. - return nil - } - err = e.client.Endpoints(namespace).Delete(name, nil) - if err != nil && !errors.IsNotFound(err) { - utilruntime.HandleError(fmt.Errorf("Error deleting endpoint %q: %v", key, err)) - return err - } - return nil - } - - service := obj.(*api.Service) - if service.Spec.Selector == nil { - // services without a selector receive no endpoints from this controller; - // these services will receive the endpoints that are created out-of-band via the REST API. - return nil - } - - glog.V(5).Infof("About to update endpoints for service %q", key) - pods, err := e.podStore.Pods(service.Namespace).List(labels.Set(service.Spec.Selector).AsSelector()) - if err != nil { - // Since we're getting stuff from a local cache, it is - // basically impossible to get this error. - utilruntime.HandleError(fmt.Errorf("Error syncing service %q: %v", key, err)) - return err - } - - subsets := []api.EndpointSubset{} - containerPortAnnotations := map[string]string{} // by : - for i := range pods { - // TODO: Do we need to copy here? - pod := &(*pods[i]) - - for i := range service.Spec.Ports { - servicePort := &service.Spec.Ports[i] - - portName := servicePort.Name - portProto := servicePort.Protocol - portNum, containerPort, err := findPort(pod, servicePort) - if err != nil { - glog.V(4).Infof("Failed to find port for service %s/%s: %v", service.Namespace, service.Name, err) - continue - } - // HACK(jdef): use HostIP instead of pod.CurrentState.PodIP for generic mesos compat - if len(pod.Status.HostIP) == 0 { - glog.V(4).Infof("Failed to find a host IP for pod %s/%s", pod.Namespace, pod.Name) - continue - } - if pod.DeletionTimestamp != nil { - glog.V(5).Infof("Pod is being deleted %s/%s", pod.Namespace, pod.Name) - continue - } - - if !api.IsPodReady(pod) { - glog.V(5).Infof("Pod is out of service: %v/%v", pod.Namespace, pod.Name) - continue - } - - // HACK(jdef): use HostIP instead of pod.CurrentState.PodIP for generic mesos compat - epp := api.EndpointPort{Name: portName, Port: int32(portNum), Protocol: portProto} - epa := api.EndpointAddress{IP: pod.Status.HostIP, TargetRef: &api.ObjectReference{ - Kind: "Pod", - Namespace: pod.ObjectMeta.Namespace, - Name: pod.ObjectMeta.Name, - UID: pod.ObjectMeta.UID, - ResourceVersion: pod.ObjectMeta.ResourceVersion, - }} - subsets = append(subsets, api.EndpointSubset{Addresses: []api.EndpointAddress{epa}, Ports: []api.EndpointPort{epp}}) - containerPortAnnotations[fmt.Sprintf(meta.ContainerPortKeyFormat, portProto, pod.Status.HostIP, portNum)] = strconv.Itoa(containerPort) - } - } - subsets = endpoints.RepackSubsets(subsets) - - // See if there's actually an update here. - currentEndpoints, err := e.client.Endpoints(service.Namespace).Get(service.Name) - if err != nil { - if errors.IsNotFound(err) { - currentEndpoints = &api.Endpoints{ - ObjectMeta: api.ObjectMeta{ - Name: service.Name, - Labels: service.Labels, - }, - } - } else { - utilruntime.HandleError(fmt.Errorf("Error getting endpoints: %v", err)) - return err - } - } - if reflect.DeepEqual(currentEndpoints.Subsets, subsets) && reflect.DeepEqual(currentEndpoints.Labels, service.Labels) { - glog.V(5).Infof("Endpoints are equal for %s/%s, skipping update", service.Namespace, service.Name) - return nil - } - newEndpoints := currentEndpoints - newEndpoints.Subsets = subsets - newEndpoints.Labels = service.Labels - - if newEndpoints.Annotations == nil { - newEndpoints.Annotations = map[string]string{} - } - for hostIpPort, containerPort := range containerPortAnnotations { - newEndpoints.Annotations[hostIpPort] = containerPort - } - - if len(currentEndpoints.ResourceVersion) == 0 { - // No previous endpoints, create them - _, err = e.client.Endpoints(service.Namespace).Create(newEndpoints) - } else { - // Pre-existing - _, err = e.client.Endpoints(service.Namespace).Update(newEndpoints) - } - if err != nil { - utilruntime.HandleError(fmt.Errorf("Error updating endpoints: %v", err)) - return err - } - return nil -} - -// checkLeftoverEndpoints lists all currently existing endpoints and adds their -// service to the queue. This will detect endpoints that exist with no -// corresponding service; these endpoints need to be deleted. We only need to -// do this once on startup, because in steady-state these are detected (but -// some stragglers could have been left behind if the endpoint controller -// reboots). -func (e *endpointController) checkLeftoverEndpoints() { - list, err := e.client.Endpoints(api.NamespaceAll).List(api.ListOptions{}) - if err != nil { - utilruntime.HandleError(fmt.Errorf("Unable to list endpoints (%v); orphaned endpoints will not be cleaned up. (They're pretty harmless, but you can restart this component if you want another attempt made.)", err)) - return - } - for i := range list.Items { - ep := &list.Items[i] - key, err := keyFunc(ep) - if err != nil { - utilruntime.HandleError(fmt.Errorf("Unable to get key for endpoint %#v", ep)) - continue - } - e.queue.Add(key) - } -} - -// findPort locates the container port for the given pod and portName. If the -// targetPort is a number, use that. If the targetPort is a string, look that -// string up in all named ports in all containers in the target pod. If no -// match is found, fail. -// -// HACK(jdef): return the HostPort in addition to the ContainerPort for generic mesos compatibility -func findPort(pod *api.Pod, svcPort *api.ServicePort) (int, int, error) { - portName := svcPort.TargetPort - switch portName.Type { - case intstr.String: - name := portName.StrVal - for _, container := range pod.Spec.Containers { - for _, port := range container.Ports { - if port.Name == name && port.Protocol == svcPort.Protocol { - hostPort, err := findMappedPortName(pod, port.Protocol, name) - return hostPort, int(port.ContainerPort), err - } - } - } - case intstr.Int: - // HACK(jdef): slightly different semantics from upstream here: - // we ensure that if the user spec'd a port in the service that - // it actually maps to a host-port assigned to the pod. upstream - // doesn't check this and happily returns the container port spec'd - // in the service, but that doesn't align w/ mesos port mgmt. - p := portName.IntValue() - for _, container := range pod.Spec.Containers { - for _, port := range container.Ports { - if int(port.ContainerPort) == p && port.Protocol == svcPort.Protocol { - hostPort, err := findMappedPort(pod, port.Protocol, p) - return hostPort, int(port.ContainerPort), err - } - } - } - } - return 0, 0, fmt.Errorf("no suitable port for manifest: %s", pod.UID) -} - -func findMappedPort(pod *api.Pod, protocol api.Protocol, port int) (int, error) { - if len(pod.Annotations) > 0 { - key := fmt.Sprintf(meta.PortMappingKeyFormat, string(protocol), port) - if value, found := pod.Annotations[key]; found { - return strconv.Atoi(value) - } - } - return 0, fmt.Errorf("failed to find mapped container %s port: %d", protocol, port) -} - -func findMappedPortName(pod *api.Pod, protocol api.Protocol, portName string) (int, error) { - if len(pod.Annotations) > 0 { - key := fmt.Sprintf(meta.PortNameMappingKeyFormat, string(protocol), portName) - if value, found := pod.Annotations[key]; found { - return strconv.Atoi(value) - } - } - return 0, fmt.Errorf("failed to find mapped container %s port name: %q", protocol, portName) -} diff --git a/contrib/mesos/pkg/service/endpoints_controller_test.go b/contrib/mesos/pkg/service/endpoints_controller_test.go deleted file mode 100644 index c20c7c091ca..00000000000 --- a/contrib/mesos/pkg/service/endpoints_controller_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "testing" - - "k8s.io/kubernetes/pkg/api" -) - -func TestFindMappedPort(t *testing.T) { - pod := &api.Pod{} - port, err := findMappedPort(pod, api.ProtocolTCP, 80) - if err == nil { - t.Fatalf("expected error since port tcp/80 is not mapped") - } - port, err = findMappedPortName(pod, api.ProtocolUDP, "foo") - if err == nil { - t.Fatalf("expected error since port udp/'foo' is not mapped") - } - - pod.Annotations = make(map[string]string) - pod.Annotations["k8s.mesosphere.io/port_TCP_80"] = "123" - pod.Annotations["k8s.mesosphere.io/portName_UDP_foo"] = "456" - - port, err = findMappedPort(pod, api.ProtocolUDP, 80) - if err == nil { - t.Fatalf("expected error since port udp/80 is not mapped") - } - port, err = findMappedPort(pod, api.ProtocolTCP, 80) - if err != nil { - t.Fatalf("expected that port 80 is mapped") - } - if port != 123 { - t.Fatalf("expected mapped port == 123") - } - - port, err = findMappedPortName(pod, api.ProtocolTCP, "foo") - if err == nil { - t.Fatalf("expected error since port tcp/'foo' is not mapped") - } - port, err = findMappedPortName(pod, api.ProtocolUDP, "foo") - if err != nil { - t.Fatalf("expected that port udp/'foo' is mapped") - } - if port != 456 { - t.Fatalf("expected mapped port == 456") - } -} diff --git a/contrib/mesos/target.sh b/contrib/mesos/target.sh deleted file mode 100644 index a51a7ccc39f..00000000000 --- a/contrib/mesos/target.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The set of server targets that we are only building for Linux -# Used by hack/lib/golang.sh -kube::contrib::mesos::server_targets() { - local -r targets=( - contrib/mesos/cmd/k8sm-scheduler - contrib/mesos/cmd/k8sm-executor - contrib/mesos/cmd/k8sm-controller-manager - contrib/mesos/cmd/km - ) - echo "${targets[@]}" -} - -# The set of test targets that we are building for all platforms -# Used by hack/lib/golang.sh -kube::contrib::mesos::test_targets() { - true -} - -# The set of source targets to include in the kube-build image -# Used by build/common.sh -kube::contrib::mesos::source_targets() { - local -r targets=( - contrib/mesos - ) - echo "${targets[@]}" -} diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index cdd6996ff00..539cc7ff3c2 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -18,13 +18,6 @@ readonly KUBE_GO_PACKAGE=k8s.io/kubernetes readonly KUBE_GOPATH="${KUBE_OUTPUT}/go" -# Load contrib target functions -if [ -n "${KUBERNETES_CONTRIB:-}" ]; then - for contrib in "${KUBERNETES_CONTRIB}"; do - source "${KUBE_ROOT}/contrib/${contrib}/target.sh" - done -fi - # The set of server targets that we are only building for Linux # Note: if you are adding something here, you might need to add it to # kube::build::source_targets in build/common.sh as well. @@ -40,11 +33,6 @@ kube::golang::server_targets() { cmd/kube-discovery plugin/cmd/kube-scheduler ) - if [ -n "${KUBERNETES_CONTRIB:-}" ]; then - for contrib in "${KUBERNETES_CONTRIB}"; do - targets+=($(eval "kube::contrib::${contrib}::server_targets")) - done - fi echo "${targets[@]}" } @@ -126,11 +114,6 @@ kube::golang::test_targets() { vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test ) - if [ -n "${KUBERNETES_CONTRIB:-}" ]; then - for contrib in "${KUBERNETES_CONTRIB}"; do - targets+=($(eval "kube::contrib::${contrib}::test_targets")) - done - fi echo "${targets[@]}" } readonly KUBE_TEST_TARGETS=($(kube::golang::test_targets)) diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh index 3f33e227525..690d7e04e86 100755 --- a/hack/make-rules/test.sh +++ b/hack/make-rules/test.sh @@ -32,7 +32,6 @@ kube::test::find_dirs() { -o -path './_output/*' \ -o -path './_gopath/*' \ -o -path './contrib/podex/*' \ - -o -path './contrib/mesos/*' \ -o -path './output/*' \ -o -path './release/*' \ -o -path './target/*' \