HPA scale-to-zero for custom object/external metrics
Add support for scaling to zero pods minReplicas is allowed to be zero condition is set once Based on https://github.com/kubernetes/kubernetes/pull/61423 set original valid condition add scale to/from zero and invalid metric tests Scaling up from zero pods ignores tolerance validate metrics when minReplicas is 0 Document HPA behaviour when minReplicas is 0 Documented minReplicas field in autoscaling APIs
This commit is contained in:
Rinat Shigapov
@@ -230,16 +230,11 @@ func (a *HorizontalController) processNextWorkItem() bool {
|
||||
}
|
||||
|
||||
// computeReplicasForMetrics computes the desired number of replicas for the metric specifications listed in the HPA,
|
||||
// returning the maximum of the computed replica counts, a description of the associated metric, and the statuses of
|
||||
// returning the maximum of the computed replica counts, a description of the associated metric, and the statuses of
|
||||
// all metrics computed.
|
||||
func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler, scale *autoscalingv1.Scale,
|
||||
metricSpecs []autoscalingv2.MetricSpec) (replicas int32, metric string, statuses []autoscalingv2.MetricStatus, timestamp time.Time, err error) {
|
||||
|
||||
specReplicas := scale.Spec.Replicas
|
||||
statusReplicas := scale.Status.Replicas
|
||||
|
||||
statuses = make([]autoscalingv2.MetricStatus, len(metricSpecs))
|
||||
|
||||
if scale.Status.Selector == "" {
|
||||
errMsg := "selector is required"
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "SelectorRequired", errMsg)
|
||||
@@ -255,15 +250,23 @@ func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.Hori
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf(errMsg)
|
||||
}
|
||||
|
||||
specReplicas := scale.Spec.Replicas
|
||||
statusReplicas := scale.Status.Replicas
|
||||
statuses = make([]autoscalingv2.MetricStatus, len(metricSpecs))
|
||||
|
||||
invalidMetricsCount := 0
|
||||
var invalidMetricError error
|
||||
var invalidMetricCondition autoscalingv2.HorizontalPodAutoscalerCondition
|
||||
|
||||
for i, metricSpec := range metricSpecs {
|
||||
replicaCountProposal, metricNameProposal, timestampProposal, err := a.computeReplicasForMetric(hpa, metricSpec, specReplicas, statusReplicas, selector, &statuses[i])
|
||||
replicaCountProposal, metricNameProposal, timestampProposal, condition, err := a.computeReplicasForMetric(hpa, metricSpec, specReplicas, statusReplicas, selector, &statuses[i])
|
||||
|
||||
if err != nil {
|
||||
if invalidMetricsCount <= 0 {
|
||||
invalidMetricCondition = condition
|
||||
invalidMetricError = err
|
||||
}
|
||||
invalidMetricsCount++
|
||||
invalidMetricError = err
|
||||
}
|
||||
if err == nil && (replicas == 0 || replicaCountProposal > replicas) {
|
||||
timestamp = timestampProposal
|
||||
@@ -272,62 +275,59 @@ func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.Hori
|
||||
}
|
||||
}
|
||||
|
||||
// If all metrics are invalid or some are invalid and we would scale down,
|
||||
// return an error and set the condition of the hpa based on the first invalid metric.
|
||||
// Otherwise set the condition as scaling active as we're going to scale
|
||||
if invalidMetricsCount >= len(metricSpecs) || (invalidMetricsCount > 0 && replicas < specReplicas) {
|
||||
return 0, "", statuses, time.Time{}, fmt.Errorf("Invalid metrics (%v invalid out of %v), last error was: %v", invalidMetricsCount, len(metricSpecs), invalidMetricError)
|
||||
} else {
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionTrue, "ValidMetricFound", "the HPA was able to successfully calculate a replica count from %v", metric)
|
||||
// If all metrics are invalid return error and set condition on hpa based on first invalid metric.
|
||||
if invalidMetricsCount >= len(metricSpecs) {
|
||||
setCondition(hpa, invalidMetricCondition.Type, invalidMetricCondition.Status, invalidMetricCondition.Reason, invalidMetricCondition.Message)
|
||||
return 0, "", statuses, time.Time{}, fmt.Errorf("Invalid metrics (%v invalid out of %v), first error is: %v", invalidMetricsCount, len(metricSpecs), invalidMetricError)
|
||||
}
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionTrue, "ValidMetricFound", "the HPA was able to successfully calculate a replica count from %s", metric)
|
||||
return replicas, metric, statuses, timestamp, nil
|
||||
}
|
||||
|
||||
// computeReplicasForMetric computes the desired number of replicas for for a specific hpa and single metric specification.
|
||||
// Computes the desired number of replicas for for a specific hpa and metric specification,
|
||||
// returning the metric status and a proposed condition to be set on the HPA object.
|
||||
func (a *HorizontalController) computeReplicasForMetric(hpa *autoscalingv2.HorizontalPodAutoscaler, spec autoscalingv2.MetricSpec,
|
||||
specReplicas, statusReplicas int32, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, metricNameProposal string,
|
||||
timestampProposal time.Time, err error) {
|
||||
timestampProposal time.Time, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) {
|
||||
|
||||
switch spec.Type {
|
||||
case autoscalingv2.ObjectMetricSourceType:
|
||||
metricSelector, err := metav1.LabelSelectorAsSelector(spec.Object.Metric.Selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", time.Time{}, fmt.Errorf("failed to get object metric value: %v", err)
|
||||
condition := a.getUnableComputeReplicaCountCondition(hpa, "FailedGetObjectMetric", err)
|
||||
return 0, "", time.Time{}, condition, fmt.Errorf("failed to get object metric value: %v", err)
|
||||
}
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForObjectMetric(specReplicas, statusReplicas, spec, hpa, selector, status, metricSelector)
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, condition, err = a.computeStatusForObjectMetric(specReplicas, statusReplicas, spec, hpa, selector, status, metricSelector)
|
||||
if err != nil {
|
||||
return 0, "", time.Time{}, fmt.Errorf("failed to get object metric value: %v", err)
|
||||
return 0, "", time.Time{}, condition, fmt.Errorf("failed to get object metric value: %v", err)
|
||||
}
|
||||
case autoscalingv2.PodsMetricSourceType:
|
||||
metricSelector, err := metav1.LabelSelectorAsSelector(spec.Pods.Metric.Selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetPodsMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetPodsMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", time.Time{}, fmt.Errorf("failed to get pods metric value: %v", err)
|
||||
condition := a.getUnableComputeReplicaCountCondition(hpa, "FailedGetPodsMetric", err)
|
||||
return 0, "", time.Time{}, condition, fmt.Errorf("failed to get pods metric value: %v", err)
|
||||
}
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForPodsMetric(specReplicas, spec, hpa, selector, status, metricSelector)
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, condition, err = a.computeStatusForPodsMetric(specReplicas, spec, hpa, selector, status, metricSelector)
|
||||
if err != nil {
|
||||
return 0, "", time.Time{}, fmt.Errorf("failed to get object metric value: %v", err)
|
||||
return 0, "", time.Time{}, condition, fmt.Errorf("failed to get pods metric value: %v", err)
|
||||
}
|
||||
case autoscalingv2.ResourceMetricSourceType:
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForResourceMetric(specReplicas, spec, hpa, selector, status)
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, condition, err = a.computeStatusForResourceMetric(specReplicas, spec, hpa, selector, status)
|
||||
if err != nil {
|
||||
return 0, "", time.Time{}, err
|
||||
return 0, "", time.Time{}, condition, err
|
||||
}
|
||||
case autoscalingv2.ExternalMetricSourceType:
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForExternalMetric(specReplicas, statusReplicas, spec, hpa, selector, status)
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, condition, err = a.computeStatusForExternalMetric(specReplicas, statusReplicas, spec, hpa, selector, status)
|
||||
if err != nil {
|
||||
return 0, "", time.Time{}, err
|
||||
return 0, "", time.Time{}, condition, err
|
||||
}
|
||||
default:
|
||||
errMsg := fmt.Sprintf("unknown metric source type %q", string(spec.Type))
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "InvalidMetricSourceType", errMsg)
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "InvalidMetricSourceType", "the HPA was unable to compute the replica count: %v", fmt.Errorf(errMsg))
|
||||
return 0, "", time.Time{}, fmt.Errorf(errMsg)
|
||||
err = fmt.Errorf(errMsg)
|
||||
condition := a.getUnableComputeReplicaCountCondition(hpa, "InvalidMetricSourceType", err)
|
||||
return 0, "", time.Time{}, condition, err
|
||||
}
|
||||
return replicaCountProposal, metricNameProposal, timestampProposal, nil
|
||||
return replicaCountProposal, metricNameProposal, timestampProposal, autoscalingv2.HorizontalPodAutoscalerCondition{}, nil
|
||||
}
|
||||
|
||||
func (a *HorizontalController) reconcileKey(key string) (deleted bool, err error) {
|
||||
@@ -347,13 +347,12 @@ func (a *HorizontalController) reconcileKey(key string) (deleted bool, err error
|
||||
}
|
||||
|
||||
// computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType.
|
||||
func (a *HorizontalController) computeStatusForObjectMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (int32, time.Time, string, error) {
|
||||
func (a *HorizontalController) computeStatusForObjectMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (replicas int32, timestamp time.Time, metricName string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) {
|
||||
if metricSpec.Object.Target.Type == autoscalingv2.ValueMetricType {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetObjectMetricReplicas(specReplicas, metricSpec.Object.Target.Value.MilliValue(), metricSpec.Object.Metric.Name, hpa.Namespace, &metricSpec.Object.DescribedObject, selector, metricSelector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, timestampProposal, "", err
|
||||
condition := a.getUnableComputeReplicaCountCondition(hpa, "FailedGetObjectMetric", err)
|
||||
return 0, timestampProposal, "", condition, err
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ObjectMetricSourceType,
|
||||
@@ -368,13 +367,12 @@ func (a *HorizontalController) computeStatusForObjectMetric(specReplicas, status
|
||||
},
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("%s metric %s", metricSpec.Object.DescribedObject.Kind, metricSpec.Object.Metric.Name), nil
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("%s metric %s", metricSpec.Object.DescribedObject.Kind, metricSpec.Object.Metric.Name), autoscalingv2.HorizontalPodAutoscalerCondition{}, nil
|
||||
} else if metricSpec.Object.Target.Type == autoscalingv2.AverageValueMetricType {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetObjectPerPodMetricReplicas(statusReplicas, metricSpec.Object.Target.AverageValue.MilliValue(), metricSpec.Object.Metric.Name, hpa.Namespace, &metricSpec.Object.DescribedObject, metricSelector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get %s object metric: %v", metricSpec.Object.Metric.Name, err)
|
||||
condition := a.getUnableComputeReplicaCountCondition(hpa, "FailedGetObjectMetric", err)
|
||||
return 0, time.Time{}, "", condition, fmt.Errorf("failed to get %s object metric: %v", metricSpec.Object.Metric.Name, err)
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ObjectMetricSourceType,
|
||||
@@ -388,21 +386,20 @@ func (a *HorizontalController) computeStatusForObjectMetric(specReplicas, status
|
||||
},
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.Object.Metric.Name, metricSpec.Object.Metric.Selector), nil
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.Object.Metric.Name, metricSpec.Object.Metric.Selector), autoscalingv2.HorizontalPodAutoscalerCondition{}, nil
|
||||
}
|
||||
errMsg := "invalid object metric source: neither a value target nor an average value target was set"
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", errMsg)
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %s", errMsg)
|
||||
return 0, time.Time{}, "", fmt.Errorf(errMsg)
|
||||
err = fmt.Errorf(errMsg)
|
||||
condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetObjectMetric", err)
|
||||
return 0, time.Time{}, "", condition, err
|
||||
}
|
||||
|
||||
// computeStatusForPodsMetric computes the desired number of replicas for the specified metric of type PodsMetricSourceType.
|
||||
func (a *HorizontalController) computeStatusForPodsMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (int32, time.Time, string, error) {
|
||||
func (a *HorizontalController) computeStatusForPodsMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetMetricReplicas(currentReplicas, metricSpec.Pods.Target.AverageValue.MilliValue(), metricSpec.Pods.Metric.Name, hpa.Namespace, selector, metricSelector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetPodsMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetPodsMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, timestampProposal, "", err
|
||||
condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetPodsMetric", err)
|
||||
return 0, timestampProposal, "", condition, err
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.PodsMetricSourceType,
|
||||
@@ -417,20 +414,19 @@ func (a *HorizontalController) computeStatusForPodsMetric(currentReplicas int32,
|
||||
},
|
||||
}
|
||||
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("pods metric %s", metricSpec.Pods.Metric.Name), nil
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("pods metric %s", metricSpec.Pods.Metric.Name), autoscalingv2.HorizontalPodAutoscalerCondition{}, nil
|
||||
}
|
||||
|
||||
// computeStatusForResourceMetric computes the desired number of replicas for the specified metric of type ResourceMetricSourceType.
|
||||
func (a *HorizontalController) computeStatusForResourceMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) {
|
||||
func (a *HorizontalController) computeStatusForResourceMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) {
|
||||
if metricSpec.Resource.Target.AverageValue != nil {
|
||||
var rawProposal int64
|
||||
replicaCountProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetRawResourceReplicas(currentReplicas, metricSpec.Resource.Target.AverageValue.MilliValue(), metricSpec.Resource.Name, hpa.Namespace, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
|
||||
condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetResourceMetric", err)
|
||||
return 0, time.Time{}, "", condition, fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
|
||||
}
|
||||
metricNameProposal := fmt.Sprintf("%s resource", metricSpec.Resource.Name)
|
||||
metricNameProposal = fmt.Sprintf("%s resource", metricSpec.Resource.Name)
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ResourceMetricSourceType,
|
||||
Resource: &autoscalingv2.ResourceMetricStatus{
|
||||
@@ -440,24 +436,23 @@ func (a *HorizontalController) computeStatusForResourceMetric(currentReplicas in
|
||||
},
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, metricNameProposal, nil
|
||||
return replicaCountProposal, timestampProposal, metricNameProposal, autoscalingv2.HorizontalPodAutoscalerCondition{}, nil
|
||||
} else {
|
||||
if metricSpec.Resource.Target.AverageUtilization == nil {
|
||||
errMsg := "invalid resource metric source: neither a utilization target nor a value target was set"
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", errMsg)
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %s", errMsg)
|
||||
return 0, time.Time{}, "", fmt.Errorf(errMsg)
|
||||
err = fmt.Errorf(errMsg)
|
||||
condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetResourceMetric", err)
|
||||
return 0, time.Time{}, "", condition, fmt.Errorf(errMsg)
|
||||
}
|
||||
targetUtilization := *metricSpec.Resource.Target.AverageUtilization
|
||||
var percentageProposal int32
|
||||
var rawProposal int64
|
||||
replicaCountProposal, percentageProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetResourceReplicas(currentReplicas, targetUtilization, metricSpec.Resource.Name, hpa.Namespace, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
|
||||
condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetResourceMetric", err)
|
||||
return 0, time.Time{}, "", condition, fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
|
||||
}
|
||||
metricNameProposal := fmt.Sprintf("%s resource utilization (percentage of request)", metricSpec.Resource.Name)
|
||||
metricNameProposal = fmt.Sprintf("%s resource utilization (percentage of request)", metricSpec.Resource.Name)
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ResourceMetricSourceType,
|
||||
Resource: &autoscalingv2.ResourceMetricStatus{
|
||||
@@ -468,18 +463,17 @@ func (a *HorizontalController) computeStatusForResourceMetric(currentReplicas in
|
||||
},
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, metricNameProposal, nil
|
||||
return replicaCountProposal, timestampProposal, metricNameProposal, autoscalingv2.HorizontalPodAutoscalerCondition{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// computeStatusForExternalMetric computes the desired number of replicas for the specified metric of type ExternalMetricSourceType.
|
||||
func (a *HorizontalController) computeStatusForExternalMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) {
|
||||
func (a *HorizontalController) computeStatusForExternalMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition autoscalingv2.HorizontalPodAutoscalerCondition, err error) {
|
||||
if metricSpec.External.Target.AverageValue != nil {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalPerPodMetricReplicas(statusReplicas, metricSpec.External.Target.AverageValue.MilliValue(), metricSpec.External.Metric.Name, hpa.Namespace, metricSpec.External.Metric.Selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get %s external metric: %v", metricSpec.External.Metric.Name, err)
|
||||
condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetExternalMetric", err)
|
||||
return 0, time.Time{}, "", condition, fmt.Errorf("failed to get %s external metric: %v", metricSpec.External.Metric.Name, err)
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ExternalMetricSourceType,
|
||||
@@ -493,14 +487,13 @@ func (a *HorizontalController) computeStatusForExternalMetric(specReplicas, stat
|
||||
},
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.Metric.Name, metricSpec.External.Metric.Selector), nil
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.Metric.Name, metricSpec.External.Metric.Selector), autoscalingv2.HorizontalPodAutoscalerCondition{}, nil
|
||||
}
|
||||
if metricSpec.External.Target.Value != nil {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalMetricReplicas(specReplicas, metricSpec.External.Target.Value.MilliValue(), metricSpec.External.Metric.Name, hpa.Namespace, metricSpec.External.Metric.Selector, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get external metric %s: %v", metricSpec.External.Metric.Name, err)
|
||||
condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetExternalMetric", err)
|
||||
return 0, time.Time{}, "", condition, fmt.Errorf("failed to get external metric %s: %v", metricSpec.External.Metric.Name, err)
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ExternalMetricSourceType,
|
||||
@@ -514,12 +507,12 @@ func (a *HorizontalController) computeStatusForExternalMetric(specReplicas, stat
|
||||
},
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.Metric.Name, metricSpec.External.Metric.Selector), nil
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.Metric.Name, metricSpec.External.Metric.Selector), autoscalingv2.HorizontalPodAutoscalerCondition{}, nil
|
||||
}
|
||||
errMsg := "invalid external metric source: neither a value target nor an average value target was set"
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", errMsg)
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %s", errMsg)
|
||||
return 0, time.Time{}, "", fmt.Errorf(errMsg)
|
||||
err = fmt.Errorf(errMsg)
|
||||
condition = a.getUnableComputeReplicaCountCondition(hpa, "FailedGetExternalMetric", err)
|
||||
return 0, time.Time{}, "", condition, fmt.Errorf(errMsg)
|
||||
}
|
||||
|
||||
func (a *HorizontalController) recordInitialRecommendation(currentReplicas int32, key string) {
|
||||
@@ -583,9 +576,18 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
||||
desiredReplicas := int32(0)
|
||||
rescaleReason := ""
|
||||
|
||||
var minReplicas int32
|
||||
|
||||
if hpa.Spec.MinReplicas != nil {
|
||||
minReplicas = *hpa.Spec.MinReplicas
|
||||
} else {
|
||||
// Default value
|
||||
minReplicas = 1
|
||||
}
|
||||
|
||||
rescale := true
|
||||
|
||||
if scale.Spec.Replicas == 0 {
|
||||
if scale.Spec.Replicas == 0 && minReplicas != 0 {
|
||||
// Autoscaling is disabled for this resource
|
||||
desiredReplicas = 0
|
||||
rescale = false
|
||||
@@ -593,12 +595,9 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
||||
} else if currentReplicas > hpa.Spec.MaxReplicas {
|
||||
rescaleReason = "Current number of replicas above Spec.MaxReplicas"
|
||||
desiredReplicas = hpa.Spec.MaxReplicas
|
||||
} else if hpa.Spec.MinReplicas != nil && currentReplicas < *hpa.Spec.MinReplicas {
|
||||
} else if currentReplicas < minReplicas {
|
||||
rescaleReason = "Current number of replicas below Spec.MinReplicas"
|
||||
desiredReplicas = *hpa.Spec.MinReplicas
|
||||
} else if currentReplicas == 0 {
|
||||
rescaleReason = "Current number of replicas must be greater than 0"
|
||||
desiredReplicas = 1
|
||||
desiredReplicas = minReplicas
|
||||
} else {
|
||||
var metricTimestamp time.Time
|
||||
metricDesiredReplicas, metricName, metricStatuses, metricTimestamp, err = a.computeReplicasForMetrics(hpa, scale, hpa.Spec.Metrics)
|
||||
@@ -624,7 +623,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
||||
if desiredReplicas < currentReplicas {
|
||||
rescaleReason = "All metrics below target"
|
||||
}
|
||||
desiredReplicas = a.normalizeDesiredReplicas(hpa, key, currentReplicas, desiredReplicas)
|
||||
desiredReplicas = a.normalizeDesiredReplicas(hpa, key, currentReplicas, desiredReplicas, minReplicas)
|
||||
rescale = desiredReplicas != currentReplicas
|
||||
}
|
||||
|
||||
@@ -679,19 +678,13 @@ func (a *HorizontalController) stabilizeRecommendation(key string, prenormalized
|
||||
|
||||
// normalizeDesiredReplicas takes the metrics desired replicas value and normalizes it based on the appropriate conditions (i.e. < maxReplicas, >
|
||||
// minReplicas, etc...)
|
||||
func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, key string, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 {
|
||||
func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, key string, currentReplicas int32, prenormalizedDesiredReplicas int32, minReplicas int32) int32 {
|
||||
stabilizedRecommendation := a.stabilizeRecommendation(key, prenormalizedDesiredReplicas)
|
||||
if stabilizedRecommendation != prenormalizedDesiredReplicas {
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ScaleDownStabilized", "recent recommendations were higher than current one, applying the highest recent recommendation")
|
||||
} else {
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "recommended size matches current size")
|
||||
}
|
||||
var minReplicas int32
|
||||
if hpa.Spec.MinReplicas != nil {
|
||||
minReplicas = *hpa.Spec.MinReplicas
|
||||
} else {
|
||||
minReplicas = 0
|
||||
}
|
||||
|
||||
desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, stabilizedRecommendation, minReplicas, hpa.Spec.MaxReplicas)
|
||||
|
||||
@@ -704,6 +697,16 @@ func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.Horiz
|
||||
return desiredReplicas
|
||||
}
|
||||
|
||||
func (a *HorizontalController) getUnableComputeReplicaCountCondition(hpa *autoscalingv2.HorizontalPodAutoscaler, reason string, err error) (condition autoscalingv2.HorizontalPodAutoscalerCondition) {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, reason, err.Error())
|
||||
return autoscalingv2.HorizontalPodAutoscalerCondition{
|
||||
Type: autoscalingv2.ScalingActive,
|
||||
Status: v1.ConditionFalse,
|
||||
Reason: reason,
|
||||
Message: fmt.Sprintf("the HPA was unable to compute the replica count: %v", err),
|
||||
}
|
||||
}
|
||||
|
||||
// convertDesiredReplicas performs the actual normalization, without depending on `HorizontalController` or `HorizontalPodAutoscaler`
|
||||
func convertDesiredReplicasWithRules(currentReplicas, desiredReplicas, hpaMinReplicas, hpaMaxReplicas int32) (int32, string, string) {
|
||||
|
||||
@@ -713,13 +716,8 @@ func convertDesiredReplicasWithRules(currentReplicas, desiredReplicas, hpaMinRep
|
||||
var possibleLimitingCondition string
|
||||
var possibleLimitingReason string
|
||||
|
||||
if hpaMinReplicas == 0 {
|
||||
minimumAllowedReplicas = 1
|
||||
possibleLimitingReason = "the desired replica count is zero"
|
||||
} else {
|
||||
minimumAllowedReplicas = hpaMinReplicas
|
||||
possibleLimitingReason = "the desired replica count is less than the minimum replica count"
|
||||
}
|
||||
minimumAllowedReplicas = hpaMinReplicas
|
||||
possibleLimitingReason = "the desired replica count is less than the minimum replica count"
|
||||
|
||||
// Do not upscale too much to prevent incorrect rapid increase of the number of master replicas caused by
|
||||
// bogus CPU usage report from heapster/kubelet (like in issue #32304).
|
||||
|
||||
Reference in New Issue
Block a user