Merge pull request #44959 from ktsakalozos/bug/change-config

Automatic merge from submit-queue

Retry calls we report config changes quickly.

**What this PR does / why we need it**: In Juju deployments of Kubernetes the status of the charms is updated when a status-update is triggered periodically. As a result changes in config variables may take up to 10 minutes to be reflected on the charms status. See bug below.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #
https://github.com/juju-solutions/bundle-canonical-kubernetes/issues/263

**Special notes for your reviewer**:

**Release note**:

```
Kubernetes clusters deployed with Juju pick up config changes faster.
```

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -39,6 +39,7 @@ from charms.reactive import is_state
 from charms.reactive import when, when_any, when_not
 from charms.reactive.helpers import data_changed
 from charms.kubernetes.common import get_version
+from charms.kubernetes.common import retry
 from charms.kubernetes.flagmanager import FlagManager
 
 from charmhelpers.core import hookenv
@@ -414,6 +415,7 @@ def push_api_data(kube_api):
 @when('kubernetes-master.components.started')
 def configure_cdk_addons():
     ''' Configure CDK addons '''
+    remove_state('cdk-addons.configured')
     dbEnabled = str(hookenv.config('enable-dashboard-addons')).lower()
     args = [
         'arch=' + arch(),
@@ -422,15 +424,31 @@ def configure_cdk_addons():
         'enable-dashboard=' + dbEnabled
     ]
     check_call(['snap', 'set', 'cdk-addons'] + args)
-    try:
-        check_call(['cdk-addons.apply'])
-    except CalledProcessError:
+    if not addons_ready():
         hookenv.status_set('waiting', 'Waiting to retry addon deployment')
         remove_state('cdk-addons.configured')
         return
+
     set_state('cdk-addons.configured')
 
 
+@retry(times=3, delay_secs=20)
+def addons_ready():
+    """
+    Test if the add ons got installed
+
+    Returns: True is the addons got applied
+
+    """
+    try:
+        check_call(['cdk-addons.apply'])
+        return True
+    except CalledProcessError:
+        hookenv.log("Addons are not ready yet.")
+        return False
+
+
+
 @when('loadbalancer.available', 'certificates.ca.available',
       'certificates.client.cert.available')
 def loadbalancer_kubeconfig(loadbalancer, ca, client):
@@ -838,6 +856,7 @@ def setup_tokens(token, username, user):
         stream.write('{0},{1},{2}\n'.format(token, username, user))
 
 
+@retry(times=3, delay_secs=10)
 def all_kube_system_pods_running():
     ''' Check pod status in the kube-system namespace. Returns True if all
     pods are running, False otherwise. '''