From 3ce173800f56f984f5b9725c608c569bd9c5af9a Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Thu, 8 Oct 2020 16:04:22 +0200 Subject: [PATCH] Make casctl settle wait for udev and add cores/start caches Signed-off-by: Jan Musial --- .../test_helper_functions_01.py | 523 +++++++++++++++--- utils/casctl | 14 +- utils/opencas.py | 93 +++- 3 files changed, 530 insertions(+), 100 deletions(-) diff --git a/test/utils_tests/opencas-py-tests/test_helper_functions_01.py b/test/utils_tests/opencas-py-tests/test_helper_functions_01.py index 0aa5c96..ec4fe9a 100644 --- a/test/utils_tests/opencas-py-tests/test_helper_functions_01.py +++ b/test/utils_tests/opencas-py-tests/test_helper_functions_01.py @@ -4,7 +4,7 @@ # import pytest -from unittest.mock import patch +from unittest.mock import patch, Mock import time import opencas @@ -24,45 +24,50 @@ def test_cas_settle_no_config(mock_config): @patch("opencas.cas_config.from_file") @patch("opencas.get_caches_list") -def test_cas_settle_cores_didnt_start_01(mock_list, mock_config): +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +def test_cas_settle_cores_didnt_start_01(mock_add, mock_exists, mock_run, mock_list, mock_config): """ Check if properly returns uninitialized cores and waits for given time Single core in config, no devices in runtime config. """ - mock_config.return_value.get_startup_cores.return_value = [ - opencas.cas_config.core_config(42, 13, "/dev/dummy") - ] + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={}, + cores=[opencas.cas_config.core_config(42, 13, "/dev/dummy")], + ) time_start = time.time() - result = opencas.wait_for_startup(timeout=5, interval=1) + result = opencas.wait_for_startup(timeout=3, interval=1) time_stop = time.time() assert len(result) == 1, "didn't return single uninitialized core" - assert ( - result[0].cache_id == 42 - and result[0].core_id == 13 - and result[0].device == "/dev/dummy" - ) - assert 4.5 < time_stop - time_start < 5.5, "didn't wait the right amount of time" - assert mock_list.call_count == 5 + assert result[0].cache_id == 42 and result[0].core_id == 13 and result[0].device == "/dev/dummy" + assert 2.5 < time_stop - time_start < 3.5, "didn't wait the right amount of time" @patch("opencas.cas_config.from_file") @patch("opencas.get_caches_list") -def test_cas_settle_cores_didnt_start_02(mock_list, mock_config): +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +def test_cas_settle_cores_didnt_start_02(mock_add, mock_exists, mock_run, mock_list, mock_config): """ Check if properly returns uninitialized cores and waits for given time Single device in config, one device in runtime config, but not the configured core """ - mock_config.return_value.get_startup_cores.return_value = [ - opencas.cas_config.core_config(1, 1, "/dev/dummy") - ] + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={}, + cores=[opencas.cas_config.core_config(1, 1, "/dev/dummy")], + ) mock_list.return_value = [ { @@ -75,28 +80,28 @@ def test_cas_settle_cores_didnt_start_02(mock_list, mock_config): } ] - time_start = time.time() - - result = opencas.wait_for_startup(timeout=1, interval=0.1) - - time_stop = time.time() + result = opencas.wait_for_startup(timeout=0, interval=0) assert len(result) == 1, "didn't return uninitialized core" - assert 0.5 < time_stop - time_start < 1.5, "didn't wait the right amount of time" @patch("opencas.cas_config.from_file") @patch("opencas.get_caches_list") -def test_cas_settle_cores_didnt_start_02(mock_list, mock_config): +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +def test_cas_settle_cores_didnt_start_03(mock_add, mock_exists, mock_run, mock_list, mock_config): """ Check if properly returns uninitialized cores and waits for given time The device waited for is in core pool. """ - mock_config.return_value.get_startup_cores.return_value = [ - opencas.cas_config.core_config(1, 1, "/dev/dummy") - ] + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={}, + cores=[opencas.cas_config.core_config(1, 1, "/dev/dummy")], + ) mock_list.return_value = [ { @@ -133,30 +138,28 @@ def test_cas_settle_cores_didnt_start_02(mock_list, mock_config): }, ] - time_start = time.time() + result = opencas.wait_for_startup(timeout=0, interval=0) - result = opencas.wait_for_startup(timeout=1, interval=0.1) - - time_stop = time.time() - - assert len(result) == 1, "didn't return uninitialized core" - assert 0.5 < time_stop - time_start < 1.5, "didn't wait the right amount of time" - # Assert the call count is within some small range in case something freezes up for a second - assert 9 <= mock_list.call_count <= 11 + assert len(result) == 0 @patch("opencas.cas_config.from_file") @patch("opencas.get_caches_list") -def test_cas_settle_cores_didnt_start_03(mock_list, mock_config): +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +def test_cas_settle_cores_didnt_start_04(mock_add, mock_exists, mock_run, mock_list, mock_config): """ Check if properly returns uninitialized cores and waits for given time The device waited for is not present, but its cache device is already started. """ - mock_config.return_value.get_startup_cores.return_value = [ - opencas.cas_config.core_config(1, 1, "/dev/dummy") - ] + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={}, + cores=[opencas.cas_config.core_config(1, 1, "/dev/dummy")], + ) mock_list.return_value = [ { @@ -209,31 +212,31 @@ def test_cas_settle_cores_didnt_start_03(mock_list, mock_config): }, ] - time_start = time.time() - - result = opencas.wait_for_startup(timeout=1, interval=0.1) - - time_stop = time.time() + result = opencas.wait_for_startup(timeout=0, interval=0) assert len(result) == 1, "didn't return uninitialized core" - assert 0.5 < time_stop - time_start < 1.5, "didn't wait the right amount of time" - # Assert the call count is within some small range in case something freezes up for a second - assert 9 <= mock_list.call_count <= 11 @patch("opencas.cas_config.from_file") @patch("opencas.get_caches_list") -def test_cas_settle_cores_didnt_start_04(mock_list, mock_config): +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +def test_cas_settle_cores_didnt_start_05(mock_add, mock_exists, mock_run, mock_list, mock_config): """ Check if properly returns uninitialized cores Two devices configured, both not present. """ - mock_config.return_value.get_startup_cores.return_value = [ - opencas.cas_config.core_config(1, 1, "/dev/dummy"), - opencas.cas_config.core_config(4, 44, "/dev/dosko"), - ] + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={}, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/dummy"), + opencas.cas_config.core_config(4, 44, "/dev/dosko"), + ], + ) mock_list.return_value = [ { @@ -278,24 +281,164 @@ def test_cas_settle_cores_didnt_start_04(mock_list, mock_config): }, ] - result = opencas.wait_for_startup(timeout=1, interval=0.1) + result = opencas.wait_for_startup(timeout=0, interval=0) assert len(result) == 2, "didn't return uninitialized cores" @patch("opencas.cas_config.from_file") @patch("opencas.get_caches_list") -def test_cas_settle_core_started_01(mock_list, mock_config): +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.start_cache") +def test_cas_settle_caches_didnt_start_01( + mock_start, mock_exists, mock_run, mock_list, mock_config +): + """ + Check if properly returns uninitialized caches and waits for given time + + Single cache in config, no devices in runtime config. + """ + + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + cores=[], + caches={42: opencas.cas_config.cache_config(42, "/dev/dummy", "wt")}, + ) + + result = opencas.wait_for_startup(timeout=0, interval=0) + + assert len(result) == 1, "didn't return single uninitialized cache" + assert result[0].cache_id == 42 and result[0].device == "/dev/dummy" + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.start_cache") +def test_cas_settle_caches_didnt_start_02( + mock_start, mock_exists, mock_run, mock_list, mock_config +): + """ + Check if properly returns uninitialized cache and waits for given time + + Single device in config, one device in runtime config, but not the configured cache + """ + + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + cores=[], + caches={1: opencas.cas_config.cache_config(1, "/dev/dummy", "wt")}, + ) + + mock_list.return_value = [ + { + "type": "cache", + "id": "3", + "disk": "/dev/dummy_cache", + "status": "Active", + "write policy": "wt", + "device": "-", + } + ] + + result = opencas.wait_for_startup(timeout=0, interval=0) + + assert len(result) == 1, "didn't return uninitialized core" + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.start_cache") +def test_cas_settle_caches_didnt_start_03( + mock_start, mock_exists, mock_run, mock_list, mock_config +): + """ + Check if properly returns uninitialized caches + + Two devices configured, both not present. + """ + + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + cores=[], + caches={ + 1: opencas.cas_config.cache_config(1, "/dev/dummy", "wt"), + 4: opencas.cas_config.cache_config(4, "/dev/dosko", "wo"), + }, + ) + + mock_list.return_value = [ + { + "type": "cache", + "id": "8", + "disk": "/dev/dummy_cache", + "status": "Incomplete", + "write policy": "wt", + "device": "-", + }, + { + "type": "core", + "id": "1", + "disk": "/dev/yes", + "status": "Inactive", + "write policy": "-", + "device": "/dev/cas1-1", + }, + { + "type": "core", + "id": "2", + "disk": "/dev/dummy3", + "status": "Active", + "write policy": "-", + "device": "/dev/cas1-2", + }, + { + "type": "cache", + "id": "2", + "disk": "/dev/dummy_cache2", + "status": "Running", + "write policy": "wb", + "device": "-", + }, + { + "type": "core", + "id": "3", + "disk": "/dev/dummy2", + "status": "Active", + "write policy": "-", + "device": "/dev/cas2-3", + }, + ] + + result = opencas.wait_for_startup(timeout=0, interval=0) + + assert len(result) == 2, "didn't return uninitialized cores" + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +def test_cas_settle_core_started_01(mock_add, mock_exists, mock_run, mock_list, mock_config): """ Check if properly returns uninitialized cores and doesn't return initialized ones Two devices configured, one present, one not present. """ - mock_config.return_value.get_startup_cores.return_value = [ - opencas.cas_config.core_config(1, 1, "/dev/dummy"), - opencas.cas_config.core_config(4, 44, "/dev/dosko"), - ] + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={}, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/dummy"), + opencas.cas_config.core_config(4, 44, "/dev/dosko"), + ], + ) mock_list.return_value = [ { @@ -348,24 +491,31 @@ def test_cas_settle_core_started_01(mock_list, mock_config): }, ] - result = opencas.wait_for_startup(timeout=1, interval=0.1) + result = opencas.wait_for_startup(timeout=0, interval=0) assert len(result) == 1, "didn't return uninitialized core" @patch("opencas.cas_config.from_file") @patch("opencas.get_caches_list") -def test_cas_settle_core_started_02(mock_list, mock_config): +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +def test_cas_settle_core_started_02(mock_add, mock_exists, mock_run, mock_list, mock_config): """ Check if properly returns uninitialized cores and doesn't return initialized ones Two devices configured, both present and added. """ - mock_config.return_value.get_startup_cores.return_value = [ - opencas.cas_config.core_config(1, 1, "/dev/dummy"), - opencas.cas_config.core_config(4, 44, "/dev/dosko"), - ] + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={}, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/dummy"), + opencas.cas_config.core_config(4, 44, "/dev/dosko"), + ], + ) mock_list.return_value = [ { @@ -434,14 +584,17 @@ def test_cas_settle_core_started_02(mock_list, mock_config): }, ] - result = opencas.wait_for_startup(timeout=1, interval=0.1) + result = opencas.wait_for_startup(timeout=0, interval=0) assert len(result) == 0, "no cores should remain uninitialized" @patch("opencas.cas_config.from_file") @patch("opencas.get_caches_list") -def test_cas_settle_core_started_03(mock_list, mock_config): +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +def test_cas_settle_core_started_03(mock_add, mock_exists, mock_run, mock_list, mock_config): """ Check if properly returns uninitialized cores and doesn't return initialized ones @@ -449,10 +602,14 @@ def test_cas_settle_core_started_03(mock_list, mock_config): get_caches_list() """ - mock_config.return_value.get_startup_cores.return_value = [ - opencas.cas_config.core_config(1, 1, "/dev/dummy"), - opencas.cas_config.core_config(2, 1, "/dev/dosko"), - ] + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={}, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/dummy"), + opencas.cas_config.core_config(2, 1, "/dev/dosko"), + ], + ) mock_list.side_effect = [ [], @@ -578,6 +735,228 @@ def test_cas_settle_core_started_03(mock_list, mock_config): ], ] - result = opencas.wait_for_startup(timeout=1, interval=0.1) + result = opencas.wait_for_startup(timeout=1, interval=0.01) assert len(result) == 0, "no cores should remain uninitialized" + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +@patch("opencas.start_cache") +def test_last_resort_add_01(mock_start, mock_add, mock_exists, mock_run, mock_list, mock_config): + """ + Check if adding cores/starting caches is not attempted while waiting for startup if paths to + devices don't exist. + + """ + mock_config.return_value = Mock( + spec_set=opencas.cas_config(), + caches={ + 1: opencas.cas_config.cache_config(1, "/dev/lizards", "wt"), + 2: opencas.cas_config.cache_config(2, "/dev/chemtrails", "wo"), + }, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/dummy"), + opencas.cas_config.core_config(2, 1, "/dev/dosko"), + ], + ) + + mock_exists.return_value = False + + result = opencas.wait_for_startup(timeout=0, interval=0) + + mock_add.assert_not_called() + mock_start.assert_not_called() + mock_run.assert_called_with(["udevadm", "settle"]) + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +@patch("opencas.start_cache") +def test_last_resort_add_02(mock_start, mock_add, mock_exists, mock_run, mock_list, mock_config): + """ + Check if adding cores/starting caches is attempted while waiting for startup. + + """ + config = Mock( + spec_set=opencas.cas_config(), + caches={ + 1: opencas.cas_config.cache_config(1, "/dev/lizards", "wt"), + 2: opencas.cas_config.cache_config(2, "/dev/wartortle", "wo"), + }, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/dummy"), + opencas.cas_config.core_config(2, 1, "/dev/dosko"), + ], + ) + + mock_config.return_value = config + + mock_exists.return_value = True + + result = opencas.wait_for_startup(timeout=0, interval=0) + + mock_start.assert_any_call(config.caches[1], True) + mock_start.assert_any_call(config.caches[2], True) + mock_add.assert_any_call(config.cores[0], True) + mock_add.assert_any_call(config.cores[1], True) + mock_run.assert_called_with(["udevadm", "settle"]) + + +def _exists_mock(timeout): + def mock(path): + if time.time() > timeout: + return True + else: + return False + + return mock + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +@patch("opencas.start_cache") +def test_last_resort_add_03(mock_start, mock_add, mock_exists, mock_run, mock_list, mock_config): + """ + Check if adding cores/starting caches is not attempted while waiting for startup if paths to + devices show up after expiring waiting timeout. + + """ + config = Mock( + spec_set=opencas.cas_config(), + caches={ + 1: opencas.cas_config.cache_config(1, "/dev/lizards", "wt"), + 2: opencas.cas_config.cache_config(2, "/dev/aerodactyl", "wo"), + }, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/dummy"), + opencas.cas_config.core_config(2, 1, "/dev/dosko"), + ], + ) + + mock_config.return_value = config + + mock_exists.side_effect = _exists_mock(time.time() + 10) + + result = opencas.wait_for_startup(timeout=0.5, interval=0.1) + + mock_start.assert_not_called() + mock_add.assert_not_called() + mock_run.assert_called_with(["udevadm", "settle"]) + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +@patch("opencas.start_cache") +def test_last_resort_add_04(mock_start, mock_add, mock_exists, mock_run, mock_list, mock_config): + """ + Check if adding cores/starting caches is attempted while waiting for startup if paths to + devices show up after half of the waiting timeout expires. + """ + config = Mock( + spec_set=opencas.cas_config(), + caches={ + 1: opencas.cas_config.cache_config(1, "/dev/lizards", "wt"), + 2: opencas.cas_config.cache_config(2, "/dev/chemtrails", "wo"), + }, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/sandshrew"), + opencas.cas_config.core_config(2, 1, "/dev/dosko"), + ], + ) + + mock_config.return_value = config + + mock_exists.side_effect = _exists_mock(time.time() + 1) + + result = opencas.wait_for_startup(timeout=2, interval=0.1) + + mock_start.assert_any_call(config.caches[1], True) + mock_start.assert_any_call(config.caches[2], True) + mock_add.assert_any_call(config.cores[0], True) + mock_add.assert_any_call(config.cores[1], True) + mock_run.assert_called_with(["udevadm", "settle"]) + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +@patch("opencas.start_cache") +def test_last_resort_add_04(mock_start, mock_add, mock_exists, mock_run, mock_list, mock_config): + """ + Check if adding cores/starting caches is attempted while waiting for startup for lazy_startup + devices once before returning. + """ + config = Mock( + spec_set=opencas.cas_config(), + caches={ + 1: opencas.cas_config.cache_config(1, "/dev/lizards", "wt", lazy_startup="true"), + 2: opencas.cas_config.cache_config(2, "/dev/chemtrails", "wo", lazy_startup="true"), + }, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/sandshrew", lazy_startup="true"), + opencas.cas_config.core_config(2, 1, "/dev/dosko", lazy_startup="true"), + ], + ) + + mock_config.return_value = config + + mock_exists.return_value = True + + result = opencas.wait_for_startup(timeout=0.5, interval=0.1) + + mock_start.assert_any_call(config.caches[1], True) + mock_start.assert_any_call(config.caches[2], True) + assert mock_start.call_count == 2, "start cache was called more than once per device" + mock_add.assert_any_call(config.cores[0], True) + mock_add.assert_any_call(config.cores[1], True) + assert mock_add.call_count == 2, "add core was called more than once per device" + mock_run.assert_called_with(["udevadm", "settle"]) + + +@patch("opencas.cas_config.from_file") +@patch("opencas.get_caches_list") +@patch("subprocess.run") +@patch("os.path.exists") +@patch("opencas.add_core") +@patch("opencas.start_cache") +def test_last_resort_add_05(mock_start, mock_add, mock_exists, mock_run, mock_list, mock_config): + """ + Check if adding cores/starting caches is not attempted while waiting for startup for lazy + startup devices if paths show up after half of the startup timeout expires. + """ + config = Mock( + spec_set=opencas.cas_config(), + caches={ + 1: opencas.cas_config.cache_config(1, "/dev/lizards", "wt", lazy_startup="true"), + 2: opencas.cas_config.cache_config(2, "/dev/chemtrails", "wo", lazy_startup="true"), + }, + cores=[ + opencas.cas_config.core_config(1, 1, "/dev/sandshrew", lazy_startup="true"), + opencas.cas_config.core_config(2, 1, "/dev/dosko", lazy_startup="true"), + ], + ) + + mock_config.return_value = config + + mock_exists.side_effect = _exists_mock(time.time() + 1) + + result = opencas.wait_for_startup(timeout=2, interval=0.5) + + mock_start.assert_not_called() + mock_add.assert_not_called() + mock_run.assert_called_with(["udevadm", "settle"]) diff --git a/utils/casctl b/utils/casctl index f5da426..a53012e 100755 --- a/utils/casctl +++ b/utils/casctl @@ -113,17 +113,15 @@ def settle(timeout, interval): # Don't fail the boot if we're missing the config exit(0) + fail = False if not_initialized: - eprint("Open CAS initialization failed. Couldn't set up all required devices") for device in not_initialized: - eprint( - "Couldn't add device {} as core {} in cache {}".format( - device.device, device.core_id, device.cache_id - ) - ) - exit(1) + fail = fail or not device.is_lazy() + eprint("Couldn't initialize device {}".format(device.device)) - exit(0) + eprint("Open CAS initialization failed. Couldn't set up all required devices") + + exit(1 if fail else 0) # Stop - detach cores and stop caches diff --git a/utils/opencas.py b/utils/opencas.py index 3c0195a..d43b0b2 100644 --- a/utils/opencas.py +++ b/utils/opencas.py @@ -26,7 +26,7 @@ class casadm: class CasadmError(Exception): def __init__(self, result): - super(casadm.CasadmError, self).__init__('casadm error') + super(casadm.CasadmError, self).__init__('casadm error: {}'.format(result.stderr)) self.result = result @classmethod @@ -172,7 +172,12 @@ class cas_config(object): @staticmethod def get_by_id_path(path): + blocklist = ["lvm", "md-name"] + for id_path in os.listdir('/dev/disk/by-id'): + if any([id_path.startswith(x) for x in blocklist]): + continue + full_path = '/dev/disk/by-id/{0}'.format(id_path) if os.path.realpath(full_path) == os.path.realpath(path): return full_path @@ -247,6 +252,8 @@ class cas_config(object): self.check_promotion_policy_valid(param_value) elif param_name == 'cache_line_size': self.check_cache_line_size_valid(param_value) + elif param_name == "lazy_startup": + self.check_lazy_startup(param_value) else: raise ValueError('{0} is invalid parameter name'.format(param_name)) @@ -278,6 +285,10 @@ class cas_config(object): raise ValueError('{0} is invalid cleaning policy name'.format( cleaning_policy)) + def check_lazy_startup_valid(self, lazy_startup): + if param_value.lower() not in ["true", "false"]: + raise ValueError('{0} is invalid lazy_startup value'.format(lazy_startup)) + def check_promotion_policy_valid(self, promotion_policy): if promotion_policy.lower() not in ['always', 'nhit']: raise ValueError('{0} is invalid promotion policy name'.format( @@ -314,6 +325,9 @@ class cas_config(object): return ret + def is_lazy(self): + return self.params.get("lazy_startup", "false").lower() == "true" + class core_config(object): def __init__(self, cache_id, core_id, path, **params): self.cache_id = int(cache_id) @@ -395,6 +409,9 @@ class cas_config(object): return ret + def is_lazy(self): + return self.params.get("lazy_startup", "false").lower() == "true" + def __init__(self, caches=None, cores=None, version_tag=None): self.caches = caches if caches else dict() @@ -534,13 +551,6 @@ class cas_config(object): except: raise Exception('Couldn\'t write config file') - def get_startup_cores(self): - return [ - core - for core in self.cores - if core.params.get("lazy_startup", "false") == "false" - ] - # Config helper functions @@ -740,7 +750,7 @@ def stop(flush): def get_devices_state(): device_list = get_caches_list() - devices = {"core_pool": [], "caches": {}, "cores": {}} + devices = {"core_pool": {}, "caches": {}, "cores": {}} core_pool = False prev_cache_id = -1 @@ -764,7 +774,12 @@ def get_devices_state(): elif device["type"] == "core": core = {"device": device["disk"], "status": device["status"]} if core_pool: - devices["core_pool"].append(core) + try: + device_path = os.path.realpath(core["device"]) + except ValueError: + device_path = core["device"] + + devices["core_pool"].update({device_path: core}) else: core.update({"cache_id": prev_cache_id}) devices["cores"].update( @@ -781,7 +796,42 @@ def wait_for_cas_ctrl(): time.sleep(1) +def _get_uninitialized_devices(target_dev_state): + not_initialized = [] + + runtime_dev_state = get_devices_state() + + for core in target_dev_state.cores: + try: + runtime_state = ( + runtime_dev_state["cores"].get((core.cache_id, core.core_id)) + or runtime_dev_state["core_pool"].get(os.path.realpath(core.device)) + ) + except ValueError: + runtime_state = None + + if not runtime_state or runtime_state["status"] == "Inactive": + not_initialized.append(core) + + for cache in target_dev_state.caches.values(): + runtime_state = runtime_dev_state["caches"].get(cache.cache_id) + + if not runtime_state: + not_initialized.append(cache) + + return not_initialized + + def wait_for_startup(timeout=300, interval=5): + def start_device(dev): + if os.path.exists(dev.device): + if type(dev) is cas_config.core_config: + add_core(dev, True) + elif type(dev) is cas_config.cache_config: + start_cache(dev, True) + + stop_time = time.time() + int(timeout) + try: config = cas_config.from_file( cas_config.default_location, allow_incomplete=True @@ -789,21 +839,24 @@ def wait_for_startup(timeout=300, interval=5): except Exception as e: raise Exception("Unable to load opencas config. Reason: {0}".format(str(e))) - stop_time = time.time() + int(timeout) + not_initialized = _get_uninitialized_devices(config) + if not not_initialized: + return [] - not_initialized = None - target_core_state = config.get_startup_cores() + result = subprocess.run(["udevadm", "settle"]) + + for dev in not_initialized: + start_device(dev) while stop_time > time.time(): - not_initialized = [] - runtime_core_state = get_devices_state()["cores"] + not_initialized = _get_uninitialized_devices(config) + wait = False - for core in target_core_state: - runtime_state = runtime_core_state.get((core.cache_id, core.core_id), None) - if not runtime_state or runtime_state["status"] != "Active": - not_initialized.append(core) + for dev in not_initialized: + wait = wait or not dev.is_lazy() + start_device(dev) - if not not_initialized: + if not wait: break time.sleep(interval)