github/open-cas-linux
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1135 from robertbaldyga/core-remove-fixes

Browse Source 
Core remove fixes
This commit is contained in:
Robert Baldyga 2022-03-28 13:51:44 +02:00 committed by GitHub
commit 4ca0eb0530
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 133 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
casadm/cas_lib.c
View File

@ -2751,6 +2751,7 @@ int list_caches(unsigned int list_format, bool by_id_path)
char status_buf[CACHE_STATE_LENGHT];
const char *tmp_status;
char mode_string[12];
char exp_obj[32];
char cache_ctrl_dev[MAX_STR_LEN] = "-";
float cache_flush_prog;
float core_flush_prog;
@ -2815,15 +2816,17 @@ int list_caches(unsigned int list_format, bool by_id_path)
tmp_status = get_core_state_name(curr_core->info.state);
}
snprintf(exp_obj, sizeof(exp_obj), "/dev/cas%d-%d",
curr_cache->id, curr_core->id);
fprintf(intermediate_file[1], TAG(TREE_LEAF)
"%s,%u,%s,%s,%s,/dev/cas%d-%d\n",
"%s,%u,%s,%s,%s,%s\n",
"core", /* type */
curr_core->id, /* id */
core_path, /* path to core*/
tmp_status, /* core status */
"-", /* write policy */
curr_cache->id, /* core id (part of path)*/
curr_core->id /* cache id (part of path)*/ );
curr_core->info.exp_obj_exists ? exp_obj : "-" /* exported object path */);
}
}

5
casadm/extended_err_msg.c
View File

@ -187,6 +187,11 @@ struct {
"Core volume size does not match"
" the size stored in cache metadata"
},
{
OCF_ERR_CORE_NOT_REMOVED,
"Unable fully remove core due to metadata flush error.\n"
"Exported object was removed, but core object still exists."
},
/* CAS kernel error mappings*/
{

8
casadm/statistics_model.c
View File

@ -188,15 +188,19 @@ static void print_core_conf(const struct kcas_core_info *info, FILE *outfile)
{
uint64_t core_size;
float core_size_gb;
char exp_obj[32];
core_size = info->info.core_size_bytes / KiB / 4;
core_size_gb = calc_gb(core_size);
snprintf(exp_obj, sizeof(exp_obj), "/dev/cas%d-%d",
info->cache_id, info->core_id);
print_kv_pair(outfile, "Core Id", "%i", info->core_id);
print_kv_pair(outfile, "Core Device", "%s",
info->core_path_name);
print_kv_pair(outfile, "Exported Object", "/dev/cas%d-%d",
info->cache_id, info->core_id);
print_kv_pair(outfile, "Exported Object", "%s",
info->exp_obj_exists ? exp_obj : "-");
print_kv_pair(outfile, "Core Size", "%lu, [" UNIT_BLOCKS "], %.2f, [GiB]",
core_size, core_size_gb);
print_kv_pair_time(outfile, "Dirty for", info->info.dirty_for);

8
modules/cas_cache/layer_cache_management.c
View File

@ -1567,7 +1567,7 @@ int cache_mngt_remove_core_from_cache(struct kcas_remove_core *cmd)
wait_for_completion(&context.cmpl);
if (!result && !cmd->detach)
if (result != -OCF_ERR_CORE_NOT_REMOVED && !cmd->detach)
mark_core_id_free(cache, cmd->core_id);
unlock:
@ -3123,6 +3123,8 @@ int cache_mngt_get_core_info(struct kcas_core_info *info)
ocf_cache_t cache;
ocf_core_t core;
const struct ocf_volume_uuid *uuid;
ocf_volume_t vol;
struct bd_object *bdvol;
int result;
result = mngt_get_cache_by_id(cas_ctx, info->cache_id, &cache);
@ -3152,6 +3154,10 @@ int cache_mngt_get_core_info(struct kcas_core_info *info)
info->state = ocf_core_get_state(core);
vol = ocf_core_get_volume(core);
bdvol = bd_object(vol);
info->exp_obj_exists = bdvol->expobj_valid;
unlock:
ocf_mngt_cache_read_unlock(cache);
put:

3
modules/cas_cache/volume/vol_block_dev_top.c
View File

@ -695,6 +695,9 @@ static int kcas_core_lock_exported_object(ocf_core_t core, void *cntx)
struct bd_object *bvol = bd_object(
ocf_core_get_volume(core));
if (!bvol->expobj_valid)
return 0;
result = casdisk_functions.casdsk_exp_obj_lock(bvol->dsk);
if (-EBUSY == result) {

13
modules/include/cas_ioctl_codes.h
View File

@ -205,6 +205,8 @@ struct kcas_core_info {
ocf_core_state_t state;
bool exp_obj_exists;
int ext_err_code;
};
@ -393,7 +395,7 @@ struct kcas_standby_activate
* 22 * KCAS_IOCTL_INSERT_CORE * OK *
* 23 * KCAS_IOCTL_REMOVE_CORE * OK *
* 24 * KCAS_IOCTL_CACHE_INFO * OK *
* 25 * KCAS_IOCTL_CORE_INFO * OK *
* 25 * KCAS_IOCTL_CORE_INFO * DEPERCATED *
* 26 * KCAS_IOCTL_GET_CORE_POOL_COUNT * OK *
* 27 * KCAS_IOCTL_GET_CORE_POOL_PATHS * OK *
* 28 * KCAS_IOCTL_CORE_POOL_REMOVE * OK *
@ -406,6 +408,9 @@ struct kcas_standby_activate
* 35 * KCAS_IOCTL_PURGE_CACHE * OK *
* 36 * KCAS_IOCTL_PURGE_CORE * OK *
* 37 * KCAS_IOCTL_REMOVE_INACTIVE * OK *
* 38 * KCAS_IOCTL_STANDBY_DETACH * OK *
* 39 * KCAS_IOCTL_STANDBY_ACTIVATE * OK *
* 40 * KCAS_IOCTL_CORE_INFO * OK *
*******************************************************************************
*/
@ -457,9 +462,6 @@ struct kcas_standby_activate
/** Retrieve properties of a running cache instance (incl. mode etc.) */
#define KCAS_IOCTL_CACHE_INFO _IOWR(KCAS_IOCTL_MAGIC, 24, struct kcas_cache_info)
/** Rretrieve statisting of a given core object */
#define KCAS_IOCTL_CORE_INFO _IOWR(KCAS_IOCTL_MAGIC, 25, struct kcas_core_info)
/** Get core pool count */
#define KCAS_IOCTL_GET_CORE_POOL_COUNT _IOR(KCAS_IOCTL_MAGIC, 26, struct kcas_core_pool_count)
@ -504,6 +506,9 @@ struct kcas_standby_activate
/** Activate failover standby cache instance */
#define KCAS_IOCTL_STANDBY_ACTIVATE _IOWR(KCAS_IOCTL_MAGIC, 39, struct kcas_standby_activate)
/** Rretrieve statisting of a given core object */
#define KCAS_IOCTL_CORE_INFO _IOWR(KCAS_IOCTL_MAGIC, 40, struct kcas_core_info)
/**
* Extended kernel CAS error codes
*/

2
ocf

@ -1 +1 @@
Subproject commit 4690cd60fea0645cf8207d7fbba2b318783b10f4
Subproject commit e741160f8d147163b9ad02db899d201ddfde686e

54
test/smoke_test/cas_functions
View File

@ -126,10 +126,56 @@ remove_core() {
clear_options
}
check_device_state() {
check_cache_state() {
check_options ${FUNCNAME[0]}
local COMMAND="$CAS --list-caches -o csv | grep `readlink -m $DEVICE_ID_OPTION` | cut -d',' -f4 | grep $DEMANDED_STATE_OPTION &> /dev/null"
get_cache_device_entry() {
$CAS --list-caches -o csv | \
awk -F, -v cache=$CACHE_ID_OPTION '{
if ($1 == "cache" && $2 == cache) print $0;
}'
}
local COMMAND="get_cache_device_entry | grep $DEMANDED_STATE_OPTION &> /dev/null"
run_cmd $COMMAND
clear_options
}
check_core_state() {
check_options ${FUNCNAME[0]}
get_core_device_entry() {
$CAS --list-caches -o csv | \
awk -F, -v cache=$CACHE_ID_OPTION -v core=$CORE_ID_OPTION '{
if ($1 == "cache") curr_cache = $2;
if ($1 == "core") curr_core = $2;
if (curr_cache == cache && curr_core == core) print $0;
}'
}
local COMMAND="get_core_device_entry | grep $DEMANDED_STATE_OPTION &> /dev/null"
run_cmd $COMMAND
clear_options
}
check_device_in_core_pool() {
check_options ${FUNCNAME[0]}
get_core_pool_entry() {
$CAS --list-caches -o csv | \
awk -F, -v device=$(readlink -m $DEVICE_ID_OPTION) '{
if ($1 == "core pool") pool = 1;
if ($1 == "cache") pool = 0;
if (pool == 1 && $3 == device) print $0;
}'
}
local COMMAND="get_core_pool_entry | grep "Detached" &> /dev/null"
run_cmd $COMMAND
@ -694,7 +740,9 @@ export -f set_cache_mode
export -f add_core
export -f try_add_core
export -f remove_core
export -f check_device_state
export -f check_cache_state
export -f check_core_state
export -f check_device_in_core_pool
export -f get_stats_value
export -f init
export -f iteration

4
test/smoke_test/cas_options
View File

@ -37,7 +37,9 @@ export START_CACHE_REQUIRED_OPTIONS="CACHE_DEVICE_OPTION"
export STOP_CACHE_REQUIRED_OPTIONS="CACHE_ID_OPTION"
export ADD_CORE_REQUIRED_OPTIONS="CACHE_ID_OPTION CORE_DEVICE_OPTION"
export TRY_ADD_CORE_REQUIRED_OPTIONS="CACHE_ID_OPTION CORE_ID_OPTION CORE_DEVICE_OPTION"
export CHECK_DEVICE_STATE_REQUIRED_OPTIONS="DEVICE_ID_OPTION DEMANDED_STATE_OPTION"
export CHECK_CACHE_STATE_REQUIRED_OPTIONS="CACHE_ID_OPTION DEMANDED_STATE_OPTION"
export CHECK_CORE_STATE_REQUIRED_OPTIONS="CACHE_ID_OPTION CORE_ID_OPTION DEMANDED_STATE_OPTION"
export CHECK_DEVICE_IN_CORE_POOL_REQUIRED_OPTIONS="DEVICE_ID_OPTION"
export REMOVE_CORE_REQUIRED_OPTIONS="CACHE_ID_OPTION CORE_ID_OPTION"
export REMOVE_PARTITIONS_REQUIRED_OPTIONS="TARGET_DEVICE_OPTION"
export FLUSH_CACHE_REQUIRED_OPTIONS="CACHE_ID_OPTION"

22
test/smoke_test/incremental_load/01
View File

@ -36,10 +36,10 @@ CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part1" add_core
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part2" add_core
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part3" add_core
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Running" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-1" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-2" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-3" DEMANDED_STATE_OPTION="Active" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Running" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Active" check_core_state
# Stop prepared setup
CACHE_ID_OPTION="1" stop_cache
@ -47,21 +47,21 @@ CACHE_ID_OPTION="1" stop_cache
# Add cores to pool, then load cache and check if cache is running
# Try to add core devices and check their states
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part1" CORE_ID_OPTION="1" try_add_core
DEVICE_ID_OPTION="${CORE_DEVICE}-part1" DEMANDED_STATE_OPTION="Detached" check_device_state
DEVICE_ID_OPTION="${CORE_DEVICE}-part1" check_device_in_core_pool
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part2" CORE_ID_OPTION="2" try_add_core
DEVICE_ID_OPTION="${CORE_DEVICE}-part2" DEMANDED_STATE_OPTION="Detached" check_device_state
DEVICE_ID_OPTION="${CORE_DEVICE}-part2" check_device_in_core_pool
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part3" CORE_ID_OPTION="3" try_add_core
DEVICE_ID_OPTION="${CORE_DEVICE}-part3" DEMANDED_STATE_OPTION="Detached" check_device_state
DEVICE_ID_OPTION="${CORE_DEVICE}-part3" check_device_in_core_pool
# Try to load cache device, check if it is running and if all cores status is appropirate
CACHE_DEVICE_OPTION="${CACHE_DEVICE}-part1" CACHE_LOAD_METADATA_OPTION="y" start_cache
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Running" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-1" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-2" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-3" DEMANDED_STATE_OPTION="Active" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Running" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Active" check_core_state
# Stop prepared setup

30
test/smoke_test/incremental_load/02
View File

@ -36,10 +36,10 @@ CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part1" add_core
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part2" add_core
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part3" add_core
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Running" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-1" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-2" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-3" DEMANDED_STATE_OPTION="Active" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Running" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Active" check_core_state
# Stop prepared setup
CACHE_ID_OPTION="1" stop_cache
@ -49,10 +49,10 @@ sleep 1
# Load cache, then add cores and check if chache is running
# Try to load cache device, check its state and cores state
CACHE_DEVICE_OPTION="${CACHE_DEVICE}-part1" CACHE_LOAD_METADATA_OPTION="y" start_cache
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Incomplete" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-1" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-2" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-3" DEMANDED_STATE_OPTION="Inactive" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Incomplete" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Inactive" check_core_state
# Prepare partition for core device to be added
CACHE_ID_OPTION="1" CACHE_DONT_FLUSH_DATA_OPTION="1" stop_cache
@ -62,16 +62,16 @@ sleep 1
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part1" CORE_ID_OPTION="1" try_add_core
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part2" CORE_ID_OPTION="2" try_add_core
CACHE_ID_OPTION="1" CORE_DEVICE_OPTION="${CORE_DEVICE}-part3" CORE_ID_OPTION="3" try_add_core
DEVICE_ID_OPTION="${CORE_DEVICE}-part1" DEMANDED_STATE_OPTION="Detached" check_device_state
DEVICE_ID_OPTION="${CORE_DEVICE}-part2" DEMANDED_STATE_OPTION="Detached" check_device_state
DEVICE_ID_OPTION="${CORE_DEVICE}-part3" DEMANDED_STATE_OPTION="Detached" check_device_state
DEVICE_ID_OPTION="${CORE_DEVICE}-part1" check_device_in_core_pool
DEVICE_ID_OPTION="${CORE_DEVICE}-part2" check_device_in_core_pool
DEVICE_ID_OPTION="${CORE_DEVICE}-part3" check_device_in_core_pool
CACHE_DEVICE_OPTION="${CACHE_DEVICE}-part1" CACHE_LOAD_METADATA_OPTION="y" start_cache
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Running" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-1" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-2" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-3" DEMANDED_STATE_OPTION="Active" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Running" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Active" check_core_state
# Clean up after test
CACHE_ID_OPTION="1" stop_cache

16
test/smoke_test/init_script/01
View File

@ -42,10 +42,10 @@ run_cmd "casctl init"
run_cmd "udevadm settle"
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Running" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-1" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-2" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-3" DEMANDED_STATE_OPTION="Active" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Running" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Active" check_core_state
run_cmd "casctl stop"
@ -53,10 +53,10 @@ check_no_cache_running
run_cmd "casctl start"
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Running" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-1" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-2" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}1-3" DEMANDED_STATE_OPTION="Active" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Running" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Active" check_core_state
run_cmd "casctl stop"

16
test/smoke_test/init_script/02
View File

@ -44,10 +44,10 @@ EOM
run_cmd "casctl init"
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Running" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}16384-0" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}16384-4095" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}16384-4094" DEMANDED_STATE_OPTION="Active" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Running" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Active" check_core_state
run_cmd "casadm -G -n cleaning -i 16384 | grep "acp" &>/dev/null"
@ -57,10 +57,10 @@ check_no_cache_running
run_cmd "casctl start"
DEVICE_ID_OPTION="${CACHE_DEVICE}-part1" DEMANDED_STATE_OPTION="Running" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}16384-0" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}16384-4095" DEMANDED_STATE_OPTION="Active" check_device_state
DEVICE_ID_OPTION="${DEVICE_NAME}16384-4094" DEMANDED_STATE_OPTION="Active" check_device_state
CACHE_ID_OPTION="1" DEMANDED_STATE_OPTION="Running" check_cache_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="1" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="2" DEMANDED_STATE_OPTION="Active" check_core_state
CACHE_ID_OPTION="1" CORE_ID_OPTION="3" DEMANDED_STATE_OPTION="Active" check_core_state
run_cmd "casctl stop"