open-cas-linux/test/smoke_test/cas_functions

#!/bin/bash

#
# Copyright(c) 2012-2021 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause
#

############################################################
#                        CAS API                           #
############################################################

clear_config() {
    check_options ${FUNCNAME[0]}

    if [ -n "$STORE_CONFIG_OPTION" ] ; then
        cp $CAS_CONFIG_PATH $TMP_DIR/`basename $CAS_CONFIG_PATH`
    fi

    echo "version=`$CAS -V -o csv | grep CLI | cut -d, -f2`" > "$CAS_CONFIG_PATH"

    export CAS_CONFIG_VERSION_TAG=`head $CAS_CONFIG_PATH`

    clear_options
}

restore_config() {
    cp $TMP_DIR/`basename $CAS_CONFIG_PATH` $CAS_CONFIG_PATH
}

start_cache() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --start-cache --cache-device $CACHE_DEVICE_OPTION"

    if [ -n "$CACHE_ID_OPTION" ] ; then
        COMMAND="$COMMAND --cache-id $CACHE_ID_OPTION"
    fi
    if [ -n "$CACHE_FORCE_OPTION" ] ; then
        COMMAND="$COMMAND --force"
    fi
    if [ -n "$CACHE_MODE_OPTION" ] ; then
        COMMAND="$COMMAND --cache-mode $CACHE_MODE_OPTION"
    fi
    if [ -n "$CACHE_LOAD_METADATA_OPTION" ] ; then
        COMMAND="$COMMAND --load"
    fi
    if [ -n "$CACHE_LINE_SIZE" ] ; then
        COMMAND="$COMMAND --cache-line-size $CACHE_LINE_SIZE"
    fi

    run_cmd $COMMAND
    clear_options
}

stop_cache() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --stop-cache --cache-id $CACHE_ID_OPTION"

    if [ -n "$CACHE_DONT_FLUSH_DATA_OPTION" ] ; then
        COMMAND="$COMMAND --no-data-flush"
    fi

    run_cmd $COMMAND
    clear_options
}

set_cache_mode() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS -Q -c $CACHE_MODE_OPTION --cache-id $CACHE_ID_OPTION"

    if [ -n "$CACHE_MODE_FLUSH_OPTION" ] ; then
        COMMAND="$COMMAND --flush-cache yes"
    else
        COMMAND="$COMMAND --flush-cache no"
    fi

    run_cmd $COMMAND
    clear_options
}

try_add_core() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --script --add-core --try-add --cache-id $CACHE_ID_OPTION --core-id $CORE_ID_OPTION --core-device $CORE_DEVICE_OPTION"

    run_cmd $COMMAND
}

add_core() {
    local RETRY=10
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --add-core --cache-id $CACHE_ID_OPTION --core-device $CORE_DEVICE_OPTION"

    if [ -n "$CORE_TRY_ADD_OPTION" ] ; then
        COMMAND="$COMMAND --try-add"
    fi

    run_cmd $COMMAND
    if [ -n "$NEGATIVE_TEST_OPTION" ]; then
        clear_options
        return 0
    fi
    local i=0
    local SHORT_LINK=$(realpath $CORE_DEVICE_OPTION)
    local CAS_DEV=` casadm -L | egrep "^.core +[0-9]+ +$SHORT_LINK" | awk '{print $NF}'`
    clear_options
    while [ ! -e $CAS_DEV ]; do
        sleep 2
        let i++
        if [ $i -gt $RETRY ]; then
            error "Add core timeout"
            end_test 1
        fi
    done
}

remove_core() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --remove-core --cache-id $CACHE_ID_OPTION --core-id $CORE_ID_OPTION"

    run_cmd $COMMAND
    clear_options
}


check_cache_state() {
    check_options ${FUNCNAME[0]}

    get_cache_device_entry() {
        $CAS --list-caches -o csv | \
            awk -F, -v cache=$CACHE_ID_OPTION '{
            if ($1 == "cache" && $2 == cache) print $0;
        }'
    }

    local COMMAND="get_cache_device_entry | grep $DEMANDED_STATE_OPTION &> /dev/null"

    run_cmd $COMMAND

    clear_options
}

check_core_state() {
    check_options ${FUNCNAME[0]}

    get_core_device_entry() {
        $CAS --list-caches -o csv | \
            awk -F, -v cache=$CACHE_ID_OPTION -v core=$CORE_ID_OPTION '{
            if ($1 == "cache") curr_cache = $2;
            if ($1 == "core") curr_core = $2;
            if (curr_cache == cache && curr_core == core) print $0;
        }'
    }

    local COMMAND="get_core_device_entry | grep $DEMANDED_STATE_OPTION &> /dev/null"

    run_cmd $COMMAND

    clear_options
}

check_device_in_core_pool() {
    check_options ${FUNCNAME[0]}

    get_core_pool_entry() {
        $CAS --list-caches -o csv | \
            awk -F, -v device=$(readlink -m $DEVICE_ID_OPTION) '{
                if ($1 == "core pool") pool = 1;
                if ($1 == "cache") pool = 0;
                if (pool == 1 && $3 == device) print $0;
            }'
    }

    local COMMAND="get_core_pool_entry | grep "Detached" &> /dev/null"

    run_cmd $COMMAND

    clear_options
}

get_stats_value() {
    check_options ${FUNCNAME[0]}

    if [ -n "$CORE_ID_OPTION" ] ; then
        CORE_ID_OPTION="--core-id $CORE_ID_OPTION"
    fi

    if [ "$IO_CLASS_ID" = "all" ]; then
        PART_ID_OPTION="--io-class-id"
    elif [ -n "$IO_CLASS_ID" ] ; then
        PART_ID_OPTION="--io-class-id $IO_CLASS_ID"
    fi

    SILENT_COMMAND_OPTION="1" run_cmd "$CAS --stats --output-format csv \
    --cache-id $CACHE_ID_OPTION $CORE_ID_OPTION $PART_ID_OPTION | tail -2 | tr \",\" \"\n\""

    local IFS_DEFAULT=$IFS
    IFS=$'\n'

    local STATS_ARRAY=()
    local i=0
    for line in $OUTPUT; do
        STATS_ARRAY[$i]="$line"
        i=$((i+1))
    done

    STATS_VALUES_LENGTH=${#STATS_ARRAY[@]}
    STATS_VALUES_OFFSET=$((STATS_VALUES_LENGTH / 2))

    local hits=0
    for (( i=0; i<$STATS_VALUES_OFFSET; i++ ))
    do
        echo "${STATS_ARRAY[$i]}" | grep -i "$STAT_NAME_OPTION" | grep -v "%" \
            | grep -i "$STAT_UNIT_OPTION" &> /dev/null
        if [ $? -eq 0 ]; then
            local value_id=$(( STATS_VALUES_OFFSET + i ))
            OUTPUT="${STATS_ARRAY[$value_id]}"
            hits=$((hits + 1))
        fi
    done

    IFS=$IFS_DEFAULT

    if [[ $hits -gt 1 ]] ; then
        error "Given statistic name or unit is ambiguous!"
        OUTPUT=""
        end_test 1
    fi

    if [[ $hits -eq 0 ]] ; then
        error "Given statistic name or unit not found!"
        OUTPUT=""
        end_test 1
    fi

    clear_options
}

init() {
    check_options ${FUNCNAME[0]}

    local L_NUMBER_OF_CACHE_PARTITIONS=1
    local L_NUMBER_OF_CORE_PARTITIONS=1
    local L_MAKE_FILE_SYSTEM
    local MAX_NUMBER_OF_CORE_PARTITIONS=4
    local L_CACHE_FORCE_OPTION
    local L_CACHE_MODE_OPTION
    local L_CACHE_LOAD_METADATA_OPTION
    local L_CACHE_LINE_SIZE
    local L_MAKE_PARTITIONS=${MAKE_PARTITIONS}

    if [ -n "$NUMBER_OF_CACHE_PARTITIONS" ] ; then
        L_NUMBER_OF_CACHE_PARTITIONS="$NUMBER_OF_CACHE_PARTITIONS"
    fi
    if [ -n "$NUMBER_OF_CORE_PARTITIONS" ] ; then
        L_NUMBER_OF_CORE_PARTITIONS="$NUMBER_OF_CORE_PARTITIONS"
    fi
    if [ -n "$MAKE_FILE_SYSTEM" ] ; then
        L_MAKE_FILE_SYSTEM="$MAKE_FILE_SYSTEM"
    fi
    if [ -n "$CACHE_FORCE_OPTION" ] ; then
        L_CACHE_FORCE_OPTION="$CACHE_FORCE_OPTION"
    fi
    if [ -n "$CACHE_MODE_OPTION" ] ; then
        L_CACHE_MODE_OPTION="$CACHE_MODE_OPTION"
    fi
    if [ -n "$CACHE_LOAD_METADATA_OPTION" ] ; then
        L_CACHE_LOAD_METADATA_OPTION="$CACHE_LOAD_METADATA_OPTION"
    fi
    if [ -n "$CACHE_LINE_SIZE" ] ; then
        L_CACHE_LINE_SIZE="$CACHE_LINE_SIZE"
    fi

    if [ $L_NUMBER_OF_CACHE_PARTITIONS -gt $MAX_NUMBER_OF_CORE_PARTITIONS ] ; then
        echo "You are trying start too many cache device!"
        echo "Max number of cache device is 4."
        L_NUMBER_OF_CACHE_PARTITIONS=4
    fi

    if [ $L_NUMBER_OF_CACHE_PARTITIONS -eq 4 ] || [ $L_NUMBER_OF_CACHE_PARTITIONS -eq 3 ] ; then
        MAX_NUMBER_OF_CORE_PARTITIONS=1
    fi

    if [ $L_NUMBER_OF_CACHE_PARTITIONS -eq 2 ] ; then
        MAX_NUMBER_OF_CORE_PARTITIONS=2
    fi

    if [ -n "$PARTITION_CORE_SIZE_OPTION" ] ; then
        L_PARTITION_CORE_SIZE_OPTION="$PARTITION_CORE_SIZE_OPTION"
    else
        L_PARTITION_CORE_SIZE_OPTION="$DEFAULT_CORE_SIZE"
    fi

    if [ -n "$PARTITION_CACHE_SIZE_OPTION" ] ; then
        L_PARTITION_CACHE_SIZE_OPTION="$PARTITION_CACHE_SIZE_OPTION"
    else
        L_PARTITION_CACHE_SIZE_OPTION="$DEFAULT_CACHE_SIZE"
    fi

    if [ "1" == "$L_MAKE_PARTITIONS" ]
    then
        TARGET_DEVICE_OPTION="$CACHE_DEVICE" remove_partitions
        TARGET_DEVICE_OPTION="$CORE_DEVICE" remove_partitions

        local L_PARTITION_NUMBERS=$(seq 1 $L_NUMBER_OF_CACHE_PARTITIONS)
        TARGET_DEVICE_OPTION="$CACHE_DEVICE" PARTITION_SIZE_OPTION="$L_PARTITION_CACHE_SIZE_OPTION" PARTITION_IDS_OPTION="$L_PARTITION_NUMBERS" make_primary_partitions

        local L_PARTITION_NUMBERS=$(seq 1 $(( $L_NUMBER_OF_CACHE_PARTITIONS * $L_NUMBER_OF_CORE_PARTITIONS )))
        TARGET_DEVICE_OPTION="$CORE_DEVICE" PARTITION_SIZE_OPTION="$L_PARTITION_CORE_SIZE_OPTION" PARTITION_IDS_OPTION="$L_PARTITION_NUMBERS" make_primary_partitions
    fi

    k=1
    for ((i = 1 ; i <= L_NUMBER_OF_CACHE_PARTITIONS; i++)); do

        if [ -n "$L_CACHE_FORCE_OPTION" ] ; then
            CACHE_FORCE_OPTION="$L_CACHE_FORCE_OPTION"
        fi
        if [ -n "$L_CACHE_MODE_OPTION" ] ; then
            CACHE_MODE_OPTION="$L_CACHE_MODE_OPTION"
        fi
        if [ -n "$L_CACHE_LOAD_METADATA_OPTION" ] ; then
            CACHE_LOAD_METADATA_OPTION="$L_CACHE_LOAD_METADATA_OPTION"
        fi
        if [ -n "$L_CACHE_LINE_SIZE" ] ; then
            CACHE_LINE_SIZE="$L_CACHE_LINE_SIZE"
        fi

        CACHE_ID_OPTION="$i" CACHE_DEVICE_OPTION="${CACHE_DEVICE}-part$i" start_cache

        for ((j = 1; j <= L_NUMBER_OF_CORE_PARTITIONS && j <= MAX_NUMBER_OF_CORE_PARTITIONS; j++)); do
            CACHE_ID_OPTION="$i" CORE_DEVICE_OPTION="${CORE_DEVICE}-part$k" add_core
            k=`expr $k \+ 1`
        done
    done

    if [ -n "$L_MAKE_FILE_SYSTEM" ] ; then
        for ((i=1 ; i <= L_NUMBER_OF_CACHE_PARTITIONS; i++)); do
            for ((j=1 ; j <= L_NUMBER_OF_CORE_PARTITIONS && j <= MAX_NUMBER_OF_CORE_PARTITIONS; j++)); do
                TARGET_DEVICE_OPTION="${DEVICE_NAME}$i-$j" FILESYSTEM_TYPE="$L_MAKE_FILE_SYSTEM" make_filesystem
                run_cmd "mount ${DEVICE_NAME}$i-$j ${MOUNTPOINT}-$i-$j"
            done
        done
    fi

    clear_options
}

iteration() {
    check_options ${FUNCNAME[0]}

    local L_NUMBER_OF_CACHE_PARTITIONS
    local L_NUMBER_OF_CORE_PARTITIONS
    local L_MAKE_FILE_SYSTEM="off"
    local MAX_NUMBER_OF_CORE_PARTITIONS=4
    local L_CACHE_FORCE_OPTION="off"
    local L_CACHE_MODE_OPTION="off"
    local L_CACHE_LOAD_METADATA_OPTION="off"
    local L_CACHE_LINE_SIZE="off"
    local L_MAKE_PARTITIONS=1

    if [ -n "$NUMBER_OF_CACHE_PARTITIONS" ] ; then
        L_NUMBER_OF_CACHE_PARTITIONS="$NUMBER_OF_CACHE_PARTITIONS"
    fi
    if [ -n "$NUMBER_OF_CORE_PARTITIONS" ] ; then
        L_NUMBER_OF_CORE_PARTITIONS="$NUMBER_OF_CORE_PARTITIONS"
    fi
    if [ -n "$MAKE_FILE_SYSTEM" ] ; then
        if [ "$MAKE_FILE_SYSTEM" == "all" ] ; then
            L_MAKE_FILE_SYSTEM="ext3 ext4 xfs"
        else
            L_MAKE_FILE_SYSTEM="$MAKE_FILE_SYSTEM"
        fi
    fi
    if [ -n "$CACHE_FORCE_OPTION" ] ; then
        if [ "$CACHE_FORCE_OPTION" == "all" ] ; then
            L_CACHE_MODE_OPTION="off on"
        else
            L_CACHE_FORCE_OPTION="$CACHE_FORCE_OPTION"
        fi
    fi
    if [ -n "$CACHE_MODE_OPTION" ] ; then
        if [ "$CACHE_MODE_OPTION" == "all" ] ; then
            L_CACHE_MODE_OPTION="wa wb wt pt wo"
        else
            L_CACHE_MODE_OPTION="$CACHE_MODE_OPTION"
        fi
    fi
    if [ -n "$CACHE_LOAD_METADATA_OPTION" ] ; then
        if [ "$CACHE_LOAD_METADATA_OPTION" == "all" ] ; then
            L_CACHE_LOAD_METADATA_OPTION="off on"
        else
            L_CACHE_LOAD_METADATA_OPTION="$CACHE_LOAD_METADATA_OPTION"
        fi
    fi
    if [ -n "$CACHE_LINE_SIZE" ] ; then
        if [ "$CACHE_LINE_SIZE" == "all" ] ; then
            L_CACHE_LINE_SIZE="4 8 16 32 64"
        else
            L_CACHE_LINE_SIZE="$CACHE_LINE_SIZE"
        fi
    fi

    iteration_number=1
    for mode in $L_CACHE_MODE_OPTION; do
        for cache_line_size_mode in $L_CACHE_LINE_SIZE; do
            for filesystem in $L_MAKE_FILE_SYSTEM; do
                for force in $L_CACHE_FORCE_OPTION; do
                    for load_metadata in $L_CACHE_LOAD_METADATA_OPTION; do
                        echo "--- Iteration $iteration_number ---"
                        echo "Configuration in this iteration is:"

                        if [ -n "$L_NUMBER_OF_CACHE_PARTITIONS" ] ; then
                            NUMBER_OF_CACHE_PARTITIONS="$L_NUMBER_OF_CACHE_PARTITIONS"
                        fi
                        if [ -n "$L_NUMBER_OF_CORE_PARTITIONS" ] ; then
                            NUMBER_OF_CORE_PARTITIONS="$L_NUMBER_OF_CORE_PARTITIONS"
                        fi
                        if [ "$mode" != "off" ] ; then
                            CACHE_MODE_OPTION="$mode"
                            echo "Cache mode : $mode"
                        fi
                        if [ "$cache_line_size_mode" != "off" ] ; then
                            CACHE_LINE_SIZE="$cache_line_size_mode"
                            echo "Cache_line_size : $cache_line_size_mode"
                        fi
                        if [ "$filesystem" != "off" ] ; then
                            MAKE_FILE_SYSTEM="$filesystem"
                            echo "Filesystem type : $filesystem"
                        fi
                        if [ "$force" == "on" ] ; then
                            CACHE_FORCE_OPTION="$force"
                            echo "Force option: on"
                        fi
                        if [ "$load_metadata" == "on" ] ; then
                            CACHE_LOAD_METADATA_OPTION="$load_metadata"
                            echo "Load metadata option: on"
                        else
                            L_MAKE_PARTITIONS=1
                        fi

                        MAKE_PARTITIONS=${L_MAKE_PARTITIONS}
                        init
                        $1
                        if [ $? -ne 0 ]
                        then
                            end_test 1
                        fi
                        L_MAKE_PARTITIONS=0

                        if [ -n "$L_NUMBER_OF_CACHE_PARTITIONS" ] ; then
                            NUMBER_OF_CACHE_PARTITIONS="$L_NUMBER_OF_CACHE_PARTITIONS"
                        fi

                        if [ -n "$L_NUMBER_OF_CORE_PARTITIONS" ] ; then
                            NUMBER_OF_CORE_PARTITIONS="$L_NUMBER_OF_CORE_PARTITIONS"
                        fi

                        deinit
                        iteration_number=`expr $iteration_number \+ 1`
                    done
                done
            done
        done
    done

    clear_options
}

deinit() {
    check_options ${FUNCNAME[0]}

    local L_NUMBER_OF_CACHE_PARTITIONS=1
    local L_NUMBER_OF_CORE_PARTITIONS=1
    local MAX_NUMBER_OF_CORE_PARTITIONS=4

    if [ -n "$NUMBER_OF_CACHE_PARTITIONS" ] ; then
        L_NUMBER_OF_CACHE_PARTITIONS="$NUMBER_OF_CACHE_PARTITIONS"
    fi

    if [ -n "$NUMBER_OF_CORE_PARTITIONS" ] ; then
        L_NUMBER_OF_CORE_PARTITIONS="$NUMBER_OF_CORE_PARTITIONS"
    fi

    if [ $L_NUMBER_OF_CACHE_PARTITIONS -gt $MAX_NUMBER_OF_CORE_PARTITIONS ] ; then
        echo "You are trying start too many cache device!"
        echo "Max number of cache device is 4."
        L_NUMBER_OF_CACHE_PARTITIONS=4
    fi

    if [ $L_NUMBER_OF_CACHE_PARTITIONS -eq 4 ] || [ $L_NUMBER_OF_CACHE_PARTITIONS -eq 3 ] ; then
        MAX_NUMBER_OF_CORE_PARTITIONS=1
    fi

    if [ $L_NUMBER_OF_CACHE_PARTITIONS -eq 2 ] ; then
        MAX_NUMBER_OF_CORE_PARTITIONS=2
    fi

    for ((i=1 ; i <= L_NUMBER_OF_CACHE_PARTITIONS; i++)); do
        for ((j=1 ; j <= L_NUMBER_OF_CORE_PARTITIONS && j <= MAX_NUMBER_OF_CORE_PARTITIONS; j++)); do
            CHECK_MOUNTPOINT=`mount | grep  ${MOUNTPOINT}-$i-$j`
            if [ -n "$CHECK_MOUNTPOINT" ] ; then
                run_cmd "umount ${MOUNTPOINT}-$i-$j"
            fi
        done
    done

    for ((i=1 ; i <= L_NUMBER_OF_CACHE_PARTITIONS; i++)); do
        for ((j=1 ; j <= L_NUMBER_OF_CORE_PARTITIONS && j <= MAX_NUMBER_OF_CORE_PARTITIONS; j++)); do
            CACHE_ID_OPTION="$i" CORE_ID_OPTION="$j" remove_core
        done
        CACHE_ID_OPTION="$i" stop_cache
    done

    clear_options
}

flush_cache() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --flush-cache --cache-id $CACHE_ID_OPTION"

    run_cmd $COMMAND
    clear_options
}

flush_core() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --flush-core --cache-id $CACHE_ID_OPTION --core-id $CORE_ID_OPTION"

    run_cmd $COMMAND
    clear_options
}

set_cleaning_policy() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --set-param --name cleaning --cache-id $CACHE_ID_OPTION \
                   --policy $CLEAN_POL_OPTION"

    run_cmd $COMMAND
    clear_options
}

get_cleaning_policy() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --get-param --name cleaning --cache-id $CACHE_ID_OPTION \
                   --output-format csv"

    run_cmd $COMMAND
    clear_options
}

set_promotion_policy() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --set-param --name promotion --cache-id $CACHE_ID_OPTION \
                   --policy $PROMO_POL_OPTION"

    run_cmd $COMMAND
    clear_options
}

check_promotion_policy() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --get-param --name promotion --cache-id $CACHE_ID_OPTION \
                   --output-format csv"

    echo -n "$(date +%Y-%m-%d_%H:%M:%S)  "
    echo -n "Checking if promotion policy type set to '$PROMO_POL_OPTION'. "

    PROMO_POL_VALUE=$($COMMAND | grep -i type | cut -d ',' -f2)

    if [ -n "$NEGATIVE_TEST_OPTION" ] && [ "$NEGATIVE_TEST_OPTION" -ne 0 ] ; then
        echo -n "(negative test) "
        if [[ "$PROMO_POL_OPTION" != "$PROMO_POL_VALUE" ]] ; then
            success
        else
            error
            echo "Promotion policy type set to inadmissible value '$PROMO_POL_VALUE'!"
            if [ -z $DONT_FAIL_ON_ERROR_OPTION ]; then
                end_test 1
            fi
            return 1
        fi
    else
        if [[ "$PROMO_POL_OPTION" == "$PROMO_POL_VALUE" ]] ; then
            success
        else
            error
            echo "Promotion policy type do not match!"
            if [ -z $DONT_FAIL_ON_ERROR_OPTION ]; then
                end_test 1
            fi
            return 1
        fi
    fi

    clear_options
}

set_flush_params() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --set-param --name $CLEAN_POL_NS_OPTION --cache-id $CACHE_ID_OPTION"

    if [ -n "$WAKE_UP_OPTION" ] ; then
        COMMAND="$COMMAND --wake-up $WAKE_UP_OPTION"
    fi
    if [ -n "$STALE_TIME_OPTION" ] ; then
        COMMAND="$COMMAND --staleness-time $STALE_TIME_OPTION"
    fi
    if [ -n "$FLUSH_BUFFERS_OPTION" ] ; then
        COMMAND="$COMMAND --flush-max-buffers $FLUSH_BUFFERS_OPTION"
    fi
    if [ -n "$ACTIVITY_THRESH_OPTION" ] ; then
        COMMAND="$COMMAND --activity-threshold $ACTIVITY_THRESH_OPTION"
    fi

    run_cmd $COMMAND
    clear_options
}

get_flush_params() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --get-param --name $CLEAN_POL_NS_OPTION \
                   --cache-id $CACHE_ID_OPTION --output-format csv"

    run_cmd $COMMAND
    clear_options
}

set_promotion_params() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --set-param --name $PROMO_POL_NS_OPTION --cache-id $CACHE_ID_OPTION"

    if [ -n "$THRESHOLD_OPTION" ] ; then
        COMMAND="$COMMAND --threshold $THRESHOLD_OPTION"
    fi
    if [ -n "$TRIGGER_OPTION" ] ; then
        COMMAND="$COMMAND --trigger $TRIGGER_OPTION"
    fi

    run_cmd $COMMAND
    clear_options
}

check_promotion_params() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --get-param --name $PROMO_POL_NS_OPTION \
                   --cache-id $CACHE_ID_OPTION --output-format csv"

    echo -n "$(date +%Y-%m-%d_%H:%M:%S)  "
    echo -n "Checking threshold and/or trigger values. "
    if [ -z "$THRESHOLD_OPTION" ] && [ -z "$TRIGGER_OPTION" ] ; then
        error
        echo "No defined promotion policy option to check! "
        if [ -z $DONT_FAIL_ON_ERROR_OPTION ]; then
            end_test 1
        fi
        return 1
    fi

    if [ -n "$THRESHOLD_OPTION" ] ; then
        echo -n "THRESHOLD value = '$THRESHOLD_OPTION' "
        THRESHOLD_VALUE=$($COMMAND | grep -i threshold | cut -d ',' -f2)
        if [ "$THRESHOLD_OPTION" -ne "$THRESHOLD_VALUE" ] ; then
            THRESHOLD_VALUE_ERROR="Threshold value do not match!"
        fi
    fi
    if [ -n "$TRIGGER_OPTION" ] ; then
        echo -n "TRIGGER value = '$TRIGGER_OPTION' "
        TRIGGER_VALUE=$($COMMAND | grep -i trigger | cut -d ',' -f2)
        if [ "$TRIGGER_OPTION" -ne "$TRIGGER_VALUE" ] ; then
            TRIGGER_VALUE_ERROR="Trigger value do not match!"
        fi
    fi
    if [ -n "$NEGATIVE_TEST_OPTION" ] && [ "$NEGATIVE_TEST_OPTION" -ne 0 ] ; then
        echo -n "(negative test) "
        if [ -n "$THRESHOLD_VALUE_ERROR" ] || [ -n "$TRIGGER_VALUE_ERROR" ] ; then
            success
        else
            error
            echo "Given values match!"
            if [ -z $DONT_FAIL_ON_ERROR_OPTION ]; then
                end_test 1
            fi
            return 1
        fi
    else
        if [ -z "$THRESHOLD_VALUE_ERROR" ] && [ -z "$TRIGGER_VALUE_ERROR" ] ; then
            success
        else
            error
            echo -e "$THRESHOLD_VALUE_ERROR\n$TRIGGER_VALUE_ERROR"
            if [ -z $DONT_FAIL_ON_ERROR_OPTION ]; then
                end_test 1
            fi
            return 1
        fi
    fi

    clear_options
}

dirty_stop() {
    check_options ${FUNCNAME[0]}
    local L_CACHE_DEVICE_OPTION = $CACHE_DEVICE_OPTION
    CACHE_DEVICE_OPTION=$L_CACHE_DEVICE_OPTION turn_off_device

    local COMMAND="$CAS --stop-cache --cache-id $CACHE_ID_OPTION"

    sleep 1
    run_cmd $COMMAND
    sleep 1

    CACHE_DEVICE_OPTION=$L_CACHE_DEVICE_OPTION turn_on_device
    clear_options
}

check_no_cache_running() {
    run_cmd "$CAS -L | grep 'No caches running'"
    clear_options
}

export -f clear_config
export -f restore_config
export -f start_cache
export -f stop_cache
export -f set_cache_mode
export -f add_core
export -f try_add_core
export -f remove_core
export -f check_cache_state
export -f check_core_state
export -f check_device_in_core_pool
export -f get_stats_value
export -f init
export -f iteration
export -f deinit
export -f flush_core
export -f flush_cache
export -f set_flush_params
export -f get_flush_params
export -f set_cleaning_policy
export -f get_cleaning_policy
export -f set_promotion_params
export -f check_promotion_params
export -f set_promotion_policy
export -f check_promotion_policy
export -f dirty_stop
export -f check_no_cache_running

############################################################
#                     SYSTEM FUNCTIONS                     #
############################################################

mount_cache() {
    check_options ${FUNCNAME[0]}

    DIR="${MOUNTPOINT}-${CACHE_ID_OPTION}-${CORE_ID_OPTION}"

    if [ ! -d $DIR ] ; then
        mkdir $DIR
    fi

    if [ -n $(mount | grep "$DIR") ] ; then
        umount $DIR 2> /dev/null
    fi

    local COMMAND="mount ${DEVICE_NAME}${CACHE_ID_OPTION}-${CORE_ID_OPTION} $DIR"

    run_cmd $COMMAND
    clear_options
}

remove_partitions() {
    check_options ${FUNCNAME[0]}

    local DEV_NAME=$(echo $TARGET_DEVICE_OPTION | sed 's/\/.*\///')
    local SFDISK_CFG="$TMP_DIR/${DEV_NAME}_sfdisk.cfg"
    local COMMAND="dd if=/dev/zero of=$TARGET_DEVICE_OPTION bs=4k count=10"

    yes | rm -r $SFDISK_CFG 2>/dev/null
    run_cmd $COMMAND
    clear_options
}

make_primary_partitions() {
    check_options ${FUNCNAME[0]}
    local RETRY=40
    local DEV_NAME=${TARGET_DEVICE_OPTION}
    local SFDISK_CFG="$TMP_DIR/${DEV_NAME}_sfdisk.cfg"
    local COMMAND="sfdisk -D -uM $TARGET_DEVICE_OPTION < $SFDISK_CFG"
    local MAX_RETRY=4
    local START="2048"
    local END=""
    local PART_NUM=0
    local RET=1
    local i

    run_cmd parted -s $DEV_NAME mktable gpt

    for PARTITION_ID in $PARTITION_IDS_OPTION ; do
        i=0

        if [[ $PARTITION_SIZE_OPTION == *[^0-9] ]] ; then
            local BYTES=$(($(get_bytes $PARTITION_SIZE_OPTION)/512))
        fi

        if [[ $PARTITION_ID -eq 1 ]] ; then
            END=$(($BYTES + $START))
        fi

        while [[ $RET -ne 0 && $i -lt $MAX_RETRY ]] ; do
            echo -n "parted -s -a optimal $DEV_NAME mkpart primary ${START}s ${END}s "
            parted -s -a optimal $DEV_NAME mkpart primary ${START}s ${END}s
            RET=$?
            if [[ $RET -ne 0 ]] ; then
                sleep 2
                let i++
            fi
        done

        if [[ $RET -ne 0 ]] ; then
            warning "Parted can't inform kernel about changes in partition table"
        else
            success
            RET=1
        fi

        START=$(($END + 2048))
        END=$(($START + $BYTES))
        let PART_NUM++
    done
    run_cmd "udevadm settle"

    i=0

    for ID in `seq 1 $PART_NUM`; do
        local i=0
        local TEST_DEV="${DEV_NAME}-part${ID}"
        local TEST_DEV_P="${DEV_NAME}-part${ID}"
        while ! [[ -L $TEST_DEV  || -L $TEST_DEV_P ]] ; do
            # make sure that partition is detected if it was created
            partprobe
            sleep 1
            let i++
            if [ $i -gt $MAX_RETRY ]; then
                clear_options
                error "Creating partition ${TEST_DEV} failed"
                end_test 1
            fi
        done
        # Create symlink for NVMe
        if [[ $TARGET_DEVICE_OPTION == "/dev/nvme"* ]] ; then
            ln -sf "$TEST_DEV_P" "$TEST_DEV"
        fi
        # partition successfully created.,
        # erase all filesystem/cas cache metadata that may have existed on it
        # before.

        if [ -L $TEST_DEV ]
        then
            run_cmd dd if=/dev/zero of="${TEST_DEV}" bs=1M count=1 oflag=direct
        else
            run_cmd dd if=/dev/zero of="${TEST_DEV_P}" bs=1M count=1 oflag=direct
        fi

    done

    run_cmd "udevadm settle"
    clear_options
}

make_filesystem() {
    check_options ${FUNCNAME[0]}
    local L_LABEL=""

    if [ "$FILESYSTEM_LABEL" != "" ]; then
            L_LABEL="-L $FILESYSTEM_LABEL"
    fi

    case $FILESYSTEM_TYPE in
        "ext3" )
            local COMMAND="mkfs.ext3 ${L_LABEL} $TARGET_DEVICE_OPTION"
            ;;
        "ext4" )
            hash mkfs.ext4 2> /dev/null && [ ! -e /etc/SuSE-release ]   #ext4 is not supported on SLES
            if [ $? -eq 0 ] ; then
                local COMMAND="mkfs.ext4 ${L_LABEL} $TARGET_DEVICE_OPTION"
            else
                local COMMAND="mkfs.ext3 ${L_LABEL} $TARGET_DEVICE_OPTION"
                warning "EXT4 not found or SLES detected , using EXT3 instead"
            fi
            ;;
        "xfs" )
            hash mkfs.xfs 2> /dev/null
            if [ $? -eq 0 ] ; then
                local COMMAND="mkfs.xfs ${L_LABEL} -f -b size=4096 $TARGET_DEVICE_OPTION"
            else
                local COMMAND="mkfs.ext3 ${L_LABEL} $TARGET_DEVICE_OPTION"
                warning "XFS not found, using EXT3 instead"
            fi
            ;;
        * )
            error "Unrecognized filesystem $FILESYSTEM_TYPE"
    esac

    run_cmd $COMMAND
    clear_options
}

# Removes all caches which use the cache device. This should be used in cleanup after each test
# so that if the test fails, it won't make other tests fail.
remove_caches() {
    SILENT_COMMAND_OPTION="1"

    CACHE_IDS_TO_REMOVE=$(${CAS} -L | grep $CACHE_DEVICE | awk '{print $2}')
    for ID in $CACHE_IDS_TO_REMOVE ; do
        # Umount all mounted instances first
        DEVICES_TO_UMOUNT="$(mount | grep "/dev/${DEVICE_NAME}${ID}-" | awk '{print $1}')"
        for DEVICE_TO_UMOUNT in $DEVICES_TO_UMOUNT ; do
            umount $DEVICE_TO_UMOUNT
        done
        CACHE_ID_OPTION="$ID"
        CACHE_DONT_FLUSH_DATA_OPTION="yes"
        # Stop the cache!
        stop_cache
    done

    # Remove detached cores from core pool
    CORE_DEVICES_TO_REMOVE=$(${CAS} -L | grep $CORE_DEVICE | awk '{print $3}')
    for DEVICE in $CORE_DEVICES_TO_REMOVE ; do
        local COMMAND="$CAS --remove-detached -d $DEVICE"
        run_cmd $COMMAND
    done

    if [ -n "$CACHE_IDS_TO_REMOVE" ] || [ -n "$CORE_DEVICES_TO_REMOVE" ] ; then
        warning "Had to remove some caches in cleanup - \
            either the test failed or it doesn't clean up after itself!"
    fi
    clear_options
}

turn_on_device() {
    # Use realpath resolved before turning off device
    check_options ${FUNCNAME[0]}
    if [[ $CACHE_DEVICE_OPTION == "/dev/nvme"* ]] ; then
        turn_on_nvme_device
    else
        COMMAND="echo 'running' > /sys/block/${CACHE_DEVICE_OPTION:4}/device/state"
        run_cmd $COMMAND
    fi
    clear_options
}

turn_off_device() {
    check_options ${FUNCNAME[0]}
    SHORT_LINK=$(realpath $CACHE_DEVICE_OPTION)
    if [[ $SHORT_LINK == "/dev/nvme"* ]] ; then
        turn_off_nvme_device
    else
        local COMMAND="echo 'offline' > /sys/block/${SHORT_LINK:4}/device/state"
        run_cmd $COMMAND
    fi
}

turn_off_nvme_device() {
    check_options ${FUNCNAME[0]}
    SHORT_LINK=$(realpath $CACHE_DEVICE_OPTION)
    COMMAND="echo '1' > /sys/block/${SHORT_LINK:4}/device/device/remove"
    run_cmd $COMMAND
    clear_options
}

turn_on_nvme_device() {
    check_options ${FUNCNAME[0]}
    local COMMAND="echo '1' > /sys/bus/pci/rescan"
    run_cmd $COMMAND
    sleep 30
}

check_is_nvme_atomic() {
    check_options ${FUNCNAME[0]}
    nvme id-ns $DEVICE_OPTION | grep "ms:8.*ds:9.*(in use)" &> /dev/null
    return $?
}

io_class_list() {
    check_options ${FUNCNAME[0]}
    local COMMAND="CASADM_NO_LINE_BREAK='t' LANG='C' $CAS --io-class --list --cache-id $CACHE_ID_OPTION"

    if [ -n "$CSV_FILE" ] ; then
        COMMAND="$COMMAND --output-format csv > $CSV_FILE"
    fi

    run_cmd $COMMAND

    clear_options
}

io_class_load() {
    check_options ${FUNCNAME[0]}
    run_cmd $CAS --io-class --load-config --file $CSV_FILE --cache-id $CACHE_ID_OPTION
    clear_options
}

io_class_stats() {
    check_options ${FUNCNAME[0]}

    local COMMAND="$CAS --stats --cache-id $CACHE_ID_OPTION --io-class-id"

    if [ -n "$IO_CLASS_ID" ] ; then
        COMMAND="$COMMAND $IO_CLASS_ID"
    fi

    if [ -n "$CSV_FILE" ] ; then
        COMMAND="$COMMAND --output-format csv > $CSV_FILE"
    fi

    run_cmd $COMMAND
    clear_options
}

stats() {
    check_options ${FUNCNAME[0]}
    run_cmd $CAS --stats --cache-id $CACHE_ID_OPTION
    clear_options
}

check_fio_ver() {
    check_options ${FUNCNAME[0]}

    fio_ver="$(fio -v)"
    if [ -z $fio_ver ] ; then
    echo_yellow "Fio is not installed, skipping test!"
    return 1
    fi

    major=`echo $fio_ver | cut -d '.' -f 1 | cut -d '-' -f 2`
    minor=`echo $fio_ver | cut -d '.' -f 2`
    if [[ $major -lt $FIO_MAJOR_OPTION ]] ; then
    echo_yellow "Invalid fio version, skipping test!"
    return 1
    elif [[ $major -eq $FIO_MAJOR_OPTION ]] && \
        [[ $minor -lt $FIO_MINOR_OPTION ]] ; then
    echo_yellow "Invalid fio minor revision, skipping test!"
    return 1
    fi

    clear_options
    return 0
}

check_kernel_ver() {
    check_options ${FUNCNAME[0]}

    kernel_ver="$(uname -r)"
    kernel_version=`echo ${kernel_ver} | cut  -d '.' -f 1`
    major=`echo ${kernel_ver} | cut  -d '.' -f 2`
    if [[ $kernel_version -lt $KERNEL_VER_OPTION ]] ; then
    echo_yellow "Invalid kernel version, skipping test!"
    return 1
    elif [[ $kernel_version -eq $KERNEL_VER_OPTION ]] && \
        [[ $major -lt $KERNEL_MAJOR_OPTION ]] ; then
    echo_yellow "Invalid kernel major revision, skipping test!"
    return 1
    fi

    clear_options
    return 0
}

export -f mount_cache
export -f remove_partitions
export -f make_primary_partitions
export -f remove_caches
export -f make_filesystem
export -f turn_on_device
export -f turn_off_device
export -f turn_on_nvme_device
export -f turn_off_nvme_device
export -f io_class_list
export -f io_class_load
export -f stats
export -f io_class_stats
export -f check_is_nvme_atomic
export -f check_fio_ver
export -f check_kernel_ver

export CAS_FUNCTIONS_LOADED="1"

echo "--- Open CAS API library loaded ---"