389 lines
13 KiB
Python
389 lines
13 KiB
Python
#
|
|
# Copyright(c) 2019-2022 Intel Corporation
|
|
# SPDX-License-Identifier: BSD-3-Clause
|
|
#
|
|
|
|
import datetime
|
|
import json
|
|
import secrets
|
|
from enum import Enum
|
|
from types import SimpleNamespace as Namespace
|
|
|
|
from connection.base_executor import BaseExecutor
|
|
from core.test_run import TestRun
|
|
from storage_devices.device import Device
|
|
from test_tools.fio.fio_result import FioResult
|
|
from test_utils.linux_command import LinuxCommand
|
|
from test_utils.size import Size
|
|
|
|
|
|
class CpusAllowedPolicy(Enum):
|
|
shared = 0,
|
|
split = 1
|
|
|
|
|
|
class ErrorFilter(Enum):
|
|
none = 0,
|
|
read = 1,
|
|
write = 2,
|
|
io = 3,
|
|
verify = 4,
|
|
all = 5
|
|
|
|
|
|
class FioOutput(Enum):
|
|
normal = 'normal'
|
|
terse = 'terse'
|
|
json = 'json'
|
|
jsonplus = 'json+'
|
|
|
|
|
|
class IoEngine(Enum):
|
|
# Basic read or write I/O. fseek is used to position the I/O location.
|
|
sync = 0,
|
|
# Linux native asynchronous I/O.
|
|
libaio = 1,
|
|
# Basic pread or pwrite I/O.
|
|
psync = 2,
|
|
# Basic readv or writev I/O.
|
|
# Will emulate queuing by coalescing adjacent IOs into a single submission.
|
|
vsync = 3,
|
|
# Basic preadv or pwritev I/O.
|
|
pvsync = 4,
|
|
# POSIX asynchronous I/O using aio_read and aio_write.
|
|
posixaio = 5,
|
|
# File is memory mapped with mmap and data copied using memcpy.
|
|
mmap = 6,
|
|
# RADOS Block Device
|
|
rbd = 7,
|
|
# SPDK Block Device
|
|
spdk_bdev = 8
|
|
|
|
|
|
class ReadWrite(Enum):
|
|
randread = 0,
|
|
randrw = 1,
|
|
randwrite = 2,
|
|
read = 3,
|
|
readwrite = 4,
|
|
write = 5,
|
|
trim = 6,
|
|
randtrim = 7,
|
|
trimwrite = 8
|
|
|
|
|
|
class VerifyMethod(Enum):
|
|
# Use an md5 sum of the data area and store it in the header of each block.
|
|
md5 = 0,
|
|
# Use an experimental crc64 sum of the data area and store it in the header of each block.
|
|
crc64 = 1,
|
|
# Use optimized sha1 as the checksum function.
|
|
sha1 = 2,
|
|
# Verify a strict pattern.
|
|
# Normally fio includes a header with some basic information and a checksum, but if this
|
|
# option is set, only the specific pattern set with verify_pattern is verified.
|
|
pattern = 3,
|
|
# Write extra information about each I/O (timestamp, block number, etc.).
|
|
# The block number is verified.
|
|
meta = 4
|
|
|
|
|
|
class RandomGenerator(Enum):
|
|
tausworthe = 0,
|
|
lfsr = 1,
|
|
tausworthe64 = 2
|
|
|
|
|
|
class FioParam(LinuxCommand):
|
|
def __init__(self, fio, command_executor: BaseExecutor, command_name):
|
|
LinuxCommand.__init__(self, command_executor, command_name)
|
|
self.verification_pattern = ''
|
|
self.fio = fio
|
|
|
|
def get_verification_pattern(self):
|
|
if not self.verification_pattern:
|
|
self.verification_pattern = f'0x{secrets.token_hex(32)}'
|
|
return self.verification_pattern
|
|
|
|
def allow_mounted_write(self, value: bool = True):
|
|
return self.set_param('allow_mounted_write', int(value))
|
|
|
|
# example: "bs=8k,32k" => 8k for reads, 32k for writes and trims
|
|
def block_size(self, *sizes: Size):
|
|
return self.set_param('blocksize', *[int(size) for size in sizes])
|
|
|
|
def blocksize_range(self, ranges):
|
|
value = []
|
|
for bs_range in ranges:
|
|
str_range = str(int(bs_range[0])) + '-' + str(int(bs_range[1]))
|
|
value.append(str_range)
|
|
return self.set_param('blocksize_range', ",".join(value))
|
|
|
|
def bs_split(self, value):
|
|
return self.set_param('bssplit', value)
|
|
|
|
def buffer_pattern(self, pattern):
|
|
return self.set_param('buffer_pattern', pattern)
|
|
|
|
def continue_on_error(self, value: ErrorFilter):
|
|
return self.set_param('continue_on_error', value.name)
|
|
|
|
def cpus_allowed(self, value):
|
|
return self.set_param('cpus_allowed', ",".join(value))
|
|
|
|
def cpus_allowed_policy(self, value: CpusAllowedPolicy):
|
|
return self.set_param('cpus_allowed_policy', value.name)
|
|
|
|
def direct(self, value: bool = True):
|
|
if 'buffered' in self.command_param:
|
|
self.remove_param('buffered')
|
|
return self.set_param('direct', int(value))
|
|
|
|
def directory(self, directory):
|
|
return self.set_param('directory', directory)
|
|
|
|
def do_verify(self, value: bool = True):
|
|
return self.set_param('do_verify', int(value))
|
|
|
|
def exit_all_on_error(self, value: bool = True):
|
|
return self.set_flags('exitall_on_error') if value \
|
|
else self.remove_flag('exitall_on_error')
|
|
|
|
def group_reporting(self, value: bool = True):
|
|
return self.set_flags('group_reporting') if value else self.remove_flag('group_reporting')
|
|
|
|
def file_name(self, path):
|
|
return self.set_param('filename', path)
|
|
|
|
def file_size(self, size: Size):
|
|
return self.set_param('filesize', int(size))
|
|
|
|
def file_size_range(self, ranges):
|
|
value = []
|
|
for bs_range in ranges:
|
|
str_range = str(int(bs_range[0])) + '-' + str(int(bs_range[1]))
|
|
value.append(str_range)
|
|
return self.set_param('filesize', ",".join(value))
|
|
|
|
def fsync(self, value: int):
|
|
return self.set_param('fsync', value)
|
|
|
|
def ignore_errors(self, read_errors, write_errors, verify_errors):
|
|
separator = ':'
|
|
return self.set_param(
|
|
'ignore_error',
|
|
separator.join(str(err) for err in read_errors),
|
|
separator.join(str(err) for err in write_errors),
|
|
separator.join(str(err) for err in verify_errors))
|
|
|
|
def io_depth(self, value: int):
|
|
if value != 1:
|
|
if 'ioengine' in self.command_param and \
|
|
self.command_param['ioengine'] == 'sync':
|
|
TestRun.LOGGER.warning("Setting iodepth will have no effect with "
|
|
"'ioengine=sync' setting")
|
|
return self.set_param('iodepth', value)
|
|
|
|
def io_engine(self, value: IoEngine):
|
|
if value == IoEngine.sync:
|
|
if 'iodepth' in self.command_param and self.command_param['iodepth'] != 1:
|
|
TestRun.LOGGER.warning("Setting 'ioengine=sync' will cause iodepth setting "
|
|
"to be ignored")
|
|
return self.set_param('ioengine', value.name)
|
|
|
|
def io_size(self, value: Size):
|
|
return self.set_param('io_size', int(value.get_value()))
|
|
|
|
def loops(self, value: int):
|
|
return self.set_param('loops', value)
|
|
|
|
def no_random_map(self, value: bool = True):
|
|
if 'verify' in self.command_param:
|
|
raise ValueError("'NoRandomMap' parameter is mutually exclusive with verify")
|
|
if value:
|
|
return self.set_flags('norandommap')
|
|
else:
|
|
return self.remove_flag('norandommap')
|
|
|
|
def nr_files(self, value: int):
|
|
return self.set_param('nrfiles', value)
|
|
|
|
def num_ios(self, value: int):
|
|
return self.set_param('number_ios', value)
|
|
|
|
def num_jobs(self, value: int):
|
|
return self.set_param('numjobs', value)
|
|
|
|
def offset(self, value: Size):
|
|
return self.set_param('offset', int(value.get_value()))
|
|
|
|
def offset_increment(self, value: Size):
|
|
return self.set_param('offset_increment', f"{value.value}{value.unit.get_short_name()}")
|
|
|
|
def percentage_random(self, value: int):
|
|
if value <= 100:
|
|
return self.set_param('percentage_random', value)
|
|
raise ValueError("Argument out of range. Should be 0-100.")
|
|
|
|
def pool(self, value):
|
|
return self.set_param('pool', value)
|
|
|
|
def ramp_time(self, value: datetime.timedelta):
|
|
return self.set_param('ramp_time', int(value.total_seconds()))
|
|
|
|
def random_distribution(self, value):
|
|
return self.set_param('random_distribution', value)
|
|
|
|
def rand_repeat(self, value: int):
|
|
return self.set_param('randrepeat', value)
|
|
|
|
def rand_seed(self, value: int):
|
|
return self.set_param('randseed', value)
|
|
|
|
def read_write(self, rw: ReadWrite):
|
|
return self.set_param('readwrite', rw.name)
|
|
|
|
def run_time(self, value: datetime.timedelta):
|
|
if value.total_seconds() == 0:
|
|
raise ValueError("Runtime parameter must not be set to 0.")
|
|
return self.set_param('runtime', int(value.total_seconds()))
|
|
|
|
def serialize_overlap(self, value: bool = True):
|
|
return self.set_param('serialize_overlap', int(value))
|
|
|
|
def size(self, value: Size):
|
|
return self.set_param('size', int(value.get_value()))
|
|
|
|
def stonewall(self, value: bool = True):
|
|
return self.set_flags('stonewall') if value else self.remove_param('stonewall')
|
|
|
|
def sync(self, value: bool = True):
|
|
return self.set_param('sync', int(value))
|
|
|
|
def time_based(self, value: bool = True):
|
|
return self.set_flags('time_based') if value else self.remove_flag('time_based')
|
|
|
|
def thread(self, value: bool = True):
|
|
return self.set_flags('thread') if value else self.remove_param('thread')
|
|
|
|
def lat_percentiles(self, value: bool):
|
|
return self.set_param('lat_percentiles', int(value))
|
|
|
|
def scramble_buffers(self, value: bool):
|
|
return self.set_param('scramble_buffers', int(value))
|
|
|
|
def slat_percentiles(self, value: bool):
|
|
return self.set_param('slat_percentiles', int(value))
|
|
|
|
def spdk_core_mask(self, value: str):
|
|
return self.set_param('spdk_core_mask', value)
|
|
|
|
def spdk_json_conf(self, path):
|
|
return self.set_param('spdk_json_conf', path)
|
|
|
|
def clat_percentiles(self, value: bool):
|
|
return self.set_param('clat_percentiles', int(value))
|
|
|
|
def percentile_list(self, value: []):
|
|
val = ':'.join(value) if len(value) > 0 else '100'
|
|
return self.set_param('percentile_list', val)
|
|
|
|
def verification_with_pattern(self, pattern=None):
|
|
if pattern is not None and pattern != '':
|
|
self.verification_pattern = pattern
|
|
return self.verify(VerifyMethod.pattern) \
|
|
.set_param('verify_pattern', self.get_verification_pattern()) \
|
|
.do_verify()
|
|
|
|
def verify(self, value: VerifyMethod):
|
|
return self.set_param('verify', value.name)
|
|
|
|
def create_only(self, value: bool = False):
|
|
return self.set_param('create_only', int(value))
|
|
|
|
def verify_pattern(self, pattern=None):
|
|
return self.set_param('verify_pattern', pattern or self.get_verification_pattern())
|
|
|
|
def verify_backlog(self, value: int):
|
|
return self.set_param('verify_backlog', value)
|
|
|
|
def verify_dump(self, value: bool = True):
|
|
return self.set_param('verify_dump', int(value))
|
|
|
|
def verify_fatal(self, value: bool = True):
|
|
return self.set_param('verify_fatal', int(value))
|
|
|
|
def verify_only(self, value: bool = True):
|
|
return self.set_flags('verify_only') if value else self.remove_param('verify_only')
|
|
|
|
def write_hint(self, value: str):
|
|
return self.set_param('write_hint', value)
|
|
|
|
def write_percentage(self, value: int):
|
|
if value <= 100:
|
|
return self.set_param('rwmixwrite', value)
|
|
raise ValueError("Argument out of range. Should be 0-100.")
|
|
|
|
def random_generator(self, value: RandomGenerator):
|
|
return self.set_param('random_generator', value.name)
|
|
|
|
def target(self, target):
|
|
if isinstance(target, Device):
|
|
return self.file_name(target.path)
|
|
return self.file_name(target)
|
|
|
|
def add_job(self, job_name=None):
|
|
if not job_name:
|
|
job_name = f'job{len(self.fio.jobs)}'
|
|
new_job = FioParamConfig(self.fio, self.command_executor, f'[{job_name}]')
|
|
self.fio.jobs.append(new_job)
|
|
return new_job
|
|
|
|
def clear_jobs(self):
|
|
self.fio.jobs = []
|
|
|
|
return self
|
|
|
|
def edit_global(self):
|
|
return self.fio.global_cmd_parameters
|
|
|
|
def run(self, fio_timeout: datetime.timedelta = None):
|
|
if "per_job_logs" in self.fio.global_cmd_parameters.command_param:
|
|
self.fio.global_cmd_parameters.set_param("per_job_logs", '0')
|
|
fio_output = self.fio.run(fio_timeout)
|
|
if fio_output.exit_code != 0:
|
|
raise Exception(f"Exception occurred while trying to execute fio, exit_code:"
|
|
f"{fio_output.exit_code}.\n"
|
|
f"stdout: {fio_output.stdout}\nstderr: {fio_output.stderr}")
|
|
TestRun.executor.run(f"sed -i '/^[[:alnum:]]/d' {self.fio.fio_file}") # Remove warnings
|
|
out = self.command_executor.run_expect_success(f"cat {self.fio.fio_file}").stdout
|
|
return self.get_results(out)
|
|
|
|
def run_in_background(self):
|
|
if "per_job_logs" in self.fio.global_cmd_parameters.command_param:
|
|
self.fio.global_cmd_parameters.set_param("per_job_logs", '0')
|
|
return self.fio.run_in_background()
|
|
|
|
@staticmethod
|
|
def get_results(result):
|
|
data = json.loads(result, object_hook=lambda d: Namespace(**d))
|
|
jobs_list = []
|
|
if hasattr(data, 'jobs'):
|
|
jobs = data.jobs
|
|
for job in jobs:
|
|
job_result = FioResult(data, job)
|
|
jobs_list.append(job_result)
|
|
return jobs_list
|
|
|
|
|
|
class FioParamCmd(FioParam):
|
|
def __init__(self, fio, command_executor: BaseExecutor, command_name='fio'):
|
|
FioParam.__init__(self, fio, command_executor, command_name)
|
|
self.param_name_prefix = "--"
|
|
|
|
|
|
class FioParamConfig(FioParam):
|
|
def __init__(self, fio, command_executor: BaseExecutor, command_name='[global]'):
|
|
FioParam.__init__(self, fio, command_executor, command_name)
|
|
self.param_name_prefix = "\n"
|