a1af1809d8
Resetting cache_error/core_error in ocf_req_forward_* functions may lead to overwriting already reported error if the forward is being done in the loop. To avoid this potential problem, introduce set of forward init functions intended to be called before the entire forward operation, which resets the error code and sets a forward callback. Signed-off-by: Robert Baldyga <robert.baldyga@huawei.com>
493 lines
12 KiB
C
493 lines
12 KiB
C
/*
|
|
* Copyright(c) 2012-2022 Intel Corporation
|
|
* Copyright(c) 2023-2024 Huawei Technologies
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
*/
|
|
|
|
#include "ocf/ocf.h"
|
|
#include "metadata/metadata.h"
|
|
#include "metadata/metadata.h"
|
|
#include "engine/cache_engine.h"
|
|
#include "utils/utils_cache_line.h"
|
|
#include "ocf_request.h"
|
|
#include "utils/utils_user_part.h"
|
|
#include "ocf_priv.h"
|
|
#include "ocf_cache_priv.h"
|
|
#include "ocf_queue_priv.h"
|
|
#include "utils/utils_stats.h"
|
|
#include "ocf/ocf_debug.h"
|
|
|
|
ocf_volume_t ocf_cache_get_volume(ocf_cache_t cache)
|
|
{
|
|
return cache->device ? &cache->device->volume : NULL;
|
|
}
|
|
|
|
ocf_volume_t ocf_cache_get_front_volume(ocf_cache_t cache)
|
|
{
|
|
return cache->device ? &cache->device->front_volume : NULL;
|
|
}
|
|
|
|
uint32_t ocf_cache_get_queue_count(ocf_cache_t cache)
|
|
{
|
|
ocf_queue_t queue = NULL;
|
|
uint32_t cnt = 0;
|
|
|
|
list_for_each_entry(queue, &cache->io_queues, list)
|
|
cnt++;
|
|
|
|
return cnt;
|
|
}
|
|
|
|
int ocf_cache_set_name(ocf_cache_t cache, const char *src, size_t src_size)
|
|
{
|
|
int result;
|
|
|
|
OCF_CHECK_NULL(cache);
|
|
|
|
result = env_strncpy(cache->name, OCF_CACHE_NAME_SIZE, src, src_size);
|
|
if (result)
|
|
return result;
|
|
|
|
return env_strncpy(cache->conf_meta->name, OCF_CACHE_NAME_SIZE,
|
|
src, src_size);
|
|
}
|
|
|
|
bool ocf_cache_mode_is_valid(ocf_cache_mode_t mode)
|
|
{
|
|
return mode >= ocf_cache_mode_wt && mode < ocf_cache_mode_max;
|
|
}
|
|
|
|
const char *ocf_cache_get_name(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return cache->name;
|
|
}
|
|
|
|
bool ocf_cache_is_incomplete(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return env_bit_test(ocf_cache_state_incomplete, &cache->cache_state);
|
|
}
|
|
|
|
bool ocf_cache_is_running(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return env_bit_test(ocf_cache_state_running, &cache->cache_state);
|
|
}
|
|
|
|
bool ocf_cache_is_detached(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return env_bit_test(ocf_cache_state_detached, &cache->cache_state);
|
|
}
|
|
|
|
bool ocf_cache_is_standby(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return env_bit_test(ocf_cache_state_standby, &cache->cache_state);
|
|
}
|
|
|
|
bool ocf_cache_is_device_attached(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return !!cache->device;
|
|
}
|
|
|
|
ocf_cache_mode_t ocf_cache_get_mode(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
|
|
return cache->conf_meta->cache_mode;
|
|
}
|
|
|
|
static uint64_t _calc_dirty_for(uint64_t dirty_since)
|
|
{
|
|
uint64_t current_time = env_ticks_to_secs(env_get_tick_count());
|
|
|
|
return dirty_since ? (current_time - dirty_since) : 0;
|
|
}
|
|
|
|
int ocf_cache_get_info(ocf_cache_t cache, struct ocf_cache_info *info)
|
|
{
|
|
uint32_t cache_occupancy_total = 0;
|
|
uint32_t dirty_blocks_total = 0;
|
|
uint32_t initial_dirty_blocks_total = 0;
|
|
uint32_t flushed_total = 0;
|
|
uint32_t curr_dirty_cnt;
|
|
uint64_t dirty_since = 0;
|
|
uint32_t init_dirty_cnt;
|
|
uint64_t core_dirty_since;
|
|
uint32_t dirty_blocks_inactive = 0;
|
|
uint32_t cache_occupancy_inactive = 0;
|
|
ocf_core_t core;
|
|
ocf_core_id_t core_id;
|
|
|
|
OCF_CHECK_NULL(cache);
|
|
|
|
if (!info)
|
|
return -OCF_ERR_INVAL;
|
|
|
|
ENV_BUG_ON(env_memset(info, sizeof(*info), 0));
|
|
|
|
_ocf_stats_zero(&info->inactive);
|
|
|
|
info->attached = ocf_cache_is_device_attached(cache);
|
|
info->standby_detached = ocf_cache_is_standby(cache) &&
|
|
ocf_refcnt_frozen(&cache->refcnt.metadata);
|
|
if (info->attached && !info->standby_detached) {
|
|
info->volume_type = ocf_ctx_get_volume_type_id(cache->owner,
|
|
cache->device->volume.type);
|
|
info->size = cache->conf_meta->cachelines;
|
|
}
|
|
info->state = cache->cache_state;
|
|
info->cache_line_size = ocf_line_size(cache);
|
|
info->metadata_end_offset = ocf_cache_is_device_attached(cache) ?
|
|
cache->device->metadata_offset / PAGE_SIZE : 0;
|
|
info->metadata_footprint = ocf_cache_is_device_attached(cache) ?
|
|
ocf_metadata_size_of(cache) : 0;
|
|
|
|
if (ocf_cache_is_standby(cache))
|
|
return 0;
|
|
|
|
info->core_count = cache->conf_meta->core_count;
|
|
|
|
info->cache_mode = ocf_cache_get_mode(cache);
|
|
|
|
/* iterate through all possibly valid core objcts, as list of
|
|
* valid objects may be not continuous
|
|
*/
|
|
for_each_core(cache, core, core_id) {
|
|
/* If current dirty blocks exceeds saved initial dirty
|
|
* blocks then update the latter
|
|
*/
|
|
curr_dirty_cnt = env_atomic_read(
|
|
&core->runtime_meta->dirty_clines);
|
|
init_dirty_cnt = env_atomic_read(
|
|
&core->runtime_meta->initial_dirty_clines);
|
|
if (init_dirty_cnt && (curr_dirty_cnt > init_dirty_cnt)) {
|
|
env_atomic_set(
|
|
&core->runtime_meta->initial_dirty_clines,
|
|
env_atomic_read(
|
|
&core->runtime_meta->dirty_clines));
|
|
}
|
|
cache_occupancy_total += env_atomic_read(
|
|
&core->runtime_meta->cached_clines);
|
|
|
|
dirty_blocks_total += env_atomic_read(
|
|
&core->runtime_meta->dirty_clines);
|
|
initial_dirty_blocks_total += env_atomic_read(
|
|
&core->runtime_meta->initial_dirty_clines);
|
|
|
|
if (!core->opened) {
|
|
cache_occupancy_inactive += env_atomic_read(
|
|
&core->runtime_meta->cached_clines);
|
|
|
|
dirty_blocks_inactive += env_atomic_read(
|
|
&core->runtime_meta->dirty_clines);
|
|
}
|
|
core_dirty_since = env_atomic64_read(
|
|
&core->runtime_meta->dirty_since);
|
|
if (core_dirty_since) {
|
|
dirty_since = (dirty_since ?
|
|
OCF_MIN(dirty_since, core_dirty_since) :
|
|
core_dirty_since);
|
|
}
|
|
|
|
flushed_total += env_atomic_read(&core->flushed);
|
|
}
|
|
|
|
info->dirty = dirty_blocks_total;
|
|
info->dirty_initial = initial_dirty_blocks_total;
|
|
info->occupancy = cache_occupancy_total;
|
|
info->dirty_for = _calc_dirty_for(dirty_since);
|
|
|
|
if (info->attached) {
|
|
_set(&info->inactive.occupancy,
|
|
_lines4k(cache_occupancy_inactive, ocf_line_size(cache)),
|
|
_lines4k(info->size, ocf_line_size(cache)));
|
|
_set(&info->inactive.clean,
|
|
_lines4k(cache_occupancy_inactive - dirty_blocks_inactive,
|
|
ocf_line_size(cache)),
|
|
_lines4k(cache_occupancy_total, ocf_line_size(cache)));
|
|
_set(&info->inactive.dirty,
|
|
_lines4k(dirty_blocks_inactive, ocf_line_size(cache)),
|
|
_lines4k(cache_occupancy_total, ocf_line_size(cache)));
|
|
}
|
|
|
|
info->flushed = (env_atomic_read(&cache->flush_in_progress)) ?
|
|
flushed_total : 0;
|
|
|
|
info->fallback_pt.status = ocf_fallback_pt_is_on(cache);
|
|
info->fallback_pt.error_counter =
|
|
env_atomic_read(&cache->fallback_pt_error_counter);
|
|
|
|
info->cleaning_policy = cache->cleaner.policy;
|
|
info->promotion_policy = cache->conf_meta->promotion_policy_type;
|
|
info->cache_line_size = ocf_line_size(cache);
|
|
|
|
return 0;
|
|
}
|
|
|
|
const struct ocf_volume_uuid *ocf_cache_get_uuid(ocf_cache_t cache)
|
|
{
|
|
if (!ocf_cache_is_device_attached(cache))
|
|
return NULL;
|
|
|
|
return ocf_volume_get_uuid(ocf_cache_get_volume(cache));
|
|
}
|
|
|
|
uint8_t ocf_cache_get_type_id(ocf_cache_t cache)
|
|
{
|
|
if (!ocf_cache_is_device_attached(cache))
|
|
return 0xff;
|
|
|
|
return ocf_ctx_get_volume_type_id(ocf_cache_get_ctx(cache),
|
|
ocf_volume_get_type(ocf_cache_get_volume(cache)));
|
|
}
|
|
|
|
ocf_cache_line_size_t ocf_cache_get_line_size(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return ocf_line_size(cache);
|
|
}
|
|
|
|
uint64_t ocf_cache_bytes_2_lines(ocf_cache_t cache, uint64_t bytes)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return ocf_bytes_2_lines(cache, bytes);
|
|
}
|
|
|
|
uint32_t ocf_cache_get_core_count(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return cache->conf_meta->core_count;
|
|
}
|
|
|
|
ocf_ctx_t ocf_cache_get_ctx(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return cache->owner;
|
|
}
|
|
|
|
void ocf_cache_set_priv(ocf_cache_t cache, void *priv)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
cache->priv = priv;
|
|
}
|
|
|
|
void *ocf_cache_get_priv(ocf_cache_t cache)
|
|
{
|
|
OCF_CHECK_NULL(cache);
|
|
return cache->priv;
|
|
}
|
|
|
|
struct ocf_cache_volume {
|
|
ocf_cache_t cache;
|
|
};
|
|
|
|
static inline ocf_cache_t ocf_volume_to_cache(ocf_volume_t volume)
|
|
{
|
|
struct ocf_cache_volume *cache_volume = ocf_volume_get_priv(volume);
|
|
|
|
return cache_volume->cache;
|
|
}
|
|
|
|
static void ocf_cache_volume_io_complete_generic(struct ocf_request *req,
|
|
int error)
|
|
{
|
|
ocf_cache_t cache = req->cache;
|
|
|
|
ocf_refcnt_dec(&cache->refcnt.metadata);
|
|
ocf_io_end_func(req, error);
|
|
}
|
|
|
|
static void ocf_cache_io_complete(struct ocf_request *req, int error)
|
|
{
|
|
ocf_cache_t cache = req->cache;
|
|
|
|
if (error)
|
|
req->error = req->error ?: error;
|
|
|
|
if (env_atomic_dec_return(&req->req_remaining))
|
|
return;
|
|
|
|
ocf_refcnt_dec(&cache->refcnt.metadata);
|
|
ocf_io_end_func(req, req->error);
|
|
}
|
|
|
|
static void ocf_cache_volume_submit_io(ocf_io_t io)
|
|
{
|
|
struct ocf_request *req = ocf_io_to_req(io);
|
|
ocf_cache_t cache = req->cache;
|
|
int result;
|
|
|
|
if (!ocf_refcnt_inc(&cache->refcnt.metadata)) {
|
|
ocf_io_end_func(io, -OCF_ERR_IO);
|
|
return;
|
|
}
|
|
if (unlikely(!ocf_cache_is_standby(cache))) {
|
|
ocf_io_end_func(io, -OCF_ERR_CACHE_NOT_STANDBY);
|
|
return;
|
|
}
|
|
|
|
env_atomic_set(&req->req_remaining, 3);
|
|
|
|
ocf_req_forward_cache_init(req, ocf_cache_io_complete);
|
|
ocf_req_forward_cache_io(req, req->rw, req->addr,
|
|
req->bytes, req->offset);
|
|
|
|
req->complete = ocf_cache_io_complete;
|
|
result = ocf_metadata_passive_update(req);
|
|
if (result) {
|
|
ocf_cache_log(cache, log_crit,
|
|
"Metadata update error (error=%d)!\n", result);
|
|
}
|
|
|
|
// TODO why the result is not passed to io_cmpl???
|
|
ocf_cache_io_complete(req, 0);
|
|
}
|
|
|
|
static void ocf_cache_volume_submit_flush(ocf_io_t io)
|
|
{
|
|
struct ocf_request *req = ocf_io_to_req(io);
|
|
ocf_cache_t cache = req->cache;
|
|
|
|
if (!ocf_refcnt_inc(&cache->refcnt.metadata)) {
|
|
ocf_io_end_func(io, -OCF_ERR_IO);
|
|
return;
|
|
}
|
|
if (unlikely(!ocf_cache_is_standby(cache))) {
|
|
ocf_io_end_func(io, -OCF_ERR_CACHE_NOT_STANDBY);
|
|
return;
|
|
}
|
|
|
|
ocf_req_forward_cache_init(req, ocf_cache_volume_io_complete_generic);
|
|
ocf_req_forward_cache_flush(req);
|
|
}
|
|
|
|
|
|
static void ocf_cache_volume_submit_discard(ocf_io_t io)
|
|
{
|
|
struct ocf_request *req = ocf_io_to_req(io);
|
|
ocf_cache_t cache = req->cache;
|
|
|
|
if (!ocf_refcnt_inc(&cache->refcnt.metadata)) {
|
|
ocf_io_end_func(io, -OCF_ERR_IO);
|
|
return;
|
|
}
|
|
if (unlikely(!ocf_cache_is_standby(cache))) {
|
|
ocf_io_end_func(io, -OCF_ERR_CACHE_NOT_STANDBY);
|
|
return;
|
|
}
|
|
|
|
ocf_req_forward_cache_init(req, ocf_cache_volume_io_complete_generic);
|
|
ocf_req_forward_cache_discard(req, req->addr,
|
|
req->bytes);
|
|
}
|
|
|
|
/* *** VOLUME OPS *** */
|
|
|
|
static int ocf_cache_volume_open(ocf_volume_t volume, void *volume_params)
|
|
{
|
|
struct ocf_cache_volume *cache_volume = ocf_volume_get_priv(volume);
|
|
const struct ocf_volume_uuid *uuid = ocf_volume_get_uuid(volume);
|
|
ocf_cache_t cache = (ocf_cache_t)uuid->data;
|
|
|
|
cache_volume->cache = cache;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void ocf_cache_volume_close(ocf_volume_t volume)
|
|
{
|
|
struct ocf_cache_volume *cache_volume = ocf_volume_get_priv(volume);
|
|
|
|
cache_volume->cache = NULL;
|
|
}
|
|
|
|
static unsigned int ocf_cache_volume_get_max_io_size(ocf_volume_t volume)
|
|
{
|
|
ocf_cache_t cache = ocf_volume_to_cache(volume);
|
|
|
|
return ocf_volume_get_max_io_size(ocf_cache_get_volume(cache));
|
|
}
|
|
|
|
static uint64_t ocf_cache_volume_get_byte_length(ocf_volume_t volume)
|
|
{
|
|
ocf_cache_t cache = ocf_volume_to_cache(volume);
|
|
|
|
return ocf_volume_get_length(ocf_cache_get_volume(cache));
|
|
}
|
|
|
|
const struct ocf_volume_properties ocf_cache_volume_properties = {
|
|
.name = "OCF_Cache",
|
|
.volume_priv_size = sizeof(struct ocf_cache_volume),
|
|
.caps = {
|
|
.atomic_writes = 0,
|
|
},
|
|
.ops = {
|
|
.submit_io = ocf_cache_volume_submit_io,
|
|
.submit_flush = ocf_cache_volume_submit_flush,
|
|
.submit_discard = ocf_cache_volume_submit_discard,
|
|
.submit_metadata = NULL,
|
|
|
|
.open = ocf_cache_volume_open,
|
|
.close = ocf_cache_volume_close,
|
|
.get_max_io_size = ocf_cache_volume_get_max_io_size,
|
|
.get_length = ocf_cache_volume_get_byte_length,
|
|
},
|
|
.deinit = NULL,
|
|
};
|
|
|
|
static int ocf_cache_io_allocator_init(ocf_io_allocator_t allocator,
|
|
const char *name)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static void ocf_cache_io_allocator_deinit(ocf_io_allocator_t allocator)
|
|
{
|
|
}
|
|
|
|
static void *ocf_cache_io_allocator_new(ocf_io_allocator_t allocator,
|
|
ocf_volume_t volume, ocf_queue_t queue,
|
|
uint64_t addr, uint32_t bytes, uint32_t dir)
|
|
{
|
|
ocf_cache_t cache = ocf_volume_to_cache(volume);
|
|
|
|
return ocf_req_new_cache(cache, queue, addr, bytes, dir);
|
|
}
|
|
|
|
static void ocf_cache_io_allocator_del(ocf_io_allocator_t allocator, void *obj)
|
|
{
|
|
struct ocf_request *req = obj;
|
|
|
|
ocf_req_put(req);
|
|
}
|
|
|
|
const struct ocf_io_allocator_type ocf_cache_io_allocator_type = {
|
|
.ops = {
|
|
.allocator_init = ocf_cache_io_allocator_init,
|
|
.allocator_deinit = ocf_cache_io_allocator_deinit,
|
|
.allocator_new = ocf_cache_io_allocator_new,
|
|
.allocator_del = ocf_cache_io_allocator_del,
|
|
},
|
|
};
|
|
|
|
const struct ocf_volume_extended ocf_cache_volume_extended = {
|
|
.allocator_type = &ocf_cache_io_allocator_type,
|
|
};
|
|
|
|
int ocf_cache_volume_type_init(ocf_ctx_t ctx)
|
|
{
|
|
return ocf_ctx_register_volume_type_internal(ctx, OCF_VOLUME_TYPE_CACHE,
|
|
&ocf_cache_volume_properties,
|
|
&ocf_cache_volume_extended);
|
|
}
|
|
|
|
bool ocf_dbg_cache_is_settled(ocf_cache_t cache)
|
|
{
|
|
return ocf_refcnt_zeroed(&cache->refcnt.metadata) &&
|
|
ocf_refcnt_zeroed(&cache->refcnt.d2c);
|
|
}
|