Dynamic I/O queue management

- Queue allocation is now separated from starting cache.
- Queue can be created and destroyed in runtime.
- All queue ops accept queue handle instead of queue id.
- Cache stores queues as list instead of array.

Signed-off-by: Michal Mielewczyk <michal.mielewczyk@intel.com>
Signed-off-by: Robert Baldyga <robert.baldyga@intel.com>

inc/ocf_cache.h

@@ -109,18 +109,6 @@ ocf_volume_t ocf_cache_get_volume(ocf_cache_t cache);
  */
 ocf_cache_id_t ocf_cache_get_id(ocf_cache_t cache);
 
-/**
- * @brief Get queue object associated with cache
- *
- * @param[in] cache Cache object
- * @param[in] id Queue id
- * @param[out] q Queue object
- *
- * @retval 0 Success
- * @retval Non-zero Fail
- */
-int ocf_cache_get_queue(ocf_cache_t cache, unsigned id, ocf_queue_t *q);
-
 /**
  * @brief Set name of given cache object
  *

inc/ocf_cleaner.h

@@ -34,16 +34,17 @@ void ocf_cleaner_set_cmpl(ocf_cleaner_t cleaner, ocf_cleaner_end_t fn);
  * @brief Set cleaner queue
  *
  * @param[in] cleaner Cleaner instance
- * @param[in] io_queue Queue number
+ * @param[in] io_queue Queue handle
  */
-void ocf_cleaner_set_io_queue(ocf_cleaner_t cleaner, uint32_t io_queue);
+void ocf_cleaner_set_io_queue(ocf_cleaner_t cleaner, ocf_queue_t io_queue);
 
 /**
  * @brief Run cleaner
  *
  * @param[in] c Cleaner instance to run
+ * @param[in] queue IO queue handle
  */
-void ocf_cleaner_run(ocf_cleaner_t c);
+void ocf_cleaner_run(ocf_cleaner_t c, ocf_queue_t queue);
 
 /**
  * @brief Set cleaner private data

inc/ocf_ctx.h

@@ -128,54 +128,6 @@ struct ocf_data_ops {
 	void (*secure_erase)(ctx_data_t *dst);
 };
 
-/**
- * @brief I/O queue operations
- */
-struct ocf_queue_ops {
-	/**
-	 * @brief Initialize I/O queue.
-	 *
-	 * This function should create worker, thread or any other queue
-	 * processing related stuff specific to given environment.
-	 *
-	 * @param[in] q I/O queue to be initialized
-	 *
-	 * @retval 0 I/O queue has been initializaed successfully
-	 * @retval Non-zero I/O queue initialization failure
-	 */
-	int (*init)(ocf_queue_t q);
-
-	/**
-	 * @brief Kick I/O queue processing
-	 *
-	 * This function should inform worker, thread or any other queue
-	 * processing mechanism, that there are new requests in queue to
-	 * be processed. Processing requests inside current call is not allowed.
-	 *
-	 * @param[in] q I/O queue to be kicked
-	 */
-	void (*kick)(ocf_queue_t q);
-
-	/**
-	 * @brief Kick I/O queue processing
-	 *
-	 * This function should inform worker, thread or any other queue
-	 * processing mechanism, that there are new requests in queue to
-	 * be processed. Kick function is allowed to process requests in current
-	 * call
-	 *
-	 * @param[in] q I/O queue to be kicked
-	 */
-	void (*kick_sync)(ocf_queue_t q);
-
-	/**
-	 * @brief Stop I/O queue
-	 *
-	 * @param[in] q I/O queue beeing stopped
-	 */
-	void (*stop)(ocf_queue_t q);
-};
-
 /**
  * @brief Cleaner operations
  */
@@ -243,9 +195,6 @@ struct ocf_ctx_ops {
 	/* Context data operations */
 	struct ocf_data_ops data;
 
-	/* Queue operations */
-	struct ocf_queue_ops queue;
-
 	/* Cleaner operations */
 	struct ocf_cleaner_ops cleaner;
 

inc/ocf_io.h

@@ -84,9 +84,9 @@ struct ocf_io {
 	uint32_t dir;
 
 	/**
-	 * @brief Queue id
+	 * @brief Queue handle
 	 */
-	uint32_t io_queue;
+	ocf_queue_t io_queue;
 
 	/**
 	 * @brief OCF IO start function
@@ -258,12 +258,12 @@ static inline ctx_data_t *ocf_io_get_data(struct ocf_io *io)
 }
 
 /**
- * @brief Set queue id to which IO should be submitted
+ * @brief Set queue to which IO should be submitted
  *
  * @param[in] io OCF IO to set up
- * @param[in] queue IO queue id
+ * @param[in] queue IO queue handle
  */
-static inline void ocf_io_set_queue(struct ocf_io *io, uint32_t queue)
+static inline void ocf_io_set_queue(struct ocf_io *io, ocf_queue_t queue)
 {
 	io->io_queue = queue;
 }

inc/ocf_mngt.h

@@ -240,11 +240,6 @@ struct ocf_mngt_cache_config {
 		 uint32_t queue_unblock_size;
 	} backfill;
 
-	/**
-	 * @brief Number of I/O queues to be created
-	 */
-	uint32_t io_queues;
-
 	/**
 	 * @brief Start cache and keep it locked
 	 *
@@ -360,14 +355,12 @@ int ocf_mngt_cache_detach(ocf_cache_t cache);
  *
  * @param[in] ctx OCF context
  * @param[out] cache Cache handle
- * @param[in] cfg Cache configuration
  * @param[in] device_cfg Caching device configuration
  *
  * @retval 0 Cache successfully loaded
  * @retval Non-zero Error occurred during loading cache
  */
 int ocf_mngt_cache_load(ocf_ctx_t ctx, ocf_cache_t *cache,
-		struct ocf_mngt_cache_config *cfg,
 		struct ocf_mngt_cache_device_config *device_cfg);
 
 /* Adding and removing cores */

inc/ocf_queue.h

@@ -11,6 +11,72 @@
  * @brief OCF queues API
  */
 
+/**
+ * @brief I/O queue operations
+ */
+struct ocf_queue_ops {
+	/**
+	 * @brief Kick I/O queue processing
+	 *
+	 * This function should inform worker, thread or any other queue
+	 * processing mechanism, that there are new requests in queue to
+	 * be processed. Processing requests synchronously in this function
+	 * is not allowed.
+	 *
+	 * @param[in] q I/O queue to be kicked
+	 */
+	void (*kick)(ocf_queue_t q);
+
+	/**
+	 * @brief Kick I/O queue processing
+	 *
+	 * This function should inform worker, thread or any other queue
+	 * processing mechanism, that there are new requests in queue to
+	 * be processed. Function kick_sync is allowed to process requests
+	 * synchronously without delegating them to the worker.
+	 *
+	 * @param[in] q I/O queue to be kicked
+	 */
+	void (*kick_sync)(ocf_queue_t q);
+
+	/**
+	 * @brief Stop I/O queue
+	 *
+	 * @param[in] q I/O queue beeing stopped
+	 */
+	void (*stop)(ocf_queue_t q);
+};
+
+/**
+ * @brief Allocate IO queue and add it to list in cache
+ *
+ * @param[in] cache Handle to cache instance
+ * @param[out] queue Handle to created queue
+ * @param[in] ops Queue operations
+ *
+ * @return Zero on success, otherwise error code
+ */
+int ocf_queue_create(ocf_cache_t cache, ocf_queue_t *queue,
+		const struct ocf_queue_ops *ops);
+
+/**
+ * @brief Increase reference counter in queue
+ *
+ * @param[in] queue Queue
+ *
+ */
+void ocf_queue_get(ocf_queue_t queue);
+
+/**
+ * @brief Decrease reference counter in queue
+ *
+ * @note If queue don't have any reference - deallocate it
+ *
+ * @param[in] queue Queue
+ *
+ */
+void ocf_queue_put(ocf_queue_t queue);
+
 /**
  * @brief Process single request from queue
  *
@@ -60,13 +126,4 @@ uint32_t ocf_queue_pending_io(ocf_queue_t q);
  */
 ocf_cache_t ocf_queue_get_cache(ocf_queue_t q);
 
-/**
- * @brief Get I/O queue id
- *
- * @param[in] q I/O queue
- *
- * @retval I/O queue id
- */
-uint32_t ocf_queue_get_id(ocf_queue_t q);
-
 #endif

inc/ocf_trace.h

@@ -74,9 +74,6 @@ struct ocf_event_cache_desc {
 	/** Number of cores */
 	uint32_t cores_no;
 
-	/** Number of IO queues */
-	uint32_t io_queues_no;
-
 	/** Trace version */
 	uint32_t version;
 };
@@ -152,7 +149,7 @@ struct ocf_event_io_cmpl {
  *
  * @param[in] cache OCF cache
  * @param[in] trace_ctx Tracing context
- * @param[in] qid Queue Id
+ * @param[in] queue Queue handle
  * @param[out] trace Event log
  * @param[out] size Size of event log
  *
@@ -160,7 +157,7 @@ struct ocf_event_io_cmpl {
  * @return Non-zero error
  */
 typedef void (*ocf_trace_callback_t)(ocf_cache_t cache, void *trace_ctx,
-		uint32_t qid, const void* trace, const uint32_t size);
+		ocf_queue_t queue, const void* trace, const uint32_t size);
 
 /**
  * @brief Start tracing