accel: accel buffer allocation functions

The data buffers backed by these accel buffers aren't allocated
immediately, but only when they're necessary to execute a given
operation.  It allows users to append operations to a sequence, without
actually reserving large space for the data.  That way, if some of these
buffers aren't needed to execute a sequence, they won't be allocated.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Ieeea8a011b40c7f2f33e9a6f03fe34264e9316f3
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15746
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>

include/spdk/accel.h

@@ -373,6 +373,37 @@ void spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq);
  */
 void spdk_accel_sequence_abort(struct spdk_accel_sequence *seq);
 
+/**
+ * Allocate a buffer from accel domain.  These buffers can be only used with operations appended to
+ * a sequence.  The actual data buffer won't be allocated immediately, but only when it's necessary
+ * to execute a given operation.  In some cases, this might even mean that a data buffer won't be
+ * allocated at all, if a sequence can be executed without it.
+ *
+ * A buffer can only be a part of one sequence, but it can be used by multiple operations within
+ * that sequence.
+ *
+ * \param ch I/O channel.
+ * \param len Length of the buffer to allocate.
+ * \param buf Pointer to the allocated buffer.
+ * \param domain Memory domain in which the buffer is allocated.
+ * \param domain_ctx Memory domain context related to the allocated buffer.
+ *
+ * \return 0 if a buffer was successfully allocated, negative errno otherwise.
+ */
+int spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,
+		       struct spdk_memory_domain **domain, void **domain_ctx);
+
+/**
+ * Release a buffer allocated via `spdk_accel_get_buf()`.
+ *
+ * \param ch I/O channel.
+ * \param buf Buffer allocated via `spdk_accel_get_buf()`.
+ * \param domain Memory domain in which the buffer is allocated.
+ * \param domain_ctx Memory domain context related to the allocated buffer.
+ */
+void spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf,
+			struct spdk_memory_domain *domain, void *domain_ctx);
+
 /**
  * Return the name of the module assigned to a specific opcode.
  *

lib/accel/accel.c

@@ -30,6 +30,8 @@
 #define MAX_TASKS_PER_CHANNEL		0x800
 #define ACCEL_SMALL_CACHE_SIZE		128
 #define ACCEL_LARGE_CACHE_SIZE		16
+/* Set MSB, so we don't return NULL pointers as buffers */
+#define ACCEL_BUFFER_BASE		((void *)(1ull << 63))
 
 /* Largest context size for all accel modules */
 static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task);
@@ -53,12 +55,19 @@ static const char *g_opcode_strings[ACCEL_OPC_LAST] = {
 	"compress", "decompress"
 };
 
+struct accel_buffer {
+	uint64_t			len;
+	TAILQ_ENTRY(accel_buffer)	link;
+};
+
 struct accel_io_channel {
 	struct spdk_io_channel			*module_ch[ACCEL_OPC_LAST];
 	void					*task_pool_base;
 	struct spdk_accel_sequence		*seq_pool_base;
+	struct accel_buffer			*buf_pool_base;
 	TAILQ_HEAD(, spdk_accel_task)		task_pool;
 	TAILQ_HEAD(, spdk_accel_sequence)	seq_pool;
+	TAILQ_HEAD(, accel_buffer)		buf_pool;
 	struct spdk_iobuf_channel		iobuf;
 };
 
@@ -517,6 +526,28 @@ spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs,
 	return 0;
 }
 
+static inline struct accel_buffer *
+accel_get_buf(struct accel_io_channel *ch, uint64_t len)
+{
+	struct accel_buffer *buf;
+
+	buf = TAILQ_FIRST(&ch->buf_pool);
+	if (spdk_unlikely(buf == NULL)) {
+		return NULL;
+	}
+
+	TAILQ_REMOVE(&ch->buf_pool, buf, link);
+	buf->len = len;
+
+	return buf;
+}
+
+static inline void
+accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf)
+{
+	TAILQ_INSERT_HEAD(&ch->buf_pool, buf, link);
+}
+
 static inline struct spdk_accel_sequence *
 accel_sequence_get(struct accel_io_channel *ch)
 {
@@ -718,6 +749,39 @@ spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_c
 	return 0;
 }
 
+int
+spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,
+		   struct spdk_memory_domain **domain, void **domain_ctx)
+{
+	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
+	struct accel_buffer *accel_buf;
+
+	accel_buf = accel_get_buf(accel_ch, len);
+	if (spdk_unlikely(accel_buf == NULL)) {
+		return -ENOMEM;
+	}
+
+	/* We always return the same pointer and identify the buffers through domain_ctx */
+	*buf = ACCEL_BUFFER_BASE;
+	*domain_ctx = accel_buf;
+	*domain = g_accel_domain;
+
+	return 0;
+}
+
+void
+spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf,
+		   struct spdk_memory_domain *domain, void *domain_ctx)
+{
+	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
+	struct accel_buffer *accel_buf = domain_ctx;
+
+	assert(domain == g_accel_domain);
+	assert(buf == ACCEL_BUFFER_BASE);
+
+	accel_put_buf(accel_ch, accel_buf);
+}
+
 static void
 accel_sequence_complete_tasks(struct spdk_accel_sequence *seq)
 {
@@ -1004,6 +1068,7 @@ accel_create_channel(void *io_device, void *ctx_buf)
 	struct accel_io_channel	*accel_ch = ctx_buf;
 	struct spdk_accel_task *accel_task;
 	struct spdk_accel_sequence *seq;
+	struct accel_buffer *buf;
 	uint8_t *task_mem;
 	int i = 0, j, rc;
 
@@ -1017,14 +1082,22 @@ accel_create_channel(void *io_device, void *ctx_buf)
 		goto err;
 	}
 
+	accel_ch->buf_pool_base = calloc(MAX_TASKS_PER_CHANNEL, sizeof(struct accel_buffer));
+	if (accel_ch->buf_pool_base == NULL) {
+		goto err;
+	}
+
 	TAILQ_INIT(&accel_ch->task_pool);
 	TAILQ_INIT(&accel_ch->seq_pool);
+	TAILQ_INIT(&accel_ch->buf_pool);
 	task_mem = accel_ch->task_pool_base;
 	for (i = 0 ; i < MAX_TASKS_PER_CHANNEL; i++) {
 		accel_task = (struct spdk_accel_task *)task_mem;
 		seq = &accel_ch->seq_pool_base[i];
+		buf = &accel_ch->buf_pool_base[i];
 		TAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link);
 		TAILQ_INSERT_TAIL(&accel_ch->seq_pool, seq, link);
+		TAILQ_INSERT_TAIL(&accel_ch->buf_pool, buf, link);
 		task_mem += g_max_accel_module_size;
 	}
 
@@ -1051,6 +1124,7 @@ err:
 	}
 	free(accel_ch->task_pool_base);
 	free(accel_ch->seq_pool_base);
+	free(accel_ch->buf_pool_base);
 	return -ENOMEM;
 }
 
@@ -1071,6 +1145,7 @@ accel_destroy_channel(void *io_device, void *ctx_buf)
 
 	free(accel_ch->task_pool_base);
 	free(accel_ch->seq_pool_base);
+	free(accel_ch->buf_pool_base);
 }
 
 struct spdk_io_channel *

lib/accel/spdk_accel.map

@@ -24,6 +24,8 @@
 	spdk_accel_sequence_finish;
 	spdk_accel_sequence_abort;
 	spdk_accel_sequence_reverse;
+	spdk_accel_get_buf;
+	spdk_accel_put_buf;
 
 	# functions needed by modules
 	spdk_accel_module_list_add;