util: add spdk_thread and spdk_io_channel

This patch adds a basic framework for creating I/O channels
for I/O devices.  An spdk_io_channel represents a one-to-one
mapping between a calling thread (represented by spdk_thread)
and an I/O device that the thread will perform I/O operations
on.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I658ab7f995cc962f4e2a204e058cdd3ad3fd735d
This commit is contained in:
Jim Harris 2016-09-15 12:27:10 -07:00
parent 884d076451
commit ee66e58e0a
7 changed files with 576 additions and 2 deletions

117
include/spdk/io_channel.h Normal file
View File

@ -0,0 +1,117 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* IO channel
*/
#ifndef SPDK_IO_CHANNEL_H_
#define SPDK_IO_CHANNEL_H_
#include <inttypes.h>
#include "spdk/queue.h"
#define SPDK_IO_PRIORITY_DEFAULT 100
struct spdk_io_channel;
typedef int (*io_channel_create_cb_t)(void *io_device, uint32_t priority, void *ctx_buf);
typedef void (*io_channel_destroy_cb_t)(void *io_device, void *ctx_buf);
/**
* \brief Initializes the calling thread for I/O channel allocation.
*/
void spdk_allocate_thread(void);
/**
* \brief Releases any resources related to the calling thread for I/O channel allocation.
*
* All I/O channel references related to the calling thread must be released using
* spdk_put_io_channel() prior to calling this function.
*/
void spdk_free_thread(void);
/**
* \brief Register the opaque io_device context as an I/O device.
*
* After an I/O device is registered, it can return I/O channels using the
* spdk_get_io_channel() function. create_cb is the callback function invoked
* to allocate any resources required for a new I/O channel. destroy_cb is the
* callback function invoked to release the resources for an I/O channel. ctx_size
* is the size of the context buffer allocated to store references to allocated I/O
* channel resources.
*/
void spdk_io_device_register(void *io_device, io_channel_create_cb_t create_cb,
io_channel_destroy_cb_t destroy_cb, uint32_t ctx_size);
/**
* \brief Unregister the opaque io_device context as an I/O device.
*
* Callers must ensure they release references to any I/O channel related to this
* device before calling this function.
*/
void spdk_io_device_unregister(void *io_device);
/**
* \brief Gets an I/O channel for the specified io_device to be used by the calling thread.
*
* The io_device context pointer specified must have previously been registered using
* spdk_io_device_register(). If an existing I/O channel does not exist yet for the given
* io_device on the calling thread, it will allocate an I/O channel and invoke the create_cb
* function pointer specified in spdk_io_device_register(). If an I/O channel already
* exists for the given io_device on the calling thread, its reference is returned rather
* than creating a new I/O channel.
*
* The priority parameter allows callers to create different I/O channels to the same
* I/O device with varying priorities. Currently this value must be set to
* SPDK_IO_PRIORITY_DEFAULT.
*/
struct spdk_io_channel *spdk_get_io_channel(void *io_device, uint32_t priority);
/**
* \brief Releases a reference to an I/O channel.
*
* Must be called from the same thread that called spdk_get_io_channel() for the specified
* I/O channel. If this releases the last reference to the I/O channel, The destroy_cb
* function specified in spdk_io_device_register() will be invoked to release any
* associated resources.
*/
void spdk_put_io_channel(struct spdk_io_channel *ch);
/**
* \brief Returns the context buffer associated with an I/O channel.
*/
void *spdk_io_channel_get_ctx(struct spdk_io_channel *ch);
#endif /* SPDK_IO_CHANNEL_H_ */

View File

@ -35,7 +35,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
CFLAGS += $(DPDK_INC)
C_SRCS = bit_array.c fd.c string.c pci.c
C_SRCS = bit_array.c fd.c io_channel.c string.c pci.c
LIBNAME = util
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

210
lib/util/io_channel.c Normal file
View File

@ -0,0 +1,210 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdbool.h>
#include <pthread.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "spdk/io_channel.h"
#include "spdk/log.h"
static pthread_mutex_t g_devlist_mutex = PTHREAD_MUTEX_INITIALIZER;
struct io_device {
void *io_device_ctx;
io_channel_create_cb_t create_cb;
io_channel_destroy_cb_t destroy_cb;
uint32_t ctx_size;
TAILQ_ENTRY(io_device) tailq;
};
static TAILQ_HEAD(, io_device) g_io_devices = TAILQ_HEAD_INITIALIZER(g_io_devices);
struct spdk_io_channel {
pthread_t thread_id;
void *io_device;
uint32_t ref;
uint32_t priority;
TAILQ_ENTRY(spdk_io_channel) tailq;
io_channel_destroy_cb_t destroy_cb;
/*
* Modules will allocate extra memory off the end of this structure
* to store references to hardware-specific references (i.e. NVMe queue
* pairs, or references to child device spdk_io_channels (i.e.
* virtual bdevs).
*/
};
static __thread TAILQ_HEAD(, spdk_io_channel) g_io_channels;
void
spdk_allocate_thread(void)
{
TAILQ_INIT(&g_io_channels);
}
void
spdk_free_thread(void)
{
assert(TAILQ_EMPTY(&g_io_channels));
}
void
spdk_io_device_register(void *io_device, io_channel_create_cb_t create_cb,
io_channel_destroy_cb_t destroy_cb, uint32_t ctx_size)
{
struct io_device *dev, *tmp;
dev = calloc(1, sizeof(struct io_device));
if (dev == NULL) {
SPDK_ERRLOG("could not allocate io_device\n");
return;
}
dev->io_device_ctx = io_device;
dev->create_cb = create_cb;
dev->destroy_cb = destroy_cb;
dev->ctx_size = ctx_size;
pthread_mutex_lock(&g_devlist_mutex);
TAILQ_FOREACH(tmp, &g_io_devices, tailq) {
if (tmp->io_device_ctx == io_device) {
SPDK_ERRLOG("io_device %p already registered\n", io_device);
free(dev);
pthread_mutex_unlock(&g_devlist_mutex);
return;
}
}
TAILQ_INSERT_TAIL(&g_io_devices, dev, tailq);
pthread_mutex_unlock(&g_devlist_mutex);
}
void
spdk_io_device_unregister(void *io_device)
{
struct io_device *dev;
pthread_mutex_lock(&g_devlist_mutex);
TAILQ_FOREACH(dev, &g_io_devices, tailq) {
if (dev->io_device_ctx == io_device) {
TAILQ_REMOVE(&g_io_devices, dev, tailq);
pthread_mutex_unlock(&g_devlist_mutex);
return;
}
}
SPDK_ERRLOG("io_device %p not found\n", io_device);
pthread_mutex_unlock(&g_devlist_mutex);
}
struct spdk_io_channel *
spdk_get_io_channel(void *io_device, uint32_t priority)
{
struct spdk_io_channel *ch;
struct io_device *dev;
int rc;
if (priority != SPDK_IO_PRIORITY_DEFAULT) {
SPDK_ERRLOG("priority must be set to SPDK_IO_PRIORITY_DEFAULT\n");
return NULL;
}
pthread_mutex_lock(&g_devlist_mutex);
TAILQ_FOREACH(dev, &g_io_devices, tailq) {
if (dev->io_device_ctx == io_device) {
break;
}
}
if (dev == NULL) {
SPDK_ERRLOG("could not find io_device %p\n", io_device);
pthread_mutex_unlock(&g_devlist_mutex);
return NULL;
}
pthread_mutex_unlock(&g_devlist_mutex);
TAILQ_FOREACH(ch, &g_io_channels, tailq) {
if (ch->io_device == io_device && ch->priority == priority) {
ch->ref++;
/*
* An I/O channel already exists for this device on this
* thread, so return it.
*/
return ch;
}
}
ch = calloc(1, sizeof(*ch) + dev->ctx_size);
if (ch == NULL) {
SPDK_ERRLOG("could not calloc spdk_io_channel\n");
return NULL;
}
rc = dev->create_cb(io_device, priority, (uint8_t *)ch + sizeof(*ch));
if (rc == -1) {
free(ch);
return NULL;
}
ch->io_device = io_device;
ch->destroy_cb = dev->destroy_cb;
ch->thread_id = pthread_self();
ch->priority = priority;
ch->ref = 1;
TAILQ_INSERT_TAIL(&g_io_channels, ch, tailq);
return ch;
}
void
spdk_put_io_channel(struct spdk_io_channel *ch)
{
if (ch->ref == 0) {
SPDK_ERRLOG("ref already zero\n");
return;
}
ch->ref--;
if (ch->ref == 0) {
TAILQ_REMOVE(&g_io_channels, ch, tailq);
ch->destroy_cb(ch->io_device, (uint8_t *)ch + sizeof(*ch));
free(ch);
}
}
void *
spdk_io_channel_get_ctx(struct spdk_io_channel *ch)
{
return (uint8_t *)ch + sizeof(*ch);
}

View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = bit_array
DIRS-y = bit_array io_channel
.PHONY: all clean $(DIRS-y)

1
test/lib/util/io_channel/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
io_channel_ut

View File

@ -0,0 +1,58 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
$(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
$(SPDK_ROOT_DIR)/lib/cunit/libspdk_cunit.a
CFLAGS += $(DPDK_INC)
CFLAGS += -I$(SPDK_ROOT_DIR)/test
CFLAGS += -I$(SPDK_ROOT_DIR)/lib
LIBS += $(SPDK_LIBS)
LIBS += -lcunit
APP = io_channel_ut
C_SRCS = $(APP).c
all: $(APP)
$(APP): $(OBJS) $(SPDK_LIBS)
$(LINK_C)
clean:
$(CLEAN_C) $(APP)
include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

View File

@ -0,0 +1,188 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "spdk_cunit.h"
#include "util/io_channel.c"
static void
thread_alloc(void)
{
spdk_allocate_thread();
spdk_free_thread();
}
static uint64_t device1;
static uint64_t device2;
static uint64_t device3;
static uint64_t ctx1 = 0x1111;
static uint64_t ctx2 = 0x2222;
static int g_create_cb_calls = 0;
static int g_destroy_cb_calls = 0;
static int
create_cb_1(void *io_device, uint32_t priority, void *ctx_buf)
{
CU_ASSERT(io_device == &device1);
CU_ASSERT(priority == SPDK_IO_PRIORITY_DEFAULT);
*(uint64_t *)ctx_buf = ctx1;
g_create_cb_calls++;
return 0;
}
static void
destroy_cb_1(void *io_device, void *ctx_buf)
{
CU_ASSERT(io_device == &device1);
CU_ASSERT(*(uint64_t *)ctx_buf == ctx1);
g_destroy_cb_calls++;
}
static int
create_cb_2(void *io_device, uint32_t priority, void *ctx_buf)
{
CU_ASSERT(io_device == &device2);
CU_ASSERT(priority == SPDK_IO_PRIORITY_DEFAULT);
*(uint64_t *)ctx_buf = ctx2;
g_create_cb_calls++;
return 0;
}
static void
destroy_cb_2(void *io_device, void *ctx_buf)
{
CU_ASSERT(io_device == &device2);
CU_ASSERT(*(uint64_t *)ctx_buf == ctx2);
g_destroy_cb_calls++;
}
static int
create_cb_null(void *io_device, uint32_t priority, void *ctx_buf)
{
return -1;
}
static void
channel(void)
{
struct spdk_io_channel *ch1, *ch2;
void *ctx;
spdk_allocate_thread();
spdk_io_device_register(&device1, create_cb_1, destroy_cb_1, sizeof(ctx1));
spdk_io_device_register(&device2, create_cb_2, destroy_cb_2, sizeof(ctx2));
spdk_io_device_register(&device3, create_cb_null, NULL, 0);
g_create_cb_calls = 0;
ch1 = spdk_get_io_channel(&device1, SPDK_IO_PRIORITY_DEFAULT);
CU_ASSERT(g_create_cb_calls == 1);
CU_ASSERT(ch1 != NULL);
g_create_cb_calls = 0;
ch2 = spdk_get_io_channel(&device1, SPDK_IO_PRIORITY_DEFAULT);
CU_ASSERT(g_create_cb_calls == 0);
CU_ASSERT(ch1 == ch2);
CU_ASSERT(ch2 != NULL);
g_destroy_cb_calls = 0;
spdk_put_io_channel(ch2);
CU_ASSERT(g_destroy_cb_calls == 0);
g_create_cb_calls = 0;
ch2 = spdk_get_io_channel(&device2, SPDK_IO_PRIORITY_DEFAULT);
CU_ASSERT(g_create_cb_calls == 1);
CU_ASSERT(ch1 != ch2);
CU_ASSERT(ch2 != NULL);
ctx = spdk_io_channel_get_ctx(ch2);
CU_ASSERT(*(uint64_t *)ctx == ctx2);
g_destroy_cb_calls = 0;
spdk_put_io_channel(ch1);
CU_ASSERT(g_destroy_cb_calls == 1);
g_destroy_cb_calls = 0;
spdk_put_io_channel(ch2);
CU_ASSERT(g_destroy_cb_calls == 1);
ch1 = spdk_get_io_channel(&device3, SPDK_IO_PRIORITY_DEFAULT);
CU_ASSERT(ch1 == NULL);
/* Confirm failure if user specifies an invalid I/O priority. */
ch1 = spdk_get_io_channel(&device1, SPDK_IO_PRIORITY_DEFAULT + 1);
CU_ASSERT(ch1 == NULL);
spdk_io_device_unregister(&device1);
spdk_io_device_unregister(&device2);
spdk_io_device_unregister(&device3);
CU_ASSERT(TAILQ_EMPTY(&g_io_devices));
CU_ASSERT(TAILQ_EMPTY(&g_io_channels));
spdk_free_thread();
}
int
main(int argc, char **argv)
{
CU_pSuite suite = NULL;
unsigned int num_failures;
if (CU_initialize_registry() != CUE_SUCCESS) {
return CU_get_error();
}
suite = CU_add_suite("io_channel", NULL, NULL);
if (suite == NULL) {
CU_cleanup_registry();
return CU_get_error();
}
if (
CU_add_test(suite, "thread_alloc", thread_alloc) == NULL ||
CU_add_test(suite, "channel", channel) == NULL
) {
CU_cleanup_registry();
return CU_get_error();
}
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();
num_failures = CU_get_number_of_failures();
CU_cleanup_registry();
return num_failures;
}