ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fio_plugin: switch to LD_PRELOAD instead of dynamically loading

Browse Source 
fio has a race between reap_threads() and free_ioengine().  free_ioengine()
will call the ioengine's cleanup routine and then dlclose it if it
is dynamically linked (like the spdk fio plugin).  free_ioengine() does
not set td->io_ops = NULL though until after dlclose() is complete.  If
reap_threads() tries to dereference td->io_ops after our plugin has been
closed but before io_ops was set to NULL, it will segfault.

Solution (until an upstream fio fix is available) is to use LD_PRELOAD
instead.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: Ide4eb3cb92a636513289107fc211fdf1f98b616f

Reviewed-on: https://review.gerrithub.io/365272
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Ziye Yang <optimistyzy@gmail.com>
This commit is contained in:
Jim Harris 2017-06-13 11:50:32 -07:00
parent c7e5d9f1b7
commit 7b7799bf9c
6 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
examples/nvme/fio_plugin/README.md
View File

@ -27,10 +27,11 @@ as this README.
Usage Usage
------ ------
To use the SPDK fio plugin with fio, simply set the following in the fio configuration file To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD when running
(see example_config.fio in the same directory as this README). fio and set ioengine=spdk in the fio configuration file (see example_config.fio in the same
directory as this README).
ioengine=<path to fio_plugin binary> LD_PRELOAD=<path to fio_plugin binary> fio
To select NVMe devices, you simply pass an identifier as the filename in the format To select NVMe devices, you simply pass an identifier as the filename in the format
@ -45,6 +46,9 @@ NVMe namespaces start at 1, not 0! And it should be put on the end. For example,
Currently the SPDK fio plugin is limited to thread usage model, so fio jobs must also specify thread=1 Currently the SPDK fio plugin is limited to thread usage model, so fio jobs must also specify thread=1
when using the SPDK fio plugin. when using the SPDK fio plugin.
fio also currently has a race condition on shutdown if dynamically loading the ioengine by specifying the
engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avoid this race condition.
When testing random workloads, it is recommended to set norandommap=1. fio's random map When testing random workloads, it is recommended to set norandommap=1. fio's random map
processing consumes extra CPU cycles which will degrade performance over time with processing consumes extra CPU cycles which will degrade performance over time with
the fio_plugin since all I/O are submitted and completed on a single CPU core. the fio_plugin since all I/O are submitted and completed on a single CPU core.

2
examples/nvme/fio_plugin/example_config.fio
View File

@ -1,5 +1,5 @@
[global] [global]
ioengine=./examples/nvme/fio_plugin/fio_plugin ioengine=spdk
thread=1 thread=1
group_reporting=1 group_reporting=1
direct=1 direct=1

12
examples/nvme/fio_plugin/fio_plugin.c
View File

@ -486,7 +486,7 @@ static void spdk_fio_cleanup(struct thread_data *td)
/* FIO imports this structure using dlsym */ /* FIO imports this structure using dlsym */
struct ioengine_ops ioengine = { struct ioengine_ops ioengine = {
.name = "spdk_fio", .name = "spdk",
.version = FIO_IOOPS_VERSION, .version = FIO_IOOPS_VERSION,
.queue = spdk_fio_queue, .queue = spdk_fio_queue,
.getevents = spdk_fio_getevents, .getevents = spdk_fio_getevents,
@ -502,3 +502,13 @@ struct ioengine_ops ioengine = {
.io_u_free = spdk_fio_io_u_free, .io_u_free = spdk_fio_io_u_free,
.flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN, .flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN,
}; };
static void fio_init fio_spdk_register(void)
{
register_ioengine(&ioengine);
}
static void fio_exit fio_spdk_unregister(void)
{
unregister_ioengine(&ioengine);
}

2
examples/nvme/fio_plugin/full_bench.fio
View File

@ -1,5 +1,5 @@
[global] [global]
ioengine=./examples/nvme/fio_plugin/fio_plugin ioengine=spdk
thread=1 thread=1
group_reporting=1 group_reporting=1
direct=1 direct=1

4
test/lib/nvme/nvme.sh
View File

@ -45,10 +45,12 @@ timing_enter overhead
$rootdir/test/lib/nvme/overhead/overhead -s 4096 -t 1 $rootdir/test/lib/nvme/overhead/overhead -s 4096 -t 1
timing_exit overhead timing_exit overhead
PLUGIN_DIR=$rootdir/examples/nvme/fio_plugin
if [ -d /usr/src/fio ]; then if [ -d /usr/src/fio ]; then
timing_enter fio_plugin timing_enter fio_plugin
for bdf in $(linux_iter_pci 0108); do for bdf in $(linux_iter_pci 0108); do
/usr/src/fio/fio $rootdir/examples/nvme/fio_plugin/example_config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=1" LD_PRELOAD=$PLUGIN_DIR/fio_plugin /usr/src/fio/fio $PLUGIN_DIR/example_config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=1"
break break
done done

4
test/nvmf/host/fio.sh
View File

@ -32,7 +32,9 @@ timing_exit start_nvmf_tgt
$rpc_py construct_nvmf_subsystem Direct nqn.2016-06.io.spdk:cnode1 'transport:RDMA traddr:192.168.100.8 trsvcid:4420' '' -p "*" $rpc_py construct_nvmf_subsystem Direct nqn.2016-06.io.spdk:cnode1 'transport:RDMA traddr:192.168.100.8 trsvcid:4420' '' -p "*"
/usr/src/fio/fio $rootdir/examples/nvme/fio_plugin/example_config.fio --filename="trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1" PLUGIN_DIR=$rootdir/examples/nvme/fio_plugin
LD_PRELOAD=$PLUGIN_DIR/fio_plugin /usr/src/fio/fio $PLUGIN_DIR/example_config.fio --filename="trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1"
sync sync