From 7b7799bf9c5c4aa82652cabad09fbd58386159fd Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Tue, 13 Jun 2017 11:50:32 -0700 Subject: [PATCH] fio_plugin: switch to LD_PRELOAD instead of dynamically loading fio has a race between reap_threads() and free_ioengine(). free_ioengine() will call the ioengine's cleanup routine and then dlclose it if it is dynamically linked (like the spdk fio plugin). free_ioengine() does not set td->io_ops = NULL though until after dlclose() is complete. If reap_threads() tries to dereference td->io_ops after our plugin has been closed but before io_ops was set to NULL, it will segfault. Solution (until an upstream fio fix is available) is to use LD_PRELOAD instead. Signed-off-by: Jim Harris Change-Id: Ide4eb3cb92a636513289107fc211fdf1f98b616f Reviewed-on: https://review.gerrithub.io/365272 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Ben Walker Reviewed-by: Ziye Yang --- examples/nvme/fio_plugin/README.md | 10 +++++++--- examples/nvme/fio_plugin/example_config.fio | 2 +- examples/nvme/fio_plugin/fio_plugin.c | 12 +++++++++++- examples/nvme/fio_plugin/full_bench.fio | 2 +- test/lib/nvme/nvme.sh | 4 +++- test/nvmf/host/fio.sh | 4 +++- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/examples/nvme/fio_plugin/README.md b/examples/nvme/fio_plugin/README.md index 7d92fadc7..4827b35f7 100644 --- a/examples/nvme/fio_plugin/README.md +++ b/examples/nvme/fio_plugin/README.md @@ -27,10 +27,11 @@ as this README. Usage ------ -To use the SPDK fio plugin with fio, simply set the following in the fio configuration file -(see example_config.fio in the same directory as this README). +To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD when running +fio and set ioengine=spdk in the fio configuration file (see example_config.fio in the same +directory as this README). - ioengine= + LD_PRELOAD= fio To select NVMe devices, you simply pass an identifier as the filename in the format @@ -45,6 +46,9 @@ NVMe namespaces start at 1, not 0! And it should be put on the end. For example, Currently the SPDK fio plugin is limited to thread usage model, so fio jobs must also specify thread=1 when using the SPDK fio plugin. +fio also currently has a race condition on shutdown if dynamically loading the ioengine by specifying the +engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avoid this race condition. + When testing random workloads, it is recommended to set norandommap=1. fio's random map processing consumes extra CPU cycles which will degrade performance over time with the fio_plugin since all I/O are submitted and completed on a single CPU core. diff --git a/examples/nvme/fio_plugin/example_config.fio b/examples/nvme/fio_plugin/example_config.fio index 5b6a15304..a8e62ccb9 100644 --- a/examples/nvme/fio_plugin/example_config.fio +++ b/examples/nvme/fio_plugin/example_config.fio @@ -1,5 +1,5 @@ [global] -ioengine=./examples/nvme/fio_plugin/fio_plugin +ioengine=spdk thread=1 group_reporting=1 direct=1 diff --git a/examples/nvme/fio_plugin/fio_plugin.c b/examples/nvme/fio_plugin/fio_plugin.c index 2aa4506f9..f4bb8b6ea 100644 --- a/examples/nvme/fio_plugin/fio_plugin.c +++ b/examples/nvme/fio_plugin/fio_plugin.c @@ -486,7 +486,7 @@ static void spdk_fio_cleanup(struct thread_data *td) /* FIO imports this structure using dlsym */ struct ioengine_ops ioengine = { - .name = "spdk_fio", + .name = "spdk", .version = FIO_IOOPS_VERSION, .queue = spdk_fio_queue, .getevents = spdk_fio_getevents, @@ -502,3 +502,13 @@ struct ioengine_ops ioengine = { .io_u_free = spdk_fio_io_u_free, .flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN, }; + +static void fio_init fio_spdk_register(void) +{ + register_ioengine(&ioengine); +} + +static void fio_exit fio_spdk_unregister(void) +{ + unregister_ioengine(&ioengine); +} diff --git a/examples/nvme/fio_plugin/full_bench.fio b/examples/nvme/fio_plugin/full_bench.fio index af4fd3756..f74d23ae3 100644 --- a/examples/nvme/fio_plugin/full_bench.fio +++ b/examples/nvme/fio_plugin/full_bench.fio @@ -1,5 +1,5 @@ [global] -ioengine=./examples/nvme/fio_plugin/fio_plugin +ioengine=spdk thread=1 group_reporting=1 direct=1 diff --git a/test/lib/nvme/nvme.sh b/test/lib/nvme/nvme.sh index 1c8c7ac96..25c0361a5 100755 --- a/test/lib/nvme/nvme.sh +++ b/test/lib/nvme/nvme.sh @@ -45,10 +45,12 @@ timing_enter overhead $rootdir/test/lib/nvme/overhead/overhead -s 4096 -t 1 timing_exit overhead +PLUGIN_DIR=$rootdir/examples/nvme/fio_plugin + if [ -d /usr/src/fio ]; then timing_enter fio_plugin for bdf in $(linux_iter_pci 0108); do - /usr/src/fio/fio $rootdir/examples/nvme/fio_plugin/example_config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=1" + LD_PRELOAD=$PLUGIN_DIR/fio_plugin /usr/src/fio/fio $PLUGIN_DIR/example_config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=1" break done diff --git a/test/nvmf/host/fio.sh b/test/nvmf/host/fio.sh index 7a092f9e6..5924a2b8a 100755 --- a/test/nvmf/host/fio.sh +++ b/test/nvmf/host/fio.sh @@ -32,7 +32,9 @@ timing_exit start_nvmf_tgt $rpc_py construct_nvmf_subsystem Direct nqn.2016-06.io.spdk:cnode1 'transport:RDMA traddr:192.168.100.8 trsvcid:4420' '' -p "*" -/usr/src/fio/fio $rootdir/examples/nvme/fio_plugin/example_config.fio --filename="trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1" +PLUGIN_DIR=$rootdir/examples/nvme/fio_plugin + +LD_PRELOAD=$PLUGIN_DIR/fio_plugin /usr/src/fio/fio $PLUGIN_DIR/example_config.fio --filename="trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1" sync