From 47c4304d83bea9b122aa659aede1407492bd8564 Mon Sep 17 00:00:00 2001 From: Tomasz Zawadzki Date: Tue, 23 Mar 2021 07:10:10 -0400 Subject: [PATCH] subsystem/nvmf: remove cpu_mask hint when creating poll group Passing cpu_mask hints that match only single core were usefull to prevent any accidents when doing round-robin in case of 'static' scheduler. In practice this is not required in case of 'static' scheduler, the threads will be spread out over all reactors anyway. This hinders other schedulers which try to respect the cpu_mask hints, as they would not move the thread to any reactor. Preventing bunching up less used threads on single reactor. Drawback of this patch is that poll group names will not match the cores they are on. Signed-off-by: Tomasz Zawadzki Change-Id: I5fb308362dd045228ea9fcca24f988388854c054 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7028 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Shuhei Matsumoto Reviewed-by: Ben Walker Reviewed-by: Reviewed-by: Aleksey Marchuk --- mk/spdk.lib_deps.mk | 2 +- module/event/subsystems/nvmf/nvmf_tgt.c | 6 +----- test/scheduler/idle.sh | 8 ++------ 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index ece8233ff..7b2912e06 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -166,7 +166,7 @@ DEPDIRS-event_vmd := event vmd $(JSON_LIBS) log thread DEPDIRS-event_bdev := event bdev event_accel event_vmd event_sock DEPDIRS-event_nbd := event nbd event_bdev -DEPDIRS-event_nvmf := event nvmf event_bdev event_sock $(BDEV_DEPS_THREAD) +DEPDIRS-event_nvmf := event nvmf event_bdev event_sock thread log bdev $(JSON_LIBS) DEPDIRS-event_scsi := event scsi event_bdev DEPDIRS-event_iscsi := event iscsi event_scsi event_sock diff --git a/module/event/subsystems/nvmf/nvmf_tgt.c b/module/event/subsystems/nvmf/nvmf_tgt.c index 5f5ea5dbd..4ac81f3e5 100644 --- a/module/event/subsystems/nvmf/nvmf_tgt.c +++ b/module/event/subsystems/nvmf/nvmf_tgt.c @@ -38,7 +38,6 @@ #include "spdk/log.h" #include "spdk/nvme.h" #include "spdk/nvmf_cmd.h" -#include "spdk/util.h" enum nvmf_tgt_state { NVMF_TGT_INIT_NONE = 0, @@ -186,7 +185,6 @@ nvmf_tgt_create_poll_group(void *ctx) static void nvmf_tgt_create_poll_groups(void) { - struct spdk_cpuset tmp_cpumask = {}; uint32_t i; char thread_name[32]; struct spdk_thread *thread; @@ -195,11 +193,9 @@ nvmf_tgt_create_poll_groups(void) assert(g_tgt_init_thread != NULL); SPDK_ENV_FOREACH_CORE(i) { - spdk_cpuset_zero(&tmp_cpumask); - spdk_cpuset_set_cpu(&tmp_cpumask, i, true); snprintf(thread_name, sizeof(thread_name), "nvmf_tgt_poll_group_%u", i); - thread = spdk_thread_create(thread_name, &tmp_cpumask); + thread = spdk_thread_create(thread_name, NULL); assert(thread != NULL); spdk_thread_send_msg(thread, nvmf_tgt_create_poll_group, NULL); diff --git a/test/scheduler/idle.sh b/test/scheduler/idle.sh index 23f4d2852..830580241 100755 --- a/test/scheduler/idle.sh +++ b/test/scheduler/idle.sh @@ -38,13 +38,10 @@ idle() { # The expectation here is that when SPDK app is idle the following is true: # - all threads are assigned to main lcore # - threads are not being moved between lcores - # - each thread has a mask pinned to a single cpu - - local all_set xtrace_disable while ((samples++ < 5)); do - all_set=0 cpusmask=0 + cpusmask=0 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') threads=($( jq -r "select(.lcore == $spdk_main_core) | .lw_threads[].name" <<< "$reactor_framework" @@ -56,10 +53,9 @@ idle() { done printf 'SPDK cpumask: %x Threads cpumask: %x\n' "$spdk_cpusmask" "$cpusmask" - thread_stats && ((cpusmask == spdk_cpusmask)) && all_set=1 + thread_stats done - ((all_set == 1)) xtrace_restore }