From 777bd353d22f044e4bec1b79bd0ca98994093b01 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Mon, 25 May 2020 07:15:57 +0900 Subject: [PATCH] example/nvmf: Add periodic round robin poll group migration Add simple periodic round robin poll group migration to the example application to exercise SPDK thread dynamic scheduling. Add CLI option to enable the feature manually. Default setting is disabled, and the period is specified as micro second. Signed-off-by: Shuhei Matsumoto Change-Id: I4639e94ce00611a600ff068e754ee6f31905565c Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2556 Community-CI: Mellanox Build Bot Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Aleksey Marchuk --- examples/nvmf/nvmf/nvmf.c | 48 +++++++++++++++++++++++++++++++- test/nvmf/target/nvmf_example.sh | 4 +-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/examples/nvmf/nvmf/nvmf.c b/examples/nvmf/nvmf/nvmf.c index 3d5a13cce..8c0a748dd 100644 --- a/examples/nvmf/nvmf/nvmf.c +++ b/examples/nvmf/nvmf/nvmf.c @@ -106,6 +106,9 @@ static bool g_reactors_exit = false; static enum nvmf_target_state g_target_state; static bool g_intr_received = false; +static uint32_t g_migrate_pg_period_us = 0; +static struct spdk_poller *g_migrate_pg_poller = NULL; + static void nvmf_target_advance_state(void); static int nvmf_schedule_spdk_thread(struct spdk_thread *thread); @@ -114,6 +117,7 @@ usage(char *program_name) { printf("%s options", program_name); printf("\n"); + printf("\t[-g period of round robin poll group migration (us) (default: 0 (disabled))]\n"); printf("\t[-h show this usage]\n"); printf("\t[-i shared memory ID (optional)]\n"); printf("\t[-m core mask for DPDK]\n"); @@ -130,8 +134,16 @@ parse_args(int argc, char **argv, struct spdk_env_opts *opts) int op; long int value; - while ((op = getopt(argc, argv, "i:m:n:p:r:s:u:h")) != -1) { + while ((op = getopt(argc, argv, "g:i:m:n:p:r:s:u:h")) != -1) { switch (op) { + case 'g': + value = spdk_strtol(optarg, 10); + if (value < 0) { + fprintf(stderr, "converting a string to integer failed\n"); + return -EINVAL; + } + g_migrate_pg_period_us = value; + break; case 'i': value = spdk_strtol(optarg, 10); if (value < 0) { @@ -773,6 +785,35 @@ nvmf_subsystem_init_done(int rc, void *cb_arg) nvmf_target_advance_state(); } +static void +migrate_poll_group_by_rr(void *ctx) +{ + uint32_t current_core, next_core; + struct spdk_cpuset cpumask = {}; + + current_core = spdk_env_get_current_core(); + next_core = spdk_env_get_next_core(current_core); + if (next_core == UINT32_MAX) { + next_core = spdk_env_get_first_core(); + } + + spdk_cpuset_set_cpu(&cpumask, next_core, true); + + spdk_thread_set_cpumask(&cpumask); +} + +static int +migrate_poll_groups_by_rr(void *ctx) +{ + struct nvmf_target_poll_group *pg; + + TAILQ_FOREACH(pg, &g_poll_groups, link) { + spdk_thread_send_msg(pg->thread, migrate_poll_group_by_rr, NULL); + } + + return 1; +} + static void nvmf_target_advance_state(void) { @@ -803,8 +844,13 @@ nvmf_target_advance_state(void) break; case NVMF_RUNNING: fprintf(stdout, "nvmf target is running\n"); + if (g_migrate_pg_period_us != 0) { + g_migrate_pg_poller = SPDK_POLLER_REGISTER(migrate_poll_groups_by_rr, NULL, + g_migrate_pg_period_us); + } break; case NVMF_FINI_STOP_SUBSYSTEMS: + spdk_poller_unregister(&g_migrate_pg_poller); nvmf_tgt_stop_subsystems(&g_nvmf_tgt); break; case NVMF_FINI_POLL_GROUPS: diff --git a/test/nvmf/target/nvmf_example.sh b/test/nvmf/target/nvmf_example.sh index 0a3f05f05..3cb5aa86f 100755 --- a/test/nvmf/target/nvmf_example.sh +++ b/test/nvmf/target/nvmf_example.sh @@ -12,9 +12,9 @@ MALLOC_BLOCK_SIZE=512 function build_nvmf_example_args() { if [ $SPDK_RUN_NON_ROOT -eq 1 ]; then - echo "sudo -u $(logname) ./examples/nvmf/nvmf/nvmf -i $NVMF_APP_SHM_ID" + echo "sudo -u $(logname) ./examples/nvmf/nvmf/nvmf -i $NVMF_APP_SHM_ID" -g 10000 else - echo "./examples/nvmf/nvmf/nvmf -i $NVMF_APP_SHM_ID" + echo "./examples/nvmf/nvmf/nvmf -i $NVMF_APP_SHM_ID" -g 10000 fi }