ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename instance_id to shm_id and make it default to pid

Browse Source 
By default, all SPDK applications will not share memory.
To share memory, start the applications with the same
shared memory id.

Change-Id: Ib6180369ef0ed12d05983a21d7943e467402b21a
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Ben Walker 2017-02-01 14:34:45 -07:00 committed by Daniel Verkamp
parent 26c8e7ccef
commit 25270f1d7c
14 changed files with 116 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/iscsi_tgt/iscsi_tgt.c
View File

@ -71,7 +71,7 @@ usage(char *executable_name)
printf(" -c config config file (default %s)\n", SPDK_ISCSI_DEFAULT_CONFIG);
printf(" -e mask tracepoint group mask for spdk trace buffers (default 0x0)\n");
printf(" -m mask core mask for DPDK\n");
printf(" -i instance ID\n");
printf(" -i shared memory ID (optional)\n");
printf(" -l facility use specific syslog facility (default %s)\n",
opts.log_facility);
printf(" -n channel number of memory channels used for DPDK\n");
@ -116,7 +116,7 @@ main(int argc, char **argv)
opts.config_file = optarg;
break;
case 'i':
opts.instance_id = atoi(optarg);
opts.shm_id = atoi(optarg);
break;
case 'l':
opts.log_facility = optarg;

12
app/iscsi_top/iscsi_top.cpp
View File

@ -54,14 +54,14 @@ extern "C" {
}
static char *exe_name;
static int g_instance_id = 0;
static int g_shm_id = 0;
static void usage(void)
{
fprintf(stderr, "usage:\n");
fprintf(stderr, " %s <option>\n", exe_name);
fprintf(stderr, " option = '-i' to specify the instance ID,"
" (default: 0)\n");
fprintf(stderr, " option = '-i' to specify the shared memory ID,"
" (required)\n");
}
static bool
@ -90,7 +90,7 @@ print_connections(void)
int fd, i;
char shm_name[64];
sprintf(shm_name, "spdk_iscsi_conns.%d", g_instance_id);
sprintf(shm_name, "spdk_iscsi_conns.%d", g_shm_id);
fd = shm_open(shm_name, O_RDONLY, 0600);
if (fd < 0) {
fprintf(stderr, "Cannot open shared memory: %s\n", shm_name);
@ -150,7 +150,7 @@ int main(int argc, char **argv)
while ((op = getopt(argc, argv, "i:")) != -1) {
switch (op) {
case 'i':
g_instance_id = atoi(optarg);
g_shm_id = atoi(optarg);
break;
default:
usage();
@ -158,7 +158,7 @@ int main(int argc, char **argv)
}
}
sprintf(spdk_trace_shm_name, "/iscsi_trace.%d", g_instance_id);
sprintf(spdk_trace_shm_name, "/iscsi_trace.%d", g_shm_id);
history_fd = shm_open(spdk_trace_shm_name, O_RDONLY, 0600);
if (history_fd < 0) {
fprintf(stderr, "Unable to open history shm %s\n", spdk_trace_shm_name);

4
app/nvmf_tgt/nvmf_tgt.c
View File

@ -305,7 +305,7 @@ usage(void)
printf(" -c config - config file (default %s)\n", SPDK_NVMF_DEFAULT_CONFIG);
printf(" -e mask - tracepoint group mask for spdk trace buffers (default 0x0)\n");
printf(" -m mask - core mask for DPDK\n");
printf(" -i instance ID\n");
printf(" -i shared memory ID (optional)\n");
printf(" -l facility - use specific syslog facility (default %s)\n",
opts.log_facility);
printf(" -n channel number of memory channels used for DPDK\n");
@ -408,7 +408,7 @@ main(int argc, char **argv)
opts.config_file = optarg;
break;
case 'i':
opts.instance_id = atoi(optarg);
opts.shm_id = atoi(optarg);
break;
case 'l':
opts.log_facility = optarg;

8
app/trace/trace.cpp
View File

@ -94,7 +94,7 @@ struct object_stats g_stats[SPDK_TRACE_MAX_OBJECT];
static char *exe_name;
static int verbose = 1;
static int g_instance_id = 0;
static int g_shm_id = 0;
static int g_fudge_factor = 20;
static uint64_t tsc_rate;
@ -306,7 +306,7 @@ static void usage(void)
fprintf(stderr, " '-c' to display single lcore history\n");
fprintf(stderr, " '-f' to specify number of events to ignore at\n");
fprintf(stderr, " beginning and end of trace (default: 20)\n");
fprintf(stderr, " '-i' to specify the instance ID, (default: 0)\n");
fprintf(stderr, " '-i' to specify the shared memory ID, (required)\n");
}
int main(int argc, char **argv)
@ -336,7 +336,7 @@ int main(int argc, char **argv)
g_fudge_factor = atoi(optarg);
break;
case 'i':
g_instance_id = atoi(optarg);
g_shm_id = atoi(optarg);
break;
case 'q':
verbose = 0;
@ -350,7 +350,7 @@ int main(int argc, char **argv)
}
}
snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", app_name, g_instance_id);
snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", app_name, g_shm_id);
fd = shm_open(shm_name, O_RDONLY, 0600);
if (fd < 0) {
fprintf(stderr, "Could not open shm %s.\n", shm_name);

6
etc/spdk/iscsi.conf.in
View File

@ -10,9 +10,9 @@
# Bracketed ([]) names define sections
[Global]
# Instance ID for multi-process support
# Default: 0
#InstanceID 0
# Shared Memory Group ID. SPDK applications with the same ID will share memory.
# Default: <the process PID>
#SharedMemoryID 0
# Users can restrict work items to only run on certain cores by
# specifying a ReactorMask. Default is to allow work items to run

37
examples/nvme/arbitration/arbitration.c
View File

@ -87,6 +87,7 @@ struct worker_thread {
};
struct arb_context {
int shm_id;
int outstanding_commands;
int num_namespaces;
int num_workers;
@ -119,6 +120,7 @@ static struct worker_thread *g_workers = NULL;
static struct feature features[256];
static struct arb_context g_arbitration = {
.shm_id = -1,
.outstanding_commands = 0,
.num_workers = 0,
.num_namespaces = 0,
@ -521,7 +523,8 @@ usage(char *program_name)
printf("\t\t(2: vendor specific mechanism)]\n");
printf("\t[-b enable arbitration user configuration, default: disabled]\n");
printf("\t\t(0 - disabled; 1 - enabled)\n");
printf("\t[-i subjected IOs for performance comparison]\n");
printf("\t[-n subjected IOs for performance comparison]\n");
printf("\t[-i shared memory group ID]\n");
}
static const char *
@ -682,11 +685,14 @@ parse_args(int argc, char **argv)
int op = 0;
bool mix_specified = false;
while ((op = getopt(argc, argv, "c:l:m:q:s:t:w:M:a:b:i:h")) != -1) {
while ((op = getopt(argc, argv, "c:l:i:m:q:s:t:w:M:a:b:n:h")) != -1) {
switch (op) {
case 'c':
g_arbitration.core_mask = optarg;
break;
case 'i':
g_arbitration.shm_id = atoi(optarg);
break;
case 'l':
g_arbitration.latency_tracking_enable = atoi(optarg);
break;
@ -715,7 +721,7 @@ parse_args(int argc, char **argv)
case 'b':
g_arbitration.arbitration_config = atoi(optarg);
break;
case 'i':
case 'n':
g_arbitration.io_count = atoi(optarg);
break;
case 'h':
@ -1082,6 +1088,7 @@ static char *ealargs[] = {
"--proc-type=auto",
#ifdef __linux__
"--base-virtaddr=0x1000000000",
"", /* May be replaced by --file-prefix */
#endif
};
@ -1092,6 +1099,9 @@ main(int argc, char **argv)
struct worker_thread *worker;
char task_pool_name[30];
uint32_t task_count;
size_t argcount;
char *core_mask = NULL;
char *file_prefix = NULL;
rc = parse_args(argc, argv);
if (rc != 0) {
@ -1099,15 +1109,30 @@ main(int argc, char **argv)
}
/* Default 4 cores for (urgent / high / medium / low) 4 kinds of queues respectively */
ealargs[1] = spdk_sprintf_alloc("-c %s", g_arbitration.core_mask);
ealargs[1] = core_mask = spdk_sprintf_alloc("-c %s", g_arbitration.core_mask);
if (ealargs[1] == NULL) {
perror("ealargs spdk_sprintf_alloc");
return 1;
}
rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs);
free(ealargs[1]);
argcount = (sizeof(ealargs) / sizeof(ealargs[0])) - 1;
#ifdef __linux__
if (g_arbitration.shm_id >= 0) {
ealargs[5] = file_prefix = spdk_sprintf_alloc("--file-prefix=spdk%d",
g_arbitration.shm_id);
argcount++;
}
#endif
rc = rte_eal_init(argcount, ealargs);
free(core_mask);
if (file_prefix) {
free(file_prefix);
}
if (rc < 0) {
fprintf(stderr, "could not initialize dpdk\n");

37
examples/nvme/perf/perf.c
View File

@ -148,6 +148,7 @@ static int g_queue_depth;
static int g_time_in_sec;
static uint32_t g_max_completions;
static int g_dpdk_mem;
static int g_shm_id = -1;
static const char *g_core_mask;
@ -661,6 +662,7 @@ static void usage(char *program_name)
printf("\t[-d DPDK huge memory size in MB.]\n");
printf("\t[-m max completions per poll]\n");
printf("\t\t(default: 0 - unlimited)\n");
printf("\t[-i shared memory group ID]\n");
}
static void
@ -854,7 +856,7 @@ parse_args(int argc, char **argv)
g_core_mask = NULL;
g_max_completions = 0;
while ((op = getopt(argc, argv, "c:d:lm:q:r:s:t:w:M:")) != -1) {
while ((op = getopt(argc, argv, "c:d:i:lm:q:r:s:t:w:M:")) != -1) {
switch (op) {
case 'c':
g_core_mask = optarg;
@ -862,6 +864,9 @@ parse_args(int argc, char **argv)
case 'd':
g_dpdk_mem = atoi(optarg);
break;
case 'i':
g_shm_id = atoi(optarg);
break;
case 'l':
g_latency_tracking_enable = true;
break;
@ -1196,7 +1201,9 @@ static char *ealargs[] = {
"--proc-type=auto",
#ifdef __linux__
"--base-virtaddr=0x1000000000",
"", /* May be replaced by --file-prefix */
#endif
};
int main(int argc, char **argv)
@ -1205,29 +1212,47 @@ int main(int argc, char **argv)
struct worker_thread *worker;
char task_pool_name[30];
uint32_t task_count;
size_t argcount;
char *core_mask = NULL;
char *mem_size = NULL;
char *file_prefix = NULL;
rc = parse_args(argc, argv);
if (rc != 0) {
return rc;
}
ealargs[1] = spdk_sprintf_alloc("-c %s", g_core_mask ? g_core_mask : "0x1");
ealargs[1] = core_mask = spdk_sprintf_alloc("-c %s", g_core_mask ? g_core_mask : "0x1");
if (ealargs[1] == NULL) {
perror("ealargs spdk_sprintf_alloc");
return 1;
}
ealargs[3] = spdk_sprintf_alloc("-m %d", g_dpdk_mem ? g_dpdk_mem : 512);
ealargs[3] = mem_size = spdk_sprintf_alloc("-m %d", g_dpdk_mem ? g_dpdk_mem : 512);
if (ealargs[3] == NULL) {
free(ealargs[1]);
perror("ealargs spdk_sprintf_alloc");
return 1;
}
rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs);
argcount = (sizeof(ealargs) / sizeof(ealargs[0])) - 1;
free(ealargs[1]);
free(ealargs[3]);
#ifdef __linux__
if (g_shm_id >= 0) {
ealargs[6] = file_prefix = spdk_sprintf_alloc("--file-prefix=spdk%d",
g_shm_id);
argcount++;
}
#endif
rc = rte_eal_init(argcount, ealargs);
free(core_mask);
free(mem_size);
if (file_prefix) {
free(file_prefix);
}
if (rc < 0) {
fprintf(stderr, "could not initialize dpdk\n");

6
include/spdk/event.h
View File

@ -75,7 +75,7 @@ struct spdk_app_opts {
const char *log_facility;
const char *tpoint_group_mask;
int instance_id;
int shm_id;
spdk_app_shutdown_cb shutdown_cb;
spdk_sighandler_t usr1_handler;
@ -139,9 +139,9 @@ void spdk_app_stop(int rc);
int spdk_app_get_running_config(char **config_str, char *name);
/**
* \brief Return the instance id for this application.
* \brief Return the shared memory id for this application.
*/
int spdk_app_get_instance_id(void);
int spdk_app_get_shm_id(void);
/**
* \brief Convert a string containing a CPU core mask into a bitmask

22
lib/event/app.c
View File

@ -68,7 +68,7 @@
struct spdk_app {
struct spdk_conf *config;
char pidfile[SPDK_APP_PIDFILE_MAX_LENGTH];
int instance_id;
int shm_id;
spdk_app_shutdown_cb shutdown_cb;
int rc;
};
@ -80,9 +80,9 @@ static int spdk_app_write_pidfile(void);
static void spdk_app_remove_pidfile(void);
int
spdk_app_get_instance_id(void)
spdk_app_get_shm_id(void)
{
return g_spdk_app.instance_id;
return g_spdk_app.shm_id;
}
/* Global section */
@ -224,7 +224,7 @@ spdk_app_opts_init(struct spdk_app_opts *opts)
opts->log_facility = SPDK_APP_DEFAULT_LOG_FACILITY;
opts->enable_coredump = true;
opts->instance_id = -1;
opts->shm_id = -1;
opts->dpdk_mem_size = SPDK_APP_DPDK_DEFAULT_MEM_SIZE;
opts->dpdk_master_core = SPDK_APP_DPDK_DEFAULT_MASTER_CORE;
opts->dpdk_mem_channel = SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL;
@ -268,23 +268,23 @@ spdk_app_init(struct spdk_app_opts *opts)
}
spdk_conf_set_as_default(config);
if (opts->instance_id == -1) {
if (opts->shm_id == -1) {
sp = spdk_conf_find_section(config, "Global");
if (sp != NULL) {
opts->instance_id = spdk_conf_section_get_intval(sp, "InstanceID");
opts->shm_id = spdk_conf_section_get_intval(sp, "SharedMemoryID");
}
}
if (opts->instance_id < 0) {
opts->instance_id = 0;
if (opts->shm_id < 0) {
opts->shm_id = getpid();
}
memset(&g_spdk_app, 0, sizeof(g_spdk_app));
g_spdk_app.config = config;
g_spdk_app.instance_id = opts->instance_id;
g_spdk_app.shm_id = opts->shm_id;
g_spdk_app.shutdown_cb = opts->shutdown_cb;
snprintf(g_spdk_app.pidfile, sizeof(g_spdk_app.pidfile), "%s/%s.pid.%d",
SPDK_APP_PIDFILE_PREFIX, opts->name, opts->instance_id);
SPDK_APP_PIDFILE_PREFIX, opts->name, opts->shm_id);
spdk_app_write_pidfile();
/* open log files */
@ -391,7 +391,7 @@ spdk_app_init(struct spdk_app_opts *opts)
sigaddset(&signew, SIGHUP);
pthread_sigmask(SIG_SETMASK, &signew, NULL);
snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", opts->name, opts->instance_id);
snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", opts->name, opts->shm_id);
spdk_trace_init(shm_name);
if (opts->tpoint_group_mask == NULL) {

4
lib/event/dpdk_init.c
View File

@ -146,8 +146,8 @@ spdk_build_eal_cmdline(struct spdk_app_opts *opts)
#ifdef __linux__
/* set the hugepage file prefix */
g_arg_strings[EAL_FILE_PREFIX_ARG] = spdk_sprintf_alloc("--file-prefix=rte%d",
opts->instance_id);
g_arg_strings[EAL_FILE_PREFIX_ARG] = spdk_sprintf_alloc("--file-prefix=spdk%d",
opts->shm_id);
#else
/* --file-prefix is not required on FreeBSD */
g_arg_strings[EAL_FILE_PREFIX_ARG] = strdup("");

2
lib/iscsi/conn.c
View File

@ -249,7 +249,7 @@ int spdk_initialize_iscsi_conns(void)
return -1;
}
snprintf(g_shm_name, sizeof(g_shm_name), "spdk_iscsi_conns.%d", spdk_app_get_instance_id());
snprintf(g_shm_name, sizeof(g_shm_name), "spdk_iscsi_conns.%d", spdk_app_get_shm_id());
conns_array_fd = shm_open(g_shm_name, O_RDWR | O_CREAT, 0600);
if (conns_array_fd < 0) {
SPDK_ERRLOG("could not shm_open %s\n", g_shm_name);

2
test/iscsi_tgt/ip_migration/ip_migration.sh
View File

@ -51,7 +51,7 @@ do
cp $testdir/iscsi.conf $testdir/iscsi.conf.$i
port=$(($RPC_PORT + $i))
echo "Listen 127.0.0.1:$port" >> $testdir/iscsi.conf.$i
$exe -c $testdir/iscsi.conf.$i -i $i -s 1000 &
$exe -c $testdir/iscsi.conf.$i -s 1000 -i $i &
pid=$!
echo "Process pid: $pid"

8
test/lib/nvme/nvme.sh
View File

@ -57,22 +57,22 @@ timing_exit arbitration
if [ $(uname -s) = Linux ]; then
timing_enter multi_process
$rootdir/examples/nvme/arbitration/arbitration -s 4096 -t 10 -c 0xf &
$rootdir/examples/nvme/arbitration/arbitration -i 0 -s 4096 -t 10 -c 0xf &
pid=$!
sleep 3
$rootdir/examples/nvme/perf/perf -q 1 -w randread -s 4096 -t 10 -c 0x10 &
$rootdir/examples/nvme/perf/perf -i 0 -q 1 -w randread -s 4096 -t 10 -c 0x10 &
sleep 1
kill -9 $!
count=0
while [ $count -le 2 ]; do
$rootdir/examples/nvme/perf/perf -q 1 -w read -s 4096 -t 1 -c 0x10
$rootdir/examples/nvme/perf/perf -i 0 -q 1 -w read -s 4096 -t 1 -c 0x10
count=$(($count + 1))
done
count=0
while [ $count -le 1 ]; do
core=$((1 << (($count + 4))))
printf -v hexcore "0x%x" "$core"
$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 1 -c $hexcore &
$rootdir/examples/nvme/perf/perf -i 0 -q 128 -w read -s 4096 -t 1 -c $hexcore &
count=$(($count + 1))
done
wait $pid

30
test/lib/nvme/nvmemp.sh
View File

@ -10,26 +10,26 @@ if [ $(uname -s) = Linux ]; then
timing_enter nvme_mp
timing_enter mp_func_test
$rootdir/examples/nvme/arbitration/arbitration -s 4096 -t 5 -c 0xf &
$rootdir/examples/nvme/arbitration/arbitration -i 0 -s 4096 -t 5 -c 0xf &
sleep 3
$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 1 -c 0x10
$rootdir/examples/nvme/perf/perf -i 0 -q 128 -w read -s 4096 -t 1 -c 0x10
wait $!
timing_exit mp_func_test
timing_enter mp_fault_test
timing_enter mp_fault_test_1
$rootdir/examples/nvme/arbitration/arbitration -s 4096 -t 5 -c 0xf &
$rootdir/examples/nvme/arbitration/arbitration -i 0 -s 4096 -t 5 -c 0xf &
sleep 3
$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 3 -c 0x10
$rootdir/examples/nvme/perf/perf -i 0 -q 128 -w read -s 4096 -t 3 -c 0x10
sleep 1
kill -9 $!
timing_exit mp_fault_test_1
timing_enter mp_fault_test_2
$rootdir/examples/nvme/arbitration/arbitration -s 4096 -t 7 -c 0xf &
$rootdir/examples/nvme/arbitration/arbitration -i 0 -s 4096 -t 7 -c 0xf &
pid=$!
sleep 3
$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 3 -c 0x10 &
$rootdir/examples/nvme/perf/perf -i 0 -q 128 -w read -s 4096 -t 3 -c 0x10 &
sleep 2
kill -9 $!
wait $pid
@ -38,23 +38,23 @@ if [ $(uname -s) = Linux ]; then
timing_enter mp_stress_test
timing_enter mp_stress_test_1
$rootdir/examples/nvme/arbitration/arbitration -s 4096 -t 10 -c 0xf &
$rootdir/examples/nvme/arbitration/arbitration -i 0 -s 4096 -t 10 -c 0xf &
sleep 3
count=0
while [ $count -le 4 ]; do
$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 1 -c 0x10
$rootdir/examples/nvme/perf/perf -i 0 -q 128 -w read -s 4096 -t 1 -c 0x10
count=$(( $count + 1 ))
done
wait $!
timing_exit mp_stress_test_1
timing_enter mp_stress_test_2
$rootdir/examples/nvme/arbitration/arbitration -s 4096 -t 15 -c 0xf &
$rootdir/examples/nvme/arbitration/arbitration -i 0 -s 4096 -t 15 -c 0xf &
pid=$!
sleep 3
count=0
while [ $count -le 4 ]; do
$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 3 -c 0x10 &
$rootdir/examples/nvme/perf/perf -i 0 -q 128 -w read -s 4096 -t 3 -c 0x10 &
sleep 2
kill -9 $!
count=$(( $count + 1 ))
@ -63,14 +63,14 @@ if [ $(uname -s) = Linux ]; then
timing_exit mp_stress_test_2
timing_enter mp_stress_test_3
$rootdir/examples/nvme/arbitration/arbitration -s 4096 -t 10 &
$rootdir/examples/nvme/arbitration/arbitration -i 0 -s 4096 -t 10 &
pid=$!
sleep 3
count=0
while [ $count -le 4 ]; do
core=$((1 << (($count + 4))))
printf -v hexcore "0x%x" "$core"
$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 1 -c $hexcore &
$rootdir/examples/nvme/perf/perf -i 0 -q 128 -w read -s 4096 -t 1 -c $hexcore &
count=$(( $count + 1 ))
done
wait $pid
@ -78,12 +78,12 @@ if [ $(uname -s) = Linux ]; then
timing_exit mp_stress_test
timing_enter mp_perf_test
$rootdir/examples/nvme/perf/perf -q 1 -w randread -s 4096 -t 5 -c 0x3
$rootdir/examples/nvme/perf/perf -i 0 -q 1 -w randread -s 4096 -t 5 -c 0x3
sleep 3
$rootdir/examples/nvme/perf/perf -q 1 -w randread -s 4096 -t 8 -c 0x1 &
$rootdir/examples/nvme/perf/perf -i 0 -q 1 -w randread -s 4096 -t 8 -c 0x1 &
sleep 3
$rootdir/examples/nvme/perf/perf -q 1 -w randread -s 4096 -t 3 -c 0x2
$rootdir/examples/nvme/perf/perf -i 0 -q 1 -w randread -s 4096 -t 3 -c 0x2
wait $!
timing_exit mp_perf_test