ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add event-driven application framework

Browse Source 
Change-Id: Iba90db6d8853dde972b4eec2c35eb44eeddae780
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp 2016-05-24 11:04:20 -07:00 committed by Benjamin Walker
parent ab1f6bdc54
commit eeeac6676d
20 changed files with 2395 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
autotest.sh
View File

@ -44,6 +44,7 @@ timing_exit afterboot
timing_enter lib
time test/lib/event/event.sh
time test/lib/nvme/nvme.sh
time test/lib/memory/memory.sh
time test/lib/ioat/ioat.sh

290
include/spdk/event.h Normal file
View File

@ -0,0 +1,290 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* Event framework public API.
*
* This is a framework for writing asynchronous, polled-mode, shared-nothing
* server applications. The framework relies on DPDK for much of its underlying
* architecture. The framework defines several concepts - reactors, events, pollers,
* and subsystems - that are described in the following sections.
*
* The framework runs one thread per core (the user provides a core mask), where
* each thread is a tight loop. The threads never block for any reason. These threads
* are called reactors and their main responsibility is to process incoming events
* from a queue.
*
* An event, defined by \ref spdk_event is a bundled function pointer and arguments that
* can be sent to a different core and executed. The function pointer is executed only once,
* and then the entire event is freed. These functions should never block and preferably
* should execute very quickly. Events also have a pointer to a 'next' event that will be
* executed upon completion of the given event, which allows chaining. This is
* very much a simplified version of futures, promises, and continuations designed within
* the constraints of the C programming language.
*
* The framework also defines another type of function called a poller. Pollers are also
* functions with arguments that can be bundled and sent to a different core to be executed,
* but they are instead executed repeatedly on that core until unregistered. The reactor
* will handle interspersing calls to the pollers with other event processing automatically.
* Pollers are intended to poll hardware as a replacement for interrupts and they should not
* generally be used for any other purpose.
*
* The framework also defines an interface for subsystems, which are libraries of code that
* depend on this framework. A library can register itself as a subsystem and provide
* pointers to initialize and destroy itself which will be called at the appropriate time.
* This is purely for sequencing initialization code in a convenient manner within the
* framework.
*
* The framework itself is bundled into a higher level abstraction called an "app". Once
* \ref spdk_app_start is called it will block the current thread until the application
* terminates (by calling \ref spdk_app_stop).
*/
#ifndef SPDK_EVENT_H
#define SPDK_EVENT_H
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include "spdk/queue.h"
#define SPDK_APP_DEFAULT_LOG_FACILITY "local7"
#define SPDK_APP_DEFAULT_LOG_PRIORITY "info"
typedef struct spdk_event *spdk_event_t;
typedef void (*spdk_event_fn)(spdk_event_t);
/**
* \brief An event is a function that is passed to and called on an lcore.
*/
struct spdk_event {
uint32_t lcore;
spdk_event_fn fn;
void *arg1;
void *arg2;
struct spdk_event *next;
};
typedef void (*spdk_poller_fn)(void *arg);
/**
* \brief A poller is a function that is repeatedly called on an lcore.
*/
struct spdk_poller {
uint32_t lcore;
spdk_poller_fn fn;
void *arg;
};
#define SPDK_POLLER_RING_SIZE 4096
/*
* -1 accounts for the empty slot needed to differentiate between ring empty
* and ring full.
*/
#define SPDK_MAX_POLLERS_PER_CORE (SPDK_POLLER_RING_SIZE - 1)
typedef void (*spdk_app_shutdown_cb)(void);
typedef void (*spdk_sighandler_t)(int);
#define SPDK_APP_DPDK_DEFAULT_MEM_SIZE 2048
#define SPDK_APP_DPDK_DEFAULT_MASTER_CORE 0
#define SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL 4
#define SPDK_APP_DPDK_DEFAULT_CORE_MASK "0x1"
/**
* \brief Event framework initialization options
*/
struct spdk_app_opts {
const char *name;
const char *config_file;
const char *reactor_mask;
const char *log_facility;
const char *tpoint_group_mask;
int instance_id;
spdk_app_shutdown_cb shutdown_cb;
spdk_event_fn start_fn;
spdk_sighandler_t usr1_handler;
bool enable_coredump;
uint32_t dpdk_mem_channel;
uint32_t dpdk_master_core;
int dpdk_mem_size;
};
/**
* \brief Initialize the default value of opts
*/
void spdk_app_opts_init(struct spdk_app_opts *opts);
/**
* \brief Initialize DPDK via opts.
*/
void spdk_init_dpdk(struct spdk_app_opts *opts);
/**
* \brief Initialize an application to use the event framework. This must be called prior to using
* any other functions in this library.
*/
void spdk_app_init(struct spdk_app_opts *opts);
/**
* \brief Perform final shutdown operations on an application using the event framework.
*/
void spdk_app_fini(void);
/**
* \brief Start the framework. Once started, the framework will call start_fn on the master
* core with the arguments provided. This call will block until \ref spdk_app_stop is called.
*/
int spdk_app_start(spdk_event_fn start_fn, void *arg1, void *arg2);
/**
* \brief Stop the framework. This does not wait for all threads to exit. Instead, it kicks off
* the shutdown process and returns. Once the shutdown process is complete, \ref spdk_app_start will return.
*/
void spdk_app_stop(int rc);
/**
* \brief Generate a configuration file that corresponds to the current running state.
*/
int spdk_app_get_running_config(char **config_str, char *name);
/**
* \brief Return the instance id for this application.
*/
int spdk_app_get_instance_id(void);
/**
* \brief Convert a string containing a CPU core mask into a bitmask
*/
int spdk_app_parse_core_mask(const char *mask, uint64_t *cpumask);
/**
* \brief Return a mask of the CPU cores active for this application
*/
uint64_t spdk_app_get_core_mask(void);
/**
* \brief Return the number of CPU cores utilized by this application
*/
int spdk_app_get_core_count(void);
/**
* \brief Allocate an event to be passed to \ref spdk_event_call
*/
spdk_event_t spdk_event_allocate(uint32_t lcore, spdk_event_fn fn,
void *arg1, void *arg2,
spdk_event_t next);
/**
* \brief Pass the given event to the associated lcore and call the function.
*/
void spdk_event_call(spdk_event_t event);
#define spdk_event_get_next(event) (event)->next
#define spdk_event_get_arg1(event) (event)->arg1
#define spdk_event_get_arg2(event) (event)->arg2
/* TODO: This is only used by tests and should be made private */
void spdk_event_queue_run_all(uint32_t lcore);
/**
* \brief Register a poller on the given lcore.
*/
void spdk_poller_register(struct spdk_poller *poller,
uint32_t lcore,
struct spdk_event *complete);
/**
* \brief Unregister a poller on the given lcore.
*/
void spdk_poller_unregister(struct spdk_poller *poller,
struct spdk_event *complete);
/**
* \brief Move a poller from its current lcore to a new lcore.
*/
void spdk_poller_migrate(struct spdk_poller *poller, int new_lcore,
struct spdk_event *complete);
struct spdk_subsystem {
const char *name;
int (*init)(void);
int (*fini)(void);
void (*config)(FILE *fp);
TAILQ_ENTRY(spdk_subsystem) tailq;
};
struct spdk_subsystem_depend {
const char *name;
const char *depends_on;
TAILQ_ENTRY(spdk_subsystem_depend) tailq;
};
void spdk_add_subsystem(struct spdk_subsystem *subsystem);
void spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend);
/**
* \brief Register a new subsystem
*/
#define SPDK_SUBSYSTEM_REGISTER(_name, _init, _fini, _config) \
static struct spdk_subsystem __subsystem_ ## _name = { \
.name = #_name, \
.init = _init, \
.fini = _fini, \
.config = _config, \
}; \
__attribute__((constructor)) static void _name ## _register(void) \
{ \
spdk_add_subsystem(&__subsystem_ ## _name); \
}
/**
* \brief Declare that a subsystem depends on another subsystem.
*/
#define SPDK_SUBSYSTEM_DEPEND(_name, _depends_on) \
static struct spdk_subsystem_depend __subsystem_ ## _name ## _depend_on ## _depends_on = { \
.name = #_name, \
.depends_on = #_depends_on, \
}; \
__attribute__((constructor)) static void _name ## _depend_on ## _depends_on(void) \
{ \
spdk_add_subsystem_depend(&__subsystem_ ## _name ## _depend_on ## _depends_on); \
}
#endif

2
lib/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y += conf cunit json jsonrpc log memory trace util nvme ioat
DIRS-y += conf cunit event json jsonrpc log memory trace util nvme ioat
.PHONY: all clean $(DIRS-y)

40
lib/event/Makefile Normal file
View File

@ -0,0 +1,40 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
CFLAGS += $(DPDK_INC)
LIBNAME = event
C_SRCS = app.c dpdk_init.c reactor.c subsystem.c
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

478
lib/event/app.c Normal file
View File

@ -0,0 +1,478 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/event.h"
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <fcntl.h>
#include <signal.h>
#include <rte_debug.h>
#include "spdk/log.h"
#include "spdk/conf.h"
#include "spdk/trace.h"
#include "reactor.h"
#include "subsystem.h"
/* Add enough here to append ".pid" plus 2 digit instance ID */
#define SPDK_APP_PIDFILE_MAX_LENGTH 40
#define SPDK_APP_PIDFILE_PREFIX "/var/run"
struct spdk_app {
struct spdk_conf *config;
char pidfile[SPDK_APP_PIDFILE_MAX_LENGTH];
int instance_id;
spdk_app_shutdown_cb shutdown_cb;
int rc;
};
static struct spdk_app g_spdk_app;
static spdk_event_t g_shutdown_event = NULL;
static int spdk_app_write_pidfile(void);
static void spdk_app_remove_pidfile(void);
int
spdk_app_get_instance_id(void)
{
return g_spdk_app.instance_id;
}
/* Global section */
#define GLOBAL_CONFIG_TMPL \
"# Configuration file\n" \
"#\n" \
"# Please write all parameters using ASCII.\n" \
"# The parameter must be quoted if it includes whitespace.\n" \
"#\n" \
"# Configuration syntax:\n" \
"# Spaces at head of line are deleted, other spaces are as separator\n" \
"# Lines starting with '#' are comments and not evaluated.\n" \
"# Lines ending with '\\' are concatenated with the next line.\n" \
"# Bracketed keys are section keys grouping the following value keys.\n" \
"# Number of section key is used as a tag number.\n" \
"# Ex. [TargetNode1] = TargetNode section key with tag number 1\n" \
"[Global]\n" \
" Comment \"Global section\"\n" \
"\n" \
" # Users can restrict work items to only run on certain cores by\n" \
" # specifying a ReactorMask. Default is to allow work items to run\n" \
" # on all cores. Core 0 must be set in the mask if one is specified.\n" \
" # Default: 0xFFFF (cores 0-15)\n" \
" ReactorMask \"0x%" PRIX64 "\"\n" \
"\n" \
" # Tracepoint group mask for spdk trace buffers\n" \
" # Default: 0x0 (all tracepoint groups disabled)\n" \
" # Set to 0xFFFFFFFFFFFFFFFF to enable all tracepoint groups.\n" \
" TpointGroupMask \"0x%" PRIX64 "\"\n" \
"\n" \
" # syslog facility\n" \
" LogFacility \"%s\"\n" \
"\n"
static void
spdk_app_config_dump_global_section(FILE *fp)
{
if (NULL == fp)
return;
/* FIXME - lookup log facility and put it in place of "local7" below */
fprintf(fp, GLOBAL_CONFIG_TMPL,
spdk_app_get_core_mask(), spdk_trace_get_tpoint_group_mask(),
"local7");
}
int
spdk_app_get_running_config(char **config_str, char *name)
{
FILE *fp = NULL;
int fd = -1;
long length = 0, ret = 0;
char vbuf[BUFSIZ];
char config_template[64];
snprintf(config_template, sizeof(config_template), "/tmp/%s.XXXXXX", name);
/* Create temporary file to hold config */
fd = mkstemp(config_template);
if (fd == -1) {
fprintf(stderr, "mkstemp failed\n");
return -1;
}
fp = fdopen(fd, "wb+");
if (NULL == fp) {
fprintf(stderr, "error opening tmpfile fd = %d\n", fd);
return -1;
}
/* Buffered IO */
setvbuf(fp, vbuf, _IOFBF, BUFSIZ);
spdk_app_config_dump_global_section(fp);
spdk_subsystem_config(fp);
length = ftell(fp);
*config_str = malloc(length + 1);
if (!*config_str) {
perror("config_str");
fclose(fp);
return -1;
}
fseek(fp, 0, SEEK_SET);
ret = fread(*config_str, sizeof(char), length, fp);
if (ret < length)
fprintf(stderr, "%s: warning - short read\n", __func__);
fclose(fp);
(*config_str)[length] = '\0';
return 0;
}
static const char *
spdk_get_log_facility(struct spdk_conf *config)
{
struct spdk_conf_section *sp;
const char *logfacility;
sp = spdk_conf_find_section(config, "Global");
if (sp == NULL) {
return SPDK_APP_DEFAULT_LOG_FACILITY;
}
logfacility = spdk_conf_section_get_val(sp, "LogFacility");
if (logfacility == NULL) {
return SPDK_APP_DEFAULT_LOG_FACILITY;
}
return logfacility;
}
static void
__shutdown_signal(int signo)
{
/*
* Call pre-allocated shutdown event. Note that it is not
* safe to allocate the event within the signal handlers
* context, since that context is not a DPDK thread so
* buffer allocation is not permitted.
*/
if (g_shutdown_event != NULL) {
spdk_event_call(g_shutdown_event);
}
}
static void
__shutdown_event_cb(spdk_event_t event)
{
g_spdk_app.shutdown_cb();
}
void
spdk_app_opts_init(struct spdk_app_opts *opts)
{
if (!opts)
return;
memset(opts, 0, sizeof(*opts));
opts->enable_coredump = true;
opts->instance_id = -1;
opts->dpdk_mem_size = -1;
opts->dpdk_master_core = SPDK_APP_DPDK_DEFAULT_MASTER_CORE;
opts->dpdk_mem_channel = SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL;
opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK;
}
void
spdk_app_init(struct spdk_app_opts *opts)
{
struct spdk_conf *config;
struct spdk_conf_section *sp;
struct sigaction sigact;
sigset_t signew;
char shm_name[64];
int rc;
uint64_t tpoint_group_mask;
char *end;
if (opts->enable_coredump) {
struct rlimit core_limits;
core_limits.rlim_cur = core_limits.rlim_max = RLIM_INFINITY;
setrlimit(RLIMIT_CORE, &core_limits);
}
config = spdk_conf_allocate();
RTE_VERIFY(config != NULL);
if (opts->config_file) {
rc = spdk_conf_read(config, opts->config_file);
if (rc != 0) {
fprintf(stderr, "Could not read config file %s\n", opts->config_file);
exit(EXIT_FAILURE);
}
if (config->section == NULL) {
fprintf(stderr, "Invalid config file %s\n", opts->config_file);
exit(EXIT_FAILURE);
}
}
spdk_conf_set_as_default(config);
if (opts->instance_id == -1) {
sp = spdk_conf_find_section(config, "Global");
if (sp != NULL) {
opts->instance_id = spdk_conf_section_get_intval(sp, "InstanceID");
}
}
if (opts->instance_id < 0) {
opts->instance_id = 0;
}
memset(&g_spdk_app, 0, sizeof(g_spdk_app));
g_spdk_app.config = config;
g_spdk_app.instance_id = opts->instance_id;
g_spdk_app.shutdown_cb = opts->shutdown_cb;
snprintf(g_spdk_app.pidfile, sizeof(g_spdk_app.pidfile), "%s/%s.pid.%d",
SPDK_APP_PIDFILE_PREFIX, opts->name, opts->instance_id);
spdk_app_write_pidfile();
/* open log files */
if (opts->log_facility == NULL) {
opts->log_facility = spdk_get_log_facility(g_spdk_app.config);
if (opts->log_facility == NULL) {
fprintf(stderr, "NULL logfacility\n");
spdk_conf_free(g_spdk_app.config);
exit(EXIT_FAILURE);
}
}
rc = spdk_set_log_facility(opts->log_facility);
if (rc < 0) {
fprintf(stderr, "log facility error\n");
spdk_conf_free(g_spdk_app.config);
exit(EXIT_FAILURE);
}
rc = spdk_set_log_priority(SPDK_APP_DEFAULT_LOG_PRIORITY);
if (rc < 0) {
fprintf(stderr, "log priority error\n");
spdk_conf_free(g_spdk_app.config);
exit(EXIT_FAILURE);
}
spdk_open_log();
if (opts->reactor_mask == NULL) {
sp = spdk_conf_find_section(g_spdk_app.config, "Global");
if (sp != NULL) {
if (spdk_conf_section_get_val(sp, "WorkerMask")) {
fprintf(stderr, "WorkerMask not valid key name."
" Use ReactorMask instead.\n");
spdk_conf_free(g_spdk_app.config);
exit(EXIT_FAILURE);
}
opts->reactor_mask = spdk_conf_section_get_val(sp, "ReactorMask");
}
}
/*
* If mask not specified on command line or in configuration file,
* reactor_mask will be NULL which will enable all cores to run
* reactors.
*/
if (spdk_reactor_subsystem_init(opts->reactor_mask)) {
fprintf(stderr, "Invalid reactor mask.\n");
exit(EXIT_FAILURE);
}
/* setup signal handler thread */
pthread_sigmask(SIG_SETMASK, NULL, &signew);
memset(&sigact, 0, sizeof(sigact));
sigact.sa_handler = SIG_IGN;
sigemptyset(&sigact.sa_mask);
rc = sigaction(SIGPIPE, &sigact, NULL);
if (rc < 0) {
SPDK_ERRLOG("sigaction(SIGPIPE) failed\n");
exit(EXIT_FAILURE);
}
if (opts->shutdown_cb != NULL) {
g_shutdown_event = spdk_event_allocate(rte_lcore_id(), __shutdown_event_cb,
NULL, NULL, NULL);
sigact.sa_handler = __shutdown_signal;
sigemptyset(&sigact.sa_mask);
rc = sigaction(SIGINT, &sigact, NULL);
if (rc < 0) {
SPDK_ERRLOG("sigaction(SIGINT) failed\n");
exit(EXIT_FAILURE);
}
sigaddset(&signew, SIGINT);
sigact.sa_handler = __shutdown_signal;
sigemptyset(&sigact.sa_mask);
rc = sigaction(SIGTERM, &sigact, NULL);
if (rc < 0) {
SPDK_ERRLOG("sigaction(SIGTERM) failed\n");
exit(EXIT_FAILURE);
}
sigaddset(&signew, SIGTERM);
}
if (opts->usr1_handler != NULL) {
sigact.sa_handler = opts->usr1_handler;
sigemptyset(&sigact.sa_mask);
rc = sigaction(SIGUSR1, &sigact, NULL);
if (rc < 0) {
SPDK_ERRLOG("sigaction(SIGUSR1) failed\n");
exit(EXIT_FAILURE);
}
sigaddset(&signew, SIGUSR1);
}
sigaddset(&signew, SIGQUIT);
sigaddset(&signew, SIGHUP);
pthread_sigmask(SIG_SETMASK, &signew, NULL);
snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", opts->name, opts->instance_id);
spdk_trace_init(shm_name);
if (opts->tpoint_group_mask == NULL) {
sp = spdk_conf_find_section(g_spdk_app.config, "Global");
if (sp != NULL) {
opts->tpoint_group_mask = spdk_conf_section_get_val(sp, "TpointGroupMask");
}
}
if (opts->tpoint_group_mask != NULL) {
errno = 0;
tpoint_group_mask = strtoull(opts->tpoint_group_mask, &end, 16);
if (*end != '\0' || errno) {
SPDK_ERRLOG("invalid tpoint mask %s\n", opts->tpoint_group_mask);
} else {
spdk_trace_set_tpoint_group_mask(tpoint_group_mask);
}
}
rc = spdk_subsystem_init();
if (rc < 0) {
SPDK_ERRLOG("spdk_subsystem_init() failed\n");
exit(EXIT_FAILURE);
}
}
void
spdk_app_fini(void)
{
spdk_subsystem_fini();
spdk_trace_cleanup();
spdk_app_remove_pidfile();
spdk_conf_free(g_spdk_app.config);
spdk_close_log();
}
int
spdk_app_start(spdk_event_fn start_fn, void *arg1, void *arg2)
{
spdk_event_t event;
g_spdk_app.rc = 0;
event = spdk_event_allocate(rte_get_master_lcore(), start_fn,
arg1, arg2, NULL);
/* Queues up the event, but can't run it until the reactors start */
spdk_event_call(event);
/* This blocks until spdk_app_stop is called */
spdk_reactor_subsystem_start();
return g_spdk_app.rc;
}
void
spdk_app_stop(int rc)
{
spdk_reactor_subsystem_stop();
g_spdk_app.rc = rc;
}
static int
spdk_app_write_pidfile(void)
{
FILE *fp;
pid_t pid;
struct flock lock = {
.l_type = F_WRLCK,
.l_whence = SEEK_SET,
.l_start = 0,
.l_len = 0,
};
fp = fopen(g_spdk_app.pidfile, "w");
if (fp == NULL) {
SPDK_ERRLOG("pidfile open error %d\n", errno);
return -1;
}
if (fcntl(fileno(fp), F_SETLK, &lock) != 0) {
fprintf(stderr, "Cannot create lock on file %s, probably you"
" should use different instance id\n", g_spdk_app.pidfile);
exit(EXIT_FAILURE);
}
pid = getpid();
fprintf(fp, "%d\n", (int)pid);
fclose(fp);
return 0;
}
static void
spdk_app_remove_pidfile(void)
{
int rc;
rc = remove(g_spdk_app.pidfile);
if (rc != 0) {
SPDK_ERRLOG("pidfile remove error %d\n", errno);
/* ignore error */
}
}

194
lib/event/dpdk_init.c Normal file
View File

@ -0,0 +1,194 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/event.h"
#include <errno.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <rte_debug.h>
#include <rte_config.h>
#include <rte_per_lcore.h>
#include <rte_eal.h>
#include <rte_launch.h>
#include <rte_common.h>
#include "spdk/string.h"
enum dpdk_eal_args {
EAL_PROGNAME_ARG = 0,
EAL_COREMASK_ARG,
EAL_MEMCHAN_ARG,
EAL_MEMSIZE_ARG,
EAL_MASTER_CORE_ARG,
EAL_FILE_PREFIX_ARG,
EAL_ARG_COUNT
};
/* g_arg_strings contains the original pointers allocated via
* spdk_sprintf_alloc(). These pointers are copied to g_ealargs
* for passing to DPDK rte_eal_init(). Since DPDK may modify the
* pointer values, we use g_arg_strings() to free the strings after
* rte_eal_init() completes.
*/
static char *g_arg_strings[EAL_ARG_COUNT];
static char *g_ealargs[EAL_ARG_COUNT];
static void
spdk_free_ealargs(void)
{
int i;
for (i = 0; i < EAL_ARG_COUNT; i++)
free(g_arg_strings[i]);
}
static unsigned long long
spdk_get_eal_coremask(const char *coremask)
{
unsigned long long core_mask, max_coremask = 0;
int num_cores_online;
num_cores_online = sysconf(_SC_NPROCESSORS_ONLN);
if (num_cores_online > 0) {
if (num_cores_online > RTE_MAX_LCORE) {
num_cores_online = RTE_MAX_LCORE;
}
if (num_cores_online >= 64) {
max_coremask = ~0ULL;
} else {
max_coremask = (1ULL << num_cores_online) - 1;
}
}
core_mask = strtoull(coremask, NULL, 16);
core_mask &= max_coremask;
return core_mask;
}
static void
spdk_build_eal_cmdline(struct spdk_app_opts *opts)
{
unsigned long long core_mask;
/* set the program name */
g_arg_strings[EAL_PROGNAME_ARG] = spdk_sprintf_alloc("%s", opts->name);
if (g_arg_strings[EAL_PROGNAME_ARG] == NULL) {
rte_exit(EXIT_FAILURE, "g_arg_strings spdk_sprintf_alloc");
}
/*set the coremask */
core_mask = spdk_get_eal_coremask(opts->reactor_mask);
g_arg_strings[EAL_COREMASK_ARG] = spdk_sprintf_alloc("-c %llx", core_mask);
if (g_arg_strings[EAL_COREMASK_ARG] == NULL) {
spdk_free_ealargs();
rte_exit(EXIT_FAILURE, "g_arg_strings spdk_sprintf_alloc");
}
/* set the memory channel number */
g_arg_strings[EAL_MEMCHAN_ARG] = spdk_sprintf_alloc("-n %d", opts->dpdk_mem_channel);
if (g_arg_strings[EAL_MEMCHAN_ARG] == NULL) {
spdk_free_ealargs();
rte_exit(EXIT_FAILURE, "g_arg_strings spdk_sprintf_alloc");
}
/* set the memory size */
if (opts->dpdk_mem_size == -1)
opts->dpdk_mem_size = SPDK_APP_DPDK_DEFAULT_MEM_SIZE;
g_arg_strings[EAL_MEMSIZE_ARG] = spdk_sprintf_alloc("-m %d", opts->dpdk_mem_size);
if (g_arg_strings[EAL_MEMSIZE_ARG] == NULL) {
spdk_free_ealargs();
rte_exit(EXIT_FAILURE, "g_arg_strings spdk_sprintf_alloc");
}
/* set the master core */
g_arg_strings[EAL_MASTER_CORE_ARG] = spdk_sprintf_alloc("--master-lcore=%d",
opts->dpdk_master_core);
if (g_arg_strings[EAL_MASTER_CORE_ARG] == NULL) {
spdk_free_ealargs();
rte_exit(EXIT_FAILURE, "g_arg_strings spdk_sprintf_alloc");
}
#ifdef __linux__
/* set the hugepage file prefix */
g_arg_strings[EAL_FILE_PREFIX_ARG] = spdk_sprintf_alloc("--file-prefix=rte%d",
opts->instance_id);
#else
/* --file-prefix is not required on FreeBSD */
g_arg_strings[EAL_FILE_PREFIX_ARG] = strdup("");
#endif
if (g_arg_strings[EAL_FILE_PREFIX_ARG] == NULL) {
spdk_free_ealargs();
rte_exit(EXIT_FAILURE, "ealargs spdk_sprintf_alloc");
}
memcpy(g_ealargs, g_arg_strings, sizeof(g_arg_strings));
}
void
spdk_init_dpdk(struct spdk_app_opts *opts)
{
int i, rc;
static bool g_dpdk_initialized = false;
/* to make sure DPDK is only initialized once */
if (g_dpdk_initialized)
rte_exit(EXIT_FAILURE, "DPDK is already initialized\n");
spdk_build_eal_cmdline(opts);
printf("Starting Intel(R) DPDK initialization ... \n");
printf("[ DPDK EAL parameters: ");
for (i = 0; i < EAL_ARG_COUNT; i++) {
printf("%s ", g_ealargs[i]);
}
printf("]\n");
fflush(stdout);
rc = rte_eal_init(EAL_ARG_COUNT, g_ealargs);
spdk_free_ealargs();
if (rc < 0)
rte_exit(EXIT_FAILURE, "Invalid EAL arguments for DPDK\n");
g_dpdk_initialized = true;
printf("done.\n");
}

577
lib/event/reactor.c Normal file
View File

@ -0,0 +1,577 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/event.h"
#include <stdio.h>
#include <stdbool.h>
#include <string.h>
#include <unistd.h>
#ifdef __linux__
#include <sys/prctl.h>
#endif
#ifdef __FreeBSD__
#include <pthread_np.h>
#endif
#include <rte_config.h>
#include <rte_debug.h>
#include <rte_mempool.h>
#include <rte_ring.h>
#include <rte_timer.h>
#include "reactor.h"
#include "spdk/log.h"
enum spdk_reactor_state {
SPDK_REACTOR_STATE_INVALID = 0,
SPDK_REACTOR_STATE_INITIALIZED = 1,
SPDK_REACTOR_STATE_RUNNING = 2,
SPDK_REACTOR_STATE_EXITING = 3,
SPDK_REACTOR_STATE_SHUTDOWN = 4,
};
struct spdk_reactor {
/* Logical core number for this reactor. */
uint32_t lcore;
/*
* Contains pollers actively running on this reactor. Pollers
* are run round-robin. The reactor takes one poller from the head
* of the ring, executes it, then puts it back at the tail of
* the ring.
*/
struct rte_ring *active_pollers;
struct rte_ring *events;
};
static struct spdk_reactor g_reactors[RTE_MAX_LCORE];
static uint64_t g_reactor_mask = 0;
static int g_reactor_count = 0;
static enum spdk_reactor_state g_reactor_state = SPDK_REACTOR_STATE_INVALID;
static void spdk_reactor_construct(struct spdk_reactor *w, uint32_t lcore);
struct rte_mempool *g_spdk_event_mempool;
/** \file
*/
static struct spdk_reactor *
spdk_reactor_get(uint32_t lcore)
{
struct spdk_reactor *reactor;
reactor = &g_reactors[lcore];
return reactor;
}
spdk_event_t
spdk_event_allocate(uint32_t lcore, spdk_event_fn fn, void *arg1, void *arg2,
spdk_event_t next)
{
struct spdk_event *event = NULL;
int rc;
rc = rte_mempool_get(g_spdk_event_mempool, (void **)&event);
RTE_VERIFY((rc == 0) && (event != NULL));
event->lcore = lcore;
event->fn = fn;
event->arg1 = arg1;
event->arg2 = arg2;
event->next = next;
return event;
}
static void
spdk_event_free(struct spdk_event *event)
{
rte_mempool_put(g_spdk_event_mempool, (void *)event);
}
void
spdk_event_call(spdk_event_t event)
{
int rc;
struct spdk_reactor *reactor;
reactor = spdk_reactor_get(event->lcore);
RTE_VERIFY(reactor->events != NULL);
rc = rte_ring_enqueue(reactor->events, event);
RTE_VERIFY(rc == 0);
}
static uint32_t
spdk_event_queue_count(uint32_t lcore)
{
struct spdk_reactor *reactor;
reactor = spdk_reactor_get(lcore);
if (reactor->events == NULL) {
return 0;
}
return rte_ring_count(reactor->events);
}
static void
spdk_event_queue_run_single(uint32_t lcore)
{
struct spdk_event *event = NULL;
struct spdk_reactor *reactor;
int rc;
reactor = spdk_reactor_get(lcore);
RTE_VERIFY(reactor->events != NULL);
rc = rte_ring_dequeue(reactor->events, (void **)&event);
if ((rc != 0) || event == NULL) {
return;
}
event->fn(event);
spdk_event_free(event);
}
static void
spdk_event_queue_run(uint32_t lcore, uint32_t count)
{
while (count--) {
spdk_event_queue_run_single(lcore);
}
}
void
spdk_event_queue_run_all(uint32_t lcore)
{
uint32_t count;
count = spdk_event_queue_count(lcore);
spdk_event_queue_run(lcore, count);
}
/**
\brief Set current reactor thread name to "reactor <cpu #>".
This makes the reactor threads distinguishable in top and gdb.
*/
static void set_reactor_thread_name(void)
{
char thread_name[16];
snprintf(thread_name, sizeof(thread_name), "reactor %d",
rte_lcore_id());
#if defined(__linux__)
prctl(PR_SET_NAME, thread_name, 0, 0, 0);
#elif defined(__FreeBSD__)
pthread_set_name_np(pthread_self(), thread_name);
#else
#error missing platform support for thread name
#endif
}
/**
\brief This is the main function of the reactor thread.
\code
while (1)
if (new work items to be scheduled)
dequeue work item from new work item ring
enqueue work item to active work item ring
else if (active work item count > 0)
dequeue work item from active work item ring
invoke work item function pointer
if (work item state == RUNNING)
enqueue work item to active work item ring
else if (application state != RUNNING)
# exit the reactor loop
break
else
sleep for 100ms
\endcode
Note that new work items are posted to a separate ring so that the
active work item ring can be kept single producer/single consumer and
only be touched by reactor itself. This avoids atomic operations
on the active work item ring which would hurt performance.
*/
static int
_spdk_reactor_run(void *arg)
{
struct spdk_reactor *reactor = arg;
struct spdk_poller *poller = NULL;
int rc;
set_reactor_thread_name();
SPDK_NOTICELOG("waiting for work item to arrive...\n");
while (1) {
spdk_event_queue_run_all(rte_lcore_id());
rte_timer_manage();
if (rte_ring_dequeue(reactor->active_pollers, (void **)&poller) == 0) {
poller->fn(poller->arg);
rc = rte_ring_enqueue(reactor->active_pollers,
(void *)poller);
if (rc != 0) {
SPDK_ERRLOG("poller could not be enqueued\n");
exit(EXIT_FAILURE);
}
}
if (g_reactor_state != SPDK_REACTOR_STATE_RUNNING) {
break;
}
}
return 0;
}
static void
spdk_reactor_construct(struct spdk_reactor *reactor, uint32_t lcore)
{
char ring_name[64];
reactor->lcore = lcore;
snprintf(ring_name, sizeof(ring_name), "spdk_active_pollers_%d", lcore);
reactor->active_pollers =
rte_ring_create(ring_name, SPDK_POLLER_RING_SIZE, rte_lcore_to_socket_id(lcore),
RING_F_SP_ENQ | RING_F_SC_DEQ);
snprintf(ring_name, sizeof(ring_name) - 1, "spdk_event_queue_%u", lcore);
reactor->events =
rte_ring_create(ring_name, 65536, rte_lcore_to_socket_id(lcore), RING_F_SC_DEQ);
RTE_VERIFY(reactor->events != NULL);
}
static void
spdk_reactor_start(struct spdk_reactor *reactor)
{
if (reactor->lcore != rte_get_master_lcore()) {
switch (rte_eal_get_lcore_state(reactor->lcore)) {
case FINISHED:
rte_eal_wait_lcore(reactor->lcore);
/* drop through */
case WAIT:
rte_eal_remote_launch(_spdk_reactor_run, (void *)reactor, reactor->lcore);
break;
case RUNNING:
printf("Something already running on lcore %d\n", reactor->lcore);
break;
}
} else {
_spdk_reactor_run(reactor);
}
}
int
spdk_app_get_core_count(void)
{
return g_reactor_count;
}
int
spdk_app_parse_core_mask(const char *mask, uint64_t *cpumask)
{
unsigned int i;
char *end;
if (mask == NULL || cpumask == NULL) {
return -1;
}
errno = 0;
*cpumask = strtoull(mask, &end, 16);
if (*end != '\0' || errno) {
return -1;
}
for (i = 0; i < RTE_MAX_LCORE && i < 64; i++) {
if ((*cpumask & (1ULL << i)) && !rte_lcore_is_enabled(i)) {
*cpumask &= ~(1ULL << i);
}
}
return 0;
}
static int
spdk_reactor_parse_mask(const char *mask)
{
int i;
int ret = 0;
uint32_t master_core = rte_get_master_lcore();
if (g_reactor_state >= SPDK_REACTOR_STATE_INITIALIZED) {
SPDK_ERRLOG("cannot set reactor mask after application has started\n");
return -1;
}
g_reactor_mask = 0;
if (mask == NULL) {
/* No mask specified so use the same mask as DPDK. */
RTE_LCORE_FOREACH(i) {
g_reactor_mask |= (1ULL << i);
}
} else {
ret = spdk_app_parse_core_mask(mask, &g_reactor_mask);
if (ret != 0) {
SPDK_ERRLOG("reactor mask %s specified on command line "
"is invalid\n", mask);
return ret;
}
if (!(g_reactor_mask & (1ULL << master_core))) {
SPDK_ERRLOG("master_core %d must be set in core mask\n", master_core);
return -1;
}
}
return 0;
}
uint64_t
spdk_app_get_core_mask(void)
{
return g_reactor_mask;
}
static uint64_t
spdk_reactor_get_socket_mask(void)
{
int i;
uint32_t socket_id;
uint64_t socket_info = 0;
RTE_LCORE_FOREACH(i) {
if (((1ULL << i) & g_reactor_mask)) {
socket_id = rte_lcore_to_socket_id(i);
socket_info |= (1ULL << socket_id);
}
}
return socket_info;
}
void
spdk_reactor_subsystem_start(void)
{
struct spdk_reactor *reactor;
uint32_t i;
RTE_VERIFY(rte_get_master_lcore() == rte_lcore_id());
g_reactor_state = SPDK_REACTOR_STATE_RUNNING;
RTE_LCORE_FOREACH_SLAVE(i) {
if (((1ULL << i) & spdk_app_get_core_mask())) {
reactor = spdk_reactor_get(i);
spdk_reactor_start(reactor);
}
}
/* Start the master reactor */
reactor = spdk_reactor_get(rte_get_master_lcore());
spdk_reactor_start(reactor);
rte_eal_mp_wait_lcore();
g_reactor_state = SPDK_REACTOR_STATE_SHUTDOWN;
}
void spdk_reactor_subsystem_stop(void)
{
g_reactor_state = SPDK_REACTOR_STATE_EXITING;
}
int
spdk_reactor_subsystem_init(const char *mask)
{
uint32_t i;
int rc;
struct spdk_reactor *reactor;
rc = spdk_reactor_parse_mask(mask);
if (rc < 0) {
return rc;
}
printf("Occupied cpu core mask is 0x%lx\n", spdk_app_get_core_mask());
printf("Occupied cpu socket mask is 0x%lx\n", spdk_reactor_get_socket_mask());
RTE_LCORE_FOREACH(i) {
if (((1ULL << i) & spdk_app_get_core_mask())) {
reactor = spdk_reactor_get(i);
spdk_reactor_construct(reactor, i);
g_reactor_count++;
}
}
/* TODO: separate event mempools per socket */
g_spdk_event_mempool = rte_mempool_create("spdk_event_mempool", 262144,
sizeof(struct spdk_event), 128, 0,
NULL, NULL, NULL, NULL,
SOCKET_ID_ANY, 0);
if (g_spdk_event_mempool == NULL) {
SPDK_ERRLOG("spdk_event_mempool allocation failed\n");
return -1;
}
g_reactor_state = SPDK_REACTOR_STATE_INITIALIZED;
return rc;
}
int
spdk_reactor_subsystem_fini(void)
{
/* TODO: free rings and mempool */
return 0;
}
static void
_spdk_event_add_poller(spdk_event_t event)
{
struct spdk_reactor *reactor = spdk_event_get_arg1(event);
struct spdk_poller *poller = spdk_event_get_arg2(event);
struct spdk_event *next = spdk_event_get_next(event);
poller->lcore = reactor->lcore;
rte_ring_enqueue(reactor->active_pollers, (void *)poller);
if (next) {
spdk_event_call(next);
}
}
void
spdk_poller_register(struct spdk_poller *poller,
uint32_t lcore, spdk_event_t complete)
{
struct spdk_reactor *reactor;
struct spdk_event *event;
reactor = spdk_reactor_get(lcore);
event = spdk_event_allocate(lcore, _spdk_event_add_poller, reactor, poller, complete);
spdk_event_call(event);
}
static void
_spdk_event_remove_poller(spdk_event_t event)
{
struct spdk_reactor *reactor = spdk_event_get_arg1(event);
struct spdk_poller *poller = spdk_event_get_arg2(event);
struct spdk_event *next = spdk_event_get_next(event);
struct spdk_poller *tmp = NULL;
uint32_t i;
int rc;
/* Loop over all pollers, without breaking early, so that
* the list of pollers stays in the same order. */
for (i = 0; i < rte_ring_count(reactor->active_pollers); i++) {
rte_ring_dequeue(reactor->active_pollers, (void **)&tmp);
if (tmp != poller) {
rc = rte_ring_enqueue(reactor->active_pollers, (void *)tmp);
if (rc != 0) {
SPDK_ERRLOG("poller could not be enqueued\n");
exit(EXIT_FAILURE);
}
}
}
if (next) {
spdk_event_call(next);
}
}
void
spdk_poller_unregister(struct spdk_poller *poller,
struct spdk_event *complete)
{
struct spdk_reactor *reactor;
struct spdk_event *event;
reactor = spdk_reactor_get(poller->lcore);
event = spdk_event_allocate(poller->lcore, _spdk_event_remove_poller, reactor, poller, complete);
spdk_event_call(event);
}
static void
_spdk_poller_migrate(spdk_event_t event)
{
struct spdk_poller *poller = spdk_event_get_arg1(event);
struct spdk_event *next = spdk_event_get_next(event);
/* Register the poller on the current lcore. This works
* because we already set this event up so that it is called
* on the new_lcore.
*/
spdk_poller_register(poller, rte_lcore_id(), next);
}
void
spdk_poller_migrate(struct spdk_poller *poller, int new_lcore,
struct spdk_event *complete)
{
struct spdk_event *event;
RTE_VERIFY(spdk_app_get_core_mask() & (1ULL << new_lcore));
RTE_VERIFY(poller != NULL);
event = spdk_event_allocate(new_lcore, _spdk_poller_migrate, poller, NULL, complete);
spdk_poller_unregister(poller, event);
}

43
lib/event/reactor.h Normal file
View File

@ -0,0 +1,43 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef SPDK_REACTOR_H_
#define SPDK_REACTOR_H_
int spdk_reactor_subsystem_init(const char *mask);
int spdk_reactor_subsystem_fini(void);
void spdk_reactor_subsystem_start(void);
void spdk_reactor_subsystem_stop(void);
#endif

172
lib/event/subsystem.c Normal file
View File

@ -0,0 +1,172 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/event.h"
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include "subsystem.h"
static TAILQ_HEAD(spdk_subsystem_list, spdk_subsystem) g_subsystems =
TAILQ_HEAD_INITIALIZER(g_subsystems);
static TAILQ_HEAD(subsystem_depend, spdk_subsystem_depend) g_depends =
TAILQ_HEAD_INITIALIZER(g_depends);
void
spdk_add_subsystem(struct spdk_subsystem *subsystem)
{
TAILQ_INSERT_TAIL(&g_subsystems, subsystem, tailq);
}
void
spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend)
{
TAILQ_INSERT_TAIL(&g_depends, depend, tailq);
}
static struct spdk_subsystem *
spdk_subsystem_find(struct spdk_subsystem_list *list, const char *name)
{
struct spdk_subsystem *iter;
TAILQ_FOREACH(iter, list, tailq) {
if (strcmp(name, iter->name) == 0) {
return iter;
}
}
return NULL;
}
static void
subsystem_sort(void)
{
bool depends_on, depends_on_sorted;
struct spdk_subsystem *subsystem, *subsystem_tmp;
struct spdk_subsystem_depend *subsystem_dep;
struct spdk_subsystem_list subsystems_list = TAILQ_HEAD_INITIALIZER(subsystems_list);
while (!TAILQ_EMPTY(&g_subsystems)) {
TAILQ_FOREACH_SAFE(subsystem, &g_subsystems, tailq, subsystem_tmp) {
depends_on = false;
TAILQ_FOREACH(subsystem_dep, &g_depends, tailq) {
if (strcmp(subsystem->name, subsystem_dep->name) == 0) {
depends_on = true;
depends_on_sorted = !!spdk_subsystem_find(&subsystems_list, subsystem_dep->depends_on);
if (depends_on_sorted)
continue;
break;
}
}
if (depends_on == false) {
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
TAILQ_INSERT_TAIL(&subsystems_list, subsystem, tailq);
} else {
if (depends_on_sorted == true) {
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
TAILQ_INSERT_TAIL(&subsystems_list, subsystem, tailq);
}
}
}
}
TAILQ_FOREACH_SAFE(subsystem, &subsystems_list, tailq, subsystem_tmp) {
TAILQ_REMOVE(&subsystems_list, subsystem, tailq);
TAILQ_INSERT_TAIL(&g_subsystems, subsystem, tailq);
}
}
int
spdk_subsystem_init(void)
{
int rc = 0;
struct spdk_subsystem *subsystem;
struct spdk_subsystem_depend *dep;
/* Verify that all dependency name and depends_on subsystems are registered */
TAILQ_FOREACH(dep, &g_depends, tailq) {
if (!spdk_subsystem_find(&g_subsystems, dep->name)) {
fprintf(stderr, "subsystem %s is missing\n", dep->name);
return -1;
}
if (!spdk_subsystem_find(&g_subsystems, dep->depends_on)) {
fprintf(stderr, "subsystem %s dependency %s is missing\n",
dep->name, dep->depends_on);
return -1;
}
}
subsystem_sort();
TAILQ_FOREACH(subsystem, &g_subsystems, tailq) {
if (subsystem->init) {
rc = subsystem->init();
if (rc)
return rc;
}
}
return rc;
}
int
spdk_subsystem_fini(void)
{
int rc = 0;
struct spdk_subsystem *cur;
cur = TAILQ_LAST(&g_subsystems, spdk_subsystem_list);
do {
if (cur->fini) {
rc = cur->fini();
if (rc)
return rc;
}
cur = TAILQ_PREV(cur, spdk_subsystem_list, tailq);
} while (cur);
return rc;
}
void
spdk_subsystem_config(FILE *fp)
{
struct spdk_subsystem *subsystem;
TAILQ_FOREACH(subsystem, &g_subsystems, tailq) {
if (subsystem->config)
subsystem->config(fp);
}
}

43
lib/event/subsystem.h Normal file
View File

@ -0,0 +1,43 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef SPDK_SUBSYSTEM_H_
#define SPDK_SUBSYSTEM_H_
#include <stdio.h>
int spdk_subsystem_init(void);
int spdk_subsystem_fini(void);
void spdk_subsystem_config(FILE *fp);
#endif

2
mk/spdk.common.mk
View File

@ -151,7 +151,7 @@ DPDK_INC_DIR ?= $(DPDK_DIR_ABS)/include
DPDK_LIB_DIR ?= $(DPDK_DIR_ABS)/lib
DPDK_INC = -I$(DPDK_INC_DIR)
DPDK_LIB = -L$(DPDK_LIB_DIR) -lrte_eal -lrte_mempool -lrte_ring -Wl,-rpath=$(DPDK_LIB_DIR)
DPDK_LIB = -L$(DPDK_LIB_DIR) -lrte_eal -lrte_mempool -lrte_ring -lrte_timer -Wl,-rpath=$(DPDK_LIB_DIR)
# librte_malloc was removed after DPDK 2.1. Link this library conditionally based on its
# existence to maintain backward compatibility.
ifneq ($(wildcard $(DPDK_DIR_ABS)/lib/librte_malloc.*),)

2
test/lib/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = log json jsonrpc nvme memory ioat
DIRS-y = event log json jsonrpc nvme memory ioat
.PHONY: all clean $(DIRS-y)

44
test/lib/event/Makefile Normal file
View File

@ -0,0 +1,44 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = event subsystem
.PHONY: all clean $(DIRS-y)
all: $(DIRS-y)
clean: $(DIRS-y)
include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

10
test/lib/event/event.sh Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
testdir=$(readlink -f $(dirname $0))
rootdir=$testdir/../../..
source $rootdir/scripts/autotest_common.sh
timing_enter event
$testdir/event/event -m 0xF -t 5
$testdir/subsystem/subsystem_ut
timing_exit event

1
test/lib/event/event/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
event

57
test/lib/event/event/Makefile Normal file
View File

@ -0,0 +1,57 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
CFLAGS += $(DPDK_INC)
APP = event
C_SRCS := event.c
SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/event/libspdk_event.a \
$(SPDK_ROOT_DIR)/lib/trace/libspdk_trace.a \
$(SPDK_ROOT_DIR)/lib/conf/libspdk_conf.a \
$(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
$(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
LIBS += $(SPDK_LIBS) -lpthread $(DPDK_LIB) -lrt
all : $(APP)
$(APP) : $(OBJS) $(SPDK_LIBS)
$(LINK_C)
clean :
$(CLEAN_C) $(APP)
include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

178
test/lib/event/event/event.c Normal file
View File

@ -0,0 +1,178 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <unistd.h>
#include <rte_config.h>
#include <rte_eal.h>
#include <rte_debug.h>
#include <rte_mempool.h>
#include <rte_cycles.h>
#include <rte_malloc.h>
#include <rte_ring.h>
#include <rte_lcore.h>
#include "spdk/event.h"
#include "spdk/log.h"
static uint64_t g_tsc_rate;
static uint64_t g_tsc_us_rate;
static int g_time_in_sec;
static __thread uint64_t __call_count = 0;
static uint64_t call_count[RTE_MAX_LCORE];
static void
submit_new_event(spdk_event_t event)
{
static __thread uint32_t next_lcore = RTE_MAX_LCORE;
if (next_lcore == RTE_MAX_LCORE) {
next_lcore = rte_get_next_lcore(rte_lcore_id(), 0, 1);
}
++__call_count;
event = spdk_event_allocate(next_lcore, submit_new_event, NULL, NULL, NULL);
spdk_event_call(event);
}
static int
event_work_fn(void *arg)
{
uint64_t tsc_end;
tsc_end = rte_get_timer_cycles() + g_time_in_sec * g_tsc_rate;
submit_new_event(NULL);
submit_new_event(NULL);
submit_new_event(NULL);
submit_new_event(NULL);
while (1) {
spdk_event_queue_run_all(rte_lcore_id());
if (rte_get_timer_cycles() > tsc_end) {
break;
}
}
call_count[rte_lcore_id()] = __call_count;
return 0;
}
static void
usage(char *program_name)
{
printf("%s options\n", program_name);
printf("\t[-m core mask for distributing I/O submission/completion work\n");
printf("\t\t(default: 0x1 - use core 0 only)]\n");
printf("\t[-t time in seconds]\n");
}
static void
performance_dump(int io_time)
{
uint32_t i;
printf("\n");
RTE_LCORE_FOREACH(i) {
printf("lcore %2d: %8ju\n", i, call_count[i] / g_time_in_sec);
}
fflush(stdout);
}
int
main(int argc, char **argv)
{
struct spdk_app_opts opts;
int op;
int i;
spdk_app_opts_init(&opts);
opts.name = "event";
g_time_in_sec = 0;
while ((op = getopt(argc, argv, "m:t:")) != -1) {
switch (op) {
case 'm':
opts.reactor_mask = optarg;
break;
case 't':
g_time_in_sec = atoi(optarg);
break;
default:
usage(argv[0]);
exit(1);
}
}
if (!g_time_in_sec) {
usage(argv[0]);
exit(1);
}
optind = 1; /*reset the optind */
spdk_init_dpdk(&opts);
spdk_app_init(&opts);
g_tsc_rate = rte_get_timer_hz();
g_tsc_us_rate = g_tsc_rate / (1000 * 1000);
printf("Running I/O for %d seconds...", g_time_in_sec);
fflush(stdout);
/* call event_work_fn on each slave lcore */
RTE_LCORE_FOREACH_SLAVE(i) {
rte_eal_remote_launch(event_work_fn, NULL, i);
}
/* call event_work_fn on lcore0 */
event_work_fn(NULL);
rte_eal_mp_wait_lcore();
performance_dump(g_time_in_sec);
printf("done.\n");
return 0;
}

1
test/lib/event/subsystem/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
subsystem_ut

57
test/lib/event/subsystem/Makefile Normal file
View File

@ -0,0 +1,57 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
CFLAGS += -I$(SPDK_ROOT_DIR)/lib/event
APP = subsystem_ut
C_SRCS := subsystem_ut.c
SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/event/libspdk_event.a \
$(SPDK_ROOT_DIR)/lib/trace/libspdk_trace.a \
$(SPDK_ROOT_DIR)/lib/conf/libspdk_conf.a \
$(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
$(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
LIBS += $(SPDK_LIBS) -lpthread -lrt -lcunit
all : $(APP)
$(APP) : $(OBJS) $(SPDK_LIBS)
$(LINK_C)
clean :
$(CLEAN_C) $(APP)
include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

206
test/lib/event/subsystem/subsystem_ut.c Normal file
View File

@ -0,0 +1,206 @@
/*-
* <COPYRIGHT_TAG>
*/
#include <stdint.h>
#include <stdlib.h>
#include <CUnit/Basic.h>
#include "subsystem.c"
static struct spdk_subsystem g_ut_subsystems[8];
static struct spdk_subsystem_depend g_ut_subsystem_deps[8];
static void
set_up_subsystem(struct spdk_subsystem *subsystem, const char *name)
{
subsystem->init = NULL;
subsystem->fini = NULL;
subsystem->config = NULL;
subsystem->name = name;
}
static void
set_up_depends(struct spdk_subsystem_depend *depend, const char *subsystem_name,
const char *dpends_on_name)
{
depend->name = subsystem_name;
depend->depends_on = dpends_on_name;
}
static void
subsystem_clear(void)
{
struct spdk_subsystem *subsystem, *subsystem_tmp;
struct spdk_subsystem_depend *subsystem_dep, *subsystem_dep_tmp;
TAILQ_FOREACH_SAFE(subsystem, &g_subsystems, tailq, subsystem_tmp) {
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
}
TAILQ_FOREACH_SAFE(subsystem_dep, &g_depends, tailq, subsystem_dep_tmp) {
TAILQ_REMOVE(&g_depends, subsystem_dep, tailq);
}
}
static void
subsystem_sort_test_depends_on_single(void)
{
struct spdk_subsystem *subsystem;
int i;
char subsystem_name[16];
spdk_subsystem_init();
i = 4;
TAILQ_FOREACH(subsystem, &g_subsystems, tailq) {
snprintf(subsystem_name, sizeof(subsystem_name), "subsystem%d", i);
i--;
CU_ASSERT(strcmp(subsystem_name, subsystem->name) == 0);
}
}
static void
subsystem_sort_test_depends_on_multiple(void)
{
int i;
struct spdk_subsystem *subsystem;
subsystem_clear();
set_up_subsystem(&g_ut_subsystems[0], "iscsi");
set_up_subsystem(&g_ut_subsystems[1], "nvmf");
set_up_subsystem(&g_ut_subsystems[2], "sock");
set_up_subsystem(&g_ut_subsystems[3], "bdev");
set_up_subsystem(&g_ut_subsystems[4], "rpc");
set_up_subsystem(&g_ut_subsystems[5], "scsi");
set_up_subsystem(&g_ut_subsystems[6], "interface");
set_up_subsystem(&g_ut_subsystems[7], "copy");
for (i = 0; i < 8; i++) {
spdk_add_subsystem(&g_ut_subsystems[i]);
}
set_up_depends(&g_ut_subsystem_deps[0], "bdev", "copy");
set_up_depends(&g_ut_subsystem_deps[1], "scsi", "bdev");
set_up_depends(&g_ut_subsystem_deps[2], "rpc", "interface");
set_up_depends(&g_ut_subsystem_deps[3], "sock", "interface");
set_up_depends(&g_ut_subsystem_deps[4], "nvmf", "interface");
set_up_depends(&g_ut_subsystem_deps[5], "iscsi", "scsi");
set_up_depends(&g_ut_subsystem_deps[6], "iscsi", "sock");
set_up_depends(&g_ut_subsystem_deps[7], "iscsi", "rpc");
for (i = 0; i < 8; i++) {
spdk_add_subsystem_depend(&g_ut_subsystem_deps[i]);
}
spdk_subsystem_init();
subsystem = TAILQ_FIRST(&g_subsystems);
CU_ASSERT(strcmp(subsystem->name, "interface") == 0);
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
subsystem = TAILQ_FIRST(&g_subsystems);
CU_ASSERT(strcmp(subsystem->name, "copy") == 0);
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
subsystem = TAILQ_FIRST(&g_subsystems);
CU_ASSERT(strcmp(subsystem->name, "nvmf") == 0);
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
subsystem = TAILQ_FIRST(&g_subsystems);
CU_ASSERT(strcmp(subsystem->name, "sock") == 0);
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
subsystem = TAILQ_FIRST(&g_subsystems);
CU_ASSERT(strcmp(subsystem->name, "bdev") == 0);
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
subsystem = TAILQ_FIRST(&g_subsystems);
CU_ASSERT(strcmp(subsystem->name, "rpc") == 0);
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
subsystem = TAILQ_FIRST(&g_subsystems);
CU_ASSERT(strcmp(subsystem->name, "scsi") == 0);
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
subsystem = TAILQ_FIRST(&g_subsystems);
CU_ASSERT(strcmp(subsystem->name, "iscsi") == 0);
TAILQ_REMOVE(&g_subsystems, subsystem, tailq);
}
SPDK_SUBSYSTEM_REGISTER(subsystem1, NULL, NULL, NULL)
SPDK_SUBSYSTEM_REGISTER(subsystem2, NULL, NULL, NULL)
SPDK_SUBSYSTEM_REGISTER(subsystem3, NULL, NULL, NULL)
SPDK_SUBSYSTEM_REGISTER(subsystem4, NULL, NULL, NULL)
SPDK_SUBSYSTEM_DEPEND(subsystem1, subsystem2)
SPDK_SUBSYSTEM_DEPEND(subsystem2, subsystem3)
SPDK_SUBSYSTEM_DEPEND(subsystem3, subsystem4)
static void
subsystem_sort_test_missing_dependency(void)
{
/*
* A depends on B, but B is missing
*/
subsystem_clear();
set_up_subsystem(&g_ut_subsystems[0], "A");
spdk_add_subsystem(&g_ut_subsystems[0]);
set_up_depends(&g_ut_subsystem_deps[0], "A", "B");
spdk_add_subsystem_depend(&g_ut_subsystem_deps[0]);
CU_ASSERT(spdk_subsystem_init() != 0);
/*
* Dependency from C to A is defined, but C is missing
*/
subsystem_clear();
set_up_subsystem(&g_ut_subsystems[0], "A");
spdk_add_subsystem(&g_ut_subsystems[0]);
set_up_depends(&g_ut_subsystem_deps[0], "C", "A");
spdk_add_subsystem_depend(&g_ut_subsystem_deps[0]);
CU_ASSERT(spdk_subsystem_init() != 0);
}
int
main(int argc, char **argv)
{
CU_pSuite suite = NULL;
unsigned int num_failures;
if (CU_initialize_registry() != CUE_SUCCESS) {
return CU_get_error();
}
suite = CU_add_suite("subsystem_suite", NULL, NULL);
if (suite == NULL) {
CU_cleanup_registry();
return CU_get_error();
}
if (
CU_add_test(suite, "subsystem_sort_test_depends_on_single",
subsystem_sort_test_depends_on_single) == NULL
|| CU_add_test(suite, "subsystem_sort_test_depends_on_multiple",
subsystem_sort_test_depends_on_multiple) == NULL
|| CU_add_test(suite, "subsystem_sort_test_missing_dependency",
subsystem_sort_test_missing_dependency) == NULL
) {
CU_cleanup_registry();
return CU_get_error();
}
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();
num_failures = CU_get_number_of_failures();
CU_cleanup_registry();
return num_failures;
}