ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

util: Add spdk_fd_group_nest() and spdk_fd_group_unnest

Browse Source 
These provide a way to nest one fd_group into another in a more
efficient manner than just adding the fd_group's fd to the parent. It
also keeps track of which events belong to which group, so the unnest
operation can be implemented.

Change-Id: I63d63365f1160cce8b4b6388a0ea2003ef424b9e
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15473
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
This commit is contained in:
Ben Walker 2022-11-16 12:33:59 -07:00 committed by Jim Harris
parent acad335521
commit d83e476240
3 changed files with 230 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
include/spdk/fd_group.h
View File

@ -75,6 +75,30 @@ int spdk_fd_group_wait(struct spdk_fd_group *fgrp, int timeout);
*/
int spdk_fd_group_get_fd(struct spdk_fd_group *fgrp);
/**
* Nest the child fd_group in the parent fd_group. After this operation
* completes, calling spdk_fd_group_wait() on the parent will include events
* from the child.
*
* \param parent The parent fd_group.
* \param child The child fd_group.
*
* \return 0 on success. Negated errno on failure. However, on all errno values other
* than -ENOTRECOVERABLE, the operation has not changed the state of the fd_group.
*/
int spdk_fd_group_nest(struct spdk_fd_group *parent, struct spdk_fd_group *child);
/**
* Remove the nested child from the parent.
*
* \param parent The parent fd_group.
* \param child The child fd_group.
*
* \return 0 on success. Negated errno on failure. However, on all errno values other
* than -ENOTRECOVERABLE, the operation has not changed the state of the fd_group.
*/
int spdk_fd_group_unnest(struct spdk_fd_group *parent, struct spdk_fd_group *child);
/**
* Register one event source to specified fgrp.
*

209
lib/util/fd_group.c
View File

@ -41,6 +41,7 @@ struct event_handler {
void *fn_arg;
/* file descriptor of the interrupt event */
int fd;
uint32_t events;
char name[SPDK_MAX_EVENT_NAME_LEN + 1];
};
@ -48,6 +49,8 @@ struct spdk_fd_group {
int epfd;
int num_fds; /* Number of fds registered in this group. */
struct spdk_fd_group *parent;
/* interrupt sources list */
TAILQ_HEAD(, event_handler) event_handlers;
};
@ -72,6 +75,155 @@ spdk_fd_group_get_epoll_event(struct epoll_event *event)
return 0;
}
static int
_fd_group_del_all(int epfd, struct spdk_fd_group *grp)
{
struct event_handler *ehdlr = NULL;
struct epoll_event epevent = {0};
int rc;
int ret = 0;
TAILQ_FOREACH(ehdlr, &grp->event_handlers, next) {
rc = epoll_ctl(epfd, EPOLL_CTL_DEL, ehdlr->fd, NULL);
if (rc < 0) {
if (errno == ENOENT) {
/* This is treated as success. It happens if there are multiple
* attempts to remove fds from the group.
*/
continue;
}
ret = -errno;
SPDK_ERRLOG("Failed to remove fd %d from group: %s\n", ehdlr->fd, strerror(errno));
goto recover;
}
}
return 0;
recover:
/* We failed to remove everything. Let's try to get everything put back into
* the original group. */
TAILQ_FOREACH(ehdlr, &grp->event_handlers, next) {
epevent.events = ehdlr->events;
epevent.data.ptr = ehdlr;
rc = epoll_ctl(epfd, EPOLL_CTL_ADD, ehdlr->fd, &epevent);
if (rc < 0) {
if (errno == EEXIST) {
/* This is fine. Keep going. */
continue;
}
/* Continue on even though we've failed. But indicate
* this is a fatal error. */
SPDK_ERRLOG("Failed to recover fd_group_del_all: %s\n", strerror(errno));
ret = -ENOTRECOVERABLE;
}
}
return ret;
}
static int
_fd_group_add_all(int epfd, struct spdk_fd_group *grp)
{
struct event_handler *ehdlr = NULL;
struct epoll_event epevent = {0};
int rc;
int ret = 0;
/* Hoist the fds from the child up into the parent */
TAILQ_FOREACH(ehdlr, &grp->event_handlers, next) {
epevent.events = ehdlr->events;
epevent.data.ptr = ehdlr;
rc = epoll_ctl(epfd, EPOLL_CTL_ADD, ehdlr->fd, &epevent);
if (rc < 0) {
if (errno == EEXIST) {
/* This is treated as success */
continue;
}
ret = -errno;
SPDK_ERRLOG("Failed to add fd to fd group: %s\n", strerror(errno));
goto recover;
}
}
return 0;
recover:
/* We failed to add everything, so try to remove what we did add. */
TAILQ_FOREACH(ehdlr, &grp->event_handlers, next) {
rc = epoll_ctl(epfd, EPOLL_CTL_DEL, ehdlr->fd, NULL);
if (rc < 0) {
if (errno == ENOENT) {
/* This is treated as success. */
continue;
}
/* Continue on even though we've failed. But indicate
* this is a fatal error. */
SPDK_ERRLOG("Failed to recover fd_group_del_all: %s\n", strerror(errno));
ret = -ENOTRECOVERABLE;
}
}
return ret;
}
int
spdk_fd_group_unnest(struct spdk_fd_group *parent, struct spdk_fd_group *child)
{
int rc;
if (parent == NULL || child == NULL) {
return -EINVAL;
}
if (child->parent != parent) {
return -EINVAL;
}
rc = _fd_group_del_all(parent->epfd, child);
if (rc < 0) {
return rc;
}
child->parent = NULL;
return _fd_group_add_all(child->epfd, child);
}
int
spdk_fd_group_nest(struct spdk_fd_group *parent, struct spdk_fd_group *child)
{
int rc;
if (parent == NULL || child == NULL) {
return -EINVAL;
}
if (child->parent) {
return -EINVAL;
}
if (parent->parent) {
/* More than one layer of nesting is not currently supported */
assert(false);
return -ENOTSUP;
}
rc = _fd_group_del_all(child->epfd, child);
if (rc < 0) {
return rc;
}
child->parent = parent;
return _fd_group_add_all(parent->epfd, child);
}
int
spdk_fd_group_add(struct spdk_fd_group *fgrp, int efd, spdk_fd_fn fn,
void *arg, const char *name)
@ -79,6 +231,7 @@ spdk_fd_group_add(struct spdk_fd_group *fgrp, int efd, spdk_fd_fn fn,
struct event_handler *ehdlr = NULL;
struct epoll_event epevent = {0};
int rc;
int epfd;
/* parameter checking */
if (fgrp == NULL || efd < 0 || fn == NULL) {
@ -102,11 +255,18 @@ spdk_fd_group_add(struct spdk_fd_group *fgrp, int efd, spdk_fd_fn fn,
ehdlr->fn = fn;
ehdlr->fn_arg = arg;
ehdlr->state = EVENT_HANDLER_STATE_WAITING;
ehdlr->events = EPOLLIN;
snprintf(ehdlr->name, sizeof(ehdlr->name), "%s", name);
epevent.events = EPOLLIN;
if (fgrp->parent) {
epfd = fgrp->parent->epfd;
} else {
epfd = fgrp->epfd;
}
epevent.events = ehdlr->events;
epevent.data.ptr = ehdlr;
rc = epoll_ctl(fgrp->epfd, EPOLL_CTL_ADD, efd, &epevent);
rc = epoll_ctl(epfd, EPOLL_CTL_ADD, efd, &epevent);
if (rc < 0) {
free(ehdlr);
return -errno;
@ -123,6 +283,7 @@ spdk_fd_group_remove(struct spdk_fd_group *fgrp, int efd)
{
struct event_handler *ehdlr;
int rc;
int epfd;
if (fgrp == NULL || efd < 0) {
SPDK_ERRLOG("Invalid to remvoe efd(%d) from fd_group(%p).\n", efd, fgrp);
@ -144,7 +305,13 @@ spdk_fd_group_remove(struct spdk_fd_group *fgrp, int efd)
assert(ehdlr->state != EVENT_HANDLER_STATE_REMOVED);
rc = epoll_ctl(fgrp->epfd, EPOLL_CTL_DEL, ehdlr->fd, NULL);
if (fgrp->parent) {
epfd = fgrp->parent->epfd;
} else {
epfd = fgrp->epfd;
}
rc = epoll_ctl(epfd, EPOLL_CTL_DEL, ehdlr->fd, NULL);
if (rc < 0) {
SPDK_ERRLOG("Failed to delete the fd(%d) from the epoll group(%p)\n", efd, fgrp);
return;
@ -168,6 +335,7 @@ spdk_fd_group_event_modify(struct spdk_fd_group *fgrp,
{
struct epoll_event epevent;
struct event_handler *ehdlr;
int epfd;
if (fgrp == NULL || efd < 0) {
return -EINVAL;
@ -185,10 +353,18 @@ spdk_fd_group_event_modify(struct spdk_fd_group *fgrp,
assert(ehdlr->state != EVENT_HANDLER_STATE_REMOVED);
epevent.events = event_types;
ehdlr->events = event_types;
if (fgrp->parent) {
epfd = fgrp->parent->epfd;
} else {
epfd = fgrp->epfd;
}
epevent.events = ehdlr->events;
epevent.data.ptr = ehdlr;
return epoll_ctl(fgrp->epfd, EPOLL_CTL_MOD, ehdlr->fd, &epevent);
return epoll_ctl(epfd, EPOLL_CTL_MOD, ehdlr->fd, &epevent);
}
int
@ -244,6 +420,17 @@ spdk_fd_group_wait(struct spdk_fd_group *fgrp, int timeout)
int n;
int nfds;
if (fgrp->parent != NULL) {
if (timeout < 0) {
SPDK_ERRLOG("Calling spdk_fd_group_wait on a group nested in another group without a timeout will block indefinitely.\n");
assert(false);
return -EINVAL;
} else {
SPDK_WARNLOG("Calling spdk_fd_group_wait on a group nested in another group will never find any events\n");
return 0;
}
}
nfds = epoll_wait(fgrp->epfd, events, totalfds, timeout);
if (nfds < 0) {
if (errno != EINTR) {
@ -348,4 +535,16 @@ spdk_fd_group_wait(struct spdk_fd_group *fgrp, int timeout)
return -ENOTSUP;
}
int
spdk_fd_group_unnest(struct spdk_fd_group *parent, struct spdk_fd_group *child)
{
return -ENOTSUP;
}
int
spdk_fd_group_nest(struct spdk_fd_group *parent, struct spdk_fd_group *child)
{
return -ENOTSUP;
}
#endif

2
lib/util/spdk_util.map
View File

@ -165,6 +165,8 @@
spdk_fd_group_remove;
spdk_fd_group_event_modify;
spdk_fd_group_get_fd;
spdk_fd_group_nest;
spdk_fd_group_unnest;
# public functions in xor.h
spdk_xor_gen;