From 0a04c076ea470671144f8b1693760c6d97ad4a96 Mon Sep 17 00:00:00 2001 From: Jan Kryl Date: Wed, 18 Sep 2019 20:16:15 +0000 Subject: [PATCH] nvmf: Add context parameter to new_qpair() callback It can be useful for passing additional information about nvmf target to a handler for new nvmf connections. Context can be stored in globals as it is currently done in nvmf code. However in case of multiple targets or languages where accessing global state is challenging (i.e. Rust), this becomes inconvenient. Change-Id: Ia6a2fdba4601531822b3e5fda7ac5ab89d46f6c5 Signed-off-by: Jan Kryl Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/469263 Community-CI: Broadcom SPDK FC-NVMe CI Tested-by: SPDK CI Jenkins Reviewed-by: Seth Howell Reviewed-by: Jim Harris Reviewed-by: Ben Walker Reviewed-by: Sasha Kotchubievsky --- CHANGELOG.md | 4 ++++ include/spdk/nvmf.h | 6 ++++-- lib/nvmf/fc.c | 2 +- lib/nvmf/fc_ls.c | 2 +- lib/nvmf/nvmf.c | 4 ++-- lib/nvmf/nvmf_fc.h | 1 + lib/nvmf/rdma.c | 12 ++++++------ lib/nvmf/tcp.c | 13 +++++++------ lib/nvmf/transport.c | 4 ++-- lib/nvmf/transport.h | 5 +++-- module/event/subsystems/nvmf/nvmf_tgt.c | 4 ++-- test/unit/lib/nvmf/fc_ls.c/fc_ls_ut.c | 2 +- 12 files changed, 34 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5387e2d1c..6f83d13c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,10 @@ Three new header functions have also been added to help deal with multiple targe `spdk_nvmf_get_first_target` takes no arguments and returns the first target in the global list. `spdk_nvmf_get_next_tgt` takes a target pointer as an argument and returns the next one in the global list. +The `spdk_nvmf_tgt_accept` takes additional argument allowing to pass arbitrary context +information to the `new_qpair` callback. This will simplify the code when having multiple +nvmf targets or when retrieving the context information from globals is not suitable. + ### bdev A new spdk_bdev_open_ext function has been added and spdk_bdev_open function has been deprecated. diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index 2de14c0c5..d188d4e02 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -218,8 +218,9 @@ void spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, * Function to be called for each newly discovered qpair. * * \param qpair The newly discovered qpair. + * \param cb_arg A context argument passed to this function. */ -typedef void (*new_qpair_fn)(struct spdk_nvmf_qpair *qpair); +typedef void (*new_qpair_fn)(struct spdk_nvmf_qpair *qpair, void *cb_arg); /** * Poll the target for incoming connections. @@ -230,8 +231,9 @@ typedef void (*new_qpair_fn)(struct spdk_nvmf_qpair *qpair); * * \param tgt The target associated with the listen address. * \param cb_fn Called for each newly discovered qpair. + * \param cb_arg A context argument passed to cb_fn. */ -void spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn); +void spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn, void *cb_arg); /** * Create a poll group. diff --git a/lib/nvmf/fc.c b/lib/nvmf/fc.c index 77da54d0c..8409805f9 100644 --- a/lib/nvmf/fc.c +++ b/lib/nvmf/fc.c @@ -1937,7 +1937,7 @@ nvmf_fc_stop_listen(struct spdk_nvmf_transport *transport, } static void -nvmf_fc_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) +nvmf_fc_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg) { struct spdk_nvmf_fc_port *fc_port = NULL; static bool start_lld = false; diff --git a/lib/nvmf/fc_ls.c b/lib/nvmf/fc_ls.c index ed67191ab..08011eb5d 100644 --- a/lib/nvmf/fc_ls.c +++ b/lib/nvmf/fc_ls.c @@ -595,7 +595,7 @@ nvmf_fc_ls_add_conn_to_poller( /* Let the nvmf_tgt decide which pollgroup to use. */ fc_conn->create_opd = opd; - fc_port->new_qp_cb(&fc_conn->qpair); + fc_port->new_qp_cb(&fc_conn->qpair, fc_port->new_qp_arg); } /* Delete association functions */ diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index f29a93358..ff5a2096a 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -677,12 +677,12 @@ spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, enum spdk_nvme_transport_ } void -spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn) +spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn, void *cb_arg) { struct spdk_nvmf_transport *transport, *tmp; TAILQ_FOREACH_SAFE(transport, &tgt->transports, link, tmp) { - spdk_nvmf_transport_accept(transport, cb_fn); + spdk_nvmf_transport_accept(transport, cb_fn, cb_arg); } } diff --git a/lib/nvmf/nvmf_fc.h b/lib/nvmf/nvmf_fc.h index d92499a01..f2408d525 100644 --- a/lib/nvmf/nvmf_fc.h +++ b/lib/nvmf/nvmf_fc.h @@ -311,6 +311,7 @@ struct spdk_nvmf_fc_port { uint16_t fcp_rq_id; struct spdk_nvmf_fc_hwqp ls_queue; new_qpair_fn new_qp_cb; + void *new_qp_arg; uint32_t num_io_queues; struct spdk_nvmf_fc_hwqp *io_queues; diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 6d4b4e96c..3fbabe73c 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -1244,7 +1244,7 @@ spdk_nvmf_rdma_event_reject(struct rdma_cm_id *id, enum spdk_nvmf_rdma_transport static int nvmf_rdma_connect(struct spdk_nvmf_transport *transport, struct rdma_cm_event *event, - new_qpair_fn cb_fn) + new_qpair_fn cb_fn, void *cb_arg) { struct spdk_nvmf_rdma_transport *rtransport; struct spdk_nvmf_rdma_qpair *rqpair = NULL; @@ -1340,7 +1340,7 @@ nvmf_rdma_connect(struct spdk_nvmf_transport *transport, struct rdma_cm_event *e event->id->context = &rqpair->qpair; - cb_fn(&rqpair->qpair); + cb_fn(&rqpair->qpair, cb_arg); return 0; } @@ -2926,7 +2926,7 @@ nvmf_rdma_handle_last_wqe_reached(void *ctx) } static void -spdk_nvmf_process_cm_event(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) +spdk_nvmf_process_cm_event(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg) { struct spdk_nvmf_rdma_transport *rtransport; struct rdma_cm_event *event; @@ -2953,7 +2953,7 @@ spdk_nvmf_process_cm_event(struct spdk_nvmf_transport *transport, new_qpair_fn c /* No action required. The target never attempts to resolve routes. */ break; case RDMA_CM_EVENT_CONNECT_REQUEST: - rc = nvmf_rdma_connect(transport, event, cb_fn); + rc = nvmf_rdma_connect(transport, event, cb_fn, cb_arg); if (rc < 0) { SPDK_ERRLOG("Unable to process connect event. rc: %d\n", rc); break; @@ -3091,7 +3091,7 @@ spdk_nvmf_process_ib_event(struct spdk_nvmf_rdma_device *device) } static void -spdk_nvmf_rdma_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) +spdk_nvmf_rdma_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg) { int nfds, i = 0; struct spdk_nvmf_rdma_transport *rtransport; @@ -3106,7 +3106,7 @@ spdk_nvmf_rdma_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) /* The first poll descriptor is RDMA CM event */ if (rtransport->poll_fds[i++].revents & POLLIN) { - spdk_nvmf_process_cm_event(transport, cb_fn); + spdk_nvmf_process_cm_event(transport, cb_fn, cb_arg); nfds--; } diff --git a/lib/nvmf/tcp.c b/lib/nvmf/tcp.c index 1241a8614..bcbbbb15f 100644 --- a/lib/nvmf/tcp.c +++ b/lib/nvmf/tcp.c @@ -1095,7 +1095,8 @@ spdk_nvmf_tcp_qpair_sock_init(struct spdk_nvmf_tcp_qpair *tqpair) static void _spdk_nvmf_tcp_handle_connect(struct spdk_nvmf_transport *transport, struct spdk_nvmf_tcp_port *port, - struct spdk_sock *sock, new_qpair_fn cb_fn) + struct spdk_sock *sock, + new_qpair_fn cb_fn, void *cb_arg) { struct spdk_nvmf_tcp_qpair *tqpair; int rc; @@ -1136,12 +1137,12 @@ _spdk_nvmf_tcp_handle_connect(struct spdk_nvmf_transport *transport, return; } - cb_fn(&tqpair->qpair); + cb_fn(&tqpair->qpair, cb_arg); } static void spdk_nvmf_tcp_port_accept(struct spdk_nvmf_transport *transport, struct spdk_nvmf_tcp_port *port, - new_qpair_fn cb_fn) + new_qpair_fn cb_fn, void *cb_arg) { struct spdk_sock *sock; int i; @@ -1149,13 +1150,13 @@ spdk_nvmf_tcp_port_accept(struct spdk_nvmf_transport *transport, struct spdk_nvm for (i = 0; i < NVMF_TCP_MAX_ACCEPT_SOCK_ONE_TIME; i++) { sock = spdk_sock_accept(port->listen_sock); if (sock) { - _spdk_nvmf_tcp_handle_connect(transport, port, sock, cb_fn); + _spdk_nvmf_tcp_handle_connect(transport, port, sock, cb_fn, cb_arg); } } } static void -spdk_nvmf_tcp_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) +spdk_nvmf_tcp_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg) { struct spdk_nvmf_tcp_transport *ttransport; struct spdk_nvmf_tcp_port *port; @@ -1163,7 +1164,7 @@ spdk_nvmf_tcp_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); TAILQ_FOREACH(port, &ttransport->ports, link) { - spdk_nvmf_tcp_port_accept(transport, port, cb_fn); + spdk_nvmf_tcp_port_accept(transport, port, cb_fn, cb_arg); } } diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index 04229bf88..a9e550de7 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -171,9 +171,9 @@ spdk_nvmf_transport_stop_listen(struct spdk_nvmf_transport *transport, } void -spdk_nvmf_transport_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn) +spdk_nvmf_transport_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg) { - transport->ops->accept(transport, cb_fn); + transport->ops->accept(transport, cb_fn, cb_arg); } void diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index 778dbc8fd..06af852d5 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -87,7 +87,7 @@ struct spdk_nvmf_transport_ops { /** * Check for new connections on the transport. */ - void (*accept)(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn); + void (*accept)(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg); /** * Fill out a discovery log entry for a specific listen address. @@ -183,7 +183,8 @@ struct spdk_nvmf_transport_ops { int spdk_nvmf_transport_stop_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_transport_id *trid); -void spdk_nvmf_transport_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn); +void spdk_nvmf_transport_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, + void *cb_arg); void spdk_nvmf_transport_listener_discover(struct spdk_nvmf_transport *transport, struct spdk_nvme_transport_id *trid, diff --git a/module/event/subsystems/nvmf/nvmf_tgt.c b/module/event/subsystems/nvmf/nvmf_tgt.c index 7b89b9dbc..d8b5767a4 100644 --- a/module/event/subsystems/nvmf/nvmf_tgt.c +++ b/module/event/subsystems/nvmf/nvmf_tgt.c @@ -250,7 +250,7 @@ nvmf_tgt_poll_group_add(void *_ctx) } static void -new_qpair(struct spdk_nvmf_qpair *qpair) +new_qpair(struct spdk_nvmf_qpair *qpair, void *cb_arg) { struct nvmf_tgt_pg_ctx *ctx; struct nvmf_tgt_poll_group *pg; @@ -294,7 +294,7 @@ acceptor_poll(void *arg) { struct spdk_nvmf_tgt *tgt = arg; - spdk_nvmf_tgt_accept(tgt, new_qpair); + spdk_nvmf_tgt_accept(tgt, new_qpair, NULL); return -1; } diff --git a/test/unit/lib/nvmf/fc_ls.c/fc_ls_ut.c b/test/unit/lib/nvmf/fc_ls.c/fc_ls_ut.c index 1ad8e6bf8..60b40e860 100644 --- a/test/unit/lib/nvmf/fc_ls.c/fc_ls_ut.c +++ b/test/unit/lib/nvmf/fc_ls.c/fc_ls_ut.c @@ -143,7 +143,7 @@ spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_ } static void -new_qpair(struct spdk_nvmf_qpair *qpair) +new_qpair(struct spdk_nvmf_qpair *qpair, void *cb_arg) { uint32_t i; struct spdk_nvmf_fc_conn *fc_conn;