diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index dc2776adb..79e892610 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -969,7 +969,78 @@ nvme_qpair_free_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req) STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq); } -void nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child); +static inline void +nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child) +{ + assert(parent != NULL); + assert(child != NULL); + assert(child->parent == parent); + assert(parent->num_children != 0); + + parent->num_children--; + TAILQ_REMOVE(&parent->children, child, child_tailq); +} + +static inline void +nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl) +{ + struct nvme_request *child = child_arg; + struct nvme_request *parent = child->parent; + + nvme_request_remove_child(parent, child); + + if (spdk_nvme_cpl_is_error(cpl)) { + memcpy(&parent->parent_status, cpl, sizeof(*cpl)); + } + + if (parent->num_children == 0) { + nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair, + parent, &parent->parent_status); + nvme_free_request(parent); + } +} + +static inline void +nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child) +{ + assert(parent->num_children != UINT16_MAX); + + if (parent->num_children == 0) { + /* + * Defer initialization of the children TAILQ since it falls + * on a separate cacheline. This ensures we do not touch this + * cacheline except on request splitting cases, which are + * relatively rare. + */ + TAILQ_INIT(&parent->children); + parent->parent = NULL; + memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl)); + } + + parent->num_children++; + TAILQ_INSERT_TAIL(&parent->children, child, child_tailq); + child->parent = parent; + child->cb_fn = nvme_cb_complete_child; + child->cb_arg = child; +} + +static inline void +nvme_request_free_children(struct nvme_request *req) +{ + struct nvme_request *child, *tmp; + + if (req->num_children == 0) { + return; + } + + /* free all child nvme_request */ + TAILQ_FOREACH_SAFE(child, &req->children, child_tailq, tmp) { + nvme_request_remove_child(req, child); + nvme_request_free_children(child); + nvme_free_request(child); + } +} + int nvme_request_check_timeout(struct nvme_request *req, uint16_t cid, struct spdk_nvme_ctrlr_process *active_proc, uint64_t now_tick); uint64_t nvme_get_quirks(const struct spdk_pci_id *id); diff --git a/lib/nvme/nvme_ns_cmd.c b/lib/nvme/nvme_ns_cmd.c index 0ed3b1408..f9debf36f 100644 --- a/lib/nvme/nvme_ns_cmd.c +++ b/lib/nvme/nvme_ns_cmd.c @@ -62,78 +62,6 @@ spdk_nvme_ns_check_request_length(uint32_t lba_count, uint32_t sectors_per_max_i return child_per_io >= qdepth; } -static void -nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl) -{ - struct nvme_request *child = child_arg; - struct nvme_request *parent = child->parent; - - nvme_request_remove_child(parent, child); - - if (spdk_nvme_cpl_is_error(cpl)) { - memcpy(&parent->parent_status, cpl, sizeof(*cpl)); - } - - if (parent->num_children == 0) { - nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair, - parent, &parent->parent_status); - nvme_free_request(parent); - } -} - -static void -nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child) -{ - assert(parent->num_children != UINT16_MAX); - - if (parent->num_children == 0) { - /* - * Defer initialization of the children TAILQ since it falls - * on a separate cacheline. This ensures we do not touch this - * cacheline except on request splitting cases, which are - * relatively rare. - */ - TAILQ_INIT(&parent->children); - parent->parent = NULL; - memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl)); - } - - parent->num_children++; - TAILQ_INSERT_TAIL(&parent->children, child, child_tailq); - child->parent = parent; - child->cb_fn = nvme_cb_complete_child; - child->cb_arg = child; -} - -void -nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child) -{ - assert(parent != NULL); - assert(child != NULL); - assert(child->parent == parent); - assert(parent->num_children != 0); - - parent->num_children--; - TAILQ_REMOVE(&parent->children, child, child_tailq); -} - -static void -nvme_request_free_children(struct nvme_request *req) -{ - struct nvme_request *child, *tmp; - - if (req->num_children == 0) { - return; - } - - /* free all child nvme_request */ - TAILQ_FOREACH_SAFE(child, &req->children, child_tailq, tmp) { - nvme_request_remove_child(req, child); - nvme_request_free_children(child); - nvme_free_request(child); - } -} - static struct nvme_request * _nvme_add_child_request(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, const struct nvme_payload *payload, diff --git a/test/unit/lib/nvme/nvme_qpair.c/nvme_qpair_ut.c b/test/unit/lib/nvme/nvme_qpair.c/nvme_qpair_ut.c index 9034df6f1..16e0d974c 100644 --- a/test/unit/lib/nvme/nvme_qpair.c/nvme_qpair_ut.c +++ b/test/unit/lib/nvme/nvme_qpair.c/nvme_qpair_ut.c @@ -48,14 +48,6 @@ struct nvme_driver _g_nvme_driver = { .lock = PTHREAD_MUTEX_INITIALIZER, }; -void -nvme_request_remove_child(struct nvme_request *parent, - struct nvme_request *child) -{ - parent->num_children--; - TAILQ_REMOVE(&parent->children, child, child_tailq); -} - void nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr) {