diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 1c3b56c36..1436690c5 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -769,8 +769,7 @@ static void ftl_wptr_pad_band(struct ftl_wptr *wptr) { struct spdk_ftl_dev *dev = wptr->dev; - size_t size = ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_INTERNAL) + - ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_USER); + size_t size = ftl_rwb_num_pending(dev->rwb); size_t blocks_left, rwb_size, pad_size; blocks_left = ftl_wptr_user_lbks_left(wptr); @@ -787,8 +786,7 @@ static void ftl_wptr_process_shutdown(struct ftl_wptr *wptr) { struct spdk_ftl_dev *dev = wptr->dev; - size_t size = ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_INTERNAL) + - ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_USER); + size_t size = ftl_rwb_num_pending(dev->rwb); size_t num_active = dev->xfer_size * ftl_rwb_get_active_batches(dev->rwb); num_active = num_active ? num_active : dev->xfer_size; diff --git a/lib/ftl/ftl_rwb.c b/lib/ftl/ftl_rwb.c index 854c9ae23..20e098c13 100644 --- a/lib/ftl/ftl_rwb.c +++ b/lib/ftl/ftl_rwb.c @@ -83,6 +83,8 @@ struct ftl_rwb { /* Number of acquired entries */ unsigned int num_acquired[FTL_RWB_TYPE_MAX]; + /* Number of acquired but not yet submitted entries */ + unsigned int num_pending; /* User/internal limits */ size_t limits[FTL_RWB_TYPE_MAX]; @@ -302,7 +304,7 @@ ftl_rwb_batch_release(struct ftl_rwb_batch *batch) num_acquired = __atomic_fetch_sub(&rwb->num_acquired[ftl_rwb_entry_type(entry)], 1, __ATOMIC_SEQ_CST); entry->band = NULL; - assert(num_acquired > 0); + assert(num_acquired > 0); } pthread_spin_lock(&rwb->lock); @@ -371,6 +373,14 @@ ftl_rwb_batch_revert(struct ftl_rwb_batch *batch) if (spdk_ring_enqueue(rwb->prio_queue, (void **)&batch, 1, NULL) != 1) { assert(0 && "Should never happen"); } + + __atomic_fetch_add(&rwb->num_pending, rwb->xfer_size, __ATOMIC_SEQ_CST); +} + +unsigned int +ftl_rwb_num_pending(struct ftl_rwb *rwb) +{ + return __atomic_load_n(&rwb->num_pending, __ATOMIC_SEQ_CST); } void @@ -457,6 +467,7 @@ ftl_rwb_acquire(struct ftl_rwb *rwb, enum ftl_rwb_entry_type type) pthread_spin_unlock(&rwb->lock); __atomic_fetch_add(&rwb->num_acquired[type], 1, __ATOMIC_SEQ_CST); + __atomic_fetch_add(&rwb->num_pending, 1, __ATOMIC_SEQ_CST); return entry; error: pthread_spin_unlock(&rwb->lock); @@ -491,12 +502,19 @@ struct ftl_rwb_batch * ftl_rwb_pop(struct ftl_rwb *rwb) { struct ftl_rwb_batch *batch = NULL; + unsigned int num_pending __attribute__((unused)); if (spdk_ring_dequeue(rwb->prio_queue, (void **)&batch, 1) == 1) { + num_pending = __atomic_fetch_sub(&rwb->num_pending, rwb->xfer_size, + __ATOMIC_SEQ_CST); + assert(num_pending > 0); return batch; } if (spdk_ring_dequeue(rwb->submit_queue, (void **)&batch, 1) == 1) { + num_pending = __atomic_fetch_sub(&rwb->num_pending, rwb->xfer_size, + __ATOMIC_SEQ_CST); + assert(num_pending > 0); return batch; } diff --git a/lib/ftl/ftl_rwb.h b/lib/ftl/ftl_rwb.h index ead170c59..cc7e0f1a9 100644 --- a/lib/ftl/ftl_rwb.h +++ b/lib/ftl/ftl_rwb.h @@ -118,6 +118,7 @@ struct ftl_rwb_entry *ftl_rwb_batch_first_entry(struct ftl_rwb_batch *batch); void *ftl_rwb_batch_get_data(struct ftl_rwb_batch *batch); void *ftl_rwb_batch_get_md(struct ftl_rwb_batch *batch); void ftl_rwb_disable_interleaving(struct ftl_rwb *rwb); +unsigned int ftl_rwb_num_pending(struct ftl_rwb *rwb); static inline void _ftl_rwb_entry_set_valid(struct ftl_rwb_entry *entry, bool valid)