ftl: I/O statistics

Add gathering of some performance counters and RPC for printing them.

Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Change-Id: I2e77d37fb66459240ff2e241f2b1f77c60f4eef4
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13390
Community-CI: Mellanox Build Bot
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

doc/jsonrpc.md

@@ -470,6 +470,7 @@ Example response:
     "bdev_ftl_create",
     "bdev_ftl_load",
     "bdev_ftl_unmap",
+    "bdev_ftl_get_stats",
     "bdev_lvol_get_lvstores",
     "bdev_lvol_delete",
     "bdev_lvol_resize",
@@ -4954,6 +4955,179 @@ Example response:
   "result": true
 }
 ~~~
+
+### bdev_ftl_get_stats {#rpc_bdev_ftl_get_stats}
+
+Get IO statistics for FTL bdev
+
+This RPC is subject to change.
+
+#### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+name                    | Required | string      | Bdev name
+
+#### Response
+
+The response is an object containing IO statistics for an FTL instance, split into multiple subobjects:
+
+- `user` - contains information about number of IOs, and errors for any incoming requests,
+- `cmp` - information about IO for the compaction process,
+- `gc` - information about IO for the garbage collection process,
+- `md_base` - internal metadata requests to the base FTL device,
+- `md_nv_cache` - internal metadata requests to the cache device,
+- `l2p` - requests done on the L2P cache region.
+
+Each subobject contains the following information:
+
+- `ios` - describes the total number of IOs requested,
+- `blocks` - the total number of requested blocks,
+- `errors` - describes the number of detected errors for a given operation, with the following distinctions:
+  - `media` - media errors,
+  - `crc` - mismatch in calculated CRC versus saved checksum in the metadata,
+  - `other` - any other errors.
+
+#### Example
+
+Example request:
+
+~~~json
+{
+  "params": {
+    "name": "ftl0"
+  },
+  "jsonrpc": "2.0",
+  "method": "bdev_ftl_get_stats",
+  "id": 1
+}
+~~~
+
+Example response:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": {
+      "name": "ftl0",
+      "user": {
+        "read": {
+          "ios": 0,
+          "blocks": 0,
+          "errors": {
+            "media": 0,
+            "crc": 0,
+            "other": 0
+          }
+        },
+        "write": {
+          "ios": 318707,
+          "blocks": 318707,
+          "errors": {
+            "media": 0,
+            "other": 0
+          }
+        }
+      },
+      "cmp": {
+        "read": {
+          "ios": 0,
+          "blocks": 0,
+          "errors": {
+            "media": 0,
+            "crc": 0,
+            "other": 0
+          }
+        },
+        "write": {
+          "ios": 0,
+          "blocks": 0,
+          "errors": {
+            "media": 0,
+            "other": 0
+          }
+        }
+      },
+      "gc": {
+        "read": {
+          "ios": 0,
+          "blocks": 0,
+          "errors": {
+            "media": 0,
+            "crc": 0,
+            "other": 0
+          }
+        },
+        "write": {
+          "ios": 0,
+          "blocks": 0,
+          "errors": {
+            "media": 0,
+            "other": 0
+          }
+        }
+      },
+      "md_base": {
+        "read": {
+          "ios": 0,
+          "blocks": 0,
+          "errors": {
+            "media": 0,
+            "crc": 0,
+            "other": 0
+          }
+        },
+        "write": {
+          "ios": 1,
+          "blocks": 32,
+          "errors": {
+            "media": 0,
+            "other": 0
+          }
+        }
+      },
+      "md_nv_cache": {
+        "read": {
+          "ios": 0,
+          "blocks": 0,
+          "errors": {
+            "media": 0,
+            "crc": 0,
+            "other": 0
+          }
+        },
+        "write": {
+          "ios": 1064,
+          "blocks": 1073896,
+          "errors": {
+            "media": 0,
+            "other": 0
+          }
+        }
+      },
+      "l2p": {
+        "read": {
+          "ios": 240659,
+          "blocks": 240659,
+          "errors": {
+            "media": 0,
+            "crc": 0,
+            "other": 0
+          }
+        },
+        "write": {
+          "ios": 235745,
+          "blocks": 235745,
+          "errors": {
+            "media": 0,
+            "other": 0
+          }
+        }
+      }
+    }
+}
+~~~
 ### bdev_pmem_create_pool {#rpc_bdev_pmem_create_pool}
 
 Create a @ref bdev_config_pmem blk pool file. It is equivalent of following `pmempool create` command:

include/spdk/ftl.h

@@ -27,6 +27,52 @@ enum {
 	SPDK_FTL_LIMIT_MAX
 };
 
+struct ftl_stats_error {
+	uint64_t media;
+	uint64_t crc;
+	uint64_t other;
+};
+
+struct ftl_stats_group {
+	uint64_t ios;
+	uint64_t blocks;
+	struct ftl_stats_error errors;
+};
+
+struct ftl_stats_entry {
+	struct ftl_stats_group read;
+	struct ftl_stats_group write;
+};
+
+enum ftl_stats_type {
+	FTL_STATS_TYPE_USER = 0,
+	FTL_STATS_TYPE_CMP,
+	FTL_STATS_TYPE_GC,
+	FTL_STATS_TYPE_MD_BASE,
+	FTL_STATS_TYPE_MD_NV_CACHE,
+	FTL_STATS_TYPE_L2P,
+	FTL_STATS_TYPE_MAX,
+};
+
+struct ftl_stats {
+	/* Number of times write limits were triggered by FTL writers
+	 * (gc and compaction) dependent on number of free bands. GC starts at
+	 * SPDK_FTL_LIMIT_START level, while at SPDK_FTL_LIMIT_CRIT compaction stops
+	 * and only GC is allowed to work.
+	 */
+	uint64_t		limits[SPDK_FTL_LIMIT_MAX];
+
+	/* Total number of blocks with IO to the underlying devices
+	 * 1. nv cache read/write
+	 * 2. base bdev read/write
+	 */
+	uint64_t		io_activity_total;
+
+	struct ftl_stats_entry	entries[FTL_STATS_TYPE_MAX];
+};
+
+typedef void (*spdk_ftl_stats_fn)(struct ftl_stats *stats, void *cb_arg);
+
 struct spdk_ftl_conf {
 	/* Device's name */
 	char					*name;
@@ -47,7 +93,7 @@ struct spdk_ftl_conf {
 	size_t					user_io_pool_size;
 
 	/* User writes limits */
-	size_t			                limits[SPDK_FTL_LIMIT_MAX];
+	size_t					limits[SPDK_FTL_LIMIT_MAX];
 
 	/* FTL startup mode mask, see spdk_ftl_mode enum for possible values */
 	uint32_t				mode;
@@ -234,6 +280,19 @@ size_t spdk_ftl_io_size(void);
  */
 void spdk_ftl_dev_set_fast_shutdown(struct spdk_ftl_dev *dev, bool fast_shutdown);
 
+/*
+ * Returns current FTL I/O statistics.
+ *
+ * \param dev Device
+ * \param stats Allocated ftl_stats
+ * \param cb_fn Callback function to invoke when the call is completed
+ * \param cb_arg Argument to pass to the callback function
+ *
+ * \return 0 if successfully submitted, negative errno otherwise.
+ */
+int spdk_ftl_get_stats(struct spdk_ftl_dev *dev, struct ftl_stats *stats, spdk_ftl_stats_fn cb_fn,
+		       void *cb_arg);
+
 #ifdef __cplusplus
 }
 #endif

lib/ftl/ftl_band_ops.c

@@ -15,6 +15,10 @@ static void
 write_rq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
 {
 	struct ftl_rq *rq = arg;
+	struct spdk_ftl_dev *dev = rq->dev;
+
+	ftl_stats_bdev_io_completed(dev, rq->owner.compaction ? FTL_STATS_TYPE_CMP : FTL_STATS_TYPE_GC,
+				    bdev_io);
 
 	rq->success = success;
 
@@ -61,7 +65,7 @@ ftl_band_rq_write(struct ftl_band *band, struct ftl_rq *rq)
 	ftl_band_rq_bdev_write(rq);
 
 	band->queue_depth++;
-	dev->io_activity_total += rq->num_blocks;
+	dev->stats.io_activity_total += rq->num_blocks;
 
 	ftl_band_iter_advance(band, rq->num_blocks);
 	if (ftl_band_filled(band, band->md->iter.offset)) {
@@ -79,6 +83,8 @@ read_rq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
 	struct ftl_band *band = entry->io.band;
 	struct ftl_rq *rq = ftl_rq_from_entry(entry);
 
+	ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_GC, bdev_io);
+
 	rq->success = success;
 	if (spdk_unlikely(!success)) {
 		ftl_band_rq_bdev_read(entry);
@@ -134,7 +140,7 @@ ftl_band_rq_read(struct ftl_band *band, struct ftl_rq *rq)
 
 	ftl_band_rq_bdev_read(entry);
 
-	dev->io_activity_total += rq->num_blocks;
+	dev->stats.io_activity_total += rq->num_blocks;
 	band->queue_depth++;
 }
 
@@ -144,6 +150,8 @@ write_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
 	struct ftl_basic_rq *brq = arg;
 	struct ftl_band *band = brq->io.band;
 
+	ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_MD_BASE, bdev_io);
+
 	brq->success = success;
 
 	assert(band->queue_depth > 0);
@@ -188,7 +196,7 @@ ftl_band_basic_rq_write(struct ftl_band *band, struct ftl_basic_rq *brq)
 
 	ftl_band_brq_bdev_write(brq);
 
-	dev->io_activity_total += brq->num_blocks;
+	dev->stats.io_activity_total += brq->num_blocks;
 	band->queue_depth++;
 	ftl_band_iter_advance(band, brq->num_blocks);
 	if (ftl_band_filled(band, band->md->iter.offset)) {
@@ -203,6 +211,8 @@ read_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
 	struct ftl_basic_rq *brq = arg;
 	struct ftl_band *band = brq->io.band;
 
+	ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_MD_BASE, bdev_io);
+
 	brq->success = success;
 
 	assert(band->queue_depth > 0);
@@ -245,7 +255,7 @@ ftl_band_basic_rq_read(struct ftl_band *band, struct ftl_basic_rq *brq)
 	ftl_band_brq_bdev_read(brq);
 
 	brq->io.band->queue_depth++;
-	dev->io_activity_total += brq->num_blocks;
+	dev->stats.io_activity_total += brq->num_blocks;
 }
 
 static void
@@ -404,6 +414,8 @@ read_md_cb(struct ftl_basic_rq *brq)
 	if (band->md->p2l_map_checksum && band->md->p2l_map_checksum != band_map_crc) {
 		FTL_ERRLOG(dev, "GC error, inconsistent P2L map CRC\n");
 		success = false;
+
+		ftl_stats_crc_error(band->dev, FTL_STATS_TYPE_GC);
 	}
 	band->owner.ops_fn = NULL;
 	band->owner.priv = NULL;

lib/ftl/ftl_core.c

@@ -30,6 +30,9 @@ static void
 ftl_io_cmpl_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 {
 	struct ftl_io *io = cb_arg;
+	struct spdk_ftl_dev *dev = io->dev;
+
+	ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_USER, bdev_io);
 
 	if (spdk_unlikely(!success)) {
 		io->status = -EIO;
@@ -111,6 +114,7 @@ void
 ftl_apply_limits(struct spdk_ftl_dev *dev)
 {
 	size_t limit;
+	struct ftl_stats *stats = &dev->stats;
 	int i;
 
 	/*  Clear existing limit */
@@ -120,6 +124,7 @@ ftl_apply_limits(struct spdk_ftl_dev *dev)
 		limit = ftl_get_limit(dev, i);
 
 		if (dev->num_free <= limit) {
+			stats->limits[i]++;
 			dev->limit = i;
 			break;
 		}
@@ -663,7 +668,7 @@ int
 ftl_core_poller(void *ctx)
 {
 	struct spdk_ftl_dev *dev = ctx;
-	uint64_t io_activity_total_old = dev->io_activity_total;
+	uint64_t io_activity_total_old = dev->stats.io_activity_total;
 
 	if (dev->halt && ftl_shutdown_complete(dev)) {
 		spdk_poller_unregister(&dev->core_poller);
@@ -677,7 +682,7 @@ ftl_core_poller(void *ctx)
 	ftl_nv_cache_process(dev);
 	ftl_l2p_process(dev);
 
-	if (io_activity_total_old != dev->io_activity_total) {
+	if (io_activity_total_old != dev->stats.io_activity_total) {
 		return SPDK_POLLER_BUSY;
 	}
 
@@ -734,10 +739,101 @@ spdk_ftl_dev_set_fast_shutdown(struct spdk_ftl_dev *dev, bool fast_shutdown)
 	dev->conf.fast_shutdown = fast_shutdown;
 }
 
+void
+ftl_stats_bdev_io_completed(struct spdk_ftl_dev *dev, enum ftl_stats_type type,
+			    struct spdk_bdev_io *bdev_io)
+{
+	struct ftl_stats_entry *stats_entry = &dev->stats.entries[type];
+	struct ftl_stats_group *stats_group;
+	uint32_t cdw0;
+	int sct;
+	int sc;
+
+	switch (bdev_io->type) {
+	case SPDK_BDEV_IO_TYPE_READ:
+		stats_group = &stats_entry->read;
+		break;
+	case SPDK_BDEV_IO_TYPE_WRITE:
+	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
+		stats_group = &stats_entry->write;
+		break;
+	default:
+		return;
+	}
+
+	spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
+
+	if (sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_SUCCESS) {
+		stats_group->ios++;
+		stats_group->blocks += bdev_io->u.bdev.num_blocks;
+	} else if (sct == SPDK_NVME_SCT_MEDIA_ERROR) {
+		stats_group->errors.media++;
+	} else {
+		stats_group->errors.other++;
+	}
+}
+
 struct spdk_io_channel *
 spdk_ftl_get_io_channel(struct spdk_ftl_dev *dev)
 {
 	return spdk_get_io_channel(dev);
 }
 
+void
+ftl_stats_crc_error(struct spdk_ftl_dev *dev, enum ftl_stats_type type)
+{
+
+	struct ftl_stats_entry *stats_entry = &dev->stats.entries[type];
+	struct ftl_stats_group *stats_group = &stats_entry->read;
+
+	stats_group->errors.crc++;
+}
+
+struct ftl_get_stats_ctx {
+	struct spdk_ftl_dev *dev;
+	struct ftl_stats *stats;
+	spdk_ftl_stats_fn cb_fn;
+	void *cb_arg;
+};
+
+static void
+_ftl_get_stats(void *_ctx)
+{
+	struct ftl_get_stats_ctx *stats_ctx = _ctx;
+
+	*stats_ctx->stats = stats_ctx->dev->stats;
+
+	stats_ctx->cb_fn(stats_ctx->stats, stats_ctx->cb_arg);
+	free(stats_ctx);
+}
+
+int
+spdk_ftl_get_stats(struct spdk_ftl_dev *dev, struct ftl_stats *stats, spdk_ftl_stats_fn cb_fn,
+		   void *cb_arg)
+{
+	struct ftl_get_stats_ctx *stats_ctx;
+	int rc;
+
+	stats_ctx = calloc(1, sizeof(struct ftl_get_stats_ctx));
+	if (!stats_ctx) {
+		return -ENOMEM;
+	}
+
+	stats_ctx->dev = dev;
+	stats_ctx->stats = stats;
+	stats_ctx->cb_fn = cb_fn;
+	stats_ctx->cb_arg = cb_arg;
+
+	rc = spdk_thread_send_msg(dev->core_thread, _ftl_get_stats, stats_ctx);
+	if (rc) {
+		goto stats_allocated;
+	}
+
+	return 0;
+
+stats_allocated:
+	free(stats_ctx);
+	return rc;
+}
+
 SPDK_LOG_REGISTER_COMPONENT(ftl_core)

lib/ftl/ftl_core.h

@@ -86,11 +86,8 @@ struct spdk_ftl_dev {
 	/* Band md memory pool */
 	struct ftl_mempool		*band_md_pool;
 
-	/* counters for poller busy, include
+	/* Statistics */
-	   1. nv cache read/write
+	struct ftl_stats		stats;
-	   2. metadata read/write
-	   3. base bdev read/write */
-	uint64_t			io_activity_total;
 
 	/* Array of bands */
 	struct ftl_band			*bands;
@@ -205,6 +202,11 @@ void ftl_set_unmap_map(struct spdk_ftl_dev *dev, uint64_t lba, uint64_t num_bloc
 
 void ftl_recover_max_seq(struct spdk_ftl_dev *dev);
 
+void ftl_stats_bdev_io_completed(struct spdk_ftl_dev *dev, enum ftl_stats_type type,
+				 struct spdk_bdev_io *bdev_io);
+
+void ftl_stats_crc_error(struct spdk_ftl_dev *dev, enum ftl_stats_type type);
+
 int ftl_unmap(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch,
 	      uint64_t lba, size_t lba_cnt, spdk_ftl_fn cb_fn, void *cb_arg);
 

lib/ftl/ftl_debug.c

@@ -180,6 +180,16 @@ ftl_dev_dump_stats(const struct spdk_ftl_dev *dev)
 {
 	uint64_t i, total = 0;
 	char uuid[SPDK_UUID_STRING_LEN];
+	double waf;
+	uint64_t write_user, write_total;
+	const char *limits[] = {
+		[SPDK_FTL_LIMIT_CRIT]  = "crit",
+		[SPDK_FTL_LIMIT_HIGH]  = "high",
+		[SPDK_FTL_LIMIT_LOW]   = "low",
+		[SPDK_FTL_LIMIT_START] = "start"
+	};
+
+	(void)limits;
 
 	if (!dev->bands) {
 		return;
@@ -190,8 +200,24 @@ ftl_dev_dump_stats(const struct spdk_ftl_dev *dev)
 		total += dev->bands[i].p2l_map.num_valid;
 	}
 
+	write_user = dev->stats.entries[FTL_STATS_TYPE_CMP].write.blocks;
+	write_total = write_user +
+		      dev->stats.entries[FTL_STATS_TYPE_GC].write.blocks +
+		      dev->stats.entries[FTL_STATS_TYPE_MD_BASE].write.blocks;
+
+	waf = (double)write_total / (double)write_user;
+
 	spdk_uuid_fmt_lower(uuid, sizeof(uuid), &dev->conf.uuid);
 	FTL_NOTICELOG(dev, "\n");
 	FTL_NOTICELOG(dev, "device UUID:         %s\n", uuid);
 	FTL_NOTICELOG(dev, "total valid LBAs:    %zu\n", total);
+	FTL_NOTICELOG(dev, "total writes:        %"PRIu64"\n", write_total);
+	FTL_NOTICELOG(dev, "user writes:         %"PRIu64"\n", write_user);
+	FTL_NOTICELOG(dev, "WAF:                 %.4lf\n", waf);
+#ifdef DEBUG
+	FTL_NOTICELOG(dev, "limits:\n");
+	for (i = 0; i < SPDK_FTL_LIMIT_MAX; ++i) {
+		FTL_NOTICELOG(dev, " %5s: %"PRIu64"\n", limits[i], dev->stats.limits[i]);
+	}
+#endif
 }

lib/ftl/ftl_l2p_cache.c

@@ -605,6 +605,7 @@ process_persist_page_out_cb(struct spdk_bdev_io *bdev_io, bool success, void *ar
 	struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
 
 	assert(bdev_io);
+	ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_L2P, bdev_io);
 	spdk_bdev_free_io(bdev_io);
 
 	if (!success) {
@@ -678,6 +679,7 @@ process_unmap_page_out_cb(struct spdk_bdev_io *bdev_io, bool success, void *ctx_
 	struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
 
 	assert(bdev_io);
+	ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_L2P, bdev_io);
 	spdk_bdev_free_io(bdev_io);
 
 	if (!success) {
@@ -702,6 +704,7 @@ process_unmap_page_in_cb(struct spdk_bdev_io *bdev_io, bool success, void *ctx_p
 	struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
 
 	if (bdev_io) {
+		ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_L2P, bdev_io);
 		spdk_bdev_free_io(bdev_io);
 	}
 	if (success) {
@@ -1228,6 +1231,7 @@ page_in_io_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 	struct ftl_l2p_cache *cache = page->ctx.cache;
 	struct spdk_ftl_dev *dev = cache->dev;
 
+	ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_L2P, bdev_io);
 	spdk_bdev_free_io(bdev_io);
 	page_in_io_complete(dev, cache, page, success);
 }
@@ -1408,6 +1412,7 @@ page_out_io_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 	struct ftl_l2p_cache *cache = page->ctx.cache;
 	struct spdk_ftl_dev *dev = cache->dev;
 
+	ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_L2P, bdev_io);
 	spdk_bdev_free_io(bdev_io);
 	page_out_io_complete(dev, cache, page, success);
 }

lib/ftl/ftl_nv_cache.c

@@ -599,6 +599,9 @@ compaction_process_read_cb(struct spdk_bdev_io *bdev_io,
 			   bool success, void *cb_arg)
 {
 	struct ftl_nv_cache_compactor *compactor = cb_arg;
+	struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(compactor->nv_cache, struct spdk_ftl_dev, nv_cache);
+
+	ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_CMP, bdev_io);
 
 	spdk_bdev_free_io(bdev_io);
 
@@ -996,6 +999,8 @@ ftl_nv_cache_submit_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 {
 	struct ftl_io *io = cb_arg;
 
+	ftl_stats_bdev_io_completed(io->dev, FTL_STATS_TYPE_USER, bdev_io);
+
 	spdk_bdev_free_io(bdev_io);
 
 	if (spdk_unlikely(!success)) {
@@ -1502,6 +1507,8 @@ write_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
 	struct ftl_basic_rq *brq = arg;
 	struct ftl_nv_cache_chunk *chunk = brq->io.chunk;
 
+	ftl_stats_bdev_io_completed(brq->dev, FTL_STATS_TYPE_MD_NV_CACHE, bdev_io);
+
 	brq->success = success;
 	if (spdk_likely(success)) {
 		chunk_advance_blocks(chunk->nv_cache, chunk, brq->num_blocks);
@@ -1547,7 +1554,7 @@ ftl_chunk_basic_rq_write(struct ftl_nv_cache_chunk *chunk, struct ftl_basic_rq *
 	_ftl_chunk_basic_rq_write(brq);
 
 	chunk->md->write_pointer += brq->num_blocks;
-	dev->io_activity_total += brq->num_blocks;
+	dev->stats.io_activity_total += brq->num_blocks;
 }
 
 static void
@@ -1555,6 +1562,8 @@ read_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
 {
 	struct ftl_basic_rq *brq = arg;
 
+	ftl_stats_bdev_io_completed(brq->dev, FTL_STATS_TYPE_MD_NV_CACHE, bdev_io);
+
 	brq->success = success;
 
 	brq->owner.cb(brq);
@@ -1575,7 +1584,7 @@ ftl_chunk_basic_rq_read(struct ftl_nv_cache_chunk *chunk, struct ftl_basic_rq *b
 			brq->io_payload, NULL, brq->io.addr, brq->num_blocks, read_brq_end, brq);
 
 	if (spdk_likely(!rc)) {
-		dev->io_activity_total += brq->num_blocks;
+		dev->stats.io_activity_total += brq->num_blocks;
 	}
 
 	return rc;
@@ -1779,6 +1788,8 @@ read_open_chunk_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 	ftl_addr addr = ftl_addr_from_nvc_offset(dev, cache_offset);
 	int rc;
 
+	ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_USER, bdev_io);
+
 	spdk_bdev_free_io(bdev_io);
 
 	if (!success) {

lib/ftl/ftl_p2l.c

@@ -401,6 +401,7 @@ ftl_mngt_p2l_ckpt_restore(struct ftl_band *band, uint32_t md_region, uint64_t se
 		if (page_md_buf->p2l_ckpt.p2l_checksum &&
 		    page_md_buf->p2l_ckpt.p2l_checksum != spdk_crc32c_update(page,
 				    FTL_NUM_LBA_IN_BLOCK * sizeof(struct ftl_p2l_map_entry), 0)) {
+			ftl_stats_crc_error(band->dev, FTL_STATS_TYPE_MD_NV_CACHE);
 			return -EINVAL;
 		}
 

lib/ftl/mngt/ftl_mngt_misc.c

@@ -180,6 +180,10 @@ ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mng
 		dev->unmap_in_progress = true;
 	}
 
+	/* Clear the limit applications as they're incremented incorrectly by
+	 * the initialization code.
+	 */
+	memset(dev->stats.limits, 0, sizeof(dev->stats.limits));
 	dev->initialized = 1;
 	dev->sb_shm->shm_ready = true;
 

lib/ftl/mngt/ftl_mngt_recovery.c

@@ -419,6 +419,7 @@ restore_band_l2p_cb(struct ftl_band *band, void *cntx, enum ftl_md_status status
 	/* P2L map is only valid if the band state is closed */
 	if (FTL_BAND_STATE_CLOSED == band->md->state && band->md->p2l_map_checksum != band_map_crc) {
 		FTL_ERRLOG(dev, "L2P band restore error, inconsistent P2L map CRC\n");
+		ftl_stats_crc_error(dev, FTL_STATS_TYPE_MD_BASE);
 		rc = -EINVAL;
 		goto cleanup;
 	}
@@ -510,6 +511,7 @@ restore_chunk_l2p_cb(struct ftl_nv_cache_chunk *chunk, void *ctx)
 	chunk_map_crc = spdk_crc32c_update(chunk->p2l_map.chunk_map,
 					   ftl_nv_cache_chunk_tail_md_num_blocks(chunk->nv_cache) * FTL_BLOCK_SIZE, 0);
 	if (chunk->md->p2l_map_checksum != chunk_map_crc) {
+		ftl_stats_crc_error(dev, FTL_STATS_TYPE_MD_NV_CACHE);
 		return -1;
 	}
 

lib/ftl/spdk_ftl.map

@@ -17,6 +17,7 @@
 	spdk_ftl_writev;
 	spdk_ftl_unmap;
 	spdk_ftl_dev_set_fast_shutdown;
+	spdk_ftl_get_stats;
 
 	local: *;
 };

lib/ftl/utils/ftl_md.c

@@ -423,6 +423,19 @@ exception(void *arg)
 	io_cleanup(md);
 }
 
+static inline enum ftl_stats_type
+get_bdev_io_ftl_stats_type(struct spdk_ftl_dev *dev, struct spdk_bdev_io *bdev_io) {
+	struct spdk_bdev *nvc = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc);
+
+	if (bdev_io->bdev == nvc)
+	{
+		return FTL_STATS_TYPE_MD_NV_CACHE;
+	} else
+	{
+		return FTL_STATS_TYPE_MD_BASE;
+	}
+}
+
 static void
 audit_md_vss_version(struct ftl_md *md, uint64_t blocks)
 {
@@ -440,6 +453,8 @@ read_write_blocks_cb(struct spdk_bdev_io *bdev_io, bool success, void *arg)
 {
 	struct ftl_md *md = arg;
 
+	ftl_stats_bdev_io_completed(md->dev, get_bdev_io_ftl_stats_type(md->dev, bdev_io), bdev_io);
+
 	if (spdk_unlikely(!success)) {
 		if (md->io.op == FTL_MD_OP_RESTORE && has_mirror(md)) {
 			md->io.status = -EAGAIN;
@@ -666,6 +681,9 @@ static void
 persist_entry_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 {
 	struct ftl_md_io_entry_ctx *ctx = cb_arg;
+	struct ftl_md *md = ctx->md;
+
+	ftl_stats_bdev_io_completed(md->dev, get_bdev_io_ftl_stats_type(md->dev, bdev_io), bdev_io);
 
 	spdk_bdev_free_io(bdev_io);
 
@@ -769,6 +787,8 @@ read_entry_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 	struct ftl_md_io_entry_ctx *ctx = cb_arg;
 	struct ftl_md *md = ctx->md;
 
+	ftl_stats_bdev_io_completed(md->dev, get_bdev_io_ftl_stats_type(md->dev, bdev_io), bdev_io);
+
 	spdk_bdev_free_io(bdev_io);
 
 	if (!success) {

module/bdev/ftl/bdev_ftl.c

@@ -548,6 +548,66 @@ not_found:
 	cb_fn(cb_arg, rc);
 }
 
+static void
+bdev_ftl_get_stats_cb(struct ftl_stats *stats, void *ctx)
+{
+	struct rpc_ftl_stats_ctx *ftl_stats_ctx = ctx;
+
+	ftl_stats_ctx->cb(ftl_stats_ctx);
+
+	spdk_bdev_close(ftl_stats_ctx->ftl_bdev_desc);
+	free(ftl_stats_ctx);
+}
+
+
+int
+bdev_ftl_get_stats(const char *name, ftl_bdev_thread_fn cb, struct spdk_jsonrpc_request *request,
+		   struct ftl_stats *stats)
+{
+	struct spdk_bdev_desc *ftl_bdev_desc;
+	struct spdk_bdev *bdev;
+	struct ftl_bdev *ftl;
+	struct rpc_ftl_stats_ctx *ftl_stats_ctx;
+	int rc;
+
+	rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc);
+	if (rc) {
+		goto not_found;
+	}
+
+	bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc);
+	if (bdev->module != &g_ftl_if) {
+		rc = -ENODEV;
+		goto bdev_opened;
+	}
+
+	ftl_stats_ctx = calloc(1, sizeof(*ftl_stats_ctx));
+	if (!ftl_stats_ctx) {
+		SPDK_ERRLOG("Could not allocate ftl_stats_ctx\n");
+		rc = -ENOMEM;
+		goto bdev_opened;
+	}
+
+	ftl = bdev->ctxt;
+	ftl_stats_ctx->request = request;
+	ftl_stats_ctx->ftl_bdev_desc = ftl_bdev_desc;
+	ftl_stats_ctx->cb = cb;
+	ftl_stats_ctx->ftl_stats = stats;
+
+	rc = spdk_ftl_get_stats(ftl->dev, stats, bdev_ftl_get_stats_cb, ftl_stats_ctx);
+	if (rc) {
+		goto stats_allocated;
+	}
+
+	return 0;
+stats_allocated:
+	free(ftl_stats_ctx);
+bdev_opened:
+	spdk_bdev_close(ftl_bdev_desc);
+not_found:
+	return rc;
+}
+
 static void
 bdev_ftl_finish(void)
 {

module/bdev/ftl/bdev_ftl.h

@@ -17,14 +17,24 @@ struct ftl_bdev_info {
 	struct spdk_uuid	uuid;
 };
 
-typedef void (*ftl_bdev_init_fn)(const struct ftl_bdev_info *, void *, int);
 typedef void (*ftl_bdev_thread_fn)(void *);
 
+struct rpc_ftl_stats_ctx {
+	struct spdk_bdev_desc		*ftl_bdev_desc;
+	ftl_bdev_thread_fn		cb;
+	struct spdk_jsonrpc_request	*request;
+	struct ftl_stats		*ftl_stats;
+};
+
+typedef void (*ftl_bdev_init_fn)(const struct ftl_bdev_info *, void *, int);
+
 int bdev_ftl_create_bdev(const struct spdk_ftl_conf *conf, ftl_bdev_init_fn cb, void *cb_arg);
 void bdev_ftl_delete_bdev(const char *name, bool fast_shutdown, spdk_bdev_unregister_cb cb_fn,
 			  void *cb_arg);
 int bdev_ftl_defer_init(const struct spdk_ftl_conf *conf);
 void bdev_ftl_unmap(const char *name, uint64_t lba, uint64_t num_blocks, spdk_ftl_fn cb_fn,
 		    void *cb_arg);
+int bdev_ftl_get_stats(const char *name, ftl_bdev_thread_fn cb,
+		       struct spdk_jsonrpc_request *request, struct ftl_stats *stats);
 
 #endif /* SPDK_BDEV_FTL_H */

module/bdev/ftl/bdev_ftl_rpc.c

@@ -210,3 +210,109 @@ invalid:
 }
 
 SPDK_RPC_REGISTER("bdev_ftl_unmap", rpc_bdev_ftl_unmap, SPDK_RPC_RUNTIME)
+
+struct rpc_ftl_stats {
+	char *name;
+};
+
+static const struct spdk_json_object_decoder rpc_ftl_stats_decoders[] = {
+	{"name", offsetof(struct rpc_ftl_stats, name), spdk_json_decode_string},
+};
+
+static void
+_rpc_bdev_ftl_get_stats(void *cntx)
+{
+	struct rpc_ftl_stats_ctx *ftl_stats = cntx;
+	struct spdk_jsonrpc_request *request = ftl_stats->request;
+	struct ftl_stats *stats = ftl_stats->ftl_stats;
+	struct spdk_json_write_ctx *w = spdk_jsonrpc_begin_result(request);
+
+	spdk_json_write_object_begin(w);
+	spdk_json_write_named_string(w, "name", spdk_bdev_desc_get_bdev(ftl_stats->ftl_bdev_desc)->name);
+
+	/* TODO: Instead of named objects, store them in an array with the name being an attribute */
+	for (uint64_t i = 0; i < FTL_STATS_TYPE_MAX; i++) {
+		switch (i) {
+		case FTL_STATS_TYPE_USER:
+			spdk_json_write_named_object_begin(w, "user");
+			break;
+		case FTL_STATS_TYPE_CMP:
+			spdk_json_write_named_object_begin(w, "cmp");
+			break;
+		case FTL_STATS_TYPE_GC:
+			spdk_json_write_named_object_begin(w, "gc");
+			break;
+		case FTL_STATS_TYPE_MD_BASE:
+			spdk_json_write_named_object_begin(w, "md_base");
+			break;
+		case FTL_STATS_TYPE_MD_NV_CACHE:
+			spdk_json_write_named_object_begin(w, "md_nv_cache");
+			break;
+		case FTL_STATS_TYPE_L2P:
+			spdk_json_write_named_object_begin(w, "l2p");
+			break;
+		default:
+			assert(false);
+			continue;
+		}
+
+		spdk_json_write_named_object_begin(w, "read");
+		spdk_json_write_named_uint64(w, "ios", stats->entries[i].read.ios);
+		spdk_json_write_named_uint64(w, "blocks", stats->entries[i].read.blocks);
+		spdk_json_write_named_object_begin(w, "errors");
+		spdk_json_write_named_uint64(w, "media", stats->entries[i].read.errors.media);
+		spdk_json_write_named_uint64(w, "crc", stats->entries[i].read.errors.crc);
+		spdk_json_write_named_uint64(w, "other", stats->entries[i].read.errors.other);
+		spdk_json_write_object_end(w);
+		spdk_json_write_object_end(w);
+
+		spdk_json_write_named_object_begin(w, "write");
+		spdk_json_write_named_uint64(w, "ios", stats->entries[i].write.ios);
+		spdk_json_write_named_uint64(w, "blocks", stats->entries[i].write.blocks);
+		spdk_json_write_named_object_begin(w, "errors");
+		spdk_json_write_named_uint64(w, "media", stats->entries[i].write.errors.media);
+		spdk_json_write_named_uint64(w, "other", stats->entries[i].write.errors.other);
+		spdk_json_write_object_end(w);
+		spdk_json_write_object_end(w);
+
+		spdk_json_write_object_end(w);
+	}
+
+	spdk_json_write_object_end(w);
+	spdk_jsonrpc_end_result(request, w);
+
+	free(stats);
+}
+
+static void
+rpc_bdev_ftl_get_stats(struct spdk_jsonrpc_request *request,
+		       const struct spdk_json_val *params)
+{
+	struct ftl_stats *stats;
+	struct rpc_ftl_stats attrs = {};
+	int rc;
+
+	if (spdk_json_decode_object(params, rpc_ftl_stats_decoders, SPDK_COUNTOF(rpc_ftl_stats_decoders),
+				    &attrs)) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");
+		goto invalid;
+	}
+
+	stats = calloc(1, sizeof(struct ftl_stats));
+	if (!stats) {
+		spdk_jsonrpc_send_bool_response(request, false);
+		goto invalid;
+	}
+
+	rc = bdev_ftl_get_stats(attrs.name, _rpc_bdev_ftl_get_stats, request, stats);
+	if (rc) {
+		free(stats);
+		spdk_jsonrpc_send_bool_response(request, false);
+		goto invalid;
+	}
+
+invalid:
+	free(attrs.name);
+}
+
+SPDK_RPC_REGISTER("bdev_ftl_get_stats", rpc_bdev_ftl_get_stats, SPDK_RPC_RUNTIME)

python/spdk/rpc/bdev.py

@@ -1420,6 +1420,17 @@ def bdev_ftl_unmap(client, name, lba, num_blocks):
     return client.call('bdev_ftl_unmap', params)
 
 
+def bdev_ftl_get_stats(client, name):
+    """get FTL stats
+
+    Args:
+        name: name of the bdev
+    """
+    params = {'name': name}
+
+    return client.call('bdev_ftl_get_stats', params)
+
+
 def bdev_get_bdevs(client, name=None, timeout=None):
     """Get information about block devices.
 

scripts/rpc.py

@@ -2086,6 +2086,13 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
     p.add_argument('--num-blocks', help='num blocks', required=True, type=int)
     p.set_defaults(func=bdev_ftl_unmap)
 
+    def bdev_ftl_get_stats(args):
+        print_dict(rpc.bdev.bdev_ftl_get_stats(args.client, name=args.name))
+
+    p = subparsers.add_parser('bdev_ftl_get_stats', help='print ftl stats')
+    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
+    p.set_defaults(func=bdev_ftl_get_stats)
+
     # vmd
     def vmd_enable(args):
         print_dict(rpc.vmd.vmd_enable(args.client))

test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c

@@ -133,6 +133,8 @@ DEFINE_STUB(ftl_md_get_buffer, void *, (struct ftl_md *md), NULL);
 DEFINE_STUB(ftl_md_get_vss_buffer, union ftl_md_vss *, (struct ftl_md *md), NULL);
 DEFINE_STUB(ftl_nv_cache_acquire_trim_seq_id, uint64_t, (struct ftl_nv_cache *nv_cache), 0);
 DEFINE_STUB_V(ftl_md_persist, (struct ftl_md *md));
+DEFINE_STUB_V(spdk_bdev_io_get_nvme_status, (const struct spdk_bdev_io *bdev_io, uint32_t *cdw0,
+		int *sct, int *sc));
 
 static void
 adjust_bitmap(struct ftl_bitmap **bitmap, uint64_t *bit)