nvme: Add the priority field in struct spdk_nvme_transport_id

Purpose: To set the priority of the NVMe-oF connection especially
for TCP connection.

For example, the previous example can be:

trtype:TCP adrfam:IPv4 traddr:10.67.110.181 trsvcid:4420

With the change, it could be:

trtype:TCP adrfam:IPv4 traddr:10.67.110.181 trsvcid:4420 priority:2

The priority is optional. We try to change
spdk_nvme_transport_id but not in spdk_nvme_ctrlr_opts since
the opts in spdk_nvme_ctrlr_opts will reflect in every nvme ctrlr,
this is short of flexibility.

Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Signed-off-by: Sudheer Mogilappagari <sudheer.mogilappagari@intel.com>
Change-Id: I1ba364c714a95f2dbeab2b3fcc832b0222b48a15
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1875
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

CHANGELOG.md

@@ -36,6 +36,9 @@ New version of OCF comes with API changes and bug fixes
 Export internal nvme_ctrlr_cmd_security_receive/send() APIs as public APIs with "spdk_"
 prefix.
 
+Added `priority` field in `spdk_nvme_transport_id`, this field is used to specify the priority
+of the NVMe-oF connection, and currently it is used for NVMe-oF tcp connection.
+
 ### copy
 
 The copy engine library, modules and public APIs have been renamed. Use of the word `copy`

include/spdk/nvme.h

@@ -53,6 +53,8 @@ extern "C" {
 #define SPDK_NVME_TRANSPORT_NAME_RDMA	"RDMA"
 #define SPDK_NVME_TRANSPORT_NAME_TCP	"TCP"
 
+#define SPDK_NVMF_PRIORITY_MAX_LEN 4
+
 /**
  * Opaque handle to a controller. Returned by spdk_nvme_probe()'s attach_cb.
  */
@@ -360,6 +362,13 @@ struct spdk_nvme_transport_id {
 	 * Subsystem NQN of the NVMe over Fabrics endpoint. May be a zero length string.
 	 */
 	char subnqn[SPDK_NVMF_NQN_MAX_LEN + 1];
+
+	/**
+	 * The Transport connection priority of the NVMe-oF endpoint. Currently this is
+	 * only supported by posix based sock implementation on Kernel TCP stack. More
+	 * information of this field can be found from the socket(7) man page.
+	 */
+	int priority;
 };
 
 /**

lib/nvme/nvme.c

@@ -32,6 +32,7 @@
  */
 
 #include "spdk/nvmf_spec.h"
+#include "spdk/string.h"
 #include "nvme_internal.h"
 #include "nvme_io_msg.h"
 
@@ -975,6 +976,13 @@ spdk_nvme_transport_id_parse(struct spdk_nvme_transport_id *trid, const char *st
 				return -EINVAL;
 			}
 			memcpy(trid->trsvcid, val, val_len + 1);
+		} else if (strcasecmp(key, "priority") == 0) {
+			if (val_len > SPDK_NVMF_PRIORITY_MAX_LEN) {
+				SPDK_ERRLOG("priority length %zu greater than maximum allowed %u\n",
+					    val_len, SPDK_NVMF_PRIORITY_MAX_LEN);
+				return -EINVAL;
+			}
+			trid->priority = spdk_strtol(val, 10);
 		} else if (strcasecmp(key, "subnqn") == 0) {
 			if (val_len > SPDK_NVMF_NQN_MAX_LEN) {
 				SPDK_ERRLOG("subnqn length %zu greater than maximum allowed %u\n",
@@ -1046,6 +1054,8 @@ spdk_nvme_host_id_parse(struct spdk_nvme_host_id *hostid, const char *str)
 			continue;
 		} else if (strcasecmp(key, "subnqn") == 0) {
 			continue;
+		} else if (strcasecmp(key, "priority") == 0) {
+			continue;
 		} else if (strcasecmp(key, "ns") == 0) {
 			continue;
 		} else if (strcasecmp(key, "hostaddr") == 0) {

lib/nvme/nvme_fabric.c

@@ -166,7 +166,8 @@ nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint
 
 static void
 nvme_fabric_discover_probe(struct spdk_nvmf_discovery_log_page_entry *entry,
-			   struct spdk_nvme_probe_ctx *probe_ctx)
+			   struct spdk_nvme_probe_ctx *probe_ctx,
+			   int discover_priority)
 {
 	struct spdk_nvme_transport_id trid;
 	uint8_t *end;
@@ -221,6 +222,9 @@ nvme_fabric_discover_probe(struct spdk_nvmf_discovery_log_page_entry *entry,
 		      trid.subnqn, trid.trtype,
 		      trid.traddr, trid.trsvcid);
 
+	/* Copy the priority from the discovery ctrlr */
+	trid.priority = discover_priority;
+
 	nvme_ctrlr_probe(&trid, probe_ctx, NULL);
 }
 
@@ -378,7 +382,7 @@ nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr,
 		}
 
 		for (i = 0; i < numrec; i++) {
-			nvme_fabric_discover_probe(log_page_entry++, probe_ctx);
+			nvme_fabric_discover_probe(log_page_entry++, probe_ctx, ctrlr->trid.priority);
 		}
 		remaining_num_rec -= numrec;
 		log_page_offset += numrec * sizeof(struct spdk_nvmf_discovery_log_page_entry);

lib/nvme/nvme_tcp.c

@@ -1487,6 +1487,7 @@ nvme_tcp_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpa
 	struct nvme_tcp_qpair *tqpair;
 	int family;
 	long int port;
+	struct spdk_sock_opts opts;
 
 	tqpair = nvme_tcp_qpair(qpair);
 
@@ -1528,7 +1529,10 @@ nvme_tcp_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpa
 		return -1;
 	}
 
-	tqpair->sock = spdk_sock_connect(ctrlr->trid.traddr, port, NULL);
+	opts.opts_size = sizeof(opts);
+	spdk_sock_get_default_opts(&opts);
+	opts.priority = ctrlr->trid.priority;
+	tqpair->sock = spdk_sock_connect_ext(ctrlr->trid.traddr, port, NULL, &opts);
 	if (!tqpair->sock) {
 		SPDK_ERRLOG("sock connection error of tqpair=%p with addr=%s, port=%ld\n",
 			    tqpair, ctrlr->trid.traddr, port);

lib/nvmf/tcp.c

@@ -594,6 +594,7 @@ spdk_nvmf_tcp_listen(struct spdk_nvmf_transport *transport,
 	struct spdk_nvmf_tcp_port *port;
 	int trsvcid_int;
 	uint8_t adrfam;
+	struct spdk_sock_opts opts;
 
 	ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport);
 
@@ -612,7 +613,11 @@ spdk_nvmf_tcp_listen(struct spdk_nvmf_transport *transport,
 	}
 
 	port->trid = trid;
-	port->listen_sock = spdk_sock_listen(trid->traddr, trsvcid_int, NULL);
+	opts.opts_size = sizeof(opts);
+	spdk_sock_get_default_opts(&opts);
+	opts.priority = transport->opts.sock_priority;
+	port->listen_sock = spdk_sock_listen_ext(trid->traddr, trsvcid_int,
+			    NULL, &opts);
 	if (port->listen_sock == NULL) {
 		SPDK_ERRLOG("spdk_sock_listen(%s, %d) failed: %s (%d)\n",
 			    trid->traddr, trsvcid_int,

module/bdev/nvme/bdev_nvme_rpc.c

@@ -173,6 +173,7 @@ struct rpc_bdev_nvme_attach_controller {
 	char *adrfam;
 	char *traddr;
 	char *trsvcid;
+	char *priority;
 	char *subnqn;
 	char *hostnqn;
 	char *hostaddr;
@@ -189,6 +190,7 @@ free_rpc_bdev_nvme_attach_controller(struct rpc_bdev_nvme_attach_controller *req
 	free(req->adrfam);
 	free(req->traddr);
 	free(req->trsvcid);
+	free(req->priority);
 	free(req->subnqn);
 	free(req->hostnqn);
 	free(req->hostaddr);
@@ -202,6 +204,7 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_attach_controller_dec
 
 	{"adrfam", offsetof(struct rpc_bdev_nvme_attach_controller, adrfam), spdk_json_decode_string, true},
 	{"trsvcid", offsetof(struct rpc_bdev_nvme_attach_controller, trsvcid), spdk_json_decode_string, true},
+	{"priority", offsetof(struct rpc_bdev_nvme_attach_controller, priority), spdk_json_decode_string, true},
 	{"subnqn", offsetof(struct rpc_bdev_nvme_attach_controller, subnqn), spdk_json_decode_string, true},
 	{"hostnqn", offsetof(struct rpc_bdev_nvme_attach_controller, hostnqn), spdk_json_decode_string, true},
 	{"hostaddr", offsetof(struct rpc_bdev_nvme_attach_controller, hostaddr), spdk_json_decode_string, true},
@@ -303,6 +306,11 @@ spdk_rpc_bdev_nvme_attach_controller(struct spdk_jsonrpc_request *request,
 		snprintf(trid.trsvcid, sizeof(trid.trsvcid), "%s", ctx->req.trsvcid);
 	}
 
+	/* Parse priority for the NVMe-oF transport connection */
+	if (ctx->req.priority) {
+		trid.priority = spdk_strtol(ctx->req.priority, 10);
+	}
+
 	/* Parse subnqn */
 	if (ctx->req.subnqn) {
 		snprintf(trid.subnqn, sizeof(trid.subnqn), "%s", ctx->req.subnqn);

scripts/rpc.py

@@ -425,6 +425,7 @@ if __name__ == "__main__":
                                                          traddr=args.traddr,
                                                          adrfam=args.adrfam,
                                                          trsvcid=args.trsvcid,
+                                                         priority=args.priority,
                                                          subnqn=args.subnqn,
                                                          hostnqn=args.hostnqn,
                                                          hostaddr=args.hostaddr,
@@ -443,6 +444,8 @@ if __name__ == "__main__":
                    help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
     p.add_argument('-s', '--trsvcid',
                    help='NVMe-oF target trsvcid: e.g., a port number')
+    p.add_argument('-p', '--priority',
+                   help='NVMe-oF connection priority: e.g., a priority number')
     p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn')
     p.add_argument('-q', '--hostnqn', help='NVMe-oF host subnqn')
     p.add_argument('-i', '--hostaddr',

scripts/rpc/bdev.py

@@ -432,8 +432,8 @@ def bdev_nvme_set_hotplug(client, enable, period_us=None):
 
 @deprecated_alias('construct_nvme_bdev')
 def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvcid=None,
-                                subnqn=None, hostnqn=None, hostaddr=None, hostsvcid=None,
+                                priority=None, subnqn=None, hostnqn=None, hostaddr=None,
-                                prchk_reftag=None, prchk_guard=None):
+                                hostsvcid=None, prchk_reftag=None, prchk_guard=None):
     """Construct block device for each NVMe namespace in the attached controller.
 
     Args:
@@ -442,6 +442,7 @@ def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvc
         traddr: transport address (PCI BDF or IP address)
         adrfam: address family ("IPv4", "IPv6", "IB", or "FC") (optional for PCIe)
         trsvcid: transport service ID (port number for IP-based addresses; optional for PCIe)
+        priority: transport connection priority (Sock priority for TCP-based transports; optional)
         subnqn: subsystem NQN to connect to (optional)
         hostnqn: NQN to connect from (optional)
         hostaddr: host transport address (IP address for IP-based transports, NULL for PCIe or FC; optional)
@@ -471,6 +472,9 @@ def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvc
     if trsvcid:
         params['trsvcid'] = trsvcid
 
+    if priority:
+        params['priority'] = priority
+
     if subnqn:
         params['subnqn'] = subnqn
 

test/common/lib/test_sock.c

@@ -40,8 +40,13 @@ DEFINE_STUB(spdk_sock_getaddr, int, (struct spdk_sock *sock, char *saddr, int sl
 				     char *caddr, int clen, uint16_t *cport), 0);
 DEFINE_STUB(spdk_sock_connect, struct spdk_sock *, (const char *ip, int port, char *impl_name),
 	    NULL);
+DEFINE_STUB(spdk_sock_connect_ext, struct spdk_sock *, (const char *ip, int port, char *impl_name,
+		struct spdk_sock_opts *opts), NULL);
 DEFINE_STUB(spdk_sock_listen, struct spdk_sock *, (const char *ip, int port, char *impl_name),
 	    NULL);
+DEFINE_STUB(spdk_sock_listen_ext, struct spdk_sock *, (const char *ip, int port, char *impl_name,
+		struct spdk_sock_opts *opts), NULL);
+DEFINE_STUB_V(spdk_sock_get_default_opts, (struct spdk_sock_opts *opts));
 DEFINE_STUB(spdk_sock_accept, struct spdk_sock *, (struct spdk_sock *sock), NULL);
 DEFINE_STUB(spdk_sock_close, int, (struct spdk_sock **sock), 0);
 DEFINE_STUB(spdk_sock_recv, ssize_t, (struct spdk_sock *sock, void *buf, size_t len), 0);

test/unit/lib/nvme/nvme.c/nvme_ut.c

@@ -971,6 +971,15 @@ test_trid_parse_and_compare(void)
 	CU_ASSERT(spdk_nvme_transport_id_parse(&trid1, "trtype=PCIe traddr=0000:04:00.0") == 0);
 	CU_ASSERT(spdk_nvme_transport_id_parse(&trid2, "trtype=PCIe traddr=05:00.0") == 0);
 	CU_ASSERT(spdk_nvme_transport_id_compare(&trid1, &trid2) < 0);
+
+	CU_ASSERT(spdk_nvme_transport_id_parse(&trid1,
+					       "trtype:tcp\n"
+					       "adrfam:ipv4\n"
+					       "traddr:192.168.100.8\n"
+					       "trsvcid:4420\n"
+					       "priority:2\n"
+					       "subnqn:nqn.2014-08.org.nvmexpress.discovery") == 0);
+	CU_ASSERT(trid1.priority == 2);
 }
 
 static void

test/unit/lib/nvme/nvme_tcp.c/nvme_tcp_ut.c

@@ -45,6 +45,9 @@ SPDK_LOG_REGISTER_COMPONENT("nvme", SPDK_LOG_NVME);
 DEFINE_STUB(nvme_qpair_submit_request,
 	    int, (struct spdk_nvme_qpair *qpair, struct nvme_request *req), 0);
 
+DEFINE_STUB(spdk_sock_set_priority,
+	    int, (struct spdk_sock *sock, int priority), 0);
+
 static void
 test_nvme_tcp_pdu_set_data_buf(void)
 {