nvme/tcp: Allow to choose SSL socket implementation

Adding `psk` field to `spdk_nvme_ctrlr_opts`

Adding `psk` parameter to `bdev_nvme_attach_controller` RPC

Change-Id: Ie6f0d8b04ce472e6153934e985c026acded6cdfc
Signed-off-by: Boris Glimcher <Boris.Glimcher@emc.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14046
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

CHANGELOG.md

@@ -43,6 +43,9 @@ Calculate num_md_pages from num_md_pages_per_cluster_ratio, and pass it to spdk_
 
 ### rpc
 
+Added `psk` parameter to `bdev_nvme_attach_controller` RPC in order to enable SSL socket implementation
+of TCP connection and set the PSK. Applicable for TCP transport only.
+
 New options `enable_ktls` and `tls_version` were added to the `sock_impl_set_options` structure.
 New options `psk_key` and `psk_identity` were added to the `sock_impl_set_options` structure.
 
@@ -81,6 +84,9 @@ tell the driver to not read the CHANGED_NS_LIST log page in response to a NS_ATT
 AEN.  When called the application is required to read this log page instead to clear the
 AEN.
 
+Added `psk` field to `spdk_nvme_ctrlr_opts` struct in order to enable SSL socket implementation
+of TCP connection and set the PSK. Applicable for TCP transport only.
+
 ### util
 
 Added new functions: `spdk_hexlify` and `spdk_unhexlify`.

doc/jsonrpc.md

@@ -3245,6 +3245,7 @@ num_io_queues              | Optional | uint32_t    | The number of IO queues to
 ctrlr_loss_timeout_sec     | Optional | number      | Time to wait until ctrlr is reconnected before deleting ctrlr.  -1 means infinite reconnects. 0 means no reconnect.
 reconnect_delay_sec        | Optional | number      | Time to delay a reconnect trial. 0 means no reconnect.
 fast_io_fail_timeout_sec   | Optional | number      | Time to wait until ctrlr is reconnected before failing I/O to ctrlr. 0 means no such timeout.
+psk                        | Optional | string      | PSK in hexadecimal digits, e.g. 1234567890ABCDEF (Enables SSL socket implementation for TCP)
 
 #### Example
 

include/spdk/nvme.h

@@ -266,8 +266,17 @@ struct spdk_nvme_ctrlr_opts {
 	 * Default is `false` (CHANGED_NS_LIST log page is read).
 	 */
 	uint8_t disable_read_changed_ns_list_log_page;
+
+	/**
+	 * Set PSK and enable SSL socket implementation for NVMe/TCP only.
+	 *
+	 * If empty, a default socket implementation will be used.
+	 * The TLS PSK interchange format is: NVMeTLSkey-1:xx:[Base64 encoded string]:
+	 * 12B (header) + 2B (hash) + 176B (base64 for 1024b + crc32) + 3B (colons) + 1B (NULL) + 6B (extra space for future)
+	 */
+	char psk[200];
 } __attribute__((packed));
-SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_ctrlr_opts) == 617, "Incorrect size");
+SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_ctrlr_opts) == 817, "Incorrect size");
 
 /**
  * NVMe acceleration operation callback.

lib/nvme/nvme.c

@@ -962,6 +962,7 @@ nvme_ctrlr_opts_init(struct spdk_nvme_ctrlr_opts *opts,
 	SET_FIELD(fabrics_connect_timeout_us);
 	SET_FIELD(disable_read_ana_log_page);
 	SET_FIELD(disable_read_changed_ns_list_log_page);
+	SET_FIELD_ARRAY(psk);
 
 #undef FIELD_OK
 #undef SET_FIELD

lib/nvme/nvme_ctrlr.c

@@ -234,6 +234,10 @@ spdk_nvme_ctrlr_get_default_ctrlr_opts(struct spdk_nvme_ctrlr_opts *opts, size_t
 	SET_FIELD(disable_read_ana_log_page, false);
 	SET_FIELD(disable_read_changed_ns_list_log_page, false);
 
+	if (FIELD_OK(psk)) {
+		memset(opts->psk, 0, sizeof(opts->psk));
+	}
+
 #undef FIELD_OK
 #undef SET_FIELD
 }

lib/nvme/nvme_tcp.c

@@ -1882,6 +1882,9 @@ nvme_tcp_qpair_connect_sock(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpai
 	struct nvme_tcp_qpair *tqpair;
 	int family;
 	long int port;
+	char *sock_impl_name;
+	struct spdk_sock_impl_opts impl_opts;
+	size_t impl_opts_size = sizeof(impl_opts);
 	struct spdk_sock_opts opts;
 
 	tqpair = nvme_tcp_qpair(qpair);
@@ -1926,6 +1929,17 @@ nvme_tcp_qpair_connect_sock(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpai
 		return rc;
 	}
 
+	sock_impl_name = ctrlr->opts.psk[0] ? "ssl" : NULL;
+	SPDK_DEBUGLOG(nvme, "sock_impl_name is %s\n", sock_impl_name);
+
+	spdk_sock_impl_get_opts(sock_impl_name, &impl_opts, &impl_opts_size);
+	impl_opts.enable_ktls = false;
+	impl_opts.tls_version = SPDK_TLS_VERSION_1_3;
+	/* TODO: Change current PSK HEX string format to TLS PSK Interchange Format */
+	impl_opts.psk_key = ctrlr->opts.psk;
+	/* TODO: generate identity from hostnqn instead */
+	impl_opts.psk_identity = "psk.spdk.io";
+
 	opts.opts_size = sizeof(opts);
 	spdk_sock_get_default_opts(&opts);
 	opts.priority = ctrlr->trid.priority;
@@ -1933,7 +1947,11 @@ nvme_tcp_qpair_connect_sock(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpai
 	if (ctrlr->opts.transport_ack_timeout) {
 		opts.ack_timeout = 1ULL << ctrlr->opts.transport_ack_timeout;
 	}
-	tqpair->sock = spdk_sock_connect_ext(ctrlr->trid.traddr, port, NULL, &opts);
+	if (sock_impl_name) {
+		opts.impl_opts = &impl_opts;
+		opts.impl_opts_size = sizeof(impl_opts);
+	}
+	tqpair->sock = spdk_sock_connect_ext(ctrlr->trid.traddr, port, sock_impl_name, &opts);
 	if (!tqpair->sock) {
 		SPDK_ERRLOG("sock connection error of tqpair=%p with addr=%s, port=%ld\n",
 			    tqpair, ctrlr->trid.traddr, port);

lib/nvmf/tcp.c

@@ -777,6 +777,7 @@ nvmf_tcp_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_tr
 	opts.opts_size = sizeof(opts);
 	spdk_sock_get_default_opts(&opts);
 	opts.priority = ttransport->tcp_opts.sock_priority;
+	/* TODO: also add impl_opts like on the initiator side */
 	port->listen_sock = spdk_sock_listen_ext(trid->traddr, trsvcid_int,
 			    NULL, &opts);
 	if (port->listen_sock == NULL) {

module/bdev/nvme/bdev_nvme_rpc.c

@@ -163,6 +163,7 @@ struct rpc_bdev_nvme_attach_controller {
 	char *hostnqn;
 	char *hostaddr;
 	char *hostsvcid;
+	char *psk;
 	enum bdev_nvme_multipath_mode multipath;
 	struct nvme_ctrlr_opts bdev_opts;
 	struct spdk_nvme_ctrlr_opts drv_opts;
@@ -181,6 +182,7 @@ free_rpc_bdev_nvme_attach_controller(struct rpc_bdev_nvme_attach_controller *req
 	free(req->hostnqn);
 	free(req->hostaddr);
 	free(req->hostsvcid);
+	free(req->psk);
 }
 
 static int
@@ -256,6 +258,7 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_attach_controller_dec
 	{"ctrlr_loss_timeout_sec", offsetof(struct rpc_bdev_nvme_attach_controller, bdev_opts.ctrlr_loss_timeout_sec), spdk_json_decode_int32, true},
 	{"reconnect_delay_sec", offsetof(struct rpc_bdev_nvme_attach_controller, bdev_opts.reconnect_delay_sec), spdk_json_decode_uint32, true},
 	{"fast_io_fail_timeout_sec", offsetof(struct rpc_bdev_nvme_attach_controller, bdev_opts.fast_io_fail_timeout_sec), spdk_json_decode_uint32, true},
+	{"psk", offsetof(struct rpc_bdev_nvme_attach_controller, psk), spdk_json_decode_string, true},
 };
 
 #define NVME_MAX_BDEVS_PER_RPC 128
@@ -407,6 +410,11 @@ rpc_bdev_nvme_attach_controller(struct spdk_jsonrpc_request *request,
 			 ctx->req.hostnqn);
 	}
 
+	if (ctx->req.psk) {
+		snprintf(ctx->req.drv_opts.psk, sizeof(ctx->req.drv_opts.psk), "%s",
+			 ctx->req.psk);
+	}
+
 	if (ctx->req.hostaddr) {
 		maxlen = sizeof(ctx->req.drv_opts.src_addr);
 		len = strnlen(ctx->req.hostaddr, maxlen);

python/spdk/rpc/bdev.py

@@ -600,7 +600,7 @@ def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvc
                                 hostsvcid=None, prchk_reftag=None, prchk_guard=None,
                                 hdgst=None, ddgst=None, fabrics_timeout=None, multipath=None, num_io_queues=None,
                                 ctrlr_loss_timeout_sec=None, reconnect_delay_sec=None,
-                                fast_io_fail_timeout_sec=None):
+                                fast_io_fail_timeout_sec=None, psk=None):
     """Construct block device for each NVMe namespace in the attached controller.
 
     Args:
@@ -635,6 +635,7 @@ def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvc
         0 means no such timeout.
         If fast_io_fail_timeout_sec is not zero, it has to be not less than reconnect_delay_sec and less than
         ctrlr_loss_timeout_sec if ctrlr_loss_timeout_sec is not -1. (optional)
+        psk: Set PSK and enable TCP SSL socket implementation (optional)
 
     Returns:
         Names of created block devices.
@@ -694,6 +695,9 @@ def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvc
     if fast_io_fail_timeout_sec is not None:
         params['fast_io_fail_timeout_sec'] = fast_io_fail_timeout_sec
 
+    if psk:
+        params['psk'] = psk
+
     return client.call('bdev_nvme_attach_controller', params)
 
 

scripts/rpc.py

@@ -608,7 +608,8 @@ if __name__ == "__main__":
                                                          num_io_queues=args.num_io_queues,
                                                          ctrlr_loss_timeout_sec=args.ctrlr_loss_timeout_sec,
                                                          reconnect_delay_sec=args.reconnect_delay_sec,
-                                                         fast_io_fail_timeout_sec=args.fast_io_fail_timeout_sec))
+                                                         fast_io_fail_timeout_sec=args.fast_io_fail_timeout_sec,
+                                                         psk=args.psk))
 
     p = subparsers.add_parser('bdev_nvme_attach_controller', help='Add bdevs with nvme backend')
     p.add_argument('-b', '--name', help="Name of the NVMe controller, prefix for each bdev name", required=True)
@@ -659,6 +660,8 @@ if __name__ == "__main__":
                    If fast_io_fail_timeout_sec is not zero, it has to be not less than reconnect_delay_sec and
                    less than ctrlr_loss_timeout_sec if ctrlr_loss_timeout_sec is not -1.""",
                    type=int)
+    p.add_argument('-k', '--psk',
+                   help='Set PSK and enable TCP SSL socket implementation: e.g., 1234567890ABCDEF')
     p.set_defaults(func=bdev_nvme_attach_controller)
 
     def bdev_nvme_get_controllers(args):

test/common/lib/test_sock.c

@@ -17,6 +17,8 @@ DEFINE_STUB(spdk_sock_listen, struct spdk_sock *, (const char *ip, int port, con
 DEFINE_STUB(spdk_sock_listen_ext, struct spdk_sock *, (const char *ip, int port,
 		const char *impl_name, struct spdk_sock_opts *opts), NULL);
 DEFINE_STUB_V(spdk_sock_get_default_opts, (struct spdk_sock_opts *opts));
+DEFINE_STUB(spdk_sock_impl_get_opts, int, (const char *impl_name, struct spdk_sock_impl_opts *opts,
+		size_t *len), 0);
 DEFINE_STUB(spdk_sock_accept, struct spdk_sock *, (struct spdk_sock *sock), NULL);
 DEFINE_STUB(spdk_sock_close, int, (struct spdk_sock **sock), 0);
 DEFINE_STUB(spdk_sock_recv, ssize_t, (struct spdk_sock *sock, void *buf, size_t len), 1);

test/nvmf/target/tls.sh

@@ -116,5 +116,20 @@ $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 malloc0 -n 1
 	-r "trtype:${TEST_TRANSPORT} adrfam:IPv4 traddr:${NVMF_FIRST_TARGET_IP} trsvcid:${NVMF_PORT} \
 subnqn:nqn.2016-06.io.spdk:cnode1" --psk-key 1234567890ABCDEF --psk-identity psk.spdk.io
 
+# use bdevperf to test "bdev_nvme_attach_controller"
+bdevperf_rpc_sock=/var/tmp/bdevperf.sock
+$rootdir/test/bdev/bdevperf/bdevperf -m 0x4 -z -r $bdevperf_rpc_sock -q 128 -o 4096 -w verify -t 10 &
+bdevperf_pid=$!
+
+trap 'process_shm --id $NVMF_APP_SHM_ID; killprocess $bdevperf_pid; nvmftestfini; exit 1' SIGINT SIGTERM EXIT
+waitforlisten $bdevperf_pid $bdevperf_rpc_sock
+# send RPC
+$rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b TLSTEST -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP \
+	-s $NVMF_PORT -f ipv4 -n nqn.2016-06.io.spdk:cnode1 --psk 1234567890ABCDEF
+# run I/O and wait
+$rootdir/test/bdev/bdevperf/bdevperf.py -t 20 -s $bdevperf_rpc_sock perform_tests
+# finish
+killprocess $bdevperf_pid
+
 trap - SIGINT SIGTERM EXIT
 nvmftestfini