sock: Add ktls and tls_version to spdk_sock_opts

See https://docs.kernel.org/networking/tls-offload.html
See https://www.openssl.org/docs/man3.0/man3/SSL_set_options.html

Change-Id: I2fb433cbc34061cb03e1591bb0b47063fcafc68c
Signed-off-by: Boris Glimcher <Boris.Glimcher@emc.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13071
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>

CHANGELOG.md

@@ -17,6 +17,10 @@ in the future.
 Add num_md_pages_per_cluster_ratio parameter to the bdev_lvol_create_lvstore RPC.
 Calculate num_md_pages from num_md_pages_per_cluster_ratio, and pass it to spdk_bs_opts.
 
+### rpc
+
+New options `ktls` and `tls_version` were added to the `spdk_sock_opts` structure.
+
 ## v22.05
 
 ### sock

examples/sock/hello_world/hello_sock.c

@@ -24,6 +24,8 @@ static char *g_sock_impl_name;
 static int g_port;
 static bool g_is_server;
 static int g_zcopy;
+static int g_ktls;
+static int g_tls_version;
 static bool g_verbose;
 
 /*
@@ -36,6 +38,8 @@ struct hello_context_t {
 	char *sock_impl_name;
 	int port;
 	int zcopy;
+	int ktls;
+	int tls_version;
 
 	bool verbose;
 	int bytes_in;
@@ -61,6 +65,9 @@ hello_sock_usage(void)
 	printf(" -P port       port number\n");
 	printf(" -N sock_impl  socket implementation, e.g., -N posix or -N uring\n");
 	printf(" -S            start in server mode\n");
+	printf(" -T tls_ver    TLS version, e.g., -T 12 or -T 13. If omitted, auto-negotiation will take place\n");
+	printf(" -k            disable KTLS for the given sock implementation (default)\n");
+	printf(" -K            enable KTLS for the given sock implementation\n");
 	printf(" -V            print out additional informations\n");
 	printf(" -z            disable zero copy send for the given sock implementation\n");
 	printf(" -Z            enable zero copy send for the given sock implementation\n");
@@ -89,6 +96,19 @@ hello_sock_parse_arg(int ch, char *arg)
 	case 'S':
 		g_is_server = 1;
 		break;
+	case 'K':
+		g_ktls = 1;
+		break;
+	case 'k':
+		g_ktls = 0;
+		break;
+	case 'T':
+		g_tls_version = spdk_strtol(arg, 10);
+		if (g_tls_version < 0) {
+			fprintf(stderr, "Invalid TLS version\n");
+			return g_tls_version;
+		}
+		break;
 	case 'V':
 		g_verbose = true;
 		break;
@@ -203,6 +223,8 @@ hello_sock_connect(struct hello_context_t *ctx)
 	opts.opts_size = sizeof(opts);
 	spdk_sock_get_default_opts(&opts);
 	opts.zcopy = ctx->zcopy;
+	opts.ktls = ctx->ktls;
+	opts.tls_version = ctx->tls_version;
 
 	SPDK_NOTICELOG("Connecting to the server on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port,
 		       ctx->sock_impl_name);
@@ -339,6 +361,8 @@ hello_sock_listen(struct hello_context_t *ctx)
 	opts.opts_size = sizeof(opts);
 	spdk_sock_get_default_opts(&opts);
 	opts.zcopy = ctx->zcopy;
+	opts.ktls = ctx->ktls;
+	opts.tls_version = ctx->tls_version;
 
 	ctx->sock = spdk_sock_listen_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts);
 	if (ctx->sock == NULL) {
@@ -407,7 +431,7 @@ main(int argc, char **argv)
 	opts.name = "hello_sock";
 	opts.shutdown_cb = hello_sock_shutdown_cb;
 
-	if ((rc = spdk_app_parse_args(argc, argv, &opts, "H:N:P:SVzZ", NULL, hello_sock_parse_arg,
+	if ((rc = spdk_app_parse_args(argc, argv, &opts, "H:kKN:P:ST:VzZ", NULL, hello_sock_parse_arg,
 				      hello_sock_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) {
 		exit(rc);
 	}
@@ -416,6 +440,8 @@ main(int argc, char **argv)
 	hello_context.sock_impl_name = g_sock_impl_name;
 	hello_context.port = g_port;
 	hello_context.zcopy = g_zcopy;
+	hello_context.ktls = g_ktls;
+	hello_context.tls_version = g_tls_version;
 	hello_context.verbose = g_verbose;
 
 	rc = spdk_app_start(&opts, hello_start, &hello_context);

include/spdk/sock.h

@@ -65,6 +65,10 @@ enum spdk_placement_mode {
 	PLACEMENT_MARK,
 };
 
+#define SPDK_TLS_VERSION_1_1 11
+#define SPDK_TLS_VERSION_1_2 12
+#define SPDK_TLS_VERSION_1_3 13
+
 /**
  * SPDK socket implementation options.
  *
@@ -150,6 +154,17 @@ struct spdk_sock_opts {
 	 * Time in msec to wait ack until connection is closed forcefully.
 	 */
 	uint32_t ack_timeout;
+
+	/**
+	 * TLS protocol version. Used by posix socket module.
+	 */
+	uint32_t tls_version;
+
+	/**
+	 * Used to enable or disable KTLS for ssl posix socket module.
+	 */
+	bool ktls;
+
 };
 
 /**

lib/sock/Makefile

@@ -7,7 +7,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 6
+SO_VER := 7
 SO_MINOR := 0
 
 C_SRCS = sock.c sock_rpc.c

lib/sock/sock.c

@@ -15,6 +15,8 @@
 #define SPDK_SOCK_DEFAULT_PRIORITY 0
 #define SPDK_SOCK_DEFAULT_ZCOPY true
 #define SPDK_SOCK_DEFAULT_ACK_TIMEOUT 0
+#define SPDK_SOCK_DEFAULT_TLS_VERSION 0
+#define SPDK_SOCK_DEFAULT_KTLS false
 
 #define SPDK_SOCK_OPTS_FIELD_OK(opts, field) (offsetof(struct spdk_sock_opts, field) + sizeof(opts->field) <= (opts->opts_size))
 
@@ -245,6 +247,14 @@ spdk_sock_get_default_opts(struct spdk_sock_opts *opts)
 	if (SPDK_SOCK_OPTS_FIELD_OK(opts, ack_timeout)) {
 		opts->ack_timeout = SPDK_SOCK_DEFAULT_ACK_TIMEOUT;
 	}
+
+	if (SPDK_SOCK_OPTS_FIELD_OK(opts, tls_version)) {
+		opts->tls_version = SPDK_SOCK_DEFAULT_TLS_VERSION;
+	}
+
+	if (SPDK_SOCK_OPTS_FIELD_OK(opts, ktls)) {
+		opts->ktls = SPDK_SOCK_DEFAULT_KTLS;
+	}
 }
 
 /*
@@ -273,6 +283,14 @@ sock_init_opts(struct spdk_sock_opts *opts, struct spdk_sock_opts *opts_user)
 	if (SPDK_SOCK_OPTS_FIELD_OK(opts, ack_timeout)) {
 		opts->ack_timeout = opts_user->ack_timeout;
 	}
+
+	if (SPDK_SOCK_OPTS_FIELD_OK(opts, tls_version)) {
+		opts->tls_version = opts_user->tls_version;
+	}
+
+	if (SPDK_SOCK_OPTS_FIELD_OK(opts, ktls)) {
+		opts->ktls = opts_user->ktls;
+	}
 }
 
 struct spdk_sock *

module/sock/posix/posix.c

@@ -506,9 +506,14 @@ err:
 }
 
 static SSL_CTX *
-posix_sock_create_ssl_context(const SSL_METHOD *method)
+posix_sock_create_ssl_context(const SSL_METHOD *method, struct spdk_sock_opts *opts)
 {
 	SSL_CTX *ctx;
+	int tls_version = 0;
+	bool ktls_enabled = false;
+#ifdef SSL_OP_ENABLE_KTLS
+	long options;
+#endif
 
 	SSL_library_init();
 	OpenSSL_add_all_algorithms();
@@ -516,11 +521,59 @@ posix_sock_create_ssl_context(const SSL_METHOD *method)
 	/* Produce a SSL CTX in SSL V2 and V3 standards compliant way */
 	ctx = SSL_CTX_new(method);
 	if (!ctx) {
-		SPDK_ERRLOG("SSL_CTX_new() failed, errno = %d\n", errno);
+		SPDK_ERRLOG("SSL_CTX_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL));
 		return NULL;
 	}
 	SPDK_DEBUGLOG(sock_posix, "SSL context created\n");
+
+	switch (opts->tls_version) {
+	case 0:
+		/* auto-negotioation */
+		break;
+	case SPDK_TLS_VERSION_1_1:
+		tls_version = TLS1_1_VERSION;
+		break;
+	case SPDK_TLS_VERSION_1_2:
+		tls_version = TLS1_2_VERSION;
+		break;
+	case SPDK_TLS_VERSION_1_3:
+		tls_version = TLS1_3_VERSION;
+		break;
+	default:
+		SPDK_ERRLOG("Incorrect TLS version provided: %d\n", opts->tls_version);
+		goto err;
+	}
+
+	if (tls_version) {
+		SPDK_DEBUGLOG(sock_posix, "Hardening TLS version to '%d'='0x%X'\n", opts->tls_version, tls_version);
+		if (!SSL_CTX_set_min_proto_version(ctx, tls_version)) {
+			SPDK_ERRLOG("Unable to set Min TLS version to '%d'='0x%X\n", opts->tls_version, tls_version);
+			goto err;
+		}
+		if (!SSL_CTX_set_max_proto_version(ctx, tls_version)) {
+			SPDK_ERRLOG("Unable to set Max TLS version to '%d'='0x%X\n", opts->tls_version, tls_version);
+			goto err;
+		}
+	}
+	if (opts->ktls) {
+		SPDK_DEBUGLOG(sock_posix, "Enabling kTLS offload\n");
+#ifdef SSL_OP_ENABLE_KTLS
+		options = SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS);
+		ktls_enabled = options & SSL_OP_ENABLE_KTLS;
+#else
+		ktls_enabled = false;
+#endif
+		if (!ktls_enabled) {
+			SPDK_ERRLOG("Unable to set kTLS offload via SSL_CTX_set_options(). Configure openssl with 'enable-ktls'\n");
+			goto err;
+		}
+	}
+
 	return ctx;
+
+err:
+	SSL_CTX_free(ctx);
+	return NULL;
 }
 
 static SSL *
@@ -532,7 +585,7 @@ ssl_sock_connect_loop(SSL_CTX *ctx, int fd)
 
 	ssl = SSL_new(ctx);
 	if (!ssl) {
-		SPDK_ERRLOG("SSL_new() failed, errno = %d\n", errno);
+		SPDK_ERRLOG("SSL_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL));
 		return NULL;
 	}
 	SSL_set_fd(ssl, fd);
@@ -570,7 +623,7 @@ ssl_sock_accept_loop(SSL_CTX *ctx, int fd)
 
 	ssl = SSL_new(ctx);
 	if (!ssl) {
-		SPDK_ERRLOG("SSL_new() failed, errno = %d\n", errno);
+		SPDK_ERRLOG("SSL_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL));
 		return NULL;
 	}
 	SSL_set_fd(ssl, fd);
@@ -742,7 +795,7 @@ retry:
 		}
 		if (type == SPDK_SOCK_CREATE_LISTEN) {
 			if (enable_ssl) {
-				ctx = posix_sock_create_ssl_context(TLS_server_method());
+				ctx = posix_sock_create_ssl_context(TLS_server_method(), opts);
 				if (!ctx) {
 					SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno);
 					close(fd);
@@ -791,7 +844,7 @@ retry:
 			}
 			enable_zcopy_impl_opts = g_spdk_posix_sock_impl_opts.enable_zerocopy_send_client;
 			if (enable_ssl) {
-				ctx = posix_sock_create_ssl_context(TLS_client_method());
+				ctx = posix_sock_create_ssl_context(TLS_client_method(), opts);
 				if (!ctx) {
 					SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno);
 					close(fd);

test/iscsi_tgt/sock/sock.sh

@@ -131,6 +131,36 @@ if ! echo "$response" | grep -q "$message"; then
 	exit 1
 fi
 
+# send message using hello_sock client using TLS 1.3
+message="**MESSAGE:This is a test message from the hello_sock client with ssl using TLS 1.3**"
+response=$(echo $message | $HELLO_SOCK_APP -H $TARGET_IP -P $ISCSI_PORT -N "ssl" -T 13)
+if ! echo "$response" | grep -q "$message"; then
+	exit 1
+fi
+
+# send message using hello_sock client using TLS 1.2
+message="**MESSAGE:This is a test message from the hello_sock client with ssl using TLS 1.2**"
+response=$(echo $message | $HELLO_SOCK_APP -H $TARGET_IP -P $ISCSI_PORT -N "ssl" -T 12)
+if ! echo "$response" | grep -q "$message"; then
+	exit 1
+fi
+
+# send message using hello_sock client using incorrect TLS 7
+message="**MESSAGE:This is a test message from the hello_sock client with ssl using incorrect TLS 7**"
+echo $message | $HELLO_SOCK_APP -H $TARGET_IP -P $ISCSI_PORT -N "ssl" -T 7 && exit 1
+
+# send message using hello_sock client with KTLS disabled
+message="**MESSAGE:This is a test message from the hello_sock client with KTLS disabled**"
+response=$(echo $message | $HELLO_SOCK_APP -H $TARGET_IP -P $ISCSI_PORT -N "ssl" -k)
+if ! echo "$response" | grep -q "$message"; then
+	exit 1
+fi
+
+# send message using hello_sock client with KTLS enabled
+message="**MESSAGE:This is a test message from the hello_sock client with KTLS enabled**"
+# UT infrastructure so far doesn't support new openssl-3 with this option, so expect a failure
+echo $message | $HELLO_SOCK_APP -H $TARGET_IP -P $ISCSI_PORT -N "ssl" -K && exit 1
+
 # send message using openssl client using TLS 1.3
 message="**MESSAGE:This is a test message from the openssl client using TLS 1.3**"
 response=$( (