sock: Add ktls and tls_version to spdk_sock_impl_opts

Since `sock_impl_opts` was added to `sock_opts`
Can remove `ktls` and `tls_version` from spdk_sock_opts

Example:
  rpc.py sock_impl_set_options -i ssl --enable-ktls
  rpc.py sock_impl_set_options -i ssl --disable-ktls
  rpc.py sock_impl_set_options -i ssl --tls-version=12

  ./build/examples/perf --enable-ktls
  ./build/examples/perf --disable-ktls
  ./build/examples/perf --tls-version=12

Check kTLS statistics here: /proc/net/tls_stat

Change-Id: Icf7ee822bad92fda149710be77feb77fc8d4f163
Signed-off-by: Boris Glimcher <Boris.Glimcher@emc.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13510
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Boris Glimcher 2022-05-02 19:08:50 +03:00 committed by Konrad Sztyber
parent d0038b70df
commit 806744b7c8
12 changed files with 183 additions and 55 deletions

View File

@ -23,7 +23,7 @@ Calculate num_md_pages from num_md_pages_per_cluster_ratio, and pass it to spdk_
### rpc
New options `ktls` and `tls_version` were added to the `spdk_sock_opts` structure.
New options `enable_ktls` and `tls_version` were added to the `sock_impl_set_options` structure.
Added warning message for `bdev_rbd_create`, if it is used without -c.
`bdev_rbd_create()` API without specifying -c is deprecated and will be removed in future release.

View File

@ -9518,7 +9518,9 @@ Example response:
"enable_placement_id": 0,
"enable_zerocopy_send_server": true,
"enable_zerocopy_send_client": false,
"zerocopy_threshold": 0
"zerocopy_threshold": 0,
"tls_version": 13,
"enable_ktls": false
}
}
~~~
@ -9541,6 +9543,8 @@ enable_zerocopy_send_server | Optional | boolean | Enable or disable zero co
enable_zerocopy_send_client | Optional | boolean | Enable or disable zero copy on send for client sockets
zerocopy_threshold | Optional | number | Set zerocopy_threshold in bytes. A consecutive sequence of requests' iovecs
that fall below this threshold may be sent without zerocopy flag set
tls_version | Optional | number | TLS protocol version, e.g. 13 for v1.3 (only applies when impl_name == ssl)
enable_ktls | Optional | boolean | Enable or disable Kernel TLS (only applies when impl_name == ssl)
#### Response
@ -9564,7 +9568,9 @@ Example request:
"enable_placement_id": 0,
"enable_zerocopy_send_server": true,
"enable_zerocopy_send_client": false,
"zerocopy_threshold": 10240
"zerocopy_threshold": 10240,
"tls_version": 13,
"enable_ktls": false
}
}
~~~

View File

@ -291,7 +291,7 @@ static int g_file_optind; /* Index of first filename in argv */
static inline void task_complete(struct perf_task *task);
static void
perf_set_sock_zcopy(const char *impl_name, bool enable)
perf_set_sock_opts(const char *impl_name, const char *field, uint32_t val)
{
struct spdk_sock_impl_opts sock_opts = {};
size_t opts_size = sizeof(sock_opts);
@ -314,11 +314,23 @@ perf_set_sock_zcopy(const char *impl_name, bool enable)
opts_size = sizeof(sock_opts);
}
sock_opts.enable_zerocopy_send_client = enable;
if (!field) {
fprintf(stderr, "Warning: no socket opts field specified\n");
return;
} else if (strcmp(field, "enable_zerocopy_send_client") == 0) {
sock_opts.enable_zerocopy_send_client = val;
} else if (strcmp(field, "tls_version") == 0) {
sock_opts.tls_version = val;
} else if (strcmp(field, "ktls") == 0) {
sock_opts.enable_ktls = val;
} else {
fprintf(stderr, "Warning: invalid or unprocessed socket opts field: %s\n", field);
return;
}
if (spdk_sock_impl_set_opts(impl_name, &sock_opts, opts_size)) {
fprintf(stderr, "Failed to %s zcopy send for sock impl %s: error %d (%s)\n",
enable ? "enable" : "disable", impl_name, errno, strerror(errno));
fprintf(stderr, "Failed to set %s: %d for sock impl %s : error %d (%s)\n", field, val, impl_name,
errno, strerror(errno));
}
}
@ -1775,6 +1787,9 @@ usage(char *program_name)
printf("\t[--transport-stats dump transport statistics]\n");
printf("\t[--iova-mode <mode> specify DPDK IOVA mode: va|pa]\n");
printf("\t[--io-queue-size <val> size of NVMe IO queue. Default: maximum allowed by controller]\n");
printf("\t[--disable-ktls disable Kernel TLS. Only valid for ssl impl. Default for ssl impl]\n");
printf("\t[--enable-ktls enable Kernel TLS. Only valid for ssl impl]\n");
printf("\t[--tls-version <val> TLS version to use. Only valid for ssl impl. Default: 0 (auto-negotiation)]\n");
}
static void
@ -2267,6 +2282,12 @@ static const struct option g_perf_cmdline_opts[] = {
{"iova-mode", required_argument, NULL, PERF_IOVA_MODE},
#define PERF_IO_QUEUE_SIZE 259
{"io-queue-size", required_argument, NULL, PERF_IO_QUEUE_SIZE},
#define PERF_DISABLE_KTLS 260
{"disable-ktls", no_argument, NULL, PERF_DISABLE_KTLS},
#define PERF_ENABLE_KTLS 261
{"enable-ktls", no_argument, NULL, PERF_ENABLE_KTLS},
#define PERF_TLS_VERSION 262
{"tls-version", required_argument, NULL, PERF_TLS_VERSION},
/* Should be the last element */
{0, 0, 0, 0}
};
@ -2447,11 +2468,25 @@ parse_args(int argc, char **argv, struct spdk_env_opts *env_opts)
case PERF_ENABLE_VMD:
g_vmd = true;
break;
case PERF_DISABLE_KTLS:
perf_set_sock_opts(optarg, "ktls", 0);
break;
case PERF_ENABLE_KTLS:
perf_set_sock_opts(optarg, "ktls", 1);
break;
case PERF_TLS_VERSION:
val = spdk_strtol(optarg, 10);
if (val < 0) {
fprintf(stderr, "Illegal tls version value %s\n", optarg);
return val;
}
perf_set_sock_opts(optarg, "tls_version", val);
break;
case PERF_DISABLE_ZCOPY:
perf_set_sock_zcopy(optarg, false);
perf_set_sock_opts(optarg, "enable_zerocopy_send_client", 0);
break;
case PERF_ENABLE_ZCOPY:
perf_set_sock_zcopy(optarg, true);
perf_set_sock_opts(optarg, "enable_zerocopy_send_client", 1);
break;
case PERF_DEFAULT_SOCK_IMPL:
rc = spdk_sock_set_default_impl(optarg);

View File

@ -218,13 +218,19 @@ hello_sock_connect(struct hello_context_t *ctx)
int rc;
char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN];
uint16_t cport, sport;
struct spdk_sock_impl_opts impl_opts;
size_t impl_opts_size = sizeof(impl_opts);
struct spdk_sock_opts opts;
spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size);
impl_opts.enable_ktls = ctx->ktls;
impl_opts.tls_version = ctx->tls_version;
opts.opts_size = sizeof(opts);
spdk_sock_get_default_opts(&opts);
opts.zcopy = ctx->zcopy;
opts.ktls = ctx->ktls;
opts.tls_version = ctx->tls_version;
opts.impl_opts = &impl_opts;
opts.impl_opts_size = sizeof(impl_opts);
SPDK_NOTICELOG("Connecting to the server on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port,
ctx->sock_impl_name);
@ -356,13 +362,19 @@ hello_sock_group_poll(void *arg)
static int
hello_sock_listen(struct hello_context_t *ctx)
{
struct spdk_sock_impl_opts impl_opts;
size_t impl_opts_size = sizeof(impl_opts);
struct spdk_sock_opts opts;
spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size);
impl_opts.enable_ktls = ctx->ktls;
impl_opts.tls_version = ctx->tls_version;
opts.opts_size = sizeof(opts);
spdk_sock_get_default_opts(&opts);
opts.zcopy = ctx->zcopy;
opts.ktls = ctx->ktls;
opts.tls_version = ctx->tls_version;
opts.impl_opts = &impl_opts;
opts.impl_opts_size = sizeof(impl_opts);
ctx->sock = spdk_sock_listen_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts);
if (ctx->sock == NULL) {

View File

@ -128,6 +128,16 @@ struct spdk_sock_impl_opts {
* threshold may be sent without zerocopy flag set.
*/
uint32_t zerocopy_threshold;
/**
* TLS protocol version. Used by ssl socket module.
*/
uint32_t tls_version;
/**
* Enable or disable kernel TLS. Used by ssl socket modules.
*/
bool enable_ktls;
};
/**
@ -159,16 +169,6 @@ struct spdk_sock_opts {
*/
uint32_t ack_timeout;
/**
* TLS protocol version. Used by posix socket module.
*/
uint32_t tls_version;
/**
* Used to enable or disable KTLS for ssl posix socket module.
*/
bool ktls;
/**
* Socket implementation options. If non-NULL, these will override those set by
* spdk_sock_impl_set_opts(). The library copies this structure internally, so the user can

View File

@ -15,8 +15,6 @@
#define SPDK_SOCK_DEFAULT_PRIORITY 0
#define SPDK_SOCK_DEFAULT_ZCOPY true
#define SPDK_SOCK_DEFAULT_ACK_TIMEOUT 0
#define SPDK_SOCK_DEFAULT_TLS_VERSION 0
#define SPDK_SOCK_DEFAULT_KTLS false
#define SPDK_SOCK_OPTS_FIELD_OK(opts, field) (offsetof(struct spdk_sock_opts, field) + sizeof(opts->field) <= (opts->opts_size))
@ -248,14 +246,6 @@ spdk_sock_get_default_opts(struct spdk_sock_opts *opts)
opts->ack_timeout = SPDK_SOCK_DEFAULT_ACK_TIMEOUT;
}
if (SPDK_SOCK_OPTS_FIELD_OK(opts, tls_version)) {
opts->tls_version = SPDK_SOCK_DEFAULT_TLS_VERSION;
}
if (SPDK_SOCK_OPTS_FIELD_OK(opts, ktls)) {
opts->ktls = SPDK_SOCK_DEFAULT_KTLS;
}
if (SPDK_SOCK_OPTS_FIELD_OK(opts, impl_opts)) {
opts->impl_opts = NULL;
}
@ -292,14 +282,6 @@ sock_init_opts(struct spdk_sock_opts *opts, struct spdk_sock_opts *opts_user)
opts->ack_timeout = opts_user->ack_timeout;
}
if (SPDK_SOCK_OPTS_FIELD_OK(opts, tls_version)) {
opts->tls_version = opts_user->tls_version;
}
if (SPDK_SOCK_OPTS_FIELD_OK(opts, ktls)) {
opts->ktls = opts_user->ktls;
}
if (SPDK_SOCK_OPTS_FIELD_OK(opts, impl_opts)) {
opts->impl_opts = opts_user->impl_opts;
}
@ -883,6 +865,8 @@ spdk_sock_write_config_json(struct spdk_json_write_ctx *w)
spdk_json_write_named_bool(w, "enable_zerocopy_send_server", opts.enable_zerocopy_send_server);
spdk_json_write_named_bool(w, "enable_zerocopy_send_client", opts.enable_zerocopy_send_client);
spdk_json_write_named_uint32(w, "zerocopy_threshold", opts.zerocopy_threshold);
spdk_json_write_named_uint32(w, "tls_version", opts.tls_version);
spdk_json_write_named_bool(w, "enable_ktls", opts.enable_ktls);
spdk_json_write_object_end(w);
spdk_json_write_object_end(w);
} else {

View File

@ -52,6 +52,8 @@ rpc_sock_impl_get_options(struct spdk_jsonrpc_request *request,
spdk_json_write_named_bool(w, "enable_zerocopy_send_server", sock_opts.enable_zerocopy_send_server);
spdk_json_write_named_bool(w, "enable_zerocopy_send_client", sock_opts.enable_zerocopy_send_client);
spdk_json_write_named_uint32(w, "zerocopy_threshold", sock_opts.zerocopy_threshold);
spdk_json_write_named_uint32(w, "tls_version", sock_opts.tls_version);
spdk_json_write_named_bool(w, "enable_ktls", sock_opts.enable_ktls);
spdk_json_write_object_end(w);
spdk_jsonrpc_end_result(request, w);
free(impl_name);
@ -100,6 +102,14 @@ static const struct spdk_json_object_decoder rpc_sock_impl_set_opts_decoders[] =
{
"zerocopy_threshold", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.zerocopy_threshold),
spdk_json_decode_uint32, true
},
{
"tls_version", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.tls_version),
spdk_json_decode_uint32, true
},
{
"enable_ktls", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_ktls),
spdk_json_decode_bool, true
}
};

View File

@ -76,7 +76,9 @@ static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = {
.enable_placement_id = PLACEMENT_NONE,
.enable_zerocopy_send_server = true,
.enable_zerocopy_send_client = false,
.zerocopy_threshold = 0
.zerocopy_threshold = 0,
.tls_version = 0,
.enable_ktls = false
};
static struct spdk_sock_map g_map = {
@ -114,6 +116,8 @@ posix_sock_copy_impl_opts(struct spdk_sock_impl_opts *dest, const struct spdk_so
SET_FIELD(enable_zerocopy_send_server);
SET_FIELD(enable_zerocopy_send_client);
SET_FIELD(zerocopy_threshold);
SET_FIELD(tls_version);
SET_FIELD(enable_ktls);
#undef SET_FIELD
#undef FIELD_OK
@ -577,7 +581,8 @@ err:
}
static SSL_CTX *
posix_sock_create_ssl_context(const SSL_METHOD *method, struct spdk_sock_opts *opts)
posix_sock_create_ssl_context(const SSL_METHOD *method, struct spdk_sock_opts *opts,
struct spdk_sock_impl_opts *impl_opts)
{
SSL_CTX *ctx;
int tls_version = 0;
@ -597,7 +602,7 @@ posix_sock_create_ssl_context(const SSL_METHOD *method, struct spdk_sock_opts *o
}
SPDK_DEBUGLOG(sock_posix, "SSL context created\n");
switch (opts->tls_version) {
switch (impl_opts->tls_version) {
case 0:
/* auto-negotioation */
break;
@ -611,22 +616,23 @@ posix_sock_create_ssl_context(const SSL_METHOD *method, struct spdk_sock_opts *o
tls_version = TLS1_3_VERSION;
break;
default:
SPDK_ERRLOG("Incorrect TLS version provided: %d\n", opts->tls_version);
SPDK_ERRLOG("Incorrect TLS version provided: %d\n", impl_opts->tls_version);
goto err;
}
if (tls_version) {
SPDK_DEBUGLOG(sock_posix, "Hardening TLS version to '%d'='0x%X'\n", opts->tls_version, tls_version);
SPDK_DEBUGLOG(sock_posix, "Hardening TLS version to '%d'='0x%X'\n", impl_opts->tls_version,
tls_version);
if (!SSL_CTX_set_min_proto_version(ctx, tls_version)) {
SPDK_ERRLOG("Unable to set Min TLS version to '%d'='0x%X\n", opts->tls_version, tls_version);
SPDK_ERRLOG("Unable to set Min TLS version to '%d'='0x%X\n", impl_opts->tls_version, tls_version);
goto err;
}
if (!SSL_CTX_set_max_proto_version(ctx, tls_version)) {
SPDK_ERRLOG("Unable to set Max TLS version to '%d'='0x%X\n", opts->tls_version, tls_version);
SPDK_ERRLOG("Unable to set Max TLS version to '%d'='0x%X\n", impl_opts->tls_version, tls_version);
goto err;
}
}
if (opts->ktls) {
if (impl_opts->enable_ktls) {
SPDK_DEBUGLOG(sock_posix, "Enabling kTLS offload\n");
#ifdef SSL_OP_ENABLE_KTLS
options = SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS);
@ -868,7 +874,7 @@ retry:
}
if (type == SPDK_SOCK_CREATE_LISTEN) {
if (enable_ssl) {
ctx = posix_sock_create_ssl_context(TLS_server_method(), opts);
ctx = posix_sock_create_ssl_context(TLS_server_method(), opts, &impl_opts);
if (!ctx) {
SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno);
close(fd);
@ -917,7 +923,7 @@ retry:
}
enable_zcopy_impl_opts = impl_opts.enable_zerocopy_send_client;
if (enable_ssl) {
ctx = posix_sock_create_ssl_context(TLS_client_method(), opts);
ctx = posix_sock_create_ssl_context(TLS_client_method(), opts, &impl_opts);
if (!ctx) {
SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno);
close(fd);

View File

@ -95,7 +95,9 @@ static struct spdk_sock_impl_opts g_spdk_uring_sock_impl_opts = {
.enable_placement_id = PLACEMENT_NONE,
.enable_zerocopy_send_server = false,
.enable_zerocopy_send_client = false,
.zerocopy_threshold = 0
.zerocopy_threshold = 0,
.tls_version = 0,
.enable_ktls = false
};
static struct spdk_sock_map g_map = {
@ -134,6 +136,8 @@ uring_sock_copy_impl_opts(struct spdk_sock_impl_opts *dest, const struct spdk_so
SET_FIELD(enable_zerocopy_send_server);
SET_FIELD(enable_zerocopy_send_client);
SET_FIELD(zerocopy_threshold);
SET_FIELD(tls_version);
SET_FIELD(enable_ktls);
#undef SET_FIELD
#undef FIELD_OK

View File

@ -20,7 +20,9 @@ def sock_impl_set_options(client,
enable_placement_id=None,
enable_zerocopy_send_server=None,
enable_zerocopy_send_client=None,
zerocopy_threshold=None):
zerocopy_threshold=None,
tls_version=None,
enable_ktls=None):
"""Set parameters for the socket layer implementation.
Args:
@ -33,6 +35,8 @@ def sock_impl_set_options(client,
enable_zerocopy_send_server: enable or disable zerocopy on send for server sockets(optional)
enable_zerocopy_send_client: enable or disable zerocopy on send for client sockets(optional)
zerocopy_threshold: set zerocopy_threshold in bytes(optional)
tls_version: set TLS protocol version (optional)
enable_ktls: enable or disable Kernel TLS (optional)
"""
params = {}
@ -53,6 +57,10 @@ def sock_impl_set_options(client,
params['enable_zerocopy_send_client'] = enable_zerocopy_send_client
if zerocopy_threshold is not None:
params['zerocopy_threshold'] = zerocopy_threshold
if tls_version is not None:
params['tls_version'] = tls_version
if enable_ktls is not None:
params['enable_ktls'] = enable_ktls
return client.call('sock_impl_set_options', params)

View File

@ -2823,7 +2823,9 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
enable_placement_id=args.enable_placement_id,
enable_zerocopy_send_server=args.enable_zerocopy_send_server,
enable_zerocopy_send_client=args.enable_zerocopy_send_client,
zerocopy_threshold=args.zerocopy_threshold)
zerocopy_threshold=args.zerocopy_threshold,
tls_version=args.tls_version,
enable_ktls=args.enable_ktls)
p = subparsers.add_parser('sock_impl_set_options', help="""Set options of socket layer implementation""")
p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
@ -2847,9 +2849,14 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
p.add_argument('--disable-zerocopy-send-client', help='Disable zerocopy on send for client sockets',
action='store_false', dest='enable_zerocopy_send_client')
p.add_argument('--zerocopy-threshold', help='Set zerocopy_threshold in bytes', type=int)
p.add_argument('--tls-version', help='TLS protocol version', type=int)
p.add_argument('--enable-ktls', help='Enable Kernel TLS',
action='store_true', dest='enable_ktls')
p.add_argument('--disable-ktls', help='Disable Kernel TLS',
action='store_false', dest='enable_ktls')
p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None, enable_quickack=None,
enable_placement_id=None, enable_zerocopy_send_server=None, enable_zerocopy_send_client=None,
zerocopy_threshold=None)
zerocopy_threshold=None, tls_version=None, enable_ktls=None)
def sock_set_default_impl(args):
print_json(rpc.sock.sock_set_default_impl(args.client,

View File

@ -17,6 +17,62 @@ if [ "$TEST_TRANSPORT" != tcp ]; then
fi
$rpc_py sock_set_default_impl -i ssl
# Check default TLS version
version=$($rpc_py sock_impl_get_options -i ssl | jq -r .tls_version)
if [[ "$version" != "0" ]]; then
echo "TLS version was not set correctly $version != 0"
exit 1
fi
# Check TLS version set to 13
$rpc_py sock_impl_set_options -i ssl --tls-version 13
version=$($rpc_py sock_impl_get_options -i ssl | jq -r .tls_version)
if [[ "$version" != "13" ]]; then
echo "TLS version was not set correctly $version != 13"
exit 1
fi
# Check TLS version set to 12
$rpc_py sock_impl_set_options -i ssl --tls-version 12
version=$($rpc_py sock_impl_get_options -i ssl | jq -r .tls_version)
if [[ "$version" != "12" ]]; then
echo "TLS version was not set correctly $version != 12"
exit 1
fi
# Check incorrect TLS version set to 7
$rpc_py sock_impl_set_options -i ssl --tls-version 7
version=$($rpc_py sock_impl_get_options -i ssl | jq -r .tls_version)
if [[ "$version" != "7" ]]; then
echo "TLS version was not set correctly $version != 7"
exit 1
fi
# Check default KTLS is disabled
ktls=$($rpc_py sock_impl_get_options -i ssl | jq -r .enable_ktls)
if [[ "$ktls" != "false" ]]; then
echo "KTLS was not set correctly $ktls != false"
exit 1
fi
# Check KTLS enable
$rpc_py sock_impl_set_options -i ssl --enable-ktls
ktls=$($rpc_py sock_impl_get_options -i ssl | jq -r .enable_ktls)
if [[ "$ktls" != "true" ]]; then
echo "KTLS was not set correctly $ktls != true"
exit 1
fi
# Check KTLS disable
$rpc_py sock_impl_set_options -i ssl --disable-ktls
ktls=$($rpc_py sock_impl_get_options -i ssl | jq -r .enable_ktls)
if [[ "$ktls" != "false" ]]; then
echo "KTLS was not set correctly $ktls != false"
exit 1
fi
$rpc_py sock_impl_set_options -i ssl --tls-version 13
$rpc_py framework_start_init
$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS
$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -m 10