ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sock: Add sock_impl option to disable receive pipe

Browse Source 
Receive pipe reduces number of system calls and gives significant
performance improvement with kernel TCP stack and relatively small IO
sizes. With user space TCP/IP implementations there are no system
calls and double buffering introduced by pipe has negative impact on
performance. Receive pipe remains enabled by default.

Signed-off-by: Evgeniy Kochetov <evgeniik@mellanox.com>
Change-Id: Ic5ddee42293df2c233ba7ffbe6662de7917ac586
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3343
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Evgeniy Kochetov 2020-07-14 16:28:30 +03:00 committed by Jim Harris
parent 59b5ba5ca9
commit 63c5e51ebc
8 changed files with 43 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -107,6 +107,9 @@ Added `uring` based socket implementation, the code is located in module/sock/ur
available in Linux which requires kernel version is greater than 5.4.3. Currently, our CI pool added the uring
based socket tests for iSCSI target and also the tests for SPDK NVMe-oF tcp transport.
Added `enable_recv_pipe` socket layer option to allow disabling of double buffering on receive.
New option is used only in posix implementation.
### vhost
The function `spdk_vhost_blk_get_dev` has been removed.

7
doc/jsonrpc.md
View File

@ -6585,7 +6585,8 @@ Example response:
"id": 1,
"result": {
"recv_buf_size": 2097152,
"send_buf_size": 2097152
"send_buf_size": 2097152,
"enable_recv_pipe": true
}
}
~~~
@ -6601,6 +6602,7 @@ Name | Optional | Type | Description
impl_name | Required | string | Name of socket implementation, e.g. posix
recv_buf_size | Optional | number | Size of socket receive buffer in bytes
send_buf_size | Optional | number | Size of socket send buffer in bytes
enable_recv_pipe | Optional | boolean | Enable or disable receive pipe
### Response
@ -6618,7 +6620,8 @@ Example request:
"params": {
"impl_name": "posix",
"recv_buf_size": 2097152,
"send_buf_size": 2097152
"send_buf_size": 2097152,
"enable_recv_pipe": false
}
}
~~~

5
include/spdk/sock.h
View File

@ -97,6 +97,11 @@ struct spdk_sock_impl_opts {
* Size of sock send buffer. Used by posix socket module.
*/
uint32_t send_buf_size;
/**
* Enable or disable receive pipe. Used by posix socket module.
*/
bool enable_recv_pipe;
};
/**

1
lib/sock/sock.c
View File

@ -775,6 +775,7 @@ spdk_sock_write_config_json(struct spdk_json_write_ctx *w)
spdk_json_write_named_string(w, "impl_name", impl->name);
spdk_json_write_named_uint32(w, "recv_buf_size", opts.recv_buf_size);
spdk_json_write_named_uint32(w, "send_buf_size", opts.send_buf_size);
spdk_json_write_named_bool(w, "enable_recv_pipe", opts.enable_recv_pipe);
spdk_json_write_object_end(w);
spdk_json_write_object_end(w);
} else {

5
lib/sock/sock_rpc.c
View File

@ -73,6 +73,7 @@ rpc_sock_impl_get_options(struct spdk_jsonrpc_request *request,
spdk_json_write_object_begin(w);
spdk_json_write_named_uint32(w, "recv_buf_size", sock_opts.recv_buf_size);
spdk_json_write_named_uint32(w, "send_buf_size", sock_opts.send_buf_size);
spdk_json_write_named_bool(w, "enable_recv_pipe", sock_opts.enable_recv_pipe);
spdk_json_write_object_end(w);
spdk_jsonrpc_end_result(request, w);
free(impl_name);
@ -98,6 +99,10 @@ static const struct spdk_json_object_decoder rpc_sock_impl_set_opts_decoders[] =
"send_buf_size", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.send_buf_size),
spdk_json_decode_uint32, true
},
{
"enable_recv_pipe", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_recv_pipe),
spdk_json_decode_bool, true
},
};
static void

13
module/sock/posix/posix.c
View File

@ -81,7 +81,8 @@ struct spdk_posix_sock_group_impl {
static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = {
.recv_buf_size = MIN_SO_RCVBUF_SIZE,
.send_buf_size = MIN_SO_SNDBUF_SIZE
.send_buf_size = MIN_SO_SNDBUF_SIZE,
.enable_recv_pipe = true
};
static int
@ -267,9 +268,11 @@ posix_sock_set_recvbuf(struct spdk_sock *_sock, int sz)
assert(sock != NULL);
rc = posix_sock_alloc_pipe(sock, sz);
if (rc) {
return rc;
if (g_spdk_posix_sock_impl_opts.enable_recv_pipe) {
rc = posix_sock_alloc_pipe(sock, sz);
if (rc) {
return rc;
}
}
/* Set kernel buffer size to be at least MIN_SO_RCVBUF_SIZE */
@ -1332,6 +1335,7 @@ posix_sock_impl_get_opts(struct spdk_sock_impl_opts *opts, size_t *len)
GET_FIELD(recv_buf_size);
GET_FIELD(send_buf_size);
GET_FIELD(enable_recv_pipe);
#undef GET_FIELD
#undef FIELD_OK
@ -1358,6 +1362,7 @@ posix_sock_impl_set_opts(const struct spdk_sock_impl_opts *opts, size_t len)
SET_FIELD(recv_buf_size);
SET_FIELD(send_buf_size);
SET_FIELD(enable_recv_pipe);
#undef SET_FIELD
#undef FIELD_OK

9
scripts/rpc.py
View File

@ -2397,13 +2397,18 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
rpc.sock.sock_impl_set_options(args.client,
impl_name=args.impl,
recv_buf_size=args.recv_buf_size,
send_buf_size=args.send_buf_size)
send_buf_size=args.send_buf_size,
enable_recv_pipe=args.enable_recv_pipe)
p = subparsers.add_parser('sock_impl_set_options', help="""Set options of socket layer implementation""")
p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
p.add_argument('-r', '--recv-buf-size', help='Size of receive buffer on socket in bytes', type=int)
p.add_argument('-s', '--send-buf-size', help='Size of send buffer on socket in bytes', type=int)
p.set_defaults(func=sock_impl_set_options)
p.add_argument('--enable-recv-pipe', help='Enable receive pipe',
action='store_true', dest='enable_recv_pipe')
p.add_argument('--disable-recv-pipe', help='Disable receive pipe',
action='store_false', dest='enable_recv_pipe')
p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None)
def check_called_name(name):
if name in deprecated_aliases:

9
scripts/rpc/sock.py
View File

@ -11,13 +11,18 @@ def sock_impl_get_options(client, impl_name=None):
return client.call('sock_impl_get_options', params)
def sock_impl_set_options(client, impl_name=None, recv_buf_size=None, send_buf_size=None):
def sock_impl_set_options(client,
impl_name=None,
recv_buf_size=None,
send_buf_size=None,
enable_recv_pipe=None):
"""Set parameters for the socket layer implementation.
Args:
impl_name: name of socket implementation, e.g. posix
recv_buf_size: size of socket receive buffer in bytes (optional)
send_buf_size: size of socket send buffer in bytes (optional)
enable_recv_pipe: enable or disable receive pipe (optional)
"""
params = {}
@ -26,5 +31,7 @@ def sock_impl_set_options(client, impl_name=None, recv_buf_size=None, send_buf_s
params['recv_buf_size'] = recv_buf_size
if send_buf_size is not None:
params['send_buf_size'] = send_buf_size
if enable_recv_pipe is not None:
params['enable_recv_pipe'] = enable_recv_pipe
return client.call('sock_impl_set_options', params)