From 61cd9d308eb5237749bd18cf3c40535fdd0568e5 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Thu, 30 Jul 2020 10:59:57 +0900 Subject: [PATCH] lib/sock: Add option to enable or disable quick ACK TCP delayed ACK can be disabled or enabled by enabling or disabling quick ACK, respectively. The recently added spdk_sock_impl_opts is helpful for sock library to control quick ACK. Hence this patch adds and uses an option enable_quickack. The option is effective only for the POSIX sock module. We have spdk_sock_opts now too but spdk_sock_impl_opts will be better for this case. This option is not supported on FreeBSD. FreeBSD users can set the option globally via sysctl if desired. Signed-off-by: Shuhei Matsumoto Change-Id: Ic89620267acce5872dc8ecaf7a99bb70ae97e993 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3603 Reviewed-by: Jim Harris Reviewed-by: Tomasz Zawadzki Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins --- doc/jsonrpc.md | 6 ++++-- include/spdk/sock.h | 5 +++++ lib/sock/sock_rpc.c | 5 +++++ module/sock/posix/posix.c | 25 ++++++++++++++++++++----- scripts/rpc.py | 10 ++++++++-- scripts/rpc/sock.py | 6 +++++- 6 files changed, 47 insertions(+), 10 deletions(-) diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 1f4335e3b..b018245c7 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -6608,6 +6608,7 @@ recv_buf_size | Optional | number | Size of socket receive buffer send_buf_size | Optional | number | Size of socket send buffer in bytes enable_recv_pipe | Optional | boolean | Enable or disable receive pipe enable_zerocopy_send | Optional | boolean | Enable or disable zero copy on send +enable_quick_ack | Optional | boolean | Enable or disable quick ACK ### Response @@ -6626,8 +6627,9 @@ Example request: "impl_name": "posix", "recv_buf_size": 2097152, "send_buf_size": 2097152, - "enable_recv_pipe": false - "enable_zerocopy_send": true + "enable_recv_pipe": false, + "enable_zerocopy_send": true, + "enable_quick_ack": false } } ~~~ diff --git a/include/spdk/sock.h b/include/spdk/sock.h index f70a2ac39..8c57dd2ce 100644 --- a/include/spdk/sock.h +++ b/include/spdk/sock.h @@ -107,6 +107,11 @@ struct spdk_sock_impl_opts { * Enable or disable use of zero copy flow on send. Used by posix socket module. */ bool enable_zerocopy_send; + + /** + * Enable or disable quick ACK. Used by posix socket module. + */ + bool enable_quickack; }; /** diff --git a/lib/sock/sock_rpc.c b/lib/sock/sock_rpc.c index c8686a068..726ab54ae 100644 --- a/lib/sock/sock_rpc.c +++ b/lib/sock/sock_rpc.c @@ -75,6 +75,7 @@ rpc_sock_impl_get_options(struct spdk_jsonrpc_request *request, spdk_json_write_named_uint32(w, "send_buf_size", sock_opts.send_buf_size); spdk_json_write_named_bool(w, "enable_recv_pipe", sock_opts.enable_recv_pipe); spdk_json_write_named_bool(w, "enable_zerocopy_send", sock_opts.enable_zerocopy_send); + spdk_json_write_named_bool(w, "enable_quickack", sock_opts.enable_quickack); spdk_json_write_object_end(w); spdk_jsonrpc_end_result(request, w); free(impl_name); @@ -108,6 +109,10 @@ static const struct spdk_json_object_decoder rpc_sock_impl_set_opts_decoders[] = "enable_zerocopy_send", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_zerocopy_send), spdk_json_decode_bool, true }, + { + "enable_quickack", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_quickack), + spdk_json_decode_bool, true + }, }; static void diff --git a/module/sock/posix/posix.c b/module/sock/posix/posix.c index 4eb1bf106..d146706bc 100644 --- a/module/sock/posix/posix.c +++ b/module/sock/posix/posix.c @@ -83,7 +83,8 @@ static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = { .recv_buf_size = MIN_SO_RCVBUF_SIZE, .send_buf_size = MIN_SO_SNDBUF_SIZE, .enable_recv_pipe = true, - .enable_zerocopy_send = true + .enable_zerocopy_send = true, + .enable_quickack = false, }; static int @@ -313,9 +314,9 @@ static struct spdk_posix_sock * posix_sock_alloc(int fd, bool enable_zero_copy) { struct spdk_posix_sock *sock; -#ifdef SPDK_ZEROCOPY - int rc; +#if defined(SPDK_ZEROCOPY) || defined(__linux__) int flag; + int rc; #endif sock = calloc(1, sizeof(*sock)); @@ -326,19 +327,31 @@ posix_sock_alloc(int fd, bool enable_zero_copy) sock->fd = fd; -#ifdef SPDK_ZEROCOPY +#if defined(SPDK_ZEROCOPY) + flag = 1; + if (!enable_zero_copy || !g_spdk_posix_sock_impl_opts.enable_zerocopy_send) { return sock; } /* Try to turn on zero copy sends */ - flag = 1; rc = setsockopt(sock->fd, SOL_SOCKET, SO_ZEROCOPY, &flag, sizeof(flag)); if (rc == 0) { sock->zcopy = true; } #endif +#if defined(__linux__) + flag = 1; + + if (g_spdk_posix_sock_impl_opts.enable_quickack) { + rc = setsockopt(sock->fd, IPPROTO_TCP, TCP_QUICKACK, &flag, sizeof(flag)); + if (rc != 0) { + SPDK_ERRLOG("quickack was failed to set\n"); + } + } +#endif + return sock; } @@ -1338,6 +1351,7 @@ posix_sock_impl_get_opts(struct spdk_sock_impl_opts *opts, size_t *len) GET_FIELD(send_buf_size); GET_FIELD(enable_recv_pipe); GET_FIELD(enable_zerocopy_send); + GET_FIELD(enable_quickack); #undef GET_FIELD #undef FIELD_OK @@ -1366,6 +1380,7 @@ posix_sock_impl_set_opts(const struct spdk_sock_impl_opts *opts, size_t len) SET_FIELD(send_buf_size); SET_FIELD(enable_recv_pipe); SET_FIELD(enable_zerocopy_send); + SET_FIELD(enable_quickack); #undef SET_FIELD #undef FIELD_OK diff --git a/scripts/rpc.py b/scripts/rpc.py index 140bfefd9..667fb26c2 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2401,7 +2401,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse recv_buf_size=args.recv_buf_size, send_buf_size=args.send_buf_size, enable_recv_pipe=args.enable_recv_pipe, - enable_zerocopy_send=args.enable_zerocopy_send) + enable_zerocopy_send=args.enable_zerocopy_send, + enable_quickack=args.enable_quickack) p = subparsers.add_parser('sock_impl_set_options', help="""Set options of socket layer implementation""") p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True) @@ -2415,7 +2416,12 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse action='store_true', dest='enable_zerocopy_send') p.add_argument('--disable-zerocopy-send', help='Disable zerocopy on send', action='store_false', dest='enable_zerocopy_send') - p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None, enable_zerocopy_send=None) + p.add_argument('--enable-quickack', help='Enable quick ACK', + action='store_true', dest='enable_quickack') + p.add_argument('--disable-quickack', help='Disable quick ACK', + action='store_false', dest='enable_quickack') + p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None, enable_zerocopy_send=None, + enable_quickack=None) def check_called_name(name): if name in deprecated_aliases: diff --git a/scripts/rpc/sock.py b/scripts/rpc/sock.py index 34d7f100d..526bac698 100644 --- a/scripts/rpc/sock.py +++ b/scripts/rpc/sock.py @@ -16,7 +16,8 @@ def sock_impl_set_options(client, recv_buf_size=None, send_buf_size=None, enable_recv_pipe=None, - enable_zerocopy_send=None): + enable_zerocopy_send=None, + enable_quickack=None): """Set parameters for the socket layer implementation. Args: @@ -25,6 +26,7 @@ def sock_impl_set_options(client, send_buf_size: size of socket send buffer in bytes (optional) enable_recv_pipe: enable or disable receive pipe (optional) enable_zerocopy_send: enable or disable zerocopy on send (optional) + enable_quickack: enable or disable quickack (optional) """ params = {} @@ -37,5 +39,7 @@ def sock_impl_set_options(client, params['enable_recv_pipe'] = enable_recv_pipe if enable_zerocopy_send is not None: params['enable_zerocopy_send'] = enable_zerocopy_send + if enable_quickack is not None: + params['enable_quickack'] = enable_quickack return client.call('sock_impl_set_options', params)