From 6ad6a1131b1e1a0e58492f38fc4c392ba4e41f81 Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Mon, 15 Jul 2019 20:58:16 +0800 Subject: [PATCH] nvmf/tcp: Add a feature to allow set the sock priority of the connection. This priority is used to differentiate the sock priority on the TCP connections between NVMe-oF TCP target and other TCP based applications. Signed-off-by: Ziye Yang Change-Id: I6ee294e647420b56d1d91a07c2e37bf34ce24e03 Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/461801 Tested-by: SPDK CI Jenkins Reviewed-by: Changpeng Liu Reviewed-by: Shuhei Matsumoto --- CHANGELOG.md | 5 +++++ doc/jsonrpc.md | 1 + include/spdk/nvmf.h | 1 + lib/event/subsystems/nvmf/nvmf_rpc.c | 5 +++++ lib/nvmf/tcp.c | 25 +++++++++++++++++++++++-- scripts/rpc.py | 4 +++- scripts/rpc/nvmf.py | 5 ++++- test/unit/lib/nvmf/tcp.c/tcp_ut.c | 5 +++++ 8 files changed, 47 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23513e229..8552418ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,11 @@ for reads is turned on by default. A config knob was added to allow disabling the optimization. This will mostly be used for integration testing with 5.0.x kernels while some compatibility fixes make their way down the pipeline for 5.1.x kernels. +The sock priority setting of the TCP connection owned by the tcp transport is added. It is +used to optimize the TCP connection performance under designated traffic classes. And the +priority is used to differeniate the sock priority between SPDK NVMe-oF TCP target application +and other TCP based applications. + Shared receive queue can now be disabled even for NICs that support it using the `nvmf_create_transport` RPC method parameter `no_srq`. The actual use of a shared receive queue is predicated on hardware support when this flag is not used. diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 2a47b6ec4..b493dd16b 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -3616,6 +3616,7 @@ max_srq_depth | Optional | number | The number of elements in a p no_srq | Optional | boolean | Disable shared receive queue even for devices that support it. (RDMA only) c2h_success | Optional | boolean | Disable C2H success optimization (TCP only) dif_insert_or_strip | Optional | boolean | Enable DIF insert for write I/O and DIF strip for read I/O DIF (TCP only) +sock_priority | Optional | number | The socket priority of the connection owned by this transport (TCP only) ### Example: diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index db225e0b3..39192744c 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -76,6 +76,7 @@ struct spdk_nvmf_transport_opts { bool no_srq; bool c2h_success; bool dif_insert_or_strip; + uint32_t sock_priority; }; struct spdk_nvmf_poll_group_stat { diff --git a/lib/event/subsystems/nvmf/nvmf_rpc.c b/lib/event/subsystems/nvmf/nvmf_rpc.c index 654d7ebf7..c54c8ecbc 100644 --- a/lib/event/subsystems/nvmf/nvmf_rpc.c +++ b/lib/event/subsystems/nvmf/nvmf_rpc.c @@ -1458,6 +1458,10 @@ static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[] "dif_insert_or_strip", offsetof(struct nvmf_rpc_create_transport_ctx, opts.dif_insert_or_strip), spdk_json_decode_bool, true }, + { + "sock_priority", offsetof(struct nvmf_rpc_create_transport_ctx, opts.sock_priority), + spdk_json_decode_uint32, true + }, }; static void @@ -1597,6 +1601,7 @@ dump_nvmf_transport(struct spdk_json_write_ctx *w, struct spdk_nvmf_transport *t } else if (type == SPDK_NVME_TRANSPORT_TCP) { spdk_json_write_named_bool(w, "c2h_success", opts->c2h_success); spdk_json_write_named_bool(w, "dif_insert_or_strip", opts->dif_insert_or_strip); + spdk_json_write_named_uint32(w, "sock_priority", opts->sock_priority); } spdk_json_write_object_end(w); diff --git a/lib/nvmf/tcp.c b/lib/nvmf/tcp.c index 8d7ed0a3b..74c1a787c 100644 --- a/lib/nvmf/tcp.c +++ b/lib/nvmf/tcp.c @@ -54,6 +54,7 @@ #define NVMF_TCP_PDU_MAX_H2C_DATA_SIZE 131072 #define NVMF_TCP_PDU_MAX_C2H_DATA_SIZE 131072 #define NVMF_TCP_QPAIR_MAX_C2H_PDU_NUM 64 /* Maximal c2h_data pdu number for ecah tqpair */ +#define SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY 6 /* spdk nvmf related structure */ enum spdk_nvmf_tcp_req_state { @@ -540,7 +541,7 @@ spdk_nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) " max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n" " in_capsule_data_size=%d, max_aq_depth=%d\n" " num_shared_buffers=%d, c2h_success=%d,\n" - " dif_insert_or_strip=%d\n", + " dif_insert_or_strip=%d, sock_priority=%d\n", opts->max_queue_depth, opts->max_io_size, opts->max_qpairs_per_ctrlr, @@ -549,7 +550,16 @@ spdk_nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) opts->max_aq_depth, opts->num_shared_buffers, opts->c2h_success, - opts->dif_insert_or_strip); + opts->dif_insert_or_strip, + opts->sock_priority); + + if (opts->sock_priority > SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY) { + SPDK_ERRLOG("Unsupported socket_priority=%d, the current range is: 0 to %d\n" + "you can use man 7 socket to view the range of priority under SO_PRIORITY item\n", + opts->sock_priority, SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY); + free(ttransport); + return NULL; + } /* I/O unit size cannot be larger than max I/O size */ if (opts->io_unit_size > opts->max_io_size) { @@ -1096,6 +1106,15 @@ _spdk_nvmf_tcp_handle_connect(struct spdk_nvmf_transport *transport, SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "New connection accepted on %s port %s\n", port->trid.traddr, port->trid.trsvcid); + if (transport->opts.sock_priority) { + rc = spdk_sock_set_priority(sock, transport->opts.sock_priority); + if (rc) { + SPDK_ERRLOG("Failed to set the priority of the socket\n"); + spdk_sock_close(&sock); + return; + } + } + tqpair = calloc(1, sizeof(struct spdk_nvmf_tcp_qpair)); if (tqpair == NULL) { SPDK_ERRLOG("Could not allocate new connection.\n"); @@ -2866,6 +2885,7 @@ spdk_nvmf_tcp_qpair_set_sq_size(struct spdk_nvmf_qpair *qpair) #define SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE 32 #define SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION true #define SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP false +#define SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY 0 static void spdk_nvmf_tcp_opts_init(struct spdk_nvmf_transport_opts *opts) @@ -2880,6 +2900,7 @@ spdk_nvmf_tcp_opts_init(struct spdk_nvmf_transport_opts *opts) opts->buf_cache_size = SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE; opts->c2h_success = SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION; opts->dif_insert_or_strip = SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP; + opts->sock_priority = SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY; } const struct spdk_nvmf_transport_ops spdk_nvmf_transport_tcp = { diff --git a/scripts/rpc.py b/scripts/rpc.py index 990ca4d5d..367dcbe25 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1419,7 +1419,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse max_srq_depth=args.max_srq_depth, no_srq=args.no_srq, c2h_success=args.c2h_success, - dif_insert_or_strip=args.dif_insert_or_strip) + dif_insert_or_strip=args.dif_insert_or_strip, + sock_priority=args.sock_priority) p = subparsers.add_parser('nvmf_create_transport', help='Create NVMf transport') p.add_argument('-t', '--trtype', help='Transport type (ex. RDMA)', type=str, required=True) @@ -1435,6 +1436,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p.add_argument('-r', '--no-srq', action='store_true', help='Disable per-thread shared receive queue. Relevant only for RDMA transport') p.add_argument('-o', '--c2h-success', action='store_false', help='Disable C2H success optimization. Relevant only for TCP transport') p.add_argument('-f', '--dif-insert-or-strip', action='store_true', help='Enable DIF insert/strip. Relevant only for TCP transport') + p.add_argument('-y', '--sock-priority', help='The sock priority of the tcp connection. Relevant only for TCP transport', type=int) p.set_defaults(func=nvmf_create_transport) def get_nvmf_transports(args): diff --git a/scripts/rpc/nvmf.py b/scripts/rpc/nvmf.py index 76c56ace6..e0214eeb7 100644 --- a/scripts/rpc/nvmf.py +++ b/scripts/rpc/nvmf.py @@ -48,7 +48,8 @@ def nvmf_create_transport(client, max_srq_depth=None, no_srq=False, c2h_success=True, - dif_insert_or_strip=None): + dif_insert_or_strip=None, + sock_priority=None): """NVMf Transport Create options. Args: @@ -96,6 +97,8 @@ def nvmf_create_transport(client, params['c2h_success'] = c2h_success if dif_insert_or_strip: params['dif_insert_or_strip'] = dif_insert_or_strip + if sock_priority: + params['sock_priority'] = sock_priority return client.call('nvmf_create_transport', params) diff --git a/test/unit/lib/nvmf/tcp.c/tcp_ut.c b/test/unit/lib/nvmf/tcp.c/tcp_ut.c index ac96c1887..bf5f3458f 100644 --- a/test/unit/lib/nvmf/tcp.c/tcp_ut.c +++ b/test/unit/lib/nvmf/tcp.c/tcp_ut.c @@ -176,6 +176,11 @@ DEFINE_STUB(spdk_sock_group_get_ctx, (struct spdk_sock_group *group), NULL); +DEFINE_STUB(spdk_sock_set_priority, + int, + (struct spdk_sock *sock, int priority), + 0); + DEFINE_STUB_V(spdk_nvmf_ns_reservation_request, (void *ctx)); struct spdk_trace_histories *g_trace_histories;