ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf/tcp: Add a feature to allow set the sock priority of the connection.

Browse Source 
This priority is used to differentiate the sock priority on the TCP connections
between  NVMe-oF TCP target and other TCP based applications.

Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Change-Id: I6ee294e647420b56d1d91a07c2e37bf34ce24e03
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/461801
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Ziye Yang 2019-07-15 20:58:16 +08:00 committed by Changpeng Liu
parent 96b759297f
commit 6ad6a1131b
8 changed files with 47 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -35,6 +35,11 @@ for reads is turned on by default. A config knob was added to allow disabling
the optimization. This will mostly be used for integration testing with 5.0.x kernels
while some compatibility fixes make their way down the pipeline for 5.1.x kernels.
The sock priority setting of the TCP connection owned by the tcp transport is added. It is
used to optimize the TCP connection performance under designated traffic classes. And the
priority is used to differeniate the sock priority between SPDK NVMe-oF TCP target application
and other TCP based applications.
Shared receive queue can now be disabled even for NICs that support it using the
`nvmf_create_transport` RPC method parameter `no_srq`. The actual use of a shared
receive queue is predicated on hardware support when this flag is not used.

1
doc/jsonrpc.md
View File

@ -3616,6 +3616,7 @@ max_srq_depth | Optional | number | The number of elements in a p
no_srq | Optional | boolean | Disable shared receive queue even for devices that support it. (RDMA only)
c2h_success | Optional | boolean | Disable C2H success optimization (TCP only)
dif_insert_or_strip | Optional | boolean | Enable DIF insert for write I/O and DIF strip for read I/O DIF (TCP only)
sock_priority | Optional | number | The socket priority of the connection owned by this transport (TCP only)
### Example:

1
include/spdk/nvmf.h
View File

@ -76,6 +76,7 @@ struct spdk_nvmf_transport_opts {
bool no_srq;
bool c2h_success;
bool dif_insert_or_strip;
uint32_t sock_priority;
};
struct spdk_nvmf_poll_group_stat {

5
lib/event/subsystems/nvmf/nvmf_rpc.c
View File

@ -1458,6 +1458,10 @@ static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[]
"dif_insert_or_strip", offsetof(struct nvmf_rpc_create_transport_ctx, opts.dif_insert_or_strip),
spdk_json_decode_bool, true
},
{
"sock_priority", offsetof(struct nvmf_rpc_create_transport_ctx, opts.sock_priority),
spdk_json_decode_uint32, true
},
};
static void
@ -1597,6 +1601,7 @@ dump_nvmf_transport(struct spdk_json_write_ctx *w, struct spdk_nvmf_transport *t
} else if (type == SPDK_NVME_TRANSPORT_TCP) {
spdk_json_write_named_bool(w, "c2h_success", opts->c2h_success);
spdk_json_write_named_bool(w, "dif_insert_or_strip", opts->dif_insert_or_strip);
spdk_json_write_named_uint32(w, "sock_priority", opts->sock_priority);
}
spdk_json_write_object_end(w);

25
lib/nvmf/tcp.c
View File

@ -54,6 +54,7 @@
#define NVMF_TCP_PDU_MAX_H2C_DATA_SIZE 131072
#define NVMF_TCP_PDU_MAX_C2H_DATA_SIZE 131072
#define NVMF_TCP_QPAIR_MAX_C2H_PDU_NUM 64 /* Maximal c2h_data pdu number for ecah tqpair */
#define SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY 6
/* spdk nvmf related structure */
enum spdk_nvmf_tcp_req_state {
@ -540,7 +541,7 @@ spdk_nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts)
" max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n"
" in_capsule_data_size=%d, max_aq_depth=%d\n"
" num_shared_buffers=%d, c2h_success=%d,\n"
" dif_insert_or_strip=%d\n",
" dif_insert_or_strip=%d, sock_priority=%d\n",
opts->max_queue_depth,
opts->max_io_size,
opts->max_qpairs_per_ctrlr,
@ -549,7 +550,16 @@ spdk_nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts)
opts->max_aq_depth,
opts->num_shared_buffers,
opts->c2h_success,
opts->dif_insert_or_strip);
opts->dif_insert_or_strip,
opts->sock_priority);
if (opts->sock_priority > SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY) {
SPDK_ERRLOG("Unsupported socket_priority=%d, the current range is: 0 to %d\n"
"you can use man 7 socket to view the range of priority under SO_PRIORITY item\n",
opts->sock_priority, SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY);
free(ttransport);
return NULL;
}
/* I/O unit size cannot be larger than max I/O size */
if (opts->io_unit_size > opts->max_io_size) {
@ -1096,6 +1106,15 @@ _spdk_nvmf_tcp_handle_connect(struct spdk_nvmf_transport *transport,
SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "New connection accepted on %s port %s\n",
port->trid.traddr, port->trid.trsvcid);
if (transport->opts.sock_priority) {
rc = spdk_sock_set_priority(sock, transport->opts.sock_priority);
if (rc) {
SPDK_ERRLOG("Failed to set the priority of the socket\n");
spdk_sock_close(&sock);
return;
}
}
tqpair = calloc(1, sizeof(struct spdk_nvmf_tcp_qpair));
if (tqpair == NULL) {
SPDK_ERRLOG("Could not allocate new connection.\n");
@ -2866,6 +2885,7 @@ spdk_nvmf_tcp_qpair_set_sq_size(struct spdk_nvmf_qpair *qpair)
#define SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE 32
#define SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION true
#define SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP false
#define SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY 0
static void
spdk_nvmf_tcp_opts_init(struct spdk_nvmf_transport_opts *opts)
@ -2880,6 +2900,7 @@ spdk_nvmf_tcp_opts_init(struct spdk_nvmf_transport_opts *opts)
opts->buf_cache_size = SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE;
opts->c2h_success = SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION;
opts->dif_insert_or_strip = SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP;
opts->sock_priority = SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY;
}
const struct spdk_nvmf_transport_ops spdk_nvmf_transport_tcp = {

4
scripts/rpc.py
View File

@ -1419,7 +1419,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
max_srq_depth=args.max_srq_depth,
no_srq=args.no_srq,
c2h_success=args.c2h_success,
dif_insert_or_strip=args.dif_insert_or_strip)
dif_insert_or_strip=args.dif_insert_or_strip,
sock_priority=args.sock_priority)
p = subparsers.add_parser('nvmf_create_transport', help='Create NVMf transport')
p.add_argument('-t', '--trtype', help='Transport type (ex. RDMA)', type=str, required=True)
@ -1435,6 +1436,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
p.add_argument('-r', '--no-srq', action='store_true', help='Disable per-thread shared receive queue. Relevant only for RDMA transport')
p.add_argument('-o', '--c2h-success', action='store_false', help='Disable C2H success optimization. Relevant only for TCP transport')
p.add_argument('-f', '--dif-insert-or-strip', action='store_true', help='Enable DIF insert/strip. Relevant only for TCP transport')
p.add_argument('-y', '--sock-priority', help='The sock priority of the tcp connection. Relevant only for TCP transport', type=int)
p.set_defaults(func=nvmf_create_transport)
def get_nvmf_transports(args):

5
scripts/rpc/nvmf.py
View File

@ -48,7 +48,8 @@ def nvmf_create_transport(client,
max_srq_depth=None,
no_srq=False,
c2h_success=True,
dif_insert_or_strip=None):
dif_insert_or_strip=None,
sock_priority=None):
"""NVMf Transport Create options.
Args:
@ -96,6 +97,8 @@ def nvmf_create_transport(client,
params['c2h_success'] = c2h_success
if dif_insert_or_strip:
params['dif_insert_or_strip'] = dif_insert_or_strip
if sock_priority:
params['sock_priority'] = sock_priority
return client.call('nvmf_create_transport', params)

5
test/unit/lib/nvmf/tcp.c/tcp_ut.c
View File

@ -176,6 +176,11 @@ DEFINE_STUB(spdk_sock_group_get_ctx,
(struct spdk_sock_group *group),
NULL);
DEFINE_STUB(spdk_sock_set_priority,
int,
(struct spdk_sock *sock, int priority),
0);
DEFINE_STUB_V(spdk_nvmf_ns_reservation_request, (void *ctx));
struct spdk_trace_histories *g_trace_histories;