From 20dc7f7d197dab42effa22bfabca1dea5a259b0d Mon Sep 17 00:00:00 2001 From: Ben Walker Date: Tue, 25 Oct 2016 12:53:43 -0700 Subject: [PATCH] nvmf: For iWARP, register buffers with IBV_ACCESS_REMOTE_WRITE For iWARP devices, buffers that are intended to be the target of an RDMA read initiated by the target must additionally have IBV_ACCESS_REMOTE_WRITE permission. This is because iWARP's RDMA read path essentially requests the remote side to do an RDMA write. This is unfortunate because there is no way to differentiate between memory that the remote side can do an RDMA write to and memory that will only be the target of RDMA reads initiated by the target. There is nothing we can do about this serious deficiency in the specification, however, so we have to live with it. Change-Id: I3d2f2814ce0cb1df4e5347296ef371db4d16be21 Signed-off-by: Ben Walker --- lib/nvmf/rdma.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 03db128b1..39423a70a 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -297,7 +297,8 @@ spdk_nvmf_rdma_conn_create(struct rdma_cm_id *id, struct ibv_comp_channel *chann 0); rdma_conn->bufs_mr = ibv_reg_mr(id->pd, rdma_conn->bufs, max_queue_depth * g_rdma.in_capsule_data_size, - IBV_ACCESS_LOCAL_WRITE); + IBV_ACCESS_LOCAL_WRITE | + IBV_ACCESS_REMOTE_WRITE); if (!rdma_conn->cmds_mr || !rdma_conn->cpls_mr || !rdma_conn->bufs_mr) { SPDK_ERRLOG("Unable to register required memory for RDMA queue.\n"); spdk_nvmf_rdma_conn_destroy(rdma_conn); @@ -1250,7 +1251,8 @@ spdk_nvmf_rdma_session_add_conn(struct spdk_nvmf_session *session, rdma_sess->verbs = rdma_conn->cm_id->verbs; rdma_sess->buf_mr = ibv_reg_mr(rdma_conn->cm_id->pd, rdma_sess->buf, g_rdma.max_queue_depth * g_rdma.max_io_size, - IBV_ACCESS_LOCAL_WRITE); + IBV_ACCESS_LOCAL_WRITE | + IBV_ACCESS_REMOTE_WRITE); if (!rdma_sess->buf_mr) { SPDK_ERRLOG("Large buffer pool registration failed (%d x %d)\n", g_rdma.max_queue_depth, g_rdma.max_io_size);