From dfc54617412c492a4145b7490404933abc91bf52 Mon Sep 17 00:00:00 2001 From: Sebastian Brzezinka Date: Fri, 2 Sep 2022 15:18:47 +0200 Subject: [PATCH] sma: support bdev-based QoS for NVMe/vfiouser devices test/sma verifies that bdev-based QoS settings are correctly applied on vfio-user devices. Signed-off-by: Sebastian Brzezinka Change-Id: I0109bfdbcb95d6e683c45c3dbdb2c3c175f10aa2 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14337 Tested-by: SPDK CI Jenkins Reviewed-by: Tomasz Zawadzki Reviewed-by: Konrad Sztyber --- python/spdk/sma/device/nvmf_vfiouser.py | 32 +++++++++++++++ test/sma/vfiouser_qemu.sh | 52 +++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/python/spdk/sma/device/nvmf_vfiouser.py b/python/spdk/sma/device/nvmf_vfiouser.py index 4b0ea8902..a42d09e81 100644 --- a/python/spdk/sma/device/nvmf_vfiouser.py +++ b/python/spdk/sma/device/nvmf_vfiouser.py @@ -7,6 +7,7 @@ from socket import AddressFamily import grpc from google.protobuf import wrappers_pb2 as wrap from spdk.rpc.client import JSONRPCException +from spdk.sma import qos from ..common import format_volume_id, volume_id_to_nguid from ..proto import sma_pb2 @@ -279,3 +280,34 @@ class NvmfVfioDeviceManager(DeviceManager): def owns_device(self, id): return id.startswith(self._prefix) + + def set_qos(self, request): + nqn = request.device_handle[len(f'{self._prefix}:'):] + volume = format_volume_id(request.volume_id) + if volume is None: + raise DeviceException(grpc.StatusCode.INVALID_ARGUMENT, + 'Invalid volume ID') + try: + with self._client() as client: + # Make sure that a volume exists and is attached to the device + bdev = self._get_bdev(client, volume) + if bdev is None: + raise DeviceException(grpc.StatusCode.NOT_FOUND, + 'No volume associated with volume_id could be found') + subsys = self._get_subsys(client, nqn) + if subsys is None: + raise DeviceException(grpc.StatusCode.NOT_FOUND, + 'No device associated with device_handle could be found') + ns = self._get_ns(bdev, subsys) + if ns is None: + raise DeviceException(grpc.StatusCode.INVALID_ARGUMENT, + 'Specified volume is not attached to the device') + qos.set_volume_bdev_qos(client, request) + except qos.QosException as ex: + raise DeviceException(ex.code, ex.message) + except JSONRPCException: + raise DeviceException(grpc.StatusCode.INTERNAL, + 'Failed to set QoS') + + def get_qos_capabilities(self, request): + return qos.get_bdev_qos_capabilities() diff --git a/test/sma/vfiouser_qemu.sh b/test/sma/vfiouser_qemu.sh index 77c4feab0..4449966eb 100755 --- a/test/sma/vfiouser_qemu.sh +++ b/test/sma/vfiouser_qemu.sh @@ -295,5 +295,57 @@ detach_volume "$device0" "$uuid0" delete_device "$device0" [[ $(rpc_cmd bdev_get_bdevs | jq -r '.[] | select(.product_name == "crypto")' | jq -r length) -eq 0 ]] +# Test qos +device_vfio_user=1 +device0=$(create_device 0 0 | jq -r '.handle') +attach_volume "$device0" "$uuid0" + +# First check the capabilities +diff <(get_qos_caps $device_vfio_user | jq --sort-keys) <( + jq --sort-keys <<- CAPS + { + "max_volume_caps": { + "rw_iops": true, + "rd_bandwidth": true, + "wr_bandwidth": true, + "rw_bandwidth": true + } + } + CAPS +) + +"$rootdir/scripts/sma-client.py" <<- EOF + { + "method": "SetQos", + "params": { + "device_handle": "$device0", + "volume_id": "$(uuid2base64 $uuid0)", + "maximum": { + "rd_iops": 0, + "wr_iops": 0, + "rw_iops": 3, + "rd_bandwidth": 4, + "wr_bandwidth": 5, + "rw_bandwidth": 6 + } + } + } +EOF + +# Make sure that limits were changed +diff <(rpc_cmd bdev_get_bdevs -b null0 | jq --sort-keys '.[].assigned_rate_limits') <( + jq --sort-keys <<- EOF + { + "rw_ios_per_sec": 3000, + "rw_mbytes_per_sec": 6, + "r_mbytes_per_sec": 4, + "w_mbytes_per_sec": 5 + } + EOF +) + +detach_volume "$device0" "$uuid0" +delete_device "$device0" + cleanup trap - SIGINT SIGTERM EXIT