ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
acd2cc94e7
Spdk/test/sma/vfiouser_qemu.sh
paul luse eb53c23236 add (c) and SPDX header to bash files as needed 
per Intel policy to include file commit date using git cmd
below.  The policy does not apply to non-Intel (C) notices.

git log --follow -C90% --format=%ad --date default <file> | tail -1

and then pull just the year from the result.

Intel copyrights were not added to files where Intel either had
no contribution ot the contribution lacked substance (ie license
header updates, formatting changes, etc)

For intel copyrights added, --follow and -C95% were used.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I2ef86976095b88a9bf5b1003e59f3943cd6bbe4c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15209
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Krzysztof Karas <krzysztof.karas@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
2022-11-29 08:27:51 +00:00

367 lines
11 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (C) 2022 Intel Corporation
# All rights reserved.
#
testdir=$(readlink -f "$(dirname "$0")")
rootdir=$(readlink -f "$testdir/../..")
source "$rootdir/test/common/autotest_common.sh"
source "$rootdir/test/vfio_user/common.sh"
source "$testdir/common.sh"
function create_device() {
local pfid=${1:-1}
local vfid=${2:-0}
"$rootdir/scripts/sma-client.py" <<- CREATE
{
"method": "CreateDevice",
"params": {
"nvme": {
"physical_id": "$pfid",
"virtual_id": "$vfid"
}
}
}
CREATE
}
function delete_device() {
"$rootdir/scripts/sma-client.py" <<- DELETE
{
"method": "DeleteDevice",
"params": {
"handle": "$1"
}
}
DELETE
}
function attach_volume() {
"$rootdir/scripts/sma-client.py" <<- ATTACH
{
"method": "AttachVolume",
"params": {
"device_handle": "$1",
"volume": {
"volume_id": "$(uuid2base64 $2)"
}
}
}
ATTACH
}
function detach_volume() {
"$rootdir/scripts/sma-client.py" <<- DETACH
{
"method": "DetachVolume",
"params": {
"device_handle": "$1",
"volume_id": "$(uuid2base64 $2)"
}
}
DETACH
}
function vm_count_nvme() {
vm_exec $1 "grep -l SPDK /sys/class/nvme/*/model" | wc -l
}
function vm_check_subsys_volume() {
local vm_id=$1
local nqn=$2
local uuid=$3
nvme="$(vm_exec $vm_id "grep -l $nqn /sys/class/nvme/*/subsysnqn" | awk -F/ '{print $5}')"
if [[ -z "$nvme" ]]; then
error "FAILED no NVMe on vm=$vm_id with nqn=$nqn"
return 1
fi
tmpuuid="$(vm_exec $vm_id "grep -l $uuid /sys/class/nvme/$nvme/nvme*/uuid")"
if [[ -z "$tmpuuid" ]]; then
return 1
fi
}
function vm_check_subsys_nqn() {
sleep 1
nqn=$(vm_exec $1 "grep -l $2 /sys/class/nvme/*/subsysnqn")
if [[ -z "$nqn" ]]; then
error "FAILED no NVMe on vm=$1 with nqn=$2"
return 1
fi
}
function cleanup() {
vm_kill_all
killprocess $tgtpid
killprocess $smapid
if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi
}
trap "cleanup; exit 1" SIGINT SIGTERM EXIT
# SSH VM Password
VM_PASSWORD=root
vm_no=0
VFO_ROOT_PATH="/tmp/sma/vfio-user/qemu"
if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi
mkdir -p "${VFO_ROOT_PATH}"
# Cleanup old VM:
used_vms=$vm_no
vm_kill_all
vm_setup --os="$VM_IMAGE" --disk-type=virtio --force=$vm_no --qemu-args="-qmp tcp:localhost:10005,server,nowait \
-device pci-bridge,chassis_nr=1,id=pci.spdk.0 \
-device pci-bridge,chassis_nr=2,id=pci.spdk.1"
# Run pre-configured VM and wait for them to start
vm_run $vm_no
vm_wait_for_boot 300 $vm_no
# Start SPDK
$rootdir/build/bin/spdk_tgt &
tgtpid=$!
waitforlisten $tgtpid
# Prepare the target
rpc_cmd bdev_null_create null0 100 4096
rpc_cmd bdev_null_create null1 100 4096
# Start SMA server
$rootdir/scripts/sma.py -c <(
cat <<- EOF
devices:
- name: 'vfiouser'
params:
buses:
- name: 'pci.spdk.0'
count: 32
- name: 'pci.spdk.1'
count: 32
qmp_addr: 127.0.0.1
qmp_port: 10005
crypto:
name: 'bdev_crypto'
params:
driver: 'crypto_aesni_mb'
EOF
) &
smapid=$!
# Wait until the SMA starts listening
sma_waitforlisten
# Make sure a TCP transport has been created
rpc_cmd nvmf_get_transports --trtype VFIOUSER
# Make sure no nvme subsystems are present
[[ $(vm_exec ${vm_no} nvme list-subsys -o json | jq -r '.Subsystems | length') -eq 0 ]]
# Create a couple of devices and verify them via RPC and SSH
device0=$(create_device 0 0 | jq -r '.handle')
rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0
vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-0
# Check that there are two subsystems (1 created above + discovery)
[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]]
device1=$(create_device 1 0 | jq -r '.handle')
rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0
rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1
[[ "$device0" != "$device1" ]]
vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-1
# Check that there are three subsystems (2 created above + discovery)
[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]]
# Verify the method is idempotent and sending the same gRPCs won't create new
# devices and will return the same IDs
tmp0=$(create_device 0 0 | jq -r '.handle')
tmp1=$(create_device 1 0 | jq -r '.handle')
[[ $(vm_count_nvme ${vm_no}) -eq 2 ]]
[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]]
[[ "$tmp0" == "$device0" ]]
[[ "$tmp1" == "$device1" ]]
# Now remove them verifying via RPC
delete_device "$device0"
NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0
rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1
[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]]
[[ $(vm_count_nvme ${vm_no}) -eq 1 ]]
delete_device "$device1"
NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0
NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1
[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 1 ]]
[[ $(vm_count_nvme ${vm_no}) -eq 0 ]]
# Finally check that removing a non-existing device is also sucessful
delete_device "$device0"
delete_device "$device1"
# Check volume attach/detach
device0=$(create_device 0 0 | jq -r '.handle')
device1=$(create_device 1 0 | jq -r '.handle')
uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid')
uuid1=$(rpc_cmd bdev_get_bdevs -b null1 | jq -r '.[].uuid')
# Attach the first volume to a first subsystem
attach_volume "$device0" "$uuid0"
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]]
vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0
attach_volume "$device1" "$uuid1"
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]]
vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1
# Attach the same device again and see that it won't fail
attach_volume "$device0" "$uuid0"
attach_volume "$device1" "$uuid1"
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]]
vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0
NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1
vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1
NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0
# Cross detach volumes and verify they not fail and have not been removed from the subsystems
detach_volume "$device0" "$uuid1"
detach_volume "$device1" "$uuid0"
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]]
vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0
NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1
vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1
NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0
# Detach volumes and verify they have been removed from the subsystems
detach_volume "$device0" "$uuid0"
detach_volume "$device1" "$uuid1"
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 0 ]]
[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]]
NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0
NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1
# Detach volumes once again and verify they will not fail
detach_volume "$device0" "$uuid0"
detach_volume "$device1" "$uuid1"
detach_volume "$device0" "$uuid1"
detach_volume "$device1" "$uuid0"
delete_device "$device0"
delete_device "$device1"
# Create device with allocation on second bus
device3=$(create_device 42 0 | jq -r '.handle')
vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-42
# Verify that device can be found on second bus and properly deleted
delete_device "$device3"
NOT vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-42
key0=1234567890abcdef1234567890abcdef
device0=$(create_device 0 0 | jq -r '.handle')
uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid')
# Now check vfio-user attach with bdev crypto
"$rootdir/scripts/sma-client.py" <<- ATTACH
{
"method": "AttachVolume",
"params": {
"device_handle": "$device0",
"volume": {
"volume_id": "$(uuid2base64 $uuid0)",
"crypto": {
"cipher": "$(get_cipher AES_CBC)",
"key": "$(format_key $key0)"
}
}
}
}
ATTACH
# Make sure that the namespace exposed in the subsystem is a crypto bdev
ns_bdev=$(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].name')
[[ $(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[0].product_name') == "crypto" ]]
crypto_bdev=$(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[] | select(.product_name == "crypto")')
[[ $(rpc_cmd bdev_get_bdevs | jq -r '[.[] | select(.product_name == "crypto")] | length') -eq 1 ]]
[[ $(jq -r '.driver_specific.crypto.key' <<< "$crypto_bdev") == "$key0" ]]
detach_volume "$device0" "$uuid0"
delete_device "$device0"
[[ $(rpc_cmd bdev_get_bdevs | jq -r '.[] | select(.product_name == "crypto")' | jq -r length) -eq 0 ]]
# Test qos
device_vfio_user=1
device0=$(create_device 0 0 | jq -r '.handle')
attach_volume "$device0" "$uuid0"
# First check the capabilities
diff <(get_qos_caps $device_vfio_user | jq --sort-keys) <(
jq --sort-keys <<- CAPS
{
"max_volume_caps": {
"rw_iops": true,
"rd_bandwidth": true,
"wr_bandwidth": true,
"rw_bandwidth": true
}
}
CAPS
)
"$rootdir/scripts/sma-client.py" <<- EOF
{
"method": "SetQos",
"params": {
"device_handle": "$device0",
"volume_id": "$(uuid2base64 $uuid0)",
"maximum": {
"rd_iops": 0,
"wr_iops": 0,
"rw_iops": 3,
"rd_bandwidth": 4,
"wr_bandwidth": 5,
"rw_bandwidth": 6
}
}
}
EOF
# Make sure that limits were changed
diff <(rpc_cmd bdev_get_bdevs -b null0 | jq --sort-keys '.[].assigned_rate_limits') <(
jq --sort-keys <<- EOF
{
"rw_ios_per_sec": 3000,
"rw_mbytes_per_sec": 6,
"r_mbytes_per_sec": 4,
"w_mbytes_per_sec": 5
}
EOF
)
detach_volume "$device0" "$uuid0"
delete_device "$device0"
cleanup
trap - SIGINT SIGTERM EXIT
Reference in New Issue View Git Blame Copy Permalink