From 052ea0baac1d3b3a9f44091e5a245ac305bda44c Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Tue, 5 Jul 2022 10:50:42 +0200 Subject: [PATCH] vmd: method for removing devices behind VMD Added new RPC, vmd_remove_device, that allows users to remove a PCI device managed by the VMD library simulating a hot-remove. Signed-off-by: Konrad Sztyber Change-Id: Ifb84818ce8d147d1d586b52590527e85fe9c10de Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13957 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Tomasz Zawadzki Reviewed-by: Jim Harris Reviewed-by: Tom Nabarro --- doc/jsonrpc.md | 33 +++++++++++++++++ include/spdk/vmd.h | 10 +++++ lib/vmd/Makefile | 2 +- lib/vmd/spdk_vmd.map | 1 + lib/vmd/vmd.c | 16 ++++++++ mk/spdk.lib_deps.mk | 2 +- module/event/subsystems/vmd/event_vmd.h | 1 + module/event/subsystems/vmd/vmd.c | 6 +++ module/event/subsystems/vmd/vmd_rpc.c | 49 +++++++++++++++++++++++++ python/spdk/rpc/vmd.py | 5 +++ scripts/rpc.py | 7 ++++ 11 files changed, 130 insertions(+), 2 deletions(-) diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 3ef69e2a4..ed61f203b 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -9997,6 +9997,39 @@ Example response: } ~~~ +### vmd_remove_device {#rpc_vmd_remove_device} + +Remove a device behind a VMD. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +addr | Required | string | Address of the device to remove. + +### Example + +~~~json +{ + "jsonrpc": "2.0", + "method": "vmd_remove_device", + "params": { + "addr": "5d0505:01:00.0" + } + "id": 1 +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + ### spdk_get_version {#rpc_spdk_get_version} Get the version info of the running SPDK application. diff --git a/include/spdk/vmd.h b/include/spdk/vmd.h index a8110333c..ae073b14d 100644 --- a/include/spdk/vmd.h +++ b/include/spdk/vmd.h @@ -80,6 +80,16 @@ int spdk_vmd_get_led_state(struct spdk_pci_device *pci_device, enum spdk_vmd_led */ int spdk_vmd_hotplug_monitor(void); +/** + * Removes a given device from the PCI subsystem simulating a hot-remove. If the device is being + * actively used by another module, the actual detach might be deferred. + * + * \param addr Address of a PCI device to remove. + * + * \return 0 if the device was successfully removed, negative errno otherwise. + */ +int spdk_vmd_remove_device(const struct spdk_pci_addr *addr); + #ifdef __cplusplus } #endif diff --git a/lib/vmd/Makefile b/lib/vmd/Makefile index 4f2e47f3c..bb38c508d 100644 --- a/lib/vmd/Makefile +++ b/lib/vmd/Makefile @@ -7,7 +7,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk SO_VER := 4 -SO_MINOR := 0 +SO_MINOR := 1 C_SRCS = vmd.c led.c LIBNAME = vmd diff --git a/lib/vmd/spdk_vmd.map b/lib/vmd/spdk_vmd.map index 036d079b5..75eebc31e 100644 --- a/lib/vmd/spdk_vmd.map +++ b/lib/vmd/spdk_vmd.map @@ -8,6 +8,7 @@ spdk_vmd_set_led_state; spdk_vmd_get_led_state; spdk_vmd_hotplug_monitor; + spdk_vmd_remove_device; local: *; }; diff --git a/lib/vmd/vmd.c b/lib/vmd/vmd.c index 2b6bc4650..ee8c04a7d 100644 --- a/lib/vmd/vmd.c +++ b/lib/vmd/vmd.c @@ -1419,6 +1419,22 @@ spdk_vmd_hotplug_monitor(void) return num_hotplugs; } +int +spdk_vmd_remove_device(const struct spdk_pci_addr *addr) +{ + struct vmd_pci_device *device; + + device = vmd_find_device(addr); + if (device == NULL) { + return -ENODEV; + } + + assert(strcmp(spdk_pci_device_get_type(&device->pci), "vmd") == 0); + vmd_remove_device(device); + + return 0; +} + static int vmd_attach_device(const struct spdk_pci_addr *addr) { diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index 4be41fe9c..85f3ef248 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -144,7 +144,7 @@ DEPDIRS-bdev_xnvme := $(BDEV_DEPS_THREAD) # are not related to symbols, but are defined directly in # the SPDK event subsystem code. DEPDIRS-event_accel := init accel -DEPDIRS-event_vmd := init vmd $(JSON_LIBS) log thread +DEPDIRS-event_vmd := init vmd $(JSON_LIBS) log thread util DEPDIRS-event_bdev := init bdev event_accel event_vmd event_sock diff --git a/module/event/subsystems/vmd/event_vmd.h b/module/event/subsystems/vmd/event_vmd.h index c725b1565..4b5a269e8 100644 --- a/module/event/subsystems/vmd/event_vmd.h +++ b/module/event/subsystems/vmd/event_vmd.h @@ -7,5 +7,6 @@ #define EVENT_VMD_H void vmd_subsystem_enable(void); +bool vmd_subsystem_is_enabled(void); #endif diff --git a/module/event/subsystems/vmd/vmd.c b/module/event/subsystems/vmd/vmd.c index 6062bf3fe..027448446 100644 --- a/module/event/subsystems/vmd/vmd.c +++ b/module/event/subsystems/vmd/vmd.c @@ -24,6 +24,12 @@ vmd_subsystem_enable(void) g_enabled = true; } +bool +vmd_subsystem_is_enabled(void) +{ + return g_enabled; +} + static int vmd_hotplug_monitor(void *ctx) { diff --git a/module/event/subsystems/vmd/vmd_rpc.c b/module/event/subsystems/vmd/vmd_rpc.c index df7164071..57a7903fe 100644 --- a/module/event/subsystems/vmd/vmd_rpc.c +++ b/module/event/subsystems/vmd/vmd_rpc.c @@ -5,7 +5,9 @@ #include "spdk/vmd.h" +#include "spdk/env.h" #include "spdk/rpc.h" +#include "spdk/string.h" #include "spdk/util.h" #include "spdk/log.h" @@ -20,3 +22,50 @@ rpc_vmd_enable(struct spdk_jsonrpc_request *request, const struct spdk_json_val } SPDK_RPC_REGISTER("vmd_enable", rpc_vmd_enable, SPDK_RPC_STARTUP) SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vmd_enable, enable_vmd) + +struct rpc_vmd_remove_device { + char *addr; +}; + +static const struct spdk_json_object_decoder rpc_vmd_remove_device_decoders[] = { + {"addr", offsetof(struct rpc_vmd_remove_device, addr), spdk_json_decode_string}, +}; + +static void +rpc_vmd_remove_device(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_vmd_remove_device req = {}; + struct spdk_pci_addr addr; + int rc; + + if (!vmd_subsystem_is_enabled()) { + spdk_jsonrpc_send_error_response(request, -EPERM, "VMD subsystem is disabled"); + return; + } + + rc = spdk_json_decode_object(params, rpc_vmd_remove_device_decoders, + SPDK_COUNTOF(rpc_vmd_remove_device_decoders), + &req); + if (rc != 0) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "spdk_json_decode_object failed"); + return; + } + + rc = spdk_pci_addr_parse(&addr, req.addr); + if (rc != 0) { + spdk_jsonrpc_send_error_response(request, -EINVAL, "Failed to parse PCI address"); + goto out; + } + + rc = spdk_vmd_remove_device(&addr); + if (rc != 0) { + spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc)); + goto out; + } + + spdk_jsonrpc_send_bool_response(request, true); +out: + free(req.addr); +} +SPDK_RPC_REGISTER("vmd_remove_device", rpc_vmd_remove_device, SPDK_RPC_RUNTIME) diff --git a/python/spdk/rpc/vmd.py b/python/spdk/rpc/vmd.py index cfcaafe65..3b3452de2 100644 --- a/python/spdk/rpc/vmd.py +++ b/python/spdk/rpc/vmd.py @@ -5,3 +5,8 @@ from .helpers import deprecated_alias def vmd_enable(client): """Enable VMD enumeration.""" return client.call('vmd_enable') + + +def vmd_remove_device(client, addr): + """Remove a device behind VMD""" + return client.call('vmd_remove_device', {'addr': addr}) diff --git a/scripts/rpc.py b/scripts/rpc.py index 7834ac059..31c7092d5 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2073,6 +2073,13 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p = subparsers.add_parser('vmd_enable', aliases=['enable_vmd'], help='Enable VMD enumeration') p.set_defaults(func=vmd_enable) + def vmd_remove_device(args): + print_dict(rpc.vmd.vmd_remove_device(args.client, addr=args.addr)) + + p = subparsers.add_parser('vmd_remove_device', help='Remove a device behind VMD') + p.add_argument('addr', help='Address of the device to remove', type=str) + p.set_defaults(func=vmd_remove_device) + # nbd def nbd_start_disk(args): print(rpc.nbd.nbd_start_disk(args.client,