vmd: method for removing devices behind VMD

Added new RPC, vmd_remove_device, that allows users to remove a PCI
device managed by the VMD library simulating a hot-remove.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Ifb84818ce8d147d1d586b52590527e85fe9c10de
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13957
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tom Nabarro <tom.nabarro@intel.com>

doc/jsonrpc.md

@@ -9997,6 +9997,39 @@ Example response:
 }
 ~~~
 
+### vmd_remove_device {#rpc_vmd_remove_device}
+
+Remove a device behind a VMD.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+addr                    | Required | string      | Address of the device to remove.
+
+### Example
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "method": "vmd_remove_device",
+  "params": {
+    "addr": "5d0505:01:00.0"
+  }
+  "id": 1
+}
+~~~
+
+Example response:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~
+
 ### spdk_get_version {#rpc_spdk_get_version}
 
 Get the version info of the running SPDK application.

include/spdk/vmd.h

@@ -80,6 +80,16 @@ int spdk_vmd_get_led_state(struct spdk_pci_device *pci_device, enum spdk_vmd_led
  */
 int spdk_vmd_hotplug_monitor(void);
 
+/**
+ * Removes a given device from the PCI subsystem simulating a hot-remove.  If the device is being
+ * actively used by another module, the actual detach might be deferred.
+ *
+ * \param addr Address of a PCI device to remove.
+ *
+ * \return 0 if the device was successfully removed, negative errno otherwise.
+ */
+int spdk_vmd_remove_device(const struct spdk_pci_addr *addr);
+
 #ifdef __cplusplus
 }
 #endif

lib/vmd/Makefile

@@ -7,7 +7,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 SO_VER := 4
-SO_MINOR := 0
+SO_MINOR := 1
 
 C_SRCS = vmd.c led.c
 LIBNAME = vmd

lib/vmd/spdk_vmd.map

@@ -8,6 +8,7 @@
 	spdk_vmd_set_led_state;
 	spdk_vmd_get_led_state;
 	spdk_vmd_hotplug_monitor;
+	spdk_vmd_remove_device;
 
 	local: *;
 };

lib/vmd/vmd.c

@@ -1419,6 +1419,22 @@ spdk_vmd_hotplug_monitor(void)
 	return num_hotplugs;
 }
 
+int
+spdk_vmd_remove_device(const struct spdk_pci_addr *addr)
+{
+	struct vmd_pci_device *device;
+
+	device = vmd_find_device(addr);
+	if (device == NULL) {
+		return -ENODEV;
+	}
+
+	assert(strcmp(spdk_pci_device_get_type(&device->pci), "vmd") == 0);
+	vmd_remove_device(device);
+
+	return 0;
+}
+
 static int
 vmd_attach_device(const struct spdk_pci_addr *addr)
 {

mk/spdk.lib_deps.mk

@@ -144,7 +144,7 @@ DEPDIRS-bdev_xnvme := $(BDEV_DEPS_THREAD)
 # are not related to symbols, but are defined directly in
 # the SPDK event subsystem code.
 DEPDIRS-event_accel := init accel
-DEPDIRS-event_vmd := init vmd $(JSON_LIBS) log thread
+DEPDIRS-event_vmd := init vmd $(JSON_LIBS) log thread util
 
 DEPDIRS-event_bdev := init bdev event_accel event_vmd event_sock
 

module/event/subsystems/vmd/event_vmd.h

@@ -7,5 +7,6 @@
 #define EVENT_VMD_H
 
 void vmd_subsystem_enable(void);
+bool vmd_subsystem_is_enabled(void);
 
 #endif

module/event/subsystems/vmd/vmd.c

@@ -24,6 +24,12 @@ vmd_subsystem_enable(void)
 	g_enabled = true;
 }
 
+bool
+vmd_subsystem_is_enabled(void)
+{
+	return g_enabled;
+}
+
 static int
 vmd_hotplug_monitor(void *ctx)
 {

module/event/subsystems/vmd/vmd_rpc.c

@@ -5,7 +5,9 @@
 
 #include "spdk/vmd.h"
 
+#include "spdk/env.h"
 #include "spdk/rpc.h"
+#include "spdk/string.h"
 #include "spdk/util.h"
 
 #include "spdk/log.h"
@@ -20,3 +22,50 @@ rpc_vmd_enable(struct spdk_jsonrpc_request *request, const struct spdk_json_val
 }
 SPDK_RPC_REGISTER("vmd_enable", rpc_vmd_enable, SPDK_RPC_STARTUP)
 SPDK_RPC_REGISTER_ALIAS_DEPRECATED(vmd_enable, enable_vmd)
+
+struct rpc_vmd_remove_device {
+	char *addr;
+};
+
+static const struct spdk_json_object_decoder rpc_vmd_remove_device_decoders[] = {
+	{"addr", offsetof(struct rpc_vmd_remove_device, addr), spdk_json_decode_string},
+};
+
+static void
+rpc_vmd_remove_device(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params)
+{
+	struct rpc_vmd_remove_device req = {};
+	struct spdk_pci_addr addr;
+	int rc;
+
+	if (!vmd_subsystem_is_enabled()) {
+		spdk_jsonrpc_send_error_response(request, -EPERM, "VMD subsystem is disabled");
+		return;
+	}
+
+	rc = spdk_json_decode_object(params, rpc_vmd_remove_device_decoders,
+				     SPDK_COUNTOF(rpc_vmd_remove_device_decoders),
+				     &req);
+	if (rc != 0) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "spdk_json_decode_object failed");
+		return;
+	}
+
+	rc = spdk_pci_addr_parse(&addr, req.addr);
+	if (rc != 0) {
+		spdk_jsonrpc_send_error_response(request, -EINVAL, "Failed to parse PCI address");
+		goto out;
+	}
+
+	rc = spdk_vmd_remove_device(&addr);
+	if (rc != 0) {
+		spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));
+		goto out;
+	}
+
+	spdk_jsonrpc_send_bool_response(request, true);
+out:
+	free(req.addr);
+}
+SPDK_RPC_REGISTER("vmd_remove_device", rpc_vmd_remove_device, SPDK_RPC_RUNTIME)

python/spdk/rpc/vmd.py

@@ -5,3 +5,8 @@ from .helpers import deprecated_alias
 def vmd_enable(client):
     """Enable VMD enumeration."""
     return client.call('vmd_enable')
+
+
+def vmd_remove_device(client, addr):
+    """Remove a device behind VMD"""
+    return client.call('vmd_remove_device', {'addr': addr})

scripts/rpc.py

@@ -2073,6 +2073,13 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
     p = subparsers.add_parser('vmd_enable', aliases=['enable_vmd'], help='Enable VMD enumeration')
     p.set_defaults(func=vmd_enable)
 
+    def vmd_remove_device(args):
+        print_dict(rpc.vmd.vmd_remove_device(args.client, addr=args.addr))
+
+    p = subparsers.add_parser('vmd_remove_device', help='Remove a device behind VMD')
+    p.add_argument('addr', help='Address of the device to remove', type=str)
+    p.set_defaults(func=vmd_remove_device)
+
     # nbd
     def nbd_start_disk(args):
         print(rpc.nbd.nbd_start_disk(args.client,