diff --git a/lib/rte_vhost/socket.c b/lib/rte_vhost/socket.c
index 1d69e630e..2b2716a5d 100644
--- a/lib/rte_vhost/socket.c
+++ b/lib/rte_vhost/socket.c
@@ -809,8 +809,23 @@ rte_vhost_driver_start(const char *path)
 		return -1;
 
 	if (fdset_tid == 0) {
-		int ret = pthread_create(&fdset_tid, NULL, fdset_event_dispatch,
+		rte_cpuset_t orig_cpuset;
+		rte_cpuset_t tmp_cpuset;
+		long num_cores, i;
+		int ret;
+
+		CPU_ZERO(&tmp_cpuset);
+		num_cores = sysconf(_SC_NPROCESSORS_CONF);
+		/* Create a mask containing all CPUs */
+		for (i = 0; i < num_cores; i++) {
+			CPU_SET(i, &tmp_cpuset);
+		}
+
+		rte_thread_get_affinity(&orig_cpuset);
+		rte_thread_set_affinity(&tmp_cpuset);
+		ret = pthread_create(&fdset_tid, NULL, fdset_event_dispatch,
 				     &vhost_user.fdset);
+		rte_thread_set_affinity(&orig_cpuset);
 		if (ret < 0)
 			RTE_LOG(ERR, VHOST_CONFIG,
 				"failed to create fdset handling thread");
diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c
index c07b14664..8d46240e9 100644
--- a/lib/vhost/vhost.c
+++ b/lib/vhost/vhost.c
@@ -670,18 +670,6 @@ vhost_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask)
 	return 0;
 }
 
-static void *
-_start_rte_driver(void *arg)
-{
-	char *path = arg;
-
-	if (rte_vhost_driver_start(path) != 0) {
-		return NULL;
-	}
-
-	return path;
-}
-
 int
 vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *mask_str,
 		   const struct spdk_vhost_dev_backend *backend)
@@ -783,12 +771,7 @@ vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *ma
 
 	vhost_dev_install_rte_compat_hooks(vdev);
 
-	/* The following might start a POSIX thread that polls for incoming
-	 * socket connections and calls backend->start/stop_device. These backend
-	 * callbacks are also protected by the global SPDK vhost mutex, so we're
-	 * safe with not initializing the vdev just yet.
-	 */
-	if (spdk_call_unaffinitized(_start_rte_driver, path) == NULL) {
+	if (rte_vhost_driver_start(path) != 0) {
 		SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s\n",
 			    name, errno, spdk_strerror(errno));
 		rte_vhost_driver_unregister(path);