diff --git a/examples/idxd/perf/perf.c b/examples/idxd/perf/perf.c index 464c7a94e..514d9ab06 100644 --- a/examples/idxd/perf/perf.c +++ b/examples/idxd/perf/perf.c @@ -166,13 +166,20 @@ attach_cb(void *cb_ctx, struct spdk_idxd_device *idxd) g_num_devices++; } +static bool +probe_cb(void *cb_ctx, struct spdk_pci_device *dev) +{ + /* this tool will gladly claim all types of IDXD devices. */ + return true; +} + static int idxd_init(void) { spdk_idxd_set_config(g_idxd_kernel_mode); - if (spdk_idxd_probe(NULL, attach_cb) != 0) { - fprintf(stderr, "idxd_probe() failed\n"); + if (spdk_idxd_probe(NULL, attach_cb, probe_cb) != 0) { + fprintf(stderr, "spdk_idxd_probe() failed\n"); return 1; } diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index d709fb99d..eac69c8b9 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -104,6 +104,15 @@ typedef void (*spdk_idxd_req_cb)(void *arg, int status); */ typedef void (*spdk_idxd_attach_cb)(void *cb_ctx, struct spdk_idxd_device *idxd); +/** + * Callback for spdk_idxd_probe() to report a device that has been found. + * + * \param cb_ctx User-specified opaque value corresponding to cb_ctx from spdk_idxd_probe(). + * \param dev PCI device that is in question. + * \return true if the caller wants the device, false if not.. + */ +typedef bool (*spdk_idxd_probe_cb)(void *cb_ctx, struct spdk_pci_device *dev); + /** * Enumerate the IDXD devices attached to the system and attach the userspace * IDXD driver to them if desired. @@ -116,12 +125,14 @@ typedef void (*spdk_idxd_attach_cb)(void *cb_ctx, struct spdk_idxd_device *idxd) * * \param cb_ctx Opaque value which will be passed back in cb_ctx parameter of * the callbacks. + * \param probe_cb callback to determine if the device being probe should be attached. * \param attach_cb will be called for devices for which probe_cb returned true * once the IDXD controller has been attached to the userspace driver. * * \return 0 on success, -1 on failure. */ -int spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb); +int spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb, + spdk_idxd_probe_cb probe_cb); /** * Detach specified device returned by spdk_idxd_probe() from the IDXD driver. diff --git a/include/spdk/idxd_spec.h b/include/spdk/idxd_spec.h index b8532cbcd..8bb081d33 100644 --- a/include/spdk/idxd_spec.h +++ b/include/spdk/idxd_spec.h @@ -100,6 +100,32 @@ enum dsa_completion_status { DSA_COMP_TRANSLATION_FAIL = 34, }; +enum iaa_completion_status { + IAA_COMP_NONE = 0, + IAA_COMP_SUCCESS = 1, + IAA_COMP_PAGE_FAULT_IR = 4, + IAA_COMP_OUTBUF_OVERFLOW = 5, + IAA_COMP_BAD_OPCODE = 16, + IAA_COMP_INVALID_FLAGS = 17, + IAA_COMP_NOZERO_RESERVE = 18, + IAA_COMP_INVALID_SIZE = 19, + IAA_COMP_OVERLAP_BUFFERS = 22, + IAA_COMP_INT_HANDLE_INVAL = 25, + IAA_COMP_CRA_XLAT = 32, + IAA_COMP_CRA_ALIGN = 33, + IAA_COMP_ADDR_ALIGN = 34, + IAA_COMP_PRIV_BAD = 35, + IAA_COMP_TRAFFIC_CLASS_CONF = 36, + IAA_COMP_PFAULT_RDBA = 37, + IAA_COMP_HW_ERR1 = 38, + IAA_COMP_TRANSLATION_FAIL = 39, + IAA_COMP_PRS_TIMEOUT = 40, + IAA_COMP_WATCHDOG = 41, + IAA_COMP_INVALID_COMP_FLAG = 48, + IAA_COMP_INVALID_FILTER_FLAG = 49, + IAA_COMP_INVALID_NUM_ELEMS = 50, +}; + enum idxd_wq_state { WQ_DISABLED = 0, WQ_ENABLED = 1, @@ -188,6 +214,7 @@ struct idxd_hw_desc { uint64_t completion_addr; union { uint64_t src_addr; + uint64_t src1_addr; uint64_t readback_addr; uint64_t pattern; uint64_t desc_list_addr; @@ -199,12 +226,24 @@ struct idxd_hw_desc { uint64_t comp_pattern; }; union { + uint32_t src1_size; uint32_t xfer_size; uint32_t desc_count; }; uint16_t int_handle; - uint16_t rsvd1; union { + uint16_t rsvd1; + uint16_t compr_flags; + uint16_t decompr_flags; + }; + union { + struct { + uint64_t src2_addr; + uint32_t max_dst_size; + uint32_t src2_size; + uint32_t filter_flags; + uint32_t num_inputs; + } iaa; uint8_t expected_res; struct { uint64_t addr; @@ -288,6 +327,22 @@ struct dsa_hw_comp_record { }; SPDK_STATIC_ASSERT(sizeof(struct dsa_hw_comp_record) == 32, "size mismatch"); +struct iaa_hw_comp_record { + volatile uint8_t status; + uint8_t error_code; + uint16_t rsvd; + uint32_t bytes_completed; + uint64_t fault_addr; + uint32_t invalid_flags; + uint32_t rsvd2; + uint32_t output_size; + uint8_t output_bits; + uint8_t rsvd3; + uint16_t rsvd4; + uint64_t rsvd5[4]; +}; +SPDK_STATIC_ASSERT(sizeof(struct iaa_hw_comp_record) == 64, "size mismatch"); + union idxd_gencap_register { struct { uint64_t block_on_fault: 1; diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index 2b3f56c74..b267ab3a3 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -348,14 +348,15 @@ idxd_device_destruct(struct spdk_idxd_device *idxd) } int -spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb) +spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb, + spdk_idxd_probe_cb probe_cb) { if (g_idxd_impl == NULL) { SPDK_ERRLOG("No idxd impl is selected\n"); return -1; } - return g_idxd_impl->probe(cb_ctx, attach_cb); + return g_idxd_impl->probe(cb_ctx, attach_cb, probe_cb); } void diff --git a/lib/idxd/idxd.h b/lib/idxd/idxd.h index 20ffaf1e7..29e7f99b1 100644 --- a/lib/idxd/idxd.h +++ b/lib/idxd/idxd.h @@ -69,6 +69,11 @@ static inline void movdir64b(void *dst, const void *src) #define WQ_PRIORITY_1 1 #define IDXD_MAX_QUEUES 64 +enum idxd_dev { + IDXD_DEV_TYPE_DSA = 0, + IDXD_DEV_TYPE_IAA = 1, +}; + /* Each pre-allocated batch structure goes on a per channel list and * contains the memory for both user descriptors. */ @@ -124,7 +129,10 @@ struct pci_dev_id { * size and must be 32 byte aligned. */ struct idxd_ops { - struct dsa_hw_comp_record hw; + union { + struct dsa_hw_comp_record hw; + struct iaa_hw_comp_record iaa_hw; + }; void *cb_arg; spdk_idxd_req_cb cb_fn; struct idxd_batch *batch; @@ -134,11 +142,12 @@ struct idxd_ops { uint32_t count; STAILQ_ENTRY(idxd_ops) link; }; -SPDK_STATIC_ASSERT(sizeof(struct idxd_ops) == 96, "size mismatch"); +SPDK_STATIC_ASSERT(sizeof(struct idxd_ops) == 128, "size mismatch"); struct spdk_idxd_impl { const char *name; - int (*probe)(void *cb_ctx, spdk_idxd_attach_cb attach_cb); + int (*probe)(void *cb_ctx, spdk_idxd_attach_cb attach_cb, + spdk_idxd_probe_cb probe_cb); void (*destruct)(struct spdk_idxd_device *idxd); void (*dump_sw_error)(struct spdk_idxd_device *idxd, void *portal); char *(*portal_get_addr)(struct spdk_idxd_device *idxd); @@ -154,6 +163,7 @@ struct spdk_idxd_device { uint32_t total_wq_size; uint32_t chan_per_device; pthread_mutex_t num_channels_lock; + enum idxd_dev type; }; void idxd_impl_register(struct spdk_idxd_impl *impl); diff --git a/lib/idxd/idxd_kernel.c b/lib/idxd/idxd_kernel.c index ce676a18d..ac25967b7 100644 --- a/lib/idxd/idxd_kernel.c +++ b/lib/idxd/idxd_kernel.c @@ -82,7 +82,7 @@ kernel_idxd_device_destruct(struct spdk_idxd_device *idxd) static struct spdk_idxd_impl g_kernel_idxd_impl; static int -kernel_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb) +kernel_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb, spdk_idxd_probe_cb probe_cb) { int rc; struct accfg_ctx *ctx; diff --git a/lib/idxd/idxd_user.c b/lib/idxd/idxd_user.c index 06998eeeb..be05f1380 100644 --- a/lib/idxd/idxd_user.c +++ b/lib/idxd/idxd_user.c @@ -51,8 +51,6 @@ struct spdk_user_idxd_device { struct idxd_registers *registers; }; -typedef bool (*spdk_idxd_probe_cb)(void *cb_ctx, struct spdk_pci_device *pci_dev); - #define __user_idxd(idxd) (struct spdk_user_idxd_device *)idxd pthread_mutex_t g_driver_lock = PTHREAD_MUTEX_INITIALIZER; @@ -329,7 +327,8 @@ idxd_device_configure(struct spdk_user_idxd_device *user_idxd) } if ((rc == 0) && (gensts_reg.state == IDXD_DEVICE_STATE_ENABLED)) { - SPDK_DEBUGLOG(idxd, "Device enabled\n"); + SPDK_DEBUGLOG(idxd, "Device enabled VID 0x%x DID 0x%x\n", + user_idxd->device->id.vendor_id, user_idxd->device->id.device_id); } return rc; @@ -364,27 +363,6 @@ struct idxd_enum_ctx { void *cb_ctx; }; -/* This function must only be called while holding g_driver_lock */ -static int -idxd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) -{ - struct idxd_enum_ctx *enum_ctx = ctx; - struct spdk_idxd_device *idxd; - - if (enum_ctx->probe_cb(enum_ctx->cb_ctx, pci_dev)) { - idxd = idxd_attach(pci_dev); - if (idxd == NULL) { - SPDK_ERRLOG("idxd_attach() failed\n"); - return -EINVAL; - } - - enum_ctx->attach_cb(enum_ctx->cb_ctx, idxd); - } - - return 0; -} - - static bool probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) { @@ -409,8 +387,37 @@ probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) return true; } +/* This function must only be called while holding g_driver_lock */ static int -user_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb) +idxd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) +{ + struct idxd_enum_ctx *enum_ctx = ctx; + struct spdk_idxd_device *idxd; + + /* Call the user probe_cb to see if they want this device or not, if not + * skip it with a positive return code. + */ + if (enum_ctx->probe_cb(enum_ctx->cb_ctx, pci_dev) == false) { + return 1; + } + + if (probe_cb(enum_ctx->cb_ctx, pci_dev)) { + idxd = idxd_attach(pci_dev); + if (idxd == NULL) { + SPDK_ERRLOG("idxd_attach() failed\n"); + return -EINVAL; + } + + enum_ctx->attach_cb(enum_ctx->cb_ctx, idxd); + } + + return 0; +} + +/* The IDXD driver supports 2 distinct HW units, DSA and IAA. */ +static int +user_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb, + spdk_idxd_probe_cb probe_cb) { int rc; struct idxd_enum_ctx enum_ctx; @@ -422,6 +429,7 @@ user_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb) pthread_mutex_lock(&g_driver_lock); rc = spdk_pci_enumerate(spdk_pci_idxd_get_driver(), idxd_enum_cb, &enum_ctx); pthread_mutex_unlock(&g_driver_lock); + assert(rc == 0); return rc; } @@ -464,6 +472,7 @@ idxd_attach(struct spdk_pci_device *device) { struct spdk_user_idxd_device *user_idxd; struct spdk_idxd_device *idxd; + uint16_t did = device->id.device_id; uint32_t cmd_reg; int rc; @@ -474,6 +483,12 @@ idxd_attach(struct spdk_pci_device *device) } idxd = &user_idxd->idxd; + if (did == PCI_DEVICE_ID_INTEL_DSA) { + idxd->type = IDXD_DEV_TYPE_DSA; + } else if (did == PCI_DEVICE_ID_INTEL_IAA) { + idxd->type = IDXD_DEV_TYPE_IAA; + } + user_idxd->device = device; idxd->impl = &g_user_idxd_impl; idxd->socket_id = device->socket_id; diff --git a/module/accel/dsa/accel_engine_dsa.c b/module/accel/dsa/accel_engine_dsa.c index 89e629d46..04f0a7a76 100644 --- a/module/accel/dsa/accel_engine_dsa.c +++ b/module/accel/dsa/accel_engine_dsa.c @@ -404,6 +404,16 @@ accel_engine_dsa_enable_probe(bool kernel_mode) spdk_idxd_set_config(g_kernel_mode); } +static bool +probe_cb(void *cb_ctx, struct spdk_pci_device *dev) +{ + if (dev->id.device_id == PCI_DEVICE_ID_INTEL_DSA) { + return true; + } + + return false; +} + static int accel_engine_dsa_init(void) { @@ -411,7 +421,7 @@ accel_engine_dsa_init(void) return -EINVAL; } - if (spdk_idxd_probe(NULL, attach_cb) != 0) { + if (spdk_idxd_probe(NULL, attach_cb, probe_cb) != 0) { SPDK_ERRLOG("spdk_idxd_probe() failed\n"); return -EINVAL; }