RE: [PATCH] dmaengine: idxd: add IAX configuration support in the IDXD driver

["Jiang, Dave" <dave.jiang@intel.com>]

> -----Original Message-----
> From: Dave Jiang <dave.jiang@intel.com>
> Sent: Tuesday, November 17, 2020 1:39 PM
> To: vkoul@kernel.org
> Cc: dmaengine@vger.kernel.org; Luck, Tony <tony.luck@intel.com>;
> Williams, Dan J <dan.j.williams@intel.com>
> Subject: [PATCH] dmaengine: idxd: add IAX configuration support in the IDXD
> driver
> 
> Add support to allow configuration of Intel Analytics Accelerator (IAX) in
> addition to the Intel Data Streaming Accelerator (DSA). The IAX hardware
> has the same configuration interface as DSA. The main difference
> is the type of operations it performs. We can support the DSA and
> IAX devices on the same driver with some tweaks.
> 
> IAX has a 64B completion record that needs to be 64B aligned, as opposed to
> a 32B completion record that is 32B aligned for DSA. IAX also does not
> support token management.
> 
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>

Hi Vinod,
Very much appreciate all your reviews. Will you please consider this patch for the 5.11 merge window? Thank you!


> ---
>  drivers/dma/idxd/cdev.c      |    1 +
>  drivers/dma/idxd/device.c    |   37 +++++++++++++++-----
>  drivers/dma/idxd/idxd.h      |   24 +++++++++++--
>  drivers/dma/idxd/init.c      |   14 +++++++
>  drivers/dma/idxd/registers.h |    1 +
>  drivers/dma/idxd/submit.c    |    2 +
>  drivers/dma/idxd/sysfs.c     |   46 +++++++++++++++++++++++-
>  include/uapi/linux/idxd.h    |   79
> ++++++++++++++++++++++++++++++++++++++++++
>  8 files changed, 187 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c
> index 010b820d8f74..0db9b82ed8cf 100644
> --- a/drivers/dma/idxd/cdev.c
> +++ b/drivers/dma/idxd/cdev.c
> @@ -28,6 +28,7 @@ struct idxd_cdev_context {
>   */
>  static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
>  	{ .name = "dsa" },
> +	{ .name = "iax" }
>  };
> 
>  struct idxd_user_context {
> diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
> index b75f9a09666e..47ff8e387172 100644
> --- a/drivers/dma/idxd/device.c
> +++ b/drivers/dma/idxd/device.c
> @@ -131,6 +131,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
>  	struct idxd_device *idxd = wq->idxd;
>  	struct device *dev = &idxd->pdev->dev;
>  	int rc, num_descs, i;
> +	int align;
> +	u64 tmp;
> 
>  	if (wq->type != IDXD_WQT_KERNEL)
>  		return 0;
> @@ -142,14 +144,27 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
>  	if (rc < 0)
>  		return rc;
> 
> -	wq->compls_size = num_descs * sizeof(struct
> dsa_completion_record);
> -	wq->compls = dma_alloc_coherent(dev, wq->compls_size,
> -					&wq->compls_addr, GFP_KERNEL);
> -	if (!wq->compls) {
> +	if (idxd->type == IDXD_TYPE_DSA)
> +		align = 32;
> +	else if (idxd->type == IDXD_TYPE_IAX)
> +		align = 64;
> +	else
> +		return -ENODEV;
> +
> +	wq->compls_size = num_descs * idxd->compl_size + align;
> +	wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size,
> +					    &wq->compls_addr_raw,
> GFP_KERNEL);
> +	if (!wq->compls_raw) {
>  		rc = -ENOMEM;
>  		goto fail_alloc_compls;
>  	}
> 
> +	/* Adjust alignment */
> +	wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align -
> 1);
> +	tmp = (u64)wq->compls_raw;
> +	tmp = (tmp + (align - 1)) & ~(align - 1);
> +	wq->compls = (struct dsa_completion_record *)tmp;
> +
>  	rc = alloc_descs(wq, num_descs);
>  	if (rc < 0)
>  		goto fail_alloc_descs;
> @@ -163,9 +178,11 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
>  		struct idxd_desc *desc = wq->descs[i];
> 
>  		desc->hw = wq->hw_descs[i];
> -		desc->completion = &wq->compls[i];
> -		desc->compl_dma  = wq->compls_addr +
> -			sizeof(struct dsa_completion_record) * i;
> +		if (idxd->type == IDXD_TYPE_DSA)
> +			desc->completion = &wq->compls[i];
> +		else if (idxd->type == IDXD_TYPE_IAX)
> +			desc->iax_completion = &wq->iax_compls[i];
> +		desc->compl_dma = wq->compls_addr + idxd->compl_size *
> i;
>  		desc->id = i;
>  		desc->wq = wq;
>  		desc->cpu = -1;
> @@ -178,7 +195,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
>   fail_sbitmap_init:
>  	free_descs(wq);
>   fail_alloc_descs:
> -	dma_free_coherent(dev, wq->compls_size, wq->compls, wq-
> >compls_addr);
> +	dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
> +			  wq->compls_addr_raw);
>   fail_alloc_compls:
>  	free_hw_descs(wq);
>  	return rc;
> @@ -193,7 +211,8 @@ void idxd_wq_free_resources(struct idxd_wq *wq)
> 
>  	free_hw_descs(wq);
>  	free_descs(wq);
> -	dma_free_coherent(dev, wq->compls_size, wq->compls, wq-
> >compls_addr);
> +	dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
> +			  wq->compls_addr_raw);
>  	sbitmap_queue_free(&wq->sbq);
>  }
> 
> diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
> index 149934f8d097..5a50e91c71bf 100644
> --- a/drivers/dma/idxd/idxd.h
> +++ b/drivers/dma/idxd/idxd.h
> @@ -20,7 +20,8 @@ extern struct kmem_cache *idxd_desc_pool;
>  enum idxd_type {
>  	IDXD_TYPE_UNKNOWN = -1,
>  	IDXD_TYPE_DSA = 0,
> -	IDXD_TYPE_MAX
> +	IDXD_TYPE_IAX,
> +	IDXD_TYPE_MAX,
>  };
> 
>  #define IDXD_NAME_SIZE		128
> @@ -114,8 +115,13 @@ struct idxd_wq {
>  	u32 vec_ptr;		/* interrupt steering */
>  	struct dsa_hw_desc **hw_descs;
>  	int num_descs;
> -	struct dsa_completion_record *compls;
> +	union {
> +		struct dsa_completion_record *compls;
> +		struct iax_completion_record *iax_compls;
> +	};
> +	void *compls_raw;
>  	dma_addr_t compls_addr;
> +	dma_addr_t compls_addr_raw;
>  	int compls_size;
>  	struct idxd_desc **descs;
>  	struct sbitmap_queue sbq;
> @@ -196,6 +202,7 @@ struct idxd_device {
>  	int token_limit;
>  	int nr_tokens;		/* non-reserved tokens */
>  	unsigned int wqcfg_size;
> +	int compl_size;
> 
>  	union sw_err_reg sw_err;
>  	wait_queue_head_t cmd_waitq;
> @@ -210,9 +217,15 @@ struct idxd_device {
> 
>  /* IDXD software descriptor */
>  struct idxd_desc {
> -	struct dsa_hw_desc *hw;
> +	union {
> +		struct dsa_hw_desc *hw;
> +		struct iax_hw_desc *iax_hw;
> +	};
>  	dma_addr_t desc_dma;
> -	struct dsa_completion_record *completion;
> +	union {
> +		struct dsa_completion_record *completion;
> +		struct iax_completion_record *iax_completion;
> +	};
>  	dma_addr_t compl_dma;
>  	struct dma_async_tx_descriptor txd;
>  	struct llist_node llnode;
> @@ -226,6 +239,7 @@ struct idxd_desc {
>  #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
> 
>  extern struct bus_type dsa_bus_type;
> +extern struct bus_type iax_bus_type;
> 
>  extern bool support_enqcmd;
> 
> @@ -271,6 +285,8 @@ static inline void idxd_set_type(struct idxd_device
> *idxd)
> 
>  	if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
>  		idxd->type = IDXD_TYPE_DSA;
> +	else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0)
> +		idxd->type = IDXD_TYPE_IAX;
>  	else
>  		idxd->type = IDXD_TYPE_UNKNOWN;
>  }
> diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
> index 45b0eac640c3..2c051e07c34c 100644
> --- a/drivers/dma/idxd/init.c
> +++ b/drivers/dma/idxd/init.c
> @@ -36,12 +36,16 @@ static struct mutex idxd_idr_lock;
>  static struct pci_device_id idxd_pci_tbl[] = {
>  	/* DSA ver 1.0 platforms */
>  	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) },
> +
> +	/* IAX ver 1.0 platforms */
> +	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) },
>  	{ 0, }
>  };
>  MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
> 
>  static char *idxd_name[] = {
>  	"dsa",
> +	"iax"
>  };
> 
>  const char *idxd_get_dev_name(struct idxd_device *idxd)
> @@ -377,6 +381,14 @@ static int idxd_probe(struct idxd_device *idxd)
>  	return rc;
>  }
> 
> +static void idxd_type_init(struct idxd_device *idxd)
> +{
> +	if (idxd->type == IDXD_TYPE_DSA)
> +		idxd->compl_size = sizeof(struct dsa_completion_record);
> +	else if (idxd->type == IDXD_TYPE_IAX)
> +		idxd->compl_size = sizeof(struct iax_completion_record);
> +}
> +
>  static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id
> *id)
>  {
>  	struct device *dev = &pdev->dev;
> @@ -412,6 +424,8 @@ static int idxd_pci_probe(struct pci_dev *pdev, const
> struct pci_device_id *id)
> 
>  	idxd_set_type(idxd);
> 
> +	idxd_type_init(idxd);
> +
>  	dev_dbg(dev, "Set PCI master\n");
>  	pci_set_master(pdev);
>  	pci_set_drvdata(pdev, idxd);
> diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h
> index 0cdc5405bc53..23c41fe52215 100644
> --- a/drivers/dma/idxd/registers.h
> +++ b/drivers/dma/idxd/registers.h
> @@ -5,6 +5,7 @@
> 
>  /* PCI Config */
>  #define PCI_DEVICE_ID_INTEL_DSA_SPR0	0x0b25
> +#define PCI_DEVICE_ID_INTEL_IAX_SPR0	0x0cfe
> 
>  #define IDXD_MMIO_BAR		0
>  #define IDXD_WQ_BAR		2
> diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c
> index efca5d8468a6..0ff64eeb84be 100644
> --- a/drivers/dma/idxd/submit.c
> +++ b/drivers/dma/idxd/submit.c
> @@ -15,7 +15,7 @@ static struct idxd_desc *__get_desc(struct idxd_wq
> *wq, int idx, int cpu)
> 
>  	desc = wq->descs[idx];
>  	memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
> -	memset(desc->completion, 0, sizeof(struct
> dsa_completion_record));
> +	memset(desc->completion, 0, idxd->compl_size);
>  	desc->cpu = cpu;
> 
>  	if (device_pasid_enabled(idxd))
> diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c
> index 3af83f1fd36e..266423a2cabc 100644
> --- a/drivers/dma/idxd/sysfs.c
> +++ b/drivers/dma/idxd/sysfs.c
> @@ -41,14 +41,24 @@ static struct device_type dsa_device_type = {
>  	.release = idxd_conf_device_release,
>  };
> 
> +static struct device_type iax_device_type = {
> +	.name = "iax",
> +	.release = idxd_conf_device_release,
> +};
> +
>  static inline bool is_dsa_dev(struct device *dev)
>  {
>  	return dev ? dev->type == &dsa_device_type : false;
>  }
> 
> +static inline bool is_iax_dev(struct device *dev)
> +{
> +	return dev ? dev->type == &iax_device_type : false;
> +}
> +
>  static inline bool is_idxd_dev(struct device *dev)
>  {
> -	return is_dsa_dev(dev);
> +	return is_dsa_dev(dev) || is_iax_dev(dev);
>  }
> 
>  static inline bool is_idxd_wq_dev(struct device *dev)
> @@ -359,8 +369,17 @@ struct bus_type dsa_bus_type = {
>  	.shutdown = idxd_config_bus_shutdown,
>  };
> 
> +struct bus_type iax_bus_type = {
> +	.name = "iax",
> +	.match = idxd_config_bus_match,
> +	.probe = idxd_config_bus_probe,
> +	.remove = idxd_config_bus_remove,
> +	.shutdown = idxd_config_bus_shutdown,
> +};
> +
>  static struct bus_type *idxd_bus_types[] = {
> -	&dsa_bus_type
> +	&dsa_bus_type,
> +	&iax_bus_type
>  };
> 
>  static struct idxd_device_driver dsa_drv = {
> @@ -372,8 +391,18 @@ static struct idxd_device_driver dsa_drv = {
>  	},
>  };
> 
> +static struct idxd_device_driver iax_drv = {
> +	.drv = {
> +		.name = "iax",
> +		.bus = &iax_bus_type,
> +		.owner = THIS_MODULE,
> +		.mod_name = KBUILD_MODNAME,
> +	},
> +};
> +
>  static struct idxd_device_driver *idxd_drvs[] = {
> -	&dsa_drv
> +	&dsa_drv,
> +	&iax_drv
>  };
> 
>  struct bus_type *idxd_get_bus_type(struct idxd_device *idxd)
> @@ -385,6 +414,8 @@ static struct device_type
> *idxd_get_device_type(struct idxd_device *idxd)
>  {
>  	if (idxd->type == IDXD_TYPE_DSA)
>  		return &dsa_device_type;
> +	else if (idxd->type == IDXD_TYPE_IAX)
> +		return &iax_device_type;
>  	else
>  		return NULL;
>  }
> @@ -525,6 +556,9 @@ static ssize_t group_tokens_reserved_store(struct
> device *dev,
>  	if (rc < 0)
>  		return -EINVAL;
> 
> +	if (idxd->type == IDXD_TYPE_IAX)
> +		return -EOPNOTSUPP;
> +
>  	if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
>  		return -EPERM;
> 
> @@ -570,6 +604,9 @@ static ssize_t group_tokens_allowed_store(struct
> device *dev,
>  	if (rc < 0)
>  		return -EINVAL;
> 
> +	if (idxd->type == IDXD_TYPE_IAX)
> +		return -EOPNOTSUPP;
> +
>  	if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
>  		return -EPERM;
> 
> @@ -612,6 +649,9 @@ static ssize_t group_use_token_limit_store(struct
> device *dev,
>  	if (rc < 0)
>  		return -EINVAL;
> 
> +	if (idxd->type == IDXD_TYPE_IAX)
> +		return -EOPNOTSUPP;
> +
>  	if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
>  		return -EPERM;
> 
> diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h
> index fdcdfe414223..236d437947bc 100644
> --- a/include/uapi/linux/idxd.h
> +++ b/include/uapi/linux/idxd.h
> @@ -26,6 +26,9 @@
>  #define IDXD_OP_FLAG_DRDBK	0x4000
>  #define IDXD_OP_FLAG_DSTS	0x8000
> 
> +/* IAX */
> +#define IDXD_OP_FLAG_RD_SRC2_AECS	0x010000
> +
>  /* Opcode */
>  enum dsa_opcode {
>  	DSA_OPCODE_NOOP = 0,
> @@ -47,6 +50,14 @@ enum dsa_opcode {
>  	DSA_OPCODE_CFLUSH = 0x20,
>  };
> 
> +enum iax_opcode {
> +	IAX_OPCODE_NOOP = 0,
> +	IAX_OPCODE_DRAIN = 2,
> +	IAX_OPCODE_MEMMOVE,
> +	IAX_OPCODE_DECOMPRESS = 0x42,
> +	IAX_OPCODE_COMPRESS,
> +};
> +
>  /* Completion record status */
>  enum dsa_completion_status {
>  	DSA_COMP_NONE = 0,
> @@ -80,6 +91,33 @@ enum dsa_completion_status {
>  	DSA_COMP_TRANSLATION_FAIL,
>  };
> 
> +enum iax_completion_status {
> +	IAX_COMP_NONE = 0,
> +	IAX_COMP_SUCCESS,
> +	IAX_COMP_PAGE_FAULT_IR = 0x04,
> +	IAX_COMP_OUTBUF_OVERFLOW,
> +	IAX_COMP_BAD_OPCODE = 0x10,
> +	IAX_COMP_INVALID_FLAGS,
> +	IAX_COMP_NOZERO_RESERVE,
> +	IAX_COMP_INVALID_SIZE,
> +	IAX_COMP_OVERLAP_BUFFERS = 0x16,
> +	IAX_COMP_INT_HANDLE_INVAL = 0x19,
> +	IAX_COMP_CRA_XLAT,
> +	IAX_COMP_CRA_ALIGN,
> +	IAX_COMP_ADDR_ALIGN,
> +	IAX_COMP_PRIV_BAD,
> +	IAX_COMP_TRAFFIC_CLASS_CONF,
> +	IAX_COMP_PFAULT_RDBA,
> +	IAX_COMP_HW_ERR1,
> +	IAX_COMP_HW_ERR_DRB,
> +	IAX_COMP_TRANSLATION_FAIL,
> +	IAX_COMP_PRS_TIMEOUT,
> +	IAX_COMP_WATCHDOG,
> +	IAX_COMP_INVALID_COMP_FLAG = 0x30,
> +	IAX_COMP_INVALID_FILTER_FLAG,
> +	IAX_COMP_INVALID_NUM_ELEMS = 0x33,
> +};
> +
>  #define DSA_COMP_STATUS_MASK		0x7f
>  #define DSA_COMP_STATUS_WRITE		0x80
> 
> @@ -163,6 +201,28 @@ struct dsa_hw_desc {
>  	};
>  } __attribute__((packed));
> 
> +struct iax_hw_desc {
> +	uint32_t        pasid:20;
> +	uint32_t        rsvd:11;
> +	uint32_t        priv:1;
> +	uint32_t        flags:24;
> +	uint32_t        opcode:8;
> +	uint64_t        completion_addr;
> +	uint64_t        src1_addr;
> +	uint64_t        dst_addr;
> +	uint32_t        src1_size;
> +	uint16_t        int_handle;
> +	union {
> +		uint16_t        compr_flags;
> +		uint16_t        decompr_flags;
> +	};
> +	uint64_t        src2_addr;
> +	uint32_t        max_dst_size;
> +	uint32_t        src2_size;
> +	uint32_t	filter_flags;
> +	uint32_t	num_inputs;
> +} __attribute__((packed));
> +
>  struct dsa_raw_desc {
>  	uint64_t	field[8];
>  } __attribute__((packed));
> @@ -223,4 +283,23 @@ struct dsa_raw_completion_record {
>  	uint64_t	field[4];
>  } __attribute__((packed));
> 
> +struct iax_completion_record {
> +	volatile uint8_t        status;
> +	uint8_t                 error_code;
> +	uint16_t                rsvd;
> +	uint32_t                bytes_completed;
> +	uint64_t                fault_addr;
> +	uint32_t                invalid_flags;
> +	uint32_t                rsvd2;
> +	uint32_t                output_size;
> +	uint8_t                 output_bits;
> +	uint8_t                 rsvd3;
> +	uint16_t                rsvd4;
> +	uint64_t                rsvd5[4];
> +} __attribute__((packed));
> +
> +struct iax_raw_completion_record {
> +	uint64_t	field[8];
> +} __attribute__((packed));
> +
>  #endif
>