Re: [RFC PATCH net-next 14/19] pds_vdpa: Add new PCI VF device for PDS vDPA services

[Jason Wang <jasowang@redhat.com>]

在 2022/11/19 06:56, Shannon Nelson 写道:
> This is the initial PCI driver framework for the new pds_vdpa VF
> device driver, an auxiliary_bus client of the pds_core driver.
> This does the very basics of registering for the new PCI
> device 1dd8:100b, setting up debugfs entries, and registering
> with devlink.
>
> The new PCI device id has not made it to the official PCI ID Repository
> yet, but will soon be registered there.
>
> Signed-off-by: Shannon Nelson <snelson@pensando.io>
> ---
>   drivers/vdpa/pds/Makefile       |   7 +
>   drivers/vdpa/pds/debugfs.c      |  44 +++++++
>   drivers/vdpa/pds/debugfs.h      |  22 ++++
>   drivers/vdpa/pds/pci_drv.c      | 143 +++++++++++++++++++++
>   drivers/vdpa/pds/pci_drv.h      |  46 +++++++
>   include/linux/pds/pds_core_if.h |   1 +
>   include/linux/pds/pds_vdpa.h    | 219 ++++++++++++++++++++++++++++++++
>   7 files changed, 482 insertions(+)
>   create mode 100644 drivers/vdpa/pds/Makefile
>   create mode 100644 drivers/vdpa/pds/debugfs.c
>   create mode 100644 drivers/vdpa/pds/debugfs.h
>   create mode 100644 drivers/vdpa/pds/pci_drv.c
>   create mode 100644 drivers/vdpa/pds/pci_drv.h
>   create mode 100644 include/linux/pds/pds_vdpa.h
>
> diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
> new file mode 100644
> index 000000000000..3ba28a875574
> --- /dev/null
> +++ b/drivers/vdpa/pds/Makefile
> @@ -0,0 +1,7 @@
> +# SPDX-License-Identifier: GPL-2.0-only
> +# Copyright(c) 2022 Pensando Systems, Inc
> +
> +obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
> +
> +pds_vdpa-y := pci_drv.o	\
> +	      debugfs.o
> diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
> new file mode 100644
> index 000000000000..f5b6654ae89b
> --- /dev/null
> +++ b/drivers/vdpa/pds/debugfs.c
> @@ -0,0 +1,44 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/* Copyright(c) 2022 Pensando Systems, Inc */
> +
> +#include <linux/module.h>
> +#include <linux/pci.h>
> +#include <linux/types.h>
> +
> +#include <linux/pds/pds_core_if.h>
> +#include <linux/pds/pds_vdpa.h>
> +
> +#include "pci_drv.h"
> +#include "debugfs.h"
> +
> +#ifdef CONFIG_DEBUG_FS
> +
> +static struct dentry *dbfs_dir;
> +
> +void
> +pds_vdpa_debugfs_create(void)
> +{
> +	dbfs_dir = debugfs_create_dir(PDS_VDPA_DRV_NAME, NULL);
> +}
> +
> +void
> +pds_vdpa_debugfs_destroy(void)
> +{
> +	debugfs_remove_recursive(dbfs_dir);
> +	dbfs_dir = NULL;
> +}
> +
> +void
> +pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_pci_device *vdpa_pdev)
> +{
> +	vdpa_pdev->dentry = debugfs_create_dir(pci_name(vdpa_pdev->pdev), dbfs_dir);
> +}
> +
> +void
> +pds_vdpa_debugfs_del_pcidev(struct pds_vdpa_pci_device *vdpa_pdev)
> +{
> +	debugfs_remove_recursive(vdpa_pdev->dentry);
> +	vdpa_pdev->dentry = NULL;
> +}
> +
> +#endif /* CONFIG_DEBUG_FS */
> diff --git a/drivers/vdpa/pds/debugfs.h b/drivers/vdpa/pds/debugfs.h
> new file mode 100644
> index 000000000000..ac31ab47746b
> --- /dev/null
> +++ b/drivers/vdpa/pds/debugfs.h
> @@ -0,0 +1,22 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2022 Pensando Systems, Inc */
> +
> +#ifndef _PDS_VDPA_DEBUGFS_H_
> +#define _PDS_VDPA_DEBUGFS_H_
> +
> +#include <linux/debugfs.h>
> +
> +#ifdef CONFIG_DEBUG_FS
> +
> +void pds_vdpa_debugfs_create(void);
> +void pds_vdpa_debugfs_destroy(void);
> +void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_pci_device *vdpa_pdev);
> +void pds_vdpa_debugfs_del_pcidev(struct pds_vdpa_pci_device *vdpa_pdev);
> +#else
> +static inline void pds_vdpa_debugfs_create(void) { }
> +static inline void pds_vdpa_debugfs_destroy(void) { }
> +static inline void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_pci_device *vdpa_pdev) { }
> +static inline void pds_vdpa_debugfs_del_pcidev(struct pds_vdpa_pci_device *vdpa_pdev) { }
> +#endif
> +
> +#endif /* _PDS_VDPA_DEBUGFS_H_ */
> diff --git a/drivers/vdpa/pds/pci_drv.c b/drivers/vdpa/pds/pci_drv.c
> new file mode 100644
> index 000000000000..369e11153f21
> --- /dev/null
> +++ b/drivers/vdpa/pds/pci_drv.c
> @@ -0,0 +1,143 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/* Copyright(c) 2022 Pensando Systems, Inc */
> +
> +#include <linux/module.h>
> +#include <linux/pci.h>
> +#include <linux/aer.h>
> +#include <linux/types.h>
> +#include <linux/vdpa.h>
> +
> +#include <linux/pds/pds_core_if.h>
> +#include <linux/pds/pds_vdpa.h>
> +
> +#include "pci_drv.h"
> +#include "debugfs.h"
> +
> +static void
> +pds_vdpa_dma_action(void *data)
> +{
> +	pci_free_irq_vectors((struct pci_dev *)data);
> +}


Nit: since we're release irq vectors, it might be better to use 
"pds_vdpa_irq_action"


> +
> +static int
> +pds_vdpa_pci_probe(struct pci_dev *pdev,
> +		   const struct pci_device_id *id)
> +{
> +	struct pds_vdpa_pci_device *vdpa_pdev;
> +	struct device *dev = &pdev->dev;
> +	int err;
> +
> +	vdpa_pdev = kzalloc(sizeof(*vdpa_pdev), GFP_KERNEL);
> +	if (!vdpa_pdev)
> +		return -ENOMEM;
> +	pci_set_drvdata(pdev, vdpa_pdev);
> +
> +	vdpa_pdev->pdev = pdev;
> +	vdpa_pdev->vf_id = pci_iov_vf_id(pdev);
> +	vdpa_pdev->pci_id = PCI_DEVID(pdev->bus->number, pdev->devfn);
> +
> +	/* Query system for DMA addressing limitation for the device. */
> +	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
> +	if (err) {
> +		dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting. %pe\n",
> +			ERR_PTR(err));
> +		goto err_out_free_mem;
> +	}
> +
> +	pci_enable_pcie_error_reporting(pdev);
> +
> +	/* Use devres management */
> +	err = pcim_enable_device(pdev);
> +	if (err) {
> +		dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
> +		goto err_out_free_mem;
> +	}
> +
> +	err = devm_add_action_or_reset(dev, pds_vdpa_dma_action, pdev);
> +	if (err) {
> +		dev_err(dev, "Failed adding devres for freeing irq vectors: %pe\n",
> +			ERR_PTR(err));
> +		goto err_out_pci_release_device;
> +	}
> +
> +	pci_set_master(pdev);
> +
> +	pds_vdpa_debugfs_add_pcidev(vdpa_pdev);
> +
> +	dev_info(dev, "%s: PF %#04x VF %#04x (%d) vf_id %d domain %d vdpa_aux %p vdpa_pdev %p\n",
> +		 __func__, pci_dev_id(vdpa_pdev->pdev->physfn),
> +		 vdpa_pdev->pci_id, vdpa_pdev->pci_id, vdpa_pdev->vf_id,
> +		 pci_domain_nr(pdev->bus), vdpa_pdev->vdpa_aux, vdpa_pdev);
> +
> +	return 0;
> +
> +err_out_pci_release_device:
> +	pci_disable_device(pdev);


Do we still need to care about this consider we use 
devres/pcim_enable_device()?


> +err_out_free_mem:
> +	pci_disable_pcie_error_reporting(pdev);
> +	kfree(vdpa_pdev);
> +	return err;
> +}
> +
> +static void
> +pds_vdpa_pci_remove(struct pci_dev *pdev)
> +{
> +	struct pds_vdpa_pci_device *vdpa_pdev = pci_get_drvdata(pdev);
> +
> +	pds_vdpa_debugfs_del_pcidev(vdpa_pdev);
> +	pci_clear_master(pdev);
> +	pci_disable_pcie_error_reporting(pdev);
> +	pci_disable_device(pdev);
> +	kfree(vdpa_pdev);
> +
> +	dev_info(&pdev->dev, "Removed\n");
> +}
> +
> +static const struct pci_device_id
> +pds_vdpa_pci_table[] = {
> +	{ PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
> +	{ 0, }
> +};
> +MODULE_DEVICE_TABLE(pci, pds_vdpa_pci_table);
> +
> +static struct pci_driver
> +pds_vdpa_pci_driver = {
> +	.name = PDS_VDPA_DRV_NAME,
> +	.id_table = pds_vdpa_pci_table,
> +	.probe = pds_vdpa_pci_probe,
> +	.remove = pds_vdpa_pci_remove
> +};
> +
> +static void __exit
> +pds_vdpa_pci_cleanup(void)
> +{
> +	pci_unregister_driver(&pds_vdpa_pci_driver);
> +
> +	pds_vdpa_debugfs_destroy();
> +}
> +module_exit(pds_vdpa_pci_cleanup);
> +
> +static int __init
> +pds_vdpa_pci_init(void)
> +{
> +	int err;
> +
> +	pds_vdpa_debugfs_create();
> +
> +	err = pci_register_driver(&pds_vdpa_pci_driver);
> +	if (err) {
> +		pr_err("%s: pci driver register failed: %pe\n", __func__, ERR_PTR(err));
> +		goto err_pci;
> +	}
> +
> +	return 0;
> +
> +err_pci:
> +	pds_vdpa_debugfs_destroy();
> +	return err;
> +}
> +module_init(pds_vdpa_pci_init);
> +
> +MODULE_DESCRIPTION(PDS_VDPA_DRV_DESCRIPTION);
> +MODULE_AUTHOR("Pensando Systems, Inc");
> +MODULE_LICENSE("GPL");
> diff --git a/drivers/vdpa/pds/pci_drv.h b/drivers/vdpa/pds/pci_drv.h
> new file mode 100644
> index 000000000000..747809e0df9e
> --- /dev/null
> +++ b/drivers/vdpa/pds/pci_drv.h
> @@ -0,0 +1,46 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/* Copyright(c) 2022 Pensando Systems, Inc */
> +
> +#ifndef _PCI_DRV_H
> +#define _PCI_DRV_H
> +
> +#include <linux/pci.h>
> +#include <linux/virtio_pci_modern.h>
> +
> +#define PDS_VDPA_DRV_NAME           "pds_vdpa"
> +#define PDS_VDPA_DRV_DESCRIPTION    "Pensando vDPA VF Device Driver"
> +
> +#define PDS_VDPA_BAR_BASE	0
> +#define PDS_VDPA_BAR_INTR	2
> +#define PDS_VDPA_BAR_DBELL	4
> +
> +struct pds_dev_bar {
> +	int           index;
> +	void __iomem  *vaddr;
> +	phys_addr_t   pa;
> +	unsigned long len;
> +};
> +
> +struct pds_vdpa_intr_info {
> +	int index;
> +	int irq;
> +	int qid;
> +	char name[32];
> +};
> +
> +struct pds_vdpa_pci_device {
> +	struct pci_dev *pdev;
> +	struct pds_vdpa_aux *vdpa_aux;
> +
> +	int vf_id;
> +	int pci_id;
> +
> +	int nintrs;
> +	struct pds_vdpa_intr_info *intrs;
> +
> +	struct dentry *dentry;
> +
> +	struct virtio_pci_modern_device vd_mdev;
> +};
> +
> +#endif /* _PCI_DRV_H */
> diff --git a/include/linux/pds/pds_core_if.h b/include/linux/pds/pds_core_if.h
> index 6333ec351e14..6e92697657e4 100644
> --- a/include/linux/pds/pds_core_if.h
> +++ b/include/linux/pds/pds_core_if.h
> @@ -8,6 +8,7 @@
>   
>   #define PCI_VENDOR_ID_PENSANDO			0x1dd8
>   #define PCI_DEVICE_ID_PENSANDO_CORE_PF		0x100c
> +#define PCI_DEVICE_ID_PENSANDO_VDPA_VF          0x100b
>   
>   #define PDS_CORE_BARS_MAX			4
>   #define PDS_CORE_PCI_BAR_DBELL			1
> diff --git a/include/linux/pds/pds_vdpa.h b/include/linux/pds/pds_vdpa.h
> new file mode 100644
> index 000000000000..7ecef890f175
> --- /dev/null
> +++ b/include/linux/pds/pds_vdpa.h
> @@ -0,0 +1,219 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/* Copyright(c) 2022 Pensando Systems, Inc */
> +
> +#ifndef _PDS_VDPA_IF_H_
> +#define _PDS_VDPA_IF_H_
> +
> +#include <linux/pds/pds_common.h>
> +
> +#define PDS_DEV_TYPE_VDPA_STR	"vDPA"
> +#define PDS_VDPA_DEV_NAME	PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_VDPA_STR
> +
> +/*
> + * enum pds_vdpa_cmd_opcode - vDPA Device commands
> + */
> +enum pds_vdpa_cmd_opcode {
> +	PDS_VDPA_CMD_INIT		= 48,
> +	PDS_VDPA_CMD_IDENT		= 49,
> +	PDS_VDPA_CMD_RESET		= 51,
> +	PDS_VDPA_CMD_VQ_RESET		= 52,
> +	PDS_VDPA_CMD_VQ_INIT		= 53,
> +	PDS_VDPA_CMD_STATUS_UPDATE	= 54,
> +	PDS_VDPA_CMD_SET_FEATURES	= 55,
> +	PDS_VDPA_CMD_SET_ATTR		= 56,
> +};
> +
> +/**
> + * struct pds_vdpa_cmd - generic command
> + * @opcode:	Opcode
> + * @vdpa_index:	Index for vdpa subdevice
> + * @vf_id:	VF id
> + */
> +struct pds_vdpa_cmd {
> +	u8     opcode;
> +	u8     vdpa_index;
> +	__le16 vf_id;
> +};
> +
> +/**
> + * struct pds_vdpa_comp - generic command completion
> + * @status:	Status of the command (enum pds_core_status_code)
> + * @rsvd:	Word boundary padding
> + * @color:	Color bit
> + */
> +struct pds_vdpa_comp {
> +	u8 status;
> +	u8 rsvd[14];
> +	u8 color;
> +};
> +
> +/**
> + * struct pds_vdpa_init_cmd - INIT command
> + * @opcode:	Opcode PDS_VDPA_CMD_INIT
> + * @vdpa_index: Index for vdpa subdevice
> + * @vf_id:	VF id
> + * @len:	length of config info DMA space
> + * @config_pa:	address for DMA of virtio_net_config struct


Looks like the structure is not specific to net, if yes, we may tweak 
the above comment to say it's the address of the device configuration space.


> + */
> +struct pds_vdpa_init_cmd {
> +	u8     opcode;
> +	u8     vdpa_index;
> +	__le16 vf_id;
> +	__le32 len;
> +	__le64 config_pa;
> +};
> +
> +/**
> + * struct pds_vdpa_ident - vDPA identification data
> + * @hw_features:	vDPA features supported by device
> + * @max_vqs:		max queues available (2 queues for a single queuepair)
> + * @max_qlen:		log(2) of maximum number of descriptors
> + * @min_qlen:		log(2) of minimum number of descriptors


Note that is you have the plan to support packed virtqueue, the qlen is 
not necessarily the power of 2.


> + *
> + * This struct is used in a DMA block that is set up for the PDS_VDPA_CMD_IDENT
> + * transaction.  Set up the DMA block and send the address in the IDENT cmd
> + * data, the DSC will write the ident information, then we can remove the DMA
> + * block after reading the answer.  If the completion status is 0, then there
> + * is valid information, else there was an error and the data should be invalid.
> + */
> +struct pds_vdpa_ident {
> +	__le64 hw_features;
> +	__le16 max_vqs;
> +	__le16 max_qlen;
> +	__le16 min_qlen;
> +};
> +
> +/**
> + * struct pds_vdpa_ident_cmd - IDENT command
> + * @opcode:	Opcode PDS_VDPA_CMD_IDENT
> + * @rsvd:       Word boundary padding
> + * @vf_id:	VF id
> + * @len:	length of ident info DMA space
> + * @ident_pa:	address for DMA of ident info (struct pds_vdpa_ident)
> + *			only used for this transaction, then forgotten by DSC
> + */
> +struct pds_vdpa_ident_cmd {
> +	u8     opcode;
> +	u8     rsvd;
> +	__le16 vf_id;
> +	__le32 len;
> +	__le64 ident_pa;
> +};
> +
> +/**
> + * struct pds_vdpa_status_cmd - STATUS_UPDATE command
> + * @opcode:	Opcode PDS_VDPA_CMD_STATUS_UPDATE
> + * @vdpa_index: Index for vdpa subdevice
> + * @vf_id:	VF id
> + * @status:	new status bits
> + */
> +struct pds_vdpa_status_cmd {
> +	u8     opcode;
> +	u8     vdpa_index;
> +	__le16 vf_id;
> +	u8     status;
> +};
> +
> +/**
> + * enum pds_vdpa_attr - List of VDPA device attributes
> + * @PDS_VDPA_ATTR_MAC:          MAC address
> + * @PDS_VDPA_ATTR_MAX_VQ_PAIRS: Max virtqueue pairs
> + */
> +enum pds_vdpa_attr {
> +	PDS_VDPA_ATTR_MAC          = 1,
> +	PDS_VDPA_ATTR_MAX_VQ_PAIRS = 2,
> +};
> +
> +/**
> + * struct pds_vdpa_setattr_cmd - SET_ATTR command
> + * @opcode:		Opcode PDS_VDPA_CMD_SET_ATTR
> + * @vdpa_index:		Index for vdpa subdevice
> + * @vf_id:		VF id
> + * @attr:		attribute to be changed (enum pds_vdpa_attr)
> + * @pad:		Word boundary padding
> + * @mac:		new mac address to be assigned as vdpa device address
> + * @max_vq_pairs:	new limit of virtqueue pairs
> + */
> +struct pds_vdpa_setattr_cmd {
> +	u8     opcode;
> +	u8     vdpa_index;
> +	__le16 vf_id;
> +	u8     attr;
> +	u8     pad[3];
> +	union {
> +		u8 mac[6];
> +		__le16 max_vq_pairs;


So does this mean if we want to set both mac and max_vq_paris, we need 
two commands? The seems to be less efficient, since the mgmt layer could 
provision more attributes here. Can we pack all attributes into a single 
command?


> +	} __packed;
> +};
> +
> +/**
> + * struct pds_vdpa_vq_init_cmd - queue init command
> + * @opcode: Opcode PDS_VDPA_CMD_VQ_INIT
> + * @vdpa_index:	Index for vdpa subdevice
> + * @vf_id:	VF id
> + * @qid:	Queue id (bit0 clear = rx, bit0 set = tx, qid=N is ctrlq)


I wonder any reason we need to design it like this, it would be better 
to make it general to be used by other type of virtio devices.


> + * @len:	log(2) of max descriptor count
> + * @desc_addr:	DMA address of descriptor area
> + * @avail_addr:	DMA address of available descriptors (aka driver area)
> + * @used_addr:	DMA address of used descriptors (aka device area)
> + * @intr_index:	interrupt index


Is this something like MSI-X vector?


> + */
> +struct pds_vdpa_vq_init_cmd {
> +	u8     opcode;
> +	u8     vdpa_index;
> +	__le16 vf_id;
> +	__le16 qid;
> +	__le16 len;
> +	__le64 desc_addr;
> +	__le64 avail_addr;
> +	__le64 used_addr;
> +	__le16 intr_index;
> +};
> +
> +/**
> + * struct pds_vdpa_vq_init_comp - queue init completion
> + * @status:	Status of the command (enum pds_core_status_code)
> + * @hw_qtype:	HW queue type, used in doorbell selection
> + * @hw_qindex:	HW queue index, used in doorbell selection
> + * @rsvd:	Word boundary padding
> + * @color:	Color bit


More comment is needed to know the how to use this color bit.


> + */
> +struct pds_vdpa_vq_init_comp {
> +	u8     status;
> +	u8     hw_qtype;
> +	__le16 hw_qindex;
> +	u8     rsvd[11];
> +	u8     color;
> +};
> +
> +/**
> + * struct pds_vdpa_vq_reset_cmd - queue reset command
> + * @opcode:	Opcode PDS_VDPA_CMD_VQ_RESET


Is there a chance that we could have more type of opcode here?

Thanks


> + * @vdpa_index:	Index for vdpa subdevice
> + * @vf_id:	VF id
> + * @qid:	Queue id
> + */
> +struct pds_vdpa_vq_reset_cmd {
> +	u8     opcode;
> +	u8     vdpa_index;
> +	__le16 vf_id;
> +	__le16 qid;
> +};
> +
> +/**
> + * struct pds_vdpa_set_features_cmd - set hw features
> + * @opcode: Opcode PDS_VDPA_CMD_SET_FEATURES
> + * @vdpa_index:	Index for vdpa subdevice
> + * @vf_id:	VF id
> + * @rsvd:       Word boundary padding
> + * @features:	Feature bit mask
> + */
> +struct pds_vdpa_set_features_cmd {
> +	u8     opcode;
> +	u8     vdpa_index;
> +	__le16 vf_id;
> +	__le32 rsvd;
> +	__le64 features;
> +};
> +
> +#endif /* _PDS_VDPA_IF_H_ */