[PATCH v4 0/2] vdpasim: support doorbell mapping

[PATCH v4 0/2] vdpasim: support doorbell mapping

[Longpeng(Mike)]

From: Longpeng <longpeng2@huawei.com>

This patchset supports doorbell mapping for vdpasim devices.

v3: https://lore.kernel.org/lkml/20230213070446-mutt-send-email-mst@kernel.org/T/
v2: https://lore.kernel.org/lkml/CACGkMEtdT5fG=ffbpQadkGmzHf6Ax-+L50LsriYqJaW++natMg@mail.gmail.com/T/

Changes v3->v4:
    - use WRITE_ONCE  [Jason]
    - start/stop periodic work when create/destroy vdpasim device  [Jason]

Changes v2->v3:
    - add a new callback named get_vq_notification_pgprot to vdpa_config_ops [Jason]
    - remove the new added module parameter 'parameter' [Jason]
    - opencode the schedule/cancel_delayed() [Jason]

Changes v1->v2:
    - support both kick mode and passthrough mode. [Jason]
    - poll the notify register first. [Jason, Michael]

Longpeng (2):
  vdpa: support specify the pgprot of vq notification area
  vdpasim: support doorbell mapping

 drivers/vdpa/vdpa_sim/vdpa_sim.c | 67 ++++++++++++++++++++++++++++++++
 drivers/vdpa/vdpa_sim/vdpa_sim.h |  3 ++
 drivers/vhost/vdpa.c             |  4 +-
 include/linux/vdpa.h             |  9 +++++
 4 files changed, 82 insertions(+), 1 deletion(-)

-- 
2.23.0

[PATCH v4 1/2] vdpa: support specify the pgprot of vq notification area

[Longpeng(Mike)]

From: Longpeng <longpeng2@huawei.com>

Adds get_vq_notification_pgprot operation to vdpa_config_ops to support
specify the pgprot of vq norification area. It's an optional operation,
the vdpa framework will treat the pgprot of vq notification area as
noncached as default as usual.

Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: Longpeng <longpeng2@huawei.com>
---
 drivers/vhost/vdpa.c | 4 +++-
 include/linux/vdpa.h | 9 +++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index ec32f785dfde..8b595521c958 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -1273,7 +1273,9 @@ static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
 
 	notify = ops->get_vq_notification(vdpa, index);
 
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_page_prot = ops->get_vq_notification_pgprot ?
+		ops->get_vq_notification_pgprot(vdpa, index, vma->vm_page_prot) :
+		pgprot_noncached(vma->vm_page_prot);
 	if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
 			    PFN_DOWN(notify.addr), PAGE_SIZE,
 			    vma->vm_page_prot))
diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
index 6d0f5e4e82c2..07fcf5e6abc8 100644
--- a/include/linux/vdpa.h
+++ b/include/linux/vdpa.h
@@ -169,6 +169,12 @@ struct vdpa_map_file {
  *				@vdev: vdpa device
  *				@idx: virtqueue index
  *				Returns the notifcation area
+ * @get_vq_notification_pgprot:	Get the pgprot of the vq's notification area (optional)
+ *				@vdev: vdpa device
+ *				@idx: virtqueue index
+ *				@prot: original page protection value of the
+ *				       notification area
+ *				Returns pgprot_t: the pgprot of the notification area
  * @get_vq_irq:			Get the irq number of a virtqueue (optional,
  *				but must implemented if require vq irq offloading)
  *				@vdev: vdpa device
@@ -305,6 +311,9 @@ struct vdpa_config_ops {
 				   struct netlink_ext_ack *extack);
 	struct vdpa_notification_area
 	(*get_vq_notification)(struct vdpa_device *vdev, u16 idx);
+	pgprot_t (*get_vq_notification_pgprot)(struct vdpa_device *vdev,
+					       u16 idx,
+					       pgprot_t prot);
 	/* vq irq is not expected to be changed once DRIVER_OK is set */
 	int (*get_vq_irq)(struct vdpa_device *vdev, u16 idx);
 
-- 
2.23.0

[PATCH v4 2/2] vdpasim: support doorbell mapping

[Longpeng(Mike)]

From: Longpeng <longpeng2@huawei.com>

Support doorbell mapping for vdpasim devices, then we can test the notify
passthrough feature even if there's no real hardware on hand.

Allocates a dummy page which is used to emulate the notify page of the device,
all VQs share the same notify register  that initiated to 0xffff. A  periodic
work will check whether there're requests need to process ( the value of the
notify register is 0xffff or not ).

We can test on QEMU with:
 -device vhost-vdpa-device-pci,vhostdev=/dev/vhost-vdpa-0,page-per-vq=on

Signed-off-by: Longpeng <longpeng2@huawei.com>
---
 drivers/vdpa/vdpa_sim/vdpa_sim.c | 67 ++++++++++++++++++++++++++++++++
 drivers/vdpa/vdpa_sim/vdpa_sim.h |  3 ++
 2 files changed, 70 insertions(+)

diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index cb88891b44a8..5a8c820b179f 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -39,6 +39,8 @@ MODULE_PARM_DESC(max_iotlb_entries,
 #define VDPASIM_QUEUE_ALIGN PAGE_SIZE
 #define VDPASIM_QUEUE_MAX 256
 #define VDPASIM_VENDOR_ID 0
+#define VDPASIM_VRING_POLL_PERIOD 100 /* ms */
+#define VDPASIM_NOTIFY_DEFVAL 0xffff
 
 static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)
 {
@@ -245,6 +247,28 @@ static const struct dma_map_ops vdpasim_dma_ops = {
 static const struct vdpa_config_ops vdpasim_config_ops;
 static const struct vdpa_config_ops vdpasim_batch_config_ops;
 
+static void vdpasim_notify_work(struct work_struct *work)
+{
+	struct vdpasim *vdpasim;
+	u16 *val;
+
+	vdpasim = container_of(work, struct vdpasim, notify_work.work);
+
+	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
+		goto out;
+
+	if (!vdpasim->running)
+		goto out;
+
+	val = (u16 *)vdpasim->notify;
+	if (xchg(val, VDPASIM_NOTIFY_DEFVAL) != VDPASIM_NOTIFY_DEFVAL)
+		schedule_work(&vdpasim->work);
+
+out:
+	schedule_delayed_work(&vdpasim->notify_work,
+			      msecs_to_jiffies(VDPASIM_VRING_POLL_PERIOD));
+}
+
 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
 			       const struct vdpa_dev_set_config *config)
 {
@@ -286,6 +310,13 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
 	set_dma_ops(dev, &vdpasim_dma_ops);
 	vdpasim->vdpa.mdev = dev_attr->mgmt_dev;
 
+	INIT_DELAYED_WORK(&vdpasim->notify_work, vdpasim_notify_work);
+
+	vdpasim->notify = (u16 *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+	if (!vdpasim->notify)
+		goto err_iommu;
+	WRITE_ONCE(*vdpasim->notify, VDPASIM_NOTIFY_DEFVAL);
+
 	vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL);
 	if (!vdpasim->config)
 		goto err_iommu;
@@ -320,6 +351,13 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
 
 	vdpasim->vdpa.dma_dev = dev;
 
+	/*
+ 	 * Start periodic (every 100ms) notify work, it won't introduce
+ 	 * any overhead if the device is not started.
+	 */
+	schedule_delayed_work(&vdpasim->notify_work,
+			      msecs_to_jiffies(VDPASIM_VRING_POLL_PERIOD));
+
 	return vdpasim;
 
 err_iommu:
@@ -671,11 +709,34 @@ static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid,
 	return 0;
 }
 
+static pgprot_t vdpasim_get_vq_notification_pgprot(struct vdpa_device *vdpa,
+						   u16 qid, pgprot_t prot)
+{
+	/*
+	 * We use normal RAM pages to emulate the vq notification area, so
+	 * just keep the pgprot as it mmaped.
+	 */
+	return prot;
+}
+
+static struct vdpa_notification_area
+vdpasim_get_vq_notification(struct vdpa_device *vdpa, u16 qid)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpa_notification_area notify;
+
+	notify.addr = virt_to_phys((void *)vdpasim->notify);
+	notify.size = PAGE_SIZE;
+
+	return notify;
+}
+
 static void vdpasim_free(struct vdpa_device *vdpa)
 {
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
 	int i;
 
+	cancel_delayed_work_sync(&vdpasim->notify_work);
 	cancel_work_sync(&vdpasim->work);
 
 	for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
@@ -694,6 +755,8 @@ static void vdpasim_free(struct vdpa_device *vdpa)
 	kfree(vdpasim->iommu);
 	kfree(vdpasim->vqs);
 	kfree(vdpasim->config);
+	if (vdpasim->notify)
+		free_page((unsigned long)vdpasim->notify);
 }
 
 static const struct vdpa_config_ops vdpasim_config_ops = {
@@ -705,6 +768,8 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
 	.get_vq_ready           = vdpasim_get_vq_ready,
 	.set_vq_state           = vdpasim_set_vq_state,
 	.get_vq_state           = vdpasim_get_vq_state,
+	.get_vq_notification    = vdpasim_get_vq_notification,
+	.get_vq_notification_pgprot = vdpasim_get_vq_notification_pgprot,
 	.get_vq_align           = vdpasim_get_vq_align,
 	.get_vq_group           = vdpasim_get_vq_group,
 	.get_device_features    = vdpasim_get_device_features,
@@ -738,6 +803,8 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
 	.get_vq_ready           = vdpasim_get_vq_ready,
 	.set_vq_state           = vdpasim_set_vq_state,
 	.get_vq_state           = vdpasim_get_vq_state,
+	.get_vq_notification    = vdpasim_get_vq_notification,
+	.get_vq_notification_pgprot = vdpasim_get_vq_notification_pgprot,
 	.get_vq_align           = vdpasim_get_vq_align,
 	.get_vq_group           = vdpasim_get_vq_group,
 	.get_device_features    = vdpasim_get_device_features,
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
index 0e78737dcc16..786e902de0b2 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
@@ -69,6 +69,9 @@ struct vdpasim {
 	bool running;
 	/* spinlock to synchronize iommu table */
 	spinlock_t iommu_lock;
+	/* dummy notify page */
+	u16 *notify;
+	struct delayed_work notify_work;
 };
 
 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
-- 
2.23.0

Re: [PATCH v4 2/2] vdpasim: support doorbell mapping

[kernel test robot]

Hi Longpeng(Mike),

Thank you for the patch! Yet something to improve:

[auto build test ERROR on v6.2]
[also build test ERROR on next-20230227]
[cannot apply to mst-vhost/linux-next linus/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Longpeng-Mike/vdpa-support-specify-the-pgprot-of-vq-notification-area/20230227-172516
patch link:    https://lore.kernel.org/r/20230227091857.2406-3-longpeng2%40huawei.com
patch subject: [PATCH v4 2/2] vdpasim: support doorbell mapping
config: arm-randconfig-r046-20230227 (https://download.01.org/0day-ci/archive/20230227/202302272333.Jioo8IEs-lkp@intel.com/config)
compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project db89896bbbd2251fff457699635acbbedeead27f)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # install arm cross compiling tool for clang build
        # apt-get install binutils-arm-linux-gnueabi
        # https://github.com/intel-lab-lkp/linux/commit/a472c7ad92f68b5b596fd68e1936b2d47fe2ea0b
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Longpeng-Mike/vdpa-support-specify-the-pgprot-of-vq-notification-area/20230227-172516
        git checkout a472c7ad92f68b5b596fd68e1936b2d47fe2ea0b
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=arm olddefconfig
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=arm SHELL=/bin/bash

If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>
| Link: https://lore.kernel.org/oe-kbuild-all/202302272333.Jioo8IEs-lkp@intel.com/

All errors (new ones prefixed by >>):

>> ld.lld: error: undefined symbol: __bad_xchg
   >>> referenced by cmpxchg.h:110 (arch/arm/include/asm/cmpxchg.h:110)
   >>>               drivers/vdpa/vdpa_sim/vdpa_sim.o:(vdpasim_notify_work) in archive vmlinux.a

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests

Re: [PATCH v4 2/2] vdpasim: support doorbell mapping

[kernel test robot]

Hi Longpeng(Mike),

Thank you for the patch! Yet something to improve:

[auto build test ERROR on v6.2]
[cannot apply to mst-vhost/linux-next linus/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Longpeng-Mike/vdpa-support-specify-the-pgprot-of-vq-notification-area/20230227-172516
patch link:    https://lore.kernel.org/r/20230227091857.2406-3-longpeng2%40huawei.com
patch subject: [PATCH v4 2/2] vdpasim: support doorbell mapping
config: arm-allmodconfig (https://download.01.org/0day-ci/archive/20230227/202302272338.vXR8d1Bb-lkp@intel.com/config)
compiler: arm-linux-gnueabi-gcc (GCC) 12.1.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/intel-lab-lkp/linux/commit/a472c7ad92f68b5b596fd68e1936b2d47fe2ea0b
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Longpeng-Mike/vdpa-support-specify-the-pgprot-of-vq-notification-area/20230227-172516
        git checkout a472c7ad92f68b5b596fd68e1936b2d47fe2ea0b
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=arm olddefconfig
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=arm SHELL=/bin/bash

If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>
| Link: https://lore.kernel.org/oe-kbuild-all/202302272338.vXR8d1Bb-lkp@intel.com/

All errors (new ones prefixed by >>, old ones prefixed by <<):

>> ERROR: modpost: "__bad_xchg" [drivers/vdpa/vdpa_sim/vdpa_sim.ko] undefined!

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests

Re: [PATCH v4 2/2] vdpasim: support doorbell mapping

[kernel test robot]

Hi Longpeng(Mike),

Thank you for the patch! Yet something to improve:

[auto build test ERROR on v6.2]
[cannot apply to mst-vhost/linux-next linus/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Longpeng-Mike/vdpa-support-specify-the-pgprot-of-vq-notification-area/20230227-172516
patch link:    https://lore.kernel.org/r/20230227091857.2406-3-longpeng2%40huawei.com
patch subject: [PATCH v4 2/2] vdpasim: support doorbell mapping
config: arm-allyesconfig (https://download.01.org/0day-ci/archive/20230228/202302280613.0lyqDCJr-lkp@intel.com/config)
compiler: arm-linux-gnueabi-gcc (GCC) 12.1.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/intel-lab-lkp/linux/commit/a472c7ad92f68b5b596fd68e1936b2d47fe2ea0b
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Longpeng-Mike/vdpa-support-specify-the-pgprot-of-vq-notification-area/20230227-172516
        git checkout a472c7ad92f68b5b596fd68e1936b2d47fe2ea0b
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=arm olddefconfig
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=arm SHELL=/bin/bash

If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>
| Link: https://lore.kernel.org/oe-kbuild-all/202302280613.0lyqDCJr-lkp@intel.com/

All errors (new ones prefixed by >>):

   arm-linux-gnueabi-ld: drivers/vdpa/vdpa_sim/vdpa_sim.o: in function `vdpasim_notify_work':
>> vdpa_sim.c:(.text+0x12d0): undefined reference to `__bad_xchg'

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests

Re: [PATCH v4 2/2] vdpasim: support doorbell mapping

[Jason Wang]

在 2023/2/27 17:18, Longpeng(Mike) 写道:
> From: Longpeng <longpeng2@huawei.com>
>
> Support doorbell mapping for vdpasim devices, then we can test the notify
> passthrough feature even if there's no real hardware on hand.
>
> Allocates a dummy page which is used to emulate the notify page of the device,
> all VQs share the same notify register  that initiated to 0xffff. A  periodic
> work will check whether there're requests need to process ( the value of the
> notify register is 0xffff or not ).
>
> We can test on QEMU with:
>   -device vhost-vdpa-device-pci,vhostdev=/dev/vhost-vdpa-0,page-per-vq=on
>
> Signed-off-by: Longpeng <longpeng2@huawei.com>


Acked-by: Jason Wang <jasowang@redhat.com>

Thanks


> ---
>   drivers/vdpa/vdpa_sim/vdpa_sim.c | 67 ++++++++++++++++++++++++++++++++
>   drivers/vdpa/vdpa_sim/vdpa_sim.h |  3 ++
>   2 files changed, 70 insertions(+)
>
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> index cb88891b44a8..5a8c820b179f 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> @@ -39,6 +39,8 @@ MODULE_PARM_DESC(max_iotlb_entries,
>   #define VDPASIM_QUEUE_ALIGN PAGE_SIZE
>   #define VDPASIM_QUEUE_MAX 256
>   #define VDPASIM_VENDOR_ID 0
> +#define VDPASIM_VRING_POLL_PERIOD 100 /* ms */
> +#define VDPASIM_NOTIFY_DEFVAL 0xffff
>   
>   static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)
>   {
> @@ -245,6 +247,28 @@ static const struct dma_map_ops vdpasim_dma_ops = {
>   static const struct vdpa_config_ops vdpasim_config_ops;
>   static const struct vdpa_config_ops vdpasim_batch_config_ops;
>   
> +static void vdpasim_notify_work(struct work_struct *work)
> +{
> +	struct vdpasim *vdpasim;
> +	u16 *val;
> +
> +	vdpasim = container_of(work, struct vdpasim, notify_work.work);
> +
> +	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
> +		goto out;
> +
> +	if (!vdpasim->running)
> +		goto out;
> +
> +	val = (u16 *)vdpasim->notify;
> +	if (xchg(val, VDPASIM_NOTIFY_DEFVAL) != VDPASIM_NOTIFY_DEFVAL)
> +		schedule_work(&vdpasim->work);
> +
> +out:
> +	schedule_delayed_work(&vdpasim->notify_work,
> +			      msecs_to_jiffies(VDPASIM_VRING_POLL_PERIOD));
> +}
> +
>   struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
>   			       const struct vdpa_dev_set_config *config)
>   {
> @@ -286,6 +310,13 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
>   	set_dma_ops(dev, &vdpasim_dma_ops);
>   	vdpasim->vdpa.mdev = dev_attr->mgmt_dev;
>   
> +	INIT_DELAYED_WORK(&vdpasim->notify_work, vdpasim_notify_work);
> +
> +	vdpasim->notify = (u16 *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
> +	if (!vdpasim->notify)
> +		goto err_iommu;
> +	WRITE_ONCE(*vdpasim->notify, VDPASIM_NOTIFY_DEFVAL);
> +
>   	vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL);
>   	if (!vdpasim->config)
>   		goto err_iommu;
> @@ -320,6 +351,13 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
>   
>   	vdpasim->vdpa.dma_dev = dev;
>   
> +	/*
> + 	 * Start periodic (every 100ms) notify work, it won't introduce
> + 	 * any overhead if the device is not started.
> +	 */
> +	schedule_delayed_work(&vdpasim->notify_work,
> +			      msecs_to_jiffies(VDPASIM_VRING_POLL_PERIOD));
> +
>   	return vdpasim;
>   
>   err_iommu:
> @@ -671,11 +709,34 @@ static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid,
>   	return 0;
>   }
>   
> +static pgprot_t vdpasim_get_vq_notification_pgprot(struct vdpa_device *vdpa,
> +						   u16 qid, pgprot_t prot)
> +{
> +	/*
> +	 * We use normal RAM pages to emulate the vq notification area, so
> +	 * just keep the pgprot as it mmaped.
> +	 */
> +	return prot;
> +}
> +
> +static struct vdpa_notification_area
> +vdpasim_get_vq_notification(struct vdpa_device *vdpa, u16 qid)
> +{
> +	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> +	struct vdpa_notification_area notify;
> +
> +	notify.addr = virt_to_phys((void *)vdpasim->notify);
> +	notify.size = PAGE_SIZE;
> +
> +	return notify;
> +}
> +
>   static void vdpasim_free(struct vdpa_device *vdpa)
>   {
>   	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
>   	int i;
>   
> +	cancel_delayed_work_sync(&vdpasim->notify_work);
>   	cancel_work_sync(&vdpasim->work);
>   
>   	for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
> @@ -694,6 +755,8 @@ static void vdpasim_free(struct vdpa_device *vdpa)
>   	kfree(vdpasim->iommu);
>   	kfree(vdpasim->vqs);
>   	kfree(vdpasim->config);
> +	if (vdpasim->notify)
> +		free_page((unsigned long)vdpasim->notify);
>   }
>   
>   static const struct vdpa_config_ops vdpasim_config_ops = {
> @@ -705,6 +768,8 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
>   	.get_vq_ready           = vdpasim_get_vq_ready,
>   	.set_vq_state           = vdpasim_set_vq_state,
>   	.get_vq_state           = vdpasim_get_vq_state,
> +	.get_vq_notification    = vdpasim_get_vq_notification,
> +	.get_vq_notification_pgprot = vdpasim_get_vq_notification_pgprot,
>   	.get_vq_align           = vdpasim_get_vq_align,
>   	.get_vq_group           = vdpasim_get_vq_group,
>   	.get_device_features    = vdpasim_get_device_features,
> @@ -738,6 +803,8 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
>   	.get_vq_ready           = vdpasim_get_vq_ready,
>   	.set_vq_state           = vdpasim_set_vq_state,
>   	.get_vq_state           = vdpasim_get_vq_state,
> +	.get_vq_notification    = vdpasim_get_vq_notification,
> +	.get_vq_notification_pgprot = vdpasim_get_vq_notification_pgprot,
>   	.get_vq_align           = vdpasim_get_vq_align,
>   	.get_vq_group           = vdpasim_get_vq_group,
>   	.get_device_features    = vdpasim_get_device_features,
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> index 0e78737dcc16..786e902de0b2 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> @@ -69,6 +69,9 @@ struct vdpasim {
>   	bool running;
>   	/* spinlock to synchronize iommu table */
>   	spinlock_t iommu_lock;
> +	/* dummy notify page */
> +	u16 *notify;
> +	struct delayed_work notify_work;
>   };
>   
>   struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,

Re: [PATCH v4 0/2] vdpasim: support doorbell mapping

[Michael S. Tsirkin]

On Mon, Feb 27, 2023 at 05:18:55PM +0800, Longpeng(Mike) wrote:
> From: Longpeng <longpeng2@huawei.com>
> 
> This patchset supports doorbell mapping for vdpasim devices.
> 
> v3: https://lore.kernel.org/lkml/20230213070446-mutt-send-email-mst@kernel.org/T/
> v2: https://lore.kernel.org/lkml/CACGkMEtdT5fG=ffbpQadkGmzHf6Ax-+L50LsriYqJaW++natMg@mail.gmail.com/T/

Pls fix issues found by 0-day and repost.

> Changes v3->v4:
>     - use WRITE_ONCE  [Jason]
>     - start/stop periodic work when create/destroy vdpasim device  [Jason]
> 
> Changes v2->v3:
>     - add a new callback named get_vq_notification_pgprot to vdpa_config_ops [Jason]
>     - remove the new added module parameter 'parameter' [Jason]
>     - opencode the schedule/cancel_delayed() [Jason]
> 
> Changes v1->v2:
>     - support both kick mode and passthrough mode. [Jason]
>     - poll the notify register first. [Jason, Michael]
> 
> Longpeng (2):
>   vdpa: support specify the pgprot of vq notification area
>   vdpasim: support doorbell mapping
> 
>  drivers/vdpa/vdpa_sim/vdpa_sim.c | 67 ++++++++++++++++++++++++++++++++
>  drivers/vdpa/vdpa_sim/vdpa_sim.h |  3 ++
>  drivers/vhost/vdpa.c             |  4 +-
>  include/linux/vdpa.h             |  9 +++++
>  4 files changed, 82 insertions(+), 1 deletion(-)
> 
> -- 
> 2.23.0