[RFC v1 0/8] vhost-vdpa: add support for iommufd

[RFC v1 0/8] vhost-vdpa: add support for iommufd

[Cindy Lu]

Hi All
This code provides the iommufd support for vdpa device
This code fixes the bugs from the last version and also add the asid support. rebase on kernel
v6,6-rc3
Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net), 
I will continue working on it

The kernel code is
https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1

Signed-off-by: Cindy Lu <lulu@redhat.com>


Cindy Lu (8):
  vhost/iommufd: Add the functions support iommufd
  Kconfig: Add the new file vhost/iommufd
  vhost: Add 3 new uapi to support iommufd
  vdpa: Add new vdpa_config_ops to support iommufd
  vdpa_sim :Add support for iommufd
  vdpa: change the map/unmap process to support iommufd
  vp_vdpa::Add support for iommufd
  iommu: expose the function iommu_device_use_default_domain

 drivers/iommu/iommu.c             |   2 +
 drivers/vdpa/vdpa_sim/vdpa_sim.c  |   8 ++
 drivers/vdpa/virtio_pci/vp_vdpa.c |   4 +
 drivers/vhost/Kconfig             |   1 +
 drivers/vhost/Makefile            |   1 +
 drivers/vhost/iommufd.c           | 178 +++++++++++++++++++++++++
 drivers/vhost/vdpa.c              | 210 +++++++++++++++++++++++++++++-
 drivers/vhost/vhost.h             |  21 +++
 include/linux/vdpa.h              |  38 +++++-
 include/uapi/linux/vhost.h        |  66 ++++++++++
 10 files changed, 525 insertions(+), 4 deletions(-)
 create mode 100644 drivers/vhost/iommufd.c

-- 
2.34.3

[RFC v1 1/8] vhost/iommufd: Add the functions support iommufd

[Cindy Lu]

Add a new file vhost/iommufd.c to support the function of
iommufd, This file contains iommufd function of emulated device and
the physical device. 

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 drivers/vhost/iommufd.c | 178 ++++++++++++++++++++++++++++++++++++++++
 drivers/vhost/vhost.h   |  21 +++++
 2 files changed, 199 insertions(+)
 create mode 100644 drivers/vhost/iommufd.c

diff --git a/drivers/vhost/iommufd.c b/drivers/vhost/iommufd.c
new file mode 100644
index 000000000000..113dda50a9b6
--- /dev/null
+++ b/drivers/vhost/iommufd.c
@@ -0,0 +1,178 @@
+#include <linux/vdpa.h>
+#include <linux/iommufd.h>
+
+#include "vhost.h"
+
+MODULE_IMPORT_NS(IOMMUFD);
+
+int vdpa_iommufd_bind(struct vdpa_device *vdpa, struct iommufd_ctx *ictx,
+		      u32 *ioas_id, u32 *device_id)
+{
+	int ret;
+
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	/*
+        * If the driver doesn't provide this op then it means the device does
+        * not do DMA at all. So nothing to do.
+        */
+	if (!vdpa->config->bind_iommufd)
+		return 0;
+	ret = vdpa->config->bind_iommufd(vdpa, ictx, device_id);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void vdpa_iommufd_unbind(struct vdpa_device *vdpa)
+{
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	if (vdpa->config->unbind_iommufd)
+		vdpa->config->unbind_iommufd(vdpa);
+}
+
+int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
+			       struct iommufd_ctx *ictx, u32 *out_device_id)
+{
+	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
+	struct iommufd_device *idev;
+
+	idev = iommufd_device_bind(ictx, dma_dev, out_device_id);
+	if (IS_ERR(idev))
+		return PTR_ERR(idev);
+	vdpa->iommufd_device = idev;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_bind);
+
+void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa)
+{
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	if (vdpa->iommufd_attached) {
+		iommufd_device_detach(vdpa->iommufd_device);
+		vdpa->iommufd_attached = false;
+	}
+	iommufd_device_unbind(vdpa->iommufd_device);
+	vdpa->iommufd_device = NULL;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_unbind);
+
+int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa,
+				      u32 *iommufd_ioasid)
+{
+	int rc;
+
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	if (WARN_ON(!vdpa->iommufd_device))
+		return -EINVAL;
+
+	if (vdpa->iommufd_attached)
+		rc = iommufd_device_replace(vdpa->iommufd_device,
+					    iommufd_ioasid);
+	else
+		rc = iommufd_device_attach(vdpa->iommufd_device,
+					   iommufd_ioasid);
+	if (rc)
+		return rc;
+	vdpa->iommufd_attached = true;
+
+	return 0;
+}
+
+EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_attach_ioas);
+int vdpa_iommufd_physical_detach_ioas(struct vdpa_device *vdpa)
+{
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	if (WARN_ON(!vdpa->iommufd_device) || !vdpa->iommufd_attached)
+		return -1;
+
+	iommufd_device_detach(vdpa->iommufd_device);
+	vdpa->iommufd_attached = false;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_detach_ioas);
+
+static void vdpa_emulated_unmap(void *data, unsigned long iova,
+				unsigned long length)
+{
+	struct vdpa_device *vdpa = data;
+	/* todo: need to unmap the iova-lenth in all ASID*/
+
+	//	vdpa->config->dma_unmap(vdpa, 0, iova, length);
+}
+
+static const struct iommufd_access_ops vdpa_user_ops = {
+	.needs_pin_pages = 1,
+	.unmap = vdpa_emulated_unmap,
+};
+
+int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
+			       struct iommufd_ctx *ictx, u32 *out_device_id)
+{
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	struct iommufd_access *user;
+
+	user = iommufd_access_create(ictx, &vdpa_user_ops, vdpa, out_device_id);
+	if (IS_ERR(user))
+		return PTR_ERR(user);
+	vdpa->iommufd_access = user;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_emulated_bind);
+
+void vdpa_iommufd_emulated_unbind(struct vdpa_device *vdpa)
+{
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	if (vdpa->iommufd_access) {
+		iommufd_access_destroy(vdpa->iommufd_access);
+		vdpa->iommufd_attached = false;
+		vdpa->iommufd_access = NULL;
+	}
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_emulated_unbind);
+
+int vdpa_iommufd_emulated_attach_ioas(struct vdpa_device *vdpa,
+				      u32 *iommufd_ioasid)
+{
+	int rc;
+
+	struct iommufd_access *user;
+
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	if (vdpa->iommufd_attached) {
+		rc = iommufd_access_replace(vdpa->iommufd_access,
+					    *iommufd_ioasid);
+	} else {
+		rc = iommufd_access_attach(vdpa->iommufd_access,
+					   *iommufd_ioasid);
+	}
+	user = vdpa->iommufd_access;
+
+	if (rc)
+		return rc;
+	vdpa->iommufd_attached = true;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_emulated_attach_ioas);
+
+int vdpa_iommufd_emulated_detach_ioas(struct vdpa_device *vdpa)
+{
+	vhost_vdpa_lockdep_assert_held(vdpa);
+
+	if (WARN_ON(!vdpa->iommufd_access) || !vdpa->iommufd_attached)
+		return -1;
+
+	iommufd_access_detach(vdpa->iommufd_access);
+	vdpa->iommufd_attached = false;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_emulated_detach_ioas);
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index f60d5f7bef94..179012e350f9 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -310,6 +310,27 @@ static inline bool vhost_is_little_endian(struct vhost_virtqueue *vq)
 }
 #endif
 
+struct iommufd_ctx;
+struct vdpa_device;
+void vhost_vdpa_lockdep_assert_held(struct vdpa_device *vdpa);
+
+#if IS_ENABLED(CONFIG_IOMMUFD)
+int vdpa_iommufd_bind(struct vdpa_device *vdpa, struct iommufd_ctx *ictx,
+		      u32 *ioas_id, u32 *device_id);
+void vdpa_iommufd_unbind(struct vdpa_device *vdpa);
+#else
+static inline int vdpa_iommufd_bind(struct vdpa_device *vdpa,
+				    struct iommufd_ctx *ictx, u32 *ioas_id,
+				    u32 *device_id)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline void vdpa_iommufd_unbind(struct vdpa_device *vdpa)
+{
+}
+#endif
+
 /* Memory accessors */
 static inline u16 vhost16_to_cpu(struct vhost_virtqueue *vq, __virtio16 val)
 {
-- 
2.34.3

[RFC v1 2/8] Kconfig: Add the new file vhost/iommufd

[Cindy Lu]

Change the makefile and Kconfig, to add the
new file vhost/iommufd.c

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 drivers/vhost/Kconfig  | 1 +
 drivers/vhost/Makefile | 1 +
 2 files changed, 2 insertions(+)

diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig
index b455d9ab6f3d..a4becfb36d77 100644
--- a/drivers/vhost/Kconfig
+++ b/drivers/vhost/Kconfig
@@ -72,6 +72,7 @@ config VHOST_VDPA
 	select VHOST
 	select IRQ_BYPASS_MANAGER
 	depends on VDPA
+	depends on IOMMUFD || !IOMMUFD
 	help
 	  This kernel module can be loaded in host kernel to accelerate
 	  guest virtio devices with the vDPA-based backends.
diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile
index f3e1897cce85..cda7f6b7f8da 100644
--- a/drivers/vhost/Makefile
+++ b/drivers/vhost/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_VHOST_RING) += vringh.o
 
 obj-$(CONFIG_VHOST_VDPA) += vhost_vdpa.o
 vhost_vdpa-y := vdpa.o
+vhost_vdpa-$(CONFIG_IOMMUFD) += iommufd.o
 
 obj-$(CONFIG_VHOST)	+= vhost.o
 
-- 
2.34.3

Re: [RFC v1 2/8] Kconfig: Add the new file vhost/iommufd

[Yi Liu]

On 2023/11/4 01:16, Cindy Lu wrote:
> Change the makefile and Kconfig, to add the
> new file vhost/iommufd.c

why not merge it with patch 1?

> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
>   drivers/vhost/Kconfig  | 1 +
>   drivers/vhost/Makefile | 1 +
>   2 files changed, 2 insertions(+)
> 
> diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig
> index b455d9ab6f3d..a4becfb36d77 100644
> --- a/drivers/vhost/Kconfig
> +++ b/drivers/vhost/Kconfig
> @@ -72,6 +72,7 @@ config VHOST_VDPA
>   	select VHOST
>   	select IRQ_BYPASS_MANAGER
>   	depends on VDPA
> +	depends on IOMMUFD || !IOMMUFD
>   	help
>   	  This kernel module can be loaded in host kernel to accelerate
>   	  guest virtio devices with the vDPA-based backends.
> diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile
> index f3e1897cce85..cda7f6b7f8da 100644
> --- a/drivers/vhost/Makefile
> +++ b/drivers/vhost/Makefile
> @@ -12,6 +12,7 @@ obj-$(CONFIG_VHOST_RING) += vringh.o
>   
>   obj-$(CONFIG_VHOST_VDPA) += vhost_vdpa.o
>   vhost_vdpa-y := vdpa.o
> +vhost_vdpa-$(CONFIG_IOMMUFD) += iommufd.o
>   
>   obj-$(CONFIG_VHOST)	+= vhost.o
>   

-- 
Regards,
Yi Liu

Re: [RFC v1 2/8] Kconfig: Add the new file vhost/iommufd

[Cindy Lu]

On Mon, Nov 6, 2023 at 4:50 PM Yi Liu <yi.l.liu@intel.com> wrote:
>
> On 2023/11/4 01:16, Cindy Lu wrote:
> > Change the makefile and Kconfig, to add the
> > new file vhost/iommufd.c
>
> why not merge it with patch 1?
>
sure will change this

thanks
cindy
> > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > ---
> >   drivers/vhost/Kconfig  | 1 +
> >   drivers/vhost/Makefile | 1 +
> >   2 files changed, 2 insertions(+)
> >
> > diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig
> > index b455d9ab6f3d..a4becfb36d77 100644
> > --- a/drivers/vhost/Kconfig
> > +++ b/drivers/vhost/Kconfig
> > @@ -72,6 +72,7 @@ config VHOST_VDPA
> >       select VHOST
> >       select IRQ_BYPASS_MANAGER
> >       depends on VDPA
> > +     depends on IOMMUFD || !IOMMUFD
> >       help
> >         This kernel module can be loaded in host kernel to accelerate
> >         guest virtio devices with the vDPA-based backends.
> > diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile
> > index f3e1897cce85..cda7f6b7f8da 100644
> > --- a/drivers/vhost/Makefile
> > +++ b/drivers/vhost/Makefile
> > @@ -12,6 +12,7 @@ obj-$(CONFIG_VHOST_RING) += vringh.o
> >
> >   obj-$(CONFIG_VHOST_VDPA) += vhost_vdpa.o
> >   vhost_vdpa-y := vdpa.o
> > +vhost_vdpa-$(CONFIG_IOMMUFD) += iommufd.o
> >
> >   obj-$(CONFIG_VHOST) += vhost.o
> >
>
> --
> Regards,
> Yi Liu
>

[RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Cindy Lu]

VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device

VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
address space specified by IOAS id.

VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
from the iommufd address space

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 drivers/vhost/vdpa.c       | 171 +++++++++++++++++++++++++++++++++++++
 include/uapi/linux/vhost.h |  66 ++++++++++++++
 2 files changed, 237 insertions(+)

diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 78379ffd2336..dfaddd833364 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -18,6 +18,7 @@
 #include <linux/mm.h>
 #include <linux/slab.h>
 #include <linux/iommu.h>
+#include <linux/iommufd.h>
 #include <linux/uuid.h>
 #include <linux/vdpa.h>
 #include <linux/nospec.h>
@@ -25,6 +26,8 @@
 
 #include "vhost.h"
 
+MODULE_IMPORT_NS(IOMMUFD);
+
 enum {
 	VHOST_VDPA_BACKEND_FEATURES =
 	(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
@@ -69,6 +72,15 @@ static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
 				   struct vhost_iotlb *iotlb, u64 start,
 				   u64 last, u32 asid);
 
+void vhost_vdpa_lockdep_assert_held(struct vdpa_device *vdpa)
+{
+	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
+
+	if (WARN_ON(!v))
+		return;
+	lockdep_assert_held(&v->vdev.mutex);
+}
+
 static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
 {
 	struct vhost_vdpa_as *as = container_of(iotlb, struct
@@ -551,6 +563,149 @@ static long vhost_vdpa_suspend(struct vhost_vdpa *v)
 
 	return ops->suspend(vdpa);
 }
+static long vhost_vdpa_iommufd_set_device(struct vhost_vdpa *v,
+					  void __user *argp)
+{
+	struct device *dma_dev = vdpa_get_dma_dev(v->vdpa);
+	struct vhost_vdpa_set_iommufd set_iommufd;
+	struct vdpa_device *vdpa = v->vdpa;
+	struct iommufd_ctx *ictx;
+	unsigned long minsz;
+	u32 ioas_id, dev_id;
+	struct fd f;
+	long r = 0;
+
+	minsz = offsetofend(struct vhost_vdpa_set_iommufd, iommufd_ioasid);
+	if (copy_from_user(&set_iommufd, argp, minsz))
+		return -EFAULT;
+
+	/* Unset IOMMUFD */
+	if (set_iommufd.iommufd < 0) {
+		if (!vdpa->iommufd_ictx || !vdpa->iommufd_device)
+			return -EINVAL;
+		if (atomic_read(&vdpa->iommufd_users)) {
+			atomic_dec(&vdpa->iommufd_users);
+			return 0;
+		}
+		vdpa_iommufd_unbind(v->vdpa);
+		vdpa->iommufd_device = NULL;
+		vdpa->iommufd_ictx = NULL;
+		return iommu_attach_device(v->domain, dma_dev);
+	}
+
+	/* For same device but different groups, ++refcount only */
+	if (vdpa->iommufd_device)
+		goto out_inc;
+
+	r = -EBADF;
+	f = fdget(set_iommufd.iommufd);
+	if (!f.file)
+		goto out;
+
+	r = -EINVAL;
+	ictx = iommufd_ctx_from_file(f.file);
+	if (IS_ERR(ictx))
+		goto out_fdput;
+
+	if (v->domain) {
+		iommu_device_unuse_default_domain(dma_dev);
+		iommu_detach_device(v->domain, dma_dev);
+	}
+
+	ioas_id = set_iommufd.iommufd_ioasid;
+	r = vdpa_iommufd_bind(vdpa, ictx, &ioas_id, &dev_id);
+	if (r)
+		goto out_reattach;
+
+	set_iommufd.out_dev_id = dev_id;
+	r = copy_to_user(argp + minsz, &set_iommufd.out_dev_id,
+			 sizeof(set_iommufd.out_dev_id)) ?
+		    -EFAULT :
+		    0;
+	if (r)
+		goto out_device_unbind;
+
+	vdpa->iommufd_ictx = ictx;
+
+out_inc:
+	atomic_inc(&vdpa->iommufd_users);
+
+	goto out_fdput;
+
+out_device_unbind:
+
+	vdpa_iommufd_unbind(vdpa);
+out_reattach:
+	iommu_device_use_default_domain(dma_dev);
+	iommu_attach_device(v->domain, dma_dev);
+	iommufd_ctx_put(ictx);
+out_fdput:
+	fdput(f);
+out:
+	return r;
+}
+int vhost_vdpa_iommufd_ioas_attach(struct vhost_vdpa *v, void __user *arg)
+{
+	struct vdpa_device_attach_iommufd_as attach;
+	unsigned long minsz;
+	int ret;
+
+	minsz = offsetofend(struct vdpa_device_attach_iommufd_as, ioas_id);
+
+	if (copy_from_user(&attach, (void __user *)arg, minsz))
+		return -EFAULT;
+
+	if (attach.argsz < minsz || attach.flags)
+		return -EINVAL;
+
+	if (!v->vdpa->config->bind_iommufd)
+		return -ENODEV;
+
+	if (!v->vdpa->iommufd_ictx) {
+		ret = -EINVAL;
+		return ret;
+	}
+
+	ret = v->vdpa->config->attach_ioas(v->vdpa, &attach.ioas_id);
+
+	if (ret)
+		return ret;
+
+	ret = copy_to_user(
+		      (void __user *)arg +
+			      offsetofend(struct vdpa_device_attach_iommufd_as,
+					  flags),
+		      &attach.ioas_id, sizeof(attach.ioas_id)) ?
+		      -EFAULT :
+		      0;
+
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int vhost_vdpa_iommufd_ioas_detach(struct vhost_vdpa *v, void __user *arg)
+{
+	struct vdpa_device_detach_iommufd_as detach;
+	unsigned long minsz;
+
+	minsz = offsetofend(struct vdpa_device_detach_iommufd_as, flags);
+
+	if (copy_from_user(&detach, (void __user *)arg, minsz))
+		return -EFAULT;
+
+	if (detach.argsz < minsz || detach.flags)
+		return -EINVAL;
+
+	if (!v->vdpa->config->bind_iommufd)
+		return -ENODEV;
+
+	if (v->vdpa->iommufd_ictx) {
+		return -EINVAL;
+	}
+	return v->vdpa->config->detach_ioas(v->vdpa);
+}
 
 /* After a successful return of this ioctl the device resumes processing
  * virtqueue descriptors. The device becomes fully operational the same way it
@@ -744,6 +899,18 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
 	case VHOST_SET_LOG_FD:
 		r = -ENOIOCTLCMD;
 		break;
+	case VHOST_VDPA_SET_IOMMU_FD:
+
+		r = vhost_vdpa_iommufd_set_device(v, argp);
+		break;
+	case VDPA_DEVICE_ATTACH_IOMMUFD_AS:
+		r = vhost_vdpa_iommufd_ioas_attach(v, (void __user *)arg);
+		break;
+
+	case VDPA_DEVICE_DETACH_IOMMUFD_AS:
+		r = vhost_vdpa_iommufd_ioas_detach(v, (void __user *)arg);
+		break;
+
 	case VHOST_VDPA_SET_CONFIG_CALL:
 		r = vhost_vdpa_set_config_call(v, argp);
 		break;
@@ -896,6 +1063,10 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 	} else if (ops->set_map) {
 		if (!v->in_batch)
 			r = ops->set_map(vdpa, asid, iotlb);
+	} else if (!vdpa->iommufd_ictx) {
+		/* Legacy iommu domain pathway without IOMMUFD */
+		r = iommu_map(v->domain, iova, pa, size,
+			      perm_to_iommu_flags(perm), GFP_KERNEL);
 	} else {
 		r = iommu_map(v->domain, iova, pa, size,
 			      perm_to_iommu_flags(perm), GFP_KERNEL);
diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
index f5c48b61ab62..07e1b2c443ca 100644
--- a/include/uapi/linux/vhost.h
+++ b/include/uapi/linux/vhost.h
@@ -219,4 +219,70 @@
  */
 #define VHOST_VDPA_RESUME		_IO(VHOST_VIRTIO, 0x7E)
 
+/* vhost_vdpa_set_iommufd
+ * Input parameters:
+ * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
+ * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
+ * Output parameters:
+ * @out_dev_id: device identifier
+ */
+struct vhost_vdpa_set_iommufd {
+	__s32 iommufd;
+	__u32 iommufd_ioasid;
+	__u32 out_dev_id;
+};
+
+#define VHOST_VDPA_SET_IOMMU_FD \
+	_IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
+
+/*
+ * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
+ * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
+ *
+ * Attach a vdpa device to an iommufd address space specified by IOAS
+ * id.
+ *
+ * Available only after a device has been bound to iommufd via
+ * VHOST_VDPA_SET_IOMMU_FD
+ *
+ * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
+ *
+ * @argsz:	user filled size of this data.
+ * @flags:	must be 0.
+ * @ioas_id:	Input the target id which can represent an ioas
+ *		allocated via iommufd subsystem.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vdpa_device_attach_iommufd_as {
+	__u32 argsz;
+	__u32 flags;
+	__u32 ioas_id;
+};
+
+#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
+	_IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
+
+/*
+ * VDPA_DEVICE_DETACH_IOMMUFD_AS
+ *
+ * Detach a vdpa device from the iommufd address space it has been
+ * attached to. After it, device should be in a blocking DMA state.
+ *
+ * Available only after a device has been bound to iommufd via
+ * VHOST_VDPA_SET_IOMMU_FD
+ *
+ * @argsz:	user filled size of this data.
+ * @flags:	must be 0.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vdpa_device_detach_iommufd_as {
+	__u32 argsz;
+	__u32 flags;
+};
+
+#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
+	_IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
+
 #endif
-- 
2.34.3

Re: [RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Jason Wang]

On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
>
> VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
>
> VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> address space specified by IOAS id.
>
> VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> from the iommufd address space
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>

As discussed in the previous version, any reason/advantages of this
compared to just having a single  VDPA_DEVICE_ATTACH_IOMMUFD_AS?

Thanks

Re: [RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Jason Wang]

On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
>
> VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
>
> VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> address space specified by IOAS id.
>
> VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> from the iommufd address space
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---

[...]

> diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> index f5c48b61ab62..07e1b2c443ca 100644
> --- a/include/uapi/linux/vhost.h
> +++ b/include/uapi/linux/vhost.h
> @@ -219,4 +219,70 @@
>   */
>  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
>
> +/* vhost_vdpa_set_iommufd
> + * Input parameters:
> + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> + * Output parameters:
> + * @out_dev_id: device identifier
> + */
> +struct vhost_vdpa_set_iommufd {
> +       __s32 iommufd;
> +       __u32 iommufd_ioasid;
> +       __u32 out_dev_id;
> +};
> +
> +#define VHOST_VDPA_SET_IOMMU_FD \
> +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> +
> +/*
> + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> + *
> + * Attach a vdpa device to an iommufd address space specified by IOAS
> + * id.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> + *
> + * @argsz:     user filled size of this data.
> + * @flags:     must be 0.
> + * @ioas_id:   Input the target id which can represent an ioas
> + *             allocated via iommufd subsystem.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_attach_iommufd_as {
> +       __u32 argsz;
> +       __u32 flags;
> +       __u32 ioas_id;
> +};

I think we need to map ioas to vDPA AS, so there should be an ASID
from the view of vDPA?

Thanks

> +
> +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> +
> +/*
> + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> + *
> + * Detach a vdpa device from the iommufd address space it has been
> + * attached to. After it, device should be in a blocking DMA state.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * @argsz:     user filled size of this data.
> + * @flags:     must be 0.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_detach_iommufd_as {
> +       __u32 argsz;
> +       __u32 flags;
> +};
> +
> +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> +
>  #endif
> --
> 2.34.3
>

Re: [RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Cindy Lu]

On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
>
> On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> >
> > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> >
> > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > address space specified by IOAS id.
> >
> > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > from the iommufd address space
> >
> > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > ---
>
> [...]
>
> > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > index f5c48b61ab62..07e1b2c443ca 100644
> > --- a/include/uapi/linux/vhost.h
> > +++ b/include/uapi/linux/vhost.h
> > @@ -219,4 +219,70 @@
> >   */
> >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> >
> > +/* vhost_vdpa_set_iommufd
> > + * Input parameters:
> > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > + * Output parameters:
> > + * @out_dev_id: device identifier
> > + */
> > +struct vhost_vdpa_set_iommufd {
> > +       __s32 iommufd;
> > +       __u32 iommufd_ioasid;
> > +       __u32 out_dev_id;
> > +};
> > +
> > +#define VHOST_VDPA_SET_IOMMU_FD \
> > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > +
> > +/*
> > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > + *
> > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > + * id.
> > + *
> > + * Available only after a device has been bound to iommufd via
> > + * VHOST_VDPA_SET_IOMMU_FD
> > + *
> > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > + *
> > + * @argsz:     user filled size of this data.
> > + * @flags:     must be 0.
> > + * @ioas_id:   Input the target id which can represent an ioas
> > + *             allocated via iommufd subsystem.
> > + *
> > + * Return: 0 on success, -errno on failure.
> > + */
> > +struct vdpa_device_attach_iommufd_as {
> > +       __u32 argsz;
> > +       __u32 flags;
> > +       __u32 ioas_id;
> > +};
>
> I think we need to map ioas to vDPA AS, so there should be an ASID
> from the view of vDPA?
>
> Thanks
>
The qemu will have a structure save and  maintain this information,So
I didn't add this
 in kernel，we can add this but maybe only for check?
this in
Thanks
Cindy
> > +
> > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > +
> > +/*
> > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > + *
> > + * Detach a vdpa device from the iommufd address space it has been
> > + * attached to. After it, device should be in a blocking DMA state.
> > + *
> > + * Available only after a device has been bound to iommufd via
> > + * VHOST_VDPA_SET_IOMMU_FD
> > + *
> > + * @argsz:     user filled size of this data.
> > + * @flags:     must be 0.
> > + *
> > + * Return: 0 on success, -errno on failure.
> > + */
> > +struct vdpa_device_detach_iommufd_as {
> > +       __u32 argsz;
> > +       __u32 flags;
> > +};
> > +
> > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > +
> >  #endif
> > --
> > 2.34.3
> >
>

Re: [RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Jason Wang]

On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
>
> On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > >
> > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > >
> > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > address space specified by IOAS id.
> > >
> > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > from the iommufd address space
> > >
> > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > ---
> >
> > [...]
> >
> > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > index f5c48b61ab62..07e1b2c443ca 100644
> > > --- a/include/uapi/linux/vhost.h
> > > +++ b/include/uapi/linux/vhost.h
> > > @@ -219,4 +219,70 @@
> > >   */
> > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > >
> > > +/* vhost_vdpa_set_iommufd
> > > + * Input parameters:
> > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > + * Output parameters:
> > > + * @out_dev_id: device identifier
> > > + */
> > > +struct vhost_vdpa_set_iommufd {
> > > +       __s32 iommufd;
> > > +       __u32 iommufd_ioasid;
> > > +       __u32 out_dev_id;
> > > +};
> > > +
> > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > +
> > > +/*
> > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > + *
> > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > + * id.
> > > + *
> > > + * Available only after a device has been bound to iommufd via
> > > + * VHOST_VDPA_SET_IOMMU_FD
> > > + *
> > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > + *
> > > + * @argsz:     user filled size of this data.
> > > + * @flags:     must be 0.
> > > + * @ioas_id:   Input the target id which can represent an ioas
> > > + *             allocated via iommufd subsystem.
> > > + *
> > > + * Return: 0 on success, -errno on failure.
> > > + */
> > > +struct vdpa_device_attach_iommufd_as {
> > > +       __u32 argsz;
> > > +       __u32 flags;
> > > +       __u32 ioas_id;
> > > +};
> >
> > I think we need to map ioas to vDPA AS, so there should be an ASID
> > from the view of vDPA?
> >
> > Thanks
> >
> The qemu will have a structure save and  maintain this information,So
> I didn't add this
>  in kernel，we can add this but maybe only for check?

I meant for example, a simulator has two AS. How can we attach an ioas
to a specific AS with the above uAPI?

Thanks

> this in
> Thanks
> Cindy
> > > +
> > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > +
> > > +/*
> > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > + *
> > > + * Detach a vdpa device from the iommufd address space it has been
> > > + * attached to. After it, device should be in a blocking DMA state.
> > > + *
> > > + * Available only after a device has been bound to iommufd via
> > > + * VHOST_VDPA_SET_IOMMU_FD
> > > + *
> > > + * @argsz:     user filled size of this data.
> > > + * @flags:     must be 0.
> > > + *
> > > + * Return: 0 on success, -errno on failure.
> > > + */
> > > +struct vdpa_device_detach_iommufd_as {
> > > +       __u32 argsz;
> > > +       __u32 flags;
> > > +};
> > > +
> > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > +
> > >  #endif
> > > --
> > > 2.34.3
> > >
> >
>

Re: [RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Cindy Lu]

On Wed, Nov 8, 2023 at 11:03 AM Jason Wang <jasowang@redhat.com> wrote:
>
> On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
> >
> > On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> > >
> > > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > > >
> > > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > > >
> > > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > > address space specified by IOAS id.
> > > >
> > > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > > from the iommufd address space
> > > >
> > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > ---
> > >
> > > [...]
> > >
> > > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > > index f5c48b61ab62..07e1b2c443ca 100644
> > > > --- a/include/uapi/linux/vhost.h
> > > > +++ b/include/uapi/linux/vhost.h
> > > > @@ -219,4 +219,70 @@
> > > >   */
> > > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > > >
> > > > +/* vhost_vdpa_set_iommufd
> > > > + * Input parameters:
> > > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > > + * Output parameters:
> > > > + * @out_dev_id: device identifier
> > > > + */
> > > > +struct vhost_vdpa_set_iommufd {
> > > > +       __s32 iommufd;
> > > > +       __u32 iommufd_ioasid;
> > > > +       __u32 out_dev_id;
> > > > +};
> > > > +
> > > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > > +
> > > > +/*
> > > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > > + *
> > > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > > + * id.
> > > > + *
> > > > + * Available only after a device has been bound to iommufd via
> > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > + *
> > > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > > + *
> > > > + * @argsz:     user filled size of this data.
> > > > + * @flags:     must be 0.
> > > > + * @ioas_id:   Input the target id which can represent an ioas
> > > > + *             allocated via iommufd subsystem.
> > > > + *
> > > > + * Return: 0 on success, -errno on failure.
> > > > + */
> > > > +struct vdpa_device_attach_iommufd_as {
> > > > +       __u32 argsz;
> > > > +       __u32 flags;
> > > > +       __u32 ioas_id;
> > > > +};
> > >
> > > I think we need to map ioas to vDPA AS, so there should be an ASID
> > > from the view of vDPA?
> > >
> > > Thanks
> > >
> > The qemu will have a structure save and  maintain this information,So
> > I didn't add this
> >  in kernel，we can add this but maybe only for check?
>
> I meant for example, a simulator has two AS. How can we attach an ioas
> to a specific AS with the above uAPI?
>
> Thank>
this   __u32 ioas_id here is alloc from the iommufd system. maybe I
need to change to new name iommuds_asid to
make this more clear
the process in qemu is

1) qemu want to use AS 0 (for example)
2) checking the existing asid. the asid 0 not used before
3 )alloc new asid from iommufd system, get new ioas_id (maybe 3 for example)
qemu will save this relation 3<-->0 in the driver.
4) setting the ioctl VDPA_DEVICE_ATTACH_IOMMUFD_AS to attach new ASID
to the kernel
5） while map the memory， qemu will use ASID 3 to map /umap
and use ASID 0 for legacy mode map/umap

So kernel here will not maintain the ioas_id from iommufd，
and this also make the code strange since there will 2 different asid
for the same AS, maybe we can save these information in the kernel
Thanks
cindy
> > Thanks
> > Cindy
> > > > +
> > > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > > +
> > > > +/*
> > > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > > + *
> > > > + * Detach a vdpa device from the iommufd address space it has been
> > > > + * attached to. After it, device should be in a blocking DMA state.
> > > > + *
> > > > + * Available only after a device has been bound to iommufd via
> > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > + *
> > > > + * @argsz:     user filled size of this data.
> > > > + * @flags:     must be 0.
> > > > + *
> > > > + * Return: 0 on success, -errno on failure.
> > > > + */
> > > > +struct vdpa_device_detach_iommufd_as {
> > > > +       __u32 argsz;
> > > > +       __u32 flags;
> > > > +};
> > > > +
> > > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > > +
> > > >  #endif
> > > > --
> > > > 2.34.3
> > > >
> > >
> >
>

Re: [RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Jason Wang]

On Wed, Nov 8, 2023 at 2:39 PM Cindy Lu <lulu@redhat.com> wrote:
>
> On Wed, Nov 8, 2023 at 11:03 AM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
> > >
> > > On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> > > >
> > > > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > > > >
> > > > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > > > >
> > > > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > > > address space specified by IOAS id.
> > > > >
> > > > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > > > from the iommufd address space
> > > > >
> > > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > > ---
> > > >
> > > > [...]
> > > >
> > > > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > > > index f5c48b61ab62..07e1b2c443ca 100644
> > > > > --- a/include/uapi/linux/vhost.h
> > > > > +++ b/include/uapi/linux/vhost.h
> > > > > @@ -219,4 +219,70 @@
> > > > >   */
> > > > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > > > >
> > > > > +/* vhost_vdpa_set_iommufd
> > > > > + * Input parameters:
> > > > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > > > + * Output parameters:
> > > > > + * @out_dev_id: device identifier
> > > > > + */
> > > > > +struct vhost_vdpa_set_iommufd {
> > > > > +       __s32 iommufd;
> > > > > +       __u32 iommufd_ioasid;
> > > > > +       __u32 out_dev_id;
> > > > > +};
> > > > > +
> > > > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > > > +
> > > > > +/*
> > > > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > > > + *
> > > > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > > > + * id.
> > > > > + *
> > > > > + * Available only after a device has been bound to iommufd via
> > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > + *
> > > > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > > > + *
> > > > > + * @argsz:     user filled size of this data.
> > > > > + * @flags:     must be 0.
> > > > > + * @ioas_id:   Input the target id which can represent an ioas
> > > > > + *             allocated via iommufd subsystem.
> > > > > + *
> > > > > + * Return: 0 on success, -errno on failure.
> > > > > + */
> > > > > +struct vdpa_device_attach_iommufd_as {
> > > > > +       __u32 argsz;
> > > > > +       __u32 flags;
> > > > > +       __u32 ioas_id;
> > > > > +};
> > > >
> > > > I think we need to map ioas to vDPA AS, so there should be an ASID
> > > > from the view of vDPA?
> > > >
> > > > Thanks
> > > >
> > > The qemu will have a structure save and  maintain this information,So
> > > I didn't add this
> > >  in kernel，we can add this but maybe only for check?
> >
> > I meant for example, a simulator has two AS. How can we attach an ioas
> > to a specific AS with the above uAPI?
> >
> > Thank>
> this   __u32 ioas_id here is alloc from the iommufd system. maybe I
> need to change to new name iommuds_asid to
> make this more clear
> the process in qemu is
>
> 1) qemu want to use AS 0 (for example)
> 2) checking the existing asid. the asid 0 not used before
> 3 )alloc new asid from iommufd system, get new ioas_id (maybe 3 for example)
> qemu will save this relation 3<-->0 in the driver.
> 4) setting the ioctl VDPA_DEVICE_ATTACH_IOMMUFD_AS to attach new ASID
> to the kernel

So if we want to map IOMMUFD AS 3 to VDPA AS 0, how can it be done?

For example I didn't see a vDPA AS parameter in the above uAPI.

vhost_vdpa_set_iommufd has iommufd_ioasid which is obviously not the vDPA AS.

And ioas_id of vdpa_device_attach_iommufd_as (as you explained above)
is not vDPA AS.

Thanks


> 5） while map the memory， qemu will use ASID 3 to map /umap
> and use ASID 0 for legacy mode map/umap
>
> So kernel here will not maintain the ioas_id from iommufd，
> and this also make the code strange since there will 2 different asid
> for the same AS, maybe we can save these information in the kernel
> Thanks
> cindy
> > > Thanks
> > > Cindy
> > > > > +
> > > > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > > > +
> > > > > +/*
> > > > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > > > + *
> > > > > + * Detach a vdpa device from the iommufd address space it has been
> > > > > + * attached to. After it, device should be in a blocking DMA state.
> > > > > + *
> > > > > + * Available only after a device has been bound to iommufd via
> > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > + *
> > > > > + * @argsz:     user filled size of this data.
> > > > > + * @flags:     must be 0.
> > > > > + *
> > > > > + * Return: 0 on success, -errno on failure.
> > > > > + */
> > > > > +struct vdpa_device_detach_iommufd_as {
> > > > > +       __u32 argsz;
> > > > > +       __u32 flags;
> > > > > +};
> > > > > +
> > > > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > > > +
> > > > >  #endif
> > > > > --
> > > > > 2.34.3
> > > > >
> > > >
> > >
> >
>

Re: [RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Jason Wang]

On Wed, Nov 8, 2023 at 3:09 PM Jason Wang <jasowang@redhat.com> wrote:
>
> On Wed, Nov 8, 2023 at 2:39 PM Cindy Lu <lulu@redhat.com> wrote:
> >
> > On Wed, Nov 8, 2023 at 11:03 AM Jason Wang <jasowang@redhat.com> wrote:
> > >
> > > On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
> > > >
> > > > On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> > > > >
> > > > > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > > > > >
> > > > > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > > > > >
> > > > > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > > > > address space specified by IOAS id.
> > > > > >
> > > > > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > > > > from the iommufd address space
> > > > > >
> > > > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > > > ---
> > > > >
> > > > > [...]
> > > > >
> > > > > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > > > > index f5c48b61ab62..07e1b2c443ca 100644
> > > > > > --- a/include/uapi/linux/vhost.h
> > > > > > +++ b/include/uapi/linux/vhost.h
> > > > > > @@ -219,4 +219,70 @@
> > > > > >   */
> > > > > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > > > > >
> > > > > > +/* vhost_vdpa_set_iommufd
> > > > > > + * Input parameters:
> > > > > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > > > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > > > > + * Output parameters:
> > > > > > + * @out_dev_id: device identifier
> > > > > > + */
> > > > > > +struct vhost_vdpa_set_iommufd {
> > > > > > +       __s32 iommufd;
> > > > > > +       __u32 iommufd_ioasid;
> > > > > > +       __u32 out_dev_id;
> > > > > > +};
> > > > > > +
> > > > > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > > > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > > > > +
> > > > > > +/*
> > > > > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > > > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > > > > + *
> > > > > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > > > > + * id.
> > > > > > + *
> > > > > > + * Available only after a device has been bound to iommufd via
> > > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > > + *
> > > > > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > > > > + *
> > > > > > + * @argsz:     user filled size of this data.
> > > > > > + * @flags:     must be 0.
> > > > > > + * @ioas_id:   Input the target id which can represent an ioas
> > > > > > + *             allocated via iommufd subsystem.
> > > > > > + *
> > > > > > + * Return: 0 on success, -errno on failure.
> > > > > > + */
> > > > > > +struct vdpa_device_attach_iommufd_as {
> > > > > > +       __u32 argsz;
> > > > > > +       __u32 flags;
> > > > > > +       __u32 ioas_id;
> > > > > > +};
> > > > >
> > > > > I think we need to map ioas to vDPA AS, so there should be an ASID
> > > > > from the view of vDPA?
> > > > >
> > > > > Thanks
> > > > >
> > > > The qemu will have a structure save and  maintain this information,So
> > > > I didn't add this
> > > >  in kernel，we can add this but maybe only for check?
> > >
> > > I meant for example, a simulator has two AS. How can we attach an ioas
> > > to a specific AS with the above uAPI?
> > >
> > > Thank>
> > this   __u32 ioas_id here is alloc from the iommufd system. maybe I
> > need to change to new name iommuds_asid to
> > make this more clear
> > the process in qemu is
> >
> > 1) qemu want to use AS 0 (for example)
> > 2) checking the existing asid. the asid 0 not used before
> > 3 )alloc new asid from iommufd system, get new ioas_id (maybe 3 for example)
> > qemu will save this relation 3<-->0 in the driver.
> > 4) setting the ioctl VDPA_DEVICE_ATTACH_IOMMUFD_AS to attach new ASID
> > to the kernel
>
> So if we want to map IOMMUFD AS 3 to VDPA AS 0, how can it be done?
>
> For example I didn't see a vDPA AS parameter in the above uAPI.
>
> vhost_vdpa_set_iommufd has iommufd_ioasid which is obviously not the vDPA AS.
>
> And ioas_id of vdpa_device_attach_iommufd_as (as you explained above)
> is not vDPA AS.

For example, the simulator/mlx5e has two ASes. It needs to know the
mapping between vDPA AS and iommufd AS. Otherwise the translation will
be problematic.

Thanks

>
> Thanks
>
>
> > 5） while map the memory， qemu will use ASID 3 to map /umap
> > and use ASID 0 for legacy mode map/umap
> >
> > So kernel here will not maintain the ioas_id from iommufd，
> > and this also make the code strange since there will 2 different asid
> > for the same AS, maybe we can save these information in the kernel
> > Thanks
> > cindy
> > > > Thanks
> > > > Cindy
> > > > > > +
> > > > > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > > > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > > > > +
> > > > > > +/*
> > > > > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > > > > + *
> > > > > > + * Detach a vdpa device from the iommufd address space it has been
> > > > > > + * attached to. After it, device should be in a blocking DMA state.
> > > > > > + *
> > > > > > + * Available only after a device has been bound to iommufd via
> > > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > > + *
> > > > > > + * @argsz:     user filled size of this data.
> > > > > > + * @flags:     must be 0.
> > > > > > + *
> > > > > > + * Return: 0 on success, -errno on failure.
> > > > > > + */
> > > > > > +struct vdpa_device_detach_iommufd_as {
> > > > > > +       __u32 argsz;
> > > > > > +       __u32 flags;
> > > > > > +};
> > > > > > +
> > > > > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > > > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > > > > +
> > > > > >  #endif
> > > > > > --
> > > > > > 2.34.3
> > > > > >
> > > > >
> > > >
> > >
> >

Re: [RFC v1 3/8] vhost: Add 3 new uapi to support iommufd

[Cindy Lu]

Jason Wang <jasowang@redhat.com> 于2023年11月10日周五 10:32写道：
>
> On Wed, Nov 8, 2023 at 3:09 PM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Wed, Nov 8, 2023 at 2:39 PM Cindy Lu <lulu@redhat.com> wrote:
> > >
> > > On Wed, Nov 8, 2023 at 11:03 AM Jason Wang <jasowang@redhat.com> wrote:
> > > >
> > > > On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
> > > > >
> > > > > On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> > > > > >
> > > > > > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > > > > > >
> > > > > > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > > > > > >
> > > > > > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > > > > > address space specified by IOAS id.
> > > > > > >
> > > > > > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > > > > > from the iommufd address space
> > > > > > >
> > > > > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > > > > ---
> > > > > >
> > > > > > [...]
> > > > > >
> > > > > > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > > > > > index f5c48b61ab62..07e1b2c443ca 100644
> > > > > > > --- a/include/uapi/linux/vhost.h
> > > > > > > +++ b/include/uapi/linux/vhost.h
> > > > > > > @@ -219,4 +219,70 @@
> > > > > > >   */
> > > > > > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > > > > > >
> > > > > > > +/* vhost_vdpa_set_iommufd
> > > > > > > + * Input parameters:
> > > > > > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > > > > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > > > > > + * Output parameters:
> > > > > > > + * @out_dev_id: device identifier
> > > > > > > + */
> > > > > > > +struct vhost_vdpa_set_iommufd {
> > > > > > > +       __s32 iommufd;
> > > > > > > +       __u32 iommufd_ioasid;
> > > > > > > +       __u32 out_dev_id;
> > > > > > > +};
> > > > > > > +
> > > > > > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > > > > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > > > > > +
> > > > > > > +/*
> > > > > > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > > > > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > > > > > + *
> > > > > > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > > > > > + * id.
> > > > > > > + *
> > > > > > > + * Available only after a device has been bound to iommufd via
> > > > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > > > + *
> > > > > > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > > > > > + *
> > > > > > > + * @argsz:     user filled size of this data.
> > > > > > > + * @flags:     must be 0.
> > > > > > > + * @ioas_id:   Input the target id which can represent an ioas
> > > > > > > + *             allocated via iommufd subsystem.
> > > > > > > + *
> > > > > > > + * Return: 0 on success, -errno on failure.
> > > > > > > + */
> > > > > > > +struct vdpa_device_attach_iommufd_as {
> > > > > > > +       __u32 argsz;
> > > > > > > +       __u32 flags;
> > > > > > > +       __u32 ioas_id;
> > > > > > > +};
> > > > > >
> > > > > > I think we need to map ioas to vDPA AS, so there should be an ASID
> > > > > > from the view of vDPA?
> > > > > >
> > > > > > Thanks
> > > > > >
> > > > > The qemu will have a structure save and  maintain this information,So
> > > > > I didn't add this
> > > > >  in kernel，we can add this but maybe only for check?
> > > >
> > > > I meant for example, a simulator has two AS. How can we attach an ioas
> > > > to a specific AS with the above uAPI?
> > > >
> > > > Thank>
> > > this   __u32 ioas_id here is alloc from the iommufd system. maybe I
> > > need to change to new name iommuds_asid to
> > > make this more clear
> > > the process in qemu is
> > >
> > > 1) qemu want to use AS 0 (for example)
> > > 2) checking the existing asid. the asid 0 not used before
> > > 3 )alloc new asid from iommufd system, get new ioas_id (maybe 3 for example)
> > > qemu will save this relation 3<-->0 in the driver.
> > > 4) setting the ioctl VDPA_DEVICE_ATTACH_IOMMUFD_AS to attach new ASID
> > > to the kernel
> >
> > So if we want to map IOMMUFD AS 3 to VDPA AS 0, how can it be done?
> >
> > For example I didn't see a vDPA AS parameter in the above uAPI.
> >
> > vhost_vdpa_set_iommufd has iommufd_ioasid which is obviously not the vDPA AS.
> >
> > And ioas_id of vdpa_device_attach_iommufd_as (as you explained above)
> > is not vDPA AS.
>
> For example, the simulator/mlx5e has two ASes. It needs to know the
> mapping between vDPA AS and iommufd AS. Otherwise the translation will
> be problematic.
>
> Thanks
>
Got it, thanks Jason. I will re-write this part
Thanks
Cindy
> >
> > Thanks
> >
> >
> > > 5） while map the memory， qemu will use ASID 3 to map /umap
> > > and use ASID 0 for legacy mode map/umap
> > >
> > > So kernel here will not maintain the ioas_id from iommufd，
> > > and this also make the code strange since there will 2 different asid
> > > for the same AS, maybe we can save these information in the kernel
> > > Thanks
> > > cindy
> > > > > Thanks
> > > > > Cindy
> > > > > > > +
> > > > > > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > > > > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > > > > > +
> > > > > > > +/*
> > > > > > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > > > > > + *
> > > > > > > + * Detach a vdpa device from the iommufd address space it has been
> > > > > > > + * attached to. After it, device should be in a blocking DMA state.
> > > > > > > + *
> > > > > > > + * Available only after a device has been bound to iommufd via
> > > > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > > > + *
> > > > > > > + * @argsz:     user filled size of this data.
> > > > > > > + * @flags:     must be 0.
> > > > > > > + *
> > > > > > > + * Return: 0 on success, -errno on failure.
> > > > > > > + */
> > > > > > > +struct vdpa_device_detach_iommufd_as {
> > > > > > > +       __u32 argsz;
> > > > > > > +       __u32 flags;
> > > > > > > +};
> > > > > > > +
> > > > > > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > > > > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > > > > > +
> > > > > > >  #endif
> > > > > > > --
> > > > > > > 2.34.3
> > > > > > >
> > > > > >
> > > > >
> > > >
> > >
>

[RFC v1 4/8] vdpa: Add new vdpa_config_ops to support iommufd

[Cindy Lu]

Add 4 new vdpa_config_ops function to support iommufd

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 include/linux/vdpa.h | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
index 0e652026b776..233d80f9d910 100644
--- a/include/linux/vdpa.h
+++ b/include/linux/vdpa.h
@@ -5,6 +5,7 @@
 #include <linux/kernel.h>
 #include <linux/device.h>
 #include <linux/interrupt.h>
+#include <linux/iommufd.h>
 #include <linux/vhost_iotlb.h>
 #include <linux/virtio_net.h>
 #include <linux/if_ether.h>
@@ -97,6 +98,12 @@ struct vdpa_device {
 	struct vdpa_mgmt_dev *mdev;
 	unsigned int ngroups;
 	unsigned int nas;
+	struct iommufd_access *iommufd_access;
+	struct iommufd_device *iommufd_device;
+	struct iommufd_ctx *iommufd_ictx;
+	unsigned long *vq_bitmap;
+	atomic_t iommufd_users;
+	bool iommufd_attached;
 };
 
 /**
@@ -332,6 +339,17 @@ struct vdpa_map_file {
  *				@vdev: vdpa device
  * @free:			Free resources that belongs to vDPA (optional)
  *				@vdev: vdpa device
+ * @bind_iommufd:              use vdpa_iommufd_physical_bind for an IOMMU
+ *                             backed device.
+ *                             otherwise use vdpa_iommufd_emulated_bind
+ * @unbind_iommufd:            use vdpa_iommufd_physical_unbind for an IOMMU
+ *                             backed device.
+ *                             otherwise, use vdpa_iommufd_emulated_unbind
+ * @attach_ioas:               use vdpa_iommufd_physical_attach_ioas for an
+ *                             IOMMU backed device.
+ * @detach_ioas:               Opposite of attach_ioas
+ * @free:			Free resources that belongs to vDPA (optional)
+ *				@vdev: vdpa device
  */
 struct vdpa_config_ops {
 	/* Virtqueue ops */
@@ -402,6 +420,13 @@ struct vdpa_config_ops {
 
 	/* Free device resources */
 	void (*free)(struct vdpa_device *vdev);
+	/* IOMMUFD ops */
+	int (*bind_iommufd)(struct vdpa_device *vdev, struct iommufd_ctx *ictx,
+			    u32 *out_device_id);
+	void (*unbind_iommufd)(struct vdpa_device *vdev);
+	int (*attach_ioas)(struct vdpa_device *vdev, u32 *pt_id);
+	int (*detach_ioas)(struct vdpa_device *vdev);
+
 };
 
 struct vdpa_device *__vdpa_alloc_device(struct device *parent,
@@ -570,4 +595,15 @@ struct vdpa_mgmt_dev {
 int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev);
 void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev);
 
-#endif /* _LINUX_VDPA_H */
+int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
+			       struct iommufd_ctx *ictx, u32 *out_device_id);
+void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa);
+int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
+int vdpa_iommufd_physical_detach_ioas(struct vdpa_device *vdpa);
+int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
+			       struct iommufd_ctx *ictx, u32 *out_device_id);
+void vdpa_iommufd_emulated_unbind(struct vdpa_device *vdpa);
+int vdpa_iommufd_emulated_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
+int vdpa_iommufd_emulated_detach_ioas(struct vdpa_device *vdpa);
+
+#endif
-- 
2.34.3

Re: [RFC v1 4/8] vdpa: Add new vdpa_config_ops to support iommufd

[Yi Liu]

On 2023/11/4 01:16, Cindy Lu wrote:
> Add 4 new vdpa_config_ops function to support iommufd
> 
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
>   include/linux/vdpa.h | 38 +++++++++++++++++++++++++++++++++++++-
>   1 file changed, 37 insertions(+), 1 deletion(-)
> 
> diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
> index 0e652026b776..233d80f9d910 100644
> --- a/include/linux/vdpa.h
> +++ b/include/linux/vdpa.h
> @@ -5,6 +5,7 @@
>   #include <linux/kernel.h>
>   #include <linux/device.h>
>   #include <linux/interrupt.h>
> +#include <linux/iommufd.h>
>   #include <linux/vhost_iotlb.h>
>   #include <linux/virtio_net.h>
>   #include <linux/if_ether.h>
> @@ -97,6 +98,12 @@ struct vdpa_device {
>   	struct vdpa_mgmt_dev *mdev;
>   	unsigned int ngroups;
>   	unsigned int nas;
> +	struct iommufd_access *iommufd_access;
> +	struct iommufd_device *iommufd_device;
> +	struct iommufd_ctx *iommufd_ictx;
> +	unsigned long *vq_bitmap;
> +	atomic_t iommufd_users;
> +	bool iommufd_attached;
>   };
>   
>   /**
> @@ -332,6 +339,17 @@ struct vdpa_map_file {
>    *				@vdev: vdpa device
>    * @free:			Free resources that belongs to vDPA (optional)
>    *				@vdev: vdpa device
> + * @bind_iommufd:              use vdpa_iommufd_physical_bind for an IOMMU
> + *                             backed device.
> + *                             otherwise use vdpa_iommufd_emulated_bind
> + * @unbind_iommufd:            use vdpa_iommufd_physical_unbind for an IOMMU
> + *                             backed device.
> + *                             otherwise, use vdpa_iommufd_emulated_unbind
> + * @attach_ioas:               use vdpa_iommufd_physical_attach_ioas for an
> + *                             IOMMU backed device.
> + * @detach_ioas:               Opposite of attach_ioas
> + * @free:			Free resources that belongs to vDPA (optional)
> + *				@vdev: vdpa device

duplicated kdoc.

>    */
>   struct vdpa_config_ops {
>   	/* Virtqueue ops */
> @@ -402,6 +420,13 @@ struct vdpa_config_ops {
>   
>   	/* Free device resources */
>   	void (*free)(struct vdpa_device *vdev);
> +	/* IOMMUFD ops */
> +	int (*bind_iommufd)(struct vdpa_device *vdev, struct iommufd_ctx *ictx,
> +			    u32 *out_device_id);
> +	void (*unbind_iommufd)(struct vdpa_device *vdev);
> +	int (*attach_ioas)(struct vdpa_device *vdev, u32 *pt_id);
> +	int (*detach_ioas)(struct vdpa_device *vdev);
> +
>   };
>   
>   struct vdpa_device *__vdpa_alloc_device(struct device *parent,
> @@ -570,4 +595,15 @@ struct vdpa_mgmt_dev {
>   int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev);
>   void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev);
>   
> -#endif /* _LINUX_VDPA_H */
> +int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
> +			       struct iommufd_ctx *ictx, u32 *out_device_id);
> +void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa);
> +int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
> +int vdpa_iommufd_physical_detach_ioas(struct vdpa_device *vdpa);
> +int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
> +			       struct iommufd_ctx *ictx, u32 *out_device_id);
> +void vdpa_iommufd_emulated_unbind(struct vdpa_device *vdpa);
> +int vdpa_iommufd_emulated_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
> +int vdpa_iommufd_emulated_detach_ioas(struct vdpa_device *vdpa);
> +
> +#endif

-- 
Regards,
Yi Liu

[RFC v1 5/8] vdpa_sim :Add support for iommufd

[Cindy Lu]

Add new vdpa_config_ops function to support iommufd

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index 76d41058add9..9400ec32ec41 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -762,6 +762,10 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
 	.bind_mm		= vdpasim_bind_mm,
 	.unbind_mm		= vdpasim_unbind_mm,
 	.free                   = vdpasim_free,
+	.bind_iommufd           = vdpa_iommufd_emulated_bind,
+	.unbind_iommufd         = vdpa_iommufd_emulated_unbind,
+	.attach_ioas            = vdpa_iommufd_emulated_attach_ioas,
+	.detach_ioas            = vdpa_iommufd_emulated_detach_ioas,
 };
 
 static const struct vdpa_config_ops vdpasim_batch_config_ops = {
@@ -799,6 +803,10 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
 	.bind_mm		= vdpasim_bind_mm,
 	.unbind_mm		= vdpasim_unbind_mm,
 	.free                   = vdpasim_free,
+	.bind_iommufd        = vdpa_iommufd_emulated_bind,
+	.unbind_iommufd        = vdpa_iommufd_emulated_unbind,
+	.attach_ioas           = vdpa_iommufd_emulated_attach_ioas,
+	.detach_ioas           = vdpa_iommufd_emulated_detach_ioas,
 };
 
 MODULE_VERSION(DRV_VERSION);
-- 
2.34.3

[RFC v1 6/8] vdpa: change the map/unmap process to support iommufd

[Cindy Lu]

Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
then will use the Legacy iommu domain pathway

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 drivers/vhost/vdpa.c | 43 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 38 insertions(+), 5 deletions(-)

diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index dfaddd833364..0e2dba59e1ce 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -1067,9 +1067,6 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 		/* Legacy iommu domain pathway without IOMMUFD */
 		r = iommu_map(v->domain, iova, pa, size,
 			      perm_to_iommu_flags(perm), GFP_KERNEL);
-	} else {
-		r = iommu_map(v->domain, iova, pa, size,
-			      perm_to_iommu_flags(perm), GFP_KERNEL);
 	}
 	if (r) {
 		vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
@@ -1095,8 +1092,10 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
 	if (ops->set_map) {
 		if (!v->in_batch)
 			ops->set_map(vdpa, asid, iotlb);
+	} else if (!vdpa->iommufd_ictx) {
+		/* Legacy iommu domain pathway without IOMMUFD */
+		iommu_unmap(v->domain, iova, size);
 	}
-
 }
 
 static int vhost_vdpa_va_map(struct vhost_vdpa *v,
@@ -1149,7 +1148,36 @@ static int vhost_vdpa_va_map(struct vhost_vdpa *v,
 
 	return ret;
 }
+#if 0
+int vhost_pin_pages(struct vdpa_device *device, dma_addr_t iova, int npage,
+		    int prot, struct page **pages)
+{
+	if (!pages || !npage)
+		return -EINVAL;
+	//if (!device->config->dma_unmap)
+	//return -EINVAL;
+
+	if (0) { //device->iommufd_access) {
+		int ret;
+
+		if (iova > ULONG_MAX)
+			return -EINVAL;
 
+		ret = iommufd_access_pin_pages(
+			device->iommufd_access, iova, npage * PAGE_SIZE, pages,
+			(prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
+		if (ret) {
+
+			return ret;
+		}
+
+		return npage;
+	} else {
+		return pin_user_pages(iova, npage, prot, pages);
+	}
+	return -EINVAL;
+}
+#endif
 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
 			     struct vhost_iotlb *iotlb,
 			     u64 iova, u64 size, u64 uaddr, u32 perm)
@@ -1418,9 +1446,13 @@ static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
 
 	if (v->domain) {
-		iommu_detach_device(v->domain, dma_dev);
+		if (!vdpa->iommufd_ictx) {
+			iommu_detach_device(v->domain, dma_dev);
+		}
 		iommu_domain_free(v->domain);
 	}
+	if (vdpa->iommufd_ictx)
+		vdpa_iommufd_unbind(vdpa);
 
 	v->domain = NULL;
 }
@@ -1645,6 +1677,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa)
 	}
 
 	atomic_set(&v->opened, 0);
+	atomic_set(&vdpa->iommufd_users, 0);
 	v->minor = minor;
 	v->vdpa = vdpa;
 	v->nvqs = vdpa->nvqs;
-- 
2.34.3

Re: [RFC v1 6/8] vdpa: change the map/unmap process to support iommufd

[Yi Liu]

On 2023/11/4 01:16, Cindy Lu wrote:
> Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
> then will use the Legacy iommu domain pathway
> 
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
>   drivers/vhost/vdpa.c | 43 ++++++++++++++++++++++++++++++++++++++-----
>   1 file changed, 38 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
> index dfaddd833364..0e2dba59e1ce 100644
> --- a/drivers/vhost/vdpa.c
> +++ b/drivers/vhost/vdpa.c
> @@ -1067,9 +1067,6 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
>   		/* Legacy iommu domain pathway without IOMMUFD */
>   		r = iommu_map(v->domain, iova, pa, size,
>   			      perm_to_iommu_flags(perm), GFP_KERNEL);
> -	} else {
> -		r = iommu_map(v->domain, iova, pa, size,
> -			      perm_to_iommu_flags(perm), GFP_KERNEL);
>   	}
>   	if (r) {
>   		vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
> @@ -1095,8 +1092,10 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
>   	if (ops->set_map) {
>   		if (!v->in_batch)
>   			ops->set_map(vdpa, asid, iotlb);
> +	} else if (!vdpa->iommufd_ictx) {
> +		/* Legacy iommu domain pathway without IOMMUFD */
> +		iommu_unmap(v->domain, iova, size);
>   	}
> -
>   }
>   
>   static int vhost_vdpa_va_map(struct vhost_vdpa *v,
> @@ -1149,7 +1148,36 @@ static int vhost_vdpa_va_map(struct vhost_vdpa *v,
>   
>   	return ret;
>   }
> +#if 0
> +int vhost_pin_pages(struct vdpa_device *device, dma_addr_t iova, int npage,
> +		    int prot, struct page **pages)
> +{
> +	if (!pages || !npage)
> +		return -EINVAL;
> +	//if (!device->config->dma_unmap)
> +	//return -EINVAL;
> +
> +	if (0) { //device->iommufd_access) {
> +		int ret;
> +
> +		if (iova > ULONG_MAX)
> +			return -EINVAL;
>   
> +		ret = iommufd_access_pin_pages(
> +			device->iommufd_access, iova, npage * PAGE_SIZE, pages,
> +			(prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
> +		if (ret) {
> +
> +			return ret;
> +		}
> +
> +		return npage;
> +	} else {
> +		return pin_user_pages(iova, npage, prot, pages);
> +	}
> +	return -EINVAL;
> +}
> +#endif

Is above code needed or not?

>   static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
>   			     struct vhost_iotlb *iotlb,
>   			     u64 iova, u64 size, u64 uaddr, u32 perm)
> @@ -1418,9 +1446,13 @@ static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
>   	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
>   
>   	if (v->domain) {
> -		iommu_detach_device(v->domain, dma_dev);
> +		if (!vdpa->iommufd_ictx) {
> +			iommu_detach_device(v->domain, dma_dev);
> +		}
>   		iommu_domain_free(v->domain);
>   	}
> +	if (vdpa->iommufd_ictx)
> +		vdpa_iommufd_unbind(vdpa);
>   
>   	v->domain = NULL;
>   }
> @@ -1645,6 +1677,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa)
>   	}
>   
>   	atomic_set(&v->opened, 0);
> +	atomic_set(&vdpa->iommufd_users, 0);
>   	v->minor = minor;
>   	v->vdpa = vdpa;
>   	v->nvqs = vdpa->nvqs;

-- 
Regards,
Yi Liu

Re: [RFC v1 6/8] vdpa: change the map/unmap process to support iommufd

[Cindy Lu]

On Mon, Nov 6, 2023 at 4:52 PM Yi Liu <yi.l.liu@intel.com> wrote:
>
> On 2023/11/4 01:16, Cindy Lu wrote:
> > Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
> > then will use the Legacy iommu domain pathway
> >
> > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > ---
> >   drivers/vhost/vdpa.c | 43 ++++++++++++++++++++++++++++++++++++++-----
> >   1 file changed, 38 insertions(+), 5 deletions(-)
> >
> > diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
> > index dfaddd833364..0e2dba59e1ce 100644
> > --- a/drivers/vhost/vdpa.c
> > +++ b/drivers/vhost/vdpa.c
> > @@ -1067,9 +1067,6 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
> >               /* Legacy iommu domain pathway without IOMMUFD */
> >               r = iommu_map(v->domain, iova, pa, size,
> >                             perm_to_iommu_flags(perm), GFP_KERNEL);
> > -     } else {
> > -             r = iommu_map(v->domain, iova, pa, size,
> > -                           perm_to_iommu_flags(perm), GFP_KERNEL);
> >       }
> >       if (r) {
> >               vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
> > @@ -1095,8 +1092,10 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
> >       if (ops->set_map) {
> >               if (!v->in_batch)
> >                       ops->set_map(vdpa, asid, iotlb);
> > +     } else if (!vdpa->iommufd_ictx) {
> > +             /* Legacy iommu domain pathway without IOMMUFD */
> > +             iommu_unmap(v->domain, iova, size);
> >       }
> > -
> >   }
> >
> >   static int vhost_vdpa_va_map(struct vhost_vdpa *v,
> > @@ -1149,7 +1148,36 @@ static int vhost_vdpa_va_map(struct vhost_vdpa *v,
> >
> >       return ret;
> >   }
> > +#if 0
> > +int vhost_pin_pages(struct vdpa_device *device, dma_addr_t iova, int npage,
> > +                 int prot, struct page **pages)
> > +{
> > +     if (!pages || !npage)
> > +             return -EINVAL;
> > +     //if (!device->config->dma_unmap)
> > +     //return -EINVAL;
> > +
> > +     if (0) { //device->iommufd_access) {
> > +             int ret;
> > +
> > +             if (iova > ULONG_MAX)
> > +                     return -EINVAL;
> >
> > +             ret = iommufd_access_pin_pages(
> > +                     device->iommufd_access, iova, npage * PAGE_SIZE, pages,
> > +                     (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
> > +             if (ret) {
> > +
> > +                     return ret;
> > +             }
> > +
> > +             return npage;
> > +     } else {
> > +             return pin_user_pages(iova, npage, prot, pages);
> > +     }
> > +     return -EINVAL;
> > +}
> > +#endif
>
> Is above code needed or not?
this code is for simulator， and this device still has some bugs I will
continue working in it,
Thanks
cindy
>
> >   static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
> >                            struct vhost_iotlb *iotlb,
> >                            u64 iova, u64 size, u64 uaddr, u32 perm)
> > @@ -1418,9 +1446,13 @@ static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
> >       struct device *dma_dev = vdpa_get_dma_dev(vdpa);
> >
> >       if (v->domain) {
> > -             iommu_detach_device(v->domain, dma_dev);
> > +             if (!vdpa->iommufd_ictx) {
> > +                     iommu_detach_device(v->domain, dma_dev);
> > +             }
> >               iommu_domain_free(v->domain);
> >       }
> > +     if (vdpa->iommufd_ictx)
> > +             vdpa_iommufd_unbind(vdpa);
> >
> >       v->domain = NULL;
> >   }
> > @@ -1645,6 +1677,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa)
> >       }
> >
> >       atomic_set(&v->opened, 0);
> > +     atomic_set(&vdpa->iommufd_users, 0);
> >       v->minor = minor;
> >       v->vdpa = vdpa;
> >       v->nvqs = vdpa->nvqs;
>
> --
> Regards,
> Yi Liu
>

[RFC v1 7/8] vp_vdpa::Add support for iommufd

[Cindy Lu]

Add new vdpa_config_ops function to support iommufd

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 drivers/vdpa/virtio_pci/vp_vdpa.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
index 281287fae89f..dd2c372d36a6 100644
--- a/drivers/vdpa/virtio_pci/vp_vdpa.c
+++ b/drivers/vdpa/virtio_pci/vp_vdpa.c
@@ -460,6 +460,10 @@ static const struct vdpa_config_ops vp_vdpa_ops = {
 	.set_config	= vp_vdpa_set_config,
 	.set_config_cb  = vp_vdpa_set_config_cb,
 	.get_vq_irq	= vp_vdpa_get_vq_irq,
+	.bind_iommufd = vdpa_iommufd_physical_bind,
+	.unbind_iommufd = vdpa_iommufd_physical_unbind,
+	.attach_ioas = vdpa_iommufd_physical_attach_ioas,
+	.detach_ioas = vdpa_iommufd_physical_detach_ioas,
 };
 
 static void vp_vdpa_free_irq_vectors(void *data)
-- 
2.34.3

Re: [RFC v1 7/8] vp_vdpa::Add support for iommufd

[Jason Wang]

On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
>
> Add new vdpa_config_ops function to support iommufd
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
>  drivers/vdpa/virtio_pci/vp_vdpa.c | 4 ++++
>  1 file changed, 4 insertions(+)
>
> diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
> index 281287fae89f..dd2c372d36a6 100644
> --- a/drivers/vdpa/virtio_pci/vp_vdpa.c
> +++ b/drivers/vdpa/virtio_pci/vp_vdpa.c
> @@ -460,6 +460,10 @@ static const struct vdpa_config_ops vp_vdpa_ops = {
>         .set_config     = vp_vdpa_set_config,
>         .set_config_cb  = vp_vdpa_set_config_cb,
>         .get_vq_irq     = vp_vdpa_get_vq_irq,
> +       .bind_iommufd = vdpa_iommufd_physical_bind,
> +       .unbind_iommufd = vdpa_iommufd_physical_unbind,
> +       .attach_ioas = vdpa_iommufd_physical_attach_ioas,
> +       .detach_ioas = vdpa_iommufd_physical_detach_ioas,

For the device that depends on the platform IOMMU, any reason we still
bother a per device config ops here just as an indirection?

Thanks

>  };
>
>  static void vp_vdpa_free_irq_vectors(void *data)
> --
> 2.34.3
>

[RFC v1 8/8] iommu: expose the function iommu_device_use_default_domain

[Cindy Lu]

Expose the function iommu_device_use_default_domain() and
iommu_device_unuse_default_domain()，
While vdpa bind the iommufd device and detach the iommu device,
vdpa need to call the function
iommu_device_unuse_default_domain() to release the owner

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 drivers/iommu/iommu.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 3bfc56df4f78..987cbf8c9a87 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -3164,6 +3164,7 @@ int iommu_device_use_default_domain(struct device *dev)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(iommu_device_use_default_domain);
 
 /**
  * iommu_device_unuse_default_domain() - Device driver stops handling device
@@ -3187,6 +3188,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
 	mutex_unlock(&group->mutex);
 	iommu_group_put(group);
 }
+EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain);
 
 static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
 {
-- 
2.34.3

Re: [RFC v1 8/8] iommu: expose the function iommu_device_use_default_domain

[Jason Gunthorpe]

On Sat, Nov 04, 2023 at 01:16:41AM +0800, Cindy Lu wrote:
> Expose the function iommu_device_use_default_domain() and
> iommu_device_unuse_default_domain()，
> While vdpa bind the iommufd device and detach the iommu device,
> vdpa need to call the function
> iommu_device_unuse_default_domain() to release the owner

Definately not. You need to set the driver_managed_dma flag.

Jason

Re: [RFC v1 8/8] iommu: expose the function iommu_device_use_default_domain

[Jason Wang]

On Sat, Nov 4, 2023 at 1:18 AM Cindy Lu <lulu@redhat.com> wrote:
>
> Expose the function iommu_device_use_default_domain() and
> iommu_device_unuse_default_domain()，
> While vdpa bind the iommufd device and detach the iommu device,
> vdpa need to call the function
> iommu_device_unuse_default_domain() to release the owner
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>

This is the end of the series, who is the user then?

Thanks

> ---
>  drivers/iommu/iommu.c | 2 ++
>  1 file changed, 2 insertions(+)
>
> diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> index 3bfc56df4f78..987cbf8c9a87 100644
> --- a/drivers/iommu/iommu.c
> +++ b/drivers/iommu/iommu.c
> @@ -3164,6 +3164,7 @@ int iommu_device_use_default_domain(struct device *dev)
>
>         return ret;
>  }
> +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain);
>
>  /**
>   * iommu_device_unuse_default_domain() - Device driver stops handling device
> @@ -3187,6 +3188,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
>         mutex_unlock(&group->mutex);
>         iommu_group_put(group);
>  }
> +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain);
>
>  static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
>  {
> --
> 2.34.3
>

Re: [RFC v1 8/8] iommu: expose the function iommu_device_use_default_domain

[Cindy Lu]

On Mon, Nov 6, 2023 at 3:26 PM Jason Wang <jasowang@redhat.com> wrote:
>
> On Sat, Nov 4, 2023 at 1:18 AM Cindy Lu <lulu@redhat.com> wrote:
> >
> > Expose the function iommu_device_use_default_domain() and
> > iommu_device_unuse_default_domain()，
> > While vdpa bind the iommufd device and detach the iommu device,
> > vdpa need to call the function
> > iommu_device_unuse_default_domain() to release the owner
> >
> > Signed-off-by: Cindy Lu <lulu@redhat.com>
>
> This is the end of the series, who is the user then?
>
> Thanks
>
hi Jason
These 2 functions was called in vhost_vdpa_iommufd_set_device(), Vdpa need to
release the dma owner, otherwise, the function will fail when
iommufd called iommu_device_claim_dma_owner() in iommufd_device_bind()
I will change this sequence, Or maybe will find some other way to fix
this problem
thanks
cindy


> > ---
> >  drivers/iommu/iommu.c | 2 ++
> >  1 file changed, 2 insertions(+)
> >
> > diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> > index 3bfc56df4f78..987cbf8c9a87 100644
> > --- a/drivers/iommu/iommu.c
> > +++ b/drivers/iommu/iommu.c
> > @@ -3164,6 +3164,7 @@ int iommu_device_use_default_domain(struct device *dev)
> >
> >         return ret;
> >  }
> > +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain);
> >
> >  /**
> >   * iommu_device_unuse_default_domain() - Device driver stops handling device
> > @@ -3187,6 +3188,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
> >         mutex_unlock(&group->mutex);
> >         iommu_group_put(group);
> >  }
> > +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain);
> >
> >  static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
> >  {
> > --
> > 2.34.3
> >
>

Re: [RFC v1 8/8] iommu: expose the function iommu_device_use_default_domain

[Jason Wang]

On Tue, Nov 7, 2023 at 2:10 PM Cindy Lu <lulu@redhat.com> wrote:
>
> On Mon, Nov 6, 2023 at 3:26 PM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Sat, Nov 4, 2023 at 1:18 AM Cindy Lu <lulu@redhat.com> wrote:
> > >
> > > Expose the function iommu_device_use_default_domain() and
> > > iommu_device_unuse_default_domain()，
> > > While vdpa bind the iommufd device and detach the iommu device,
> > > vdpa need to call the function
> > > iommu_device_unuse_default_domain() to release the owner
> > >
> > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> >
> > This is the end of the series, who is the user then?
> >
> > Thanks
> >
> hi Jason
> These 2 functions was called in vhost_vdpa_iommufd_set_device(), Vdpa need to
> release the dma owner, otherwise, the function will fail when
> iommufd called iommu_device_claim_dma_owner() in iommufd_device_bind()
> I will change this sequence, Or maybe will find some other way to fix
> this problem
> thanks

I meant exporting helpers needs to be done before the real users.

Thanks

> cindy
>
>
> > > ---
> > >  drivers/iommu/iommu.c | 2 ++
> > >  1 file changed, 2 insertions(+)
> > >
> > > diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> > > index 3bfc56df4f78..987cbf8c9a87 100644
> > > --- a/drivers/iommu/iommu.c
> > > +++ b/drivers/iommu/iommu.c
> > > @@ -3164,6 +3164,7 @@ int iommu_device_use_default_domain(struct device *dev)
> > >
> > >         return ret;
> > >  }
> > > +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain);
> > >
> > >  /**
> > >   * iommu_device_unuse_default_domain() - Device driver stops handling device
> > > @@ -3187,6 +3188,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
> > >         mutex_unlock(&group->mutex);
> > >         iommu_group_put(group);
> > >  }
> > > +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain);
> > >
> > >  static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
> > >  {
> > > --
> > > 2.34.3
> > >
> >
>

Re: [RFC v1 8/8] iommu: expose the function iommu_device_use_default_domain

[Cindy Lu]

On Wed, Nov 8, 2023 at 11:04 AM Jason Wang <jasowang@redhat.com> wrote:
>
> On Tue, Nov 7, 2023 at 2:10 PM Cindy Lu <lulu@redhat.com> wrote:
> >
> > On Mon, Nov 6, 2023 at 3:26 PM Jason Wang <jasowang@redhat.com> wrote:
> > >
> > > On Sat, Nov 4, 2023 at 1:18 AM Cindy Lu <lulu@redhat.com> wrote:
> > > >
> > > > Expose the function iommu_device_use_default_domain() and
> > > > iommu_device_unuse_default_domain()，
> > > > While vdpa bind the iommufd device and detach the iommu device,
> > > > vdpa need to call the function
> > > > iommu_device_unuse_default_domain() to release the owner
> > > >
> > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > >
> > > This is the end of the series, who is the user then?
> > >
> > > Thanks
> > >
> > hi Jason
> > These 2 functions was called in vhost_vdpa_iommufd_set_device(), Vdpa need to
> > release the dma owner, otherwise, the function will fail when
> > iommufd called iommu_device_claim_dma_owner() in iommufd_device_bind()
> > I will change this sequence, Or maybe will find some other way to fix
> > this problem
> > thanks
>
> I meant exporting helpers needs to be done before the real users.
>
> Thanks
>
sure will fix
Thanks
Cindy
> > cindy
> >
> >
> > > > ---
> > > >  drivers/iommu/iommu.c | 2 ++
> > > >  1 file changed, 2 insertions(+)
> > > >
> > > > diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> > > > index 3bfc56df4f78..987cbf8c9a87 100644
> > > > --- a/drivers/iommu/iommu.c
> > > > +++ b/drivers/iommu/iommu.c
> > > > @@ -3164,6 +3164,7 @@ int iommu_device_use_default_domain(struct device *dev)
> > > >
> > > >         return ret;
> > > >  }
> > > > +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain);
> > > >
> > > >  /**
> > > >   * iommu_device_unuse_default_domain() - Device driver stops handling device
> > > > @@ -3187,6 +3188,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
> > > >         mutex_unlock(&group->mutex);
> > > >         iommu_group_put(group);
> > > >  }
> > > > +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain);
> > > >
> > > >  static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
> > > >  {
> > > > --
> > > > 2.34.3
> > > >
> > >
> >
>

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Jason Wang]

On Sat, Nov 4, 2023 at 1:16 AM Cindy Lu <lulu@redhat.com> wrote:
>
>
> Hi All
> This code provides the iommufd support for vdpa device
> This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> v6,6-rc3
> Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net),
> I will continue working on it
>
> The kernel code is
> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>

It would be better to have a change summary here.

Thanks

>
>
> Cindy Lu (8):
>   vhost/iommufd: Add the functions support iommufd
>   Kconfig: Add the new file vhost/iommufd
>   vhost: Add 3 new uapi to support iommufd
>   vdpa: Add new vdpa_config_ops to support iommufd
>   vdpa_sim :Add support for iommufd
>   vdpa: change the map/unmap process to support iommufd
>   vp_vdpa::Add support for iommufd
>   iommu: expose the function iommu_device_use_default_domain
>
>  drivers/iommu/iommu.c             |   2 +
>  drivers/vdpa/vdpa_sim/vdpa_sim.c  |   8 ++
>  drivers/vdpa/virtio_pci/vp_vdpa.c |   4 +
>  drivers/vhost/Kconfig             |   1 +
>  drivers/vhost/Makefile            |   1 +
>  drivers/vhost/iommufd.c           | 178 +++++++++++++++++++++++++
>  drivers/vhost/vdpa.c              | 210 +++++++++++++++++++++++++++++-
>  drivers/vhost/vhost.h             |  21 +++
>  include/linux/vdpa.h              |  38 +++++-
>  include/uapi/linux/vhost.h        |  66 ++++++++++
>  10 files changed, 525 insertions(+), 4 deletions(-)
>  create mode 100644 drivers/vhost/iommufd.c
>
> --
> 2.34.3
>

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Yi Liu]

On 2023/11/6 12:11, Jason Wang wrote:
> On Sat, Nov 4, 2023 at 1:16 AM Cindy Lu <lulu@redhat.com> wrote:
>>
>>
>> Hi All
>> This code provides the iommufd support for vdpa device
>> This code fixes the bugs from the last version and also add the asid support. rebase on kernel
>> v6,6-rc3
>> Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net),
>> I will continue working on it
>>
>> The kernel code is
>> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
>>
>> Signed-off-by: Cindy Lu <lulu@redhat.com>
> 
> It would be better to have a change summary here.

yeah, the version number is also incorrect.

> Thanks
> 
>>
>>
>> Cindy Lu (8):
>>    vhost/iommufd: Add the functions support iommufd
>>    Kconfig: Add the new file vhost/iommufd
>>    vhost: Add 3 new uapi to support iommufd
>>    vdpa: Add new vdpa_config_ops to support iommufd
>>    vdpa_sim :Add support for iommufd
>>    vdpa: change the map/unmap process to support iommufd
>>    vp_vdpa::Add support for iommufd
>>    iommu: expose the function iommu_device_use_default_domain
>>
>>   drivers/iommu/iommu.c             |   2 +
>>   drivers/vdpa/vdpa_sim/vdpa_sim.c  |   8 ++
>>   drivers/vdpa/virtio_pci/vp_vdpa.c |   4 +
>>   drivers/vhost/Kconfig             |   1 +
>>   drivers/vhost/Makefile            |   1 +
>>   drivers/vhost/iommufd.c           | 178 +++++++++++++++++++++++++
>>   drivers/vhost/vdpa.c              | 210 +++++++++++++++++++++++++++++-
>>   drivers/vhost/vhost.h             |  21 +++
>>   include/linux/vdpa.h              |  38 +++++-
>>   include/uapi/linux/vhost.h        |  66 ++++++++++
>>   10 files changed, 525 insertions(+), 4 deletions(-)
>>   create mode 100644 drivers/vhost/iommufd.c
>>
>> --
>> 2.34.3
>>
> 

-- 
Regards,
Yi Liu

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Michael S. Tsirkin]

On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
> 
> Hi All
> This code provides the iommufd support for vdpa device
> This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> v6,6-rc3
> Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net), 

What kind of problems? Understanding that will make it easier
to figure out whether this is worth reviewing.

> I will continue working on it
> 
> The kernel code is
> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
> 
> Signed-off-by: Cindy Lu <lulu@redhat.com>

Please also Cc iommufd maintainers:

Jason Gunthorpe <jgg@ziepe.ca> (maintainer:IOMMUFD)
Kevin Tian <kevin.tian@intel.com> (maintainer:IOMMUFD)
Joerg Roedel <joro@8bytes.org> (maintainer:IOMMU SUBSYSTEM)
Will Deacon <will@kernel.org> (maintainer:IOMMU SUBSYSTEM)
Robin Murphy <robin.murphy@arm.com> (reviewer:IOMMU SUBSYSTEM)
iommu@lists.linux.dev (open list:IOMMUFD)
linux-kernel@vger.kernel.org (open list)

-- 
MST

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Jason Gunthorpe]

On Tue, Nov 07, 2023 at 02:30:34AM -0500, Michael S. Tsirkin wrote:
> On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
> > 
> > Hi All
> > This code provides the iommufd support for vdpa device
> > This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> > v6,6-rc3
> > Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net), 
> 
> What kind of problems? Understanding that will make it easier
> to figure out whether this is worth reviewing.

IMHO, this patch series needs to spend more time internally to Red Hat
before it is presented to the community. It is too far away from
something worth reviewing at this point.

Jason

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Michael S. Tsirkin]

On Tue, Nov 07, 2023 at 08:49:02AM -0400, Jason Gunthorpe wrote:
> On Tue, Nov 07, 2023 at 02:30:34AM -0500, Michael S. Tsirkin wrote:
> > On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
> > > 
> > > Hi All
> > > This code provides the iommufd support for vdpa device
> > > This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> > > v6,6-rc3
> > > Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net), 
> > 
> > What kind of problems? Understanding that will make it easier
> > to figure out whether this is worth reviewing.
> 
> IMHO, this patch series needs to spend more time internally to Red Hat
> before it is presented to the community. It is too far away from
> something worth reviewing at this point.
> 
> Jason

I am always trying to convince people to post RFCs early
instead of working for months behind closed doors only
to be told to rewrite everything in Rust.

Why does it have to be internal to a specific company?
I see Yi Liu from Intel is helping Cindy get it into shape
and that's classic open source ethos.

I know some subsystems ignore the RFC tag but I didn't realize
iommu is one of these. Is that really true?

-- 
MST

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Jason Gunthorpe]

On Tue, Nov 07, 2023 at 08:28:32AM -0500, Michael S. Tsirkin wrote:
> On Tue, Nov 07, 2023 at 08:49:02AM -0400, Jason Gunthorpe wrote:
> > On Tue, Nov 07, 2023 at 02:30:34AM -0500, Michael S. Tsirkin wrote:
> > > On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
> > > > 
> > > > Hi All
> > > > This code provides the iommufd support for vdpa device
> > > > This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> > > > v6,6-rc3
> > > > Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net), 
> > > 
> > > What kind of problems? Understanding that will make it easier
> > > to figure out whether this is worth reviewing.
> > 
> > IMHO, this patch series needs to spend more time internally to Red Hat
> > before it is presented to the community. It is too far away from
> > something worth reviewing at this point.
> > 
> > Jason
> 
> I am always trying to convince people to post RFCs early
> instead of working for months behind closed doors only
> to be told to rewrite everything in Rust.

The community has a limited review bandwidth, things should meet a
minimum standard before there is an expectation that other people
should review it on the list.

> Why does it have to be internal to a specific company?
> I see Yi Liu from Intel is helping Cindy get it into shape
> and that's classic open source ethos.

Big company's should take the responsibility to train and provide
skill development for their own staff.

> I know some subsystems ignore the RFC tag but I didn't realize
> iommu is one of these. Is that really true?

At least I've looked at this twice now and been disappointed. Neither
have been a well thought out RFC, this is a brain dump of unfinished
work.

Jason

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Michael S. Tsirkin]

On Tue, Nov 07, 2023 at 10:12:37AM -0400, Jason Gunthorpe wrote:
> Big company's should take the responsibility to train and provide
> skill development for their own staff.

That would result in a beautiful cathedral of a patch. I know this is
how some companies work. We are doing more of a bazaar thing here,
though. In a bunch of subsystems it seems that you don't get the
necessary skills until you have been publically shouted at by
maintainers - better to start early ;). Not a nice environment for
novices, for sure.

-- 
MST

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Jason Gunthorpe]

On Tue, Nov 07, 2023 at 09:30:21AM -0500, Michael S. Tsirkin wrote:
> On Tue, Nov 07, 2023 at 10:12:37AM -0400, Jason Gunthorpe wrote:
> > Big company's should take the responsibility to train and provide
> > skill development for their own staff.
> 
> That would result in a beautiful cathedral of a patch. I know this is
> how some companies work. We are doing more of a bazaar thing here,
> though. In a bunch of subsystems it seems that you don't get the
> necessary skills until you have been publically shouted at by
> maintainers - better to start early ;). Not a nice environment for
> novices, for sure.

In my view the "shouting from maintainers" is harmful to the people
buidling skills and it is an unkind thing to dump employees into that
kind of situation.

They should have help to establish the basic level of competence where
they may do the wrong thing, but all the process and presentation of
the wrong thing is top notch. You get a much better reception.

Jason

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Michael S. Tsirkin]

On Tue, Nov 07, 2023 at 11:52:17AM -0400, Jason Gunthorpe wrote:
> On Tue, Nov 07, 2023 at 09:30:21AM -0500, Michael S. Tsirkin wrote:
> > On Tue, Nov 07, 2023 at 10:12:37AM -0400, Jason Gunthorpe wrote:
> > > Big company's should take the responsibility to train and provide
> > > skill development for their own staff.
> > 
> > That would result in a beautiful cathedral of a patch. I know this is
> > how some companies work. We are doing more of a bazaar thing here,
> > though. In a bunch of subsystems it seems that you don't get the
> > necessary skills until you have been publically shouted at by
> > maintainers - better to start early ;). Not a nice environment for
> > novices, for sure.
> 
> In my view the "shouting from maintainers" is harmful to the people
> buidling skills and it is an unkind thing to dump employees into that
> kind of situation.
> 
> They should have help to establish the basic level of competence where
> they may do the wrong thing, but all the process and presentation of
> the wrong thing is top notch. You get a much better reception.
> 
> Jason

What - like e.g. mechanically fixing checkpatch warnings without
understanding? I actually very much dislike it when people take a bad
patch and just polish the presentation
- it is easier for me if I can judge patch quality quickly from the
presentation. Matter of taste I guess.

-- 
MST

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Jason Gunthorpe]

On Thu, Nov 09, 2023 at 06:48:46PM -0500, Michael S. Tsirkin wrote:
> On Tue, Nov 07, 2023 at 11:52:17AM -0400, Jason Gunthorpe wrote:
> > On Tue, Nov 07, 2023 at 09:30:21AM -0500, Michael S. Tsirkin wrote:
> > > On Tue, Nov 07, 2023 at 10:12:37AM -0400, Jason Gunthorpe wrote:
> > > > Big company's should take the responsibility to train and provide
> > > > skill development for their own staff.
> > > 
> > > That would result in a beautiful cathedral of a patch. I know this is
> > > how some companies work. We are doing more of a bazaar thing here,
> > > though. In a bunch of subsystems it seems that you don't get the
> > > necessary skills until you have been publically shouted at by
> > > maintainers - better to start early ;). Not a nice environment for
> > > novices, for sure.
> > 
> > In my view the "shouting from maintainers" is harmful to the people
> > buidling skills and it is an unkind thing to dump employees into that
> > kind of situation.
> > 
> > They should have help to establish the basic level of competence where
> > they may do the wrong thing, but all the process and presentation of
> > the wrong thing is top notch. You get a much better reception.
> 
> What - like e.g. mechanically fixing checkpatch warnings without
> understanding? 

No, not at all. I mean actually going through and explaining what the
idea is to another person and ensuing that the commit messages convey
that idea, that the patches reflect the idea, that everything is
convayed, and it isn't obviously internally illogical.

Like, why did this series have a giant block of #ifdef 0'd code with
no explanation at all? That isn't checkpatch nitpicks, that is not
meeting the minimum standard to convey an idea in an RFC.

Jason

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Jakub Kicinski]

On Tue, 7 Nov 2023 08:28:32 -0500 Michael S. Tsirkin wrote:
> I am always trying to convince people to post RFCs early
> instead of working for months behind closed doors only
> to be told to rewrite everything in Rust.
> 
> Why does it have to be internal to a specific company?
> I see Yi Liu from Intel is helping Cindy get it into shape
> and that's classic open source ethos.

+1, FWIW.

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Michael S. Tsirkin]

On Tue, Nov 07, 2023 at 08:49:02AM -0400, Jason Gunthorpe wrote:
> IMHO, this patch series needs to spend more time internally to Red Hat
> before it is presented to the community.

Just to add an example why I think this "internal review" is a bad idea
I seem to recall that someone internal to nvidia at some point
attempted to implement this already. The only output from that
work we have is that "it's tough" - no pointers to what's tough,
no code to study even as a bad path to follow.
And while Red Hat might be big, the virt team is rather smaller.

-- 
MST

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Jason Gunthorpe]

On Tue, Nov 07, 2023 at 09:55:26AM -0500, Michael S. Tsirkin wrote:
> On Tue, Nov 07, 2023 at 08:49:02AM -0400, Jason Gunthorpe wrote:
> > IMHO, this patch series needs to spend more time internally to Red Hat
> > before it is presented to the community.
> 
> Just to add an example why I think this "internal review" is a bad idea
> I seem to recall that someone internal to nvidia at some point
> attempted to implement this already. The only output from that
> work we have is that "it's tough" - no pointers to what's tough,
> no code to study even as a bad path to follow.
> And while Red Hat might be big, the virt team is rather smaller.

I don't think Nicolin got to a presentable code point.

But you can start to see the issues even in this series, like
simulator is complicated. mlx5 is complicated. Deciding to omit those
is one path. Come with a proposal and justification to take it out,
not a patch with an unexplained #ifdef.

Again, I'm not talking about big impactful decisions I'm saying RH
should take it internally to get a RFC proposal to the level where it
is actually an RFC proposal and not a brain dump. Make sure it has
logical commit messages, make sure the basic thinking about the idea
is done and the proposal is self consistent and explained. Make sure
the patches and series construction meet a kernel standard.

The purpose of the RFC is to clearly articulate what it is you are
asking to do, why you want to do it, and how you intend to get
there. There is still alot of basic work to achieve this and properly
communicate it.

Training to do that should rightly come from the employeer, not the
community. We've seen some big blow ups because some companies have
been trying to externalize their training to the community.

Jason

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Michael S. Tsirkin]

On Tue, Nov 07, 2023 at 11:48:48AM -0400, Jason Gunthorpe wrote:
> On Tue, Nov 07, 2023 at 09:55:26AM -0500, Michael S. Tsirkin wrote:
> > On Tue, Nov 07, 2023 at 08:49:02AM -0400, Jason Gunthorpe wrote:
> > > IMHO, this patch series needs to spend more time internally to Red Hat
> > > before it is presented to the community.
> > 
> > Just to add an example why I think this "internal review" is a bad idea
> > I seem to recall that someone internal to nvidia at some point
> > attempted to implement this already. The only output from that
> > work we have is that "it's tough" - no pointers to what's tough,
> > no code to study even as a bad path to follow.
> > And while Red Hat might be big, the virt team is rather smaller.
> 
> I don't think Nicolin got to a presentable code point.
> 
> But you can start to see the issues even in this series, like
> simulator is complicated. mlx5 is complicated. Deciding to omit those
> is one path. Come with a proposal and justification to take it out,
> not a patch with an unexplained #ifdef.

Right. Simulator I don't think we need to support, or at least
not necessarily to get this merged - it does not really
benefit from any iommufd features.

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Michael S. Tsirkin]

On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
> Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net), 

I'm not sure there's even value in bothering with iommufd for the
simulator. Just find a way to disable it and fail gracefully.

-- 
MST

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Michael S. Tsirkin]

On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
> 
> Hi All
> This code provides the iommufd support for vdpa device
> This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> v6,6-rc3
> Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net), 
> I will continue working on it
> 
> The kernel code is
> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
> 
> Signed-off-by: Cindy Lu <lulu@redhat.com>

Was this abandoned?

> 
> Cindy Lu (8):
>   vhost/iommufd: Add the functions support iommufd
>   Kconfig: Add the new file vhost/iommufd
>   vhost: Add 3 new uapi to support iommufd
>   vdpa: Add new vdpa_config_ops to support iommufd
>   vdpa_sim :Add support for iommufd
>   vdpa: change the map/unmap process to support iommufd
>   vp_vdpa::Add support for iommufd
>   iommu: expose the function iommu_device_use_default_domain
> 
>  drivers/iommu/iommu.c             |   2 +
>  drivers/vdpa/vdpa_sim/vdpa_sim.c  |   8 ++
>  drivers/vdpa/virtio_pci/vp_vdpa.c |   4 +
>  drivers/vhost/Kconfig             |   1 +
>  drivers/vhost/Makefile            |   1 +
>  drivers/vhost/iommufd.c           | 178 +++++++++++++++++++++++++
>  drivers/vhost/vdpa.c              | 210 +++++++++++++++++++++++++++++-
>  drivers/vhost/vhost.h             |  21 +++
>  include/linux/vdpa.h              |  38 +++++-
>  include/uapi/linux/vhost.h        |  66 ++++++++++
>  10 files changed, 525 insertions(+), 4 deletions(-)
>  create mode 100644 drivers/vhost/iommufd.c
> 
> -- 
> 2.34.3

Re: [RFC v1 0/8] vhost-vdpa: add support for iommufd

[Cindy Lu]

On Thu, Jan 11, 2024 at 6:25 AM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Sat, Nov 04, 2023 at 01:16:33AM +0800, Cindy Lu wrote:
> >
> > Hi All
> > This code provides the iommufd support for vdpa device
> > This code fixes the bugs from the last version and also add the asid support. rebase on kernel
> > v6,6-rc3
> > Test passed in the physical device (vp_vdpa), but  there are still some problems in the emulated device (vdpa_sim_net),
> > I will continue working on it
> >
> > The kernel code is
> > https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC_v1
> >
> > Signed-off-by: Cindy Lu <lulu@redhat.com>
>
> Was this abandoned?
>
Thanks Micheal. I'm really sorry for the delay. I will continue working on this
Thanks
Cindy
> >
> > Cindy Lu (8):
> >   vhost/iommufd: Add the functions support iommufd
> >   Kconfig: Add the new file vhost/iommufd
> >   vhost: Add 3 new uapi to support iommufd
> >   vdpa: Add new vdpa_config_ops to support iommufd
> >   vdpa_sim :Add support for iommufd
> >   vdpa: change the map/unmap process to support iommufd
> >   vp_vdpa::Add support for iommufd
> >   iommu: expose the function iommu_device_use_default_domain
> >
> >  drivers/iommu/iommu.c             |   2 +
> >  drivers/vdpa/vdpa_sim/vdpa_sim.c  |   8 ++
> >  drivers/vdpa/virtio_pci/vp_vdpa.c |   4 +
> >  drivers/vhost/Kconfig             |   1 +
> >  drivers/vhost/Makefile            |   1 +
> >  drivers/vhost/iommufd.c           | 178 +++++++++++++++++++++++++
> >  drivers/vhost/vdpa.c              | 210 +++++++++++++++++++++++++++++-
> >  drivers/vhost/vhost.h             |  21 +++
> >  include/linux/vdpa.h              |  38 +++++-
> >  include/uapi/linux/vhost.h        |  66 ++++++++++
> >  10 files changed, 525 insertions(+), 4 deletions(-)
> >  create mode 100644 drivers/vhost/iommufd.c
> >
> > --
> > 2.34.3
>