* [RFC 1/7] vhost/iommufd: Add the functions support iommufd
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
@ 2023-09-23 17:05 ` Cindy Lu
2023-09-26 2:47 ` Jason Wang
2023-09-23 17:05 ` [RFC 2/7] Kconfig: Add the new file vhost/iommufd Cindy Lu
` (7 subsequent siblings)
8 siblings, 1 reply; 24+ messages in thread
From: Cindy Lu @ 2023-09-23 17:05 UTC (permalink / raw)
To: lulu, jasowang, mst, yi.l.liu, jgg, linux-kernel, virtualization,
netdev
Add a new file vhost/iommufd.c to support the function of
iommufd, This file contains iommufd function of emulated device and
the physical device.
Signed-off-by: Cindy Lu <lulu@redhat.com>
---
drivers/vhost/iommufd.c | 151 ++++++++++++++++++++++++++++++++++++++++
drivers/vhost/vhost.h | 21 ++++++
2 files changed, 172 insertions(+)
create mode 100644 drivers/vhost/iommufd.c
diff --git a/drivers/vhost/iommufd.c b/drivers/vhost/iommufd.c
new file mode 100644
index 000000000000..080858f76fd5
--- /dev/null
+++ b/drivers/vhost/iommufd.c
@@ -0,0 +1,151 @@
+#include <linux/vdpa.h>
+#include <linux/iommufd.h>
+
+#include "vhost.h"
+
+MODULE_IMPORT_NS(IOMMUFD);
+
+int vdpa_iommufd_bind(struct vdpa_device *vdpa, struct iommufd_ctx *ictx,
+ u32 *ioas_id, u32 *device_id)
+{
+ int ret;
+
+ vhost_vdpa_lockdep_assert_held(vdpa);
+
+ /*
+ * If the driver doesn't provide this op then it means the device does
+ * not do DMA at all. So nothing to do.
+ */
+ if (!vdpa->config->bind_iommufd)
+ return 0;
+
+ ret = vdpa->config->bind_iommufd(vdpa, ictx, device_id);
+ if (ret)
+ return ret;
+
+ ret = vdpa->config->attach_ioas(vdpa, ioas_id);
+ if (ret)
+ goto err_unbind;
+ vdpa->iommufd_attached = true;
+
+ return 0;
+
+err_unbind:
+ if (vdpa->config->unbind_iommufd)
+ vdpa->config->unbind_iommufd(vdpa);
+ return ret;
+}
+
+void vdpa_iommufd_unbind(struct vdpa_device *vdpa)
+{
+ vhost_vdpa_lockdep_assert_held(vdpa);
+
+ if (vdpa->config->unbind_iommufd)
+ vdpa->config->unbind_iommufd(vdpa);
+}
+
+int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
+ struct iommufd_ctx *ictx, u32 *out_device_id)
+{
+ struct device *dma_dev = vdpa_get_dma_dev(vdpa);
+ struct iommufd_device *idev;
+
+ idev = iommufd_device_bind(ictx, dma_dev, out_device_id);
+ if (IS_ERR(idev))
+ return PTR_ERR(idev);
+ vdpa->iommufd_device = idev;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_bind);
+
+void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa)
+{
+ vhost_vdpa_lockdep_assert_held(vdpa);
+
+ if (vdpa->iommufd_attached) {
+ iommufd_device_detach(vdpa->iommufd_device);
+ vdpa->iommufd_attached = false;
+ }
+ iommufd_device_unbind(vdpa->iommufd_device);
+ vdpa->iommufd_device = NULL;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_unbind);
+
+int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id)
+{
+ unsigned int flags = 0;
+
+ return iommufd_device_attach(vdpa->iommufd_device, pt_id);
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_attach_ioas);
+
+static void vdpa_emulated_unmap(void *data, unsigned long iova,
+ unsigned long length)
+{
+ struct vdpa_device *vdpa = data;
+
+ vdpa->config->dma_unmap(vdpa, 0, iova, length);
+}
+
+static const struct iommufd_access_ops vdpa_user_ops = {
+ .needs_pin_pages = 1,
+ .unmap = vdpa_emulated_unmap,
+};
+
+int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
+ struct iommufd_ctx *ictx, u32 *out_device_id)
+{
+ vhost_vdpa_lockdep_assert_held(vdpa);
+
+ vdpa->iommufd_ictx = ictx;
+ iommufd_ctx_get(ictx);
+ struct iommufd_device *idev;
+
+ idev = iommufd_device_bind(ictx, vdpa->dma_dev, out_device_id);
+
+ if (IS_ERR(idev))
+ return PTR_ERR(idev);
+ vdpa->iommufd_device = idev;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_emulated_bind);
+
+void vdpa_iommufd_emulated_unbind(struct vdpa_device *vdpa)
+{
+ vhost_vdpa_lockdep_assert_held(vdpa);
+
+ if (vdpa->iommufd_access) {
+ iommufd_access_destroy(vdpa->iommufd_access);
+ vdpa->iommufd_access = NULL;
+ }
+ iommufd_ctx_put(vdpa->iommufd_ictx);
+ vdpa->iommufd_ictx = NULL;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_emulated_unbind);
+
+int vdpa_iommufd_emulated_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id)
+{
+ struct iommufd_access *user;
+
+ vhost_vdpa_lockdep_assert_held(vdpa);
+
+ user = iommufd_access_create(vdpa->iommufd_ictx, *pt_id, &vdpa_user_ops,
+ vdpa);
+ if (IS_ERR(user))
+ return PTR_ERR(user);
+ vdpa->iommufd_access = user;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_emulated_attach_ioas);
+int vdpa_iommufd_emulated_detach_ioas(struct vdpa_device *vdpa)
+{
+ vhost_vdpa_lockdep_assert_held(vdpa);
+
+ if (!vdpa->iommufd_ictx || !vdpa->iommufd_access)
+ return -1;
+
+ iommufd_access_destroy(vdpa->iommufd_access);
+ vdpa->iommufd_access = NULL;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(vdpa_iommufd_emulated_detach_ioas);
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 790b296271f1..c470a5596d9c 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -291,6 +291,27 @@ static inline bool vhost_is_little_endian(struct vhost_virtqueue *vq)
}
#endif
+struct iommufd_ctx;
+struct vdpa_device;
+void vhost_vdpa_lockdep_assert_held(struct vdpa_device *vdpa);
+
+#if IS_ENABLED(CONFIG_IOMMUFD)
+int vdpa_iommufd_bind(struct vdpa_device *vdpa, struct iommufd_ctx *ictx,
+ u32 *ioas_id, u32 *device_id);
+void vdpa_iommufd_unbind(struct vdpa_device *vdpa);
+#else
+static inline int vdpa_iommufd_bind(struct vdpa_device *vdpa,
+ struct iommufd_ctx *ictx, u32 *ioas_id,
+ u32 *device_id)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline void vdpa_iommufd_unbind(struct vdpa_device *vdpa)
+{
+}
+#endif
+
/* Memory accessors */
static inline u16 vhost16_to_cpu(struct vhost_virtqueue *vq, __virtio16 val)
{
--
2.34.3
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [RFC 1/7] vhost/iommufd: Add the functions support iommufd
2023-09-23 17:05 ` [RFC 1/7] vhost/iommufd: Add the functions support iommufd Cindy Lu
@ 2023-09-26 2:47 ` Jason Wang
0 siblings, 0 replies; 24+ messages in thread
From: Jason Wang @ 2023-09-26 2:47 UTC (permalink / raw)
To: Cindy Lu; +Cc: mst, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 1:05 AM Cindy Lu <lulu@redhat.com> wrote:
>
> Add a new file vhost/iommufd.c to support the function of
> iommufd, This file contains iommufd function of emulated device and
> the physical device.
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
> drivers/vhost/iommufd.c | 151 ++++++++++++++++++++++++++++++++++++++++
> drivers/vhost/vhost.h | 21 ++++++
> 2 files changed, 172 insertions(+)
> create mode 100644 drivers/vhost/iommufd.c
>
> diff --git a/drivers/vhost/iommufd.c b/drivers/vhost/iommufd.c
> new file mode 100644
> index 000000000000..080858f76fd5
> --- /dev/null
> +++ b/drivers/vhost/iommufd.c
> @@ -0,0 +1,151 @@
> +#include <linux/vdpa.h>
> +#include <linux/iommufd.h>
> +
> +#include "vhost.h"
> +
> +MODULE_IMPORT_NS(IOMMUFD);
> +
> +int vdpa_iommufd_bind(struct vdpa_device *vdpa, struct iommufd_ctx *ictx,
> + u32 *ioas_id, u32 *device_id)
> +{
> + int ret;
> +
> + vhost_vdpa_lockdep_assert_held(vdpa);
> +
> + /*
> + * If the driver doesn't provide this op then it means the device does
> + * not do DMA at all. So nothing to do.
> + */
> + if (!vdpa->config->bind_iommufd)
> + return 0;
> +
> + ret = vdpa->config->bind_iommufd(vdpa, ictx, device_id);
> + if (ret)
> + return ret;
> +
> + ret = vdpa->config->attach_ioas(vdpa, ioas_id);
> + if (ret)
> + goto err_unbind;
> + vdpa->iommufd_attached = true;
> +
> + return 0;
> +
> +err_unbind:
> + if (vdpa->config->unbind_iommufd)
> + vdpa->config->unbind_iommufd(vdpa);
> + return ret;
> +}
> +
> +void vdpa_iommufd_unbind(struct vdpa_device *vdpa)
> +{
> + vhost_vdpa_lockdep_assert_held(vdpa);
> +
> + if (vdpa->config->unbind_iommufd)
> + vdpa->config->unbind_iommufd(vdpa);
> +}
> +
> +int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
> + struct iommufd_ctx *ictx, u32 *out_device_id)
> +{
> + struct device *dma_dev = vdpa_get_dma_dev(vdpa);
> + struct iommufd_device *idev;
> +
> + idev = iommufd_device_bind(ictx, dma_dev, out_device_id);
> + if (IS_ERR(idev))
> + return PTR_ERR(idev);
> + vdpa->iommufd_device = idev;
> + return 0;
> +}
> +EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_bind);
> +
> +void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa)
> +{
> + vhost_vdpa_lockdep_assert_held(vdpa);
> +
> + if (vdpa->iommufd_attached) {
> + iommufd_device_detach(vdpa->iommufd_device);
> + vdpa->iommufd_attached = false;
> + }
> + iommufd_device_unbind(vdpa->iommufd_device);
> + vdpa->iommufd_device = NULL;
> +}
> +EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_unbind);
> +
> +int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id)
> +{
> + unsigned int flags = 0;
> +
> + return iommufd_device_attach(vdpa->iommufd_device, pt_id);
> +}
> +EXPORT_SYMBOL_GPL(vdpa_iommufd_physical_attach_ioas);
> +
> +static void vdpa_emulated_unmap(void *data, unsigned long iova,
> + unsigned long length)
> +{
> + struct vdpa_device *vdpa = data;
> +
> + vdpa->config->dma_unmap(vdpa, 0, iova, length);
> +}
> +
> +static const struct iommufd_access_ops vdpa_user_ops = {
> + .needs_pin_pages = 1,
Note that simulators support VA, so no page pinning in that case while rebasing.
static bool use_va = true;
module_param(use_va, bool, 0444);
MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA");
So we need to handle that case as well.
(Note that it looks like VA mode is broken, I may need some time to fix that).
> + .unmap = vdpa_emulated_unmap,
> +};
> +
> +int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
> + struct iommufd_ctx *ictx, u32 *out_device_id)
> +{
> + vhost_vdpa_lockdep_assert_held(vdpa);
> +
> + vdpa->iommufd_ictx = ictx;
> + iommufd_ctx_get(ictx);
> + struct iommufd_device *idev;
> +
> + idev = iommufd_device_bind(ictx, vdpa->dma_dev, out_device_id);
This seems not appropriate for emulated devices as it deals with the
concepts that only exist in physical devices like the IOMMU domain
etc.
If possible, please refer how VFIO handles this (I guess it should
have something)
Thanks
^ permalink raw reply [flat|nested] 24+ messages in thread
* [RFC 2/7] Kconfig: Add the new file vhost/iommufd
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
2023-09-23 17:05 ` [RFC 1/7] vhost/iommufd: Add the functions support iommufd Cindy Lu
@ 2023-09-23 17:05 ` Cindy Lu
2023-09-23 17:05 ` [RFC 3/7] vhost: Add 3 new uapi to support iommufd Cindy Lu
` (6 subsequent siblings)
8 siblings, 0 replies; 24+ messages in thread
From: Cindy Lu @ 2023-09-23 17:05 UTC (permalink / raw)
To: lulu, jasowang, mst, yi.l.liu, jgg, linux-kernel, virtualization,
netdev
Change the makefile and Kconfig, to add the
new file vhost/iommufd.c
Signed-off-by: Cindy Lu <lulu@redhat.com>
---
drivers/vhost/Kconfig | 1 +
drivers/vhost/Makefile | 1 +
2 files changed, 2 insertions(+)
diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig
index 587fbae06182..c54cea6c03f0 100644
--- a/drivers/vhost/Kconfig
+++ b/drivers/vhost/Kconfig
@@ -67,6 +67,7 @@ config VHOST_VDPA
select VHOST
select IRQ_BYPASS_MANAGER
depends on VDPA
+ depends on IOMMUFD || !IOMMUFD
help
This kernel module can be loaded in host kernel to accelerate
guest virtio devices with the vDPA-based backends.
diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile
index f3e1897cce85..cda7f6b7f8da 100644
--- a/drivers/vhost/Makefile
+++ b/drivers/vhost/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_VHOST_RING) += vringh.o
obj-$(CONFIG_VHOST_VDPA) += vhost_vdpa.o
vhost_vdpa-y := vdpa.o
+vhost_vdpa-$(CONFIG_IOMMUFD) += iommufd.o
obj-$(CONFIG_VHOST) += vhost.o
--
2.34.3
^ permalink raw reply related [flat|nested] 24+ messages in thread* [RFC 3/7] vhost: Add 3 new uapi to support iommufd
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
2023-09-23 17:05 ` [RFC 1/7] vhost/iommufd: Add the functions support iommufd Cindy Lu
2023-09-23 17:05 ` [RFC 2/7] Kconfig: Add the new file vhost/iommufd Cindy Lu
@ 2023-09-23 17:05 ` Cindy Lu
2023-09-25 13:49 ` Jason Gunthorpe
2023-09-26 2:47 ` Jason Wang
2023-09-23 17:05 ` [RFC 4/7] vdpa: change the map/unmap process " Cindy Lu
` (5 subsequent siblings)
8 siblings, 2 replies; 24+ messages in thread
From: Cindy Lu @ 2023-09-23 17:05 UTC (permalink / raw)
To: lulu, jasowang, mst, yi.l.liu, jgg, linux-kernel, virtualization,
netdev
VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
address space specified by IOAS id.
VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
from the iommufd address space
Signed-off-by: Cindy Lu <lulu@redhat.com>
---
drivers/vhost/vdpa.c | 191 +++++++++++++++++++++++++++++++++++++
include/uapi/linux/vhost.h | 71 ++++++++++++++
2 files changed, 262 insertions(+)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index ec32f785dfde..91da012084e9 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -18,6 +18,7 @@
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/iommu.h>
+#include <linux/iommufd.h>
#include <linux/uuid.h>
#include <linux/vdpa.h>
#include <linux/nospec.h>
@@ -25,6 +26,8 @@
#include "vhost.h"
+MODULE_IMPORT_NS(IOMMUFD);
+
enum {
VHOST_VDPA_BACKEND_FEATURES =
(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
@@ -69,6 +72,15 @@ static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
struct vhost_iotlb *iotlb, u64 start,
u64 last, u32 asid);
+void vhost_vdpa_lockdep_assert_held(struct vdpa_device *vdpa)
+{
+ struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
+
+ if (WARN_ON(!v))
+ return;
+ lockdep_assert_held(&v->vdev.mutex);
+}
+
static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
{
struct vhost_vdpa_as *as = container_of(iotlb, struct
@@ -497,6 +509,173 @@ static long vhost_vdpa_suspend(struct vhost_vdpa *v)
return ops->suspend(vdpa);
}
+static long vhost_vdpa_tmp_set_iommufd(struct vhost_vdpa *v, void __user *argp)
+{
+ struct device *dma_dev = vdpa_get_dma_dev(v->vdpa);
+ struct vhost_vdpa_set_iommufd set_iommufd;
+ struct vdpa_device *vdpa = v->vdpa;
+ struct iommufd_ctx *ictx;
+ unsigned long minsz;
+ u32 pt_id, dev_id;
+ struct fd f;
+ long r = 0;
+ int idx;
+
+ minsz = offsetofend(struct vhost_vdpa_set_iommufd, ioas_id);
+ if (copy_from_user(&set_iommufd, argp, minsz))
+ return -EFAULT;
+
+ if (set_iommufd.group_id >= v->nvqs)
+ return -ENOBUFS;
+
+ idx = array_index_nospec(set_iommufd.group_id, v->nvqs);
+
+ /* Unset IOMMUFD */
+ if (set_iommufd.iommufd < 0) {
+ if (!test_bit(idx, vdpa->vq_bitmap))
+ return -EINVAL;
+
+ if (!vdpa->iommufd_ictx || !vdpa->iommufd_device)
+ return -EINVAL;
+ if (atomic_read(&vdpa->iommufd_users)) {
+ atomic_dec(&vdpa->iommufd_users);
+ return 0;
+ }
+ vdpa_iommufd_unbind(v->vdpa);
+ vdpa->iommufd_device = NULL;
+ vdpa->iommufd_ictx = NULL;
+ clear_bit(idx, vdpa->vq_bitmap);
+ return iommu_attach_device(v->domain, dma_dev);
+ }
+ /* First opened virtqueue of this vdpa device */
+
+ if (!vdpa->vq_bitmap) {
+ vdpa->vq_bitmap = bitmap_alloc(v->nvqs, GFP_KERNEL);
+ }
+ ///if (test_bit(idx, vdpa->vq_bitmap))
+ //return -EBUSY;
+
+ /* For same device but different groups, ++refcount only */
+ if (vdpa->iommufd_device)
+ goto out_inc;
+
+ r = -EBADF;
+ f = fdget(set_iommufd.iommufd);
+ if (!f.file)
+ goto out_bitmap_free;
+
+ r = -EINVAL;
+ ictx = iommufd_ctx_from_file(f.file);
+ if (IS_ERR(ictx))
+ goto out_fdput;
+
+ if (v->domain)
+ iommu_detach_device(v->domain, dma_dev);
+
+#if 0
+ iommu_group_add_device(iommu_group_alloc(), &vdpa->dev);
+#endif
+ pt_id = set_iommufd.ioas_id;
+ r = vdpa_iommufd_bind(vdpa, ictx, &pt_id, &dev_id);
+ if (r)
+ goto out_reattach;
+
+ set_iommufd.out_dev_id = dev_id;
+ set_iommufd.out_hwpt_id = pt_id;
+ r = copy_to_user(argp + minsz, &set_iommufd.out_dev_id,
+ sizeof(set_iommufd.out_dev_id) +
+ sizeof(set_iommufd.out_hwpt_id)) ?
+ -EFAULT :
+ 0;
+ if (r)
+ goto out_device_unbind;
+ printk(KERN_ERR "[%s] %d called %p\n", __func__, __LINE__,
+ vdpa->iommufd_ictx);
+
+ vdpa->iommufd_ictx = ictx;
+
+out_inc:
+ atomic_inc(&vdpa->iommufd_users);
+ set_bit(idx, vdpa->vq_bitmap);
+
+ goto out_fdput;
+
+out_device_unbind:
+
+ vdpa_iommufd_unbind(vdpa);
+out_reattach:
+
+ iommu_attach_device(v->domain, dma_dev);
+ iommufd_ctx_put(ictx);
+out_fdput:
+ fdput(f);
+out_bitmap_free:
+ bitmap_free(vdpa->vq_bitmap);
+ return r;
+}
+int vdpa_ioctl_device_attach(struct vhost_vdpa *v, void __user *arg)
+{
+ struct vdpa_device_attach_iommufd_as attach;
+ unsigned long minsz;
+ int ret;
+
+ minsz = offsetofend(struct vdpa_device_attach_iommufd_as, ioas_id);
+
+ if (copy_from_user(&attach, (void __user *)arg, minsz))
+ return -EFAULT;
+
+ if (attach.argsz < minsz || attach.flags ||
+ attach.ioas_id == IOMMUFD_INVALID_ID)
+ return -EINVAL;
+
+ if (!v->vdpa->config->bind_iommufd)
+ return -ENODEV;
+
+ if (!v->vdpa->iommufd_ictx) {
+ ret = -EINVAL;
+ return ret;
+ }
+
+ ret = v->vdpa->config->attach_ioas(v->vdpa, &attach.ioas_id);
+
+ if (ret)
+ return ret;
+
+ ret = copy_to_user(
+ (void __user *)arg +
+ offsetofend(struct vdpa_device_attach_iommufd_as,
+ flags),
+ &attach.ioas_id, sizeof(attach.ioas_id)) ?
+ -EFAULT :
+ 0;
+
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+int vdpa_ioctl_device_detach(struct vhost_vdpa *v, void __user *arg)
+{
+ struct vdpa_device_detach_iommufd_as detach;
+ unsigned long minsz;
+
+ minsz = offsetofend(struct vdpa_device_detach_iommufd_as, flags);
+
+ if (copy_from_user(&detach, (void __user *)arg, minsz))
+ return -EFAULT;
+
+ if (detach.argsz < minsz || detach.flags)
+ return -EINVAL;
+
+ if (!v->vdpa->config->bind_iommufd)
+ return -ENODEV;
+
+ if (v->vdpa->iommufd_ictx) {
+ return -EINVAL;
+ }
+ return v->vdpa->config->detach_ioas(v->vdpa);
+}
static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
void __user *argp)
@@ -655,6 +834,18 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
case VHOST_SET_LOG_FD:
r = -ENOIOCTLCMD;
break;
+ case VHOST_VDPA_SET_IOMMU_FD:
+
+ r = vhost_vdpa_tmp_set_iommufd(v, argp);
+ break;
+ case VDPA_DEVICE_ATTACH_IOMMUFD_AS:
+ r = vdpa_ioctl_device_attach(v, (void __user *)arg);
+ break;
+
+ case VDPA_DEVICE_DETACH_IOMMUFD_AS:
+ r = vdpa_ioctl_device_detach(v, (void __user *)arg);
+ break;
+
case VHOST_VDPA_SET_CONFIG_CALL:
r = vhost_vdpa_set_config_call(v, argp);
break;
diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
index f9f115a7c75b..cdda0c1860d8 100644
--- a/include/uapi/linux/vhost.h
+++ b/include/uapi/linux/vhost.h
@@ -45,6 +45,7 @@
#define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64)
/* Specify an eventfd file descriptor to signal on log write. */
#define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int)
+/* Specify an iommufd file descriptor for IO address translation */
/* Ring setup. */
/* Set number of descriptors in ring. This parameter can not
@@ -180,4 +181,74 @@
*/
#define VHOST_VDPA_SUSPEND _IO(VHOST_VIRTIO, 0x7D)
+/* vhost vdpa set iommufd
+ * Input parameters:
+ * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
+ * @group_id: identifier of the group that a virtqueue belongs to
+ * @ioas_id: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
+ * Output parameters:
+ * @out_dev_id: device identifier
+ * @out_hwpt_id: hardware IO pagetable identifier
+ */
+struct vhost_vdpa_set_iommufd {
+ __s32 iommufd;
+ __u32 group_id;
+ __u32 ioas_id;
+ __u32 out_dev_id;
+ __u32 out_hwpt_id;
+};
+
+#define VHOST_VDPA_SET_IOMMU_FD \
+ _IOW(VHOST_VIRTIO, 0x7e, struct vhost_vdpa_set_iommufd)
+
+/*
+ * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
+ * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
+ *
+ * Attach a vdpa device to an iommufd address space specified by IOAS
+ * id.
+ *
+ * Available only after a device has been bound to iommufd via
+ * VHOST_VDPA_SET_IOMMU_FD
+ *
+ * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
+ *
+ * @argsz: user filled size of this data.
+ * @flags: must be 0.
+ * @ioas_id: Input the target id which can represent an ioas
+ * allocated via iommufd subsystem.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vdpa_device_attach_iommufd_as {
+ __u32 argsz;
+ __u32 flags;
+ __u32 ioas_id;
+};
+
+#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
+ _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
+
+/*
+ * VDPA_DEVICE_DETACH_IOMMUFD_AS
+ *
+ * Detach a vdpa device from the iommufd address space it has been
+ * attached to. After it, device should be in a blocking DMA state.
+ *
+ * Available only after a device has been bound to iommufd via
+ * VHOST_VDPA_SET_IOMMU_FD
+ *
+ * @argsz: user filled size of this data.
+ * @flags: must be 0.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vdpa_device_detach_iommufd_as {
+ __u32 argsz;
+ __u32 flags;
+};
+
+#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
+ _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
+
#endif
--
2.34.3
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [RFC 3/7] vhost: Add 3 new uapi to support iommufd
2023-09-23 17:05 ` [RFC 3/7] vhost: Add 3 new uapi to support iommufd Cindy Lu
@ 2023-09-25 13:49 ` Jason Gunthorpe
2023-09-26 2:47 ` Jason Wang
1 sibling, 0 replies; 24+ messages in thread
From: Jason Gunthorpe @ 2023-09-25 13:49 UTC (permalink / raw)
To: Cindy Lu; +Cc: jasowang, mst, yi.l.liu, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 01:05:36AM +0800, Cindy Lu wrote:
> +/* vhost vdpa set iommufd
> + * Input parameters:
> + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> + * @group_id: identifier of the group that a virtqueue belongs to
> + * @ioas_id: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> + * Output parameters:
> + * @out_dev_id: device identifier
> + * @out_hwpt_id: hardware IO pagetable identifier
> + */
> +struct vhost_vdpa_set_iommufd {
> + __s32 iommufd;
> + __u32 group_id;
> + __u32 ioas_id;
> + __u32 out_dev_id;
> + __u32 out_hwpt_id;
> +};
> +
> +#define VHOST_VDPA_SET_IOMMU_FD \
> + _IOW(VHOST_VIRTIO, 0x7e, struct vhost_vdpa_set_iommufd)
> +
> +/*
> + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> + *
> + * Attach a vdpa device to an iommufd address space specified by IOAS
> + * id.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> + *
> + * @argsz: user filled size of this data.
> + * @flags: must be 0.
> + * @ioas_id: Input the target id which can represent an ioas
> + * allocated via iommufd subsystem.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_attach_iommufd_as {
> + __u32 argsz;
> + __u32 flags;
> + __u32 ioas_id;
> +};
I don't know this at all, but it seems really weird that the API
taking in the iommfd is more specific (eg includes the group_id) than
the API taking in the ioas.
I'd expect that you have one iommfd setup per virtio FD.
And the various virtio queues would be able to be linked to their own
IOAS.
I'm not sure multiple iommufd's per virito FD is worthwhile.
Jason
^ permalink raw reply [flat|nested] 24+ messages in thread* Re: [RFC 3/7] vhost: Add 3 new uapi to support iommufd
2023-09-23 17:05 ` [RFC 3/7] vhost: Add 3 new uapi to support iommufd Cindy Lu
2023-09-25 13:49 ` Jason Gunthorpe
@ 2023-09-26 2:47 ` Jason Wang
1 sibling, 0 replies; 24+ messages in thread
From: Jason Wang @ 2023-09-26 2:47 UTC (permalink / raw)
To: Cindy Lu; +Cc: mst, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 1:05 AM Cindy Lu <lulu@redhat.com> wrote:
>
> VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
>
> VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> address space specified by IOAS id.
>
> VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> from the iommufd address space
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
> drivers/vhost/vdpa.c | 191 +++++++++++++++++++++++++++++++++++++
> include/uapi/linux/vhost.h | 71 ++++++++++++++
> 2 files changed, 262 insertions(+)
>
> diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
> index ec32f785dfde..91da012084e9 100644
> --- a/drivers/vhost/vdpa.c
> +++ b/drivers/vhost/vdpa.c
> @@ -18,6 +18,7 @@
> #include <linux/mm.h>
> #include <linux/slab.h>
> #include <linux/iommu.h>
> +#include <linux/iommufd.h>
> #include <linux/uuid.h>
> #include <linux/vdpa.h>
> #include <linux/nospec.h>
> @@ -25,6 +26,8 @@
>
> #include "vhost.h"
>
> +MODULE_IMPORT_NS(IOMMUFD);
> +
> enum {
> VHOST_VDPA_BACKEND_FEATURES =
> (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
> @@ -69,6 +72,15 @@ static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
> struct vhost_iotlb *iotlb, u64 start,
> u64 last, u32 asid);
>
> +void vhost_vdpa_lockdep_assert_held(struct vdpa_device *vdpa)
> +{
> + struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
> +
> + if (WARN_ON(!v))
> + return;
> + lockdep_assert_held(&v->vdev.mutex);
> +}
> +
> static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
> {
> struct vhost_vdpa_as *as = container_of(iotlb, struct
> @@ -497,6 +509,173 @@ static long vhost_vdpa_suspend(struct vhost_vdpa *v)
>
> return ops->suspend(vdpa);
> }
> +static long vhost_vdpa_tmp_set_iommufd(struct vhost_vdpa *v, void __user *argp)
> +{
> + struct device *dma_dev = vdpa_get_dma_dev(v->vdpa);
> + struct vhost_vdpa_set_iommufd set_iommufd;
> + struct vdpa_device *vdpa = v->vdpa;
> + struct iommufd_ctx *ictx;
> + unsigned long minsz;
> + u32 pt_id, dev_id;
> + struct fd f;
> + long r = 0;
> + int idx;
> +
> + minsz = offsetofend(struct vhost_vdpa_set_iommufd, ioas_id);
> + if (copy_from_user(&set_iommufd, argp, minsz))
> + return -EFAULT;
> +
> + if (set_iommufd.group_id >= v->nvqs)
> + return -ENOBUFS;
Needs to be v->ngroups but as replied below, but I think it should be asid.
> +
> + idx = array_index_nospec(set_iommufd.group_id, v->nvqs);
> +
> + /* Unset IOMMUFD */
> + if (set_iommufd.iommufd < 0) {
> + if (!test_bit(idx, vdpa->vq_bitmap))
> + return -EINVAL;
> +
> + if (!vdpa->iommufd_ictx || !vdpa->iommufd_device)
> + return -EINVAL;
> + if (atomic_read(&vdpa->iommufd_users)) {
> + atomic_dec(&vdpa->iommufd_users);
> + return 0;
> + }
> + vdpa_iommufd_unbind(v->vdpa);
> + vdpa->iommufd_device = NULL;
> + vdpa->iommufd_ictx = NULL;
> + clear_bit(idx, vdpa->vq_bitmap);
> + return iommu_attach_device(v->domain, dma_dev);
> + }
> + /* First opened virtqueue of this vdpa device */
> +
> + if (!vdpa->vq_bitmap) {
> + vdpa->vq_bitmap = bitmap_alloc(v->nvqs, GFP_KERNEL);
> + }
> + ///if (test_bit(idx, vdpa->vq_bitmap))
> + //return -EBUSY;
> +
> + /* For same device but different groups, ++refcount only */
> + if (vdpa->iommufd_device)
> + goto out_inc;
> +
> + r = -EBADF;
> + f = fdget(set_iommufd.iommufd);
> + if (!f.file)
> + goto out_bitmap_free;
> +
> + r = -EINVAL;
> + ictx = iommufd_ctx_from_file(f.file);
> + if (IS_ERR(ictx))
> + goto out_fdput;
> +
> + if (v->domain)
> + iommu_detach_device(v->domain, dma_dev);
> +
> +#if 0
> + iommu_group_add_device(iommu_group_alloc(), &vdpa->dev);
> +#endif
> + pt_id = set_iommufd.ioas_id;
> + r = vdpa_iommufd_bind(vdpa, ictx, &pt_id, &dev_id);
> + if (r)
> + goto out_reattach;
> +
> + set_iommufd.out_dev_id = dev_id;
> + set_iommufd.out_hwpt_id = pt_id;
> + r = copy_to_user(argp + minsz, &set_iommufd.out_dev_id,
> + sizeof(set_iommufd.out_dev_id) +
> + sizeof(set_iommufd.out_hwpt_id)) ?
> + -EFAULT :
> + 0;
> + if (r)
> + goto out_device_unbind;
> + printk(KERN_ERR "[%s] %d called %p\n", __func__, __LINE__,
> + vdpa->iommufd_ictx);
> +
> + vdpa->iommufd_ictx = ictx;
> +
> +out_inc:
> + atomic_inc(&vdpa->iommufd_users);
> + set_bit(idx, vdpa->vq_bitmap);
> +
> + goto out_fdput;
> +
> +out_device_unbind:
> +
> + vdpa_iommufd_unbind(vdpa);
> +out_reattach:
> +
> + iommu_attach_device(v->domain, dma_dev);
> + iommufd_ctx_put(ictx);
> +out_fdput:
> + fdput(f);
> +out_bitmap_free:
> + bitmap_free(vdpa->vq_bitmap);
> + return r;
> +}
> +int vdpa_ioctl_device_attach(struct vhost_vdpa *v, void __user *arg)
> +{
> + struct vdpa_device_attach_iommufd_as attach;
> + unsigned long minsz;
> + int ret;
> +
> + minsz = offsetofend(struct vdpa_device_attach_iommufd_as, ioas_id);
> +
> + if (copy_from_user(&attach, (void __user *)arg, minsz))
> + return -EFAULT;
> +
> + if (attach.argsz < minsz || attach.flags ||
> + attach.ioas_id == IOMMUFD_INVALID_ID)
> + return -EINVAL;
> +
> + if (!v->vdpa->config->bind_iommufd)
> + return -ENODEV;
> +
> + if (!v->vdpa->iommufd_ictx) {
> + ret = -EINVAL;
> + return ret;
> + }
> +
> + ret = v->vdpa->config->attach_ioas(v->vdpa, &attach.ioas_id);
> +
> + if (ret)
> + return ret;
> +
> + ret = copy_to_user(
> + (void __user *)arg +
> + offsetofend(struct vdpa_device_attach_iommufd_as,
> + flags),
> + &attach.ioas_id, sizeof(attach.ioas_id)) ?
> + -EFAULT :
> + 0;
> +
> + if (ret)
> + return ret;
> +
> + return 0;
> +}
> +
> +int vdpa_ioctl_device_detach(struct vhost_vdpa *v, void __user *arg)
> +{
> + struct vdpa_device_detach_iommufd_as detach;
> + unsigned long minsz;
> +
> + minsz = offsetofend(struct vdpa_device_detach_iommufd_as, flags);
> +
> + if (copy_from_user(&detach, (void __user *)arg, minsz))
> + return -EFAULT;
> +
> + if (detach.argsz < minsz || detach.flags)
> + return -EINVAL;
> +
> + if (!v->vdpa->config->bind_iommufd)
> + return -ENODEV;
> +
> + if (v->vdpa->iommufd_ictx) {
> + return -EINVAL;
> + }
> + return v->vdpa->config->detach_ioas(v->vdpa);
> +}
>
> static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
> void __user *argp)
> @@ -655,6 +834,18 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
> case VHOST_SET_LOG_FD:
> r = -ENOIOCTLCMD;
> break;
> + case VHOST_VDPA_SET_IOMMU_FD:
> +
> + r = vhost_vdpa_tmp_set_iommufd(v, argp);
> + break;
> + case VDPA_DEVICE_ATTACH_IOMMUFD_AS:
> + r = vdpa_ioctl_device_attach(v, (void __user *)arg);
> + break;
> +
> + case VDPA_DEVICE_DETACH_IOMMUFD_AS:
> + r = vdpa_ioctl_device_detach(v, (void __user *)arg);
> + break;
> +
> case VHOST_VDPA_SET_CONFIG_CALL:
> r = vhost_vdpa_set_config_call(v, argp);
> break;
> diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> index f9f115a7c75b..cdda0c1860d8 100644
> --- a/include/uapi/linux/vhost.h
> +++ b/include/uapi/linux/vhost.h
> @@ -45,6 +45,7 @@
> #define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64)
> /* Specify an eventfd file descriptor to signal on log write. */
> #define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int)
> +/* Specify an iommufd file descriptor for IO address translation */
>
> /* Ring setup. */
> /* Set number of descriptors in ring. This parameter can not
> @@ -180,4 +181,74 @@
> */
> #define VHOST_VDPA_SUSPEND _IO(VHOST_VIRTIO, 0x7D)
>
> +/* vhost vdpa set iommufd
> + * Input parameters:
> + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> + * @group_id: identifier of the group that a virtqueue belongs to
> + * @ioas_id: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> + * Output parameters:
> + * @out_dev_id: device identifier
> + * @out_hwpt_id: hardware IO pagetable identifier
> + */
> +struct vhost_vdpa_set_iommufd {
> + __s32 iommufd;
> + __u32 group_id;
This needs to be ASID of a vdpa device, as the current address space
could be shared among various groups. And the device do translation
based on:
virtqueue -> group -> AS -> IOAS of iommufd
Thanks
> + __u32 ioas_id;
> + __u32 out_dev_id;
> + __u32 out_hwpt_id;
> +};
> +
> +#define VHOST_VDPA_SET_IOMMU_FD \
> + _IOW(VHOST_VIRTIO, 0x7e, struct vhost_vdpa_set_iommufd)
> +
> +/*
> + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> + *
> + * Attach a vdpa device to an iommufd address space specified by IOAS
> + * id.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> + *
> + * @argsz: user filled size of this data.
> + * @flags: must be 0.
> + * @ioas_id: Input the target id which can represent an ioas
> + * allocated via iommufd subsystem.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_attach_iommufd_as {
> + __u32 argsz;
> + __u32 flags;
> + __u32 ioas_id;
> +};
> +
> +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> + _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> +
> +/*
> + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> + *
> + * Detach a vdpa device from the iommufd address space it has been
> + * attached to. After it, device should be in a blocking DMA state.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * @argsz: user filled size of this data.
> + * @flags: must be 0.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_detach_iommufd_as {
> + __u32 argsz;
> + __u32 flags;
> +};
> +
> +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> + _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> +
> #endif
> --
> 2.34.3
>
^ permalink raw reply [flat|nested] 24+ messages in thread
* [RFC 4/7] vdpa: change the map/unmap process to support iommufd
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
` (2 preceding siblings ...)
2023-09-23 17:05 ` [RFC 3/7] vhost: Add 3 new uapi to support iommufd Cindy Lu
@ 2023-09-23 17:05 ` Cindy Lu
2023-09-25 13:45 ` Jason Gunthorpe
2023-09-23 17:05 ` [RFC 5/7] vdpa: Add new vdpa_config_ops Cindy Lu
` (4 subsequent siblings)
8 siblings, 1 reply; 24+ messages in thread
From: Cindy Lu @ 2023-09-23 17:05 UTC (permalink / raw)
To: lulu, jasowang, mst, yi.l.liu, jgg, linux-kernel, virtualization,
netdev
Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
then will use the Legacy iommu domain pathway
Signed-off-by: Cindy Lu <lulu@redhat.com>
---
drivers/vhost/vdpa.c | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 91da012084e9..8d1ad89d4671 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -981,6 +981,10 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
} else if (ops->set_map) {
if (!v->in_batch)
r = ops->set_map(vdpa, asid, iotlb);
+ } else if (!vdpa->iommufd_ictx) {
+ /* Legacy iommu domain pathway without IOMMUFD */
+ r = iommu_map(v->domain, iova, pa, size,
+ perm_to_iommu_flags(perm));
} else {
r = iommu_map(v->domain, iova, pa, size,
perm_to_iommu_flags(perm));
@@ -1009,6 +1013,9 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
if (ops->set_map) {
if (!v->in_batch)
ops->set_map(vdpa, asid, iotlb);
+ } else if (!vdpa->iommufd_ictx) {
+ /* Legacy iommu domain pathway without IOMMUFD */
+ iommu_unmap(v->domain, iova, size);
}
/* If we are in the middle of batch processing, delay the free
* of AS until BATCH_END.
@@ -1337,6 +1344,8 @@ static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
iommu_detach_device(v->domain, dma_dev);
iommu_domain_free(v->domain);
}
+ if (vdpa->iommufd_ictx)
+ vdpa_iommufd_unbind(vdpa);
v->domain = NULL;
}
@@ -1560,6 +1569,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa)
}
atomic_set(&v->opened, 0);
+ atomic_set(&vdpa->iommufd_users, 0);
v->minor = minor;
v->vdpa = vdpa;
v->nvqs = vdpa->nvqs;
--
2.34.3
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [RFC 4/7] vdpa: change the map/unmap process to support iommufd
2023-09-23 17:05 ` [RFC 4/7] vdpa: change the map/unmap process " Cindy Lu
@ 2023-09-25 13:45 ` Jason Gunthorpe
2023-09-26 8:09 ` Cindy Lu
0 siblings, 1 reply; 24+ messages in thread
From: Jason Gunthorpe @ 2023-09-25 13:45 UTC (permalink / raw)
To: Cindy Lu; +Cc: jasowang, mst, yi.l.liu, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 01:05:37AM +0800, Cindy Lu wrote:
> Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
> then will use the Legacy iommu domain pathway
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
> drivers/vhost/vdpa.c | 10 ++++++++++
> 1 file changed, 10 insertions(+)
>
> diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
> index 91da012084e9..8d1ad89d4671 100644
> --- a/drivers/vhost/vdpa.c
> +++ b/drivers/vhost/vdpa.c
> @@ -981,6 +981,10 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
> } else if (ops->set_map) {
> if (!v->in_batch)
> r = ops->set_map(vdpa, asid, iotlb);
> + } else if (!vdpa->iommufd_ictx) {
> + /* Legacy iommu domain pathway without IOMMUFD */
> + r = iommu_map(v->domain, iova, pa, size,
> + perm_to_iommu_flags(perm));
> } else {
> r = iommu_map(v->domain, iova, pa, size,
> perm_to_iommu_flags(perm));
Um, why is the 2nd else the same as the new one?
Jason
^ permalink raw reply [flat|nested] 24+ messages in thread* Re: [RFC 4/7] vdpa: change the map/unmap process to support iommufd
2023-09-25 13:45 ` Jason Gunthorpe
@ 2023-09-26 8:09 ` Cindy Lu
0 siblings, 0 replies; 24+ messages in thread
From: Cindy Lu @ 2023-09-26 8:09 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: jasowang, mst, yi.l.liu, linux-kernel, virtualization, netdev
On Mon, Sep 25, 2023 at 9:45 PM Jason Gunthorpe <jgg@nvidia.com> wrote:
>
> On Sun, Sep 24, 2023 at 01:05:37AM +0800, Cindy Lu wrote:
> > Add the check for iommufd_ictx,If vdpa don't have the iommufd_ictx
> > then will use the Legacy iommu domain pathway
> >
> > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > ---
> > drivers/vhost/vdpa.c | 10 ++++++++++
> > 1 file changed, 10 insertions(+)
> >
> > diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
> > index 91da012084e9..8d1ad89d4671 100644
> > --- a/drivers/vhost/vdpa.c
> > +++ b/drivers/vhost/vdpa.c
> > @@ -981,6 +981,10 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
> > } else if (ops->set_map) {
> > if (!v->in_batch)
> > r = ops->set_map(vdpa, asid, iotlb);
> > + } else if (!vdpa->iommufd_ictx) {
> > + /* Legacy iommu domain pathway without IOMMUFD */
> > + r = iommu_map(v->domain, iova, pa, size,
> > + perm_to_iommu_flags(perm));
> > } else {
> > r = iommu_map(v->domain, iova, pa, size,
> > perm_to_iommu_flags(perm));
>
> Um, why is the 2nd else the same as the new one?
>
the code here seems not in a good logic， I will try to optimize it in
the next version
Thanks
Cindy
> Jason
>
^ permalink raw reply [flat|nested] 24+ messages in thread
* [RFC 5/7] vdpa: Add new vdpa_config_ops
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
` (3 preceding siblings ...)
2023-09-23 17:05 ` [RFC 4/7] vdpa: change the map/unmap process " Cindy Lu
@ 2023-09-23 17:05 ` Cindy Lu
2023-09-26 2:47 ` Jason Wang
2023-09-23 17:05 ` [RFC 6/7] vdpa_sim :Add support for iommufd Cindy Lu
` (3 subsequent siblings)
8 siblings, 1 reply; 24+ messages in thread
From: Cindy Lu @ 2023-09-23 17:05 UTC (permalink / raw)
To: lulu, jasowang, mst, yi.l.liu, jgg, linux-kernel, virtualization,
netdev
Add new vdpa_config_ops to support iommufd
Signed-off-by: Cindy Lu <lulu@redhat.com>
---
include/linux/vdpa.h | 34 +++++++++++++++++++++++++++++++++-
1 file changed, 33 insertions(+), 1 deletion(-)
diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
index 6d0f5e4e82c2..4ada5bd6f90e 100644
--- a/include/linux/vdpa.h
+++ b/include/linux/vdpa.h
@@ -5,6 +5,7 @@
#include <linux/kernel.h>
#include <linux/device.h>
#include <linux/interrupt.h>
+#include <linux/iommufd.h>
#include <linux/vhost_iotlb.h>
#include <linux/virtio_net.h>
#include <linux/if_ether.h>
@@ -91,6 +92,12 @@ struct vdpa_device {
struct vdpa_mgmt_dev *mdev;
unsigned int ngroups;
unsigned int nas;
+ struct iommufd_access *iommufd_access;
+ struct iommufd_device *iommufd_device;
+ struct iommufd_ctx *iommufd_ictx;
+ unsigned long *vq_bitmap;
+ atomic_t iommufd_users;
+ bool iommufd_attached;
};
/**
@@ -282,6 +289,15 @@ struct vdpa_map_file {
* @iova: iova to be unmapped
* @size: size of the area
* Returns integer: success (0) or error (< 0)
+ * @bind_iommufd: use vdpa_iommufd_physical_bind for an IOMMU
+ * backed device.
+ * otherwise use vdpa_iommufd_emulated_bind
+ * @unbind_iommufd: use vdpa_iommufd_physical_unbind for an IOMMU
+ * backed device.
+ * otherwise, use vdpa_iommufd_emulated_unbind
+ * @attach_ioas: use vdpa_iommufd_physical_attach_ioas for an
+ * IOMMU backed device.
+ * @detach_ioas: Opposite of attach_ioas
* @free: Free resources that belongs to vDPA (optional)
* @vdev: vdpa device
*/
@@ -341,6 +357,12 @@ struct vdpa_config_ops {
u64 iova, u64 size);
int (*set_group_asid)(struct vdpa_device *vdev, unsigned int group,
unsigned int asid);
+ /* IOMMUFD ops */
+ int (*bind_iommufd)(struct vdpa_device *vdev, struct iommufd_ctx *ictx,
+ u32 *out_device_id);
+ void (*unbind_iommufd)(struct vdpa_device *vdev);
+ int (*attach_ioas)(struct vdpa_device *vdev, u32 *pt_id);
+ int (*detach_ioas)(struct vdpa_device *vdev);
/* Free device resources */
void (*free)(struct vdpa_device *vdev);
@@ -510,4 +532,14 @@ struct vdpa_mgmt_dev {
int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev);
void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev);
-#endif /* _LINUX_VDPA_H */
+int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
+ struct iommufd_ctx *ictx, u32 *out_device_id);
+void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa);
+int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
+int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
+ struct iommufd_ctx *ictx, u32 *out_device_id);
+void vdpa_iommufd_emulated_unbind(struct vdpa_device *vdpa);
+int vdpa_iommufd_emulated_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
+int vdpa_iommufd_emulated_detach_ioas(struct vdpa_device *vdpa);
+
+#endif
--
2.34.3
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [RFC 5/7] vdpa: Add new vdpa_config_ops
2023-09-23 17:05 ` [RFC 5/7] vdpa: Add new vdpa_config_ops Cindy Lu
@ 2023-09-26 2:47 ` Jason Wang
0 siblings, 0 replies; 24+ messages in thread
From: Jason Wang @ 2023-09-26 2:47 UTC (permalink / raw)
To: Cindy Lu; +Cc: mst, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 1:06 AM Cindy Lu <lulu@redhat.com> wrote:
>
> Add new vdpa_config_ops to support iommufd
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
> include/linux/vdpa.h | 34 +++++++++++++++++++++++++++++++++-
> 1 file changed, 33 insertions(+), 1 deletion(-)
>
> diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
> index 6d0f5e4e82c2..4ada5bd6f90e 100644
> --- a/include/linux/vdpa.h
> +++ b/include/linux/vdpa.h
> @@ -5,6 +5,7 @@
> #include <linux/kernel.h>
> #include <linux/device.h>
> #include <linux/interrupt.h>
> +#include <linux/iommufd.h>
> #include <linux/vhost_iotlb.h>
> #include <linux/virtio_net.h>
> #include <linux/if_ether.h>
> @@ -91,6 +92,12 @@ struct vdpa_device {
> struct vdpa_mgmt_dev *mdev;
> unsigned int ngroups;
> unsigned int nas;
> + struct iommufd_access *iommufd_access;
> + struct iommufd_device *iommufd_device;
> + struct iommufd_ctx *iommufd_ictx;
> + unsigned long *vq_bitmap;
> + atomic_t iommufd_users;
> + bool iommufd_attached;
> };
>
> /**
> @@ -282,6 +289,15 @@ struct vdpa_map_file {
> * @iova: iova to be unmapped
> * @size: size of the area
> * Returns integer: success (0) or error (< 0)
> + * @bind_iommufd: use vdpa_iommufd_physical_bind for an IOMMU
> + * backed device.
> + * otherwise use vdpa_iommufd_emulated_bind
> + * @unbind_iommufd: use vdpa_iommufd_physical_unbind for an IOMMU
> + * backed device.
> + * otherwise, use vdpa_iommufd_emulated_unbind
> + * @attach_ioas: use vdpa_iommufd_physical_attach_ioas for an
> + * IOMMU backed device.
> + * @detach_ioas: Opposite of attach_ioas
Those should be marked as mandatory only for parents with specific
translations (e.g simulator and mlx5_vdpa).
Or anything I missed?
Thanks
> * @free: Free resources that belongs to vDPA (optional)
> * @vdev: vdpa device
> */
> @@ -341,6 +357,12 @@ struct vdpa_config_ops {
> u64 iova, u64 size);
> int (*set_group_asid)(struct vdpa_device *vdev, unsigned int group,
> unsigned int asid);
> + /* IOMMUFD ops */
> + int (*bind_iommufd)(struct vdpa_device *vdev, struct iommufd_ctx *ictx,
> + u32 *out_device_id);
> + void (*unbind_iommufd)(struct vdpa_device *vdev);
> + int (*attach_ioas)(struct vdpa_device *vdev, u32 *pt_id);
> + int (*detach_ioas)(struct vdpa_device *vdev);
>
> /* Free device resources */
> void (*free)(struct vdpa_device *vdev);
> @@ -510,4 +532,14 @@ struct vdpa_mgmt_dev {
> int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev);
> void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev);
>
> -#endif /* _LINUX_VDPA_H */
> +int vdpa_iommufd_physical_bind(struct vdpa_device *vdpa,
> + struct iommufd_ctx *ictx, u32 *out_device_id);
> +void vdpa_iommufd_physical_unbind(struct vdpa_device *vdpa);
> +int vdpa_iommufd_physical_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
> +int vdpa_iommufd_emulated_bind(struct vdpa_device *vdpa,
> + struct iommufd_ctx *ictx, u32 *out_device_id);
> +void vdpa_iommufd_emulated_unbind(struct vdpa_device *vdpa);
> +int vdpa_iommufd_emulated_attach_ioas(struct vdpa_device *vdpa, u32 *pt_id);
> +int vdpa_iommufd_emulated_detach_ioas(struct vdpa_device *vdpa);
> +
> +#endif
> --
> 2.34.3
>
^ permalink raw reply [flat|nested] 24+ messages in thread
* [RFC 6/7] vdpa_sim :Add support for iommufd
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
` (4 preceding siblings ...)
2023-09-23 17:05 ` [RFC 5/7] vdpa: Add new vdpa_config_ops Cindy Lu
@ 2023-09-23 17:05 ` Cindy Lu
2023-09-25 13:46 ` Jason Gunthorpe
2023-09-23 17:05 ` [RFC 7/7] iommufd: Skip the CACHE_COHERENCY and iommu group check Cindy Lu
` (2 subsequent siblings)
8 siblings, 1 reply; 24+ messages in thread
From: Cindy Lu @ 2023-09-23 17:05 UTC (permalink / raw)
To: lulu, jasowang, mst, yi.l.liu, jgg, linux-kernel, virtualization,
netdev
Add new vdpa_config_ops to support iommufd
Signed-off-by: Cindy Lu <lulu@redhat.com>
---
drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index cb88891b44a8..55e6f45bb274 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -727,6 +727,10 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
.dma_map = vdpasim_dma_map,
.dma_unmap = vdpasim_dma_unmap,
.free = vdpasim_free,
+ .bind_iommufd = vdpa_iommufd_emulated_bind,
+ .unbind_iommufd = vdpa_iommufd_emulated_unbind,
+ .attach_ioas = vdpa_iommufd_emulated_attach_ioas,
+ .detach_ioas = vdpa_iommufd_emulated_detach_ioas,
};
static const struct vdpa_config_ops vdpasim_batch_config_ops = {
@@ -759,6 +763,10 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
.set_group_asid = vdpasim_set_group_asid,
.set_map = vdpasim_set_map,
.free = vdpasim_free,
+ .bind_iommufd = vdpa_iommufd_emulated_bind,
+ .unbind_iommufd = vdpa_iommufd_emulated_unbind,
+ .attach_ioas = vdpa_iommufd_emulated_attach_ioas,
+ .detach_ioas = vdpa_iommufd_emulated_detach_ioas,
};
MODULE_VERSION(DRV_VERSION);
--
2.34.3
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [RFC 6/7] vdpa_sim :Add support for iommufd
2023-09-23 17:05 ` [RFC 6/7] vdpa_sim :Add support for iommufd Cindy Lu
@ 2023-09-25 13:46 ` Jason Gunthorpe
0 siblings, 0 replies; 24+ messages in thread
From: Jason Gunthorpe @ 2023-09-25 13:46 UTC (permalink / raw)
To: Cindy Lu; +Cc: jasowang, mst, yi.l.liu, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 01:05:39AM +0800, Cindy Lu wrote:
> Add new vdpa_config_ops to support iommufd
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
> drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++++++++
> 1 file changed, 8 insertions(+)
>
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> index cb88891b44a8..55e6f45bb274 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> @@ -727,6 +727,10 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
> .dma_map = vdpasim_dma_map,
> .dma_unmap = vdpasim_dma_unmap,
> .free = vdpasim_free,
> + .bind_iommufd = vdpa_iommufd_emulated_bind,
> + .unbind_iommufd = vdpa_iommufd_emulated_unbind,
> + .attach_ioas = vdpa_iommufd_emulated_attach_ioas,
> + .detach_ioas = vdpa_iommufd_emulated_detach_ioas,
> };
Er, this series never calls iommufd_access_pin_pages() or
iommufd_access_rw(), so something pretty big is missing
Jason
^ permalink raw reply [flat|nested] 24+ messages in thread
* [RFC 7/7] iommufd: Skip the CACHE_COHERENCY and iommu group check
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
` (5 preceding siblings ...)
2023-09-23 17:05 ` [RFC 6/7] vdpa_sim :Add support for iommufd Cindy Lu
@ 2023-09-23 17:05 ` Cindy Lu
2023-09-25 13:50 ` Jason Gunthorpe
2023-09-26 2:46 ` [RFC 0/7] vdpa: Add support for iommufd Jason Wang
2023-10-26 6:42 ` Michael S. Tsirkin
8 siblings, 1 reply; 24+ messages in thread
From: Cindy Lu @ 2023-09-23 17:05 UTC (permalink / raw)
To: lulu, jasowang, mst, yi.l.liu, jgg, linux-kernel, virtualization,
netdev
This is just the work arround for vdpa, I Will
fix these problems in the next version.
Skip these 2 checks:
1.IOMMU_CAP_CACHE_COHERENCY check
2.iommu_group_get check
Signed-off-by: Cindy Lu <lulu@redhat.com>
---
drivers/iommu/iommufd/device.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index f7cb353fd9c7..0224d751f503 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -71,8 +71,8 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
* to restore cache coherency.
*/
if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY))
- return ERR_PTR(-EINVAL);
-
+ //return ERR_PTR(-EINVAL);
+#if 0
group = iommu_group_get(dev);
if (!group)
return ERR_PTR(-ENODEV);
@@ -80,7 +80,7 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
rc = iommu_device_claim_dma_owner(dev, ictx);
if (rc)
goto out_group_put;
-
+#endif
idev = iommufd_object_alloc(ictx, idev, IOMMUFD_OBJ_DEVICE);
if (IS_ERR(idev)) {
rc = PTR_ERR(idev);
--
2.34.3
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [RFC 7/7] iommufd: Skip the CACHE_COHERENCY and iommu group check
2023-09-23 17:05 ` [RFC 7/7] iommufd: Skip the CACHE_COHERENCY and iommu group check Cindy Lu
@ 2023-09-25 13:50 ` Jason Gunthorpe
2023-09-26 8:02 ` Cindy Lu
0 siblings, 1 reply; 24+ messages in thread
From: Jason Gunthorpe @ 2023-09-25 13:50 UTC (permalink / raw)
To: Cindy Lu; +Cc: jasowang, mst, yi.l.liu, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 01:05:40AM +0800, Cindy Lu wrote:
> This is just the work arround for vdpa, I Will
> fix these problems in the next version.
>
> Skip these 2 checks:
> 1.IOMMU_CAP_CACHE_COHERENCY check
> 2.iommu_group_get check
Uuh, something has gone really, really wrong if you need to skip these
checks in the core iommufd code..
Jason
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [RFC 7/7] iommufd: Skip the CACHE_COHERENCY and iommu group check
2023-09-25 13:50 ` Jason Gunthorpe
@ 2023-09-26 8:02 ` Cindy Lu
0 siblings, 0 replies; 24+ messages in thread
From: Cindy Lu @ 2023-09-26 8:02 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: jasowang, mst, yi.l.liu, linux-kernel, virtualization, netdev
On Mon, Sep 25, 2023 at 9:50 PM Jason Gunthorpe <jgg@nvidia.com> wrote:
>
> On Sun, Sep 24, 2023 at 01:05:40AM +0800, Cindy Lu wrote:
> > This is just the work arround for vdpa, I Will
> > fix these problems in the next version.
> >
> > Skip these 2 checks:
> > 1.IOMMU_CAP_CACHE_COHERENCY check
> > 2.iommu_group_get check
>
> Uuh, something has gone really, really wrong if you need to skip these
> checks in the core iommufd code..
>
there are problems in this code, I will continue working in this.
Thanks
Cindy
> Jason
>
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [RFC 0/7] vdpa: Add support for iommufd
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
` (6 preceding siblings ...)
2023-09-23 17:05 ` [RFC 7/7] iommufd: Skip the CACHE_COHERENCY and iommu group check Cindy Lu
@ 2023-09-26 2:46 ` Jason Wang
2023-10-26 6:42 ` Michael S. Tsirkin
8 siblings, 0 replies; 24+ messages in thread
From: Jason Wang @ 2023-09-26 2:46 UTC (permalink / raw)
To: Cindy Lu; +Cc: mst, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 1:05 AM Cindy Lu <lulu@redhat.com> wrote:
>
> Hi All
> Really apologize for the delay, this is the draft RFC for
> iommufd support for vdpa, This code provides the basic function
> for iommufd support
>
> The code was tested and passed in device vdpa_sim_net
> The qemu code is
> https://gitlab.com/lulu6/gitlabqemutmp/-/tree/iommufdRFC
> The kernel code is
> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC
>
> ToDo
> 1. this code is out of date and needs to clean and rebase on the latest code
> 2. this code has some workaround, I Skip the check for
> iommu_group and CACHE_COHERENCY, also some misc issues like need to add
> mutex for iommfd operations
> 3. only test in emulated device, other modes not tested yet
>
> After addressed these problems I will send out a new version for RFC. I will
> provide the code in 3 weeks
Something more needs to be done after a quick glance at the codes.
1) The support for device with platform IOMMU support
2) The support for multiple ASes per device
...
Thanks
>
> Thanks
> Cindy
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> The test step is
> 1. create vdpa_sim device
> ...
> vdpa dev add name vdpa15 mgmtdev vdpasim_net
> ...
> 2. load the VM with the command
> -object iommufd,id=iommufd0 \
> -device virtio-net-pci,netdev=vhost-vdpa1,disable-legacy=on,disable-modern=off\
> -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-0,id=vhost-vdpa1,iommufd=iommufd0\
>
> 3. in guest VM you can find the vdpa_sim port works well.
> [root@ubuntunew ~]# ifconfig eth0
> eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
> inet6 fe80::5054:ff:fe12:3456 prefixlen 64 scopeid 0x20<link>
> ether 52:54:00:12:34:56 txqueuelen 1000 (Ethernet)
> RX packets 53 bytes 9108 (8.8 KiB)
> RX errors 0 dropped 0 overruns 0 frame 0
> TX packets 53 bytes 9108 (8.8 KiB)
> TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
>
> [root@ubuntunew ~]# ./test.sh eth0
> [ 172.815279] pktgen: Packet Generator for packet performance testing. Version: 2.75
> Adding queue 0 of eth0
> Configuring devices eth0@0
> Running... ctrl^C to stop
>
> [root@ubuntunew ~]# ifconfig eth0
> eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
> inet6 fe80::5054:ff:fe12:3456 prefixlen 64 scopeid 0x20<link>
> ether 52:54:00:12:34:56 txqueuelen 1000 (Ethernet)
> RX packets 183455 bytes 11748533 (11.2 MiB)
> RX errors 0 dropped 0 overruns 0 frame 0
> TX packets 183473 bytes 11749685 (11.2 MiB)
> TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
>
> Cindy Lu (7):
> vhost/iommufd: Add the functions support iommufd
> Kconfig: Add the new file vhost/iommufd
> vhost: Add 3 new uapi to support iommufd
> vdpa: change the map/unmap process to support iommufd
> vdpa: Add new vdpa_config_ops
> vdpa_sim :Add support for iommufd
> iommufd: Skip the CACHE_COHERENCY and iommu group check
>
> drivers/iommu/iommufd/device.c | 6 +-
> drivers/vdpa/vdpa_sim/vdpa_sim.c | 8 ++
> drivers/vhost/Kconfig | 1 +
> drivers/vhost/Makefile | 1 +
> drivers/vhost/iommufd.c | 151 +++++++++++++++++++++++
> drivers/vhost/vdpa.c | 201 +++++++++++++++++++++++++++++++
> drivers/vhost/vhost.h | 21 ++++
> include/linux/vdpa.h | 34 +++++-
> include/uapi/linux/vhost.h | 71 +++++++++++
> 9 files changed, 490 insertions(+), 4 deletions(-)
> create mode 100644 drivers/vhost/iommufd.c
>
> --
> 2.34.3
>
^ permalink raw reply [flat|nested] 24+ messages in thread* Re: [RFC 0/7] vdpa: Add support for iommufd
2023-09-23 17:05 [RFC 0/7] vdpa: Add support for iommufd Cindy Lu
` (7 preceding siblings ...)
2023-09-26 2:46 ` [RFC 0/7] vdpa: Add support for iommufd Jason Wang
@ 2023-10-26 6:42 ` Michael S. Tsirkin
2023-10-26 6:48 ` Cindy Lu
8 siblings, 1 reply; 24+ messages in thread
From: Michael S. Tsirkin @ 2023-10-26 6:42 UTC (permalink / raw)
To: Cindy Lu; +Cc: jasowang, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Sun, Sep 24, 2023 at 01:05:33AM +0800, Cindy Lu wrote:
> Hi All
> Really apologize for the delay, this is the draft RFC for
> iommufd support for vdpa, This code provides the basic function
> for iommufd support
>
> The code was tested and passed in device vdpa_sim_net
> The qemu code is
> https://gitlab.com/lulu6/gitlabqemutmp/-/tree/iommufdRFC
> The kernel code is
> https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC
>
> ToDo
> 1. this code is out of date and needs to clean and rebase on the latest code
> 2. this code has some workaround, I Skip the check for
> iommu_group and CACHE_COHERENCY, also some misc issues like need to add
> mutex for iommfd operations
> 3. only test in emulated device, other modes not tested yet
>
> After addressed these problems I will send out a new version for RFC. I will
> provide the code in 3 weeks
What's the status here?
--
MST
^ permalink raw reply [flat|nested] 24+ messages in thread* Re: [RFC 0/7] vdpa: Add support for iommufd
2023-10-26 6:42 ` Michael S. Tsirkin
@ 2023-10-26 6:48 ` Cindy Lu
2023-10-26 6:49 ` Michael S. Tsirkin
2023-11-02 10:02 ` Michael S. Tsirkin
0 siblings, 2 replies; 24+ messages in thread
From: Cindy Lu @ 2023-10-26 6:48 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: jasowang, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Thu, Oct 26, 2023 at 2:42 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Sun, Sep 24, 2023 at 01:05:33AM +0800, Cindy Lu wrote:
> > Hi All
> > Really apologize for the delay, this is the draft RFC for
> > iommufd support for vdpa, This code provides the basic function
> > for iommufd support
> >
> > The code was tested and passed in device vdpa_sim_net
> > The qemu code is
> > https://gitlab.com/lulu6/gitlabqemutmp/-/tree/iommufdRFC
> > The kernel code is
> > https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC
> >
> > ToDo
> > 1. this code is out of date and needs to clean and rebase on the latest code
> > 2. this code has some workaround, I Skip the check for
> > iommu_group and CACHE_COHERENCY, also some misc issues like need to add
> > mutex for iommfd operations
> > 3. only test in emulated device, other modes not tested yet
> >
> > After addressed these problems I will send out a new version for RFC. I will
> > provide the code in 3 weeks
>
> What's the status here?
>
Hi Michael
The code is finished, but I found some bug after adding the support for ASID,
will post the new version after this bug is fixed, should be next week
Thanks
Cindy
> --
> MST
>
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [RFC 0/7] vdpa: Add support for iommufd
2023-10-26 6:48 ` Cindy Lu
@ 2023-10-26 6:49 ` Michael S. Tsirkin
2023-10-26 7:03 ` Cindy Lu
2023-11-02 10:02 ` Michael S. Tsirkin
1 sibling, 1 reply; 24+ messages in thread
From: Michael S. Tsirkin @ 2023-10-26 6:49 UTC (permalink / raw)
To: Cindy Lu; +Cc: jasowang, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Thu, Oct 26, 2023 at 02:48:07PM +0800, Cindy Lu wrote:
> On Thu, Oct 26, 2023 at 2:42 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> >
> > On Sun, Sep 24, 2023 at 01:05:33AM +0800, Cindy Lu wrote:
> > > Hi All
> > > Really apologize for the delay, this is the draft RFC for
> > > iommufd support for vdpa, This code provides the basic function
> > > for iommufd support
> > >
> > > The code was tested and passed in device vdpa_sim_net
> > > The qemu code is
> > > https://gitlab.com/lulu6/gitlabqemutmp/-/tree/iommufdRFC
> > > The kernel code is
> > > https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC
> > >
> > > ToDo
> > > 1. this code is out of date and needs to clean and rebase on the latest code
> > > 2. this code has some workaround, I Skip the check for
> > > iommu_group and CACHE_COHERENCY, also some misc issues like need to add
> > > mutex for iommfd operations
> > > 3. only test in emulated device, other modes not tested yet
> > >
> > > After addressed these problems I will send out a new version for RFC. I will
> > > provide the code in 3 weeks
> >
> > What's the status here?
> >
> Hi Michael
> The code is finished, but I found some bug after adding the support for ASID,
> will post the new version after this bug is fixed, should be next week
> Thanks
> Cindy
We'll miss this merge window then.
> > --
> > MST
> >
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [RFC 0/7] vdpa: Add support for iommufd
2023-10-26 6:49 ` Michael S. Tsirkin
@ 2023-10-26 7:03 ` Cindy Lu
0 siblings, 0 replies; 24+ messages in thread
From: Cindy Lu @ 2023-10-26 7:03 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: jasowang, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Thu, Oct 26, 2023 at 2:49 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Thu, Oct 26, 2023 at 02:48:07PM +0800, Cindy Lu wrote:
> > On Thu, Oct 26, 2023 at 2:42 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > >
> > > On Sun, Sep 24, 2023 at 01:05:33AM +0800, Cindy Lu wrote:
> > > > Hi All
> > > > Really apologize for the delay, this is the draft RFC for
> > > > iommufd support for vdpa, This code provides the basic function
> > > > for iommufd support
> > > >
> > > > The code was tested and passed in device vdpa_sim_net
> > > > The qemu code is
> > > > https://gitlab.com/lulu6/gitlabqemutmp/-/tree/iommufdRFC
> > > > The kernel code is
> > > > https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC
> > > >
> > > > ToDo
> > > > 1. this code is out of date and needs to clean and rebase on the latest code
> > > > 2. this code has some workaround, I Skip the check for
> > > > iommu_group and CACHE_COHERENCY, also some misc issues like need to add
> > > > mutex for iommfd operations
> > > > 3. only test in emulated device, other modes not tested yet
> > > >
> > > > After addressed these problems I will send out a new version for RFC. I will
> > > > provide the code in 3 weeks
> > >
> > > What's the status here?
> > >
> > Hi Michael
> > The code is finished, but I found some bug after adding the support for ASID,
> > will post the new version after this bug is fixed, should be next week
> > Thanks
> > Cindy
>
>
> We'll miss this merge window then.
>
thanks Micheal， I will try my best. will post the new version as soon as I can
Thanks
Cindy
> > > --
> > > MST
> > >
>
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [RFC 0/7] vdpa: Add support for iommufd
2023-10-26 6:48 ` Cindy Lu
2023-10-26 6:49 ` Michael S. Tsirkin
@ 2023-11-02 10:02 ` Michael S. Tsirkin
2023-11-02 12:09 ` Cindy Lu
1 sibling, 1 reply; 24+ messages in thread
From: Michael S. Tsirkin @ 2023-11-02 10:02 UTC (permalink / raw)
To: Cindy Lu; +Cc: jasowang, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Thu, Oct 26, 2023 at 02:48:07PM +0800, Cindy Lu wrote:
> On Thu, Oct 26, 2023 at 2:42 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> >
> > On Sun, Sep 24, 2023 at 01:05:33AM +0800, Cindy Lu wrote:
> > > Hi All
> > > Really apologize for the delay, this is the draft RFC for
> > > iommufd support for vdpa, This code provides the basic function
> > > for iommufd support
> > >
> > > The code was tested and passed in device vdpa_sim_net
> > > The qemu code is
> > > https://gitlab.com/lulu6/gitlabqemutmp/-/tree/iommufdRFC
> > > The kernel code is
> > > https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC
> > >
> > > ToDo
> > > 1. this code is out of date and needs to clean and rebase on the latest code
> > > 2. this code has some workaround, I Skip the check for
> > > iommu_group and CACHE_COHERENCY, also some misc issues like need to add
> > > mutex for iommfd operations
> > > 3. only test in emulated device, other modes not tested yet
> > >
> > > After addressed these problems I will send out a new version for RFC. I will
> > > provide the code in 3 weeks
> >
> > What's the status here?
> >
> Hi Michael
> The code is finished, but I found some bug after adding the support for ASID,
> will post the new version after this bug is fixed, should be next week
> Thanks
> Cindy
The week is almost gone, what's going on?
> > --
> > MST
> >
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [RFC 0/7] vdpa: Add support for iommufd
2023-11-02 10:02 ` Michael S. Tsirkin
@ 2023-11-02 12:09 ` Cindy Lu
0 siblings, 0 replies; 24+ messages in thread
From: Cindy Lu @ 2023-11-02 12:09 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: jasowang, yi.l.liu, jgg, linux-kernel, virtualization, netdev
On Thu, Nov 2, 2023 at 6:02 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Thu, Oct 26, 2023 at 02:48:07PM +0800, Cindy Lu wrote:
> > On Thu, Oct 26, 2023 at 2:42 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > >
> > > On Sun, Sep 24, 2023 at 01:05:33AM +0800, Cindy Lu wrote:
> > > > Hi All
> > > > Really apologize for the delay, this is the draft RFC for
> > > > iommufd support for vdpa, This code provides the basic function
> > > > for iommufd support
> > > >
> > > > The code was tested and passed in device vdpa_sim_net
> > > > The qemu code is
> > > > https://gitlab.com/lulu6/gitlabqemutmp/-/tree/iommufdRFC
> > > > The kernel code is
> > > > https://gitlab.com/lulu6/vhost/-/tree/iommufdRFC
> > > >
> > > > ToDo
> > > > 1. this code is out of date and needs to clean and rebase on the latest code
> > > > 2. this code has some workaround, I Skip the check for
> > > > iommu_group and CACHE_COHERENCY, also some misc issues like need to add
> > > > mutex for iommfd operations
> > > > 3. only test in emulated device, other modes not tested yet
> > > >
> > > > After addressed these problems I will send out a new version for RFC. I will
> > > > provide the code in 3 weeks
> > >
> > > What's the status here?
> > >
> > Hi Michael
> > The code is finished, but I found some bug after adding the support for ASID,
> > will post the new version after this bug is fixed, should be next week
> > Thanks
> > Cindy
>
> The week is almost gone, what's going on?
>
thanks, Micheal, I will send it out tomorrow
Thanks
Cindy
>
> > > --
> > > MST
> > >
>
^ permalink raw reply [flat|nested] 24+ messages in thread