* [PULL 01/47] vfio: Introduce base object for VFIOContainer and targeted interface
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
@ 2023-12-19 18:55 ` Cédric Le Goater
2023-12-19 18:55 ` [PULL 02/47] vfio/container: Introduce a empty VFIOIOMMUOps Cédric Le Goater
` (46 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:55 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Introduce a dumb VFIOContainerBase object and its targeted interface.
This is willingly not a QOM object because we don't want it to be
visible from the user interface. The VFIOContainerBase will be
smoothly populated in subsequent patches as well as interfaces.
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 8 ++---
include/hw/vfio/vfio-container-base.h | 50 +++++++++++++++++++++++++++
2 files changed, 52 insertions(+), 6 deletions(-)
create mode 100644 include/hw/vfio/vfio-container-base.h
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index a4a22accb9434c5d04bef73172a0ccbe182405cb..586d153c12b58f5eaabe8e36ad91fad6abd4af10 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -30,6 +30,7 @@
#include <linux/vfio.h>
#endif
#include "sysemu/sysemu.h"
+#include "hw/vfio/vfio-container-base.h"
#define VFIO_MSG_PREFIX "vfio %s: "
@@ -81,6 +82,7 @@ typedef struct VFIOAddressSpace {
struct VFIOGroup;
typedef struct VFIOContainer {
+ VFIOContainerBase bcontainer;
VFIOAddressSpace *space;
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
MemoryListener listener;
@@ -201,12 +203,6 @@ typedef struct VFIODisplay {
} dmabuf;
} VFIODisplay;
-typedef struct {
- unsigned long *bitmap;
- hwaddr size;
- hwaddr pages;
-} VFIOBitmap;
-
VFIOAddressSpace *vfio_get_address_space(AddressSpace *as);
void vfio_put_address_space(VFIOAddressSpace *space);
bool vfio_devices_all_running_and_saving(VFIOContainer *container);
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
new file mode 100644
index 0000000000000000000000000000000000000000..1d6daaea5d54775221d36bff98590472e0cf25e2
--- /dev/null
+++ b/include/hw/vfio/vfio-container-base.h
@@ -0,0 +1,50 @@
+/*
+ * VFIO BASE CONTAINER
+ *
+ * Copyright (C) 2023 Intel Corporation.
+ * Copyright Red Hat, Inc. 2023
+ *
+ * Authors: Yi Liu <yi.l.liu@intel.com>
+ * Eric Auger <eric.auger@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef HW_VFIO_VFIO_CONTAINER_BASE_H
+#define HW_VFIO_VFIO_CONTAINER_BASE_H
+
+#include "exec/memory.h"
+
+typedef struct VFIODevice VFIODevice;
+typedef struct VFIOIOMMUOps VFIOIOMMUOps;
+
+typedef struct {
+ unsigned long *bitmap;
+ hwaddr size;
+ hwaddr pages;
+} VFIOBitmap;
+
+/*
+ * This is the base object for vfio container backends
+ */
+typedef struct VFIOContainerBase {
+ const VFIOIOMMUOps *ops;
+} VFIOContainerBase;
+
+struct VFIOIOMMUOps {
+ /* basic feature */
+ int (*dma_map)(VFIOContainerBase *bcontainer,
+ hwaddr iova, ram_addr_t size,
+ void *vaddr, bool readonly);
+ int (*dma_unmap)(VFIOContainerBase *bcontainer,
+ hwaddr iova, ram_addr_t size,
+ IOMMUTLBEntry *iotlb);
+ int (*attach_device)(const char *name, VFIODevice *vbasedev,
+ AddressSpace *as, Error **errp);
+ void (*detach_device)(VFIODevice *vbasedev);
+ /* migration feature */
+ int (*set_dirty_page_tracking)(VFIOContainerBase *bcontainer, bool start);
+ int (*query_dirty_bitmap)(VFIOContainerBase *bcontainer, VFIOBitmap *vbmap,
+ hwaddr iova, hwaddr size);
+};
+#endif /* HW_VFIO_VFIO_CONTAINER_BASE_H */
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 02/47] vfio/container: Introduce a empty VFIOIOMMUOps
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
2023-12-19 18:55 ` [PULL 01/47] vfio: Introduce base object for VFIOContainer and targeted interface Cédric Le Goater
@ 2023-12-19 18:55 ` Cédric Le Goater
2023-12-19 18:55 ` [PULL 03/47] vfio/container: Switch to dma_map|unmap API Cédric Le Goater
` (45 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:55 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
This empty VFIOIOMMUOps named vfio_legacy_ops will hold all general
IOMMU ops of legacy container.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 2 +-
hw/vfio/container.c | 5 +++++
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 586d153c12b58f5eaabe8e36ad91fad6abd4af10..678161f2079b20aae9317d67e5a94696ea269248 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -255,7 +255,7 @@ typedef QLIST_HEAD(VFIOGroupList, VFIOGroup) VFIOGroupList;
typedef QLIST_HEAD(VFIODeviceList, VFIODevice) VFIODeviceList;
extern VFIOGroupList vfio_group_list;
extern VFIODeviceList vfio_device_list;
-
+extern const VFIOIOMMUOps vfio_legacy_ops;
extern const MemoryListener vfio_memory_listener;
extern int vfio_kvm_device_fd;
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 242010036af33faa325a34008af40c2cc67a02ea..4bc43ddfa4441dc76950203903f22e5475574a47 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -472,6 +472,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
Error **errp)
{
VFIOContainer *container;
+ VFIOContainerBase *bcontainer;
int ret, fd;
VFIOAddressSpace *space;
@@ -552,6 +553,8 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
container->iova_ranges = NULL;
QLIST_INIT(&container->giommu_list);
QLIST_INIT(&container->vrdl_list);
+ bcontainer = &container->bcontainer;
+ bcontainer->ops = &vfio_legacy_ops;
ret = vfio_init_container(container, group->fd, errp);
if (ret) {
@@ -933,3 +936,5 @@ void vfio_detach_device(VFIODevice *vbasedev)
vfio_put_base_device(vbasedev);
vfio_put_group(group);
}
+
+const VFIOIOMMUOps vfio_legacy_ops;
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 03/47] vfio/container: Switch to dma_map|unmap API
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
2023-12-19 18:55 ` [PULL 01/47] vfio: Introduce base object for VFIOContainer and targeted interface Cédric Le Goater
2023-12-19 18:55 ` [PULL 02/47] vfio/container: Introduce a empty VFIOIOMMUOps Cédric Le Goater
@ 2023-12-19 18:55 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 04/47] vfio/common: Introduce vfio_container_init/destroy helper Cédric Le Goater
` (44 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:55 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 4 ---
include/hw/vfio/vfio-container-base.h | 7 +++++
hw/vfio/common.c | 45 +++++++++++++++------------
hw/vfio/container-base.c | 32 +++++++++++++++++++
hw/vfio/container.c | 22 ++++++++-----
hw/vfio/meson.build | 1 +
hw/vfio/trace-events | 2 +-
7 files changed, 81 insertions(+), 32 deletions(-)
create mode 100644 hw/vfio/container-base.c
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 678161f2079b20aae9317d67e5a94696ea269248..24a26345e5aa6094706dcf1301eeaf44ac91840d 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -208,10 +208,6 @@ void vfio_put_address_space(VFIOAddressSpace *space);
bool vfio_devices_all_running_and_saving(VFIOContainer *container);
/* container->fd */
-int vfio_dma_unmap(VFIOContainer *container, hwaddr iova,
- ram_addr_t size, IOMMUTLBEntry *iotlb);
-int vfio_dma_map(VFIOContainer *container, hwaddr iova,
- ram_addr_t size, void *vaddr, bool readonly);
int vfio_set_dirty_page_tracking(VFIOContainer *container, bool start);
int vfio_query_dirty_bitmap(VFIOContainer *container, VFIOBitmap *vbmap,
hwaddr iova, hwaddr size);
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 1d6daaea5d54775221d36bff98590472e0cf25e2..56b033f59f13684a56824929c16d9445a6efc5ff 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -31,6 +31,13 @@ typedef struct VFIOContainerBase {
const VFIOIOMMUOps *ops;
} VFIOContainerBase;
+int vfio_container_dma_map(VFIOContainerBase *bcontainer,
+ hwaddr iova, ram_addr_t size,
+ void *vaddr, bool readonly);
+int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
+ hwaddr iova, ram_addr_t size,
+ IOMMUTLBEntry *iotlb);
+
struct VFIOIOMMUOps {
/* basic feature */
int (*dma_map)(VFIOContainerBase *bcontainer,
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index e70fdf5e0cacf756fb8dcce2bfd906c65ae71684..e610771888491c8cd19883cdf91ad1589c843e27 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -292,7 +292,7 @@ static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr,
static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
{
VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n);
- VFIOContainer *container = giommu->container;
+ VFIOContainerBase *bcontainer = &giommu->container->bcontainer;
hwaddr iova = iotlb->iova + giommu->iommu_offset;
void *vaddr;
int ret;
@@ -322,21 +322,22 @@ static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
* of vaddr will always be there, even if the memory object is
* destroyed and its backing memory munmap-ed.
*/
- ret = vfio_dma_map(container, iova,
- iotlb->addr_mask + 1, vaddr,
- read_only);
+ ret = vfio_container_dma_map(bcontainer, iova,
+ iotlb->addr_mask + 1, vaddr,
+ read_only);
if (ret) {
- error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
+ error_report("vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx", %p) = %d (%s)",
- container, iova,
+ bcontainer, iova,
iotlb->addr_mask + 1, vaddr, ret, strerror(-ret));
}
} else {
- ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1, iotlb);
+ ret = vfio_container_dma_unmap(bcontainer, iova,
+ iotlb->addr_mask + 1, iotlb);
if (ret) {
- error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+ error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx") = %d (%s)",
- container, iova,
+ bcontainer, iova,
iotlb->addr_mask + 1, ret, strerror(-ret));
vfio_set_migration_error(ret);
}
@@ -355,9 +356,10 @@ static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl,
int ret;
/* Unmap with a single call. */
- ret = vfio_dma_unmap(vrdl->container, iova, size , NULL);
+ ret = vfio_container_dma_unmap(&vrdl->container->bcontainer,
+ iova, size , NULL);
if (ret) {
- error_report("%s: vfio_dma_unmap() failed: %s", __func__,
+ error_report("%s: vfio_container_dma_unmap() failed: %s", __func__,
strerror(-ret));
}
}
@@ -385,8 +387,8 @@ static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl,
section->offset_within_address_space;
vaddr = memory_region_get_ram_ptr(section->mr) + start;
- ret = vfio_dma_map(vrdl->container, iova, next - start,
- vaddr, section->readonly);
+ ret = vfio_container_dma_map(&vrdl->container->bcontainer, iova,
+ next - start, vaddr, section->readonly);
if (ret) {
/* Rollback */
vfio_ram_discard_notify_discard(rdl, section);
@@ -684,10 +686,11 @@ static void vfio_listener_region_add(MemoryListener *listener,
}
}
- ret = vfio_dma_map(container, iova, int128_get64(llsize),
- vaddr, section->readonly);
+ ret = vfio_container_dma_map(&container->bcontainer,
+ iova, int128_get64(llsize), vaddr,
+ section->readonly);
if (ret) {
- error_setg(&err, "vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
+ error_setg(&err, "vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx", %p) = %d (%s)",
container, iova, int128_get64(llsize), vaddr, ret,
strerror(-ret));
@@ -784,18 +787,20 @@ static void vfio_listener_region_del(MemoryListener *listener,
if (int128_eq(llsize, int128_2_64())) {
/* The unmap ioctl doesn't accept a full 64-bit span. */
llsize = int128_rshift(llsize, 1);
- ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL);
+ ret = vfio_container_dma_unmap(&container->bcontainer, iova,
+ int128_get64(llsize), NULL);
if (ret) {
- error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+ error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx") = %d (%s)",
container, iova, int128_get64(llsize), ret,
strerror(-ret));
}
iova += int128_get64(llsize);
}
- ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL);
+ ret = vfio_container_dma_unmap(&container->bcontainer, iova,
+ int128_get64(llsize), NULL);
if (ret) {
- error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+ error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx") = %d (%s)",
container, iova, int128_get64(llsize), ret,
strerror(-ret));
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
new file mode 100644
index 0000000000000000000000000000000000000000..55d3a35fa42b66e6590da8d699e60850c88ed74d
--- /dev/null
+++ b/hw/vfio/container-base.c
@@ -0,0 +1,32 @@
+/*
+ * VFIO BASE CONTAINER
+ *
+ * Copyright (C) 2023 Intel Corporation.
+ * Copyright Red Hat, Inc. 2023
+ *
+ * Authors: Yi Liu <yi.l.liu@intel.com>
+ * Eric Auger <eric.auger@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/error-report.h"
+#include "hw/vfio/vfio-container-base.h"
+
+int vfio_container_dma_map(VFIOContainerBase *bcontainer,
+ hwaddr iova, ram_addr_t size,
+ void *vaddr, bool readonly)
+{
+ g_assert(bcontainer->ops->dma_map);
+ return bcontainer->ops->dma_map(bcontainer, iova, size, vaddr, readonly);
+}
+
+int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
+ hwaddr iova, ram_addr_t size,
+ IOMMUTLBEntry *iotlb)
+{
+ g_assert(bcontainer->ops->dma_unmap);
+ return bcontainer->ops->dma_unmap(bcontainer, iova, size, iotlb);
+}
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 4bc43ddfa4441dc76950203903f22e5475574a47..c04df26323bd115af67ec10f851d61bf686534c2 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -115,9 +115,11 @@ unmap_exit:
/*
* DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
*/
-int vfio_dma_unmap(VFIOContainer *container, hwaddr iova,
- ram_addr_t size, IOMMUTLBEntry *iotlb)
+static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
+ ram_addr_t size, IOMMUTLBEntry *iotlb)
{
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
struct vfio_iommu_type1_dma_unmap unmap = {
.argsz = sizeof(unmap),
.flags = 0,
@@ -151,7 +153,7 @@ int vfio_dma_unmap(VFIOContainer *container, hwaddr iova,
*/
if (errno == EINVAL && unmap.size && !(unmap.iova + unmap.size) &&
container->iommu_type == VFIO_TYPE1v2_IOMMU) {
- trace_vfio_dma_unmap_overflow_workaround();
+ trace_vfio_legacy_dma_unmap_overflow_workaround();
unmap.size -= 1ULL << ctz64(container->pgsizes);
continue;
}
@@ -170,9 +172,11 @@ int vfio_dma_unmap(VFIOContainer *container, hwaddr iova,
return 0;
}
-int vfio_dma_map(VFIOContainer *container, hwaddr iova,
- ram_addr_t size, void *vaddr, bool readonly)
+static int vfio_legacy_dma_map(VFIOContainerBase *bcontainer, hwaddr iova,
+ ram_addr_t size, void *vaddr, bool readonly)
{
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
struct vfio_iommu_type1_dma_map map = {
.argsz = sizeof(map),
.flags = VFIO_DMA_MAP_FLAG_READ,
@@ -191,7 +195,8 @@ int vfio_dma_map(VFIOContainer *container, hwaddr iova,
* the VGA ROM space.
*/
if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
- (errno == EBUSY && vfio_dma_unmap(container, iova, size, NULL) == 0 &&
+ (errno == EBUSY &&
+ vfio_legacy_dma_unmap(bcontainer, iova, size, NULL) == 0 &&
ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
return 0;
}
@@ -937,4 +942,7 @@ void vfio_detach_device(VFIODevice *vbasedev)
vfio_put_group(group);
}
-const VFIOIOMMUOps vfio_legacy_ops;
+const VFIOIOMMUOps vfio_legacy_ops = {
+ .dma_map = vfio_legacy_dma_map,
+ .dma_unmap = vfio_legacy_dma_unmap,
+};
diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build
index 2a6912c94027d6213144f910d64625a469cc2b1f..eb6ce6229daef67206d562e981898d62b105ece0 100644
--- a/hw/vfio/meson.build
+++ b/hw/vfio/meson.build
@@ -2,6 +2,7 @@ vfio_ss = ss.source_set()
vfio_ss.add(files(
'helpers.c',
'common.c',
+ 'container-base.c',
'container.c',
'spapr.c',
'migration.c',
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index 0eb2387cf24c920b0904ec4012b0fcd3f2e8b3cf..9f7fedee986de18d3f97f86bfb5db21b77175573 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -116,7 +116,7 @@ vfio_region_unmap(const char *name, unsigned long offset, unsigned long end) "Re
vfio_region_sparse_mmap_header(const char *name, int index, int nr_areas) "Device %s region %d: %d sparse mmap entries"
vfio_region_sparse_mmap_entry(int i, unsigned long start, unsigned long end) "sparse entry %d [0x%lx - 0x%lx]"
vfio_get_dev_region(const char *name, int index, uint32_t type, uint32_t subtype) "%s index %d, %08x/%08x"
-vfio_dma_unmap_overflow_workaround(void) ""
+vfio_legacy_dma_unmap_overflow_workaround(void) ""
vfio_get_dirty_bitmap(int fd, uint64_t iova, uint64_t size, uint64_t bitmap_size, uint64_t start, uint64_t dirty_pages) "container fd=%d, iova=0x%"PRIx64" size= 0x%"PRIx64" bitmap_size=0x%"PRIx64" start=0x%"PRIx64" dirty_pages=%"PRIu64
vfio_iommu_map_dirty_notify(uint64_t iova_start, uint64_t iova_end) "iommu dirty @ 0x%"PRIx64" - 0x%"PRIx64
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 04/47] vfio/common: Introduce vfio_container_init/destroy helper
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (2 preceding siblings ...)
2023-12-19 18:55 ` [PULL 03/47] vfio/container: Switch to dma_map|unmap API Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 05/47] vfio/common: Move giommu_list in base container Cédric Le Goater
` (43 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
This adds two helper functions vfio_container_init/destroy which will be
used by both legacy and iommufd containers to do base container specific
initialization and release.
No functional change intended.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-container-base.h | 4 ++++
hw/vfio/container-base.c | 9 +++++++++
hw/vfio/container.c | 4 +++-
3 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 56b033f59f13684a56824929c16d9445a6efc5ff..577f52ccbc0e6d3b48d733c3bec0c1a2435d3e32 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -38,6 +38,10 @@ int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
hwaddr iova, ram_addr_t size,
IOMMUTLBEntry *iotlb);
+void vfio_container_init(VFIOContainerBase *bcontainer,
+ const VFIOIOMMUOps *ops);
+void vfio_container_destroy(VFIOContainerBase *bcontainer);
+
struct VFIOIOMMUOps {
/* basic feature */
int (*dma_map)(VFIOContainerBase *bcontainer,
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index 55d3a35fa42b66e6590da8d699e60850c88ed74d..e929435751d3bef179218f667f48e0419c352df4 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -30,3 +30,12 @@ int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
g_assert(bcontainer->ops->dma_unmap);
return bcontainer->ops->dma_unmap(bcontainer, iova, size, iotlb);
}
+
+void vfio_container_init(VFIOContainerBase *bcontainer, const VFIOIOMMUOps *ops)
+{
+ bcontainer->ops = ops;
+}
+
+void vfio_container_destroy(VFIOContainerBase *bcontainer)
+{
+}
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index c04df26323bd115af67ec10f851d61bf686534c2..32a0251dd1673ef9f47f8a0db9f7dd113841b7d8 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -559,7 +559,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
QLIST_INIT(&container->giommu_list);
QLIST_INIT(&container->vrdl_list);
bcontainer = &container->bcontainer;
- bcontainer->ops = &vfio_legacy_ops;
+ vfio_container_init(bcontainer, &vfio_legacy_ops);
ret = vfio_init_container(container, group->fd, errp);
if (ret) {
@@ -661,6 +661,7 @@ put_space_exit:
static void vfio_disconnect_container(VFIOGroup *group)
{
VFIOContainer *container = group->container;
+ VFIOContainerBase *bcontainer = &container->bcontainer;
QLIST_REMOVE(group, container_next);
group->container = NULL;
@@ -695,6 +696,7 @@ static void vfio_disconnect_container(VFIOGroup *group)
QLIST_REMOVE(giommu, giommu_next);
g_free(giommu);
}
+ vfio_container_destroy(bcontainer);
trace_vfio_disconnect_container(container->fd);
close(container->fd);
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 05/47] vfio/common: Move giommu_list in base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (3 preceding siblings ...)
2023-12-19 18:56 ` [PULL 04/47] vfio/common: Introduce vfio_container_init/destroy helper Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 06/47] vfio/container: Move space field to " Cédric Le Goater
` (42 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
Move the giommu_list field in the base container and store
the base container in the VFIOGuestIOMMU.
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 9 ---------
include/hw/vfio/vfio-container-base.h | 9 +++++++++
hw/vfio/common.c | 17 +++++++++++------
hw/vfio/container-base.c | 9 +++++++++
hw/vfio/container.c | 8 --------
5 files changed, 29 insertions(+), 23 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 24a26345e5aa6094706dcf1301eeaf44ac91840d..6be082b8f20c2dfa28dbed85c343667ea20e2b73 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -95,7 +95,6 @@ typedef struct VFIOContainer {
uint64_t max_dirty_bitmap_size;
unsigned long pgsizes;
unsigned int dma_max_mappings;
- QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
QLIST_HEAD(, VFIORamDiscardListener) vrdl_list;
@@ -104,14 +103,6 @@ typedef struct VFIOContainer {
GList *iova_ranges;
} VFIOContainer;
-typedef struct VFIOGuestIOMMU {
- VFIOContainer *container;
- IOMMUMemoryRegion *iommu_mr;
- hwaddr iommu_offset;
- IOMMUNotifier n;
- QLIST_ENTRY(VFIOGuestIOMMU) giommu_next;
-} VFIOGuestIOMMU;
-
typedef struct VFIORamDiscardListener {
VFIOContainer *container;
MemoryRegion *mr;
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 577f52ccbc0e6d3b48d733c3bec0c1a2435d3e32..a11aec575544d7a4f29c0e33200ed962568372cf 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -29,8 +29,17 @@ typedef struct {
*/
typedef struct VFIOContainerBase {
const VFIOIOMMUOps *ops;
+ QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
} VFIOContainerBase;
+typedef struct VFIOGuestIOMMU {
+ VFIOContainerBase *bcontainer;
+ IOMMUMemoryRegion *iommu_mr;
+ hwaddr iommu_offset;
+ IOMMUNotifier n;
+ QLIST_ENTRY(VFIOGuestIOMMU) giommu_next;
+} VFIOGuestIOMMU;
+
int vfio_container_dma_map(VFIOContainerBase *bcontainer,
hwaddr iova, ram_addr_t size,
void *vaddr, bool readonly);
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index e610771888491c8cd19883cdf91ad1589c843e27..43580bcc43c72336a50b002e17f4415035aa8798 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -292,7 +292,7 @@ static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr,
static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
{
VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n);
- VFIOContainerBase *bcontainer = &giommu->container->bcontainer;
+ VFIOContainerBase *bcontainer = giommu->bcontainer;
hwaddr iova = iotlb->iova + giommu->iommu_offset;
void *vaddr;
int ret;
@@ -569,6 +569,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
MemoryRegionSection *section)
{
VFIOContainer *container = container_of(listener, VFIOContainer, listener);
+ VFIOContainerBase *bcontainer = &container->bcontainer;
hwaddr iova, end;
Int128 llend, llsize;
void *vaddr;
@@ -612,7 +613,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
giommu->iommu_mr = iommu_mr;
giommu->iommu_offset = section->offset_within_address_space -
section->offset_within_region;
- giommu->container = container;
+ giommu->bcontainer = bcontainer;
llend = int128_add(int128_make64(section->offset_within_region),
section->size);
llend = int128_sub(llend, int128_one());
@@ -647,7 +648,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
g_free(giommu);
goto fail;
}
- QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
+ QLIST_INSERT_HEAD(&bcontainer->giommu_list, giommu, giommu_next);
memory_region_iommu_replay(giommu->iommu_mr, &giommu->n);
return;
@@ -732,6 +733,7 @@ static void vfio_listener_region_del(MemoryListener *listener,
MemoryRegionSection *section)
{
VFIOContainer *container = container_of(listener, VFIOContainer, listener);
+ VFIOContainerBase *bcontainer = &container->bcontainer;
hwaddr iova, end;
Int128 llend, llsize;
int ret;
@@ -744,7 +746,7 @@ static void vfio_listener_region_del(MemoryListener *listener,
if (memory_region_is_iommu(section->mr)) {
VFIOGuestIOMMU *giommu;
- QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) {
+ QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) {
if (MEMORY_REGION(giommu->iommu_mr) == section->mr &&
giommu->n.start == section->offset_within_region) {
memory_region_unregister_iommu_notifier(section->mr,
@@ -1206,7 +1208,9 @@ static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
vfio_giommu_dirty_notifier *gdn = container_of(n,
vfio_giommu_dirty_notifier, n);
VFIOGuestIOMMU *giommu = gdn->giommu;
- VFIOContainer *container = giommu->container;
+ VFIOContainerBase *bcontainer = giommu->bcontainer;
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
hwaddr iova = iotlb->iova + giommu->iommu_offset;
ram_addr_t translated_addr;
int ret = -EINVAL;
@@ -1284,12 +1288,13 @@ static int vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainer *container,
static int vfio_sync_dirty_bitmap(VFIOContainer *container,
MemoryRegionSection *section)
{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
ram_addr_t ram_addr;
if (memory_region_is_iommu(section->mr)) {
VFIOGuestIOMMU *giommu;
- QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) {
+ QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) {
if (MEMORY_REGION(giommu->iommu_mr) == section->mr &&
giommu->n.start == section->offset_within_region) {
Int128 llend;
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index e929435751d3bef179218f667f48e0419c352df4..20bcb9669a6122f951119c025b9950854c53f88e 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -34,8 +34,17 @@ int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
void vfio_container_init(VFIOContainerBase *bcontainer, const VFIOIOMMUOps *ops)
{
bcontainer->ops = ops;
+ QLIST_INIT(&bcontainer->giommu_list);
}
void vfio_container_destroy(VFIOContainerBase *bcontainer)
{
+ VFIOGuestIOMMU *giommu, *tmp;
+
+ QLIST_FOREACH_SAFE(giommu, &bcontainer->giommu_list, giommu_next, tmp) {
+ memory_region_unregister_iommu_notifier(
+ MEMORY_REGION(giommu->iommu_mr), &giommu->n);
+ QLIST_REMOVE(giommu, giommu_next);
+ g_free(giommu);
+ }
}
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 32a0251dd1673ef9f47f8a0db9f7dd113841b7d8..133d3c8f5c08e6476cebb3c707ebd115a068af2d 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -556,7 +556,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
container->dirty_pages_supported = false;
container->dma_max_mappings = 0;
container->iova_ranges = NULL;
- QLIST_INIT(&container->giommu_list);
QLIST_INIT(&container->vrdl_list);
bcontainer = &container->bcontainer;
vfio_container_init(bcontainer, &vfio_legacy_ops);
@@ -686,16 +685,9 @@ static void vfio_disconnect_container(VFIOGroup *group)
if (QLIST_EMPTY(&container->group_list)) {
VFIOAddressSpace *space = container->space;
- VFIOGuestIOMMU *giommu, *tmp;
QLIST_REMOVE(container, next);
- QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) {
- memory_region_unregister_iommu_notifier(
- MEMORY_REGION(giommu->iommu_mr), &giommu->n);
- QLIST_REMOVE(giommu, giommu_next);
- g_free(giommu);
- }
vfio_container_destroy(bcontainer);
trace_vfio_disconnect_container(container->fd);
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 06/47] vfio/container: Move space field to base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (4 preceding siblings ...)
2023-12-19 18:56 ` [PULL 05/47] vfio/common: Move giommu_list in base container Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 07/47] vfio/container: Switch to IOMMU BE set_dirty_page_tracking/query_dirty_bitmap API Cédric Le Goater
` (41 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
Move the space field to the base object. Also the VFIOAddressSpace
now contains a list of base containers.
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 8 --------
include/hw/vfio/vfio-container-base.h | 9 +++++++++
hw/ppc/spapr_pci_vfio.c | 10 +++++-----
hw/vfio/common.c | 4 ++--
hw/vfio/container-base.c | 6 +++++-
hw/vfio/container.c | 18 ++++++++----------
6 files changed, 29 insertions(+), 26 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 6be082b8f20c2dfa28dbed85c343667ea20e2b73..bd4de6cb3aff7cb85e34960fe4238d82b30867cd 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -73,17 +73,10 @@ typedef struct VFIOMigration {
bool initial_data_sent;
} VFIOMigration;
-typedef struct VFIOAddressSpace {
- AddressSpace *as;
- QLIST_HEAD(, VFIOContainer) containers;
- QLIST_ENTRY(VFIOAddressSpace) list;
-} VFIOAddressSpace;
-
struct VFIOGroup;
typedef struct VFIOContainer {
VFIOContainerBase bcontainer;
- VFIOAddressSpace *space;
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
MemoryListener listener;
MemoryListener prereg_listener;
@@ -98,7 +91,6 @@ typedef struct VFIOContainer {
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
QLIST_HEAD(, VFIORamDiscardListener) vrdl_list;
- QLIST_ENTRY(VFIOContainer) next;
QLIST_HEAD(, VFIODevice) device_list;
GList *iova_ranges;
} VFIOContainer;
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index a11aec575544d7a4f29c0e33200ed962568372cf..c7cc6ec9c55e93e4f7c6be5576ed59b5dea58b1e 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -24,12 +24,20 @@ typedef struct {
hwaddr pages;
} VFIOBitmap;
+typedef struct VFIOAddressSpace {
+ AddressSpace *as;
+ QLIST_HEAD(, VFIOContainerBase) containers;
+ QLIST_ENTRY(VFIOAddressSpace) list;
+} VFIOAddressSpace;
+
/*
* This is the base object for vfio container backends
*/
typedef struct VFIOContainerBase {
const VFIOIOMMUOps *ops;
+ VFIOAddressSpace *space;
QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
+ QLIST_ENTRY(VFIOContainerBase) next;
} VFIOContainerBase;
typedef struct VFIOGuestIOMMU {
@@ -48,6 +56,7 @@ int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
IOMMUTLBEntry *iotlb);
void vfio_container_init(VFIOContainerBase *bcontainer,
+ VFIOAddressSpace *space,
const VFIOIOMMUOps *ops);
void vfio_container_destroy(VFIOContainerBase *bcontainer);
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index f283f7e38d619fd65985146b5e1572c221f95c53..d1d07bec4644da4ae6a99d3357d6d17ff66264de 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -84,27 +84,27 @@ static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op)
static VFIOContainer *vfio_eeh_as_container(AddressSpace *as)
{
VFIOAddressSpace *space = vfio_get_address_space(as);
- VFIOContainer *container = NULL;
+ VFIOContainerBase *bcontainer = NULL;
if (QLIST_EMPTY(&space->containers)) {
/* No containers to act on */
goto out;
}
- container = QLIST_FIRST(&space->containers);
+ bcontainer = QLIST_FIRST(&space->containers);
- if (QLIST_NEXT(container, next)) {
+ if (QLIST_NEXT(bcontainer, next)) {
/*
* We don't yet have logic to synchronize EEH state across
* multiple containers
*/
- container = NULL;
+ bcontainer = NULL;
goto out;
}
out:
vfio_put_address_space(space);
- return container;
+ return container_of(bcontainer, VFIOContainer, bcontainer);
}
static bool vfio_eeh_as_ok(AddressSpace *as)
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 43580bcc43c72336a50b002e17f4415035aa8798..1d8202537ea4f27f49f2d477537d6c7c557bd61f 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -145,7 +145,7 @@ void vfio_unblock_multiple_devices_migration(void)
bool vfio_viommu_preset(VFIODevice *vbasedev)
{
- return vbasedev->container->space->as != &address_space_memory;
+ return vbasedev->container->bcontainer.space->as != &address_space_memory;
}
static void vfio_set_migration_error(int err)
@@ -922,7 +922,7 @@ static void vfio_dirty_tracking_init(VFIOContainer *container,
dirty.container = container;
memory_listener_register(&dirty.listener,
- container->space->as);
+ container->bcontainer.space->as);
*ranges = dirty.ranges;
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index 20bcb9669a6122f951119c025b9950854c53f88e..3933391e0db26116401509c5917eaceabcd98443 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -31,9 +31,11 @@ int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
return bcontainer->ops->dma_unmap(bcontainer, iova, size, iotlb);
}
-void vfio_container_init(VFIOContainerBase *bcontainer, const VFIOIOMMUOps *ops)
+void vfio_container_init(VFIOContainerBase *bcontainer, VFIOAddressSpace *space,
+ const VFIOIOMMUOps *ops)
{
bcontainer->ops = ops;
+ bcontainer->space = space;
QLIST_INIT(&bcontainer->giommu_list);
}
@@ -41,6 +43,8 @@ void vfio_container_destroy(VFIOContainerBase *bcontainer)
{
VFIOGuestIOMMU *giommu, *tmp;
+ QLIST_REMOVE(bcontainer, next);
+
QLIST_FOREACH_SAFE(giommu, &bcontainer->giommu_list, giommu_next, tmp) {
memory_region_unregister_iommu_notifier(
MEMORY_REGION(giommu->iommu_mr), &giommu->n);
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 133d3c8f5c08e6476cebb3c707ebd115a068af2d..f12fcb6fe11d578838b1052f70ee5099b9018e94 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -514,7 +514,8 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
* details once we know which type of IOMMU we are using.
*/
- QLIST_FOREACH(container, &space->containers, next) {
+ QLIST_FOREACH(bcontainer, &space->containers, next) {
+ container = container_of(bcontainer, VFIOContainer, bcontainer);
if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
ret = vfio_ram_block_discard_disable(container, true);
if (ret) {
@@ -550,7 +551,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
}
container = g_malloc0(sizeof(*container));
- container->space = space;
container->fd = fd;
container->error = NULL;
container->dirty_pages_supported = false;
@@ -558,7 +558,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
container->iova_ranges = NULL;
QLIST_INIT(&container->vrdl_list);
bcontainer = &container->bcontainer;
- vfio_container_init(bcontainer, &vfio_legacy_ops);
+ vfio_container_init(bcontainer, space, &vfio_legacy_ops);
ret = vfio_init_container(container, group->fd, errp);
if (ret) {
@@ -613,14 +613,14 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
vfio_kvm_device_add_group(group);
QLIST_INIT(&container->group_list);
- QLIST_INSERT_HEAD(&space->containers, container, next);
+ QLIST_INSERT_HEAD(&space->containers, bcontainer, next);
group->container = container;
QLIST_INSERT_HEAD(&container->group_list, group, container_next);
container->listener = vfio_memory_listener;
- memory_listener_register(&container->listener, container->space->as);
+ memory_listener_register(&container->listener, bcontainer->space->as);
if (container->error) {
ret = -1;
@@ -634,7 +634,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
return 0;
listener_release_exit:
QLIST_REMOVE(group, container_next);
- QLIST_REMOVE(container, next);
+ QLIST_REMOVE(bcontainer, next);
vfio_kvm_device_del_group(group);
memory_listener_unregister(&container->listener);
if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU ||
@@ -684,9 +684,7 @@ static void vfio_disconnect_container(VFIOGroup *group)
}
if (QLIST_EMPTY(&container->group_list)) {
- VFIOAddressSpace *space = container->space;
-
- QLIST_REMOVE(container, next);
+ VFIOAddressSpace *space = bcontainer->space;
vfio_container_destroy(bcontainer);
@@ -707,7 +705,7 @@ static VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp)
QLIST_FOREACH(group, &vfio_group_list, next) {
if (group->groupid == groupid) {
/* Found it. Now is it already in the right context? */
- if (group->container->space->as == as) {
+ if (group->container->bcontainer.space->as == as) {
return group;
} else {
error_setg(errp, "group %d used in multiple address spaces",
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 07/47] vfio/container: Switch to IOMMU BE set_dirty_page_tracking/query_dirty_bitmap API
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (5 preceding siblings ...)
2023-12-19 18:56 ` [PULL 06/47] vfio/container: Move space field to " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 08/47] vfio/container: Move per container device list in base container Cédric Le Goater
` (40 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
dirty_pages_supported field is also moved to the base container
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 6 ------
include/hw/vfio/vfio-container-base.h | 6 ++++++
hw/vfio/common.c | 12 ++++++++----
hw/vfio/container-base.c | 16 ++++++++++++++++
hw/vfio/container.c | 21 ++++++++++++++-------
5 files changed, 44 insertions(+), 17 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index bd4de6cb3aff7cb85e34960fe4238d82b30867cd..60f2785fe07cf71e3c65e85d933536c09170f4c6 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -83,7 +83,6 @@ typedef struct VFIOContainer {
unsigned iommu_type;
Error *error;
bool initialized;
- bool dirty_pages_supported;
uint64_t dirty_pgsizes;
uint64_t max_dirty_bitmap_size;
unsigned long pgsizes;
@@ -190,11 +189,6 @@ VFIOAddressSpace *vfio_get_address_space(AddressSpace *as);
void vfio_put_address_space(VFIOAddressSpace *space);
bool vfio_devices_all_running_and_saving(VFIOContainer *container);
-/* container->fd */
-int vfio_set_dirty_page_tracking(VFIOContainer *container, bool start);
-int vfio_query_dirty_bitmap(VFIOContainer *container, VFIOBitmap *vbmap,
- hwaddr iova, hwaddr size);
-
/* SPAPR specific */
int vfio_container_add_section_window(VFIOContainer *container,
MemoryRegionSection *section,
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index c7cc6ec9c55e93e4f7c6be5576ed59b5dea58b1e..f244f003d080a30e95b6daa97a132eeb5a271ae4 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -36,6 +36,7 @@ typedef struct VFIOAddressSpace {
typedef struct VFIOContainerBase {
const VFIOIOMMUOps *ops;
VFIOAddressSpace *space;
+ bool dirty_pages_supported;
QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
QLIST_ENTRY(VFIOContainerBase) next;
} VFIOContainerBase;
@@ -54,6 +55,11 @@ int vfio_container_dma_map(VFIOContainerBase *bcontainer,
int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
hwaddr iova, ram_addr_t size,
IOMMUTLBEntry *iotlb);
+int vfio_container_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
+ bool start);
+int vfio_container_query_dirty_bitmap(VFIOContainerBase *bcontainer,
+ VFIOBitmap *vbmap,
+ hwaddr iova, hwaddr size);
void vfio_container_init(VFIOContainerBase *bcontainer,
VFIOAddressSpace *space,
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 1d8202537ea4f27f49f2d477537d6c7c557bd61f..b1a875ca93a44347f3652298995b89b5d9aed045 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -1079,7 +1079,8 @@ static void vfio_listener_log_global_start(MemoryListener *listener)
if (vfio_devices_all_device_dirty_tracking(container)) {
ret = vfio_devices_dma_logging_start(container);
} else {
- ret = vfio_set_dirty_page_tracking(container, true);
+ ret = vfio_container_set_dirty_page_tracking(&container->bcontainer,
+ true);
}
if (ret) {
@@ -1097,7 +1098,8 @@ static void vfio_listener_log_global_stop(MemoryListener *listener)
if (vfio_devices_all_device_dirty_tracking(container)) {
vfio_devices_dma_logging_stop(container);
} else {
- ret = vfio_set_dirty_page_tracking(container, false);
+ ret = vfio_container_set_dirty_page_tracking(&container->bcontainer,
+ false);
}
if (ret) {
@@ -1165,7 +1167,8 @@ int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
VFIOBitmap vbmap;
int ret;
- if (!container->dirty_pages_supported && !all_device_dirty_tracking) {
+ if (!container->bcontainer.dirty_pages_supported &&
+ !all_device_dirty_tracking) {
cpu_physical_memory_set_dirty_range(ram_addr, size,
tcg_enabled() ? DIRTY_CLIENTS_ALL :
DIRTY_CLIENTS_NOCODE);
@@ -1180,7 +1183,8 @@ int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
if (all_device_dirty_tracking) {
ret = vfio_devices_query_dirty_bitmap(container, &vbmap, iova, size);
} else {
- ret = vfio_query_dirty_bitmap(container, &vbmap, iova, size);
+ ret = vfio_container_query_dirty_bitmap(&container->bcontainer, &vbmap,
+ iova, size);
}
if (ret) {
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index 3933391e0db26116401509c5917eaceabcd98443..5d654ae17233fe06c235d33d981eb8fa92128536 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -31,11 +31,27 @@ int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
return bcontainer->ops->dma_unmap(bcontainer, iova, size, iotlb);
}
+int vfio_container_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
+ bool start)
+{
+ g_assert(bcontainer->ops->set_dirty_page_tracking);
+ return bcontainer->ops->set_dirty_page_tracking(bcontainer, start);
+}
+
+int vfio_container_query_dirty_bitmap(VFIOContainerBase *bcontainer,
+ VFIOBitmap *vbmap,
+ hwaddr iova, hwaddr size)
+{
+ g_assert(bcontainer->ops->query_dirty_bitmap);
+ return bcontainer->ops->query_dirty_bitmap(bcontainer, vbmap, iova, size);
+}
+
void vfio_container_init(VFIOContainerBase *bcontainer, VFIOAddressSpace *space,
const VFIOIOMMUOps *ops)
{
bcontainer->ops = ops;
bcontainer->space = space;
+ bcontainer->dirty_pages_supported = false;
QLIST_INIT(&bcontainer->giommu_list);
}
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index f12fcb6fe11d578838b1052f70ee5099b9018e94..3ab74e26151efd7ae4027ed3b4cf04a835083ccc 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -131,7 +131,7 @@ static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
if (iotlb && vfio_devices_all_running_and_mig_active(container)) {
if (!vfio_devices_all_device_dirty_tracking(container) &&
- container->dirty_pages_supported) {
+ container->bcontainer.dirty_pages_supported) {
return vfio_dma_unmap_bitmap(container, iova, size, iotlb);
}
@@ -205,14 +205,17 @@ static int vfio_legacy_dma_map(VFIOContainerBase *bcontainer, hwaddr iova,
return -errno;
}
-int vfio_set_dirty_page_tracking(VFIOContainer *container, bool start)
+static int vfio_legacy_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
+ bool start)
{
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
int ret;
struct vfio_iommu_type1_dirty_bitmap dirty = {
.argsz = sizeof(dirty),
};
- if (!container->dirty_pages_supported) {
+ if (!bcontainer->dirty_pages_supported) {
return 0;
}
@@ -232,9 +235,12 @@ int vfio_set_dirty_page_tracking(VFIOContainer *container, bool start)
return ret;
}
-int vfio_query_dirty_bitmap(VFIOContainer *container, VFIOBitmap *vbmap,
- hwaddr iova, hwaddr size)
+static int vfio_legacy_query_dirty_bitmap(VFIOContainerBase *bcontainer,
+ VFIOBitmap *vbmap,
+ hwaddr iova, hwaddr size)
{
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
struct vfio_iommu_type1_dirty_bitmap *dbitmap;
struct vfio_iommu_type1_dirty_bitmap_get *range;
int ret;
@@ -461,7 +467,7 @@ static void vfio_get_iommu_info_migration(VFIOContainer *container,
* qemu_real_host_page_size to mark those dirty.
*/
if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) {
- container->dirty_pages_supported = true;
+ container->bcontainer.dirty_pages_supported = true;
container->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size;
container->dirty_pgsizes = cap_mig->pgsize_bitmap;
}
@@ -553,7 +559,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
container = g_malloc0(sizeof(*container));
container->fd = fd;
container->error = NULL;
- container->dirty_pages_supported = false;
container->dma_max_mappings = 0;
container->iova_ranges = NULL;
QLIST_INIT(&container->vrdl_list);
@@ -937,4 +942,6 @@ void vfio_detach_device(VFIODevice *vbasedev)
const VFIOIOMMUOps vfio_legacy_ops = {
.dma_map = vfio_legacy_dma_map,
.dma_unmap = vfio_legacy_dma_unmap,
+ .set_dirty_page_tracking = vfio_legacy_set_dirty_page_tracking,
+ .query_dirty_bitmap = vfio_legacy_query_dirty_bitmap,
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 08/47] vfio/container: Move per container device list in base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (6 preceding siblings ...)
2023-12-19 18:56 ` [PULL 07/47] vfio/container: Switch to IOMMU BE set_dirty_page_tracking/query_dirty_bitmap API Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 09/47] vfio/container: Convert functions to " Cédric Le Goater
` (39 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
VFIO Device is also changed to point to base container instead of
legacy container.
No functional change intended.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 3 +--
include/hw/vfio/vfio-container-base.h | 1 +
hw/vfio/common.c | 23 +++++++++++++++--------
hw/vfio/container.c | 12 ++++++------
4 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 60f2785fe07cf71e3c65e85d933536c09170f4c6..9740cf9fbc65831647b30dcc415d3d1db5835661 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -90,7 +90,6 @@ typedef struct VFIOContainer {
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
QLIST_HEAD(, VFIORamDiscardListener) vrdl_list;
- QLIST_HEAD(, VFIODevice) device_list;
GList *iova_ranges;
} VFIOContainer;
@@ -118,7 +117,7 @@ typedef struct VFIODevice {
QLIST_ENTRY(VFIODevice) container_next;
QLIST_ENTRY(VFIODevice) global_next;
struct VFIOGroup *group;
- VFIOContainer *container;
+ VFIOContainerBase *bcontainer;
char *sysfsdev;
char *name;
DeviceState *dev;
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index f244f003d080a30e95b6daa97a132eeb5a271ae4..709096249642c7c597aa789264a09389e53664f3 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -39,6 +39,7 @@ typedef struct VFIOContainerBase {
bool dirty_pages_supported;
QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
QLIST_ENTRY(VFIOContainerBase) next;
+ QLIST_HEAD(, VFIODevice) device_list;
} VFIOContainerBase;
typedef struct VFIOGuestIOMMU {
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index b1a875ca93a44347f3652298995b89b5d9aed045..9415395ed9becc337cf25c5714c6b3c4c452c81b 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -145,7 +145,7 @@ void vfio_unblock_multiple_devices_migration(void)
bool vfio_viommu_preset(VFIODevice *vbasedev)
{
- return vbasedev->container->bcontainer.space->as != &address_space_memory;
+ return vbasedev->bcontainer->space->as != &address_space_memory;
}
static void vfio_set_migration_error(int err)
@@ -179,6 +179,7 @@ bool vfio_device_state_is_precopy(VFIODevice *vbasedev)
static bool vfio_devices_all_dirty_tracking(VFIOContainer *container)
{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
MigrationState *ms = migrate_get_current();
@@ -187,7 +188,7 @@ static bool vfio_devices_all_dirty_tracking(VFIOContainer *container)
return false;
}
- QLIST_FOREACH(vbasedev, &container->device_list, container_next) {
+ QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
VFIOMigration *migration = vbasedev->migration;
if (!migration) {
@@ -205,9 +206,10 @@ static bool vfio_devices_all_dirty_tracking(VFIOContainer *container)
bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container)
{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
- QLIST_FOREACH(vbasedev, &container->device_list, container_next) {
+ QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
if (!vbasedev->dirty_pages_supported) {
return false;
}
@@ -222,13 +224,14 @@ bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container)
*/
bool vfio_devices_all_running_and_mig_active(VFIOContainer *container)
{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
if (!migration_is_active(migrate_get_current())) {
return false;
}
- QLIST_FOREACH(vbasedev, &container->device_list, container_next) {
+ QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
VFIOMigration *migration = vbasedev->migration;
if (!migration) {
@@ -833,12 +836,13 @@ static bool vfio_section_is_vfio_pci(MemoryRegionSection *section,
VFIOContainer *container)
{
VFIOPCIDevice *pcidev;
+ VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
Object *owner;
owner = memory_region_owner(section->mr);
- QLIST_FOREACH(vbasedev, &container->device_list, container_next) {
+ QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) {
continue;
}
@@ -939,13 +943,14 @@ static void vfio_devices_dma_logging_stop(VFIOContainer *container)
uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature),
sizeof(uint64_t))] = {};
struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
+ VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
feature->argsz = sizeof(buf);
feature->flags = VFIO_DEVICE_FEATURE_SET |
VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP;
- QLIST_FOREACH(vbasedev, &container->device_list, container_next) {
+ QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
if (!vbasedev->dirty_tracking) {
continue;
}
@@ -1036,6 +1041,7 @@ static int vfio_devices_dma_logging_start(VFIOContainer *container)
{
struct vfio_device_feature *feature;
VFIODirtyRanges ranges;
+ VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
int ret = 0;
@@ -1046,7 +1052,7 @@ static int vfio_devices_dma_logging_start(VFIOContainer *container)
return -errno;
}
- QLIST_FOREACH(vbasedev, &container->device_list, container_next) {
+ QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
if (vbasedev->dirty_tracking) {
continue;
}
@@ -1139,10 +1145,11 @@ int vfio_devices_query_dirty_bitmap(VFIOContainer *container,
VFIOBitmap *vbmap, hwaddr iova,
hwaddr size)
{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
int ret;
- QLIST_FOREACH(vbasedev, &container->device_list, container_next) {
+ QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
ret = vfio_device_dma_logging_report(vbasedev, iova, size,
vbmap->bitmap);
if (ret) {
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 3ab74e26151efd7ae4027ed3b4cf04a835083ccc..63a906de933574706beddf54b78885e9a6c659df 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -888,7 +888,7 @@ int vfio_attach_device(char *name, VFIODevice *vbasedev,
int groupid = vfio_device_groupid(vbasedev, errp);
VFIODevice *vbasedev_iter;
VFIOGroup *group;
- VFIOContainer *container;
+ VFIOContainerBase *bcontainer;
int ret;
if (groupid < 0) {
@@ -915,9 +915,9 @@ int vfio_attach_device(char *name, VFIODevice *vbasedev,
return ret;
}
- container = group->container;
- vbasedev->container = container;
- QLIST_INSERT_HEAD(&container->device_list, vbasedev, container_next);
+ bcontainer = &group->container->bcontainer;
+ vbasedev->bcontainer = bcontainer;
+ QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next);
QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next);
return ret;
@@ -927,13 +927,13 @@ void vfio_detach_device(VFIODevice *vbasedev)
{
VFIOGroup *group = vbasedev->group;
- if (!vbasedev->container) {
+ if (!vbasedev->bcontainer) {
return;
}
QLIST_REMOVE(vbasedev, global_next);
QLIST_REMOVE(vbasedev, container_next);
- vbasedev->container = NULL;
+ vbasedev->bcontainer = NULL;
trace_vfio_detach_device(vbasedev->name, group->groupid);
vfio_put_base_device(vbasedev);
vfio_put_group(group);
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 09/47] vfio/container: Convert functions to base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (7 preceding siblings ...)
2023-12-19 18:56 ` [PULL 08/47] vfio/container: Move per container device list in base container Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 10/47] vfio/container: Move pgsizes and dma_max_mappings " Cédric Le Goater
` (38 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
In the prospect to get rid of VFIOContainer refs
in common.c lets convert misc functions to use the base
container object instead:
vfio_devices_all_dirty_tracking
vfio_devices_all_device_dirty_tracking
vfio_devices_all_running_and_mig_active
vfio_devices_query_dirty_bitmap
vfio_get_dirty_bitmap
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 9 ++++----
hw/vfio/common.c | 42 +++++++++++++++--------------------
hw/vfio/container.c | 6 ++---
hw/vfio/trace-events | 2 +-
4 files changed, 26 insertions(+), 33 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 9740cf9fbc65831647b30dcc415d3d1db5835661..bc67e1316ceeaf4439aa9dd71d8c35cc5f34366c 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -186,7 +186,6 @@ typedef struct VFIODisplay {
VFIOAddressSpace *vfio_get_address_space(AddressSpace *as);
void vfio_put_address_space(VFIOAddressSpace *space);
-bool vfio_devices_all_running_and_saving(VFIOContainer *container);
/* SPAPR specific */
int vfio_container_add_section_window(VFIOContainer *container,
@@ -260,11 +259,11 @@ bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp);
void vfio_migration_exit(VFIODevice *vbasedev);
int vfio_bitmap_alloc(VFIOBitmap *vbmap, hwaddr size);
-bool vfio_devices_all_running_and_mig_active(VFIOContainer *container);
-bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container);
-int vfio_devices_query_dirty_bitmap(VFIOContainer *container,
+bool vfio_devices_all_running_and_mig_active(VFIOContainerBase *bcontainer);
+bool vfio_devices_all_device_dirty_tracking(VFIOContainerBase *bcontainer);
+int vfio_devices_query_dirty_bitmap(VFIOContainerBase *bcontainer,
VFIOBitmap *vbmap, hwaddr iova,
hwaddr size);
-int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
+int vfio_get_dirty_bitmap(VFIOContainerBase *bcontainer, uint64_t iova,
uint64_t size, ram_addr_t ram_addr);
#endif /* HW_VFIO_VFIO_COMMON_H */
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 9415395ed9becc337cf25c5714c6b3c4c452c81b..cf6618f6ed3d16a4c76d641d7bdb4958dd2bd8ef 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -177,9 +177,8 @@ bool vfio_device_state_is_precopy(VFIODevice *vbasedev)
migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P;
}
-static bool vfio_devices_all_dirty_tracking(VFIOContainer *container)
+static bool vfio_devices_all_dirty_tracking(VFIOContainerBase *bcontainer)
{
- VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
MigrationState *ms = migrate_get_current();
@@ -204,9 +203,8 @@ static bool vfio_devices_all_dirty_tracking(VFIOContainer *container)
return true;
}
-bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container)
+bool vfio_devices_all_device_dirty_tracking(VFIOContainerBase *bcontainer)
{
- VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
@@ -222,9 +220,8 @@ bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container)
* Check if all VFIO devices are running and migration is active, which is
* essentially equivalent to the migration being in pre-copy phase.
*/
-bool vfio_devices_all_running_and_mig_active(VFIOContainer *container)
+bool vfio_devices_all_running_and_mig_active(VFIOContainerBase *bcontainer)
{
- VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
if (!migration_is_active(migrate_get_current())) {
@@ -1082,7 +1079,7 @@ static void vfio_listener_log_global_start(MemoryListener *listener)
VFIOContainer *container = container_of(listener, VFIOContainer, listener);
int ret;
- if (vfio_devices_all_device_dirty_tracking(container)) {
+ if (vfio_devices_all_device_dirty_tracking(&container->bcontainer)) {
ret = vfio_devices_dma_logging_start(container);
} else {
ret = vfio_container_set_dirty_page_tracking(&container->bcontainer,
@@ -1101,7 +1098,7 @@ static void vfio_listener_log_global_stop(MemoryListener *listener)
VFIOContainer *container = container_of(listener, VFIOContainer, listener);
int ret = 0;
- if (vfio_devices_all_device_dirty_tracking(container)) {
+ if (vfio_devices_all_device_dirty_tracking(&container->bcontainer)) {
vfio_devices_dma_logging_stop(container);
} else {
ret = vfio_container_set_dirty_page_tracking(&container->bcontainer,
@@ -1141,11 +1138,10 @@ static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova,
return 0;
}
-int vfio_devices_query_dirty_bitmap(VFIOContainer *container,
+int vfio_devices_query_dirty_bitmap(VFIOContainerBase *bcontainer,
VFIOBitmap *vbmap, hwaddr iova,
hwaddr size)
{
- VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
int ret;
@@ -1165,17 +1161,16 @@ int vfio_devices_query_dirty_bitmap(VFIOContainer *container,
return 0;
}
-int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
+int vfio_get_dirty_bitmap(VFIOContainerBase *bcontainer, uint64_t iova,
uint64_t size, ram_addr_t ram_addr)
{
bool all_device_dirty_tracking =
- vfio_devices_all_device_dirty_tracking(container);
+ vfio_devices_all_device_dirty_tracking(bcontainer);
uint64_t dirty_pages;
VFIOBitmap vbmap;
int ret;
- if (!container->bcontainer.dirty_pages_supported &&
- !all_device_dirty_tracking) {
+ if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) {
cpu_physical_memory_set_dirty_range(ram_addr, size,
tcg_enabled() ? DIRTY_CLIENTS_ALL :
DIRTY_CLIENTS_NOCODE);
@@ -1188,10 +1183,9 @@ int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
}
if (all_device_dirty_tracking) {
- ret = vfio_devices_query_dirty_bitmap(container, &vbmap, iova, size);
+ ret = vfio_devices_query_dirty_bitmap(bcontainer, &vbmap, iova, size);
} else {
- ret = vfio_container_query_dirty_bitmap(&container->bcontainer, &vbmap,
- iova, size);
+ ret = vfio_container_query_dirty_bitmap(bcontainer, &vbmap, iova, size);
}
if (ret) {
@@ -1201,8 +1195,7 @@ int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr,
vbmap.pages);
- trace_vfio_get_dirty_bitmap(container->fd, iova, size, vbmap.size,
- ram_addr, dirty_pages);
+ trace_vfio_get_dirty_bitmap(iova, size, vbmap.size, ram_addr, dirty_pages);
out:
g_free(vbmap.bitmap);
@@ -1236,8 +1229,8 @@ static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
rcu_read_lock();
if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) {
- ret = vfio_get_dirty_bitmap(container, iova, iotlb->addr_mask + 1,
- translated_addr);
+ ret = vfio_get_dirty_bitmap(&container->bcontainer, iova,
+ iotlb->addr_mask + 1, translated_addr);
if (ret) {
error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx") = %d (%s)",
@@ -1266,7 +1259,8 @@ static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section,
* Sync the whole mapped region (spanning multiple individual mappings)
* in one go.
*/
- return vfio_get_dirty_bitmap(vrdl->container, iova, size, ram_addr);
+ return vfio_get_dirty_bitmap(&vrdl->container->bcontainer, iova, size,
+ ram_addr);
}
static int vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainer *container,
@@ -1335,7 +1329,7 @@ static int vfio_sync_dirty_bitmap(VFIOContainer *container,
ram_addr = memory_region_get_ram_addr(section->mr) +
section->offset_within_region;
- return vfio_get_dirty_bitmap(container,
+ return vfio_get_dirty_bitmap(&container->bcontainer,
REAL_HOST_PAGE_ALIGN(section->offset_within_address_space),
int128_get64(section->size), ram_addr);
}
@@ -1350,7 +1344,7 @@ static void vfio_listener_log_sync(MemoryListener *listener,
return;
}
- if (vfio_devices_all_dirty_tracking(container)) {
+ if (vfio_devices_all_dirty_tracking(&container->bcontainer)) {
ret = vfio_sync_dirty_bitmap(container, section);
if (ret) {
error_report("vfio: Failed to sync dirty bitmap, err: %d (%s)", ret,
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 63a906de933574706beddf54b78885e9a6c659df..7bd81eab09008530294757a81429bfea1b147815 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -129,8 +129,8 @@ static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
bool need_dirty_sync = false;
int ret;
- if (iotlb && vfio_devices_all_running_and_mig_active(container)) {
- if (!vfio_devices_all_device_dirty_tracking(container) &&
+ if (iotlb && vfio_devices_all_running_and_mig_active(bcontainer)) {
+ if (!vfio_devices_all_device_dirty_tracking(bcontainer) &&
container->bcontainer.dirty_pages_supported) {
return vfio_dma_unmap_bitmap(container, iova, size, iotlb);
}
@@ -162,7 +162,7 @@ static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
}
if (need_dirty_sync) {
- ret = vfio_get_dirty_bitmap(container, iova, size,
+ ret = vfio_get_dirty_bitmap(bcontainer, iova, size,
iotlb->translated_addr);
if (ret) {
return ret;
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index 9f7fedee986de18d3f97f86bfb5db21b77175573..08a1f9dfa4508b8378a41c35feed67123ba4b942 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -117,7 +117,7 @@ vfio_region_sparse_mmap_header(const char *name, int index, int nr_areas) "Devic
vfio_region_sparse_mmap_entry(int i, unsigned long start, unsigned long end) "sparse entry %d [0x%lx - 0x%lx]"
vfio_get_dev_region(const char *name, int index, uint32_t type, uint32_t subtype) "%s index %d, %08x/%08x"
vfio_legacy_dma_unmap_overflow_workaround(void) ""
-vfio_get_dirty_bitmap(int fd, uint64_t iova, uint64_t size, uint64_t bitmap_size, uint64_t start, uint64_t dirty_pages) "container fd=%d, iova=0x%"PRIx64" size= 0x%"PRIx64" bitmap_size=0x%"PRIx64" start=0x%"PRIx64" dirty_pages=%"PRIu64
+vfio_get_dirty_bitmap(uint64_t iova, uint64_t size, uint64_t bitmap_size, uint64_t start, uint64_t dirty_pages) "iova=0x%"PRIx64" size= 0x%"PRIx64" bitmap_size=0x%"PRIx64" start=0x%"PRIx64" dirty_pages=%"PRIu64
vfio_iommu_map_dirty_notify(uint64_t iova_start, uint64_t iova_end) "iommu dirty @ 0x%"PRIx64" - 0x%"PRIx64
# platform.c
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 10/47] vfio/container: Move pgsizes and dma_max_mappings to base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (8 preceding siblings ...)
2023-12-19 18:56 ` [PULL 09/47] vfio/container: Convert functions to " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 11/47] vfio/container: Move vrdl_list " Cédric Le Goater
` (37 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 2 --
include/hw/vfio/vfio-container-base.h | 2 ++
hw/vfio/common.c | 17 +++++++++--------
hw/vfio/container-base.c | 1 +
hw/vfio/container.c | 11 +++++------
hw/vfio/spapr.c | 10 ++++++----
6 files changed, 23 insertions(+), 20 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index bc67e1316ceeaf4439aa9dd71d8c35cc5f34366c..d3dc2f9dcbde9abf57fa1592604f60caa3cb0137 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -85,8 +85,6 @@ typedef struct VFIOContainer {
bool initialized;
uint64_t dirty_pgsizes;
uint64_t max_dirty_bitmap_size;
- unsigned long pgsizes;
- unsigned int dma_max_mappings;
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
QLIST_HEAD(, VFIORamDiscardListener) vrdl_list;
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 709096249642c7c597aa789264a09389e53664f3..85ec7e1a561af641ae2261c775332afb8e76c838 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -36,6 +36,8 @@ typedef struct VFIOAddressSpace {
typedef struct VFIOContainerBase {
const VFIOIOMMUOps *ops;
VFIOAddressSpace *space;
+ unsigned long pgsizes;
+ unsigned int dma_max_mappings;
bool dirty_pages_supported;
QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
QLIST_ENTRY(VFIOContainerBase) next;
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index cf6618f6ed3d16a4c76d641d7bdb4958dd2bd8ef..1cb53d369e871090b4d58dc6d5dd4d6cdfe455c4 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -401,6 +401,7 @@ static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl,
static void vfio_register_ram_discard_listener(VFIOContainer *container,
MemoryRegionSection *section)
{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr);
VFIORamDiscardListener *vrdl;
@@ -419,8 +420,8 @@ static void vfio_register_ram_discard_listener(VFIOContainer *container,
section->mr);
g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity));
- g_assert(container->pgsizes &&
- vrdl->granularity >= 1ULL << ctz64(container->pgsizes));
+ g_assert(bcontainer->pgsizes &&
+ vrdl->granularity >= 1ULL << ctz64(bcontainer->pgsizes));
ram_discard_listener_init(&vrdl->listener,
vfio_ram_discard_notify_populate,
@@ -441,7 +442,7 @@ static void vfio_register_ram_discard_listener(VFIOContainer *container,
* number of sections in the address space we could have over time,
* also consuming DMA mappings.
*/
- if (container->dma_max_mappings) {
+ if (bcontainer->dma_max_mappings) {
unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512;
#ifdef CONFIG_KVM
@@ -462,11 +463,11 @@ static void vfio_register_ram_discard_listener(VFIOContainer *container,
}
if (vrdl_mappings + max_memslots - vrdl_count >
- container->dma_max_mappings) {
+ bcontainer->dma_max_mappings) {
warn_report("%s: possibly running out of DMA mappings. E.g., try"
" increasing the 'block-size' of virtio-mem devies."
" Maximum possible DMA mappings: %d, Maximum possible"
- " memslots: %d", __func__, container->dma_max_mappings,
+ " memslots: %d", __func__, bcontainer->dma_max_mappings,
max_memslots);
}
}
@@ -626,7 +627,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
iommu_idx);
ret = memory_region_iommu_set_page_size_mask(giommu->iommu_mr,
- container->pgsizes,
+ bcontainer->pgsizes,
&err);
if (ret) {
g_free(giommu);
@@ -675,7 +676,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
llsize = int128_sub(llend, int128_make64(iova));
if (memory_region_is_ram_device(section->mr)) {
- hwaddr pgmask = (1ULL << ctz64(container->pgsizes)) - 1;
+ hwaddr pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1;
if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) {
trace_vfio_listener_region_add_no_dma_map(
@@ -777,7 +778,7 @@ static void vfio_listener_region_del(MemoryListener *listener,
if (memory_region_is_ram_device(section->mr)) {
hwaddr pgmask;
- pgmask = (1ULL << ctz64(container->pgsizes)) - 1;
+ pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1;
try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask));
} else if (memory_region_has_ram_discard_manager(section->mr)) {
vfio_unregister_ram_discard_listener(container, section);
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index 5d654ae17233fe06c235d33d981eb8fa92128536..dcce111349f9ded6beb72efc7a4ddd86ab7df184 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -52,6 +52,7 @@ void vfio_container_init(VFIOContainerBase *bcontainer, VFIOAddressSpace *space,
bcontainer->ops = ops;
bcontainer->space = space;
bcontainer->dirty_pages_supported = false;
+ bcontainer->dma_max_mappings = 0;
QLIST_INIT(&bcontainer->giommu_list);
}
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 7bd81eab09008530294757a81429bfea1b147815..c5a62628826c6426e8cd67ff53e73c502d7f73ba 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -154,7 +154,7 @@ static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
if (errno == EINVAL && unmap.size && !(unmap.iova + unmap.size) &&
container->iommu_type == VFIO_TYPE1v2_IOMMU) {
trace_vfio_legacy_dma_unmap_overflow_workaround();
- unmap.size -= 1ULL << ctz64(container->pgsizes);
+ unmap.size -= 1ULL << ctz64(bcontainer->pgsizes);
continue;
}
error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno));
@@ -559,7 +559,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
container = g_malloc0(sizeof(*container));
container->fd = fd;
container->error = NULL;
- container->dma_max_mappings = 0;
container->iova_ranges = NULL;
QLIST_INIT(&container->vrdl_list);
bcontainer = &container->bcontainer;
@@ -589,13 +588,13 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
}
if (info->flags & VFIO_IOMMU_INFO_PGSIZES) {
- container->pgsizes = info->iova_pgsizes;
+ bcontainer->pgsizes = info->iova_pgsizes;
} else {
- container->pgsizes = qemu_real_host_page_size();
+ bcontainer->pgsizes = qemu_real_host_page_size();
}
- if (!vfio_get_info_dma_avail(info, &container->dma_max_mappings)) {
- container->dma_max_mappings = 65535;
+ if (!vfio_get_info_dma_avail(info, &bcontainer->dma_max_mappings)) {
+ bcontainer->dma_max_mappings = 65535;
}
vfio_get_info_iova_range(info, container);
diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c
index 83da2f7ec213dab2acca9b96a1d07a22a49d22c4..4f76bdd3cadfd282b7f2a01c19ee772be5627068 100644
--- a/hw/vfio/spapr.c
+++ b/hw/vfio/spapr.c
@@ -226,6 +226,7 @@ static int vfio_spapr_create_window(VFIOContainer *container,
hwaddr *pgsize)
{
int ret = 0;
+ VFIOContainerBase *bcontainer = &container->bcontainer;
IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr);
uint64_t pagesize = memory_region_iommu_get_min_page_size(iommu_mr), pgmask;
unsigned entries, bits_total, bits_per_level, max_levels;
@@ -239,13 +240,13 @@ static int vfio_spapr_create_window(VFIOContainer *container,
if (pagesize > rampagesize) {
pagesize = rampagesize;
}
- pgmask = container->pgsizes & (pagesize | (pagesize - 1));
+ pgmask = bcontainer->pgsizes & (pagesize | (pagesize - 1));
pagesize = pgmask ? (1ULL << (63 - clz64(pgmask))) : 0;
if (!pagesize) {
error_report("Host doesn't support page size 0x%"PRIx64
", the supported mask is 0x%lx",
memory_region_iommu_get_min_page_size(iommu_mr),
- container->pgsizes);
+ bcontainer->pgsizes);
return -EINVAL;
}
@@ -421,6 +422,7 @@ void vfio_container_del_section_window(VFIOContainer *container,
int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
struct vfio_iommu_spapr_tce_info info;
bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU;
int ret, fd = container->fd;
@@ -461,7 +463,7 @@ int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
}
if (v2) {
- container->pgsizes = info.ddw.pgsizes;
+ bcontainer->pgsizes = info.ddw.pgsizes;
/*
* There is a default window in just created container.
* To make region_add/del simpler, we better remove this
@@ -476,7 +478,7 @@ int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
}
} else {
/* The default table uses 4K pages */
- container->pgsizes = 0x1000;
+ bcontainer->pgsizes = 0x1000;
vfio_host_win_add(container, info.dma32_window_start,
info.dma32_window_start +
info.dma32_window_size - 1,
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 11/47] vfio/container: Move vrdl_list to base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (9 preceding siblings ...)
2023-12-19 18:56 ` [PULL 10/47] vfio/container: Move pgsizes and dma_max_mappings " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 12/47] vfio/container: Move listener " Cédric Le Goater
` (36 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
No functional change intended.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 11 --------
include/hw/vfio/vfio-container-base.h | 11 ++++++++
hw/vfio/common.c | 38 +++++++++++++--------------
hw/vfio/container-base.c | 1 +
hw/vfio/container.c | 1 -
5 files changed, 31 insertions(+), 31 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index d3dc2f9dcbde9abf57fa1592604f60caa3cb0137..8a607a4c1762afbd2f9525c69bb9ba83d194d8f6 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -87,20 +87,9 @@ typedef struct VFIOContainer {
uint64_t max_dirty_bitmap_size;
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
- QLIST_HEAD(, VFIORamDiscardListener) vrdl_list;
GList *iova_ranges;
} VFIOContainer;
-typedef struct VFIORamDiscardListener {
- VFIOContainer *container;
- MemoryRegion *mr;
- hwaddr offset_within_address_space;
- hwaddr size;
- uint64_t granularity;
- RamDiscardListener listener;
- QLIST_ENTRY(VFIORamDiscardListener) next;
-} VFIORamDiscardListener;
-
typedef struct VFIOHostDMAWindow {
hwaddr min_iova;
hwaddr max_iova;
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 85ec7e1a561af641ae2261c775332afb8e76c838..8e05b5ac5a1b252365c39310c64e77fd63dbb9ee 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -40,6 +40,7 @@ typedef struct VFIOContainerBase {
unsigned int dma_max_mappings;
bool dirty_pages_supported;
QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
+ QLIST_HEAD(, VFIORamDiscardListener) vrdl_list;
QLIST_ENTRY(VFIOContainerBase) next;
QLIST_HEAD(, VFIODevice) device_list;
} VFIOContainerBase;
@@ -52,6 +53,16 @@ typedef struct VFIOGuestIOMMU {
QLIST_ENTRY(VFIOGuestIOMMU) giommu_next;
} VFIOGuestIOMMU;
+typedef struct VFIORamDiscardListener {
+ VFIOContainerBase *bcontainer;
+ MemoryRegion *mr;
+ hwaddr offset_within_address_space;
+ hwaddr size;
+ uint64_t granularity;
+ RamDiscardListener listener;
+ QLIST_ENTRY(VFIORamDiscardListener) next;
+} VFIORamDiscardListener;
+
int vfio_container_dma_map(VFIOContainerBase *bcontainer,
hwaddr iova, ram_addr_t size,
void *vaddr, bool readonly);
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 1cb53d369e871090b4d58dc6d5dd4d6cdfe455c4..f15665789fa5f99e0ec440696434679a8142fb48 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -351,13 +351,13 @@ static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl,
{
VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener,
listener);
+ VFIOContainerBase *bcontainer = vrdl->bcontainer;
const hwaddr size = int128_get64(section->size);
const hwaddr iova = section->offset_within_address_space;
int ret;
/* Unmap with a single call. */
- ret = vfio_container_dma_unmap(&vrdl->container->bcontainer,
- iova, size , NULL);
+ ret = vfio_container_dma_unmap(bcontainer, iova, size , NULL);
if (ret) {
error_report("%s: vfio_container_dma_unmap() failed: %s", __func__,
strerror(-ret));
@@ -369,6 +369,7 @@ static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl,
{
VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener,
listener);
+ VFIOContainerBase *bcontainer = vrdl->bcontainer;
const hwaddr end = section->offset_within_region +
int128_get64(section->size);
hwaddr start, next, iova;
@@ -387,8 +388,8 @@ static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl,
section->offset_within_address_space;
vaddr = memory_region_get_ram_ptr(section->mr) + start;
- ret = vfio_container_dma_map(&vrdl->container->bcontainer, iova,
- next - start, vaddr, section->readonly);
+ ret = vfio_container_dma_map(bcontainer, iova, next - start,
+ vaddr, section->readonly);
if (ret) {
/* Rollback */
vfio_ram_discard_notify_discard(rdl, section);
@@ -398,10 +399,9 @@ static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl,
return 0;
}
-static void vfio_register_ram_discard_listener(VFIOContainer *container,
+static void vfio_register_ram_discard_listener(VFIOContainerBase *bcontainer,
MemoryRegionSection *section)
{
- VFIOContainerBase *bcontainer = &container->bcontainer;
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr);
VFIORamDiscardListener *vrdl;
@@ -412,7 +412,7 @@ static void vfio_register_ram_discard_listener(VFIOContainer *container,
g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE));
vrdl = g_new0(VFIORamDiscardListener, 1);
- vrdl->container = container;
+ vrdl->bcontainer = bcontainer;
vrdl->mr = section->mr;
vrdl->offset_within_address_space = section->offset_within_address_space;
vrdl->size = int128_get64(section->size);
@@ -427,7 +427,7 @@ static void vfio_register_ram_discard_listener(VFIOContainer *container,
vfio_ram_discard_notify_populate,
vfio_ram_discard_notify_discard, true);
ram_discard_manager_register_listener(rdm, &vrdl->listener, section);
- QLIST_INSERT_HEAD(&container->vrdl_list, vrdl, next);
+ QLIST_INSERT_HEAD(&bcontainer->vrdl_list, vrdl, next);
/*
* Sanity-check if we have a theoretically problematic setup where we could
@@ -451,7 +451,7 @@ static void vfio_register_ram_discard_listener(VFIOContainer *container,
}
#endif
- QLIST_FOREACH(vrdl, &container->vrdl_list, next) {
+ QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) {
hwaddr start, end;
start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space,
@@ -473,13 +473,13 @@ static void vfio_register_ram_discard_listener(VFIOContainer *container,
}
}
-static void vfio_unregister_ram_discard_listener(VFIOContainer *container,
+static void vfio_unregister_ram_discard_listener(VFIOContainerBase *bcontainer,
MemoryRegionSection *section)
{
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr);
VFIORamDiscardListener *vrdl = NULL;
- QLIST_FOREACH(vrdl, &container->vrdl_list, next) {
+ QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) {
if (vrdl->mr == section->mr &&
vrdl->offset_within_address_space ==
section->offset_within_address_space) {
@@ -663,7 +663,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
* about changes.
*/
if (memory_region_has_ram_discard_manager(section->mr)) {
- vfio_register_ram_discard_listener(container, section);
+ vfio_register_ram_discard_listener(bcontainer, section);
return;
}
@@ -781,7 +781,7 @@ static void vfio_listener_region_del(MemoryListener *listener,
pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1;
try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask));
} else if (memory_region_has_ram_discard_manager(section->mr)) {
- vfio_unregister_ram_discard_listener(container, section);
+ vfio_unregister_ram_discard_listener(bcontainer, section);
/* Unregistering will trigger an unmap. */
try_unmap = false;
}
@@ -1260,17 +1260,17 @@ static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section,
* Sync the whole mapped region (spanning multiple individual mappings)
* in one go.
*/
- return vfio_get_dirty_bitmap(&vrdl->container->bcontainer, iova, size,
- ram_addr);
+ return vfio_get_dirty_bitmap(vrdl->bcontainer, iova, size, ram_addr);
}
-static int vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainer *container,
- MemoryRegionSection *section)
+static int
+vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section)
{
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr);
VFIORamDiscardListener *vrdl = NULL;
- QLIST_FOREACH(vrdl, &container->vrdl_list, next) {
+ QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) {
if (vrdl->mr == section->mr &&
vrdl->offset_within_address_space ==
section->offset_within_address_space) {
@@ -1324,7 +1324,7 @@ static int vfio_sync_dirty_bitmap(VFIOContainer *container,
}
return 0;
} else if (memory_region_has_ram_discard_manager(section->mr)) {
- return vfio_sync_ram_discard_listener_dirty_bitmap(container, section);
+ return vfio_sync_ram_discard_listener_dirty_bitmap(bcontainer, section);
}
ram_addr = memory_region_get_ram_addr(section->mr) +
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index dcce111349f9ded6beb72efc7a4ddd86ab7df184..584eee4ba1170e103fb45003c514084ee3326353 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -54,6 +54,7 @@ void vfio_container_init(VFIOContainerBase *bcontainer, VFIOAddressSpace *space,
bcontainer->dirty_pages_supported = false;
bcontainer->dma_max_mappings = 0;
QLIST_INIT(&bcontainer->giommu_list);
+ QLIST_INIT(&bcontainer->vrdl_list);
}
void vfio_container_destroy(VFIOContainerBase *bcontainer)
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index c5a62628826c6426e8cd67ff53e73c502d7f73ba..6ba2e2f8c41e72d24b32f3efa31c9683da18387c 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -560,7 +560,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
container->fd = fd;
container->error = NULL;
container->iova_ranges = NULL;
- QLIST_INIT(&container->vrdl_list);
bcontainer = &container->bcontainer;
vfio_container_init(bcontainer, space, &vfio_legacy_ops);
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 12/47] vfio/container: Move listener to base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (10 preceding siblings ...)
2023-12-19 18:56 ` [PULL 11/47] vfio/container: Move vrdl_list " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 13/47] vfio/container: Move dirty_pgsizes and max_dirty_bitmap_size " Cédric Le Goater
` (35 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
Move listener to base container. Also error and initialized fields
are moved at the same time.
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 3 -
include/hw/vfio/vfio-container-base.h | 3 +
hw/vfio/common.c | 110 +++++++++++++-------------
hw/vfio/container-base.c | 1 +
hw/vfio/container.c | 19 +++--
hw/vfio/spapr.c | 11 +--
6 files changed, 74 insertions(+), 73 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 8a607a4c1762afbd2f9525c69bb9ba83d194d8f6..922022cbc6c69ddb39c7241fc983b959be566d33 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -78,11 +78,8 @@ struct VFIOGroup;
typedef struct VFIOContainer {
VFIOContainerBase bcontainer;
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
- MemoryListener listener;
MemoryListener prereg_listener;
unsigned iommu_type;
- Error *error;
- bool initialized;
uint64_t dirty_pgsizes;
uint64_t max_dirty_bitmap_size;
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 8e05b5ac5a1b252365c39310c64e77fd63dbb9ee..95f8d319e0c6cf490a3e92dbf1735ad12b53fc65 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -36,6 +36,9 @@ typedef struct VFIOAddressSpace {
typedef struct VFIOContainerBase {
const VFIOIOMMUOps *ops;
VFIOAddressSpace *space;
+ MemoryListener listener;
+ Error *error;
+ bool initialized;
unsigned long pgsizes;
unsigned int dma_max_mappings;
bool dirty_pages_supported;
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index f15665789fa5f99e0ec440696434679a8142fb48..be623e544b641c0bc202b30ed350911b3ec99078 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -541,7 +541,7 @@ static bool vfio_listener_valid_section(MemoryRegionSection *section,
return true;
}
-static bool vfio_get_section_iova_range(VFIOContainer *container,
+static bool vfio_get_section_iova_range(VFIOContainerBase *bcontainer,
MemoryRegionSection *section,
hwaddr *out_iova, hwaddr *out_end,
Int128 *out_llend)
@@ -569,8 +569,10 @@ static bool vfio_get_section_iova_range(VFIOContainer *container,
static void vfio_listener_region_add(MemoryListener *listener,
MemoryRegionSection *section)
{
- VFIOContainer *container = container_of(listener, VFIOContainer, listener);
- VFIOContainerBase *bcontainer = &container->bcontainer;
+ VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
+ listener);
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
hwaddr iova, end;
Int128 llend, llsize;
void *vaddr;
@@ -581,7 +583,8 @@ static void vfio_listener_region_add(MemoryListener *listener,
return;
}
- if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) {
+ if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end,
+ &llend)) {
if (memory_region_is_ram_device(section->mr)) {
trace_vfio_listener_region_add_no_dma_map(
memory_region_name(section->mr),
@@ -688,13 +691,12 @@ static void vfio_listener_region_add(MemoryListener *listener,
}
}
- ret = vfio_container_dma_map(&container->bcontainer,
- iova, int128_get64(llsize), vaddr,
- section->readonly);
+ ret = vfio_container_dma_map(bcontainer, iova, int128_get64(llsize),
+ vaddr, section->readonly);
if (ret) {
error_setg(&err, "vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx", %p) = %d (%s)",
- container, iova, int128_get64(llsize), vaddr, ret,
+ bcontainer, iova, int128_get64(llsize), vaddr, ret,
strerror(-ret));
if (memory_region_is_ram_device(section->mr)) {
/* Allow unexpected mappings not to be fatal for RAM devices */
@@ -716,9 +718,9 @@ fail:
* can gracefully fail. Runtime, there's not much we can do other
* than throw a hardware error.
*/
- if (!container->initialized) {
- if (!container->error) {
- error_propagate_prepend(&container->error, err,
+ if (!bcontainer->initialized) {
+ if (!bcontainer->error) {
+ error_propagate_prepend(&bcontainer->error, err,
"Region %s: ",
memory_region_name(section->mr));
} else {
@@ -733,8 +735,10 @@ fail:
static void vfio_listener_region_del(MemoryListener *listener,
MemoryRegionSection *section)
{
- VFIOContainer *container = container_of(listener, VFIOContainer, listener);
- VFIOContainerBase *bcontainer = &container->bcontainer;
+ VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
+ listener);
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
hwaddr iova, end;
Int128 llend, llsize;
int ret;
@@ -767,7 +771,8 @@ static void vfio_listener_region_del(MemoryListener *listener,
*/
}
- if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) {
+ if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end,
+ &llend)) {
return;
}
@@ -790,22 +795,22 @@ static void vfio_listener_region_del(MemoryListener *listener,
if (int128_eq(llsize, int128_2_64())) {
/* The unmap ioctl doesn't accept a full 64-bit span. */
llsize = int128_rshift(llsize, 1);
- ret = vfio_container_dma_unmap(&container->bcontainer, iova,
+ ret = vfio_container_dma_unmap(bcontainer, iova,
int128_get64(llsize), NULL);
if (ret) {
error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx") = %d (%s)",
- container, iova, int128_get64(llsize), ret,
+ bcontainer, iova, int128_get64(llsize), ret,
strerror(-ret));
}
iova += int128_get64(llsize);
}
- ret = vfio_container_dma_unmap(&container->bcontainer, iova,
+ ret = vfio_container_dma_unmap(bcontainer, iova,
int128_get64(llsize), NULL);
if (ret) {
error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx") = %d (%s)",
- container, iova, int128_get64(llsize), ret,
+ bcontainer, iova, int128_get64(llsize), ret,
strerror(-ret));
}
}
@@ -825,16 +830,15 @@ typedef struct VFIODirtyRanges {
} VFIODirtyRanges;
typedef struct VFIODirtyRangesListener {
- VFIOContainer *container;
+ VFIOContainerBase *bcontainer;
VFIODirtyRanges ranges;
MemoryListener listener;
} VFIODirtyRangesListener;
static bool vfio_section_is_vfio_pci(MemoryRegionSection *section,
- VFIOContainer *container)
+ VFIOContainerBase *bcontainer)
{
VFIOPCIDevice *pcidev;
- VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
Object *owner;
@@ -863,7 +867,7 @@ static void vfio_dirty_tracking_update(MemoryListener *listener,
hwaddr iova, end, *min, *max;
if (!vfio_listener_valid_section(section, "tracking_update") ||
- !vfio_get_section_iova_range(dirty->container, section,
+ !vfio_get_section_iova_range(dirty->bcontainer, section,
&iova, &end, NULL)) {
return;
}
@@ -887,7 +891,7 @@ static void vfio_dirty_tracking_update(MemoryListener *listener,
* The alternative would be an IOVATree but that has a much bigger runtime
* overhead and unnecessary complexity.
*/
- if (vfio_section_is_vfio_pci(section, dirty->container) &&
+ if (vfio_section_is_vfio_pci(section, dirty->bcontainer) &&
iova >= UINT32_MAX) {
min = &range->minpci64;
max = &range->maxpci64;
@@ -911,7 +915,7 @@ static const MemoryListener vfio_dirty_tracking_listener = {
.region_add = vfio_dirty_tracking_update,
};
-static void vfio_dirty_tracking_init(VFIOContainer *container,
+static void vfio_dirty_tracking_init(VFIOContainerBase *bcontainer,
VFIODirtyRanges *ranges)
{
VFIODirtyRangesListener dirty;
@@ -921,10 +925,10 @@ static void vfio_dirty_tracking_init(VFIOContainer *container,
dirty.ranges.min64 = UINT64_MAX;
dirty.ranges.minpci64 = UINT64_MAX;
dirty.listener = vfio_dirty_tracking_listener;
- dirty.container = container;
+ dirty.bcontainer = bcontainer;
memory_listener_register(&dirty.listener,
- container->bcontainer.space->as);
+ bcontainer->space->as);
*ranges = dirty.ranges;
@@ -936,12 +940,11 @@ static void vfio_dirty_tracking_init(VFIOContainer *container,
memory_listener_unregister(&dirty.listener);
}
-static void vfio_devices_dma_logging_stop(VFIOContainer *container)
+static void vfio_devices_dma_logging_stop(VFIOContainerBase *bcontainer)
{
uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature),
sizeof(uint64_t))] = {};
struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
- VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
feature->argsz = sizeof(buf);
@@ -962,7 +965,7 @@ static void vfio_devices_dma_logging_stop(VFIOContainer *container)
}
static struct vfio_device_feature *
-vfio_device_feature_dma_logging_start_create(VFIOContainer *container,
+vfio_device_feature_dma_logging_start_create(VFIOContainerBase *bcontainer,
VFIODirtyRanges *tracking)
{
struct vfio_device_feature *feature;
@@ -1035,16 +1038,15 @@ static void vfio_device_feature_dma_logging_start_destroy(
g_free(feature);
}
-static int vfio_devices_dma_logging_start(VFIOContainer *container)
+static int vfio_devices_dma_logging_start(VFIOContainerBase *bcontainer)
{
struct vfio_device_feature *feature;
VFIODirtyRanges ranges;
- VFIOContainerBase *bcontainer = &container->bcontainer;
VFIODevice *vbasedev;
int ret = 0;
- vfio_dirty_tracking_init(container, &ranges);
- feature = vfio_device_feature_dma_logging_start_create(container,
+ vfio_dirty_tracking_init(bcontainer, &ranges);
+ feature = vfio_device_feature_dma_logging_start_create(bcontainer,
&ranges);
if (!feature) {
return -errno;
@@ -1067,7 +1069,7 @@ static int vfio_devices_dma_logging_start(VFIOContainer *container)
out:
if (ret) {
- vfio_devices_dma_logging_stop(container);
+ vfio_devices_dma_logging_stop(bcontainer);
}
vfio_device_feature_dma_logging_start_destroy(feature);
@@ -1077,14 +1079,14 @@ out:
static void vfio_listener_log_global_start(MemoryListener *listener)
{
- VFIOContainer *container = container_of(listener, VFIOContainer, listener);
+ VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
+ listener);
int ret;
- if (vfio_devices_all_device_dirty_tracking(&container->bcontainer)) {
- ret = vfio_devices_dma_logging_start(container);
+ if (vfio_devices_all_device_dirty_tracking(bcontainer)) {
+ ret = vfio_devices_dma_logging_start(bcontainer);
} else {
- ret = vfio_container_set_dirty_page_tracking(&container->bcontainer,
- true);
+ ret = vfio_container_set_dirty_page_tracking(bcontainer, true);
}
if (ret) {
@@ -1096,14 +1098,14 @@ static void vfio_listener_log_global_start(MemoryListener *listener)
static void vfio_listener_log_global_stop(MemoryListener *listener)
{
- VFIOContainer *container = container_of(listener, VFIOContainer, listener);
+ VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
+ listener);
int ret = 0;
- if (vfio_devices_all_device_dirty_tracking(&container->bcontainer)) {
- vfio_devices_dma_logging_stop(container);
+ if (vfio_devices_all_device_dirty_tracking(bcontainer)) {
+ vfio_devices_dma_logging_stop(bcontainer);
} else {
- ret = vfio_container_set_dirty_page_tracking(&container->bcontainer,
- false);
+ ret = vfio_container_set_dirty_page_tracking(bcontainer, false);
}
if (ret) {
@@ -1214,8 +1216,6 @@ static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
vfio_giommu_dirty_notifier, n);
VFIOGuestIOMMU *giommu = gdn->giommu;
VFIOContainerBase *bcontainer = giommu->bcontainer;
- VFIOContainer *container = container_of(bcontainer, VFIOContainer,
- bcontainer);
hwaddr iova = iotlb->iova + giommu->iommu_offset;
ram_addr_t translated_addr;
int ret = -EINVAL;
@@ -1230,12 +1230,12 @@ static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
rcu_read_lock();
if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) {
- ret = vfio_get_dirty_bitmap(&container->bcontainer, iova,
- iotlb->addr_mask + 1, translated_addr);
+ ret = vfio_get_dirty_bitmap(bcontainer, iova, iotlb->addr_mask + 1,
+ translated_addr);
if (ret) {
error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", "
"0x%"HWADDR_PRIx") = %d (%s)",
- container, iova, iotlb->addr_mask + 1, ret,
+ bcontainer, iova, iotlb->addr_mask + 1, ret,
strerror(-ret));
}
}
@@ -1291,10 +1291,9 @@ vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainerBase *bcontainer,
&vrdl);
}
-static int vfio_sync_dirty_bitmap(VFIOContainer *container,
+static int vfio_sync_dirty_bitmap(VFIOContainerBase *bcontainer,
MemoryRegionSection *section)
{
- VFIOContainerBase *bcontainer = &container->bcontainer;
ram_addr_t ram_addr;
if (memory_region_is_iommu(section->mr)) {
@@ -1330,7 +1329,7 @@ static int vfio_sync_dirty_bitmap(VFIOContainer *container,
ram_addr = memory_region_get_ram_addr(section->mr) +
section->offset_within_region;
- return vfio_get_dirty_bitmap(&container->bcontainer,
+ return vfio_get_dirty_bitmap(bcontainer,
REAL_HOST_PAGE_ALIGN(section->offset_within_address_space),
int128_get64(section->size), ram_addr);
}
@@ -1338,15 +1337,16 @@ static int vfio_sync_dirty_bitmap(VFIOContainer *container,
static void vfio_listener_log_sync(MemoryListener *listener,
MemoryRegionSection *section)
{
- VFIOContainer *container = container_of(listener, VFIOContainer, listener);
+ VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
+ listener);
int ret;
if (vfio_listener_skipped_section(section)) {
return;
}
- if (vfio_devices_all_dirty_tracking(&container->bcontainer)) {
- ret = vfio_sync_dirty_bitmap(container, section);
+ if (vfio_devices_all_dirty_tracking(bcontainer)) {
+ ret = vfio_sync_dirty_bitmap(bcontainer, section);
if (ret) {
error_report("vfio: Failed to sync dirty bitmap, err: %d (%s)", ret,
strerror(-ret));
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index 584eee4ba1170e103fb45003c514084ee3326353..7f508669f5dbdab211bed217c143c2c8387b5ae4 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -51,6 +51,7 @@ void vfio_container_init(VFIOContainerBase *bcontainer, VFIOAddressSpace *space,
{
bcontainer->ops = ops;
bcontainer->space = space;
+ bcontainer->error = NULL;
bcontainer->dirty_pages_supported = false;
bcontainer->dma_max_mappings = 0;
QLIST_INIT(&bcontainer->giommu_list);
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 6ba2e2f8c41e72d24b32f3efa31c9683da18387c..5c1dee8c9f889fb6746449fe4fe611289cb247db 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -453,6 +453,7 @@ static void vfio_get_iommu_info_migration(VFIOContainer *container,
{
struct vfio_info_cap_header *hdr;
struct vfio_iommu_type1_info_cap_migration *cap_mig;
+ VFIOContainerBase *bcontainer = &container->bcontainer;
hdr = vfio_get_iommu_info_cap(info, VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION);
if (!hdr) {
@@ -467,7 +468,7 @@ static void vfio_get_iommu_info_migration(VFIOContainer *container,
* qemu_real_host_page_size to mark those dirty.
*/
if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) {
- container->bcontainer.dirty_pages_supported = true;
+ bcontainer->dirty_pages_supported = true;
container->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size;
container->dirty_pgsizes = cap_mig->pgsize_bitmap;
}
@@ -558,7 +559,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
container = g_malloc0(sizeof(*container));
container->fd = fd;
- container->error = NULL;
container->iova_ranges = NULL;
bcontainer = &container->bcontainer;
vfio_container_init(bcontainer, space, &vfio_legacy_ops);
@@ -621,25 +621,24 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
group->container = container;
QLIST_INSERT_HEAD(&container->group_list, group, container_next);
- container->listener = vfio_memory_listener;
-
- memory_listener_register(&container->listener, bcontainer->space->as);
+ bcontainer->listener = vfio_memory_listener;
+ memory_listener_register(&bcontainer->listener, bcontainer->space->as);
- if (container->error) {
+ if (bcontainer->error) {
ret = -1;
- error_propagate_prepend(errp, container->error,
+ error_propagate_prepend(errp, bcontainer->error,
"memory listener initialization failed: ");
goto listener_release_exit;
}
- container->initialized = true;
+ bcontainer->initialized = true;
return 0;
listener_release_exit:
QLIST_REMOVE(group, container_next);
QLIST_REMOVE(bcontainer, next);
vfio_kvm_device_del_group(group);
- memory_listener_unregister(&container->listener);
+ memory_listener_unregister(&bcontainer->listener);
if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU ||
container->iommu_type == VFIO_SPAPR_TCE_IOMMU) {
vfio_spapr_container_deinit(container);
@@ -674,7 +673,7 @@ static void vfio_disconnect_container(VFIOGroup *group)
* group.
*/
if (QLIST_EMPTY(&container->group_list)) {
- memory_listener_unregister(&container->listener);
+ memory_listener_unregister(&bcontainer->listener);
if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU ||
container->iommu_type == VFIO_SPAPR_TCE_IOMMU) {
vfio_spapr_container_deinit(container);
diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c
index 4f76bdd3cadfd282b7f2a01c19ee772be5627068..7a50975f2598b5e58a3dfae552ee49999516df99 100644
--- a/hw/vfio/spapr.c
+++ b/hw/vfio/spapr.c
@@ -46,6 +46,7 @@ static void vfio_prereg_listener_region_add(MemoryListener *listener,
{
VFIOContainer *container = container_of(listener, VFIOContainer,
prereg_listener);
+ VFIOContainerBase *bcontainer = &container->bcontainer;
const hwaddr gpa = section->offset_within_address_space;
hwaddr end;
int ret;
@@ -88,9 +89,9 @@ static void vfio_prereg_listener_region_add(MemoryListener *listener,
* can gracefully fail. Runtime, there's not much we can do other
* than throw a hardware error.
*/
- if (!container->initialized) {
- if (!container->error) {
- error_setg_errno(&container->error, -ret,
+ if (!bcontainer->initialized) {
+ if (!bcontainer->error) {
+ error_setg_errno(&bcontainer->error, -ret,
"Memory registering failed");
}
} else {
@@ -445,9 +446,9 @@ int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
memory_listener_register(&container->prereg_listener,
&address_space_memory);
- if (container->error) {
+ if (bcontainer->error) {
ret = -1;
- error_propagate_prepend(errp, container->error,
+ error_propagate_prepend(errp, bcontainer->error,
"RAM memory listener initialization failed: ");
goto listener_unregister_exit;
}
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 13/47] vfio/container: Move dirty_pgsizes and max_dirty_bitmap_size to base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (11 preceding siblings ...)
2023-12-19 18:56 ` [PULL 12/47] vfio/container: Move listener " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 14/47] vfio/container: Move iova_ranges " Cédric Le Goater
` (34 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 2 --
include/hw/vfio/vfio-container-base.h | 2 ++
hw/vfio/container.c | 9 +++++----
3 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 922022cbc6c69ddb39c7241fc983b959be566d33..b1c9fe711bb9cb661ca036214a63bb1607e956a3 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -80,8 +80,6 @@ typedef struct VFIOContainer {
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
MemoryListener prereg_listener;
unsigned iommu_type;
- uint64_t dirty_pgsizes;
- uint64_t max_dirty_bitmap_size;
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
GList *iova_ranges;
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 95f8d319e0c6cf490a3e92dbf1735ad12b53fc65..80e4a993c5cfbceb9ab8366eae4ba5bafe26b208 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -39,6 +39,8 @@ typedef struct VFIOContainerBase {
MemoryListener listener;
Error *error;
bool initialized;
+ uint64_t dirty_pgsizes;
+ uint64_t max_dirty_bitmap_size;
unsigned long pgsizes;
unsigned int dma_max_mappings;
bool dirty_pages_supported;
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 5c1dee8c9f889fb6746449fe4fe611289cb247db..c8088a8174f18e58c6704e09cf9a7c09a0124175 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -64,6 +64,7 @@ static int vfio_dma_unmap_bitmap(VFIOContainer *container,
hwaddr iova, ram_addr_t size,
IOMMUTLBEntry *iotlb)
{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
struct vfio_iommu_type1_dma_unmap *unmap;
struct vfio_bitmap *bitmap;
VFIOBitmap vbmap;
@@ -91,7 +92,7 @@ static int vfio_dma_unmap_bitmap(VFIOContainer *container,
bitmap->size = vbmap.size;
bitmap->data = (__u64 *)vbmap.bitmap;
- if (vbmap.size > container->max_dirty_bitmap_size) {
+ if (vbmap.size > bcontainer->max_dirty_bitmap_size) {
error_report("UNMAP: Size of bitmap too big 0x%"PRIx64, vbmap.size);
ret = -E2BIG;
goto unmap_exit;
@@ -131,7 +132,7 @@ static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
if (iotlb && vfio_devices_all_running_and_mig_active(bcontainer)) {
if (!vfio_devices_all_device_dirty_tracking(bcontainer) &&
- container->bcontainer.dirty_pages_supported) {
+ bcontainer->dirty_pages_supported) {
return vfio_dma_unmap_bitmap(container, iova, size, iotlb);
}
@@ -469,8 +470,8 @@ static void vfio_get_iommu_info_migration(VFIOContainer *container,
*/
if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) {
bcontainer->dirty_pages_supported = true;
- container->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size;
- container->dirty_pgsizes = cap_mig->pgsize_bitmap;
+ bcontainer->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size;
+ bcontainer->dirty_pgsizes = cap_mig->pgsize_bitmap;
}
}
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 14/47] vfio/container: Move iova_ranges to base container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (12 preceding siblings ...)
2023-12-19 18:56 ` [PULL 13/47] vfio/container: Move dirty_pgsizes and max_dirty_bitmap_size " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 15/47] vfio/container: Implement attach/detach_device Cédric Le Goater
` (33 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Meanwhile remove the helper function vfio_free_container as it
only calls g_free now.
No functional change intended.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 1 -
include/hw/vfio/vfio-container-base.h | 1 +
hw/vfio/common.c | 5 +++--
hw/vfio/container-base.c | 3 +++
hw/vfio/container.c | 19 ++++++-------------
5 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index b1c9fe711bb9cb661ca036214a63bb1607e956a3..b9e5a0e64b46bacae99c2c542f1b7c7560957080 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -82,7 +82,6 @@ typedef struct VFIOContainer {
unsigned iommu_type;
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
- GList *iova_ranges;
} VFIOContainer;
typedef struct VFIOHostDMAWindow {
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 80e4a993c5cfbceb9ab8366eae4ba5bafe26b208..9658ffb526f61734611fea2a2a8401fdd77b02fb 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -48,6 +48,7 @@ typedef struct VFIOContainerBase {
QLIST_HEAD(, VFIORamDiscardListener) vrdl_list;
QLIST_ENTRY(VFIOContainerBase) next;
QLIST_HEAD(, VFIODevice) device_list;
+ GList *iova_ranges;
} VFIOContainerBase;
typedef struct VFIOGuestIOMMU {
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index be623e544b641c0bc202b30ed350911b3ec99078..8ef2e7967d6572e3ad3d53d6d5b5d3d0c95301ae 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -637,9 +637,10 @@ static void vfio_listener_region_add(MemoryListener *listener,
goto fail;
}
- if (container->iova_ranges) {
+ if (bcontainer->iova_ranges) {
ret = memory_region_iommu_set_iova_ranges(giommu->iommu_mr,
- container->iova_ranges, &err);
+ bcontainer->iova_ranges,
+ &err);
if (ret) {
g_free(giommu);
goto fail;
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index 7f508669f5dbdab211bed217c143c2c8387b5ae4..0177f4374132d4bbdc5f992e9c3fddb138809307 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -54,6 +54,7 @@ void vfio_container_init(VFIOContainerBase *bcontainer, VFIOAddressSpace *space,
bcontainer->error = NULL;
bcontainer->dirty_pages_supported = false;
bcontainer->dma_max_mappings = 0;
+ bcontainer->iova_ranges = NULL;
QLIST_INIT(&bcontainer->giommu_list);
QLIST_INIT(&bcontainer->vrdl_list);
}
@@ -70,4 +71,6 @@ void vfio_container_destroy(VFIOContainerBase *bcontainer)
QLIST_REMOVE(giommu, giommu_next);
g_free(giommu);
}
+
+ g_list_free_full(bcontainer->iova_ranges, g_free);
}
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index c8088a8174f18e58c6704e09cf9a7c09a0124175..721c0d7375bd1e07f04e33558e8ca3b3f234412d 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -308,7 +308,7 @@ bool vfio_get_info_dma_avail(struct vfio_iommu_type1_info *info,
}
static bool vfio_get_info_iova_range(struct vfio_iommu_type1_info *info,
- VFIOContainer *container)
+ VFIOContainerBase *bcontainer)
{
struct vfio_info_cap_header *hdr;
struct vfio_iommu_type1_info_cap_iova_range *cap;
@@ -326,8 +326,8 @@ static bool vfio_get_info_iova_range(struct vfio_iommu_type1_info *info,
range_set_bounds(range, cap->iova_ranges[i].start,
cap->iova_ranges[i].end);
- container->iova_ranges =
- range_list_insert(container->iova_ranges, range);
+ bcontainer->iova_ranges =
+ range_list_insert(bcontainer->iova_ranges, range);
}
return true;
@@ -475,12 +475,6 @@ static void vfio_get_iommu_info_migration(VFIOContainer *container,
}
}
-static void vfio_free_container(VFIOContainer *container)
-{
- g_list_free_full(container->iova_ranges, g_free);
- g_free(container);
-}
-
static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
Error **errp)
{
@@ -560,7 +554,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
container = g_malloc0(sizeof(*container));
container->fd = fd;
- container->iova_ranges = NULL;
bcontainer = &container->bcontainer;
vfio_container_init(bcontainer, space, &vfio_legacy_ops);
@@ -597,7 +590,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
bcontainer->dma_max_mappings = 65535;
}
- vfio_get_info_iova_range(info, container);
+ vfio_get_info_iova_range(info, bcontainer);
vfio_get_iommu_info_migration(container, info);
g_free(info);
@@ -649,7 +642,7 @@ enable_discards_exit:
vfio_ram_block_discard_disable(container, false);
free_container_exit:
- vfio_free_container(container);
+ g_free(container);
close_fd_exit:
close(fd);
@@ -693,7 +686,7 @@ static void vfio_disconnect_container(VFIOGroup *group)
trace_vfio_disconnect_container(container->fd);
close(container->fd);
- vfio_free_container(container);
+ g_free(container);
vfio_put_address_space(space);
}
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 15/47] vfio/container: Implement attach/detach_device
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (13 preceding siblings ...)
2023-12-19 18:56 ` [PULL 14/47] vfio/container: Move iova_ranges " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 16/47] vfio/spapr: Introduce spapr backend and target interface Cédric Le Goater
` (32 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Yi Sun,
Cédric Le Goater
From: Eric Auger <eric.auger@redhat.com>
No functional change intended.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Yi Sun <yi.y.sun@linux.intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/common.c | 16 ++++++++++++++++
hw/vfio/container.c | 12 +++++-------
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 8ef2e7967d6572e3ad3d53d6d5b5d3d0c95301ae..483ba820890460cba0ec7a4ecfed72d66433d41e 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -1498,3 +1498,19 @@ retry:
return info;
}
+
+int vfio_attach_device(char *name, VFIODevice *vbasedev,
+ AddressSpace *as, Error **errp)
+{
+ const VFIOIOMMUOps *ops = &vfio_legacy_ops;
+
+ return ops->attach_device(name, vbasedev, as, errp);
+}
+
+void vfio_detach_device(VFIODevice *vbasedev)
+{
+ if (!vbasedev->bcontainer) {
+ return;
+ }
+ vbasedev->bcontainer->ops->detach_device(vbasedev);
+}
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 721c0d7375bd1e07f04e33558e8ca3b3f234412d..6bacf38222ad9d95e05897572995e925e46b54dc 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -873,8 +873,8 @@ static int vfio_device_groupid(VFIODevice *vbasedev, Error **errp)
* @name and @vbasedev->name are likely to be different depending
* on the type of the device, hence the need for passing @name
*/
-int vfio_attach_device(char *name, VFIODevice *vbasedev,
- AddressSpace *as, Error **errp)
+static int vfio_legacy_attach_device(const char *name, VFIODevice *vbasedev,
+ AddressSpace *as, Error **errp)
{
int groupid = vfio_device_groupid(vbasedev, errp);
VFIODevice *vbasedev_iter;
@@ -914,14 +914,10 @@ int vfio_attach_device(char *name, VFIODevice *vbasedev,
return ret;
}
-void vfio_detach_device(VFIODevice *vbasedev)
+static void vfio_legacy_detach_device(VFIODevice *vbasedev)
{
VFIOGroup *group = vbasedev->group;
- if (!vbasedev->bcontainer) {
- return;
- }
-
QLIST_REMOVE(vbasedev, global_next);
QLIST_REMOVE(vbasedev, container_next);
vbasedev->bcontainer = NULL;
@@ -933,6 +929,8 @@ void vfio_detach_device(VFIODevice *vbasedev)
const VFIOIOMMUOps vfio_legacy_ops = {
.dma_map = vfio_legacy_dma_map,
.dma_unmap = vfio_legacy_dma_unmap,
+ .attach_device = vfio_legacy_attach_device,
+ .detach_device = vfio_legacy_detach_device,
.set_dirty_page_tracking = vfio_legacy_set_dirty_page_tracking,
.query_dirty_bitmap = vfio_legacy_query_dirty_bitmap,
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 16/47] vfio/spapr: Introduce spapr backend and target interface
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (14 preceding siblings ...)
2023-12-19 18:56 ` [PULL 15/47] vfio/container: Implement attach/detach_device Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 17/47] vfio/spapr: switch to spapr IOMMU BE add/del_section_window Cédric Le Goater
` (31 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Introduce an empty spapr backend which will hold spapr specific
content, currently only prereg_listener and hostwin_list.
Also introduce two spapr specific callbacks add/del_window into
VFIOIOMMUOps. Instantiate a spapr ops with a helper setup_spapr_ops
and assign it to bcontainer->ops.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-container-base.h | 6 ++++++
hw/vfio/spapr.c | 14 ++++++++++++++
2 files changed, 20 insertions(+)
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 9658ffb526f61734611fea2a2a8401fdd77b02fb..f62a14ac7388de4e682181eb141c71f61b43791b 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -101,5 +101,11 @@ struct VFIOIOMMUOps {
int (*set_dirty_page_tracking)(VFIOContainerBase *bcontainer, bool start);
int (*query_dirty_bitmap)(VFIOContainerBase *bcontainer, VFIOBitmap *vbmap,
hwaddr iova, hwaddr size);
+ /* SPAPR specific */
+ int (*add_window)(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section,
+ Error **errp);
+ void (*del_window)(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section);
};
#endif /* HW_VFIO_VFIO_CONTAINER_BASE_H */
diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c
index 7a50975f2598b5e58a3dfae552ee49999516df99..e1a6b3556347ddd29fdbe13f96ac6db9a2a869e5 100644
--- a/hw/vfio/spapr.c
+++ b/hw/vfio/spapr.c
@@ -24,6 +24,10 @@
#include "qapi/error.h"
#include "trace.h"
+typedef struct VFIOSpaprContainer {
+ VFIOContainer container;
+} VFIOSpaprContainer;
+
static bool vfio_prereg_listener_skipped_section(MemoryRegionSection *section)
{
if (memory_region_is_iommu(section->mr)) {
@@ -421,6 +425,14 @@ void vfio_container_del_section_window(VFIOContainer *container,
}
}
+static VFIOIOMMUOps vfio_iommu_spapr_ops;
+
+static void setup_spapr_ops(VFIOContainerBase *bcontainer)
+{
+ vfio_iommu_spapr_ops = *bcontainer->ops;
+ bcontainer->ops = &vfio_iommu_spapr_ops;
+}
+
int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
{
VFIOContainerBase *bcontainer = &container->bcontainer;
@@ -486,6 +498,8 @@ int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
0x1000);
}
+ setup_spapr_ops(bcontainer);
+
return 0;
listener_unregister_exit:
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 17/47] vfio/spapr: switch to spapr IOMMU BE add/del_section_window
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (15 preceding siblings ...)
2023-12-19 18:56 ` [PULL 16/47] vfio/spapr: Introduce spapr backend and target interface Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 18/47] vfio/spapr: Move prereg_listener into spapr container Cédric Le Goater
` (30 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
No functional change intended.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 5 -----
include/hw/vfio/vfio-container-base.h | 5 +++++
hw/vfio/common.c | 8 ++------
hw/vfio/container-base.c | 21 +++++++++++++++++++++
hw/vfio/spapr.c | 19 ++++++++++++++-----
5 files changed, 42 insertions(+), 16 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index b9e5a0e64b46bacae99c2c542f1b7c7560957080..055f6793635e8c0faa2bbd910737bf8db4e45c88 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -169,11 +169,6 @@ VFIOAddressSpace *vfio_get_address_space(AddressSpace *as);
void vfio_put_address_space(VFIOAddressSpace *space);
/* SPAPR specific */
-int vfio_container_add_section_window(VFIOContainer *container,
- MemoryRegionSection *section,
- Error **errp);
-void vfio_container_del_section_window(VFIOContainer *container,
- MemoryRegionSection *section);
int vfio_spapr_container_init(VFIOContainer *container, Error **errp);
void vfio_spapr_container_deinit(VFIOContainer *container);
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index f62a14ac7388de4e682181eb141c71f61b43791b..4b6f017c6f15de1bb1fb42bcae62a1f062895bdd 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -75,6 +75,11 @@ int vfio_container_dma_map(VFIOContainerBase *bcontainer,
int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
hwaddr iova, ram_addr_t size,
IOMMUTLBEntry *iotlb);
+int vfio_container_add_section_window(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section,
+ Error **errp);
+void vfio_container_del_section_window(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section);
int vfio_container_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
bool start);
int vfio_container_query_dirty_bitmap(VFIOContainerBase *bcontainer,
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 483ba820890460cba0ec7a4ecfed72d66433d41e..572ae7c934070b9fdc66915f6831fb0fa1323941 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -571,8 +571,6 @@ static void vfio_listener_region_add(MemoryListener *listener,
{
VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
listener);
- VFIOContainer *container = container_of(bcontainer, VFIOContainer,
- bcontainer);
hwaddr iova, end;
Int128 llend, llsize;
void *vaddr;
@@ -595,7 +593,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
return;
}
- if (vfio_container_add_section_window(container, section, &err)) {
+ if (vfio_container_add_section_window(bcontainer, section, &err)) {
goto fail;
}
@@ -738,8 +736,6 @@ static void vfio_listener_region_del(MemoryListener *listener,
{
VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
listener);
- VFIOContainer *container = container_of(bcontainer, VFIOContainer,
- bcontainer);
hwaddr iova, end;
Int128 llend, llsize;
int ret;
@@ -818,7 +814,7 @@ static void vfio_listener_region_del(MemoryListener *listener,
memory_region_unref(section->mr);
- vfio_container_del_section_window(container, section);
+ vfio_container_del_section_window(bcontainer, section);
}
typedef struct VFIODirtyRanges {
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index 0177f4374132d4bbdc5f992e9c3fddb138809307..71f727497314b06185fd0201d1472b83aafda703 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -31,6 +31,27 @@ int vfio_container_dma_unmap(VFIOContainerBase *bcontainer,
return bcontainer->ops->dma_unmap(bcontainer, iova, size, iotlb);
}
+int vfio_container_add_section_window(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section,
+ Error **errp)
+{
+ if (!bcontainer->ops->add_window) {
+ return 0;
+ }
+
+ return bcontainer->ops->add_window(bcontainer, section, errp);
+}
+
+void vfio_container_del_section_window(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section)
+{
+ if (!bcontainer->ops->del_window) {
+ return;
+ }
+
+ return bcontainer->ops->del_window(bcontainer, section);
+}
+
int vfio_container_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
bool start)
{
diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c
index e1a6b3556347ddd29fdbe13f96ac6db9a2a869e5..5be1911aadccc3b3300b84b614a212aff39d57e0 100644
--- a/hw/vfio/spapr.c
+++ b/hw/vfio/spapr.c
@@ -319,10 +319,13 @@ static int vfio_spapr_create_window(VFIOContainer *container,
return 0;
}
-int vfio_container_add_section_window(VFIOContainer *container,
- MemoryRegionSection *section,
- Error **errp)
+static int
+vfio_spapr_container_add_section_window(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section,
+ Error **errp)
{
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
VFIOHostDMAWindow *hostwin;
hwaddr pgsize = 0;
int ret;
@@ -407,9 +410,13 @@ int vfio_container_add_section_window(VFIOContainer *container,
return 0;
}
-void vfio_container_del_section_window(VFIOContainer *container,
- MemoryRegionSection *section)
+static void
+vfio_spapr_container_del_section_window(VFIOContainerBase *bcontainer,
+ MemoryRegionSection *section)
{
+ VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
+
if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) {
return;
}
@@ -430,6 +437,8 @@ static VFIOIOMMUOps vfio_iommu_spapr_ops;
static void setup_spapr_ops(VFIOContainerBase *bcontainer)
{
vfio_iommu_spapr_ops = *bcontainer->ops;
+ vfio_iommu_spapr_ops.add_window = vfio_spapr_container_add_section_window;
+ vfio_iommu_spapr_ops.del_window = vfio_spapr_container_del_section_window;
bcontainer->ops = &vfio_iommu_spapr_ops;
}
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 18/47] vfio/spapr: Move prereg_listener into spapr container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (16 preceding siblings ...)
2023-12-19 18:56 ` [PULL 17/47] vfio/spapr: switch to spapr IOMMU BE add/del_section_window Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 19/47] vfio/spapr: Move hostwin_list " Cédric Le Goater
` (29 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
No functional changes intended.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 1 -
hw/vfio/spapr.c | 24 ++++++++++++++++--------
2 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 055f6793635e8c0faa2bbd910737bf8db4e45c88..ed6148c058be03f6c1898ff680072f23dbe51ba2 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -78,7 +78,6 @@ struct VFIOGroup;
typedef struct VFIOContainer {
VFIOContainerBase bcontainer;
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
- MemoryListener prereg_listener;
unsigned iommu_type;
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c
index 5be1911aadccc3b3300b84b614a212aff39d57e0..68c3dd6c75678dcfa901b8e7bc241a56047c0fbe 100644
--- a/hw/vfio/spapr.c
+++ b/hw/vfio/spapr.c
@@ -26,6 +26,7 @@
typedef struct VFIOSpaprContainer {
VFIOContainer container;
+ MemoryListener prereg_listener;
} VFIOSpaprContainer;
static bool vfio_prereg_listener_skipped_section(MemoryRegionSection *section)
@@ -48,8 +49,9 @@ static void *vfio_prereg_gpa_to_vaddr(MemoryRegionSection *section, hwaddr gpa)
static void vfio_prereg_listener_region_add(MemoryListener *listener,
MemoryRegionSection *section)
{
- VFIOContainer *container = container_of(listener, VFIOContainer,
- prereg_listener);
+ VFIOSpaprContainer *scontainer = container_of(listener, VFIOSpaprContainer,
+ prereg_listener);
+ VFIOContainer *container = &scontainer->container;
VFIOContainerBase *bcontainer = &container->bcontainer;
const hwaddr gpa = section->offset_within_address_space;
hwaddr end;
@@ -107,8 +109,9 @@ static void vfio_prereg_listener_region_add(MemoryListener *listener,
static void vfio_prereg_listener_region_del(MemoryListener *listener,
MemoryRegionSection *section)
{
- VFIOContainer *container = container_of(listener, VFIOContainer,
- prereg_listener);
+ VFIOSpaprContainer *scontainer = container_of(listener, VFIOSpaprContainer,
+ prereg_listener);
+ VFIOContainer *container = &scontainer->container;
const hwaddr gpa = section->offset_within_address_space;
hwaddr end;
int ret;
@@ -445,6 +448,8 @@ static void setup_spapr_ops(VFIOContainerBase *bcontainer)
int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
{
VFIOContainerBase *bcontainer = &container->bcontainer;
+ VFIOSpaprContainer *scontainer = container_of(container, VFIOSpaprContainer,
+ container);
struct vfio_iommu_spapr_tce_info info;
bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU;
int ret, fd = container->fd;
@@ -463,9 +468,9 @@ int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
return -errno;
}
} else {
- container->prereg_listener = vfio_prereg_listener;
+ scontainer->prereg_listener = vfio_prereg_listener;
- memory_listener_register(&container->prereg_listener,
+ memory_listener_register(&scontainer->prereg_listener,
&address_space_memory);
if (bcontainer->error) {
ret = -1;
@@ -513,7 +518,7 @@ int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
listener_unregister_exit:
if (v2) {
- memory_listener_unregister(&container->prereg_listener);
+ memory_listener_unregister(&scontainer->prereg_listener);
}
return ret;
}
@@ -523,7 +528,10 @@ void vfio_spapr_container_deinit(VFIOContainer *container)
VFIOHostDMAWindow *hostwin, *next;
if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
- memory_listener_unregister(&container->prereg_listener);
+ VFIOSpaprContainer *scontainer = container_of(container,
+ VFIOSpaprContainer,
+ container);
+ memory_listener_unregister(&scontainer->prereg_listener);
}
QLIST_FOREACH_SAFE(hostwin, &container->hostwin_list, hostwin_next,
next) {
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 19/47] vfio/spapr: Move hostwin_list into spapr container
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (17 preceding siblings ...)
2023-12-19 18:56 ` [PULL 18/47] vfio/spapr: Move prereg_listener into spapr container Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 20/47] backends/iommufd: Introduce the iommufd object Cédric Le Goater
` (28 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
No functional changes intended.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 1 -
hw/vfio/spapr.c | 36 +++++++++++++++++++----------------
2 files changed, 20 insertions(+), 17 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index ed6148c058be03f6c1898ff680072f23dbe51ba2..24ecc0e7ee6243ad9ee2003d197e04390b780d2c 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -79,7 +79,6 @@ typedef struct VFIOContainer {
VFIOContainerBase bcontainer;
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
unsigned iommu_type;
- QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
} VFIOContainer;
diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c
index 68c3dd6c75678dcfa901b8e7bc241a56047c0fbe..5c6426e6973bec606667ebcaca5b0585b184a214 100644
--- a/hw/vfio/spapr.c
+++ b/hw/vfio/spapr.c
@@ -27,6 +27,7 @@
typedef struct VFIOSpaprContainer {
VFIOContainer container;
MemoryListener prereg_listener;
+ QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
} VFIOSpaprContainer;
static bool vfio_prereg_listener_skipped_section(MemoryRegionSection *section)
@@ -154,12 +155,12 @@ static const MemoryListener vfio_prereg_listener = {
.region_del = vfio_prereg_listener_region_del,
};
-static void vfio_host_win_add(VFIOContainer *container, hwaddr min_iova,
+static void vfio_host_win_add(VFIOSpaprContainer *scontainer, hwaddr min_iova,
hwaddr max_iova, uint64_t iova_pgsizes)
{
VFIOHostDMAWindow *hostwin;
- QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
+ QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) {
if (ranges_overlap(hostwin->min_iova,
hostwin->max_iova - hostwin->min_iova + 1,
min_iova,
@@ -173,15 +174,15 @@ static void vfio_host_win_add(VFIOContainer *container, hwaddr min_iova,
hostwin->min_iova = min_iova;
hostwin->max_iova = max_iova;
hostwin->iova_pgsizes = iova_pgsizes;
- QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next);
+ QLIST_INSERT_HEAD(&scontainer->hostwin_list, hostwin, hostwin_next);
}
-static int vfio_host_win_del(VFIOContainer *container,
+static int vfio_host_win_del(VFIOSpaprContainer *scontainer,
hwaddr min_iova, hwaddr max_iova)
{
VFIOHostDMAWindow *hostwin;
- QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
+ QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) {
if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) {
QLIST_REMOVE(hostwin, hostwin_next);
g_free(hostwin);
@@ -192,7 +193,7 @@ static int vfio_host_win_del(VFIOContainer *container,
return -1;
}
-static VFIOHostDMAWindow *vfio_find_hostwin(VFIOContainer *container,
+static VFIOHostDMAWindow *vfio_find_hostwin(VFIOSpaprContainer *container,
hwaddr iova, hwaddr end)
{
VFIOHostDMAWindow *hostwin;
@@ -329,6 +330,8 @@ vfio_spapr_container_add_section_window(VFIOContainerBase *bcontainer,
{
VFIOContainer *container = container_of(bcontainer, VFIOContainer,
bcontainer);
+ VFIOSpaprContainer *scontainer = container_of(container, VFIOSpaprContainer,
+ container);
VFIOHostDMAWindow *hostwin;
hwaddr pgsize = 0;
int ret;
@@ -344,7 +347,7 @@ vfio_spapr_container_add_section_window(VFIOContainerBase *bcontainer,
iova = section->offset_within_address_space;
end = iova + int128_get64(section->size) - 1;
- if (!vfio_find_hostwin(container, iova, end)) {
+ if (!vfio_find_hostwin(scontainer, iova, end)) {
error_setg(errp, "Container %p can't map guest IOVA region"
" 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container,
iova, end);
@@ -358,7 +361,7 @@ vfio_spapr_container_add_section_window(VFIOContainerBase *bcontainer,
}
/* For now intersections are not allowed, we may relax this later */
- QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
+ QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) {
if (ranges_overlap(hostwin->min_iova,
hostwin->max_iova - hostwin->min_iova + 1,
section->offset_within_address_space,
@@ -380,7 +383,7 @@ vfio_spapr_container_add_section_window(VFIOContainerBase *bcontainer,
return ret;
}
- vfio_host_win_add(container, section->offset_within_address_space,
+ vfio_host_win_add(scontainer, section->offset_within_address_space,
section->offset_within_address_space +
int128_get64(section->size) - 1, pgsize);
#ifdef CONFIG_KVM
@@ -419,6 +422,8 @@ vfio_spapr_container_del_section_window(VFIOContainerBase *bcontainer,
{
VFIOContainer *container = container_of(bcontainer, VFIOContainer,
bcontainer);
+ VFIOSpaprContainer *scontainer = container_of(container, VFIOSpaprContainer,
+ container);
if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) {
return;
@@ -426,7 +431,7 @@ vfio_spapr_container_del_section_window(VFIOContainerBase *bcontainer,
vfio_spapr_remove_window(container,
section->offset_within_address_space);
- if (vfio_host_win_del(container,
+ if (vfio_host_win_del(scontainer,
section->offset_within_address_space,
section->offset_within_address_space +
int128_get64(section->size) - 1) < 0) {
@@ -454,7 +459,7 @@ int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU;
int ret, fd = container->fd;
- QLIST_INIT(&container->hostwin_list);
+ QLIST_INIT(&scontainer->hostwin_list);
/*
* The host kernel code implementing VFIO_IOMMU_DISABLE is called
@@ -506,7 +511,7 @@ int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
} else {
/* The default table uses 4K pages */
bcontainer->pgsizes = 0x1000;
- vfio_host_win_add(container, info.dma32_window_start,
+ vfio_host_win_add(scontainer, info.dma32_window_start,
info.dma32_window_start +
info.dma32_window_size - 1,
0x1000);
@@ -525,15 +530,14 @@ listener_unregister_exit:
void vfio_spapr_container_deinit(VFIOContainer *container)
{
+ VFIOSpaprContainer *scontainer = container_of(container, VFIOSpaprContainer,
+ container);
VFIOHostDMAWindow *hostwin, *next;
if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
- VFIOSpaprContainer *scontainer = container_of(container,
- VFIOSpaprContainer,
- container);
memory_listener_unregister(&scontainer->prereg_listener);
}
- QLIST_FOREACH_SAFE(hostwin, &container->hostwin_list, hostwin_next,
+ QLIST_FOREACH_SAFE(hostwin, &scontainer->hostwin_list, hostwin_next,
next) {
QLIST_REMOVE(hostwin, hostwin_next);
g_free(hostwin);
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 20/47] backends/iommufd: Introduce the iommufd object
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (18 preceding siblings ...)
2023-12-19 18:56 ` [PULL 19/47] vfio/spapr: Move hostwin_list " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-21 16:00 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 21/47] util/char_dev: Add open_cdev() Cédric Le Goater
` (27 subsequent siblings)
47 siblings, 1 reply; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Cédric Le Goater,
Nicolin Chen
From: Eric Auger <eric.auger@redhat.com>
Introduce an iommufd object which allows the interaction
with the host /dev/iommu device.
The /dev/iommu can have been already pre-opened outside of qemu,
in which case the fd can be passed directly along with the
iommufd object:
This allows the iommufd object to be shared accross several
subsystems (VFIO, VDPA, ...). For example, libvirt would open
the /dev/iommu once.
If no fd is passed along with the iommufd object, the /dev/iommu
is opened by the qemu code.
Suggested-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
MAINTAINERS | 8 ++
qapi/qom.json | 19 +++
include/sysemu/iommufd.h | 38 ++++++
backends/iommufd.c | 245 +++++++++++++++++++++++++++++++++++++++
backends/Kconfig | 4 +
backends/meson.build | 1 +
backends/trace-events | 10 ++
qemu-options.hx | 12 ++
8 files changed, 337 insertions(+)
create mode 100644 include/sysemu/iommufd.h
create mode 100644 backends/iommufd.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 695e0bd34fbba253d77570e5b3ef8dabe7a174b3..a5a446914a1cf505131bfe113540fd501511abb8 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2167,6 +2167,14 @@ F: hw/vfio/ap.c
F: docs/system/s390x/vfio-ap.rst
L: qemu-s390x@nongnu.org
+iommufd
+M: Yi Liu <yi.l.liu@intel.com>
+M: Eric Auger <eric.auger@redhat.com>
+M: Zhenzhong Duan <zhenzhong.duan@intel.com>
+S: Supported
+F: backends/iommufd.c
+F: include/sysemu/iommufd.h
+
vhost
M: Michael S. Tsirkin <mst@redhat.com>
S: Supported
diff --git a/qapi/qom.json b/qapi/qom.json
index c53ef978ff7e6a81f8c926159fe52c0d349696ac..95516ba325e541e5eeb3e1f588474cdf63ad68a5 100644
--- a/qapi/qom.json
+++ b/qapi/qom.json
@@ -794,6 +794,23 @@
{ 'struct': 'VfioUserServerProperties',
'data': { 'socket': 'SocketAddress', 'device': 'str' } }
+##
+# @IOMMUFDProperties:
+#
+# Properties for iommufd objects.
+#
+# @fd: file descriptor name previously passed via 'getfd' command,
+# which represents a pre-opened /dev/iommu. This allows the
+# iommufd object to be shared accross several subsystems
+# (VFIO, VDPA, ...), and the file descriptor to be shared
+# with other process, e.g. DPDK. (default: QEMU opens
+# /dev/iommu by itself)
+#
+# Since: 9.0
+##
+{ 'struct': 'IOMMUFDProperties',
+ 'data': { '*fd': 'str' } }
+
##
# @RngProperties:
#
@@ -934,6 +951,7 @@
'input-barrier',
{ 'name': 'input-linux',
'if': 'CONFIG_LINUX' },
+ 'iommufd',
'iothread',
'main-loop',
{ 'name': 'memory-backend-epc',
@@ -1003,6 +1021,7 @@
'input-barrier': 'InputBarrierProperties',
'input-linux': { 'type': 'InputLinuxProperties',
'if': 'CONFIG_LINUX' },
+ 'iommufd': 'IOMMUFDProperties',
'iothread': 'IothreadProperties',
'main-loop': 'MainLoopProperties',
'memory-backend-epc': { 'type': 'MemoryBackendEpcProperties',
diff --git a/include/sysemu/iommufd.h b/include/sysemu/iommufd.h
new file mode 100644
index 0000000000000000000000000000000000000000..9c5524b0ed15ef5f81be159415bc216572a283d8
--- /dev/null
+++ b/include/sysemu/iommufd.h
@@ -0,0 +1,38 @@
+#ifndef SYSEMU_IOMMUFD_H
+#define SYSEMU_IOMMUFD_H
+
+#include "qom/object.h"
+#include "qemu/thread.h"
+#include "exec/hwaddr.h"
+#include "exec/cpu-common.h"
+
+#define TYPE_IOMMUFD_BACKEND "iommufd"
+OBJECT_DECLARE_TYPE(IOMMUFDBackend, IOMMUFDBackendClass, IOMMUFD_BACKEND)
+
+struct IOMMUFDBackendClass {
+ ObjectClass parent_class;
+};
+
+struct IOMMUFDBackend {
+ Object parent;
+
+ /*< protected >*/
+ int fd; /* /dev/iommu file descriptor */
+ bool owned; /* is the /dev/iommu opened internally */
+ QemuMutex lock;
+ uint32_t users;
+
+ /*< public >*/
+};
+
+int iommufd_backend_connect(IOMMUFDBackend *be, Error **errp);
+void iommufd_backend_disconnect(IOMMUFDBackend *be);
+
+int iommufd_backend_alloc_ioas(IOMMUFDBackend *be, uint32_t *ioas_id,
+ Error **errp);
+void iommufd_backend_free_id(IOMMUFDBackend *be, uint32_t id);
+int iommufd_backend_map_dma(IOMMUFDBackend *be, uint32_t ioas_id, hwaddr iova,
+ ram_addr_t size, void *vaddr, bool readonly);
+int iommufd_backend_unmap_dma(IOMMUFDBackend *be, uint32_t ioas_id,
+ hwaddr iova, ram_addr_t size);
+#endif
diff --git a/backends/iommufd.c b/backends/iommufd.c
new file mode 100644
index 0000000000000000000000000000000000000000..ba58a0eb0d0ba9aae625c987eb728547543dba66
--- /dev/null
+++ b/backends/iommufd.c
@@ -0,0 +1,245 @@
+/*
+ * iommufd container backend
+ *
+ * Copyright (C) 2023 Intel Corporation.
+ * Copyright Red Hat, Inc. 2023
+ *
+ * Authors: Yi Liu <yi.l.liu@intel.com>
+ * Eric Auger <eric.auger@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "qemu/osdep.h"
+#include "sysemu/iommufd.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/module.h"
+#include "qom/object_interfaces.h"
+#include "qemu/error-report.h"
+#include "monitor/monitor.h"
+#include "trace.h"
+#include <sys/ioctl.h>
+#include <linux/iommufd.h>
+
+static void iommufd_backend_init(Object *obj)
+{
+ IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
+
+ be->fd = -1;
+ be->users = 0;
+ be->owned = true;
+ qemu_mutex_init(&be->lock);
+}
+
+static void iommufd_backend_finalize(Object *obj)
+{
+ IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
+
+ if (be->owned) {
+ close(be->fd);
+ be->fd = -1;
+ }
+}
+
+static void iommufd_backend_set_fd(Object *obj, const char *str, Error **errp)
+{
+ IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
+ int fd = -1;
+
+ fd = monitor_fd_param(monitor_cur(), str, errp);
+ if (fd == -1) {
+ error_prepend(errp, "Could not parse remote object fd %s:", str);
+ return;
+ }
+ qemu_mutex_lock(&be->lock);
+ be->fd = fd;
+ be->owned = false;
+ qemu_mutex_unlock(&be->lock);
+ trace_iommu_backend_set_fd(be->fd);
+}
+
+static bool iommufd_backend_can_be_deleted(UserCreatable *uc)
+{
+ IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);
+
+ return !be->users;
+}
+
+static void iommufd_backend_class_init(ObjectClass *oc, void *data)
+{
+ UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+ ucc->can_be_deleted = iommufd_backend_can_be_deleted;
+
+ object_class_property_add_str(oc, "fd", NULL, iommufd_backend_set_fd);
+}
+
+int iommufd_backend_connect(IOMMUFDBackend *be, Error **errp)
+{
+ int fd, ret = 0;
+
+ qemu_mutex_lock(&be->lock);
+ if (be->users == UINT32_MAX) {
+ error_setg(errp, "too many connections");
+ ret = -E2BIG;
+ goto out;
+ }
+ if (be->owned && !be->users) {
+ fd = qemu_open_old("/dev/iommu", O_RDWR);
+ if (fd < 0) {
+ error_setg_errno(errp, errno, "/dev/iommu opening failed");
+ ret = fd;
+ goto out;
+ }
+ be->fd = fd;
+ }
+ be->users++;
+out:
+ trace_iommufd_backend_connect(be->fd, be->owned,
+ be->users, ret);
+ qemu_mutex_unlock(&be->lock);
+ return ret;
+}
+
+void iommufd_backend_disconnect(IOMMUFDBackend *be)
+{
+ qemu_mutex_lock(&be->lock);
+ if (!be->users) {
+ goto out;
+ }
+ be->users--;
+ if (!be->users && be->owned) {
+ close(be->fd);
+ be->fd = -1;
+ }
+out:
+ trace_iommufd_backend_disconnect(be->fd, be->users);
+ qemu_mutex_unlock(&be->lock);
+}
+
+int iommufd_backend_alloc_ioas(IOMMUFDBackend *be, uint32_t *ioas_id,
+ Error **errp)
+{
+ int ret, fd = be->fd;
+ struct iommu_ioas_alloc alloc_data = {
+ .size = sizeof(alloc_data),
+ .flags = 0,
+ };
+
+ ret = ioctl(fd, IOMMU_IOAS_ALLOC, &alloc_data);
+ if (ret) {
+ error_setg_errno(errp, errno, "Failed to allocate ioas");
+ return ret;
+ }
+
+ *ioas_id = alloc_data.out_ioas_id;
+ trace_iommufd_backend_alloc_ioas(fd, *ioas_id, ret);
+
+ return ret;
+}
+
+void iommufd_backend_free_id(IOMMUFDBackend *be, uint32_t id)
+{
+ int ret, fd = be->fd;
+ struct iommu_destroy des = {
+ .size = sizeof(des),
+ .id = id,
+ };
+
+ ret = ioctl(fd, IOMMU_DESTROY, &des);
+ trace_iommufd_backend_free_id(fd, id, ret);
+ if (ret) {
+ error_report("Failed to free id: %u %m", id);
+ }
+}
+
+int iommufd_backend_map_dma(IOMMUFDBackend *be, uint32_t ioas_id, hwaddr iova,
+ ram_addr_t size, void *vaddr, bool readonly)
+{
+ int ret, fd = be->fd;
+ struct iommu_ioas_map map = {
+ .size = sizeof(map),
+ .flags = IOMMU_IOAS_MAP_READABLE |
+ IOMMU_IOAS_MAP_FIXED_IOVA,
+ .ioas_id = ioas_id,
+ .__reserved = 0,
+ .user_va = (uintptr_t)vaddr,
+ .iova = iova,
+ .length = size,
+ };
+
+ if (!readonly) {
+ map.flags |= IOMMU_IOAS_MAP_WRITEABLE;
+ }
+
+ ret = ioctl(fd, IOMMU_IOAS_MAP, &map);
+ trace_iommufd_backend_map_dma(fd, ioas_id, iova, size,
+ vaddr, readonly, ret);
+ if (ret) {
+ ret = -errno;
+
+ /* TODO: Not support mapping hardware PCI BAR region for now. */
+ if (errno == EFAULT) {
+ warn_report("IOMMU_IOAS_MAP failed: %m, PCI BAR?");
+ } else {
+ error_report("IOMMU_IOAS_MAP failed: %m");
+ }
+ }
+ return ret;
+}
+
+int iommufd_backend_unmap_dma(IOMMUFDBackend *be, uint32_t ioas_id,
+ hwaddr iova, ram_addr_t size)
+{
+ int ret, fd = be->fd;
+ struct iommu_ioas_unmap unmap = {
+ .size = sizeof(unmap),
+ .ioas_id = ioas_id,
+ .iova = iova,
+ .length = size,
+ };
+
+ ret = ioctl(fd, IOMMU_IOAS_UNMAP, &unmap);
+ /*
+ * IOMMUFD takes mapping as some kind of object, unmapping
+ * nonexistent mapping is treated as deleting a nonexistent
+ * object and return ENOENT. This is different from legacy
+ * backend which allows it. vIOMMU may trigger a lot of
+ * redundant unmapping, to avoid flush the log, treat them
+ * as succeess for IOMMUFD just like legacy backend.
+ */
+ if (ret && errno == ENOENT) {
+ trace_iommufd_backend_unmap_dma_non_exist(fd, ioas_id, iova, size, ret);
+ ret = 0;
+ } else {
+ trace_iommufd_backend_unmap_dma(fd, ioas_id, iova, size, ret);
+ }
+
+ if (ret) {
+ ret = -errno;
+ error_report("IOMMU_IOAS_UNMAP failed: %m");
+ }
+ return ret;
+}
+
+static const TypeInfo iommufd_backend_info = {
+ .name = TYPE_IOMMUFD_BACKEND,
+ .parent = TYPE_OBJECT,
+ .instance_size = sizeof(IOMMUFDBackend),
+ .instance_init = iommufd_backend_init,
+ .instance_finalize = iommufd_backend_finalize,
+ .class_size = sizeof(IOMMUFDBackendClass),
+ .class_init = iommufd_backend_class_init,
+ .interfaces = (InterfaceInfo[]) {
+ { TYPE_USER_CREATABLE },
+ { }
+ }
+};
+
+static void register_types(void)
+{
+ type_register_static(&iommufd_backend_info);
+}
+
+type_init(register_types);
diff --git a/backends/Kconfig b/backends/Kconfig
index f35abc16092808b1fe5b033a346908e2d66bff0b..2cb23f62fa1526cedafedcc99a032e098075b846 100644
--- a/backends/Kconfig
+++ b/backends/Kconfig
@@ -1 +1,5 @@
source tpm/Kconfig
+
+config IOMMUFD
+ bool
+ depends on VFIO
diff --git a/backends/meson.build b/backends/meson.build
index 914c7c4afb905cfe710ad23dd1ee42907f6d1679..9a5cea480d172d50a641e4d9179093e8155f2db1 100644
--- a/backends/meson.build
+++ b/backends/meson.build
@@ -20,6 +20,7 @@ if have_vhost_user
system_ss.add(when: 'CONFIG_VIRTIO', if_true: files('vhost-user.c'))
endif
system_ss.add(when: 'CONFIG_VIRTIO_CRYPTO', if_true: files('cryptodev-vhost.c'))
+system_ss.add(when: 'CONFIG_IOMMUFD', if_true: files('iommufd.c'))
if have_vhost_user_crypto
system_ss.add(when: 'CONFIG_VIRTIO_CRYPTO', if_true: files('cryptodev-vhost-user.c'))
endif
diff --git a/backends/trace-events b/backends/trace-events
index 652eb76a5723e2053fe97338c481309c58284d6a..d45c6e31a67ed66d94787f60eb08a525cf6ff68b 100644
--- a/backends/trace-events
+++ b/backends/trace-events
@@ -5,3 +5,13 @@ dbus_vmstate_pre_save(void)
dbus_vmstate_post_load(int version_id) "version_id: %d"
dbus_vmstate_loading(const char *id) "id: %s"
dbus_vmstate_saving(const char *id) "id: %s"
+
+# iommufd.c
+iommufd_backend_connect(int fd, bool owned, uint32_t users, int ret) "fd=%d owned=%d users=%d (%d)"
+iommufd_backend_disconnect(int fd, uint32_t users) "fd=%d users=%d"
+iommu_backend_set_fd(int fd) "pre-opened /dev/iommu fd=%d"
+iommufd_backend_map_dma(int iommufd, uint32_t ioas, uint64_t iova, uint64_t size, void *vaddr, bool readonly, int ret) " iommufd=%d ioas=%d iova=0x%"PRIx64" size=0x%"PRIx64" addr=%p readonly=%d (%d)"
+iommufd_backend_unmap_dma_non_exist(int iommufd, uint32_t ioas, uint64_t iova, uint64_t size, int ret) " Unmap nonexistent mapping: iommufd=%d ioas=%d iova=0x%"PRIx64" size=0x%"PRIx64" (%d)"
+iommufd_backend_unmap_dma(int iommufd, uint32_t ioas, uint64_t iova, uint64_t size, int ret) " iommufd=%d ioas=%d iova=0x%"PRIx64" size=0x%"PRIx64" (%d)"
+iommufd_backend_alloc_ioas(int iommufd, uint32_t ioas, int ret) " iommufd=%d ioas=%d (%d)"
+iommufd_backend_free_id(int iommufd, uint32_t id, int ret) " iommufd=%d id=%d (%d)"
diff --git a/qemu-options.hx b/qemu-options.hx
index 42fd09e4de96e962cd5873c49501f6e1dbb5e346..5fe8ea57d2d2f9390a976ef2fefe86463e888bb1 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -5224,6 +5224,18 @@ SRST
The ``share`` boolean option is on by default with memfd.
+ ``-object iommufd,id=id[,fd=fd]``
+ Creates an iommufd backend which allows control of DMA mapping
+ through the ``/dev/iommu`` device.
+
+ The ``id`` parameter is a unique ID which frontends (such as
+ vfio-pci of vdpa) will use to connect with the iommufd backend.
+
+ The ``fd`` parameter is an optional pre-opened file descriptor
+ resulting from ``/dev/iommu`` opening. Usually the iommufd is shared
+ across all subsystems, bringing the benefit of centralized
+ reference counting.
+
``-object rng-builtin,id=id``
Creates a random number generator backend which obtains entropy
from QEMU builtin functions. The ``id`` parameter is a unique ID
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* Re: [PULL 20/47] backends/iommufd: Introduce the iommufd object
2023-12-19 18:56 ` [PULL 20/47] backends/iommufd: Introduce the iommufd object Cédric Le Goater
@ 2023-12-21 16:00 ` Cédric Le Goater
2023-12-21 17:14 ` Eric Auger
2023-12-22 2:41 ` Duan, Zhenzhong
0 siblings, 2 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-21 16:00 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Nicolin Chen
[ ... ]
> +static void iommufd_backend_init(Object *obj)
> +{
> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
> +
> + be->fd = -1;
> + be->users = 0;
> + be->owned = true;
> + qemu_mutex_init(&be->lock);> +}
> +
> +static void iommufd_backend_finalize(Object *obj)
> +{
> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
> +
> + if (be->owned) {
> + close(be->fd);
> + be->fd = -1;
> + }
> +}
> +
> +static void iommufd_backend_set_fd(Object *obj, const char *str, Error **errp)
> +{
> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
> + int fd = -1;
> +
> + fd = monitor_fd_param(monitor_cur(), str, errp);
> + if (fd == -1) {
> + error_prepend(errp, "Could not parse remote object fd %s:", str);
> + return;
> + }
> + qemu_mutex_lock(&be->lock);
> + be->fd = fd;
> + be->owned = false;
> + qemu_mutex_unlock(&be->lock);
> + trace_iommu_backend_set_fd(be->fd);
> +}
> +
> +static bool iommufd_backend_can_be_deleted(UserCreatable *uc)
> +{
> + IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);
> +
> + return !be->users;
Coverity CID 1531549 reports a concurrent data access violation because
be->users is being accessed without holding the mutex.
I wonder how useful is this mutex anyhow, since the code paths should
be protected by the BQL lock. If you agree, I will send an update to
simply drop be->lock and solve this report.
Thanks,
C.
> +}
> +
> +static void iommufd_backend_class_init(ObjectClass *oc, void *data)
> +{
> + UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
> +
> + ucc->can_be_deleted = iommufd_backend_can_be_deleted;
> +
> + object_class_property_add_str(oc, "fd", NULL, iommufd_backend_set_fd);
> +}
> +
> +int iommufd_backend_connect(IOMMUFDBackend *be, Error **errp)
> +{
> + int fd, ret = 0;
> +
> + qemu_mutex_lock(&be->lock);
> + if (be->users == UINT32_MAX) {
> + error_setg(errp, "too many connections");
> + ret = -E2BIG;
> + goto out;
> + }
> + if (be->owned && !be->users) {
> + fd = qemu_open_old("/dev/iommu", O_RDWR);
> + if (fd < 0) {
> + error_setg_errno(errp, errno, "/dev/iommu opening failed");
> + ret = fd;
> + goto out;
> + }
> + be->fd = fd;
> + }
> + be->users++;
> +out:
> + trace_iommufd_backend_connect(be->fd, be->owned,
> + be->users, ret);
> + qemu_mutex_unlock(&be->lock);
> + return ret;
> +}
> +
> +void iommufd_backend_disconnect(IOMMUFDBackend *be)
> +{
> + qemu_mutex_lock(&be->lock);
> + if (!be->users) {
> + goto out;
> + }
> + be->users--;
> + if (!be->users && be->owned) {
> + close(be->fd);
> + be->fd = -1;
> + }
> +out:
> + trace_iommufd_backend_disconnect(be->fd, be->users);
> + qemu_mutex_unlock(&be->lock);
> +}
> +
> +int iommufd_backend_alloc_ioas(IOMMUFDBackend *be, uint32_t *ioas_id,
> + Error **errp)
> +{
> + int ret, fd = be->fd;
> + struct iommu_ioas_alloc alloc_data = {
> + .size = sizeof(alloc_data),
> + .flags = 0,
> + };
> +
> + ret = ioctl(fd, IOMMU_IOAS_ALLOC, &alloc_data);
> + if (ret) {
> + error_setg_errno(errp, errno, "Failed to allocate ioas");
> + return ret;
> + }
> +
> + *ioas_id = alloc_data.out_ioas_id;
> + trace_iommufd_backend_alloc_ioas(fd, *ioas_id, ret);
> +
> + return ret;
> +}
> +
> +void iommufd_backend_free_id(IOMMUFDBackend *be, uint32_t id)
> +{
> + int ret, fd = be->fd;
> + struct iommu_destroy des = {
> + .size = sizeof(des),
> + .id = id,
> + };
> +
> + ret = ioctl(fd, IOMMU_DESTROY, &des);
> + trace_iommufd_backend_free_id(fd, id, ret);
> + if (ret) {
> + error_report("Failed to free id: %u %m", id);
> + }
> +}
> +
> +int iommufd_backend_map_dma(IOMMUFDBackend *be, uint32_t ioas_id, hwaddr iova,
> + ram_addr_t size, void *vaddr, bool readonly)
> +{
> + int ret, fd = be->fd;
> + struct iommu_ioas_map map = {
> + .size = sizeof(map),
> + .flags = IOMMU_IOAS_MAP_READABLE |
> + IOMMU_IOAS_MAP_FIXED_IOVA,
> + .ioas_id = ioas_id,
> + .__reserved = 0,
> + .user_va = (uintptr_t)vaddr,
> + .iova = iova,
> + .length = size,
> + };
> +
> + if (!readonly) {
> + map.flags |= IOMMU_IOAS_MAP_WRITEABLE;
> + }
> +
> + ret = ioctl(fd, IOMMU_IOAS_MAP, &map);
> + trace_iommufd_backend_map_dma(fd, ioas_id, iova, size,
> + vaddr, readonly, ret);
> + if (ret) {
> + ret = -errno;
> +
> + /* TODO: Not support mapping hardware PCI BAR region for now. */
> + if (errno == EFAULT) {
> + warn_report("IOMMU_IOAS_MAP failed: %m, PCI BAR?");
> + } else {
> + error_report("IOMMU_IOAS_MAP failed: %m");
> + }
> + }
> + return ret;
> +}
> +
> +int iommufd_backend_unmap_dma(IOMMUFDBackend *be, uint32_t ioas_id,
> + hwaddr iova, ram_addr_t size)
> +{
> + int ret, fd = be->fd;
> + struct iommu_ioas_unmap unmap = {
> + .size = sizeof(unmap),
> + .ioas_id = ioas_id,
> + .iova = iova,
> + .length = size,
> + };
> +
> + ret = ioctl(fd, IOMMU_IOAS_UNMAP, &unmap);
> + /*
> + * IOMMUFD takes mapping as some kind of object, unmapping
> + * nonexistent mapping is treated as deleting a nonexistent
> + * object and return ENOENT. This is different from legacy
> + * backend which allows it. vIOMMU may trigger a lot of
> + * redundant unmapping, to avoid flush the log, treat them
> + * as succeess for IOMMUFD just like legacy backend.
> + */
> + if (ret && errno == ENOENT) {
> + trace_iommufd_backend_unmap_dma_non_exist(fd, ioas_id, iova, size, ret);
> + ret = 0;
> + } else {
> + trace_iommufd_backend_unmap_dma(fd, ioas_id, iova, size, ret);
> + }
> +
> + if (ret) {
> + ret = -errno;
> + error_report("IOMMU_IOAS_UNMAP failed: %m");
> + }
> + return ret;
> +}
> +
> +static const TypeInfo iommufd_backend_info = {
> + .name = TYPE_IOMMUFD_BACKEND,
> + .parent = TYPE_OBJECT,
> + .instance_size = sizeof(IOMMUFDBackend),
> + .instance_init = iommufd_backend_init,
> + .instance_finalize = iommufd_backend_finalize,
> + .class_size = sizeof(IOMMUFDBackendClass),
> + .class_init = iommufd_backend_class_init,
> + .interfaces = (InterfaceInfo[]) {
> + { TYPE_USER_CREATABLE },
> + { }
> + }
> +};
> +
> +static void register_types(void)
> +{
> + type_register_static(&iommufd_backend_info);
> +}
> +
> +type_init(register_types);
> diff --git a/backends/Kconfig b/backends/Kconfig
> index f35abc16092808b1fe5b033a346908e2d66bff0b..2cb23f62fa1526cedafedcc99a032e098075b846 100644
> --- a/backends/Kconfig
> +++ b/backends/Kconfig
> @@ -1 +1,5 @@
> source tpm/Kconfig
> +
> +config IOMMUFD
> + bool
> + depends on VFIO
> diff --git a/backends/meson.build b/backends/meson.build
> index 914c7c4afb905cfe710ad23dd1ee42907f6d1679..9a5cea480d172d50a641e4d9179093e8155f2db1 100644
> --- a/backends/meson.build
> +++ b/backends/meson.build
> @@ -20,6 +20,7 @@ if have_vhost_user
> system_ss.add(when: 'CONFIG_VIRTIO', if_true: files('vhost-user.c'))
> endif
> system_ss.add(when: 'CONFIG_VIRTIO_CRYPTO', if_true: files('cryptodev-vhost.c'))
> +system_ss.add(when: 'CONFIG_IOMMUFD', if_true: files('iommufd.c'))
> if have_vhost_user_crypto
> system_ss.add(when: 'CONFIG_VIRTIO_CRYPTO', if_true: files('cryptodev-vhost-user.c'))
> endif
> diff --git a/backends/trace-events b/backends/trace-events
> index 652eb76a5723e2053fe97338c481309c58284d6a..d45c6e31a67ed66d94787f60eb08a525cf6ff68b 100644
> --- a/backends/trace-events
> +++ b/backends/trace-events
> @@ -5,3 +5,13 @@ dbus_vmstate_pre_save(void)
> dbus_vmstate_post_load(int version_id) "version_id: %d"
> dbus_vmstate_loading(const char *id) "id: %s"
> dbus_vmstate_saving(const char *id) "id: %s"
> +
> +# iommufd.c
> +iommufd_backend_connect(int fd, bool owned, uint32_t users, int ret) "fd=%d owned=%d users=%d (%d)"
> +iommufd_backend_disconnect(int fd, uint32_t users) "fd=%d users=%d"
> +iommu_backend_set_fd(int fd) "pre-opened /dev/iommu fd=%d"
> +iommufd_backend_map_dma(int iommufd, uint32_t ioas, uint64_t iova, uint64_t size, void *vaddr, bool readonly, int ret) " iommufd=%d ioas=%d iova=0x%"PRIx64" size=0x%"PRIx64" addr=%p readonly=%d (%d)"
> +iommufd_backend_unmap_dma_non_exist(int iommufd, uint32_t ioas, uint64_t iova, uint64_t size, int ret) " Unmap nonexistent mapping: iommufd=%d ioas=%d iova=0x%"PRIx64" size=0x%"PRIx64" (%d)"
> +iommufd_backend_unmap_dma(int iommufd, uint32_t ioas, uint64_t iova, uint64_t size, int ret) " iommufd=%d ioas=%d iova=0x%"PRIx64" size=0x%"PRIx64" (%d)"
> +iommufd_backend_alloc_ioas(int iommufd, uint32_t ioas, int ret) " iommufd=%d ioas=%d (%d)"
> +iommufd_backend_free_id(int iommufd, uint32_t id, int ret) " iommufd=%d id=%d (%d)"
> diff --git a/qemu-options.hx b/qemu-options.hx
> index 42fd09e4de96e962cd5873c49501f6e1dbb5e346..5fe8ea57d2d2f9390a976ef2fefe86463e888bb1 100644
> --- a/qemu-options.hx
> +++ b/qemu-options.hx
> @@ -5224,6 +5224,18 @@ SRST
>
> The ``share`` boolean option is on by default with memfd.
>
> + ``-object iommufd,id=id[,fd=fd]``
> + Creates an iommufd backend which allows control of DMA mapping
> + through the ``/dev/iommu`` device.
> +
> + The ``id`` parameter is a unique ID which frontends (such as
> + vfio-pci of vdpa) will use to connect with the iommufd backend.
> +
> + The ``fd`` parameter is an optional pre-opened file descriptor
> + resulting from ``/dev/iommu`` opening. Usually the iommufd is shared
> + across all subsystems, bringing the benefit of centralized
> + reference counting.
> +
> ``-object rng-builtin,id=id``
> Creates a random number generator backend which obtains entropy
> from QEMU builtin functions. The ``id`` parameter is a unique ID
^ permalink raw reply [flat|nested] 57+ messages in thread* Re: [PULL 20/47] backends/iommufd: Introduce the iommufd object
2023-12-21 16:00 ` Cédric Le Goater
@ 2023-12-21 17:14 ` Eric Auger
2023-12-21 21:23 ` Cédric Le Goater
2023-12-22 2:41 ` Duan, Zhenzhong
1 sibling, 1 reply; 57+ messages in thread
From: Eric Auger @ 2023-12-21 17:14 UTC (permalink / raw)
To: Cédric Le Goater, qemu-devel
Cc: Zhenzhong Duan, Peter Maydell, Richard Henderson, Nicholas Piggin,
Harsh Prateek Bora, Thomas Huth, Eric Farman, Alex Williamson,
Matthew Rosato, Yi Liu, Nicolin Chen
Hi Cédric,
On 12/21/23 17:00, Cédric Le Goater wrote:
> [ ... ]
>
>
>> +static void iommufd_backend_init(Object *obj)
>> +{
>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>> +
>> + be->fd = -1;
>> + be->users = 0;
>> + be->owned = true;
>> + qemu_mutex_init(&be->lock);> +}
>> +
>> +static void iommufd_backend_finalize(Object *obj)
>> +{
>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>> +
>> + if (be->owned) {
>> + close(be->fd);
>> + be->fd = -1;
>> + }
>> +}
>> +
>> +static void iommufd_backend_set_fd(Object *obj, const char *str,
>> Error **errp)
>> +{
>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>> + int fd = -1;
>> +
>> + fd = monitor_fd_param(monitor_cur(), str, errp);
>> + if (fd == -1) {
>> + error_prepend(errp, "Could not parse remote object fd %s:",
>> str);
>> + return;
>> + }
>> + qemu_mutex_lock(&be->lock);
>> + be->fd = fd;
>> + be->owned = false;
>> + qemu_mutex_unlock(&be->lock);
>> + trace_iommu_backend_set_fd(be->fd);
>> +}
>> +
>> +static bool iommufd_backend_can_be_deleted(UserCreatable *uc)
>> +{
>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);
>> +
>> + return !be->users;
>
> Coverity CID 1531549 reports a concurrent data access violation because
> be->users is being accessed without holding the mutex.
>
> I wonder how useful is this mutex anyhow, since the code paths should
> be protected by the BQL lock. If you agree, I will send an update to
> simply drop be->lock and solve this report.
I am not totally comfortable with the fact BQL covers the same
protection? Please can you elaborate.
Thanks
Eric
>
> Thanks,
>
> C.
>
>
>
>> +}
>> +
>> +static void iommufd_backend_class_init(ObjectClass *oc, void *data)
>> +{
>> + UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
>> +
>> + ucc->can_be_deleted = iommufd_backend_can_be_deleted;
>> +
>> + object_class_property_add_str(oc, "fd", NULL,
>> iommufd_backend_set_fd);
>> +}
>> +
>> +int iommufd_backend_connect(IOMMUFDBackend *be, Error **errp)
>> +{
>> + int fd, ret = 0;
>> +
>> + qemu_mutex_lock(&be->lock);
>> + if (be->users == UINT32_MAX) {
>> + error_setg(errp, "too many connections");
>> + ret = -E2BIG;
>> + goto out;
>> + }
>> + if (be->owned && !be->users) {
>> + fd = qemu_open_old("/dev/iommu", O_RDWR);
>> + if (fd < 0) {
>> + error_setg_errno(errp, errno, "/dev/iommu opening failed");
>> + ret = fd;
>> + goto out;
>> + }
>> + be->fd = fd;
>> + }
>> + be->users++;
>> +out:
>> + trace_iommufd_backend_connect(be->fd, be->owned,
>> + be->users, ret);
>> + qemu_mutex_unlock(&be->lock);
>> + return ret;
>> +}
>> +
>> +void iommufd_backend_disconnect(IOMMUFDBackend *be)
>> +{
>> + qemu_mutex_lock(&be->lock);
>> + if (!be->users) {
>> + goto out;
>> + }
>> + be->users--;
>> + if (!be->users && be->owned) {
>> + close(be->fd);
>> + be->fd = -1;
>> + }
>> +out:
>> + trace_iommufd_backend_disconnect(be->fd, be->users);
>> + qemu_mutex_unlock(&be->lock);
>> +}
>> +
>> +int iommufd_backend_alloc_ioas(IOMMUFDBackend *be, uint32_t *ioas_id,
>> + Error **errp)
>> +{
>> + int ret, fd = be->fd;
>> + struct iommu_ioas_alloc alloc_data = {
>> + .size = sizeof(alloc_data),
>> + .flags = 0,
>> + };
>> +
>> + ret = ioctl(fd, IOMMU_IOAS_ALLOC, &alloc_data);
>> + if (ret) {
>> + error_setg_errno(errp, errno, "Failed to allocate ioas");
>> + return ret;
>> + }
>> +
>> + *ioas_id = alloc_data.out_ioas_id;
>> + trace_iommufd_backend_alloc_ioas(fd, *ioas_id, ret);
>> +
>> + return ret;
>> +}
>> +
>> +void iommufd_backend_free_id(IOMMUFDBackend *be, uint32_t id)
>> +{
>> + int ret, fd = be->fd;
>> + struct iommu_destroy des = {
>> + .size = sizeof(des),
>> + .id = id,
>> + };
>> +
>> + ret = ioctl(fd, IOMMU_DESTROY, &des);
>> + trace_iommufd_backend_free_id(fd, id, ret);
>> + if (ret) {
>> + error_report("Failed to free id: %u %m", id);
>> + }
>> +}
>> +
>> +int iommufd_backend_map_dma(IOMMUFDBackend *be, uint32_t ioas_id,
>> hwaddr iova,
>> + ram_addr_t size, void *vaddr, bool
>> readonly)
>> +{
>> + int ret, fd = be->fd;
>> + struct iommu_ioas_map map = {
>> + .size = sizeof(map),
>> + .flags = IOMMU_IOAS_MAP_READABLE |
>> + IOMMU_IOAS_MAP_FIXED_IOVA,
>> + .ioas_id = ioas_id,
>> + .__reserved = 0,
>> + .user_va = (uintptr_t)vaddr,
>> + .iova = iova,
>> + .length = size,
>> + };
>> +
>> + if (!readonly) {
>> + map.flags |= IOMMU_IOAS_MAP_WRITEABLE;
>> + }
>> +
>> + ret = ioctl(fd, IOMMU_IOAS_MAP, &map);
>> + trace_iommufd_backend_map_dma(fd, ioas_id, iova, size,
>> + vaddr, readonly, ret);
>> + if (ret) {
>> + ret = -errno;
>> +
>> + /* TODO: Not support mapping hardware PCI BAR region for
>> now. */
>> + if (errno == EFAULT) {
>> + warn_report("IOMMU_IOAS_MAP failed: %m, PCI BAR?");
>> + } else {
>> + error_report("IOMMU_IOAS_MAP failed: %m");
>> + }
>> + }
>> + return ret;
>> +}
>> +
>> +int iommufd_backend_unmap_dma(IOMMUFDBackend *be, uint32_t ioas_id,
>> + hwaddr iova, ram_addr_t size)
>> +{
>> + int ret, fd = be->fd;
>> + struct iommu_ioas_unmap unmap = {
>> + .size = sizeof(unmap),
>> + .ioas_id = ioas_id,
>> + .iova = iova,
>> + .length = size,
>> + };
>> +
>> + ret = ioctl(fd, IOMMU_IOAS_UNMAP, &unmap);
>> + /*
>> + * IOMMUFD takes mapping as some kind of object, unmapping
>> + * nonexistent mapping is treated as deleting a nonexistent
>> + * object and return ENOENT. This is different from legacy
>> + * backend which allows it. vIOMMU may trigger a lot of
>> + * redundant unmapping, to avoid flush the log, treat them
>> + * as succeess for IOMMUFD just like legacy backend.
>> + */
>> + if (ret && errno == ENOENT) {
>> + trace_iommufd_backend_unmap_dma_non_exist(fd, ioas_id, iova,
>> size, ret);
>> + ret = 0;
>> + } else {
>> + trace_iommufd_backend_unmap_dma(fd, ioas_id, iova, size, ret);
>> + }
>> +
>> + if (ret) {
>> + ret = -errno;
>> + error_report("IOMMU_IOAS_UNMAP failed: %m");
>> + }
>> + return ret;
>> +}
>> +
>> +static const TypeInfo iommufd_backend_info = {
>> + .name = TYPE_IOMMUFD_BACKEND,
>> + .parent = TYPE_OBJECT,
>> + .instance_size = sizeof(IOMMUFDBackend),
>> + .instance_init = iommufd_backend_init,
>> + .instance_finalize = iommufd_backend_finalize,
>> + .class_size = sizeof(IOMMUFDBackendClass),
>> + .class_init = iommufd_backend_class_init,
>> + .interfaces = (InterfaceInfo[]) {
>> + { TYPE_USER_CREATABLE },
>> + { }
>> + }
>> +};
>> +
>> +static void register_types(void)
>> +{
>> + type_register_static(&iommufd_backend_info);
>> +}
>> +
>> +type_init(register_types);
>> diff --git a/backends/Kconfig b/backends/Kconfig
>> index
>> f35abc16092808b1fe5b033a346908e2d66bff0b..2cb23f62fa1526cedafedcc99a032e098075b846
>> 100644
>> --- a/backends/Kconfig
>> +++ b/backends/Kconfig
>> @@ -1 +1,5 @@
>> source tpm/Kconfig
>> +
>> +config IOMMUFD
>> + bool
>> + depends on VFIO
>> diff --git a/backends/meson.build b/backends/meson.build
>> index
>> 914c7c4afb905cfe710ad23dd1ee42907f6d1679..9a5cea480d172d50a641e4d9179093e8155f2db1
>> 100644
>> --- a/backends/meson.build
>> +++ b/backends/meson.build
>> @@ -20,6 +20,7 @@ if have_vhost_user
>> system_ss.add(when: 'CONFIG_VIRTIO', if_true: files('vhost-user.c'))
>> endif
>> system_ss.add(when: 'CONFIG_VIRTIO_CRYPTO', if_true:
>> files('cryptodev-vhost.c'))
>> +system_ss.add(when: 'CONFIG_IOMMUFD', if_true: files('iommufd.c'))
>> if have_vhost_user_crypto
>> system_ss.add(when: 'CONFIG_VIRTIO_CRYPTO', if_true:
>> files('cryptodev-vhost-user.c'))
>> endif
>> diff --git a/backends/trace-events b/backends/trace-events
>> index
>> 652eb76a5723e2053fe97338c481309c58284d6a..d45c6e31a67ed66d94787f60eb08a525cf6ff68b
>> 100644
>> --- a/backends/trace-events
>> +++ b/backends/trace-events
>> @@ -5,3 +5,13 @@ dbus_vmstate_pre_save(void)
>> dbus_vmstate_post_load(int version_id) "version_id: %d"
>> dbus_vmstate_loading(const char *id) "id: %s"
>> dbus_vmstate_saving(const char *id) "id: %s"
>> +
>> +# iommufd.c
>> +iommufd_backend_connect(int fd, bool owned, uint32_t users, int ret)
>> "fd=%d owned=%d users=%d (%d)"
>> +iommufd_backend_disconnect(int fd, uint32_t users) "fd=%d users=%d"
>> +iommu_backend_set_fd(int fd) "pre-opened /dev/iommu fd=%d"
>> +iommufd_backend_map_dma(int iommufd, uint32_t ioas, uint64_t iova,
>> uint64_t size, void *vaddr, bool readonly, int ret) " iommufd=%d
>> ioas=%d iova=0x%"PRIx64" size=0x%"PRIx64" addr=%p readonly=%d (%d)"
>> +iommufd_backend_unmap_dma_non_exist(int iommufd, uint32_t ioas,
>> uint64_t iova, uint64_t size, int ret) " Unmap nonexistent mapping:
>> iommufd=%d ioas=%d iova=0x%"PRIx64" size=0x%"PRIx64" (%d)"
>> +iommufd_backend_unmap_dma(int iommufd, uint32_t ioas, uint64_t iova,
>> uint64_t size, int ret) " iommufd=%d ioas=%d iova=0x%"PRIx64"
>> size=0x%"PRIx64" (%d)"
>> +iommufd_backend_alloc_ioas(int iommufd, uint32_t ioas, int ret) "
>> iommufd=%d ioas=%d (%d)"
>> +iommufd_backend_free_id(int iommufd, uint32_t id, int ret) "
>> iommufd=%d id=%d (%d)"
>> diff --git a/qemu-options.hx b/qemu-options.hx
>> index
>> 42fd09e4de96e962cd5873c49501f6e1dbb5e346..5fe8ea57d2d2f9390a976ef2fefe86463e888bb1
>> 100644
>> --- a/qemu-options.hx
>> +++ b/qemu-options.hx
>> @@ -5224,6 +5224,18 @@ SRST
>> The ``share`` boolean option is on by default with memfd.
>> + ``-object iommufd,id=id[,fd=fd]``
>> + Creates an iommufd backend which allows control of DMA mapping
>> + through the ``/dev/iommu`` device.
>> +
>> + The ``id`` parameter is a unique ID which frontends (such as
>> + vfio-pci of vdpa) will use to connect with the iommufd backend.
>> +
>> + The ``fd`` parameter is an optional pre-opened file descriptor
>> + resulting from ``/dev/iommu`` opening. Usually the iommufd
>> is shared
>> + across all subsystems, bringing the benefit of centralized
>> + reference counting.
>> +
>> ``-object rng-builtin,id=id``
>> Creates a random number generator backend which obtains
>> entropy
>> from QEMU builtin functions. The ``id`` parameter is a
>> unique ID
>
^ permalink raw reply [flat|nested] 57+ messages in thread* Re: [PULL 20/47] backends/iommufd: Introduce the iommufd object
2023-12-21 17:14 ` Eric Auger
@ 2023-12-21 21:23 ` Cédric Le Goater
2023-12-22 10:09 ` Eric Auger
0 siblings, 1 reply; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-21 21:23 UTC (permalink / raw)
To: eric.auger, qemu-devel
Cc: Zhenzhong Duan, Peter Maydell, Richard Henderson, Nicholas Piggin,
Harsh Prateek Bora, Thomas Huth, Eric Farman, Alex Williamson,
Matthew Rosato, Yi Liu, Nicolin Chen
On 12/21/23 18:14, Eric Auger wrote:
> Hi Cédric,
>
> On 12/21/23 17:00, Cédric Le Goater wrote:
>> [ ... ]
>>
>>
>>> +static void iommufd_backend_init(Object *obj)
>>> +{
>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>> +
>>> + be->fd = -1;
>>> + be->users = 0;
>>> + be->owned = true;
>>> + qemu_mutex_init(&be->lock);> +}
>>> +
>>> +static void iommufd_backend_finalize(Object *obj)
>>> +{
>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>> +
>>> + if (be->owned) {
>>> + close(be->fd);
>>> + be->fd = -1;
>>> + }
>>> +}
>>> +
>>> +static void iommufd_backend_set_fd(Object *obj, const char *str,
>>> Error **errp)
>>> +{
>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>> + int fd = -1;
>>> +
>>> + fd = monitor_fd_param(monitor_cur(), str, errp);
>>> + if (fd == -1) {
>>> + error_prepend(errp, "Could not parse remote object fd %s:",
>>> str);
>>> + return;
>>> + }
>>> + qemu_mutex_lock(&be->lock);
>>> + be->fd = fd;
>>> + be->owned = false;
>>> + qemu_mutex_unlock(&be->lock);
>>> + trace_iommu_backend_set_fd(be->fd);
>>> +}
>>> +
>>> +static bool iommufd_backend_can_be_deleted(UserCreatable *uc)
>>> +{
>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);
>>> +
>>> + return !be->users;
>>
>> Coverity CID 1531549 reports a concurrent data access violation because
>> be->users is being accessed without holding the mutex.
>>
>> I wonder how useful is this mutex anyhow, since the code paths should
>> be protected by the BQL lock. If you agree, I will send an update to
>> simply drop be->lock and solve this report.
> I am not totally comfortable with the fact BQL covers the same
> protection? Please can you elaborate.
These routines are called when a device is created which is called
from the QEMU main thread which exits holding the BQL. It should be
fine.
Thanks,
C.
^ permalink raw reply [flat|nested] 57+ messages in thread* Re: [PULL 20/47] backends/iommufd: Introduce the iommufd object
2023-12-21 21:23 ` Cédric Le Goater
@ 2023-12-22 10:09 ` Eric Auger
2023-12-22 10:34 ` Cédric Le Goater
0 siblings, 1 reply; 57+ messages in thread
From: Eric Auger @ 2023-12-22 10:09 UTC (permalink / raw)
To: Cédric Le Goater, qemu-devel
Cc: Zhenzhong Duan, Peter Maydell, Richard Henderson, Nicholas Piggin,
Harsh Prateek Bora, Thomas Huth, Eric Farman, Alex Williamson,
Matthew Rosato, Yi Liu, Nicolin Chen
Hi Cédric,
On 12/21/23 22:23, Cédric Le Goater wrote:
> On 12/21/23 18:14, Eric Auger wrote:
>> Hi Cédric,
>>
>> On 12/21/23 17:00, Cédric Le Goater wrote:
>>> [ ... ]
>>>
>>>
>>>> +static void iommufd_backend_init(Object *obj)
>>>> +{
>>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>>> +
>>>> + be->fd = -1;
>>>> + be->users = 0;
>>>> + be->owned = true;
>>>> + qemu_mutex_init(&be->lock);> +}
>>>> +
>>>> +static void iommufd_backend_finalize(Object *obj)
>>>> +{
>>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>>> +
>>>> + if (be->owned) {
>>>> + close(be->fd);
>>>> + be->fd = -1;
>>>> + }
>>>> +}
>>>> +
>>>> +static void iommufd_backend_set_fd(Object *obj, const char *str,
>>>> Error **errp)
>>>> +{
>>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>>> + int fd = -1;
>>>> +
>>>> + fd = monitor_fd_param(monitor_cur(), str, errp);
>>>> + if (fd == -1) {
>>>> + error_prepend(errp, "Could not parse remote object fd %s:",
>>>> str);
>>>> + return;
>>>> + }
>>>> + qemu_mutex_lock(&be->lock);
>>>> + be->fd = fd;
>>>> + be->owned = false;
>>>> + qemu_mutex_unlock(&be->lock);
>>>> + trace_iommu_backend_set_fd(be->fd);
>>>> +}
>>>> +
>>>> +static bool iommufd_backend_can_be_deleted(UserCreatable *uc)
>>>> +{
>>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);
>>>> +
>>>> + return !be->users;
>>>
>>> Coverity CID 1531549 reports a concurrent data access violation because
>>> be->users is being accessed without holding the mutex.
>>>
>>> I wonder how useful is this mutex anyhow, since the code paths should
>>> be protected by the BQL lock. If you agree, I will send an update to
>>> simply drop be->lock and solve this report.
>> I am not totally comfortable with the fact BQL covers the same
>> protection? Please can you elaborate.
>
> These routines are called when a device is created which is called
> from the QEMU main thread which exits holding the BQL. It should be
> fine.
OK fine for me as well
Thanks
Eric
>
> Thanks,
>
> C.
>
>
^ permalink raw reply [flat|nested] 57+ messages in thread* Re: [PULL 20/47] backends/iommufd: Introduce the iommufd object
2023-12-22 10:09 ` Eric Auger
@ 2023-12-22 10:34 ` Cédric Le Goater
0 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-22 10:34 UTC (permalink / raw)
To: eric.auger, qemu-devel
Cc: Zhenzhong Duan, Peter Maydell, Richard Henderson, Nicholas Piggin,
Harsh Prateek Bora, Thomas Huth, Eric Farman, Alex Williamson,
Matthew Rosato, Yi Liu, Nicolin Chen
On 12/22/23 11:09, Eric Auger wrote:
> Hi Cédric,
>
> On 12/21/23 22:23, Cédric Le Goater wrote:
>> On 12/21/23 18:14, Eric Auger wrote:
>>> Hi Cédric,
>>>
>>> On 12/21/23 17:00, Cédric Le Goater wrote:
>>>> [ ... ]
>>>>
>>>>
>>>>> +static void iommufd_backend_init(Object *obj)
>>>>> +{
>>>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>>>> +
>>>>> + be->fd = -1;
>>>>> + be->users = 0;
>>>>> + be->owned = true;
>>>>> + qemu_mutex_init(&be->lock);> +}
>>>>> +
>>>>> +static void iommufd_backend_finalize(Object *obj)
>>>>> +{
>>>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>>>> +
>>>>> + if (be->owned) {
>>>>> + close(be->fd);
>>>>> + be->fd = -1;
>>>>> + }
>>>>> +}
>>>>> +
>>>>> +static void iommufd_backend_set_fd(Object *obj, const char *str,
>>>>> Error **errp)
>>>>> +{
>>>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>>>>> + int fd = -1;
>>>>> +
>>>>> + fd = monitor_fd_param(monitor_cur(), str, errp);
>>>>> + if (fd == -1) {
>>>>> + error_prepend(errp, "Could not parse remote object fd %s:",
>>>>> str);
>>>>> + return;
>>>>> + }
>>>>> + qemu_mutex_lock(&be->lock);
>>>>> + be->fd = fd;
>>>>> + be->owned = false;
>>>>> + qemu_mutex_unlock(&be->lock);
>>>>> + trace_iommu_backend_set_fd(be->fd);
>>>>> +}
>>>>> +
>>>>> +static bool iommufd_backend_can_be_deleted(UserCreatable *uc)
>>>>> +{
>>>>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);
>>>>> +
>>>>> + return !be->users;
>>>>
>>>> Coverity CID 1531549 reports a concurrent data access violation because
>>>> be->users is being accessed without holding the mutex.
>>>>
>>>> I wonder how useful is this mutex anyhow, since the code paths should
>>>> be protected by the BQL lock. If you agree, I will send an update to
>>>> simply drop be->lock and solve this report.
>>> I am not totally comfortable with the fact BQL covers the same
>>> protection? Please can you elaborate.
>>
>> These routines are called when a device is created which is called
>> from the QEMU main thread which exits holding the BQL. It should be
>> fine.
>
> OK fine for me as well
I pushed 2 patches on vfio-9.0 that I will send next year. Time for a
break !
Cheers,
C.
^ permalink raw reply [flat|nested] 57+ messages in thread
* RE: [PULL 20/47] backends/iommufd: Introduce the iommufd object
2023-12-21 16:00 ` Cédric Le Goater
2023-12-21 17:14 ` Eric Auger
@ 2023-12-22 2:41 ` Duan, Zhenzhong
1 sibling, 0 replies; 57+ messages in thread
From: Duan, Zhenzhong @ 2023-12-22 2:41 UTC (permalink / raw)
To: Cédric Le Goater, qemu-devel@nongnu.org
Cc: Eric Auger, Peter Maydell, Richard Henderson, Nicholas Piggin,
Harsh Prateek Bora, Thomas Huth, Eric Farman, Alex Williamson,
Matthew Rosato, Liu, Yi L, Nicolin Chen
Hi Cédric,
>-----Original Message-----
>From: Cédric Le Goater <clg@redhat.com>
>Sent: Friday, December 22, 2023 12:01 AM
>Subject: Re: [PULL 20/47] backends/iommufd: Introduce the iommufd object
>
>[ ... ]
>
>
>> +static void iommufd_backend_init(Object *obj)
>> +{
>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>> +
>> + be->fd = -1;
>> + be->users = 0;
>> + be->owned = true;
>> + qemu_mutex_init(&be->lock);> +}
>> +
>> +static void iommufd_backend_finalize(Object *obj)
>> +{
>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>> +
>> + if (be->owned) {
>> + close(be->fd);
>> + be->fd = -1;
>> + }
>> +}
>> +
>> +static void iommufd_backend_set_fd(Object *obj, const char *str, Error
>**errp)
>> +{
>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
>> + int fd = -1;
>> +
>> + fd = monitor_fd_param(monitor_cur(), str, errp);
>> + if (fd == -1) {
>> + error_prepend(errp, "Could not parse remote object fd %s:", str);
>> + return;
>> + }
>> + qemu_mutex_lock(&be->lock);
>> + be->fd = fd;
>> + be->owned = false;
>> + qemu_mutex_unlock(&be->lock);
>> + trace_iommu_backend_set_fd(be->fd);
>> +}
>> +
>> +static bool iommufd_backend_can_be_deleted(UserCreatable *uc)
>> +{
>> + IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);
>> +
>> + return !be->users;
>
>Coverity CID 1531549 reports a concurrent data access violation because
>be->users is being accessed without holding the mutex.
>
>I wonder how useful is this mutex anyhow, since the code paths should
>be protected by the BQL lock. If you agree, I will send an update to
>simply drop be->lock and solve this report.
Make sense on my side, thanks for help fixing.
BRs.
Zhenzhong
^ permalink raw reply [flat|nested] 57+ messages in thread
* [PULL 21/47] util/char_dev: Add open_cdev()
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (19 preceding siblings ...)
2023-12-19 18:56 ` [PULL 20/47] backends/iommufd: Introduce the iommufd object Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 22/47] vfio/common: return early if space isn't empty Cédric Le Goater
` (26 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Jason Gunthorpe,
Cédric Le Goater, Nicolin Chen
From: Yi Liu <yi.l.liu@intel.com>
/dev/vfio/devices/vfioX may not exist. In that case it is still possible
to open /dev/char/$major:$minor instead. Add helper function to abstract
the cdev open.
Suggested-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
MAINTAINERS | 2 +
include/qemu/chardev_open.h | 16 ++++++++
util/chardev_open.c | 81 +++++++++++++++++++++++++++++++++++++
util/meson.build | 1 +
4 files changed, 100 insertions(+)
create mode 100644 include/qemu/chardev_open.h
create mode 100644 util/chardev_open.c
diff --git a/MAINTAINERS b/MAINTAINERS
index a5a446914a1cf505131bfe113540fd501511abb8..ca70bb4e6415fc3af110cc7fd37ac67be5ab8c9d 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2174,6 +2174,8 @@ M: Zhenzhong Duan <zhenzhong.duan@intel.com>
S: Supported
F: backends/iommufd.c
F: include/sysemu/iommufd.h
+F: include/qemu/chardev_open.h
+F: util/chardev_open.c
vhost
M: Michael S. Tsirkin <mst@redhat.com>
diff --git a/include/qemu/chardev_open.h b/include/qemu/chardev_open.h
new file mode 100644
index 0000000000000000000000000000000000000000..64e8fcfdcb239e47774d55dfc57efff9101ce0a8
--- /dev/null
+++ b/include/qemu/chardev_open.h
@@ -0,0 +1,16 @@
+/*
+ * QEMU Chardev Helper
+ *
+ * Copyright (C) 2023 Intel Corporation.
+ *
+ * Authors: Yi Liu <yi.l.liu@intel.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_CHARDEV_OPEN_H
+#define QEMU_CHARDEV_OPEN_H
+
+int open_cdev(const char *devpath, dev_t cdev);
+#endif
diff --git a/util/chardev_open.c b/util/chardev_open.c
new file mode 100644
index 0000000000000000000000000000000000000000..f7764297882f59411c81b304a1ab5d16e903a389
--- /dev/null
+++ b/util/chardev_open.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+ * Copyright (C) 2023 Intel Corporation.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors: Yi Liu <yi.l.liu@intel.com>
+ *
+ * Copied from
+ * https://github.com/linux-rdma/rdma-core/blob/master/util/open_cdev.c
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/chardev_open.h"
+
+static int open_cdev_internal(const char *path, dev_t cdev)
+{
+ struct stat st;
+ int fd;
+
+ fd = qemu_open_old(path, O_RDWR);
+ if (fd == -1) {
+ return -1;
+ }
+ if (fstat(fd, &st) || !S_ISCHR(st.st_mode) ||
+ (cdev != 0 && st.st_rdev != cdev)) {
+ close(fd);
+ return -1;
+ }
+ return fd;
+}
+
+static int open_cdev_robust(dev_t cdev)
+{
+ g_autofree char *devpath = NULL;
+
+ /*
+ * This assumes that udev is being used and is creating the /dev/char/
+ * symlinks.
+ */
+ devpath = g_strdup_printf("/dev/char/%u:%u", major(cdev), minor(cdev));
+ return open_cdev_internal(devpath, cdev);
+}
+
+int open_cdev(const char *devpath, dev_t cdev)
+{
+ int fd;
+
+ fd = open_cdev_internal(devpath, cdev);
+ if (fd == -1 && cdev != 0) {
+ return open_cdev_robust(cdev);
+ }
+ return fd;
+}
diff --git a/util/meson.build b/util/meson.build
index c2322ef6e71a1de643d44dd3ad4bad497bc975ec..174c133368a8584315f488786a3d4b59f6cede80 100644
--- a/util/meson.build
+++ b/util/meson.build
@@ -108,6 +108,7 @@ if have_block
util_ss.add(files('filemonitor-stub.c'))
endif
util_ss.add(when: 'CONFIG_LINUX', if_true: files('vfio-helpers.c'))
+ util_ss.add(when: 'CONFIG_LINUX', if_true: files('chardev_open.c'))
endif
if cpu == 'aarch64'
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 22/47] vfio/common: return early if space isn't empty
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (20 preceding siblings ...)
2023-12-19 18:56 ` [PULL 21/47] util/char_dev: Add open_cdev() Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 23/47] vfio/iommufd: Implement the iommufd backend Cédric Le Goater
` (25 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
This is a trivial optimization. If there is active container in space,
vfio_reset_handler will never be unregistered. So revert the check of
space->containers and return early.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/common.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 572ae7c934070b9fdc66915f6831fb0fa1323941..934f4f5446e9fc7decd991148f3f7992bf653a4a 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -1462,10 +1462,13 @@ VFIOAddressSpace *vfio_get_address_space(AddressSpace *as)
void vfio_put_address_space(VFIOAddressSpace *space)
{
- if (QLIST_EMPTY(&space->containers)) {
- QLIST_REMOVE(space, list);
- g_free(space);
+ if (!QLIST_EMPTY(&space->containers)) {
+ return;
}
+
+ QLIST_REMOVE(space, list);
+ g_free(space);
+
if (QLIST_EMPTY(&vfio_address_spaces)) {
qemu_unregister_reset(vfio_reset_handler, NULL);
}
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 23/47] vfio/iommufd: Implement the iommufd backend
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (21 preceding siblings ...)
2023-12-19 18:56 ` [PULL 22/47] vfio/common: return early if space isn't empty Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 24/47] vfio/iommufd: Relax assert check for " Cédric Le Goater
` (24 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Cédric Le Goater,
Nicolin Chen
From: Yi Liu <yi.l.liu@intel.com>
The iommufd backend is implemented based on the new /dev/iommu user API.
This backend obviously depends on CONFIG_IOMMUFD.
So far, the iommufd backend doesn't support dirty page sync yet.
Co-authored-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 11 +
hw/vfio/common.c | 6 +
hw/vfio/iommufd.c | 422 ++++++++++++++++++++++++++++++++++
hw/vfio/meson.build | 3 +
hw/vfio/trace-events | 10 +
5 files changed, 452 insertions(+)
create mode 100644 hw/vfio/iommufd.c
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 24ecc0e7ee6243ad9ee2003d197e04390b780d2c..3dac5c167efa1fc6afefc103539ce5e01cceb602 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -89,6 +89,14 @@ typedef struct VFIOHostDMAWindow {
QLIST_ENTRY(VFIOHostDMAWindow) hostwin_next;
} VFIOHostDMAWindow;
+typedef struct IOMMUFDBackend IOMMUFDBackend;
+
+typedef struct VFIOIOMMUFDContainer {
+ VFIOContainerBase bcontainer;
+ IOMMUFDBackend *be;
+ uint32_t ioas_id;
+} VFIOIOMMUFDContainer;
+
typedef struct VFIODeviceOps VFIODeviceOps;
typedef struct VFIODevice {
@@ -116,6 +124,8 @@ typedef struct VFIODevice {
OnOffAuto pre_copy_dirty_page_tracking;
bool dirty_pages_supported;
bool dirty_tracking;
+ int devid;
+ IOMMUFDBackend *iommufd;
} VFIODevice;
struct VFIODeviceOps {
@@ -201,6 +211,7 @@ typedef QLIST_HEAD(VFIODeviceList, VFIODevice) VFIODeviceList;
extern VFIOGroupList vfio_group_list;
extern VFIODeviceList vfio_device_list;
extern const VFIOIOMMUOps vfio_legacy_ops;
+extern const VFIOIOMMUOps vfio_iommufd_ops;
extern const MemoryListener vfio_memory_listener;
extern int vfio_kvm_device_fd;
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 934f4f5446e9fc7decd991148f3f7992bf653a4a..6569732b7a1d894ae08e38adb93dfd94aeca4075 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -19,6 +19,7 @@
*/
#include "qemu/osdep.h"
+#include CONFIG_DEVICES /* CONFIG_IOMMUFD */
#include <sys/ioctl.h>
#ifdef CONFIG_KVM
#include <linux/kvm.h>
@@ -1503,6 +1504,11 @@ int vfio_attach_device(char *name, VFIODevice *vbasedev,
{
const VFIOIOMMUOps *ops = &vfio_legacy_ops;
+#ifdef CONFIG_IOMMUFD
+ if (vbasedev->iommufd) {
+ ops = &vfio_iommufd_ops;
+ }
+#endif
return ops->attach_device(name, vbasedev, as, errp);
}
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
new file mode 100644
index 0000000000000000000000000000000000000000..6d31aeac7bd8781a103328f8a438c011cdc2db1e
--- /dev/null
+++ b/hw/vfio/iommufd.c
@@ -0,0 +1,422 @@
+/*
+ * iommufd container backend
+ *
+ * Copyright (C) 2023 Intel Corporation.
+ * Copyright Red Hat, Inc. 2023
+ *
+ * Authors: Yi Liu <yi.l.liu@intel.com>
+ * Eric Auger <eric.auger@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "qemu/osdep.h"
+#include <sys/ioctl.h>
+#include <linux/vfio.h>
+#include <linux/iommufd.h>
+
+#include "hw/vfio/vfio-common.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+#include "qapi/error.h"
+#include "sysemu/iommufd.h"
+#include "hw/qdev-core.h"
+#include "sysemu/reset.h"
+#include "qemu/cutils.h"
+#include "qemu/chardev_open.h"
+
+static int iommufd_cdev_map(VFIOContainerBase *bcontainer, hwaddr iova,
+ ram_addr_t size, void *vaddr, bool readonly)
+{
+ VFIOIOMMUFDContainer *container =
+ container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
+
+ return iommufd_backend_map_dma(container->be,
+ container->ioas_id,
+ iova, size, vaddr, readonly);
+}
+
+static int iommufd_cdev_unmap(VFIOContainerBase *bcontainer,
+ hwaddr iova, ram_addr_t size,
+ IOMMUTLBEntry *iotlb)
+{
+ VFIOIOMMUFDContainer *container =
+ container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
+
+ /* TODO: Handle dma_unmap_bitmap with iotlb args (migration) */
+ return iommufd_backend_unmap_dma(container->be,
+ container->ioas_id, iova, size);
+}
+
+static int iommufd_cdev_kvm_device_add(VFIODevice *vbasedev, Error **errp)
+{
+ return vfio_kvm_device_add_fd(vbasedev->fd, errp);
+}
+
+static void iommufd_cdev_kvm_device_del(VFIODevice *vbasedev)
+{
+ Error *err = NULL;
+
+ if (vfio_kvm_device_del_fd(vbasedev->fd, &err)) {
+ error_report_err(err);
+ }
+}
+
+static int iommufd_cdev_connect_and_bind(VFIODevice *vbasedev, Error **errp)
+{
+ IOMMUFDBackend *iommufd = vbasedev->iommufd;
+ struct vfio_device_bind_iommufd bind = {
+ .argsz = sizeof(bind),
+ .flags = 0,
+ };
+ int ret;
+
+ ret = iommufd_backend_connect(iommufd, errp);
+ if (ret) {
+ return ret;
+ }
+
+ /*
+ * Add device to kvm-vfio to be prepared for the tracking
+ * in KVM. Especially for some emulated devices, it requires
+ * to have kvm information in the device open.
+ */
+ ret = iommufd_cdev_kvm_device_add(vbasedev, errp);
+ if (ret) {
+ goto err_kvm_device_add;
+ }
+
+ /* Bind device to iommufd */
+ bind.iommufd = iommufd->fd;
+ ret = ioctl(vbasedev->fd, VFIO_DEVICE_BIND_IOMMUFD, &bind);
+ if (ret) {
+ error_setg_errno(errp, errno, "error bind device fd=%d to iommufd=%d",
+ vbasedev->fd, bind.iommufd);
+ goto err_bind;
+ }
+
+ vbasedev->devid = bind.out_devid;
+ trace_iommufd_cdev_connect_and_bind(bind.iommufd, vbasedev->name,
+ vbasedev->fd, vbasedev->devid);
+ return ret;
+err_bind:
+ iommufd_cdev_kvm_device_del(vbasedev);
+err_kvm_device_add:
+ iommufd_backend_disconnect(iommufd);
+ return ret;
+}
+
+static void iommufd_cdev_unbind_and_disconnect(VFIODevice *vbasedev)
+{
+ /* Unbind is automatically conducted when device fd is closed */
+ iommufd_cdev_kvm_device_del(vbasedev);
+ iommufd_backend_disconnect(vbasedev->iommufd);
+}
+
+static int iommufd_cdev_getfd(const char *sysfs_path, Error **errp)
+{
+ long int ret = -ENOTTY;
+ char *path, *vfio_dev_path = NULL, *vfio_path = NULL;
+ DIR *dir = NULL;
+ struct dirent *dent;
+ gchar *contents;
+ struct stat st;
+ gsize length;
+ int major, minor;
+ dev_t vfio_devt;
+
+ path = g_strdup_printf("%s/vfio-dev", sysfs_path);
+ if (stat(path, &st) < 0) {
+ error_setg_errno(errp, errno, "no such host device");
+ goto out_free_path;
+ }
+
+ dir = opendir(path);
+ if (!dir) {
+ error_setg_errno(errp, errno, "couldn't open directory %s", path);
+ goto out_free_path;
+ }
+
+ while ((dent = readdir(dir))) {
+ if (!strncmp(dent->d_name, "vfio", 4)) {
+ vfio_dev_path = g_strdup_printf("%s/%s/dev", path, dent->d_name);
+ break;
+ }
+ }
+
+ if (!vfio_dev_path) {
+ error_setg(errp, "failed to find vfio-dev/vfioX/dev");
+ goto out_close_dir;
+ }
+
+ if (!g_file_get_contents(vfio_dev_path, &contents, &length, NULL)) {
+ error_setg(errp, "failed to load \"%s\"", vfio_dev_path);
+ goto out_free_dev_path;
+ }
+
+ if (sscanf(contents, "%d:%d", &major, &minor) != 2) {
+ error_setg(errp, "failed to get major:minor for \"%s\"", vfio_dev_path);
+ goto out_free_dev_path;
+ }
+ g_free(contents);
+ vfio_devt = makedev(major, minor);
+
+ vfio_path = g_strdup_printf("/dev/vfio/devices/%s", dent->d_name);
+ ret = open_cdev(vfio_path, vfio_devt);
+ if (ret < 0) {
+ error_setg(errp, "Failed to open %s", vfio_path);
+ }
+
+ trace_iommufd_cdev_getfd(vfio_path, ret);
+ g_free(vfio_path);
+
+out_free_dev_path:
+ g_free(vfio_dev_path);
+out_close_dir:
+ closedir(dir);
+out_free_path:
+ if (*errp) {
+ error_prepend(errp, VFIO_MSG_PREFIX, path);
+ }
+ g_free(path);
+
+ return ret;
+}
+
+static int iommufd_cdev_attach_ioas_hwpt(VFIODevice *vbasedev, uint32_t id,
+ Error **errp)
+{
+ int ret, iommufd = vbasedev->iommufd->fd;
+ struct vfio_device_attach_iommufd_pt attach_data = {
+ .argsz = sizeof(attach_data),
+ .flags = 0,
+ .pt_id = id,
+ };
+
+ /* Attach device to an IOAS or hwpt within iommufd */
+ ret = ioctl(vbasedev->fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data);
+ if (ret) {
+ error_setg_errno(errp, errno,
+ "[iommufd=%d] error attach %s (%d) to id=%d",
+ iommufd, vbasedev->name, vbasedev->fd, id);
+ } else {
+ trace_iommufd_cdev_attach_ioas_hwpt(iommufd, vbasedev->name,
+ vbasedev->fd, id);
+ }
+ return ret;
+}
+
+static int iommufd_cdev_detach_ioas_hwpt(VFIODevice *vbasedev, Error **errp)
+{
+ int ret, iommufd = vbasedev->iommufd->fd;
+ struct vfio_device_detach_iommufd_pt detach_data = {
+ .argsz = sizeof(detach_data),
+ .flags = 0,
+ };
+
+ ret = ioctl(vbasedev->fd, VFIO_DEVICE_DETACH_IOMMUFD_PT, &detach_data);
+ if (ret) {
+ error_setg_errno(errp, errno, "detach %s failed", vbasedev->name);
+ } else {
+ trace_iommufd_cdev_detach_ioas_hwpt(iommufd, vbasedev->name);
+ }
+ return ret;
+}
+
+static int iommufd_cdev_attach_container(VFIODevice *vbasedev,
+ VFIOIOMMUFDContainer *container,
+ Error **errp)
+{
+ return iommufd_cdev_attach_ioas_hwpt(vbasedev, container->ioas_id, errp);
+}
+
+static void iommufd_cdev_detach_container(VFIODevice *vbasedev,
+ VFIOIOMMUFDContainer *container)
+{
+ Error *err = NULL;
+
+ if (iommufd_cdev_detach_ioas_hwpt(vbasedev, &err)) {
+ error_report_err(err);
+ }
+}
+
+static void iommufd_cdev_container_destroy(VFIOIOMMUFDContainer *container)
+{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
+
+ if (!QLIST_EMPTY(&bcontainer->device_list)) {
+ return;
+ }
+ memory_listener_unregister(&bcontainer->listener);
+ vfio_container_destroy(bcontainer);
+ iommufd_backend_free_id(container->be, container->ioas_id);
+ g_free(container);
+}
+
+static int iommufd_cdev_ram_block_discard_disable(bool state)
+{
+ /*
+ * We support coordinated discarding of RAM via the RamDiscardManager.
+ */
+ return ram_block_uncoordinated_discard_disable(state);
+}
+
+static int iommufd_cdev_attach(const char *name, VFIODevice *vbasedev,
+ AddressSpace *as, Error **errp)
+{
+ VFIOContainerBase *bcontainer;
+ VFIOIOMMUFDContainer *container;
+ VFIOAddressSpace *space;
+ struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
+ int ret, devfd;
+ uint32_t ioas_id;
+ Error *err = NULL;
+
+ devfd = iommufd_cdev_getfd(vbasedev->sysfsdev, errp);
+ if (devfd < 0) {
+ return devfd;
+ }
+ vbasedev->fd = devfd;
+
+ ret = iommufd_cdev_connect_and_bind(vbasedev, errp);
+ if (ret) {
+ goto err_connect_bind;
+ }
+
+ space = vfio_get_address_space(as);
+
+ /* try to attach to an existing container in this space */
+ QLIST_FOREACH(bcontainer, &space->containers, next) {
+ container = container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
+ if (bcontainer->ops != &vfio_iommufd_ops ||
+ vbasedev->iommufd != container->be) {
+ continue;
+ }
+ if (iommufd_cdev_attach_container(vbasedev, container, &err)) {
+ const char *msg = error_get_pretty(err);
+
+ trace_iommufd_cdev_fail_attach_existing_container(msg);
+ error_free(err);
+ err = NULL;
+ } else {
+ ret = iommufd_cdev_ram_block_discard_disable(true);
+ if (ret) {
+ error_setg(errp,
+ "Cannot set discarding of RAM broken (%d)", ret);
+ goto err_discard_disable;
+ }
+ goto found_container;
+ }
+ }
+
+ /* Need to allocate a new dedicated container */
+ ret = iommufd_backend_alloc_ioas(vbasedev->iommufd, &ioas_id, errp);
+ if (ret < 0) {
+ goto err_alloc_ioas;
+ }
+
+ trace_iommufd_cdev_alloc_ioas(vbasedev->iommufd->fd, ioas_id);
+
+ container = g_malloc0(sizeof(*container));
+ container->be = vbasedev->iommufd;
+ container->ioas_id = ioas_id;
+
+ bcontainer = &container->bcontainer;
+ vfio_container_init(bcontainer, space, &vfio_iommufd_ops);
+ QLIST_INSERT_HEAD(&space->containers, bcontainer, next);
+
+ ret = iommufd_cdev_attach_container(vbasedev, container, errp);
+ if (ret) {
+ goto err_attach_container;
+ }
+
+ ret = iommufd_cdev_ram_block_discard_disable(true);
+ if (ret) {
+ goto err_discard_disable;
+ }
+
+ bcontainer->pgsizes = qemu_real_host_page_size();
+
+ bcontainer->listener = vfio_memory_listener;
+ memory_listener_register(&bcontainer->listener, bcontainer->space->as);
+
+ if (bcontainer->error) {
+ ret = -1;
+ error_propagate_prepend(errp, bcontainer->error,
+ "memory listener initialization failed: ");
+ goto err_listener_register;
+ }
+
+ bcontainer->initialized = true;
+
+found_container:
+ ret = ioctl(devfd, VFIO_DEVICE_GET_INFO, &dev_info);
+ if (ret) {
+ error_setg_errno(errp, errno, "error getting device info");
+ goto err_listener_register;
+ }
+
+ /*
+ * TODO: examine RAM_BLOCK_DISCARD stuff, should we do group level
+ * for discarding incompatibility check as well?
+ */
+ if (vbasedev->ram_block_discard_allowed) {
+ iommufd_cdev_ram_block_discard_disable(false);
+ }
+
+ vbasedev->group = 0;
+ vbasedev->num_irqs = dev_info.num_irqs;
+ vbasedev->num_regions = dev_info.num_regions;
+ vbasedev->flags = dev_info.flags;
+ vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
+ vbasedev->bcontainer = bcontainer;
+ QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next);
+ QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next);
+
+ trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev->num_irqs,
+ vbasedev->num_regions, vbasedev->flags);
+ return 0;
+
+err_listener_register:
+ iommufd_cdev_ram_block_discard_disable(false);
+err_discard_disable:
+ iommufd_cdev_detach_container(vbasedev, container);
+err_attach_container:
+ iommufd_cdev_container_destroy(container);
+err_alloc_ioas:
+ vfio_put_address_space(space);
+ iommufd_cdev_unbind_and_disconnect(vbasedev);
+err_connect_bind:
+ close(vbasedev->fd);
+ return ret;
+}
+
+static void iommufd_cdev_detach(VFIODevice *vbasedev)
+{
+ VFIOContainerBase *bcontainer = vbasedev->bcontainer;
+ VFIOAddressSpace *space = bcontainer->space;
+ VFIOIOMMUFDContainer *container = container_of(bcontainer,
+ VFIOIOMMUFDContainer,
+ bcontainer);
+ QLIST_REMOVE(vbasedev, global_next);
+ QLIST_REMOVE(vbasedev, container_next);
+ vbasedev->bcontainer = NULL;
+
+ if (!vbasedev->ram_block_discard_allowed) {
+ iommufd_cdev_ram_block_discard_disable(false);
+ }
+
+ iommufd_cdev_detach_container(vbasedev, container);
+ iommufd_cdev_container_destroy(container);
+ vfio_put_address_space(space);
+
+ iommufd_cdev_unbind_and_disconnect(vbasedev);
+ close(vbasedev->fd);
+}
+
+const VFIOIOMMUOps vfio_iommufd_ops = {
+ .dma_map = iommufd_cdev_map,
+ .dma_unmap = iommufd_cdev_unmap,
+ .attach_device = iommufd_cdev_attach,
+ .detach_device = iommufd_cdev_detach,
+};
diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build
index eb6ce6229daef67206d562e981898d62b105ece0..e5d98b6adc223061f6b0c3e1a7db3ba93d4eef16 100644
--- a/hw/vfio/meson.build
+++ b/hw/vfio/meson.build
@@ -7,6 +7,9 @@ vfio_ss.add(files(
'spapr.c',
'migration.c',
))
+vfio_ss.add(when: 'CONFIG_IOMMUFD', if_true: files(
+ 'iommufd.c',
+))
vfio_ss.add(when: 'CONFIG_VFIO_PCI', if_true: files(
'display.c',
'pci-quirks.c',
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index 08a1f9dfa4508b8378a41c35feed67123ba4b942..3340c93af0e9d366e04e6fe573f1010ab2a861a6 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -164,3 +164,13 @@ vfio_state_pending_estimate(const char *name, uint64_t precopy, uint64_t postcop
vfio_state_pending_exact(const char *name, uint64_t precopy, uint64_t postcopy, uint64_t stopcopy_size, uint64_t precopy_init_size, uint64_t precopy_dirty_size) " (%s) precopy 0x%"PRIx64" postcopy 0x%"PRIx64" stopcopy size 0x%"PRIx64" precopy initial size 0x%"PRIx64" precopy dirty size 0x%"PRIx64
vfio_vmstate_change(const char *name, int running, const char *reason, const char *dev_state) " (%s) running %d reason %s device state %s"
vfio_vmstate_change_prepare(const char *name, int running, const char *reason, const char *dev_state) " (%s) running %d reason %s device state %s"
+
+#iommufd.c
+
+iommufd_cdev_connect_and_bind(int iommufd, const char *name, int devfd, int devid) " [iommufd=%d] Successfully bound device %s (fd=%d): output devid=%d"
+iommufd_cdev_getfd(const char *dev, int devfd) " %s (fd=%d)"
+iommufd_cdev_attach_ioas_hwpt(int iommufd, const char *name, int devfd, int id) " [iommufd=%d] Successfully attached device %s (%d) to id=%d"
+iommufd_cdev_detach_ioas_hwpt(int iommufd, const char *name) " [iommufd=%d] Successfully detached %s"
+iommufd_cdev_fail_attach_existing_container(const char *msg) " %s"
+iommufd_cdev_alloc_ioas(int iommufd, int ioas_id) " [iommufd=%d] new IOMMUFD container with ioasid=%d"
+iommufd_cdev_device_info(char *name, int devfd, int num_irqs, int num_regions, int flags) " %s (%d) num_irqs=%d num_regions=%d flags=%d"
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 24/47] vfio/iommufd: Relax assert check for iommufd backend
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (22 preceding siblings ...)
2023-12-19 18:56 ` [PULL 23/47] vfio/iommufd: Implement the iommufd backend Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 25/47] vfio/iommufd: Add support for iova_ranges and pgsizes Cédric Le Goater
` (23 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Currently iommufd doesn't support dirty page sync yet,
but it will not block us doing live migration if VFIO
migration is force enabled.
So in this case we allow set_dirty_page_tracking to be NULL.
Note we don't need same change for query_dirty_bitmap because
when dirty page sync isn't supported, query_dirty_bitmap will
never be called.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/container-base.c | 4 ++++
hw/vfio/container.c | 4 ----
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index 71f727497314b06185fd0201d1472b83aafda703..eee2dcfe760b5530ef428fe341d376a85b47344a 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -55,6 +55,10 @@ void vfio_container_del_section_window(VFIOContainerBase *bcontainer,
int vfio_container_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
bool start)
{
+ if (!bcontainer->dirty_pages_supported) {
+ return 0;
+ }
+
g_assert(bcontainer->ops->set_dirty_page_tracking);
return bcontainer->ops->set_dirty_page_tracking(bcontainer, start);
}
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 6bacf38222ad9d95e05897572995e925e46b54dc..ed2d721b2bc6d175ecd3fb9498cbdc0d7ba6769d 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -216,10 +216,6 @@ static int vfio_legacy_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
.argsz = sizeof(dirty),
};
- if (!bcontainer->dirty_pages_supported) {
- return 0;
- }
-
if (start) {
dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START;
} else {
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 25/47] vfio/iommufd: Add support for iova_ranges and pgsizes
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (23 preceding siblings ...)
2023-12-19 18:56 ` [PULL 24/47] vfio/iommufd: Relax assert check for " Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 26/47] vfio/pci: Extract out a helper vfio_pci_get_pci_hot_reset_info Cédric Le Goater
` (22 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Some vIOMMU such as virtio-iommu use IOVA ranges from host side to
setup reserved ranges for passthrough device, so that guest will not
use an IOVA range beyond host support.
Use an uAPI of IOMMUFD to get IOVA ranges of host side and pass to
vIOMMU just like the legacy backend, if this fails, fallback to
64bit IOVA range.
Also use out_iova_alignment returned from uAPI as pgsizes instead of
qemu_real_host_page_size() as a fallback.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/iommufd.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 55 insertions(+), 1 deletion(-)
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
index 6d31aeac7bd8781a103328f8a438c011cdc2db1e..01b448e840581e0dd6d3df1897169665f79dcbe3 100644
--- a/hw/vfio/iommufd.c
+++ b/hw/vfio/iommufd.c
@@ -261,6 +261,53 @@ static int iommufd_cdev_ram_block_discard_disable(bool state)
return ram_block_uncoordinated_discard_disable(state);
}
+static int iommufd_cdev_get_info_iova_range(VFIOIOMMUFDContainer *container,
+ uint32_t ioas_id, Error **errp)
+{
+ VFIOContainerBase *bcontainer = &container->bcontainer;
+ struct iommu_ioas_iova_ranges *info;
+ struct iommu_iova_range *iova_ranges;
+ int ret, sz, fd = container->be->fd;
+
+ info = g_malloc0(sizeof(*info));
+ info->size = sizeof(*info);
+ info->ioas_id = ioas_id;
+
+ ret = ioctl(fd, IOMMU_IOAS_IOVA_RANGES, info);
+ if (ret && errno != EMSGSIZE) {
+ goto error;
+ }
+
+ sz = info->num_iovas * sizeof(struct iommu_iova_range);
+ info = g_realloc(info, sizeof(*info) + sz);
+ info->allowed_iovas = (uintptr_t)(info + 1);
+
+ ret = ioctl(fd, IOMMU_IOAS_IOVA_RANGES, info);
+ if (ret) {
+ goto error;
+ }
+
+ iova_ranges = (struct iommu_iova_range *)(uintptr_t)info->allowed_iovas;
+
+ for (int i = 0; i < info->num_iovas; i++) {
+ Range *range = g_new(Range, 1);
+
+ range_set_bounds(range, iova_ranges[i].start, iova_ranges[i].last);
+ bcontainer->iova_ranges =
+ range_list_insert(bcontainer->iova_ranges, range);
+ }
+ bcontainer->pgsizes = info->out_iova_alignment;
+
+ g_free(info);
+ return 0;
+
+error:
+ ret = -errno;
+ g_free(info);
+ error_setg_errno(errp, errno, "Cannot get IOVA ranges");
+ return ret;
+}
+
static int iommufd_cdev_attach(const char *name, VFIODevice *vbasedev,
AddressSpace *as, Error **errp)
{
@@ -335,7 +382,14 @@ static int iommufd_cdev_attach(const char *name, VFIODevice *vbasedev,
goto err_discard_disable;
}
- bcontainer->pgsizes = qemu_real_host_page_size();
+ ret = iommufd_cdev_get_info_iova_range(container, ioas_id, &err);
+ if (ret) {
+ error_append_hint(&err,
+ "Fallback to default 64bit IOVA range and 4K page size\n");
+ warn_report_err(err);
+ err = NULL;
+ bcontainer->pgsizes = qemu_real_host_page_size();
+ }
bcontainer->listener = vfio_memory_listener;
memory_listener_register(&bcontainer->listener, bcontainer->space->as);
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 26/47] vfio/pci: Extract out a helper vfio_pci_get_pci_hot_reset_info
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (24 preceding siblings ...)
2023-12-19 18:56 ` [PULL 25/47] vfio/iommufd: Add support for iova_ranges and pgsizes Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 27/47] vfio/pci: Introduce a vfio pci hot reset interface Cédric Le Goater
` (21 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
This helper will be used by both legacy and iommufd backends.
No functional changes intended.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/pci.h | 3 +++
hw/vfio/pci.c | 54 +++++++++++++++++++++++++++++++++++----------------
2 files changed, 40 insertions(+), 17 deletions(-)
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index fba8737ab2cb23118c0819f600379773d718ed18..1006061afb0cc13d78d804afa54e79cfb0f5c972 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -218,6 +218,9 @@ void vfio_probe_igd_bar4_quirk(VFIOPCIDevice *vdev, int nr);
extern const PropertyInfo qdev_prop_nv_gpudirect_clique;
+int vfio_pci_get_pci_hot_reset_info(VFIOPCIDevice *vdev,
+ struct vfio_pci_hot_reset_info **info_p);
+
int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp);
int vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev,
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index c62c02f7b692c98bba1b931ebb1a4254a7f56061..eb55e8ae88e1686d31f821df73789212bde46884 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2445,22 +2445,13 @@ static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
return (strcmp(tmp, name) == 0);
}
-static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
+int vfio_pci_get_pci_hot_reset_info(VFIOPCIDevice *vdev,
+ struct vfio_pci_hot_reset_info **info_p)
{
- VFIOGroup *group;
struct vfio_pci_hot_reset_info *info;
- struct vfio_pci_dependent_device *devices;
- struct vfio_pci_hot_reset *reset;
- int32_t *fds;
- int ret, i, count;
- bool multi = false;
+ int ret, count;
- trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
-
- if (!single) {
- vfio_pci_pre_reset(vdev);
- }
- vdev->vbasedev.needs_reset = false;
+ assert(info_p && !*info_p);
info = g_malloc0(sizeof(*info));
info->argsz = sizeof(*info);
@@ -2468,24 +2459,53 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
if (ret && errno != ENOSPC) {
ret = -errno;
+ g_free(info);
if (!vdev->has_pm_reset) {
error_report("vfio: Cannot reset device %s, "
"no available reset mechanism.", vdev->vbasedev.name);
}
- goto out_single;
+ return ret;
}
count = info->count;
- info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
- info->argsz = sizeof(*info) + (count * sizeof(*devices));
- devices = &info->devices[0];
+ info = g_realloc(info, sizeof(*info) + (count * sizeof(info->devices[0])));
+ info->argsz = sizeof(*info) + (count * sizeof(info->devices[0]));
ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
if (ret) {
ret = -errno;
+ g_free(info);
error_report("vfio: hot reset info failed: %m");
+ return ret;
+ }
+
+ *info_p = info;
+ return 0;
+}
+
+static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
+{
+ VFIOGroup *group;
+ struct vfio_pci_hot_reset_info *info = NULL;
+ struct vfio_pci_dependent_device *devices;
+ struct vfio_pci_hot_reset *reset;
+ int32_t *fds;
+ int ret, i, count;
+ bool multi = false;
+
+ trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
+
+ if (!single) {
+ vfio_pci_pre_reset(vdev);
+ }
+ vdev->vbasedev.needs_reset = false;
+
+ ret = vfio_pci_get_pci_hot_reset_info(vdev, &info);
+
+ if (ret) {
goto out_single;
}
+ devices = &info->devices[0];
trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 27/47] vfio/pci: Introduce a vfio pci hot reset interface
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (25 preceding siblings ...)
2023-12-19 18:56 ` [PULL 26/47] vfio/pci: Extract out a helper vfio_pci_get_pci_hot_reset_info Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 28/47] vfio/iommufd: Enable pci hot reset through iommufd cdev interface Cédric Le Goater
` (20 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Legacy vfio pci and iommufd cdev have different process to hot reset
vfio device, expand current code to abstract out pci_hot_reset callback
for legacy vfio, this same interface will also be used by iommufd
cdev vfio device.
Rename vfio_pci_hot_reset to vfio_legacy_pci_hot_reset and move it
into container.c.
vfio_pci_[pre/post]_reset and vfio_pci_host_match are exported so
they could be called in legacy and iommufd pci_hot_reset callback.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/pci.h | 3 +
include/hw/vfio/vfio-container-base.h | 3 +
hw/vfio/container.c | 170 ++++++++++++++++++++++++++
hw/vfio/pci.c | 168 +------------------------
4 files changed, 182 insertions(+), 162 deletions(-)
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index 1006061afb0cc13d78d804afa54e79cfb0f5c972..6e64a2654e690af11b72710530a41135b726e96f 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -218,6 +218,9 @@ void vfio_probe_igd_bar4_quirk(VFIOPCIDevice *vdev, int nr);
extern const PropertyInfo qdev_prop_nv_gpudirect_clique;
+void vfio_pci_pre_reset(VFIOPCIDevice *vdev);
+void vfio_pci_post_reset(VFIOPCIDevice *vdev);
+bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name);
int vfio_pci_get_pci_hot_reset_info(VFIOPCIDevice *vdev,
struct vfio_pci_hot_reset_info **info_p);
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 4b6f017c6f15de1bb1fb42bcae62a1f062895bdd..45bb19c7670e76ae142dbcd2bfc96f6e74ae160b 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -106,6 +106,9 @@ struct VFIOIOMMUOps {
int (*set_dirty_page_tracking)(VFIOContainerBase *bcontainer, bool start);
int (*query_dirty_bitmap)(VFIOContainerBase *bcontainer, VFIOBitmap *vbmap,
hwaddr iova, hwaddr size);
+ /* PCI specific */
+ int (*pci_hot_reset)(VFIODevice *vbasedev, bool single);
+
/* SPAPR specific */
int (*add_window)(VFIOContainerBase *bcontainer,
MemoryRegionSection *section,
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index ed2d721b2bc6d175ecd3fb9498cbdc0d7ba6769d..1dbf9b9a17e070ded853931757f5852d1e4568aa 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -33,6 +33,7 @@
#include "trace.h"
#include "qapi/error.h"
#include "migration/migration.h"
+#include "pci.h"
VFIOGroupList vfio_group_list =
QLIST_HEAD_INITIALIZER(vfio_group_list);
@@ -922,6 +923,174 @@ static void vfio_legacy_detach_device(VFIODevice *vbasedev)
vfio_put_group(group);
}
+static int vfio_legacy_pci_hot_reset(VFIODevice *vbasedev, bool single)
+{
+ VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
+ VFIOGroup *group;
+ struct vfio_pci_hot_reset_info *info = NULL;
+ struct vfio_pci_dependent_device *devices;
+ struct vfio_pci_hot_reset *reset;
+ int32_t *fds;
+ int ret, i, count;
+ bool multi = false;
+
+ trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
+
+ if (!single) {
+ vfio_pci_pre_reset(vdev);
+ }
+ vdev->vbasedev.needs_reset = false;
+
+ ret = vfio_pci_get_pci_hot_reset_info(vdev, &info);
+
+ if (ret) {
+ goto out_single;
+ }
+ devices = &info->devices[0];
+
+ trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
+
+ /* Verify that we have all the groups required */
+ for (i = 0; i < info->count; i++) {
+ PCIHostDeviceAddress host;
+ VFIOPCIDevice *tmp;
+ VFIODevice *vbasedev_iter;
+
+ host.domain = devices[i].segment;
+ host.bus = devices[i].bus;
+ host.slot = PCI_SLOT(devices[i].devfn);
+ host.function = PCI_FUNC(devices[i].devfn);
+
+ trace_vfio_pci_hot_reset_dep_devices(host.domain,
+ host.bus, host.slot, host.function, devices[i].group_id);
+
+ if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
+ continue;
+ }
+
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ if (group->groupid == devices[i].group_id) {
+ break;
+ }
+ }
+
+ if (!group) {
+ if (!vdev->has_pm_reset) {
+ error_report("vfio: Cannot reset device %s, "
+ "depends on group %d which is not owned.",
+ vdev->vbasedev.name, devices[i].group_id);
+ }
+ ret = -EPERM;
+ goto out;
+ }
+
+ /* Prep dependent devices for reset and clear our marker. */
+ QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
+ if (!vbasedev_iter->dev->realized ||
+ vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
+ continue;
+ }
+ tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
+ if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
+ if (single) {
+ ret = -EINVAL;
+ goto out_single;
+ }
+ vfio_pci_pre_reset(tmp);
+ tmp->vbasedev.needs_reset = false;
+ multi = true;
+ break;
+ }
+ }
+ }
+
+ if (!single && !multi) {
+ ret = -EINVAL;
+ goto out_single;
+ }
+
+ /* Determine how many group fds need to be passed */
+ count = 0;
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ for (i = 0; i < info->count; i++) {
+ if (group->groupid == devices[i].group_id) {
+ count++;
+ break;
+ }
+ }
+ }
+
+ reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
+ reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
+ fds = &reset->group_fds[0];
+
+ /* Fill in group fds */
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ for (i = 0; i < info->count; i++) {
+ if (group->groupid == devices[i].group_id) {
+ fds[reset->count++] = group->fd;
+ break;
+ }
+ }
+ }
+
+ /* Bus reset! */
+ ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
+ g_free(reset);
+ if (ret) {
+ ret = -errno;
+ }
+
+ trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
+ ret ? strerror(errno) : "Success");
+
+out:
+ /* Re-enable INTx on affected devices */
+ for (i = 0; i < info->count; i++) {
+ PCIHostDeviceAddress host;
+ VFIOPCIDevice *tmp;
+ VFIODevice *vbasedev_iter;
+
+ host.domain = devices[i].segment;
+ host.bus = devices[i].bus;
+ host.slot = PCI_SLOT(devices[i].devfn);
+ host.function = PCI_FUNC(devices[i].devfn);
+
+ if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
+ continue;
+ }
+
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ if (group->groupid == devices[i].group_id) {
+ break;
+ }
+ }
+
+ if (!group) {
+ break;
+ }
+
+ QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
+ if (!vbasedev_iter->dev->realized ||
+ vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
+ continue;
+ }
+ tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
+ if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
+ vfio_pci_post_reset(tmp);
+ break;
+ }
+ }
+ }
+out_single:
+ if (!single) {
+ vfio_pci_post_reset(vdev);
+ }
+ g_free(info);
+
+ return ret;
+}
+
const VFIOIOMMUOps vfio_legacy_ops = {
.dma_map = vfio_legacy_dma_map,
.dma_unmap = vfio_legacy_dma_unmap,
@@ -929,4 +1098,5 @@ const VFIOIOMMUOps vfio_legacy_ops = {
.detach_device = vfio_legacy_detach_device,
.set_dirty_page_tracking = vfio_legacy_set_dirty_page_tracking,
.query_dirty_bitmap = vfio_legacy_query_dirty_bitmap,
+ .pci_hot_reset = vfio_legacy_pci_hot_reset,
};
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index eb55e8ae88e1686d31f821df73789212bde46884..d00c3472c7086dde71ee883537c8d14a1d4db830 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2374,7 +2374,7 @@ static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp)
return 0;
}
-static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
+void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
{
PCIDevice *pdev = &vdev->pdev;
uint16_t cmd;
@@ -2411,7 +2411,7 @@ static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
}
-static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
+void vfio_pci_post_reset(VFIOPCIDevice *vdev)
{
Error *err = NULL;
int nr;
@@ -2435,7 +2435,7 @@ static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
vfio_quirk_reset(vdev);
}
-static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
+bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
{
char tmp[13];
@@ -2485,166 +2485,10 @@ int vfio_pci_get_pci_hot_reset_info(VFIOPCIDevice *vdev,
static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
{
- VFIOGroup *group;
- struct vfio_pci_hot_reset_info *info = NULL;
- struct vfio_pci_dependent_device *devices;
- struct vfio_pci_hot_reset *reset;
- int32_t *fds;
- int ret, i, count;
- bool multi = false;
-
- trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
-
- if (!single) {
- vfio_pci_pre_reset(vdev);
- }
- vdev->vbasedev.needs_reset = false;
-
- ret = vfio_pci_get_pci_hot_reset_info(vdev, &info);
-
- if (ret) {
- goto out_single;
- }
- devices = &info->devices[0];
-
- trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
-
- /* Verify that we have all the groups required */
- for (i = 0; i < info->count; i++) {
- PCIHostDeviceAddress host;
- VFIOPCIDevice *tmp;
- VFIODevice *vbasedev_iter;
-
- host.domain = devices[i].segment;
- host.bus = devices[i].bus;
- host.slot = PCI_SLOT(devices[i].devfn);
- host.function = PCI_FUNC(devices[i].devfn);
-
- trace_vfio_pci_hot_reset_dep_devices(host.domain,
- host.bus, host.slot, host.function, devices[i].group_id);
-
- if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
- continue;
- }
-
- QLIST_FOREACH(group, &vfio_group_list, next) {
- if (group->groupid == devices[i].group_id) {
- break;
- }
- }
-
- if (!group) {
- if (!vdev->has_pm_reset) {
- error_report("vfio: Cannot reset device %s, "
- "depends on group %d which is not owned.",
- vdev->vbasedev.name, devices[i].group_id);
- }
- ret = -EPERM;
- goto out;
- }
-
- /* Prep dependent devices for reset and clear our marker. */
- QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
- if (!vbasedev_iter->dev->realized ||
- vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
- continue;
- }
- tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
- if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
- if (single) {
- ret = -EINVAL;
- goto out_single;
- }
- vfio_pci_pre_reset(tmp);
- tmp->vbasedev.needs_reset = false;
- multi = true;
- break;
- }
- }
- }
-
- if (!single && !multi) {
- ret = -EINVAL;
- goto out_single;
- }
-
- /* Determine how many group fds need to be passed */
- count = 0;
- QLIST_FOREACH(group, &vfio_group_list, next) {
- for (i = 0; i < info->count; i++) {
- if (group->groupid == devices[i].group_id) {
- count++;
- break;
- }
- }
- }
-
- reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
- reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
- fds = &reset->group_fds[0];
-
- /* Fill in group fds */
- QLIST_FOREACH(group, &vfio_group_list, next) {
- for (i = 0; i < info->count; i++) {
- if (group->groupid == devices[i].group_id) {
- fds[reset->count++] = group->fd;
- break;
- }
- }
- }
-
- /* Bus reset! */
- ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
- g_free(reset);
-
- trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
- ret ? strerror(errno) : "Success");
-
-out:
- /* Re-enable INTx on affected devices */
- for (i = 0; i < info->count; i++) {
- PCIHostDeviceAddress host;
- VFIOPCIDevice *tmp;
- VFIODevice *vbasedev_iter;
-
- host.domain = devices[i].segment;
- host.bus = devices[i].bus;
- host.slot = PCI_SLOT(devices[i].devfn);
- host.function = PCI_FUNC(devices[i].devfn);
-
- if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
- continue;
- }
-
- QLIST_FOREACH(group, &vfio_group_list, next) {
- if (group->groupid == devices[i].group_id) {
- break;
- }
- }
-
- if (!group) {
- break;
- }
-
- QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
- if (!vbasedev_iter->dev->realized ||
- vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
- continue;
- }
- tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
- if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
- vfio_pci_post_reset(tmp);
- break;
- }
- }
- }
-out_single:
- if (!single) {
- vfio_pci_post_reset(vdev);
- }
- g_free(info);
+ VFIODevice *vbasedev = &vdev->vbasedev;
+ const VFIOIOMMUOps *ops = vbasedev->bcontainer->ops;
- return ret;
+ return ops->pci_hot_reset(vbasedev, single);
}
/*
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 28/47] vfio/iommufd: Enable pci hot reset through iommufd cdev interface
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (26 preceding siblings ...)
2023-12-19 18:56 ` [PULL 27/47] vfio/pci: Introduce a vfio pci hot reset interface Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 29/47] vfio/pci: Allow the selection of a given iommu backend Cédric Le Goater
` (19 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Nicolin Chen,
Cédric Le Goater
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Implement the newly introduced pci_hot_reset callback named
iommufd_cdev_pci_hot_reset to do iommufd specific check and
reset operation.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/iommufd.c | 150 +++++++++++++++++++++++++++++++++++++++++++
hw/vfio/trace-events | 1 +
2 files changed, 151 insertions(+)
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
index 01b448e840581e0dd6d3df1897169665f79dcbe3..6e53e013ef57b6d7e3be58e61356fbabacbe8bf3 100644
--- a/hw/vfio/iommufd.c
+++ b/hw/vfio/iommufd.c
@@ -24,6 +24,7 @@
#include "sysemu/reset.h"
#include "qemu/cutils.h"
#include "qemu/chardev_open.h"
+#include "pci.h"
static int iommufd_cdev_map(VFIOContainerBase *bcontainer, hwaddr iova,
ram_addr_t size, void *vaddr, bool readonly)
@@ -468,9 +469,158 @@ static void iommufd_cdev_detach(VFIODevice *vbasedev)
close(vbasedev->fd);
}
+static VFIODevice *iommufd_cdev_pci_find_by_devid(__u32 devid)
+{
+ VFIODevice *vbasedev_iter;
+
+ QLIST_FOREACH(vbasedev_iter, &vfio_device_list, global_next) {
+ if (vbasedev_iter->bcontainer->ops != &vfio_iommufd_ops) {
+ continue;
+ }
+ if (devid == vbasedev_iter->devid) {
+ return vbasedev_iter;
+ }
+ }
+ return NULL;
+}
+
+static VFIOPCIDevice *
+iommufd_cdev_dep_get_realized_vpdev(struct vfio_pci_dependent_device *dep_dev,
+ VFIODevice *reset_dev)
+{
+ VFIODevice *vbasedev_tmp;
+
+ if (dep_dev->devid == reset_dev->devid ||
+ dep_dev->devid == VFIO_PCI_DEVID_OWNED) {
+ return NULL;
+ }
+
+ vbasedev_tmp = iommufd_cdev_pci_find_by_devid(dep_dev->devid);
+ if (!vbasedev_tmp || !vbasedev_tmp->dev->realized ||
+ vbasedev_tmp->type != VFIO_DEVICE_TYPE_PCI) {
+ return NULL;
+ }
+
+ return container_of(vbasedev_tmp, VFIOPCIDevice, vbasedev);
+}
+
+static int iommufd_cdev_pci_hot_reset(VFIODevice *vbasedev, bool single)
+{
+ VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
+ struct vfio_pci_hot_reset_info *info = NULL;
+ struct vfio_pci_dependent_device *devices;
+ struct vfio_pci_hot_reset *reset;
+ int ret, i;
+ bool multi = false;
+
+ trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
+
+ if (!single) {
+ vfio_pci_pre_reset(vdev);
+ }
+ vdev->vbasedev.needs_reset = false;
+
+ ret = vfio_pci_get_pci_hot_reset_info(vdev, &info);
+
+ if (ret) {
+ goto out_single;
+ }
+
+ assert(info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID);
+
+ devices = &info->devices[0];
+
+ if (!(info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED)) {
+ if (!vdev->has_pm_reset) {
+ for (i = 0; i < info->count; i++) {
+ if (devices[i].devid == VFIO_PCI_DEVID_NOT_OWNED) {
+ error_report("vfio: Cannot reset device %s, "
+ "depends on device %04x:%02x:%02x.%x "
+ "which is not owned.",
+ vdev->vbasedev.name, devices[i].segment,
+ devices[i].bus, PCI_SLOT(devices[i].devfn),
+ PCI_FUNC(devices[i].devfn));
+ }
+ }
+ }
+ ret = -EPERM;
+ goto out_single;
+ }
+
+ trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
+
+ for (i = 0; i < info->count; i++) {
+ VFIOPCIDevice *tmp;
+
+ trace_iommufd_cdev_pci_hot_reset_dep_devices(devices[i].segment,
+ devices[i].bus,
+ PCI_SLOT(devices[i].devfn),
+ PCI_FUNC(devices[i].devfn),
+ devices[i].devid);
+
+ /*
+ * If a VFIO cdev device is resettable, all the dependent devices
+ * are either bound to same iommufd or within same iommu_groups as
+ * one of the iommufd bound devices.
+ */
+ assert(devices[i].devid != VFIO_PCI_DEVID_NOT_OWNED);
+
+ tmp = iommufd_cdev_dep_get_realized_vpdev(&devices[i], &vdev->vbasedev);
+ if (!tmp) {
+ continue;
+ }
+
+ if (single) {
+ ret = -EINVAL;
+ goto out_single;
+ }
+ vfio_pci_pre_reset(tmp);
+ tmp->vbasedev.needs_reset = false;
+ multi = true;
+ }
+
+ if (!single && !multi) {
+ ret = -EINVAL;
+ goto out_single;
+ }
+
+ /* Use zero length array for hot reset with iommufd backend */
+ reset = g_malloc0(sizeof(*reset));
+ reset->argsz = sizeof(*reset);
+
+ /* Bus reset! */
+ ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
+ g_free(reset);
+ if (ret) {
+ ret = -errno;
+ }
+
+ trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
+ ret ? strerror(errno) : "Success");
+
+ /* Re-enable INTx on affected devices */
+ for (i = 0; i < info->count; i++) {
+ VFIOPCIDevice *tmp;
+
+ tmp = iommufd_cdev_dep_get_realized_vpdev(&devices[i], &vdev->vbasedev);
+ if (!tmp) {
+ continue;
+ }
+ vfio_pci_post_reset(tmp);
+ }
+out_single:
+ if (!single) {
+ vfio_pci_post_reset(vdev);
+ }
+ g_free(info);
+
+ return ret;
+}
+
const VFIOIOMMUOps vfio_iommufd_ops = {
.dma_map = iommufd_cdev_map,
.dma_unmap = iommufd_cdev_unmap,
.attach_device = iommufd_cdev_attach,
.detach_device = iommufd_cdev_detach,
+ .pci_hot_reset = iommufd_cdev_pci_hot_reset,
};
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index 3340c93af0e9d366e04e6fe573f1010ab2a861a6..8fdde5445697789edeb4c6383566c1b417cc1595 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -174,3 +174,4 @@ iommufd_cdev_detach_ioas_hwpt(int iommufd, const char *name) " [iommufd=%d] Succ
iommufd_cdev_fail_attach_existing_container(const char *msg) " %s"
iommufd_cdev_alloc_ioas(int iommufd, int ioas_id) " [iommufd=%d] new IOMMUFD container with ioasid=%d"
iommufd_cdev_device_info(char *name, int devfd, int num_irqs, int num_regions, int flags) " %s (%d) num_irqs=%d num_regions=%d flags=%d"
+iommufd_cdev_pci_hot_reset_dep_devices(int domain, int bus, int slot, int function, int dev_id) "\t%04x:%02x:%02x.%x devid %d"
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 29/47] vfio/pci: Allow the selection of a given iommu backend
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (27 preceding siblings ...)
2023-12-19 18:56 ` [PULL 28/47] vfio/iommufd: Enable pci hot reset through iommufd cdev interface Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 30/47] vfio/pci: Make vfio cdev pre-openable by passing a file handle Cédric Le Goater
` (18 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Yi Liu, Cédric Le Goater,
Nicolin Chen
From: Eric Auger <eric.auger@redhat.com>
Now we support two types of iommu backends, let's add the capability
to select one of them. This depends on whether an iommufd object has
been linked with the vfio-pci device:
If the user wants to use the legacy backend, it shall not
link the vfio-pci device with any iommufd object:
-device vfio-pci,host=0000:02:00.0
This is called the legacy mode/backend.
If the user wants to use the iommufd backend (/dev/iommu) it
shall pass an iommufd object id in the vfio-pci device options:
-object iommufd,id=iommufd0
-device vfio-pci,host=0000:02:00.0,iommufd=iommufd0
Suggested-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/pci.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index d00c3472c7086dde71ee883537c8d14a1d4db830..c5984b0598d26e7dd31fdb12dccac2e3ca81adf3 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -19,6 +19,7 @@
*/
#include "qemu/osdep.h"
+#include CONFIG_DEVICES /* CONFIG_IOMMUFD */
#include <linux/vfio.h>
#include <sys/ioctl.h>
@@ -42,6 +43,7 @@
#include "qapi/error.h"
#include "migration/blocker.h"
#include "migration/qemu-file.h"
+#include "sysemu/iommufd.h"
#define TYPE_VFIO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
@@ -3386,6 +3388,10 @@ static Property vfio_pci_dev_properties[] = {
* DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
* DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
*/
+#ifdef CONFIG_IOMMUFD
+ DEFINE_PROP_LINK("iommufd", VFIOPCIDevice, vbasedev.iommufd,
+ TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
+#endif
DEFINE_PROP_END_OF_LIST(),
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 30/47] vfio/pci: Make vfio cdev pre-openable by passing a file handle
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (28 preceding siblings ...)
2023-12-19 18:56 ` [PULL 29/47] vfio/pci: Allow the selection of a given iommu backend Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 31/47] vfio/platform: Allow the selection of a given iommu backend Cédric Le Goater
` (17 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
This gives management tools like libvirt a chance to open the vfio
cdev with privilege and pass FD to qemu. This way qemu never needs
to have privilege to open a VFIO or iommu cdev node.
Together with the earlier support of pre-opening /dev/iommu device,
now we have full support of passing a vfio device to unprivileged
qemu by management tool. This mode is no more considered for the
legacy backend. So let's remove the "TODO" comment.
Add helper functions vfio_device_set_fd() and vfio_device_get_name()
to set fd and get device name, they will also be used by other vfio
devices.
There is no easy way to check if a device is mdev with FD passing,
so fail the x-balloon-allowed check unconditionally in this case.
There is also no easy way to get BDF as name with FD passing, so
we fake a name by VFIO_FD[fd].
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 4 ++++
hw/vfio/helpers.c | 43 +++++++++++++++++++++++++++++++++++
hw/vfio/iommufd.c | 12 ++++++----
hw/vfio/pci.c | 28 +++++++++++++----------
4 files changed, 71 insertions(+), 16 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 3dac5c167efa1fc6afefc103539ce5e01cceb602..697bf24a350d5880e59811322d9037575a90d9a2 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -251,4 +251,8 @@ int vfio_devices_query_dirty_bitmap(VFIOContainerBase *bcontainer,
hwaddr size);
int vfio_get_dirty_bitmap(VFIOContainerBase *bcontainer, uint64_t iova,
uint64_t size, ram_addr_t ram_addr);
+
+/* Returns 0 on success, or a negative errno. */
+int vfio_device_get_name(VFIODevice *vbasedev, Error **errp);
+void vfio_device_set_fd(VFIODevice *vbasedev, const char *str, Error **errp);
#endif /* HW_VFIO_VFIO_COMMON_H */
diff --git a/hw/vfio/helpers.c b/hw/vfio/helpers.c
index 168847e7c51ef35afbea276745c1aa7e6cd94ce0..3592c3d54ecd68d4bfd23d4c3402a393fb1f2eb0 100644
--- a/hw/vfio/helpers.c
+++ b/hw/vfio/helpers.c
@@ -27,6 +27,7 @@
#include "trace.h"
#include "qapi/error.h"
#include "qemu/error-report.h"
+#include "monitor/monitor.h"
/*
* Common VFIO interrupt disable
@@ -609,3 +610,45 @@ bool vfio_has_region_cap(VFIODevice *vbasedev, int region, uint16_t cap_type)
return ret;
}
+
+int vfio_device_get_name(VFIODevice *vbasedev, Error **errp)
+{
+ struct stat st;
+
+ if (vbasedev->fd < 0) {
+ if (stat(vbasedev->sysfsdev, &st) < 0) {
+ error_setg_errno(errp, errno, "no such host device");
+ error_prepend(errp, VFIO_MSG_PREFIX, vbasedev->sysfsdev);
+ return -errno;
+ }
+ /* User may specify a name, e.g: VFIO platform device */
+ if (!vbasedev->name) {
+ vbasedev->name = g_path_get_basename(vbasedev->sysfsdev);
+ }
+ } else {
+ if (!vbasedev->iommufd) {
+ error_setg(errp, "Use FD passing only with iommufd backend");
+ return -EINVAL;
+ }
+ /*
+ * Give a name with fd so any function printing out vbasedev->name
+ * will not break.
+ */
+ if (!vbasedev->name) {
+ vbasedev->name = g_strdup_printf("VFIO_FD%d", vbasedev->fd);
+ }
+ }
+
+ return 0;
+}
+
+void vfio_device_set_fd(VFIODevice *vbasedev, const char *str, Error **errp)
+{
+ int fd = monitor_fd_param(monitor_cur(), str, errp);
+
+ if (fd < 0) {
+ error_prepend(errp, "Could not parse remote object fd %s:", str);
+ return;
+ }
+ vbasedev->fd = fd;
+}
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
index 6e53e013ef57b6d7e3be58e61356fbabacbe8bf3..5accd2648444defcd698bd6d0cefe11d255b4cfb 100644
--- a/hw/vfio/iommufd.c
+++ b/hw/vfio/iommufd.c
@@ -320,11 +320,15 @@ static int iommufd_cdev_attach(const char *name, VFIODevice *vbasedev,
uint32_t ioas_id;
Error *err = NULL;
- devfd = iommufd_cdev_getfd(vbasedev->sysfsdev, errp);
- if (devfd < 0) {
- return devfd;
+ if (vbasedev->fd < 0) {
+ devfd = iommufd_cdev_getfd(vbasedev->sysfsdev, errp);
+ if (devfd < 0) {
+ return devfd;
+ }
+ vbasedev->fd = devfd;
+ } else {
+ devfd = vbasedev->fd;
}
- vbasedev->fd = devfd;
ret = iommufd_cdev_connect_and_bind(vbasedev, errp);
if (ret) {
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index c5984b0598d26e7dd31fdb12dccac2e3ca81adf3..445d58c8e59b0a00a8336d092c2b8cec7e39b396 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2944,17 +2944,19 @@ static void vfio_realize(PCIDevice *pdev, Error **errp)
VFIODevice *vbasedev = &vdev->vbasedev;
char *tmp, *subsys;
Error *err = NULL;
- struct stat st;
int i, ret;
bool is_mdev;
char uuid[UUID_STR_LEN];
char *name;
- if (!vbasedev->sysfsdev) {
+ if (vbasedev->fd < 0 && !vbasedev->sysfsdev) {
if (!(~vdev->host.domain || ~vdev->host.bus ||
~vdev->host.slot || ~vdev->host.function)) {
error_setg(errp, "No provided host device");
error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F "
+#ifdef CONFIG_IOMMUFD
+ "or -device vfio-pci,fd=DEVICE_FD "
+#endif
"or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
return;
}
@@ -2964,13 +2966,9 @@ static void vfio_realize(PCIDevice *pdev, Error **errp)
vdev->host.slot, vdev->host.function);
}
- if (stat(vbasedev->sysfsdev, &st) < 0) {
- error_setg_errno(errp, errno, "no such host device");
- error_prepend(errp, VFIO_MSG_PREFIX, vbasedev->sysfsdev);
+ if (vfio_device_get_name(vbasedev, errp) < 0) {
return;
}
-
- vbasedev->name = g_path_get_basename(vbasedev->sysfsdev);
vbasedev->ops = &vfio_pci_ops;
vbasedev->type = VFIO_DEVICE_TYPE_PCI;
vbasedev->dev = DEVICE(vdev);
@@ -3330,6 +3328,7 @@ static void vfio_instance_init(Object *obj)
vdev->host.bus = ~0U;
vdev->host.slot = ~0U;
vdev->host.function = ~0U;
+ vdev->vbasedev.fd = -1;
vdev->nv_gpudirect_clique = 0xFF;
@@ -3383,11 +3382,6 @@ static Property vfio_pci_dev_properties[] = {
qdev_prop_nv_gpudirect_clique, uint8_t),
DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice, msix_relo,
OFF_AUTOPCIBAR_OFF),
- /*
- * TODO - support passed fds... is this necessary?
- * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
- * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
- */
#ifdef CONFIG_IOMMUFD
DEFINE_PROP_LINK("iommufd", VFIOPCIDevice, vbasedev.iommufd,
TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
@@ -3395,6 +3389,13 @@ static Property vfio_pci_dev_properties[] = {
DEFINE_PROP_END_OF_LIST(),
};
+#ifdef CONFIG_IOMMUFD
+static void vfio_pci_set_fd(Object *obj, const char *str, Error **errp)
+{
+ vfio_device_set_fd(&VFIO_PCI(obj)->vbasedev, str, errp);
+}
+#endif
+
static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
@@ -3402,6 +3403,9 @@ static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
dc->reset = vfio_pci_reset;
device_class_set_props(dc, vfio_pci_dev_properties);
+#ifdef CONFIG_IOMMUFD
+ object_class_property_add_str(klass, "fd", NULL, vfio_pci_set_fd);
+#endif
dc->desc = "VFIO-based PCI device assignment";
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
pdc->realize = vfio_realize;
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 31/47] vfio/platform: Allow the selection of a given iommu backend
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (29 preceding siblings ...)
2023-12-19 18:56 ` [PULL 30/47] vfio/pci: Make vfio cdev pre-openable by passing a file handle Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 32/47] vfio/platform: Make vfio cdev pre-openable by passing a file handle Cédric Le Goater
` (16 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Now we support two types of iommu backends, let's add the capability
to select one of them. This depends on whether an iommufd object has
been linked with the vfio-platform device:
If the user wants to use the legacy backend, it shall not
link the vfio-platform device with any iommufd object:
-device vfio-platform,host=XXX
This is called the legacy mode/backend.
If the user wants to use the iommufd backend (/dev/iommu) it
shall pass an iommufd object id in the vfio-platform device options:
-object iommufd,id=iommufd0
-device vfio-platform,host=XXX,iommufd=iommufd0
Suggested-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/platform.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/hw/vfio/platform.c b/hw/vfio/platform.c
index 8e3d4ac45824ec69afb523f8f0e668327122cd02..98ae4bc655b401250bd4cb943e4efec97f486a5f 100644
--- a/hw/vfio/platform.c
+++ b/hw/vfio/platform.c
@@ -15,11 +15,13 @@
*/
#include "qemu/osdep.h"
+#include CONFIG_DEVICES /* CONFIG_IOMMUFD */
#include "qapi/error.h"
#include <sys/ioctl.h>
#include <linux/vfio.h>
#include "hw/vfio/vfio-platform.h"
+#include "sysemu/iommufd.h"
#include "migration/vmstate.h"
#include "qemu/error-report.h"
#include "qemu/lockable.h"
@@ -649,6 +651,10 @@ static Property vfio_platform_dev_properties[] = {
DEFINE_PROP_UINT32("mmap-timeout-ms", VFIOPlatformDevice,
mmap_timeout, 1100),
DEFINE_PROP_BOOL("x-irqfd", VFIOPlatformDevice, irqfd_allowed, true),
+#ifdef CONFIG_IOMMUFD
+ DEFINE_PROP_LINK("iommufd", VFIOPlatformDevice, vbasedev.iommufd,
+ TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
+#endif
DEFINE_PROP_END_OF_LIST(),
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 32/47] vfio/platform: Make vfio cdev pre-openable by passing a file handle
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (30 preceding siblings ...)
2023-12-19 18:56 ` [PULL 31/47] vfio/platform: Allow the selection of a given iommu backend Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 33/47] vfio/ap: Allow the selection of a given iommu backend Cédric Le Goater
` (15 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
This gives management tools like libvirt a chance to open the vfio
cdev with privilege and pass FD to qemu. This way qemu never needs
to have privilege to open a VFIO or iommu cdev node.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/platform.c | 32 ++++++++++++++++++++++++--------
1 file changed, 24 insertions(+), 8 deletions(-)
diff --git a/hw/vfio/platform.c b/hw/vfio/platform.c
index 98ae4bc655b401250bd4cb943e4efec97f486a5f..a97d9c6234ccac0ba26b44d3e2290616eda5debd 100644
--- a/hw/vfio/platform.c
+++ b/hw/vfio/platform.c
@@ -531,14 +531,13 @@ static VFIODeviceOps vfio_platform_ops = {
*/
static int vfio_base_device_init(VFIODevice *vbasedev, Error **errp)
{
- struct stat st;
int ret;
- /* @sysfsdev takes precedence over @host */
- if (vbasedev->sysfsdev) {
+ /* @fd takes precedence over @sysfsdev which takes precedence over @host */
+ if (vbasedev->fd < 0 && vbasedev->sysfsdev) {
g_free(vbasedev->name);
vbasedev->name = g_path_get_basename(vbasedev->sysfsdev);
- } else {
+ } else if (vbasedev->fd < 0) {
if (!vbasedev->name || strchr(vbasedev->name, '/')) {
error_setg(errp, "wrong host device name");
return -EINVAL;
@@ -548,10 +547,9 @@ static int vfio_base_device_init(VFIODevice *vbasedev, Error **errp)
vbasedev->name);
}
- if (stat(vbasedev->sysfsdev, &st) < 0) {
- error_setg_errno(errp, errno,
- "failed to get the sysfs host device file status");
- return -errno;
+ ret = vfio_device_get_name(vbasedev, errp);
+ if (ret) {
+ return ret;
}
ret = vfio_attach_device(vbasedev->name, vbasedev,
@@ -658,6 +656,20 @@ static Property vfio_platform_dev_properties[] = {
DEFINE_PROP_END_OF_LIST(),
};
+static void vfio_platform_instance_init(Object *obj)
+{
+ VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(obj);
+
+ vdev->vbasedev.fd = -1;
+}
+
+#ifdef CONFIG_IOMMUFD
+static void vfio_platform_set_fd(Object *obj, const char *str, Error **errp)
+{
+ vfio_device_set_fd(&VFIO_PLATFORM_DEVICE(obj)->vbasedev, str, errp);
+}
+#endif
+
static void vfio_platform_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
@@ -665,6 +677,9 @@ static void vfio_platform_class_init(ObjectClass *klass, void *data)
dc->realize = vfio_platform_realize;
device_class_set_props(dc, vfio_platform_dev_properties);
+#ifdef CONFIG_IOMMUFD
+ object_class_property_add_str(klass, "fd", NULL, vfio_platform_set_fd);
+#endif
dc->vmsd = &vfio_platform_vmstate;
dc->desc = "VFIO-based platform device assignment";
sbc->connect_irq_notifier = vfio_start_irqfd_injection;
@@ -677,6 +692,7 @@ static const TypeInfo vfio_platform_dev_info = {
.name = TYPE_VFIO_PLATFORM,
.parent = TYPE_SYS_BUS_DEVICE,
.instance_size = sizeof(VFIOPlatformDevice),
+ .instance_init = vfio_platform_instance_init,
.class_init = vfio_platform_class_init,
.class_size = sizeof(VFIOPlatformDeviceClass),
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 33/47] vfio/ap: Allow the selection of a given iommu backend
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (31 preceding siblings ...)
2023-12-19 18:56 ` [PULL 32/47] vfio/platform: Make vfio cdev pre-openable by passing a file handle Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 34/47] vfio/ap: Make vfio cdev pre-openable by passing a file handle Cédric Le Goater
` (14 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Now we support two types of iommu backends, let's add the capability
to select one of them. This depends on whether an iommufd object has
been linked with the vfio-ap device:
if the user wants to use the legacy backend, it shall not
link the vfio-ap device with any iommufd object:
-device vfio-ap,sysfsdev=/sys/bus/mdev/devices/XXX
This is called the legacy mode/backend.
If the user wants to use the iommufd backend (/dev/iommu) it
shall pass an iommufd object id in the vfio-ap device options:
-object iommufd,id=iommufd0
-device vfio-ap,sysfsdev=/sys/bus/mdev/devices/XXX,iommufd=iommufd0
Suggested-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Matthew Rosato <mjrosato@linux.ibm.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/ap.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index bbf69ff55ae8a922d0ab6d6f966a9a2283cdd2a3..80629609aebbff9156f78c9bbde5dc6c293ac84e 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -11,10 +11,12 @@
*/
#include "qemu/osdep.h"
+#include CONFIG_DEVICES /* CONFIG_IOMMUFD */
#include <linux/vfio.h>
#include <sys/ioctl.h>
#include "qapi/error.h"
#include "hw/vfio/vfio-common.h"
+#include "sysemu/iommufd.h"
#include "hw/s390x/ap-device.h"
#include "qemu/error-report.h"
#include "qemu/event_notifier.h"
@@ -204,6 +206,10 @@ static void vfio_ap_unrealize(DeviceState *dev)
static Property vfio_ap_properties[] = {
DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+#ifdef CONFIG_IOMMUFD
+ DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
+ TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
+#endif
DEFINE_PROP_END_OF_LIST(),
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 34/47] vfio/ap: Make vfio cdev pre-openable by passing a file handle
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (32 preceding siblings ...)
2023-12-19 18:56 ` [PULL 33/47] vfio/ap: Allow the selection of a given iommu backend Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 35/47] vfio/ccw: Allow the selection of a given iommu backend Cédric Le Goater
` (13 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
This gives management tools like libvirt a chance to open the vfio
cdev with privilege and pass FD to qemu. This way qemu never needs
to have privilege to open a VFIO or iommu cdev node.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Matthew Rosato <mjrosato@linux.ibm.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/ap.c | 23 ++++++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 80629609aebbff9156f78c9bbde5dc6c293ac84e..f180e4a32aa00f391f02e49b6c1bc2e8ebd2fecb 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -160,7 +160,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(dev);
VFIODevice *vbasedev = &vapdev->vdev;
- vbasedev->name = g_path_get_basename(vbasedev->sysfsdev);
+ if (vfio_device_get_name(vbasedev, errp) < 0) {
+ return;
+ }
+
vbasedev->ops = &vfio_ap_ops;
vbasedev->type = VFIO_DEVICE_TYPE_AP;
vbasedev->dev = dev;
@@ -230,11 +233,28 @@ static const VMStateDescription vfio_ap_vmstate = {
.unmigratable = 1,
};
+static void vfio_ap_instance_init(Object *obj)
+{
+ VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
+
+ vapdev->vdev.fd = -1;
+}
+
+#ifdef CONFIG_IOMMUFD
+static void vfio_ap_set_fd(Object *obj, const char *str, Error **errp)
+{
+ vfio_device_set_fd(&VFIO_AP_DEVICE(obj)->vdev, str, errp);
+}
+#endif
+
static void vfio_ap_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
device_class_set_props(dc, vfio_ap_properties);
+#ifdef CONFIG_IOMMUFD
+ object_class_property_add_str(klass, "fd", NULL, vfio_ap_set_fd);
+#endif
dc->vmsd = &vfio_ap_vmstate;
dc->desc = "VFIO-based AP device assignment";
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
@@ -249,6 +269,7 @@ static const TypeInfo vfio_ap_info = {
.name = TYPE_VFIO_AP_DEVICE,
.parent = TYPE_AP_DEVICE,
.instance_size = sizeof(VFIOAPDevice),
+ .instance_init = vfio_ap_instance_init,
.class_init = vfio_ap_class_init,
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 35/47] vfio/ccw: Allow the selection of a given iommu backend
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (33 preceding siblings ...)
2023-12-19 18:56 ` [PULL 34/47] vfio/ap: Make vfio cdev pre-openable by passing a file handle Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 36/47] vfio/ccw: Make vfio cdev pre-openable by passing a file handle Cédric Le Goater
` (12 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Now we support two types of iommu backends, let's add the capability
to select one of them. This depends on whether an iommufd object has
been linked with the vfio-ccw device:
If the user wants to use the legacy backend, it shall not
link the vfio-ccw device with any iommufd object:
-device vfio-ccw,sysfsdev=/sys/bus/mdev/devices/XXX
This is called the legacy mode/backend.
If the user wants to use the iommufd backend (/dev/iommu) it
shall pass an iommufd object id in the vfio-ccw device options:
-object iommufd,id=iommufd0
-device vfio-ccw,sysfsdev=/sys/bus/mdev/devices/XXX,iommufd=iommufd0
Suggested-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Matthew Rosato <mjrosato@linux.ibm.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Farman <farman@linux.ibm.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/ccw.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/hw/vfio/ccw.c b/hw/vfio/ccw.c
index d857bb8d0fe4ff71b3cf635d6da43eccbe593d5f..d2d58bb677cfad38a6965ea6a43783256407207b 100644
--- a/hw/vfio/ccw.c
+++ b/hw/vfio/ccw.c
@@ -15,12 +15,14 @@
*/
#include "qemu/osdep.h"
+#include CONFIG_DEVICES /* CONFIG_IOMMUFD */
#include <linux/vfio.h>
#include <linux/vfio_ccw.h>
#include <sys/ioctl.h>
#include "qapi/error.h"
#include "hw/vfio/vfio-common.h"
+#include "sysemu/iommufd.h"
#include "hw/s390x/s390-ccw.h"
#include "hw/s390x/vfio-ccw.h"
#include "hw/qdev-properties.h"
@@ -677,6 +679,10 @@ static void vfio_ccw_unrealize(DeviceState *dev)
static Property vfio_ccw_properties[] = {
DEFINE_PROP_STRING("sysfsdev", VFIOCCWDevice, vdev.sysfsdev),
DEFINE_PROP_BOOL("force-orb-pfch", VFIOCCWDevice, force_orb_pfch, false),
+#ifdef CONFIG_IOMMUFD
+ DEFINE_PROP_LINK("iommufd", VFIOCCWDevice, vdev.iommufd,
+ TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
+#endif
DEFINE_PROP_END_OF_LIST(),
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 36/47] vfio/ccw: Make vfio cdev pre-openable by passing a file handle
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (34 preceding siblings ...)
2023-12-19 18:56 ` [PULL 35/47] vfio/ccw: Allow the selection of a given iommu backend Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 37/47] vfio: Make VFIOContainerBase poiner parameter const in VFIOIOMMUOps callbacks Cédric Le Goater
` (11 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
This gives management tools like libvirt a chance to open the vfio
cdev with privilege and pass FD to qemu. This way qemu never needs
to have privilege to open a VFIO or iommu cdev node.
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Matthew Rosato <mjrosato@linux.ibm.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Farman <farman@linux.ibm.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/ccw.c | 25 ++++++++++++++++++++++---
1 file changed, 22 insertions(+), 3 deletions(-)
diff --git a/hw/vfio/ccw.c b/hw/vfio/ccw.c
index d2d58bb677cfad38a6965ea6a43783256407207b..2afdf17dbe1e09dbb2f4f2d3d3b34f2a3d32dc5c 100644
--- a/hw/vfio/ccw.c
+++ b/hw/vfio/ccw.c
@@ -590,11 +590,12 @@ static void vfio_ccw_realize(DeviceState *dev, Error **errp)
}
}
+ if (vfio_device_get_name(vbasedev, errp) < 0) {
+ return;
+ }
+
vbasedev->ops = &vfio_ccw_ops;
vbasedev->type = VFIO_DEVICE_TYPE_CCW;
- vbasedev->name = g_strdup_printf("%x.%x.%04x", vcdev->cdev.hostid.cssid,
- vcdev->cdev.hostid.ssid,
- vcdev->cdev.hostid.devid);
vbasedev->dev = dev;
/*
@@ -691,12 +692,29 @@ static const VMStateDescription vfio_ccw_vmstate = {
.unmigratable = 1,
};
+static void vfio_ccw_instance_init(Object *obj)
+{
+ VFIOCCWDevice *vcdev = VFIO_CCW(obj);
+
+ vcdev->vdev.fd = -1;
+}
+
+#ifdef CONFIG_IOMMUFD
+static void vfio_ccw_set_fd(Object *obj, const char *str, Error **errp)
+{
+ vfio_device_set_fd(&VFIO_CCW(obj)->vdev, str, errp);
+}
+#endif
+
static void vfio_ccw_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
S390CCWDeviceClass *cdc = S390_CCW_DEVICE_CLASS(klass);
device_class_set_props(dc, vfio_ccw_properties);
+#ifdef CONFIG_IOMMUFD
+ object_class_property_add_str(klass, "fd", NULL, vfio_ccw_set_fd);
+#endif
dc->vmsd = &vfio_ccw_vmstate;
dc->desc = "VFIO-based subchannel assignment";
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
@@ -714,6 +732,7 @@ static const TypeInfo vfio_ccw_info = {
.name = TYPE_VFIO_CCW,
.parent = TYPE_S390_CCW,
.instance_size = sizeof(VFIOCCWDevice),
+ .instance_init = vfio_ccw_instance_init,
.class_init = vfio_ccw_class_init,
};
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 37/47] vfio: Make VFIOContainerBase poiner parameter const in VFIOIOMMUOps callbacks
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (35 preceding siblings ...)
2023-12-19 18:56 ` [PULL 36/47] vfio/ccw: Make vfio cdev pre-openable by passing a file handle Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 38/47] hw/arm: Activate IOMMUFD for virt machines Cédric Le Goater
` (10 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Some of the callbacks in VFIOIOMMUOps pass VFIOContainerBase poiner,
those callbacks only need read access to the sub object of VFIOContainerBase.
So make VFIOContainerBase, VFIOContainer and VFIOIOMMUFDContainer as const
in these callbacks.
Local functions called by those callbacks also need same changes to avoid
build error.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 12 ++++++----
include/hw/vfio/vfio-container-base.h | 12 ++++++----
hw/vfio/common.c | 9 +++----
hw/vfio/container-base.c | 2 +-
hw/vfio/container.c | 34 ++++++++++++++-------------
hw/vfio/iommufd.c | 8 +++----
6 files changed, 42 insertions(+), 35 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 697bf24a350d5880e59811322d9037575a90d9a2..efcba19f6663af28bffacf954469a9986b8ae8f0 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -244,13 +244,15 @@ bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp);
void vfio_migration_exit(VFIODevice *vbasedev);
int vfio_bitmap_alloc(VFIOBitmap *vbmap, hwaddr size);
-bool vfio_devices_all_running_and_mig_active(VFIOContainerBase *bcontainer);
-bool vfio_devices_all_device_dirty_tracking(VFIOContainerBase *bcontainer);
-int vfio_devices_query_dirty_bitmap(VFIOContainerBase *bcontainer,
+bool
+vfio_devices_all_running_and_mig_active(const VFIOContainerBase *bcontainer);
+bool
+vfio_devices_all_device_dirty_tracking(const VFIOContainerBase *bcontainer);
+int vfio_devices_query_dirty_bitmap(const VFIOContainerBase *bcontainer,
VFIOBitmap *vbmap, hwaddr iova,
hwaddr size);
-int vfio_get_dirty_bitmap(VFIOContainerBase *bcontainer, uint64_t iova,
- uint64_t size, ram_addr_t ram_addr);
+int vfio_get_dirty_bitmap(const VFIOContainerBase *bcontainer, uint64_t iova,
+ uint64_t size, ram_addr_t ram_addr);
/* Returns 0 on success, or a negative errno. */
int vfio_device_get_name(VFIODevice *vbasedev, Error **errp);
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index 45bb19c7670e76ae142dbcd2bfc96f6e74ae160b..2ae297ccda93fd97986c852a8329b390fa1ab91f 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -82,7 +82,7 @@ void vfio_container_del_section_window(VFIOContainerBase *bcontainer,
MemoryRegionSection *section);
int vfio_container_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
bool start);
-int vfio_container_query_dirty_bitmap(VFIOContainerBase *bcontainer,
+int vfio_container_query_dirty_bitmap(const VFIOContainerBase *bcontainer,
VFIOBitmap *vbmap,
hwaddr iova, hwaddr size);
@@ -93,18 +93,20 @@ void vfio_container_destroy(VFIOContainerBase *bcontainer);
struct VFIOIOMMUOps {
/* basic feature */
- int (*dma_map)(VFIOContainerBase *bcontainer,
+ int (*dma_map)(const VFIOContainerBase *bcontainer,
hwaddr iova, ram_addr_t size,
void *vaddr, bool readonly);
- int (*dma_unmap)(VFIOContainerBase *bcontainer,
+ int (*dma_unmap)(const VFIOContainerBase *bcontainer,
hwaddr iova, ram_addr_t size,
IOMMUTLBEntry *iotlb);
int (*attach_device)(const char *name, VFIODevice *vbasedev,
AddressSpace *as, Error **errp);
void (*detach_device)(VFIODevice *vbasedev);
/* migration feature */
- int (*set_dirty_page_tracking)(VFIOContainerBase *bcontainer, bool start);
- int (*query_dirty_bitmap)(VFIOContainerBase *bcontainer, VFIOBitmap *vbmap,
+ int (*set_dirty_page_tracking)(const VFIOContainerBase *bcontainer,
+ bool start);
+ int (*query_dirty_bitmap)(const VFIOContainerBase *bcontainer,
+ VFIOBitmap *vbmap,
hwaddr iova, hwaddr size);
/* PCI specific */
int (*pci_hot_reset)(VFIODevice *vbasedev, bool single);
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 6569732b7a1d894ae08e38adb93dfd94aeca4075..08a3e576725b1fc9f2f7e425375df3b827c4fe56 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -204,7 +204,7 @@ static bool vfio_devices_all_dirty_tracking(VFIOContainerBase *bcontainer)
return true;
}
-bool vfio_devices_all_device_dirty_tracking(VFIOContainerBase *bcontainer)
+bool vfio_devices_all_device_dirty_tracking(const VFIOContainerBase *bcontainer)
{
VFIODevice *vbasedev;
@@ -221,7 +221,8 @@ bool vfio_devices_all_device_dirty_tracking(VFIOContainerBase *bcontainer)
* Check if all VFIO devices are running and migration is active, which is
* essentially equivalent to the migration being in pre-copy phase.
*/
-bool vfio_devices_all_running_and_mig_active(VFIOContainerBase *bcontainer)
+bool
+vfio_devices_all_running_and_mig_active(const VFIOContainerBase *bcontainer)
{
VFIODevice *vbasedev;
@@ -1139,7 +1140,7 @@ static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova,
return 0;
}
-int vfio_devices_query_dirty_bitmap(VFIOContainerBase *bcontainer,
+int vfio_devices_query_dirty_bitmap(const VFIOContainerBase *bcontainer,
VFIOBitmap *vbmap, hwaddr iova,
hwaddr size)
{
@@ -1162,7 +1163,7 @@ int vfio_devices_query_dirty_bitmap(VFIOContainerBase *bcontainer,
return 0;
}
-int vfio_get_dirty_bitmap(VFIOContainerBase *bcontainer, uint64_t iova,
+int vfio_get_dirty_bitmap(const VFIOContainerBase *bcontainer, uint64_t iova,
uint64_t size, ram_addr_t ram_addr)
{
bool all_device_dirty_tracking =
diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c
index eee2dcfe760b5530ef428fe341d376a85b47344a..1ffd25bbfa8bd3d404e43b96357273b95f5a0031 100644
--- a/hw/vfio/container-base.c
+++ b/hw/vfio/container-base.c
@@ -63,7 +63,7 @@ int vfio_container_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
return bcontainer->ops->set_dirty_page_tracking(bcontainer, start);
}
-int vfio_container_query_dirty_bitmap(VFIOContainerBase *bcontainer,
+int vfio_container_query_dirty_bitmap(const VFIOContainerBase *bcontainer,
VFIOBitmap *vbmap,
hwaddr iova, hwaddr size)
{
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 1dbf9b9a17e070ded853931757f5852d1e4568aa..b22feb8ded0a0d9ed98d6e206b78c0c6e2554d5c 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -61,11 +61,11 @@ static int vfio_ram_block_discard_disable(VFIOContainer *container, bool state)
}
}
-static int vfio_dma_unmap_bitmap(VFIOContainer *container,
+static int vfio_dma_unmap_bitmap(const VFIOContainer *container,
hwaddr iova, ram_addr_t size,
IOMMUTLBEntry *iotlb)
{
- VFIOContainerBase *bcontainer = &container->bcontainer;
+ const VFIOContainerBase *bcontainer = &container->bcontainer;
struct vfio_iommu_type1_dma_unmap *unmap;
struct vfio_bitmap *bitmap;
VFIOBitmap vbmap;
@@ -117,11 +117,12 @@ unmap_exit:
/*
* DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
*/
-static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
- ram_addr_t size, IOMMUTLBEntry *iotlb)
+static int vfio_legacy_dma_unmap(const VFIOContainerBase *bcontainer,
+ hwaddr iova, ram_addr_t size,
+ IOMMUTLBEntry *iotlb)
{
- VFIOContainer *container = container_of(bcontainer, VFIOContainer,
- bcontainer);
+ const VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
struct vfio_iommu_type1_dma_unmap unmap = {
.argsz = sizeof(unmap),
.flags = 0,
@@ -174,11 +175,11 @@ static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
return 0;
}
-static int vfio_legacy_dma_map(VFIOContainerBase *bcontainer, hwaddr iova,
+static int vfio_legacy_dma_map(const VFIOContainerBase *bcontainer, hwaddr iova,
ram_addr_t size, void *vaddr, bool readonly)
{
- VFIOContainer *container = container_of(bcontainer, VFIOContainer,
- bcontainer);
+ const VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
struct vfio_iommu_type1_dma_map map = {
.argsz = sizeof(map),
.flags = VFIO_DMA_MAP_FLAG_READ,
@@ -207,11 +208,12 @@ static int vfio_legacy_dma_map(VFIOContainerBase *bcontainer, hwaddr iova,
return -errno;
}
-static int vfio_legacy_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
- bool start)
+static int
+vfio_legacy_set_dirty_page_tracking(const VFIOContainerBase *bcontainer,
+ bool start)
{
- VFIOContainer *container = container_of(bcontainer, VFIOContainer,
- bcontainer);
+ const VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
int ret;
struct vfio_iommu_type1_dirty_bitmap dirty = {
.argsz = sizeof(dirty),
@@ -233,12 +235,12 @@ static int vfio_legacy_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
return ret;
}
-static int vfio_legacy_query_dirty_bitmap(VFIOContainerBase *bcontainer,
+static int vfio_legacy_query_dirty_bitmap(const VFIOContainerBase *bcontainer,
VFIOBitmap *vbmap,
hwaddr iova, hwaddr size)
{
- VFIOContainer *container = container_of(bcontainer, VFIOContainer,
- bcontainer);
+ const VFIOContainer *container = container_of(bcontainer, VFIOContainer,
+ bcontainer);
struct vfio_iommu_type1_dirty_bitmap *dbitmap;
struct vfio_iommu_type1_dirty_bitmap_get *range;
int ret;
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
index 5accd2648444defcd698bd6d0cefe11d255b4cfb..87a561c54580adc6d7b2711331a00940ff13bd43 100644
--- a/hw/vfio/iommufd.c
+++ b/hw/vfio/iommufd.c
@@ -26,10 +26,10 @@
#include "qemu/chardev_open.h"
#include "pci.h"
-static int iommufd_cdev_map(VFIOContainerBase *bcontainer, hwaddr iova,
+static int iommufd_cdev_map(const VFIOContainerBase *bcontainer, hwaddr iova,
ram_addr_t size, void *vaddr, bool readonly)
{
- VFIOIOMMUFDContainer *container =
+ const VFIOIOMMUFDContainer *container =
container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
return iommufd_backend_map_dma(container->be,
@@ -37,11 +37,11 @@ static int iommufd_cdev_map(VFIOContainerBase *bcontainer, hwaddr iova,
iova, size, vaddr, readonly);
}
-static int iommufd_cdev_unmap(VFIOContainerBase *bcontainer,
+static int iommufd_cdev_unmap(const VFIOContainerBase *bcontainer,
hwaddr iova, ram_addr_t size,
IOMMUTLBEntry *iotlb)
{
- VFIOIOMMUFDContainer *container =
+ const VFIOIOMMUFDContainer *container =
container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
/* TODO: Handle dma_unmap_bitmap with iotlb args (migration) */
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 38/47] hw/arm: Activate IOMMUFD for virt machines
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (36 preceding siblings ...)
2023-12-19 18:56 ` [PULL 37/47] vfio: Make VFIOContainerBase poiner parameter const in VFIOIOMMUOps callbacks Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 39/47] kconfig: Activate IOMMUFD for s390x machines Cédric Le Goater
` (9 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
Signed-off-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/arm/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/hw/arm/Kconfig b/hw/arm/Kconfig
index 3ada335a24374d4d543bc14c19b0a0b0bd1505b3..660f49db49859115e1eaba8e798cee398e4cf14e 100644
--- a/hw/arm/Kconfig
+++ b/hw/arm/Kconfig
@@ -8,6 +8,7 @@ config ARM_VIRT
imply TPM_TIS_SYSBUS
imply TPM_TIS_I2C
imply NVDIMM
+ imply IOMMUFD
select ARM_GIC
select ACPI
select ARM_SMMUV3
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 39/47] kconfig: Activate IOMMUFD for s390x machines
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (37 preceding siblings ...)
2023-12-19 18:56 ` [PULL 38/47] hw/arm: Activate IOMMUFD for virt machines Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 40/47] hw/i386: Activate IOMMUFD for q35 machines Cédric Le Goater
` (8 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
Signed-off-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Matthew Rosato <mjrosato@linux.ibm.com>
Reviewed-by: Eric Farman <farman@linux.ibm.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/s390x/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/hw/s390x/Kconfig b/hw/s390x/Kconfig
index 4c068d7960b973474b84d3cef9d4a3bf31ca4f9f..26ad1044858ce9ba3e78c65ffb9f295c31d7ab4c 100644
--- a/hw/s390x/Kconfig
+++ b/hw/s390x/Kconfig
@@ -6,6 +6,7 @@ config S390_CCW_VIRTIO
imply VFIO_CCW
imply WDT_DIAG288
imply PCIE_DEVICES
+ imply IOMMUFD
select PCI_EXPRESS
select S390_FLIC
select S390_FLIC_KVM if KVM
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 40/47] hw/i386: Activate IOMMUFD for q35 machines
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (38 preceding siblings ...)
2023-12-19 18:56 ` [PULL 39/47] kconfig: Activate IOMMUFD for s390x machines Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 41/47] vfio/pci: Move VFIODevice initializations in vfio_instance_init Cédric Le Goater
` (7 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
Signed-off-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/i386/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/hw/i386/Kconfig b/hw/i386/Kconfig
index 55850791df4148f5535eb06b76e09dabf75d84f1..a1846be6f761b1c40f63213df536d825a9c9a966 100644
--- a/hw/i386/Kconfig
+++ b/hw/i386/Kconfig
@@ -95,6 +95,7 @@ config Q35
imply E1000E_PCI_EXPRESS
imply VMPORT
imply VMMOUSE
+ imply IOMMUFD
select PC_PCI
select PC_ACPI
select PCI_EXPRESS_Q35
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 41/47] vfio/pci: Move VFIODevice initializations in vfio_instance_init
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (39 preceding siblings ...)
2023-12-19 18:56 ` [PULL 40/47] hw/i386: Activate IOMMUFD for q35 machines Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 42/47] vfio/platform: Move VFIODevice initializations in vfio_platform_instance_init Cédric Le Goater
` (6 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Philippe Mathieu-Daudé, Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Some of the VFIODevice initializations is in vfio_realize,
move all of them in vfio_instance_init.
No functional change intended.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/pci.c | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 445d58c8e59b0a00a8336d092c2b8cec7e39b396..87405584d774810c8349825fafec34945a915121 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2969,9 +2969,6 @@ static void vfio_realize(PCIDevice *pdev, Error **errp)
if (vfio_device_get_name(vbasedev, errp) < 0) {
return;
}
- vbasedev->ops = &vfio_pci_ops;
- vbasedev->type = VFIO_DEVICE_TYPE_PCI;
- vbasedev->dev = DEVICE(vdev);
/*
* Mediated devices *might* operate compatibly with discarding of RAM, but
@@ -3320,6 +3317,7 @@ static void vfio_instance_init(Object *obj)
{
PCIDevice *pci_dev = PCI_DEVICE(obj);
VFIOPCIDevice *vdev = VFIO_PCI(obj);
+ VFIODevice *vbasedev = &vdev->vbasedev;
device_add_bootindex_property(obj, &vdev->bootindex,
"bootindex", NULL,
@@ -3328,7 +3326,11 @@ static void vfio_instance_init(Object *obj)
vdev->host.bus = ~0U;
vdev->host.slot = ~0U;
vdev->host.function = ~0U;
- vdev->vbasedev.fd = -1;
+
+ vbasedev->type = VFIO_DEVICE_TYPE_PCI;
+ vbasedev->ops = &vfio_pci_ops;
+ vbasedev->dev = DEVICE(vdev);
+ vbasedev->fd = -1;
vdev->nv_gpudirect_clique = 0xFF;
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 42/47] vfio/platform: Move VFIODevice initializations in vfio_platform_instance_init
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (40 preceding siblings ...)
2023-12-19 18:56 ` [PULL 41/47] vfio/pci: Move VFIODevice initializations in vfio_instance_init Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 43/47] vfio/ap: Move VFIODevice initializations in vfio_ap_instance_init Cédric Le Goater
` (5 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Philippe Mathieu-Daudé, Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Some of the VFIODevice initializations is in vfio_platform_realize,
move all of them in vfio_platform_instance_init.
No functional change intended.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/platform.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/hw/vfio/platform.c b/hw/vfio/platform.c
index a97d9c6234ccac0ba26b44d3e2290616eda5debd..506eb8193f2b047bc810e50236ae82fb7dfc7693 100644
--- a/hw/vfio/platform.c
+++ b/hw/vfio/platform.c
@@ -581,10 +581,6 @@ static void vfio_platform_realize(DeviceState *dev, Error **errp)
VFIODevice *vbasedev = &vdev->vbasedev;
int i, ret;
- vbasedev->type = VFIO_DEVICE_TYPE_PLATFORM;
- vbasedev->dev = dev;
- vbasedev->ops = &vfio_platform_ops;
-
qemu_mutex_init(&vdev->intp_mutex);
trace_vfio_platform_realize(vbasedev->sysfsdev ?
@@ -659,8 +655,12 @@ static Property vfio_platform_dev_properties[] = {
static void vfio_platform_instance_init(Object *obj)
{
VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(obj);
+ VFIODevice *vbasedev = &vdev->vbasedev;
- vdev->vbasedev.fd = -1;
+ vbasedev->type = VFIO_DEVICE_TYPE_PLATFORM;
+ vbasedev->ops = &vfio_platform_ops;
+ vbasedev->dev = DEVICE(vdev);
+ vbasedev->fd = -1;
}
#ifdef CONFIG_IOMMUFD
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 43/47] vfio/ap: Move VFIODevice initializations in vfio_ap_instance_init
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (41 preceding siblings ...)
2023-12-19 18:56 ` [PULL 42/47] vfio/platform: Move VFIODevice initializations in vfio_platform_instance_init Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 44/47] vfio/ccw: Move VFIODevice initializations in vfio_ccw_instance_init Cédric Le Goater
` (4 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Philippe Mathieu-Daudé, Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Some of the VFIODevice initializations is in vfio_ap_realize,
move all of them in vfio_ap_instance_init.
No functional change intended.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Reviewed-by: Eric Farman <farman@linux.ibm.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/ap.c | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index f180e4a32aa00f391f02e49b6c1bc2e8ebd2fecb..95fe7cd98b6fdad5d8ca12a798f9aff91ca5b3f4 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -164,18 +164,6 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
return;
}
- vbasedev->ops = &vfio_ap_ops;
- vbasedev->type = VFIO_DEVICE_TYPE_AP;
- vbasedev->dev = dev;
-
- /*
- * vfio-ap devices operate in a way compatible with discarding of
- * memory in RAM blocks, as no pages are pinned in the host.
- * This needs to be set before vfio_get_device() for vfio common to
- * handle ram_block_discard_disable().
- */
- vapdev->vdev.ram_block_discard_allowed = true;
-
ret = vfio_attach_device(vbasedev->name, vbasedev,
&address_space_memory, errp);
if (ret) {
@@ -236,8 +224,20 @@ static const VMStateDescription vfio_ap_vmstate = {
static void vfio_ap_instance_init(Object *obj)
{
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
+ VFIODevice *vbasedev = &vapdev->vdev;
- vapdev->vdev.fd = -1;
+ vbasedev->type = VFIO_DEVICE_TYPE_AP;
+ vbasedev->ops = &vfio_ap_ops;
+ vbasedev->dev = DEVICE(vapdev);
+ vbasedev->fd = -1;
+
+ /*
+ * vfio-ap devices operate in a way compatible with discarding of
+ * memory in RAM blocks, as no pages are pinned in the host.
+ * This needs to be set before vfio_get_device() for vfio common to
+ * handle ram_block_discard_disable().
+ */
+ vbasedev->ram_block_discard_allowed = true;
}
#ifdef CONFIG_IOMMUFD
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 44/47] vfio/ccw: Move VFIODevice initializations in vfio_ccw_instance_init
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (42 preceding siblings ...)
2023-12-19 18:56 ` [PULL 43/47] vfio/ap: Move VFIODevice initializations in vfio_ap_instance_init Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 45/47] vfio: Introduce a helper function to initialize VFIODevice Cédric Le Goater
` (3 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Philippe Mathieu-Daudé, Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Some of the VFIODevice initializations is in vfio_ccw_realize,
move all of them in vfio_ccw_instance_init.
No functional change intended.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Reviewed-by: Eric Farman <farman@linux.ibm.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/vfio/ccw.c | 30 +++++++++++++++---------------
1 file changed, 15 insertions(+), 15 deletions(-)
diff --git a/hw/vfio/ccw.c b/hw/vfio/ccw.c
index 2afdf17dbe1e09dbb2f4f2d3d3b34f2a3d32dc5c..6305a4c1b86412c9706027939460176b62a6345d 100644
--- a/hw/vfio/ccw.c
+++ b/hw/vfio/ccw.c
@@ -594,20 +594,6 @@ static void vfio_ccw_realize(DeviceState *dev, Error **errp)
return;
}
- vbasedev->ops = &vfio_ccw_ops;
- vbasedev->type = VFIO_DEVICE_TYPE_CCW;
- vbasedev->dev = dev;
-
- /*
- * All vfio-ccw devices are believed to operate in a way compatible with
- * discarding of memory in RAM blocks, ie. pages pinned in the host are
- * in the current working set of the guest driver and therefore never
- * overlap e.g., with pages available to the guest balloon driver. This
- * needs to be set before vfio_get_device() for vfio common to handle
- * ram_block_discard_disable().
- */
- vbasedev->ram_block_discard_allowed = true;
-
ret = vfio_attach_device(cdev->mdevid, vbasedev,
&address_space_memory, errp);
if (ret) {
@@ -695,8 +681,22 @@ static const VMStateDescription vfio_ccw_vmstate = {
static void vfio_ccw_instance_init(Object *obj)
{
VFIOCCWDevice *vcdev = VFIO_CCW(obj);
+ VFIODevice *vbasedev = &vcdev->vdev;
+
+ vbasedev->type = VFIO_DEVICE_TYPE_CCW;
+ vbasedev->ops = &vfio_ccw_ops;
+ vbasedev->dev = DEVICE(vcdev);
+ vbasedev->fd = -1;
- vcdev->vdev.fd = -1;
+ /*
+ * All vfio-ccw devices are believed to operate in a way compatible with
+ * discarding of memory in RAM blocks, ie. pages pinned in the host are
+ * in the current working set of the guest driver and therefore never
+ * overlap e.g., with pages available to the guest balloon driver. This
+ * needs to be set before vfio_get_device() for vfio common to handle
+ * ram_block_discard_disable().
+ */
+ vbasedev->ram_block_discard_allowed = true;
}
#ifdef CONFIG_IOMMUFD
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 45/47] vfio: Introduce a helper function to initialize VFIODevice
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (43 preceding siblings ...)
2023-12-19 18:56 ` [PULL 44/47] vfio/ccw: Move VFIODevice initializations in vfio_ccw_instance_init Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 46/47] docs/devel: Add VFIO iommufd backend documentation Cédric Le Goater
` (2 subsequent siblings)
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Introduce a helper function to replace the common code to initialize
VFIODevice in pci, platform, ap and ccw VFIO device.
No functional change intended.
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 2 ++
hw/vfio/ap.c | 8 ++------
hw/vfio/ccw.c | 8 ++------
hw/vfio/helpers.c | 11 +++++++++++
hw/vfio/pci.c | 6 ++----
hw/vfio/platform.c | 6 ++----
6 files changed, 21 insertions(+), 20 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index efcba19f6663af28bffacf954469a9986b8ae8f0..b8aa8a549532442a31c8e85ce385c992d84f6bd5 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -257,4 +257,6 @@ int vfio_get_dirty_bitmap(const VFIOContainerBase *bcontainer, uint64_t iova,
/* Returns 0 on success, or a negative errno. */
int vfio_device_get_name(VFIODevice *vbasedev, Error **errp);
void vfio_device_set_fd(VFIODevice *vbasedev, const char *str, Error **errp);
+void vfio_device_init(VFIODevice *vbasedev, int type, VFIODeviceOps *ops,
+ DeviceState *dev, bool ram_discard);
#endif /* HW_VFIO_VFIO_COMMON_H */
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 95fe7cd98b6fdad5d8ca12a798f9aff91ca5b3f4..e157aa1ff79c1ad96b9102d54d4c11da1bd719ba 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -226,18 +226,14 @@ static void vfio_ap_instance_init(Object *obj)
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
VFIODevice *vbasedev = &vapdev->vdev;
- vbasedev->type = VFIO_DEVICE_TYPE_AP;
- vbasedev->ops = &vfio_ap_ops;
- vbasedev->dev = DEVICE(vapdev);
- vbasedev->fd = -1;
-
/*
* vfio-ap devices operate in a way compatible with discarding of
* memory in RAM blocks, as no pages are pinned in the host.
* This needs to be set before vfio_get_device() for vfio common to
* handle ram_block_discard_disable().
*/
- vbasedev->ram_block_discard_allowed = true;
+ vfio_device_init(vbasedev, VFIO_DEVICE_TYPE_AP, &vfio_ap_ops,
+ DEVICE(vapdev), true);
}
#ifdef CONFIG_IOMMUFD
diff --git a/hw/vfio/ccw.c b/hw/vfio/ccw.c
index 6305a4c1b86412c9706027939460176b62a6345d..90e4a534371684c08e112364e1537eb8979f73f4 100644
--- a/hw/vfio/ccw.c
+++ b/hw/vfio/ccw.c
@@ -683,11 +683,6 @@ static void vfio_ccw_instance_init(Object *obj)
VFIOCCWDevice *vcdev = VFIO_CCW(obj);
VFIODevice *vbasedev = &vcdev->vdev;
- vbasedev->type = VFIO_DEVICE_TYPE_CCW;
- vbasedev->ops = &vfio_ccw_ops;
- vbasedev->dev = DEVICE(vcdev);
- vbasedev->fd = -1;
-
/*
* All vfio-ccw devices are believed to operate in a way compatible with
* discarding of memory in RAM blocks, ie. pages pinned in the host are
@@ -696,7 +691,8 @@ static void vfio_ccw_instance_init(Object *obj)
* needs to be set before vfio_get_device() for vfio common to handle
* ram_block_discard_disable().
*/
- vbasedev->ram_block_discard_allowed = true;
+ vfio_device_init(vbasedev, VFIO_DEVICE_TYPE_CCW, &vfio_ccw_ops,
+ DEVICE(vcdev), true);
}
#ifdef CONFIG_IOMMUFD
diff --git a/hw/vfio/helpers.c b/hw/vfio/helpers.c
index 3592c3d54ecd68d4bfd23d4c3402a393fb1f2eb0..678987080228e9f69890bd955de452fe7ec06cca 100644
--- a/hw/vfio/helpers.c
+++ b/hw/vfio/helpers.c
@@ -652,3 +652,14 @@ void vfio_device_set_fd(VFIODevice *vbasedev, const char *str, Error **errp)
}
vbasedev->fd = fd;
}
+
+void vfio_device_init(VFIODevice *vbasedev, int type, VFIODeviceOps *ops,
+ DeviceState *dev, bool ram_discard)
+{
+ vbasedev->type = type;
+ vbasedev->ops = ops;
+ vbasedev->dev = dev;
+ vbasedev->fd = -1;
+
+ vbasedev->ram_block_discard_allowed = ram_discard;
+}
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 87405584d774810c8349825fafec34945a915121..1874ec1aba987cac6cb83f86650e7a5e1968c327 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -3327,10 +3327,8 @@ static void vfio_instance_init(Object *obj)
vdev->host.slot = ~0U;
vdev->host.function = ~0U;
- vbasedev->type = VFIO_DEVICE_TYPE_PCI;
- vbasedev->ops = &vfio_pci_ops;
- vbasedev->dev = DEVICE(vdev);
- vbasedev->fd = -1;
+ vfio_device_init(vbasedev, VFIO_DEVICE_TYPE_PCI, &vfio_pci_ops,
+ DEVICE(vdev), false);
vdev->nv_gpudirect_clique = 0xFF;
diff --git a/hw/vfio/platform.c b/hw/vfio/platform.c
index 506eb8193f2b047bc810e50236ae82fb7dfc7693..a8d9b7da633e0717421acbe9a951334b074b6607 100644
--- a/hw/vfio/platform.c
+++ b/hw/vfio/platform.c
@@ -657,10 +657,8 @@ static void vfio_platform_instance_init(Object *obj)
VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(obj);
VFIODevice *vbasedev = &vdev->vbasedev;
- vbasedev->type = VFIO_DEVICE_TYPE_PLATFORM;
- vbasedev->ops = &vfio_platform_ops;
- vbasedev->dev = DEVICE(vdev);
- vbasedev->fd = -1;
+ vfio_device_init(vbasedev, VFIO_DEVICE_TYPE_PLATFORM, &vfio_platform_ops,
+ DEVICE(vdev), false);
}
#ifdef CONFIG_IOMMUFD
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 46/47] docs/devel: Add VFIO iommufd backend documentation
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (44 preceding siblings ...)
2023-12-19 18:56 ` [PULL 45/47] vfio: Introduce a helper function to initialize VFIODevice Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-19 18:56 ` [PULL 47/47] hw/ppc/Kconfig: Imply VFIO_PCI Cédric Le Goater
2023-12-20 16:03 ` [PULL 00/47] vfio queue Stefan Hajnoczi
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater, Yi Liu,
Nicolin Chen
From: Zhenzhong Duan <zhenzhong.duan@intel.com>
Suggested-by: Cédric Le Goater <clg@redhat.com>
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
MAINTAINERS | 1 +
docs/devel/index-internals.rst | 1 +
docs/devel/vfio-iommufd.rst | 166 +++++++++++++++++++++++++++++++++
3 files changed, 168 insertions(+)
create mode 100644 docs/devel/vfio-iommufd.rst
diff --git a/MAINTAINERS b/MAINTAINERS
index ca70bb4e6415fc3af110cc7fd37ac67be5ab8c9d..0ddb20a35f205dba3b437c33bf489a53ecfc36b0 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2176,6 +2176,7 @@ F: backends/iommufd.c
F: include/sysemu/iommufd.h
F: include/qemu/chardev_open.h
F: util/chardev_open.c
+F: docs/devel/vfio-iommufd.rst
vhost
M: Michael S. Tsirkin <mst@redhat.com>
diff --git a/docs/devel/index-internals.rst b/docs/devel/index-internals.rst
index 6f81df92bcaba790477aff1ccb51048409331950..3def4a138bae5eca5b564e0044c1c2e80b5bc07a 100644
--- a/docs/devel/index-internals.rst
+++ b/docs/devel/index-internals.rst
@@ -18,5 +18,6 @@ Details about QEMU's various subsystems including how to add features to them.
s390-dasd-ipl
tracing
vfio-migration
+ vfio-iommufd
writing-monitor-commands
virtio-backends
diff --git a/docs/devel/vfio-iommufd.rst b/docs/devel/vfio-iommufd.rst
new file mode 100644
index 0000000000000000000000000000000000000000..3d1c11f175e5968e9f1519da70c9a0a6ced03995
--- /dev/null
+++ b/docs/devel/vfio-iommufd.rst
@@ -0,0 +1,166 @@
+===============================
+IOMMUFD BACKEND usage with VFIO
+===============================
+
+(Same meaning for backend/container/BE)
+
+With the introduction of iommufd, the Linux kernel provides a generic
+interface for user space drivers to propagate their DMA mappings to kernel
+for assigned devices. While the legacy kernel interface is group-centric,
+the new iommufd interface is device-centric, relying on device fd and iommufd.
+
+To support both interfaces in the QEMU VFIO device, introduce a base container
+to abstract the common part of VFIO legacy and iommufd container. So that the
+generic VFIO code can use either container.
+
+The base container implements generic functions such as memory_listener and
+address space management whereas the derived container implements callbacks
+specific to either legacy or iommufd. Each container has its own way to setup
+secure context and dma management interface. The below diagram shows how it
+looks like with both containers.
+
+::
+
+ VFIO AddressSpace/Memory
+ +-------+ +----------+ +-----+ +-----+
+ | pci | | platform | | ap | | ccw |
+ +---+---+ +----+-----+ +--+--+ +--+--+ +----------------------+
+ | | | | | AddressSpace |
+ | | | | +------------+---------+
+ +---V-----------V-----------V--------V----+ /
+ | VFIOAddressSpace | <------------+
+ | | | MemoryListener
+ | VFIOContainerBase list |
+ +-------+----------------------------+----+
+ | |
+ | |
+ +-------V------+ +--------V----------+
+ | iommufd | | vfio legacy |
+ | container | | container |
+ +-------+------+ +--------+----------+
+ | |
+ | /dev/iommu | /dev/vfio/vfio
+ | /dev/vfio/devices/vfioX | /dev/vfio/$group_id
+ Userspace | |
+ ============+============================+===========================
+ Kernel | device fd |
+ +---------------+ | group/container fd
+ | (BIND_IOMMUFD | | (SET_CONTAINER/SET_IOMMU)
+ | ATTACH_IOAS) | | device fd
+ | | |
+ | +-------V------------V-----------------+
+ iommufd | | vfio |
+ (map/unmap | +---------+--------------------+-------+
+ ioas_copy) | | | map/unmap
+ | | |
+ +------V------+ +-----V------+ +------V--------+
+ | iommfd core | | device | | vfio iommu |
+ +-------------+ +------------+ +---------------+
+
+* Secure Context setup
+
+ - iommufd BE: uses device fd and iommufd to setup secure context
+ (bind_iommufd, attach_ioas)
+ - vfio legacy BE: uses group fd and container fd to setup secure context
+ (set_container, set_iommu)
+
+* Device access
+
+ - iommufd BE: device fd is opened through ``/dev/vfio/devices/vfioX``
+ - vfio legacy BE: device fd is retrieved from group fd ioctl
+
+* DMA Mapping flow
+
+ 1. VFIOAddressSpace receives MemoryRegion add/del via MemoryListener
+ 2. VFIO populates DMA map/unmap via the container BEs
+ * iommufd BE: uses iommufd
+ * vfio legacy BE: uses container fd
+
+Example configuration
+=====================
+
+Step 1: configure the host device
+---------------------------------
+
+It's exactly same as the VFIO device with legacy VFIO container.
+
+Step 2: configure QEMU
+----------------------
+
+Interactions with the ``/dev/iommu`` are abstracted by a new iommufd
+object (compiled in with the ``CONFIG_IOMMUFD`` option).
+
+Any QEMU device (e.g. VFIO device) wishing to use ``/dev/iommu`` must
+be linked with an iommufd object. It gets a new optional property
+named iommufd which allows to pass an iommufd object. Take ``vfio-pci``
+device for example:
+
+.. code-block:: bash
+
+ -object iommufd,id=iommufd0
+ -device vfio-pci,host=0000:02:00.0,iommufd=iommufd0
+
+Note the ``/dev/iommu`` and VFIO cdev can be externally opened by a
+management layer. In such a case the fd is passed, the fd supports a
+string naming the fd or a number, for example:
+
+.. code-block:: bash
+
+ -object iommufd,id=iommufd0,fd=22
+ -device vfio-pci,iommufd=iommufd0,fd=23
+
+If the ``fd`` property is not passed, the fd is opened by QEMU.
+
+If no ``iommufd`` object is passed to the ``vfio-pci`` device, iommufd
+is not used and the user gets the behavior based on the legacy VFIO
+container:
+
+.. code-block:: bash
+
+ -device vfio-pci,host=0000:02:00.0
+
+Supported platform
+==================
+
+Supports x86, ARM and s390x currently.
+
+Caveats
+=======
+
+Dirty page sync
+---------------
+
+Dirty page sync with iommufd backend is unsupported yet, live migration is
+disabled by default. But it can be force enabled like below, low efficient
+though.
+
+.. code-block:: bash
+
+ -object iommufd,id=iommufd0
+ -device vfio-pci,host=0000:02:00.0,iommufd=iommufd0,enable-migration=on
+
+P2P DMA
+-------
+
+PCI p2p DMA is unsupported as IOMMUFD doesn't support mapping hardware PCI
+BAR region yet. Below warning shows for assigned PCI device, it's not a bug.
+
+.. code-block:: none
+
+ qemu-system-x86_64: warning: IOMMU_IOAS_MAP failed: Bad address, PCI BAR?
+ qemu-system-x86_64: vfio_container_dma_map(0x560cb6cb1620, 0xe000000021000, 0x3000, 0x7f32ed55c000) = -14 (Bad address)
+
+FD passing with mdev
+--------------------
+
+``vfio-pci`` device checks sysfsdev property to decide if backend is a mdev.
+If FD passing is used, there is no way to know that and the mdev is treated
+like a real PCI device. There is an error as below if user wants to enable
+RAM discarding for mdev.
+
+.. code-block:: none
+
+ qemu-system-x86_64: -device vfio-pci,iommufd=iommufd0,x-balloon-allowed=on,fd=9: vfio VFIO_FD9: x-balloon-allowed only potentially compatible with mdev devices
+
+``vfio-ap`` and ``vfio-ccw`` devices don't have same issue as their backend
+devices are always mdev and RAM discarding is force enabled.
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* [PULL 47/47] hw/ppc/Kconfig: Imply VFIO_PCI
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (45 preceding siblings ...)
2023-12-19 18:56 ` [PULL 46/47] docs/devel: Add VFIO iommufd backend documentation Cédric Le Goater
@ 2023-12-19 18:56 ` Cédric Le Goater
2023-12-20 16:03 ` [PULL 00/47] vfio queue Stefan Hajnoczi
47 siblings, 0 replies; 57+ messages in thread
From: Cédric Le Goater @ 2023-12-19 18:56 UTC (permalink / raw)
To: qemu-devel
Cc: Eric Auger, Zhenzhong Duan, Peter Maydell, Richard Henderson,
Nicholas Piggin, Harsh Prateek Bora, Thomas Huth, Eric Farman,
Alex Williamson, Matthew Rosato, Cédric Le Goater,
Philippe Mathieu-Daudé
When the legacy and iommufd backends were introduced, a set of common
vfio-pci routines were exported in pci.c for both backends to use :
vfio_pci_pre_reset
vfio_pci_get_pci_hot_reset_info
vfio_pci_host_match
vfio_pci_post_reset
This introduced a build failure on PPC when --without-default-devices
is use because VFIO is always selected in ppc/Kconfig but VFIO_PCI is
not.
Use an 'imply VFIO_PCI' in ppc/Kconfig and bypass compilation of the
VFIO EEH hooks routines defined in hw/ppc/spapr_pci_vfio.c with
CONFIG_VFIO_PCI.
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
hw/ppc/spapr_pci_vfio.c | 36 ++++++++++++++++++++++++++++++++++++
hw/ppc/Kconfig | 2 +-
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index d1d07bec4644da4ae6a99d3357d6d17ff66264de..76b2a3487b5d6f21528e9c301341eb27bc8fec1d 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -26,10 +26,12 @@
#include "hw/pci/pci_device.h"
#include "hw/vfio/vfio-common.h"
#include "qemu/error-report.h"
+#include CONFIG_DEVICES /* CONFIG_VFIO_PCI */
/*
* Interfaces for IBM EEH (Enhanced Error Handling)
*/
+#ifdef CONFIG_VFIO_PCI
static bool vfio_eeh_container_ok(VFIOContainer *container)
{
/*
@@ -314,3 +316,37 @@ int spapr_phb_vfio_eeh_configure(SpaprPhbState *sphb)
return RTAS_OUT_SUCCESS;
}
+
+#else
+
+bool spapr_phb_eeh_available(SpaprPhbState *sphb)
+{
+ return false;
+}
+
+void spapr_phb_vfio_reset(DeviceState *qdev)
+{
+}
+
+int spapr_phb_vfio_eeh_set_option(SpaprPhbState *sphb,
+ unsigned int addr, int option)
+{
+ return RTAS_OUT_NOT_SUPPORTED;
+}
+
+int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb, int *state)
+{
+ return RTAS_OUT_NOT_SUPPORTED;
+}
+
+int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
+{
+ return RTAS_OUT_NOT_SUPPORTED;
+}
+
+int spapr_phb_vfio_eeh_configure(SpaprPhbState *sphb)
+{
+ return RTAS_OUT_NOT_SUPPORTED;
+}
+
+#endif /* CONFIG_VFIO_PCI */
diff --git a/hw/ppc/Kconfig b/hw/ppc/Kconfig
index 56f0475a8e2e46e433d9df02460aec65e26d77a5..44263a58c4d3202afbe8bb44c67701bded855a57 100644
--- a/hw/ppc/Kconfig
+++ b/hw/ppc/Kconfig
@@ -3,11 +3,11 @@ config PSERIES
imply PCI_DEVICES
imply TEST_DEVICES
imply VIRTIO_VGA
+ imply VFIO_PCI if LINUX # needed by spapr_pci_vfio.c
select NVDIMM
select DIMM
select PCI
select SPAPR_VSCSI
- select VFIO if LINUX # needed by spapr_pci_vfio.c
select XICS
select XIVE
select MSI_NONBROKEN
--
2.43.0
^ permalink raw reply related [flat|nested] 57+ messages in thread* Re: [PULL 00/47] vfio queue
2023-12-19 18:55 [PULL 00/47] vfio queue Cédric Le Goater
` (46 preceding siblings ...)
2023-12-19 18:56 ` [PULL 47/47] hw/ppc/Kconfig: Imply VFIO_PCI Cédric Le Goater
@ 2023-12-20 16:03 ` Stefan Hajnoczi
47 siblings, 0 replies; 57+ messages in thread
From: Stefan Hajnoczi @ 2023-12-20 16:03 UTC (permalink / raw)
To: Cédric Le Goater
Cc: qemu-devel, Eric Auger, Zhenzhong Duan, Peter Maydell,
Richard Henderson, Nicholas Piggin, Harsh Prateek Bora,
Thomas Huth, Eric Farman, Alex Williamson, Matthew Rosato,
Cédric Le Goater
[-- Attachment #1: Type: text/plain, Size: 115 bytes --]
Applied, thanks.
Please update the changelog at https://wiki.qemu.org/ChangeLog/8.2 for any user-visible changes.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]
^ permalink raw reply [flat|nested] 57+ messages in thread