* [PULL 1/2] vfio: register container for cpr
2024-03-10 19:19 [PULL 0/2] vfio queue Cédric Le Goater
@ 2024-03-10 19:19 ` Cédric Le Goater
2024-03-10 19:19 ` [PULL 2/2] vfio: allow cpr-reboot migration if suspended Cédric Le Goater
2024-03-12 11:05 ` [PULL 0/2] vfio queue Peter Maydell
2 siblings, 0 replies; 4+ messages in thread
From: Cédric Le Goater @ 2024-03-10 19:19 UTC (permalink / raw)
To: qemu-devel; +Cc: Steve Sistare, Cédric Le Goater
From: Steve Sistare <steven.sistare@oracle.com>
Define entry points to perform per-container cpr-specific initialization
and teardown.
Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-common.h | 3 +++
hw/vfio/container.c | 11 ++++++++++-
hw/vfio/cpr.c | 19 +++++++++++++++++++
hw/vfio/iommufd.c | 6 ++++++
hw/vfio/meson.build | 1 +
5 files changed, 39 insertions(+), 1 deletion(-)
create mode 100644 hw/vfio/cpr.c
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 4a6c262f778b9bf4bf5760ffd548b9b6ea5bad33..b9da6c08ef41174610eb92726c590309a53696a3 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -205,6 +205,9 @@ void vfio_detach_device(VFIODevice *vbasedev);
int vfio_kvm_device_add_fd(int fd, Error **errp);
int vfio_kvm_device_del_fd(int fd, Error **errp);
+int vfio_cpr_register_container(VFIOContainerBase *bcontainer, Error **errp);
+void vfio_cpr_unregister_container(VFIOContainerBase *bcontainer);
+
extern const MemoryRegionOps vfio_region_ops;
typedef QLIST_HEAD(VFIOGroupList, VFIOGroup) VFIOGroupList;
typedef QLIST_HEAD(VFIODeviceList, VFIODevice) VFIODeviceList;
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index bd25b9fbad2e717e63c2ab0e331186e5f63cef49..096d77eac3946a9c38fc2a98116b93353f71f06e 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -621,10 +621,15 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
goto free_container_exit;
}
+ ret = vfio_cpr_register_container(bcontainer, errp);
+ if (ret) {
+ goto free_container_exit;
+ }
+
ret = vfio_ram_block_discard_disable(container, true);
if (ret) {
error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken");
- goto free_container_exit;
+ goto unregister_container_exit;
}
assert(bcontainer->ops->setup);
@@ -667,6 +672,9 @@ listener_release_exit:
enable_discards_exit:
vfio_ram_block_discard_disable(container, false);
+unregister_container_exit:
+ vfio_cpr_unregister_container(bcontainer);
+
free_container_exit:
g_free(container);
@@ -710,6 +718,7 @@ static void vfio_disconnect_container(VFIOGroup *group)
vfio_container_destroy(bcontainer);
trace_vfio_disconnect_container(container->fd);
+ vfio_cpr_unregister_container(bcontainer);
close(container->fd);
g_free(container);
diff --git a/hw/vfio/cpr.c b/hw/vfio/cpr.c
new file mode 100644
index 0000000000000000000000000000000000000000..3bede5494804e4aa291663c1e7c01d5dc5d3a407
--- /dev/null
+++ b/hw/vfio/cpr.c
@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2021-2024 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/vfio/vfio-common.h"
+#include "qapi/error.h"
+
+int vfio_cpr_register_container(VFIOContainerBase *bcontainer, Error **errp)
+{
+ return 0;
+}
+
+void vfio_cpr_unregister_container(VFIOContainerBase *bcontainer)
+{
+}
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
index 9bfddc1360895413176a9f170e29e89027384a66..e1be2244947172367e2a1f85c59967c20bfcff77 100644
--- a/hw/vfio/iommufd.c
+++ b/hw/vfio/iommufd.c
@@ -411,6 +411,11 @@ found_container:
goto err_listener_register;
}
+ ret = vfio_cpr_register_container(bcontainer, errp);
+ if (ret) {
+ goto err_listener_register;
+ }
+
/*
* TODO: examine RAM_BLOCK_DISCARD stuff, should we do group level
* for discarding incompatibility check as well?
@@ -461,6 +466,7 @@ static void iommufd_cdev_detach(VFIODevice *vbasedev)
iommufd_cdev_ram_block_discard_disable(false);
}
+ vfio_cpr_unregister_container(bcontainer);
iommufd_cdev_detach_container(vbasedev, container);
iommufd_cdev_container_destroy(container);
vfio_put_address_space(space);
diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build
index bb98493b53e858c53181e224f9cb46892838a8be..bba776f75cc718956b37f68cb542b378a509d77f 100644
--- a/hw/vfio/meson.build
+++ b/hw/vfio/meson.build
@@ -5,6 +5,7 @@ vfio_ss.add(files(
'container-base.c',
'container.c',
'migration.c',
+ 'cpr.c',
))
vfio_ss.add(when: 'CONFIG_PSERIES', if_true: files('spapr.c'))
vfio_ss.add(when: 'CONFIG_IOMMUFD', if_true: files(
--
2.44.0
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PULL 2/2] vfio: allow cpr-reboot migration if suspended
2024-03-10 19:19 [PULL 0/2] vfio queue Cédric Le Goater
2024-03-10 19:19 ` [PULL 1/2] vfio: register container for cpr Cédric Le Goater
@ 2024-03-10 19:19 ` Cédric Le Goater
2024-03-12 11:05 ` [PULL 0/2] vfio queue Peter Maydell
2 siblings, 0 replies; 4+ messages in thread
From: Cédric Le Goater @ 2024-03-10 19:19 UTC (permalink / raw)
To: qemu-devel; +Cc: Steve Sistare, Cédric Le Goater
From: Steve Sistare <steven.sistare@oracle.com>
Allow cpr-reboot for vfio if the guest is in the suspended runstate. The
guest drivers' suspend methods flush outstanding requests and re-initialize
the devices, and thus there is no device state to save and restore. The
user is responsible for suspending the guest before initiating cpr, such as
by issuing guest-suspend-ram to the qemu guest agent.
Relax the vfio blocker so it does not apply to cpr, and add a notifier that
verifies the guest is suspended.
Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
---
include/hw/vfio/vfio-container-base.h | 1 +
hw/vfio/common.c | 2 +-
hw/vfio/cpr.c | 20 ++++++++++++++++++++
hw/vfio/migration.c | 2 +-
4 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h
index b2813b0c117985425c842d91f011bb895955d738..3582d5f97a37877b2adfc0d0b06996c82403f8b7 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -49,6 +49,7 @@ typedef struct VFIOContainerBase {
QLIST_ENTRY(VFIOContainerBase) next;
QLIST_HEAD(, VFIODevice) device_list;
GList *iova_ranges;
+ NotifierWithReturn cpr_reboot_notifier;
} VFIOContainerBase;
typedef struct VFIOGuestIOMMU {
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 059bfdc07a85e2eb908df828c1f42104d683e911..ff88c3f31ca660b3c0a790601100fdc6116192a0 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -128,7 +128,7 @@ int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp)
error_setg(&multiple_devices_migration_blocker,
"Multiple VFIO devices migration is supported only if all of "
"them support P2P migration");
- ret = migrate_add_blocker(&multiple_devices_migration_blocker, errp);
+ ret = migrate_add_blocker_normal(&multiple_devices_migration_blocker, errp);
return ret;
}
diff --git a/hw/vfio/cpr.c b/hw/vfio/cpr.c
index 3bede5494804e4aa291663c1e7c01d5dc5d3a407..392c2dd95d14664bade0ae74655e91bc839504c0 100644
--- a/hw/vfio/cpr.c
+++ b/hw/vfio/cpr.c
@@ -7,13 +7,33 @@
#include "qemu/osdep.h"
#include "hw/vfio/vfio-common.h"
+#include "migration/misc.h"
#include "qapi/error.h"
+#include "sysemu/runstate.h"
+
+static int vfio_cpr_reboot_notifier(NotifierWithReturn *notifier,
+ MigrationEvent *e, Error **errp)
+{
+ if (e->type == MIG_EVENT_PRECOPY_SETUP &&
+ !runstate_check(RUN_STATE_SUSPENDED) && !vm_get_suspended()) {
+
+ error_setg(errp,
+ "VFIO device only supports cpr-reboot for runstate suspended");
+
+ return -1;
+ }
+ return 0;
+}
int vfio_cpr_register_container(VFIOContainerBase *bcontainer, Error **errp)
{
+ migration_add_notifier_mode(&bcontainer->cpr_reboot_notifier,
+ vfio_cpr_reboot_notifier,
+ MIG_MODE_CPR_REBOOT);
return 0;
}
void vfio_cpr_unregister_container(VFIOContainerBase *bcontainer)
{
+ migration_remove_notifier(&bcontainer->cpr_reboot_notifier);
}
diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 50140eda872b3d5c9438459e207682de9be06511..2050ac8897231ff89cc223f0570d5c7a65dede9e 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -889,7 +889,7 @@ static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp)
vbasedev->migration_blocker = error_copy(err);
error_free(err);
- return migrate_add_blocker(&vbasedev->migration_blocker, errp);
+ return migrate_add_blocker_normal(&vbasedev->migration_blocker, errp);
}
/* ---------------------------------------------------------------------- */
--
2.44.0
^ permalink raw reply related [flat|nested] 4+ messages in thread