* Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
2026-04-10 8:19 ` Cédric Le Goater
@ 2026-04-10 10:47 ` Anthony Krowiak
2026-04-11 7:20 ` Cédric Le Goater
2026-04-10 10:50 ` Anthony Krowiak
2026-04-13 15:23 ` Anthony Krowiak
2 siblings, 1 reply; 12+ messages in thread
From: Anthony Krowiak @ 2026-04-10 10:47 UTC (permalink / raw)
To: Cédric Le Goater, qemu-devel
Cc: qemu-s390x, mjrosato, jjherne, pasic, farman, borntraeger, alex,
cohuck
On 4/10/26 4:19 AM, Cédric Le Goater wrote:
> On 4/9/26 16:13, Anthony Krowiak wrote:
>> From: Anthony Krowiak <akrowiak@linux.ibm.com>
>>
>> Implements the callbacks used by the VFIO migration framework to migrate
>> guests with pass-through access to s390 crypto devices.
>>
>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>> ---
>> hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>> 1 file changed, 39 insertions(+), 4 deletions(-)
>>
>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>> index fd68983702..49947b18c3 100644
>> --- a/hw/vfio/ap.c
>> +++ b/hw/vfio/ap.c
>> @@ -32,6 +32,7 @@
>> #include "hw/s390x/ap-bridge.h"
>> #include "system/address-spaces.h"
>> #include "qom/object.h"
>> +#include "vfio-migration-internal.h"
>> #define TYPE_VFIO_AP_DEVICE "vfio-ap"
>> @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device
>> = {
>> .name = "vfio-ap-device",
>> .version_id = 1,
>> .minimum_version_id = 1,
>> + .unmigratable = 0,
>
> unneeded.
I'll remove it
>
>> .fields = (const VMStateField[]) {
>> VMSTATE_END_OF_LIST()
>> },
>> @@ -66,17 +68,44 @@ static void __attribute__((constructor))
>> vfio_ap_global_init(void)
>> qemu_mutex_init(&cfg_chg_events_lock);
>> }
>> +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f,
>> Error **errp)
>> +{
>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +
>> + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL,
>> errp);
>> +}
>> +
>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>> +{
>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> + Error *err = NULL;
>> + int ret;
>> +
>> + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>> + if (ret) {
>> + error_report_err(err);
>> + }
>> +
>> + return ret;
>> +}
>> +
>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>> +{
>> + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>> +
>> + return OBJECT(vdev);
>> +}
>> +
>> static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>> {
>> vdev->needs_reset = false;
>> }
>> -/*
>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>> - * vfio-ap device now.
>> - */
>> struct VFIODeviceOps vfio_ap_ops = {
>> .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>> + .vfio_save_config = vfio_ap_save_config,
>> + .vfio_load_config = vfio_ap_load_config,
>> + .vfio_get_object = vfio_ap_get_object,
>> };
>> static void vfio_ap_req_notifier_handler(void *opaque)
>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev,
>> Error **errp)
>> goto error;
>> }
>> + if (!vfio_migration_realize(vbasedev, errp)) {
>> + goto error;
>> + }
>> +
>> if (!vfio_ap_register_irq_notifier(vapdev,
>> VFIO_AP_REQ_IRQ_INDEX, &err)) {
>> /*
>> * Report this error, but do not make it a failing condition.
>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>> static const Property vfio_ap_properties[] = {
>> DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>> + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>> + vdev.enable_migration, ON_OFF_AUTO_ON),
>
> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
> has migration support. If not, the VM would still start but migration
> would be blocked.
I will make that change but I'm wondering; how does QEMU know whether the
kernel has support for migration of vfio-ap devices or not?
>
>
> thanks,
>
> C.
>
>
>> #ifdef CONFIG_IOMMUFD
>> DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
>> TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
>
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
2026-04-10 10:47 ` Anthony Krowiak
@ 2026-04-11 7:20 ` Cédric Le Goater
0 siblings, 0 replies; 12+ messages in thread
From: Cédric Le Goater @ 2026-04-11 7:20 UTC (permalink / raw)
To: Anthony Krowiak, qemu-devel
Cc: qemu-s390x, mjrosato, jjherne, pasic, farman, borntraeger, alex,
cohuck
On 4/10/26 12:47, Anthony Krowiak wrote:
>
>
> On 4/10/26 4:19 AM, Cédric Le Goater wrote:
>> On 4/9/26 16:13, Anthony Krowiak wrote:
>>> From: Anthony Krowiak <akrowiak@linux.ibm.com>
>>>
>>> Implements the callbacks used by the VFIO migration framework to migrate
>>> guests with pass-through access to s390 crypto devices.
>>>
>>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>>> ---
>>> hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>> 1 file changed, 39 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>>> index fd68983702..49947b18c3 100644
>>> --- a/hw/vfio/ap.c
>>> +++ b/hw/vfio/ap.c
>>> @@ -32,6 +32,7 @@
>>> #include "hw/s390x/ap-bridge.h"
>>> #include "system/address-spaces.h"
>>> #include "qom/object.h"
>>> +#include "vfio-migration-internal.h"
>>> #define TYPE_VFIO_AP_DEVICE "vfio-ap"
>>> @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
>>> .name = "vfio-ap-device",
>>> .version_id = 1,
>>> .minimum_version_id = 1,
>>> + .unmigratable = 0,
>>
>> unneeded.
>
> I'll remove it
>
>>
>>> .fields = (const VMStateField[]) {
>>> VMSTATE_END_OF_LIST()
>>> },
>>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
>>> qemu_mutex_init(&cfg_chg_events_lock);
>>> }
>>> +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
>>> +{
>>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +
>>> + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
>>> +}
>>> +
>>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>>> +{
>>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> + Error *err = NULL;
>>> + int ret;
>>> +
>>> + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>>> + if (ret) {
>>> + error_report_err(err);
>>> + }
>>> +
>>> + return ret;
>>> +}
>>> +
>>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>>> +{
>>> + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>>> +
>>> + return OBJECT(vdev);
>>> +}
>>> +
>>> static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>> {
>>> vdev->needs_reset = false;
>>> }
>>> -/*
>>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>>> - * vfio-ap device now.
>>> - */
>>> struct VFIODeviceOps vfio_ap_ops = {
>>> .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>>> + .vfio_save_config = vfio_ap_save_config,
>>> + .vfio_load_config = vfio_ap_load_config,
>>> + .vfio_get_object = vfio_ap_get_object,
>>> };
>>> static void vfio_ap_req_notifier_handler(void *opaque)
>>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
>>> goto error;
>>> }
>>> + if (!vfio_migration_realize(vbasedev, errp)) {
>>> + goto error;
>>> + }
>>> +
>>> if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>> /*
>>> * Report this error, but do not make it a failing condition.
>>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>> static const Property vfio_ap_properties[] = {
>>> DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>>> + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>>> + vdev.enable_migration, ON_OFF_AUTO_ON),
>>
>> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
>> has migration support. If not, the VM would still start but migration
>> would be blocked.
>
> I will make that change but I'm wondering; how does QEMU know whether the
> kernel has support for migration of vfio-ap devices or not?
vfio_migration_init() will probe the kernel for migration support
and install a migration blocker in case of failure.
You should see a message like :
VFIO migration init failed: migration is not supported in kernel
Thanks,
C.
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
2026-04-10 8:19 ` Cédric Le Goater
2026-04-10 10:47 ` Anthony Krowiak
@ 2026-04-10 10:50 ` Anthony Krowiak
2026-04-11 7:56 ` Cédric Le Goater
2026-04-13 15:23 ` Anthony Krowiak
2 siblings, 1 reply; 12+ messages in thread
From: Anthony Krowiak @ 2026-04-10 10:50 UTC (permalink / raw)
To: Cédric Le Goater, qemu-devel
Cc: qemu-s390x, mjrosato, jjherne, pasic, farman, borntraeger, alex,
cohuck
On 4/10/26 4:19 AM, Cédric Le Goater wrote:
> On 4/9/26 16:13, Anthony Krowiak wrote:
>> From: Anthony Krowiak <akrowiak@linux.ibm.com>
>>
>> Implements the callbacks used by the VFIO migration framework to migrate
>> guests with pass-through access to s390 crypto devices.
>>
>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>> ---
>> hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>> 1 file changed, 39 insertions(+), 4 deletions(-)
>>
>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>> index fd68983702..49947b18c3 100644
>> --- a/hw/vfio/ap.c
>> +++ b/hw/vfio/ap.c
>> @@ -32,6 +32,7 @@
>> #include "hw/s390x/ap-bridge.h"
>> #include "system/address-spaces.h"
>> #include "qom/object.h"
>> +#include "vfio-migration-internal.h"
>> #define TYPE_VFIO_AP_DEVICE "vfio-ap"
>> @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device
>> = {
>> .name = "vfio-ap-device",
>> .version_id = 1,
>> .minimum_version_id = 1,
>> + .unmigratable = 0,
>
> unneeded.
>
>> .fields = (const VMStateField[]) {
>> VMSTATE_END_OF_LIST()
>> },
>> @@ -66,17 +68,44 @@ static void __attribute__((constructor))
>> vfio_ap_global_init(void)
>> qemu_mutex_init(&cfg_chg_events_lock);
>> }
>> +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f,
>> Error **errp)
>> +{
>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +
>> + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL,
>> errp);
>> +}
>> +
>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>> +{
>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> + Error *err = NULL;
>> + int ret;
>> +
>> + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>> + if (ret) {
>> + error_report_err(err);
>> + }
>> +
>> + return ret;
>> +}
>> +
>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>> +{
>> + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>> +
>> + return OBJECT(vdev);
>> +}
>> +
>> static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>> {
>> vdev->needs_reset = false;
>> }
>> -/*
>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>> - * vfio-ap device now.
>> - */
>> struct VFIODeviceOps vfio_ap_ops = {
>> .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>> + .vfio_save_config = vfio_ap_save_config,
>> + .vfio_load_config = vfio_ap_load_config,
>> + .vfio_get_object = vfio_ap_get_object,
>> };
>> static void vfio_ap_req_notifier_handler(void *opaque)
>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev,
>> Error **errp)
>> goto error;
>> }
>> + if (!vfio_migration_realize(vbasedev, errp)) {
>> + goto error;
>> + }
>> +
>> if (!vfio_ap_register_irq_notifier(vapdev,
>> VFIO_AP_REQ_IRQ_INDEX, &err)) {
>> /*
>> * Report this error, but do not make it a failing condition.
>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>> static const Property vfio_ap_properties[] = {
>> DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>> + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>> + vdev.enable_migration, ON_OFF_AUTO_ON),
>
> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
> has migration support. If not, the VM would still start but migration
> would be blocked.
Will this allow the vfio-ap device migration patches to get merged before
the kernel/KVM patches?
>
> thanks,
>
> C.
>
>
>> #ifdef CONFIG_IOMMUFD
>> DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
>> TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
>
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
2026-04-10 10:50 ` Anthony Krowiak
@ 2026-04-11 7:56 ` Cédric Le Goater
0 siblings, 0 replies; 12+ messages in thread
From: Cédric Le Goater @ 2026-04-11 7:56 UTC (permalink / raw)
To: Anthony Krowiak, qemu-devel
Cc: qemu-s390x, mjrosato, jjherne, pasic, farman, borntraeger, alex,
cohuck
On 4/10/26 12:50, Anthony Krowiak wrote:
>
>
> On 4/10/26 4:19 AM, Cédric Le Goater wrote:
>> On 4/9/26 16:13, Anthony Krowiak wrote:
>>> From: Anthony Krowiak <akrowiak@linux.ibm.com>
>>>
>>> Implements the callbacks used by the VFIO migration framework to migrate
>>> guests with pass-through access to s390 crypto devices.
>>>
>>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>>> ---
>>> hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>> 1 file changed, 39 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>>> index fd68983702..49947b18c3 100644
>>> --- a/hw/vfio/ap.c
>>> +++ b/hw/vfio/ap.c
>>> @@ -32,6 +32,7 @@
>>> #include "hw/s390x/ap-bridge.h"
>>> #include "system/address-spaces.h"
>>> #include "qom/object.h"
>>> +#include "vfio-migration-internal.h"
>>> #define TYPE_VFIO_AP_DEVICE "vfio-ap"
>>> @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
>>> .name = "vfio-ap-device",
>>> .version_id = 1,
>>> .minimum_version_id = 1,
>>> + .unmigratable = 0,
>>
>> unneeded.
>>
>>> .fields = (const VMStateField[]) {
>>> VMSTATE_END_OF_LIST()
>>> },
>>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
>>> qemu_mutex_init(&cfg_chg_events_lock);
>>> }
>>> +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
>>> +{
>>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +
>>> + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
>>> +}
>>> +
>>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>>> +{
>>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> + Error *err = NULL;
>>> + int ret;
>>> +
>>> + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>>> + if (ret) {
>>> + error_report_err(err);
>>> + }
>>> +
>>> + return ret;
>>> +}
>>> +
>>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>>> +{
>>> + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>>> +
>>> + return OBJECT(vdev);
>>> +}
>>> +
>>> static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>> {
>>> vdev->needs_reset = false;
>>> }
>>> -/*
>>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>>> - * vfio-ap device now.
>>> - */
>>> struct VFIODeviceOps vfio_ap_ops = {
>>> .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>>> + .vfio_save_config = vfio_ap_save_config,
>>> + .vfio_load_config = vfio_ap_load_config,
>>> + .vfio_get_object = vfio_ap_get_object,
>>> };
>>> static void vfio_ap_req_notifier_handler(void *opaque)
>>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
>>> goto error;
>>> }
>>> + if (!vfio_migration_realize(vbasedev, errp)) {
>>> + goto error;
>>> + }
>>> +
>>> if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>> /*
>>> * Report this error, but do not make it a failing condition.
>>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>> static const Property vfio_ap_properties[] = {
>>> DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>>> + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>>> + vdev.enable_migration, ON_OFF_AUTO_ON),
>>
>> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
>> has migration support. If not, the VM would still start but migration
>> would be blocked.
>
> Will this allow the vfio-ap device migration patches to get merged before
> the kernel/KVM patches?
It's possible. Several reasons could explain the lack of migration
support. The most obvious is a lack of kernel support, but it could
also be a firmware support issue, an incompatible hardware state of
the adapter, adapter model, etc.
Do all CEXxS crypto cards support migration ?
Do you expect kernel support [*] to be merged in v7.1 ?
Also, I think the QEMU support lacks vfio_migration_exit().
Thanks,
C.
[*] https://lore.kernel.org/r/20260407205100.331150-1-akrowiak@linux.ibm.com
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
2026-04-10 8:19 ` Cédric Le Goater
2026-04-10 10:47 ` Anthony Krowiak
2026-04-10 10:50 ` Anthony Krowiak
@ 2026-04-13 15:23 ` Anthony Krowiak
2026-04-16 15:05 ` Cédric Le Goater
2 siblings, 1 reply; 12+ messages in thread
From: Anthony Krowiak @ 2026-04-13 15:23 UTC (permalink / raw)
To: Cédric Le Goater, qemu-devel
Cc: qemu-s390x, mjrosato, jjherne, pasic, farman, borntraeger, alex,
cohuck
On 4/10/26 4:19 AM, Cédric Le Goater wrote:
> On 4/9/26 16:13, Anthony Krowiak wrote:
>> From: Anthony Krowiak <akrowiak@linux.ibm.com>
>>
>> Implements the callbacks used by the VFIO migration framework to migrate
>> guests with pass-through access to s390 crypto devices.
>>
>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>> ---
>> hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>> 1 file changed, 39 insertions(+), 4 deletions(-)
>>
>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>> index fd68983702..49947b18c3 100644
>> --- a/hw/vfio/ap.c
>> +++ b/hw/vfio/ap.c
>> @@ -32,6 +32,7 @@
>> #include "hw/s390x/ap-bridge.h"
>> #include "system/address-spaces.h"
>> #include "qom/object.h"
>> +#include "vfio-migration-internal.h"
>> #define TYPE_VFIO_AP_DEVICE "vfio-ap"
>> @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device
>> = {
>> .name = "vfio-ap-device",
>> .version_id = 1,
>> .minimum_version_id = 1,
>> + .unmigratable = 0,
>
> unneeded.
>
>> .fields = (const VMStateField[]) {
>> VMSTATE_END_OF_LIST()
>> },
>> @@ -66,17 +68,44 @@ static void __attribute__((constructor))
>> vfio_ap_global_init(void)
>> qemu_mutex_init(&cfg_chg_events_lock);
>> }
>> +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f,
>> Error **errp)
>> +{
>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +
>> + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL,
>> errp);
>> +}
>> +
>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>> +{
>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> + Error *err = NULL;
>> + int ret;
>> +
>> + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>> + if (ret) {
>> + error_report_err(err);
>> + }
>> +
>> + return ret;
>> +}
>> +
>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>> +{
>> + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>> +
>> + return OBJECT(vdev);
>> +}
>> +
>> static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>> {
>> vdev->needs_reset = false;
>> }
>> -/*
>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>> - * vfio-ap device now.
>> - */
>> struct VFIODeviceOps vfio_ap_ops = {
>> .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>> + .vfio_save_config = vfio_ap_save_config,
>> + .vfio_load_config = vfio_ap_load_config,
>> + .vfio_get_object = vfio_ap_get_object,
>> };
>> static void vfio_ap_req_notifier_handler(void *opaque)
>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev,
>> Error **errp)
>> goto error;
>> }
>> + if (!vfio_migration_realize(vbasedev, errp)) {
>> + goto error;
>> + }
>> +
>> if (!vfio_ap_register_irq_notifier(vapdev,
>> VFIO_AP_REQ_IRQ_INDEX, &err)) {
>> /*
>> * Report this error, but do not make it a failing condition.
>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>> static const Property vfio_ap_properties[] = {
>> DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>> + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>> + vdev.enable_migration, ON_OFF_AUTO_ON),
>
> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
> has migration support. If not, the VM would still start but migration
> would be blocked.
I made this change and migration failed on the destination host with the
following error message:
error: Requested operation is not valid: cannot migrate domain:
62177883-f1bb-47f0-914d-32a22e3a8804: VFIO device doesn't support device
and IOMMU dirty tracking
This despite having the following property defined:
DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOAPDevice,
vdev.device_dirty_page_tracking, ON_OFF_AUTO_ON)
>
> thanks,
>
> C.
>
>
>> #ifdef CONFIG_IOMMUFD
>> DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
>> TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
>
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices
2026-04-13 15:23 ` Anthony Krowiak
@ 2026-04-16 15:05 ` Cédric Le Goater
0 siblings, 0 replies; 12+ messages in thread
From: Cédric Le Goater @ 2026-04-16 15:05 UTC (permalink / raw)
To: Anthony Krowiak, qemu-devel
Cc: qemu-s390x, mjrosato, jjherne, pasic, farman, borntraeger, alex,
cohuck
On 4/13/26 17:23, Anthony Krowiak wrote:
>
>
> On 4/10/26 4:19 AM, Cédric Le Goater wrote:
>> On 4/9/26 16:13, Anthony Krowiak wrote:
>>> From: Anthony Krowiak <akrowiak@linux.ibm.com>
>>>
>>> Implements the callbacks used by the VFIO migration framework to migrate
>>> guests with pass-through access to s390 crypto devices.
>>>
>>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>>> ---
>>> hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>> 1 file changed, 39 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>>> index fd68983702..49947b18c3 100644
>>> --- a/hw/vfio/ap.c
>>> +++ b/hw/vfio/ap.c
>>> @@ -32,6 +32,7 @@
>>> #include "hw/s390x/ap-bridge.h"
>>> #include "system/address-spaces.h"
>>> #include "qom/object.h"
>>> +#include "vfio-migration-internal.h"
>>> #define TYPE_VFIO_AP_DEVICE "vfio-ap"
>>> @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
>>> .name = "vfio-ap-device",
>>> .version_id = 1,
>>> .minimum_version_id = 1,
>>> + .unmigratable = 0,
>>
>> unneeded.
>>
>>> .fields = (const VMStateField[]) {
>>> VMSTATE_END_OF_LIST()
>>> },
>>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
>>> qemu_mutex_init(&cfg_chg_events_lock);
>>> }
>>> +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
>>> +{
>>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +
>>> + return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
>>> +}
>>> +
>>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>>> +{
>>> + VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> + Error *err = NULL;
>>> + int ret;
>>> +
>>> + ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>>> + if (ret) {
>>> + error_report_err(err);
>>> + }
>>> +
>>> + return ret;
>>> +}
>>> +
>>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>>> +{
>>> + VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>>> +
>>> + return OBJECT(vdev);
>>> +}
>>> +
>>> static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>> {
>>> vdev->needs_reset = false;
>>> }
>>> -/*
>>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>>> - * vfio-ap device now.
>>> - */
>>> struct VFIODeviceOps vfio_ap_ops = {
>>> .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>>> + .vfio_save_config = vfio_ap_save_config,
>>> + .vfio_load_config = vfio_ap_load_config,
>>> + .vfio_get_object = vfio_ap_get_object,
>>> };
>>> static void vfio_ap_req_notifier_handler(void *opaque)
>>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
>>> goto error;
>>> }
>>> + if (!vfio_migration_realize(vbasedev, errp)) {
>>> + goto error;
>>> + }
>>> +
>>> if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>> /*
>>> * Report this error, but do not make it a failing condition.
>>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>> static const Property vfio_ap_properties[] = {
>>> DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>>> + DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>>> + vdev.enable_migration, ON_OFF_AUTO_ON),
>>
>> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
>> has migration support. If not, the VM would still start but migration
>> would be blocked.
>
> I made this change and migration failed on the destination host with the
> following error message:
>
> error: Requested operation is not valid: cannot migrate domain: 62177883-f1bb-47f0-914d-32a22e3a8804: VFIO device doesn't support device and IOMMU dirty tracking
>
> This despite having the following property defined:
>
> DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOAPDevice, vdev.device_dirty_page_tracking, ON_OFF_AUTO_ON)
try adding dummy vfio_log_ops handlers in the driver.
static int mtty_log_start(struct vfio_device *vdev,
struct rb_root_cached *ranges,
u32 nnodes, u64 *page_size)
{
return 0;
}
static int mtty_log_stop(struct vfio_device *vdev)
{
return 0;
}
static int mtty_log_read_and_clear(struct vfio_device *vdev,
unsigned long iova, unsigned long length,
struct iova_bitmap *dirty)
{
return 0;
}
static const struct vfio_log_ops mtty_log_ops = {
.log_start = mtty_log_start,
.log_stop = mtty_log_stop,
.log_read_and_clear = mtty_log_read_and_clear,
};
See samples/vfio-mdev/mtty.c.
C.
^ permalink raw reply [flat|nested] 12+ messages in thread