[PATCH v2 0/2] Enable live migration of vfio-ap devices

[PATCH v2 0/2] Enable live migration of vfio-ap devices

[Anthony Krowiak]

This patch series enables live guest migration for s390 guests using
AP (Adjunct Processor) crypto devices passed through via the VFIO
framework.

The s390 AP architecture provides cryptographic coprocessor and accelerator
functions to guests. The vfio_ap device driver allows these AP devices to
be passed through to KVM guests, giving them access to the cryptographic 
capabilities of CEXxS crypto cards.

Previously, guests using vfio-ap devices could not be migrated because the
device was marked as unmigratable. This series implements the necessary
QEMU-side support to enable migration by:

* Removing the migration blocker and adding proper VMState infrastructure
* Implementing the VFIO migration callbacks that coordinate with the kernel
  vfio_ap driver to save and restore device state

The actual AP device state (queue configurations, crypto adapter
assignments, etc.) is managed entirely by the kernel vfio_ap driver and
transferred via the VFIO migration protocol. QEMU's role is to coordinate
the migration process and ensure proper sequencing of state transfers.

This implementation follows the standard VFIO migration framework used by
other VFIO device types (GPUs, network adapters, etc.) and integrates
cleanly with QEMU's existing migration infrastructure.

Dependencies:
------------
* Requires corresponding kernel patches that implement the VFIO migration
  protocol in the vfio_ap driver
    - see kernel patch series at https://lkml.org/lkml/2026/4/7/1991
* Kernel must support VFIO_DEVICE_FEATURE_MIGRATION for vfio-ap devices

Anthony  Krowiak (1):
  hw/vfio/ap: Callbacks for migration of guests with pass-through access
    to AP devices

Anthony Krowiak (1):
  hw/vfio/ap: Extend vfio device state description to facilitate live
    migration

 hw/vfio/ap.c | 60 ++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 49 insertions(+), 11 deletions(-)

-- 
2.52.0

[PATCH v2 1/2] hw/vfio/ap: Extend vfio device state description to facilitate live migration

[Anthony Krowiak]

Make the changes needed to the VFIOAPDevice state description to facilitate
live guest migration.

Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
Reviewed-by: Jason J. Herne <jjherne@linux.ibm.com>
---
 hw/vfio/ap.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 5c8f305653..fd68983702 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -41,6 +41,16 @@ struct VFIOAPDevice {
     EventNotifier req_notifier;
     EventNotifier cfg_notifier;
 };
+OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
+
+static const VMStateDescription vmstate_ap_device = {
+    .name = "vfio-ap-device",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (const VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    },
+};
 
 typedef struct APConfigChgEvent {
     QTAILQ_ENTRY(APConfigChgEvent) next;
@@ -56,8 +66,6 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
     qemu_mutex_init(&cfg_chg_events_lock);
 }
 
-OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
-
 static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
 {
     vdev->needs_reset = false;
@@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev)
     }
 }
 
-static const VMStateDescription vfio_ap_vmstate = {
-    .name = "vfio-ap",
-    .unmigratable = 1,
-};
-
 static void vfio_ap_instance_init(Object *obj)
 {
     VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
@@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass *klass, const void *data)
 #ifdef CONFIG_IOMMUFD
     object_class_property_add_str(klass, "fd", NULL, vfio_ap_set_fd);
 #endif
-    dc->vmsd = &vfio_ap_vmstate;
+    dc->vmsd = &vmstate_ap_device;
     dc->desc = "VFIO-based AP device assignment";
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     dc->realize = vfio_ap_realize;
-- 
2.52.0

[PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

[Anthony Krowiak]

From: Anthony  Krowiak <akrowiak@linux.ibm.com>

Implements the callbacks used by the VFIO migration framework to migrate
guests with pass-through access to s390 crypto devices.

Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
---
 hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index fd68983702..49947b18c3 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -32,6 +32,7 @@
 #include "hw/s390x/ap-bridge.h"
 #include "system/address-spaces.h"
 #include "qom/object.h"
+#include "vfio-migration-internal.h"
 
 #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
 
@@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
     .name = "vfio-ap-device",
     .version_id = 1,
     .minimum_version_id = 1,
+    .unmigratable = 0,
     .fields = (const VMStateField[]) {
         VMSTATE_END_OF_LIST()
     },
@@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
     qemu_mutex_init(&cfg_chg_events_lock);
 }
 
+static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+
+    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
+}
+
+static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
+{
+    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
+    Error *err = NULL;
+    int ret;
+
+    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
+    if (ret) {
+        error_report_err(err);
+    }
+
+    return ret;
+}
+
+static Object *vfio_ap_get_object(VFIODevice *vbasedev)
+{
+    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
+
+    return OBJECT(vdev);
+}
+
 static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
 {
     vdev->needs_reset = false;
 }
 
-/*
- * We don't need vfio_hot_reset_multi and vfio_eoi operations for
- * vfio-ap device now.
- */
 struct VFIODeviceOps vfio_ap_ops = {
     .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+    .vfio_save_config = vfio_ap_save_config,
+    .vfio_load_config = vfio_ap_load_config,
+    .vfio_get_object = vfio_ap_get_object,
 };
 
 static void vfio_ap_req_notifier_handler(void *opaque)
@@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
         goto error;
     }
 
+    if (!vfio_migration_realize(vbasedev, errp)) {
+        goto error;
+    }
+
     if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
         /*
          * Report this error, but do not make it a failing condition.
@@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
 
 static const Property vfio_ap_properties[] = {
     DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
+                            vdev.enable_migration, ON_OFF_AUTO_ON),
 #ifdef CONFIG_IOMMUFD
     DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
                      TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
-- 
2.52.0

Re: [PATCH v2 1/2] hw/vfio/ap: Extend vfio device state description to facilitate live migration

[Cédric Le Goater]

On 4/9/26 16:13, Anthony Krowiak wrote:
> Make the changes needed to the VFIOAPDevice state description to facilitate
> live guest migration.
> 
> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
> Reviewed-by: Jason J. Herne <jjherne@linux.ibm.com>
> ---
>   hw/vfio/ap.c | 19 +++++++++++--------
>   1 file changed, 11 insertions(+), 8 deletions(-)
> 
> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
> index 5c8f305653..fd68983702 100644
> --- a/hw/vfio/ap.c
> +++ b/hw/vfio/ap.c
> @@ -41,6 +41,16 @@ struct VFIOAPDevice {
>       EventNotifier req_notifier;
>       EventNotifier cfg_notifier;
>   };
> +OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
> +
> +static const VMStateDescription vmstate_ap_device = {
> +    .name = "vfio-ap-device",
> +    .version_id = 1,
> +    .minimum_version_id = 1,
> +    .fields = (const VMStateField[]) {
> +        VMSTATE_END_OF_LIST()
> +    },
> +};
>   
>   typedef struct APConfigChgEvent {
>       QTAILQ_ENTRY(APConfigChgEvent) next;
> @@ -56,8 +66,6 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
>       qemu_mutex_init(&cfg_chg_events_lock);
>   }
>   
> -OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
> -
>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>   {
>       vdev->needs_reset = false;
> @@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev)
>       }
>   }
>   
> -static const VMStateDescription vfio_ap_vmstate = {
> -    .name = "vfio-ap",
> -    .unmigratable = 1,
> -};
> -
>   static void vfio_ap_instance_init(Object *obj)
>   {
>       VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
> @@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass *klass, const void *data)
>   #ifdef CONFIG_IOMMUFD
>       object_class_property_add_str(klass, "fd", NULL, vfio_ap_set_fd);
>   #endif

You will need to rebase on 'vfio-next' or 'master' when the changes
pending in the vfio queue are merged.


> -    dc->vmsd = &vfio_ap_vmstate;
> +    dc->vmsd = &vmstate_ap_device;
>       dc->desc = "VFIO-based AP device assignment";
>       set_bit(DEVICE_CATEGORY_MISC, dc->categories);
>       dc->realize = vfio_ap_realize;

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

[Cédric Le Goater]

On 4/9/26 16:13, Anthony Krowiak wrote:
> From: Anthony  Krowiak <akrowiak@linux.ibm.com>
> 
> Implements the callbacks used by the VFIO migration framework to migrate
> guests with pass-through access to s390 crypto devices.
> 
> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
> ---
>   hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>   1 file changed, 39 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
> index fd68983702..49947b18c3 100644
> --- a/hw/vfio/ap.c
> +++ b/hw/vfio/ap.c
> @@ -32,6 +32,7 @@
>   #include "hw/s390x/ap-bridge.h"
>   #include "system/address-spaces.h"
>   #include "qom/object.h"
> +#include "vfio-migration-internal.h"
>   
>   #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
>   
> @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
>       .name = "vfio-ap-device",
>       .version_id = 1,
>       .minimum_version_id = 1,
> +    .unmigratable = 0,

unneeded.

>       .fields = (const VMStateField[]) {
>           VMSTATE_END_OF_LIST()
>       },
> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
>       qemu_mutex_init(&cfg_chg_events_lock);
>   }
>   
> +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
> +{
> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
> +
> +    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
> +}
> +
> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
> +{
> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
> +    Error *err = NULL;
> +    int ret;
> +
> +    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
> +    if (ret) {
> +        error_report_err(err);
> +    }
> +
> +    return ret;
> +}
> +
> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
> +{
> +    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
> +
> +    return OBJECT(vdev);
> +}
> +
>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>   {
>       vdev->needs_reset = false;
>   }
>   
> -/*
> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
> - * vfio-ap device now.
> - */
>   struct VFIODeviceOps vfio_ap_ops = {
>       .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
> +    .vfio_save_config = vfio_ap_save_config,
> +    .vfio_load_config = vfio_ap_load_config,
> +    .vfio_get_object = vfio_ap_get_object,
>   };
>   
>   static void vfio_ap_req_notifier_handler(void *opaque)
> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
>           goto error;
>       }
>   
> +    if (!vfio_migration_realize(vbasedev, errp)) {
> +        goto error;
> +    }
> +
>       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
>           /*
>            * Report this error, but do not make it a failing condition.
> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>   
>   static const Property vfio_ap_properties[] = {
>       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
> +    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
> +                            vdev.enable_migration, ON_OFF_AUTO_ON),

Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
has migration support. If not, the VM would still start but migration
would be blocked.

thanks,

C.


>   #ifdef CONFIG_IOMMUFD
>       DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
>                        TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

[Anthony Krowiak]

On 4/10/26 4:19 AM, Cédric Le Goater wrote:
> On 4/9/26 16:13, Anthony Krowiak wrote:
>> From: Anthony  Krowiak <akrowiak@linux.ibm.com>
>>
>> Implements the callbacks used by the VFIO migration framework to migrate
>> guests with pass-through access to s390 crypto devices.
>>
>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>> ---
>>   hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>   1 file changed, 39 insertions(+), 4 deletions(-)
>>
>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>> index fd68983702..49947b18c3 100644
>> --- a/hw/vfio/ap.c
>> +++ b/hw/vfio/ap.c
>> @@ -32,6 +32,7 @@
>>   #include "hw/s390x/ap-bridge.h"
>>   #include "system/address-spaces.h"
>>   #include "qom/object.h"
>> +#include "vfio-migration-internal.h"
>>     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
>>   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device 
>> = {
>>       .name = "vfio-ap-device",
>>       .version_id = 1,
>>       .minimum_version_id = 1,
>> +    .unmigratable = 0,
>
> unneeded.

I'll remove it

>
>>       .fields = (const VMStateField[]) {
>>           VMSTATE_END_OF_LIST()
>>       },
>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) 
>> vfio_ap_global_init(void)
>>       qemu_mutex_init(&cfg_chg_events_lock);
>>   }
>>   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, 
>> Error **errp)
>> +{
>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +
>> +    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, 
>> errp);
>> +}
>> +
>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>> +{
>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +    Error *err = NULL;
>> +    int ret;
>> +
>> +    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>> +    if (ret) {
>> +        error_report_err(err);
>> +    }
>> +
>> +    return ret;
>> +}
>> +
>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>> +{
>> +    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>> +
>> +    return OBJECT(vdev);
>> +}
>> +
>>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>   {
>>       vdev->needs_reset = false;
>>   }
>>   -/*
>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>> - * vfio-ap device now.
>> - */
>>   struct VFIODeviceOps vfio_ap_ops = {
>>       .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>> +    .vfio_save_config = vfio_ap_save_config,
>> +    .vfio_load_config = vfio_ap_load_config,
>> +    .vfio_get_object = vfio_ap_get_object,
>>   };
>>     static void vfio_ap_req_notifier_handler(void *opaque)
>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, 
>> Error **errp)
>>           goto error;
>>       }
>>   +    if (!vfio_migration_realize(vbasedev, errp)) {
>> +        goto error;
>> +    }
>> +
>>       if (!vfio_ap_register_irq_notifier(vapdev, 
>> VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>           /*
>>            * Report this error, but do not make it a failing condition.
>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>     static const Property vfio_ap_properties[] = {
>>       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>> +    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>> +                            vdev.enable_migration, ON_OFF_AUTO_ON),
>
> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
> has migration support. If not, the VM would still start but migration
> would be blocked.

I will make that change but I'm wondering; how does QEMU know whether the
kernel has support for migration of vfio-ap devices or not?

>
>
> thanks,
>
> C.
>
>
>>   #ifdef CONFIG_IOMMUFD
>>       DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
>>                        TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
>

Re: [PATCH v2 1/2] hw/vfio/ap: Extend vfio device state description to facilitate live migration

[Anthony Krowiak]

On 4/10/26 4:17 AM, Cédric Le Goater wrote:
> On 4/9/26 16:13, Anthony Krowiak wrote:
>> Make the changes needed to the VFIOAPDevice state description to 
>> facilitate
>> live guest migration.
>>
>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>> Reviewed-by: Jason J. Herne <jjherne@linux.ibm.com>
>> ---
>>   hw/vfio/ap.c | 19 +++++++++++--------
>>   1 file changed, 11 insertions(+), 8 deletions(-)
>>
>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>> index 5c8f305653..fd68983702 100644
>> --- a/hw/vfio/ap.c
>> +++ b/hw/vfio/ap.c
>> @@ -41,6 +41,16 @@ struct VFIOAPDevice {
>>       EventNotifier req_notifier;
>>       EventNotifier cfg_notifier;
>>   };
>> +OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
>> +
>> +static const VMStateDescription vmstate_ap_device = {
>> +    .name = "vfio-ap-device",
>> +    .version_id = 1,
>> +    .minimum_version_id = 1,
>> +    .fields = (const VMStateField[]) {
>> +        VMSTATE_END_OF_LIST()
>> +    },
>> +};
>>     typedef struct APConfigChgEvent {
>>       QTAILQ_ENTRY(APConfigChgEvent) next;
>> @@ -56,8 +66,6 @@ static void __attribute__((constructor)) 
>> vfio_ap_global_init(void)
>>       qemu_mutex_init(&cfg_chg_events_lock);
>>   }
>>   -OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
>> -
>>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>   {
>>       vdev->needs_reset = false;
>> @@ -297,11 +305,6 @@ static void vfio_ap_reset(DeviceState *dev)
>>       }
>>   }
>>   -static const VMStateDescription vfio_ap_vmstate = {
>> -    .name = "vfio-ap",
>> -    .unmigratable = 1,
>> -};
>> -
>>   static void vfio_ap_instance_init(Object *obj)
>>   {
>>       VFIOAPDevice *vapdev = VFIO_AP_DEVICE(obj);
>> @@ -335,7 +338,7 @@ static void vfio_ap_class_init(ObjectClass 
>> *klass, const void *data)
>>   #ifdef CONFIG_IOMMUFD
>>       object_class_property_add_str(klass, "fd", NULL, vfio_ap_set_fd);
>>   #endif
>
> You will need to rebase on 'vfio-next' or 'master' when the changes
> pending in the vfio queue are merged.

Will do. Thanks for the review.

>
>
>> -    dc->vmsd = &vfio_ap_vmstate;
>> +    dc->vmsd = &vmstate_ap_device;
>>       dc->desc = "VFIO-based AP device assignment";
>>       set_bit(DEVICE_CATEGORY_MISC, dc->categories);
>>       dc->realize = vfio_ap_realize;
>

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

[Anthony Krowiak]

On 4/10/26 4:19 AM, Cédric Le Goater wrote:
> On 4/9/26 16:13, Anthony Krowiak wrote:
>> From: Anthony  Krowiak <akrowiak@linux.ibm.com>
>>
>> Implements the callbacks used by the VFIO migration framework to migrate
>> guests with pass-through access to s390 crypto devices.
>>
>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>> ---
>>   hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>   1 file changed, 39 insertions(+), 4 deletions(-)
>>
>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>> index fd68983702..49947b18c3 100644
>> --- a/hw/vfio/ap.c
>> +++ b/hw/vfio/ap.c
>> @@ -32,6 +32,7 @@
>>   #include "hw/s390x/ap-bridge.h"
>>   #include "system/address-spaces.h"
>>   #include "qom/object.h"
>> +#include "vfio-migration-internal.h"
>>     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
>>   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device 
>> = {
>>       .name = "vfio-ap-device",
>>       .version_id = 1,
>>       .minimum_version_id = 1,
>> +    .unmigratable = 0,
>
> unneeded.
>
>>       .fields = (const VMStateField[]) {
>>           VMSTATE_END_OF_LIST()
>>       },
>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) 
>> vfio_ap_global_init(void)
>>       qemu_mutex_init(&cfg_chg_events_lock);
>>   }
>>   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, 
>> Error **errp)
>> +{
>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +
>> +    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, 
>> errp);
>> +}
>> +
>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>> +{
>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +    Error *err = NULL;
>> +    int ret;
>> +
>> +    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>> +    if (ret) {
>> +        error_report_err(err);
>> +    }
>> +
>> +    return ret;
>> +}
>> +
>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>> +{
>> +    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>> +
>> +    return OBJECT(vdev);
>> +}
>> +
>>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>   {
>>       vdev->needs_reset = false;
>>   }
>>   -/*
>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>> - * vfio-ap device now.
>> - */
>>   struct VFIODeviceOps vfio_ap_ops = {
>>       .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>> +    .vfio_save_config = vfio_ap_save_config,
>> +    .vfio_load_config = vfio_ap_load_config,
>> +    .vfio_get_object = vfio_ap_get_object,
>>   };
>>     static void vfio_ap_req_notifier_handler(void *opaque)
>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, 
>> Error **errp)
>>           goto error;
>>       }
>>   +    if (!vfio_migration_realize(vbasedev, errp)) {
>> +        goto error;
>> +    }
>> +
>>       if (!vfio_ap_register_irq_notifier(vapdev, 
>> VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>           /*
>>            * Report this error, but do not make it a failing condition.
>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>     static const Property vfio_ap_properties[] = {
>>       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>> +    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>> +                            vdev.enable_migration, ON_OFF_AUTO_ON),
>
> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
> has migration support. If not, the VM would still start but migration
> would be blocked.

Will this allow the vfio-ap device migration patches to get merged before
the kernel/KVM patches?

>
> thanks,
>
> C.
>
>
>>   #ifdef CONFIG_IOMMUFD
>>       DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
>>                        TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
>

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

[Cédric Le Goater]

On 4/10/26 12:47, Anthony Krowiak wrote:
> 
> 
> On 4/10/26 4:19 AM, Cédric Le Goater wrote:
>> On 4/9/26 16:13, Anthony Krowiak wrote:
>>> From: Anthony  Krowiak <akrowiak@linux.ibm.com>
>>>
>>> Implements the callbacks used by the VFIO migration framework to migrate
>>> guests with pass-through access to s390 crypto devices.
>>>
>>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>>> ---
>>>   hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>>   1 file changed, 39 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>>> index fd68983702..49947b18c3 100644
>>> --- a/hw/vfio/ap.c
>>> +++ b/hw/vfio/ap.c
>>> @@ -32,6 +32,7 @@
>>>   #include "hw/s390x/ap-bridge.h"
>>>   #include "system/address-spaces.h"
>>>   #include "qom/object.h"
>>> +#include "vfio-migration-internal.h"
>>>     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
>>>   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
>>>       .name = "vfio-ap-device",
>>>       .version_id = 1,
>>>       .minimum_version_id = 1,
>>> +    .unmigratable = 0,
>>
>> unneeded.
> 
> I'll remove it
> 
>>
>>>       .fields = (const VMStateField[]) {
>>>           VMSTATE_END_OF_LIST()
>>>       },
>>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
>>>       qemu_mutex_init(&cfg_chg_events_lock);
>>>   }
>>>   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
>>> +{
>>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +
>>> +    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
>>> +}
>>> +
>>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>>> +{
>>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +    Error *err = NULL;
>>> +    int ret;
>>> +
>>> +    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>>> +    if (ret) {
>>> +        error_report_err(err);
>>> +    }
>>> +
>>> +    return ret;
>>> +}
>>> +
>>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>>> +{
>>> +    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>>> +
>>> +    return OBJECT(vdev);
>>> +}
>>> +
>>>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>>   {
>>>       vdev->needs_reset = false;
>>>   }
>>>   -/*
>>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>>> - * vfio-ap device now.
>>> - */
>>>   struct VFIODeviceOps vfio_ap_ops = {
>>>       .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>>> +    .vfio_save_config = vfio_ap_save_config,
>>> +    .vfio_load_config = vfio_ap_load_config,
>>> +    .vfio_get_object = vfio_ap_get_object,
>>>   };
>>>     static void vfio_ap_req_notifier_handler(void *opaque)
>>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
>>>           goto error;
>>>       }
>>>   +    if (!vfio_migration_realize(vbasedev, errp)) {
>>> +        goto error;
>>> +    }
>>> +
>>>       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>>           /*
>>>            * Report this error, but do not make it a failing condition.
>>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>>     static const Property vfio_ap_properties[] = {
>>>       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>>> +    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>>> +                            vdev.enable_migration, ON_OFF_AUTO_ON),
>>
>> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
>> has migration support. If not, the VM would still start but migration
>> would be blocked.
> 
> I will make that change but I'm wondering; how does QEMU know whether the
> kernel has support for migration of vfio-ap devices or not?

vfio_migration_init() will probe the kernel for migration support
and install a migration blocker in case of failure.

You should see a message like :

   VFIO migration init failed: migration is not supported in kernel


Thanks,

C.

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

[Cédric Le Goater]

On 4/10/26 12:50, Anthony Krowiak wrote:
> 
> 
> On 4/10/26 4:19 AM, Cédric Le Goater wrote:
>> On 4/9/26 16:13, Anthony Krowiak wrote:
>>> From: Anthony  Krowiak <akrowiak@linux.ibm.com>
>>>
>>> Implements the callbacks used by the VFIO migration framework to migrate
>>> guests with pass-through access to s390 crypto devices.
>>>
>>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>>> ---
>>>   hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>>   1 file changed, 39 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>>> index fd68983702..49947b18c3 100644
>>> --- a/hw/vfio/ap.c
>>> +++ b/hw/vfio/ap.c
>>> @@ -32,6 +32,7 @@
>>>   #include "hw/s390x/ap-bridge.h"
>>>   #include "system/address-spaces.h"
>>>   #include "qom/object.h"
>>> +#include "vfio-migration-internal.h"
>>>     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
>>>   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
>>>       .name = "vfio-ap-device",
>>>       .version_id = 1,
>>>       .minimum_version_id = 1,
>>> +    .unmigratable = 0,
>>
>> unneeded.
>>
>>>       .fields = (const VMStateField[]) {
>>>           VMSTATE_END_OF_LIST()
>>>       },
>>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
>>>       qemu_mutex_init(&cfg_chg_events_lock);
>>>   }
>>>   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
>>> +{
>>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +
>>> +    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
>>> +}
>>> +
>>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>>> +{
>>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +    Error *err = NULL;
>>> +    int ret;
>>> +
>>> +    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>>> +    if (ret) {
>>> +        error_report_err(err);
>>> +    }
>>> +
>>> +    return ret;
>>> +}
>>> +
>>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>>> +{
>>> +    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>>> +
>>> +    return OBJECT(vdev);
>>> +}
>>> +
>>>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>>   {
>>>       vdev->needs_reset = false;
>>>   }
>>>   -/*
>>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>>> - * vfio-ap device now.
>>> - */
>>>   struct VFIODeviceOps vfio_ap_ops = {
>>>       .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>>> +    .vfio_save_config = vfio_ap_save_config,
>>> +    .vfio_load_config = vfio_ap_load_config,
>>> +    .vfio_get_object = vfio_ap_get_object,
>>>   };
>>>     static void vfio_ap_req_notifier_handler(void *opaque)
>>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
>>>           goto error;
>>>       }
>>>   +    if (!vfio_migration_realize(vbasedev, errp)) {
>>> +        goto error;
>>> +    }
>>> +
>>>       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>>           /*
>>>            * Report this error, but do not make it a failing condition.
>>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>>     static const Property vfio_ap_properties[] = {
>>>       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>>> +    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>>> +                            vdev.enable_migration, ON_OFF_AUTO_ON),
>>
>> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
>> has migration support. If not, the VM would still start but migration
>> would be blocked.
> 
> Will this allow the vfio-ap device migration patches to get merged before
> the kernel/KVM patches?

It's possible. Several reasons could explain the lack of migration
support. The most obvious is a lack of kernel support, but it could
also be a firmware support issue, an incompatible hardware state of
the adapter, adapter model, etc.

Do all CEXxS crypto cards support migration ?

Do you expect kernel support [*] to be merged in v7.1 ?

Also, I think the QEMU support lacks vfio_migration_exit().

Thanks,

C.

[*]  https://lore.kernel.org/r/20260407205100.331150-1-akrowiak@linux.ibm.com

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

[Anthony Krowiak]

On 4/10/26 4:19 AM, Cédric Le Goater wrote:
> On 4/9/26 16:13, Anthony Krowiak wrote:
>> From: Anthony  Krowiak <akrowiak@linux.ibm.com>
>>
>> Implements the callbacks used by the VFIO migration framework to migrate
>> guests with pass-through access to s390 crypto devices.
>>
>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>> ---
>>   hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>   1 file changed, 39 insertions(+), 4 deletions(-)
>>
>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>> index fd68983702..49947b18c3 100644
>> --- a/hw/vfio/ap.c
>> +++ b/hw/vfio/ap.c
>> @@ -32,6 +32,7 @@
>>   #include "hw/s390x/ap-bridge.h"
>>   #include "system/address-spaces.h"
>>   #include "qom/object.h"
>> +#include "vfio-migration-internal.h"
>>     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
>>   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device 
>> = {
>>       .name = "vfio-ap-device",
>>       .version_id = 1,
>>       .minimum_version_id = 1,
>> +    .unmigratable = 0,
>
> unneeded.
>
>>       .fields = (const VMStateField[]) {
>>           VMSTATE_END_OF_LIST()
>>       },
>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) 
>> vfio_ap_global_init(void)
>>       qemu_mutex_init(&cfg_chg_events_lock);
>>   }
>>   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, 
>> Error **errp)
>> +{
>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +
>> +    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, 
>> errp);
>> +}
>> +
>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>> +{
>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>> +    Error *err = NULL;
>> +    int ret;
>> +
>> +    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>> +    if (ret) {
>> +        error_report_err(err);
>> +    }
>> +
>> +    return ret;
>> +}
>> +
>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>> +{
>> +    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>> +
>> +    return OBJECT(vdev);
>> +}
>> +
>>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>   {
>>       vdev->needs_reset = false;
>>   }
>>   -/*
>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>> - * vfio-ap device now.
>> - */
>>   struct VFIODeviceOps vfio_ap_ops = {
>>       .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>> +    .vfio_save_config = vfio_ap_save_config,
>> +    .vfio_load_config = vfio_ap_load_config,
>> +    .vfio_get_object = vfio_ap_get_object,
>>   };
>>     static void vfio_ap_req_notifier_handler(void *opaque)
>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, 
>> Error **errp)
>>           goto error;
>>       }
>>   +    if (!vfio_migration_realize(vbasedev, errp)) {
>> +        goto error;
>> +    }
>> +
>>       if (!vfio_ap_register_irq_notifier(vapdev, 
>> VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>           /*
>>            * Report this error, but do not make it a failing condition.
>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>     static const Property vfio_ap_properties[] = {
>>       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>> +    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>> +                            vdev.enable_migration, ON_OFF_AUTO_ON),
>
> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
> has migration support. If not, the VM would still start but migration
> would be blocked.

I made this change and migration failed on the destination host with the
following error message:

error: Requested operation is not valid: cannot migrate domain: 
62177883-f1bb-47f0-914d-32a22e3a8804: VFIO device doesn't support device 
and IOMMU dirty tracking

This despite having the following property defined:

DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOAPDevice, 
vdev.device_dirty_page_tracking, ON_OFF_AUTO_ON)


>
> thanks,
>
> C.
>
>
>>   #ifdef CONFIG_IOMMUFD
>>       DEFINE_PROP_LINK("iommufd", VFIOAPDevice, vdev.iommufd,
>>                        TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
>

Re: [PATCH v2 2/2] hw/vfio/ap: Callbacks for migration of guests with pass-through access to AP devices

[Cédric Le Goater]

On 4/13/26 17:23, Anthony Krowiak wrote:
> 
> 
> On 4/10/26 4:19 AM, Cédric Le Goater wrote:
>> On 4/9/26 16:13, Anthony Krowiak wrote:
>>> From: Anthony  Krowiak <akrowiak@linux.ibm.com>
>>>
>>> Implements the callbacks used by the VFIO migration framework to migrate
>>> guests with pass-through access to s390 crypto devices.
>>>
>>> Signed-off-by: Anthony Krowiak <akrowiak@linux.ibm.com>
>>> ---
>>>   hw/vfio/ap.c | 43 +++++++++++++++++++++++++++++++++++++++----
>>>   1 file changed, 39 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
>>> index fd68983702..49947b18c3 100644
>>> --- a/hw/vfio/ap.c
>>> +++ b/hw/vfio/ap.c
>>> @@ -32,6 +32,7 @@
>>>   #include "hw/s390x/ap-bridge.h"
>>>   #include "system/address-spaces.h"
>>>   #include "qom/object.h"
>>> +#include "vfio-migration-internal.h"
>>>     #define TYPE_VFIO_AP_DEVICE      "vfio-ap"
>>>   @@ -47,6 +48,7 @@ static const VMStateDescription vmstate_ap_device = {
>>>       .name = "vfio-ap-device",
>>>       .version_id = 1,
>>>       .minimum_version_id = 1,
>>> +    .unmigratable = 0,
>>
>> unneeded.
>>
>>>       .fields = (const VMStateField[]) {
>>>           VMSTATE_END_OF_LIST()
>>>       },
>>> @@ -66,17 +68,44 @@ static void __attribute__((constructor)) vfio_ap_global_init(void)
>>>       qemu_mutex_init(&cfg_chg_events_lock);
>>>   }
>>>   +static int vfio_ap_save_config(VFIODevice *vdev, QEMUFile *f, Error **errp)
>>> +{
>>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +
>>> +    return vmstate_save_state(f, &vmstate_ap_device, vapdev, NULL, errp);
>>> +}
>>> +
>>> +static int vfio_ap_load_config(VFIODevice *vdev, QEMUFile *f)
>>> +{
>>> +    VFIOAPDevice *vapdev = container_of(vdev, VFIOAPDevice, vdev);
>>> +    Error *err = NULL;
>>> +    int ret;
>>> +
>>> +    ret = vmstate_load_state(f, &vmstate_ap_device, vapdev, 1, &err);
>>> +    if (ret) {
>>> +        error_report_err(err);
>>> +    }
>>> +
>>> +    return ret;
>>> +}
>>> +
>>> +static Object *vfio_ap_get_object(VFIODevice *vbasedev)
>>> +{
>>> +    VFIOAPDevice *vdev = container_of(vbasedev, VFIOAPDevice, vdev);
>>> +
>>> +    return OBJECT(vdev);
>>> +}
>>> +
>>>   static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
>>>   {
>>>       vdev->needs_reset = false;
>>>   }
>>>   -/*
>>> - * We don't need vfio_hot_reset_multi and vfio_eoi operations for
>>> - * vfio-ap device now.
>>> - */
>>>   struct VFIODeviceOps vfio_ap_ops = {
>>>       .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
>>> +    .vfio_save_config = vfio_ap_save_config,
>>> +    .vfio_load_config = vfio_ap_load_config,
>>> +    .vfio_get_object = vfio_ap_get_object,
>>>   };
>>>     static void vfio_ap_req_notifier_handler(void *opaque)
>>> @@ -251,6 +280,10 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
>>>           goto error;
>>>       }
>>>   +    if (!vfio_migration_realize(vbasedev, errp)) {
>>> +        goto error;
>>> +    }
>>> +
>>>       if (!vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err)) {
>>>           /*
>>>            * Report this error, but do not make it a failing condition.
>>> @@ -287,6 +320,8 @@ static void vfio_ap_unrealize(DeviceState *dev)
>>>     static const Property vfio_ap_properties[] = {
>>>       DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
>>> +    DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOAPDevice,
>>> +                            vdev.enable_migration, ON_OFF_AUTO_ON),
>>
>> Why not ON_OFF_AUTO_AUTO ? This would enable migration only if kernel
>> has migration support. If not, the VM would still start but migration
>> would be blocked.
> 
> I made this change and migration failed on the destination host with the
> following error message:
> 
> error: Requested operation is not valid: cannot migrate domain: 62177883-f1bb-47f0-914d-32a22e3a8804: VFIO device doesn't support device and IOMMU dirty tracking
> 
> This despite having the following property defined:
> 
> DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOAPDevice, vdev.device_dirty_page_tracking, ON_OFF_AUTO_ON)


try adding dummy vfio_log_ops handlers in the driver.


static int mtty_log_start(struct vfio_device *vdev,
			  struct rb_root_cached *ranges,
			  u32 nnodes, u64 *page_size)
{
	return 0;
}

static int mtty_log_stop(struct vfio_device *vdev)
{
	return 0;
}

static int mtty_log_read_and_clear(struct vfio_device *vdev,
				   unsigned long iova, unsigned long length,
				   struct iova_bitmap *dirty)
{
	return 0;
}

static const struct vfio_log_ops mtty_log_ops = {
	.log_start = mtty_log_start,
	.log_stop = mtty_log_stop,
	.log_read_and_clear = mtty_log_read_and_clear,
};

See samples/vfio-mdev/mtty.c.

C.