[PATCH V3 1/1] hw/intc/arm_gicv3_kvm: preserve pending interrupts during cpr

[PATCH V3 1/1] hw/intc/arm_gicv3_kvm: preserve pending interrupts during cpr

[Steve Sistare]

Close a race condition that causes cpr-transfer to lose VFIO
interrupts on ARM.

CPR stops VCPUs but does not disable VFIO interrupts, which may continue
to arrive throughout the transition to new QEMU.

CPR calls kvm_irqchip_remove_irqfd_notifier_gsi in old QEMU to force
future interrupts to the producer eventfd, where they are preserved.
Old QEMU then destroys the old KVM instance.  However, interrupts may
already be pending in KVM state.  To preserve them, call ioctl
KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES to flush them to guest RAM, where
they will be picked up when the new KVM+VCPU instance is created.

Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
Reviewed-by: Fabiano Rosas <farosas@suse.de>
---
 hw/intc/arm_gicv3_kvm.c            | 15 +++++++++++++++
 include/hw/intc/arm_gicv3_common.h |  3 +++
 2 files changed, 18 insertions(+)

diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
index 3be3bf6c28..5e63e69e14 100644
--- a/hw/intc/arm_gicv3_kvm.c
+++ b/hw/intc/arm_gicv3_kvm.c
@@ -30,6 +30,7 @@
 #include "gicv3_internal.h"
 #include "vgic_common.h"
 #include "migration/blocker.h"
+#include "migration/misc.h"
 #include "qom/object.h"
 #include "target/arm/cpregs.h"
 
@@ -777,6 +778,17 @@ static void vm_change_state_handler(void *opaque, bool running,
     }
 }
 
+static int kvm_arm_gicv3_notifier(NotifierWithReturn *notifier,
+                                  MigrationEvent *e, Error **errp)
+{
+    if (e->type == MIG_EVENT_PRECOPY_DONE) {
+        GICv3State *s = container_of(notifier, GICv3State, cpr_notifier);
+        return kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                                 KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES,
+                                 NULL, true, errp);
+    }
+    return 0;
+}
 
 static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
 {
@@ -890,6 +902,9 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
     if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
                               KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES)) {
         qemu_add_vm_change_state_handler(vm_change_state_handler, s);
+        migration_add_notifier_mode(&s->cpr_notifier,
+                                    kvm_arm_gicv3_notifier,
+                                    MIG_MODE_CPR_TRANSFER);
     }
 }
 
diff --git a/include/hw/intc/arm_gicv3_common.h b/include/hw/intc/arm_gicv3_common.h
index a3d6a0e507..75bbc403c7 100644
--- a/include/hw/intc/arm_gicv3_common.h
+++ b/include/hw/intc/arm_gicv3_common.h
@@ -27,6 +27,7 @@
 #include "hw/sysbus.h"
 #include "hw/intc/arm_gic_common.h"
 #include "qom/object.h"
+#include "qemu/notify.h"
 
 /*
  * Maximum number of possible interrupts, determined by the GIC architecture.
@@ -270,6 +271,8 @@ struct GICv3State {
     GICv3CPUState *cpu;
     /* List of all ITSes connected to this GIC */
     GPtrArray *itslist;
+
+    NotifierWithReturn cpr_notifier;
 };
 
 #define GICV3_BITMAP_ACCESSORS(BMP)                                     \
-- 
2.39.3

Re: [PATCH V3 1/1] hw/intc/arm_gicv3_kvm: preserve pending interrupts during cpr

[Steven Sistare]

Peter previously said "looks good" but wanted a sanity check from a
live update maintainer, which Fabiano provided.

So, which maintainer will pull this for 10.1?

- Steve

On 8/11/2025 2:19 PM, Steve Sistare wrote:
> Close a race condition that causes cpr-transfer to lose VFIO
> interrupts on ARM.
> 
> CPR stops VCPUs but does not disable VFIO interrupts, which may continue
> to arrive throughout the transition to new QEMU.
> 
> CPR calls kvm_irqchip_remove_irqfd_notifier_gsi in old QEMU to force
> future interrupts to the producer eventfd, where they are preserved.
> Old QEMU then destroys the old KVM instance.  However, interrupts may
> already be pending in KVM state.  To preserve them, call ioctl
> KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES to flush them to guest RAM, where
> they will be picked up when the new KVM+VCPU instance is created.
> 
> Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
> Reviewed-by: Fabiano Rosas <farosas@suse.de>
> ---
>   hw/intc/arm_gicv3_kvm.c            | 15 +++++++++++++++
>   include/hw/intc/arm_gicv3_common.h |  3 +++
>   2 files changed, 18 insertions(+)
> 
> diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
> index 3be3bf6c28..5e63e69e14 100644
> --- a/hw/intc/arm_gicv3_kvm.c
> +++ b/hw/intc/arm_gicv3_kvm.c
> @@ -30,6 +30,7 @@
>   #include "gicv3_internal.h"
>   #include "vgic_common.h"
>   #include "migration/blocker.h"
> +#include "migration/misc.h"
>   #include "qom/object.h"
>   #include "target/arm/cpregs.h"
>   
> @@ -777,6 +778,17 @@ static void vm_change_state_handler(void *opaque, bool running,
>       }
>   }
>   
> +static int kvm_arm_gicv3_notifier(NotifierWithReturn *notifier,
> +                                  MigrationEvent *e, Error **errp)
> +{
> +    if (e->type == MIG_EVENT_PRECOPY_DONE) {
> +        GICv3State *s = container_of(notifier, GICv3State, cpr_notifier);
> +        return kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
> +                                 KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES,
> +                                 NULL, true, errp);
> +    }
> +    return 0;
> +}
>   
>   static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
>   {
> @@ -890,6 +902,9 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
>       if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
>                                 KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES)) {
>           qemu_add_vm_change_state_handler(vm_change_state_handler, s);
> +        migration_add_notifier_mode(&s->cpr_notifier,
> +                                    kvm_arm_gicv3_notifier,
> +                                    MIG_MODE_CPR_TRANSFER);
>       }
>   }
>   
> diff --git a/include/hw/intc/arm_gicv3_common.h b/include/hw/intc/arm_gicv3_common.h
> index a3d6a0e507..75bbc403c7 100644
> --- a/include/hw/intc/arm_gicv3_common.h
> +++ b/include/hw/intc/arm_gicv3_common.h
> @@ -27,6 +27,7 @@
>   #include "hw/sysbus.h"
>   #include "hw/intc/arm_gic_common.h"
>   #include "qom/object.h"
> +#include "qemu/notify.h"
>   
>   /*
>    * Maximum number of possible interrupts, determined by the GIC architecture.
> @@ -270,6 +271,8 @@ struct GICv3State {
>       GICv3CPUState *cpu;
>       /* List of all ITSes connected to this GIC */
>       GPtrArray *itslist;
> +
> +    NotifierWithReturn cpr_notifier;
>   };
>   
>   #define GICV3_BITMAP_ACCESSORS(BMP)                                     \

Re: [PATCH V3 1/1] hw/intc/arm_gicv3_kvm: preserve pending interrupts during cpr

[Peter Maydell]

On Wed, 13 Aug 2025 at 14:32, Steven Sistare <steven.sistare@oracle.com> wrote:
>
> Peter previously said "looks good" but wanted a sanity check from a
> live update maintainer, which Fabiano provided.
>
> So, which maintainer will pull this for 10.1?

It's not going into 10.1 (it has missed that boat, I'm afraid);
it's on my list to look at and apply after we release.

thanks
-- PMM

Re: [PATCH V3 1/1] hw/intc/arm_gicv3_kvm: preserve pending interrupts during cpr

[Peter Maydell]

On Mon, 11 Aug 2025 at 19:19, Steve Sistare <steven.sistare@oracle.com> wrote:
>
> Close a race condition that causes cpr-transfer to lose VFIO
> interrupts on ARM.
>
> CPR stops VCPUs but does not disable VFIO interrupts, which may continue
> to arrive throughout the transition to new QEMU.
>
> CPR calls kvm_irqchip_remove_irqfd_notifier_gsi in old QEMU to force
> future interrupts to the producer eventfd, where they are preserved.
> Old QEMU then destroys the old KVM instance.  However, interrupts may
> already be pending in KVM state.  To preserve them, call ioctl
> KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES to flush them to guest RAM, where
> they will be picked up when the new KVM+VCPU instance is created.
>
> Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
> Reviewed-by: Fabiano Rosas <farosas@suse.de>
> ---


Applied to target-arm.next for 10.2, thanks. I added a cc:stable tag.

-- PMM