[PATCH 36/43] KVM: arm64: gic-v5: Implement save/restore mechanisms for ISTs

Sascha Bischoff
2026-04-27T16:19:32Z

When running a GICv5 VM, there are up to two ISTs that must be saved
or restored when migrating a VM.

The SPI IST is allocated by the hypervisor, as the guest presumes the
memory for the SPI state is allocated by the hardware. The LPI IST, on
the other hand, is allocated by the guest in the event that it wishes
to use LPIs. We shadow the guest's LPI IST in KVM, and therefore the
guest's memory is never directly used by the GICv5 hardware. Hence, in
both cases, the in-use ISTs are allocated by the hypervisor.

As there is no guest-allocated memory for the SPI IST, the state of
this must be saved by the VMM. Therefore, the VMM must provide a
memory buffer large enough to store/restore the SPI IST (32-bits per
SPI).

The LPI IST, if present, is stored into guest memory as the guest has
already allocated storage under the assumption that it would be used
by the GIC. Each IST Entry is written back to guest memory (skipping
metadata sections) on a save, or restored from guest memory on a
restore. The guest is only allowed to create a linear IST, so there's
a sufficiently large region of memory that is contiguous in GPA space.

On a save, the VM itself is quiesced using IRS_SAVE_VMR - this ensures
that the hardware has written all interrupt state back to the
ISTs. Following the save operation, the IRS_SAVE_VM_STATUSR is checked
to ensure that the guest has remained quiescent. In the event that it
has not, an error is propagated back to the VMM such that it can retry
the save.

On restore, the VM is first made invalid - it is not allowed to write
to any of the tables while they are valid - and then the SPI and LPI
ISTs are restored (if required) before making the VM valid again. As
part of restoring the ISTs, any pending interrupts are tracked, and
IST pending state is cleared. Once the VM is made valid, these valid
interrupts are made pending again via the GIC VDPEND system
instruction.

Signed-off-by: Sascha Bischoff <sascha.bischoff@arm.com>
---
arch/arm64/kvm/vgic/vgic-v5-tables.c | 564 ++++++++++++++++++++++++++-
arch/arm64/kvm/vgic/vgic-v5-tables.h | 38 ++
arch/arm64/kvm/vgic/vgic-v5.c | 183 +++++++++
arch/arm64/kvm/vgic/vgic.h | 2 +
include/linux/irqchip/arm-gic-v5.h | 7 +
5 files changed, 791 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c
index 77fc5fb27f30d..8e909100485bf 100644
--- a/arch/arm64/kvm/vgic/vgic-v5-tables.c
+++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c
@@ -431,6 +431,13 @@ int vgic_v5_vmte_init(struct kvm *kvm)
if (ret)
goto out_fail;

+ /*
+ * If we are restoring the state of a guest, we need to re-inject any
+ * IRQs where were pending when the state of the guest was originally
+ * saved. We use the pending_irqs list for this.
+ */
+ INIT_LIST_HEAD(&vmi->pending_irqs);
+
/* Allocate and assign the VM Descriptor, if required. */
if (vmt_info->vmd_size != 0) {
vmd = kzalloc(vmt_info->vmd_size, GFP_KERNEL);
@@ -547,9 +554,6 @@ int vgic_v5_vmte_release(struct kvm *kvm)
if (WARN_ON_ONCE(!vmi))
goto no_vmi;

- kfree(vmi->vmd_base);
- kfree(vmi->vpet_base);
-
/* If we have an LPI IST, free it */
if (vmi->h_lpi_ist)
ret = vgic_v5_lpi_ist_free(kvm);
@@ -562,6 +566,19 @@ int vgic_v5_vmte_release(struct kvm *kvm)
if (ret)
return ret;

+ kfree(vmi->vmd_base);
+ kfree(vmi->vpet_base);
+
+ /* Unlikely, but possible. Avoid leaking the memory. */
+ if (!list_empty(&vmi->pending_irqs)) {
+ struct pending_irq *pirq, *tmp;
+
+ list_for_each_entry_safe(pirq, tmp, &vmi->pending_irqs, next) {
+ list_del(&pirq->next);
+ kfree(pirq);
+ }
+ }
+
xa_erase(&vm_info, vm_id);
kfree(vmi);

@@ -1191,6 +1208,7 @@ int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits)
return ret;
}

+
/* Free the LPI IST again */
int vgic_v5_lpi_ist_free(struct kvm *kvm)
{
@@ -1206,3 +1224,543 @@ int vgic_v5_lpi_ist_free(struct kvm *kvm)
else
return vgic_v5_two_level_ist_free(kvm, false);
}
+
+/*
+ * Save the SPI IST to userspace-provided memory.
+ *
+ * Userspace (should have) has provided us with an appropriately sized buffer
+ * that we can dump the SPI IST to. We only need to write out the architected
+ * 32-bits of the IST, and can skip any and all metadata as that is
+ * implementation specific.
+ *
+ * We only ever allocate linear ISTs for SPIs, so we stride through the IST on
+ * the host (taking metadata into account, i.e., skipping it) and write the
+ * lower 32-bits of each ISTE to the host provided buffer.
+ */
+int vgic_v5_save_spi_ist(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+ u32 __user *uaddr = (u32 __user *)(unsigned long)attr->addr;
+ unsigned int host_id_bits, host_istsz, host_l2sz;
+ u16 vm_id = vgic_v5_vm_id(kvm);
+ struct vgic_v5_vm_info *vmi;
+ struct vmtl2_entry *vmte;
+ void *host_ist_base;
+ __le32 h_iste;
+ __le64 tmp;
+ int ret;
+
+ vmi = xa_load(&vm_info, vm_id);
+ if (WARN_ON_ONCE(!vmi))
+ return -ENXIO;
+
+ host_ist_base = vmi->h_spi_ist;
+
+ /* We don't have SPIs, but userspace is trying to save them. */
+ if (!host_ist_base && attr->addr)
+ return -ENOENT;
+
+ /* We have SPIs but userspace isn't trying to save them. */
+ if (host_ist_base && !attr->addr)
+ return -EINVAL;
+
+ /* No SPIs and no userspace buffer: nothing to do. */
+ if (!host_ist_base && !attr->addr)
+ return 0;
+
+ ret = vgic_v5_get_l2_vmte(vm_id, &vmte);
+ if (ret)
+ return ret;
+
+ tmp = le64_to_cpu(READ_ONCE(vmte->val[3]));
+ host_id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp);
+ host_istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp);
+ host_l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp);
+
+ /* We always use a Linear SPI IST on the host */
+ for (int i = 0; i < BIT(host_id_bits); ++i) {
+ /*
+ * We're explictitly using a void pointer here, and reinterpret
+ * it as __le64 as we only care about the lower 32 bits of the
+ * entry, and not the metadata if present. This lets us stride
+ * through the IST wil skipping the metadata.
+ */
+ __le32 *h_iste_addr = host_ist_base + i * BIT(host_istsz + 2);
+
+ h_iste = READ_ONCE(*h_iste_addr);
+ ret = put_user(h_iste, uaddr);
+ if (ret)
+ return ret;
+
+ uaddr++;
+ }
+
+ return ret;
+}
+
+/*
+ * Save the LPI IST to guest memory
+ *
+ * When a guest is using LPIs, it has allocated memory for the LPI IST. We don't
+ * let the host's IRQ directly use that memory, and instead reallocate the IST
+ * on the host. However, we're able to use the memory that the guest has
+ * allocated to save the LPI IST. There should be sufficient storage there, and
+ * if the guest hasn't done things properly, then that's on the guest - there's
+ * nothing we can do.
+ *
+ * We only store the lower 32-bits of each host ISTE as the upper bits contain
+ * the metadata, which needs to be explcitly zeroed on restore anyhow.
+ *
+ * This is a bit more complex than for the SPIs. We intentionally don't tell the
+ * guest that it is allowed to create two-level ISTs, so it should have created
+ * a linear IST for LPIs. This means that we have a contigious range in GPA
+ * space that we can iterate over when writing. HOWEVER, we (KVM) have the
+ * option of allocating a linear IST or a two-level IST. Hence, iteration is a
+ * little more complex.
+ */
+int vgic_v5_save_lpi_ist(struct kvm *kvm)
+{
+ unsigned int host_id_bits, host_istsz, host_l2sz;
+ size_t n, l2bits, h_l1_index, h_l2_index;
+ int ret, h_l1_entries, h_l2_entries;
+ u16 vm_id = vgic_v5_vm_id(kvm);
+ struct vgic_v5_vm_info *vmi;
+ struct vmtl2_entry *vmte;
+ void *h_l2_ist_base;
+ void *host_ist_base;
+ gpa_t g_entry_addr;
+ __le32 h_iste;
+ __le64 tmp;
+
+ ret = vgic_v5_check_vm_id(vm_id);
+ if (ret)
+ return ret;
+
+ vmi = xa_load(&vm_info, vm_id);
+ if (WARN_ON_ONCE(!vmi))
+ return -ENXIO;
+
+ ret = vgic_v5_get_l2_vmte(vm_id, &vmte);
+ if (ret)
+ return ret;
+
+ /* If there is no IST to save, return without error */
+ if (!kvm->arch.vgic.vgic_v5_irs_data->ist_baser.valid &&
+ !FIELD_GET(GICV5_VMTEL2E_VALID, vmte->val[2])) {
+ return 0;
+ }
+
+ /* Host says an LPI IST exists, but we have no backing object. */
+ if (FIELD_GET(GICV5_VMTEL2E_IST_VALID, vmte->val[2]) && !vmi->h_lpi_ist)
+ return -ENXIO;
+
+ if (vmi->h_lpi_ist_structure && !vmi->h_lpi_l2_ists)
+ return -ENXIO;
+
+ /*
+ * Assumption: the guest IST is Linear. This gives us a simple way to iterate
+ * over the guest's memory.
+ *
+ * Get the base address of the IST in GPA space.
+ */
+ g_entry_addr = kvm->arch.vgic.vgic_v5_irs_data->ist_baser.addr;
+
+ tmp = le64_to_cpu(READ_ONCE(vmte->val[2]));
+ host_id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp);
+ host_istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp);
+ host_l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp);
+
+ /* Linear IST on the host - the simple case */
+ if (!vmi->h_lpi_ist_structure) {
+ h_l2_entries = BIT(host_id_bits);
+ host_ist_base = vmi->h_lpi_ist;
+
+ for (h_l2_index = 0; h_l2_index < h_l2_entries; ++h_l2_index) {
+ __le32 *h_iste_addr = host_ist_base + h_l2_index * BIT(host_istsz + 2);
+
+ h_iste = *h_iste_addr;
+
+ ret = vgic_write_guest_lock(kvm, g_entry_addr, &h_iste, sizeof(h_iste));
+ if (ret)
+ return ret;
+
+ /* Advance to the next guest entry */
+ g_entry_addr += sizeof(h_iste);
+ }
+ } else {
+ /* And the two level case */
+ n = max(2, host_id_bits - ((10 - host_istsz) + (2 * host_l2sz)) + 3 - 1);
+ l2bits = (10 - host_istsz) + (2 * host_l2sz);
+ h_l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE;
+ h_l2_entries = BIT(l2bits);
+
+ /* For each L1 ISTE */
+ for (h_l1_index = 0; h_l1_index < h_l1_entries; ++h_l1_index) {
+ /*
+ * We don't do dynamic L2 IST allocation for guest ISTs
+ * - all of the memory is provisioned up-front to
+ * simplify the process. If we encounter an invalid L1
+ * ISTE things have gone wrong!
+ */
+ if (!FIELD_GET(GICV5_ISTL1E_VALID, vmi->h_lpi_ist[h_l1_index]))
+ return -ENXIO;
+
+ /* If valid, process the L2 table. For each L2 ISTE. */
+ for (h_l2_index = 0; h_l2_index < h_l2_entries; ++h_l2_index) {
+
+ h_l2_ist_base = vmi->h_lpi_l2_ists[h_l1_index];
+ if (!h_l2_ist_base)
+ return -ENXIO;
+
+ h_iste = *(__le32 *)(h_l2_ist_base +
+ h_l2_index *
+ BIT(2 + host_l2sz));
+
+ ret = vgic_write_guest_lock(kvm, g_entry_addr,
+ &h_iste, sizeof(h_iste));
+ if (ret)
+ return ret;
+
+ /* Advance to the next guest entry */
+ g_entry_addr += sizeof(__le32);
+ }
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Track any SPIs and LPIs where were marked as pending at the point where the
+ * IST was restored.
+ *
+ * Append any previously pending IRQs to the pending list as we need to mark
+ * them as non-pending when restoring the ISTs. These are then reinjected them
+ * using VDPEND prior to running the guest for the first time.
+ */
+static int vgic_v5_track_pending_irq(struct list_head *pending_irqs, u32 intid,
+ u32 type)
+{
+ struct pending_irq *pirq;
+
+ pirq = kzalloc_obj(*pirq, GFP_KERNEL);
+ if (pirq == NULL)
+ return -ENOMEM;
+
+ /* Make it in to a proper GICv5 IntID */
+ pirq->irq = FIELD_PREP(GICV5_HWIRQ_TYPE, type) |
+ FIELD_PREP(GICV5_HWIRQ_ID, intid);
+
+ INIT_LIST_HEAD(&pirq->next);
+ list_add_tail(&pirq->next, pending_irqs);
+
+ return 0;
+}
+
+/*
+ * Process and sanitise each restored ISTE.
+ *
+ * When restoring the ISTs, each ISTE needs to be processed. The HWU field needs
+ * to be explicitly zeroed - it is for hardware usage, and we might well be on
+ * different hardware now, which may use the field differently.
+ *
+ * If interrupts are marked as pending on restore, then they need to be tracked
+ * as such, and the pending state cleared. The alternative would be that the
+ * hardware needs to iterate over the whole IST post restoring, but this way is
+ * cleaner and ensures that everything is tracked correctly. The pending state
+ * for each interrupt is restored prior to running the guest for the first time.
+ */
+static int vgic_v5_process_iste(__le32 *iste, struct list_head *pending_irqs,
+ u32 intid, u32 type)
+{
+ __le64 tmp = le64_to_cpu(READ_ONCE(*iste));
+ int ret = 0;
+
+ /* Clean up the ISTE - Zero the HWU field. */
+ tmp &= ~GICV5_ISTL2E_HWU;
+
+ if (FIELD_GET(GICV5_ISTL2E_PENDING, tmp)) {
+ ret = vgic_v5_track_pending_irq(pending_irqs, intid, type);
+ if (ret)
+ return ret;
+
+ /* Now that we've tracked it, clear the pending state */
+ tmp &= ~GICV5_ISTL2E_PENDING;
+ }
+
+ WRITE_ONCE(*iste, cpu_to_le64(tmp));
+
+ return ret;
+}
+
+/*
+ * Restore the SPI IST from userspace-provided buffer to the host-allocated IST.
+ *
+ * The SPI has previously been saved to userspace-provided memory. Now,
+ * userspace has provided us with a buffer containing the SPI to restore. We
+ * need to iterate over this, and restore it to the linear SPI IST allocated by
+ * the host.
+ */
+int vgic_v5_restore_spi_ist(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+ u32 __user *uaddr = (u32 __user *)(unsigned long)attr->addr;
+ unsigned int host_id_bits, host_istsz, host_l2sz;
+ u16 vm_id = vgic_v5_vm_id(kvm);
+ struct vgic_v5_vm_info *vmi;
+ struct vmtl2_entry *vmte;
+ void *host_ist_base;
+ __le32 h_iste;
+ int ret = 0;
+ u64 tmp;
+
+ vmi = xa_load(&vm_info, vm_id);
+ if (WARN_ON_ONCE(!vmi))
+ return -ENXIO;
+
+ ret = vgic_v5_get_l2_vmte(vm_id, &vmte);
+ if (ret)
+ return ret;
+
+ host_ist_base = vmi->h_spi_ist;
+
+ /* We don't have SPIs, but userspace is trying to restore them. */
+ if (!host_ist_base && attr->addr)
+ return -ENOENT;
+
+ /* We have SPIs but userspace isn't trying to restore them. */
+ if (host_ist_base && !attr->addr)
+ return -EINVAL;
+
+ /* No SPIs and no userspace buffer: nothing to do. */
+ if (!host_ist_base && !attr->addr)
+ return 0;
+
+ tmp = le64_to_cpu(READ_ONCE(vmte->val[3]));
+ host_id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp);
+ host_istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp);
+ host_l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp);
+
+ /*
+ * The guest's SPI IST is always linear. When the SPI IST is saved, only
+ * the architected 4 bytes for the ISTE are saved, and metadata is
+ * not. This means that we can just linearly read the memory provided by
+ * userspace when restoring the IST. We stride through the
+ * host-allocated memory using the actual ISTE size, i.e, skipping
+ * metadata sections, if present.
+ */
+ for (int i = 0; i < BIT(host_id_bits); ++i) {
+ size_t host_iste_size = BIT(host_istsz + 2);
+ void *h_iste_addr = host_ist_base + i * host_iste_size;
+
+ /* Read the entry from userspace memory */
+ ret = get_user(h_iste, uaddr);
+ if (ret)
+ return ret;
+
+ /*
+ * Clean up the entry (zeroing HWU, pending state) and track if
+ * the interrupt was pending so that it can be re-injected
+ * later.
+ */
+ ret = vgic_v5_process_iste(&h_iste, &vmi->pending_irqs,
+ i, GICV5_HWIRQ_TYPE_SPI);
+ if (ret)
+ return ret;
+
+ /* Finally, write the entry to the host IST, and flush it. */
+ memset(h_iste_addr, 0, host_iste_size);
+ WRITE_ONCE(*(__le32 *)h_iste_addr, h_iste);
+ vgic_v5_clean_inval(h_iste_addr, host_iste_size, true, true);
+
+ /* Advance to the next entry in userspace memory */
+ uaddr++;
+ }
+
+ return ret;
+}
+
+/*
+ * Restore the LPI IST from guest memory to the host-allocated LPI IST.
+ *
+ * We iterate over the guest's memory to read out the saved LPI IST. KVM tells
+ * the guest that it is only allowed to create a linear IST, so the guest memory
+ * for the IST should be linear in GPA space.
+ *
+ * The host IST, on the other hand, is allowed to be two-level (but doesn't need
+ * to be). Therefore, some care needs to be taken when restoring the entries to
+ * the host's IST.
+ *
+ * Only the lower 32-bits of each ISTE are restored.
+ */
+int vgic_v5_restore_lpi_ist(struct kvm *kvm)
+{
+ unsigned int host_id_bits, host_istsz, host_l2sz;
+ size_t h_l1_index, h_l2_index, l2bits, n;
+ void *h_l2_ist_base, *host_ist_base;
+ int h_l1_entries, h_l2_entries, ret;
+ u16 vm_id = vgic_v5_vm_id(kvm);
+ struct vgic_v5_vm_info *vmi;
+ struct vmtl2_entry *vmte;
+ gpa_t g_entry_addr;
+ __le32 h_iste;
+ __le64 tmp;
+
+ ret = vgic_v5_check_vm_id(vm_id);
+ if (ret)
+ return ret;
+
+ vmi = xa_load(&vm_info, vm_id);
+ if (WARN_ON_ONCE(!vmi))
+ return -ENXIO;
+
+ ret = vgic_v5_get_l2_vmte(vm_id, &vmte);
+ if (ret)
+ return ret;
+
+ /* If there is no IST to restore, return without error */
+ if (!kvm->arch.vgic.vgic_v5_irs_data->ist_baser.valid &&
+ !FIELD_GET(GICV5_VMTEL2E_VALID, vmte->val[2])) {
+ return 0;
+ }
+
+ /* Host says an LPI IST exists, but we have no backing object. */
+ if (FIELD_GET(GICV5_VMTEL2E_IST_VALID, vmte->val[2]) && !vmi->h_lpi_ist)
+ return -ENXIO;
+
+ if (!vmi->h_lpi_ist)
+ return -ENXIO;
+
+ if (vmi->h_lpi_ist_structure && !vmi->h_lpi_l2_ists)
+ return -ENXIO;
+
+ /* The GPA of the guest's Linear LPI IST */
+ g_entry_addr = kvm->arch.vgic.vgic_v5_irs_data->ist_baser.addr;
+
+ tmp = le64_to_cpu(READ_ONCE(vmte->val[2]));
+ host_id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp);
+ host_istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp);
+ host_l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp);
+
+ /* We have a Linear IST on the host */
+ if (!vmi->h_lpi_ist_structure) {
+ h_l2_entries = BIT(host_id_bits);
+ host_ist_base = vmi->h_lpi_ist;
+
+ for (h_l2_index = 0; h_l2_index < h_l2_entries; ++h_l2_index) {
+ size_t host_iste_size = BIT(host_istsz + 2);
+ void *h_iste_addr = host_ist_base + h_l2_index * host_iste_size;
+
+ ret = kvm_read_guest_lock(kvm, g_entry_addr, &h_iste, sizeof(h_iste));
+ if (ret)
+ return ret;
+
+
+ /* Clear HWU, pending, and track if it WAS pending */
+ ret = vgic_v5_process_iste(&h_iste, &vmi->pending_irqs,
+ h_l2_index, GICV5_HWIRQ_TYPE_LPI);
+ if (ret)
+ return ret;
+
+ /* Restore the entry to the host IST */
+ memset(h_iste_addr, 0, host_iste_size);
+ WRITE_ONCE(*(__le32 *)h_iste_addr, h_iste);
+ vgic_v5_clean_inval(h_iste_addr, host_iste_size, true, true);
+
+ /* Advance to the next guest entry */
+ g_entry_addr += sizeof(h_iste);
+ }
+ } else {
+ /* A two-level host IST - the harder case */
+ n = max(2, host_id_bits - ((10 - host_istsz) + (2 * host_l2sz)) + 3 - 1);
+ l2bits = (10 - host_istsz) + (2 * host_l2sz);
+ h_l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE;
+ h_l2_entries = BIT(l2bits);
+
+ for (h_l1_index = 0; h_l1_index < h_l1_entries; ++h_l1_index) {
+ /*
+ * If the L1 ISTE is not marked valid, something is
+ * wrong; we don't do dynamic L2 IST allocation! Give up
+ * immediately.
+ */
+ if (!FIELD_GET(GICV5_ISTL1E_VALID, vmi->h_lpi_ist[h_l1_index]))
+ return -ENXIO;
+
+ h_l2_ist_base = vmi->h_lpi_l2_ists[h_l1_index];
+
+ for (h_l2_index = 0; h_l2_index < h_l2_entries; ++h_l2_index) {
+ size_t host_iste_size = BIT(host_istsz + 2);
+ void *h_iste_addr = h_l2_ist_base + h_l2_index * host_iste_size;
+
+ /* Read the guest's ISTE */
+ ret = kvm_read_guest_lock(kvm, g_entry_addr,
+ &h_iste, sizeof(h_iste));
+ if (ret)
+ return ret;
+
+ /*
+ * Clear HWU, pending, and track if it WAS
+ * pending.
+ */
+ ret = vgic_v5_process_iste(&h_iste, &vmi->pending_irqs,
+ h_l1_index * h_l2_entries + h_l2_index,
+ GICV5_HWIRQ_TYPE_LPI);
+ if (ret)
+ return ret;
+
+ /* Write the entry to the host's IST */
+ memset(h_iste_addr, 0, host_iste_size);
+ WRITE_ONCE(*(__le32 *)h_iste_addr, h_iste);
+ vgic_v5_clean_inval(h_iste_addr, host_iste_size, true, true);
+
+ /* Advance to the next guest entry */
+ g_entry_addr += sizeof(h_iste);
+ }
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Any previously pending IRQs were made non-pending when restoring guest IST
+ * state. Now that we're ready to run, we reinject that pending state for each
+ * using VDPEND.
+ */
+int vgic_v5_restore_pending_irqs(struct kvm *kvm)
+{
+ u16 vm_id = vgic_v5_vm_id(kvm);
+ struct pending_irq *pirq, *tmp;
+ struct vgic_v5_vm_info *vmi;
+
+ vmi = xa_load(&vm_info, vm_id);
+ if (WARN_ON_ONCE(!vmi))
+ return -ENXIO;
+
+ list_for_each_entry_safe(pirq, tmp, &vmi->pending_irqs, next) {
+ kvm_call_hyp(__vgic_v5_vdpend, pirq->irq, 1,
+ kvm->arch.vgic.gicv5_vm.vm_id);
+
+ list_del(&pirq->next);
+ kfree(pirq);
+ }
+
+ return 0;
+}
+
+/*
+ * Called on restore failure to clean up straggling pending state.
+ */
+void vgic_v5_scrap_pending_irqs(struct kvm *kvm)
+{
+ u16 vm_id = vgic_v5_vm_id(kvm);
+ struct pending_irq *pirq, *tmp;
+ struct vgic_v5_vm_info *vmi;
+
+ vmi = xa_load(&vm_info, vm_id);
+ if (WARN_ON_ONCE(!vmi))
+ return;
+
+ list_for_each_entry_safe(pirq, tmp, &vmi->pending_irqs, next) {
+ list_del(&pirq->next);
+ kfree(pirq);
+ }
+}
diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h
index 25e1c9fff87b4..23417e68ee24f 100644
--- a/arch/arm64/kvm/vgic/vgic-v5-tables.h
+++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h
@@ -7,6 +7,7 @@
#define __KVM_ARM_VGICV5_TABLES_H__

#include <linux/irqchip/arm-gic-v5.h>
+#include <linux/list.h>

#define VM_ID_BITS_MIN 8
#define VM_ID_BITS_MAX 16
@@ -68,6 +69,33 @@ typedef __le64 vpe_entry;
#define GICV5_VPED_ADDR_SHIFT 3ULL
#define GICV5_VPED_ADDR GENMASK_ULL(55, 3)

+// L2 IST Entry
+#define GICV5_ISTL2E_PENDING BIT(0)
+#define GICV5_ISTL2E_ACTIVE BIT(1)
+#define GICV5_ISTL2E_HM BIT(2)
+#define GICV5_ISTL2E_ENABLE BIT(3)
+#define GICV5_ISTL2E_IRM BIT(4)
+#define GICV5_ISTL2E_HWU GENMASK(10, 9)
+#define GICV5_ISTL2E_PRIORITY GENMASK(15, 11)
+#define GICV5_ISTL2E_IAFFID GENMASK(31, 16)
+
+/*
+ * Save Restore Header Format
+ *
+ * Track what has been saved into the guest's IST. Specifically, we track if the
+ * SPI and LPI ISTs have been stored, and the number of ID bits for each. This
+ * can be used to figure out where these start and end in the guest's memory.
+ */
+#define GICV5_SAVE_TABLES_IRS_IST_HEADER_SPI_IST BIT(0)
+#define GICV5_SAVE_TABLES_IRS_IST_HEADER_SPI_ID_BITS GENMASK(5, 1)
+#define GICV5_SAVE_TABLES_IRS_IST_HEADER_LPI_IST BIT(6)
+#define GICV5_SAVE_TABLES_IRS_IST_HEADER_LPI_ID_BITS GENMASK(11, 7)
+
+struct pending_irq {
+ u32 irq;
+ struct list_head next;
+};
+
struct vgic_v5_vm_info {
void __iomem *vmd_base;
vpe_entry __iomem *vpet_base;
@@ -79,6 +107,9 @@ struct vgic_v5_vm_info {
__le64 *h_lpi_ist;
__le64 **h_lpi_l2_ists;
__le64 *h_spi_ist;
+
+ /* Tracking of pending interrupts as part of IST restore */
+ struct list_head pending_irqs;
};

struct vgic_v5_vmt {
@@ -171,4 +202,11 @@ void vgic_v5_free_allocated_spi_ist(struct kvm *kvm);
int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits);
int vgic_v5_lpi_ist_free(struct kvm *kvm);

+int vgic_v5_save_spi_ist(struct kvm *kvm, struct kvm_device_attr *attr);
+int vgic_v5_save_lpi_ist(struct kvm *kvm);
+int vgic_v5_restore_spi_ist(struct kvm *kvm, struct kvm_device_attr *attr);
+int vgic_v5_restore_lpi_ist(struct kvm *kvm);
+int vgic_v5_restore_pending_irqs(struct kvm *kvm);
+void vgic_v5_scrap_pending_irqs(struct kvm *kvm);
+
#endif
diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c
index 3e435a31b463e..ff3500a634b62 100644
--- a/arch/arm64/kvm/vgic/vgic-v5.c
+++ b/arch/arm64/kvm/vgic/vgic-v5.c
@@ -580,6 +580,189 @@ static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
}
}

+/*
+ * Wait for completion a write to IRS_SAVE_VMR.
+ */
+static int vgic_v5_irs_wait_for_save_vm_op(void)
+{
+ int ret;
+ u32 statusr;
+
+ ret = readl_relaxed_poll_timeout_atomic(
+ irs_base + GICV5_IRS_SAVE_VM_STATUSR, statusr,
+ FIELD_GET(GICV5_IRS_SAVE_VM_STATUSR_IDLE, statusr), 1,
+ USEC_PER_SEC);
+
+ if (ret == -ETIMEDOUT) {
+ pr_err_ratelimited("Time out waiting for IRS Save VM Op\n");
+ return ret;
+ }
+
+ return 0;
+}
+
+static bool vgic_v5_irs_is_quiesced(u16 vm_id)
+{
+ int err;
+ u64 save_vmr;
+ u32 statusr;
+
+ save_vmr = FIELD_PREP(GICV5_IRS_SAVE_VMR_VM_ID, vm_id);
+ save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_Q, 1);
+ save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_S, 0);
+ irs_writeq_relaxed(save_vmr, GICV5_IRS_SAVE_VMR);
+
+ /* Wait for the operation */
+ err = vgic_v5_irs_wait_for_save_vm_op();
+ if (err)
+ return false;
+
+ statusr = irs_readl_relaxed(GICV5_IRS_SAVE_VM_STATUSR);
+
+ return statusr & GICV5_IRS_SAVE_VM_STATUSR_Q;
+}
+
+int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+ int ret = 0;
+ u64 save_vmr;
+ u16 vm_id = vgic_v5_vm_id(kvm);
+
+
+ mutex_lock(&kvm->lock);
+
+ if (kvm_trylock_all_vcpus(kvm)) {
+ mutex_unlock(&kvm->lock);
+ pr_err("Failed to lock VCPUs");
+ return -EBUSY;
+ }
+
+ mutex_lock(&kvm->arch.config_lock);
+
+ save_vmr = FIELD_PREP(GICV5_IRS_SAVE_VMR_VM_ID, vm_id);
+ save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_Q, 1);
+ save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_S, 1);
+ irs_writeq_relaxed(save_vmr, GICV5_IRS_SAVE_VMR);
+
+ /* Wait for the operation */
+ ret = vgic_v5_irs_wait_for_save_vm_op();
+ if (ret) {
+ pr_err("Timed out");
+ goto out_unlock;
+ }
+
+ if (!vgic_v5_irs_is_quiesced(vm_id)) {
+ pr_err("Cannot save; VM not quiesced after IRS_VM_SAVER write\n");
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+
+ /*
+ * Serialise the SPI IST to the userspace-provided memory (address in
+ * attr).
+ */
+ ret = vgic_v5_save_spi_ist(kvm, attr);
+ if (ret) {
+ pr_err("Failed to save the SPI IST!");
+ goto out_unlock;
+ }
+
+ if (!vgic_v5_irs_is_quiesced(vm_id)) {
+ pr_err("VM is not quiesed; failed to save IST(s)\n");
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+
+ /* Serialise the LPI IST to the guest's IST */
+ ret = vgic_v5_save_lpi_ist(kvm);
+ if (ret) {
+ pr_err("Failed to save the LPI IST!");
+ goto out_unlock;
+ }
+
+ if (!vgic_v5_irs_is_quiesced(vm_id)) {
+ pr_err("VM is not quiesed; failed to save IST(s)\n");
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+
+out_unlock:
+ mutex_unlock(&kvm->arch.config_lock);
+ kvm_unlock_all_vcpus(kvm);
+ mutex_unlock(&kvm->lock);
+
+ return ret;
+}
+
+int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+ int ret = 0;
+ struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
+
+ mutex_lock(&kvm->lock);
+
+ if (kvm_trylock_all_vcpus(kvm)) {
+ mutex_unlock(&kvm->lock);
+ return -EBUSY;
+ }
+
+ mutex_lock(&kvm->arch.config_lock);
+
+ /*
+ * The ISTs should not be written by us while the VM (or IST) is
+ * valid. In order to safely restore, and make sure that the GIC sees
+ * the latest and greatest state, make the VM invalid prior to
+ * restoring.
+ */
+ ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID);
+ if (ret) {
+ /*
+ * If we go wrong here, things are rather broken. VM is likely
+ * unrunnable.
+ */
+ goto out_unlock;
+ }
+
+ /*
+ * Unserialise the SPI IST from the userspace-provided memory (address
+ * in attr).
+ */
+ ret = vgic_v5_restore_spi_ist(kvm, attr);
+ if (ret) {
+ pr_err("Failed to restore the SPI IST!");
+ goto out_unlock;
+ }
+
+ /* Unserialise the ISTs from the guest's IST */
+ ret = vgic_v5_restore_lpi_ist(kvm);
+ if (ret) {
+ pr_err("Failed to restore the LPI IST!");
+ goto out_unlock;
+ }
+
+ /* ... and make the VM Valid again */
+ ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID);
+ if (ret)
+ goto out_unlock;
+
+ /*
+ * As part of restoring the ISTs, and previously pending interrupts have
+ * been tracked and made non-pending. Now that the ISTs have been
+ * restored, and the VM is valid again, restore the pending interrupts.
+ */
+ ret = vgic_v5_restore_pending_irqs(kvm);
+
+out_unlock:
+ if (ret)
+ vgic_v5_scrap_pending_irqs(kvm);
+
+ mutex_unlock(&kvm->arch.config_lock);
+ kvm_unlock_all_vcpus(kvm);
+ mutex_unlock(&kvm->lock);
+
+ return ret;
+}
+
/*
* This set of irq_chip functions is specific for doorbells.
*/
diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h
index 36604f911e089..35eb048b5a0f7 100644
--- a/arch/arm64/kvm/vgic/vgic.h
+++ b/arch/arm64/kvm/vgic/vgic.h
@@ -387,6 +387,8 @@ int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr, bool is_write);
int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz);
+int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr);
+int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr);

#define for_each_visible_v5_ppi(__i, __k) \
for_each_set_bit(__i, (__k)->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS)
diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h
index 9ea3674a6613b..431aca67f4d5f 100644
--- a/include/linux/irqchip/arm-gic-v5.h
+++ b/include/linux/irqchip/arm-gic-v5.h
@@ -322,6 +322,13 @@
#define GICV5_IRS_VMAP_VPER_VM_ID GENMASK_ULL(47, 32)
#define GICV5_IRS_VMAP_VPER_VPE_ID GENMASK_ULL(15, 0)

+#define GICV5_IRS_SAVE_VMR_VM_ID GENMASK_ULL(15, 0)
+#define GICV5_IRS_SAVE_VMR_Q BIT_ULL(62)
+#define GICV5_IRS_SAVE_VMR_S BIT_ULL(63)
+
+#define GICV5_IRS_SAVE_VM_STATUSR_IDLE BIT(0)
+#define GICV5_IRS_SAVE_VM_STATUSR_Q BIT(1)
+
#define GICV5_ISTL1E_VALID BIT_ULL(0)
#define GICV5_IRS_ISTL1E_SIZE 8UL

--
2.34.1