* [PATCH 01/15] s390x/pci: implement IOMMU replay
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-08 12:43 ` Cornelia Huck
2026-04-02 2:29 ` [PATCH 02/15] s390x/pci: Create function to contain translation status check Konstantin Shkolnyy
` (13 subsequent siblings)
14 siblings, 1 reply; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel
From: Matthew Rosato <mjrosato@linux.ibm.com>
There are a few scenarios where IOMMU replay can potentially be needed
for zPCI device, namely VFIO device reset scenarios where the guest
continues running and expects the contents of its IOMMU to be replayed
upon IOAT re-registration and migration scenarios where the destination
must reconstruct the IOMMU on the destination.
zPCI migration is not supported yet, but the IOMMU replay function is
implemented so that it can be called both from IOMMUMemoryRegionClass
now and migration post_load later.
Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 50 ++++++++++++++++++++++++++++----
hw/s390x/s390-pci-inst.c | 4 +--
include/hw/s390x/s390-pci-inst.h | 1 +
3 files changed, 47 insertions(+), 8 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 4de7b587e8..3665aba106 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -592,14 +592,52 @@ err:
return ret;
}
-static void s390_pci_iommu_replay(IOMMUMemoryRegion *iommu,
+static void s390_pci_ioat_replay(S390PCIIOMMU *iommu)
+{
+ S390IOTLBEntry entry;
+ uint16_t error = 0;
+ uint32_t dma_avail;
+ hwaddr curr, end;
+
+ curr = iommu->pba;
+ end = iommu->pal;
+
+ if (iommu->dm_mr) {
+ /* If direct mapping is used, there are no guest tables to replay */
+ return;
+ }
+
+ if (iommu->dma_limit) {
+ dma_avail = iommu->dma_limit->avail;
+ } else {
+ dma_avail = 1;
+ }
+
+ while (curr < end) {
+ error = s390_guest_io_table_walk(iommu->g_iota, curr, &entry);
+ if (error) {
+ error_report("Failure to walk table during iommu remap");
+ return;
+ }
+
+ if (entry.perm != IOMMU_NONE) {
+ if (dma_avail > 0) {
+ dma_avail = s390_pci_update_iotlb(iommu, &entry);
+ } else {
+ error_report("DMA mappings exhausted: iommu remap failed");
+ return;
+ }
+ }
+ curr += entry.len;
+ }
+}
+
+static void s390_pci_iommu_replay(IOMMUMemoryRegion *mr,
IOMMUNotifier *notifier)
{
- /* It's impossible to plug a pci device on s390x that already has iommu
- * mappings which need to be replayed, that is due to the "one iommu per
- * zpci device" construct. But when we support migration of vfio-pci
- * devices in future, we need to revisit this.
- */
+ S390PCIIOMMU *iommu = container_of(mr, S390PCIIOMMU, iommu_mr);
+
+ s390_pci_ioat_replay(iommu);
}
static S390PCIIOMMU *s390_pci_get_iommu(S390pciState *s, PCIBus *bus,
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 10066ca618..1834596076 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -613,8 +613,8 @@ int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
return 0;
}
-static uint32_t s390_pci_update_iotlb(S390PCIIOMMU *iommu,
- S390IOTLBEntry *entry)
+uint32_t s390_pci_update_iotlb(S390PCIIOMMU *iommu,
+ S390IOTLBEntry *entry)
{
S390IOTLBEntry *cache = g_hash_table_lookup(iommu->iotlb, &entry->iova);
IOMMUTLBEvent event = {
diff --git a/include/hw/s390x/s390-pci-inst.h b/include/hw/s390x/s390-pci-inst.h
index 5cb8da540b..c782990e3b 100644
--- a/include/hw/s390x/s390-pci-inst.h
+++ b/include/hw/s390x/s390-pci-inst.h
@@ -111,6 +111,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
uintptr_t ra);
void fmb_timer_free(S390PCIBusDevice *pbdev);
+uint32_t s390_pci_update_iotlb(S390PCIIOMMU *iommu, S390IOTLBEntry *entry);
#define ZPCI_IO_BAR_MIN 0
#define ZPCI_IO_BAR_MAX 5
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH 01/15] s390x/pci: implement IOMMU replay
2026-04-02 2:29 ` [PATCH 01/15] s390x/pci: implement IOMMU replay Konstantin Shkolnyy
@ 2026-04-08 12:43 ` Cornelia Huck
0 siblings, 0 replies; 21+ messages in thread
From: Cornelia Huck @ 2026-04-08 12:43 UTC (permalink / raw)
To: Konstantin Shkolnyy, mjrosato
Cc: alifm, richard.henderson, iii, david, pasic, borntraeger,
qemu-s390x, qemu-devel
On Wed, Apr 01 2026, Konstantin Shkolnyy <kshk@linux.ibm.com> wrote:
> From: Matthew Rosato <mjrosato@linux.ibm.com>
>
> There are a few scenarios where IOMMU replay can potentially be needed
> for zPCI device, namely VFIO device reset scenarios where the guest
> continues running and expects the contents of its IOMMU to be replayed
> upon IOAT re-registration and migration scenarios where the destination
> must reconstruct the IOMMU on the destination.
>
> zPCI migration is not supported yet, but the IOMMU replay function is
> implemented so that it can be called both from IOMMUMemoryRegionClass
> now and migration post_load later.
>
> Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com>
> ---
> hw/s390x/s390-pci-bus.c | 50 ++++++++++++++++++++++++++++----
> hw/s390x/s390-pci-inst.c | 4 +--
> include/hw/s390x/s390-pci-inst.h | 1 +
> 3 files changed, 47 insertions(+), 8 deletions(-)
Just a procees nit: Please make sure that the s-o-b chain is complete,
i.e. that your s-o-b is always last in the chain, even if you pick up a
patch authored by someone else.
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH 02/15] s390x/pci: Create function to contain translation status check
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 01/15] s390x/pci: implement IOMMU replay Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 12:03 ` Christian Borntraeger
2026-04-02 2:29 ` [PATCH 03/15] s390x/pci: Move iommu_mr from S390PCIIOMMU to S390PCIBusDevice Konstantin Shkolnyy
` (12 subsequent siblings)
14 siblings, 1 reply; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
Make it more clear what the bit means, and the new function will be called
from yet another place in the future.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-inst.c | 7 ++++++-
include/hw/s390x/s390-pci-bus.h | 1 +
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 1834596076..7c784b31f3 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -993,6 +993,11 @@ int pci_dereg_irqs(S390PCIBusDevice *pbdev)
return 0;
}
+bool s390_pci_is_translation_enabled(uint64_t g_iota)
+{
+ return ((g_iota >> 11) & 0x1) != 0; /* "T" bit */
+}
+
static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
uintptr_t ra)
{
@@ -1001,7 +1006,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
uint64_t pal = ldq_be_p(&fib.pal);
uint64_t g_iota = ldq_be_p(&fib.iota);
uint8_t dt = (g_iota >> 2) & 0x7;
- uint8_t t = (g_iota >> 11) & 0x1;
+ bool t = s390_pci_is_translation_enabled(g_iota);
pba &= ~0xfff;
pal |= 0xfff;
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 9228523ce8..eb15cb8b2d 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -390,6 +390,7 @@ int pci_chsc_sei_nt2_get_event(void *res);
int pci_chsc_sei_nt2_have_event(void);
void s390_pci_sclp_configure(SCCB *sccb);
void s390_pci_sclp_deconfigure(SCCB *sccb);
+bool s390_pci_is_translation_enabled(uint64_t g_iota);
void s390_pci_iommu_enable(S390PCIIOMMU *iommu);
void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu);
void s390_pci_iommu_disable(S390PCIIOMMU *iommu);
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH 02/15] s390x/pci: Create function to contain translation status check
2026-04-02 2:29 ` [PATCH 02/15] s390x/pci: Create function to contain translation status check Konstantin Shkolnyy
@ 2026-04-02 12:03 ` Christian Borntraeger
0 siblings, 0 replies; 21+ messages in thread
From: Christian Borntraeger @ 2026-04-02 12:03 UTC (permalink / raw)
To: Konstantin Shkolnyy, mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, qemu-s390x,
qemu-devel
Am 02.04.26 um 04:29 schrieb Konstantin Shkolnyy:
> Make it more clear what the bit means, and the new function will be called
> from yet another place in the future.
>
> Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
> ---
> hw/s390x/s390-pci-inst.c | 7 ++++++-
> include/hw/s390x/s390-pci-bus.h | 1 +
> 2 files changed, 7 insertions(+), 1 deletion(-)
>
> diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
> index 1834596076..7c784b31f3 100644
> --- a/hw/s390x/s390-pci-inst.c
> +++ b/hw/s390x/s390-pci-inst.c
> @@ -993,6 +993,11 @@ int pci_dereg_irqs(S390PCIBusDevice *pbdev)
> return 0;
> }
>
> +bool s390_pci_is_translation_enabled(uint64_t g_iota)
> +{
> + return ((g_iota >> 11) & 0x1) != 0; /* "T" bit */
I think from C perspective you can get rid of the != 0 part as
integers casts into bool with false=0 and true otherwise.
Apart from that
Reviewed-by: Christian Borntraeger <borntraeger@linux.ibm.com>
> +}
> +
> static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
> uintptr_t ra)
> {
> @@ -1001,7 +1006,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
> uint64_t pal = ldq_be_p(&fib.pal);
> uint64_t g_iota = ldq_be_p(&fib.iota);
> uint8_t dt = (g_iota >> 2) & 0x7;
> - uint8_t t = (g_iota >> 11) & 0x1;
> + bool t = s390_pci_is_translation_enabled(g_iota);
>
> pba &= ~0xfff;
> pal |= 0xfff;
> diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
> index 9228523ce8..eb15cb8b2d 100644
> --- a/include/hw/s390x/s390-pci-bus.h
> +++ b/include/hw/s390x/s390-pci-bus.h
> @@ -390,6 +390,7 @@ int pci_chsc_sei_nt2_get_event(void *res);
> int pci_chsc_sei_nt2_have_event(void);
> void s390_pci_sclp_configure(SCCB *sccb);
> void s390_pci_sclp_deconfigure(SCCB *sccb);
> +bool s390_pci_is_translation_enabled(uint64_t g_iota);
> void s390_pci_iommu_enable(S390PCIIOMMU *iommu);
> void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu);
> void s390_pci_iommu_disable(S390PCIIOMMU *iommu);
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH 03/15] s390x/pci: Move iommu_mr from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 01/15] s390x/pci: implement IOMMU replay Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 02/15] s390x/pci: Create function to contain translation status check Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 04/15] s390x/pci: Move dm_mr " Konstantin Shkolnyy
` (11 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 32 ++++++++++++++++++--------------
hw/s390x/s390-pci-inst.c | 28 +++++++++++++++-------------
include/hw/s390x/s390-pci-bus.h | 6 +++---
include/hw/s390x/s390-pci-inst.h | 4 ++--
4 files changed, 38 insertions(+), 32 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 3665aba106..99d503e85e 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -206,7 +206,7 @@ void s390_pci_sclp_deconfigure(SCCB *sccb)
pci_dereg_irqs(pbdev);
}
if (pbdev->iommu->enabled) {
- pci_dereg_ioat(pbdev->iommu);
+ pci_dereg_ioat(pbdev);
}
pbdev->state = ZPCI_FS_STANDBY;
rc = SCLP_RC_NORMAL_COMPLETION;
@@ -538,7 +538,8 @@ uint16_t s390_guest_io_table_walk(uint64_t g_iota, hwaddr addr,
static IOMMUTLBEntry s390_translate_iommu(IOMMUMemoryRegion *mr, hwaddr addr,
IOMMUAccessFlags flag, int iommu_idx)
{
- S390PCIIOMMU *iommu = container_of(mr, S390PCIIOMMU, iommu_mr);
+ S390PCIBusDevice *pbdev = container_of(mr, S390PCIBusDevice, iommu_mr);
+ S390PCIIOMMU *iommu = pbdev->iommu;
S390IOTLBEntry *entry;
uint64_t iova = addr & TARGET_PAGE_MASK;
uint16_t error = 0;
@@ -592,12 +593,13 @@ err:
return ret;
}
-static void s390_pci_ioat_replay(S390PCIIOMMU *iommu)
+static void s390_pci_ioat_replay(S390PCIBusDevice *pbdev)
{
S390IOTLBEntry entry;
uint16_t error = 0;
uint32_t dma_avail;
hwaddr curr, end;
+ S390PCIIOMMU *iommu = pbdev->iommu;
curr = iommu->pba;
end = iommu->pal;
@@ -622,7 +624,7 @@ static void s390_pci_ioat_replay(S390PCIIOMMU *iommu)
if (entry.perm != IOMMU_NONE) {
if (dma_avail > 0) {
- dma_avail = s390_pci_update_iotlb(iommu, &entry);
+ dma_avail = s390_pci_update_iotlb(pbdev, &entry);
} else {
error_report("DMA mappings exhausted: iommu remap failed");
return;
@@ -635,9 +637,9 @@ static void s390_pci_ioat_replay(S390PCIIOMMU *iommu)
static void s390_pci_iommu_replay(IOMMUMemoryRegion *mr,
IOMMUNotifier *notifier)
{
- S390PCIIOMMU *iommu = container_of(mr, S390PCIIOMMU, iommu_mr);
+ S390PCIBusDevice *pbdev = container_of(mr, S390PCIBusDevice, iommu_mr);
- s390_pci_ioat_replay(iommu);
+ s390_pci_ioat_replay(pbdev);
}
static S390PCIIOMMU *s390_pci_get_iommu(S390pciState *s, PCIBus *bus,
@@ -759,19 +761,20 @@ static const MemoryRegionOps s390_msi_ctrl_ops = {
.endianness = DEVICE_LITTLE_ENDIAN,
};
-void s390_pci_iommu_enable(S390PCIIOMMU *iommu)
+void s390_pci_iommu_enable(S390PCIBusDevice *pbdev)
{
+ S390PCIIOMMU *iommu = pbdev->iommu;
/*
* The iommu region is initialized against a 0-mapped address space,
* so the smallest IOMMU region we can define runs from 0 to the end
* of the PCI address space.
*/
char *name = g_strdup_printf("iommu-s390-%04x", iommu->pbdev->uid);
- memory_region_init_iommu(&iommu->iommu_mr, sizeof(iommu->iommu_mr),
+ memory_region_init_iommu(&pbdev->iommu_mr, sizeof(pbdev->iommu_mr),
TYPE_S390_IOMMU_MEMORY_REGION, OBJECT(&iommu->mr),
name, iommu->pal + 1);
iommu->enabled = true;
- memory_region_add_subregion(&iommu->mr, 0, MEMORY_REGION(&iommu->iommu_mr));
+ memory_region_add_subregion(&iommu->mr, 0, MEMORY_REGION(&pbdev->iommu_mr));
g_free(name);
}
@@ -797,8 +800,9 @@ void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu)
iommu->dm_mr);
}
-void s390_pci_iommu_disable(S390PCIIOMMU *iommu)
+void s390_pci_iommu_disable(S390PCIBusDevice *pbdev)
{
+ S390PCIIOMMU *iommu = pbdev->iommu;
iommu->enabled = false;
g_hash_table_remove_all(iommu->iotlb);
if (iommu->dm_mr) {
@@ -808,8 +812,8 @@ void s390_pci_iommu_disable(S390PCIIOMMU *iommu)
iommu->dm_mr = NULL;
} else {
memory_region_del_subregion(&iommu->mr,
- MEMORY_REGION(&iommu->iommu_mr));
- object_unparent(OBJECT(&iommu->iommu_mr));
+ MEMORY_REGION(&pbdev->iommu_mr));
+ object_unparent(OBJECT(&pbdev->iommu_mr));
}
}
@@ -1408,7 +1412,7 @@ static void s390_pcihost_reset(DeviceState *dev)
pci_dereg_irqs(pbdev);
}
if (pbdev->iommu->enabled) {
- pci_dereg_ioat(pbdev->iommu);
+ pci_dereg_ioat(pbdev);
}
pbdev->state = ZPCI_FS_STANDBY;
s390_pci_perform_unplug(pbdev);
@@ -1549,7 +1553,7 @@ static void s390_pci_device_reset(DeviceState *dev)
pci_dereg_irqs(pbdev);
}
if (pbdev->iommu->enabled) {
- pci_dereg_ioat(pbdev->iommu);
+ pci_dereg_ioat(pbdev);
}
fmb_timer_free(pbdev);
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 7c784b31f3..ed60e6100e 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -613,9 +613,10 @@ int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
return 0;
}
-uint32_t s390_pci_update_iotlb(S390PCIIOMMU *iommu,
+uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev,
S390IOTLBEntry *entry)
{
+ S390PCIIOMMU *iommu = pbdev->iommu;
S390IOTLBEntry *cache = g_hash_table_lookup(iommu->iotlb, &entry->iova);
IOMMUTLBEvent event = {
.type = entry->perm ? IOMMU_NOTIFIER_MAP : IOMMU_NOTIFIER_UNMAP,
@@ -645,7 +646,7 @@ uint32_t s390_pci_update_iotlb(S390PCIIOMMU *iommu,
event.type = IOMMU_NOTIFIER_UNMAP;
event.entry.perm = IOMMU_NONE;
- memory_region_notify_iommu(&iommu->iommu_mr, 0, event);
+ memory_region_notify_iommu(&pbdev->iommu_mr, 0, event);
event.type = IOMMU_NOTIFIER_MAP;
event.entry.perm = entry->perm;
}
@@ -663,13 +664,13 @@ uint32_t s390_pci_update_iotlb(S390PCIIOMMU *iommu,
* All associated iotlb entries have already been cleared, trigger the
* unmaps.
*/
- memory_region_notify_iommu(&iommu->iommu_mr, 0, event);
+ memory_region_notify_iommu(&pbdev->iommu_mr, 0, event);
out:
return iommu->dma_limit ? iommu->dma_limit->avail : 1;
}
-static void s390_pci_batch_unmap(S390PCIIOMMU *iommu, uint64_t iova,
+static void s390_pci_batch_unmap(S390PCIBusDevice *pbdev, uint64_t iova,
uint64_t len)
{
uint64_t remain = len, start = iova, end = start + len - 1, mask, size;
@@ -687,7 +688,7 @@ static void s390_pci_batch_unmap(S390PCIIOMMU *iommu, uint64_t iova,
size = mask + 1;
event.entry.iova = start;
event.entry.addr_mask = mask;
- memory_region_notify_iommu(&iommu->iommu_mr, 0, event);
+ memory_region_notify_iommu(&pbdev->iommu_mr, 0, event);
start += size;
remain -= size;
}
@@ -778,14 +779,14 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
coalesce += entry.len;
} else if (coalesce > 0) {
/* Unleash the coalesced unmap before processing a new map */
- s390_pci_batch_unmap(iommu, iova, coalesce);
+ s390_pci_batch_unmap(pbdev, iova, coalesce);
coalesce = 0;
}
start += entry.len;
while (entry.iova < start && entry.iova < end) {
if (dma_avail > 0 || entry.perm == IOMMU_NONE) {
- dma_avail = s390_pci_update_iotlb(iommu, &entry);
+ dma_avail = s390_pci_update_iotlb(pbdev, &entry);
entry.iova += TARGET_PAGE_SIZE;
entry.translated_addr += TARGET_PAGE_SIZE;
} else {
@@ -801,7 +802,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
}
if (coalesce) {
/* Unleash the coalesced unmap before finishing rpcit */
- s390_pci_batch_unmap(iommu, iova, coalesce);
+ s390_pci_batch_unmap(pbdev, iova, coalesce);
coalesce = 0;
}
if (again && dma_avail > 0)
@@ -1031,7 +1032,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
iommu->g_iota = g_iota;
if (t) {
- s390_pci_iommu_enable(iommu);
+ s390_pci_iommu_enable(pbdev);
} else {
s390_pci_iommu_direct_map_enable(iommu);
}
@@ -1039,9 +1040,10 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
return 0;
}
-void pci_dereg_ioat(S390PCIIOMMU *iommu)
+void pci_dereg_ioat(S390PCIBusDevice *pbdev)
{
- s390_pci_iommu_disable(iommu);
+ S390PCIIOMMU *iommu = pbdev->iommu;
+ s390_pci_iommu_disable(pbdev);
iommu->pba = 0;
iommu->pal = 0;
iommu->g_iota = 0;
@@ -1265,7 +1267,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
} else {
- pci_dereg_ioat(pbdev->iommu);
+ pci_dereg_ioat(pbdev);
}
break;
case ZPCI_MOD_FC_REREG_IOAT:
@@ -1276,7 +1278,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
} else {
- pci_dereg_ioat(pbdev->iommu);
+ pci_dereg_ioat(pbdev);
if (reg_ioat(env, pbdev, fib, ra)) {
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_INSUF_RES);
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index eb15cb8b2d..a71f562dfc 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -276,7 +276,6 @@ struct S390PCIIOMMU {
S390PCIBusDevice *pbdev;
AddressSpace as;
MemoryRegion mr;
- IOMMUMemoryRegion iommu_mr;
MemoryRegion *dm_mr;
bool enabled;
uint64_t g_iota;
@@ -354,6 +353,7 @@ struct S390PCIBusDevice {
S390MsixInfo msix;
AdapterRoutes routes;
S390PCIIOMMU *iommu;
+ IOMMUMemoryRegion iommu_mr;
MemoryRegion msix_notify_mr;
IndAddr *summary_ind;
IndAddr *indicator;
@@ -391,9 +391,9 @@ int pci_chsc_sei_nt2_have_event(void);
void s390_pci_sclp_configure(SCCB *sccb);
void s390_pci_sclp_deconfigure(SCCB *sccb);
bool s390_pci_is_translation_enabled(uint64_t g_iota);
-void s390_pci_iommu_enable(S390PCIIOMMU *iommu);
+void s390_pci_iommu_enable(S390PCIBusDevice *pbdev);
void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu);
-void s390_pci_iommu_disable(S390PCIIOMMU *iommu);
+void s390_pci_iommu_disable(S390PCIBusDevice *pbdev);
void s390_pci_generate_error_event(uint16_t pec, uint32_t fh, uint32_t fid,
uint64_t faddr, uint32_t e);
uint16_t s390_guest_io_table_walk(uint64_t g_iota, hwaddr addr,
diff --git a/include/hw/s390x/s390-pci-inst.h b/include/hw/s390x/s390-pci-inst.h
index c782990e3b..38268c256e 100644
--- a/include/hw/s390x/s390-pci-inst.h
+++ b/include/hw/s390x/s390-pci-inst.h
@@ -99,7 +99,7 @@ typedef struct ZpciFib {
} QEMU_PACKED ZpciFib;
int pci_dereg_irqs(S390PCIBusDevice *pbdev);
-void pci_dereg_ioat(S390PCIIOMMU *iommu);
+void pci_dereg_ioat(S390PCIBusDevice *pbdev);
int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra);
int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra);
int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra);
@@ -111,7 +111,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
uintptr_t ra);
void fmb_timer_free(S390PCIBusDevice *pbdev);
-uint32_t s390_pci_update_iotlb(S390PCIIOMMU *iommu, S390IOTLBEntry *entry);
+uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev, S390IOTLBEntry *entry);
#define ZPCI_IO_BAR_MIN 0
#define ZPCI_IO_BAR_MAX 5
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 04/15] s390x/pci: Move dm_mr from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (2 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 03/15] s390x/pci: Move iommu_mr from S390PCIIOMMU to S390PCIBusDevice Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 05/15] s390x/pci: Move iotlb " Konstantin Shkolnyy
` (10 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 21 +++++++++++----------
hw/s390x/s390-pci-inst.c | 2 +-
include/hw/s390x/s390-pci-bus.h | 4 ++--
3 files changed, 14 insertions(+), 13 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 99d503e85e..117fd8136e 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -604,7 +604,7 @@ static void s390_pci_ioat_replay(S390PCIBusDevice *pbdev)
curr = iommu->pba;
end = iommu->pal;
- if (iommu->dm_mr) {
+ if (pbdev->dm_mr) {
/* If direct mapping is used, there are no guest tables to replay */
return;
}
@@ -778,8 +778,9 @@ void s390_pci_iommu_enable(S390PCIBusDevice *pbdev)
g_free(name);
}
-void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu)
+void s390_pci_iommu_direct_map_enable(S390PCIBusDevice *pbdev)
{
+ S390PCIIOMMU *iommu = pbdev->iommu;
MachineState *ms = MACHINE(qdev_get_machine());
S390CcwMachineState *s390ms = S390_CCW_MACHINE(ms);
@@ -791,13 +792,13 @@ void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu)
g_autofree char *name = g_strdup_printf("iommu-dm-s390-%04x",
iommu->pbdev->uid);
- iommu->dm_mr = g_malloc0(sizeof(*iommu->dm_mr));
- memory_region_init_alias(iommu->dm_mr, OBJECT(&iommu->mr), name,
+ pbdev->dm_mr = g_malloc0(sizeof(*pbdev->dm_mr));
+ memory_region_init_alias(pbdev->dm_mr, OBJECT(&iommu->mr), name,
get_system_memory(), 0,
s390_get_memory_limit(s390ms));
iommu->enabled = true;
memory_region_add_subregion(&iommu->mr, iommu->pbdev->zpci_fn.sdma,
- iommu->dm_mr);
+ pbdev->dm_mr);
}
void s390_pci_iommu_disable(S390PCIBusDevice *pbdev)
@@ -805,11 +806,11 @@ void s390_pci_iommu_disable(S390PCIBusDevice *pbdev)
S390PCIIOMMU *iommu = pbdev->iommu;
iommu->enabled = false;
g_hash_table_remove_all(iommu->iotlb);
- if (iommu->dm_mr) {
- memory_region_del_subregion(&iommu->mr, iommu->dm_mr);
- object_unparent(OBJECT(iommu->dm_mr));
- g_free(iommu->dm_mr);
- iommu->dm_mr = NULL;
+ if (pbdev->dm_mr) {
+ memory_region_del_subregion(&iommu->mr, pbdev->dm_mr);
+ object_unparent(OBJECT(pbdev->dm_mr));
+ g_free(pbdev->dm_mr);
+ pbdev->dm_mr = NULL;
} else {
memory_region_del_subregion(&iommu->mr,
MEMORY_REGION(&pbdev->iommu_mr));
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index ed60e6100e..834a0aec95 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -1034,7 +1034,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
if (t) {
s390_pci_iommu_enable(pbdev);
} else {
- s390_pci_iommu_direct_map_enable(iommu);
+ s390_pci_iommu_direct_map_enable(pbdev);
}
return 0;
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index a71f562dfc..f310672584 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -276,7 +276,6 @@ struct S390PCIIOMMU {
S390PCIBusDevice *pbdev;
AddressSpace as;
MemoryRegion mr;
- MemoryRegion *dm_mr;
bool enabled;
uint64_t g_iota;
uint64_t pba;
@@ -354,6 +353,7 @@ struct S390PCIBusDevice {
AdapterRoutes routes;
S390PCIIOMMU *iommu;
IOMMUMemoryRegion iommu_mr;
+ MemoryRegion *dm_mr;
MemoryRegion msix_notify_mr;
IndAddr *summary_ind;
IndAddr *indicator;
@@ -392,7 +392,7 @@ void s390_pci_sclp_configure(SCCB *sccb);
void s390_pci_sclp_deconfigure(SCCB *sccb);
bool s390_pci_is_translation_enabled(uint64_t g_iota);
void s390_pci_iommu_enable(S390PCIBusDevice *pbdev);
-void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu);
+void s390_pci_iommu_direct_map_enable(S390PCIBusDevice *pbdev);
void s390_pci_iommu_disable(S390PCIBusDevice *pbdev);
void s390_pci_generate_error_event(uint16_t pec, uint32_t fh, uint32_t fid,
uint64_t faddr, uint32_t e);
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 05/15] s390x/pci: Move iotlb from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (3 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 04/15] s390x/pci: Move dm_mr " Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 06/15] s390x/pci: Remove a ptr to S390PCIBusDevice from S390PCIIOMMU Konstantin Shkolnyy
` (9 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 10 +++++-----
hw/s390x/s390-pci-inst.c | 6 +++---
include/hw/s390x/s390-pci-bus.h | 2 +-
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 117fd8136e..3f3123e62a 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -569,7 +569,7 @@ static IOMMUTLBEntry s390_translate_iommu(IOMMUMemoryRegion *mr, hwaddr addr,
goto err;
}
- entry = g_hash_table_lookup(iommu->iotlb, &iova);
+ entry = g_hash_table_lookup(pbdev->iotlb, &iova);
if (entry) {
ret.iova = entry->iova;
ret.translated_addr = entry->translated_addr;
@@ -669,8 +669,6 @@ static S390PCIIOMMU *s390_pci_get_iommu(S390pciState *s, PCIBus *bus,
PCI_FUNC(devfn));
memory_region_init(&iommu->mr, OBJECT(iommu), mr_name, UINT64_MAX);
address_space_init(&iommu->as, &iommu->mr, as_name);
- iommu->iotlb = g_hash_table_new_full(g_int64_hash, g_int64_equal,
- NULL, g_free);
table->iommu[PCI_SLOT(devfn)] = iommu;
g_free(mr_name);
@@ -805,7 +803,7 @@ void s390_pci_iommu_disable(S390PCIBusDevice *pbdev)
{
S390PCIIOMMU *iommu = pbdev->iommu;
iommu->enabled = false;
- g_hash_table_remove_all(iommu->iotlb);
+ g_hash_table_remove_all(pbdev->iotlb);
if (pbdev->dm_mr) {
memory_region_del_subregion(&iommu->mr, pbdev->dm_mr);
object_unparent(OBJECT(pbdev->dm_mr));
@@ -829,7 +827,6 @@ static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn)
}
table->iommu[PCI_SLOT(devfn)] = NULL;
- g_hash_table_destroy(iommu->iotlb);
/*
* An attached PCI device may have memory listeners, eg. VFIO PCI.
* The associated subregion will already have been unmapped in
@@ -1251,6 +1248,8 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
/* the allocated idx is actually getting used */
s->next_idx = (pbdev->idx + 1) & FH_MASK_INDEX;
pbdev->fh = pbdev->idx;
+ pbdev->iotlb = g_hash_table_new_full(g_int64_hash, g_int64_equal,
+ NULL, g_free);
QTAILQ_INSERT_TAIL(&s->zpci_devs, pbdev, link);
g_hash_table_insert(s->zpci_table, &pbdev->idx, pbdev);
} else {
@@ -1293,6 +1292,7 @@ static void s390_pcihost_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
if (pbdev->iommu && pbdev->iommu->dma_limit) {
s390_pci_end_dma_count(s, pbdev->iommu->dma_limit);
}
+ g_hash_table_destroy(pbdev->iotlb);
qdev_unrealize(dev);
}
}
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 834a0aec95..8349b44dd4 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -617,7 +617,7 @@ uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev,
S390IOTLBEntry *entry)
{
S390PCIIOMMU *iommu = pbdev->iommu;
- S390IOTLBEntry *cache = g_hash_table_lookup(iommu->iotlb, &entry->iova);
+ S390IOTLBEntry *cache = g_hash_table_lookup(pbdev->iotlb, &entry->iova);
IOMMUTLBEvent event = {
.type = entry->perm ? IOMMU_NOTIFIER_MAP : IOMMU_NOTIFIER_UNMAP,
.entry = {
@@ -633,7 +633,7 @@ uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev,
if (!cache) {
goto out;
}
- g_hash_table_remove(iommu->iotlb, &entry->iova);
+ g_hash_table_remove(pbdev->iotlb, &entry->iova);
inc_dma_avail(iommu);
/* Don't notify the iommu yet, maybe we can bundle contiguous unmaps */
goto out;
@@ -656,7 +656,7 @@ uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev,
cache->translated_addr = entry->translated_addr;
cache->len = TARGET_PAGE_SIZE;
cache->perm = entry->perm;
- g_hash_table_replace(iommu->iotlb, &cache->iova, cache);
+ g_hash_table_replace(pbdev->iotlb, &cache->iova, cache);
dec_dma_avail(iommu);
}
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index f310672584..d9bafd6859 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -281,7 +281,6 @@ struct S390PCIIOMMU {
uint64_t pba;
uint64_t pal;
uint64_t max_dma_limit;
- GHashTable *iotlb;
S390PCIDMACount *dma_limit;
};
@@ -354,6 +353,7 @@ struct S390PCIBusDevice {
S390PCIIOMMU *iommu;
IOMMUMemoryRegion iommu_mr;
MemoryRegion *dm_mr;
+ GHashTable *iotlb;
MemoryRegion msix_notify_mr;
IndAddr *summary_ind;
IndAddr *indicator;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 06/15] s390x/pci: Remove a ptr to S390PCIBusDevice from S390PCIIOMMU
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (4 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 05/15] s390x/pci: Move iotlb " Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 07/15] s390x/pci: Move/rename enabled from S390PCIIOMMU to S390PCIBusDevice Konstantin Shkolnyy
` (8 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This pointer is no longer used, after fields were moved from S390PCIIOMMU
to S390PCIBusDevice.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 15 +++++++--------
include/hw/s390x/s390-pci-bus.h | 1 -
2 files changed, 7 insertions(+), 9 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 3f3123e62a..83e9ca99f3 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -551,7 +551,7 @@ static IOMMUTLBEntry s390_translate_iommu(IOMMUMemoryRegion *mr, hwaddr addr,
.perm = IOMMU_NONE,
};
- switch (iommu->pbdev->state) {
+ switch (pbdev->state) {
case ZPCI_FS_ENABLED:
case ZPCI_FS_BLOCKED:
if (!iommu->enabled) {
@@ -586,9 +586,9 @@ static IOMMUTLBEntry s390_translate_iommu(IOMMUMemoryRegion *mr, hwaddr addr,
}
err:
if (error) {
- iommu->pbdev->state = ZPCI_FS_ERROR;
- s390_pci_generate_error_event(error, iommu->pbdev->fh,
- iommu->pbdev->fid, addr, 0);
+ pbdev->state = ZPCI_FS_ERROR;
+ s390_pci_generate_error_event(error, pbdev->fh,
+ pbdev->fid, addr, 0);
}
return ret;
}
@@ -767,7 +767,7 @@ void s390_pci_iommu_enable(S390PCIBusDevice *pbdev)
* so the smallest IOMMU region we can define runs from 0 to the end
* of the PCI address space.
*/
- char *name = g_strdup_printf("iommu-s390-%04x", iommu->pbdev->uid);
+ char *name = g_strdup_printf("iommu-s390-%04x", pbdev->uid);
memory_region_init_iommu(&pbdev->iommu_mr, sizeof(pbdev->iommu_mr),
TYPE_S390_IOMMU_MEMORY_REGION, OBJECT(&iommu->mr),
name, iommu->pal + 1);
@@ -788,14 +788,14 @@ void s390_pci_iommu_direct_map_enable(S390PCIBusDevice *pbdev)
* IOVA X + SDMA. VFIO will handle pinning via its memory listener.
*/
g_autofree char *name = g_strdup_printf("iommu-dm-s390-%04x",
- iommu->pbdev->uid);
+ pbdev->uid);
pbdev->dm_mr = g_malloc0(sizeof(*pbdev->dm_mr));
memory_region_init_alias(pbdev->dm_mr, OBJECT(&iommu->mr), name,
get_system_memory(), 0,
s390_get_memory_limit(s390ms));
iommu->enabled = true;
- memory_region_add_subregion(&iommu->mr, iommu->pbdev->zpci_fn.sdma,
+ memory_region_add_subregion(&iommu->mr, pbdev->zpci_fn.sdma,
pbdev->dm_mr);
}
@@ -1187,7 +1187,6 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
pbdev->pdev = pdev;
pbdev->iommu = s390_pci_get_iommu(s, pci_get_bus(pdev), pdev->devfn);
- pbdev->iommu->pbdev = pbdev;
pbdev->state = ZPCI_FS_DISABLED;
set_pbdev_info(pbdev);
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index d9bafd6859..fcaaf19512 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -273,7 +273,6 @@ typedef struct S390PCIDMACount {
struct S390PCIIOMMU {
Object parent_obj;
- S390PCIBusDevice *pbdev;
AddressSpace as;
MemoryRegion mr;
bool enabled;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 07/15] s390x/pci: Move/rename enabled from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (5 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 06/15] s390x/pci: Remove a ptr to S390PCIBusDevice from S390PCIIOMMU Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 08/15] s390x/pci: Move dma_limit " Konstantin Shkolnyy
` (7 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
This also allows to save/restore this field during migration.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 14 +++++++-------
hw/s390x/s390-pci-inst.c | 8 ++++----
include/hw/s390x/s390-pci-bus.h | 2 +-
3 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 83e9ca99f3..27ba3c801c 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -205,7 +205,7 @@ void s390_pci_sclp_deconfigure(SCCB *sccb)
} else if (pbdev->summary_ind) {
pci_dereg_irqs(pbdev);
}
- if (pbdev->iommu->enabled) {
+ if (pbdev->iommu_enabled) {
pci_dereg_ioat(pbdev);
}
pbdev->state = ZPCI_FS_STANDBY;
@@ -554,7 +554,7 @@ static IOMMUTLBEntry s390_translate_iommu(IOMMUMemoryRegion *mr, hwaddr addr,
switch (pbdev->state) {
case ZPCI_FS_ENABLED:
case ZPCI_FS_BLOCKED:
- if (!iommu->enabled) {
+ if (!pbdev->iommu_enabled) {
return ret;
}
break;
@@ -771,7 +771,7 @@ void s390_pci_iommu_enable(S390PCIBusDevice *pbdev)
memory_region_init_iommu(&pbdev->iommu_mr, sizeof(pbdev->iommu_mr),
TYPE_S390_IOMMU_MEMORY_REGION, OBJECT(&iommu->mr),
name, iommu->pal + 1);
- iommu->enabled = true;
+ pbdev->iommu_enabled = true;
memory_region_add_subregion(&iommu->mr, 0, MEMORY_REGION(&pbdev->iommu_mr));
g_free(name);
}
@@ -794,7 +794,7 @@ void s390_pci_iommu_direct_map_enable(S390PCIBusDevice *pbdev)
memory_region_init_alias(pbdev->dm_mr, OBJECT(&iommu->mr), name,
get_system_memory(), 0,
s390_get_memory_limit(s390ms));
- iommu->enabled = true;
+ pbdev->iommu_enabled = true;
memory_region_add_subregion(&iommu->mr, pbdev->zpci_fn.sdma,
pbdev->dm_mr);
}
@@ -802,7 +802,7 @@ void s390_pci_iommu_direct_map_enable(S390PCIBusDevice *pbdev)
void s390_pci_iommu_disable(S390PCIBusDevice *pbdev)
{
S390PCIIOMMU *iommu = pbdev->iommu;
- iommu->enabled = false;
+ pbdev->iommu_enabled = false;
g_hash_table_remove_all(pbdev->iotlb);
if (pbdev->dm_mr) {
memory_region_del_subregion(&iommu->mr, pbdev->dm_mr);
@@ -1411,7 +1411,7 @@ static void s390_pcihost_reset(DeviceState *dev)
} else if (pbdev->summary_ind) {
pci_dereg_irqs(pbdev);
}
- if (pbdev->iommu->enabled) {
+ if (pbdev->iommu_enabled) {
pci_dereg_ioat(pbdev);
}
pbdev->state = ZPCI_FS_STANDBY;
@@ -1552,7 +1552,7 @@ static void s390_pci_device_reset(DeviceState *dev)
} else if (pbdev->summary_ind) {
pci_dereg_irqs(pbdev);
}
- if (pbdev->iommu->enabled) {
+ if (pbdev->iommu_enabled) {
pci_dereg_ioat(pbdev);
}
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 8349b44dd4..4458eae681 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -1251,7 +1251,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
if (dmaas != 0) {
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
- } else if (pbdev->iommu->enabled) {
+ } else if (pbdev->iommu_enabled) {
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
} else if (reg_ioat(env, pbdev, fib, ra)) {
@@ -1263,7 +1263,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
if (dmaas != 0) {
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
- } else if (!pbdev->iommu->enabled) {
+ } else if (!pbdev->iommu_enabled) {
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
} else {
@@ -1274,7 +1274,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
if (dmaas != 0) {
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
- } else if (!pbdev->iommu->enabled) {
+ } else if (!pbdev->iommu_enabled) {
cc = ZPCI_PCI_LS_ERR;
s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
} else {
@@ -1404,7 +1404,7 @@ int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
/* fallthrough */
case ZPCI_FS_ENABLED:
fib.fc |= 0x80;
- if (pbdev->iommu->enabled) {
+ if (pbdev->iommu_enabled) {
fib.fc |= 0x10;
}
if (!(fh & FH_MASK_ENABLE)) {
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index fcaaf19512..24f6e60786 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -275,7 +275,6 @@ struct S390PCIIOMMU {
Object parent_obj;
AddressSpace as;
MemoryRegion mr;
- bool enabled;
uint64_t g_iota;
uint64_t pba;
uint64_t pal;
@@ -350,6 +349,7 @@ struct S390PCIBusDevice {
S390MsixInfo msix;
AdapterRoutes routes;
S390PCIIOMMU *iommu;
+ bool iommu_enabled;
IOMMUMemoryRegion iommu_mr;
MemoryRegion *dm_mr;
GHashTable *iotlb;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 08/15] s390x/pci: Move dma_limit from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (6 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 07/15] s390x/pci: Move/rename enabled from S390PCIIOMMU to S390PCIBusDevice Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 09/15] s390x/pci: Move g_iota " Konstantin Shkolnyy
` (6 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 10 +++++-----
hw/s390x/s390-pci-inst.c | 23 +++++++++++------------
include/hw/s390x/s390-pci-bus.h | 2 +-
3 files changed, 17 insertions(+), 18 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 27ba3c801c..4a0b31b98b 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -609,8 +609,8 @@ static void s390_pci_ioat_replay(S390PCIBusDevice *pbdev)
return;
}
- if (iommu->dma_limit) {
- dma_avail = iommu->dma_limit->avail;
+ if (pbdev->dma_limit) {
+ dma_avail = pbdev->dma_limit->avail;
} else {
dma_avail = 1;
}
@@ -1210,7 +1210,7 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
pbdev->forwarding_assist = false;
}
}
- pbdev->iommu->dma_limit = s390_pci_start_dma_count(s, pbdev);
+ pbdev->dma_limit = s390_pci_start_dma_count(s, pbdev);
/* Fill in CLP information passed via the vfio region */
s390_pci_get_clp_info(pbdev);
if (!pbdev->interp) {
@@ -1288,8 +1288,8 @@ static void s390_pcihost_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
pbdev->fid = 0;
QTAILQ_REMOVE(&s->zpci_devs, pbdev, link);
g_hash_table_remove(s->zpci_table, &pbdev->idx);
- if (pbdev->iommu && pbdev->iommu->dma_limit) {
- s390_pci_end_dma_count(s, pbdev->iommu->dma_limit);
+ if (pbdev->dma_limit) {
+ s390_pci_end_dma_count(s, pbdev->dma_limit);
}
g_hash_table_destroy(pbdev->iotlb);
qdev_unrealize(dev);
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 4458eae681..ac4e7f412d 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -28,17 +28,17 @@
#include "trace.h"
-static inline void inc_dma_avail(S390PCIIOMMU *iommu)
+static inline void inc_dma_avail(S390PCIBusDevice *pbdev)
{
- if (iommu->dma_limit) {
- iommu->dma_limit->avail++;
+ if (pbdev->dma_limit) {
+ pbdev->dma_limit->avail++;
}
}
-static inline void dec_dma_avail(S390PCIIOMMU *iommu)
+static inline void dec_dma_avail(S390PCIBusDevice *pbdev)
{
- if (iommu->dma_limit) {
- iommu->dma_limit->avail--;
+ if (pbdev->dma_limit) {
+ pbdev->dma_limit->avail--;
}
}
@@ -616,7 +616,6 @@ int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev,
S390IOTLBEntry *entry)
{
- S390PCIIOMMU *iommu = pbdev->iommu;
S390IOTLBEntry *cache = g_hash_table_lookup(pbdev->iotlb, &entry->iova);
IOMMUTLBEvent event = {
.type = entry->perm ? IOMMU_NOTIFIER_MAP : IOMMU_NOTIFIER_UNMAP,
@@ -634,7 +633,7 @@ uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev,
goto out;
}
g_hash_table_remove(pbdev->iotlb, &entry->iova);
- inc_dma_avail(iommu);
+ inc_dma_avail(pbdev);
/* Don't notify the iommu yet, maybe we can bundle contiguous unmaps */
goto out;
} else {
@@ -657,7 +656,7 @@ uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev,
cache->len = TARGET_PAGE_SIZE;
cache->perm = entry->perm;
g_hash_table_replace(pbdev->iotlb, &cache->iova, cache);
- dec_dma_avail(iommu);
+ dec_dma_avail(pbdev);
}
/*
@@ -667,7 +666,7 @@ uint32_t s390_pci_update_iotlb(S390PCIBusDevice *pbdev,
memory_region_notify_iommu(&pbdev->iommu_mr, 0, event);
out:
- return iommu->dma_limit ? iommu->dma_limit->avail : 1;
+ return pbdev->dma_limit ? pbdev->dma_limit->avail : 1;
}
static void s390_pci_batch_unmap(S390PCIBusDevice *pbdev, uint64_t iova,
@@ -744,8 +743,8 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
}
iommu = pbdev->iommu;
- if (iommu->dma_limit) {
- dma_avail = iommu->dma_limit->avail;
+ if (pbdev->dma_limit) {
+ dma_avail = pbdev->dma_limit->avail;
} else {
dma_avail = 1;
}
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 24f6e60786..126a50c98b 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -279,7 +279,6 @@ struct S390PCIIOMMU {
uint64_t pba;
uint64_t pal;
uint64_t max_dma_limit;
- S390PCIDMACount *dma_limit;
};
typedef struct S390PCIIOMMUTable {
@@ -353,6 +352,7 @@ struct S390PCIBusDevice {
IOMMUMemoryRegion iommu_mr;
MemoryRegion *dm_mr;
GHashTable *iotlb;
+ S390PCIDMACount *dma_limit;
MemoryRegion msix_notify_mr;
IndAddr *summary_ind;
IndAddr *indicator;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 09/15] s390x/pci: Move g_iota from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (7 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 08/15] s390x/pci: Move dma_limit " Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 10/15] s390x/pci: Move pba " Konstantin Shkolnyy
` (5 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
This also allows to save/restore this field during migration.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 2 +-
hw/s390x/s390-pci-inst.c | 10 +++++-----
include/hw/s390x/s390-pci-bus.h | 2 +-
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 4a0b31b98b..b784196c9e 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -616,7 +616,7 @@ static void s390_pci_ioat_replay(S390PCIBusDevice *pbdev)
}
while (curr < end) {
- error = s390_guest_io_table_walk(iommu->g_iota, curr, &entry);
+ error = s390_guest_io_table_walk(pbdev->g_iota, curr, &entry);
if (error) {
error_report("Failure to walk table during iommu remap");
return;
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index ac4e7f412d..f4ec299614 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -748,7 +748,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
} else {
dma_avail = 1;
}
- if (!iommu->g_iota) {
+ if (!pbdev->g_iota) {
error = ERR_EVENT_INVALAS;
goto err;
}
@@ -762,7 +762,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
start = sstart;
again = false;
while (start < end) {
- error = s390_guest_io_table_walk(iommu->g_iota, start, &entry);
+ error = s390_guest_io_table_walk(pbdev->g_iota, start, &entry);
if (error) {
break;
}
@@ -1028,7 +1028,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
iommu->pba = pba;
iommu->pal = pal;
- iommu->g_iota = g_iota;
+ pbdev->g_iota = g_iota;
if (t) {
s390_pci_iommu_enable(pbdev);
@@ -1045,7 +1045,7 @@ void pci_dereg_ioat(S390PCIBusDevice *pbdev)
s390_pci_iommu_disable(pbdev);
iommu->pba = 0;
iommu->pal = 0;
- iommu->g_iota = 0;
+ pbdev->g_iota = 0;
}
void fmb_timer_free(S390PCIBusDevice *pbdev)
@@ -1418,7 +1418,7 @@ int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
stq_be_p(&fib.pba, pbdev->iommu->pba);
stq_be_p(&fib.pal, pbdev->iommu->pal);
- stq_be_p(&fib.iota, pbdev->iommu->g_iota);
+ stq_be_p(&fib.iota, pbdev->g_iota);
stq_be_p(&fib.aibv, pbdev->routes.adapter.ind_addr);
stq_be_p(&fib.aisb, pbdev->routes.adapter.summary_addr);
stq_be_p(&fib.fmb_addr, pbdev->fmb_addr);
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 126a50c98b..a7d1f1ade2 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -275,7 +275,6 @@ struct S390PCIIOMMU {
Object parent_obj;
AddressSpace as;
MemoryRegion mr;
- uint64_t g_iota;
uint64_t pba;
uint64_t pal;
uint64_t max_dma_limit;
@@ -352,6 +351,7 @@ struct S390PCIBusDevice {
IOMMUMemoryRegion iommu_mr;
MemoryRegion *dm_mr;
GHashTable *iotlb;
+ uint64_t g_iota;
S390PCIDMACount *dma_limit;
MemoryRegion msix_notify_mr;
IndAddr *summary_ind;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 10/15] s390x/pci: Move pba from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (8 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 09/15] s390x/pci: Move g_iota " Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 11/15] s390x/pci: Move pal " Konstantin Shkolnyy
` (4 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
This also allows to save/restore this field during migration.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 4 ++--
hw/s390x/s390-pci-inst.c | 8 ++++----
include/hw/s390x/s390-pci-bus.h | 2 +-
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index b784196c9e..734af3a307 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -564,7 +564,7 @@ static IOMMUTLBEntry s390_translate_iommu(IOMMUMemoryRegion *mr, hwaddr addr,
trace_s390_pci_iommu_xlate(addr);
- if (addr < iommu->pba || addr > iommu->pal) {
+ if (addr < pbdev->pba || addr > iommu->pal) {
error = ERR_EVENT_OORANGE;
goto err;
}
@@ -601,7 +601,7 @@ static void s390_pci_ioat_replay(S390PCIBusDevice *pbdev)
hwaddr curr, end;
S390PCIIOMMU *iommu = pbdev->iommu;
- curr = iommu->pba;
+ curr = pbdev->pba;
end = iommu->pal;
if (pbdev->dm_mr) {
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index f4ec299614..7680d87080 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -753,7 +753,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
goto err;
}
- if (end < iommu->pba || start > iommu->pal) {
+ if (end < pbdev->pba || start > iommu->pal) {
error = ERR_EVENT_OORANGE;
goto err;
}
@@ -1026,7 +1026,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
return -EINVAL;
}
- iommu->pba = pba;
+ pbdev->pba = pba;
iommu->pal = pal;
pbdev->g_iota = g_iota;
@@ -1043,7 +1043,7 @@ void pci_dereg_ioat(S390PCIBusDevice *pbdev)
{
S390PCIIOMMU *iommu = pbdev->iommu;
s390_pci_iommu_disable(pbdev);
- iommu->pba = 0;
+ pbdev->pba = 0;
iommu->pal = 0;
pbdev->g_iota = 0;
}
@@ -1416,7 +1416,7 @@ int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
return 0;
}
- stq_be_p(&fib.pba, pbdev->iommu->pba);
+ stq_be_p(&fib.pba, pbdev->pba);
stq_be_p(&fib.pal, pbdev->iommu->pal);
stq_be_p(&fib.iota, pbdev->g_iota);
stq_be_p(&fib.aibv, pbdev->routes.adapter.ind_addr);
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index a7d1f1ade2..42f39a5cf0 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -275,7 +275,6 @@ struct S390PCIIOMMU {
Object parent_obj;
AddressSpace as;
MemoryRegion mr;
- uint64_t pba;
uint64_t pal;
uint64_t max_dma_limit;
};
@@ -352,6 +351,7 @@ struct S390PCIBusDevice {
MemoryRegion *dm_mr;
GHashTable *iotlb;
uint64_t g_iota;
+ uint64_t pba;
S390PCIDMACount *dma_limit;
MemoryRegion msix_notify_mr;
IndAddr *summary_ind;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 11/15] s390x/pci: Move pal from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (9 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 10/15] s390x/pci: Move pba " Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 12/15] s390x/pci: Move max_dma_limit " Konstantin Shkolnyy
` (3 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
This also allows to save/restore this field during migration.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 8 +++-----
hw/s390x/s390-pci-inst.c | 12 ++++--------
include/hw/s390x/s390-pci-bus.h | 2 +-
3 files changed, 8 insertions(+), 14 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 734af3a307..f339a0fd94 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -539,7 +539,6 @@ static IOMMUTLBEntry s390_translate_iommu(IOMMUMemoryRegion *mr, hwaddr addr,
IOMMUAccessFlags flag, int iommu_idx)
{
S390PCIBusDevice *pbdev = container_of(mr, S390PCIBusDevice, iommu_mr);
- S390PCIIOMMU *iommu = pbdev->iommu;
S390IOTLBEntry *entry;
uint64_t iova = addr & TARGET_PAGE_MASK;
uint16_t error = 0;
@@ -564,7 +563,7 @@ static IOMMUTLBEntry s390_translate_iommu(IOMMUMemoryRegion *mr, hwaddr addr,
trace_s390_pci_iommu_xlate(addr);
- if (addr < pbdev->pba || addr > iommu->pal) {
+ if (addr < pbdev->pba || addr > pbdev->pal) {
error = ERR_EVENT_OORANGE;
goto err;
}
@@ -599,10 +598,9 @@ static void s390_pci_ioat_replay(S390PCIBusDevice *pbdev)
uint16_t error = 0;
uint32_t dma_avail;
hwaddr curr, end;
- S390PCIIOMMU *iommu = pbdev->iommu;
curr = pbdev->pba;
- end = iommu->pal;
+ end = pbdev->pal;
if (pbdev->dm_mr) {
/* If direct mapping is used, there are no guest tables to replay */
@@ -770,7 +768,7 @@ void s390_pci_iommu_enable(S390PCIBusDevice *pbdev)
char *name = g_strdup_printf("iommu-s390-%04x", pbdev->uid);
memory_region_init_iommu(&pbdev->iommu_mr, sizeof(pbdev->iommu_mr),
TYPE_S390_IOMMU_MEMORY_REGION, OBJECT(&iommu->mr),
- name, iommu->pal + 1);
+ name, pbdev->pal + 1);
pbdev->iommu_enabled = true;
memory_region_add_subregion(&iommu->mr, 0, MEMORY_REGION(&pbdev->iommu_mr));
g_free(name);
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 7680d87080..191f549ba7 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -700,7 +700,6 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
uint32_t fh;
uint16_t error = 0;
S390PCIBusDevice *pbdev;
- S390PCIIOMMU *iommu;
S390IOTLBEntry entry;
hwaddr start, end, sstart;
uint32_t dma_avail;
@@ -742,7 +741,6 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
break;
}
- iommu = pbdev->iommu;
if (pbdev->dma_limit) {
dma_avail = pbdev->dma_limit->avail;
} else {
@@ -753,7 +751,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
goto err;
}
- if (end < pbdev->pba || start > iommu->pal) {
+ if (end < pbdev->pba || start > pbdev->pal) {
error = ERR_EVENT_OORANGE;
goto err;
}
@@ -1001,7 +999,6 @@ bool s390_pci_is_translation_enabled(uint64_t g_iota)
static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
uintptr_t ra)
{
- S390PCIIOMMU *iommu = pbdev->iommu;
uint64_t pba = ldq_be_p(&fib.pba);
uint64_t pal = ldq_be_p(&fib.pal);
uint64_t g_iota = ldq_be_p(&fib.iota);
@@ -1027,7 +1024,7 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
}
pbdev->pba = pba;
- iommu->pal = pal;
+ pbdev->pal = pal;
pbdev->g_iota = g_iota;
if (t) {
@@ -1041,10 +1038,9 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib,
void pci_dereg_ioat(S390PCIBusDevice *pbdev)
{
- S390PCIIOMMU *iommu = pbdev->iommu;
s390_pci_iommu_disable(pbdev);
pbdev->pba = 0;
- iommu->pal = 0;
+ pbdev->pal = 0;
pbdev->g_iota = 0;
}
@@ -1417,7 +1413,7 @@ int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
}
stq_be_p(&fib.pba, pbdev->pba);
- stq_be_p(&fib.pal, pbdev->iommu->pal);
+ stq_be_p(&fib.pal, pbdev->pal);
stq_be_p(&fib.iota, pbdev->g_iota);
stq_be_p(&fib.aibv, pbdev->routes.adapter.ind_addr);
stq_be_p(&fib.aisb, pbdev->routes.adapter.summary_addr);
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 42f39a5cf0..1f5d8d4bd4 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -275,7 +275,6 @@ struct S390PCIIOMMU {
Object parent_obj;
AddressSpace as;
MemoryRegion mr;
- uint64_t pal;
uint64_t max_dma_limit;
};
@@ -352,6 +351,7 @@ struct S390PCIBusDevice {
GHashTable *iotlb;
uint64_t g_iota;
uint64_t pba;
+ uint64_t pal;
S390PCIDMACount *dma_limit;
MemoryRegion msix_notify_mr;
IndAddr *summary_ind;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 12/15] s390x/pci: Move max_dma_limit from S390PCIIOMMU to S390PCIBusDevice
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (10 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 11/15] s390x/pci: Move pal " Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 13/15] s390x/pci: Add a comment explaining S390PCIIOMMU purpose Konstantin Shkolnyy
` (2 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
This field is only used when S390PCIBusDevice exists, so it can be moved
there to simplify S390PCIIOMMU which purpose is just to store the "root"
AddressSpace.
This also allows to save/restore this field during migration.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-vfio.c | 4 ++--
include/hw/s390x/s390-pci-bus.h | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c
index 7c754b656d..093743f2f0 100644
--- a/hw/s390x/s390-pci-vfio.c
+++ b/hw/s390x/s390-pci-vfio.c
@@ -90,7 +90,7 @@ S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s,
cnt->users = 1;
cnt->avail = avail;
QTAILQ_INSERT_TAIL(&s->zpci_dma_limit, cnt, link);
- pbdev->iommu->max_dma_limit = avail;
+ pbdev->max_dma_limit = avail;
return cnt;
}
@@ -151,7 +151,7 @@ static void s390_pci_read_base(S390PCIBusDevice *pbdev,
* to request that the guest free DMA mappings as necessary.
*/
if (!pbdev->rtr_avail) {
- vfio_size = pbdev->iommu->max_dma_limit << qemu_target_page_bits();
+ vfio_size = pbdev->max_dma_limit << qemu_target_page_bits();
if (vfio_size > 0 && vfio_size < cap->end_dma - cap->start_dma + 1) {
pbdev->zpci_fn.edma = cap->start_dma + vfio_size - 1;
}
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 1f5d8d4bd4..e3cbee2695 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -275,7 +275,6 @@ struct S390PCIIOMMU {
Object parent_obj;
AddressSpace as;
MemoryRegion mr;
- uint64_t max_dma_limit;
};
typedef struct S390PCIIOMMUTable {
@@ -352,6 +351,7 @@ struct S390PCIBusDevice {
uint64_t g_iota;
uint64_t pba;
uint64_t pal;
+ uint64_t max_dma_limit;
S390PCIDMACount *dma_limit;
MemoryRegion msix_notify_mr;
IndAddr *summary_ind;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 13/15] s390x/pci: Add a comment explaining S390PCIIOMMU purpose
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (11 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 12/15] s390x/pci: Move max_dma_limit " Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 14/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
2026-04-02 2:29 ` [PATCH 15/15] s390x/pci: Create function to contain fmb_timer start Konstantin Shkolnyy
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
include/hw/s390x/s390-pci-bus.h | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index e3cbee2695..2fab28e6e0 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -271,6 +271,14 @@ typedef struct S390PCIDMACount {
QTAILQ_ENTRY(S390PCIDMACount) link;
} S390PCIDMACount;
+/*
+ * This structure holds the PCI device AddressSpace that QEMU needs to link
+ * into its internal structures before the zPCI and PCI devices are fully
+ * initialized. It's a QEMU requirement to provide this "root" AddressSpace
+ * early. The AddressSpace is only actually used for I/O while the PCI
+ * device is plugged in and configured by the guest, at which time it gets
+ * additional memory subregions from zPCI device, that can do real work.
+ */
struct S390PCIIOMMU {
Object parent_obj;
AddressSpace as;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 14/15] s390x/pci: Implement migration for emulated devices
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (12 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 13/15] s390x/pci: Add a comment explaining S390PCIIOMMU purpose Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
2026-04-08 10:01 ` Thomas Huth
2026-04-02 2:29 ` [PATCH 15/15] s390x/pci: Create function to contain fmb_timer start Konstantin Shkolnyy
14 siblings, 1 reply; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
Implement zPCI device state migration, consequently enabling migration
of VMs that have emulated PCI devices, whether virtio or not.
Migration is allowed for devices whose function handle has the
FH_SHM_EMUL bit set. For these devices QEMU will save and restore the
state of its zPCI emulator.
Passthrough devices will continue to block migration.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 112 ++++++++++++++++++++++++++++++--
hw/s390x/s390-pci-inst.c | 2 +-
include/hw/s390x/s390-pci-bus.h | 2 +
3 files changed, 110 insertions(+), 6 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index f339a0fd94..b3037b9a8a 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -26,6 +26,7 @@
#include "hw/pci/pci_bridge.h"
#include "hw/pci/msi.h"
#include "exec/cpu-common.h"
+#include "migration/blocker.h"
#include "qemu/error-report.h"
#include "qemu/module.h"
#include "system/reset.h"
@@ -1116,6 +1117,28 @@ static int s390_pci_interp_plug(S390pciState *s, S390PCIBusDevice *pbdev)
return 0;
}
+static int s390_set_passthrough_migration_blocker(S390PCIBusDevice *pbdev,
+ Error **errp)
+{
+ pbdev->passthrough_migr_blocker = NULL;
+
+ if (pbdev->fh & FH_SHM_EMUL) {
+ return 0;
+ }
+ error_setg(&pbdev->passthrough_migr_blocker,
+ "Migration blocked by passthrough zPCI device "
+ "fh 0x%x uid %d fid %d", pbdev->fh, pbdev->uid, pbdev->fid);
+
+ return migrate_add_blocker(&pbdev->passthrough_migr_blocker, errp);
+}
+
+static void s390_clear_passthrough_migration_blocker(S390PCIBusDevice *pbdev)
+{
+ if (pbdev->passthrough_migr_blocker) {
+ migrate_del_blocker(&pbdev->passthrough_migr_blocker);
+ }
+}
+
static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
@@ -1235,6 +1258,11 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
return;
}
+ if (s390_set_passthrough_migration_blocker(pbdev, errp)) {
+ s390_pci_msix_free(pbdev);
+ return;
+ }
+
if (dev->hotplugged) {
s390_pci_generate_plug_event(HP_EVENT_TO_CONFIGURED ,
pbdev->fh, pbdev->fid);
@@ -1271,6 +1299,8 @@ static void s390_pcihost_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
return;
}
+ s390_clear_passthrough_migration_blocker(pbdev);
+
s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
pbdev->fh, pbdev->fid);
bus = pci_get_bus(pci_dev);
@@ -1600,13 +1630,85 @@ static const Property s390_pci_device_properties[] = {
true),
};
-static const VMStateDescription s390_pci_device_vmstate = {
- .name = TYPE_S390_PCI_DEVICE,
+static int s390_pci_device_post_load(void *opaque, int version_id)
+{
+ S390PCIBusDevice *pbdev = S390_PCI_DEVICE(opaque);
+
+ /*
+ * Now that S390PCIBusDevice fields have been restored, regenerate IOMMU
+ * state - that includes IOTLB contents and QEMU memory regions.
+ */
+ if (pbdev->iommu_enabled) {
+ assert(pbdev->iommu);
+ if (s390_pci_is_translation_enabled(pbdev->g_iota)) {
+ s390_pci_iommu_enable(pbdev);
+ s390_pci_ioat_replay(pbdev);
+ } else {
+ s390_pci_iommu_direct_map_enable(pbdev);
+ }
+ }
+
/*
- * TODO: add state handling here, so migration works at least with
- * emulated pci devices on s390x
+ * Guest sets fmb_addr by mpcifc.ZPCI_MOD_FC_SET_MEASURE instruction.
+ * The handler consequently starts fmb_timer. Now that fmb_addr has been
+ * restored, we may need to restart fmb_timer.
*/
- .unmigratable = 1,
+ if (pbdev->fmb_addr) {
+ assert(!pbdev->fmb_timer);
+ assert(pbdev->pci_group);
+ pbdev->fmb_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+ fmb_update, pbdev);
+ timer_mod(pbdev->fmb_timer,
+ qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+ pbdev->pci_group->zpci_group.mui);
+ }
+ return 0;
+}
+
+static const VMStateDescription s390_pci_device_vmstate = {
+ .name = TYPE_S390_PCI_DEVICE,
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .post_load = s390_pci_device_post_load,
+ .fields = (const VMStateField[]) {
+ VMSTATE_UINT32(state, S390PCIBusDevice),
+ VMSTATE_UINT16(uid, S390PCIBusDevice),
+ VMSTATE_UINT32(idx, S390PCIBusDevice),
+ VMSTATE_UINT32(fh, S390PCIBusDevice),
+ VMSTATE_UINT32(fid, S390PCIBusDevice),
+ VMSTATE_BOOL(fid_defined, S390PCIBusDevice),
+ VMSTATE_UINT64(fmb_addr, S390PCIBusDevice),
+ VMSTATE_UINT32(fmb.format, S390PCIBusDevice),
+ VMSTATE_UINT32(fmb.sample, S390PCIBusDevice),
+ VMSTATE_UINT64(fmb.last_update, S390PCIBusDevice),
+ VMSTATE_UINT64_ARRAY(fmb.counter, S390PCIBusDevice,
+ ARRAY_SIZE(((S390PCIBusDevice *)0)->fmb.counter)),
+ VMSTATE_UINT64(fmb.fmt0.dma_rbytes, S390PCIBusDevice),
+ VMSTATE_UINT64(fmb.fmt0.dma_wbytes, S390PCIBusDevice),
+ VMSTATE_UINT8(isc, S390PCIBusDevice),
+ VMSTATE_UINT16(noi, S390PCIBusDevice),
+ VMSTATE_UINT8(sum, S390PCIBusDevice),
+ VMSTATE_UINT8(pft, S390PCIBusDevice),
+ VMSTATE_UINT64(routes.adapter.ind_addr, S390PCIBusDevice),
+ VMSTATE_UINT64(routes.adapter.summary_addr, S390PCIBusDevice),
+ VMSTATE_UINT64(routes.adapter.ind_offset, S390PCIBusDevice),
+ VMSTATE_UINT32(routes.adapter.summary_offset, S390PCIBusDevice),
+ VMSTATE_UINT32(routes.adapter.adapter_id, S390PCIBusDevice),
+ VMSTATE_BOOL(iommu_enabled, S390PCIBusDevice),
+ VMSTATE_UINT64(g_iota, S390PCIBusDevice),
+ VMSTATE_UINT64(pba, S390PCIBusDevice),
+ VMSTATE_UINT64(pal, S390PCIBusDevice),
+ VMSTATE_UINT64(max_dma_limit, S390PCIBusDevice),
+ VMSTATE_PTR_TO_IND_ADDR(summary_ind, S390PCIBusDevice),
+ VMSTATE_PTR_TO_IND_ADDR(indicator, S390PCIBusDevice),
+ VMSTATE_BOOL(pci_unplug_request_processed, S390PCIBusDevice),
+ VMSTATE_BOOL(unplug_requested, S390PCIBusDevice),
+ VMSTATE_BOOL(interp, S390PCIBusDevice),
+ VMSTATE_BOOL(forwarding_assist, S390PCIBusDevice),
+ VMSTATE_BOOL(aif, S390PCIBusDevice),
+ VMSTATE_BOOL(rtr_avail, S390PCIBusDevice),
+ VMSTATE_END_OF_LIST()
+ }
};
static void s390_pci_device_class_init(ObjectClass *klass, const void *data)
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 191f549ba7..a9c0a4effb 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -1094,7 +1094,7 @@ static int fmb_do_update(S390PCIBusDevice *pbdev, int offset, uint64_t val,
return ret;
}
-static void fmb_update(void *opaque)
+void fmb_update(void *opaque)
{
S390PCIBusDevice *pbdev = opaque;
int64_t t = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 2fab28e6e0..2edb023112 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -337,6 +337,7 @@ struct S390PCIBusDevice {
uint16_t uid;
uint32_t idx;
uint32_t fh;
+ Error *passthrough_migr_blocker;
uint32_t fid;
bool fid_defined;
uint64_t fmb_addr;
@@ -415,5 +416,6 @@ S390PCIBusDevice *s390_pci_find_dev_by_pci(S390pciState *s,
S390PCIBusDevice *s390_pci_find_next_avail_dev(S390pciState *s,
S390PCIBusDevice *pbdev);
void s390_pci_ism_reset(void);
+void fmb_update(void *opaque);
#endif
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH 14/15] s390x/pci: Implement migration for emulated devices
2026-04-02 2:29 ` [PATCH 14/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
@ 2026-04-08 10:01 ` Thomas Huth
2026-04-08 12:28 ` Matthew Rosato
0 siblings, 1 reply; 21+ messages in thread
From: Thomas Huth @ 2026-04-08 10:01 UTC (permalink / raw)
To: Konstantin Shkolnyy, mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel
On 02/04/2026 04.29, Konstantin Shkolnyy wrote:
> Implement zPCI device state migration, consequently enabling migration
> of VMs that have emulated PCI devices, whether virtio or not.
> Migration is allowed for devices whose function handle has the
> FH_SHM_EMUL bit set. For these devices QEMU will save and restore the
> state of its zPCI emulator.
>
> Passthrough devices will continue to block migration.
>
> Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
> ---
> hw/s390x/s390-pci-bus.c | 112 ++++++++++++++++++++++++++++++--
> hw/s390x/s390-pci-inst.c | 2 +-
> include/hw/s390x/s390-pci-bus.h | 2 +
> 3 files changed, 110 insertions(+), 6 deletions(-)
>
> diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
> index f339a0fd94..b3037b9a8a 100644
> --- a/hw/s390x/s390-pci-bus.c
> +++ b/hw/s390x/s390-pci-bus.c
>...
> +static const VMStateDescription s390_pci_device_vmstate = {
> + .name = TYPE_S390_PCI_DEVICE,
> + .version_id = 1,
> + .minimum_version_id = 1,
> + .post_load = s390_pci_device_post_load,
> + .fields = (const VMStateField[]) {
> + VMSTATE_UINT32(state, S390PCIBusDevice),
> + VMSTATE_UINT16(uid, S390PCIBusDevice),
> + VMSTATE_UINT32(idx, S390PCIBusDevice),
> + VMSTATE_UINT32(fh, S390PCIBusDevice),
> + VMSTATE_UINT32(fid, S390PCIBusDevice),
> + VMSTATE_BOOL(fid_defined, S390PCIBusDevice),
> + VMSTATE_UINT64(fmb_addr, S390PCIBusDevice),
> + VMSTATE_UINT32(fmb.format, S390PCIBusDevice),
> + VMSTATE_UINT32(fmb.sample, S390PCIBusDevice),
> + VMSTATE_UINT64(fmb.last_update, S390PCIBusDevice),
> + VMSTATE_UINT64_ARRAY(fmb.counter, S390PCIBusDevice,
> + ARRAY_SIZE(((S390PCIBusDevice *)0)->fmb.counter)),
> + VMSTATE_UINT64(fmb.fmt0.dma_rbytes, S390PCIBusDevice),
> + VMSTATE_UINT64(fmb.fmt0.dma_wbytes, S390PCIBusDevice),
> + VMSTATE_UINT8(isc, S390PCIBusDevice),
> + VMSTATE_UINT16(noi, S390PCIBusDevice),
> + VMSTATE_UINT8(sum, S390PCIBusDevice),
> + VMSTATE_UINT8(pft, S390PCIBusDevice),
> + VMSTATE_UINT64(routes.adapter.ind_addr, S390PCIBusDevice),
> + VMSTATE_UINT64(routes.adapter.summary_addr, S390PCIBusDevice),
> + VMSTATE_UINT64(routes.adapter.ind_offset, S390PCIBusDevice),
> + VMSTATE_UINT32(routes.adapter.summary_offset, S390PCIBusDevice),
> + VMSTATE_UINT32(routes.adapter.adapter_id, S390PCIBusDevice),
> + VMSTATE_BOOL(iommu_enabled, S390PCIBusDevice),
> + VMSTATE_UINT64(g_iota, S390PCIBusDevice),
> + VMSTATE_UINT64(pba, S390PCIBusDevice),
> + VMSTATE_UINT64(pal, S390PCIBusDevice),
> + VMSTATE_UINT64(max_dma_limit, S390PCIBusDevice),
> + VMSTATE_PTR_TO_IND_ADDR(summary_ind, S390PCIBusDevice),
> + VMSTATE_PTR_TO_IND_ADDR(indicator, S390PCIBusDevice),
> + VMSTATE_BOOL(pci_unplug_request_processed, S390PCIBusDevice),
> + VMSTATE_BOOL(unplug_requested, S390PCIBusDevice),
> + VMSTATE_BOOL(interp, S390PCIBusDevice),
> + VMSTATE_BOOL(forwarding_assist, S390PCIBusDevice),
> + VMSTATE_BOOL(aif, S390PCIBusDevice),
> + VMSTATE_BOOL(rtr_avail, S390PCIBusDevice),
> + VMSTATE_END_OF_LIST()
> + }
> };
Hi!
I assume this will likely break backward migration with old machine types?
Could you please try whether migrating a s390-ccw-virtio-10.2 machine from
an old version of QEMU to a version of QEMU that includes your patches, and
back from the new version to the old version works as expected?
If not, you might need to add a ".needed = ..." line to the
s390_pci_device_vmstate, with a function that checks a variable from the
machine state whether the PCI state is allowed or not. And then enable that
variable only on the newest -11.1 machine type once it is available.
HTH,
Thomas
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 14/15] s390x/pci: Implement migration for emulated devices
2026-04-08 10:01 ` Thomas Huth
@ 2026-04-08 12:28 ` Matthew Rosato
2026-04-08 12:41 ` Cornelia Huck
0 siblings, 1 reply; 21+ messages in thread
From: Matthew Rosato @ 2026-04-08 12:28 UTC (permalink / raw)
To: Thomas Huth, Konstantin Shkolnyy
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel
On 4/8/26 6:01 AM, Thomas Huth wrote:
> On 02/04/2026 04.29, Konstantin Shkolnyy wrote:
>> Implement zPCI device state migration, consequently enabling migration
>> of VMs that have emulated PCI devices, whether virtio or not.
>> Migration is allowed for devices whose function handle has the
>> FH_SHM_EMUL bit set. For these devices QEMU will save and restore the
>> state of its zPCI emulator.
>>
>> Passthrough devices will continue to block migration.
>>
>> Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
>> ---
>> hw/s390x/s390-pci-bus.c | 112 ++++++++++++++++++++++++++++++--
>> hw/s390x/s390-pci-inst.c | 2 +-
>> include/hw/s390x/s390-pci-bus.h | 2 +
>> 3 files changed, 110 insertions(+), 6 deletions(-)
>>
>> diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
>> index f339a0fd94..b3037b9a8a 100644
>> --- a/hw/s390x/s390-pci-bus.c
>> +++ b/hw/s390x/s390-pci-bus.c
>> ...
>> +static const VMStateDescription s390_pci_device_vmstate = {
>> + .name = TYPE_S390_PCI_DEVICE,
>> + .version_id = 1,
>> + .minimum_version_id = 1,
>> + .post_load = s390_pci_device_post_load,
>> + .fields = (const VMStateField[]) {
>> + VMSTATE_UINT32(state, S390PCIBusDevice),
>> + VMSTATE_UINT16(uid, S390PCIBusDevice),
>> + VMSTATE_UINT32(idx, S390PCIBusDevice),
>> + VMSTATE_UINT32(fh, S390PCIBusDevice),
>> + VMSTATE_UINT32(fid, S390PCIBusDevice),
>> + VMSTATE_BOOL(fid_defined, S390PCIBusDevice),
>> + VMSTATE_UINT64(fmb_addr, S390PCIBusDevice),
>> + VMSTATE_UINT32(fmb.format, S390PCIBusDevice),
>> + VMSTATE_UINT32(fmb.sample, S390PCIBusDevice),
>> + VMSTATE_UINT64(fmb.last_update, S390PCIBusDevice),
>> + VMSTATE_UINT64_ARRAY(fmb.counter, S390PCIBusDevice,
>> + ARRAY_SIZE(((S390PCIBusDevice *)0)->fmb.counter)),
>> + VMSTATE_UINT64(fmb.fmt0.dma_rbytes, S390PCIBusDevice),
>> + VMSTATE_UINT64(fmb.fmt0.dma_wbytes, S390PCIBusDevice),
>> + VMSTATE_UINT8(isc, S390PCIBusDevice),
>> + VMSTATE_UINT16(noi, S390PCIBusDevice),
>> + VMSTATE_UINT8(sum, S390PCIBusDevice),
>> + VMSTATE_UINT8(pft, S390PCIBusDevice),
>> + VMSTATE_UINT64(routes.adapter.ind_addr, S390PCIBusDevice),
>> + VMSTATE_UINT64(routes.adapter.summary_addr, S390PCIBusDevice),
>> + VMSTATE_UINT64(routes.adapter.ind_offset, S390PCIBusDevice),
>> + VMSTATE_UINT32(routes.adapter.summary_offset, S390PCIBusDevice),
>> + VMSTATE_UINT32(routes.adapter.adapter_id, S390PCIBusDevice),
>> + VMSTATE_BOOL(iommu_enabled, S390PCIBusDevice),
>> + VMSTATE_UINT64(g_iota, S390PCIBusDevice),
>> + VMSTATE_UINT64(pba, S390PCIBusDevice),
>> + VMSTATE_UINT64(pal, S390PCIBusDevice),
>> + VMSTATE_UINT64(max_dma_limit, S390PCIBusDevice),
>> + VMSTATE_PTR_TO_IND_ADDR(summary_ind, S390PCIBusDevice),
>> + VMSTATE_PTR_TO_IND_ADDR(indicator, S390PCIBusDevice),
>> + VMSTATE_BOOL(pci_unplug_request_processed, S390PCIBusDevice),
>> + VMSTATE_BOOL(unplug_requested, S390PCIBusDevice),
>> + VMSTATE_BOOL(interp, S390PCIBusDevice),
>> + VMSTATE_BOOL(forwarding_assist, S390PCIBusDevice),
>> + VMSTATE_BOOL(aif, S390PCIBusDevice),
>> + VMSTATE_BOOL(rtr_avail, S390PCIBusDevice),
>> + VMSTATE_END_OF_LIST()
>> + }
>> };
> Hi!
>
> I assume this will likely break backward migration with old machine types?
>
> Could you please try whether migrating a s390-ccw-virtio-10.2 machine
> from an old version of QEMU to a version of QEMU that includes your
> patches, and back from the new version to the old version works as
> expected?
>
> If not, you might need to add a ".needed = ..." line to the
> s390_pci_device_vmstate, with a function that checks a variable from the
> machine state whether the PCI state is allowed or not. And then enable
> that variable only on the newest -11.1 machine type once it is available.
+1, I was having the same thought but hadn't gotten around to trying it yet.
We do already have a few examples for zPCI where we did this even though
migration was not even supported yet to kind of lay the groundwork for
backward migration ahead of time.
But honestly, looking at just this series in isolation, my concern is
things like an older machine type would not have the IOMMU replay
functionality added in patch 1.
So yes, please test to see what happens but make sure to include IOMMU
activity in that testing.
Tying this to a machine type is probably the safest bet, and given that
virtio-pci is not the default virtio transport on s390x I don't think it
would cause much hardship.
11.1 machine isn't available yet but I imagine will be soon after QEMU
11.0 release -- you can use Connie's series as a base:
https://lore.kernel.org/qemu-devel/20260331140347.653404-1-cohuck@redhat.com/
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 14/15] s390x/pci: Implement migration for emulated devices
2026-04-08 12:28 ` Matthew Rosato
@ 2026-04-08 12:41 ` Cornelia Huck
0 siblings, 0 replies; 21+ messages in thread
From: Cornelia Huck @ 2026-04-08 12:41 UTC (permalink / raw)
To: Matthew Rosato, Thomas Huth, Konstantin Shkolnyy
Cc: alifm, richard.henderson, iii, david, pasic, borntraeger,
qemu-s390x, qemu-devel
On Wed, Apr 08 2026, Matthew Rosato <mjrosato@linux.ibm.com> wrote:
> On 4/8/26 6:01 AM, Thomas Huth wrote:
>> On 02/04/2026 04.29, Konstantin Shkolnyy wrote:
>>> Implement zPCI device state migration, consequently enabling migration
>>> of VMs that have emulated PCI devices, whether virtio or not.
>>> Migration is allowed for devices whose function handle has the
>>> FH_SHM_EMUL bit set. For these devices QEMU will save and restore the
>>> state of its zPCI emulator.
>>>
>>> Passthrough devices will continue to block migration.
>>>
>>> Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
>>> ---
>>> hw/s390x/s390-pci-bus.c | 112 ++++++++++++++++++++++++++++++--
>>> hw/s390x/s390-pci-inst.c | 2 +-
>>> include/hw/s390x/s390-pci-bus.h | 2 +
>>> 3 files changed, 110 insertions(+), 6 deletions(-)
>>>
>>> diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
>>> index f339a0fd94..b3037b9a8a 100644
>>> --- a/hw/s390x/s390-pci-bus.c
>>> +++ b/hw/s390x/s390-pci-bus.c
>>> ...
>>> +static const VMStateDescription s390_pci_device_vmstate = {
>>> + .name = TYPE_S390_PCI_DEVICE,
>>> + .version_id = 1,
>>> + .minimum_version_id = 1,
>>> + .post_load = s390_pci_device_post_load,
>>> + .fields = (const VMStateField[]) {
>>> + VMSTATE_UINT32(state, S390PCIBusDevice),
>>> + VMSTATE_UINT16(uid, S390PCIBusDevice),
>>> + VMSTATE_UINT32(idx, S390PCIBusDevice),
>>> + VMSTATE_UINT32(fh, S390PCIBusDevice),
>>> + VMSTATE_UINT32(fid, S390PCIBusDevice),
>>> + VMSTATE_BOOL(fid_defined, S390PCIBusDevice),
>>> + VMSTATE_UINT64(fmb_addr, S390PCIBusDevice),
>>> + VMSTATE_UINT32(fmb.format, S390PCIBusDevice),
>>> + VMSTATE_UINT32(fmb.sample, S390PCIBusDevice),
>>> + VMSTATE_UINT64(fmb.last_update, S390PCIBusDevice),
>>> + VMSTATE_UINT64_ARRAY(fmb.counter, S390PCIBusDevice,
>>> + ARRAY_SIZE(((S390PCIBusDevice *)0)->fmb.counter)),
>>> + VMSTATE_UINT64(fmb.fmt0.dma_rbytes, S390PCIBusDevice),
>>> + VMSTATE_UINT64(fmb.fmt0.dma_wbytes, S390PCIBusDevice),
>>> + VMSTATE_UINT8(isc, S390PCIBusDevice),
>>> + VMSTATE_UINT16(noi, S390PCIBusDevice),
>>> + VMSTATE_UINT8(sum, S390PCIBusDevice),
>>> + VMSTATE_UINT8(pft, S390PCIBusDevice),
>>> + VMSTATE_UINT64(routes.adapter.ind_addr, S390PCIBusDevice),
>>> + VMSTATE_UINT64(routes.adapter.summary_addr, S390PCIBusDevice),
>>> + VMSTATE_UINT64(routes.adapter.ind_offset, S390PCIBusDevice),
>>> + VMSTATE_UINT32(routes.adapter.summary_offset, S390PCIBusDevice),
>>> + VMSTATE_UINT32(routes.adapter.adapter_id, S390PCIBusDevice),
>>> + VMSTATE_BOOL(iommu_enabled, S390PCIBusDevice),
>>> + VMSTATE_UINT64(g_iota, S390PCIBusDevice),
>>> + VMSTATE_UINT64(pba, S390PCIBusDevice),
>>> + VMSTATE_UINT64(pal, S390PCIBusDevice),
>>> + VMSTATE_UINT64(max_dma_limit, S390PCIBusDevice),
>>> + VMSTATE_PTR_TO_IND_ADDR(summary_ind, S390PCIBusDevice),
>>> + VMSTATE_PTR_TO_IND_ADDR(indicator, S390PCIBusDevice),
>>> + VMSTATE_BOOL(pci_unplug_request_processed, S390PCIBusDevice),
>>> + VMSTATE_BOOL(unplug_requested, S390PCIBusDevice),
>>> + VMSTATE_BOOL(interp, S390PCIBusDevice),
>>> + VMSTATE_BOOL(forwarding_assist, S390PCIBusDevice),
>>> + VMSTATE_BOOL(aif, S390PCIBusDevice),
>>> + VMSTATE_BOOL(rtr_avail, S390PCIBusDevice),
>>> + VMSTATE_END_OF_LIST()
>>> + }
>>> };
>> Hi!
>>
>> I assume this will likely break backward migration with old machine types?
>>
>> Could you please try whether migrating a s390-ccw-virtio-10.2 machine
>> from an old version of QEMU to a version of QEMU that includes your
>> patches, and back from the new version to the old version works as
>> expected?
>>
>> If not, you might need to add a ".needed = ..." line to the
>> s390_pci_device_vmstate, with a function that checks a variable from the
>> machine state whether the PCI state is allowed or not. And then enable
>> that variable only on the newest -11.1 machine type once it is available.
>
> +1, I was having the same thought but hadn't gotten around to trying it yet.
>
> We do already have a few examples for zPCI where we did this even though
> migration was not even supported yet to kind of lay the groundwork for
> backward migration ahead of time.
>
> But honestly, looking at just this series in isolation, my concern is
> things like an older machine type would not have the IOMMU replay
> functionality added in patch 1.
>
> So yes, please test to see what happens but make sure to include IOMMU
> activity in that testing.
>
> Tying this to a machine type is probably the safest bet, and given that
> virtio-pci is not the default virtio transport on s390x I don't think it
> would cause much hardship.
> 11.1 machine isn't available yet but I imagine will be soon after QEMU
> 11.0 release -- you can use Connie's series as a base:
> https://lore.kernel.org/qemu-devel/20260331140347.653404-1-cohuck@redhat.com/
I agree, that needs to be tied to the machine type.
I'll pick up the 11.1 machine types patch as soon as I prepare the first
s390 pull req for 11.1, so there should be a good base around soon.
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH 15/15] s390x/pci: Create function to contain fmb_timer start
2026-04-02 2:29 [PATCH 00/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
` (13 preceding siblings ...)
2026-04-02 2:29 ` [PATCH 14/15] s390x/pci: Implement migration for emulated devices Konstantin Shkolnyy
@ 2026-04-02 2:29 ` Konstantin Shkolnyy
14 siblings, 0 replies; 21+ messages in thread
From: Konstantin Shkolnyy @ 2026-04-02 2:29 UTC (permalink / raw)
To: mjrosato
Cc: alifm, richard.henderson, iii, david, cohuck, pasic, borntraeger,
qemu-s390x, qemu-devel, Konstantin Shkolnyy
fmb_timer is now started in 3 different places. The new function will encapsulate
that to make sure mui is added in all cases.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 5 ++---
hw/s390x/s390-pci-inst.c | 14 ++++++++++----
include/hw/s390x/s390-pci-bus.h | 1 +
3 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index b3037b9a8a..1451dbd340 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -1658,9 +1658,8 @@ static int s390_pci_device_post_load(void *opaque, int version_id)
assert(pbdev->pci_group);
pbdev->fmb_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
fmb_update, pbdev);
- timer_mod(pbdev->fmb_timer,
- qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
- pbdev->pci_group->zpci_group.mui);
+ s390_pci_schedule_fmb_timer(pbdev,
+ qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL));
}
return 0;
}
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index a9c0a4effb..62ecd50dcb 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -1094,9 +1094,16 @@ static int fmb_do_update(S390PCIBusDevice *pbdev, int offset, uint64_t val,
return ret;
}
+void s390_pci_schedule_fmb_timer(S390PCIBusDevice *pbdev, uint64_t start)
+{
+ timer_mod(pbdev->fmb_timer, start + pbdev->pci_group->zpci_group.mui);
+}
+
void fmb_update(void *opaque)
{
S390PCIBusDevice *pbdev = opaque;
+
+ /* Must be read before updating U bit */
int64_t t = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
int i;
@@ -1133,7 +1140,7 @@ void fmb_update(void *opaque)
sizeof(pbdev->fmb.last_update))) {
return;
}
- timer_mod(pbdev->fmb_timer, t + pbdev->pci_group->zpci_group.mui);
+ s390_pci_schedule_fmb_timer(pbdev, t);
}
static int mpcifc_reg_int_interp(S390PCIBusDevice *pbdev, ZpciFib *fib)
@@ -1326,9 +1333,8 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar,
timer_del(pbdev->fmb_timer);
}
pbdev->fmb_addr = fmb_addr;
- timer_mod(pbdev->fmb_timer,
- qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
- pbdev->pci_group->zpci_group.mui);
+ s390_pci_schedule_fmb_timer(pbdev,
+ qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL));
break;
}
default:
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 2edb023112..966ae3ade5 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -417,5 +417,6 @@ S390PCIBusDevice *s390_pci_find_next_avail_dev(S390pciState *s,
S390PCIBusDevice *pbdev);
void s390_pci_ism_reset(void);
void fmb_update(void *opaque);
+void s390_pci_schedule_fmb_timer(S390PCIBusDevice *pbdev, uint64_t start);
#endif
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread