[RFC 1/9] hw/arm/virt: Add CXL FMWS PA window for device memory

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

CXL VFIO passthrough needs a stable guest physical address range for
device memory (DPA) that falls inside a CFMWS entry the guest discovers
from ACPI CEDT. Without a dedicated range in the address map, the HDM
decoder has nowhere to point.

Add VIRT_HIGH_CXL_MMIO immediately after the second PCIe MMIO window.
It gets its own highmem_cxl_mmio flag in VirtMachineState rather than
sharing highmem_cxl, so the two slots are independently controllable
even though both are currently tied to CXL bridge presence.

The base and size flow through GPEXConfig.cxl_mmio to
acpi_dsdt_add_gpex(), which carves out a QWord memory descriptor in the
first CXL root bridge's _CRS. The CFMWS window is system-wide, so only
the first CXL bridge gets the descriptor - subsequent ones would
produce duplicate resource claims for the same range.

build_crs() already emits the bridge's own 64-bit ranges into crs.
The CFMWS window is a separate system-wide range, so only that window
is appended as a new QWord descriptor; the bridge ranges are not
re-emitted. A warn_report() fires if the CFMWS window overlaps any
existing bridge 64-bit range, since that would indicate an address
layout conflict.

Signed-off-by: Zhi Wang <zhiw@nvidia.com>
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 hw/arm/virt-acpi-build.c   |  5 +++++
 hw/arm/virt.c              |  9 +++++++++
 hw/pci-host/gpex-acpi.c    | 40 ++++++++++++++++++++++++++++++++++++++
 include/hw/arm/virt.h      |  2 ++
 include/hw/pci-host/gpex.h |  1 +
 5 files changed, 57 insertions(+)

diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
index 591cfc993c..863e0680fb 100644
--- a/hw/arm/virt-acpi-build.c
+++ b/hw/arm/virt-acpi-build.c
@@ -176,6 +176,11 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap,
         cfg.mmio64 = memmap[VIRT_HIGH_PCIE_MMIO];
     }
 
+    if (vms->highmem_cxl) {
+        cfg.cxl_mmio.base = memmap[VIRT_HIGH_CXL_MMIO].base;
+        cfg.cxl_mmio.size = memmap[VIRT_HIGH_CXL_MMIO].size;
+    }
+
     acpi_dsdt_add_gpex(scope, &cfg);
     QLIST_FOREACH(bus, &vms->bus->child, sibling) {
         if (pci_bus_is_cxl(bus)) {
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index ec0d8475ca..fa07819401 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -211,6 +211,8 @@ static const MemMapEntry base_memmap[] = {
 #define DEFAULT_HIGH_PCIE_MMIO_SIZE_GB 512
 #define DEFAULT_HIGH_PCIE_MMIO_SIZE (DEFAULT_HIGH_PCIE_MMIO_SIZE_GB * GiB)
 
+#define DEFAULT_HIGH_CXL_MMIO_SIZE  DEFAULT_HIGH_PCIE_MMIO_SIZE
+
 /*
  * Highmem IO Regions: This memory map is floating, located after the RAM.
  * Each MemMapEntry base (GPA) will be dynamically computed, depending on the
@@ -237,6 +239,11 @@ static MemMapEntry extended_memmap[] = {
     [VIRT_HIGH_PCIE_ECAM] =     { 0x0, 256 * MiB },
     /* Second PCIe window */
     [VIRT_HIGH_PCIE_MMIO] =     { 0x0, DEFAULT_HIGH_PCIE_MMIO_SIZE },
+    /*
+     * CXL FMWS guest PA window - separate from PCIe MMIO so the two are
+     * independently sizeable. Same default size for now.
+     */
+    [VIRT_HIGH_CXL_MMIO] =      { 0x0, DEFAULT_HIGH_CXL_MMIO_SIZE },
     /* Any CXL Fixed memory windows come here */
 };
 
@@ -1724,6 +1731,7 @@ static void create_cxl_host_reg_region(VirtMachineState *vms)
                        vms->memmap[VIRT_CXL_HOST].size);
     memory_region_add_subregion(sysmem, vms->memmap[VIRT_CXL_HOST].base, mr);
     vms->highmem_cxl = true;
+    vms->highmem_cxl_mmio = true;
 }
 
 static void create_platform_bus(VirtMachineState *vms)
@@ -1897,6 +1905,7 @@ static inline bool *virt_get_high_memmap_enabled(VirtMachineState *vms,
         &vms->highmem_cxl,
         &vms->highmem_ecam,
         &vms->highmem_mmio,
+        &vms->highmem_cxl_mmio,
     };
 
     assert(ARRAY_SIZE(extended_memmap) - VIRT_LOWMEMMAP_LAST ==
diff --git a/hw/pci-host/gpex-acpi.c b/hw/pci-host/gpex-acpi.c
index d9820f9b41..7de57bbc46 100644
--- a/hw/pci-host/gpex-acpi.c
+++ b/hw/pci-host/gpex-acpi.c
@@ -7,6 +7,7 @@
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pcie_host.h"
 #include "hw/acpi/cxl.h"
+#include "qemu/error-report.h"
 
 static void acpi_dsdt_add_pci_route_table(Aml *dev, uint32_t irq,
                                           Aml *scope, uint8_t bus_num)
@@ -108,6 +109,7 @@ void acpi_dsdt_add_gpex(Aml *scope, struct GPEXConfig *cfg)
     CrsRangeSet crs_range_set;
     CrsRangeEntry *entry;
     int i;
+    bool first_cxl = true;
 
     /* start to construct the tables for pxb */
     crs_range_set_init(&crs_range_set);
@@ -161,6 +163,44 @@ void acpi_dsdt_add_gpex(Aml *scope, struct GPEXConfig *cfg)
              */
             crs = build_crs(PCI_HOST_BRIDGE(BUS(bus)->parent), &crs_range_set,
                             cfg->pio.base, 0, 0, 0);
+            if (is_cxl && first_cxl && cfg->cxl_mmio.size) {
+                uint64_t cfmws_end = cfg->cxl_mmio.base +
+                                     cfg->cxl_mmio.size - 1;
+
+                /*
+                 * The CXL Fixed Memory Window (CFMWS) is a system-wide GPA
+                 * range.  Only the first CXL root bridge emits the QWord
+                 * descriptor; adding it to every bridge would give the OS
+                 * duplicate resource claims for the same range.
+                 *
+                 * build_crs() has already appended the bridge's own 64-bit
+                 * ranges into crs.  Do not copy them again here; only append
+                 * the CFMWS window itself as a new QWord descriptor.
+                 *
+                 * Warn if the CFMWS window overlaps any range already claimed
+                 * by the bridge; in the current address layout they should be
+                 * disjoint, but catch it early if the layout ever changes.
+                 */
+                for (i = 0; i < crs_range_set.mem_64bit_ranges->len; i++) {
+                    entry = g_ptr_array_index(crs_range_set.mem_64bit_ranges,
+                                              i);
+                    if (entry->base <= cfmws_end &&
+                        entry->limit >= cfg->cxl_mmio.base) {
+                        warn_report("CXL CFMWS [0x%"PRIx64"-0x%"PRIx64"] "
+                                    "overlaps CXL root bridge 64-bit range "
+                                    "[0x%"PRIx64"-0x%"PRIx64"]",
+                                    cfg->cxl_mmio.base, cfmws_end,
+                                    entry->base, entry->limit);
+                    }
+                }
+                aml_append(crs,
+                    aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED,
+                        AML_MAX_FIXED, AML_NON_CACHEABLE, AML_READ_WRITE,
+                        0x0000, cfg->cxl_mmio.base, cfmws_end, 0x0000,
+                        cfg->cxl_mmio.size));
+                first_cxl = false;
+            }
+
             aml_append(dev, aml_name_decl("_CRS", crs));
 
             if (is_cxl) {
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index 5fcbd1c76f..88bb3c0bdf 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -91,6 +91,7 @@ enum {
     VIRT_CXL_HOST,
     VIRT_HIGH_PCIE_ECAM,
     VIRT_HIGH_PCIE_MMIO,
+    VIRT_HIGH_CXL_MMIO,
 };
 
 typedef enum VirtIOMMUType {
@@ -147,6 +148,7 @@ struct VirtMachineState {
     bool highmem;
     bool highmem_compact;
     bool highmem_cxl;
+    bool highmem_cxl_mmio;  /* VIRT_HIGH_CXL_MMIO window; follows highmem_cxl */
     bool highmem_ecam;
     bool highmem_mmio;
     bool highmem_redists;
diff --git a/include/hw/pci-host/gpex.h b/include/hw/pci-host/gpex.h
index 1da9c85bce..a7c2e2edf3 100644
--- a/include/hw/pci-host/gpex.h
+++ b/include/hw/pci-host/gpex.h
@@ -43,6 +43,7 @@ struct GPEXConfig {
     MemMapEntry mmio32;
     MemMapEntry mmio64;
     MemMapEntry pio;
+    MemMapEntry cxl_mmio;
     int         irq;
     PCIBus      *bus;
     bool        pci_native_hotplug;
-- 
2.25.1