[RFC PATCH 0/3] hw/arm/virt: CXL enablement including gpex-acpi

[RFC PATCH 0/3] hw/arm/virt: CXL enablement including gpex-acpi

[Jonathan Cameron]

Very basic initial enablement of CXL on arm virt with relevant ACPI table
construction. Includes changes in gpex-acpi to ensure correct DSDT for
any pci_expand_bridge/pxb-cxl bridges.  Note not looking to do the
primary root bridge yet.

Based on Ben's tree cxl-2.0v3 at https://gitlab.com/bwidawsk/qemu

To actually get the memory appropriately exposed to the OS a few additional
changes are needed as discussed in thread
https://lore.kernel.org/qemu-devel/20210128174009.00007536@Huawei.com/

I will rebase this on future versions of Ben's series as needed.

Jonathan Cameron (3):
  hw/pci-host/gpex-acpi: Add support for dsdt construction for pxb-cxl
  hw/arm/virt: Basic CXL enablement on pci_expander_bridge instances
    pxb-cxl
  hw/cxl/cxl-device-utils: Allow incorrect read lengths

 hw/arm/Kconfig            |  1 +
 hw/arm/virt-acpi-build.c  | 27 +++++++++++++++++++++++++++
 hw/arm/virt.c             |  3 ++-
 hw/cxl/cxl-device-utils.c |  4 ++--
 hw/pci-host/gpex-acpi.c   | 22 +++++++++++++++++++---
 5 files changed, 51 insertions(+), 6 deletions(-)

-- 
2.19.1

[RFC PATCH 1/3] hw/pci-host/gpex-acpi: Add support for dsdt construction for pxb-cxl

[Jonathan Cameron]

This adds code to instantiate the slightly extended ACPI root port
description in DSDT as per the CXL 2.0 specification.

Basically a cut and paste job from the i386/pc code.

Signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
---
 hw/pci-host/gpex-acpi.c | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/hw/pci-host/gpex-acpi.c b/hw/pci-host/gpex-acpi.c
index 446912d771..58f67ee4b0 100644
--- a/hw/pci-host/gpex-acpi.c
+++ b/hw/pci-host/gpex-acpi.c
@@ -5,6 +5,7 @@
 #include "hw/pci/pci_bus.h"
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pcie_host.h"
+#include "hw/acpi/cxl.h"
 
 static void acpi_dsdt_add_pci_route_table(Aml *dev, uint32_t irq)
 {
@@ -155,6 +156,7 @@ void acpi_dsdt_add_gpex(Aml *scope, struct GPEXConfig *cfg)
         QLIST_FOREACH(bus, &bus->child, sibling) {
             uint8_t bus_num = pci_bus_num(bus);
             uint8_t numa_node = pci_bus_numa_node(bus);
+            bool is_cxl;
 
             if (!pci_bus_is_root(bus)) {
                 continue;
@@ -169,9 +171,19 @@ void acpi_dsdt_add_gpex(Aml *scope, struct GPEXConfig *cfg)
                 nr_pcie_buses = bus_num;
             }
 
+            is_cxl = pci_bus_is_cxl(bus);
+
             dev = aml_device("PC%.02X", bus_num);
-            aml_append(dev, aml_name_decl("_HID", aml_string("PNP0A08")));
-            aml_append(dev, aml_name_decl("_CID", aml_string("PNP0A03")));
+            if (is_cxl) {
+                struct Aml *pkg = aml_package(2);
+                aml_append(dev, aml_name_decl("_HID", aml_string("ACPI0016")));
+                aml_append(pkg, aml_eisaid("PNP0A08"));
+                aml_append(pkg, aml_eisaid("PNP0A03"));
+                aml_append(dev, aml_name_decl("_CID", pkg));
+            } else {
+                aml_append(dev, aml_name_decl("_HID", aml_string("PNP0A08")));
+                aml_append(dev, aml_name_decl("_CID", aml_string("PNP0A03")));
+            }
             aml_append(dev, aml_name_decl("_BBN", aml_int(bus_num)));
             aml_append(dev, aml_name_decl("_UID", aml_int(bus_num)));
             aml_append(dev, aml_name_decl("_STR", aml_unicode("pxb Device")));
@@ -190,7 +202,11 @@ void acpi_dsdt_add_gpex(Aml *scope, struct GPEXConfig *cfg)
                             cfg->pio.base, 0, 0, 0);
             aml_append(dev, aml_name_decl("_CRS", crs));
 
-            acpi_dsdt_add_pci_osc(dev);
+            if (is_cxl) {
+                build_cxl_osc_method(dev);
+            } else {
+                acpi_dsdt_add_pci_osc(dev);
+            }
 
             aml_append(scope, dev);
         }
-- 
2.19.1

[RFC PATCH 2/3] hw/arm/virt: Basic CXL enablement on pci_expander_bridge instances pxb-cxl

[Jonathan Cameron]

Code based on i386/pc enablement.
There is a small amount of directly cut and paste code so it may
make sense to unify that at some stage.

Signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
---
 hw/arm/Kconfig           |  1 +
 hw/arm/virt-acpi-build.c | 27 +++++++++++++++++++++++++++
 hw/arm/virt.c            |  3 ++-
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/hw/arm/Kconfig b/hw/arm/Kconfig
index 0a242e4c5d..c43862bb23 100644
--- a/hw/arm/Kconfig
+++ b/hw/arm/Kconfig
@@ -27,6 +27,7 @@ config ARM_VIRT
     select ACPI_HW_REDUCED
     select ACPI_NVDIMM
     select ACPI_APEI
+    select ACPI_CXL
 
 config CHEETAH
     bool
diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
index 9d9ee24053..ce7fd908d2 100644
--- a/hw/arm/virt-acpi-build.c
+++ b/hw/arm/virt-acpi-build.c
@@ -39,11 +39,13 @@
 #include "hw/acpi/aml-build.h"
 #include "hw/acpi/utils.h"
 #include "hw/acpi/pci.h"
+#include "hw/acpi/cxl.h"
 #include "hw/acpi/memory_hotplug.h"
 #include "hw/acpi/generic_event_device.h"
 #include "hw/acpi/tpm.h"
 #include "hw/pci/pcie_host.h"
 #include "hw/pci/pci.h"
+#include "hw/pci/pci_bus.h"
 #include "hw/pci-host/gpex.h"
 #include "hw/arm/virt.h"
 #include "hw/mem/nvdimm.h"
@@ -155,11 +157,27 @@ static void acpi_dsdt_add_virtio(Aml *scope,
     }
 }
 
+/* Move the i386 definition somewhere common? */
+static void build_acpi0017(Aml *table)
+{
+    Aml *dev;
+    Aml *scope;
+
+    scope =  aml_scope("_SB");
+    dev = aml_device("CXLM");
+    aml_append(dev, aml_name_decl("_HID", aml_string("ACPI0017")));
+
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
 static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap,
                               uint32_t irq, bool use_highmem, bool highmem_ecam,
                               VirtMachineState *vms)
 {
     int ecam_id = VIRT_ECAM_ID(highmem_ecam);
+    bool cxl_present = false;
+    PCIBus *bus = vms->bus;
     struct GPEXConfig cfg = {
         .mmio32 = memmap[VIRT_PCIE_MMIO],
         .pio    = memmap[VIRT_PCIE_PIO],
@@ -173,6 +191,14 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap,
     }
 
     acpi_dsdt_add_gpex(scope, &cfg);
+    QLIST_FOREACH(bus, &vms->bus->child, sibling) {
+        if (pci_bus_is_cxl(bus)) {
+            cxl_present = true;
+        }
+    }
+    if (cxl_present) {
+        build_acpi0017(scope);
+    }
 }
 
 static void acpi_dsdt_add_gpio(Aml *scope, const MemMapEntry *gpio_memmap,
@@ -724,6 +750,7 @@ void virt_acpi_build(VirtMachineState *vms, AcpiBuildTables *tables)
             build_slit(tables_blob, tables->linker, ms);
         }
     }
+    cxl_build_cedt(table_offsets, tables_blob, tables->linker);
 
     if (ms->nvdimms_state->is_enabled) {
         nvdimm_build_acpi(table_offsets, tables_blob, tables->linker,
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index a3ac20744b..4d760beea9 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -78,6 +78,7 @@
 #include "hw/acpi/generic_event_device.h"
 #include "hw/virtio/virtio-iommu.h"
 #include "hw/char/pl011.h"
+#include "hw/cxl/cxl.h"
 #include "qemu/guest-random.h"
 
 #define DEFINE_VIRT_MACHINE_LATEST(major, minor, latest) \
@@ -2481,7 +2482,7 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
     hc->unplug_request = virt_machine_device_unplug_request_cb;
     hc->unplug = virt_machine_device_unplug_cb;
     mc->nvdimm_supported = true;
-    mc->cxl_supported = false;
+    mc->cxl_supported = true;
     mc->auto_enable_numa_with_memhp = true;
     mc->auto_enable_numa_with_memdev = true;
     mc->default_ram_id = "mach-virt.ram";
-- 
2.19.1

[RFC PATCH 3/3] hw/cxl/cxl-device-utils: Allow incorrect read lengths

[Jonathan Cameron]

This is currently needed to avoid an issue in the Linux RFC
in which a read is issued that is not a multiple of DW.
On arm64 that results in byte reads being issued and a bus
error returned.

It is not yet obvious at what level this should be fixed,
so paper over it to get things working.

Not-signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
---
 hw/cxl/cxl-device-utils.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hw/cxl/cxl-device-utils.c b/hw/cxl/cxl-device-utils.c
index d0d0a47122..52dd03384a 100644
--- a/hw/cxl/cxl-device-utils.c
+++ b/hw/cxl/cxl-device-utils.c
@@ -181,11 +181,11 @@ static const MemoryRegionOps mailbox_ops = {
     .write = mailbox_reg_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
     .valid = {
-        .min_access_size = 4,
+        .min_access_size = 1,
         .max_access_size = 8,
     },
     .impl = {
-        .min_access_size = 4,
+        .min_access_size = 1,
         .max_access_size = 8,
     },
 };
-- 
2.19.1

Re: [RFC PATCH 3/3] hw/cxl/cxl-device-utils: Allow incorrect read lengths

[Ben Widawsky]

On 21-02-01 23:26:55, Jonathan Cameron wrote:
> This is currently needed to avoid an issue in the Linux RFC
> in which a read is issued that is not a multiple of DW.
> On arm64 that results in byte reads being issued and a bus
> error returned.
> 
> It is not yet obvious at what level this should be fixed,
> so paper over it to get things working.
> 
> Not-signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
> ---
>  hw/cxl/cxl-device-utils.c | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/hw/cxl/cxl-device-utils.c b/hw/cxl/cxl-device-utils.c
> index d0d0a47122..52dd03384a 100644
> --- a/hw/cxl/cxl-device-utils.c
> +++ b/hw/cxl/cxl-device-utils.c
> @@ -181,11 +181,11 @@ static const MemoryRegionOps mailbox_ops = {
>      .write = mailbox_reg_write,
>      .endianness = DEVICE_LITTLE_ENDIAN,
>      .valid = {
> -        .min_access_size = 4,
> +        .min_access_size = 1,
>          .max_access_size = 8,
>      },
>      .impl = {
> -        .min_access_size = 4,
> +        .min_access_size = 1,
>          .max_access_size = 8,
>      },
>  };

I think this is now addressed in my v3. I'm happy to carry these patches around
in my branch if it helps.

They all lgtm

Re: [RFC PATCH 3/3] hw/cxl/cxl-device-utils: Allow incorrect read lengths

[Jonathan Cameron]

On Wed, 3 Feb 2021 08:10:32 -0800
Ben Widawsky <ben@bwidawsk.net> wrote:

> On 21-02-01 23:26:55, Jonathan Cameron wrote:
> > This is currently needed to avoid an issue in the Linux RFC
> > in which a read is issued that is not a multiple of DW.
> > On arm64 that results in byte reads being issued and a bus
> > error returned.
> > 
> > It is not yet obvious at what level this should be fixed,
> > so paper over it to get things working.
> > 
> > Not-signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
> > ---
> >  hw/cxl/cxl-device-utils.c | 4 ++--
> >  1 file changed, 2 insertions(+), 2 deletions(-)
> > 
> > diff --git a/hw/cxl/cxl-device-utils.c b/hw/cxl/cxl-device-utils.c
> > index d0d0a47122..52dd03384a 100644
> > --- a/hw/cxl/cxl-device-utils.c
> > +++ b/hw/cxl/cxl-device-utils.c
> > @@ -181,11 +181,11 @@ static const MemoryRegionOps mailbox_ops = {
> >      .write = mailbox_reg_write,
> >      .endianness = DEVICE_LITTLE_ENDIAN,
> >      .valid = {
> > -        .min_access_size = 4,
> > +        .min_access_size = 1,
> >          .max_access_size = 8,
> >      },
> >      .impl = {
> > -        .min_access_size = 4,
> > +        .min_access_size = 1,
> >          .max_access_size = 8,
> >      },
> >  };  
> 
> I think this is now addressed in my v3. I'm happy to carry these patches around
> in my branch if it helps.

That would be great if you don't mind.

Thanks!

> 
> They all lgtm