Re: [PATCH v13 5/5] qtest/cxl: Add aarch64 virt test for CXL

[Jonathan Cameron <Jonathan.Cameron@huawei.com>]

On Wed, 21 May 2025 16:38:10 +0900
Itaru Kitayama <itaru.kitayama@linux.dev> wrote:

> > On May 19, 2025, at 21:54, Jonathan Cameron <Jonathan.Cameron@huawei.com> wrote:
> > 
> > On Thu, 15 May 2025 18:04:18 +0900
> > Itaru Kitayama <itaru.kitayama@linux.dev> wrote:
> >   
> >>> On May 13, 2025, at 20:14, Jonathan Cameron <Jonathan.Cameron@huawei.com> wrote:
> >>> 
> >>> Add a single complex case for aarch64 virt machine.
> >>> Given existing much more comprehensive tests for x86 cover the
> >>> common functionality, a single test should be enough to verify
> >>> that the aarch64 part continue to work.
> >>> 
> >>> Signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
> >>> ---
> >>> tests/qtest/cxl-test.c  | 59 ++++++++++++++++++++++++++++++++---------
> >>> tests/qtest/meson.build |  1 +
> >>> 2 files changed, 47 insertions(+), 13 deletions(-)
> >>> 
> >>> diff --git a/tests/qtest/cxl-test.c b/tests/qtest/cxl-test.c
> >>> index a600331843..c7189d6222 100644
> >>> --- a/tests/qtest/cxl-test.c
> >>> +++ b/tests/qtest/cxl-test.c
> >>> @@ -19,6 +19,12 @@
> >>>    "-device pxb-cxl,id=cxl.1,bus=pcie.0,bus_nr=53 " \
> >>>    "-M cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.targets.1=cxl.1,cxl-fmw.0.size=4G "
> >>> 
> >>> +#define QEMU_VIRT_2PXB_CMD \
> >>> +    "-machine virt,cxl=on -cpu max " \
> >>> +    "-device pxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=52 " \
> >>> +    "-device pxb-cxl,id=cxl.1,bus=pcie.0,bus_nr=53 " \
> >>> +    "-M cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.targets.1=cxl.1,cxl-fmw.0.size=4G "
> >>> +
> >>> #define QEMU_RP \
> >>>    "-device cxl-rp,id=rp0,bus=cxl.0,chassis=0,slot=0 "
> >>> 
> >>> @@ -197,25 +203,52 @@ static void cxl_2pxb_4rp_4t3d(void)
> >>>    qtest_end();
> >>>    rmdir(tmpfs);
> >>> }
> >>> +
> >>> +static void cxl_virt_2pxb_4rp_4t3d(void)
> >>> +{
> >>> +    g_autoptr(GString) cmdline = g_string_new(NULL);
> >>> +    char template[] = "/tmp/cxl-test-XXXXXX";
> >>> +    const char *tmpfs;
> >>> +
> >>> +    tmpfs = mkdtemp(template);
> >>> +
> >>> +    g_string_printf(cmdline, QEMU_VIRT_2PXB_CMD QEMU_4RP QEMU_4T3D,
> >>> +                    tmpfs, tmpfs, tmpfs, tmpfs, tmpfs, tmpfs,
> >>> +                    tmpfs, tmpfs);
> >>> +
> >>> +    qtest_start(cmdline->str);
> >>> +    qtest_end();
> >>> +    rmdir(tmpfs);
> >>> +}
> >>> #endif /* CONFIG_POSIX */
> >>> 
> >>> int main(int argc, char **argv)
> >>> {
> >>> -    g_test_init(&argc, &argv, NULL);
> >>> +    const char *arch = qtest_get_arch();
> >>> 
> >>> -    qtest_add_func("/pci/cxl/basic_hostbridge", cxl_basic_hb);
> >>> -    qtest_add_func("/pci/cxl/basic_pxb", cxl_basic_pxb);
> >>> -    qtest_add_func("/pci/cxl/pxb_with_window", cxl_pxb_with_window);
> >>> -    qtest_add_func("/pci/cxl/pxb_x2_with_window", cxl_2pxb_with_window);
> >>> -    qtest_add_func("/pci/cxl/rp", cxl_root_port);
> >>> -    qtest_add_func("/pci/cxl/rp_x2", cxl_2root_port);
> >>> +    g_test_init(&argc, &argv, NULL);
> >>> +    if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
> >>> +        qtest_add_func("/pci/cxl/basic_hostbridge", cxl_basic_hb);
> >>> +        qtest_add_func("/pci/cxl/basic_pxb", cxl_basic_pxb);
> >>> +        qtest_add_func("/pci/cxl/pxb_with_window", cxl_pxb_with_window);
> >>> +        qtest_add_func("/pci/cxl/pxb_x2_with_window", cxl_2pxb_with_window);
> >>> +        qtest_add_func("/pci/cxl/rp", cxl_root_port);
> >>> +        qtest_add_func("/pci/cxl/rp_x2", cxl_2root_port);
> >>> #ifdef CONFIG_POSIX
> >>> -    qtest_add_func("/pci/cxl/type3_device", cxl_t3d_deprecated);
> >>> -    qtest_add_func("/pci/cxl/type3_device_pmem", cxl_t3d_persistent);
> >>> -    qtest_add_func("/pci/cxl/type3_device_vmem", cxl_t3d_volatile);
> >>> -    qtest_add_func("/pci/cxl/type3_device_vmem_lsa", cxl_t3d_volatile_lsa);
> >>> -    qtest_add_func("/pci/cxl/rp_x2_type3_x2", cxl_1pxb_2rp_2t3d);
> >>> -    qtest_add_func("/pci/cxl/pxb_x2_root_port_x4_type3_x4", cxl_2pxb_4rp_4t3d);
> >>> +        qtest_add_func("/pci/cxl/type3_device", cxl_t3d_deprecated);
> >>> +        qtest_add_func("/pci/cxl/type3_device_pmem", cxl_t3d_persistent);
> >>> +        qtest_add_func("/pci/cxl/type3_device_vmem", cxl_t3d_volatile);
> >>> +        qtest_add_func("/pci/cxl/type3_device_vmem_lsa", cxl_t3d_volatile_lsa);
> >>> +        qtest_add_func("/pci/cxl/rp_x2_type3_x2", cxl_1pxb_2rp_2t3d);
> >>> +        qtest_add_func("/pci/cxl/pxb_x2_root_port_x4_type3_x4",
> >>> +                       cxl_2pxb_4rp_4t3d);
> >>> #endif
> >>> +    } else if (strcmp(arch, "aarch64") == 0) {
> >>> +#ifdef CONFIG_POSIX
> >>> +        qtest_add_func("/pci/cxl/virt/pxb_x2_root_port_x4_type3_x4",
> >>> +                       cxl_virt_2pxb_4rp_4t3d);
> >>> +#endif
> >>> +    }
> >>> +
> >>>    return g_test_run();
> >>> }
> >>> diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build
> >>> index 7daf619845..361000267a 100644
> >>> --- a/tests/qtest/meson.build
> >>> +++ b/tests/qtest/meson.build
> >>> @@ -258,6 +258,7 @@ qtests_aarch64 = \
> >>>  (config_all_accel.has_key('CONFIG_TCG') and                                            \
> >>>   config_all_devices.has_key('CONFIG_TPM_TIS_I2C') ? ['tpm-tis-i2c-test'] : []) + \
> >>>  (config_all_devices.has_key('CONFIG_ASPEED_SOC') ? qtests_aspeed64 : []) + \
> >>> +  qtests_cxl +                                                                                  \
> >>>  ['arm-cpu-features',
> >>>   'numa-test',
> >>>   'boot-serial-test',
> >>> -- 
> >>> 2.43.0
> >>>   
> >> 
> >> ~/projects/qemu/build$ meson test qtest-aarch64/cxl-test
> >> ninja: Entering directory `/home/realm/projects/qemu/build'
> >> [1/8] Generating qemu-version.h with a custom command (wrapped by meson to capture output)
> >> 1/1 qemu:qtest+qtest-aarch64 / qtest-aarch64/cxl-test        OK              0.17s   1 subtests passed
> >> 
> >> Ok:                 1
> >> Expected Fail:      0
> >> Fail:               0
> >> Unexpected Pass:    0
> >> Skipped:            0
> >> Timeout:            0
> >> 
> >> Tested-by: Itaru Kitayama <itaru.kitayama@fujitsu.com <mailto:itaru.kitayama@fujitsu.com>>
> >> 
> >> Jonathan, would you push your branch this series applied? I manually applied your series no issues though.  
> > 
> > I'm reluctant to push a 'normal' staging CXL tree whilst we have the TCG
> > issue outstanding (which is in mainline).
> > I can probably push one with a name that makes it clear we know it will
> > crash under some circumstances though. I'll aim to get that done later this week.
> > 
> > After talking to Richard Henderson I'm going to spin some images etc to
> > make it easier for him to replicate that TCG issue.  
> 
> While QEMU (the kernel is built off of cxl branch) boots fine and lspci shows CXL devices as shown:
> 
> root@localhost:~# lspci -mm
> 00:00.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Host bridge" -p00 "Red Hat, Inc." "Device 1100"
> 00:01.0 "SCSI storage controller" "Red Hat, Inc." "Virtio block device" -p00 "Red Hat, Inc." "Device 0002"
> 00:02.0 "Ethernet controller" "Intel Corporation" "82540EM Gigabit Ethernet Controller" -r03 -p00 "Red Hat, Inc." "QEMU Virtual Machine"
> 00:03.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Expander bridge" -p00 "Red Hat, Inc." "Device 1100"
> 00:04.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Expander bridge" -p00 "Red Hat, Inc." "Device 1100"
> 35:00.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel Corporation" "Device 0000"
> 35:01.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel Corporation" "Device 0000"
> 36:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a128" -p00 "" ""
> 36:00.1 "Serial bus controller [0c0b]" "Huawei Technologies Co., Ltd." "Device a123" -p00 "Red Hat, Inc." "Device 1100"
> 37:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
> 37:01.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
> 37:02.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
> 37:03.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
> 38:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc." "Device 1100"
> 3a:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc." "Device 1100"
> bf:00.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel Corporation" "Device 0000"
> bf:01.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel Corporation" "Device 0000"
> c0:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a128" -p00 "" ""
> c0:00.1 "Serial bus controller [0c0b]" "Huawei Technologies Co., Ltd." "Device a123" -p00 "Red Hat, Inc." "Device 1100"
> c1:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
> c1:01.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
> c1:02.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
> c1:03.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
> c2:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc." "Device 1100"
> c4:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc." "Device 1100”
> 
> but the cxl-list command takes 5-10 minutes to return the information. I’ll test with your qtest minimalist setup to see if the system is loaded. Am I seeing the TCG issue you mentioned?

I don't think so. That should only matter if the memory is hotplugged in
as normal system RAM.

> 
> root@localhost:~# cxl list -M
> [
>   {
>     "memdev":"mem10",
>     "ram_size":2147483648,
>     "ram_qos_class":42,
>     "serial":11,
>     "numa_node":0,
>     "host":"cxl_rcd.10",
>     "firmware_version":"mock fw v1 "

So this seems to be with both some emulated devices from qemu command line
and CXL test at the same time. 

I replicated a similar setup and not setting a significant delay
(half a second maybe?)  So not sure.

I did notice that there is a bug in qemu though which has surfaced
for some reason in the kernel log (it dates all the way back to 
origin CXL support series). In 
include/hw/cxl/cxl_pci.h
#define REG_LOC_DEVSEC_LENGTH is 0x24 and it should be 0x1C

That results for me in a kernel log entry about a bar not being
big enough for a huge offset (which is random data coming from
somewhere in text I think).

Seems unlikely to trigger your issue, but you never know!

Jonathan


>   },
>   {
>     "memdev":"mem5",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":6,
>     "numa_node":1,
>     "host":"cxl_mem.5",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem1",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":2,
>     "numa_node":1,
>     "host":"cxl_mem.1",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem6",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":8,
>     "numa_node":1,
>     "host":"cxl_mem.7",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem3",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":4,
>     "numa_node":1,
>     "host":"cxl_mem.3",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem4",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":5,
>     "numa_node":0,
>     "host":"cxl_mem.4",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem0",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":1,
>     "numa_node":0,
>     "host":"cxl_mem.0",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem2",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":3,
>     "numa_node":0,
>     "host":"cxl_mem.2",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem7",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":7,
>     "numa_node":0,
>     "host":"cxl_mem.6",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem8",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":9,
>     "numa_node":0,
>     "host":"cxl_mem.8",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem9",
>     "pmem_size":1073741824,
>     "pmem_qos_class":42,
>     "ram_size":1073741824,
>     "ram_qos_class":42,
>     "serial":10,
>     "numa_node":1,
>     "host":"cxl_mem.9",
>     "firmware_version":"mock fw v1 "
>   },
>   {
>     "memdev":"mem12",
>     "ram_size":268435456,
>     "serial":0,
>     "host":"0000:c4:00.0",
>     "firmware_version":"BWFW VERSION 00"
>   },
>   {
>     "memdev":"mem11",
>     "ram_size":268435456,
>     "serial":0,
>     "host":"0000:c2:00.0",
>     "firmware_version":"BWFW VERSION 00"
>   },
>   {
>     "memdev":"mem14",
>     "pmem_size":268435456,
>     "serial":0,
>     "host":"0000:3a:00.0",
>     "firmware_version":"BWFW VERSION 00"
>   },
>   {
>     "memdev":"mem13",
>     "pmem_size":268435456,
>     "serial":0,
>     "host":"0000:38:00.0",
>     "firmware_version":"BWFW VERSION 00"
>   }
> ]
> 
> Getting this shouldn’t take minutes, even with the emulator I think.
> 
> Itaru.
> 
> > 
> > Thanks for reviews.
> > 
> > Jonathan
> >   
> >> 
> >>   
> >   
> 
>