Re: [PATCH 4/4] hw/cxl: Map committed HDM decoder ranges as RAM for direct DMA

[Jonathan Cameron <jic23@kernel.org>]

On Tue,  2 Jun 2026 15:41:27 +0800
Chen Pei <cp0613@linux.alibaba.com> wrote:

> The CXL Fixed Memory Window (CFMW) is registered with
> memory_region_init_io() and has no backing ram_block.  As a result
> address_space_map() on a guest physical address inside the CFMW
> takes the bounce-buffer path and is bounded by
> DEFAULT_MAX_BOUNCE_BUFFER_SIZE (4 KiB by default for the system
> AddressSpace).  Once a Type-3 device is brought online as system RAM
> (daxctl online-memory), the kernel happily allocates DMA buffers from
> the CFMW range and any virtio operation whose scatter list exceeds
> 4 KiB or that overlaps with another in-flight transfer fails with:
> 
>     qemu-system-riscv64: virtio: bogus descriptor or out of resources

This is half the bug if you are running without the stuff I talk about
below - you can't run instructions out of that memory either.

As to the bonce buffer that should be larger anyway - there has
been some discussion in the past on how to do that for the global one
(there is support for PCI specific cases) but we never got to a
standard solution.

> 
> The bug is not RISC-V specific: CFMW registration and the bounce
> buffer limit are both arch-agnostic, so any guest that onlines CXL
> memory and issues DMA larger than 4 KiB into it is affected.  It
> shows up first on RISC-V virt because that is where the rest of
> this series enables the daxctl + virtio path end-to-end.
> 
> Reproduce on RISC-V virt with cxl=on and a single Type-3 device:
> 
>     cxl create-region -m -t ram -d decoder0.0 -w 1 mem0 -s 4G
>     daxctl online-memory dax0.0
>     free -h    # triggers the error and stalls the guest
> 
> Fix it by overlaying a RAM alias of the device's memory backend
> (hostvmem / hostpmem) at the committed HDM decoder's HPA range, with
> higher priority than the CFMW I/O region.  flatview_translate() then
> hits the alias, address_space_map() returns a direct host pointer,
> and DMA proceeds without bouncing.  This mirrors the existing QEMU
> pattern of PCI BAR and IOMMU MR overlays.  The alias is torn down on
> hdm_decoder_uncommit() so subsequent region tear-down + re-creation
> works.
> 

+CC linux-cxl as that's where you are more likely to get feedback on 
CXL aspects.

For this one there is a more comprehensive solution that restricts this
to the cases that are valid (no interleaving going on).
https://lore.kernel.org/qemu-devel/20260318171918.146-1-alireza.sanaee@huawei.com/#t
It is rather more complex as a result.

For other cases we have always intentionally left it as not supporting
what you have here because we want the interleave to be correct and testable.

As far as I'm concerned that series of Alireza's has been ready to merge for
a while. I've been a bit busy with other things so haven't been poking
for that though.  Might be a week or two before I get back on top of things.

Anyhow, please give Ali's series a go and see if that works for you.
There may well be other corners we haven't thought of!

Jonathan



> Signed-off-by: Chen Pei <cp0613@linux.alibaba.com>



> ---
>  hw/mem/cxl_type3.c          | 81 +++++++++++++++++++++++++++++++++++++
>  include/hw/cxl/cxl_device.h |  4 ++
>  2 files changed, 85 insertions(+)
> 
> diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
> index 4739239da3..f962bce66a 100644
> --- a/hw/mem/cxl_type3.c
> +++ b/hw/mem/cxl_type3.c
> @@ -24,6 +24,7 @@
>  #include "qemu/module.h"
>  #include "qemu/pmem.h"
>  #include "qemu/range.h"
> +#include "system/address-spaces.h"
>  #include "qemu/rcu.h"
>  #include "qemu/guest-random.h"
>  #include "system/hostmem.h"
> @@ -420,6 +421,11 @@ static void hdm_decoder_commit(CXLType3Dev *ct3d, int which)
>      ComponentRegisters *cregs = &ct3d->cxl_cstate.crb;
>      uint32_t *cache_mem = cregs->cache_mem_registers;
>      uint32_t ctrl;
> +    uint32_t low, high;
> +    uint64_t decoder_base, decoder_size;
> +    MemoryRegion *mr = NULL;
> +    uint64_t dpa_offset = 0;
> +    char *alias_name;
>  
>      ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc);
>      /* TODO: Sanity checks that the decoder is possible */
> @@ -427,6 +433,73 @@ static void hdm_decoder_commit(CXLType3Dev *ct3d, int which)
>      ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 1);
>  
>      stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl);
> +
> +    /*
> +     * Create a RAM alias in system memory for the committed decoder range.
> +     * This enables direct DMA mapping (address_space_map) for devices like
> +     * virtio that need to DMA to/from CXL memory.  Without this, the CFMW
> +     * I/O region would require bounce buffering which is limited to 4KB.
> +     */
> +    low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_LO + which * hdm_inc);
> +    high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_HI + which * hdm_inc);
> +    decoder_base = ((uint64_t)high << 32) | (low & 0xf0000000);
> +
> +    low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_LO + which * hdm_inc);
> +    high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_HI + which * hdm_inc);
> +    decoder_size = ((uint64_t)high << 32) | (low & 0xf0000000);
> +
> +    if (!decoder_base || !decoder_size) {
> +        return;
> +    }
> +
> +    /* Calculate DPA offset by summing sizes of preceding decoders */
> +    for (int i = 0; i < which; i++) {
> +        uint32_t prev_low, prev_high;
> +        uint64_t prev_size;
> +        uint32_t prev_ctrl;
> +
> +        prev_ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL +
> +                             i * hdm_inc);
> +        if (!FIELD_EX32(prev_ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED)) {
> +            continue;
> +        }
> +        prev_low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_LO +
> +                            i * hdm_inc);
> +        prev_high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_HI +
> +                             i * hdm_inc);
> +        prev_size = ((uint64_t)prev_high << 32) | (prev_low & 0xf0000000);
> +        dpa_offset += prev_size;
> +    }
> +
> +    /* Determine which memory backend to alias */
> +    if (ct3d->hostvmem) {
> +        MemoryRegion *vmr = host_memory_backend_get_memory(ct3d->hostvmem);
> +        uint64_t vmr_size = memory_region_size(vmr);
> +
> +        if (dpa_offset < vmr_size) {
> +            mr = vmr;
> +        }
> +    }
> +    if (!mr && ct3d->hostpmem) {
> +        MemoryRegion *pmr = host_memory_backend_get_memory(ct3d->hostpmem);
> +        uint64_t vmr_size = ct3d->hostvmem ?
> +            memory_region_size(
> +                host_memory_backend_get_memory(ct3d->hostvmem)) : 0;
> +        mr = pmr;
> +        dpa_offset -= vmr_size;
> +    }
> +
> +    if (!mr) {
> +        return;
> +    }
> +
> +    alias_name = g_strdup_printf("cxl-hdm%d-ram-alias", which);
> +    memory_region_init_alias(&ct3d->hdm_ram_alias[which], OBJECT(ct3d),
> +                             alias_name, mr, dpa_offset, decoder_size);
> +    memory_region_add_subregion_overlap(get_system_memory(), decoder_base,
> +                                        &ct3d->hdm_ram_alias[which], 1);
> +    ct3d->hdm_ram_alias_valid[which] = true;
> +    g_free(alias_name);
>  }
>  
>  static void hdm_decoder_uncommit(CXLType3Dev *ct3d, int which)
> @@ -442,6 +515,14 @@ static void hdm_decoder_uncommit(CXLType3Dev *ct3d, int which)
>      ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 0);
>  
>      stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl);
> +
> +    /* Remove the RAM alias if it was added during commit */
> +    if (ct3d->hdm_ram_alias_valid[which]) {
> +        memory_region_del_subregion(get_system_memory(),
> +                                    &ct3d->hdm_ram_alias[which]);
> +        object_unparent(OBJECT(&ct3d->hdm_ram_alias[which]));
> +        ct3d->hdm_ram_alias_valid[which] = false;
> +    }
>  }
>  
>  static int ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err)
> diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
> index 393f312217..07deef2e2c 100644
> --- a/include/hw/cxl/cxl_device.h
> +++ b/include/hw/cxl/cxl_device.h
> @@ -714,6 +714,10 @@ struct CXLType3Dev {
>      /* State */
>      AddressSpace hostvmem_as;
>      AddressSpace hostpmem_as;
> +
> +    /* RAM aliases for HDM decoders - enables direct DMA mapping */
> +    MemoryRegion hdm_ram_alias[CXL_HDM_DECODER_COUNT];
> +    bool hdm_ram_alias_valid[CXL_HDM_DECODER_COUNT];
>      CXLComponentState cxl_cstate;
>      CXLDeviceState cxl_dstate;
>      CXLCCI cci; /* Primary PCI mailbox CCI */