From: Fan Ni <nifan.cxl@gmail.com>
To: linux-cxl@vger.kernel.org, a.manzanares@samsung.com,
dave@stgolabs.net, nifan.cxl@gmail.com, fan.ni@samsung.com,
anisa.su887@gmail.com
Cc: dave.jiang@intel.com, alison.schofield@intel.com,
ira.weiny@intel.com, dan.j.williams@intel.com,
Jonathan.Cameron@huawei.com
Subject: [ISSUE]: cxl create region with one memdev fails when we have more than one HBs in the cxl topology setup
Date: Fri, 7 Mar 2025 11:24:32 -0800 [thread overview]
Message-ID: <Z8tH8BeFQ_WMYs5U@debian> (raw)
[ISSUE]: Create region with one memdev failed when we have 2 HBs in cxl topology setup
I create a cxl topology with 2 HBs like below:
CFMW
/ \
HB0 HB1
| |
RP0 RP1
| |
pmem ram
qemu command line:
/home/fan/cxl/qemu/build/qemu-system-x86_64 -s -kernel /home/fan/cxl/linux-fixes//arch/x86/boot/bzImage -append "root=/dev/sda rw console=ttyS0,115200 ignore_loglevel nokaslr cxl_acpi.dyndbg=+fplm cxl_pci.dyndbg=+fplm cxl_core.dyndbg=+fplm cxl_mem.dyndbg=+fplm cxl_pmem.dyndbg=+fplm cxl_port.dyndbg=+fplm cxl_region.dyndbg=+fplm cxl_test.dyndbg=+fplm cxl_mock.dyndbg=+fplm cxl_mock_mem.dyndbg=+fplm dax.dyndbg=+fplm dax_cxl.dyndbg=+fplm device_dax.dyndbg=+fplm" -smp 8 -accel kvm -serial mon:stdio -nographic -qmp tcp:localhost:4445,server,wait=off -netdev user,id=network0,hostfwd=tcp::2024-:22 -device e1000,netdev=network0 -monitor telnet:127.0.0.1:12346,server,nowait -drive file=/home/fan/cxl/images/qemu-image.img,index=0,media=disk,format=raw -machine q35,cxl=on -cpu qemu64,mce=on -m 8G,maxmem=64G,slots=8 -virtfs local,path=/opt/lib/modules,mount_tag=modshare,security_model=mapped -virtfs local,path=/home/fan,mount_tag=homeshare,security_model=mapped -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 -device cxl-rp,port=13,bus=cxl.1,id=root_port13,chassis=0,slot=2 -object memory-backend-file,id=hmem0,share=on,mem-path=/tmp/host//hmem0.raw,size=512M -object memory-backend-file,id=lsa0,share=on,mem-path=/tmp/host//lsa0.raw,size=2M -device cxl-type3,bus=root_port13,memdev=hmem0,lsa=lsa0,id=cxl-memdev0,sn=3840 -device pxb-cxl,bus_nr=112,bus=pcie.0,id=cxl.2 -device cxl-rp,port=14,bus=cxl.2,id=root_port14,chassis=0,slot=3 -object memory-backend-file,id=hmem1,share=on,mem-path=/tmp/host//hmem1.raw,size=512M -device cxl-type3,bus=root_port14,volatile-memdev=hmem1,id=cxl-vmemdev1,sn=3841 -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.targets.1=cxl.2,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8K
-----------------
# cxl create-region -m -d decoder0.0 -w 1 mem1 -s 512M -t pmem
cxl region: create_region: region0: set_interleave_ways failed: Invalid argument
cxl region: cmd_create_region: created 0 regions
-----------------
The reason is for the root decoder, iw = 2, while for creating the region as
above, number of ways is 1, which fails the check here:
https://web.git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/tree/drivers/cxl/core/region.c?h=fixes#n491
-----------------
if (!is_power_of_2(val / cxld->interleave_ways) ||
(val % cxld->interleave_ways)) {
dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
return -EINVAL;
}
-----------------
491 if (!is_power_of_2(val / cxld->interleave_ways) ||
(gdb) n
493 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
(gdb) p val
$8 = 1
(gdb) p cxld->interleave_ways
$9 = 2
So how can we create a region with only one memdev when there are more than 1 HBs,
but we only want to touch device under one HB?
Or do I miss something here?
#cxl list -vvv
[
{
"bus":"root0",
"provider":"ACPI.CXL",
"nr_dports":2,
"dports":[
{
"dport":"pci0000:0c",
"alias":"ACPI0016:01",
"id":12
},
{
"dport":"pci0000:70",
"alias":"ACPI0016:00",
"id":112
}
],
"ports:root0":[
{
"port":"port1",
"host":"pci0000:70",
"depth":1,
"decoders_committed":0,
"nr_dports":1,
"dports":[
{
"dport":"0000:70:00.0",
"id":14
}
],
"endpoints:port1":[
{
"endpoint":"endpoint3",
"host":"mem0",
"parent_dport":"0000:70:00.0",
"depth":2,
"decoders_committed":0,
"memdev":{
"memdev":"mem0",
"ram_size":536870912,
"serial":3841,
"host":"0000:71:00.0",
"firmware_version":"BWFW VERSION 00",
"partition_info":{
"total_size":536870912,
"volatile_only_size":536870912,
"persistent_only_size":0,
"partition_alignment_size":0
},
"firmware":{
"num_slots":2,
"active_slot":1,
"online_activate_capable":true,
"slot_1_version":"BWFW VERSION 0",
"fw_update_in_progress":false
}
},
"decoders:endpoint3":[
{
"decoder":"decoder3.0",
"interleave_ways":1,
"state":"disabled"
},
{
"decoder":"decoder3.1",
"interleave_ways":1,
"state":"disabled"
},
{
"decoder":"decoder3.2",
"interleave_ways":1,
"state":"disabled"
},
{
"decoder":"decoder3.3",
"interleave_ways":1,
"state":"disabled"
}
]
}
],
"decoders:port1":[
{
"decoder":"decoder1.0",
"interleave_ways":1,
"state":"disabled",
"nr_targets":1,
"targets":[
{
"target":"0000:70:00.0",
"position":0,
"id":14
}
]
}
]
},
{
"port":"port2",
"host":"pci0000:0c",
"depth":1,
"decoders_committed":0,
"nr_dports":1,
"dports":[
{
"dport":"0000:0c:00.0",
"id":13
}
],
"endpoints:port2":[
{
"endpoint":"endpoint4",
"host":"mem1",
"parent_dport":"0000:0c:00.0",
"depth":2,
"decoders_committed":0,
"memdev":{
"memdev":"mem1",
"pmem_size":536870912,
"serial":3840,
"host":"0000:0d:00.0",
"firmware_version":"BWFW VERSION 00",
"partition_info":{
"total_size":536870912,
"volatile_only_size":0,
"persistent_only_size":536870912,
"partition_alignment_size":0
},
"firmware":{
"num_slots":2,
"active_slot":1,
"online_activate_capable":true,
"slot_1_version":"BWFW VERSION 0",
"fw_update_in_progress":false
}
},
"decoders:endpoint4":[
{
"decoder":"decoder4.0",
"interleave_ways":1,
"state":"disabled"
},
{
"decoder":"decoder4.1",
"interleave_ways":1,
"state":"disabled"
},
{
"decoder":"decoder4.2",
"interleave_ways":1,
"state":"disabled"
},
{
"decoder":"decoder4.3",
"interleave_ways":1,
"state":"disabled"
}
]
}
],
"decoders:port2":[
{
"decoder":"decoder2.0",
"interleave_ways":1,
"state":"disabled",
"nr_targets":1,
"targets":[
{
"target":"0000:0c:00.0",
"position":0,
"id":13
}
]
}
]
}
],
"decoders:root0":[
{
"decoder":"decoder0.0",
"resource":79725330432,
"size":4294967296,
"interleave_ways":2,
"interleave_granularity":8192,
"max_available_extent":4294967296,
"pmem_capable":true,
"volatile_capable":true,
"accelmem_capable":true,
"qos_class":0,
"nr_targets":2,
"targets":[
{
"target":"pci0000:70",
"alias":"ACPI0016:00",
"position":1,
"id":112
},
{
"target":"pci0000:0c",
"alias":"ACPI0016:01",
"position":0,
"id":12
}
]
}
]
}
]
next reply other threads:[~2025-03-07 19:24 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-03-07 19:24 Fan Ni [this message]
2025-03-07 20:25 ` [ISSUE]: cxl create region with one memdev fails when we have more than one HBs in the cxl topology setup Alison Schofield
2025-03-07 20:28 ` Dan Williams
2025-03-10 17:37 ` Fan Ni
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=Z8tH8BeFQ_WMYs5U@debian \
--to=nifan.cxl@gmail.com \
--cc=Jonathan.Cameron@huawei.com \
--cc=a.manzanares@samsung.com \
--cc=alison.schofield@intel.com \
--cc=anisa.su887@gmail.com \
--cc=dan.j.williams@intel.com \
--cc=dave.jiang@intel.com \
--cc=dave@stgolabs.net \
--cc=fan.ni@samsung.com \
--cc=ira.weiny@intel.com \
--cc=linux-cxl@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox