* Re: [RFC 1/3] cxl: add tracepoints to the unwinding code path
2026-11-18 19:14 ` [RFC 1/3] cxl: add tracepoints to the unwinding code path alucerop
@ 2026-11-18 18:00 ` sashiko-bot
0 siblings, 0 replies; 10+ messages in thread
From: sashiko-bot @ 2026-11-18 18:00 UTC (permalink / raw)
To: alucerop, linux-cxl
Thank you for your contribution! Sashiko AI review found 3 potential
issues to consider:
- [High] cxl: use-after-free in unwinding tracepoints after kfree() and
put_device()
- [Low] cxl: unmatched start tracepoint on early return in
__cxl_decoder_detach()
- [Low] cxl: missing multi-read guards and convention defects in
tracepoint headers
--
commit a8dc1dbd6b765af2c082df04014f6b4732355967
Author: Alejandro Lucero <alucerop@amd.com>
cxl: add tracepoints to the unwinding code path
This commit adds start and done tracepoints under the cxl_unwinding trace
system across CXL memdev, port, decoder, and region teardown paths.
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 787988e92965a..7333fe7594bd9 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
[ ... ]
> @@ -27,9 +30,11 @@ static void cxl_memdev_release(struct device *dev)
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> struct device *parent = dev->parent;
>
> + trace_cxl_memdev_release(cxlmd, "start");
> ida_free(&cxl_memdev_ida, cxlmd->id);
> kfree(cxlmd);
> put_device(parent);
> + trace_cxl_memdev_release(cxlmd, "done");
> }
>
> static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
> @@ -752,9 +757,11 @@ static void cxl_memdev_unregister(void *_cxlmd)
> struct cxl_memdev *cxlmd = _cxlmd;
> struct device *dev = &cxlmd->dev;
>
> + trace_cxl_memdev_unregister(cxlmd, "start");
> cdev_device_del(&cxlmd->cdev, dev);
> cxl_memdev_shutdown(dev);
> put_device(dev);
> + trace_cxl_memdev_unregister(cxlmd, "done");
> }
>
> static void detach_memdev(struct work_struct *work)
> @@ -763,6 +770,7 @@ static void detach_memdev(struct work_struct *work)
>
> cxlmd = container_of(work, typeof(*cxlmd), detach_work);
>
> + trace_cxl_memdev_detach(cxlmd, "start");
> /*
> * When the creator of @cxlmd sets ->attach it indicates CXL operation
> * is required. In that case, @cxlmd detach escalates to parent device
> @@ -772,7 +780,9 @@ static void detach_memdev(struct work_struct *work)
> device_release_driver(cxlmd->dev.parent);
> else
> device_release_driver(&cxlmd->dev);
> +
> put_device(&cxlmd->dev);
> + trace_cxl_memdev_detach(cxlmd, "done");
> }
>
> static struct lock_class_key cxl_memdev_key;
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index f3705ea44db77..9ccb0d1859047 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
[ ... ]
> @@ -540,8 +544,10 @@ EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, "CXL");
>
> static void cxl_ep_release(struct cxl_ep *ep)
> {
> + trace_cxl_ep_release(ep->ep, "start");
> put_device(ep->ep);
> kfree(ep);
> + trace_cxl_ep_release(ep->ep, "done");
> }
>
> static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
> @@ -558,6 +564,7 @@ static void cxl_port_release(struct device *dev)
> unsigned long index;
> struct cxl_ep *ep;
>
> + trace_cxl_port_release(dev, "start");
> xa_for_each(&port->endpoints, index, ep)
> cxl_ep_remove(port, ep);
> xa_destroy(&port->endpoints);
> @@ -571,6 +578,7 @@ static void cxl_port_release(struct device *dev)
> put_device(dev->parent);
> kfree(port);
> }
> + trace_cxl_port_release(dev, "done");
> }
>
> static ssize_t decoders_committed_show(struct device *dev,
> @@ -1478,6 +1486,8 @@ static void delete_endpoint(void *data)
> struct cxl_port *endpoint = cxlmd->endpoint;
> struct device *host = port_to_host(endpoint);
>
> + trace_cxl_delete_endpoint(&endpoint->dev, "start");
> +
> scoped_guard(device, host) {
> if (host->driver && !endpoint->dead) {
> devm_release_action(host, cxl_unlink_parent_dport, endpoint);
> @@ -1488,6 +1498,7 @@ static void delete_endpoint(void *data)
> }
> put_device(&endpoint->dev);
> put_device(host);
> + trace_cxl_delete_endpoint(&endpoint->dev, "done");
> }
[ ... ]
> @@ -2283,6 +2296,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL");
>
> static void cxld_unregister(void *dev)
> {
> + trace_cxld_unregister(dev, "start");
> if (is_endpoint_decoder(dev))
> cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1,
> DETACH_INVALIDATE);
> @@ -2290,6 +2304,7 @@ static void cxld_unregister(void *dev)
> kill_regions(to_cxl_root_decoder(dev));
>
> device_unregister(dev);
> + trace_cxld_unregister(dev, "done");
> }
>
> int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
[ ... ]
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index a048bd5c06cb6..efbd53a37405d 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
[ ... ]
> @@ -2275,6 +2278,7 @@ __cxl_decoder_detach(struct cxl_region *cxlr,
> lockdep_assert_held_write(&cxl_rwsem.region);
>
> if (!cxled) {
> + trace_cxl_decoder_detach(&cxlr->dev, "start");
> p = &cxlr->params;
>
> if (pos >= p->interleave_ways) {
[Severity: Low]
Does emitting trace_cxl_decoder_detach(&cxlr->dev, "start") before
checking pos and p->targets[pos] leave an unmatched "start" trace event
when __cxl_decoder_detach() returns NULL early?
When unregister_region() loops over p->interleave_ways calling
detach_target(cxlr, i), any target already detached by cxld_unregister()
has p->targets[pos] == NULL and returns without a matching "done" event
(as does the dev_WARN_ONCE() return later in __cxl_decoder_detach()).
> @@ -2521,6 +2528,7 @@ static void cxl_region_release(struct device *dev)
> struct cxl_root_decoder *cxlrd = cxlr->cxlrd;
> int id = atomic_read(&cxlrd->region_id);
>
> + trace_cxl_region_release(dev, "start");
> /*
> * Try to reuse the recently idled id rather than the cached
> * next id to prevent the region id space from increasing
> @@ -2536,6 +2544,7 @@ static void cxl_region_release(struct device *dev)
> out:
> put_device(dev->parent);
> kfree(cxlr);
> + trace_cxl_region_release(dev, "done");
> }
>
> const struct device_type cxl_region_type = {
> @@ -2565,6 +2574,7 @@ static void unregister_region(struct cxl_region *cxlr)
> struct cxl_region_params *p = &cxlr->params;
> int i;
>
> + trace_cxl_unregister_region(&cxlr->dev, "start");
> xa_erase(&cxlrd->regions, cxlr->id);
> device_del(&cxlr->dev);
>
> @@ -2580,6 +2590,7 @@ static void unregister_region(struct cxl_region *cxlr)
>
> cxl_region_iomem_release(cxlr);
> put_device(&cxlr->dev);
> + trace_cxl_unregister_region(&cxlr->dev, "done");
> }
>
> static void endpoint_unregister_region(void *_cxlr)
> @@ -2587,10 +2598,12 @@ static void endpoint_unregister_region(void *_cxlr)
> struct cxl_region *cxlr = _cxlr;
> struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
>
> + trace_cxl_endpoint_unregister_region(&cxlr->dev, "start");
> guard(mutex)(&cxlrd->regions_lock);
> if (xa_load(&cxlrd->regions, cxlr->id))
> unregister_region(cxlr);
> put_device(&cxlr->dev);
> + trace_cxl_endpoint_unregister_region(&cxlr->dev, "done");
> }
[Severity: High]
Do the "done" tracepoints in these release and unregister callbacks trigger
a use-after-free?
Each of these tracepoints reads dev->kobj.name (or cxlmd->dev.kobj.name),
but the "done" call runs after kfree(), put_device(), or
device_unregister() has already freed the object and its kobject name:
- cxl_ep_release() calls kfree(ep) after put_device(ep->ep), and then
trace_cxl_ep_release(ep->ep, "done") evaluates ep->ep from the freed
struct cxl_ep even when tracing is disabled.
- cxl_memdev_release(), cxl_port_release(), and cxl_region_release() call
kfree() on the enclosing struct before the "done" tracepoint reads
dev->kobj.name.
- cxl_memdev_unregister(), detach_memdev(), delete_endpoint(),
cxld_unregister(), unregister_region(), and
endpoint_unregister_region() drop the final device reference via
put_device() or device_unregister() before calling the "done"
tracepoint.
> diff --git a/drivers/cxl/cxl_memdev_tracepoints.h b/drivers/cxl/cxl_memdev_tracepoints.h
> new file mode 100644
> index 0000000000000..6e8206cf178a0
> --- /dev/null
> +++ b/drivers/cxl/cxl_memdev_tracepoints.h
> @@ -0,0 +1,58 @@
[ ... ]
> +#undef TRACE_SYSTEM
> +#define TRACE_SYSTEM cxl_unwinding
> +
> +#include <linux/tracepoint.h>
> +#include "cxlmem.h"
> +
> +TRACE_EVENT(cxl_memdev_release,
> + TP_PROTO(struct cxl_memdev *cxlmd, char *action),
> + TP_ARGS(cxlmd, action),
> + TP_STRUCT__entry(__string(devname, cxlmd->dev.kobj.name)
> + __string(action, action)
> + ),
> + TP_fast_assign(__assign_str(devname);
> + __assign_str(action);
> + ),
> + TP_printk("%s, %s", __get_str(devname), __get_str(action))
> +);
[ ... ]
> +#undef TRACE_INCLUDE_PATH
> +#define TRACE_INCLUDE_PATH .
> +#undef TRACE_INCLUDE_FILE
> +#define TRACE_INCLUDE_FILE cxl_memdev_tracepoints
> +
> +#include <trace/define_trace.h>
> diff --git a/drivers/cxl/cxl_port_tracepoints.h b/drivers/cxl/cxl_port_tracepoints.h
> new file mode 100644
> index 0000000000000..faefa418d9684
> --- /dev/null
> +++ b/drivers/cxl/cxl_port_tracepoints.h
> @@ -0,0 +1,91 @@
[ ... ]
> +TRACE_EVENT(schedule_cxl_memdev_detach,
> + TP_PROTO(struct cxl_memdev *cxlmd),
> + TP_ARGS(cxlmd),
> + TP_STRUCT__entry(__string(devname, cxlmd->dev.kobj.name)),
> + TP_fast_assign(__assign_str(devname);
> + ),
> + TP_printk("%s", __get_str(devname))
> +);
[ ... ]
> +TRACE_EVENT(cxld_unregister,
> + TP_PROTO(struct device *dev, char *action),
[ ... ]
> diff --git a/drivers/cxl/cxl_region_tracepoints.h b/drivers/cxl/cxl_region_tracepoints.h
> new file mode 100644
> index 0000000000000..977eb45b6aad8
> --- /dev/null
> +++ b/drivers/cxl/cxl_region_tracepoints.h
> @@ -0,0 +1,78 @@
[ ... ]
> +#undef TRACE_SYSTEM
> +#define TRACE_SYSTEM cxl_unwinding
> +
> +#include <linux/tracepoint.h>
> +
> +TRACE_EVENT(cxl_decoder_detach,
> + TP_PROTO(struct device *dev, char *action),
[Severity: Low]
Should these new tracepoint headers wrap their definitions in a
"#if !defined(...) || defined(TRACE_HEADER_MULTI_READ)" guard with #endif
placed before #include <trace/define_trace.h>, like
drivers/cxl/core/trace.h?
Also, schedule_cxl_memdev_detach and cxld_unregister do not start with the
cxl_ subsystem prefix, the events read dev->kobj.name directly instead of
using dev_name(dev) (and take char *action instead of const char *action),
and the twelve identical (dev, action) TRACE_EVENT definitions could share
a DECLARE_EVENT_CLASS in drivers/cxl/core/trace.h.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261118191431.21781-2-alucerop@amd.com?part=1
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [RFC 3/3] Documentation/cxl: add document about Type2 binding and unbinding
2026-11-18 19:14 ` [RFC 3/3] Documentation/cxl: add document about Type2 binding and unbinding alucerop
@ 2026-11-18 18:04 ` sashiko-bot
0 siblings, 0 replies; 10+ messages in thread
From: sashiko-bot @ 2026-11-18 18:04 UTC (permalink / raw)
To: alucerop, linux-cxl
Thank you for your contribution! Sashiko AI review found 2 potential
issues to consider:
- [Low] Documentation/cxl: missing toctree entry for
cxl_type2_objects_unwinding.rst
- [Low] Documentation/cxl: broken RST list numbering and literal blocks
in type2 doc
--
commit b833f3f32d3c092128799e7d5375466321b53bec
Author: Alejandro Lucero <alucerop@amd.com>
Documentation/cxl: add document about Type2 binding and unbinding
This commit adds documentation describing CXL Type 2 device chain creation
and unwinding across devres and explicit unbind paths.
> diff --git a/Documentation/driver-api/cxl/type2/cxl_type2_objects_unwinding.rst b/Documentation/driver-api/cxl/type2/cxl_type2_objects_unwinding.rst
> new file mode 100644
> index 0000000000000..873f95d9c202b
> --- /dev/null
> +++ b/Documentation/driver-api/cxl/type2/cxl_type2_objects_unwinding.rst
> @@ -0,0 +1,102 @@
> +.. SPDX-License-Identifier: GPL-2.0
> +
> +=====================================
> +Compute Express Link: Type2 unwinding
> +=====================================
[Severity: Low]
Should type2/cxl_type2_objects_unwinding be added to a toctree in
Documentation/driver-api/cxl/index.rst? Without a toctree reference, make
htmldocs emits a "WARNING: document isn't included in any toctree" warning
and leaves this page unlinked from the CXL documentation index.
> +
> +The CXL subsystem relies on the linux kernel device abstraction for representing
> +the different componentes building up the subsystem itself. The core or root
[ ... ]
> + 1) The pci device owning the Type2 CXL "device" unbinds from its driver.
> + 2) The type2 driver is removed triggering the pci device unbinding.
> + 3) The cxl memdev is unbound from the cxl mem driver through sysfs.
> + 4) The cxl endpoint port is unbound from the cxl port driver through systfs.
> + 5) The cxl_region is unbound from the cxl region driver through sysfs.
> + 8) Root port linked to the device is unbound frmo the cxl port through sysfs.
> + 9) cxl acpi kernel module is removed.
[ ... ]
> +Visualizing the different unwinding cases
> +=========================================
> +
> +For helping with the debugging and/or understanding of current Type2, or even
> +Type3 devices, unwinding some tracepoints are available for enabling them all:
> +
> +
> + # echo 1 > /sys/kernel/debug/tracing/events/cxl_unwinding/enable
> +
> +or individually as for example:
> +
> + # echo 1 > /sys/kernel/debug/tracing/events/cxl_unwinding/cxl_memdev_release/enable
> +
> + or
> +
> + # echo "cxl_unwinding:cxl_memdev_release" > /sys/kernel/debug/tracing/set_event
[Severity: Low]
The numbered list jumps from 5) to 8) and 9) without a blank line, which
causes Sphinx/Docutils to fold items 8) and 9) into item 5) as
continuation text rather than separate list entries.
In the tracepoint section, the lines introducing the shell commands end
with ":" instead of "::", and "or" is indented with a tab while the
commands use three spaces, so Sphinx renders them as nested blockquotes
instead of literal code blocks.
There are also a few typos in the text: "componentes" ("components"),
"systfs" ("sysfs"), and "frmo" ("from").
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261118191431.21781-2-alucerop@amd.com?part=3
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [RFC 2/3] cxl: make explicit removal of endpoint and decoder for Type2
2026-11-18 19:14 ` [RFC 2/3] cxl: make explicit removal of endpoint and decoder for Type2 alucerop
@ 2026-11-18 18:09 ` sashiko-bot
0 siblings, 0 replies; 10+ messages in thread
From: sashiko-bot @ 2026-11-18 18:09 UTC (permalink / raw)
To: alucerop, linux-cxl
Thank you for your contribution! Sashiko AI review found 8 potential
issues to consider:
- [High] cxl: double cxld_unregister() and use-after-free in
delete_endpoint()
- [High] cxl: endpoint->dev reference leak in detach_memdev() after
delete_endpoint()
- [High] cxl: invalid pointer dereference in delete_endpoint() on Type2
probe failure
- [High] cxl: cxl_detach_ep() runs before delete_endpoint() on Type2
memdev unbind
- [High] cxl: cxlmd->detach_scheduled is never reset and races across
device locks
- [Medium] cxl: link failure on endpoint_unregister_region() when
CONFIG_CXL_REGION=n
- [Medium] cxl: reference leak in cxl_memdev_attach_region() on devm
registration error
- [Medium] cxl: use-after-free in unwinding tracepoints after
put_device()
--
commit 5ea2db5ceee405003c55ceb45874754b74107942
Author: Alejandro Lucero <alucerop@amd.com>
cxl: make explicit removal of endpoint and decoder for Type2
This commit replaces devm-based endpoint and region teardown for Type 2 CXL
devices with explicit teardown in cxl_memdev_unregister() and
delete_endpoint(), while pinning the endpoint device and adding
schedule_detach actions in cxl_memdev_attach_region().
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 7333fe7594bd9..c699c5561037e 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -758,6 +758,14 @@ static void cxl_memdev_unregister(void *_cxlmd)
> struct device *dev = &cxlmd->dev;
>
> trace_cxl_memdev_unregister(cxlmd, "start");
> +
> + /*
> + * Type2 requires to explicitly remove endpoint while Type3 relies on
> + * devm mechanisms.
> + */
> + if (cxlmd->attach)
> + delete_endpoint(cxlmd);
> +
[Severity: High]
Can this dereference an invalid cxlmd->endpoint pointer when a Type 2
memdev fails to probe cxl_mem?
cxl_memdev_alloc() initializes cxlmd->endpoint to ERR_PTR(-ENXIO). If
cxl_mem_probe() does not run or fails before registering the endpoint
(for example when !cxlds->media_ready returns -EBUSY),
__devm_cxl_add_memdev() -> cxl_memdev_autoremove() sees
cxl_memdev_attach_failed(cxlmd) and calls cxl_memdev_unregister(cxlmd).
Because cxlmd->attach is set, delete_endpoint(cxlmd) runs and immediately
dereferences cxlmd->endpoint in port_to_host(endpoint) and
trace_cxl_delete_endpoint().
> cdev_device_del(&cxlmd->cdev, dev);
> cxl_memdev_shutdown(dev);
> put_device(dev);
> @@ -776,10 +784,14 @@ static void detach_memdev(struct work_struct *work)
> * is required. In that case, @cxlmd detach escalates to parent device
> * detach.
> */
> - if (cxlmd->attach)
> + if (cxlmd->attach) {
> device_release_driver(cxlmd->dev.parent);
> - else
> +
> + /* Type2 attach region did a get. */
> + put_device(&cxlmd->endpoint->dev);
> + } else {
> device_release_driver(&cxlmd->dev);
> + }
[Severity: High]
Does put_device(&cxlmd->endpoint->dev) here leak the endpoint->dev
reference taken in cxl_memdev_attach_region()?
device_release_driver(cxlmd->dev.parent) synchronously releases devres on
the parent device, which runs cxl_memdev_unregister() ->
delete_endpoint(cxlmd). delete_endpoint() sets cxlmd->endpoint = NULL and
only drops the reference taken in cxl_endpoint_autoremove(). When control
returns to detach_memdev(), cxlmd->endpoint is NULL (and since dev is at
offset 0 of struct cxl_port, put_device(NULL) is a no-op). The extra
reference is also leaked if cxlmd->dev.parent is unbound directly without
going through detach_memdev().
>
> put_device(&cxlmd->dev);
> trace_cxl_memdev_detach(cxlmd, "done");
[Severity: Medium]
Can trace_cxl_memdev_detach(cxlmd, "done") read freed memory after
put_device(&cxlmd->dev)?
For a Type 2 device, device_release_driver(cxlmd->dev.parent) has already
run cxl_memdev_unregister() and dropped the initial reference on
cxlmd->dev, so put_device(&cxlmd->dev) here drops the final reference and
frees cxlmd before trace_cxl_memdev_detach() reads cxlmd->dev.kobj.name.
The same issue exists in endpoint_unregister_region(), cxld_unregister(),
and delete_endpoint(), where the "done" tracepoints read dev->kobj.name
after put_device() or device_unregister().
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index 9ccb0d1859047..de109fb600386 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
> @@ -1480,14 +1480,24 @@ static struct device *grandparent(struct device *dev)
> return NULL;
> }
>
> -static void delete_endpoint(void *data)
> +static void cxld_unregister(void *dev);
> +
> +void delete_endpoint(void *data)
> {
> struct cxl_memdev *cxlmd = data;
> struct cxl_port *endpoint = cxlmd->endpoint;
> struct device *host = port_to_host(endpoint);
> + struct cxl_attach_region *attach =
> + container_of(cxlmd->attach, typeof(*attach), attach);
>
> trace_cxl_delete_endpoint(&endpoint->dev, "start");
>
> + /* Type2 requires to explicitly remove region and decoder */
> + if (attach && attach->cxlr) {
> + endpoint_unregister_region(attach->cxlr);
> + cxld_unregister(&attach->cxled->cxld.dev);
> + }
[Severity: High]
Won't calling cxld_unregister(&attach->cxled->cxld.dev) directly here
cause a double unregister and use-after-free of attach->cxled?
add_hdm_decoder() still registers cxld_unregister as a devm action on
&endpoint->dev via cxl_decoder_autoremove() for both Type 2 and Type 3
devices, and cxl_memdev_attach_region() drops the device_find_child()
reference on &cxled->cxld.dev when dev goes out of scope. Calling
cxld_unregister() directly in delete_endpoint() unregisters the decoder
while leaving the devm action queued on &endpoint->dev; a few lines below,
devm_release_action(host, unregister_port, endpoint) unregisters
&endpoint->dev and runs devres_release_all(&endpoint->dev), which calls
cxld_unregister() a second time and then runs cxl_dpa_release() on the
freed decoder.
Conversely, if &endpoint->dev is unbound via sysfs first,
__device_release_driver(&endpoint->dev) runs devres_release_all() and
frees attach->cxled immediately (get_device(&endpoint->dev) only prevents
device_release(), not driver unbind devres cleanup), after which
delete_endpoint() calls cxld_unregister() on the already-freed
attach->cxled.
> +
> scoped_guard(device, host) {
> if (host->driver && !endpoint->dead) {
> devm_release_action(host, cxl_unlink_parent_dport, endpoint);
> @@ -1509,7 +1519,10 @@ int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
> get_device(host);
> get_device(&endpoint->dev);
> cxlmd->depth = endpoint->depth;
> - return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
> + /* Only Type3 relies on devm for decoder unwinding. */
> + if (cxlmd->cxlds->type == CXL_DEVTYPE_CLASSMEM)
> + return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
> + return 0;
> }
> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL");
[Severity: High]
When cxlmd->dev is unbound on a Type 2 device, doesn't skipping the
devm action here cause cxl_detach_ep() to run before delete_endpoint()?
devm_cxl_enumerate_ports() registers cxl_detach_ep on &cxlmd->dev.
Previously delete_endpoint was registered on &cxlmd->dev after
cxl_detach_ep, so devres ran delete_endpoint() first. Now, when
&cxlmd->dev unbinds (or fails in cxl_memdev_attach_region() after
devm_cxl_add_endpoint()), devres on &cxlmd->dev runs cxl_detach_ep()
before cxl_memdev_unregister() calls delete_endpoint(). cxl_detach_ep()
frees the struct cxl_ep entries in ancestor ports and deletes empty switch
ports while the endpoint port and region are still attached, causing
delete_endpoint() to hit dangling cxl_ep pointers (or NULL ep in
cxl_region_teardown_targets() / cxl_region_decode_reset()) or skip
unregister_port(endpoint) if a parent switch port's driver was detached.
> @@ -2413,6 +2426,12 @@ EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL");
> bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
> {
> trace_schedule_cxl_memdev_detach(cxlmd);
> +
> + if (cxlmd->detach_scheduled)
> + return 0;
> +
> + cxlmd->detach_scheduled = true;
> +
> return queue_work(cxl_bus_wq, &cxlmd->detach_work);
> }
> EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
[Severity: High]
Since cxlmd->detach_scheduled is never cleared in detach_memdev(), won't
a Type 3 cxl_memdev that is detached and later re-probed fail to detach on
any subsequent port removal and leak the cxlmd->dev reference taken by
cxl_endpoint_port_probe()?
In addition, cxlmd->detach_scheduled is a plain bool accessed without
synchronization from schedule_detach actions on three different devices
(&cxlr->dev, &endpoint->dev, and &cxlmd->dev), and if a Type 2 parent
driver is unbound directly while cxlmd->detach_scheduled is still false,
cxl_memdev_unregister() will trigger those schedule_detach actions and
queue detach_work while cxlmd is already being unregistered.
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index efbd53a37405d..6a188041791c2 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
[ ... ]
> @@ -4177,11 +4177,33 @@ int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
> /* Only teardown regions that pass validation, ignore the rest */
> get_device(&cxlr->dev);
>
> - rc = devm_add_action_or_reset(&endpoint->dev,
> - endpoint_unregister_region, cxlr);
> + /*
> + * While region in use by the Type2 device driver, no unbinding can
> + * happen. Keeping a reference to the endpoint device avoids release.
> + * The endpoint can be released after unwinding from the memdev forced
> + * with the below actions.
> + */
> + get_device(&endpoint->dev);
> +
> + /*
> + * All these next devm actions are necessary for dealing with unbind
> + * triggered through sysfs with the CXL objects linked to a Type2.
> + */
> + rc = devm_add_action_or_reset(&cxlr->dev, schedule_detach, cxlmd);
> + if (rc)
> + return rc;
> +
> + rc = devm_add_action_or_reset(&endpoint->dev, schedule_detach, cxlmd);
> +
> + if (rc)
> + return rc;
> +
> + rc = devm_add_action_or_reset(&cxlmd->dev, schedule_detach, cxlmd);
> if (rc)
> return rc;
>
> + attach->cxlr = cxlr;
> + attach->cxled = cxled;
[Severity: Medium]
If any of the devm_add_action_or_reset() calls above fails, aren't the
cxlr->dev and endpoint->dev references leaked?
attach->cxlr is only assigned after all three devm_add_action_or_reset()
calls succeed, so on early return delete_endpoint() sees
attach->cxlr == NULL and skips endpoint_unregister_region(attach->cxlr).
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index baa6df2d362a4..eaa663f25699a 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -942,6 +942,9 @@ void cxl_coordinates_combine(struct access_coordinate *out,
> struct access_coordinate *c1,
> struct access_coordinate *c2);
>
> +void endpoint_unregister_region(void *_cxlr);
> +void delete_endpoint(void *data);
> +
[Severity: Medium]
Does building with CONFIG_CXL_BUS enabled and CONFIG_CXL_REGION=n fail to
link on endpoint_unregister_region()?
drivers/cxl/core/port.o is built unconditionally under CONFIG_CXL_BUS and
calls endpoint_unregister_region() from delete_endpoint(), while
endpoint_unregister_region() in drivers/cxl/core/region.c is only built
when CONFIG_CXL_REGION is enabled and has no !CONFIG_CXL_REGION stub in
drivers/cxl/cxl.h.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261118191431.21781-2-alucerop@amd.com?part=2
^ permalink raw reply [flat|nested] 10+ messages in thread
* [RFC 0/3] Type2 unwinding ordering change
@ 2026-11-18 19:14 alucerop
2026-11-18 19:14 ` [RFC 1/3] cxl: add tracepoints to the unwinding code path alucerop
` (3 more replies)
0 siblings, 4 replies; 10+ messages in thread
From: alucerop @ 2026-11-18 19:14 UTC (permalink / raw)
To: linux-cxl, dave.jiang, dave, jic23, alison.schofield,
vishal.l.verma
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
This RFC requires more work mainly with the tracepoints and surely with
documentation which, I hope, will get better with help from reviewers.
Regarding the solution proposed for the described problem, I know there could
be other solutions, so I'll be happy if someone comes with a better one, but
my feeling is the current design based strongly on devm actions is adding
complexity which would be desirable to reduce somehow.
The summary
===========
Current Type2 support is safe but semantically wrong: a Type2 driver could not
be (temporarily) aware about the CXL region (the one creating the Host physical
range to work with) is not there anymore. It could because it depends on the
unwinding triggering point which changes the ordering of CXL objects unwinding.
It is safe because for current Type2 expectations the physical mapping will
remain: those HDM decoders building up the path and programmed by the BIOS will
not be reset.
While implementing multiPF support for Type2 I found this behaviour specially
annoying because potential concurrency problems due to this non deterministic
unwinding order of Type2 related objects. Adding to this the semantical problem
which will likely be a bigger problem when adhoc Type2 regions are supported, I
propose a change to how the current design works for the unwinding of objects
for a Type2 device.
I seize the necessity of explaining below the problem in detail for also
proposing to add to the CXL kernel documentation such (adjusted) explanation for
helping other to understand how the CXL subsystem works.
The long story
==============
Current CXL subsystem relies on the linux kernel device abstraction for
representing the different componentes building up the subsystem itself. The
core or root devices of the subsystem are created from data coming through ACPI
and are not discussed in this document, with the target being Type2 devices and
the binding/unbinding of those physical devices into the CXL subsystem.
Along with logical devices come CXL drivers which allow to perform this
binding/unbinding through usual kernel probe mechanisms. Moreover, current CXL
subsystem design relies on devres (managed resources) kernel functionality
linking actions to device unbinding from drivers helping with all the unwinding
required.
When a device is unbound the kernel will first invoke those devm actions linked
to the device and then, if there is no further references to the device kobject,
it will be released or freed. This implies actions linked to the unbinding
happen at the unbinding time but the final device release can happen at a later
time. Furthermore, those actions can lead to other related objects to be removed
as well as their existence depends on the current one being removed. For a Type2
device doing CXL initialization, these are the objects created with order from
left to right and with dependencies in the opposite direction:
cxl_memdev --> endpoint (cxl_port) --> cxl_decoder --> cxl_region
A CXL region defines the Host physical range where the device memory (HDM) is
mapped and someone, a client, will likely be using it somehow. For a Type2 this
client is the Type2 "generic" driver, like a network driver using CXL for some
kind of acceleration. For a Type3 the client could be the kernel memory
management or some user space functionality using such a physical range through
a dax device, with the driver taking over the device being the CXL core mem
driver. Note this last driver is also used in Type2 for the cxl_memdev created,
as next paragraph does explain.
The above object chain is created when a new memdev device is added to the
kernel device core and such a device binding to the cxl_mem driver. This will
trigger port(s) creation, at least one, a cxl_port device which will bind to
the cxl_port driver when added to the kernel device core. For a decoder only a
device is created (or as many as decoders advertises the Type2 device) but no
binding as there is no decoder driver to bind to. Finally, the region device
will be automatically created during initialization for current Type2 devices
expectation (this is called autodiscovery but work for supporting adhoc region
creation is in the way) and it will bind to the cxl_region driver. Once there
exists a driver bound to a device, it can be unbound through the sysfs files
created by default by the kernel device core.
Unwinding expectation
=====================
One could think a Type2 driver creating all those objects should control the
unwinding of all of them, not necessarily because some driver action but maybe
because the kernel signals the driver about that necessity. However, with the
current design those objects can end up being unbound through different paths
and not always the Type2 driver receives the signal or not in time to safely
stop using the CXL pathway created:
1) The pci device owning the Type2 CXL "device" unbinds from its driver.
2) The type2 driver is removed triggering the pci device unbinding.
3) The cxl memdev is unbound from the cxl mem driver through sysfs.
4) The cxl endpoint port is unbound from the cxl port driver through systfs.
5) The cxl_region is unbound from the cxl region driver through sysfs.
8) Root port linked to the device is unbound frmo the cxl port through sysfs.
9) cxl acpi kernel module is removed.
The first two cases go through the driver remove code path allowing a Type2
aware driver to stop using the CXL region memory range while all the CXL
objects are still there. Devm actions will start the CXL unwinding just after
that code path is executed with the cxl_memdev object being the first one
getting the unwinding signal, but the order here is not so relevant as there is
no client using the CXL region after the driver exit code path execution.
The next case makes ordering an issue. A cxl_memdev unbindng triggers several
demv release/clean actions:
- cxl_detach_ep
- endpoint_unregister_region
- cxld_unregister
- schedule_detach_memdev
As we can see, the chain-reaction leads to the endpoint unbinding, which in
turn leads to such endpoint unregistered from a region and that implying the
region unregistered itself and the related HDM decoder(s) "likely" being reset.
The related memdev is also scheduled for detachment. What about the Type2 driver
using the HPA linked to those objects?
Currently a Type2 driver is signalled to unbind when its memdev is going to be
removed. Following this previous chain, that happens when the memdev is going
to detach, but not when its detachment is scheduled. As stated earlier, that
implies the Type2 driver keeps using the mapping region after the objects
creating it have already disappeared. Linking a devm action between endpoint
and memdev is not so trivial and the locking issues arise quickly.
Using sysfs unbind for endpoint and region objects leads to the same problem.
Trying to change the devm chain could fix one case but easily destroying the
proper unwinding for the other ones leaving objects which should not be there
anymore.
This problem is solvable changing the way some devm action are used for Type2:
1) Any object unbinding should make the memdev to be detached first,
implying the Type2 driver will be removed as the first step in the
unwinding chain.
2) Make explicit the removal of endpoint and decoder from the memdev
detachment.
This solution implies to remove some of current devm actions linking region and
endpoint port unbinding to cxl memdev detach without allowing other actions to
happen independently of such memdev unbinding. Then some of those actions to
be invoked at the right time within the general unwinding of the chain.
For helping other people to study and visualize the problem, some tracepoints
are added hopefully showing the current unwinding and the new one after the
full patchset is applied.
Another consequence coming with this change is the state of those objects linked
to a memdev owned by a Type2 driver can only disappear through a specific order
facilitating handling potential concurrency of related functionlities like
supporting multipf where other non-PF0 functions will link to a cxl_memdev which
can onbly disappear through a well-protected and deterministic path precluding
problems with potential concurrency.
Alejandro Lucero (3):
cxl: add tracepoints to the unwinding code path
cxl: make explicit removal of endpoint and decoder for Type2
Documentation/cxl: add document about Type2 binding and unbinding
.../cxl/type2/cxl_type2_objects_unwinding.rst | 102 ++++++++++++++++++
drivers/cxl/core/memdev.c | 26 ++++-
drivers/cxl/core/port.c | 39 ++++++-
drivers/cxl/core/region.c | 42 +++++++-
drivers/cxl/cxl.h | 3 +
drivers/cxl/cxl_memdev_tracepoints.h | 58 ++++++++++
drivers/cxl/cxl_port_tracepoints.h | 91 ++++++++++++++++
drivers/cxl/cxl_region_tracepoints.h | 78 ++++++++++++++
drivers/cxl/cxlmem.h | 11 ++
drivers/cxl/port.c | 5 -
10 files changed, 443 insertions(+), 12 deletions(-)
create mode 100644 Documentation/driver-api/cxl/type2/cxl_type2_objects_unwinding.rst
create mode 100644 drivers/cxl/cxl_memdev_tracepoints.h
create mode 100644 drivers/cxl/cxl_port_tracepoints.h
create mode 100644 drivers/cxl/cxl_region_tracepoints.h
base-commit: 9d66f26fefacebfeed15de31c1123451a17ff119
--
2.34.1
^ permalink raw reply [flat|nested] 10+ messages in thread
* [RFC 1/3] cxl: add tracepoints to the unwinding code path
2026-11-18 19:14 [RFC 0/3] Type2 unwinding ordering change alucerop
@ 2026-11-18 19:14 ` alucerop
2026-11-18 18:00 ` sashiko-bot
2026-11-18 19:14 ` [RFC 2/3] cxl: make explicit removal of endpoint and decoder for Type2 alucerop
` (2 subsequent siblings)
3 siblings, 1 reply; 10+ messages in thread
From: alucerop @ 2026-11-18 19:14 UTC (permalink / raw)
To: linux-cxl, dave.jiang, dave, jic23, alison.schofield,
vishal.l.verma
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Once all CXL objects/devices are created for a Type2/3 device, the way
those same objects are removed depends on the triggering point with
several possibilities:
- Type2 driver removed.
- Through the sysfs unbind files linked to the cxl drivers.
Add tracepoints to the different functions invoked when the unwinding
happens which will help to visualize the process and debug potential
problems. As those functions can trigger further unwinding, add start
and done tracepoints for easily seeing the unwinding action graph.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/memdev.c | 10 +++
drivers/cxl/core/port.c | 16 +++++
drivers/cxl/core/region.c | 14 +++++
drivers/cxl/cxl_memdev_tracepoints.h | 58 ++++++++++++++++++
drivers/cxl/cxl_port_tracepoints.h | 91 ++++++++++++++++++++++++++++
drivers/cxl/cxl_region_tracepoints.h | 78 ++++++++++++++++++++++++
6 files changed, 267 insertions(+)
create mode 100644 drivers/cxl/cxl_memdev_tracepoints.h
create mode 100644 drivers/cxl/cxl_port_tracepoints.h
create mode 100644 drivers/cxl/cxl_region_tracepoints.h
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 787988e92965..7333fe7594bd 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -11,6 +11,9 @@
#include "trace.h"
#include "core.h"
+#define CREATE_TRACE_POINTS
+#include "cxl_memdev_tracepoints.h"
+
static DECLARE_RWSEM(cxl_memdev_rwsem);
/*
@@ -27,9 +30,11 @@ static void cxl_memdev_release(struct device *dev)
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct device *parent = dev->parent;
+ trace_cxl_memdev_release(cxlmd, "start");
ida_free(&cxl_memdev_ida, cxlmd->id);
kfree(cxlmd);
put_device(parent);
+ trace_cxl_memdev_release(cxlmd, "done");
}
static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
@@ -752,9 +757,11 @@ static void cxl_memdev_unregister(void *_cxlmd)
struct cxl_memdev *cxlmd = _cxlmd;
struct device *dev = &cxlmd->dev;
+ trace_cxl_memdev_unregister(cxlmd, "start");
cdev_device_del(&cxlmd->cdev, dev);
cxl_memdev_shutdown(dev);
put_device(dev);
+ trace_cxl_memdev_unregister(cxlmd, "done");
}
static void detach_memdev(struct work_struct *work)
@@ -763,6 +770,7 @@ static void detach_memdev(struct work_struct *work)
cxlmd = container_of(work, typeof(*cxlmd), detach_work);
+ trace_cxl_memdev_detach(cxlmd, "start");
/*
* When the creator of @cxlmd sets ->attach it indicates CXL operation
* is required. In that case, @cxlmd detach escalates to parent device
@@ -772,7 +780,9 @@ static void detach_memdev(struct work_struct *work)
device_release_driver(cxlmd->dev.parent);
else
device_release_driver(&cxlmd->dev);
+
put_device(&cxlmd->dev);
+ trace_cxl_memdev_detach(cxlmd, "done");
}
static struct lock_class_key cxl_memdev_key;
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index f3705ea44db7..9ccb0d185904 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -17,6 +17,9 @@
#include <cxl.h>
#include "core.h"
+#define CREATE_TRACE_POINTS
+#include "cxl_port_tracepoints.h"
+
/**
* DOC: cxl core
*
@@ -34,6 +37,7 @@
static DEFINE_IDA(cxl_port_ida);
static DEFINE_XARRAY(cxl_root_buses);
+
/*
* The terminal device in PCI is NULL and @platform_bus
* for platform devices (for cxl_test)
@@ -540,8 +544,10 @@ EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, "CXL");
static void cxl_ep_release(struct cxl_ep *ep)
{
+ trace_cxl_ep_release(ep->ep, "start");
put_device(ep->ep);
kfree(ep);
+ trace_cxl_ep_release(ep->ep, "done");
}
static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
@@ -558,6 +564,7 @@ static void cxl_port_release(struct device *dev)
unsigned long index;
struct cxl_ep *ep;
+ trace_cxl_port_release(dev, "start");
xa_for_each(&port->endpoints, index, ep)
cxl_ep_remove(port, ep);
xa_destroy(&port->endpoints);
@@ -571,6 +578,7 @@ static void cxl_port_release(struct device *dev)
put_device(dev->parent);
kfree(port);
}
+ trace_cxl_port_release(dev, "done");
}
static ssize_t decoders_committed_show(struct device *dev,
@@ -1478,6 +1486,8 @@ static void delete_endpoint(void *data)
struct cxl_port *endpoint = cxlmd->endpoint;
struct device *host = port_to_host(endpoint);
+ trace_cxl_delete_endpoint(&endpoint->dev, "start");
+
scoped_guard(device, host) {
if (host->driver && !endpoint->dead) {
devm_release_action(host, cxl_unlink_parent_dport, endpoint);
@@ -1488,6 +1498,7 @@ static void delete_endpoint(void *data)
}
put_device(&endpoint->dev);
put_device(host);
+ trace_cxl_delete_endpoint(&endpoint->dev, "done");
}
int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
@@ -1553,6 +1564,7 @@ static void cxl_detach_ep(void *data)
{
struct cxl_memdev *cxlmd = data;
+ trace_cxl_detach_ep(&cxlmd->dev, "start");
for (int i = cxlmd->depth - 1; i >= 1; i--) {
struct cxl_port *port, *parent_port;
struct detach_ctx ctx = {
@@ -1595,6 +1607,7 @@ static void cxl_detach_ep(void *data)
}
device_unlock(&parent_port->dev);
}
+ trace_cxl_detach_ep(&cxlmd->dev, "done");
}
static resource_size_t find_component_registers(struct device *dev)
@@ -2283,6 +2296,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL");
static void cxld_unregister(void *dev)
{
+ trace_cxld_unregister(dev, "start");
if (is_endpoint_decoder(dev))
cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1,
DETACH_INVALIDATE);
@@ -2290,6 +2304,7 @@ static void cxld_unregister(void *dev)
kill_regions(to_cxl_root_decoder(dev));
device_unregister(dev);
+ trace_cxld_unregister(dev, "done");
}
int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
@@ -2397,6 +2412,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL");
bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
{
+ trace_schedule_cxl_memdev_detach(cxlmd);
return queue_work(cxl_bus_wq, &cxlmd->detach_work);
}
EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index a048bd5c06cb..efbd53a37405 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -18,6 +18,9 @@
#include "core.h"
#include "mce.h"
+#define CREATE_TRACE_POINTS
+#include "cxl_region_tracepoints.h"
+
/**
* DOC: cxl core region
*
@@ -2275,6 +2278,7 @@ __cxl_decoder_detach(struct cxl_region *cxlr,
lockdep_assert_held_write(&cxl_rwsem.region);
if (!cxled) {
+ trace_cxl_decoder_detach(&cxlr->dev, "start");
p = &cxlr->params;
if (pos >= p->interleave_ways) {
@@ -2292,6 +2296,7 @@ __cxl_decoder_detach(struct cxl_region *cxlr,
cxl_cancel_auto_attach(cxled);
return NULL;
}
+ trace_cxl_decoder_detach(&cxlr->dev, "start");
p = &cxlr->params;
}
@@ -2329,6 +2334,7 @@ __cxl_decoder_detach(struct cxl_region *cxlr,
};
get_device(&cxlr->dev);
+ trace_cxl_decoder_detach(&cxlr->dev, "done");
return cxlr;
}
@@ -2360,6 +2366,7 @@ int cxl_decoder_detach(struct cxl_region *cxlr,
}
if (detach) {
+ trace_cxl_decoder_detach_release_driver(&detach->dev);
device_release_driver(&detach->dev);
put_device(&detach->dev);
}
@@ -2521,6 +2528,7 @@ static void cxl_region_release(struct device *dev)
struct cxl_root_decoder *cxlrd = cxlr->cxlrd;
int id = atomic_read(&cxlrd->region_id);
+ trace_cxl_region_release(dev, "start");
/*
* Try to reuse the recently idled id rather than the cached
* next id to prevent the region id space from increasing
@@ -2536,6 +2544,7 @@ static void cxl_region_release(struct device *dev)
out:
put_device(dev->parent);
kfree(cxlr);
+ trace_cxl_region_release(dev, "done");
}
const struct device_type cxl_region_type = {
@@ -2565,6 +2574,7 @@ static void unregister_region(struct cxl_region *cxlr)
struct cxl_region_params *p = &cxlr->params;
int i;
+ trace_cxl_unregister_region(&cxlr->dev, "start");
xa_erase(&cxlrd->regions, cxlr->id);
device_del(&cxlr->dev);
@@ -2580,6 +2590,7 @@ static void unregister_region(struct cxl_region *cxlr)
cxl_region_iomem_release(cxlr);
put_device(&cxlr->dev);
+ trace_cxl_unregister_region(&cxlr->dev, "done");
}
static void endpoint_unregister_region(void *_cxlr)
@@ -2587,10 +2598,12 @@ static void endpoint_unregister_region(void *_cxlr)
struct cxl_region *cxlr = _cxlr;
struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
+ trace_cxl_endpoint_unregister_region(&cxlr->dev, "start");
guard(mutex)(&cxlrd->regions_lock);
if (xa_load(&cxlrd->regions, cxlr->id))
unregister_region(cxlr);
put_device(&cxlr->dev);
+ trace_cxl_endpoint_unregister_region(&cxlr->dev, "done");
}
static struct lock_class_key cxl_region_key;
@@ -4163,6 +4176,7 @@ int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
/* Only teardown regions that pass validation, ignore the rest */
get_device(&cxlr->dev);
+
rc = devm_add_action_or_reset(&endpoint->dev,
endpoint_unregister_region, cxlr);
if (rc)
diff --git a/drivers/cxl/cxl_memdev_tracepoints.h b/drivers/cxl/cxl_memdev_tracepoints.h
new file mode 100644
index 000000000000..6e8206cf178a
--- /dev/null
+++ b/drivers/cxl/cxl_memdev_tracepoints.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Tracepoints for CXL subsystem
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cxl_unwinding
+
+#include <linux/tracepoint.h>
+#include "cxlmem.h"
+
+TRACE_EVENT(cxl_memdev_release,
+ TP_PROTO(struct cxl_memdev *cxlmd, char *action),
+ TP_ARGS(cxlmd, action),
+ TP_STRUCT__entry(__string(devname, cxlmd->dev.kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxl_memdev_unregister,
+ TP_PROTO(struct cxl_memdev *cxlmd, char *action),
+ TP_ARGS(cxlmd, action),
+ TP_STRUCT__entry(__string(devname, cxlmd->dev.kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxl_memdev_detach,
+ TP_PROTO(struct cxl_memdev *cxlmd, char *action),
+ TP_ARGS(cxlmd, action),
+ TP_STRUCT__entry(__string(devname, cxlmd->dev.kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE cxl_memdev_tracepoints
+
+#include <trace/define_trace.h>
diff --git a/drivers/cxl/cxl_port_tracepoints.h b/drivers/cxl/cxl_port_tracepoints.h
new file mode 100644
index 000000000000..faefa418d968
--- /dev/null
+++ b/drivers/cxl/cxl_port_tracepoints.h
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Tracepoints for CXL subsystem
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cxl_unwinding
+
+#include <linux/tracepoint.h>
+#include "cxlmem.h"
+
+TRACE_EVENT(schedule_cxl_memdev_detach,
+ TP_PROTO(struct cxl_memdev *cxlmd),
+ TP_ARGS(cxlmd),
+ TP_STRUCT__entry(__string(devname, cxlmd->dev.kobj.name)),
+ TP_fast_assign(__assign_str(devname);
+ ),
+ TP_printk("%s", __get_str(devname))
+);
+
+TRACE_EVENT(cxl_detach_ep,
+ TP_PROTO(struct device *ep, char *action),
+ TP_ARGS(ep, action),
+ TP_STRUCT__entry(__string(devname, ep->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxl_ep_release,
+ TP_PROTO(struct device *ep, char *action),
+ TP_ARGS(ep, action),
+ TP_STRUCT__entry(__string(devname, ep->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxl_port_release,
+ TP_PROTO(struct device *dev, char *action),
+ TP_ARGS(dev, action),
+ TP_STRUCT__entry(__string(devname, dev->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxl_delete_endpoint,
+ TP_PROTO(struct device *dev, char *action),
+ TP_ARGS(dev, action),
+ TP_STRUCT__entry(__string(devname, dev->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxld_unregister,
+ TP_PROTO(struct device *dev, char *action),
+ TP_ARGS(dev, action),
+ TP_STRUCT__entry(__string(devname, dev->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE cxl_port_tracepoints
+
+#include <trace/define_trace.h>
diff --git a/drivers/cxl/cxl_region_tracepoints.h b/drivers/cxl/cxl_region_tracepoints.h
new file mode 100644
index 000000000000..977eb45b6aad
--- /dev/null
+++ b/drivers/cxl/cxl_region_tracepoints.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Tracepoints for CXL subsystem
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cxl_unwinding
+
+#include <linux/tracepoint.h>
+
+TRACE_EVENT(cxl_decoder_detach,
+ TP_PROTO(struct device *dev, char *action),
+ TP_ARGS(dev, action),
+ TP_STRUCT__entry(__string(devname, dev->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxl_decoder_detach_release_driver,
+ TP_PROTO(struct device *ep),
+ TP_ARGS(ep),
+ TP_STRUCT__entry(__string(devname, ep->kobj.name)),
+ TP_fast_assign(__assign_str(devname);
+ ),
+ TP_printk("%s", __get_str(devname))
+);
+
+TRACE_EVENT(cxl_region_release,
+ TP_PROTO(struct device *dev, char *action),
+ TP_ARGS(dev, action),
+ TP_STRUCT__entry(__string(devname, dev->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxl_unregister_region,
+ TP_PROTO(struct device *dev, char *action),
+ TP_ARGS(dev, action),
+ TP_STRUCT__entry(__string(devname, dev->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+TRACE_EVENT(cxl_endpoint_unregister_region,
+ TP_PROTO(struct device *dev, char *action),
+ TP_ARGS(dev, action),
+ TP_STRUCT__entry(__string(devname, dev->kobj.name)
+ __string(action, action)
+ ),
+ TP_fast_assign(__assign_str(devname);
+ __assign_str(action);
+ ),
+ TP_printk("%s, %s", __get_str(devname), __get_str(action))
+);
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE cxl_region_tracepoints
+
+#include <trace/define_trace.h>
--
2.34.1
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [RFC 2/3] cxl: make explicit removal of endpoint and decoder for Type2
2026-11-18 19:14 [RFC 0/3] Type2 unwinding ordering change alucerop
2026-11-18 19:14 ` [RFC 1/3] cxl: add tracepoints to the unwinding code path alucerop
@ 2026-11-18 19:14 ` alucerop
2026-11-18 18:09 ` sashiko-bot
2026-11-18 19:14 ` [RFC 3/3] Documentation/cxl: add document about Type2 binding and unbinding alucerop
2026-11-18 23:39 ` [RFC 0/3] Type2 unwinding ordering change Dave Jiang
3 siblings, 1 reply; 10+ messages in thread
From: alucerop @ 2026-11-18 19:14 UTC (permalink / raw)
To: linux-cxl, dave.jiang, dave, jic23, alison.schofield,
vishal.l.verma
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Current design relies on devm for CXL devices unwinding but this is not
working well for Type2.
No unwinding should happen while the Type2 driver is still using the HPA
range, and although the driver is signalled about the unwinding, devm
locking interdependencies force that signalling to be asynchronous
leading to the driver still using the HPA while those CXL objects
building the path have already disappeared.
This is currently safe because Type2 does only support autodiscovery
regions implying the physical decoders building up the path are not
reset, but this is semantically wrong and it will create issues with the
coming support of adhoc region creation by Type2.
Remove devm actions for endpoint and decoder for a Type2 device. Invoke
removal of them explicitly for a Type2 when it is safe to do so.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/memdev.c | 16 ++++++++++++++--
drivers/cxl/core/port.c | 23 +++++++++++++++++++++--
drivers/cxl/core/region.c | 28 +++++++++++++++++++++++++---
drivers/cxl/cxl.h | 3 +++
drivers/cxl/cxlmem.h | 11 +++++++++++
drivers/cxl/port.c | 5 -----
6 files changed, 74 insertions(+), 12 deletions(-)
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 7333fe7594bd..c699c5561037 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -758,6 +758,14 @@ static void cxl_memdev_unregister(void *_cxlmd)
struct device *dev = &cxlmd->dev;
trace_cxl_memdev_unregister(cxlmd, "start");
+
+ /*
+ * Type2 requires to explicitly remove endpoint while Type3 relies on
+ * devm mechanisms.
+ */
+ if (cxlmd->attach)
+ delete_endpoint(cxlmd);
+
cdev_device_del(&cxlmd->cdev, dev);
cxl_memdev_shutdown(dev);
put_device(dev);
@@ -776,10 +784,14 @@ static void detach_memdev(struct work_struct *work)
* is required. In that case, @cxlmd detach escalates to parent device
* detach.
*/
- if (cxlmd->attach)
+ if (cxlmd->attach) {
device_release_driver(cxlmd->dev.parent);
- else
+
+ /* Type2 attach region did a get. */
+ put_device(&cxlmd->endpoint->dev);
+ } else {
device_release_driver(&cxlmd->dev);
+ }
put_device(&cxlmd->dev);
trace_cxl_memdev_detach(cxlmd, "done");
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 9ccb0d185904..de109fb60038 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -1480,14 +1480,24 @@ static struct device *grandparent(struct device *dev)
return NULL;
}
-static void delete_endpoint(void *data)
+static void cxld_unregister(void *dev);
+
+void delete_endpoint(void *data)
{
struct cxl_memdev *cxlmd = data;
struct cxl_port *endpoint = cxlmd->endpoint;
struct device *host = port_to_host(endpoint);
+ struct cxl_attach_region *attach =
+ container_of(cxlmd->attach, typeof(*attach), attach);
trace_cxl_delete_endpoint(&endpoint->dev, "start");
+ /* Type2 requires to explicitly remove region and decoder */
+ if (attach && attach->cxlr) {
+ endpoint_unregister_region(attach->cxlr);
+ cxld_unregister(&attach->cxled->cxld.dev);
+ }
+
scoped_guard(device, host) {
if (host->driver && !endpoint->dead) {
devm_release_action(host, cxl_unlink_parent_dport, endpoint);
@@ -1509,7 +1519,10 @@ int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
get_device(host);
get_device(&endpoint->dev);
cxlmd->depth = endpoint->depth;
- return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
+ /* Only Type3 relies on devm for decoder unwinding. */
+ if (cxlmd->cxlds->type == CXL_DEVTYPE_CLASSMEM)
+ return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
+ return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL");
@@ -2413,6 +2426,12 @@ EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL");
bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
{
trace_schedule_cxl_memdev_detach(cxlmd);
+
+ if (cxlmd->detach_scheduled)
+ return 0;
+
+ cxlmd->detach_scheduled = true;
+
return queue_work(cxl_bus_wq, &cxlmd->detach_work);
}
EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index efbd53a37405..6a188041791c 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -2593,7 +2593,7 @@ static void unregister_region(struct cxl_region *cxlr)
trace_cxl_unregister_region(&cxlr->dev, "done");
}
-static void endpoint_unregister_region(void *_cxlr)
+void endpoint_unregister_region(void *_cxlr)
{
struct cxl_region *cxlr = _cxlr;
struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
@@ -4177,11 +4177,33 @@ int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
/* Only teardown regions that pass validation, ignore the rest */
get_device(&cxlr->dev);
- rc = devm_add_action_or_reset(&endpoint->dev,
- endpoint_unregister_region, cxlr);
+ /*
+ * While region in use by the Type2 device driver, no unbinding can
+ * happen. Keeping a reference to the endpoint device avoids release.
+ * The endpoint can be released after unwinding from the memdev forced
+ * with the below actions.
+ */
+ get_device(&endpoint->dev);
+
+ /*
+ * All these next devm actions are necessary for dealing with unbind
+ * triggered through sysfs with the CXL objects linked to a Type2.
+ */
+ rc = devm_add_action_or_reset(&cxlr->dev, schedule_detach, cxlmd);
+ if (rc)
+ return rc;
+
+ rc = devm_add_action_or_reset(&endpoint->dev, schedule_detach, cxlmd);
+
+ if (rc)
+ return rc;
+
+ rc = devm_add_action_or_reset(&cxlmd->dev, schedule_detach, cxlmd);
if (rc)
return rc;
+ attach->cxlr = cxlr;
+ attach->cxled = cxled;
attach->hpa_range = (struct range) {
.start = cxlr->params.res->start,
.end = cxlr->params.res->end,
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index baa6df2d362a..eaa663f25699 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -942,6 +942,9 @@ void cxl_coordinates_combine(struct access_coordinate *out,
struct access_coordinate *c1,
struct access_coordinate *c2);
+void endpoint_unregister_region(void *_cxlr);
+void delete_endpoint(void *data);
+
bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);
struct cxl_dport *devm_cxl_add_dport_by_dev(struct cxl_port *port,
struct device *dport_dev);
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index 35268d1c387c..8e9f6ff14d98 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -51,6 +51,7 @@
* @err_rec_array: List of xarrarys to store the memdev error records to
* check attributes for a memory repair operation are from
* current boot.
+ * @detach_scheduled: shows if work does exist already for detaching the memdev.
*/
struct cxl_memdev {
struct device dev;
@@ -66,8 +67,14 @@ struct cxl_memdev {
u8 scrub_cycle;
int scrub_region_id;
struct cxl_mem_err_rec *err_rec_array;
+ bool detach_scheduled;
};
+static inline void schedule_detach(void *cxlmd)
+{
+ schedule_cxl_memdev_detach(cxlmd);
+}
+
static inline struct cxl_memdev *to_cxl_memdev(struct device *dev)
{
return container_of(dev, struct cxl_memdev, dev);
@@ -104,6 +111,8 @@ struct cxl_memdev_attach {
/**
* struct cxl_attach_region - coordinate mapping a region at memdev registration
* @attach: common core attachment descriptor
+ * @cxlr: region attached.
+ * @cxled: endpoint decoder linked to the region.
* @hpa_range: physical address range of the region
*
* For the common simple case of a CXL device with private (non-general purpose
@@ -112,6 +121,8 @@ struct cxl_memdev_attach {
*/
struct cxl_attach_region {
struct cxl_memdev_attach attach;
+ struct cxl_region *cxlr;
+ struct cxl_endpoint_decoder *cxled;
struct range hpa_range;
};
diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c
index c12fd0b89883..1b93853b0779 100644
--- a/drivers/cxl/port.c
+++ b/drivers/cxl/port.c
@@ -26,11 +26,6 @@
* PCIe topology.
*/
-static void schedule_detach(void *cxlmd)
-{
- schedule_cxl_memdev_detach(cxlmd);
-}
-
static int discover_region(struct device *dev, void *unused)
{
struct cxl_endpoint_decoder *cxled;
--
2.34.1
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [RFC 3/3] Documentation/cxl: add document about Type2 binding and unbinding
2026-11-18 19:14 [RFC 0/3] Type2 unwinding ordering change alucerop
2026-11-18 19:14 ` [RFC 1/3] cxl: add tracepoints to the unwinding code path alucerop
2026-11-18 19:14 ` [RFC 2/3] cxl: make explicit removal of endpoint and decoder for Type2 alucerop
@ 2026-11-18 19:14 ` alucerop
2026-11-18 18:04 ` sashiko-bot
2026-11-18 23:39 ` [RFC 0/3] Type2 unwinding ordering change Dave Jiang
3 siblings, 1 reply; 10+ messages in thread
From: alucerop @ 2026-11-18 19:14 UTC (permalink / raw)
To: linux-cxl, dave.jiang, dave, jic23, alison.schofield,
vishal.l.verma
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
CXL subsystem relies on kernel device model and dev managed
infrastructure for helping with device unwinding. This is not trivial
as different objects are created for each CXL Type2 or Type3 device and
with Type2 relying on an external driver to the CXL core.
Explain here:
1) the device chain creation when a Type2 device binds to its driver and
Type2 CXL API is used for CXL initialization.
2) the device chain unwinding and the potential starting points for it.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
.../cxl/type2/cxl_type2_objects_unwinding.rst | 102 ++++++++++++++++++
1 file changed, 102 insertions(+)
create mode 100644 Documentation/driver-api/cxl/type2/cxl_type2_objects_unwinding.rst
diff --git a/Documentation/driver-api/cxl/type2/cxl_type2_objects_unwinding.rst b/Documentation/driver-api/cxl/type2/cxl_type2_objects_unwinding.rst
new file mode 100644
index 000000000000..873f95d9c202
--- /dev/null
+++ b/Documentation/driver-api/cxl/type2/cxl_type2_objects_unwinding.rst
@@ -0,0 +1,102 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=====================================
+Compute Express Link: Type2 unwinding
+=====================================
+
+The CXL subsystem relies on the linux kernel device abstraction for representing
+the different componentes building up the subsystem itself. The core or root
+devices of the subsystem are created from data coming through ACPI and are not
+discussed in this document, with the target being Type2 devices and the binding
+and unbinding of those physical devices into the CXL subsystem.
+
+Along with logical devices come CXL drivers which allow to perform this binding
+and unbinding through usual kernel probe mechanisms. Moreover, current CXL
+subsystem design relies on devres (managed resources) kernel functionality
+linking actions to device unbinding from drivers helping with all the unwinding
+required.
+
+When a device is unbound the kernel will first invoke those devm actions linked
+to the device and then, if there is no further references to the device kobject,
+it will be released or freed. This implies actions linked to the unbinding
+happen at the unbinding time but the final device release can happen at a later
+time. Furthermore, those actions can lead to other related objects to be removed
+as well as their existence depends on the current one being removed. For a Type2
+device doing CXL initialization, these are the objects created with order from
+left to right and with dependencies in the opposite direction:
+
+
+ cxl_memdev --> endpoint (cxl_port) --> cxl_decoder --> cxl_region
+
+
+A CXL region defines the Host physical range where the device memory (HDM) is
+mapped and someone, a client, will likely be using it somehow. For a Type2 this
+client is the Type2 "generic" driver, like a network driver using CXL for some
+kind of acceleration. For a Type3 the client could be the kernel memory
+management or some user space functionality using such a physical range through
+a dax device, with the driver taking over the device being the CXL core mem
+driver. Note this last driver is also used in Type2 for the cxl_memdev created,
+as next paragraph does explain.
+
+The above object chain is created when a new memdev device is added to the
+kernel device core and such a device binding to the cxl_mem driver. This will
+trigger port(s) creation, at least one, a cxl_port device which will bind to
+the cxl_port driver when added to the kernel device core. For a decoder only a
+device is created (or as many as decoders advertises the Type2 device) but no
+binding as there is no decoder driver to bind to. Finally, the region device
+will be automatically created during initialization for current Type2 devices
+expectation (this is called autodiscovery and relies on the system BIOS doing
+the proper physical HDM decoders programming, but work for supporting adhoc
+region creation is in the way) and it will bind to the cxl_region driver. Once
+there exists a driver bound to a device, it can be unbound through the sysfs
+files created by default by the kernel device core.
+
+
+Unwinding expectation
+=====================
+
+One could think a Type2 driver creating all those objects should control the
+unwinding of all of them, not necessarily because some driver action but maybe
+because the kernel signals the driver about that necessity. However, with the
+current design those objects can end up being unbound through different paths:
+
+ 1) The pci device owning the Type2 CXL "device" unbinds from its driver.
+ 2) The type2 driver is removed triggering the pci device unbinding.
+ 3) The cxl memdev is unbound from the cxl mem driver through sysfs.
+ 4) The cxl endpoint port is unbound from the cxl port driver through systfs.
+ 5) The cxl_region is unbound from the cxl region driver through sysfs.
+ 8) Root port linked to the device is unbound frmo the cxl port through sysfs.
+ 9) cxl acpi kernel module is removed.
+
+The first two cases go through the driver remove code path allowing a Type2
+aware driver to stop using the CXL region memory range while all the CXL
+objects are still there. Devm actions will start the CXL unwinding just after
+that code path is executed with the cxl_memdev object being the first one
+getting the unwinding signal, but the order here is not relevant as there is
+no client using the CXL region after the driver exit code path execution.
+
+The next cases rely on devm actions linked to those logical CXL devices created
+and depending on each of them, the order changes ... for a Type3 device. A Type2
+needs the first step in any unwinding chain to be the Type2 driver stopping
+the consumption of the HPA, so devm action are different for obtaining just
+this deterministic order in any case. This implies the code linked to some
+devm actions are explicitly invoked by the unwinding code at the right point.
+In other words, the unwinding relies on both, devm and explicit calls.
+
+
+Visualizing the different unwinding cases
+=========================================
+
+For helping with the debugging and/or understanding of current Type2, or even
+Type3 devices, unwinding some tracepoints are available for enabling them all:
+
+
+ # echo 1 > /sys/kernel/debug/tracing/events/cxl_unwinding/enable
+
+or individually as for example:
+
+ # echo 1 > /sys/kernel/debug/tracing/events/cxl_unwinding/cxl_memdev_release/enable
+
+ or
+
+ # echo "cxl_unwinding:cxl_memdev_release" > /sys/kernel/debug/tracing/set_event
--
2.34.1
^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [RFC 0/3] Type2 unwinding ordering change
2026-11-18 19:14 [RFC 0/3] Type2 unwinding ordering change alucerop
` (2 preceding siblings ...)
2026-11-18 19:14 ` [RFC 3/3] Documentation/cxl: add document about Type2 binding and unbinding alucerop
@ 2026-11-18 23:39 ` Dave Jiang
2026-11-19 8:13 ` Lucero Palau, Alejandro
2026-11-19 10:00 ` Lucero Palau, Alejandro
3 siblings, 2 replies; 10+ messages in thread
From: Dave Jiang @ 2026-11-18 23:39 UTC (permalink / raw)
To: alucerop, linux-cxl, dave, jic23, alison.schofield,
vishal.l.verma
On 11/18/26 12:14 PM, alucerop@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> This RFC requires more work mainly with the tracepoints and surely with
> documentation which, I hope, will get better with help from reviewers.
>
> Regarding the solution proposed for the described problem, I know there could
> be other solutions, so I'll be happy if someone comes with a better one, but
> my feeling is the current design based strongly on devm actions is adding
> complexity which would be desirable to reduce somehow.
>
>
> The summary
> ===========
> Current Type2 support is safe but semantically wrong: a Type2 driver could not
> be (temporarily) aware about the CXL region (the one creating the Host physical
> range to work with) is not there anymore. It could because it depends on the
> unwinding triggering point which changes the ordering of CXL objects unwinding.
> It is safe because for current Type2 expectations the physical mapping will
> remain: those HDM decoders building up the path and programmed by the BIOS will
> not be reset.
Hi Alejandro,
I ran the series under cxl_test with type2_test=1 and KASAN. A plain unbind of
the accel driver oopses. delete_endpoint() unregisters the endpoint decoder,
then the devm action from add_hdm_decoder() unregisters it again. Binding the
accel with no CXL topology also oopses, on an ERR_PTR endpoint.
There's also a Type3 regression. detach_scheduled is never cleared, so after
one cxl_mem unbind/bind a Type3 memdev no longer detaches when its endpoint
is unbound.
With the crash fixed locally, the endpoint is never released after a Type2
teardown. detach_memdev() puts cxlmd->endpoint after delete_endpoint() has
set it to NULL.
The ordering isn't fixed either. schedule_detach() is async. On endpoint or
cxl_acpi unbind the decoders are torn down while the detach work runs the
driver teardown in parallel. A trace shows the two interleaved. The detach
can't run synchronously from the endpoint unbind. The PCI unbind takes the
PCI device lock and then needs the endpoint lock to remove the endpoint.
I don't think the driver needs to be unbound first. It needs to stop using
the HPA before the region goes away. Every region removal goes through
unregister_region(), so how about telling the driver there?
The diff below shows the intent. devm_cxl_probe_mem() takes an optional
revoke() callback. unregister_region() calls it after the region device is
deleted, while the decoders and HPA are still intact. A bus notifier disarms
it before the driver's own remove(). The async detach stays as it is.
Under cxl_test, mem0, endpoint, port and cxl_acpi unbind all call revoke()
before the first decoder teardown. If the driver unbind starts while
revoke() is still running, remove() waits for it to return.
I also tried your multi-PF v2 on top of this with mock functions under
cxl_test. It doesn't look like it needs a deterministic order either.
Deleting the region unbinds the other functions synchronously in
device_del(), before revoke() and before the decoders are torn down, for
every trigger above. cxl_get_range_and_link() needed changes for that. As
posted it can deadlock against PF0's unbind. The fixes are on the branch
after your v2 patches.
The branch [1] has the full set and the cxl_test changes I used. Please
take a look and see what you think.
[1]: https://git.kernel.org/pub/scm/linux/kernel/git/djiang/linux.git/log/?h=type2-revoke
DJ
---
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 787988e92965..50c2fcee54b6 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -752,6 +752,13 @@ static void cxl_memdev_unregister(void *_cxlmd)
struct cxl_memdev *cxlmd = _cxlmd;
struct device *dev = &cxlmd->dev;
+ /*
+ * The attach owner is unbinding and has already stopped using the
+ * region. Don't call back into it while the region unwinds below.
+ */
+ if (cxlmd->attach && cxlmd->attach->disarm)
+ cxlmd->attach->disarm(cxlmd);
+
cdev_device_del(&cxlmd->cdev, dev);
cxl_memdev_shutdown(dev);
put_device(dev);
@@ -834,6 +841,7 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
dev->type = &cxl_class_memdev_type;
device_set_pm_not_required(dev);
INIT_WORK(&cxlmd->detach_work, detach_memdev);
+ mutex_init(&cxlmd->attach_lock);
cdev = &cxlmd->cdev;
cdev_init(cdev, fops);
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 038233dfb142..e8a8843c2ddb 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -2559,6 +2559,41 @@ static struct cxl_region *to_cxl_region(struct device *dev)
return container_of(dev, struct cxl_region, dev);
}
+/*
+ * Tell attach owners (Type2 drivers) the region is going away while its
+ * decoders and HPA are still intact. Every region removal goes through
+ * unregister_region(), so this is the one place the owner needs to hear
+ * about it.
+ */
+static void cxl_region_notify_attach(struct cxl_region *cxlr)
+{
+ struct cxl_memdev *cxlmds[CXL_DECODER_MAX_INTERLEAVE];
+ struct cxl_region_params *p = &cxlr->params;
+ int n = 0;
+
+ lockdep_assert_held(&to_cxl_root_decoder(cxlr->dev.parent)->regions_lock);
+
+ scoped_guard(rwsem_read, &cxl_rwsem.region) {
+ for (int i = 0; i < p->interleave_ways; i++) {
+ struct cxl_endpoint_decoder *cxled = p->targets[i];
+ struct cxl_memdev *cxlmd;
+
+ if (!cxled)
+ continue;
+ cxlmd = cxled_to_memdev(cxled);
+ if (!cxlmd->attach || !cxlmd->attach->region_lost)
+ continue;
+ get_device(&cxlmd->dev);
+ cxlmds[n++] = cxlmd;
+ }
+ }
+
+ for (int i = 0; i < n; i++) {
+ cxlmds[i]->attach->region_lost(cxlmds[i], cxlr);
+ put_device(&cxlmds[i]->dev);
+ }
+}
+
static void unregister_region(struct cxl_region *cxlr)
{
struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
@@ -2568,6 +2603,13 @@ static void unregister_region(struct cxl_region *cxlr)
xa_erase(&cxlrd->regions, cxlr->id);
device_del(&cxlr->dev);
+ /*
+ * device_del() unbound any device-link consumers of the region (e.g.
+ * other functions of a multi-function device), so notify the owner
+ * last. Its decoders and HPA are still intact.
+ */
+ cxl_region_notify_attach(cxlr);
+
/*
* Now that region sysfs is shutdown, the parameter block is now
* read-only, so no need to hold the region rwsem to access the
@@ -4182,14 +4224,42 @@ int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
if (rc)
return rc;
- attach->hpa_range = (struct range) {
- .start = cxlr->params.res->start,
- .end = cxlr->params.res->end,
- };
+ scoped_guard(mutex, &cxlmd->attach_lock) {
+ attach->hpa_range = (struct range) {
+ .start = cxlr->params.res->start,
+ .end = cxlr->params.res->end,
+ };
+ cxlmd->attach_region = cxlr;
+ }
return 0;
}
EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_region, "cxl_mem");
+void cxl_memdev_attach_region_lost(struct cxl_memdev *cxlmd,
+ struct cxl_region *cxlr)
+{
+ struct cxl_attach_region *attach =
+ container_of(cxlmd->attach, typeof(*attach), attach);
+
+ guard(mutex)(&cxlmd->attach_lock);
+ if (cxlmd->attach_region != cxlr)
+ return;
+ cxlmd->attach_region = NULL;
+
+ dev_dbg(&cxlmd->dev, "%s going away, revoking %pra\n",
+ dev_name(&cxlr->dev), &attach->hpa_range);
+ if (attach->revoke)
+ attach->revoke(cxlmd->cxlds);
+}
+EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_region_lost, "cxl_mem");
+
+void cxl_memdev_attach_disarm(struct cxl_memdev *cxlmd)
+{
+ guard(mutex)(&cxlmd->attach_lock);
+ cxlmd->attach_region = NULL;
+}
+EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_disarm, "cxl_mem");
+
/*
* The presence of an attach method indicates that the region is designated for
* a purpose outside of CXL core memory expansion defaults.
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index 35268d1c387c..675d7e21f6c1 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -44,6 +44,10 @@
* @cxl_nvd: optional bridge to an nvdimm if the device supports pmem
* @endpoint: connection to the CXL port topology for this memory device
* @attach: creator of this memdev depends on CXL link attach to operate
+ * @attach_lock: protects @attach_region and serializes revoking it
+ * @attach_region: region the @attach owner uses, NULL once that region goes
+ * away or the owner unbinds. Lives with the memdev, not the
+ * owner's devm state, so a memdev reference is enough to read it.
* @id: id number of this memdev instance.
* @depth: endpoint port depth
* @scrub_cycle: current scrub cycle set for this device
@@ -61,6 +65,8 @@ struct cxl_memdev {
struct cxl_nvdimm *cxl_nvd;
struct cxl_port *endpoint;
const struct cxl_memdev_attach *attach;
+ struct mutex attach_lock; /* protects @attach_region */
+ struct cxl_region *attach_region;
int id;
int depth;
u8 scrub_cycle;
@@ -97,14 +103,30 @@ static inline bool is_cxl_endpoint(struct cxl_port *port)
return is_cxl_memdev(port->uport_dev);
}
+/**
+ * struct cxl_memdev_attach - memdev attach provider operations
+ * @probe: attach the memdev, runs in cxl_mem_probe() context
+ * @region_lost: @cxlr is being unregistered and @cxlmd is one of its
+ * targets. Called with the root decoder regions_lock held, after
+ * the region device is deleted and before its decoders are torn
+ * down.
+ * @disarm: the attach owner is unbinding; no further @region_lost
+ * notifications are wanted. Waits for an in-flight one to finish.
+ */
struct cxl_memdev_attach {
int (*probe)(struct cxl_memdev *cxlmd);
+ void (*region_lost)(struct cxl_memdev *cxlmd, struct cxl_region *cxlr);
+ void (*disarm)(struct cxl_memdev *cxlmd);
};
/**
* struct cxl_attach_region - coordinate mapping a region at memdev registration
* @attach: common core attachment descriptor
* @hpa_range: physical address range of the region
+ * @revoke: driver callback, stop all access to @hpa_range
+ * @unbind_nb: bus notifier, disarms before the driver's remove()
+ * @host: device whose unbind disarms @revoke (the cxlds->dev)
+ * @cxlmd: the memdev, set once devm_cxl_probe_mem() succeeds
*
* For the common simple case of a CXL device with private (non-general purpose
* / "accelerator") memory, enumerate firmware instantiated region, or
@@ -113,15 +135,25 @@ struct cxl_memdev_attach {
struct cxl_attach_region {
struct cxl_memdev_attach attach;
struct range hpa_range;
+ void (*revoke)(struct cxl_dev_state *cxlds);
+ struct notifier_block unbind_nb;
+ struct device *host;
+ struct cxl_memdev *cxlmd;
};
#ifdef CONFIG_CXL_REGION
int cxl_memdev_attach_region(struct cxl_memdev *cxlmd);
+void cxl_memdev_attach_region_lost(struct cxl_memdev *cxlmd,
+ struct cxl_region *cxlr);
+void cxl_memdev_attach_disarm(struct cxl_memdev *cxlmd);
#else
static inline int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
{
return -EOPNOTSUPP;
}
+
+#define cxl_memdev_attach_region_lost NULL
+#define cxl_memdev_attach_disarm NULL
#endif
struct cxl_memdev *devm_cxl_add_classdev(struct cxl_dev_state *cxlds);
diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
index 702ae9ebdcde..cd3db7646c64 100644
--- a/drivers/cxl/mem.c
+++ b/drivers/cxl/mem.c
@@ -219,10 +219,39 @@ struct cxl_memdev *devm_cxl_add_classdev(struct cxl_dev_state *cxlds)
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_classdev, "CXL");
+static int cxl_attach_unbind_notify(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ struct cxl_attach_region *attach =
+ container_of(nb, struct cxl_attach_region, unbind_nb);
+
+ if (data != attach->host)
+ return NOTIFY_DONE;
+ if (action != BUS_NOTIFY_UNBIND_DRIVER &&
+ action != BUS_NOTIFY_DRIVER_NOT_BOUND)
+ return NOTIFY_DONE;
+
+ /*
+ * Sent before remove(), or before devres unwind on probe failure:
+ * wait out an in-flight revoke, block new ones.
+ */
+ if (attach->cxlmd)
+ cxl_memdev_attach_disarm(attach->cxlmd);
+ return NOTIFY_OK;
+}
+
+static void cxl_attach_unregister_notifier(void *data)
+{
+ struct cxl_attach_region *attach = data;
+
+ bus_unregister_notifier(attach->host->bus, &attach->unbind_nb);
+}
+
/**
* devm_cxl_probe_mem - Add a CXL memory device and probe its region
* @cxlds: CXL device state to associate with the memdev
* @hpa_range: CXL.mem physical address range result
+ * @revoke: optional callback to stop HPA access before the region goes away
*
* Upon return the device will have had a chance to attach to the
* cxl_mem driver, but may fail to attach if the CXL topology is not ready
@@ -233,12 +262,29 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_classdev, "CXL");
*
* The parent of the resulting device and the devm context for allocations is
* @cxlds->dev.
+ *
+ * @revoke, if set, is called once if the region backing @hpa_range is about
+ * to be torn down for a reason other than @cxlds->dev unbinding (CXL port,
+ * memdev or region unbind / delete, cxl_acpi removal). The region, its
+ * decoders and the HPA are still intact when it runs. On return the driver
+ * must no longer access @hpa_range. The driver is unbound later,
+ * asynchronously, as before.
+ *
+ * @revoke may run before this function returns, so it needs to synchronize
+ * with the rest of the driver's probe. It never runs concurrently with or
+ * after the driver's remove() or its probe failure unwind. The driver core
+ * notification for either waits for an in-flight @revoke and blocks new
+ * ones first. @revoke is called with CXL core locks held, so it must not call
+ * back into the CXL core nor take the device lock of @cxlds->dev. It may
+ * sleep.
*/
struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
- struct range *hpa_range)
+ struct range *hpa_range,
+ void (*revoke)(struct cxl_dev_state *cxlds))
{
struct cxl_attach_region *attach = devm_kmalloc_obj(cxlds->dev, *attach);
struct cxl_memdev *cxlmd;
+ int rc;
if (!attach)
return ERR_PTR(-ENOMEM);
@@ -246,12 +292,35 @@ struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
*attach = (struct cxl_attach_region) {
.attach = {
.probe = cxl_memdev_attach_region,
+ .region_lost = cxl_memdev_attach_region_lost,
+ .disarm = cxl_memdev_attach_disarm,
},
.hpa_range = { 0, -1 },
+ .revoke = revoke,
+ .unbind_nb.notifier_call = cxl_attach_unbind_notify,
+ .host = cxlds->dev,
};
+ if (revoke) {
+ rc = bus_register_notifier(cxlds->dev->bus, &attach->unbind_nb);
+ if (rc)
+ return ERR_PTR(rc);
+ rc = devm_add_action_or_reset(cxlds->dev,
+ cxl_attach_unregister_notifier,
+ attach);
+ if (rc)
+ return ERR_PTR(rc);
+ }
cxlmd = __devm_cxl_add_memdev(cxlds, &attach->attach);
+ if (IS_ERR(cxlmd))
+ return cxlmd;
+
+ /* Don't hand out a range whose region already went away */
+ guard(mutex)(&cxlmd->attach_lock);
+ if (!cxlmd->attach_region)
+ return ERR_PTR(-ENXIO);
*hpa_range = attach->hpa_range;
+ attach->cxlmd = cxlmd;
return cxlmd;
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_probe_mem, "CXL");
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index 348d7404cd7a..a82a0e2fc609 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -74,7 +74,7 @@ int efx_cxl_init(struct efx_probe_data *probe_data)
return -ENODEV;
}
- cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &cxl_pio_range);
+ cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &cxl_pio_range, NULL);
if (IS_ERR(cxl->cxlmd)) {
pci_err(pci_dev, "CXL accel memdev creation failed\n");
return PTR_ERR(cxl->cxlmd);
diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
index 1c04a7823f64..c672ec83d820 100644
--- a/include/cxl/cxl.h
+++ b/include/cxl/cxl.h
@@ -252,7 +252,8 @@ struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
})
struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
- struct range *range);
+ struct range *range,
+ void (*revoke)(struct cxl_dev_state *cxlds));
int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity);
#endif /* __CXL_CXL_H__ */
^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [RFC 0/3] Type2 unwinding ordering change
2026-11-18 23:39 ` [RFC 0/3] Type2 unwinding ordering change Dave Jiang
@ 2026-11-19 8:13 ` Lucero Palau, Alejandro
2026-11-19 10:00 ` Lucero Palau, Alejandro
1 sibling, 0 replies; 10+ messages in thread
From: Lucero Palau, Alejandro @ 2026-11-19 8:13 UTC (permalink / raw)
To: Dave Jiang, alucerop, linux-cxl, dave, jic23, alison.schofield,
vishal.l.verma
On 18/11/2026 23:39, Dave Jiang wrote:
>
> On 11/18/26 12:14 PM, alucerop@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> This RFC requires more work mainly with the tracepoints and surely with
>> documentation which, I hope, will get better with help from reviewers.
>>
>> Regarding the solution proposed for the described problem, I know there could
>> be other solutions, so I'll be happy if someone comes with a better one, but
>> my feeling is the current design based strongly on devm actions is adding
>> complexity which would be desirable to reduce somehow.
>>
>>
>> The summary
>> ===========
>> Current Type2 support is safe but semantically wrong: a Type2 driver could not
>> be (temporarily) aware about the CXL region (the one creating the Host physical
>> range to work with) is not there anymore. It could because it depends on the
>> unwinding triggering point which changes the ordering of CXL objects unwinding.
>> It is safe because for current Type2 expectations the physical mapping will
>> remain: those HDM decoders building up the path and programmed by the BIOS will
>> not be reset.
> Hi Alejandro,
> I ran the series under cxl_test with type2_test=1 and KASAN. A plain unbind of
> the accel driver oopses. delete_endpoint() unregisters the endpoint decoder,
> then the devm action from add_hdm_decoder() unregisters it again. Binding the
> accel with no CXL topology also oopses, on an ERR_PTR endpoint.
>
> There's also a Type3 regression. detach_scheduled is never cleared, so after
> one cxl_mem unbind/bind a Type3 memdev no longer detaches when its endpoint
> is unbound.
>
> With the crash fixed locally, the endpoint is never released after a Type2
> teardown. detach_memdev() puts cxlmd->endpoint after delete_endpoint() has
> set it to NULL.
>
> The ordering isn't fixed either. schedule_detach() is async. On endpoint or
> cxl_acpi unbind the decoders are torn down while the detach work runs the
> driver teardown in parallel. A trace shows the two interleaved. The detach
> can't run synchronously from the endpoint unbind. The PCI unbind takes the
> PCI device lock and then needs the endpoint lock to remove the endpoint.
>
> I don't think the driver needs to be unbound first. It needs to stop using
> the HPA before the region goes away. Every region removal goes through
> unregister_region(), so how about telling the driver there?
Hi Dave,
I'm looking at those problems you report above. I did not see them but I
was pretty sure something was going to appear ... what makes my point
about the current complexity stronger, or at least, the need for adding
tracepoints to visualize this easily. I wanted to send the RFC for
starting the discussion after this preliminary work studying and
proposing a solution what was not so trivial as I thought.
But I wanted to quickly reply to this specific comment and the idea
proposed below. Interestingly, this was something I tried in the past:
https://lore.kernel.org/linux-cxl/20250624141355.269056-19-alejandro.lucero-palau@amd.com/
Look at the cxl_create_region interface where the Type2 driver could add
a callback action linked to cxl region removal. You can also look at the
discussion there with Dan. He did not like it and wanted to always
remove the driver in this case.
I disagreed then but I know think it is the right thing to do. It is
possible to have Type2 drivers aware of such disruption and they
changing to a degraded mode where CXL is not used, but from sfc
perspective, after internal discussion, it does not make sense.
Moreover, multipf support implies to signal all PFs using CXL and it
will need to deal with these concurrency problems originating this work.
One could argue those concurrency issues need to be addressed but my
point is to reduce that complexity where other PFs can not find device
CXL in some intermediate state requiring to be aware of all those
possibilities and add protections. I'm not saying it is not doable just
I prefer to avoid it, always presenting the device CXL state in a
all-right-state or no state at all. Relying on unwinding with Type2
driver removal first, this is/should be possible.
Anyway, thanks for looking at this. I will look at those other problems
and also the sashiko reports pointing to problems when devm actions
registration fails ...
> The diff below shows the intent. devm_cxl_probe_mem() takes an optional
> revoke() callback. unregister_region() calls it after the region device is
> deleted, while the decoders and HPA are still intact. A bus notifier disarms
> it before the driver's own remove(). The async detach stays as it is.
>
> Under cxl_test, mem0, endpoint, port and cxl_acpi unbind all call revoke()
> before the first decoder teardown. If the driver unbind starts while
> revoke() is still running, remove() waits for it to return.
>
> I also tried your multi-PF v2 on top of this with mock functions under
> cxl_test. It doesn't look like it needs a deterministic order either.
> Deleting the region unbinds the other functions synchronously in
> device_del(), before revoke() and before the decoders are torn down, for
> every trigger above. cxl_get_range_and_link() needed changes for that. As
> posted it can deadlock against PF0's unbind. The fixes are on the branch
> after your v2 patches.
>
> The branch [1] has the full set and the cxl_test changes I used. Please
> take a look and see what you think.
>
> [1]: https://git.kernel.org/pub/scm/linux/kernel/git/djiang/linux.git/log/?h=type2-revoke
>
> DJ
>
> ---
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 787988e92965..50c2fcee54b6 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -752,6 +752,13 @@ static void cxl_memdev_unregister(void *_cxlmd)
> struct cxl_memdev *cxlmd = _cxlmd;
> struct device *dev = &cxlmd->dev;
>
> + /*
> + * The attach owner is unbinding and has already stopped using the
> + * region. Don't call back into it while the region unwinds below.
> + */
> + if (cxlmd->attach && cxlmd->attach->disarm)
> + cxlmd->attach->disarm(cxlmd);
> +
> cdev_device_del(&cxlmd->cdev, dev);
> cxl_memdev_shutdown(dev);
> put_device(dev);
> @@ -834,6 +841,7 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
> dev->type = &cxl_class_memdev_type;
> device_set_pm_not_required(dev);
> INIT_WORK(&cxlmd->detach_work, detach_memdev);
> + mutex_init(&cxlmd->attach_lock);
>
> cdev = &cxlmd->cdev;
> cdev_init(cdev, fops);
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index 038233dfb142..e8a8843c2ddb 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -2559,6 +2559,41 @@ static struct cxl_region *to_cxl_region(struct device *dev)
> return container_of(dev, struct cxl_region, dev);
> }
>
> +/*
> + * Tell attach owners (Type2 drivers) the region is going away while its
> + * decoders and HPA are still intact. Every region removal goes through
> + * unregister_region(), so this is the one place the owner needs to hear
> + * about it.
> + */
> +static void cxl_region_notify_attach(struct cxl_region *cxlr)
> +{
> + struct cxl_memdev *cxlmds[CXL_DECODER_MAX_INTERLEAVE];
> + struct cxl_region_params *p = &cxlr->params;
> + int n = 0;
> +
> + lockdep_assert_held(&to_cxl_root_decoder(cxlr->dev.parent)->regions_lock);
> +
> + scoped_guard(rwsem_read, &cxl_rwsem.region) {
> + for (int i = 0; i < p->interleave_ways; i++) {
> + struct cxl_endpoint_decoder *cxled = p->targets[i];
> + struct cxl_memdev *cxlmd;
> +
> + if (!cxled)
> + continue;
> + cxlmd = cxled_to_memdev(cxled);
> + if (!cxlmd->attach || !cxlmd->attach->region_lost)
> + continue;
> + get_device(&cxlmd->dev);
> + cxlmds[n++] = cxlmd;
> + }
> + }
> +
> + for (int i = 0; i < n; i++) {
> + cxlmds[i]->attach->region_lost(cxlmds[i], cxlr);
> + put_device(&cxlmds[i]->dev);
> + }
> +}
> +
> static void unregister_region(struct cxl_region *cxlr)
> {
> struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
> @@ -2568,6 +2603,13 @@ static void unregister_region(struct cxl_region *cxlr)
> xa_erase(&cxlrd->regions, cxlr->id);
> device_del(&cxlr->dev);
>
> + /*
> + * device_del() unbound any device-link consumers of the region (e.g.
> + * other functions of a multi-function device), so notify the owner
> + * last. Its decoders and HPA are still intact.
> + */
> + cxl_region_notify_attach(cxlr);
> +
> /*
> * Now that region sysfs is shutdown, the parameter block is now
> * read-only, so no need to hold the region rwsem to access the
> @@ -4182,14 +4224,42 @@ int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
> if (rc)
> return rc;
>
> - attach->hpa_range = (struct range) {
> - .start = cxlr->params.res->start,
> - .end = cxlr->params.res->end,
> - };
> + scoped_guard(mutex, &cxlmd->attach_lock) {
> + attach->hpa_range = (struct range) {
> + .start = cxlr->params.res->start,
> + .end = cxlr->params.res->end,
> + };
> + cxlmd->attach_region = cxlr;
> + }
> return 0;
> }
> EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_region, "cxl_mem");
>
> +void cxl_memdev_attach_region_lost(struct cxl_memdev *cxlmd,
> + struct cxl_region *cxlr)
> +{
> + struct cxl_attach_region *attach =
> + container_of(cxlmd->attach, typeof(*attach), attach);
> +
> + guard(mutex)(&cxlmd->attach_lock);
> + if (cxlmd->attach_region != cxlr)
> + return;
> + cxlmd->attach_region = NULL;
> +
> + dev_dbg(&cxlmd->dev, "%s going away, revoking %pra\n",
> + dev_name(&cxlr->dev), &attach->hpa_range);
> + if (attach->revoke)
> + attach->revoke(cxlmd->cxlds);
> +}
> +EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_region_lost, "cxl_mem");
> +
> +void cxl_memdev_attach_disarm(struct cxl_memdev *cxlmd)
> +{
> + guard(mutex)(&cxlmd->attach_lock);
> + cxlmd->attach_region = NULL;
> +}
> +EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_disarm, "cxl_mem");
> +
> /*
> * The presence of an attach method indicates that the region is designated for
> * a purpose outside of CXL core memory expansion defaults.
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index 35268d1c387c..675d7e21f6c1 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> @@ -44,6 +44,10 @@
> * @cxl_nvd: optional bridge to an nvdimm if the device supports pmem
> * @endpoint: connection to the CXL port topology for this memory device
> * @attach: creator of this memdev depends on CXL link attach to operate
> + * @attach_lock: protects @attach_region and serializes revoking it
> + * @attach_region: region the @attach owner uses, NULL once that region goes
> + * away or the owner unbinds. Lives with the memdev, not the
> + * owner's devm state, so a memdev reference is enough to read it.
> * @id: id number of this memdev instance.
> * @depth: endpoint port depth
> * @scrub_cycle: current scrub cycle set for this device
> @@ -61,6 +65,8 @@ struct cxl_memdev {
> struct cxl_nvdimm *cxl_nvd;
> struct cxl_port *endpoint;
> const struct cxl_memdev_attach *attach;
> + struct mutex attach_lock; /* protects @attach_region */
> + struct cxl_region *attach_region;
> int id;
> int depth;
> u8 scrub_cycle;
> @@ -97,14 +103,30 @@ static inline bool is_cxl_endpoint(struct cxl_port *port)
> return is_cxl_memdev(port->uport_dev);
> }
>
> +/**
> + * struct cxl_memdev_attach - memdev attach provider operations
> + * @probe: attach the memdev, runs in cxl_mem_probe() context
> + * @region_lost: @cxlr is being unregistered and @cxlmd is one of its
> + * targets. Called with the root decoder regions_lock held, after
> + * the region device is deleted and before its decoders are torn
> + * down.
> + * @disarm: the attach owner is unbinding; no further @region_lost
> + * notifications are wanted. Waits for an in-flight one to finish.
> + */
> struct cxl_memdev_attach {
> int (*probe)(struct cxl_memdev *cxlmd);
> + void (*region_lost)(struct cxl_memdev *cxlmd, struct cxl_region *cxlr);
> + void (*disarm)(struct cxl_memdev *cxlmd);
> };
>
> /**
> * struct cxl_attach_region - coordinate mapping a region at memdev registration
> * @attach: common core attachment descriptor
> * @hpa_range: physical address range of the region
> + * @revoke: driver callback, stop all access to @hpa_range
> + * @unbind_nb: bus notifier, disarms before the driver's remove()
> + * @host: device whose unbind disarms @revoke (the cxlds->dev)
> + * @cxlmd: the memdev, set once devm_cxl_probe_mem() succeeds
> *
> * For the common simple case of a CXL device with private (non-general purpose
> * / "accelerator") memory, enumerate firmware instantiated region, or
> @@ -113,15 +135,25 @@ struct cxl_memdev_attach {
> struct cxl_attach_region {
> struct cxl_memdev_attach attach;
> struct range hpa_range;
> + void (*revoke)(struct cxl_dev_state *cxlds);
> + struct notifier_block unbind_nb;
> + struct device *host;
> + struct cxl_memdev *cxlmd;
> };
>
> #ifdef CONFIG_CXL_REGION
> int cxl_memdev_attach_region(struct cxl_memdev *cxlmd);
> +void cxl_memdev_attach_region_lost(struct cxl_memdev *cxlmd,
> + struct cxl_region *cxlr);
> +void cxl_memdev_attach_disarm(struct cxl_memdev *cxlmd);
> #else
> static inline int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
> {
> return -EOPNOTSUPP;
> }
> +
> +#define cxl_memdev_attach_region_lost NULL
> +#define cxl_memdev_attach_disarm NULL
> #endif
>
> struct cxl_memdev *devm_cxl_add_classdev(struct cxl_dev_state *cxlds);
> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
> index 702ae9ebdcde..cd3db7646c64 100644
> --- a/drivers/cxl/mem.c
> +++ b/drivers/cxl/mem.c
> @@ -219,10 +219,39 @@ struct cxl_memdev *devm_cxl_add_classdev(struct cxl_dev_state *cxlds)
> }
> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_classdev, "CXL");
>
> +static int cxl_attach_unbind_notify(struct notifier_block *nb,
> + unsigned long action, void *data)
> +{
> + struct cxl_attach_region *attach =
> + container_of(nb, struct cxl_attach_region, unbind_nb);
> +
> + if (data != attach->host)
> + return NOTIFY_DONE;
> + if (action != BUS_NOTIFY_UNBIND_DRIVER &&
> + action != BUS_NOTIFY_DRIVER_NOT_BOUND)
> + return NOTIFY_DONE;
> +
> + /*
> + * Sent before remove(), or before devres unwind on probe failure:
> + * wait out an in-flight revoke, block new ones.
> + */
> + if (attach->cxlmd)
> + cxl_memdev_attach_disarm(attach->cxlmd);
> + return NOTIFY_OK;
> +}
> +
> +static void cxl_attach_unregister_notifier(void *data)
> +{
> + struct cxl_attach_region *attach = data;
> +
> + bus_unregister_notifier(attach->host->bus, &attach->unbind_nb);
> +}
> +
> /**
> * devm_cxl_probe_mem - Add a CXL memory device and probe its region
> * @cxlds: CXL device state to associate with the memdev
> * @hpa_range: CXL.mem physical address range result
> + * @revoke: optional callback to stop HPA access before the region goes away
> *
> * Upon return the device will have had a chance to attach to the
> * cxl_mem driver, but may fail to attach if the CXL topology is not ready
> @@ -233,12 +262,29 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_classdev, "CXL");
> *
> * The parent of the resulting device and the devm context for allocations is
> * @cxlds->dev.
> + *
> + * @revoke, if set, is called once if the region backing @hpa_range is about
> + * to be torn down for a reason other than @cxlds->dev unbinding (CXL port,
> + * memdev or region unbind / delete, cxl_acpi removal). The region, its
> + * decoders and the HPA are still intact when it runs. On return the driver
> + * must no longer access @hpa_range. The driver is unbound later,
> + * asynchronously, as before.
> + *
> + * @revoke may run before this function returns, so it needs to synchronize
> + * with the rest of the driver's probe. It never runs concurrently with or
> + * after the driver's remove() or its probe failure unwind. The driver core
> + * notification for either waits for an in-flight @revoke and blocks new
> + * ones first. @revoke is called with CXL core locks held, so it must not call
> + * back into the CXL core nor take the device lock of @cxlds->dev. It may
> + * sleep.
> */
> struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
> - struct range *hpa_range)
> + struct range *hpa_range,
> + void (*revoke)(struct cxl_dev_state *cxlds))
> {
> struct cxl_attach_region *attach = devm_kmalloc_obj(cxlds->dev, *attach);
> struct cxl_memdev *cxlmd;
> + int rc;
>
> if (!attach)
> return ERR_PTR(-ENOMEM);
> @@ -246,12 +292,35 @@ struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
> *attach = (struct cxl_attach_region) {
> .attach = {
> .probe = cxl_memdev_attach_region,
> + .region_lost = cxl_memdev_attach_region_lost,
> + .disarm = cxl_memdev_attach_disarm,
> },
> .hpa_range = { 0, -1 },
> + .revoke = revoke,
> + .unbind_nb.notifier_call = cxl_attach_unbind_notify,
> + .host = cxlds->dev,
> };
> + if (revoke) {
> + rc = bus_register_notifier(cxlds->dev->bus, &attach->unbind_nb);
> + if (rc)
> + return ERR_PTR(rc);
> + rc = devm_add_action_or_reset(cxlds->dev,
> + cxl_attach_unregister_notifier,
> + attach);
> + if (rc)
> + return ERR_PTR(rc);
> + }
>
> cxlmd = __devm_cxl_add_memdev(cxlds, &attach->attach);
> + if (IS_ERR(cxlmd))
> + return cxlmd;
> +
> + /* Don't hand out a range whose region already went away */
> + guard(mutex)(&cxlmd->attach_lock);
> + if (!cxlmd->attach_region)
> + return ERR_PTR(-ENXIO);
> *hpa_range = attach->hpa_range;
> + attach->cxlmd = cxlmd;
> return cxlmd;
> }
> EXPORT_SYMBOL_NS_GPL(devm_cxl_probe_mem, "CXL");
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index 348d7404cd7a..a82a0e2fc609 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -74,7 +74,7 @@ int efx_cxl_init(struct efx_probe_data *probe_data)
> return -ENODEV;
> }
>
> - cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &cxl_pio_range);
> + cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &cxl_pio_range, NULL);
> if (IS_ERR(cxl->cxlmd)) {
> pci_err(pci_dev, "CXL accel memdev creation failed\n");
> return PTR_ERR(cxl->cxlmd);
> diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
> index 1c04a7823f64..c672ec83d820 100644
> --- a/include/cxl/cxl.h
> +++ b/include/cxl/cxl.h
> @@ -252,7 +252,8 @@ struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
> })
>
> struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
> - struct range *range);
> + struct range *range,
> + void (*revoke)(struct cxl_dev_state *cxlds));
>
> int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity);
> #endif /* __CXL_CXL_H__ */
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [RFC 0/3] Type2 unwinding ordering change
2026-11-18 23:39 ` [RFC 0/3] Type2 unwinding ordering change Dave Jiang
2026-11-19 8:13 ` Lucero Palau, Alejandro
@ 2026-11-19 10:00 ` Lucero Palau, Alejandro
1 sibling, 0 replies; 10+ messages in thread
From: Lucero Palau, Alejandro @ 2026-11-19 10:00 UTC (permalink / raw)
To: Dave Jiang, alucerop, linux-cxl, dave, jic23, alison.schofield,
vishal.l.verma
On 18/11/2026 23:39, Dave Jiang wrote:
>
> On 11/18/26 12:14 PM, alucerop@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> This RFC requires more work mainly with the tracepoints and surely with
>> documentation which, I hope, will get better with help from reviewers.
>>
>> Regarding the solution proposed for the described problem, I know there could
>> be other solutions, so I'll be happy if someone comes with a better one, but
>> my feeling is the current design based strongly on devm actions is adding
>> complexity which would be desirable to reduce somehow.
>>
>>
>> The summary
>> ===========
>> Current Type2 support is safe but semantically wrong: a Type2 driver could not
>> be (temporarily) aware about the CXL region (the one creating the Host physical
>> range to work with) is not there anymore. It could because it depends on the
>> unwinding triggering point which changes the ordering of CXL objects unwinding.
>> It is safe because for current Type2 expectations the physical mapping will
>> remain: those HDM decoders building up the path and programmed by the BIOS will
>> not be reset.
> Hi Alejandro,
> I ran the series under cxl_test with type2_test=1 and KASAN. A plain unbind of
> the accel driver oopses. delete_endpoint() unregisters the endpoint decoder,
> then the devm action from add_hdm_decoder() unregisters it again. Binding the
> accel with no CXL topology also oopses, on an ERR_PTR endpoint.
>
> There's also a Type3 regression. detach_scheduled is never cleared, so after
> one cxl_mem unbind/bind a Type3 memdev no longer detaches when its endpoint
> is unbound.
>
> With the crash fixed locally, the endpoint is never released after a Type2
> teardown. detach_memdev() puts cxlmd->endpoint after delete_endpoint() has
> set it to NULL.
Apologies for being so clumsy. I did miss a change required:
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index de109fb60038..27137a7032c4 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -2322,6 +2322,8 @@ static void cxld_unregister(void *dev)
int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
{
+ if (cxld->target_type == CXL_DECODER_DEVMEM)
+ return 0;
return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
}
EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, "CXL");
With this most of the unwinding possibilities should work as described.
Looking at the remaining issues now.
>
> The ordering isn't fixed either. schedule_detach() is async. On endpoint or
> cxl_acpi unbind the decoders are torn down while the detach work runs the
> driver teardown in parallel. A trace shows the two interleaved. The detach
> can't run synchronously from the endpoint unbind. The PCI unbind takes the
> PCI device lock and then needs the endpoint lock to remove the endpoint.
>
> I don't think the driver needs to be unbound first. It needs to stop using
> the HPA before the region goes away. Every region removal goes through
> unregister_region(), so how about telling the driver there?
>
> The diff below shows the intent. devm_cxl_probe_mem() takes an optional
> revoke() callback. unregister_region() calls it after the region device is
> deleted, while the decoders and HPA are still intact. A bus notifier disarms
> it before the driver's own remove(). The async detach stays as it is.
>
> Under cxl_test, mem0, endpoint, port and cxl_acpi unbind all call revoke()
> before the first decoder teardown. If the driver unbind starts while
> revoke() is still running, remove() waits for it to return.
>
> I also tried your multi-PF v2 on top of this with mock functions under
> cxl_test. It doesn't look like it needs a deterministic order either.
> Deleting the region unbinds the other functions synchronously in
> device_del(), before revoke() and before the decoders are torn down, for
> every trigger above. cxl_get_range_and_link() needed changes for that. As
> posted it can deadlock against PF0's unbind. The fixes are on the branch
> after your v2 patches.
>
> The branch [1] has the full set and the cxl_test changes I used. Please
> take a look and see what you think.
>
> [1]: https://git.kernel.org/pub/scm/linux/kernel/git/djiang/linux.git/log/?h=type2-revoke
>
> DJ
>
> ---
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 787988e92965..50c2fcee54b6 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -752,6 +752,13 @@ static void cxl_memdev_unregister(void *_cxlmd)
> struct cxl_memdev *cxlmd = _cxlmd;
> struct device *dev = &cxlmd->dev;
>
> + /*
> + * The attach owner is unbinding and has already stopped using the
> + * region. Don't call back into it while the region unwinds below.
> + */
> + if (cxlmd->attach && cxlmd->attach->disarm)
> + cxlmd->attach->disarm(cxlmd);
> +
> cdev_device_del(&cxlmd->cdev, dev);
> cxl_memdev_shutdown(dev);
> put_device(dev);
> @@ -834,6 +841,7 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
> dev->type = &cxl_class_memdev_type;
> device_set_pm_not_required(dev);
> INIT_WORK(&cxlmd->detach_work, detach_memdev);
> + mutex_init(&cxlmd->attach_lock);
>
> cdev = &cxlmd->cdev;
> cdev_init(cdev, fops);
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index 038233dfb142..e8a8843c2ddb 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -2559,6 +2559,41 @@ static struct cxl_region *to_cxl_region(struct device *dev)
> return container_of(dev, struct cxl_region, dev);
> }
>
> +/*
> + * Tell attach owners (Type2 drivers) the region is going away while its
> + * decoders and HPA are still intact. Every region removal goes through
> + * unregister_region(), so this is the one place the owner needs to hear
> + * about it.
> + */
> +static void cxl_region_notify_attach(struct cxl_region *cxlr)
> +{
> + struct cxl_memdev *cxlmds[CXL_DECODER_MAX_INTERLEAVE];
> + struct cxl_region_params *p = &cxlr->params;
> + int n = 0;
> +
> + lockdep_assert_held(&to_cxl_root_decoder(cxlr->dev.parent)->regions_lock);
> +
> + scoped_guard(rwsem_read, &cxl_rwsem.region) {
> + for (int i = 0; i < p->interleave_ways; i++) {
> + struct cxl_endpoint_decoder *cxled = p->targets[i];
> + struct cxl_memdev *cxlmd;
> +
> + if (!cxled)
> + continue;
> + cxlmd = cxled_to_memdev(cxled);
> + if (!cxlmd->attach || !cxlmd->attach->region_lost)
> + continue;
> + get_device(&cxlmd->dev);
> + cxlmds[n++] = cxlmd;
> + }
> + }
> +
> + for (int i = 0; i < n; i++) {
> + cxlmds[i]->attach->region_lost(cxlmds[i], cxlr);
> + put_device(&cxlmds[i]->dev);
> + }
> +}
> +
> static void unregister_region(struct cxl_region *cxlr)
> {
> struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
> @@ -2568,6 +2603,13 @@ static void unregister_region(struct cxl_region *cxlr)
> xa_erase(&cxlrd->regions, cxlr->id);
> device_del(&cxlr->dev);
>
> + /*
> + * device_del() unbound any device-link consumers of the region (e.g.
> + * other functions of a multi-function device), so notify the owner
> + * last. Its decoders and HPA are still intact.
> + */
> + cxl_region_notify_attach(cxlr);
> +
> /*
> * Now that region sysfs is shutdown, the parameter block is now
> * read-only, so no need to hold the region rwsem to access the
> @@ -4182,14 +4224,42 @@ int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
> if (rc)
> return rc;
>
> - attach->hpa_range = (struct range) {
> - .start = cxlr->params.res->start,
> - .end = cxlr->params.res->end,
> - };
> + scoped_guard(mutex, &cxlmd->attach_lock) {
> + attach->hpa_range = (struct range) {
> + .start = cxlr->params.res->start,
> + .end = cxlr->params.res->end,
> + };
> + cxlmd->attach_region = cxlr;
> + }
> return 0;
> }
> EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_region, "cxl_mem");
>
> +void cxl_memdev_attach_region_lost(struct cxl_memdev *cxlmd,
> + struct cxl_region *cxlr)
> +{
> + struct cxl_attach_region *attach =
> + container_of(cxlmd->attach, typeof(*attach), attach);
> +
> + guard(mutex)(&cxlmd->attach_lock);
> + if (cxlmd->attach_region != cxlr)
> + return;
> + cxlmd->attach_region = NULL;
> +
> + dev_dbg(&cxlmd->dev, "%s going away, revoking %pra\n",
> + dev_name(&cxlr->dev), &attach->hpa_range);
> + if (attach->revoke)
> + attach->revoke(cxlmd->cxlds);
> +}
> +EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_region_lost, "cxl_mem");
> +
> +void cxl_memdev_attach_disarm(struct cxl_memdev *cxlmd)
> +{
> + guard(mutex)(&cxlmd->attach_lock);
> + cxlmd->attach_region = NULL;
> +}
> +EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_disarm, "cxl_mem");
> +
> /*
> * The presence of an attach method indicates that the region is designated for
> * a purpose outside of CXL core memory expansion defaults.
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index 35268d1c387c..675d7e21f6c1 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> @@ -44,6 +44,10 @@
> * @cxl_nvd: optional bridge to an nvdimm if the device supports pmem
> * @endpoint: connection to the CXL port topology for this memory device
> * @attach: creator of this memdev depends on CXL link attach to operate
> + * @attach_lock: protects @attach_region and serializes revoking it
> + * @attach_region: region the @attach owner uses, NULL once that region goes
> + * away or the owner unbinds. Lives with the memdev, not the
> + * owner's devm state, so a memdev reference is enough to read it.
> * @id: id number of this memdev instance.
> * @depth: endpoint port depth
> * @scrub_cycle: current scrub cycle set for this device
> @@ -61,6 +65,8 @@ struct cxl_memdev {
> struct cxl_nvdimm *cxl_nvd;
> struct cxl_port *endpoint;
> const struct cxl_memdev_attach *attach;
> + struct mutex attach_lock; /* protects @attach_region */
> + struct cxl_region *attach_region;
> int id;
> int depth;
> u8 scrub_cycle;
> @@ -97,14 +103,30 @@ static inline bool is_cxl_endpoint(struct cxl_port *port)
> return is_cxl_memdev(port->uport_dev);
> }
>
> +/**
> + * struct cxl_memdev_attach - memdev attach provider operations
> + * @probe: attach the memdev, runs in cxl_mem_probe() context
> + * @region_lost: @cxlr is being unregistered and @cxlmd is one of its
> + * targets. Called with the root decoder regions_lock held, after
> + * the region device is deleted and before its decoders are torn
> + * down.
> + * @disarm: the attach owner is unbinding; no further @region_lost
> + * notifications are wanted. Waits for an in-flight one to finish.
> + */
> struct cxl_memdev_attach {
> int (*probe)(struct cxl_memdev *cxlmd);
> + void (*region_lost)(struct cxl_memdev *cxlmd, struct cxl_region *cxlr);
> + void (*disarm)(struct cxl_memdev *cxlmd);
> };
>
> /**
> * struct cxl_attach_region - coordinate mapping a region at memdev registration
> * @attach: common core attachment descriptor
> * @hpa_range: physical address range of the region
> + * @revoke: driver callback, stop all access to @hpa_range
> + * @unbind_nb: bus notifier, disarms before the driver's remove()
> + * @host: device whose unbind disarms @revoke (the cxlds->dev)
> + * @cxlmd: the memdev, set once devm_cxl_probe_mem() succeeds
> *
> * For the common simple case of a CXL device with private (non-general purpose
> * / "accelerator") memory, enumerate firmware instantiated region, or
> @@ -113,15 +135,25 @@ struct cxl_memdev_attach {
> struct cxl_attach_region {
> struct cxl_memdev_attach attach;
> struct range hpa_range;
> + void (*revoke)(struct cxl_dev_state *cxlds);
> + struct notifier_block unbind_nb;
> + struct device *host;
> + struct cxl_memdev *cxlmd;
> };
>
> #ifdef CONFIG_CXL_REGION
> int cxl_memdev_attach_region(struct cxl_memdev *cxlmd);
> +void cxl_memdev_attach_region_lost(struct cxl_memdev *cxlmd,
> + struct cxl_region *cxlr);
> +void cxl_memdev_attach_disarm(struct cxl_memdev *cxlmd);
> #else
> static inline int cxl_memdev_attach_region(struct cxl_memdev *cxlmd)
> {
> return -EOPNOTSUPP;
> }
> +
> +#define cxl_memdev_attach_region_lost NULL
> +#define cxl_memdev_attach_disarm NULL
> #endif
>
> struct cxl_memdev *devm_cxl_add_classdev(struct cxl_dev_state *cxlds);
> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
> index 702ae9ebdcde..cd3db7646c64 100644
> --- a/drivers/cxl/mem.c
> +++ b/drivers/cxl/mem.c
> @@ -219,10 +219,39 @@ struct cxl_memdev *devm_cxl_add_classdev(struct cxl_dev_state *cxlds)
> }
> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_classdev, "CXL");
>
> +static int cxl_attach_unbind_notify(struct notifier_block *nb,
> + unsigned long action, void *data)
> +{
> + struct cxl_attach_region *attach =
> + container_of(nb, struct cxl_attach_region, unbind_nb);
> +
> + if (data != attach->host)
> + return NOTIFY_DONE;
> + if (action != BUS_NOTIFY_UNBIND_DRIVER &&
> + action != BUS_NOTIFY_DRIVER_NOT_BOUND)
> + return NOTIFY_DONE;
> +
> + /*
> + * Sent before remove(), or before devres unwind on probe failure:
> + * wait out an in-flight revoke, block new ones.
> + */
> + if (attach->cxlmd)
> + cxl_memdev_attach_disarm(attach->cxlmd);
> + return NOTIFY_OK;
> +}
> +
> +static void cxl_attach_unregister_notifier(void *data)
> +{
> + struct cxl_attach_region *attach = data;
> +
> + bus_unregister_notifier(attach->host->bus, &attach->unbind_nb);
> +}
> +
> /**
> * devm_cxl_probe_mem - Add a CXL memory device and probe its region
> * @cxlds: CXL device state to associate with the memdev
> * @hpa_range: CXL.mem physical address range result
> + * @revoke: optional callback to stop HPA access before the region goes away
> *
> * Upon return the device will have had a chance to attach to the
> * cxl_mem driver, but may fail to attach if the CXL topology is not ready
> @@ -233,12 +262,29 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_classdev, "CXL");
> *
> * The parent of the resulting device and the devm context for allocations is
> * @cxlds->dev.
> + *
> + * @revoke, if set, is called once if the region backing @hpa_range is about
> + * to be torn down for a reason other than @cxlds->dev unbinding (CXL port,
> + * memdev or region unbind / delete, cxl_acpi removal). The region, its
> + * decoders and the HPA are still intact when it runs. On return the driver
> + * must no longer access @hpa_range. The driver is unbound later,
> + * asynchronously, as before.
> + *
> + * @revoke may run before this function returns, so it needs to synchronize
> + * with the rest of the driver's probe. It never runs concurrently with or
> + * after the driver's remove() or its probe failure unwind. The driver core
> + * notification for either waits for an in-flight @revoke and blocks new
> + * ones first. @revoke is called with CXL core locks held, so it must not call
> + * back into the CXL core nor take the device lock of @cxlds->dev. It may
> + * sleep.
> */
> struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
> - struct range *hpa_range)
> + struct range *hpa_range,
> + void (*revoke)(struct cxl_dev_state *cxlds))
> {
> struct cxl_attach_region *attach = devm_kmalloc_obj(cxlds->dev, *attach);
> struct cxl_memdev *cxlmd;
> + int rc;
>
> if (!attach)
> return ERR_PTR(-ENOMEM);
> @@ -246,12 +292,35 @@ struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
> *attach = (struct cxl_attach_region) {
> .attach = {
> .probe = cxl_memdev_attach_region,
> + .region_lost = cxl_memdev_attach_region_lost,
> + .disarm = cxl_memdev_attach_disarm,
> },
> .hpa_range = { 0, -1 },
> + .revoke = revoke,
> + .unbind_nb.notifier_call = cxl_attach_unbind_notify,
> + .host = cxlds->dev,
> };
> + if (revoke) {
> + rc = bus_register_notifier(cxlds->dev->bus, &attach->unbind_nb);
> + if (rc)
> + return ERR_PTR(rc);
> + rc = devm_add_action_or_reset(cxlds->dev,
> + cxl_attach_unregister_notifier,
> + attach);
> + if (rc)
> + return ERR_PTR(rc);
> + }
>
> cxlmd = __devm_cxl_add_memdev(cxlds, &attach->attach);
> + if (IS_ERR(cxlmd))
> + return cxlmd;
> +
> + /* Don't hand out a range whose region already went away */
> + guard(mutex)(&cxlmd->attach_lock);
> + if (!cxlmd->attach_region)
> + return ERR_PTR(-ENXIO);
> *hpa_range = attach->hpa_range;
> + attach->cxlmd = cxlmd;
> return cxlmd;
> }
> EXPORT_SYMBOL_NS_GPL(devm_cxl_probe_mem, "CXL");
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index 348d7404cd7a..a82a0e2fc609 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -74,7 +74,7 @@ int efx_cxl_init(struct efx_probe_data *probe_data)
> return -ENODEV;
> }
>
> - cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &cxl_pio_range);
> + cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &cxl_pio_range, NULL);
> if (IS_ERR(cxl->cxlmd)) {
> pci_err(pci_dev, "CXL accel memdev creation failed\n");
> return PTR_ERR(cxl->cxlmd);
> diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
> index 1c04a7823f64..c672ec83d820 100644
> --- a/include/cxl/cxl.h
> +++ b/include/cxl/cxl.h
> @@ -252,7 +252,8 @@ struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
> })
>
> struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
> - struct range *range);
> + struct range *range,
> + void (*revoke)(struct cxl_dev_state *cxlds));
>
> int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity);
> #endif /* __CXL_CXL_H__ */
^ permalink raw reply related [flat|nested] 10+ messages in thread
end of thread, other threads:[~2026-11-19 10:00 UTC | newest]
Thread overview: 10+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-11-18 19:14 [RFC 0/3] Type2 unwinding ordering change alucerop
2026-11-18 19:14 ` [RFC 1/3] cxl: add tracepoints to the unwinding code path alucerop
2026-11-18 18:00 ` sashiko-bot
2026-11-18 19:14 ` [RFC 2/3] cxl: make explicit removal of endpoint and decoder for Type2 alucerop
2026-11-18 18:09 ` sashiko-bot
2026-11-18 19:14 ` [RFC 3/3] Documentation/cxl: add document about Type2 binding and unbinding alucerop
2026-11-18 18:04 ` sashiko-bot
2026-11-18 23:39 ` [RFC 0/3] Type2 unwinding ordering change Dave Jiang
2026-11-19 8:13 ` Lucero Palau, Alejandro
2026-11-19 10:00 ` Lucero Palau, Alejandro
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox