From: Richard Cheng <icheng@nvidia.com>
To: Srirangan Madhavan <smadhavan@nvidia.com>
Cc: Alison Schofield <alison.schofield@intel.com>,
Bjorn Helgaas <bhelgaas@google.com>,
Dan Williams <djbw@kernel.org>,
Dave Jiang <dave.jiang@intel.com>,
Davidlohr Bueso <dave@stgolabs.net>,
Ira Weiny <ira.weiny@intel.com>,
Jonathan Cameron <jic23@kernel.org>,
Vishal Verma <vishal.l.verma@intel.com>,
linux-cxl@vger.kernel.org, linux-pci@vger.kernel.org,
linux-kernel@vger.kernel.org,
Alex Williamson <alex.williamson@redhat.com>,
vsethi@nvidia.com, alwilliamson@nvidia.com,
Sai Yashwanth Reddy Kancherla <skancherla@nvidia.com>,
Vishal Aslot <vaslot@nvidia.com>,
Manish Honap <mhonap@nvidia.com>, Jiandi An <jan@nvidia.com>,
linux-tegra@vger.kernel.org
Subject: Re: [PATCH v11 07/12] cxl: Validate HDM ranges before CXL reset
Date: Fri, 4 Sep 2026 17:19:38 +0800 [thread overview]
Message-ID: <apqMXYyHnu_jD4EK@MWDK4CY14F> (raw)
In-Reply-To: <20260902072804.665639-8-smadhavan@nvidia.com>
On Wed, Sep 02, 2026 at 07:27:59AM +0800, Srirangan Madhavan wrote:
> Before reset, require cached HDM decoder state, collect enabled decoder
> ranges, and reserve them with request_mem_region(). This rejects reset
> while affected CXL memory is busy and keeps the validation stable
> through reset.
>
> If CPU cache invalidation support is available, invalidate the affected
> ranges before reset. If the runtime backend is unavailable, continue
> after the range reservation succeeds.
>
> Reject CXL Reset when no cached HDM decoder state is available. The reset
> path needs the cached address map to validate affected ranges and perform
> CPU cache invalidation. Also reject normalized-addressing decoders for
> now because the cached decoder range is not a system physical address.
>
> Signed-off-by: Srirangan Madhavan <smadhavan@nvidia.com>
> ---
> drivers/cxl/core/resource.c | 255 +++++++++++++++++++++++++++++++++++-
> 1 file changed, 254 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/cxl/core/resource.c b/drivers/cxl/core/resource.c
> index 439071a5c34d..13bd0c3faf7e 100644
> --- a/drivers/cxl/core/resource.c
> +++ b/drivers/cxl/core/resource.c
> @@ -11,6 +11,8 @@
> #include <linux/iommu.h>
> #include <linux/jiffies.h>
> #include <linux/kernel.h>
> +#include <linux/list.h>
> +#include <linux/memregion.h>
> #include <linux/pci.h>
> #include <linux/slab.h>
>
> @@ -514,6 +516,229 @@ static const u32 cxl_reset_timeout_ms[] = {
> #define CXL_CACHE_WBI_TIMEOUT_US 100000
> #define CXL_CACHE_WBI_POLL_US 100
>
> +struct cxl_hdm_range {
> + struct list_head list;
> + struct pci_dev *pdev;
> + struct range hpa_range;
> + u64 len;
> + struct resource *res;
> +};
> +
> +struct cxl_hdm_range_context {
> + struct list_head ranges;
> +};
> +
> +static void cxl_hdm_range_context_init(struct cxl_hdm_range_context *ctx)
> +{
> + INIT_LIST_HEAD(&ctx->ranges);
> +}
> +
> +static void cxl_hdm_range_context_destroy(struct cxl_hdm_range_context *ctx)
> +{
> + struct cxl_hdm_range *range, *next;
> +
> + list_for_each_entry_safe(range, next, &ctx->ranges, list) {
> + list_del(&range->list);
> + if (range->res)
> + release_mem_region(range->hpa_range.start,
> + resource_size(range->res));
> + kfree(range);
> + }
> +}
> +
> +/*
> + * Bound the range twice: request_mem_region() takes resource_size_t while
> + * cpu_cache_invalidate_memregion() takes size_t, and the two differ on
> + * 32-bit builds with CONFIG_PHYS_ADDR_T_64BIT. range_len() can also reach
> + * RESOURCE_SIZE_MAX + 1 for a full-width range, and wraps to zero when
> + * resource_size_t is 64-bit, which the !len test catches.
> + */
> +static int cxl_hdm_range_validate(struct pci_dev *pdev,
> + const struct range *hpa_range)
> +{
> + u64 len = range_len(hpa_range);
> +
> + if (!len)
> + return -EINVAL;
> +
> + if (hpa_range->end > RESOURCE_SIZE_MAX) {
> + pci_err(pdev,
> + "CXL reset range [%#llx-%#llx] exceeds resource address size\n",
> + hpa_range->start, hpa_range->end);
> + return -EOVERFLOW;
> + }
> +
> + if (len > RESOURCE_SIZE_MAX) {
> + pci_err(pdev,
> + "CXL reset range [%#llx-%#llx] exceeds resource size\n",
> + hpa_range->start, hpa_range->end);
> + return -EOVERFLOW;
> + }
> +
> + if (len > SIZE_MAX) {
> + pci_err(pdev,
> + "CXL reset range [%#llx-%#llx] exceeds cache flush size\n",
> + hpa_range->start, hpa_range->end);
> + return -EOVERFLOW;
> + }
> +
> + return 0;
> +}
> +
> +static int cxl_hdm_range_add(struct cxl_hdm_range_context *ctx,
> + struct pci_dev *pdev, const struct range *hpa_range)
> +{
> + struct cxl_hdm_range *range;
> + int rc;
> +
> + rc = cxl_hdm_range_validate(pdev, hpa_range);
> + if (rc)
> + return rc;
> +
> + list_for_each_entry(range, &ctx->ranges, list)
> + if (range->hpa_range.start == hpa_range->start &&
> + range->hpa_range.end == hpa_range->end)
> + return 0;
> +
> + range = kzalloc_obj(*range);
> + if (!range)
> + return -ENOMEM;
> +
> + range->pdev = pdev;
> + range->hpa_range = *hpa_range;
> + range->len = range_len(hpa_range);
> + list_add_tail(&range->list, &ctx->ranges);
> +
> + return 0;
> +}
> +
> +static int cxl_hdm_ranges_collect(struct cxl_hdm_range_context *ctx,
> + struct pci_dev *pdev)
> +{
> + struct cxl_hdm_info *info;
> + int rc;
> +
> + guard(rwsem_read)(&cxl_rwsem.dpa);
> + info = pdev->hdm;
> + if (!info) {
> + pci_err(pdev, "CXL HDM decoder state unavailable\n");
> + return -ENXIO;
> + }
> +
> + for (int i = 0; i < info->decoder_count; i++) {
> + struct cxl_decoder_settings *settings = &info->settings[i];
> +
> + if (!(settings->flags & CXL_DECODER_F_ENABLE))
> + continue;
> +
> + /* A committed zero-size decoder maps no HPA. */
> + if (!range_len(&settings->hpa_range))
> + continue;
> +
> + if (settings->flags & CXL_DECODER_F_NORMALIZED_ADDRESSING) {
> + pci_err(pdev,
> + "CXL reset does not support normalized address decoders\n");
> + return -EOPNOTSUPP;
> + }
> +
> + rc = cxl_hdm_range_add(ctx, pdev, &settings->hpa_range);
> + if (rc)
> + return rc;
> + }
> +
> + return 0;
> +}
> +
> +static int cxl_hdm_range_request(struct cxl_hdm_range *range)
> +{
> + struct pci_dev *pdev = range->pdev;
> + const struct range *hpa_range = &range->hpa_range;
> +
> + range->res = request_mem_region(hpa_range->start, range->len,
> + "cxl_reset");
How does this function provide us the required quiescing ?
I do not see it revoking existing PFN/VFIO mapping or preventing a driver
from issueing a new access after cpu_cache_invalidate_memregion()
Want to know what prevents new CXL.mem requests between cache invalidation
and reset completion ?
It does not require the range to be offline or unmapped first ?
Best regards,
Richard Cheng.
> + if (!range->res) {
> + pci_err(pdev,
> + "cannot reset while CXL memory range is busy [%#llx-%#llx]\n",
> + hpa_range->start, hpa_range->end);
> + return -EBUSY;
> + }
> +
> + return 0;
> +}
> +
> +static int cxl_hdm_ranges_request(struct cxl_hdm_range_context *ctx)
> +{
> + struct cxl_hdm_range *range;
> + int rc;
> +
> + lockdep_assert_held_write(&cxl_rwsem.region);
> +
> + list_for_each_entry(range, &ctx->ranges, list) {
> + rc = cxl_hdm_range_request(range);
> + if (rc)
> + return rc;
> + }
> +
> + return 0;
> +}
> +
> +static int cxl_hdm_range_flush_cache(struct cxl_hdm_range *range)
> +{
> + struct pci_dev *pdev = range->pdev;
> + const struct range *hpa_range = &range->hpa_range;
> + int rc;
> +
> + rc = cpu_cache_invalidate_memregion(hpa_range->start, range->len);
> + if (rc)
> + pci_err(pdev,
> + "failed to invalidate CPU cache [%#llx-%#llx]: %d\n",
> + hpa_range->start, hpa_range->end, rc);
> +
> + return rc;
> +}
> +
> +static int cxl_hdm_ranges_flush_cpu_caches(struct cxl_hdm_range_context *ctx,
> + struct pci_dev *pdev)
> +{
> + struct cxl_hdm_range *range;
> + int rc;
> +
> + if (list_empty(&ctx->ranges))
> + return 0;
> +
> + if (!cpu_cache_has_invalidate_memregion()) {
> + pci_warn(pdev,
> + "CPU cache synchronization unavailable; continuing without cache invalidation\n");
> + return 0;
> + }
> +
> + list_for_each_entry(range, &ctx->ranges, list) {
> + rc = cxl_hdm_range_flush_cache(range);
> + if (rc)
> + return rc;
> + }
> +
> + return 0;
> +}
> +
> +static int cxl_hdm_ranges_prepare(struct cxl_hdm_range_context *ctx,
> + struct pci_dev *pdev)
> +{
> + int rc;
> +
> + lockdep_assert_held_write(&cxl_rwsem.region);
> +
> + rc = cxl_hdm_ranges_collect(ctx, pdev);
> + if (rc)
> + return rc;
> +
> + rc = cxl_hdm_ranges_request(ctx);
> + if (rc)
> + return rc;
> +
> + return cxl_hdm_ranges_flush_cpu_caches(ctx, pdev);
> +}
> +
> static int cxl_reset_get_dvsec(struct pci_dev *pdev, u16 *cap_out)
> {
> int dvsec, rc;
> @@ -542,6 +767,20 @@ static int cxl_reset_get_dvsec(struct pci_dev *pdev, u16 *cap_out)
> return dvsec;
> }
>
> +static bool cxl_reset_hdm_available(struct pci_dev *pdev)
> +{
> + struct cxl_hdm_info *info;
> +
> + /*
> + * pdev->hdm is owned by the PCI device and released with pci_dev, so
> + * reset-method probes and reset requests can test availability without
> + * a CXL driver bound to the device.
> + */
> + guard(rwsem_read)(&cxl_rwsem.dpa);
> + info = pdev->hdm;
> + return info && info->hdm_size;
> +}
> +
> #define CXL_RESET_CTRL2_CMD_MASK \
> (PCI_DVSEC_CXL_INIT_CACHE_WBI | PCI_DVSEC_CXL_INIT_CXL_RST)
>
> @@ -731,7 +970,9 @@ static int cxl_reset_execute(struct pci_dev *pdev, int dvsec, u16 cap)
>
> int cxl_reset_function(struct pci_dev *pdev, bool probe)
> {
> + struct cxl_hdm_range_context range_ctx;
> int dvsec;
> + int rc;
> u16 cap;
>
> dvsec = cxl_reset_get_dvsec(pdev, &cap);
> @@ -741,5 +982,17 @@ int cxl_reset_function(struct pci_dev *pdev, bool probe)
> if (probe)
> return 0;
>
> - return cxl_reset_execute(pdev, dvsec, cap);
> + if (!cxl_reset_hdm_available(pdev))
> + return -ENOTTY;
> +
> + cxl_hdm_range_context_init(&range_ctx);
> +
> + scoped_guard(rwsem_write, &cxl_rwsem.region) {
> + rc = cxl_hdm_ranges_prepare(&range_ctx, pdev);
> + if (!rc)
> + rc = cxl_reset_execute(pdev, dvsec, cap);
> + cxl_hdm_range_context_destroy(&range_ctx);
> + }
> +
> + return rc;
> }
> --
> 2.43.0
>
next prev parent reply other threads:[~2026-09-04 9:19 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-02 7:27 [PATCH v11 00/12] PCI/CXL: Add CXL reset support for Type 2 devices Srirangan Madhavan
2026-09-02 7:27 ` [PATCH v11 01/12] cxl: Move HDM decoder programming helpers Srirangan Madhavan
2026-09-02 7:27 ` [PATCH v11 02/12] cxl: Make HDM commit helpers available to reset code Srirangan Madhavan
2026-09-02 7:27 ` [PATCH v11 03/12] cxl: Share HDM decoder decode logic Srirangan Madhavan
2026-09-02 7:27 ` [PATCH v11 04/12] cxl: Cache decoder settings on PCI devices Srirangan Madhavan
2026-09-02 7:27 ` [PATCH v11 05/12] cxl: Cache endpoint decoder settings during PCI enumeration Srirangan Madhavan
2026-09-02 14:03 ` Li Ming
2026-09-02 7:27 ` [PATCH v11 06/12] cxl: Add CXL Device Reset helper Srirangan Madhavan
2026-09-02 7:27 ` [PATCH v11 07/12] cxl: Validate HDM ranges before CXL reset Srirangan Madhavan
2026-09-04 9:19 ` Richard Cheng [this message]
2026-09-02 7:28 ` [PATCH v11 08/12] PCI/CXL: Reject CXL Reset on multifunction devices Srirangan Madhavan
2026-09-04 9:26 ` Richard Cheng
2026-09-02 7:28 ` [PATCH v11 09/12] cxl: Restore CXL HDM state after PCI reset Srirangan Madhavan
2026-09-04 9:23 ` Richard Cheng
2026-09-02 7:28 ` [PATCH v11 10/12] PCI/CXL: Expose CXL Reset as a PCI reset method Srirangan Madhavan
2026-09-02 7:28 ` [PATCH v11 11/12] Documentation/ABI: Document CXL Reset " Srirangan Madhavan
2026-09-02 7:28 ` [PATCH v11 12/12] PCI/CXL: Restore HDM state after CXL bus reset Srirangan Madhavan
2026-09-04 9:15 ` [PATCH v11 00/12] PCI/CXL: Add CXL reset support for Type 2 devices Richard Cheng
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=apqMXYyHnu_jD4EK@MWDK4CY14F \
--to=icheng@nvidia.com \
--cc=alex.williamson@redhat.com \
--cc=alison.schofield@intel.com \
--cc=alwilliamson@nvidia.com \
--cc=bhelgaas@google.com \
--cc=dave.jiang@intel.com \
--cc=dave@stgolabs.net \
--cc=djbw@kernel.org \
--cc=ira.weiny@intel.com \
--cc=jan@nvidia.com \
--cc=jic23@kernel.org \
--cc=linux-cxl@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-pci@vger.kernel.org \
--cc=linux-tegra@vger.kernel.org \
--cc=mhonap@nvidia.com \
--cc=skancherla@nvidia.com \
--cc=smadhavan@nvidia.com \
--cc=vaslot@nvidia.com \
--cc=vishal.l.verma@intel.com \
--cc=vsethi@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox