From: <mhonap@nvidia.com>
To: <alex@shazbot.org>, <jgg@ziepe.ca>, <ankita@nvidia.com>,
<jic23@kernel.org>, <dave.jiang@intel.com>,
<alejandro.lucero-palau@amd.com>, <smadhavan@nvidia.com>,
<corbet@lwn.net>, <skhan@linuxfoundation.org>,
<dave@stgolabs.net>, <alison.schofield@intel.com>,
<vishal.l.verma@intel.com>, <iweiny@kernel.org>,
<ming.li@zohomail.com>, <yishaih@nvidia.com>,
<skolothumtho@nvidia.com>, <kevin.tian@intel.com>,
<bhelgaas@google.com>, <dmatlack@google.com>, <kees@kernel.org>,
<gustavoars@kernel.org>
Cc: <cjia@nvidia.com>, <kjaju@nvidia.com>, <vsethi@nvidia.com>,
<zhiw@nvidia.com>, <mhonap@nvidia.com>,
<linux-doc@vger.kernel.org>, <linux-kernel@vger.kernel.org>,
<kvm@vger.kernel.org>, <linux-cxl@vger.kernel.org>,
<linux-pci@vger.kernel.org>, <linux-kselftest@vger.kernel.org>,
<linux-hardening@vger.kernel.org>
Subject: [PATCH v4 17/27] vfio/cxl: Virtualize the CXL DVSEC
Date: Thu, 13 Aug 2026 15:06:21 +0530 [thread overview]
Message-ID: <20260813093631.2288172-18-mhonap@nvidia.com> (raw)
In-Reply-To: <20260813093631.2288172-1-mhonap@nvidia.com>
From: Manish Honap <mhonap@nvidia.com>
Serve reads of the CXL DVSEC body from the per-open shadow and keep guest
writes in the shadow rather than letting them reach the hardware, so a
guest cannot reprogram the device through the DVSEC. Accesses outside the
CXL DVSEC return -ENODEV and take the default DVSEC handling, so a device
that also exposes a vendor DVSEC is unaffected.
Route each shadow write through the CXL r4.0 field class rather than
storing it verbatim: Control stays programmable, Status is
write-1-to-clear, and Capability, Lock and the Range registers keep their
firmware snapshot. The guest can no longer set Config Lock or scribble the
capability and range fields.
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
drivers/vfio/pci/cxl/vfio_cxl_core.c | 88 ++++++++++++++++++++++++++++
drivers/vfio/pci/vfio_pci_config.c | 36 +++++++++++-
include/linux/vfio_pci_core.h | 5 ++
include/uapi/linux/pci_regs.h | 1 +
4 files changed, 129 insertions(+), 1 deletion(-)
diff --git a/drivers/vfio/pci/cxl/vfio_cxl_core.c b/drivers/vfio/pci/cxl/vfio_cxl_core.c
index 2e516a0929c6..9fed909cb9d3 100644
--- a/drivers/vfio/pci/cxl/vfio_cxl_core.c
+++ b/drivers/vfio/pci/cxl/vfio_cxl_core.c
@@ -148,11 +148,99 @@ static void vfio_cxl_close_device(struct vfio_pci_core_device *vdev)
cxl->dvsec_shadow = NULL;
}
+/* Read a 16-bit DVSEC field from the shadow; @off is DVSEC-relative. */
+static u16 vfio_cxl_dvsec16(struct vfio_cxl_state *cxl, u32 off)
+{
+ u32 dw = cxl->dvsec_shadow[off / sizeof(u32)];
+
+ return (dw >> (8 * (off % sizeof(u32)))) & 0xffff;
+}
+
+/*
+ * Apply the CXL r4.0 8.1.3 write class for the 16-bit DVSEC register at @off.
+ * Control is programmable, Status is write-1-to-clear, and Capability, Lock and
+ * the Range registers stay fixed at their firmware snapshot.
+ */
+static u16 vfio_cxl_dvsec_field(u32 off, u16 old, u16 wval, u16 wmask)
+{
+ switch (off) {
+ case PCI_DVSEC_CXL_CTRL:
+ /*
+ * CXL.mem stays enabled for as long as the guest owns the device.
+ * The HDM decoder maps the guest window to device memory, so a
+ * store to it while CXL.mem is disabled completes on the device as
+ * an error that the host fabric reports as an SError, which is
+ * fatal. The spec does not pin down accesses to a decoder whose
+ * CXL.mem is off and many hosts SError, so ignore a guest request
+ * to clear the enable and keep the bit set.
+ */
+ return ((old & ~wmask) | (wval & wmask)) | PCI_DVSEC_CXL_MEM_ENABLE;
+ case PCI_DVSEC_CXL_CTRL2:
+ return (old & ~wmask) | (wval & wmask);
+ case PCI_DVSEC_CXL_STATUS:
+ case PCI_DVSEC_CXL_STATUS2:
+ return old & ~(wval & wmask);
+ default:
+ return old;
+ }
+}
+
+/* Config accesses never cross a dword, so a single shadow entry covers them. */
+static int vfio_cxl_config_read(struct vfio_pci_core_device *vdev, int pos,
+ int count, __le32 *val)
+{
+ struct vfio_cxl_state *cxl = vdev->cxl;
+ int boff = (pos - cxl->dvsec) % sizeof(u32);
+ __le32 dword;
+
+ if (pos < cxl->dvsec || pos >= cxl->dvsec + cxl->dvsec_len)
+ return -ENODEV;
+
+ dword = cpu_to_le32(cxl->dvsec_shadow[(pos - cxl->dvsec) / sizeof(u32)]);
+ memcpy(val, (u8 *)&dword + boff, count);
+
+ return count;
+}
+
+static int vfio_cxl_config_write(struct vfio_pci_core_device *vdev, int pos,
+ int count, __le32 val)
+{
+ struct vfio_cxl_state *cxl = vdev->cxl;
+ int idx = (pos - cxl->dvsec) / sizeof(u32);
+ int boff = (pos - cxl->dvsec) % sizeof(u32);
+ u32 off = idx * sizeof(u32);
+ __le32 le_wval = 0, le_wmask = 0;
+ u32 old, wval, wmask;
+ u16 lo, hi;
+
+ if (pos < cxl->dvsec || pos >= cxl->dvsec + cxl->dvsec_len)
+ return -ENODEV;
+
+ /*
+ * Place the guest bytes and a matching byte mask at the write offset,
+ * then let the per-field class decide what actually lands in the shadow.
+ * The hardware is never touched.
+ */
+ memcpy((u8 *)&le_wval + boff, &val, count);
+ memset((u8 *)&le_wmask + boff, 0xff, count);
+ old = cxl->dvsec_shadow[idx];
+ wval = le32_to_cpu(le_wval);
+ wmask = le32_to_cpu(le_wmask);
+
+ lo = vfio_cxl_dvsec_field(off, old, wval, wmask);
+ hi = vfio_cxl_dvsec_field(off + 2, old >> 16, wval >> 16, wmask >> 16);
+ cxl->dvsec_shadow[idx] = lo | ((u32)hi << 16);
+
+ return count;
+}
+
static const struct vfio_cxl_ops vfio_cxl_ops = {
.init_device = vfio_cxl_init_device,
.release_device = vfio_cxl_release_device,
.open_device = vfio_cxl_open_device,
.close_device = vfio_cxl_close_device,
+ .config_read = vfio_cxl_config_read,
+ .config_write = vfio_cxl_config_write,
.owner = THIS_MODULE,
};
diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c
index a10ed733f0e3..f088e4ce5e07 100644
--- a/drivers/vfio/pci/vfio_pci_config.c
+++ b/drivers/vfio/pci/vfio_pci_config.c
@@ -1102,6 +1102,39 @@ void vfio_pci_uninit_perm_bits(void)
free_perm_bits(&ecap_perms[PCI_EXT_CAP_ID_PWR]);
}
+/*
+ * DVSEC accesses are dispatched to vfio-cxl for a CXL device. It handles the
+ * CXL DVSEC and returns -ENODEV for any other DVSEC, for which the default
+ * handling then applies.
+ */
+static int vfio_pci_dvsec_config_read(struct vfio_pci_core_device *vdev, int pos,
+ int count, struct perm_bits *perm,
+ int offset, __le32 *val)
+{
+ if (vdev->cxl_ops && vdev->cxl_ops->config_read) {
+ int ret = vdev->cxl_ops->config_read(vdev, pos, count, val);
+
+ if (ret != -ENODEV)
+ return ret;
+ }
+
+ return vfio_direct_config_read(vdev, pos, count, perm, offset, val);
+}
+
+static int vfio_pci_dvsec_config_write(struct vfio_pci_core_device *vdev,
+ int pos, int count, struct perm_bits *perm,
+ int offset, __le32 val)
+{
+ if (vdev->cxl_ops && vdev->cxl_ops->config_write) {
+ int ret = vdev->cxl_ops->config_write(vdev, pos, count, val);
+
+ if (ret != -ENODEV)
+ return ret;
+ }
+
+ return vfio_raw_config_write(vdev, pos, count, perm, offset, val);
+}
+
int __init vfio_pci_init_perm_bits(void)
{
int ret;
@@ -1121,7 +1154,8 @@ int __init vfio_pci_init_perm_bits(void)
ret |= init_pci_ext_cap_err_perm(&ecap_perms[PCI_EXT_CAP_ID_ERR]);
ret |= init_pci_ext_cap_pwr_perm(&ecap_perms[PCI_EXT_CAP_ID_PWR]);
ecap_perms[PCI_EXT_CAP_ID_VNDR].writefn = vfio_raw_config_write;
- ecap_perms[PCI_EXT_CAP_ID_DVSEC].writefn = vfio_raw_config_write;
+ ecap_perms[PCI_EXT_CAP_ID_DVSEC].readfn = vfio_pci_dvsec_config_read;
+ ecap_perms[PCI_EXT_CAP_ID_DVSEC].writefn = vfio_pci_dvsec_config_write;
if (ret)
vfio_pci_uninit_perm_bits();
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 7354dae1dd85..b9202cb29d96 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -72,6 +72,11 @@ struct vfio_cxl_ops {
void (*release_device)(struct vfio_pci_core_device *vdev);
int (*open_device)(struct vfio_pci_core_device *vdev);
void (*close_device)(struct vfio_pci_core_device *vdev);
+ int (*config_read)(struct vfio_pci_core_device *vdev, int pos,
+ int count, __le32 *val);
+ int (*config_write)(struct vfio_pci_core_device *vdev, int pos,
+ int count, __le32 val);
+
/* Pinned per bound CXL device so vfio-cxl cannot unload under usage */
struct module *owner;
};
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
index 4f5942ebd29b..00ca1c777b44 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -1359,6 +1359,7 @@
#define PCI_DVSEC_CXL_CTRL 0xC
#define PCI_DVSEC_CXL_CACHE_ENABLE _BITUL(0)
#define PCI_DVSEC_CXL_MEM_ENABLE _BITUL(2)
+#define PCI_DVSEC_CXL_STATUS 0xE
#define PCI_DVSEC_CXL_CTRL2 0x10
#define PCI_DVSEC_CXL_DISABLE_CACHING _BITUL(0)
#define PCI_DVSEC_CXL_INIT_CACHE_WBI _BITUL(1)
--
2.25.1
next prev parent reply other threads:[~2026-08-13 9:40 UTC|newest]
Thread overview: 28+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-08-13 9:36 [PATCH v4 00/27] vfio/pci: Add CXL Type-2 device passthrough support mhonap
2026-08-13 9:36 ` [PATCH v4 01/27] cxl: Fix resource.c include path and export cxl_restore_hdm_after_pci_reset mhonap
2026-08-13 9:36 ` [PATCH v4 02/27] cxl/regs: Skip sub-block region request for BAR-owning drivers mhonap
2026-08-13 9:36 ` [PATCH v4 03/27] cxl: Move component register defines to uapi/cxl/cxl_regs.h mhonap
2026-08-13 9:36 ` [PATCH v4 04/27] cxl: Establish media readiness in cxl_mem_probe() mhonap
2026-08-13 9:36 ` [PATCH v4 05/27] cxl: Add a function-scoped reset entry for vfio-pci mhonap
2026-08-13 9:36 ` [PATCH v4 06/27] vfio/pci: Add CXL ops registration interface mhonap
2026-08-13 9:36 ` [PATCH v4 07/27] vfio/pci: Detect CXL devices and load vfio-cxl on demand mhonap
2026-08-13 9:36 ` [PATCH v4 08/27] vfio/cxl: Add the vfio-cxl module skeleton mhonap
2026-08-13 9:36 ` [PATCH v4 09/27] vfio/cxl: Create the CXL memory device at bind mhonap
2026-08-13 9:36 ` [PATCH v4 10/27] vfio/cxl: Reject unsupported decoder topologies " mhonap
2026-08-13 9:36 ` [PATCH v4 11/27] vfio/cxl: Own the whole component register BAR mhonap
2026-08-13 9:36 ` [PATCH v4 12/27] vfio/pci: Let a provider exclude a BAR sub-range from mmap mhonap
2026-08-13 9:36 ` [PATCH v4 13/27] vfio/pci: Refuse read/write to an excluded BAR sub-range mhonap
2026-08-13 9:36 ` [PATCH v4 14/27] vfio: Add CXL region type for the HDM region mhonap
2026-08-13 9:36 ` [PATCH v4 15/27] vfio/pci: Call CXL open and close hooks around device use mhonap
2026-08-13 9:36 ` [PATCH v4 16/27] vfio/cxl: Shadow the CXL DVSEC body at open mhonap
2026-08-13 9:36 ` mhonap [this message]
2026-08-13 9:36 ` [PATCH v4 18/27] vfio/cxl: Expose the HDM memory and trap the decoder registers mhonap
2026-08-13 9:36 ` [PATCH v4 19/27] vfio/cxl: Keep the HDM decoder block off the direct BAR mapping mhonap
2026-08-13 9:36 ` [PATCH v4 20/27] vfio/cxl: Emulate the HDM decoder commit handshake mhonap
2026-08-13 9:36 ` [PATCH v4 21/27] vfio/cxl: Describe the CXL device and decoder geometry to userspace mhonap
2026-08-13 9:36 ` [PATCH v4 22/27] vfio/cxl: Revoke the HDM mapping on reset and power transitions mhonap
2026-08-13 9:36 ` [PATCH v4 23/27] vfio/cxl: Refresh the decoder snapshot after a device reset mhonap
2026-08-13 9:36 ` [PATCH v4 24/27] vfio/cxl: Service a guest-triggered CXL reset mhonap
2026-08-13 9:36 ` [PATCH v4 25/27] vfio/pci: Provide an opt-out for the CXL Type-2 extensions mhonap
2026-08-13 9:36 ` [PATCH v4 26/27] Documentation: vfio-pci: Document CXL Type-2 device passthrough mhonap
2026-08-13 9:36 ` [PATCH v4 27/27] selftests/vfio: Add CXL Type-2 passthrough corner-case tests mhonap
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260813093631.2288172-18-mhonap@nvidia.com \
--to=mhonap@nvidia.com \
--cc=alejandro.lucero-palau@amd.com \
--cc=alex@shazbot.org \
--cc=alison.schofield@intel.com \
--cc=ankita@nvidia.com \
--cc=bhelgaas@google.com \
--cc=cjia@nvidia.com \
--cc=corbet@lwn.net \
--cc=dave.jiang@intel.com \
--cc=dave@stgolabs.net \
--cc=dmatlack@google.com \
--cc=gustavoars@kernel.org \
--cc=iweiny@kernel.org \
--cc=jgg@ziepe.ca \
--cc=jic23@kernel.org \
--cc=kees@kernel.org \
--cc=kevin.tian@intel.com \
--cc=kjaju@nvidia.com \
--cc=kvm@vger.kernel.org \
--cc=linux-cxl@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-hardening@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=linux-pci@vger.kernel.org \
--cc=ming.li@zohomail.com \
--cc=skhan@linuxfoundation.org \
--cc=skolothumtho@nvidia.com \
--cc=smadhavan@nvidia.com \
--cc=vishal.l.verma@intel.com \
--cc=vsethi@nvidia.com \
--cc=yishaih@nvidia.com \
--cc=zhiw@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox