[RFC PATCH v4 27/27] cxl: add cxl_compression PCI driver

Gregory Price
2026-02-22T08:50:38Z

Add a generic CXL type-3 driver for compressed memory controllers.

The driver provides an alternative PCI binding that converts CXL
RAM regions to private-node sysram and registers them with the
CRAM subsystem for transparent demotion/promotion.

Probe flow:
1. cxl_pci_type3_probe_init() for standard CXL device setup
2. Discover/convert auto-RAM regions or create a RAM region
3. Convert to private-node sysram via devm_cxl_add_sysram()
4. Register with CRAM via cram_register_private_node()

Page flush pipeline:
When a CRAM folio is freed, the CRAM free_folio callback buffers
it into a per-CPU RCU-protected flush buffer to offload the operation.

A periodic kthread swaps the per-CPU buffers under RCU, then sends
batched Sanitize-Zero commands so the device can zero pages.

A flush_record bitmap tracks in-flight pages to avoid re-buffering on
the second free_folio entry after folio_put().

Overflow from full buffers is handled by a per-CPU workqueue fallback.

Watermark interrupts:
MSI-X vector 12 - delivers "Low" watermark interrupts
MSI-X vector 13 - delivers "High" watermark interrupts
This adjusts CRAM pressure:
Low - increases pressure.
High - reduces pressure.

A dynamic watermark mode cycles through four phases with
progressively tighter thresholds.

Static watermark mode sets pressure 0 or MAX respectively.

Teardown ordering:
pre_teardown - cram_unregister + retry-loop memory offline
post_teardown - kthread stop, drain all flush buffers via CCI

Usage:
echo $PCI_DEV > /sys/bus/pci/drivers/cxl_pci/unbind
echo $PCI_DEV > /sys/bus/pci/drivers/cxl_compression/bind

Signed-off-by: Gregory Price <gourry@gourry.net>
---
drivers/cxl/type3_drivers/Kconfig | 1 +
drivers/cxl/type3_drivers/Makefile | 1 +
.../cxl/type3_drivers/cxl_compression/Kconfig | 20 +
.../type3_drivers/cxl_compression/Makefile | 4 +
.../cxl_compression/compression.c | 1025 +++++++++++++++++
5 files changed, 1051 insertions(+)
create mode 100644 drivers/cxl/type3_drivers/cxl_compression/Kconfig
create mode 100644 drivers/cxl/type3_drivers/cxl_compression/Makefile
create mode 100644 drivers/cxl/type3_drivers/cxl_compression/compression.c

diff --git a/drivers/cxl/type3_drivers/Kconfig b/drivers/cxl/type3_drivers/Kconfig
index 369b21763856..98f73e46730e 100644
--- a/drivers/cxl/type3_drivers/Kconfig
+++ b/drivers/cxl/type3_drivers/Kconfig
@@ -1,2 +1,3 @@
# SPDX-License-Identifier: GPL-2.0
source "drivers/cxl/type3_drivers/cxl_mempolicy/Kconfig"
+source "drivers/cxl/type3_drivers/cxl_compression/Kconfig"
diff --git a/drivers/cxl/type3_drivers/Makefile b/drivers/cxl/type3_drivers/Makefile
index 2b82265ff118..f5b0766d92af 100644
--- a/drivers/cxl/type3_drivers/Makefile
+++ b/drivers/cxl/type3_drivers/Makefile
@@ -1,2 +1,3 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CXL_MEMPOLICY) += cxl_mempolicy/
+obj-$(CONFIG_CXL_COMPRESSION) += cxl_compression/
diff --git a/drivers/cxl/type3_drivers/cxl_compression/Kconfig b/drivers/cxl/type3_drivers/cxl_compression/Kconfig
new file mode 100644
index 000000000000..8c891a48b000
--- /dev/null
+++ b/drivers/cxl/type3_drivers/cxl_compression/Kconfig
@@ -0,0 +1,20 @@
+config CXL_COMPRESSION
+ tristate "CXL Compression Memory Driver"
+ depends on CXL_PCI
+ depends on CXL_REGION
+ depends on CRAM
+ help
+ This driver provides an alternative PCI binding for CXL memory
+ devices with compressed memory support. It converts CXL RAM
+ regions to sysram for direct memory hotplug and registers with
+ the CRAM subsystem for transparent compression.
+
+ Page reclamation uses the standard CXL Media Operations Zero
+ command (opcode 0x4402). If the device does not support it,
+ the driver falls back to inline CPU zeroing.
+
+ Usage: First unbind the device from cxl_pci, then bind to
+ cxl_compression. The driver will initialize the CXL device and
+ convert any RAM regions to use direct memory hotplug via sysram.
+
+ If unsure say 'n'.
diff --git a/drivers/cxl/type3_drivers/cxl_compression/Makefile b/drivers/cxl/type3_drivers/cxl_compression/Makefile
new file mode 100644
index 000000000000..46f34809bf74
--- /dev/null
+++ b/drivers/cxl/type3_drivers/cxl_compression/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CXL_COMPRESSION) += cxl_compression.o
+cxl_compression-y := compression.o
+ccflags-y += -I$(srctree)/drivers/cxl
diff --git a/drivers/cxl/type3_drivers/cxl_compression/compression.c b/drivers/cxl/type3_drivers/cxl_compression/compression.c
new file mode 100644
index 000000000000..e4c8b62227e2
--- /dev/null
+++ b/drivers/cxl/type3_drivers/cxl_compression/compression.c
@@ -0,0 +1,1025 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2026 Meta Platforms, Inc. All rights reserved. */
+/*
+ * CXL Compression Driver
+ *
+ * This driver provides an alternative binding for CXL memory devices that
+ * converts all associated RAM regions to sysram_regions for direct memory
+ * hotplug, bypassing the standard dax region path.
+ *
+ * Page reclamation uses the standard CXL Media Operations Zero command
+ * (opcode 0x4402, class 0x01, subclass 0x01). Watermark interrupts
+ * are delivered via separate MSI-X vectors (12 for lthresh, 13 for
+ * hthresh), injected externally via QMP.
+ *
+ * Usage:
+ * 1. Device initially binds to cxl_pci at boot
+ * 2. Unbind from cxl_pci:
+ * echo $PCI_DEV > /sys/bus/pci/drivers/cxl_pci/unbind
+ * 3. Bind to cxl_compression:
+ * echo $PCI_DEV > /sys/bus/pci/drivers/cxl_compression/bind
+ */
+
+#include <linux/unaligned.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/sizes.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/pci.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/bitmap.h>
+#include <linux/highmem.h>
+#include <linux/workqueue.h>
+#include <linux/kthread.h>
+#include <linux/rcupdate.h>
+#include <linux/percpu.h>
+#include <linux/sched.h>
+#include <linux/cram.h>
+#include <linux/memory_hotplug.h>
+#include <linux/xarray.h>
+#include <cxl/mailbox.h>
+#include "cxlmem.h"
+#include "cxl.h"
+
+/*
+ * Per-device compression context lookup.
+ *
+ * pci_set_drvdata() MUST store cxlds because mbox_to_cxlds() uses
+ * dev_get_drvdata() to recover the cxl_dev_state from the mailbox host
+ * device. Storing anything else in pci drvdata breaks every CXL mailbox
+ * command. Use an xarray keyed by pci_dev pointer so that multiple
+ * devices can bind concurrently without colliding.
+ */
+static DEFINE_XARRAY(comp_ctx_xa);
+
+static struct cxl_compression_ctx *pdev_to_comp_ctx(struct pci_dev *pdev)
+{
+ return xa_load(&comp_ctx_xa, (unsigned long)pdev);
+}
+
+#define CXL_MEDIA_OP_OPCODE 0x4402
+#define CXL_MEDIA_OP_CLASS_SANITIZE 0x01
+#define CXL_MEDIA_OP_SUBC_ZERO 0x01
+
+struct cxl_dpa_range {
+ __le64 starting_dpa;
+ __le64 length;
+} __packed;
+
+struct cxl_media_op_input {
+ u8 media_operation_class;
+ u8 media_operation_subclass;
+ __le16 reserved;
+ __le32 dpa_range_count;
+ struct cxl_dpa_range ranges[];
+} __packed;
+
+#define CXL_CT3_MSIX_LTHRESH 12
+#define CXL_CT3_MSIX_HTHRESH 13
+#define CXL_CT3_MSIX_VECTOR_NR 14
+#define CXL_FLUSH_INTERVAL_DEFAULT_MS 1000
+
+static unsigned int flush_buf_size;
+module_param(flush_buf_size, uint, 0444);
+MODULE_PARM_DESC(flush_buf_size,
+ "Max DPA ranges per media ops CCI command (0 = use hw max)");
+
+static unsigned int flush_interval_ms = CXL_FLUSH_INTERVAL_DEFAULT_MS;
+module_param(flush_interval_ms, uint, 0644);
+MODULE_PARM_DESC(flush_interval_ms,
+ "Flush worker interval in ms (default 1000)");
+
+struct cxl_flush_buf {
+ unsigned int count;
+ unsigned int max; /* max ranges per command */
+ struct cxl_media_op_input *cmd; /* pre-allocated CCI payload */
+ struct folio **folios; /* parallel folio tracking */
+};
+
+struct cxl_flush_ctx;
+
+struct cxl_pcpu_flush {
+ struct cxl_flush_buf __rcu *active; /* callback writes here */
+ struct cxl_flush_buf *overflow_spare; /* spare for overflow work */
+ struct work_struct overflow_work; /* per-CPU overflow flush */
+ struct cxl_flush_ctx *ctx; /* backpointer */
+};
+
+/**
+ * struct cxl_flush_ctx - Per-region flush context
+ * @flush_record: two-level bitmap, 1 bit per 4KB page, tracks in-flight ops
+ * @flush_record_pages: number of pages in the flush_record array
+ * @nr_pages: total number of 4KB pages in the region
+ * @base_pfn: starting PFN of the region (for DPA offset calculation)
+ * @buf_max: max DPA ranges per CCI command
+ * @media_ops_supported: true if device supports media operations zero
+ * @pcpu: per-CPU flush state
+ * @kthread_spares: array[nr_cpu_ids] of spare buffers for the kthread
+ * @flush_thread: round-robin kthread
+ * @mbox: pointer to CXL mailbox for sending CCI commands
+ * @dev: device for logging
+ * @nid: NUMA node of the private region
+ */
+struct cxl_flush_ctx {
+ unsigned long **flush_record;
+ unsigned int flush_record_pages;
+ unsigned long nr_pages;
+ unsigned long base_pfn;
+ unsigned int buf_max;
+ bool media_ops_supported;
+ struct cxl_pcpu_flush __percpu *pcpu;
+ struct cxl_flush_buf **kthread_spares;
+ struct task_struct *flush_thread;
+ struct cxl_mailbox *mbox;
+ struct device *dev;
+ int nid;
+};
+
+/* Bits per page-sized bitmap chunk */
+#define FLUSH_RECORD_BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE)
+#define FLUSH_RECORD_SHIFT (PAGE_SHIFT + 3)
+
+static unsigned long **flush_record_alloc(unsigned long nr_bits,
+ unsigned int *nr_pages_out)
+{
+ unsigned int nr_pages = DIV_ROUND_UP(nr_bits, FLUSH_RECORD_BITS_PER_PAGE);
+ unsigned long **pages;
+ unsigned int i;
+
+ pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL);
+ if (!pages)
+ return NULL;
+
+ for (i = 0; i < nr_pages; i++) {
+ pages[i] = (unsigned long *)get_zeroed_page(GFP_KERNEL);
+ if (!pages[i])
+ goto err;
+ }
+
+ *nr_pages_out = nr_pages;
+ return pages;
+
+err:
+ while (i--)
+ free_page((unsigned long)pages[i]);
+ kfree(pages);
+ return NULL;
+}
+
+static void flush_record_free(unsigned long **pages, unsigned int nr_pages)
+{
+ unsigned int i;
+
+ if (!pages)
+ return;
+
+ for (i = 0; i < nr_pages; i++)
+ free_page((unsigned long)pages[i]);
+ kfree(pages);
+}
+
+static inline bool flush_record_test_and_clear(unsigned long **pages,
+ unsigned long idx)
+{
+ return test_and_clear_bit(idx & (FLUSH_RECORD_BITS_PER_PAGE - 1),
+ pages[idx >> FLUSH_RECORD_SHIFT]);
+}
+
+static inline void flush_record_set(unsigned long **pages, unsigned long idx)
+{
+ set_bit(idx & (FLUSH_RECORD_BITS_PER_PAGE - 1),
+ pages[idx >> FLUSH_RECORD_SHIFT]);
+}
+
+static struct cxl_flush_buf *cxl_flush_buf_alloc(unsigned int max, int nid)
+{
+ struct cxl_flush_buf *buf;
+
+ buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, nid);
+ if (!buf)
+ return NULL;
+
+ buf->max = max;
+ buf->cmd = kvzalloc_node(struct_size(buf->cmd, ranges, max),
+ GFP_KERNEL, nid);
+ if (!buf->cmd)
+ goto err_cmd;
+
+ buf->folios = kcalloc_node(max, sizeof(struct folio *),
+ GFP_KERNEL, nid);
+ if (!buf->folios)
+ goto err_folios;
+
+ return buf;
+
+err_folios:
+ kvfree(buf->cmd);
+err_cmd:
+ kfree(buf);
+ return NULL;
+}
+
+static void cxl_flush_buf_free(struct cxl_flush_buf *buf)
+{
+ if (!buf)
+ return;
+ kvfree(buf->cmd);
+ kfree(buf->folios);
+ kfree(buf);
+}
+
+static inline void cxl_flush_buf_reset(struct cxl_flush_buf *buf)
+{
+ buf->count = 0;
+}
+
+static void cxl_flush_buf_send(struct cxl_flush_ctx *ctx,
+ struct cxl_flush_buf *buf)
+{
+ struct cxl_mbox_cmd mbox_cmd;
+ unsigned int count = buf->count;
+ unsigned int i;
+ int rc;
+
+ if (count == 0)
+ return;
+
+ if (!ctx->media_ops_supported) {
+ /* No device support, zero all folios inline */
+ for (i = 0; i < count; i++)
+ folio_zero_range(buf->folios[i], 0,
+ folio_size(buf->folios[i]));
+ goto release;
+ }
+
+ buf->cmd->media_operation_class = CXL_MEDIA_OP_CLASS_SANITIZE;
+ buf->cmd->media_operation_subclass = CXL_MEDIA_OP_SUBC_ZERO;
+ buf->cmd->reserved = 0;
+ buf->cmd->dpa_range_count = cpu_to_le32(count);
+
+ mbox_cmd = (struct cxl_mbox_cmd) {
+ .opcode = CXL_MEDIA_OP_OPCODE,
+ .payload_in = buf->cmd,
+ .size_in = struct_size(buf->cmd, ranges, count),
+ .poll_interval_ms = 1000,
+ .poll_count = 30,
+ };
+
+ rc = cxl_internal_send_cmd(ctx->mbox, &mbox_cmd);
+ if (rc) {
+ dev_warn(ctx->dev,
+ "media ops zero CCI command failed: %d\n", rc);
+
+ /* Zero all folios inline on failure */
+ for (i = 0; i < count; i++)
+ folio_zero_range(buf->folios[i], 0,
+ folio_size(buf->folios[i]));
+ }
+
+release:
+ for (i = 0; i < count; i++)
+ folio_put(buf->folios[i]);
+
+ cxl_flush_buf_reset(buf);
+}
+
+static int cxl_compression_flush_cb(struct folio *folio, void *private)
+{
+ struct cxl_flush_ctx *ctx = private;
+ unsigned long pfn = folio_pfn(folio);
+ unsigned long idx = pfn - ctx->base_pfn;
+ unsigned long nr = folio_nr_pages(folio);
+ struct cxl_pcpu_flush *pcpu;
+ struct cxl_flush_buf *buf;
+ unsigned long flags;
+ unsigned int pos;
+
+ /* Case (a): flush record bit set, resolution from our media op */
+ if (flush_record_test_and_clear(ctx->flush_record, idx))
+ return 0;
+
+ dev_dbg_ratelimited(ctx->dev,
+ "flush_cb: folio pfn=%lx order=%u idx=%lu cpu=%d\n",
+ pfn, folio_order(folio), idx,
+ raw_smp_processor_id());
+
+ local_irq_save(flags);
+ rcu_read_lock();
+
+ pcpu = this_cpu_ptr(ctx->pcpu);
+ buf = rcu_dereference(pcpu->active);
+
+ if (unlikely(!buf || buf->count >= buf->max)) {
+ rcu_read_unlock();
+ local_irq_restore(flags);
+ if (buf)
+ schedule_work_on(raw_smp_processor_id(),
+ &pcpu->overflow_work);
+ return 2;
+ }
+
+ /* Case (b): write DPA range directly into pre-formatted CCI buffer */
+ folio_get(folio);
+ flush_record_set(ctx->flush_record, idx);
+
+ pos = buf->count;
+ buf->folios[pos] = folio;
+ buf->cmd->ranges[pos].starting_dpa = cpu_to_le64((u64)idx * PAGE_SIZE);
+ buf->cmd->ranges[pos].length = cpu_to_le64((u64)nr * PAGE_SIZE);
+ buf->count = pos + 1;
+
+ rcu_read_unlock();
+ local_irq_restore(flags);
+
+ return 1;
+}
+
+static int cxl_flush_kthread_fn(void *data)
+{
+ struct cxl_flush_ctx *ctx = data;
+ struct cxl_flush_buf *dirty;
+ struct cxl_pcpu_flush *pcpu;
+ int cpu;
+ bool any_dirty;
+
+ while (!kthread_should_stop()) {
+ any_dirty = false;
+
+ /* Phase 1: Swap all per-CPU buffers */
+ for_each_possible_cpu(cpu) {
+ struct cxl_flush_buf *spare = ctx->kthread_spares[cpu];
+
+ if (!spare)
+ continue;
+
+ pcpu = per_cpu_ptr(ctx->pcpu, cpu);
+ cxl_flush_buf_reset(spare);
+ dirty = rcu_replace_pointer(pcpu->active, spare, true);
+ ctx->kthread_spares[cpu] = dirty;
+
+ if (dirty && dirty->count > 0) {
+ dev_dbg(ctx->dev,
+ "flush_kthread: cpu=%d has %u dirty ranges\n",
+ cpu, dirty->count);
+ any_dirty = true;
+ }
+ }
+
+ if (!any_dirty)
+ goto sleep;
+
+ /* Phase 2: Single synchronize_rcu for all swaps */
+ synchronize_rcu();
+
+ /* Phase 3: Send CCI commands for dirty buffers */
+ for_each_possible_cpu(cpu) {
+ dirty = ctx->kthread_spares[cpu];
+ if (dirty && dirty->count > 0)
+ cxl_flush_buf_send(ctx, dirty);
+ /* dirty is now clean, stays as kthread_spares[cpu] */
+ }
+
+sleep:
+ schedule_timeout_interruptible(
+ msecs_to_jiffies(flush_interval_ms));
+ }
+
+ return 0;
+}
+
+static void cxl_flush_overflow_work(struct work_struct *work)
+{
+ struct cxl_pcpu_flush *pcpu =
+ container_of(work, struct cxl_pcpu_flush, overflow_work);
+ struct cxl_flush_ctx *ctx = pcpu->ctx;
+ struct cxl_flush_buf *dirty, *spare;
+ unsigned long flags;
+
+ dev_dbg(ctx->dev, "flush_overflow: cpu=%d buffer full, flushing\n",
+ raw_smp_processor_id());
+
+ spare = pcpu->overflow_spare;
+ if (!spare)
+ return;
+
+ cxl_flush_buf_reset(spare);
+
+ local_irq_save(flags);
+ dirty = rcu_replace_pointer(pcpu->active, spare, true);
+ local_irq_restore(flags);
+
+ pcpu->overflow_spare = dirty;
+
+ synchronize_rcu();
+ cxl_flush_buf_send(ctx, dirty);
+}
+
+struct cxl_teardown_ctx {
+ struct cxl_flush_ctx *flush_ctx;
+ struct cxl_sysram *sysram;
+ int nid;
+};
+
+static void cxl_compression_pre_teardown(void *data)
+{
+ struct cxl_teardown_ctx *tctx = data;
+
+ if (!tctx->flush_ctx)
+ return;
+
+ /*
+ * Unregister the CRAM node before memory goes offline.
+ * node_private_clear_ops requires the node_private to still
+ * exist, which is destroyed during memory removal.
+ */
+ cram_unregister_private_node(tctx->nid);
+
+ /*
+ * Offline and remove CXL memory with retry. CXL compressed
+ * memory may have pages pinned by in-flight flush operations;
+ * keep retrying until they complete. Once done, sysram->res
+ * is NULL so the devm sysram_unregister action that follows
+ * will skip the hotplug removal.
+ */
+ if (tctx->sysram) {
+ int rc, retries = 0;
+
+ while (true) {
+ rc = cxl_sysram_offline_and_remove(tctx->sysram);
+ if (!rc)
+ break;
+ if (++retries > 60) {
+ pr_err("cxl_compression: memory offline failed after %d retries, giving up\n",
+ retries);
+ break;
+ }
+ pr_info("cxl_compression: memory offline failed (%d), retrying...\n",
+ rc);
+ msleep(1000);
+ }
+ }
+}
+
+static void cxl_compression_post_teardown(void *data)
+{
+ struct cxl_teardown_ctx *tctx = data;
+ struct cxl_flush_ctx *ctx = tctx->flush_ctx;
+ struct cxl_pcpu_flush *pcpu;
+ struct cxl_flush_buf *buf;
+ int cpu;
+
+ if (!ctx)
+ return;
+
+ /* cram_unregister_private_node already called in pre_teardown */
+
+ if (ctx->flush_thread) {
+ kthread_stop(ctx->flush_thread);
+ ctx->flush_thread = NULL;
+ }
+
+ for_each_possible_cpu(cpu) {
+ pcpu = per_cpu_ptr(ctx->pcpu, cpu);
+ cancel_work_sync(&pcpu->overflow_work);
+ }
+
+ for_each_possible_cpu(cpu) {
+ pcpu = per_cpu_ptr(ctx->pcpu, cpu);
+
+ buf = rcu_dereference_raw(pcpu->active);
+ if (buf && buf->count > 0)
+ cxl_flush_buf_send(ctx, buf);
+
+ if (pcpu->overflow_spare && pcpu->overflow_spare->count > 0)
+ cxl_flush_buf_send(ctx, pcpu->overflow_spare);
+
+ if (ctx->kthread_spares && ctx->kthread_spares[cpu]) {
+ buf = ctx->kthread_spares[cpu];
+ if (buf->count > 0)
+ cxl_flush_buf_send(ctx, buf);
+ }
+ }
+
+ for_each_possible_cpu(cpu) {
+ pcpu = per_cpu_ptr(ctx->pcpu, cpu);
+
+ buf = rcu_dereference_raw(pcpu->active);
+ cxl_flush_buf_free(buf);
+
+ cxl_flush_buf_free(pcpu->overflow_spare);
+
+ if (ctx->kthread_spares)
+ cxl_flush_buf_free(ctx->kthread_spares[cpu]);
+ }
+
+ kfree(ctx->kthread_spares);
+ free_percpu(ctx->pcpu);
+ flush_record_free(ctx->flush_record, ctx->flush_record_pages);
+}
+
+/**
+ * struct cxl_compression_ctx - Per-device context for compression driver
+ * @mbox: CXL mailbox for issuing CCI commands
+ * @pdev: PCI device
+ * @flush_ctx: Flush context for deferred page reclamation
+ * @tctx: Teardown context for devm actions
+ * @sysram: Sysram device for offline+remove in remove path
+ * @nid: NUMA node ID, NUMA_NO_NODE if unset
+ * @cxlmd: The memdev associated with this context
+ * @cxlr: Region created by this driver (NULL if pre-existing)
+ * @cxled: Endpoint decoder with DPA allocated by this driver
+ * @regions_converted: Number of regions successfully converted
+ * @media_ops_supported: Device supports media operations zero (0x4402)
+ */
+struct cxl_compression_ctx {
+ struct cxl_mailbox *mbox;
+ struct pci_dev *pdev;
+ struct cxl_flush_ctx *flush_ctx;
+ struct cxl_teardown_ctx *tctx;
+ struct cxl_sysram *sysram;
+ int nid;
+ struct cxl_memdev *cxlmd;
+ struct cxl_region *cxlr;
+ struct cxl_endpoint_decoder *cxled;
+ int regions_converted;
+ bool media_ops_supported;
+};
+
+/*
+ * Probe whether the device supports Media Operations Zero (0x4402).
+ * Send a zero-count command, a conforming device returns SUCCESS,
+ * a device that doesn't support it returns UNSUPPORTED (-ENXIO).
+ */
+static bool cxl_probe_media_ops_zero(struct cxl_mailbox *mbox,
+ struct device *dev)
+{
+ struct cxl_media_op_input probe = {
+ .media_operation_class = CXL_MEDIA_OP_CLASS_SANITIZE,
+ .media_operation_subclass = CXL_MEDIA_OP_SUBC_ZERO,
+ .dpa_range_count = 0,
+ };
+ struct cxl_mbox_cmd cmd = {
+ .opcode = CXL_MEDIA_OP_OPCODE,
+ .payload_in = &probe,
+ .size_in = sizeof(probe),
+ };
+ int rc;
+
+ rc = cxl_internal_send_cmd(mbox, &cmd);
+ if (rc) {
+ dev_info(dev,
+ "media operations zero not supported (rc=%d), using inline zeroing\n",
+ rc);
+ return false;
+ }
+
+ dev_info(dev, "media operations zero (0x4402) supported\n");
+ return true;
+}
+
+struct cxl_compression_wm_ctx {
+ struct device *dev;
+ int nid;
+};
+
+static irqreturn_t cxl_compression_lthresh_irq(int irq, void *data)
+{
+ struct cxl_compression_wm_ctx *wm = data;
+
+ dev_info(wm->dev, "lthresh watermark: pressuring node %d\n", wm->nid);
+ cram_set_pressure(wm->nid, CRAM_PRESSURE_MAX);
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t cxl_compression_hthresh_irq(int irq, void *data)
+{
+ struct cxl_compression_wm_ctx *wm = data;
+
+ dev_info(wm->dev, "hthresh watermark: resuming node %d\n", wm->nid);
+ cram_set_pressure(wm->nid, 0);
+ return IRQ_HANDLED;
+}
+
+static int convert_region_to_sysram(struct cxl_region *cxlr,
+ struct pci_dev *pdev)
+{
+ struct cxl_compression_ctx *comp_ctx = pdev_to_comp_ctx(pdev);
+ struct device *dev = cxl_region_dev(cxlr);
+ struct cxl_compression_wm_ctx *wm_ctx;
+ struct cxl_teardown_ctx *tctx;
+ struct cxl_flush_ctx *flush_ctx;
+ struct cxl_pcpu_flush *pcpu;
+ resource_size_t region_start, region_size;
+ struct range hpa_range;
+ int nid;
+ int irq;
+ int cpu;
+ int rc;
+
+ if (cxl_region_mode(cxlr) != CXL_PARTMODE_RAM) {
+ dev_dbg(dev, "skipping non-RAM region (mode=%d)\n",
+ cxl_region_mode(cxlr));
+ return 0;
+ }
+
+ dev_info(dev, "converting region to sysram\n");
+
+ rc = devm_cxl_add_sysram(cxlr, true, MMOP_ONLINE_MOVABLE);
+ if (rc) {
+ dev_err(dev, "failed to add sysram region: %d\n", rc);
+ return rc;
+ }
+
+ tctx = devm_kzalloc(dev, sizeof(*tctx), GFP_KERNEL);
+ if (!tctx)
+ return -ENOMEM;
+
+ rc = devm_add_action_or_reset(dev, cxl_compression_post_teardown, tctx);
+ if (rc)
+ return rc;
+
+ /* Find the sysram child device for pre_teardown */
+ comp_ctx->sysram = cxl_region_find_sysram(cxlr);
+ if (comp_ctx->sysram)
+ tctx->sysram = comp_ctx->sysram;
+
+ rc = cxl_get_region_range(cxlr, &hpa_range);
+ if (rc) {
+ dev_err(dev, "failed to get region range: %d\n", rc);
+ return rc;
+ }
+
+ nid = phys_to_target_node(hpa_range.start);
+ if (nid == NUMA_NO_NODE)
+ nid = memory_add_physaddr_to_nid(hpa_range.start);
+
+ region_start = hpa_range.start;
+ region_size = range_len(&hpa_range);
+
+ flush_ctx = devm_kzalloc(dev, sizeof(*flush_ctx), GFP_KERNEL);
+ if (!flush_ctx)
+ return -ENOMEM;
+
+ flush_ctx->base_pfn = PHYS_PFN(region_start);
+ flush_ctx->nr_pages = region_size >> PAGE_SHIFT;
+ flush_ctx->flush_record = flush_record_alloc(flush_ctx->nr_pages,
+ &flush_ctx->flush_record_pages);
+ if (!flush_ctx->flush_record)
+ return -ENOMEM;
+
+ flush_ctx->mbox = comp_ctx->mbox;
+ flush_ctx->dev = dev;
+ flush_ctx->nid = nid;
+ flush_ctx->media_ops_supported = comp_ctx->media_ops_supported;
+
+ /*
+ * Cap buffer at max DPA ranges that fit in one CCI payload.
+ * Header is 8 bytes (struct cxl_media_op_input), each range
+ * is 16 bytes (struct cxl_dpa_range). The module parameter
+ * flush_buf_size can further limit this (0 = use hw max).
+ */
+ flush_ctx->buf_max = (flush_ctx->mbox->payload_size -
+ sizeof(struct cxl_media_op_input)) /
+ sizeof(struct cxl_dpa_range);
+ if (flush_buf_size && flush_buf_size < flush_ctx->buf_max)
+ flush_ctx->buf_max = flush_buf_size;
+ if (flush_ctx->buf_max == 0)
+ flush_ctx->buf_max = 1;
+
+ dev_info(dev,
+ "flush buffer: %u DPA ranges per command (payload %zu bytes, media_ops %s)\n",
+ flush_ctx->buf_max, flush_ctx->mbox->payload_size,
+ flush_ctx->media_ops_supported ? "yes" : "no");
+
+ flush_ctx->pcpu = alloc_percpu(struct cxl_pcpu_flush);
+ if (!flush_ctx->pcpu)
+ return -ENOMEM;
+
+ flush_ctx->kthread_spares = kcalloc(nr_cpu_ids,
+ sizeof(struct cxl_flush_buf *),
+ GFP_KERNEL);
+ if (!flush_ctx->kthread_spares)
+ goto err_pcpu_init;
+
+ for_each_possible_cpu(cpu) {
+ struct cxl_flush_buf *active_buf, *overflow_buf, *spare_buf;
+
+ active_buf = cxl_flush_buf_alloc(flush_ctx->buf_max, nid);
+ if (!active_buf)
+ goto err_pcpu_init;
+
+ overflow_buf = cxl_flush_buf_alloc(flush_ctx->buf_max, nid);
+ if (!overflow_buf) {
+ cxl_flush_buf_free(active_buf);
+ goto err_pcpu_init;
+ }
+
+ spare_buf = cxl_flush_buf_alloc(flush_ctx->buf_max, nid);
+ if (!spare_buf) {
+ cxl_flush_buf_free(active_buf);
+ cxl_flush_buf_free(overflow_buf);
+ goto err_pcpu_init;
+ }
+
+ pcpu = per_cpu_ptr(flush_ctx->pcpu, cpu);
+ pcpu->ctx = flush_ctx;
+ rcu_assign_pointer(pcpu->active, active_buf);
+ pcpu->overflow_spare = overflow_buf;
+ INIT_WORK(&pcpu->overflow_work, cxl_flush_overflow_work);
+
+ flush_ctx->kthread_spares[cpu] = spare_buf;
+ }
+
+ flush_ctx->flush_thread = kthread_create_on_node(
+ cxl_flush_kthread_fn, flush_ctx, nid, "cxl-flush/%d", nid);
+ if (IS_ERR(flush_ctx->flush_thread)) {
+ rc = PTR_ERR(flush_ctx->flush_thread);
+ flush_ctx->flush_thread = NULL;
+ goto err_pcpu_init;
+ }
+ wake_up_process(flush_ctx->flush_thread);
+
+ rc = cram_register_private_node(nid, cxlr,
+ cxl_compression_flush_cb, flush_ctx);
+ if (rc) {
+ dev_err(dev, "failed to register cram node %d: %d\n", nid, rc);
+ goto err_pcpu_init;
+ }
+
+ tctx->flush_ctx = flush_ctx;
+ tctx->nid = nid;
+
+ rc = devm_add_action_or_reset(dev, cxl_compression_pre_teardown, tctx);
+ if (rc)
+ return rc;
+
+ comp_ctx->flush_ctx = flush_ctx;
+ comp_ctx->tctx = tctx;
+ comp_ctx->nid = nid;
+
+ /*
+ * Register watermark IRQ handlers on &pdev->dev for
+ * MSI-X vector 12 (lthresh) and vector 13 (hthresh).
+ */
+ wm_ctx = devm_kzalloc(&pdev->dev, sizeof(*wm_ctx), GFP_KERNEL);
+ if (!wm_ctx)
+ return -ENOMEM;
+
+ wm_ctx->dev = &pdev->dev;
+ wm_ctx->nid = nid;
+
+ irq = pci_irq_vector(pdev, CXL_CT3_MSIX_LTHRESH);
+ if (irq >= 0) {
+ rc = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+ cxl_compression_lthresh_irq,
+ IRQF_ONESHOT,
+ "cxl-lthresh", wm_ctx);
+ if (rc)
+ dev_warn(&pdev->dev,
+ "failed to register lthresh IRQ: %d\n", rc);
+ }
+
+ irq = pci_irq_vector(pdev, CXL_CT3_MSIX_HTHRESH);
+ if (irq >= 0) {
+ rc = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+ cxl_compression_hthresh_irq,
+ IRQF_ONESHOT,
+ "cxl-hthresh", wm_ctx);
+ if (rc)
+ dev_warn(&pdev->dev,
+ "failed to register hthresh IRQ: %d\n", rc);
+ }
+
+ return 0;
+
+err_pcpu_init:
+ if (flush_ctx->flush_thread)
+ kthread_stop(flush_ctx->flush_thread);
+ for_each_possible_cpu(cpu) {
+ struct cxl_flush_buf *buf;
+
+ pcpu = per_cpu_ptr(flush_ctx->pcpu, cpu);
+
+ buf = rcu_dereference_raw(pcpu->active);
+ cxl_flush_buf_free(buf);
+
+ cxl_flush_buf_free(pcpu->overflow_spare);
+
+ if (flush_ctx->kthread_spares)
+ cxl_flush_buf_free(flush_ctx->kthread_spares[cpu]);
+ }
+ kfree(flush_ctx->kthread_spares);
+ free_percpu(flush_ctx->pcpu);
+ flush_record_free(flush_ctx->flush_record, flush_ctx->flush_record_pages);
+ return rc ? rc : -ENOMEM;
+}
+
+static struct cxl_region *create_ram_region(struct cxl_memdev *cxlmd)
+{
+ struct cxl_root_decoder *cxlrd;
+ struct cxl_endpoint_decoder *cxled;
+ struct cxl_region *cxlr;
+ resource_size_t ram_size, avail;
+
+ ram_size = cxl_ram_size(cxlmd->cxlds);
+ if (ram_size == 0) {
+ dev_info(&cxlmd->dev, "no RAM capacity available\n");
+ return ERR_PTR(-ENODEV);
+ }
+
+ ram_size = ALIGN_DOWN(ram_size, SZ_256M);
+ if (ram_size == 0) {
+ dev_info(&cxlmd->dev, "RAM capacity too small (< 256M)\n");
+ return ERR_PTR(-ENOSPC);
+ }
+
+ dev_info(&cxlmd->dev, "creating RAM region for %lld MB\n",
+ ram_size >> 20);
+
+ cxlrd = cxl_get_hpa_freespace(cxlmd, ram_size, &avail);
+ if (IS_ERR(cxlrd)) {
+ dev_err(&cxlmd->dev, "no HPA freespace: %ld\n",
+ PTR_ERR(cxlrd));
+ return ERR_CAST(cxlrd);
+ }
+
+ cxled = cxl_request_dpa(cxlmd, CXL_PARTMODE_RAM, ram_size);
+ if (IS_ERR(cxled)) {
+ dev_err(&cxlmd->dev, "failed to request DPA: %ld\n",
+ PTR_ERR(cxled));
+ cxl_put_root_decoder(cxlrd);
+ return ERR_CAST(cxled);
+ }
+
+ cxlr = cxl_create_region(cxlrd, &cxled, 1);
+ cxl_put_root_decoder(cxlrd);
+ if (IS_ERR(cxlr)) {
+ dev_err(&cxlmd->dev, "failed to create region: %ld\n",
+ PTR_ERR(cxlr));
+ cxl_dpa_free(cxled);
+ return cxlr;
+ }
+
+ dev_info(&cxlmd->dev, "created region %s\n",
+ dev_name(cxl_region_dev(cxlr)));
+ pdev_to_comp_ctx(to_pci_dev(cxlmd->dev.parent))->cxled = cxled;
+ return cxlr;
+}
+
+static int cxl_compression_attach_probe(struct cxl_memdev *cxlmd)
+{
+ struct pci_dev *pdev = to_pci_dev(cxlmd->dev.parent);
+ struct cxl_compression_ctx *comp_ctx = pdev_to_comp_ctx(pdev);
+ struct cxl_region *regions[8];
+ struct cxl_region *cxlr;
+ int nr, i, converted = 0, errors = 0;
+ int rc;
+
+ comp_ctx->cxlmd = cxlmd;
+ comp_ctx->mbox = &cxlmd->cxlds->cxl_mbox;
+
+ /* Probe device for media operations zero support */
+ comp_ctx->media_ops_supported =
+ cxl_probe_media_ops_zero(comp_ctx->mbox,
+ &cxlmd->dev);
+
+ dev_info(&cxlmd->dev, "compression attach: looking for regions\n");
+
+ nr = cxl_get_committed_regions(cxlmd, regions, ARRAY_SIZE(regions));
+ for (i = 0; i < nr; i++) {
+ if (cxl_region_mode(regions[i]) == CXL_PARTMODE_RAM) {
+ rc = convert_region_to_sysram(regions[i], pdev);
+ if (rc)
+ errors++;
+ else
+ converted++;
+ }
+ put_device(cxl_region_dev(regions[i]));
+ }
+
+ if (converted > 0) {
+ dev_info(&cxlmd->dev,
+ "converted %d regions to sysram (%d errors)\n",
+ converted, errors);
+ return errors ? -EIO : 0;
+ }
+
+ dev_info(&cxlmd->dev, "no existing regions, creating RAM region\n");
+
+ cxlr = create_ram_region(cxlmd);
+ if (IS_ERR(cxlr)) {
+ rc = PTR_ERR(cxlr);
+ if (rc == -ENODEV) {
+ dev_info(&cxlmd->dev,
+ "could not create RAM region: %d\n", rc);
+ return 0;
+ }
+ return rc;
+ }
+
+ rc = convert_region_to_sysram(cxlr, pdev);
+ if (rc) {
+ dev_err(&cxlmd->dev,
+ "failed to convert region to sysram: %d\n", rc);
+ return rc;
+ }
+
+ comp_ctx->cxlr = cxlr;
+
+ dev_info(&cxlmd->dev, "created and converted region %s to sysram\n",
+ dev_name(cxl_region_dev(cxlr)));
+
+ return 0;
+}
+
+static const struct cxl_memdev_attach cxl_compression_attach = {
+ .probe = cxl_compression_attach_probe,
+};
+
+static int cxl_compression_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+{
+ struct cxl_compression_ctx *comp_ctx;
+ struct cxl_memdev *cxlmd;
+ int rc;
+
+ dev_info(&pdev->dev, "cxl_compression: probing device\n");
+
+ comp_ctx = devm_kzalloc(&pdev->dev, sizeof(*comp_ctx), GFP_KERNEL);
+ if (!comp_ctx)
+ return -ENOMEM;
+ comp_ctx->nid = NUMA_NO_NODE;
+ comp_ctx->pdev = pdev;
+
+ rc = xa_insert(&comp_ctx_xa, (unsigned long)pdev, comp_ctx, GFP_KERNEL);
+ if (rc)
+ return rc;
+
+ cxlmd = cxl_pci_type3_probe_init(pdev, &cxl_compression_attach);
+ if (IS_ERR(cxlmd)) {
+ xa_erase(&comp_ctx_xa, (unsigned long)pdev);
+ return PTR_ERR(cxlmd);
+ }
+
+ comp_ctx->cxlmd = cxlmd;
+ comp_ctx->mbox = &cxlmd->cxlds->cxl_mbox;
+
+ dev_info(&pdev->dev, "cxl_compression: probe complete\n");
+ return 0;
+}
+
+static void cxl_compression_remove(struct pci_dev *pdev)
+{
+ struct cxl_compression_ctx *comp_ctx = xa_erase(&comp_ctx_xa,
+ (unsigned long)pdev);
+
+ dev_info(&pdev->dev, "cxl_compression: removing device\n");
+
+ if (!comp_ctx || comp_ctx->nid == NUMA_NO_NODE)
+ return;
+
+ /*
+ * Destroy the region, devm actions on the region device handle teardown
+ * in registration-reverse order:
+ * 1. pre_teardown: cram_unregister + retry-forever memory offline
+ * 2. sysram_unregister: device_unregister (sysram->res is NULL
+ * after pre_teardown, so cxl_sysram_release skips hotplug)
+ * 3. post_teardown: kthread stop, flush cleanup
+ *
+ * PCI MMIO is still live so CCI commands in post_teardown work.
+ */
+ if (comp_ctx->cxlr) {
+ cxl_destroy_region(comp_ctx->cxlr);
+ comp_ctx->cxlr = NULL;
+ }
+
+ if (comp_ctx->cxled) {
+ cxl_dpa_free(comp_ctx->cxled);
+ comp_ctx->cxled = NULL;
+ }
+}
+
+static const struct pci_device_id cxl_compression_pci_tbl[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0d93) },
+ { /* terminate list */ },
+};
+MODULE_DEVICE_TABLE(pci, cxl_compression_pci_tbl);
+
+static struct pci_driver cxl_compression_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = cxl_compression_pci_tbl,
+ .probe = cxl_compression_probe,
+ .remove = cxl_compression_remove,
+ .driver = {
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
+ },
+};
+
+module_pci_driver(cxl_compression_driver);
+
+MODULE_DESCRIPTION("CXL: Compression Memory Driver with SysRAM regions");
+MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("CXL");
--
2.53.0