* [PATCH v20 04/14] dmaengine: qcom: bam_dma: Extend the driver's device match data
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
From: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
In preparation for supporting the pipe locking feature flag, extend the
amount of information we can carry in device match data: create a
separate structure and make the register information one of its fields.
This way, in subsequent patches, it will be just a matter of adding a
new field to the device data.
Reviewed-by: Dmitry Baryshkov <lumag@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/dma/qcom/bam_dma.c | 34 +++++++++++++++++++++++++++-------
1 file changed, 27 insertions(+), 7 deletions(-)
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index ea3df28e777f99c0532761b6aee6807ab23ab4ca..8ce0fe085c5fea6cc614edd692b5cfd264b94d5a 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -113,6 +113,10 @@ struct reg_offset_data {
unsigned int pipe_mult, evnt_mult, ee_mult;
};
+struct bam_device_data {
+ const struct reg_offset_data *reg_info;
+};
+
static const struct reg_offset_data bam_v1_3_reg_info[] = {
[BAM_CTRL] = { 0x0F80, 0x00, 0x00, 0x00 },
[BAM_REVISION] = { 0x0F84, 0x00, 0x00, 0x00 },
@@ -142,6 +146,10 @@ static const struct reg_offset_data bam_v1_3_reg_info[] = {
[BAM_P_FIFO_SIZES] = { 0x1020, 0x00, 0x40, 0x00 },
};
+static const struct bam_device_data bam_v1_3_data = {
+ .reg_info = bam_v1_3_reg_info,
+};
+
static const struct reg_offset_data bam_v1_4_reg_info[] = {
[BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 },
[BAM_REVISION] = { 0x0004, 0x00, 0x00, 0x00 },
@@ -171,6 +179,10 @@ static const struct reg_offset_data bam_v1_4_reg_info[] = {
[BAM_P_FIFO_SIZES] = { 0x1820, 0x00, 0x1000, 0x00 },
};
+static const struct bam_device_data bam_v1_4_data = {
+ .reg_info = bam_v1_4_reg_info,
+};
+
static const struct reg_offset_data bam_v1_7_reg_info[] = {
[BAM_CTRL] = { 0x00000, 0x00, 0x00, 0x00 },
[BAM_REVISION] = { 0x01000, 0x00, 0x00, 0x00 },
@@ -200,6 +212,10 @@ static const struct reg_offset_data bam_v1_7_reg_info[] = {
[BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 },
};
+static const struct bam_device_data bam_v1_7_data = {
+ .reg_info = bam_v1_7_reg_info,
+};
+
static const struct reg_offset_data bam_v2_0_reg_info[] = {
[BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 },
[BAM_REVISION] = { 0x1000, 0x00, 0x00, 0x00 },
@@ -229,6 +245,10 @@ static const struct reg_offset_data bam_v2_0_reg_info[] = {
[BAM_P_FIFO_SIZES] = { 0xC820, 0x00, 0x1000, 0x00 },
};
+static const struct bam_device_data bam_v2_0_data = {
+ .reg_info = bam_v2_0_reg_info,
+};
+
/* BAM CTRL */
#define BAM_SW_RST BIT(0)
#define BAM_EN BIT(1)
@@ -422,7 +442,7 @@ struct bam_device {
bool powered_remotely;
u32 active_channels;
- const struct reg_offset_data *layout;
+ const struct bam_device_data *dev_data;
struct clk *bamclk;
int irq;
@@ -440,7 +460,7 @@ struct bam_device {
static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe,
enum bam_reg reg)
{
- const struct reg_offset_data r = bdev->layout[reg];
+ const struct reg_offset_data r = bdev->dev_data->reg_info[reg];
return bdev->regs + r.base_offset +
r.pipe_mult * pipe +
@@ -1234,10 +1254,10 @@ static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,
}
static const struct of_device_id bam_of_match[] = {
- { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
- { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
- { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
- { .compatible = "qcom,bam-v2.0.0", .data = &bam_v2_0_reg_info },
+ { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_data },
+ { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_data },
+ { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_data },
+ { .compatible = "qcom,bam-v2.0.0", .data = &bam_v2_0_data },
{}
};
@@ -1261,7 +1281,7 @@ static int bam_dma_probe(struct platform_device *pdev)
return -ENODEV;
}
- bdev->layout = match->data;
+ bdev->dev_data = match->data;
bdev->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(bdev->regs))
--
2.47.3
^ permalink raw reply related
* [PATCH v20 06/14] dmaengine: qcom: bam_dma: add support for BAM locking
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
Add support for BAM pipe locking. To that end: when starting DMA on an RX
channel - prepend the existing queue of issued descriptors with an
additional "dummy" command descriptor with the LOCK bit set. Once the
transaction is done (no more issued descriptors), issue one more dummy
descriptor with the UNLOCK bit.
We *must* wait until the transaction is signalled as done because we
must not perform any writes into config registers while the engine is
busy.
The dummy writes must be issued into a scratchpad register of the client
so provide a mechanism to communicate the right address via descriptor
metadata.
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/dma/qcom/bam_dma.c | 189 +++++++++++++++++++++++++++++++++++++--
include/linux/dma/qcom_bam_dma.h | 14 +++
2 files changed, 196 insertions(+), 7 deletions(-)
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index f3e713a5259c2c7c24cfdcec094814eb1202971a..f4f258994264a234f60debd3e66e31a6b35d1dc5 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -28,11 +28,13 @@
#include <linux/clk.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
+#include <linux/dma/qcom_bam_dma.h>
#include <linux/dmaengine.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kernel.h>
+#include <linux/lockdep.h>
#include <linux/module.h>
#include <linux/of_address.h>
#include <linux/of_dma.h>
@@ -60,6 +62,8 @@ struct bam_desc_hw {
#define DESC_FLAG_EOB BIT(13)
#define DESC_FLAG_NWD BIT(12)
#define DESC_FLAG_CMD BIT(11)
+#define DESC_FLAG_LOCK BIT(10)
+#define DESC_FLAG_UNLOCK BIT(9)
struct bam_async_desc {
struct virt_dma_desc vd;
@@ -72,6 +76,10 @@ struct bam_async_desc {
struct bam_desc_hw *curr_desc;
+ /* BAM locking infrastructure */
+ struct scatterlist lock_sg;
+ struct bam_cmd_element lock_ce;
+
/* list node for the desc in the bam_chan list of descriptors */
struct list_head desc_node;
enum dma_transfer_direction dir;
@@ -425,6 +433,11 @@ struct bam_chan {
struct list_head desc_list;
struct list_head node;
+
+ /* BAM locking infrastructure */
+ phys_addr_t scratchpad_addr;
+ enum dma_transfer_direction direction;
+ bool bam_locked;
};
static inline struct bam_chan *to_bam_chan(struct dma_chan *common)
@@ -638,8 +651,10 @@ static void bam_free_chan(struct dma_chan *chan)
goto err;
}
- scoped_guard(spinlock_irqsave, &bchan->vc.lock)
+ scoped_guard(spinlock_irqsave, &bchan->vc.lock) {
bam_reset_channel(bchan);
+ bchan->bam_locked = false;
+ }
dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,
bchan->fifo_phys);
@@ -686,6 +701,35 @@ static int bam_slave_config(struct dma_chan *chan,
return 0;
}
+static int bam_metadata_attach(struct dma_async_tx_descriptor *desc, void *data, size_t len)
+{
+ struct bam_chan *bchan = to_bam_chan(desc->chan);
+ const struct bam_device_data *bdata = bchan->bdev->dev_data;
+ struct bam_desc_metadata *metadata = data;
+
+ if (!data)
+ return -EINVAL;
+
+ if (!bdata->pipe_lock_supported)
+ /*
+ * The client wants to use locking but this BAM version doesn't
+ * support it. Don't return an error here as this will stop the
+ * client from using DMA at all for no reason.
+ */
+ return 0;
+
+ guard(spinlock_irqsave)(&bchan->vc.lock);
+
+ bchan->scratchpad_addr = metadata->scratchpad_addr;
+ bchan->direction = metadata->direction;
+
+ return 0;
+}
+
+static const struct dma_descriptor_metadata_ops bam_metadata_ops = {
+ .attach = bam_metadata_attach,
+};
+
/**
* bam_prep_slave_sg - Prep slave sg transaction
*
@@ -702,6 +746,7 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
void *context)
{
struct bam_chan *bchan = to_bam_chan(chan);
+ struct dma_async_tx_descriptor *tx_desc;
struct bam_device *bdev = bchan->bdev;
struct bam_async_desc *async_desc;
struct scatterlist *sg;
@@ -757,7 +802,10 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
} while (remainder > 0);
}
- return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);
+ tx_desc = vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);
+ tx_desc->metadata_ops = &bam_metadata_ops;
+
+ return tx_desc;
}
/**
@@ -802,6 +850,7 @@ static int bam_dma_terminate_all(struct dma_chan *chan)
}
vchan_get_all_descriptors(&bchan->vc, &head);
+ bchan->bam_locked = false;
}
vchan_dma_desc_free_list(&bchan->vc, &head);
@@ -859,6 +908,15 @@ static int bam_resume(struct dma_chan *chan)
return 0;
}
+static void bam_dma_free_lock_desc(struct virt_dma_desc *vd)
+{
+ struct bam_async_desc *async_desc = container_of(vd, struct bam_async_desc, vd);
+ struct dma_chan *chan = vd->tx.chan;
+
+ dma_unmap_sg(chan->slave, &async_desc->lock_sg, 1, DMA_TO_DEVICE);
+ kfree(async_desc);
+}
+
/**
* process_channel_irqs - processes the channel interrupts
* @bdev: bam controller
@@ -919,13 +977,23 @@ static u32 process_channel_irqs(struct bam_device *bdev)
* push back to front of desc_issued so that
* it gets restarted by the work queue.
*/
+
+ list_del(&async_desc->desc_node);
if (!async_desc->num_desc) {
- vchan_cookie_complete(&async_desc->vd);
+ struct bam_desc_hw *hdesc = async_desc->desc;
+ u16 flags = le16_to_cpu(hdesc->flags);
+
+ if (flags & (DESC_FLAG_LOCK | DESC_FLAG_UNLOCK)) {
+ if (flags & DESC_FLAG_UNLOCK)
+ bchan->bam_locked = false;
+ bam_dma_free_lock_desc(&async_desc->vd);
+ } else {
+ vchan_cookie_complete(&async_desc->vd);
+ }
} else {
list_add(&async_desc->vd.node,
&bchan->vc.desc_issued);
}
- list_del(&async_desc->desc_node);
}
}
@@ -1046,13 +1114,101 @@ static void bam_apply_new_config(struct bam_chan *bchan,
bchan->reconfigure = 0;
}
+static struct bam_async_desc *
+bam_make_lock_desc(struct bam_chan *bchan, unsigned long flag)
+{
+ struct dma_chan *chan = &bchan->vc.chan;
+ struct bam_async_desc *async_desc;
+ struct bam_desc_hw *desc;
+ struct virt_dma_desc *vd;
+ struct virt_dma_chan *vc;
+ unsigned int mapped;
+
+ async_desc = kzalloc_flex(*async_desc, desc, 1, GFP_NOWAIT);
+ if (!async_desc) {
+ dev_err(bchan->bdev->dev, "failed to allocate the BAM lock descriptor\n");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ sg_init_table(&async_desc->lock_sg, 1);
+
+ async_desc->num_desc = 1;
+ async_desc->curr_desc = async_desc->desc;
+ async_desc->dir = DMA_MEM_TO_DEV;
+
+ desc = async_desc->desc;
+
+ bam_prep_ce_le32(&async_desc->lock_ce, bchan->scratchpad_addr, BAM_WRITE_COMMAND, 0);
+ sg_set_buf(&async_desc->lock_sg, &async_desc->lock_ce, sizeof(async_desc->lock_ce));
+
+ mapped = dma_map_sg(chan->slave, &async_desc->lock_sg, 1, DMA_TO_DEVICE);
+ if (!mapped) {
+ kfree(async_desc);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ desc->flags |= cpu_to_le16(DESC_FLAG_CMD | flag);
+ desc->addr = sg_dma_address(&async_desc->lock_sg);
+ desc->size = cpu_to_le16(sizeof(struct bam_cmd_element));
+
+ vc = &bchan->vc;
+ vd = &async_desc->vd;
+
+ dma_async_tx_descriptor_init(&vd->tx, &vc->chan);
+ vd->tx.flags = DMA_PREP_CMD;
+ vd->tx_result.result = DMA_TRANS_NOERROR;
+ vd->tx_result.residue = 0;
+
+ return async_desc;
+}
+
+static int bam_setup_pipe_lock(struct bam_chan *bchan)
+{
+ const struct bam_device_data *bdata = bchan->bdev->dev_data;
+ struct bam_async_desc *lock_desc, *unlock_desc;
+
+ lockdep_assert_held(&bchan->vc.lock);
+
+ if (!bdata->pipe_lock_supported || !bchan->scratchpad_addr ||
+ bchan->direction != DMA_MEM_TO_DEV)
+ return 0;
+
+ /*
+ * Allocate both the LOCK and the UNLOCK descriptors up-front so the
+ * operation is all-or-nothing: if either allocation fails we free both
+ * and run the sequence unlocked rather than leave the pipe locked with
+ * no matching UNLOCK.
+ *
+ * Both are queued in-band around the currently issued work: the LOCK is
+ * prepended so it enters the FIFO first, the UNLOCK is appended so it is
+ * the last descriptor of the sequence. They are loaded together with the
+ * payload in a single operation so the engine executes LOCK, the work
+ * and UNLOCK as one ordered batch.
+ */
+ lock_desc = bam_make_lock_desc(bchan, DESC_FLAG_LOCK);
+ if (IS_ERR(lock_desc))
+ return PTR_ERR(lock_desc);
+
+ unlock_desc = bam_make_lock_desc(bchan, DESC_FLAG_UNLOCK);
+ if (IS_ERR(unlock_desc)) {
+ bam_dma_free_lock_desc(&lock_desc->vd);
+ return PTR_ERR(unlock_desc);
+ }
+
+ list_add(&lock_desc->vd.node, &bchan->vc.desc_issued);
+ list_add_tail(&unlock_desc->vd.node, &bchan->vc.desc_issued);
+ bchan->bam_locked = true;
+
+ return 0;
+}
+
/**
* bam_start_dma - start next transaction
* @bchan: bam dma channel
*/
static void bam_start_dma(struct bam_chan *bchan)
{
- struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);
+ struct virt_dma_desc *vd;
struct bam_device *bdev = bchan->bdev;
struct bam_async_desc *async_desc = NULL;
struct bam_desc_hw *desc;
@@ -1064,9 +1220,23 @@ static void bam_start_dma(struct bam_chan *bchan)
lockdep_assert_held(&bchan->vc.lock);
+ vd = vchan_next_desc(&bchan->vc);
if (!vd)
return;
+ /*
+ * Wrap the issued work with a LOCK/UNLOCK pair exactly once, at the
+ * start of a fresh sequence and only when there is real work to lock
+ * around. On a re-entry after a full FIFO, we see the BAM is locked
+ * and must not add another pair we simply continue loading the
+ * remainder of the same locked sequence.
+ */
+ if (!bchan->bam_locked) {
+ ret = bam_setup_pipe_lock(bchan);
+ if (ret == 0 && bchan->bam_locked)
+ vd = vchan_next_desc(&bchan->vc);
+ }
+
ret = pm_runtime_get_sync(bdev->dev);
if (ret < 0)
return;
@@ -1191,8 +1361,12 @@ static void bam_issue_pending(struct dma_chan *chan)
*/
static void bam_dma_free_desc(struct virt_dma_desc *vd)
{
- struct bam_async_desc *async_desc = container_of(vd,
- struct bam_async_desc, vd);
+ struct bam_async_desc *async_desc = container_of(vd, struct bam_async_desc, vd);
+ struct bam_desc_hw *desc = async_desc->desc;
+ struct dma_chan *chan = vd->tx.chan;
+
+ if (le16_to_cpu(desc->flags) & (DESC_FLAG_LOCK | DESC_FLAG_UNLOCK))
+ dma_unmap_sg(chan->slave, &async_desc->lock_sg, 1, DMA_TO_DEVICE);
kfree(async_desc);
}
@@ -1384,6 +1558,7 @@ static int bam_dma_probe(struct platform_device *pdev)
bdev->common.device_terminate_all = bam_dma_terminate_all;
bdev->common.device_issue_pending = bam_issue_pending;
bdev->common.device_tx_status = bam_tx_status;
+ bdev->common.desc_metadata_modes = DESC_METADATA_CLIENT;
bdev->common.dev = bdev->dev;
ret = dma_async_device_register(&bdev->common);
diff --git a/include/linux/dma/qcom_bam_dma.h b/include/linux/dma/qcom_bam_dma.h
index 68fc0e643b1b97fe4520d5878daa322b81f4f559..a2594264b0f58c4b2b1c85e243cad0d5669c26dc 100644
--- a/include/linux/dma/qcom_bam_dma.h
+++ b/include/linux/dma/qcom_bam_dma.h
@@ -6,6 +6,8 @@
#ifndef _QCOM_BAM_DMA_H
#define _QCOM_BAM_DMA_H
+#include <linux/dmaengine.h>
+
#include <asm/byteorder.h>
/*
@@ -34,6 +36,18 @@ enum bam_command_type {
BAM_READ_COMMAND,
};
+/**
+ * struct bam_desc_metadata - DMA descriptor metadata specific to the BAM driver.
+ *
+ * @scratchpad_addr: Physical address to use for dummy write operations when
+ * queuing command descriptors with LOCK/UNLOCK bits set.
+ * @direction: Transfer direction of this channel.
+ */
+struct bam_desc_metadata {
+ phys_addr_t scratchpad_addr;
+ enum dma_transfer_direction direction;
+};
+
/*
* prep_bam_ce_le32 - Wrapper function to prepare a single BAM command
* element with the data already in le32 format.
--
2.47.3
^ permalink raw reply related
* [PATCH v20 07/14] crypto: qce - Cancel work on device detach
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
The workqueue is setup in probe() but never cancelled on error or in
remove(). Set up a devres action to clean it up. We need to move the
initialization earlier as we don't want to cancel the work before any
outstanding DMA transfer is terminated. Make sure we do terminate all
transfers in qce_dma_release() devres action.
Fixes: eb7986e5e14d ("crypto: qce - convert tasklet to workqueue")
Closes: https://sashiko.dev/#/patchset/20260427-qcom-qce-cmd-descr-v16-0-945fd1cafbbc%40oss.qualcomm.com?part=7
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/crypto/qce/core.c | 13 ++++++++++++-
drivers/crypto/qce/dma.c | 2 ++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c
index b966f3365b7de8d2a8f6707397a34aa4facdc4ac..f671946cf7351cd5f0c319909bafd87e3af701c7 100644
--- a/drivers/crypto/qce/core.c
+++ b/drivers/crypto/qce/core.c
@@ -186,6 +186,13 @@ static int qce_check_version(struct qce_device *qce)
return 0;
}
+static void qce_cancel_work(void *data)
+{
+ struct work_struct *work = data;
+
+ cancel_work_sync(work);
+}
+
static int qce_crypto_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
@@ -227,6 +234,11 @@ static int qce_crypto_probe(struct platform_device *pdev)
if (ret)
return ret;
+ INIT_WORK(&qce->done_work, qce_req_done_work);
+ ret = devm_add_action_or_reset(dev, qce_cancel_work, &qce->done_work);
+ if (ret)
+ return ret;
+
ret = devm_qce_dma_request(qce->dev, &qce->dma);
if (ret)
return ret;
@@ -239,7 +251,6 @@ static int qce_crypto_probe(struct platform_device *pdev)
if (ret)
return ret;
- INIT_WORK(&qce->done_work, qce_req_done_work);
crypto_init_queue(&qce->queue, QCE_QUEUE_LENGTH);
qce->async_req_enqueue = qce_async_request_enqueue;
diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c
index 68cafd4741ad3d91906d39e817fc7873b028d498..7ec9d72fd690fb17e03ade7efe3cc522fb47e1ac 100644
--- a/drivers/crypto/qce/dma.c
+++ b/drivers/crypto/qce/dma.c
@@ -13,6 +13,8 @@ static void qce_dma_release(void *data)
{
struct qce_dma_data *dma = data;
+ dmaengine_terminate_sync(dma->txchan);
+ dmaengine_terminate_sync(dma->rxchan);
dma_release_channel(dma->txchan);
dma_release_channel(dma->rxchan);
kfree(dma->result_buf);
--
2.47.3
^ permalink raw reply related
* [PATCH v20 08/14] crypto: qce - Include algapi.h in the core.h header
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
From: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
The header defines a struct embedding struct crypto_queue whose size
needs to be known and which is defined in crypto/algapi.h. Move the
inclusion from core.c to core.h.
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/crypto/qce/core.c | 1 -
drivers/crypto/qce/core.h | 1 +
2 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c
index f671946cf7351cd5f0c319909bafd87e3af701c7..ad37c2b8ae53a373bb248aff06c3b7946e8439a8 100644
--- a/drivers/crypto/qce/core.c
+++ b/drivers/crypto/qce/core.c
@@ -13,7 +13,6 @@
#include <linux/mod_devicetable.h>
#include <linux/platform_device.h>
#include <linux/types.h>
-#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include "core.h"
diff --git a/drivers/crypto/qce/core.h b/drivers/crypto/qce/core.h
index eb6fa7a8b64a81daf9ad5304a3ae4e5e597a70b8..f092ce2d3b04a936a37805c20ac5ba78d8fdd2df 100644
--- a/drivers/crypto/qce/core.h
+++ b/drivers/crypto/qce/core.h
@@ -8,6 +8,7 @@
#include <linux/mutex.h>
#include <linux/workqueue.h>
+#include <crypto/algapi.h>
#include "dma.h"
--
2.47.3
^ permalink raw reply related
* [PATCH v20 05/14] dmaengine: qcom: bam_dma: Add pipe_lock_supported flag support
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski,
Dmitry Baryshkov
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
From: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Extend the device match data with a flag indicating whether the IP
supports the BAM lock/unlock feature. Set it to true on BAM IP versions
1.4.0 and above.
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Dmitry Baryshkov <dmitry.baryshkov@oss.qualcomm.com>
Acked-by: Manivannan Sadhasivam <mani@kernel.org>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/dma/qcom/bam_dma.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index 8ce0fe085c5fea6cc614edd692b5cfd264b94d5a..f3e713a5259c2c7c24cfdcec094814eb1202971a 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -115,6 +115,7 @@ struct reg_offset_data {
struct bam_device_data {
const struct reg_offset_data *reg_info;
+ bool pipe_lock_supported;
};
static const struct reg_offset_data bam_v1_3_reg_info[] = {
@@ -181,6 +182,7 @@ static const struct reg_offset_data bam_v1_4_reg_info[] = {
static const struct bam_device_data bam_v1_4_data = {
.reg_info = bam_v1_4_reg_info,
+ .pipe_lock_supported = true,
};
static const struct reg_offset_data bam_v1_7_reg_info[] = {
@@ -214,6 +216,7 @@ static const struct reg_offset_data bam_v1_7_reg_info[] = {
static const struct bam_device_data bam_v1_7_data = {
.reg_info = bam_v1_7_reg_info,
+ .pipe_lock_supported = true,
};
static const struct reg_offset_data bam_v2_0_reg_info[] = {
@@ -247,6 +250,7 @@ static const struct reg_offset_data bam_v2_0_reg_info[] = {
static const struct bam_device_data bam_v2_0_data = {
.reg_info = bam_v2_0_reg_info,
+ .pipe_lock_supported = true,
};
/* BAM CTRL */
--
2.47.3
^ permalink raw reply related
* [PATCH v20 09/14] crypto: qce - Remove unused ignore_buf
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
From: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
It's unclear what the purpose of this field is. It has been here since
the initial commit but without any explanation. The driver works fine
without it. We still keep allocating more space in the result buffer, we
just don't need to store its address. While at it: move the
QCE_IGNORE_BUF_SZ definition into dma.c as it's not used outside of this
compilation unit.
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/crypto/qce/dma.c | 4 ++--
drivers/crypto/qce/dma.h | 2 --
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c
index 7ec9d72fd690fb17e03ade7efe3cc522fb47e1ac..d1daa229361aa74da5d3d7bfe1bc8ab189761e38 100644
--- a/drivers/crypto/qce/dma.c
+++ b/drivers/crypto/qce/dma.c
@@ -9,6 +9,8 @@
#include "dma.h"
+#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE)
+
static void qce_dma_release(void *data)
{
struct qce_dma_data *dma = data;
@@ -43,8 +45,6 @@ int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma)
goto error_nomem;
}
- dma->ignore_buf = dma->result_buf + QCE_RESULT_BUF_SZ;
-
return devm_add_action_or_reset(dev, qce_dma_release, dma);
error_nomem:
diff --git a/drivers/crypto/qce/dma.h b/drivers/crypto/qce/dma.h
index 31629185000e12242fa07c2cc08b95fcbd5d4b8c..fc337c435cd14917bdfb99febcf9119275afdeba 100644
--- a/drivers/crypto/qce/dma.h
+++ b/drivers/crypto/qce/dma.h
@@ -23,7 +23,6 @@ struct qce_result_dump {
u32 status2;
};
-#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE)
#define QCE_RESULT_BUF_SZ \
ALIGN(sizeof(struct qce_result_dump), QCE_BAM_BURST_SIZE)
@@ -31,7 +30,6 @@ struct qce_dma_data {
struct dma_chan *txchan;
struct dma_chan *rxchan;
struct qce_result_dump *result_buf;
- void *ignore_buf;
};
int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma);
--
2.47.3
^ permalink raw reply related
* [PATCH v20 10/14] crypto: qce - Simplify arguments of devm_qce_dma_request()
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
From: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
This function can extract all the information it needs from struct
qce_device alone so simplify its arguments. This is done in preparation
for adding support for register I/O over DMA which will require
accessing even more fields from struct qce_device.
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/crypto/qce/core.c | 2 +-
drivers/crypto/qce/dma.c | 5 ++++-
drivers/crypto/qce/dma.h | 4 +++-
3 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c
index ad37c2b8ae53a373bb248aff06c3b7946e8439a8..a0e2eadc3afd5f83e46724c8bc3e3690146b86ba 100644
--- a/drivers/crypto/qce/core.c
+++ b/drivers/crypto/qce/core.c
@@ -238,7 +238,7 @@ static int qce_crypto_probe(struct platform_device *pdev)
if (ret)
return ret;
- ret = devm_qce_dma_request(qce->dev, &qce->dma);
+ ret = devm_qce_dma_request(qce);
if (ret)
return ret;
diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c
index d1daa229361aa74da5d3d7bfe1bc8ab189761e38..d60efb5c26d88f8b0259b1dccc8724d0f75571c6 100644
--- a/drivers/crypto/qce/dma.c
+++ b/drivers/crypto/qce/dma.c
@@ -7,6 +7,7 @@
#include <linux/dmaengine.h>
#include <crypto/scatterwalk.h>
+#include "core.h"
#include "dma.h"
#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE)
@@ -22,8 +23,10 @@ static void qce_dma_release(void *data)
kfree(dma->result_buf);
}
-int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma)
+int devm_qce_dma_request(struct qce_device *qce)
{
+ struct qce_dma_data *dma = &qce->dma;
+ struct device *dev = qce->dev;
int ret;
dma->txchan = dma_request_chan(dev, "tx");
diff --git a/drivers/crypto/qce/dma.h b/drivers/crypto/qce/dma.h
index fc337c435cd14917bdfb99febcf9119275afdeba..483789d9fa98e79d1283de8297bf2fc2a773f3a7 100644
--- a/drivers/crypto/qce/dma.h
+++ b/drivers/crypto/qce/dma.h
@@ -8,6 +8,8 @@
#include <linux/dmaengine.h>
+struct qce_device;
+
/* maximum data transfer block size between BAM and CE */
#define QCE_BAM_BURST_SIZE 64
@@ -32,7 +34,7 @@ struct qce_dma_data {
struct qce_result_dump *result_buf;
};
-int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma);
+int devm_qce_dma_request(struct qce_device *qce);
int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *sg_in,
int in_ents, struct scatterlist *sg_out, int out_ents,
dma_async_tx_callback cb, void *cb_param);
--
2.47.3
^ permalink raw reply related
* [PATCH v20 11/14] crypto: qce - Use existing devres APIs in devm_qce_dma_request()
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski,
Konrad Dybcio
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
From: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Switch to devm_kmalloc() and devm_dma_alloc_chan() in
devm_qce_dma_request(). This allows us to drop two labels and shrink the
function.
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Konrad Dybcio <konrad.dybcio@oss.qualcomm.com>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/crypto/qce/dma.c | 37 +++++++++++--------------------------
1 file changed, 11 insertions(+), 26 deletions(-)
diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c
index d60efb5c26d88f8b0259b1dccc8724d0f75571c6..26347e9fc078adede712722107e74958538accdf 100644
--- a/drivers/crypto/qce/dma.c
+++ b/drivers/crypto/qce/dma.c
@@ -12,49 +12,34 @@
#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE)
-static void qce_dma_release(void *data)
+static void qce_dma_terminate(void *data)
{
struct qce_dma_data *dma = data;
dmaengine_terminate_sync(dma->txchan);
dmaengine_terminate_sync(dma->rxchan);
- dma_release_channel(dma->txchan);
- dma_release_channel(dma->rxchan);
- kfree(dma->result_buf);
}
int devm_qce_dma_request(struct qce_device *qce)
{
struct qce_dma_data *dma = &qce->dma;
struct device *dev = qce->dev;
- int ret;
- dma->txchan = dma_request_chan(dev, "tx");
+ dma->result_buf = devm_kmalloc(dev, QCE_RESULT_BUF_SZ + QCE_IGNORE_BUF_SZ, GFP_KERNEL);
+ if (!dma->result_buf)
+ return -ENOMEM;
+
+ dma->txchan = devm_dma_request_chan(dev, "tx");
if (IS_ERR(dma->txchan))
return dev_err_probe(dev, PTR_ERR(dma->txchan),
"Failed to get TX DMA channel\n");
- dma->rxchan = dma_request_chan(dev, "rx");
- if (IS_ERR(dma->rxchan)) {
- ret = dev_err_probe(dev, PTR_ERR(dma->rxchan),
- "Failed to get RX DMA channel\n");
- goto error_rx;
- }
-
- dma->result_buf = kmalloc(QCE_RESULT_BUF_SZ + QCE_IGNORE_BUF_SZ,
- GFP_KERNEL);
- if (!dma->result_buf) {
- ret = -ENOMEM;
- goto error_nomem;
- }
-
- return devm_add_action_or_reset(dev, qce_dma_release, dma);
+ dma->rxchan = devm_dma_request_chan(dev, "rx");
+ if (IS_ERR(dma->rxchan))
+ return dev_err_probe(dev, PTR_ERR(dma->rxchan),
+ "Failed to get RX DMA channel\n");
-error_nomem:
- dma_release_channel(dma->rxchan);
-error_rx:
- dma_release_channel(dma->txchan);
- return ret;
+ return devm_add_action_or_reset(dev, qce_dma_terminate, dma);
}
struct scatterlist *
--
2.47.3
^ permalink raw reply related
* [PATCH v20 12/14] crypto: qce - Map crypto memory for DMA
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
From: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
As the first step in converting the driver to using DMA for register
I/O, let's map the crypto memory range.
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/crypto/qce/core.c | 23 ++++++++++++++++++++++-
drivers/crypto/qce/core.h | 6 ++++++
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c
index a0e2eadc3afd5f83e46724c8bc3e3690146b86ba..d7b7a3dda464964afe6a6893bb329d5bd5759dcd 100644
--- a/drivers/crypto/qce/core.c
+++ b/drivers/crypto/qce/core.c
@@ -192,10 +192,19 @@ static void qce_cancel_work(void *data)
cancel_work_sync(work);
}
+static void qce_crypto_unmap_dma(void *data)
+{
+ struct qce_device *qce = data;
+
+ dma_unmap_resource(qce->dev, qce->base_dma, qce->dma_size,
+ DMA_BIDIRECTIONAL, 0);
+}
+
static int qce_crypto_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct qce_device *qce;
+ struct resource *res;
int ret;
qce = devm_kzalloc(dev, sizeof(*qce), GFP_KERNEL);
@@ -205,7 +214,7 @@ static int qce_crypto_probe(struct platform_device *pdev)
qce->dev = dev;
platform_set_drvdata(pdev, qce);
- qce->base = devm_platform_ioremap_resource(pdev, 0);
+ qce->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
if (IS_ERR(qce->base))
return PTR_ERR(qce->base);
@@ -255,6 +264,18 @@ static int qce_crypto_probe(struct platform_device *pdev)
qce->async_req_enqueue = qce_async_request_enqueue;
qce->async_req_done = qce_async_request_done;
+ qce->dma_size = resource_size(res);
+ qce->base_dma = dma_map_resource(dev, res->start, qce->dma_size,
+ DMA_BIDIRECTIONAL, 0);
+ qce->base_phys = res->start;
+ ret = dma_mapping_error(dev, qce->base_dma);
+ if (ret)
+ return ret;
+
+ ret = devm_add_action_or_reset(qce->dev, qce_crypto_unmap_dma, qce);
+ if (ret)
+ return ret;
+
return devm_qce_register_algs(qce);
}
diff --git a/drivers/crypto/qce/core.h b/drivers/crypto/qce/core.h
index f092ce2d3b04a936a37805c20ac5ba78d8fdd2df..a80e12eac6c87e5321cce16c56a4bf5003474ef0 100644
--- a/drivers/crypto/qce/core.h
+++ b/drivers/crypto/qce/core.h
@@ -27,6 +27,9 @@
* @dma: pointer to dma data
* @burst_size: the crypto burst size
* @pipe_pair_id: which pipe pair id the device using
+ * @base_dma: base DMA address
+ * @base_phys: base physical address
+ * @dma_size: size of memory mapped for DMA
* @async_req_enqueue: invoked by every algorithm to enqueue a request
* @async_req_done: invoked by every algorithm to finish its request
*/
@@ -43,6 +46,9 @@ struct qce_device {
struct qce_dma_data dma;
int burst_size;
unsigned int pipe_pair_id;
+ dma_addr_t base_dma;
+ phys_addr_t base_phys;
+ size_t dma_size;
int (*async_req_enqueue)(struct qce_device *qce,
struct crypto_async_request *req);
void (*async_req_done)(struct qce_device *qce, int ret);
--
2.47.3
^ permalink raw reply related
* [PATCH v20 13/14] crypto: qce - Add BAM DMA support for crypto register I/O
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
From: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Switch to using BAM DMA for register I/O in addition to passing data. To
that end: provide the necessary infrastructure in the driver, modify the
ordering of operations as required and replace all direct register writes
with wrappers queueing DMA command descriptors.
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/crypto/qce/aead.c | 10 ++--
drivers/crypto/qce/common.c | 20 ++++---
drivers/crypto/qce/dma.c | 120 ++++++++++++++++++++++++++++++++++++++++--
drivers/crypto/qce/dma.h | 5 ++
drivers/crypto/qce/sha.c | 10 ++--
drivers/crypto/qce/skcipher.c | 10 ++--
6 files changed, 144 insertions(+), 31 deletions(-)
diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c
index 1461a08e6c58b00e60aa35515f3392c096726f6a..544a3cf8709248a5f3eb2b669e30b09183d3a69d 100644
--- a/drivers/crypto/qce/aead.c
+++ b/drivers/crypto/qce/aead.c
@@ -463,17 +463,17 @@ qce_aead_async_req_handle(struct crypto_async_request *async_req)
src_nents = dst_nents - 1;
}
- ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, rctx->dst_sg, dst_nents,
- qce_aead_done, async_req);
+ ret = qce_start(async_req, tmpl->crypto_alg_type);
if (ret)
goto error_unmap_src;
- qce_dma_issue_pending(&qce->dma);
-
- ret = qce_start(async_req, tmpl->crypto_alg_type);
+ ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, rctx->dst_sg, dst_nents,
+ qce_aead_done, async_req);
if (ret)
goto error_terminate;
+ qce_dma_issue_pending(&qce->dma);
+
return 0;
error_terminate:
diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c
index 54a78a57f63028f01870a3edeb8e390f523bb190..37bb6f03244d317a887aeb0aa10cefe327b4ce05 100644
--- a/drivers/crypto/qce/common.c
+++ b/drivers/crypto/qce/common.c
@@ -25,7 +25,7 @@ static inline u32 qce_read(struct qce_device *qce, u32 offset)
static inline void qce_write(struct qce_device *qce, u32 offset, u32 val)
{
- writel(val, qce->base + offset);
+ qce_write_dma(qce, offset, val);
}
static inline void qce_write_array(struct qce_device *qce, u32 offset,
@@ -82,6 +82,8 @@ static void qce_setup_config(struct qce_device *qce)
{
u32 config;
+ qce_clear_bam_transaction(qce);
+
/* get big endianness */
config = qce_config_reg(qce, 0);
@@ -90,12 +92,14 @@ static void qce_setup_config(struct qce_device *qce)
qce_write(qce, REG_CONFIG, config);
}
-static inline void qce_crypto_go(struct qce_device *qce, bool result_dump)
+static inline int qce_crypto_go(struct qce_device *qce, bool result_dump)
{
if (result_dump)
qce_write(qce, REG_GOPROC, BIT(GO_SHIFT) | BIT(RESULTS_DUMP_SHIFT));
else
qce_write(qce, REG_GOPROC, BIT(GO_SHIFT));
+
+ return qce_submit_cmd_desc(qce);
}
#if defined(CONFIG_CRYPTO_DEV_QCE_SHA) || defined(CONFIG_CRYPTO_DEV_QCE_AEAD)
@@ -223,9 +227,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req)
config = qce_config_reg(qce, 1);
qce_write(qce, REG_CONFIG, config);
- qce_crypto_go(qce, true);
-
- return 0;
+ return qce_crypto_go(qce, true);
}
#endif
@@ -386,9 +388,7 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req)
config = qce_config_reg(qce, 1);
qce_write(qce, REG_CONFIG, config);
- qce_crypto_go(qce, true);
-
- return 0;
+ return qce_crypto_go(qce, true);
}
#endif
@@ -535,9 +535,7 @@ static int qce_setup_regs_aead(struct crypto_async_request *async_req)
qce_write(qce, REG_CONFIG, config);
/* Start the process */
- qce_crypto_go(qce, !IS_CCM(flags));
-
- return 0;
+ return qce_crypto_go(qce, !IS_CCM(flags));
}
#endif
diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c
index 26347e9fc078adede712722107e74958538accdf..1b43c56503334154be4b8000e5a9330b2005cb64 100644
--- a/drivers/crypto/qce/dma.c
+++ b/drivers/crypto/qce/dma.c
@@ -4,6 +4,8 @@
*/
#include <linux/device.h>
+#include <linux/dma/qcom_bam_dma.h>
+#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <crypto/scatterwalk.h>
@@ -11,6 +13,96 @@
#include "dma.h"
#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE)
+#define QCE_BAM_CMD_SGL_SIZE 128
+#define QCE_BAM_CMD_ELEMENT_SIZE 128
+
+struct qce_desc_info {
+ struct dma_async_tx_descriptor *dma_desc;
+ enum dma_data_direction dir;
+};
+
+struct qce_bam_transaction {
+ struct bam_cmd_element bam_ce[QCE_BAM_CMD_ELEMENT_SIZE];
+ struct scatterlist wr_sgl[QCE_BAM_CMD_SGL_SIZE];
+ struct qce_desc_info *desc;
+ u32 bam_ce_idx;
+ u32 pre_bam_ce_idx;
+ u32 wr_sgl_cnt;
+};
+
+void qce_clear_bam_transaction(struct qce_device *qce)
+{
+ struct qce_bam_transaction *bam_txn = qce->dma.bam_txn;
+
+ bam_txn->bam_ce_idx = 0;
+ bam_txn->wr_sgl_cnt = 0;
+ bam_txn->pre_bam_ce_idx = 0;
+}
+
+int qce_submit_cmd_desc(struct qce_device *qce)
+{
+ struct qce_desc_info *qce_desc = qce->dma.bam_txn->desc;
+ struct qce_bam_transaction *bam_txn = qce->dma.bam_txn;
+ struct dma_async_tx_descriptor *dma_desc;
+ struct dma_chan *chan = qce->dma.rxchan;
+ unsigned long attrs = DMA_PREP_CMD;
+ dma_cookie_t cookie;
+ unsigned int mapped;
+ int ret;
+
+ mapped = dma_map_sg(qce->dev, bam_txn->wr_sgl, bam_txn->wr_sgl_cnt, DMA_TO_DEVICE);
+ if (!mapped)
+ return -ENOMEM;
+
+ dma_desc = dmaengine_prep_slave_sg(chan, bam_txn->wr_sgl, mapped, DMA_MEM_TO_DEV, attrs);
+ if (!dma_desc) {
+ ret = -ENOMEM;
+ goto err_unmap_sg;
+ }
+
+ qce_desc->dma_desc = dma_desc;
+ cookie = dmaengine_submit(qce_desc->dma_desc);
+
+ ret = dma_submit_error(cookie);
+ if (ret)
+ goto err_unmap_sg;
+
+ return 0;
+
+err_unmap_sg:
+ dma_unmap_sg(qce->dev, bam_txn->wr_sgl, bam_txn->wr_sgl_cnt, DMA_TO_DEVICE);
+ return ret;
+}
+
+static void qce_prep_dma_cmd_desc(struct qce_device *qce, struct qce_dma_data *dma,
+ unsigned int addr, void *buf)
+{
+ struct qce_bam_transaction *bam_txn = dma->bam_txn;
+ struct bam_cmd_element *bam_ce_buf;
+ int bam_ce_size, cnt, idx;
+
+ idx = bam_txn->bam_ce_idx;
+ bam_ce_buf = &bam_txn->bam_ce[idx];
+ bam_prep_ce_le32(bam_ce_buf, addr, BAM_WRITE_COMMAND, *((__le32 *)buf));
+
+ bam_ce_buf = &bam_txn->bam_ce[bam_txn->pre_bam_ce_idx];
+ bam_txn->bam_ce_idx++;
+ bam_ce_size = (bam_txn->bam_ce_idx - bam_txn->pre_bam_ce_idx) * sizeof(*bam_ce_buf);
+
+ cnt = bam_txn->wr_sgl_cnt;
+
+ sg_set_buf(&bam_txn->wr_sgl[cnt], bam_ce_buf, bam_ce_size);
+
+ ++bam_txn->wr_sgl_cnt;
+ bam_txn->pre_bam_ce_idx = bam_txn->bam_ce_idx;
+}
+
+void qce_write_dma(struct qce_device *qce, unsigned int offset, u32 val)
+{
+ unsigned int reg_addr = ((unsigned int)(qce->base_phys) + offset);
+
+ qce_prep_dma_cmd_desc(qce, &qce->dma, reg_addr, &val);
+}
static void qce_dma_terminate(void *data)
{
@@ -39,6 +131,16 @@ int devm_qce_dma_request(struct qce_device *qce)
return dev_err_probe(dev, PTR_ERR(dma->rxchan),
"Failed to get RX DMA channel\n");
+ dma->bam_txn = devm_kzalloc(dev, sizeof(*dma->bam_txn), GFP_KERNEL);
+ if (!dma->bam_txn)
+ return -ENOMEM;
+
+ dma->bam_txn->desc = devm_kzalloc(dev, sizeof(*dma->bam_txn->desc), GFP_KERNEL);
+ if (!dma->bam_txn->desc)
+ return -ENOMEM;
+
+ sg_init_table(dma->bam_txn->wr_sgl, QCE_BAM_CMD_SGL_SIZE);
+
return devm_add_action_or_reset(dev, qce_dma_terminate, dma);
}
@@ -98,28 +200,36 @@ int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *rx_sg,
{
struct dma_chan *rxchan = dma->rxchan;
struct dma_chan *txchan = dma->txchan;
- unsigned long flags = DMA_PREP_INTERRUPT | DMA_CTRL_ACK;
+ unsigned long txflags = DMA_PREP_INTERRUPT | DMA_CTRL_ACK;
+ unsigned long rxflags = txflags | DMA_PREP_FENCE;
int ret;
- ret = qce_dma_prep_sg(rxchan, rx_sg, rx_nents, flags, DMA_MEM_TO_DEV,
+ ret = qce_dma_prep_sg(rxchan, rx_sg, rx_nents, rxflags, DMA_MEM_TO_DEV,
NULL, NULL);
if (ret)
return ret;
- return qce_dma_prep_sg(txchan, tx_sg, tx_nents, flags, DMA_DEV_TO_MEM,
+ return qce_dma_prep_sg(txchan, tx_sg, tx_nents, txflags, DMA_DEV_TO_MEM,
cb, cb_param);
}
void qce_dma_issue_pending(struct qce_dma_data *dma)
{
- dma_async_issue_pending(dma->rxchan);
dma_async_issue_pending(dma->txchan);
+ dma_async_issue_pending(dma->rxchan);
}
int qce_dma_terminate_all(struct qce_dma_data *dma)
{
+ struct qce_device *qce = container_of(dma, struct qce_device, dma);
+ struct qce_bam_transaction *bam_txn = dma->bam_txn;
int ret;
ret = dmaengine_terminate_all(dma->rxchan);
- return ret ?: dmaengine_terminate_all(dma->txchan);
+ if (ret)
+ return ret;
+
+ dma_unmap_sg(qce->dev, bam_txn->wr_sgl, bam_txn->wr_sgl_cnt, DMA_TO_DEVICE);
+
+ return dmaengine_terminate_all(dma->txchan);
}
diff --git a/drivers/crypto/qce/dma.h b/drivers/crypto/qce/dma.h
index 483789d9fa98e79d1283de8297bf2fc2a773f3a7..f05dfa9e6b25bd60e32f45079a8bc7e6a4cf81f9 100644
--- a/drivers/crypto/qce/dma.h
+++ b/drivers/crypto/qce/dma.h
@@ -8,6 +8,7 @@
#include <linux/dmaengine.h>
+struct qce_bam_transaction;
struct qce_device;
/* maximum data transfer block size between BAM and CE */
@@ -32,6 +33,7 @@ struct qce_dma_data {
struct dma_chan *txchan;
struct dma_chan *rxchan;
struct qce_result_dump *result_buf;
+ struct qce_bam_transaction *bam_txn;
};
int devm_qce_dma_request(struct qce_device *qce);
@@ -43,5 +45,8 @@ int qce_dma_terminate_all(struct qce_dma_data *dma);
struct scatterlist *
qce_sgtable_add(struct sg_table *sgt, struct scatterlist *sg_add,
unsigned int max_len);
+void qce_write_dma(struct qce_device *qce, unsigned int offset, u32 val);
+int qce_submit_cmd_desc(struct qce_device *qce);
+void qce_clear_bam_transaction(struct qce_device *qce);
#endif /* _DMA_H_ */
diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c
index 5476d4d30fae7eb72bbcbcdd7d8be7a76f6732c2..5cfd769a59a791a79da42e2a5b0554ad974f7631 100644
--- a/drivers/crypto/qce/sha.c
+++ b/drivers/crypto/qce/sha.c
@@ -109,17 +109,17 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req)
goto error_unmap_src;
}
- ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents,
- &rctx->result_sg, 1, qce_ahash_done, async_req);
+ ret = qce_start(async_req, tmpl->crypto_alg_type);
if (ret)
goto error_unmap_dst;
- qce_dma_issue_pending(&qce->dma);
-
- ret = qce_start(async_req, tmpl->crypto_alg_type);
+ ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents,
+ &rctx->result_sg, 1, qce_ahash_done, async_req);
if (ret)
goto error_terminate;
+ qce_dma_issue_pending(&qce->dma);
+
return 0;
error_terminate:
diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c
index a9b59e68df4b6837805d45391f5a5fe43fd47709..b4ef3748fbb4dde542b0307f32d4c871b7c33ac2 100644
--- a/drivers/crypto/qce/skcipher.c
+++ b/drivers/crypto/qce/skcipher.c
@@ -142,18 +142,18 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req)
src_nents = dst_nents - 1;
}
+ ret = qce_start(async_req, tmpl->crypto_alg_type);
+ if (ret)
+ goto error_unmap_src;
+
ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents,
rctx->dst_sg, dst_nents,
qce_skcipher_done, async_req);
if (ret)
- goto error_unmap_src;
+ goto error_terminate;
qce_dma_issue_pending(&qce->dma);
- ret = qce_start(async_req, tmpl->crypto_alg_type);
- if (ret)
- goto error_terminate;
-
return 0;
error_terminate:
--
2.47.3
^ permalink raw reply related
* [PATCH v20 14/14] crypto: qce - Communicate the base physical address to the dmaengine
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
In order to communicate to the BAM DMA engine which address should be
used as a scratchpad for dummy writes related to BAM pipe locking,
fill out and attach the provided metadata struct to the descriptor.
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/crypto/qce/dma.c | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c
index 1b43c56503334154be4b8000e5a9330b2005cb64..6410f8dc5bcf517223c768a3e8f87af245076c84 100644
--- a/drivers/crypto/qce/dma.c
+++ b/drivers/crypto/qce/dma.c
@@ -11,6 +11,7 @@
#include "core.h"
#include "dma.h"
+#include "regs-v5.h"
#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE)
#define QCE_BAM_CMD_SGL_SIZE 128
@@ -41,6 +42,10 @@ void qce_clear_bam_transaction(struct qce_device *qce)
int qce_submit_cmd_desc(struct qce_device *qce)
{
+ struct bam_desc_metadata meta = {
+ .scratchpad_addr = qce->base_phys + REG_VERSION,
+ .direction = DMA_MEM_TO_DEV,
+ };
struct qce_desc_info *qce_desc = qce->dma.bam_txn->desc;
struct qce_bam_transaction *bam_txn = qce->dma.bam_txn;
struct dma_async_tx_descriptor *dma_desc;
@@ -60,15 +65,21 @@ int qce_submit_cmd_desc(struct qce_device *qce)
goto err_unmap_sg;
}
+ ret = dmaengine_desc_attach_metadata(dma_desc, &meta, sizeof(meta));
+ if (ret)
+ goto err_free_desc;
+
qce_desc->dma_desc = dma_desc;
cookie = dmaengine_submit(qce_desc->dma_desc);
ret = dma_submit_error(cookie);
if (ret)
- goto err_unmap_sg;
+ goto err_free_desc;
return 0;
+err_free_desc:
+ dmaengine_desc_free(dma_desc);
err_unmap_sg:
dma_unmap_sg(qce->dev, bam_txn->wr_sgl, bam_txn->wr_sgl_cnt, DMA_TO_DEVICE);
return ret;
--
2.47.3
^ permalink raw reply related
* [PATCH v20 03/14] dmaengine: qcom: bam_dma: convert tasklet to a BH workqueue
From: Bartosz Golaszewski @ 2026-06-29 10:01 UTC (permalink / raw)
To: Vinod Koul, Jonathan Corbet, Thara Gopinath, Herbert Xu,
David S. Miller, Udit Tiwari, Md Sadre Alam, Dmitry Baryshkov,
Manivannan Sadhasivam, Stephan Gerhold, Bjorn Andersson,
Peter Ujfalusi, Michal Simek, Frank Li, Andy Gross,
Neil Armstrong
Cc: dmaengine, linux-doc, linux-kernel, linux-arm-msm, linux-crypto,
linux-arm-kernel, brgl, Bartosz Golaszewski, Bartosz Golaszewski,
Dmitry Baryshkov
In-Reply-To: <20260629-qcom-qce-cmd-descr-v20-0-56f67da84c05@oss.qualcomm.com>
BH workqueues are a modern mechanism, aiming to replace legacy tasklets.
Let's convert the BAM DMA driver to using the high-priority variant of
the BH workqueue.
[Vinod: suggested using the BG workqueue instead of the regular one
running in process context]
Suggested-by: Vinod Koul <vkoul@kernel.org>
Reviewed-by: Dmitry Baryshkov <dmitry.baryshkov@oss.qualcomm.com>
Reviewed-by: Bjorn Andersson <andersson@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@linaro.org>
Reviewed-by: Manivannan Sadhasivam <mani@kernel.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>
---
drivers/dma/qcom/bam_dma.c | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index fc155e0d1870cbb7e099a2c4280f9f8fbdf6cf15..ea3df28e777f99c0532761b6aee6807ab23ab4ca 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -42,6 +42,7 @@
#include <linux/pm_runtime.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
+#include <linux/workqueue.h>
#include "../dmaengine.h"
#include "../virt-dma.h"
@@ -426,8 +427,8 @@ struct bam_device {
struct clk *bamclk;
int irq;
- /* dma start transaction tasklet */
- struct tasklet_struct task;
+ /* dma start transaction workqueue */
+ struct work_struct work;
};
/**
@@ -892,7 +893,7 @@ static u32 process_channel_irqs(struct bam_device *bdev)
/*
* if complete, process cookie. Otherwise
* push back to front of desc_issued so that
- * it gets restarted by the tasklet
+ * it gets restarted by the work queue.
*/
if (!async_desc->num_desc) {
vchan_cookie_complete(&async_desc->vd);
@@ -922,9 +923,9 @@ static irqreturn_t bam_dma_irq(int irq, void *data)
srcs |= process_channel_irqs(bdev);
- /* kick off tasklet to start next dma transfer */
+ /* kick off the work queue to start next dma transfer */
if (srcs & P_IRQ)
- tasklet_schedule(&bdev->task);
+ queue_work(system_bh_highpri_wq, &bdev->work);
ret = pm_runtime_get_sync(bdev->dev);
if (ret < 0)
@@ -1120,14 +1121,14 @@ static void bam_start_dma(struct bam_chan *bchan)
}
/**
- * dma_tasklet - DMA IRQ tasklet
- * @t: tasklet argument (bam controller structure)
+ * bam_dma_work() - DMA interrupt work queue callback
+ * @work: work queue struct embedded in the BAM controller device struct
*
* Sets up next DMA operation and then processes all completed transactions
*/
-static void dma_tasklet(struct tasklet_struct *t)
+static void bam_dma_work(struct work_struct *work)
{
- struct bam_device *bdev = from_tasklet(bdev, t, task);
+ struct bam_device *bdev = from_work(bdev, work, work);
struct bam_chan *bchan;
unsigned int i;
@@ -1140,14 +1141,13 @@ static void dma_tasklet(struct tasklet_struct *t)
if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))
bam_start_dma(bchan);
}
-
}
/**
* bam_issue_pending - starts pending transactions
* @chan: dma channel
*
- * Calls tasklet directly which in turn starts any pending transactions
+ * Calls work queue directly which in turn starts any pending transactions
*/
static void bam_issue_pending(struct dma_chan *chan)
{
@@ -1316,14 +1316,14 @@ static int bam_dma_probe(struct platform_device *pdev)
if (ret)
goto err_disable_clk;
- tasklet_setup(&bdev->task, dma_tasklet);
+ INIT_WORK(&bdev->work, bam_dma_work);
bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels,
sizeof(*bdev->channels), GFP_KERNEL);
if (!bdev->channels) {
ret = -ENOMEM;
- goto err_tasklet_kill;
+ goto err_workqueue_cancel;
}
/* allocate and initialize channels */
@@ -1389,8 +1389,8 @@ static int bam_dma_probe(struct platform_device *pdev)
err_bam_channel_exit:
for (i = 0; i < bdev->num_channels; i++)
tasklet_kill(&bdev->channels[i].vc.task);
-err_tasklet_kill:
- tasklet_kill(&bdev->task);
+err_workqueue_cancel:
+ cancel_work_sync(&bdev->work);
err_disable_clk:
clk_disable_unprepare(bdev->bamclk);
@@ -1424,7 +1424,7 @@ static void bam_dma_remove(struct platform_device *pdev)
bdev->channels[i].fifo_phys);
}
- tasklet_kill(&bdev->task);
+ cancel_work_sync(&bdev->work);
clk_disable_unprepare(bdev->bamclk);
}
--
2.47.3
^ permalink raw reply related
* Re: [RFC PATCH 00/40] mm: reliable 1GB page allocation
From: Vlastimil Babka (SUSE) @ 2026-06-29 10:03 UTC (permalink / raw)
To: Lorenzo Stoakes, Rik van Riel
Cc: linux-kernel, kernel-team, linux-mm, david, willy, surenb, hannes,
ziy, usama.arif, fvdl, Andrew Morton, Jonathan Corbet,
Chris Mason, David Sterba, Steven Rostedt, Masami Hiramatsu,
Rafael J. Wysocki, Oscar Salvador, Mike Rapoport, linux-doc,
linux-btrfs, linux-trace-kernel, linux-pm, linux-cxl,
Linus Torvalds
In-Reply-To: <akIjA_dqh4OHAYo4@lucifer>
On 6/29/26 11:29, Lorenzo Stoakes wrote:
> TL;DR - please don't send unfiltered LLM code to list _at all_. If you want
> to share it, link to a repo.
>
> On Sat, Jun 27, 2026 at 09:36:51AM -0400, Rik van Riel wrote:
>> That is the one reason I sent out RFC code before it
>> is ready. I am looking for feedback on the concepts
>> in this series.
> ...
>> Once I know what I need to do, coming up with a
>> cleaner implementation is very doable.
> ...
>> The mess in the RFC is the result of trying something
>> that seemed right, watching it fail in some subtle
>> way, and trying to fix it up.
> ...
>> > But the execution has to be _completely_ rethought.
>>
>> There's no argument there.
> ...
>> > Another issue here is maintainer time - even this _extremely_ light-
>> > touch
>> > review has taken me a few hours (of my weekend :). To review it in
>> > detail
>> > would take probably DAYS of dedicated work.
>>
>> I suspect there is a mismatch in expectations here.
>>
>> I already knew this code has to be totally redone.
>
> I'm glad we are in agreement on this :)
>
> But in general I feel you have sent this and at least one other series like this
> without being as clear as you should have been.
>
> I hate to belabour the point but just to be clear:
>
> * You label one patch [DO-NOT-MERGE], but none of the others (implying they
> are candidates for being merged) [0] and the cover letter has TODOs,
> including trivia like naming, but nothing about the code.
>
> * You sent a non-RFC series with identical code quality issues [1]
> recently.
>
> * Until I pointed it out, you were responding to other review here as if
> the series was genuinely was intended for (eventual) merge:
>
> - "This is a userspace-visible removal. Writes to
> /proc/sys/vm/watermark_boost_factor will now return -ENOENT instead of
> being accepted, breaking userspace." [2]
>
> <-: "I'll just drop this patch for now." [3]
>
> - "I left a small code nit inline, but whether you take that suggestion
> or leave it, you can add Reviewed-by: ..." [4]
>
> <-: "I sent it with this series mostly because it's needed to make the
> series work, and to provide context on why it's needed. I'm happy to
> resend it with a GFP mask passed in by each caller. That would look
> better, indeed!" [5]
>
> So to be concrete, if you send really rough code, Use [pre-RFC] or [DO NOT
> MERGE] (on the series as a whole) to make that clear and say so in the
> cover letter VERY VERY clearly.
Yes please. [POC NOT-FOR-MERGE] perhaps?
> Or, you can put it in a repo somewhere and link it in an email discussing
> the concepts (like I did with scalable CoW for instance).
Indeed.
> As above, firstly make it clear that the code you are sending for review is
> not to be reviewed so people don't waste highly contended maintainer time
> on that! :)
>
> Also, you didn't respond to my point regarding cc'ing the right people -
> but that's clearly something you need to get right if you want this kind of
> feedback to start with.
>
> For instance, you didn't cc- the page allocator maintainer (Vlastimil) on a
> series that is fundamentally changing the page allocator. That's not going
> to help with feedback.
Right! Thanks a lot for adding me, Lorenzo.
> In general, this area of the page allocator and compaction isn't my
> specialism in the kernel so I can't give you the in-depth feedback you need
> on that.
>
> But I do have thoughts in general as to how to achieve what you want here:
>
> Firstly - you should try to summarise what you're doing here and what
> you're changing alongside the trade-offs as clearly as you can in the cover
> letter.
>
> Then highlight what it is you need feedback on, broken out into clear
> questions or points that make it easy for people to respond to.
Yep.
> And _you have already done this_ in your reply here:
>
> * "How do people feel about splitting up the free lists, so each gigabyte
> (well, PUD sized) chunk of memory has its own free lists?"
My immediate response is that now we'd need to search multiple sets of lists
instead of a single one? What about the overhead?
Having a POC (even vibe-coded) for measuring that overhead might be actually
useful to quickly figure out whether the idea is viable or not.
But then the code doesn't need to be sent as a huge series if it's not for
review. As Lorenzo said, git repo link is enough.
> * "How can we balance the desire for higher-order kernel allocations,
> against the desire to preserve gigabyte sized chunks of memory that can
> be used for user space?"
>
> * "How do we balance the desire to keep compaction overhead low with the
> desire to do higher order allocations almost everywhere?"
How can we have a cake and eat it too? :)
> I think a really good way of doing this would be to start out with
> something like:
>
> Right now compaction often fails to achieve what we need, with
> fragmentation occurring anyway and (for instance) THP stalling on
> the availability of higher order folios.
>
> etc. etc.
>
> Summarising _the problem_.
>
> Then a section about your proposed solution, e.g.:
>
> I propose a means by which we proactively achieve gigabyte-sized
> pageblocks with logic which maintains these as physically
> contiguous under both ordinary and contended workloads
>
> Then list out the "secret sauce" of your approach, e.g.:
>
> This works by arranging memory such that unmovable allocations are
> grouped at <blah blah blah> etc.
>
> Then raise your questions e.g.:
>
> I'd like to ask the community - how do people feel about splitting
> up the free lists, so each gigabyte (well, PUD sized) chunk of
> memory has its own free lists? <etc. etc.>
>
> Then make it clear whether this is an RFC that is ready for primetime or
> not:
>
> This series is simply intended as a proof-of-concept - PLEASE DO
> NOT REVIEW THE CODE per-se, but rather comment on the concepts!
>
> (And obviously as above, if that _is_ what you intend, underline it with
> [DO NOT MERGE] or [pre-RFC] or something like that).
Ack.
> I'd also very strongly suggest (as I did in my original reply) breaking out
> parts that can be broken out as prerequisite series.
>
> If you're doing something good or useful _anyway_ then just send that
> separately first, and have later work rely on the earlier work.
Ack.
> There's no rush, this is huge and will take time.
>
> A final KEY point:
>
> NEVER submit unfiltered code generated by an LLMs to the list in _any_
> form. If you want people to access code like that to test or something,
> then put it in a remote repo and link to it.
>
> The code is SO overly complicated and SO messy that it's really difficult
> for people to understand what's actually going on.
>
> At the heart of what you need here is CLARITY.
>
> You need to CLEARLY communicate what it is you're doing so busy maintainers
> can examine it. That's the _only_ way you're going to get something like
> this merged.
>
> The LLM-generated code is so awful that ain't nobody got the time to try to
> understand what it's doing.
Indeed.
> The workload for this really has to be on submitters, not maintainers.
>
> And what you've done, even if not intended, is workslopping, and that's
> really not acceptable. Quoting the kernel process on tool-generated content
> [6]:
>
> "If tools permit you to generate a contribution automatically, expect
> additional scrutiny in proportion to how much of it was generated.
>
> As with the output of any tooling, the result may be incorrect or
> inappropriate. You are expected to understand and to be able to defend
> everything you submit. If you are unable to do so, then do not submit the
> resulting changes.
>
> If you do so anyway, maintainers are entitled to reject your series without
> detailed review."
>
> As per this and my previous reply, AI slop doesn't scale, even as an RFC -
> I won't have time to reply like this in future, and we will just have to
> reject your series out of hand, which helps nobody.
True. Thanks a lot for going out of your way on this!
>>
>>
>> --
>> All Rights Reversed.
>
> Thanks, Lorenzo
>
> [0]:https://lore.kernel.org/all/20260520150018.2491267-41-riel@surriel.com/
> [1]:https://lore.kernel.org/linux-mm/20260616190300.1509639-1-riel@surriel.com/
> [2]:https://lore.kernel.org/all/20260526140204.1390573-1-usama.arif@linux.dev/
> [3]:https://lore.kernel.org/all/2ecf71858845e7d14c718b1a6845389cb78b986e.camel@surriel.com/
> [4]:https://lore.kernel.org/all/20260520174749.GA1458531@zen.localdomain/
> [5]:https://lore.kernel.org/all/daa29c92f055d028a5b3ec0e42cfb1ee1496a593.camel@surriel.com/
> [6]:https://docs.kernel.org/process/generated-content.html
^ permalink raw reply
* Re: [PATCH v4 0/2] arm64: errata: NVIDIA Olympus device store/load ordering
From: Vladimir Murzin @ 2026-06-29 10:45 UTC (permalink / raw)
To: Shanker Donthineni, Catalin Marinas, Will Deacon
Cc: Jason Gunthorpe, linux-arm-kernel, Mark Rutland, linux-kernel,
linux-doc, Vikram Sethi, Jason Sequeira
In-Reply-To: <20260625182425.3194066-1-sdonthineni@nvidia.com>
Hi,
On 6/25/26 19:24, Shanker Donthineni wrote:
> This series works around the NVIDIA Olympus device store/load ordering
> erratum (T410-OLY-1027): a Device-nGnR* load can be observed by a
> peripheral before an older, non-overlapping Device-nGnR* store to the
> same peripheral, breaking the program order that drivers rely on for
> MMIO and potentially leaving a device in an incorrect state.
>
> Patch 1 adds the workaround. It promotes the raw MMIO store helpers
> (__raw_writeb/w/l/q, and therefore writel()/writel_relaxed()) to
> store-release on affected CPUs, and promotes the trailing DGH of the
> write-combining __iowrite{32,64}_copy() helpers to dmb osh. Everything is
> gated on a new ARM64_WORKAROUND_DEVICE_STORE_RELEASE cpucap and patched
> in only on affected parts, so it is a no-op elsewhere.
>
> Patch 2 provides arm64 memset_io()/memcpy_toio(). The generic versions
> are built on __raw_write*(), so patch 1 would promote every store in a
> block to a store-release; as each STLR drains the write-combining buffer,
> block MMIO becomes O(n) store-releases. The arm64 versions emit plain
> STR in the loop and order the whole block with a single trailing dmb osh,
> keeping block MMIO at one-barrier cost.
>
> Performance: NVIDIA Olympus, write-combining MMIO to a device BAR, single
> PE pinned; per-call cost in ns. Consecutive writes ping-pong between two
> buffers so repeated stores are not coalesced. iowrite64/iowrite32 =
> __iowrite{64,32}_copy().
>
> Table 1 - workaround off (CONFIG_NVIDIA_OLYMPUS_1027_ERRATUM=n)
> +-------+-----------+-----------+-----------+-------------+
> | size | iowrite64 | iowrite32 | memset_io | memcpy_toio |
> +-------+-----------+-----------+-----------+-------------+
> | 8B | 67.9 ns | 67.8 ns | 3.6 ns | 3.6 ns |
> | 16B | 67.9 ns | 67.8 ns | 4.0 ns | 4.0 ns |
> | 32B | 67.9 ns | 67.9 ns | 4.6 ns | 4.6 ns |
> | 64B | 69.1 ns | 69.1 ns | 69.1 ns | 69.0 ns |
> | 128B | 138.3 ns | 138.3 ns | 138.4 ns | 138.3 ns |
> | 256B | 276.6 ns | 276.6 ns | 276.6 ns | 276.7 ns |
> | 512B | 276.6 ns | 276.5 ns | 276.6 ns | 276.6 ns |
> | 1KB | 276.6 ns | 278.4 ns | 276.6 ns | 276.6 ns |
> | 2KB | 278.4 ns | 278.4 ns | 275.9 ns | 276.6 ns |
> | 4KB | 365.7 ns | 365.7 ns | 365.7 ns | 365.7 ns |
> +-------+-----------+-----------+-----------+-------------+
> relaxed/no-flush: memset_io()/memcpy_toio() issue plain stores with no
> trailing dgh() or barrier, unlike __iowrite*_copy() which ends with dgh().
>
> Table 2 - workaround on, arm64 memset_io/memcpy_toio (this series)
> +-------+-----------+-----------+-----------+-------------+
> | size | iowrite64 | iowrite32 | memset_io | memcpy_toio |
> +-------+-----------+-----------+-----------+-------------+
> | 8B | 231.6 ns | 231.6 ns | 232.4 ns | 232.4 ns |
> | 16B | 231.7 ns | 231.9 ns | 232.7 ns | 232.6 ns |
> | 32B | 231.9 ns | 232.7 ns | 232.9 ns | 232.9 ns |
> | 64B | 232.7 ns | 235.0 ns | 233.7 ns | 233.6 ns |
> | 128B | 233.6 ns | 235.8 ns | 234.4 ns | 234.3 ns |
> | 256B | 237.7 ns | 276.8 ns | 264.0 ns | 276.7 ns |
> | 512B | 237.7 ns | 277.1 ns | 238.1 ns | 277.6 ns |
> | 1KB | 253.7 ns | 279.3 ns | 276.1 ns | 294.1 ns |
> | 2KB | 295.0 ns | 318.7 ns | 288.5 ns | 308.3 ns |
> | 4KB | 365.9 ns | 381.4 ns | 365.7 ns | 381.3 ns |
> +-------+-----------+-----------+-----------+-------------+
> all four helpers end with a single trailing barrier (dmb osh).
>
> Table 3 - workaround on, generic per-store memset_io/memcpy_toio
> +-------+-----------+-----------+-------------+--------------+
> | size | iowrite64 | iowrite32 | memset_io | memcpy_toio |
> +-------+-----------+-----------+-------------+--------------+
> | 8B | 231.6 ns | 231.6 ns | 229.0 ns | 229.0 ns |
> | 16B | 231.7 ns | 231.9 ns | 458.4 ns | 458.5 ns |
> | 32B | 231.9 ns | 232.7 ns | 917.4 ns | 917.5 ns |
> | 64B | 232.7 ns | 234.8 ns | 1835.4 ns | 1835.5 ns |
> | 128B | 233.6 ns | 235.8 ns | 3670.9 ns | 3670.8 ns |
> | 256B | 237.7 ns | 276.7 ns | 7341.6 ns | 7341.6 ns |
> | 512B | 237.7 ns | 279.4 ns | 14001.4 ns | 14001.3 ns |
> | 1KB | 253.7 ns | 279.1 ns | 28631.5 ns | 28631.8 ns |
> | 2KB | 279.4 ns | 317.9 ns | 57276.3 ns | 57275.2 ns |
> | 4KB | 365.7 ns | 381.5 ns | 114564.4 ns | 114563.6 ns |
> +-------+-----------+-----------+-------------+--------------+
> the generic memset_io()/memcpy_toio() build on __raw_write*(), which the
> workaround promotes to store-release, so every store is individually
> ordered - hence O(n) in the store count.
>
> Tables 2 and 3 show why patch 2 is needed: the generic per-store block
> writers collapse to O(n) under the workaround (4KB ~314x slower, ~115 us
> vs ~366 ns), while the arm64 versions stay flat at one-barrier cost.
That's interesting. With the way the patch set is structured, it
now looks like:
1. Fix the erratum, but cause a performance regression.
2. Restore the performance regression and (re)apply the erratum
workaround.
Would it make sense to avoid introducing the performance
regression in the first place by structuring the patch set
slightly differently?
1. (Re)introduce arm64 memset_io()/memcpy_toio().
2. Fix the erratum once for all
What do you reckon?
Cheers
Vladimir
^ permalink raw reply
* Re: [PATCH v4 2/2] arm64: io: apply the device store-release workaround once per block write
From: Vladimir Murzin @ 2026-06-29 10:48 UTC (permalink / raw)
To: Shanker Donthineni, Catalin Marinas, Will Deacon
Cc: Jason Gunthorpe, linux-arm-kernel, Mark Rutland, linux-kernel,
linux-doc, Vikram Sethi, Jason Sequeira
In-Reply-To: <20260625182425.3194066-3-sdonthineni@nvidia.com>
Hi,
On 6/25/26 19:24, Shanker Donthineni wrote:
> The generic memset_io()/memcpy_toio() are built on __raw_write*(), so on
> parts with the NVIDIA Olympus device store/load ordering erratum the
> ARM64_WORKAROUND_DEVICE_STORE_RELEASE workaround promotes every store in
> the block to a store-release. Each stlr* carries a barrier cost, so block
> MMIO becomes O(n) store-releases, making a block copy many times slower
> than a single ordered burst and growing with the transfer size.
>
> Provide arm64 memset_io()/memcpy_toio() that emit plain str* in the loop
> and order the whole block against subsequent loads with a single
> trailing dmb osh on affected CPUs (a no-op elsewhere, preserving the
> relaxed contract of these helpers). This keeps block MMIO writes at
> one-barrier cost rather than scaling with the transfer size.
>
> Performance (NVIDIA Olympus, write-combining MMIO to a device BAR, single
> PE pinned; per-call cost in ns; consecutive writes ping-pong between two
> buffers so repeated stores are not coalesced; iowrite64/iowrite32 =
> __iowrite{64,32}_copy()):
>
> Table 1 - arm64 memset_io/memcpy_toio (this patch)
> +-------+-----------+-----------+-----------+-------------+
> | size | iowrite64 | iowrite32 | memset_io | memcpy_toio |
> +-------+-----------+-----------+-----------+-------------+
> | 8B | 231.6 ns | 231.6 ns | 232.4 ns | 232.4 ns |
> | 16B | 231.7 ns | 231.9 ns | 232.7 ns | 232.6 ns |
> | 32B | 231.9 ns | 232.7 ns | 232.9 ns | 232.9 ns |
> | 64B | 232.7 ns | 235.0 ns | 233.7 ns | 233.6 ns |
> | 128B | 233.6 ns | 235.8 ns | 234.4 ns | 234.3 ns |
> | 256B | 237.7 ns | 276.8 ns | 264.0 ns | 276.7 ns |
> | 512B | 237.7 ns | 277.1 ns | 238.1 ns | 277.6 ns |
> | 1KB | 253.7 ns | 279.3 ns | 276.1 ns | 294.1 ns |
> | 2KB | 295.0 ns | 318.7 ns | 288.5 ns | 308.3 ns |
> | 4KB | 365.9 ns | 381.4 ns | 365.7 ns | 381.3 ns |
> +-------+-----------+-----------+-----------+-------------+
> all four helpers end with a single trailing barrier (dmb osh).
>
> Table 2 - generic per-store memset_io/memcpy_toio
> +-------+-----------+-----------+-------------+--------------+
> | size | iowrite64 | iowrite32 | memset_io | memcpy_toio |
> +-------+-----------+-----------+-------------+--------------+
> | 8B | 231.6 ns | 231.6 ns | 229.0 ns | 229.0 ns |
> | 16B | 231.7 ns | 231.9 ns | 458.4 ns | 458.5 ns |
> | 32B | 231.9 ns | 232.7 ns | 917.4 ns | 917.5 ns |
> | 64B | 232.7 ns | 234.8 ns | 1835.4 ns | 1835.5 ns |
> | 128B | 233.6 ns | 235.8 ns | 3670.9 ns | 3670.8 ns |
> | 256B | 237.7 ns | 276.7 ns | 7341.6 ns | 7341.6 ns |
> | 512B | 237.7 ns | 279.4 ns | 14001.4 ns | 14001.3 ns |
> | 1KB | 253.7 ns | 279.1 ns | 28631.5 ns | 28631.8 ns |
> | 2KB | 279.4 ns | 317.9 ns | 57276.3 ns | 57275.2 ns |
> | 4KB | 365.7 ns | 381.5 ns | 114564.4 ns | 114563.6 ns |
> +-------+-----------+-----------+-------------+--------------+
> the generic memset_io()/memcpy_toio() build on __raw_write*(), which the
> workaround promotes to store-release, so every store is individually
> ordered - hence O(n) in the store count.
>
> The arm64 versions stay flat at one-barrier cost while the generic
> per-store writers collapse to O(n): at 4KB ~314x slower (~115 us vs
> ~366 ns).
>
> Signed-off-by: Shanker Donthineni <sdonthineni@nvidia.com>
> ---
> arch/arm64/include/asm/io.h | 5 +++
> arch/arm64/kernel/io.c | 82 +++++++++++++++++++++++++++++++++++++
> 2 files changed, 87 insertions(+)
>
> diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
> index 69e0fa004d31..649503f347bc 100644
> --- a/arch/arm64/include/asm/io.h
> +++ b/arch/arm64/include/asm/io.h
> @@ -266,6 +266,11 @@ __iowrite64_copy(void __iomem *to, const void *from, size_t count)
> }
> #define __iowrite64_copy __iowrite64_copy
>
> +void memset_io(volatile void __iomem *dst, int c, size_t count);
> +#define memset_io memset_io
> +void memcpy_toio(volatile void __iomem *dst, const void *src, size_t count);
> +#define memcpy_toio memcpy_toio
> +
> /*
> * I/O memory mapping functions.
> */
> diff --git a/arch/arm64/kernel/io.c b/arch/arm64/kernel/io.c
> index fe86ada23c7d..b5fd9ee6d9eb 100644
> --- a/arch/arm64/kernel/io.c
> +++ b/arch/arm64/kernel/io.c
> @@ -5,9 +5,91 @@
> * Copyright (C) 2012 ARM Ltd.
> */
>
> +#include <linux/align.h>
> #include <linux/export.h>
> #include <linux/types.h>
> #include <linux/io.h>
> +#include <linux/unaligned.h>
> +
> +#include <asm/alternative.h>
> +
> +/*
> + * ARM64_WORKAROUND_DEVICE_STORE_RELEASE promotes every raw MMIO store
> + * (__raw_write*()) to a store-release on affected CPUs. The generic
> + * memset_io()/memcpy_toio() are built on those helpers, so the workaround would
> + * emit one store-release per element and turn a block write into O(n) ordered
> + * stores - far more costly than the single barrier a block actually needs.
> + *
> + * Provide arm64 versions that emit plain STR in the loop and order the whole
> + * block against subsequent loads with one trailing DMB OSH, patched in only on
> + * affected CPUs (a no-op elsewhere, so the relaxed contract of these helpers is
> + * preserved).
> + *
> + * This capability is currently enabled only for the NVIDIA Olympus device
> + * store/load ordering erratum, where a Device-nGnR* load may be observed before
> + * an older, non-overlapping Device-nGnR* store to the same peripheral.
> + */
> +static __always_inline void iomem_block_store_barrier(void)
> +{
> + asm volatile(ALTERNATIVE("nop", "dmb osh",
> + ARM64_WORKAROUND_DEVICE_STORE_RELEASE)
> + : : : "memory");
> +}
> +
> +void memset_io(volatile void __iomem *dst, int c, size_t count)
> +{
> + u64 qc = (u8)c;
> +
> + qc *= ~0ULL / 0xff;
> +
> + while (count && !IS_ALIGNED((__force unsigned long)dst, sizeof(u64))) {
> + asm volatile("strb %w0, [%1]" : : "rZ"((u8)c), "r"(dst) : "memory");
> + dst++;
> + count--;
> + }
> + while (count >= sizeof(u64)) {
> + asm volatile("str %x0, [%1]" : : "rZ"(qc), "r"(dst) : "memory");
> + dst += sizeof(u64);
> + count -= sizeof(u64);
> + }
> + while (count) {
> + asm volatile("strb %w0, [%1]" : : "rZ"((u8)c), "r"(dst) : "memory");
> + dst++;
> + count--;
> + }
> +
> + iomem_block_store_barrier();
> +}
> +EXPORT_SYMBOL(memset_io);
> +
> +void memcpy_toio(volatile void __iomem *dst, const void *src, size_t count)
> +{
> + while (count && !IS_ALIGNED((__force unsigned long)dst, sizeof(u64))) {
> + asm volatile("strb %w0, [%1]"
> + : : "rZ"(*(const u8 *)src), "r"(dst) : "memory");
> + src++;
> + dst++;
> + count--;
> + }
> + while (count >= sizeof(u64)) {
> + asm volatile("str %x0, [%1]"
> + : : "rZ"(get_unaligned((const u64 *)src)), "r"(dst)
Why do we need get_unaligned() here? I understand this came from
the generic implementation, where it needs to handle architectures
that do not support unaligned accesses. But IIUC this is not an
issue for arm64, and there was no special handling in memcpy_toio()
before 0110feaaf6d0 ("arm64: Use new fallback IO memcpy/memset").
Am I missing something?
> + : "memory");
> + src += sizeof(u64);
> + dst += sizeof(u64);
> + count -= sizeof(u64);
> + }
> + while (count) {
> + asm volatile("strb %w0, [%1]"
> + : : "rZ"(*(const u8 *)src), "r"(dst) : "memory");
> + src++;
> + dst++;
> + count--;
> + }
> +
> + iomem_block_store_barrier();
It is perhaps a matter of taste, but having the inline assembly
here (and in memset_io()) might make the code clearer. To a
casual reader, it would be obvious that the barrier is not
guaranteed and is only applicable to ARM64_WORKAROUND_DEVICE_STORE_RELEASE,
without having to jump back and forth through the code.
Obliviously maintainers might have different preference ;)
Cheers
Vladimir
> +}
> +EXPORT_SYMBOL(memcpy_toio);
>
> /*
> * This generates a memcpy that works on a from/to address which is aligned to
> -- 2.54.0.windows.1
>
^ permalink raw reply
* [PATCH v2] Documentation: admin-guide: fix brackets and translation issue
From: Manuel Ebner @ 2026-06-29 11:08 UTC (permalink / raw)
To: Jonathan Corbet, Shuah Khan, Mauro Carvalho Chehab, linux-media,
open list:DOCUMENTATION, open list
Cc: Manuel Ebner
Add missing ']' and replace 'neuer Name' with 'new Name'.
Signed-off-by: Manuel Ebner <manuelebner@mailbox.org>
---
->[v2]
Fixed one-line summary
---
Documentation/admin-guide/kernel-parameters.txt | 6 +++---
Documentation/admin-guide/media/bttv.rst | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 00375193bd26..17363d525ae3 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -6414,9 +6414,9 @@ Kernel parameters
reboot= [KNL]
Format (x86 or x86_64):
[w[arm] | c[old] | h[ard] | s[oft] | g[pio]] | d[efault] \
- [[,]s[mp]#### \
+ [[,]s[mp]####] \
[[,]b[ios] | a[cpi] | k[bd] | t[riple] | e[fi] | p[ci]] \
- [[,]f[orce]
+ [[,]f[orce]]
Where reboot_mode is one of warm (soft) or cold (hard) or gpio
(prefix with 'panic_' to set mode for panic
reboot only),
@@ -6917,7 +6917,7 @@ Kernel parameters
apic=verbose is specified.
Example: apic=debug show_lapic=all
- slab_debug[=options[,slabs][;[options[,slabs]]...] [MM]
+ slab_debug[=options[,slabs][;[options[,slabs]]...]] [MM]
Enabling slab_debug allows one to determine the
culprit if slab objects become corrupted. Enabling
slab_debug can create guard zones around objects and
diff --git a/Documentation/admin-guide/media/bttv.rst b/Documentation/admin-guide/media/bttv.rst
index 58cbaf6df694..78c3e560b806 100644
--- a/Documentation/admin-guide/media/bttv.rst
+++ b/Documentation/admin-guide/media/bttv.rst
@@ -1239,7 +1239,7 @@ Models:
- Galaxis DVB Card C CI
- Galaxis DVB Card S
- Galaxis DVB Card C
-- Galaxis plug.in S [neuer Name: Galaxis DVB Card S CI
+- Galaxis plug.in S [new Name: Galaxis DVB Card S CI]
Hauppauge
~~~~~~~~~
--
2.54.0
^ permalink raw reply related
* [PATCH v5 0/6] mm/zswap: Implement per-cgroup proactive writeback
From: Hao Jia @ 2026-06-29 11:20 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: linux-mm, linux-kernel, linux-doc, Hao Jia
From: Hao Jia <jiahao1@lixiang.com>
Zswap currently writes back pages to backing swap reactively, triggered
either by the shrinker or by the pool reaching its size limit. Although
proactive memory reclaim can automatically write back a portion of zswap
pages via the shrinker, it cannot explicitly control the amount of
writeback for a specific memory cgroup. Moreover, proactive memory reclaim
may not always be triggered during a steady state.
In certain scenarios, it is desirable to trigger writeback in advance to
free up memory. For example, users may want to prepare for an upcoming
memory-intensive workload by flushing cold memory to the backing storage
when the system is relatively idle.
This patch series introduces a "source=zswap" key to memory.reclaim
cgroup interface, allowing users to proactively write back cold compressed
data from zswap to the backing swap device. When specified, this key
bypasses standard memory reclaim and exclusively performs proactive zswap
writeback up to the requested budget. If omitted, the default reclaim
behavior remains unchanged.
Example usage:
# Write back 10MB of compressed data from zswap to the backing swap
echo "10M source=zswap" > memory.reclaim
Patch 1: Fix missing global shrinker when memory cgroup is disabled.
Patch 2: Extend shrink_memcg() to support batch writeback and update its
return value semantics, thereby improving the writeback efficiency in
the shrink_worker() path.
Patch 3: Extract a reusable writeback helper zswap_shrink_one_memcg() from
shrink_worker().
Patch 4: Extend the memory.reclaim cgroup v2 interface with a new
"source=zswap" key, allowing users to trigger proactive zswap
writeback up to a requested budget.
Patch 5: Add the zswpwb_proactive_b stat to track the compressed bytes
of proactive writeback for better monitoring and tuning.
Patch 6: Add tests for zswap proactive writeback.
v4->v5:
- Add a new patch to fix missing global shrinker when memory cgroup is disabled.
- Simplify batch writeback logic in shrink_memcg() and improve comments.
- Refactor the writeback retry helper out of shrink_worker().
- Replace the "zswap_writeback_only" memory.reclaim key with a more
extensible "source=zswap" key, leaving room for selecting other
reclaim sources in the future.
v3->v4:
- Drop the per-memcg cursor and keep the root cgroup cursor
(zswap_next_shrink) logic intact.
- Stick to using the zswap_writeback_only key, and change the proactive
writeback size to use the compressed size.
- Consolidate and reuse the logic between shrink_worker() and
shrink_memcg(). Enable batch writeback in the shrink_worker() path,
while maintaining a low writeback budget in the zswap_store() path.
v2->v3:
- Align the return value of zswap_proactive_writeback() with
memory.reclaim and update the corresponding documentation accordingly.
- Resolve conflicts in test_zswap.c on the mm-unstable branch.
- Enhance the zswap proactive writeback selftests to guard against potential
future regressions.
v1->v2:
- As suggested by Yosry and Nhat, extend the memory.reclaim cgroup v2
interface with a "zswap_writeback_only" key instead of adding a new
dedicated cgroup interface.
- Update the zswap documentation and add selftests for proactive writeback.
[v4] https://lore.kernel.org/all/20260618044857.69439-1-jiahao.kernel@gmail.com
[v3] https://lore.kernel.org/all/20260526114601.67041-1-jiahao.kernel@gmail.com
[v2] https://lore.kernel.org/all/20260525122242.36127-1-jiahao.kernel@gmail.com
[v1] https://lore.kernel.org/all/20260511105149.75584-1-jiahao.kernel@gmail.com
Hao Jia (6):
mm/zswap: Fix global shrinker when memory cgroup is disabled
mm/zswap: Support batch writeback in shrink_memcg()
mm/zswap: Extract a reusable writeback helper from shrink_worker()
mm/zswap: Implement proactive writeback
mm/zswap: Add per-memcg stat for proactive writeback
selftests/cgroup: Add tests for zswap proactive writeback
Documentation/admin-guide/cgroup-v2.rst | 22 +-
Documentation/admin-guide/mm/zswap.rst | 11 +-
include/linux/memcontrol.h | 1 +
include/linux/zswap.h | 7 +
mm/memcontrol.c | 3 +
mm/vmscan.c | 24 +-
mm/zswap.c | 239 +++++++++++++++-----
tools/testing/selftests/cgroup/test_zswap.c | 161 ++++++++++++-
8 files changed, 410 insertions(+), 58 deletions(-)
--
2.34.1
^ permalink raw reply
* [PATCH v5 1/6] mm/zswap: Fix global shrinker when memory cgroup is disabled
From: Hao Jia @ 2026-06-29 11:20 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: linux-mm, linux-kernel, linux-doc, Hao Jia, stable
In-Reply-To: <20260629112032.20423-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
When memory cgroup is disabled, mem_cgroup_iter() always returns NULL.
Therefore, the global shrinker shrink_worker() always takes the !memcg
branch. After MAX_RECLAIM_RETRIES empty walks, the worker simply gives up,
so it fails to write back anything.
Therefore, when memory cgroup is disabled, fall through with the !memcg
branch and shrink the root memcg directly. Stop the loop once
shrink_memcg() reports -ENOENT, since the root LRU is the only target and
-ENOENT means it has been exhausted.
Fixes: a65b0e7607cc ("zswap: make shrinking memcg-aware")
Cc: stable@vger.kernel.org
Reported-by: Yosry Ahmed <yosry@kernel.org>
Closes: https://lore.kernel.org/all/CAO9r8zPVzMKFbCixxD-qgtRrkFxWVrHiZZeLc=eyTPKPVQgX4g@mail.gmail.com
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
mm/zswap.c | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/mm/zswap.c b/mm/zswap.c
index 761cd699e0a3..0f8f04f22888 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -1356,7 +1356,12 @@ static void shrink_worker(struct work_struct *w)
} while (memcg && !mem_cgroup_tryget_online(memcg));
spin_unlock(&zswap_shrink_lock);
- if (!memcg) {
+ /*
+ * Reaching a NULL memcg means a full hierarchy pass completed.
+ * Exclude the memcg-disabled case, where it is always NULL, and
+ * fall through to shrink the root LRU directly.
+ */
+ if (!memcg && !mem_cgroup_disabled()) {
/*
* Continue shrinking without incrementing failures if
* we found candidate memcgs in the last tree walk.
@@ -1378,8 +1383,15 @@ static void shrink_worker(struct work_struct *w)
* with pages in zswap. Skip this without incrementing attempts
* and failures.
*/
- if (ret == -ENOENT)
+ if (ret == -ENOENT) {
+ /*
+ * With memcg disabled the root LRU is the only target, so
+ * we should abort if it has no writeback-candidate pages.
+ */
+ if (mem_cgroup_disabled())
+ break;
continue;
+ }
++attempts;
if (ret && ++failures == MAX_RECLAIM_RETRIES)
--
2.34.1
^ permalink raw reply related
* [PATCH v5 2/6] mm/zswap: Support batch writeback in shrink_memcg()
From: Hao Jia @ 2026-06-29 11:20 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260629112032.20423-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
Currently, shrink_memcg() writes back at most one entry per-node during
its traversal. This makes shrink_worker() inefficient, as it must
repeatedly re-enter shrink_memcg() to make any substantial progress.
To address this, extend shrink_memcg() and rewrite its LRU iteration logic
to support batch writeback. Introduce the nr_to_scan parameter to bound how
many pages are scanned per call. This enables batch writeback in the
shrink_worker() path, while maintaining a low scan budget in the
zswap_store() path.
Additionally, to prepare for future proactive writeback, update the return
value semantics of shrink_memcg(): a positive value now represents the
actual number of compressed bytes written back, 0 indicates that candidates
existed but no writeback succeeded, and a negative value represents an
error code.
Suggested-by: Yosry Ahmed <yosry@kernel.org>
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
mm/zswap.c | 89 ++++++++++++++++++++++++++++++++++++++++++------------
1 file changed, 69 insertions(+), 20 deletions(-)
diff --git a/mm/zswap.c b/mm/zswap.c
index 0f8f04f22888..e2c2a3f1e061 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -160,6 +160,11 @@ struct zswap_pool {
char tfm_name[CRYPTO_MAX_ALG_NAME];
};
+struct zswap_shrink_walk_arg {
+ unsigned long bytes_written;
+ bool encountered_page_in_swapcache;
+};
+
/* Global LRU lists shared by all zswap pools. */
static struct list_lru zswap_list_lru;
@@ -1089,8 +1094,9 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o
void *arg)
{
struct zswap_entry *entry = container_of(item, struct zswap_entry, lru);
- bool *encountered_page_in_swapcache = (bool *)arg;
+ struct zswap_shrink_walk_arg *walk_arg = arg;
swp_entry_t swpentry;
+ unsigned int length;
enum lru_status ret = LRU_REMOVED_RETRY;
int writeback_result;
@@ -1133,10 +1139,11 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o
/*
* Once the lru lock is dropped, the entry might get freed. The
- * swpentry is copied to the stack, and entry isn't deref'd again
- * until the entry is verified to still be alive in the tree.
+ * needed fields are copied to the stack, and entry isn't deref'd
+ * again until it is verified to still be alive in the tree.
*/
swpentry = entry->swpentry;
+ length = entry->length;
/*
* It's safe to drop the lock here because we return either
@@ -1155,12 +1162,13 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o
* into the warmer region. We should terminate shrinking (if we're in the dynamic
* shrinker context).
*/
- if (writeback_result == -EEXIST && encountered_page_in_swapcache) {
+ if (writeback_result == -EEXIST) {
ret = LRU_STOP;
- *encountered_page_in_swapcache = true;
+ walk_arg->encountered_page_in_swapcache = true;
}
} else {
zswap_written_back_pages++;
+ walk_arg->bytes_written += length;
}
return ret;
@@ -1169,8 +1177,11 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o
static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
struct shrink_control *sc)
{
+ struct zswap_shrink_walk_arg walk_arg = {
+ .bytes_written = 0,
+ .encountered_page_in_swapcache = false,
+ };
unsigned long shrink_ret;
- bool encountered_page_in_swapcache = false;
if (!zswap_shrinker_enabled ||
!mem_cgroup_zswap_writeback_enabled(sc->memcg)) {
@@ -1179,9 +1190,9 @@ static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
}
shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb,
- &encountered_page_in_swapcache);
+ &walk_arg);
- if (encountered_page_in_swapcache)
+ if (walk_arg.encountered_page_in_swapcache)
return SHRINK_STOP;
return shrink_ret ? shrink_ret : SHRINK_STOP;
@@ -1275,9 +1286,31 @@ static struct shrinker *zswap_alloc_shrinker(void)
return shrinker;
}
-static int shrink_memcg(struct mem_cgroup *memcg)
+#define NR_ZSWAP_WB_BATCH 64UL
+
+/*
+ * Scan up to @nr_to_scan pages across the per-node zswap LRUs of @memcg
+ * and write back the reclaimable ones.
+ *
+ * Since the second-chance algorithm rotates referenced entries to the
+ * LRU tail, the per-node scan is capped at the current LRU length so
+ * each entry is scanned at most once per call. It is up to the caller
+ * to handle retries, deciding whether to scan another memcg to complete
+ * the full iteration, or to rescan the current memcg to drain its zswap
+ * entries.
+ *
+ * Return: The number of compressed bytes written back (>= 0), or -ENOENT
+ * if @memcg has writeback disabled, is a zombie cgroup, or has empty
+ * zswap LRUs.
+ */
+static long shrink_memcg(struct mem_cgroup *memcg, unsigned long nr_to_scan)
{
- int nid, shrunk = 0, scanned = 0;
+ struct zswap_shrink_walk_arg walk_arg = {
+ .bytes_written = 0,
+ .encountered_page_in_swapcache = false,
+ };
+ unsigned long nr_remaining = nr_to_scan;
+ int nid;
if (!mem_cgroup_zswap_writeback_enabled(memcg))
return -ENOENT;
@@ -1290,24 +1323,40 @@ static int shrink_memcg(struct mem_cgroup *memcg)
return -ENOENT;
for_each_node_state(nid, N_NORMAL_MEMORY) {
- unsigned long nr_to_walk = 1;
+ unsigned long nr_to_walk;
- shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg,
- &shrink_memcg_cb, NULL, &nr_to_walk);
- scanned += 1 - nr_to_walk;
+ /*
+ * Cap the scan at per-node LRU length so each entry is scanned
+ * at most once per call.
+ */
+ nr_to_walk = min(nr_remaining,
+ list_lru_count_one(&zswap_list_lru, nid, memcg));
+ if (!nr_to_walk)
+ continue;
+
+ nr_remaining -= nr_to_walk;
+ list_lru_walk_one(&zswap_list_lru, nid, memcg, &shrink_memcg_cb,
+ &walk_arg, &nr_to_walk);
+ /* Return the unused share of the budget to the pool. */
+ nr_remaining += nr_to_walk;
+
+ if (!nr_remaining)
+ break;
}
- if (!scanned)
+ /* Nothing was scanned: every LRU under @memcg was empty. */
+ if (nr_remaining == nr_to_scan)
return -ENOENT;
- return shrunk ? 0 : -EAGAIN;
+ return walk_arg.bytes_written;
}
static void shrink_worker(struct work_struct *w)
{
struct mem_cgroup *memcg;
- int ret, failures = 0, attempts = 0;
+ int failures = 0, attempts = 0;
unsigned long thr;
+ long ret;
/* Reclaim down to the accept threshold */
thr = zswap_accept_thr_pages();
@@ -1373,7 +1422,7 @@ static void shrink_worker(struct work_struct *w)
goto resched;
}
- ret = shrink_memcg(memcg);
+ ret = shrink_memcg(memcg, NR_ZSWAP_WB_BATCH);
/* drop the extra reference */
mem_cgroup_put(memcg);
@@ -1394,7 +1443,7 @@ static void shrink_worker(struct work_struct *w)
}
++attempts;
- if (ret && ++failures == MAX_RECLAIM_RETRIES)
+ if (ret <= 0 && ++failures == MAX_RECLAIM_RETRIES)
break;
resched:
cond_resched();
@@ -1504,7 +1553,7 @@ bool zswap_store(struct folio *folio)
objcg = get_obj_cgroup_from_folio(folio);
if (objcg && !obj_cgroup_may_zswap(objcg)) {
memcg = get_mem_cgroup_from_objcg(objcg);
- if (shrink_memcg(memcg)) {
+ if (shrink_memcg(memcg, num_node_state(N_NORMAL_MEMORY)) <= 0) {
mem_cgroup_put(memcg);
goto put_objcg;
}
--
2.34.1
^ permalink raw reply related
* [PATCH v5 3/6] mm/zswap: Extract a reusable writeback helper from shrink_worker()
From: Hao Jia @ 2026-06-29 11:20 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260629112032.20423-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
Extract a reusable writeback helper zswap_shrink_one_memcg() from
shrink_worker(). This helper will be reused by the upcoming proactive
writeback feature.
zswap_shrink_one_memcg() takes one step of a memcg-tree writeback walk
driven by the caller's iterator. Consequently, shrink_worker() now only
needs to calculate the acceptance threshold, drive its own iteration
based on this helper, and abort the walk when zswap_shrink_one_memcg()
returns -EBUSY.
Suggested-by: Yosry Ahmed <yosry@kernel.org>
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
mm/zswap.c | 118 +++++++++++++++++++++++++++++++----------------------
1 file changed, 69 insertions(+), 49 deletions(-)
diff --git a/mm/zswap.c b/mm/zswap.c
index e2c2a3f1e061..ba01bf0e44e9 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -1351,12 +1351,70 @@ static long shrink_memcg(struct mem_cgroup *memcg, unsigned long nr_to_scan)
return walk_arg.bytes_written;
}
+/* Track progress of a memcg-tree writeback walk. */
+struct zswap_shrink_state {
+ int scans;
+ int failures;
+};
+
+/*
+ * Take one step of a memcg-tree writeback walk driven by the caller's
+ * iterator, and fold the result into @s, the retry bookkeeping shared
+ * across steps. @memcg is the iterator's current memcg, or NULL once
+ * it has wrapped around after a full pass over the tree.
+ *
+ * The function returns -EBUSY to signal the caller to abort the walk after
+ * encountering either of the following MAX_RECLAIM_RETRIES times:
+ * - No writeback-candidate memcgs were found in a memcg tree walk.
+ * - Shrinking a writeback-candidate memcg failed.
+ *
+ * Return: The number of compressed bytes written back (>= 0), or -EBUSY
+ * when the caller should abort the walk.
+ */
+static long zswap_shrink_one_memcg(struct mem_cgroup *memcg,
+ struct zswap_shrink_state *s)
+{
+ long shrunk;
+
+ /*
+ * Reaching a NULL memcg means a full hierarchy pass completed.
+ * Exclude the memcg-disabled case, where it is always NULL, and
+ * fall through to shrink the root LRU directly.
+ */
+ if (!memcg && !mem_cgroup_disabled()) {
+ /*
+ * Continue shrinking without incrementing failures if we found
+ * candidate memcgs in the last tree walk.
+ */
+ if (!s->scans && ++s->failures == MAX_RECLAIM_RETRIES)
+ return -EBUSY;
+ s->scans = 0;
+ return 0;
+ }
+
+ shrunk = shrink_memcg(memcg, NR_ZSWAP_WB_BATCH);
+
+ /*
+ * There are no writeback-candidate pages in the memcg. With memcg
+ * enabled this is not an issue as long as we can find another memcg
+ * with pages in zswap, so skip without counting it as a candidate.
+ * With memcg disabled the root LRU is the only target, so we should
+ * abort if it has no writeback-candidate pages.
+ */
+ if (shrunk == -ENOENT)
+ return mem_cgroup_disabled() ? -EBUSY : 0;
+ s->scans++;
+
+ if (shrunk <= 0 && ++s->failures == MAX_RECLAIM_RETRIES)
+ return -EBUSY;
+
+ return shrunk;
+}
+
static void shrink_worker(struct work_struct *w)
{
- struct mem_cgroup *memcg;
- int failures = 0, attempts = 0;
+ struct zswap_shrink_state s = {};
unsigned long thr;
- long ret;
/* Reclaim down to the accept threshold */
thr = zswap_accept_thr_pages();
@@ -1367,11 +1425,6 @@ static void shrink_worker(struct work_struct *w)
* writeback-disabled memcgs (memory.zswap.writeback=0) are not
* candidates for shrinking.
*
- * Shrinking will be aborted if we encounter the following
- * MAX_RECLAIM_RETRIES times:
- * - No writeback-candidate memcgs found in a memcg tree walk.
- * - Shrinking a writeback-candidate memcg failed.
- *
* We save iteration cursor memcg into zswap_next_shrink,
* which can be modified by the offline memcg cleaner
* zswap_memcg_offline_cleanup().
@@ -1386,7 +1439,11 @@ static void shrink_worker(struct work_struct *w)
* offline memcg left in zswap_next_shrink will hold the reference
* until the next run of shrink_worker().
*/
- do {
+ while (zswap_total_pages() > thr) {
+ struct mem_cgroup *memcg;
+ long ret;
+
+ cond_resched();
/*
* Start shrinking from the next memcg after zswap_next_shrink.
* When the offline cleaner has already advanced the cursor,
@@ -1405,49 +1462,12 @@ static void shrink_worker(struct work_struct *w)
} while (memcg && !mem_cgroup_tryget_online(memcg));
spin_unlock(&zswap_shrink_lock);
- /*
- * Reaching a NULL memcg means a full hierarchy pass completed.
- * Exclude the memcg-disabled case, where it is always NULL, and
- * fall through to shrink the root LRU directly.
- */
- if (!memcg && !mem_cgroup_disabled()) {
- /*
- * Continue shrinking without incrementing failures if
- * we found candidate memcgs in the last tree walk.
- */
- if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
- break;
-
- attempts = 0;
- goto resched;
- }
-
- ret = shrink_memcg(memcg, NR_ZSWAP_WB_BATCH);
+ ret = zswap_shrink_one_memcg(memcg, &s);
/* drop the extra reference */
mem_cgroup_put(memcg);
-
- /*
- * There are no writeback-candidate pages in the memcg.
- * This is not an issue as long as we can find another memcg
- * with pages in zswap. Skip this without incrementing attempts
- * and failures.
- */
- if (ret == -ENOENT) {
- /*
- * With memcg disabled the root LRU is the only target, so
- * we should abort if it has no writeback-candidate pages.
- */
- if (mem_cgroup_disabled())
- break;
- continue;
- }
- ++attempts;
-
- if (ret <= 0 && ++failures == MAX_RECLAIM_RETRIES)
+ if (ret == -EBUSY)
break;
-resched:
- cond_resched();
- } while (zswap_total_pages() > thr);
+ }
}
/*********************************
--
2.34.1
^ permalink raw reply related
* [PATCH v5 4/6] mm/zswap: Implement proactive writeback
From: Hao Jia @ 2026-06-29 11:20 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260629112032.20423-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
Zswap currently writes back pages to backing swap reactively, triggered
either by the shrinker or when the pool reaches its size limit. There is
no mechanism to control the amount of writeback for a specific memory
cgroup. However, users may want to proactively write back zswap pages,
e.g., to free up memory for other applications or to prepare for
memory-intensive workloads.
Introduce a "source=" key to the memory.reclaim cgroup interface,
currently accepting the single value "zswap". When set to "zswap", it
bypasses standard memory reclaim and exclusively performs proactive
zswap writeback up to the requested budget. If omitted, the default
reclaim behavior remains unchanged.
Example usage:
# Write back 10MB of compressed data from zswap to the backing swap
echo "10M source=zswap" > memory.reclaim
Note that the actual amount of compressed data written back may be less
than requested due to the zswap second-chance algorithm: referenced
entries are rotated on the LRU on the first encounter and only written
back on a second pass. If fewer bytes are written back than requested,
-EAGAIN is returned, matching the existing memory.reclaim semantics.
Internally, extend user_proactive_reclaim() to parse the new "source="
key and invoke the dedicated handler zswap_proactive_writeback() when it
is set to "zswap". This handler walks the target memcg subtree in a
round-robin fashion and drains each memcg's per-node zswap LRUs through
shrink_memcg(), accumulating the compressed bytes written back until the
requested budget is met.
Suggested-by: Yosry Ahmed <yosry@kernel.org>
Suggested-by: Nhat Pham <nphamcs@gmail.com>
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
Documentation/admin-guide/cgroup-v2.rst | 18 ++++++++-
Documentation/admin-guide/mm/zswap.rst | 11 +++++-
include/linux/zswap.h | 7 ++++
mm/vmscan.c | 24 +++++++++++-
mm/zswap.c | 50 +++++++++++++++++++++++++
5 files changed, 106 insertions(+), 4 deletions(-)
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 993446ab66d0..bbcc9695aa8d 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1425,9 +1425,10 @@ PAGE_SIZE multiple when read back.
The following nested keys are defined.
- ========== ================================
+ ==================== ==================================================
swappiness Swappiness value to reclaim with
- ========== ================================
+ source=zswap Only perform proactive zswap writeback
+ ==================== ==================================================
Specifying a swappiness value instructs the kernel to perform
the reclaim with that swappiness value. Note that this has the
@@ -1437,6 +1438,19 @@ The following nested keys are defined.
The valid range for swappiness is [0-200, max], setting
swappiness=max exclusively reclaims anonymous memory.
+ The source=zswap key skips ordinary memory reclaim and
+ writes back pages from zswap to the backing swap device until
+ the requested amount has been written or no further candidates
+ are found. This is useful to proactively offload cold compressed
+ data from the zswap pool to the swap device. It is only available
+ if zswap writeback is enabled. source=zswap cannot be
+ combined with swappiness; specifying both returns -EINVAL.
+
+ Example::
+
+ # Writeback up to 10MB of compressed data from zswap to the backing swap
+ echo "10M source=zswap" > memory.reclaim
+
memory.peak
A read-write single value file which exists on non-root cgroups.
diff --git a/Documentation/admin-guide/mm/zswap.rst b/Documentation/admin-guide/mm/zswap.rst
index 2464425c783d..b49b8c130389 100644
--- a/Documentation/admin-guide/mm/zswap.rst
+++ b/Documentation/admin-guide/mm/zswap.rst
@@ -131,7 +131,16 @@ User can enable it as follows::
echo Y > /sys/module/zswap/parameters/shrinker_enabled
This can be enabled at the boot time if ``CONFIG_ZSWAP_SHRINKER_DEFAULT_ON`` is
-selected.
+selected. Once enabled, the shrinker automatically writes back zswap pages to
+backing swap during memory reclaim.
+
+If users want to explicitly trigger proactive zswap writeback for a specific
+memory cgroup without invoking standard page reclaim, it can be done as follows::
+
+ echo "10M source=zswap" > /sys/fs/cgroup/<cgroup-name>/memory.reclaim
+
+Both of the methods mentioned above are subject to the ``memory.zswap.writeback``
+control. This means that ``memory.zswap.writeback`` can prevent all zswap writeback.
A debugfs interface is provided for various statistic about pool size, number
of pages stored, same-value filled pages and various counters for the reasons
diff --git a/include/linux/zswap.h b/include/linux/zswap.h
index 30c193a1207e..e5f217759894 100644
--- a/include/linux/zswap.h
+++ b/include/linux/zswap.h
@@ -35,6 +35,7 @@ void zswap_lruvec_state_init(struct lruvec *lruvec);
void zswap_folio_swapin(struct folio *folio);
bool zswap_is_enabled(void);
bool zswap_never_enabled(void);
+int zswap_proactive_writeback(struct mem_cgroup *memcg, u64 bytes_to_writeback);
#else
struct zswap_lruvec_state {};
@@ -69,6 +70,12 @@ static inline bool zswap_never_enabled(void)
return true;
}
+static inline int zswap_proactive_writeback(struct mem_cgroup *memcg,
+ u64 bytes_to_writeback)
+{
+ return -EOPNOTSUPP;
+}
+
#endif
#endif /* _LINUX_ZSWAP_H */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 35c3bb15ae96..56ed7ff48ec9 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -64,6 +64,7 @@
#include <linux/swapops.h>
#include <linux/sched/sysctl.h>
+#include <linux/zswap.h>
#include "internal.h"
#include "swap.h"
@@ -7855,11 +7856,13 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask,
enum {
MEMORY_RECLAIM_SWAPPINESS = 0,
MEMORY_RECLAIM_SWAPPINESS_MAX,
+ MEMORY_RECLAIM_SOURCE,
MEMORY_RECLAIM_NULL,
};
static const match_table_t tokens = {
{ MEMORY_RECLAIM_SWAPPINESS, "swappiness=%d"},
{ MEMORY_RECLAIM_SWAPPINESS_MAX, "swappiness=max"},
+ { MEMORY_RECLAIM_SOURCE, "source=%s"},
{ MEMORY_RECLAIM_NULL, NULL },
};
@@ -7869,9 +7872,12 @@ int user_proactive_reclaim(char *buf,
unsigned int nr_retries = MAX_RECLAIM_RETRIES;
unsigned long nr_to_reclaim, nr_reclaimed = 0;
int swappiness = -1;
+ bool zswap_writeback_only = false;
char *old_buf, *start;
+ char source[16];
substring_t args[MAX_OPT_ARGS];
gfp_t gfp_mask = GFP_KERNEL;
+ u64 nr_bytes;
if (!buf || (!memcg && !pgdat) || (memcg && pgdat))
return -EINVAL;
@@ -7879,7 +7885,8 @@ int user_proactive_reclaim(char *buf,
buf = strstrip(buf);
old_buf = buf;
- nr_to_reclaim = memparse(buf, &buf) / PAGE_SIZE;
+ nr_bytes = memparse(buf, &buf);
+ nr_to_reclaim = nr_bytes / PAGE_SIZE;
if (buf == old_buf)
return -EINVAL;
@@ -7899,11 +7906,26 @@ int user_proactive_reclaim(char *buf,
case MEMORY_RECLAIM_SWAPPINESS_MAX:
swappiness = SWAPPINESS_ANON_ONLY;
break;
+ case MEMORY_RECLAIM_SOURCE:
+ if (match_strlcpy(source, &args[0], sizeof(source)) >= sizeof(source))
+ return -EINVAL;
+ /* Only zswap is supported as a reclaim source for now. */
+ if (strcmp(source, "zswap"))
+ return -EINVAL;
+ zswap_writeback_only = true;
+ break;
default:
return -EINVAL;
}
}
+ if (zswap_writeback_only) {
+ /* source=zswap and swappiness are mutually exclusive. */
+ if (swappiness != -1)
+ return -EINVAL;
+ return zswap_proactive_writeback(memcg, nr_bytes);
+ }
+
while (nr_reclaimed < nr_to_reclaim) {
/* Will converge on zero, but reclaim enforces a minimum */
unsigned long batch_size = (nr_to_reclaim - nr_reclaimed) / 4;
diff --git a/mm/zswap.c b/mm/zswap.c
index ba01bf0e44e9..9cda96f05508 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -1713,6 +1713,56 @@ int zswap_load(struct folio *folio)
return 0;
}
+int zswap_proactive_writeback(struct mem_cgroup *memcg, u64 bytes_to_writeback)
+{
+ struct zswap_shrink_state s = {};
+ struct mem_cgroup *iter = NULL;
+ u64 bytes_written = 0;
+ int ret = 0;
+
+ if (!memcg)
+ return -EINVAL;
+ if (!mem_cgroup_zswap_writeback_enabled(memcg))
+ return -EINVAL;
+ if (!bytes_to_writeback)
+ return 0;
+
+ while (bytes_written < bytes_to_writeback) {
+ long shrunk;
+
+ cond_resched();
+
+ if (signal_pending(current)) {
+ ret = -EINTR;
+ break;
+ }
+
+ /*
+ * Use a local iterator to walk the memcg and its online descendants
+ * in a round-robin manner. Upon exiting the loop, mem_cgroup_iter_break()
+ * must be called to drop the iterator reference.
+ */
+ do {
+ iter = mem_cgroup_iter(memcg, iter, NULL);
+ } while (iter && !mem_cgroup_tryget_online(iter));
+
+ shrunk = zswap_shrink_one_memcg(iter, &s);
+ if (shrunk > 0)
+ bytes_written += shrunk;
+
+ /* drop the extra reference taken by mem_cgroup_tryget_online() */
+ mem_cgroup_put(iter);
+
+ if (shrunk == -EBUSY) {
+ ret = -EAGAIN;
+ break;
+ }
+ }
+
+ mem_cgroup_iter_break(memcg, iter);
+ return ret;
+}
+
void zswap_invalidate(swp_entry_t swp)
{
pgoff_t offset = swp_offset(swp);
--
2.34.1
^ permalink raw reply related
* [PATCH v5 5/6] mm/zswap: Add per-memcg stat for proactive writeback
From: Hao Jia @ 2026-06-29 11:20 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260629112032.20423-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
Add a new stat zswpwb_proactive_b to memory.stat. This counter is
incremented by entry->length during proactive writebacks triggered
via the source=zswap key in memory.reclaim. It tracks the
compressed size (in bytes) of pages proactively written back from
zswap to swap, allowing users to better monitor and tune the
proactive writeback mechanism.
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
Documentation/admin-guide/cgroup-v2.rst | 4 ++++
include/linux/memcontrol.h | 1 +
mm/memcontrol.c | 3 +++
mm/zswap.c | 4 +++-
4 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index bbcc9695aa8d..e1f6a4729a65 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1748,6 +1748,10 @@ The following nested keys are defined.
zswpwb
Number of pages written from zswap to swap.
+ zswpwb_proactive_b
+ Bytes of compressed data proactively written back from
+ zswap to swap via the memory.reclaim source=zswap key.
+
zswap_incomp
Number of incompressible pages currently stored in zswap
without compression. These pages could not be compressed to
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index e1f46a0016fc..56580b264dc4 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -40,6 +40,7 @@ enum memcg_stat_item {
MEMCG_ZSWAP_B,
MEMCG_ZSWAPPED,
MEMCG_ZSWAP_INCOMP,
+ MEMCG_ZSWPWB_PROACTIVE_B,
MEMCG_NR_STAT,
};
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index d20ffc827306..d81c34484bca 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -433,6 +433,7 @@ static const unsigned int memcg_stat_items[] = {
MEMCG_ZSWAP_B,
MEMCG_ZSWAPPED,
MEMCG_ZSWAP_INCOMP,
+ MEMCG_ZSWPWB_PROACTIVE_B,
};
#define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items)
@@ -1558,6 +1559,7 @@ static const struct memory_stat memory_stats[] = {
{ "zswap", MEMCG_ZSWAP_B },
{ "zswapped", MEMCG_ZSWAPPED },
{ "zswap_incomp", MEMCG_ZSWAP_INCOMP },
+ { "zswpwb_proactive_b", MEMCG_ZSWPWB_PROACTIVE_B },
#endif
{ "file_mapped", NR_FILE_MAPPED },
{ "file_dirty", NR_FILE_DIRTY },
@@ -1614,6 +1616,7 @@ static int memcg_page_state_unit(int item)
switch (item) {
case MEMCG_PERCPU_B:
case MEMCG_ZSWAP_B:
+ case MEMCG_ZSWPWB_PROACTIVE_B:
case NR_SLAB_RECLAIMABLE_B:
case NR_SLAB_UNRECLAIMABLE_B:
return 1;
diff --git a/mm/zswap.c b/mm/zswap.c
index 9cda96f05508..d356c1739c68 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -1747,8 +1747,10 @@ int zswap_proactive_writeback(struct mem_cgroup *memcg, u64 bytes_to_writeback)
} while (iter && !mem_cgroup_tryget_online(iter));
shrunk = zswap_shrink_one_memcg(iter, &s);
- if (shrunk > 0)
+ if (shrunk > 0) {
bytes_written += shrunk;
+ mod_memcg_state(iter, MEMCG_ZSWPWB_PROACTIVE_B, shrunk);
+ }
/* drop the extra reference taken by mem_cgroup_tryget_online() */
mem_cgroup_put(iter);
--
2.34.1
^ permalink raw reply related
* [PATCH v5 6/6] selftests/cgroup: Add tests for zswap proactive writeback
From: Hao Jia @ 2026-06-29 11:20 UTC (permalink / raw)
To: akpm, tj, hannes, shakeel.butt, mhocko, yosry, mkoutny, nphamcs,
chengming.zhou, muchun.song, roman.gushchin
Cc: linux-mm, linux-kernel, linux-doc, Hao Jia
In-Reply-To: <20260629112032.20423-1-jiahao.kernel@gmail.com>
From: Hao Jia <jiahao1@lixiang.com>
Add test_zswap_proactive_writeback() to cover the new memory.reclaim
"source=zswap" key. The test populates a memory cgroup zswap
pool, triggers proactive writeback, and verifies the behavior by
observing the change in zswpwb_proactive_b. Invalid input combinations
are also covered.
Extend test_zswap_writeback_one() to assert that the existing
non-proactive writeback path leaves zswpwb_proactive_b at zero.
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
---
tools/testing/selftests/cgroup/test_zswap.c | 161 +++++++++++++++++++-
1 file changed, 160 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c
index 49b36ee79160..dfd5f24b9d99 100644
--- a/tools/testing/selftests/cgroup/test_zswap.c
+++ b/tools/testing/selftests/cgroup/test_zswap.c
@@ -60,7 +60,12 @@ static int get_zswap_stored_pages(size_t *value)
static long get_cg_wb_count(const char *cg)
{
- return cg_read_key_long(cg, "memory.stat", "zswpwb");
+ return cg_read_key_long(cg, "memory.stat", "zswpwb ");
+}
+
+static long get_cg_pwb_bytes(const char *cg)
+{
+ return cg_read_key_long(cg, "memory.stat", "zswpwb_proactive_b ");
}
static long get_zswpout(const char *cgroup)
@@ -355,6 +360,7 @@ static int attempt_writeback(const char *cgroup, void *arg)
static int test_zswap_writeback_one(const char *cgroup, bool wb)
{
long zswpwb_before, zswpwb_after;
+ long pwb_bytes;
zswpwb_before = get_cg_wb_count(cgroup);
if (zswpwb_before != 0) {
@@ -362,6 +368,12 @@ static int test_zswap_writeback_one(const char *cgroup, bool wb)
return -1;
}
+ pwb_bytes = get_cg_pwb_bytes(cgroup);
+ if (pwb_bytes > 0) {
+ ksft_print_msg("zswpwb_proactive_b_before = %ld instead of 0\n", pwb_bytes);
+ return -1;
+ }
+
if (cg_run(cgroup, attempt_writeback, (void *) &wb))
return -1;
@@ -379,6 +391,17 @@ static int test_zswap_writeback_one(const char *cgroup, bool wb)
return -1;
}
+ /*
+ * attempt_writeback() does not use the proactive writeback path, so
+ * zswpwb_proactive_b must stay at zero regardless of whether
+ * writeback was enabled.
+ */
+ pwb_bytes = get_cg_pwb_bytes(cgroup);
+ if (pwb_bytes > 0) {
+ ksft_print_msg("zswpwb_proactive_b_after is %ld, expected 0\n", pwb_bytes);
+ return -1;
+ }
+
return 0;
}
@@ -770,6 +793,141 @@ static int test_zswap_incompressible(const char *root)
return ret;
}
+/*
+ * Trigger proactive zswap writeback with the following steps:
+ * 1. Allocate memory.
+ * 2. Push allocated memory into zswap.
+ * 3. Proactively write back zswap pages to swap
+ * using "source=zswap".
+ */
+static int proactive_writeback_workload(const char *cgroup, void *arg)
+{
+ long pagesize = sysconf(_SC_PAGESIZE);
+ size_t memsize = pagesize * 1024;
+ char reclaim_cmd[64];
+ char buf[pagesize];
+ long zswap_usage;
+ int ret = -1;
+ int rc;
+ char *mem;
+
+ mem = (char *)malloc(memsize);
+ if (!mem)
+ return ret;
+
+ for (int i = 0; i < pagesize; i++)
+ buf[i] = i < pagesize / 2 ? (char)i : 0;
+ for (int i = 0; i < memsize; i += pagesize)
+ memcpy(&mem[i], buf, pagesize);
+
+ /* Evict allocated memory into zswap. */
+ if (cg_write_numeric(cgroup, "memory.reclaim", memsize)) {
+ ksft_print_msg("Failed to push pages into zswap\n");
+ goto out;
+ }
+
+ zswap_usage = cg_read_long(cgroup, "memory.zswap.current");
+ if (zswap_usage <= 0) {
+ ksft_print_msg("no zswap pool to write back\n");
+ goto out;
+ }
+
+ /* Trigger proactive zswap writeback. */
+ snprintf(reclaim_cmd, sizeof(reclaim_cmd), "%ld source=zswap", zswap_usage);
+ rc = cg_write(cgroup, "memory.reclaim", reclaim_cmd);
+ if (rc && rc != -EAGAIN) {
+ ksft_print_msg("proactive zswap writeback failed: %d\n", rc);
+ goto out;
+ }
+
+ ret = 0;
+out:
+ free(mem);
+ return ret;
+}
+
+static int check_writeback_invalid_inputs(const char *cgroup)
+{
+ static char * const bad_inputs[] = {
+ "source=zswap",
+ "1M source=zswap swappiness=60",
+ "1M swappiness=60 source=zswap",
+ "1M source=zswap swappiness=max",
+ "1M swappiness=max source=zswap",
+ };
+ int i, rc;
+
+ for (i = 0; i < ARRAY_SIZE(bad_inputs); i++) {
+ rc = cg_write(cgroup, "memory.reclaim", bad_inputs[i]);
+ if (rc != -EINVAL) {
+ ksft_print_msg("memory.reclaim '%s': returned %d, expected %d\n",
+ bad_inputs[i], rc, -EINVAL);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int test_zswap_proactive_writeback(const char *root)
+{
+ long wb_before, wb_after;
+ long pwb_b_before, pwb_b_after;
+ long wb_delta, pwb_b_delta;
+ int ret = KSFT_FAIL;
+ char *test_group;
+
+ if (cg_read_strcmp(root, "memory.zswap.writeback", "1"))
+ return KSFT_SKIP;
+
+ test_group = cg_name(root, "zswap_proactive_test");
+ if (!test_group)
+ return KSFT_FAIL;
+ if (cg_create(test_group))
+ goto out;
+ /*
+ * A missing zswpwb_proactive_b stat means the kernel lacks proactive
+ * writeback support, so skip rather than fail.
+ */
+ if (get_cg_pwb_bytes(test_group) < 0) {
+ ret = KSFT_SKIP;
+ goto out;
+ }
+ if (check_writeback_invalid_inputs(test_group))
+ goto out;
+
+ pwb_b_before = get_cg_pwb_bytes(test_group);
+ wb_before = get_cg_wb_count(test_group);
+ if (pwb_b_before < 0 || wb_before < 0)
+ goto out;
+
+ if (cg_run(test_group, proactive_writeback_workload, NULL))
+ goto out;
+
+ pwb_b_after = get_cg_pwb_bytes(test_group);
+ wb_after = get_cg_wb_count(test_group);
+ if (pwb_b_after < 0 || wb_after < 0)
+ goto out;
+
+ pwb_b_delta = pwb_b_after - pwb_b_before;
+ wb_delta = wb_after - wb_before;
+
+ if (pwb_b_delta <= 0) {
+ ksft_print_msg("zswpwb_proactive_b did not increase: delta=%ld\n",
+ pwb_b_delta);
+ goto out;
+ }
+ if (wb_delta <= 0) {
+ ksft_print_msg("zswpwb did not increase: delta=%ld\n", wb_delta);
+ goto out;
+ }
+
+ ret = KSFT_PASS;
+out:
+ cg_destroy(test_group);
+ free(test_group);
+ return ret;
+}
+
#define T(x) { x, #x }
struct zswap_test {
int (*fn)(const char *root);
@@ -783,6 +941,7 @@ struct zswap_test {
T(test_no_kmem_bypass),
T(test_no_invasive_cgroup_shrink),
T(test_zswap_incompressible),
+ T(test_zswap_proactive_writeback),
};
#undef T
--
2.34.1
^ permalink raw reply related
* Re: [PATCH v17 06/10] riscv: kexec_file: Use crash_prepare_headers() helper to simplify code
From: Guo Ren @ 2026-06-29 11:28 UTC (permalink / raw)
To: Jinjie Ruan
Cc: corbet, skhan, catalin.marinas, will, chenhuacai, kernel, maddy,
mpe, npiggin, chleroy, pjw, palmer, aou, alex, tglx, mingo, bp,
dave.hansen, hpa, robh, saravanak, akpm, baoquan.he, rppt,
pasha.tatashin, pratyush, ruirui.yang, rdunlap, peterz, feng.tang,
dapeng1.mi, elver, enelsonmoore, kuba, ebiggers, lirongqing,
leitao, kees, coxu, cfsworks, jbohac, osandov, ryan.roberts,
tangyouling, sourabhjain, ritesh.list, gaohan, david, wangruikang,
muchun.song, vishal.moola, junhui.liu, namcao, rick.p.edgecombe,
djbw, liaoyuanhong, fuqiang.wang, vishal.l.verma, chenjiahao16,
x86, linux-doc, linux-kernel, linux-arm-kernel, loongarch,
linuxppc-dev, linux-riscv, devicetree, kexec
In-Reply-To: <20260629094746.191843-7-ruanjinjie@huawei.com>
On Mon, Jun 29, 2026 at 5:48 PM Jinjie Ruan <ruanjinjie@huawei.com> wrote:
>
> Use the newly introduced crash_prepare_headers() function to replace
> the existing prepare_elf_headers(), allocate cmem and exclude crash kernel
> memory in the crash core, which reduce code duplication.
>
> Only the following two architecture functions need to be implemented:
> - arch_get_system_nr_ranges(). Call get_nr_ram_ranges_callback()
> to pre-counts the max number of memory ranges.
>
> - arch_crash_populate_cmem(). Use prepare_elf64_ram_headers_callback()
> to collects the memory ranges and fills them into cmem.
>
> Cc: Paul Walmsley <pjw@kernel.org>
> Cc: Palmer Dabbelt <palmer@dabbelt.com>
> Cc: Albert Ou <aou@eecs.berkeley.edu>
> Cc: Alexandre Ghiti <alex@ghiti.fr>
> Cc: Guo Ren <guoren@kernel.org>
> Reviewed-by: Sourabh Jain <sourabhjain@linux.ibm.com>
> Acked-by: Baoquan He <bhe@redhat.com>
> Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
> Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com>
> ---
> arch/riscv/kernel/machine_kexec_file.c | 47 +++++++-------------------
> 1 file changed, 12 insertions(+), 35 deletions(-)
>
> diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c
> index fa2946aa9b8f..1dfb1d9eb691 100644
> --- a/arch/riscv/kernel/machine_kexec_file.c
> +++ b/arch/riscv/kernel/machine_kexec_file.c
> @@ -45,6 +45,15 @@ static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
> return 0;
> }
>
> +unsigned int arch_get_system_nr_ranges(void)
> +{
> + unsigned int nr_ranges = 2; /* For exclusion of crashkernel region */
> +
> + walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
> +
> + return nr_ranges;
> +}
> +
> static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
> {
> struct crash_mem *cmem = arg;
> @@ -56,41 +65,9 @@ static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
> return 0;
> }
>
> -static int prepare_elf_headers(void **addr, unsigned long *sz)
I agree to make arch_crash_populate_cmem more generic, and leave
arch_crash_populate_cmem & prepare_elf64_ram_headers_callback here as
callbacks.
Reviewed-by: Guo Ren <guoren@kernel.org>
> +int arch_crash_populate_cmem(struct crash_mem *cmem)
> {
> - struct crash_mem *cmem;
> - unsigned int nr_ranges;
> - int ret;
> -
> - nr_ranges = 2; /* For exclusion of crashkernel region */
> - walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
> -
> - cmem = kmalloc_flex(*cmem, ranges, nr_ranges);
> - if (!cmem)
> - return -ENOMEM;
> -
> - cmem->max_nr_ranges = nr_ranges;
> - cmem->nr_ranges = 0;
> - ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
> - if (ret)
> - goto out;
> -
> - /* Exclude crashkernel region */
> - ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
> - if (ret)
> - goto out;
> -
> - if (crashk_low_res.end) {
> - ret = crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end);
> - if (ret)
> - goto out;
> - }
> -
> - ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
> -
> -out:
> - kfree(cmem);
> - return ret;
> + return walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
> }
>
> static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
> @@ -282,7 +259,7 @@ int load_extra_segments(struct kimage *image, unsigned long kernel_start,
> if (image->type == KEXEC_TYPE_CRASH) {
> void *headers;
> unsigned long headers_sz;
> - ret = prepare_elf_headers(&headers, &headers_sz);
> + ret = crash_prepare_headers(true, &headers, &headers_sz, NULL);
> if (ret) {
> pr_err("Preparing elf core header failed\n");
> goto out;
> --
> 2.34.1
>
--
Best Regards
Guo Ren
^ permalink raw reply
* Re: [PATCH v4 2/5] mm/zswap: Factor writeback loop out of shrink_worker()
From: Hao Jia @ 2026-06-29 11:36 UTC (permalink / raw)
To: Yosry Ahmed
Cc: nphamcs, akpm, tj, hannes, shakeel.butt, mhocko, mkoutny,
chengming.zhou, muchun.song, roman.gushchin, linux-mm,
linux-kernel, linux-doc, Hao Jia
In-Reply-To: <CAO9r8zPVzMKFbCixxD-qgtRrkFxWVrHiZZeLc=eyTPKPVQgX4g@mail.gmail.com>
On 2026/6/27 01:09, Yosry Ahmed wrote:
>>>> /*
>>>> * Take one step of a memcg-tree writeback walk driven by the caller's
>>>> * iterator, and fold the result into @s, the retry bookkeeping shared
>>>> * across steps. @memcg is the iterator's current memcg, or NULL once
>>>> * it has wrapped around after a full pass over the tree.
>>>> *
>>>> * The function returns -EAGAIN to signal the caller to abort the walk
>>>> * after encountering the following conditions MAX_RECLAIM_RETRIES times:
>>>> * - No writeback-candidate memcgs were found in a memcg tree walk.
>>>> * - Shrinking a writeback-candidate memcg failed.
>>>
>>> Orthogonal to this patch, but I wonder if this can be simplified. I
>>> wonder if these two conditions can be replaced with "shrinking a memcg
>>> that has zswap entries failed". The "no writeback-candidate memcgs in
>>> the tree" case seems like we should abort right away instead of
>>> retrying?
>>>
>>> Nhat, WDYT?
>>>
>>
>> Perhaps something like the following is what you had in mind? I've
>> drafted the implementation below to make it easier for Nhat to compare
>> with the previous behavior.
>
> Hmm I think if we pursue this it should be in a separate patch or even
> outside of this series, ideally with numbers/proof that it's not
> introducing regressions to the scenario that lead to its introduction.
>
Got it. I will temporarily leave this part out of the current patch series.
>>
>>
>>>> *
>>>> * Return: The number of compressed bytes written back (>= 0), or -EAGAIN
>>>> * once the retry budget is exhausted and the caller should abort the walk.
>>>> */
>>>> static long zswap_shrink_one(struct mem_cgroup *memcg,
>>>
>>> Nit: zswap_shrink_one_memcg()
>>>
>>> BTW, the existing writeback logic has been broken for a while now when
>>> memcg is disabled. I think we constantly hit the !memcg case and run
>>> out of retries. Not sure if your patch changes this in any way, or if
>>> you want to fix that while you're at it :)
>>
>> Yes, I'd be happy to do that. However, would it be better to submit a
>> separate fix patch or combine it with this one?
>
> A separate patch. Feel free to send it with this series to avoid
> conflicts, but probably as patch 1 as we'll want to CC stable on it.
>
Done, and I've just submitted the v5 patch.
[v5]
https://lore.kernel.org/all/20260629112032.20423-1-jiahao.kernel@gmail.com
Please take a look when you have a chance. Thank you very much for your
review!
> [..]
>
>> /* Track progress of a memcg-tree writeback walk. */
>> struct zswap_shrink_state {
>> int scans;
>> int failures;
>> };
>>
>> /*
>> * Take one step of a memcg-tree writeback walk driven by the caller's
>> * iterator, and fold the result into @s, the retry bookkeeping shared
>> * across steps. @memcg is the iterator's current memcg, or NULL once
>> * it has wrapped around after a full pass over the tree.
>> *
>> * The function returns -EBUSY to signal the caller to abort the walk when
>> * either of the following occurs:
>> * - A full pass over the tree found no writeback-candidate memcg.
>> * - Shrinking a writeback-candidate memcg failed MAX_RECLAIM_RETRIES
>> times.
>> *
>> * When memory cgroup is disabled, the iterator always yields NULL. All
>> * zswap entries then live on the root list_lru, so NULL is treated as the
>> * root memcg and shrunk directly rather than as a completed tree pass.
>
> I think this chunk should be moved above the code returning -EBUSY
> when mem_cgroup_disabled() is true, and probably made more succinct as
> it should be obvious.
Done in v5.
>
>> *
>> * Return: The number of compressed bytes written back (>= 0), or -EBUSY
>> * when the caller should abort the walk.
>> */
>> static long zswap_shrink_one_memcg(struct mem_cgroup *memcg,
>> struct zswap_shrink_state *s)
>> {
>> bool disabled = mem_cgroup_disabled();
>
> No need to store this in a variable AFAICT, it's a static branch and
> it's clearer to just call it directly in both call sites imo.
Done, and I've just submitted the v5 patch.
Thanks,
Hao
>
>> long shrunk;
>>
>> /*
>> * If the iterator has completed a full pass, update the shrink state
>> * and check whether we should keep going.
>> * With memcg disabled the iterator always yields NULL, so fall through
>> * and shrink the root memcg directly instead.
>> */
>> if (!memcg && !disabled) {
>> /*
>> * Abort if no writeback-candidate memcgs in the last tree walk.
>> * Otherwise reset the scans count and continue.
>> */
>> if (!s->scans)
>> return -EBUSY;
>> s->scans = 0;
>> return 0;
>> }
>>
>> shrunk = shrink_memcg(memcg, NR_ZSWAP_WB_BATCH);
>>
>> /*
>> * There are no writeback-candidate pages in the memcg. With memcg
>> * enabled this is not an issue as long as we can find another memcg
>> * with pages in zswap, so skip without counting it as a candidate.
>> * With memcg disabled the root LRU is the only target, so we should
>> * abort if it has no writeback-candidate pages.
>> */
>> if (shrunk == -ENOENT)
>> return disabled ? -EBUSY : 0;
>> s->scans++;
>>
>> if (shrunk <= 0 && ++s->failures == MAX_RECLAIM_RETRIES)
>> return -EBUSY;
>>
>> return shrunk;
>> }
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox