[PATCH] drivers/char/mem: splice the zero page for /dev/zero and /dev/full

[Kris Pan <kris.pan@intel.com>]

Commit 1b057bd800c3 ("drivers/char/mem: implement splice() for /dev/zero,
/dev/full") added splice support to /dev/zero and /dev/full using
copy_splice_read(). The author noted that this could be optimized by
pushing references to the global zero page instead of allocating and
zeroing fresh pages for every pipe buffer, deferring it as "an
optimization for another day".

Implement splice_read_zero() to fulfill that optimization. Instead of
allocating a fresh page and memset()ting it to zero for every pipe buffer,
push references to the global zero page directly into the pipe.

The pipe buffers use a dedicated zero_pipe_buf_ops (mirroring the one in
mm/shmem.c) whose get() and release() are no-ops and whose try_steal()
always fails: ZERO_PAGE(0) is a reserved, permanently-referenced page that
is never freed, so no refcount is needed and the global zero page can
never be stolen, gifted, or corrupted by userspace. This also avoids the
cache-line bounce of bumping ZERO_PAGE's refcount on every splice, so
throughput keeps scaling across cores. splice_read_zero() runs under
pipe_lock() (via splice_file_to_pipe()) and returns -EAGAIN on a full
pipe, matching the ->splice_read() contract.

Measured on an Intel Core Ultra 7 155H (Meteor Lake P), splicing /dev/zero
to /dev/null through a pipe:

  Single-thread throughput:
    Baseline (copy_splice_read):  29.27 GB/s (+/- 0.25 GB/s, n=10)
    Patched  (splice_read_zero): 238.67 GB/s (+/- 0.60 GB/s, n=5)
    Delta:                        +715% (8.15x speedup)

  Multi-thread throughput (1/2/4/8/16 threads):
    Baseline:   27.6/ 53.8/106.1/171.2/142.5 GB/s (saturates, drops @16T)
    Patched:   191.7/377.5/733.5/1113.7/1379.5 GB/s (keeps scaling)

  Hardware counters (perf_event_open, 1GB transfer):
    Instructions:      258.9M ->  4.2M  (61x reduction, 0.004 instr/byte)
    Cycles:            172.0M ->  1.8M  (93x reduction)
    L1 dcache misses:    1.9M ->   248  (7662x reduction, 0.02% miss rate)
    IPC:                 1.51 ->  2.30  (+53%)

This eliminates the DRAM bus bandwidth and cache pollution from zeroing.
Stress testing 8.7 TB of spliced data showed no memory leak and no page
corruption.

Suggested-by: Max Kellermann <max.kellermann@ionos.com>
Signed-off-by: Kris Pan <kris.pan@intel.com>
---
 drivers/char/mem.c | 59 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 57 insertions(+), 2 deletions(-)

diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index 63253d1de5d70..598a4a6af8ffc 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -471,6 +471,61 @@ static ssize_t read_iter_zero(struct kiocb *iocb, struct iov_iter *iter)
 	return written;
 }
 
+static bool zero_pipe_buf_get(struct pipe_inode_info *pipe,
+			      struct pipe_buffer *buf)
+{
+	return true;
+}
+
+static void zero_pipe_buf_release(struct pipe_inode_info *pipe,
+				  struct pipe_buffer *buf)
+{
+}
+
+static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe,
+				    struct pipe_buffer *buf)
+{
+	return false;
+}
+
+static const struct pipe_buf_operations zero_pipe_buf_ops = {
+	.release	= zero_pipe_buf_release,
+	.try_steal	= zero_pipe_buf_try_steal,
+	.get		= zero_pipe_buf_get,
+};
+
+static ssize_t splice_read_zero(struct file *in, loff_t *ppos,
+				struct pipe_inode_info *pipe, size_t len,
+				unsigned int flags)
+{
+	size_t total = 0;
+	size_t used, npages;
+	struct page *page = ZERO_PAGE(0);
+
+	used = pipe_buf_usage(pipe);
+	if (used >= pipe->max_usage)
+		return -EAGAIN;
+	npages = pipe->max_usage - used;
+	len = min_t(size_t, len, npages * PAGE_SIZE);
+
+	while (len) {
+		size_t chunk = min_t(size_t, len, PAGE_SIZE);
+		struct pipe_buffer *buf = pipe_head_buf(pipe);
+
+		*buf = (struct pipe_buffer) {
+			.ops = &zero_pipe_buf_ops,
+			.page = page,
+			.offset = 0,
+			.len = chunk,
+		};
+		pipe->head++;
+		total += chunk;
+		len -= chunk;
+	}
+
+	return total;
+}
+
 static ssize_t read_zero(struct file *file, char __user *buf,
 			 size_t count, loff_t *ppos)
 {
@@ -669,7 +724,7 @@ static const struct file_operations zero_fops = {
 	.read_iter	= read_iter_zero,
 	.read		= read_zero,
 	.write_iter	= write_iter_zero,
-	.splice_read	= copy_splice_read,
+	.splice_read	= splice_read_zero,
 	.splice_write	= splice_write_zero,
 	.mmap_prepare	= mmap_zero_prepare,
 	.get_unmapped_area = get_unmapped_area_zero,
@@ -682,7 +737,7 @@ static const struct file_operations full_fops = {
 	.llseek		= full_lseek,
 	.read_iter	= read_iter_zero,
 	.write		= write_full,
-	.splice_read	= copy_splice_read,
+	.splice_read	= splice_read_zero,
 };
 
 static const struct memdev {
-- 
2.43.0