[PATCH] NFSD: Add io_cache_{read,write} controls to debugfs

[PATCH] NFSD: Add io_cache_{read,write} controls to debugfs

[Chuck Lever]

From: Chuck Lever <chuck.lever@oracle.com>

Add 'io_cache_read' to NFSD's debugfs interface so that any data
read by NFSD will either be:
- cached using page cache (NFSD_IO_BUFFERED=0)
- cached but removed from the page cache upon completion
  (NFSD_IO_DONTCACHE=1).

io_cache_read may be set by writing to:
  /sys/kernel/debug/nfsd/io_cache_read

Add 'io_cache_write' to NFSD's debugfs interface so that any data
written by NFSD will either be:
- cached using page cache (NFSD_IO_BUFFERED=0)
- cached but removed from the page cache upon completion
  (NFSD_IO_DONTCACHE=1).

io_cache_write may be set by writing to:
  /sys/kernel/debug/nfsd/io_cache_write

The default value for both settings is NFSD_IO_BUFFERED, which is
NFSD's existing behavior for both read and write. Changes to these
settings take immediate effect for all exports and NFS versions.

If NFSD_IO_DONTCACHE is specified, all exported filesystems must
implement FOP_DONTCACHE, otherwise IO flagged with RWF_DONTCACHE
will fail with -EOPNOTSUPP.

Signed-off-by: Mike Snitzer <snitzer@kernel.org>
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 fs/nfsd/debugfs.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfsd.h    |  9 +++++
 fs/nfsd/vfs.c     | 19 ++++++++++
 3 files changed, 121 insertions(+)

Changes from Mike's v9:
- Squashed the "io controls" patches together
- Removed NFSD_IO_DIRECT for the moment
- Addressed a few more checkpatch.pl nits

This gives a cleaner platform on which to build the direct I/O code
paths, and does not expose partially implemented I/O modes to users.

diff --git a/fs/nfsd/debugfs.c b/fs/nfsd/debugfs.c
index 84b0c8b559dc..2b1bb716b608 100644
--- a/fs/nfsd/debugfs.c
+++ b/fs/nfsd/debugfs.c
@@ -27,11 +27,98 @@ static int nfsd_dsr_get(void *data, u64 *val)
 static int nfsd_dsr_set(void *data, u64 val)
 {
 	nfsd_disable_splice_read = (val > 0) ? true : false;
+	if (!nfsd_disable_splice_read) {
+		/*
+		 * Must use buffered I/O if splice_read is enabled.
+		 */
+		nfsd_io_cache_read = NFSD_IO_BUFFERED;
+	}
 	return 0;
 }
 
 DEFINE_DEBUGFS_ATTRIBUTE(nfsd_dsr_fops, nfsd_dsr_get, nfsd_dsr_set, "%llu\n");
 
+/*
+ * /sys/kernel/debug/nfsd/io_cache_read
+ *
+ * Contents:
+ *   %0: NFS READ will use buffered IO
+ *   %1: NFS READ will use dontcache (buffered IO w/ dropbehind)
+ *
+ * This setting takes immediate effect for all NFS versions,
+ * all exports, and in all NFSD net namespaces.
+ */
+
+static int nfsd_io_cache_read_get(void *data, u64 *val)
+{
+	*val = nfsd_io_cache_read;
+	return 0;
+}
+
+static int nfsd_io_cache_read_set(void *data, u64 val)
+{
+	int ret = 0;
+
+	switch (val) {
+	case NFSD_IO_BUFFERED:
+		nfsd_io_cache_read = NFSD_IO_BUFFERED;
+		break;
+	case NFSD_IO_DONTCACHE:
+		/*
+		 * Must disable splice_read when enabling
+		 * NFSD_IO_DONTCACHE.
+		 */
+		nfsd_disable_splice_read = true;
+		nfsd_io_cache_read = val;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(nfsd_io_cache_read_fops, nfsd_io_cache_read_get,
+			 nfsd_io_cache_read_set, "%llu\n");
+
+/*
+ * /sys/kernel/debug/nfsd/io_cache_write
+ *
+ * Contents:
+ *   %0: NFS WRITE will use buffered IO
+ *   %1: NFS WRITE will use dontcache (buffered IO w/ dropbehind)
+ *
+ * This setting takes immediate effect for all NFS versions,
+ * all exports, and in all NFSD net namespaces.
+ */
+
+static int nfsd_io_cache_write_get(void *data, u64 *val)
+{
+	*val = nfsd_io_cache_write;
+	return 0;
+}
+
+static int nfsd_io_cache_write_set(void *data, u64 val)
+{
+	int ret = 0;
+
+	switch (val) {
+	case NFSD_IO_BUFFERED:
+	case NFSD_IO_DONTCACHE:
+		nfsd_io_cache_write = val;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(nfsd_io_cache_write_fops, nfsd_io_cache_write_get,
+			 nfsd_io_cache_write_set, "%llu\n");
+
 void nfsd_debugfs_exit(void)
 {
 	debugfs_remove_recursive(nfsd_top_dir);
@@ -44,4 +131,10 @@ void nfsd_debugfs_init(void)
 
 	debugfs_create_file("disable-splice-read", S_IWUSR | S_IRUGO,
 			    nfsd_top_dir, NULL, &nfsd_dsr_fops);
+
+	debugfs_create_file("io_cache_read", 0644, nfsd_top_dir, NULL,
+			    &nfsd_io_cache_read_fops);
+
+	debugfs_create_file("io_cache_write", 0644, nfsd_top_dir, NULL,
+			    &nfsd_io_cache_write_fops);
 }
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index 1cd0bed57bc2..809729d41e08 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -153,6 +153,15 @@ static inline void nfsd_debugfs_exit(void) {}
 
 extern bool nfsd_disable_splice_read __read_mostly;
 
+enum {
+	/* Any new NFSD_IO enum value must be added at the end */
+	NFSD_IO_BUFFERED,
+	NFSD_IO_DONTCACHE,
+};
+
+extern u64 nfsd_io_cache_read __read_mostly;
+extern u64 nfsd_io_cache_write __read_mostly;
+
 extern int nfsd_max_blksize;
 
 static inline int nfsd_v4client(struct svc_rqst *rq)
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index 3cd3b9e069f4..b081824f5528 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -49,6 +49,8 @@
 #define NFSDDBG_FACILITY		NFSDDBG_FILEOP
 
 bool nfsd_disable_splice_read __read_mostly;
+u64 nfsd_io_cache_read __read_mostly = NFSD_IO_BUFFERED;
+u64 nfsd_io_cache_write __read_mostly = NFSD_IO_BUFFERED;
 
 /**
  * nfserrno - Map Linux errnos to NFS errnos
@@ -1099,6 +1101,15 @@ __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	size_t len;
 
 	init_sync_kiocb(&kiocb, file);
+
+	switch (nfsd_io_cache_read) {
+	case NFSD_IO_BUFFERED:
+		break;
+	case NFSD_IO_DONTCACHE:
+		kiocb.ki_flags = IOCB_DONTCACHE;
+		break;
+	}
+
 	kiocb.ki_pos = offset;
 
 	v = 0;
@@ -1224,6 +1235,14 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	since = READ_ONCE(file->f_wb_err);
 	if (verf)
 		nfsd_copy_write_verifier(verf, nn);
+
+	switch (nfsd_io_cache_write) {
+	case NFSD_IO_BUFFERED:
+		break;
+	case NFSD_IO_DONTCACHE:
+		kiocb.ki_flags |= IOCB_DONTCACHE;
+		break;
+	}
 	host_err = vfs_iocb_iter_write(file, &kiocb, &iter);
 	if (host_err < 0) {
 		commit_reset_write_verifier(nn, rqstp, host_err);
-- 
2.50.0

Re: NFSD: Add io_cache_{read,write} controls to debugfs

[Mike Snitzer]

On Fri, Sep 05, 2025 at 10:55:09AM -0400, Chuck Lever wrote:
> From: Chuck Lever <chuck.lever@oracle.com>

Aside from attribution, which I suspect you didn't intend to switch
these changes from being attributed to me, this looks good (one small
nit below).

> Add 'io_cache_read' to NFSD's debugfs interface so that any data
> read by NFSD will either be:
> - cached using page cache (NFSD_IO_BUFFERED=0)
> - cached but removed from the page cache upon completion
>   (NFSD_IO_DONTCACHE=1).
> 
> io_cache_read may be set by writing to:
>   /sys/kernel/debug/nfsd/io_cache_read
> 
> Add 'io_cache_write' to NFSD's debugfs interface so that any data
> written by NFSD will either be:
> - cached using page cache (NFSD_IO_BUFFERED=0)
> - cached but removed from the page cache upon completion
>   (NFSD_IO_DONTCACHE=1).
> 
> io_cache_write may be set by writing to:
>   /sys/kernel/debug/nfsd/io_cache_write
> 
> The default value for both settings is NFSD_IO_BUFFERED, which is
> NFSD's existing behavior for both read and write. Changes to these
> settings take immediate effect for all exports and NFS versions.
> 
> If NFSD_IO_DONTCACHE is specified, all exported filesystems must
> implement FOP_DONTCACHE, otherwise IO flagged with RWF_DONTCACHE
> will fail with -EOPNOTSUPP.
> 
> Signed-off-by: Mike Snitzer <snitzer@kernel.org>
> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
> ---
>  fs/nfsd/debugfs.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++
>  fs/nfsd/nfsd.h    |  9 +++++
>  fs/nfsd/vfs.c     | 19 ++++++++++
>  3 files changed, 121 insertions(+)
> 
> Changes from Mike's v9:
> - Squashed the "io controls" patches together
> - Removed NFSD_IO_DIRECT for the moment
> - Addressed a few more checkpatch.pl nits
> 
> This gives a cleaner platform on which to build the direct I/O code
> paths, and does not expose partially implemented I/O modes to users.
>
> diff --git a/fs/nfsd/debugfs.c b/fs/nfsd/debugfs.c
> index 84b0c8b559dc..2b1bb716b608 100644
> --- a/fs/nfsd/debugfs.c
> +++ b/fs/nfsd/debugfs.c
> @@ -44,4 +131,10 @@ void nfsd_debugfs_init(void)
>  
>  	debugfs_create_file("disable-splice-read", S_IWUSR | S_IRUGO,
>  			    nfsd_top_dir, NULL, &nfsd_dsr_fops);
> +
> +	debugfs_create_file("io_cache_read", 0644, nfsd_top_dir, NULL,
> +			    &nfsd_io_cache_read_fops);
> +
> +	debugfs_create_file("io_cache_write", 0644, nfsd_top_dir, NULL,
> +			    &nfsd_io_cache_write_fops);
>  }

Relative to checkpatch warnings, this ^ code is what I'm aware of.
For consistency I stuck with "S_IWUSR | S_IRUG", whereas you honored
checkpatch's suggestion to use 0644.

Maybe update disable-splice-read to also use 0644 too? But your call!

Thanks,
Mike

Re: NFSD: Add io_cache_{read,write} controls to debugfs

[Chuck Lever]

On 9/5/25 1:32 PM, Mike Snitzer wrote:
> On Fri, Sep 05, 2025 at 10:55:09AM -0400, Chuck Lever wrote:
>> From: Chuck Lever <chuck.lever@oracle.com>
> 
> Aside from attribution, which I suspect you didn't intend to switch
> these changes from being attributed to me, this looks good (one small
> nit below).

I didn't intend to change the attribution, you're right. My tool
chain's squashing always changes the patch authorship and I don't
always remember to catch it.

I will fix that up.


>> Add 'io_cache_read' to NFSD's debugfs interface so that any data
>> read by NFSD will either be:
>> - cached using page cache (NFSD_IO_BUFFERED=0)
>> - cached but removed from the page cache upon completion
>>   (NFSD_IO_DONTCACHE=1).
>>
>> io_cache_read may be set by writing to:
>>   /sys/kernel/debug/nfsd/io_cache_read
>>
>> Add 'io_cache_write' to NFSD's debugfs interface so that any data
>> written by NFSD will either be:
>> - cached using page cache (NFSD_IO_BUFFERED=0)
>> - cached but removed from the page cache upon completion
>>   (NFSD_IO_DONTCACHE=1).
>>
>> io_cache_write may be set by writing to:
>>   /sys/kernel/debug/nfsd/io_cache_write
>>
>> The default value for both settings is NFSD_IO_BUFFERED, which is
>> NFSD's existing behavior for both read and write. Changes to these
>> settings take immediate effect for all exports and NFS versions.
>>
>> If NFSD_IO_DONTCACHE is specified, all exported filesystems must
>> implement FOP_DONTCACHE, otherwise IO flagged with RWF_DONTCACHE
>> will fail with -EOPNOTSUPP.

Btw, I'd like NFSD to fall back to buffered I/O if the file system's
FOP_DONTCACHE flag is not set. Do you have an objection to that?


>> Signed-off-by: Mike Snitzer <snitzer@kernel.org>
>> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
>> ---
>>  fs/nfsd/debugfs.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++
>>  fs/nfsd/nfsd.h    |  9 +++++
>>  fs/nfsd/vfs.c     | 19 ++++++++++
>>  3 files changed, 121 insertions(+)
>>
>> Changes from Mike's v9:
>> - Squashed the "io controls" patches together
>> - Removed NFSD_IO_DIRECT for the moment
>> - Addressed a few more checkpatch.pl nits
>>
>> This gives a cleaner platform on which to build the direct I/O code
>> paths, and does not expose partially implemented I/O modes to users.
>>
>> diff --git a/fs/nfsd/debugfs.c b/fs/nfsd/debugfs.c
>> index 84b0c8b559dc..2b1bb716b608 100644
>> --- a/fs/nfsd/debugfs.c
>> +++ b/fs/nfsd/debugfs.c
>> @@ -44,4 +131,10 @@ void nfsd_debugfs_init(void)
>>  
>>  	debugfs_create_file("disable-splice-read", S_IWUSR | S_IRUGO,
>>  			    nfsd_top_dir, NULL, &nfsd_dsr_fops);
>> +
>> +	debugfs_create_file("io_cache_read", 0644, nfsd_top_dir, NULL,
>> +			    &nfsd_io_cache_read_fops);
>> +
>> +	debugfs_create_file("io_cache_write", 0644, nfsd_top_dir, NULL,
>> +			    &nfsd_io_cache_write_fops);
>>  }
> 
> Relative to checkpatch warnings, this ^ code is what I'm aware of.
> For consistency I stuck with "S_IWUSR | S_IRUG", whereas you honored
> checkpatch's suggestion to use 0644.
> 
> Maybe update disable-splice-read to also use 0644 too? But your call!

Still debating that myself. It's a change that is not related to the
purpose of this patch, so I probably should do that kind of clean-up
separately.

-- 
Chuck Lever

Re: NFSD: Add io_cache_{read,write} controls to debugfs

[Mike Snitzer]

On Fri, Sep 05, 2025 at 02:45:37PM -0400, Chuck Lever wrote:
> On 9/5/25 1:32 PM, Mike Snitzer wrote:
> > On Fri, Sep 05, 2025 at 10:55:09AM -0400, Chuck Lever wrote:
> >> From: Chuck Lever <chuck.lever@oracle.com>
> > 
> > Aside from attribution, which I suspect you didn't intend to switch
> > these changes from being attributed to me, this looks good (one small
> > nit below).
> 
> I didn't intend to change the attribution, you're right. My tool
> chain's squashing always changes the patch authorship and I don't
> always remember to catch it.
> 
> I will fix that up.

Ack.

> >> Add 'io_cache_read' to NFSD's debugfs interface so that any data
> >> read by NFSD will either be:
> >> - cached using page cache (NFSD_IO_BUFFERED=0)
> >> - cached but removed from the page cache upon completion
> >>   (NFSD_IO_DONTCACHE=1).
> >>
> >> io_cache_read may be set by writing to:
> >>   /sys/kernel/debug/nfsd/io_cache_read
> >>
> >> Add 'io_cache_write' to NFSD's debugfs interface so that any data
> >> written by NFSD will either be:
> >> - cached using page cache (NFSD_IO_BUFFERED=0)
> >> - cached but removed from the page cache upon completion
> >>   (NFSD_IO_DONTCACHE=1).
> >>
> >> io_cache_write may be set by writing to:
> >>   /sys/kernel/debug/nfsd/io_cache_write
> >>
> >> The default value for both settings is NFSD_IO_BUFFERED, which is
> >> NFSD's existing behavior for both read and write. Changes to these
> >> settings take immediate effect for all exports and NFS versions.
> >>
> >> If NFSD_IO_DONTCACHE is specified, all exported filesystems must
> >> implement FOP_DONTCACHE, otherwise IO flagged with RWF_DONTCACHE
> >> will fail with -EOPNOTSUPP.
> 
> Btw, I'd like NFSD to fall back to buffered I/O if the file system's
> FOP_DONTCACHE flag is not set. Do you have an objection to that?

Ah yes, now that you mention it I recall you suggesting that before.
Got lost in the shuffle but I agree that it makes sense.  Just that
like the NFSD_IO_DIRECT case, doing negative checks for every IO is
excessive. SO I left sorting out the best way to implement it as TBD,
but forgot about it.

The challenge is really that the global debugfs knobs are so
coarse-grained. Handling this type of thing will be much easier once
per-export configuration is possible.

> >> Signed-off-by: Mike Snitzer <snitzer@kernel.org>
> >> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
> >> ---
> >>  fs/nfsd/debugfs.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++
> >>  fs/nfsd/nfsd.h    |  9 +++++
> >>  fs/nfsd/vfs.c     | 19 ++++++++++
> >>  3 files changed, 121 insertions(+)
> >>
> >> Changes from Mike's v9:
> >> - Squashed the "io controls" patches together
> >> - Removed NFSD_IO_DIRECT for the moment
> >> - Addressed a few more checkpatch.pl nits
> >>
> >> This gives a cleaner platform on which to build the direct I/O code
> >> paths, and does not expose partially implemented I/O modes to users.
> >>
> >> diff --git a/fs/nfsd/debugfs.c b/fs/nfsd/debugfs.c
> >> index 84b0c8b559dc..2b1bb716b608 100644
> >> --- a/fs/nfsd/debugfs.c
> >> +++ b/fs/nfsd/debugfs.c
> >> @@ -44,4 +131,10 @@ void nfsd_debugfs_init(void)
> >>  
> >>  	debugfs_create_file("disable-splice-read", S_IWUSR | S_IRUGO,
> >>  			    nfsd_top_dir, NULL, &nfsd_dsr_fops);
> >> +
> >> +	debugfs_create_file("io_cache_read", 0644, nfsd_top_dir, NULL,
> >> +			    &nfsd_io_cache_read_fops);
> >> +
> >> +	debugfs_create_file("io_cache_write", 0644, nfsd_top_dir, NULL,
> >> +			    &nfsd_io_cache_write_fops);
> >>  }
> > 
> > Relative to checkpatch warnings, this ^ code is what I'm aware of.
> > For consistency I stuck with "S_IWUSR | S_IRUG", whereas you honored
> > checkpatch's suggestion to use 0644.
> > 
> > Maybe update disable-splice-read to also use 0644 too? But your call!
> 
> Still debating that myself. It's a change that is not related to the
> purpose of this patch, so I probably should do that kind of clean-up
> separately.

Ack.