* [PATCH] NFS: correct CONFIG_NFS_V4 macro name in #endif comment
From: Ethan Nelson-Moore @ 2026-06-09 2:56 UTC (permalink / raw)
To: linux-nfs; +Cc: Ethan Nelson-Moore, Trond Myklebust, Anna Schumaker
A comment in fs/nfs/dir.c incorrectly refers to CONFIG_NFSV4 instead of
CONFIG_NFS_V4. Correct it.
Discovered while searching for CONFIG_* symbols referenced in code but
not defined in any Kconfig file.
Signed-off-by: Ethan Nelson-Moore <enelsonmoore@gmail.com>
---
fs/nfs/dir.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index e9ce1883288c..9381563c0a2a 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -2299,7 +2299,7 @@ nfs4_lookup_revalidate(struct inode *dir, const struct qstr *name,
return nfs_do_lookup_revalidate(dir, name, dentry, flags);
}
-#endif /* CONFIG_NFSV4 */
+#endif /* CONFIG_NFS_V4 */
int nfs_atomic_open_v23(struct inode *dir, struct dentry *dentry,
struct file *file, unsigned int open_flags,
--
2.43.0
^ permalink raw reply related
* Re: [PATCH 17/18] nfsd: use vfs_lookup_open() for non-creating open requests too.
From: NeilBrown @ 2026-06-09 1:03 UTC (permalink / raw)
To: Chuck Lever
Cc: Christian Brauner, Alexander Viro, Jeff Layton, Jan Kara,
linux-fsdevel, linux-nfs, Jori Koolstra, Benjamin Coddington,
Mateusz Guzik
In-Reply-To: <2f5da5fd-442f-4889-af15-58f9b5d68d71@app.fastmail.com>
On Fri, 05 Jun 2026, Chuck Lever wrote:
>
> On Sun, May 31, 2026, at 11:38 PM, NeilBrown wrote:
>
> > @@ -429,45 +438,35 @@ do_open_lookup(struct svc_rqst *rqstp, struct
> > nfsd4_compound_state *cstate, stru
> > fh_init(*resfh, NFS4_FHSIZE);
> > open->op_truncate = false;
> >
> > - status = fh_fill_pre_attrs(current_fh);
> > - if (status)
> > - goto out;
> > - if (open->op_create) {
> > - /* FIXME: check session persistence and pnfs flags.
> > - * The nfsv4.1 spec requires the following semantics:
> > - *
> > - * Persistent | pNFS | Server REQUIRED | Client Allowed
> > - * Reply Cache | server | |
> > - * -------------+--------+-----------------+--------------------
> > - * no | no | EXCLUSIVE4_1 | EXCLUSIVE4_1
> > - * | | | (SHOULD)
> > - * | | and EXCLUSIVE4 | or EXCLUSIVE4
> > - * | | | (SHOULD NOT)
> > - * no | yes | EXCLUSIVE4_1 | EXCLUSIVE4_1
> > - * yes | no | GUARDED4 | GUARDED4
> > - * yes | yes | GUARDED4 | GUARDED4
> > - */
> > + /* FIXME: check session persistence and pnfs flags.
> > + * The nfsv4.1 spec requires the following semantics:
> > + *
> > + * Persistent | pNFS | Server REQUIRED | Client Allowed
> > + * Reply Cache | server | |
> > + * -------------+--------+-----------------+--------------------
> > + * no | no | EXCLUSIVE4_1 | EXCLUSIVE4_1
> > + * | | | (SHOULD)
> > + * | | and EXCLUSIVE4 | or EXCLUSIVE4
> > + * | | | (SHOULD NOT)
> > + * no | yes | EXCLUSIVE4_1 | EXCLUSIVE4_1
> > + * yes | no | GUARDED4 | GUARDED4
> > + * yes | yes | GUARDED4 | GUARDED4
> > + */
> >
> > - current->fs->umask = open->op_umask;
> > - status = nfsd4_create_file(rqstp, current_fh, *resfh, open);
> > - current->fs->umask = 0;
> > + current->fs->umask = open->op_umask;
> > + status = nfsd4_open_file(rqstp, current_fh, *resfh, open);
> > + current->fs->umask = 0;
> > +
> > + /*
> > + * Following rfc 3530 14.2.16, and rfc 5661 18.16.4
> > + * use the returned bitmask to indicate which attributes
> > + * we used to store the verifier:
> > + */
> > + if (open->op_create && status == 0 &&
> > + nfsd4_create_is_exclusive(open->op_createmode))
> > + open->op_bmval[1] |= (FATTR4_WORD1_TIME_ACCESS |
> > + FATTR4_WORD1_TIME_MODIFY);
> >
> > - /*
> > - * Following rfc 3530 14.2.16, and rfc 5661 18.16.4
> > - * use the returned bitmask to indicate which attributes
> > - * we used to store the verifier:
> > - */
> > - if (nfsd4_create_is_exclusive(open->op_createmode) && status == 0)
> > - open->op_bmval[1] |= (FATTR4_WORD1_TIME_ACCESS |
> > - FATTR4_WORD1_TIME_MODIFY);
> > - } else {
> > - status = nfsd_lookup(rqstp, current_fh,
> > - open->op_fname, open->op_fnamelen, *resfh);
> > - /* NFSv4 protocol requires change attributes even though
> > - * no change happened.
> > - */
> > - fh_fill_post_noop(current_fh);
> > - }
> > if (status)
> > goto out;
> > status = nfsd_check_obj_isreg(*resfh, cstate->minorversion);
>
> Logic bug: Pre-series, the non-create case takes do_open_lookup()'s
> else branch and succeeds; post-series, do_open_lookup() calls
> nfsd4_open_file() unconditionally. Every CLAIM_NULL open of an
> existing file fails with NFS4ERR_EXIST.
yes - that is bad. I think
if (!open->op_created &&
- createmode == NFS4_CREATE_UNCHECKED) {
+ (open->op_create == NFS4_OPEN_NOCREATE ||
+ createmode == NFS4_CREATE_UNCHECKED)) {
/* NFSv4 protocol requires change attributes
* even though no change happened.
*/
is what I wanted to do. I've also move the clearing of ATTR_SIZE down
to where we know we did created the file, so no trunc is needed.
Thanks a lot for the review.
NeilBrown
^ permalink raw reply
* Re: [PATCH 16/18] nfsd: switch nfsd4_create_file() to use vfs_lookup_open()
From: NeilBrown @ 2026-06-09 0:44 UTC (permalink / raw)
To: Chuck Lever
Cc: Christian Brauner, Alexander Viro, Jeff Layton, Jan Kara,
linux-fsdevel, linux-nfs, Jori Koolstra, Benjamin Coddington,
Mateusz Guzik
In-Reply-To: <b86f4070-c2b8-4989-b956-ad0f40f88acb@app.fastmail.com>
On Fri, 05 Jun 2026, Chuck Lever wrote:
>
> On Sun, May 31, 2026, at 11:38 PM, NeilBrown wrote:
> > diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> > index 5bd19e5d9e34..61ecd4123817 100644
> > --- a/fs/nfsd/nfs4proc.c
> > +++ b/fs/nfsd/nfs4proc.c
>
> > @@ -302,33 +302,18 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct
> > svc_fh *fhp,
> > oflags |= O_RDONLY;
> > }
> >
> > - host_err = fh_want_write(fhp);
>
> This fh_want_write() is removed, but ...
>
>
> > - if (host_err)
> > - return nfserrno(host_err);
> > -
> > - child = start_creating(&nop_mnt_idmap, parent,
> > - &QSTR_LEN(open->op_fname, open->op_fnamelen));
> > - if (IS_ERR(child)) {
> > - status = nfserrno(PTR_ERR(child));
> > - goto out_write;
> > - }
> > - path.dentry = child;
> > -
> > - if (d_really_is_negative(child)) {
> > - open->op_filp = dentry_create(&path, oflags, open->op_iattr.ia_mode,
> > - current_cred());
> > - child = path.dentry;
> > -
> > - if (IS_ERR(open->op_filp)) {
> > - end_creating(child);
> > - status = nfserrno(PTR_ERR(open->op_filp));
> > - open->op_filp = NULL;
> > - goto out_write;
> > - }
> > -
> > - open->op_created = open->op_filp->f_mode & FMODE_CREATED;
> > + open->op_filp = vfs_lookup_open(&parent,
> > + &QSTR_LEN(open->op_fname,
> > + open->op_fnamelen),
> > + oflags,
> > + open->op_iattr.ia_mode);
> > + if (IS_ERR(open->op_filp)) {
> > + status = nfserrno(PTR_ERR(open->op_filp));
> > + open->op_filp = NULL;
> > + goto out;
> > }
> > - end_creating(child);
> > + child = open->op_filp->f_path.dentry;
> > + open->op_created = open->op_filp->f_mode & FMODE_CREATED;
> > fh_drop_write(fhp);
>
> ... the "matching" fh_drop_write remains. Harmless since nothing
> has set fhp->fh_want_write, but should be cleaned up.
Thanks for catching that!
NeilBrown
>
>
> >
> > status = fh_compose(resfhp, fhp->fh_export, child, fhp);
>
>
> --
> Chuck Lever
>
^ permalink raw reply
* Re: [PATCH] NFSD: fix up error returned by write_threads()
From: Chuck Lever @ 2026-06-09 0:33 UTC (permalink / raw)
To: jlayton, Scott Mayhew
Cc: Chuck Lever, neil, okorniev, Dai.Ngo, tom, linux-nfs
In-Reply-To: <20260608131402.95625-1-smayhew@redhat.com>
From: Chuck Lever <chuck.lever@oracle.com>
On Mon, 08 Jun 2026 09:14:02 -0400, Scott Mayhew wrote:
> Previously, writing 0 to /proc/fs/nfsd/threads would return 0 if the NFS
> server wasn't running. After commit 14282cc3cfa2, -EIO is returned.
> Existing scripts don't expect this behavior.
>
> Add a check to bypass the call to nfsd_svc() when newthreads is 0 and
> the NFS server is already stopped.
>
> [...]
Applied to nfsd-testing, thanks!
[1/1] NFSD: fix up error returned by write_threads()
commit: 36f28c2e37a6a508f5a7be62ebe9c8887ed9bd20
--
Chuck Lever <chuck.lever@oracle.com>
^ permalink raw reply
* Re: [PATCH net-next] net/sunrpc/svcauth_unix: Use strscpy() to copy strings into arrays
From: Chuck Lever @ 2026-06-09 0:33 UTC (permalink / raw)
To: Kees Cook, linux-hardening, linux-kernel, linux-nfs, netdev,
david.laight.linux
Cc: Chuck Lever, Arnd Bergmann, Anna Schumaker, David S. Miller,
Eric Dumazet, Jakub Kicinski, Jeff Layton, Paolo Abeni,
Trond Myklebust
In-Reply-To: <20260608095523.2606-16-david.laight.linux@gmail.com>
From: Chuck Lever <chuck.lever@oracle.com>
On Mon, 08 Jun 2026 10:55:00 +0100, david.laight.linux@gmail.com wrote:
> Replacing strcpy() with strscpy() ensures that overflow of the target
> buffer cannot happen.
Applied to nfsd-testing, thanks!
[1/1] net/sunrpc/svcauth_unix: Use strscpy() to copy strings into arrays
commit: cea9ba4f71e82766d782ed423c99bb12a69bbdf0
--
Chuck Lever <chuck.lever@oracle.com>
^ permalink raw reply
* Re: [PATCH 1/1] sunrpc: Use "%*phN" to dprintk() a cookie
From: Chuck Lever @ 2026-06-09 0:33 UTC (permalink / raw)
To: Jeff Layton, NeilBrown, Olga Kornievskaia, Dai Ngo, Tom Talpey,
Trond Myklebust, Anna Schumaker, linux-nfs, linux-kernel,
David Laight
Cc: Chuck Lever, Geert Uytterhoeven, Andy Shevchenko
In-Reply-To: <20260608212042.25476-1-david.laight.linux@gmail.com>
From: Chuck Lever <chuck.lever@oracle.com>
On Mon, 08 Jun 2026 22:20:42 +0100, David Laight wrote:
> Simplifies the code and removes a 'not obviously bounded' strcpy().
>
> Delete the local function nlmdbg_cookie2a() that did the equivalent.
>
> There is no need to worry about cookie->len being more than
> NLM_MAXCOOKIELEN (32), the buffer holding it is only that long.
> The existing length checks must pre-date this code being added in 2.4.26.
>
> [...]
Applied to nfsd-testing, thanks!
[1/1] lockd: Use "%*phN" to dprintk() a cookie
commit: 926ef0ef7994232faab29e6b13270d5c88b058f3
--
Chuck Lever <chuck.lever@oracle.com>
^ permalink raw reply
* [PATCH 1/1] sunrpc: Use "%*phN" to dprintk() a cookie
From: David Laight @ 2026-06-08 21:20 UTC (permalink / raw)
To: Chuck Lever, Jeff Layton, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker, linux-nfs,
linux-kernel
Cc: David Laight, Geert Uytterhoeven, Andy Shevchenko
Simplifies the code and removes a 'not obviously bounded' strcpy().
Delete the local function nlmdbg_cookie2a() that did the equivalent.
There is no need to worry about cookie->len being more than
NLM_MAXCOOKIELEN (32), the buffer holding it is only that long.
The existing length checks must pre-date this code being added in 2.4.26.
Signed-off-by: David Laight <david.laight.linux@gmail.com>
---
Found by a local build that errors strcpy() unless a constant string
is being written into an array.
fs/lockd/svclock.c | 42 +++++-------------------------------------
1 file changed, 5 insertions(+), 37 deletions(-)
diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c
index b98b1d0ada35..bcf261078aba 100644
--- a/fs/lockd/svclock.c
+++ b/fs/lockd/svclock.c
@@ -47,40 +47,6 @@ static const struct rpc_call_ops nlmsvc_grant_ops;
static LIST_HEAD(nlm_blocked);
static DEFINE_SPINLOCK(nlm_blocked_lock);
-#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
-static const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie)
-{
- /*
- * We can get away with a static buffer because this is only called
- * from lockd, which is single-threaded.
- */
- static char buf[2*NLM_MAXCOOKIELEN+1];
- unsigned int i, len = sizeof(buf);
- char *p = buf;
-
- len--; /* allow for trailing \0 */
- if (len < 3)
- return "???";
- for (i = 0 ; i < cookie->len ; i++) {
- if (len < 2) {
- strcpy(p-3, "...");
- break;
- }
- sprintf(p, "%02x", cookie->data[i]);
- p += 2;
- len -= 2;
- }
- *p = '\0';
-
- return buf;
-}
-#else
-static inline const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie)
-{
- return "???";
-}
-#endif
-
/*
* Insert a blocked lock into the global list
*/
@@ -155,11 +121,12 @@ nlmsvc_lookup_block(struct nlm_file *file, struct nlm_lock *lock)
spin_lock(&nlm_blocked_lock);
list_for_each_entry(block, &nlm_blocked, b_list) {
fl = &block->b_call->a_args.lock.fl;
- dprintk("lockd: check f=%p pd=%d %Ld-%Ld ty=%d cookie=%s\n",
+ dprintk("lockd: check f=%p pd=%d %Ld-%Ld ty=%d cookie=%*phN\n",
block->b_file, fl->c.flc_pid,
(long long)fl->fl_start,
(long long)fl->fl_end, fl->c.flc_type,
- nlmdbg_cookie2a(&block->b_call->a_args.cookie));
+ block->b_call->a_args.cookie.len,
+ block->b_call->a_args.cookie.data);
if (block->b_file == file && nlm_compare_locks(fl, &lock->fl)) {
kref_get(&block->b_count);
spin_unlock(&nlm_blocked_lock);
@@ -198,7 +165,8 @@ nlmsvc_find_block(struct nlm_cookie *cookie)
return NULL;
found:
- dprintk("nlmsvc_find_block(%s): block=%p\n", nlmdbg_cookie2a(cookie), block);
+ dprintk("nlmsvc_find_block(%*phN): block=%p\n",
+ cookie->len, cookie->data, block);
kref_get(&block->b_count);
spin_unlock(&nlm_blocked_lock);
return block;
--
2.39.5
^ permalink raw reply related
* Re: [PATCH v5 10/21] nfsd: add notification handlers for dir events
From: Chuck Lever @ 2026-06-08 20:52 UTC (permalink / raw)
To: Jeff Layton, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker, Jonathan Corbet,
Shuah Khan
Cc: Steven Rostedt, Alexander Aring, Amir Goldstein, Jan Kara,
Alexander Viro, Christian Brauner, Calum Mackay, linux-kernel,
linux-doc, linux-nfs
In-Reply-To: <20260522-dir-deleg-v5-10-542cddfad576@kernel.org>
On Fri, May 22, 2026, at 3:42 PM, Jeff Layton wrote:
> Add the necessary parts to accept a fsnotify callback for directory
> change event and create a CB_NOTIFY request for it. When a dir nfsd_file
> is created set a handle_event callback to handle the notification.
> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
> index e17488a911f7..31df04675713 100644
> --- a/fs/nfsd/nfs4xdr.c
> +++ b/fs/nfsd/nfs4xdr.c
> @@ -4172,6 +4172,127 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp,
> struct xdr_stream *xdr,
> goto out;
> }
>
> +static bool
> +nfsd4_setup_notify_entry4(struct notify_entry4 *ne, struct xdr_stream
> *xdr,
> + struct dentry *dentry, struct nfs4_delegation *dp,
> + struct nfsd_file *nf, char *name, u32 namelen)
> +{
> + uint32_t *attrmask;
> +
> + /* Reserve space for attrmask */
> + attrmask = xdr_reserve_space(xdr, 3 * sizeof(uint32_t));
> + if (!attrmask)
> + return false;
> +
> + ne->ne_file.data = name;
> + ne->ne_file.len = namelen;
> + ne->ne_attrs.attrmask.element = attrmask;
> +
> + attrmask[0] = 0;
> + attrmask[1] = 0;
> + attrmask[2] = 0;
> + ne->ne_attrs.attr_vals.data = NULL;
> + ne->ne_attrs.attr_vals.len = 0;
> + ne->ne_attrs.attrmask.count = 1;
> + return true;
> +}
> +
> +/**
> + * nfsd4_encode_notify_event - encode a notify
> + * @xdr: stream to which to encode the fattr4
> + * @nne: nfsd_notify_event to encode
> + * @dp: delegation where the event occurred
> + * @nf: nfsd_file on which event occurred
> + * @notify_mask: pointer to word where notification mask should be set
> + *
> + * Encode @nne into @xdr. Returns a pointer to the start of the event,
> or NULL if
> + * the event couldn't be encoded. The appropriate bit in the
> notify_mask will also
> + * be set on success.
> + */
> +u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct
> nfsd_notify_event *nne,
> + struct nfs4_delegation *dp, struct nfsd_file *nf,
> + u32 *notify_mask)
> +{
> + u8 *p = NULL;
> +
> + *notify_mask = 0;
> +
> + if (nne->ne_mask & FS_DELETE) {
> + struct notify_remove4 nr = { };
> +
> + if (!nfsd4_setup_notify_entry4(&nr.nrm_old_entry, xdr,
> nne->ne_dentry, dp,
> + nf, nne->ne_name, nne->ne_namelen))
> + goto out_err;
> + p = (u8 *)xdr->p;
> + if (!xdrgen_encode_notify_remove4(xdr, &nr))
> + goto out_err;
> + *notify_mask |= BIT(NOTIFY4_REMOVE_ENTRY);
> + } else if (nne->ne_mask & FS_CREATE) {
> + struct notify_add4 na = { };
> + struct notify_remove4 old = { };
> +
> + if (!nfsd4_setup_notify_entry4(&na.nad_new_entry, xdr,
> nne->ne_dentry, dp,
> + nf, nne->ne_name, nne->ne_namelen))
> + goto out_err;
> +
> + /* If a file was overwritten, report it in nad_old_entry */
> + if (nne->ne_target) {
> + if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr,
> + NULL, dp, nf,
> + nne->ne_name, nne->ne_namelen))
> + goto out_err;
> + na.nad_old_entry.count = 1;
> + na.nad_old_entry.element = &old;
> + }
> +
> + p = (u8 *)xdr->p;
> + if (!xdrgen_encode_notify_add4(xdr, &na))
> + goto out_err;
> +
> + *notify_mask |= BIT(NOTIFY4_ADD_ENTRY);
> + } else if (nne->ne_mask & FS_RENAME) {
> + struct notify_rename4 nr = { };
> + struct notify_remove4 old = { };
> + struct name_snapshot n;
> + bool ret;
> +
> + /* Don't send any attributes in the old_entry since they're the same
> in new */
> + if (!nfsd4_setup_notify_entry4(&nr.nrn_old_entry.nrm_old_entry, xdr,
> + NULL, dp, nf, nne->ne_name,
> + nne->ne_namelen))
> + goto out_err;
> +
> + take_dentry_name_snapshot(&n, nne->ne_dentry);
> + ret = nfsd4_setup_notify_entry4(&nr.nrn_new_entry.nad_new_entry, xdr,
> + nne->ne_dentry, dp, nf, (char *)n.name.name,
> + n.name.len);
Now once I got all of the previous edits in place, all three LLM
reviewers identified an issue here that might require a significant
rewrite. This is why I stopped the minor editing here and decided
it was time for you to consider restructuring (or not). I haven't
looked at patches 11-21.
I think the new name here has a time-of-use problem.
nrn_old_entry uses nne->ne_name, which alloc_nfsd_notify_event() copied
when fsnotify delivered the rename. nrn_new_entry instead reads the
live dentry via take_dentry_name_snapshot() at callback-prepare time,
which can run long after the event was queued.
CB_NOTIFY is asynchronous: nfsd_handle_dir_event() queues the event on
ncn_evt[] and nothing holds ne_dentry stable until the work runs.
d_move() reuses the same dentry and rewrites d_name in place, so a
second rename of the entry before the queued callback encodes leaves
the dget'd ne_dentry carrying the later name. An A->B event then
encodes as A->C, and a client holding the directory delegation applies
the wrong old->new mapping to its cache. The old name is immune
because it was snapshotted up front; only the new name is read late.
The new name is available at notification time -- fsnotify_move() passes
&moved->d_name as new_name, and ne_dentry is that moved dentry -- so
alloc_nfsd_notify_event() can snapshot it alongside the old name.
What I haven't assessed is whether the suggested restructuring is
now vulnerable to misbehavior during memory exhaustion.
> +
> + /* If a file was overwritten, report it in nad_old_entry */
> + if (ret && nne->ne_target) {
> + ret = nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr,
> + NULL, dp, nf,
> + (char *)n.name.name, n.name.len);
> + if (ret) {
> + nr.nrn_new_entry.nad_old_entry.count = 1;
> + nr.nrn_new_entry.nad_old_entry.element = &old;
> + }
> + }
> +
> + if (ret) {
> + p = (u8 *)xdr->p;
> + ret = xdrgen_encode_notify_rename4(xdr, &nr);
> + }
> + release_dentry_name_snapshot(&n);
> + if (!ret)
> + goto out_err;
> + *notify_mask |= BIT(NOTIFY4_RENAME_ENTRY);
> + }
> + return p;
> +out_err:
> + pr_warn("nfsd: unable to marshal notify_rename4 to xdr stream\n");
> + return NULL;
> +}
> +
> static void svcxdr_init_encode_from_buffer(struct xdr_stream *xdr,
> struct xdr_buf *buf, __be32 *p, int bytes)
> {
--
Chuck Lever
^ permalink raw reply
* Re: [PATCH v5 10/21] nfsd: add notification handlers for dir events
From: Chuck Lever @ 2026-06-08 20:40 UTC (permalink / raw)
To: Jeff Layton, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker, Jonathan Corbet,
Shuah Khan
Cc: Steven Rostedt, Alexander Aring, Amir Goldstein, Jan Kara,
Alexander Viro, Christian Brauner, Calum Mackay, linux-kernel,
linux-doc, linux-nfs
In-Reply-To: <20260522-dir-deleg-v5-10-542cddfad576@kernel.org>
On Fri, May 22, 2026, at 3:42 PM, Jeff Layton wrote:
> Add the necessary parts to accept a fsnotify callback for directory
> change event and create a CB_NOTIFY request for it. When a dir nfsd_file
> is created set a handle_event callback to handle the notification.
>
> Use that to allocate a nfsd_notify_event object and then hand off a
> reference to each delegation's CB_NOTIFY. If anything fails along the
> way, recall any affected delegations.
>
> Signed-off-by: Jeff Layton <jlayton@kernel.org>
There are some significant-looking sashiko review findings which I did
not follow up on.
> diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
> index ea3e7deb06fa..1964a213f80e 100644
> --- a/fs/nfsd/nfs4callback.c
> +++ b/fs/nfsd/nfs4callback.c
> @@ -870,21 +870,30 @@ static void nfs4_xdr_enc_cb_notify(struct
> rpc_rqst *req,
> const void *data)
> {
> const struct nfsd4_callback *cb = data;
> + struct nfsd4_cb_notify *ncn = container_of(cb, struct
> nfsd4_cb_notify, ncn_cb);
> + struct nfs4_delegation *dp = container_of(ncn, struct
> nfs4_delegation, dl_cb_notify);
> struct nfs4_cb_compound_hdr hdr = {
> .ident = 0,
> .minorversion = cb->cb_clp->cl_minorversion,
> };
> - struct CB_NOTIFY4args args = { };
> + struct CB_NOTIFY4args args;
> + __be32 *p;
>
> WARN_ON_ONCE(hdr.minorversion == 0);
>
> encode_cb_compound4args(xdr, &hdr);
> encode_cb_sequence4args(xdr, cb, &hdr);
>
> - /*
> - * FIXME: get stateid and fh from delegation. Inline the cna_changes
> - * buffer, and zero it.
> - */
> + p = xdr_reserve_space(xdr, 4);
> + *p = cpu_to_be32(OP_CB_NOTIFY);
> +
> + args.cna_stateid.seqid = dp->dl_stid.sc_stateid.si_generation;
> + memcpy(&args.cna_stateid.other, &dp->dl_stid.sc_stateid.si_opaque,
> + ARRAY_SIZE(args.cna_stateid.other));
> + args.cna_fh.len = dp->dl_stid.sc_file->fi_fhandle.fh_size;
> + args.cna_fh.data = dp->dl_stid.sc_file->fi_fhandle.fh_raw;
> + args.cna_changes.count = ncn->ncn_nf_cnt;
> + args.cna_changes.element = ncn->ncn_nf;
> WARN_ON_ONCE(!xdrgen_encode_CB_NOTIFY4args(xdr, &args));
>
> hdr.nops++;
I want to avoid the need to use xdrgen to encode the CB_NOTIFY arguments.
How about this:
+ struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb);
+ struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify);
...
+ encode_stateid4(xdr, &dp->dl_stid.sc_stateid);
+ encode_nfs_fh4(xdr, &dp->dl_stid.sc_file->fi_fhandle);
+ xdr_stream_encode_u32(xdr, ncn->ncn_nf_cnt);
+ for (u32 i = 0; i < ncn->ncn_nf_cnt; i++)
+ (void)xdrgen_encode_notify4(xdr, &ncn->ncn_nf[i]);
And then add a "pragma public notify4;" in nfs4_1.x .
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index b0652c755b3b..20477144475b 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -3461,19 +3462,131 @@ nfsd4_cb_getattr_release(struct nfsd4_callback *cb)
> nfs4_put_stid(&dp->dl_stid);
> }
>
> +static void nfsd_break_one_deleg(struct nfs4_delegation *dp)
> +{
> + bool queued;
> +
> + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags))
> + return;
> +
> + /*
> + * We're assuming the state code never drops its reference
> + * without first removing the lease. Since we're in this lease
> + * callback (and since the lease code is serialized by the
> + * flc_lock) we know the server hasn't removed the lease yet, and
> + * we know it's safe to take a reference.
> + */
> + refcount_inc(&dp->dl_stid.sc_count);
> + queued = nfsd4_run_cb(&dp->dl_recall);
> + WARN_ON_ONCE(!queued);
> + if (!queued)
> + refcount_dec(&dp->dl_stid.sc_count);
> +}
> +
> +static bool
> +nfsd4_cb_notify_prepare(struct nfsd4_callback *cb)
> +{
> + struct nfsd4_cb_notify *ncn = container_of(cb, struct
> nfsd4_cb_notify, ncn_cb);
> + struct nfs4_delegation *dp = container_of(ncn, struct
> nfs4_delegation, dl_cb_notify);
> + struct nfsd_notify_event *events[NOTIFY4_EVENT_QUEUE_SIZE];
> + struct xdr_buf xdr = { .buflen = PAGE_SIZE * NOTIFY4_PAGE_ARRAY_SIZE,
> + .pages = ncn->ncn_pages };
> + struct xdr_stream stream;
> + struct nfsd_file *nf;
> + int count, i;
> + bool error = false;
> +
> + xdr_init_encode_pages(&stream, &xdr);
> +
> + spin_lock(&ncn->ncn_lock);
> + count = ncn->ncn_evt_cnt;
> +
> + /* spurious queueing? */
> + if (count == 0) {
> + spin_unlock(&ncn->ncn_lock);
> + return false;
> + }
> +
> + /* we can't keep up! */
> + if (count > NOTIFY4_EVENT_QUEUE_SIZE) {
> + spin_unlock(&ncn->ncn_lock);
> + goto out_recall;
> + }
> +
> + memcpy(events, ncn->ncn_evt, sizeof(*events) * count);
> + ncn->ncn_evt_cnt = 0;
> + spin_unlock(&ncn->ncn_lock);
> +
> + rcu_read_lock();
> + nf =
> nfsd_file_get(rcu_dereference(dp->dl_stid.sc_file->fi_deleg_file));
> + rcu_read_unlock();
> + if (!nf) {
> + for (i = 0; i < count; ++i)
> + nfsd_notify_event_put(events[i]);
> + goto out_recall;
> + }
> +
> + for (i = 0; i < count; ++i) {
> + struct nfsd_notify_event *nne = events[i];
> +
> + if (!error) {
> + u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp));
> + u8 *p;
> +
> + if (!maskp) {
> + error = true;
> + goto put_event;
> + }
> +
> + p = nfsd4_encode_notify_event(&stream, nne, dp, nf, maskp);
> + if (!p) {
> + pr_notice("Could not generate CB_NOTIFY from fsnotify mask 0x%x\n",
> + nne->ne_mask);
> + error = true;
> + goto put_event;
> + }
> +
> + ncn->ncn_nf[i].notify_mask.count = 1;
> + ncn->ncn_nf[i].notify_mask.element = maskp;
> + ncn->ncn_nf[i].notify_vals.data = p;
> + ncn->ncn_nf[i].notify_vals.len = (u8 *)stream.p - p;
> + }
> +put_event:
> + nfsd_notify_event_put(nne);
> + }
> + if (!error) {
> + ncn->ncn_nf_cnt = count;
> + nfsd_file_put(nf);
> + return true;
> + }
> + nfsd_file_put(nf);
> +out_recall:
> + nfsd_break_one_deleg(dp);
> + return false;
> +}
> +
> static int
> nfsd4_cb_notify_done(struct nfsd4_callback *cb,
> struct rpc_task *task)
> {
> + struct nfsd4_cb_notify *ncn = container_of(cb, struct
> nfsd4_cb_notify, ncn_cb);
> + struct nfs4_delegation *dp = container_of(ncn, struct
> nfs4_delegation, dl_cb_notify);
> +
> switch (task->tk_status) {
> case -NFS4ERR_DELAY:
> rpc_delay(task, 2 * HZ);
> return 0;
> default:
> + /* For any other hard error, recall the deleg */
> + nfsd_break_one_deleg(dp);
> + fallthrough;
> + case 0:
> return 1;
> }
> }
>
> +static void nfsd4_run_cb_notify(struct nfsd4_cb_notify *ncn);
> +
> static void
> nfsd4_cb_notify_release(struct nfsd4_callback *cb)
> {
> @@ -3482,6 +3595,9 @@ nfsd4_cb_notify_release(struct nfsd4_callback *cb)
> struct nfs4_delegation *dp =
> container_of(ncn, struct nfs4_delegation, dl_cb_notify);
>
> + /* Drain events that arrived while this callback was in flight */
> + if (ncn->ncn_evt_cnt > 0)
> + nfsd4_run_cb_notify(ncn);
The above check needs to be serialized with modification of
ncn_evt_cnt:
+ bool pending;
+ /* Drain events that arrived while this callback was in flight */
+ spin_lock(&ncn->ncn_lock);
+ pending = ncn->ncn_evt_cnt > 0;
+ spin_unlock(&ncn->ncn_lock);
+ if (pending)
+ nfsd4_run_cb_notify(ncn);
> nfs4_put_stid(&dp->dl_stid);
> }
>
> @@ -9858,3 +9954,133 @@ void nfsd_update_cmtime_attr(struct file *f,
> unsigned int flags)
> MINOR(inode->i_sb->s_dev),
> inode->i_ino, ret);
> }
> +
> +static void
> +nfsd4_run_cb_notify(struct nfsd4_cb_notify *ncn)
> +{
> + struct nfs4_delegation *dp = container_of(ncn, struct
> nfs4_delegation, dl_cb_notify);
> +
> + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &ncn->ncn_cb.cb_flags))
> + return;
> +
> + if (!refcount_inc_not_zero(&dp->dl_stid.sc_count))
> + clear_bit(NFSD4_CALLBACK_RUNNING, &ncn->ncn_cb.cb_flags);
> + else
> + nfsd4_run_cb(&ncn->ncn_cb);
> +}
> +
> +static struct nfsd_notify_event *
> +alloc_nfsd_notify_event(u32 mask, const struct qstr *q, struct dentry
> *dentry,
> + struct inode *target)
> +{
> + struct nfsd_notify_event *ne;
> +
> + ne = kmalloc(sizeof(*ne) + q->len + 1, GFP_NOFS);
> + if (!ne)
> + return NULL;
> +
> + memcpy(&ne->ne_name, q->name, q->len);
> + refcount_set(&ne->ne_ref, 1);
> + ne->ne_mask = mask;
> + ne->ne_name[q->len] = '\0';
> + ne->ne_namelen = q->len;
> + ne->ne_dentry = dget(dentry);
> + ne->ne_target = target;
> + if (ne->ne_target)
> + ihold(ne->ne_target);
> + return ne;
> +}
> +
> +static bool
> +should_notify_deleg(u32 mask, struct file_lease *fl)
> +{
> + /* Don't notify the client generating the event */
> + if (nfsd_breaker_owns_lease(fl))
> + return false;
> +
> + /* Skip if this event wasn't ignored by the lease */
> + if ((mask & FS_DELETE) && !(fl->c.flc_flags & FL_IGN_DIR_DELETE))
> + return false;
> + if ((mask & FS_CREATE) && !(fl->c.flc_flags & FL_IGN_DIR_CREATE))
> + return false;
> + if ((mask & FS_RENAME) && !(fl->c.flc_flags & FL_IGN_DIR_RENAME))
> + return false;
> +
> + return true;
> +}
> +
> +static void
> +nfsd_recall_all_dir_delegs(const struct inode *dir)
> +{
> + struct file_lock_context *ctx = locks_inode_context(dir);
> + struct file_lock_core *flc;
> +
> + spin_lock(&ctx->flc_lock);
> + list_for_each_entry(flc, &ctx->flc_lease, flc_list) {
> + struct file_lease *fl = container_of(flc, struct file_lease, c);
> +
> + if (fl->fl_lmops == &nfsd_lease_mng_ops)
> + nfsd_break_deleg_cb(fl);
> + }
> + spin_unlock(&ctx->flc_lock);
> +}
> +
> +int
> +nfsd_handle_dir_event(u32 mask, const struct inode *dir, const void
> *data,
> + int data_type, const struct qstr *name)
> +{
> + struct dentry *dentry = fsnotify_data_dentry(data, data_type);
> + struct inode *target = fsnotify_data_rename_target(data, data_type);
> + struct file_lock_context *ctx;
> + struct file_lock_core *flc;
> + struct nfsd_notify_event *evt;
> +
> + /* Normalize cross-dir rename events to create/delete */
> + if (mask & FS_MOVED_FROM) {
> + mask &= ~FS_MOVED_FROM;
> + mask |= FS_DELETE;
> + }
> + if (mask & FS_MOVED_TO) {
> + mask &= ~FS_MOVED_TO;
> + mask |= FS_CREATE;
> + }
> +
I inserted an extra check here for rename notifications:
+ /*
+ * FS_RENAME fires on the source directory even for a cross-dir
+ * rename, where the moved entry now lives under a different
+ * parent. NOTIFY4_RENAME_ENTRY describes an in-place rename, so
+ * reporting it here would advertise a name absent from this
+ * directory.
+ */
+ if ((mask & FS_RENAME) && dentry && d_inode(dentry->d_parent) != dir)
+ mask &= ~FS_RENAME;
> + /* Don't do anything if this is not an expected event */
> + if (!(mask & (FS_CREATE|FS_DELETE|FS_RENAME)))
> + return 0;
> +
> + ctx = locks_inode_context(dir);
> + if (!ctx || list_empty(&ctx->flc_lease))
> + return 0;
> +
> + evt = alloc_nfsd_notify_event(mask, name, dentry, target);
> + if (!evt) {
> + nfsd_recall_all_dir_delegs(dir);
> + return 0;
> + }
> +
> + spin_lock(&ctx->flc_lock);
> + list_for_each_entry(flc, &ctx->flc_lease, flc_list) {
> + struct file_lease *fl = container_of(flc, struct file_lease, c);
> + struct nfs4_delegation *dp = flc->flc_owner;
> + struct nfsd4_cb_notify *ncn = &dp->dl_cb_notify;
> +
I added:
+ if (fl->fl_lmops != &nfsd_lease_mng_ops)
+ continue;
Otherwise the loop treats every lease on the inode as an nfsd delegation
unconditionally.
> + if (!should_notify_deleg(mask, fl))
> + continue;
> +
> + spin_lock(&ncn->ncn_lock);
> + if (ncn->ncn_evt_cnt >= NOTIFY4_EVENT_QUEUE_SIZE) {
> + /* We're generating notifications too fast. Recall. */
> + spin_unlock(&ncn->ncn_lock);
> + nfsd_break_deleg_cb(fl);
> + continue;
> + }
> + ncn->ncn_evt[ncn->ncn_evt_cnt++] = nfsd_notify_event_get(evt);
> + spin_unlock(&ncn->ncn_lock);
> +
> + nfsd4_run_cb_notify(ncn);
> + }
> + spin_unlock(&ctx->flc_lock);
> + nfsd_notify_event_put(evt);
> + return 0;
> +}
> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
> index e17488a911f7..31df04675713 100644
> --- a/fs/nfsd/nfs4xdr.c
> +++ b/fs/nfsd/nfs4xdr.c
> @@ -4172,6 +4172,127 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp,
> struct xdr_stream *xdr,
> goto out;
> }
>
> +static bool
> +nfsd4_setup_notify_entry4(struct notify_entry4 *ne, struct xdr_stream
> *xdr,
> + struct dentry *dentry, struct nfs4_delegation *dp,
> + struct nfsd_file *nf, char *name, u32 namelen)
> +{
> + uint32_t *attrmask;
> +
> + /* Reserve space for attrmask */
> + attrmask = xdr_reserve_space(xdr, 3 * sizeof(uint32_t));
> + if (!attrmask)
> + return false;
> +
> + ne->ne_file.data = name;
> + ne->ne_file.len = namelen;
> + ne->ne_attrs.attrmask.element = attrmask;
> +
> + attrmask[0] = 0;
> + attrmask[1] = 0;
> + attrmask[2] = 0;
> + ne->ne_attrs.attr_vals.data = NULL;
> + ne->ne_attrs.attr_vals.len = 0;
> + ne->ne_attrs.attrmask.count = 1;
> + return true;
> +}
> +
> +/**
> + * nfsd4_encode_notify_event - encode a notify
> + * @xdr: stream to which to encode the fattr4
> + * @nne: nfsd_notify_event to encode
> + * @dp: delegation where the event occurred
> + * @nf: nfsd_file on which event occurred
> + * @notify_mask: pointer to word where notification mask should be set
> + *
> + * Encode @nne into @xdr. Returns a pointer to the start of the event,
> or NULL if
> + * the event couldn't be encoded. The appropriate bit in the
> notify_mask will also
> + * be set on success.
> + */
Nit: Let's use the usual kdoc style to describe the return value.
+ * Encode @nne into @xdr. The matching bit in @notify_mask is set on
+ * success.
+ *
+ * Return: pointer to the start of the encoded event, or NULL if the
+ * event could not be encoded.
+ */
> +u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct
> nfsd_notify_event *nne,
> + struct nfs4_delegation *dp, struct nfsd_file *nf,
> + u32 *notify_mask)
> +{
> + u8 *p = NULL;
> +
> + *notify_mask = 0;
> +
> + if (nne->ne_mask & FS_DELETE) {
> + struct notify_remove4 nr = { };
> +
> + if (!nfsd4_setup_notify_entry4(&nr.nrm_old_entry, xdr,
> nne->ne_dentry, dp,
> + nf, nne->ne_name, nne->ne_namelen))
> + goto out_err;
> + p = (u8 *)xdr->p;
> + if (!xdrgen_encode_notify_remove4(xdr, &nr))
> + goto out_err;
> + *notify_mask |= BIT(NOTIFY4_REMOVE_ENTRY);
> + } else if (nne->ne_mask & FS_CREATE) {
> + struct notify_add4 na = { };
> + struct notify_remove4 old = { };
> +
> + if (!nfsd4_setup_notify_entry4(&na.nad_new_entry, xdr,
> nne->ne_dentry, dp,
> + nf, nne->ne_name, nne->ne_namelen))
> + goto out_err;
> +
> + /* If a file was overwritten, report it in nad_old_entry */
> + if (nne->ne_target) {
> + if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr,
> + NULL, dp, nf,
> + nne->ne_name, nne->ne_namelen))
> + goto out_err;
> + na.nad_old_entry.count = 1;
> + na.nad_old_entry.element = &old;
> + }
> +
> + p = (u8 *)xdr->p;
> + if (!xdrgen_encode_notify_add4(xdr, &na))
> + goto out_err;
> +
> + *notify_mask |= BIT(NOTIFY4_ADD_ENTRY);
> + } else if (nne->ne_mask & FS_RENAME) {
> + struct notify_rename4 nr = { };
> + struct notify_remove4 old = { };
> + struct name_snapshot n;
> + bool ret;
> +
> + /* Don't send any attributes in the old_entry since they're the same
> in new */
> + if (!nfsd4_setup_notify_entry4(&nr.nrn_old_entry.nrm_old_entry, xdr,
> + NULL, dp, nf, nne->ne_name,
> + nne->ne_namelen))
> + goto out_err;
> +
> + take_dentry_name_snapshot(&n, nne->ne_dentry);
> + ret = nfsd4_setup_notify_entry4(&nr.nrn_new_entry.nad_new_entry, xdr,
> + nne->ne_dentry, dp, nf, (char *)n.name.name,
> + n.name.len);
> +
> + /* If a file was overwritten, report it in nad_old_entry */
> + if (ret && nne->ne_target) {
> + ret = nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr,
> + NULL, dp, nf,
> + (char *)n.name.name, n.name.len);
> + if (ret) {
> + nr.nrn_new_entry.nad_old_entry.count = 1;
> + nr.nrn_new_entry.nad_old_entry.element = &old;
> + }
> + }
> +
> + if (ret) {
> + p = (u8 *)xdr->p;
> + ret = xdrgen_encode_notify_rename4(xdr, &nr);
> + }
> + release_dentry_name_snapshot(&n);
> + if (!ret)
> + goto out_err;
> + *notify_mask |= BIT(NOTIFY4_RENAME_ENTRY);
> + }
> + return p;
> +out_err:
> + pr_warn("nfsd: unable to marshal notify_rename4 to xdr stream\n");
Nit: The warning needs to match the semantics of nfsd4_encode_notify_event().
How about:
+ pr_warn("nfsd: unable to marshal notify event to xdr stream\n");
> + return NULL;
> +}
> +
--
Chuck Lever
^ permalink raw reply
* Re: [PATCH v5 09/21] nfsd: add data structures for handling CB_NOTIFY
From: Chuck Lever @ 2026-06-08 20:18 UTC (permalink / raw)
To: Jeff Layton, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker, Jonathan Corbet,
Shuah Khan
Cc: Steven Rostedt, Alexander Aring, Amir Goldstein, Jan Kara,
Alexander Viro, Christian Brauner, Calum Mackay, linux-kernel,
linux-doc, linux-nfs
In-Reply-To: <20260522-dir-deleg-v5-9-542cddfad576@kernel.org>
On Fri, May 22, 2026, at 3:42 PM, Jeff Layton wrote:
> Add the data structures, allocation helpers, and callback operations
> needed for directory delegation CB_NOTIFY support:
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index 9c6e2e7abc82..505fabf8f1bf 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -197,6 +197,44 @@ struct nfs4_cb_fattr {
> #define NOTIFY4_EVENT_QUEUE_SIZE 3
> #define NOTIFY4_PAGE_ARRAY_SIZE 1
>
> +struct nfsd_notify_event {
> + refcount_t ne_ref; // refcount
> + u32 ne_mask; // FS_* mask from fsnotify callback
> + struct dentry *ne_dentry; // dentry reference to target
> + u32 ne_namelen; // length of ne_name
> + char ne_name[]; // name of dentry being changed
Nit: checkpatch doesn't like the C++ comment style.
> +};
> +
> +static inline struct nfsd_notify_event *nfsd_notify_event_get(struct
> nfsd_notify_event *ne)
> +{
> + refcount_inc(&ne->ne_ref);
> + return ne;
> +}
> +
> +static inline void nfsd_notify_event_put(struct nfsd_notify_event *ne)
> +{
> + if (refcount_dec_and_test(&ne->ne_ref)) {
> + dput(ne->ne_dentry);
> + kfree(ne);
> + }
> +}
> +
> +/*
> + * Represents a directory delegation. The callback is for handling
> CB_NOTIFYs.
> + * As notifications from fsnotify come in, allocate a new event, take
> the ncn_lock,
> + * and add it to the ncn_evt queue. The CB_NOTIFY prepare handler will
> take the
> + * lock, clean out the list and process it.
> + */
> +struct nfsd4_cb_notify {
> + spinlock_t ncn_lock; // protects the evt queue and count
> + int ncn_evt_cnt; // count of events in ncn_evt
> + int ncn_nf_cnt; // count of valid entries in ncn_nf
> + struct nfsd_notify_event *ncn_evt[NOTIFY4_EVENT_QUEUE_SIZE]; // list
> of events
> + struct page *ncn_pages[NOTIFY4_PAGE_ARRAY_SIZE]; // for encoding
> + struct notify4 *ncn_nf; // array of notify4's to be sent
> + struct nfsd4_callback ncn_cb; // notify4 callback
> +};
Ditto.
--
Chuck Lever
^ permalink raw reply
* Re: [PATCH] vfs: add FS_USERNS_DELEGATABLE flag and set it for NFS
From: Jeff Layton @ 2026-06-08 20:08 UTC (permalink / raw)
To: Trond Myklebust, Anna Schumaker, Alexander Viro,
Christian Brauner, Jan Kara, Seth Forshee (DigitalOcean),
Alexander Mikhalitsyn
Cc: linux-nfs, linux-kernel, linux-fsdevel
In-Reply-To: <20260129-twmount-v1-1-4874ed2a15c4@kernel.org>
On Thu, 2026-01-29 at 16:47 -0500, Jeff Layton wrote:
> Commit e1c5ae59c0f2 ("fs: don't allow non-init s_user_ns for filesystems
> without FS_USERNS_MOUNT") prevents the mount of any filesystem inside a
> container that doesn't have FS_USERNS_MOUNT set.
>
> This broke NFS mounts in our containerized environment. We have a daemon
> somewhat like systemd-mountfsd running in the init_ns. A process does a
> fsopen() inside the container and passes it to the daemon via unix
> socket.
>
> The daemon then vets that the request is for an allowed NFS server and
> performs the mount. This now fails because the fc->user_ns is set to the
> value in the container and NFS doesn't set FS_USERNS_MOUNT. We don't
> want to add FS_USERNS_MOUNT to NFS since that would allow the container
> to mount any NFS server (even malicious ones).
>
> Add a new FS_USERNS_DELEGATABLE flag, and enable it on NFS.
>
> Fixes: e1c5ae59c0f2 ("fs: don't allow non-init s_user_ns for filesystems without FS_USERNS_MOUNT")
> Signed-off-by: Jeff Layton <jlayton@kernel.org>
> ---
> fs/nfs/fs_context.c | 8 ++++++--
> fs/super.c | 11 ++++++-----
> include/linux/fs.h | 1 +
> 3 files changed, 13 insertions(+), 7 deletions(-)
>
> diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c
> index b4679b7161b0968810e13f57c889052ea015bf56..128ebd48b4f4ba1c17e8b5b1b9dcefbd7a97db1a 100644
> --- a/fs/nfs/fs_context.c
> +++ b/fs/nfs/fs_context.c
> @@ -1768,7 +1768,9 @@ struct file_system_type nfs_fs_type = {
> .init_fs_context = nfs_init_fs_context,
> .parameters = nfs_fs_parameters,
> .kill_sb = nfs_kill_super,
> - .fs_flags = FS_RENAME_DOES_D_MOVE|FS_BINARY_MOUNTDATA,
> + .fs_flags = FS_RENAME_DOES_D_MOVE |
> + FS_BINARY_MOUNTDATA |
> + FS_USERNS_DELEGATABLE,
> };
> MODULE_ALIAS_FS("nfs");
> EXPORT_SYMBOL_GPL(nfs_fs_type);
> @@ -1780,7 +1782,9 @@ struct file_system_type nfs4_fs_type = {
> .init_fs_context = nfs_init_fs_context,
> .parameters = nfs_fs_parameters,
> .kill_sb = nfs_kill_super,
> - .fs_flags = FS_RENAME_DOES_D_MOVE|FS_BINARY_MOUNTDATA,
> + .fs_flags = FS_RENAME_DOES_D_MOVE |
> + FS_BINARY_MOUNTDATA |
> + FS_USERNS_DELEGATABLE,
> };
> MODULE_ALIAS_FS("nfs4");
> MODULE_ALIAS("nfs4");
> diff --git a/fs/super.c b/fs/super.c
> index 3d85265d14001d51524dbaec0778af8f12c048ac..b7f1bb2b679b43261fbdcd586971c551b85e8372 100644
> --- a/fs/super.c
> +++ b/fs/super.c
> @@ -738,12 +738,13 @@ struct super_block *sget_fc(struct fs_context *fc,
> int err;
>
> /*
> - * Never allow s_user_ns != &init_user_ns when FS_USERNS_MOUNT is
> - * not set, as the filesystem is likely unprepared to handle it.
> - * This can happen when fsconfig() is called from init_user_ns with
> - * an fs_fd opened in another user namespace.
> + * Never allow s_user_ns != &init_user_ns when FS_USERNS_MOUNT or
> + * FS_USERNS_DELEGATABLE is not set, as the filesystem is likely
> + * unprepared to handle it. This can happen when fsconfig() is called
> + * from init_user_ns with an fs_fd opened in another user namespace.
> */
> - if (user_ns != &init_user_ns && !(fc->fs_type->fs_flags & FS_USERNS_MOUNT)) {
> + if (user_ns != &init_user_ns &&
> + !(fc->fs_type->fs_flags & (FS_USERNS_MOUNT | FS_USERNS_DELEGATABLE))) {
> errorfc(fc, "VFS: Mounting from non-initial user namespace is not allowed");
> return ERR_PTR(-EPERM);
> }
> diff --git a/include/linux/fs.h b/include/linux/fs.h
> index a01621fa636a60764e1dfe83f2260caf50c4037e..94695ce5e25b5fbe4f321d5478172b8cb24e00d1 100644
> --- a/include/linux/fs.h
> +++ b/include/linux/fs.h
> @@ -2273,6 +2273,7 @@ struct file_system_type {
> #define FS_MGTIME 64 /* FS uses multigrain timestamps */
> #define FS_LBS 128 /* FS supports LBS */
> #define FS_POWER_FREEZE 256 /* Always freeze on suspend/hibernate */
> +#define FS_USERNS_DELEGATABLE 512 /* Can be mounted inside userns from outside */
> #define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */
> int (*init_fs_context)(struct fs_context *);
> const struct fs_parameter_spec *parameters;
>
> ---
> base-commit: 8dfce8991b95d8625d0a1d2896e42f93b9d7f68d
> change-id: 20260129-twmount-114ddfd43420
>
> Best regards,
Ping?
I just realized that this patch never made it into -next or a release.
Any chance we can get this into the coming v7.3 merge window? (although
it looks like we'll need to renumber the bit since
FS_USERNS_MOUNT_RESTRICTED has gone in since then.
Thanks,
--
Jeff Layton <jlayton@kernel.org>
^ permalink raw reply
* Re: [PATCH v5 08/21] nfsd: use RCU to protect fi_deleg_file
From: Chuck Lever @ 2026-06-08 17:00 UTC (permalink / raw)
To: Jeff Layton, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker, Jonathan Corbet,
Shuah Khan
Cc: Steven Rostedt, Alexander Aring, Amir Goldstein, Jan Kara,
Alexander Viro, Christian Brauner, Calum Mackay, linux-kernel,
linux-doc, linux-nfs
In-Reply-To: <20260522-dir-deleg-v5-8-542cddfad576@kernel.org>
On Fri, May 22, 2026, at 3:42 PM, Jeff Layton wrote:
> fi_deleg_file can be NULLed by put_deleg_file() when fi_delegees drops
> to zero during delegation teardown (e.g. DELEGRETURN). Concurrent
> accesses from workqueue callbacks -- such as CB_NOTIFY -- can
> dereference a NULL pointer if they race with this teardown.
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 3efc53f0dde6..bd0517dfe881 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1212,7 +1212,9 @@ static void put_deleg_file(struct nfs4_file *fp)
>
> spin_lock(&fp->fi_lock);
> if (--fp->fi_delegees == 0) {
> - swap(nf, fp->fi_deleg_file);
> + nf = rcu_dereference_protected(fp->fi_deleg_file,
> + lockdep_is_held(&fp->fi_lock));
> + rcu_assign_pointer(fp->fi_deleg_file, NULL);
Nit: For consistency, the above could be
RCU_INIT_POINTER(fp->fi_deleg_file, NULL);
> swap(rnf, fp->fi_rdeleg_file);
> }
> spin_unlock(&fp->fi_lock);
> @@ -9722,7 +9729,7 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state
> *cstate,
> }
>
> /* Something failed. Drop the lease and clean up the stid */
> - kernel_setlease(fp->fi_deleg_file->nf_file, F_UNLCK, NULL, (void **)&dp);
> + kernel_setlease(nf->nf_file, F_UNLCK, NULL, (void **)&dp);
> out_put_stid:
> nfs4_put_stid(&dp->dl_stid);
> out_delegees:
We might want:
kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file,
F_UNLCK, NULL, (void **)&dp);
instead, to avoid the Sashiko-reported UAF finding.
--
Chuck Lever
^ permalink raw reply
* Re: [PATCH v5 07/21] nfsd: add callback encoding and decoding linkages for CB_NOTIFY
From: Chuck Lever @ 2026-06-08 16:52 UTC (permalink / raw)
To: Jeff Layton, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker, Jonathan Corbet,
Shuah Khan
Cc: Steven Rostedt, Alexander Aring, Amir Goldstein, Jan Kara,
Alexander Viro, Christian Brauner, Calum Mackay, linux-kernel,
linux-doc, linux-nfs
In-Reply-To: <20260522-dir-deleg-v5-7-542cddfad576@kernel.org>
On Fri, May 22, 2026, at 3:42 PM, Jeff Layton wrote:
> Add routines for encoding and decoding CB_NOTIFY messages. These call
> into the code generated by xdrgen to do the actual encoding and
> decoding.
The commit message needs to explain that the encoder is not yet functional.
Something like: "The encoder is a stub; payload encoding (stateid, fh, and
cna_changes) is deferred."
> diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
> index 25bbf5b8814d..ea3e7deb06fa 100644
> --- a/fs/nfsd/nfs4callback.c
> +++ b/fs/nfsd/nfs4callback.c
> @@ -865,6 +865,51 @@ static void encode_stateowner(struct xdr_stream
> *xdr, struct nfs4_stateowner *so
> xdr_encode_opaque(p, so->so_owner.data, so->so_owner.len);
> }
>
> +static void nfs4_xdr_enc_cb_notify(struct rpc_rqst *req,
> + struct xdr_stream *xdr,
> + const void *data)
> +{
> + const struct nfsd4_callback *cb = data;
> + struct nfs4_cb_compound_hdr hdr = {
> + .ident = 0,
> + .minorversion = cb->cb_clp->cl_minorversion,
> + };
> + struct CB_NOTIFY4args args = { };
> +
> + WARN_ON_ONCE(hdr.minorversion == 0);
> +
> + encode_cb_compound4args(xdr, &hdr);
> + encode_cb_sequence4args(xdr, cb, &hdr);
> +
> + /*
> + * FIXME: get stateid and fh from delegation. Inline the cna_changes
> + * buffer, and zero it.
> + */
> + WARN_ON_ONCE(!xdrgen_encode_CB_NOTIFY4args(xdr, &args));
> +
> + hdr.nops++;
> + encode_cb_nops(&hdr);
> +}
There are a number of problems with this, but since there are no
callers yet, we can let some of those issues stand.
What is problematic in the longer-term is that this is a client-side
encoder (since this is the server's NFSv4 callback client).
xdrgen_encode_CB_NOTIFY4args() is an argument encoder, which is
client-side functionality, but it resides in fs/nfsd/nfs4xdr_gen.c,
which is server-side. Let's not mix these purposes.
I replaced the comment and WARN_ON with this:
+ xdr_stream_encode_u32(xdr, OP_CB_NOTIFY);
+
+ /* FIXME: encode stateid, fh, and cna_changes from delegation */
You can use xdrgen functions for individual data items, but for
full argument and response structures, only server-side is supported
at the moment. In the later patch that completes this code, I'll cover
the other fields, which can be a mix of open code and xdrgen.
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index e1c40f8b5d01..790282781243 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -190,6 +190,13 @@ struct nfs4_cb_fattr {
> u64 ncf_cur_fsize;
> };
>
> +/*
> + * FIXME: the current backchannel encoder can't handle a send buffer longer
> + * than a single page (see bc_alloc/bc_free).
> + */
Nit: The allocator function name is bc_malloc
--
Chuck Lever
^ permalink raw reply
* Re: [PATCH v5 05/21] nfsd: update the fsnotify mark when setting or removing a dir delegation
From: Chuck Lever @ 2026-06-08 16:38 UTC (permalink / raw)
To: Jeff Layton, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker, Jonathan Corbet,
Shuah Khan
Cc: Steven Rostedt, Alexander Aring, Amir Goldstein, Jan Kara,
Alexander Viro, Christian Brauner, Calum Mackay, linux-kernel,
linux-doc, linux-nfs
In-Reply-To: <20260522-dir-deleg-v5-5-542cddfad576@kernel.org>
On Fri, May 22, 2026, at 3:42 PM, Jeff Layton wrote:
> Add a new helper function that will update the mask on the nfsd_file's
> fsnotify_mark to be a union of all current directory delegations on an
> inode. Call that when directory delegations are added or removed.
This commit message repeats what the diff below says. Can it instead
explain why this change is necessary?
> Reviewed-by: Jan Kara <jack@suse.cz>
> Signed-off-by: Jeff Layton <jlayton@kernel.org>
> ---
> fs/nfsd/nfs4state.c | 34 ++++++++++++++++++++++++++++++++++
> 1 file changed, 34 insertions(+)
>
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 2a34ba457b74..efbc99f0a965 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1246,6 +1246,38 @@ static void
> nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct f
> nfsd_update_cmtime_attr(f, ATTR_ATIME);
> }
>
> +static void nfsd_fsnotify_recalc_mask(struct nfsd_file *nf)
Since nfsd_fsnotify_recalc_mask() takes a single struct nfsd_file
as an argument, should this function reside in fs/nfsd/filecache.c
instead? The question might reflect my misunderstanding of the
new function's purpose.
> +{
> + struct inode *inode = file_inode(nf->nf_file);
> + u32 lease_mask, set = 0, clear = 0;
> + struct fsnotify_mark *mark;
> +
> + /* This is only needed when adding or removing dir delegs */
> + if (!S_ISDIR(inode->i_mode) || !nf->nf_mark)
> + return;
> +
> + /* Set up notifications for any ignored delegation events */
> + lease_mask = inode_lease_ignore_mask(inode);
> + mark = &nf->nf_mark->nfm_mark;
> +
> + if (lease_mask & FL_IGN_DIR_CREATE)
> + set |= FS_CREATE | FS_MOVED_TO;
> + else
> + clear |= FS_CREATE | FS_MOVED_TO;
> +
> + if (lease_mask & FL_IGN_DIR_DELETE)
> + set |= FS_DELETE | FS_MOVED_FROM;
> + else
> + clear |= FS_DELETE | FS_MOVED_FROM;
> +
> + if (lease_mask & FL_IGN_DIR_RENAME)
> + set |= FS_RENAME;
> + else
> + clear |= FS_RENAME;
> +
> + fsnotify_modify_mark_mask(mark, set, clear);
> +}
> +
> static void nfs4_unlock_deleg_lease(struct nfs4_delegation *dp)
> {
> struct nfs4_file *fp = dp->dl_stid.sc_file;
> @@ -1255,6 +1287,7 @@ static void nfs4_unlock_deleg_lease(struct
> nfs4_delegation *dp)
>
> nfsd4_finalize_deleg_timestamps(dp, nf->nf_file);
> kernel_setlease(nf->nf_file, F_UNLCK, NULL, (void **)&dp);
> + nfsd_fsnotify_recalc_mask(nf);
> put_deleg_file(fp);
> }
>
I added the following edit to this patch>
@@ -9597,8 +9629,7 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry,
* @nf: nfsd_file opened on the directory
*
* Given a GET_DIR_DELEGATION request @gdd, attempt to acquire a delegation
- * on the directory to which @nf refers. Note that this does not set up any
- * sort of async notifications for the delegation.
+ * on the directory to which @nf refers.
*/
struct nfs4_delegation *
nfsd_get_dir_deleg
The patch makes the above kerneldoc note ("does not set up any sort of async
notifications") logically obsolete.
> @@ -9682,6 +9715,7 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate,
>
> if (!status) {
> put_nfs4_file(fp);
> + nfsd_fsnotify_recalc_mask(nf);
> return dp;
> }
>
>
> --
> 2.54.0
--
Chuck Lever
^ permalink raw reply
* Re: [PATCH v4 03/21] nfs_common: add new NOTIFY4_* flags proposed in RFC8881bis
From: Jeff Layton @ 2026-06-08 16:37 UTC (permalink / raw)
To: Chuck Lever, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker
Cc: Alexander Aring, Amir Goldstein, Jan Kara, Alexander Viro,
Christian Brauner, Calum Mackay, linux-kernel, linux-doc,
linux-nfs
In-Reply-To: <bf852bf8-2d80-4b12-8d7c-bb0770c8b379@app.fastmail.com>
On Mon, 2026-06-08 at 12:23 -0400, Chuck Lever wrote:
> Unfortunately, reviewing this series slipped off my radar. I'm just now
> getting to it. I'm going to do this in order and some of the earlier
> reviews are cosmetic. But there are some significant changes needed
> later on, so I expect we'll need to redrive this series and punt it to
> v7.3.
>
That's reasonable. We had some unrelated new bugs crop up near the end
of the cycle that made it hard to get this in and tested properly. I'd
like to propose that we keep the VFS bits
> One thing you might update before you repost is to remove the "Acked-by:
> Chuck Lever" throughout the series.
>
Ok. I think b4 added that.
>
> Nit: the Subject: prefix should be "nfsd:" not "nfs_common:"
>
include/linux/sunrpc/xdrgen/nfs4_1.h is a shared header. Anna will
presumably need this patch as well.
>
> On Fri, May 22, 2026, at 8:28 AM, Jeff Layton wrote:
> > RFC8881bis adds some new flags to GET_DIR_DELEGATION that we very much
> > need to support.
>
> Nit: Let's say: "RFC8881bis adds new flags to GET_DIR_DELEGATION
> that later patches consume."
>
> One recent private comment to me was a question about whether we can
> trust the stability of the specification, which is still a WG document
> and has been for years. This patch's commit message should address
> that.
>
That's definitely an open question. We are definitely adopting these
changes before the ink is dry. We might have to modify things later,
especially once there is more client-side experience.
I'm open to verbiage suggestions here.
>
> > diff --git a/Documentation/sunrpc/xdr/nfs4_1.x
> > b/Documentation/sunrpc/xdr/nfs4_1.x
> > index 632f5b579c39..aa14b590b524 100644
> > --- a/Documentation/sunrpc/xdr/nfs4_1.x
> > +++ b/Documentation/sunrpc/xdr/nfs4_1.x
> > @@ -416,7 +416,21 @@ enum notify_type4 {
> > NOTIFY4_REMOVE_ENTRY = 2,
> > NOTIFY4_ADD_ENTRY = 3,
> > NOTIFY4_RENAME_ENTRY = 4,
> > - NOTIFY4_CHANGE_COOKIE_VERIFIER = 5
> > + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5,
> > + /*
> > + * Added in NFSv4.1 bis document
> > + */
>
> I clarified this comment: "/* Proposed in RFC8881bis */"
>
> I rebuilt the generated source to confirm that it hasn't been altered
> by recent changes to xdrgen or this comment change. The only thing
> that changed in the output was the header's timestamp.
>
Sounds good.
--
Jeff Layton <jlayton@kernel.org>
^ permalink raw reply
* Re: [PATCH v5 04/21] nfsd: allow nfsd to get a dir lease with an ignore mask
From: Chuck Lever @ 2026-06-08 16:29 UTC (permalink / raw)
To: Jeff Layton, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker, Jonathan Corbet,
Shuah Khan
Cc: Steven Rostedt, Alexander Aring, Amir Goldstein, Jan Kara,
Alexander Viro, Christian Brauner, Calum Mackay, linux-kernel,
linux-doc, linux-nfs
In-Reply-To: <20260522-dir-deleg-v5-4-542cddfad576@kernel.org>
On Fri, May 22, 2026, at 3:42 PM, Jeff Layton wrote:
> When requesting a directory lease, enable the FL_IGN_DIR_* bits that
> correspond to the requested notification types.
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 67e163ee13a2..2a34ba457b74 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -9642,12 +9657,11 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate,
> dp->dl_stid.sc_export =
> exp_get(cstate->current_fh.fh_export);
>
> - fl = nfs4_alloc_init_lease(dp);
> + fl = nfs4_alloc_init_lease(dp, gdd->gddr_notification[0]);
Both Sashiko and my own gpt-5.5 review noted the use of a response field
here rather than a request field, but that is a false positive finding.
The commit message needs to explain why the use of gddr_notification[0]
is correct.
> if (!fl)
> goto out_put_stid;
>
> - status = kernel_setlease(nf->nf_file,
> - fl->c.flc_type, &fl, NULL);
> + status = kernel_setlease(nf->nf_file, fl->c.flc_type, &fl, NULL);
> if (fl)
> locks_free_lease(fl);
> if (status)
This last edit appears to be a clean-up that is unrelated to the purpose
of the patch.
--
Chuck Lever
^ permalink raw reply
* Re: [PATCH v4 03/21] nfs_common: add new NOTIFY4_* flags proposed in RFC8881bis
From: Chuck Lever @ 2026-06-08 16:23 UTC (permalink / raw)
To: Jeff Layton, Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo,
Tom Talpey, Trond Myklebust, Anna Schumaker
Cc: Alexander Aring, Amir Goldstein, Jan Kara, Alexander Viro,
Christian Brauner, Calum Mackay, linux-kernel, linux-doc,
linux-nfs
In-Reply-To: <20260522-dir-deleg-v4-3-2acb883ac6bc@kernel.org>
Unfortunately, reviewing this series slipped off my radar. I'm just now
getting to it. I'm going to do this in order and some of the earlier
reviews are cosmetic. But there are some significant changes needed
later on, so I expect we'll need to redrive this series and punt it to
v7.3.
One thing you might update before you repost is to remove the "Acked-by:
Chuck Lever" throughout the series.
Nit: the Subject: prefix should be "nfsd:" not "nfs_common:"
On Fri, May 22, 2026, at 8:28 AM, Jeff Layton wrote:
> RFC8881bis adds some new flags to GET_DIR_DELEGATION that we very much
> need to support.
Nit: Let's say: "RFC8881bis adds new flags to GET_DIR_DELEGATION
that later patches consume."
One recent private comment to me was a question about whether we can
trust the stability of the specification, which is still a WG document
and has been for years. This patch's commit message should address
that.
> diff --git a/Documentation/sunrpc/xdr/nfs4_1.x
> b/Documentation/sunrpc/xdr/nfs4_1.x
> index 632f5b579c39..aa14b590b524 100644
> --- a/Documentation/sunrpc/xdr/nfs4_1.x
> +++ b/Documentation/sunrpc/xdr/nfs4_1.x
> @@ -416,7 +416,21 @@ enum notify_type4 {
> NOTIFY4_REMOVE_ENTRY = 2,
> NOTIFY4_ADD_ENTRY = 3,
> NOTIFY4_RENAME_ENTRY = 4,
> - NOTIFY4_CHANGE_COOKIE_VERIFIER = 5
> + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5,
> + /*
> + * Added in NFSv4.1 bis document
> + */
I clarified this comment: "/* Proposed in RFC8881bis */"
I rebuilt the generated source to confirm that it hasn't been altered
by recent changes to xdrgen or this comment change. The only thing
that changed in the output was the header's timestamp.
--
Chuck Lever
^ permalink raw reply
* Re: [PATCH] SUNRPC: check rpc_sockaddr2uaddr() return value in rpcb_register_inet4/6
From: Jeff Layton @ 2026-06-08 14:57 UTC (permalink / raw)
To: Weiming Shi, linux-nfs, Trond Myklebust, Anna Schumaker
Cc: Chuck Lever, NeilBrown, Olga Kornievskaia, Dai Ngo, Tom Talpey,
netdev, linux-kernel, Xiang Mei
In-Reply-To: <20260607140645.2198574-2-bestswngs@gmail.com>
On Sun, 2026-06-07 at 07:06 -0700, Weiming Shi wrote:
> rpcb_register_inet4() and rpcb_register_inet6() store the result of
> rpc_sockaddr2uaddr() into map->r_addr without checking it for NULL.
> rpc_sockaddr2uaddr() returns NULL when its final kstrdup() fails, and
> the unchecked NULL is then carried into the synchronous RPCBPROC_SET
> encode path: rpcb_register_call() -> rpc_call_sync() ->
> rpcb_enc_getaddr() -> encode_rpcb_string(), whose first statement is
> strlen(string), dereferencing NULL and oopsing the kernel.
>
> The crash reproduces under failslab on v6.12; with KASAN the NULL
> dereference surfaces as a fault on the shadow of address zero:
>
> Oops: general protection fault, probably for non-canonical address
> 0xdffffc0000000000 [#1] PREEMPT SMP KASAN
> RIP: 0010:strlen (lib/string.c:409)
> Call Trace:
> encode_rpcb_string (net/sunrpc/rpcb_clnt.c:890)
> rpcb_enc_getaddr (net/sunrpc/rpcb_clnt.c:910)
> rpcauth_wrap_req_encode (net/sunrpc/auth.c:745)
> call_encode (net/sunrpc/clnt.c:1966)
> __rpc_execute (net/sunrpc/sched.c:952)
> rpc_run_task (net/sunrpc/clnt.c:1243)
> rpc_call_sync (net/sunrpc/clnt.c:1272)
> rpcb_v4_register (net/sunrpc/rpcb_clnt.c:500)
> svc_generic_rpcbind_set
> nfsd_rpcbind_set
> svc_register
> svc_setup_socket
> svc_addsock
> write_ports
> nfsctl_transaction_write
> vfs_write
>
> The crash is reachable when an in-kernel RPC service (nfsd, lockd,
> nfs-callback) registers with the local rpcbind under enough memory
> pressure for the small GFP_KERNEL kstrdup() in rpc_sockaddr2uaddr() to
> fail. The asynchronous getport path already handles this exact failure
> mode by returning -ENOMEM; only the two register helpers omit the check.
>
> Mirror that handling: bail out with -ENOMEM when rpc_sockaddr2uaddr()
> returns NULL, before the address is fed into the encoder.
>
> Fixes: d77385f23830 ("SUNRPC: Fix rpc_sockaddr2uaddr")
> Reported-by: Xiang Mei <xmei5@asu.edu>
> Assisted-by: Claude:claude-opus-4-8
> Signed-off-by: Weiming Shi <bestswngs@gmail.com>
> ---
> net/sunrpc/rpcb_clnt.c | 4 ++++
> 1 file changed, 4 insertions(+)
>
> diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c
> index 6aa372188c86..4c0b7fefee4e 100644
> --- a/net/sunrpc/rpcb_clnt.c
> +++ b/net/sunrpc/rpcb_clnt.c
> @@ -490,6 +490,8 @@ static int rpcb_register_inet4(struct sunrpc_net *sn,
> int result;
>
> map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL);
> + if (!map->r_addr)
> + return -ENOMEM;
>
> msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET];
> if (port != 0) {
> @@ -516,6 +518,8 @@ static int rpcb_register_inet6(struct sunrpc_net *sn,
> int result;
>
> map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL);
> + if (!map->r_addr)
> + return -ENOMEM;
>
> msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET];
> if (port != 0) {
Reviewed-by: Jeff Layton <jlayton@kernel.org>
^ permalink raw reply
* [PATCH v3 22/22] netfs: Combine prepare and issue ops and grab the buffers on request
From: David Howells @ 2026-06-08 14:54 UTC (permalink / raw)
To: Christian Brauner, Matthew Wilcox, Christoph Hellwig
Cc: David Howells, Paulo Alcantara, Jens Axboe, Leon Romanovsky,
Steve French, ChenXiaoSong, Marc Dionne, Eric Van Hensbergen,
Dominique Martinet, Ilya Dryomov, Trond Myklebust, netfs,
linux-afs, linux-cifs, linux-nfs, ceph-devel, v9fs, linux-erofs,
linux-fsdevel, linux-kernel
In-Reply-To: <20260608145432.681865-1-dhowells@redhat.com>
Modify the way subrequests are generated in netfslib to try and simplify
the code. The issue, primarily, is in writeback: the code has to create
multiple streams of write requests to disparate targets with different
properties (e.g. server and fscache), where not every folio needs to go to
every target (e.g. data just read from the server may only need writing to
the cache).
The current model in writeback, at least, is to go carefully through every
folio, preparing a subrequest for each stream when it was detected that
part of the current folio needed to go to that stream, and repeating this
within and across contiguous folios; then to issue subrequests as they
become full or hit boundaries after first setting up the buffer. However,
this is quite difficult to follow - and makes it tricky to handle
discontiguous folios in a request.
This is changed such that netfs now accumulates buffers and attaches them
to each stream when they become valid for that stream, then flushes the
stream when a limit or a boundary is hit. The issuing code in netfs then
loops around creating and issuing subrequests without calling a separate
prepare stage (though a function is provided to get an estimate of when
flushing should occur). The filesystem (or cache) then gets to take a
slice of the master bvec chain as its I/O buffer for each subrequest,
including discontiguities if it can support a sparse/vectored RPC (as Ceph
can).
Similar-ish changes also apply to buffered read and unbuffered read and
write, though in each of those cases there is only a single contiguous
stream. Though for buffered read this consists of interwoven requests from
multiple sources (server or cache).
To this end, netfslib is changed in the following ways:
(1) ->prepare_xxx(), buffer selection and ->issue_xxx() are now collapsed
together such that one ->issue_xxx() call is made with the subrequest
defined to the maximum extent; the filesystem/cache then reduces the
length of the subrequest and calls back to netfslib to grab a slice of
the buffer, which may reduce the subrequest further if a maximum
segment limit is set. The filesystem/cache then dispatches the
operation.
(2) Retry buffer tracking is added to the netfs_io_request struct. This
is then selected by the subrequest retry counter being non-zero.
(3) The use of iov_iter is pushed down to the filesystem. Netfslib now
provides the filesystem with a bvecq holding the buffer rather than an
iov_iter. The bvecq can be duplicated and headers/trailers attached
to hold protocol and several bvecqs can be linked together to create a
compound operation.
(4) If the ->issue_xxx() functions terminate with -ENOMEM, a flag is set
on the request to abort further subrequest generation/retrying.
(5) During writeback, netfslib now builds up an accumulation of buffered
data before issuing writes on each stream (one server, one cache). It
asks each stream for an estimate of how much data to accumulate before
it next generates subrequests on the stream. The filesystem or cache
is not required to use up all the data accumulated on a stream at that
time unless the end of the pagecache is hit.
(6) During read-gaps, in which there are two gaps on either end of a dirty
streaming write page that need to be filled, a buffer is constructed
consisting of the two ends plus a sink page repeated to cover the
middle portion. This is passed to the server as a single write. For
something like Ceph, this should probably be done either as a
vectored/sparse read or as two separate reads (if different Ceph
objects are involved).
(7) During unbuffered/DIO read/write, there is a single contiguous file
region to be read or written as a single stream. The dispatching
function just creates subrequests and calls ->issue_xxx() repeatedly
to eat through the bufferage.
(8) At the start of buffered read, the entire set of folios allocated by
VM readahead is loaded into a bvecq chain, rather than trying to do it
piecemeal as-needed. As the pages were already added and locked by
the VM, this is slightly more efficient than loading piecemeal as only
a single iteration of the xarray is required.
(9) During buffered read, there is a single contiguous file region, to
read as a single stream - however, this stream may be stitched
together from subrequests to multiple sources. Which sources are used
where is now determined by querying the cache to find the next couple
of extents in which it has data; netfslib uses this to direct the
subrequests towards the appropriate sources.
Each subrequest is given the maximum length in the current extent and
then ->issue_read() is called. The filesystem then limits the size
and slices off a piece of the buffer for that extent.
(10) Cachefiles now provides an estimation function that indicates the
standard maxima for doing DIO (MAX_RW_COUNT and BIO_MAX_VECS).
Note that sparse cachefiles still rely on the backing filesystem for
content mapping. That will need to be addressed in a future patch and is
not trivial to fix.
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Paulo Alcantara <pc@manguebit.org>
cc: Matthew Wilcox <willy@infradead.org>
cc: Christoph Hellwig <hch@infradead.org>
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
---
fs/9p/vfs_addr.c | 49 +-
fs/afs/dir.c | 11 +-
fs/afs/file.c | 27 +-
fs/afs/fsclient.c | 8 +-
fs/afs/internal.h | 6 +-
fs/afs/symlink.c | 5 +-
fs/afs/write.c | 32 +-
fs/afs/yfsclient.c | 6 +-
fs/cachefiles/io.c | 255 ++++++----
fs/ceph/Kconfig | 1 +
fs/ceph/addr.c | 119 ++---
fs/netfs/Kconfig | 3 +
fs/netfs/Makefile | 2 +-
fs/netfs/buffered_read.c | 237 +++++-----
fs/netfs/buffered_write.c | 27 +-
fs/netfs/direct_read.c | 78 +--
fs/netfs/direct_write.c | 127 +++--
fs/netfs/fscache_io.c | 6 -
fs/netfs/internal.h | 104 +++-
fs/netfs/iterator.c | 4 +-
fs/netfs/misc.c | 35 +-
fs/netfs/objects.c | 7 +-
fs/netfs/read_collect.c | 43 +-
fs/netfs/read_pgpriv2.c | 113 +++--
fs/netfs/read_retry.c | 199 ++++----
fs/netfs/read_single.c | 150 +++---
fs/netfs/write_collect.c | 58 ++-
fs/netfs/write_issue.c | 887 +++++++++++++++++++++--------------
fs/netfs/write_retry.c | 136 +++---
fs/nfs/Kconfig | 1 +
fs/nfs/fscache.c | 23 +-
fs/smb/client/cifssmb.c | 13 +-
fs/smb/client/file.c | 137 +++---
fs/smb/client/smb2ops.c | 9 +-
fs/smb/client/smb2pdu.c | 28 +-
fs/smb/client/transport.c | 15 +-
include/linux/netfs.h | 90 ++--
include/trace/events/netfs.h | 51 +-
net/9p/client.c | 8 +-
39 files changed, 1840 insertions(+), 1270 deletions(-)
diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c
index c21d33830f5f..e2f67853d74d 100644
--- a/fs/9p/vfs_addr.c
+++ b/fs/9p/vfs_addr.c
@@ -48,32 +48,71 @@ static void v9fs_begin_writeback(struct netfs_io_request *wreq)
wreq->io_streams[0].avail = true;
}
+/*
+ * Estimate how much data should be accumulated before we start issuing
+ * write subrequests.
+ */
+static int v9fs_estimate_write(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_write_estimate *estimate)
+{
+ struct p9_fid *fid = wreq->netfs_priv;
+ unsigned long long limit = ULLONG_MAX - stream->issue_from;
+ unsigned long long max_len = fid->clnt->msize - P9_IOHDRSZ;
+
+ estimate->issue_at = stream->issue_from + umin(max_len, limit);
+ return 0;
+}
+
/*
* Issue a subrequest to write to the server.
*/
static void v9fs_issue_write(struct netfs_io_subrequest *subreq)
{
+ struct iov_iter iter;
struct p9_fid *fid = subreq->rreq->netfs_priv;
int err, len;
- len = p9_client_write(fid, subreq->start, &subreq->io_iter, &err);
+ subreq->len = umin(subreq->len, fid->clnt->msize - P9_IOHDRSZ);
+
+ err = netfs_prepare_write_buffer(subreq, INT_MAX);
+ if (err < 0)
+ goto term;
+
+ iov_iter_bvec_queue(&iter, ITER_SOURCE, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset, subreq->len);
+
+ len = p9_client_write(fid, subreq->start, &iter, &err);
if (len > 0)
__set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
- netfs_write_subrequest_terminated(subreq, len ?: err);
+
+term:
+ return netfs_write_subrequest_terminated(subreq, len ?: err);
}
/**
* v9fs_issue_read - Issue a read from 9P
* @subreq: The read to make
+ * @rctx: Read generation context
*/
static void v9fs_issue_read(struct netfs_io_subrequest *subreq)
{
struct netfs_io_request *rreq = subreq->rreq;
+ struct iov_iter iter;
struct p9_fid *fid = rreq->netfs_priv;
unsigned long long pos = subreq->start + subreq->transferred;
int total, err;
- total = p9_client_read(fid, pos, &subreq->io_iter, &err);
+ err = netfs_prepare_read_buffer(subreq, INT_MAX);
+ if (err < 0)
+ goto term;
+
+ iov_iter_bvec_queue(&iter, ITER_DEST, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset, subreq->len);
+
+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
+
+ total = p9_client_read(fid, pos, &iter, &err);
/* if we just extended the file size, any portion not in
* cache won't be on server and is zeroes */
@@ -87,8 +126,9 @@ static void v9fs_issue_read(struct netfs_io_subrequest *subreq)
__set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
}
+term:
subreq->error = err;
- netfs_read_subreq_terminated(subreq);
+ return netfs_read_subreq_terminated(subreq);
}
/**
@@ -154,6 +194,7 @@ const struct netfs_request_ops v9fs_req_ops = {
.free_request = v9fs_free_request,
.issue_read = v9fs_issue_read,
.begin_writeback = v9fs_begin_writeback,
+ .estimate_write = v9fs_estimate_write,
.issue_write = v9fs_issue_write,
};
diff --git a/fs/afs/dir.c b/fs/afs/dir.c
index 774d86bf878e..4a2e4c10ba21 100644
--- a/fs/afs/dir.c
+++ b/fs/afs/dir.c
@@ -248,8 +248,8 @@ static ssize_t afs_do_read_single(struct afs_vnode *dvnode, struct file *file)
if (dvnode->directory_size < i_size) {
size_t cur_size = dvnode->directory_size;
- ret = bvecq_expand_buffer(&dvnode->directory, &cur_size, i_size,
- GFP_KERNEL);
+ ret = bvecq_expand_buffer(&dvnode->directory, &cur_size,
+ round_up(i_size, PAGE_SIZE), GFP_KERNEL);
dvnode->directory_size = cur_size;
if (ret < 0)
return ret;
@@ -2217,11 +2217,10 @@ static int afs_dir_writepages(struct address_space *mapping,
}
if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) {
+ size_t len = i_size_read(&dvnode->netfs.inode);
iov_iter_bvec_queue(&iter, ITER_SOURCE, dvnode->directory, 0, 0,
- i_size_read(&dvnode->netfs.inode));
- ret = netfs_writeback_single(mapping, wbc, &iter);
- if (ret == 1)
- ret = 0; /* Skipped write due to lock conflict. */
+ round_up(len, PAGE_SIZE));
+ ret = netfs_writeback_single(mapping, wbc, &iter, len);
}
up_read(&dvnode->validate_lock);
diff --git a/fs/afs/file.c b/fs/afs/file.c
index 67f38e99ada7..d2e75f044a7a 100644
--- a/fs/afs/file.c
+++ b/fs/afs/file.c
@@ -337,6 +337,7 @@ static void afs_issue_read(struct netfs_io_subrequest *subreq)
struct afs_operation *op;
struct afs_vnode *vnode = AFS_FS_I(subreq->rreq->inode);
struct key *key = subreq->rreq->netfs_priv;
+ int ret;
_enter("%s{%llx:%llu.%u},%x,,,",
vnode->volume->name,
@@ -345,11 +346,14 @@ static void afs_issue_read(struct netfs_io_subrequest *subreq)
vnode->fid.unique,
key_serial(key));
+ ret = netfs_prepare_read_buffer(subreq, INT_MAX);
+ if (ret < 0)
+ goto failed;
+
op = afs_alloc_operation(key, vnode->volume);
if (IS_ERR(op)) {
- subreq->error = PTR_ERR(op);
- netfs_read_subreq_terminated(subreq);
- return;
+ ret = PTR_ERR(op);
+ goto failed;
}
afs_op_set_vnode(op, 0, vnode);
@@ -364,20 +368,21 @@ static void afs_issue_read(struct netfs_io_subrequest *subreq)
op->flags |= AFS_OPERATION_ASYNC;
if (!afs_begin_vnode_operation(op)) {
- subreq->error = afs_put_operation(op);
- netfs_read_subreq_terminated(subreq);
- return;
+ ret = afs_put_operation(op);
+ goto failed;
}
- if (!afs_select_fileserver(op)) {
- afs_end_read(op);
- return;
- }
+ if (!afs_select_fileserver(op))
+ afs_end_read(op); /* Error recorded here. */
afs_issue_read_call(op);
} else {
afs_do_sync_operation(op);
}
+ return;
+failed:
+ subreq->error = ret;
+ return netfs_read_subreq_terminated(subreq);
}
static int afs_init_request(struct netfs_io_request *rreq, struct file *file)
@@ -470,7 +475,7 @@ const struct netfs_request_ops afs_req_ops = {
.update_i_size = afs_update_i_size,
.invalidate_cache = afs_netfs_invalidate_cache,
.begin_writeback = afs_begin_writeback,
- .prepare_write = afs_prepare_write,
+ .estimate_write = afs_estimate_write,
.issue_write = afs_issue_write,
.retry_request = afs_retry_request,
};
diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c
index a2ffd60889f8..c332b733d7a7 100644
--- a/fs/afs/fsclient.c
+++ b/fs/afs/fsclient.c
@@ -339,7 +339,9 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call)
if (call->remaining == 0)
goto no_more_data;
- call->iter = &subreq->io_iter;
+ iov_iter_bvec_queue(&call->def_iter, ITER_DEST, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset, subreq->len);
+
call->iov_len = umin(call->remaining, subreq->len - subreq->transferred);
call->unmarshall++;
fallthrough;
@@ -1085,7 +1087,7 @@ static void afs_fs_store_data64(struct afs_operation *op)
if (!call)
return afs_op_nomem(op);
- call->write_iter = op->store.write_iter;
+ call->write_iter = &op->store.write_iter;
/* marshall the parameters */
bp = call->request;
@@ -1139,7 +1141,7 @@ void afs_fs_store_data(struct afs_operation *op)
if (!call)
return afs_op_nomem(op);
- call->write_iter = op->store.write_iter;
+ call->write_iter = &op->store.write_iter;
/* marshall the parameters */
bp = call->request;
diff --git a/fs/afs/internal.h b/fs/afs/internal.h
index d2641efc756f..f20126000524 100644
--- a/fs/afs/internal.h
+++ b/fs/afs/internal.h
@@ -915,7 +915,7 @@ struct afs_operation {
afs_lock_type_t type;
} lock;
struct {
- struct iov_iter *write_iter;
+ struct iov_iter write_iter;
loff_t pos;
loff_t size;
loff_t i_size;
@@ -1698,7 +1698,9 @@ extern int afs_check_volume_status(struct afs_volume *, struct afs_operation *);
/*
* write.c
*/
-void afs_prepare_write(struct netfs_io_subrequest *subreq);
+int afs_estimate_write(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_write_estimate *estimate);
void afs_issue_write(struct netfs_io_subrequest *subreq);
void afs_begin_writeback(struct netfs_io_request *wreq);
void afs_retry_request(struct netfs_io_request *wreq, struct netfs_io_stream *stream);
diff --git a/fs/afs/symlink.c b/fs/afs/symlink.c
index c46d7c91250a..1a64099d03ef 100644
--- a/fs/afs/symlink.c
+++ b/fs/afs/symlink.c
@@ -243,9 +243,10 @@ int afs_symlink_writepages(struct address_space *mapping,
if (vnode->directory &&
atomic64_read(&vnode->cb_expires_at) != AFS_NO_CB_PROMISE) {
+ size_t len = i_size_read(&vnode->netfs.inode);
iov_iter_bvec_queue(&iter, ITER_SOURCE, vnode->directory, 0, 0,
- i_size_read(&vnode->netfs.inode));
- ret = netfs_writeback_single(mapping, wbc, &iter);
+ round_up(len, PAGE_SIZE));
+ ret = netfs_writeback_single(mapping, wbc, &iter, len);
}
if (ret == 0) {
diff --git a/fs/afs/write.c b/fs/afs/write.c
index 7f34b939706a..8b6053ebc2b3 100644
--- a/fs/afs/write.c
+++ b/fs/afs/write.c
@@ -83,17 +83,20 @@ static const struct afs_operation_ops afs_store_data_operation = {
};
/*
- * Prepare a subrequest to write to the server. This sets the max_len
- * parameter.
+ * Estimate the maximum size of a write we can send to the server.
*/
-void afs_prepare_write(struct netfs_io_subrequest *subreq)
+int afs_estimate_write(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_write_estimate *estimate)
{
- struct netfs_io_stream *stream = &subreq->rreq->io_streams[subreq->stream_nr];
+ unsigned long long limit = ULLONG_MAX - stream->issue_from;
+ unsigned long long max_len = 256 * 1024 * 1024;
//if (test_bit(NETFS_SREQ_RETRYING, &subreq->flags))
- // subreq->max_len = 512 * 1024;
- //else
- stream->sreq_max_len = 256 * 1024 * 1024;
+ // max_len = 512 * 1024;
+
+ estimate->issue_at = stream->issue_from + umin(max_len, limit);
+ return 0;
}
/*
@@ -139,12 +142,15 @@ static void afs_issue_write_worker(struct work_struct *work)
op->flags |= AFS_OPERATION_UNINTR;
op->ops = &afs_store_data_operation;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
afs_begin_vnode_operation(op);
- op->store.write_iter = &subreq->io_iter;
op->store.i_size = umax(pos + len, netfs_read_remote_i_size(&vnode->netfs.inode));
op->mtime = inode_get_mtime(&vnode->netfs.inode);
+ iov_iter_bvec_queue(&op->store.write_iter, ITER_SOURCE, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset, subreq->len);
+
afs_wait_for_operation(op);
ret = afs_put_operation(op);
switch (ret) {
@@ -170,6 +176,14 @@ static void afs_issue_write_worker(struct work_struct *work)
void afs_issue_write(struct netfs_io_subrequest *subreq)
{
+ int ret;
+
+ if (subreq->len > 256 * 1024 * 1024)
+ subreq->len = 256 * 1024 * 1024;
+ ret = netfs_prepare_write_buffer(subreq, INT_MAX);
+ if (ret < 0)
+ return netfs_write_subrequest_terminated(subreq, ret);
+
subreq->work.func = afs_issue_write_worker;
if (!queue_work(system_dfl_wq, &subreq->work))
WARN_ON_ONCE(1);
@@ -183,6 +197,8 @@ void afs_begin_writeback(struct netfs_io_request *wreq)
{
if (S_ISREG(wreq->inode->i_mode))
afs_get_writeback_key(wreq);
+
+ wreq->io_streams[0].avail = true;
}
/*
diff --git a/fs/afs/yfsclient.c b/fs/afs/yfsclient.c
index d941179730a9..52c588092050 100644
--- a/fs/afs/yfsclient.c
+++ b/fs/afs/yfsclient.c
@@ -385,7 +385,9 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call)
if (call->remaining == 0)
goto no_more_data;
- call->iter = &subreq->io_iter;
+ iov_iter_bvec_queue(&call->def_iter, ITER_DEST, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset, subreq->len);
+
call->iov_len = min(call->remaining, subreq->len - subreq->transferred);
call->unmarshall++;
fallthrough;
@@ -1357,7 +1359,7 @@ void yfs_fs_store_data(struct afs_operation *op)
if (!call)
return afs_op_nomem(op);
- call->write_iter = op->store.write_iter;
+ call->write_iter = &op->store.write_iter;
/* marshall the parameters */
bp = call->request;
diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c
index eebebda46a09..8256d7d66da3 100644
--- a/fs/cachefiles/io.c
+++ b/fs/cachefiles/io.c
@@ -26,7 +26,10 @@ struct cachefiles_kiocb {
};
struct cachefiles_object *object;
netfs_io_terminated_t term_func;
- void *term_func_priv;
+ union {
+ struct netfs_io_subrequest *subreq;
+ void *term_func_priv;
+ };
bool was_async;
unsigned int inval_counter; /* Copy of cookie->inval_counter */
u64 b_writing;
@@ -194,12 +197,133 @@ static int cachefiles_read(struct netfs_cache_resources *cres,
return ret;
}
+/*
+ * Handle completion of a read from the cache issued by netfslib.
+ */
+static void cachefiles_issue_read_complete(struct kiocb *iocb, long ret)
+{
+ struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
+ struct netfs_io_subrequest *subreq = ki->subreq;
+ struct inode *inode = file_inode(ki->iocb.ki_filp);
+
+ _enter("%ld", ret);
+
+ if (ret < 0) {
+ subreq->error = -ESTALE;
+ trace_cachefiles_io_error(ki->object, inode, ret,
+ cachefiles_trace_read_error);
+ }
+
+ if (ret >= 0) {
+ if (ki->object->cookie->inval_counter == ki->inval_counter) {
+ subreq->error = 0;
+ if (ret > 0) {
+ subreq->transferred += ret;
+ __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
+ }
+ } else {
+ subreq->error = -ESTALE;
+ }
+ }
+
+ netfs_read_subreq_terminated(subreq);
+ cachefiles_put_kiocb(ki);
+}
+
+/*
+ * Issue a read operation to the cache.
+ */
+static void cachefiles_issue_read(struct netfs_io_subrequest *subreq)
+{
+ struct netfs_cache_resources *cres = &subreq->rreq->cache_resources;
+ struct cachefiles_object *object;
+ struct cachefiles_kiocb *ki;
+ struct iov_iter iter;
+ struct file *file;
+ unsigned int old_nofs;
+ ssize_t ret = -ENOBUFS;
+
+ if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
+ goto failed;
+
+ fscache_count_read();
+ object = cachefiles_cres_object(cres);
+ file = cachefiles_cres_file(cres);
+
+ _enter("%pD,%lli,%llx,%zx/%llx",
+ file, file_inode(file)->i_ino, subreq->start, subreq->len,
+ i_size_read(file_inode(file)));
+
+ if (subreq->len > MAX_RW_COUNT)
+ subreq->len = MAX_RW_COUNT;
+
+ ret = netfs_prepare_read_buffer(subreq, BIO_MAX_VECS);
+ if (ret < 0)
+ goto failed;
+
+ iov_iter_bvec_queue(&iter, ITER_DEST, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset, subreq->len);
+
+ ret = -ENOMEM;
+ ki = kzalloc_obj(struct cachefiles_kiocb);
+ if (!ki)
+ goto failed;
+
+ refcount_set(&ki->ki_refcnt, 2);
+ ki->iocb.ki_filp = file;
+ ki->iocb.ki_pos = subreq->start;
+ ki->iocb.ki_flags = IOCB_DIRECT;
+ ki->iocb.ki_ioprio = get_current_ioprio();
+ ki->iocb.ki_complete = cachefiles_issue_read_complete;
+ ki->object = object;
+ ki->inval_counter = cres->inval_counter;
+ ki->subreq = subreq;
+ ki->was_async = true;
+
+ get_file(ki->iocb.ki_filp);
+ cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
+
+ trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, subreq->len);
+ old_nofs = memalloc_nofs_save();
+ ret = cachefiles_inject_read_error();
+ if (ret == 0)
+ ret = vfs_iocb_iter_read(file, &ki->iocb, &iter);
+ memalloc_nofs_restore(old_nofs);
+
+ switch (ret) {
+ case -EIOCBQUEUED:
+ break;
+
+ case -ERESTARTSYS:
+ case -ERESTARTNOINTR:
+ case -ERESTARTNOHAND:
+ case -ERESTART_RESTARTBLOCK:
+ /* There's no easy way to restart the syscall since other AIO's
+ * may be already running. Just fail this IO with EINTR.
+ */
+ ret = -EINTR;
+ fallthrough;
+ default:
+ ki->was_async = false;
+ cachefiles_issue_read_complete(&ki->iocb, ret);
+ break;
+ }
+
+ cachefiles_put_kiocb(ki);
+ _leave(" = %zd", ret);
+ return;
+failed:
+ subreq->error = ret;
+ return netfs_read_subreq_terminated(subreq);
+}
+
/*
* Query the occupancy of the cache in a region, returning the extent of the
- * next two chunks of cached data and the next hole.
+ * next two chunks of cached data and the next hole. The occupancy map is
+ * preloaded to show just one giant hole.
*/
-static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
- struct fscache_occupancy *occ)
+static void cachefiles_query_occupancy(struct netfs_cache_resources *cres,
+ struct fscache_occupancy *occ)
{
struct cachefiles_object *object;
struct inode *inode;
@@ -209,7 +333,7 @@ static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
int i;
if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
- return -ENOBUFS;
+ return;
object = cachefiles_cres_object(cres);
file = cachefiles_cres_file(cres);
@@ -244,7 +368,7 @@ static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
ret = vfs_llseek(file, occ->query_from, SEEK_DATA);
if (IS_ERR_VALUE_LL(ret)) {
if (ret != -ENXIO)
- return ret;
+ goto done;
occ->query_from = ULLONG_MAX;
goto done;
}
@@ -257,7 +381,7 @@ static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
ret = vfs_llseek(file, occ->query_from, SEEK_HOLE);
if (IS_ERR_VALUE_LL(ret)) {
if (ret != -ENXIO)
- return ret;
+ goto done;
occ->query_from = ULLONG_MAX;
goto done;
}
@@ -270,7 +394,6 @@ static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
done:
_debug("query[0] %llx-%llx", occ->cached_from[0], occ->cached_to[0]);
_debug("query[1] %llx-%llx", occ->cached_from[1], occ->cached_to[1]);
- return 0;
}
/*
@@ -610,47 +733,13 @@ int __cachefiles_prepare_write(struct cachefiles_object *object,
cachefiles_has_space_for_write);
}
-static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
- loff_t *_start, size_t *_len, size_t upper_len,
- loff_t i_size, bool no_space_allocated_yet)
+static int cachefiles_estimate_write(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_write_estimate *estimate)
{
- struct cachefiles_object *object = cachefiles_cres_object(cres);
- struct cachefiles_cache *cache = object->volume->cache;
- const struct cred *saved_cred;
- int ret;
-
- if (!cachefiles_cres_file(cres)) {
- if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
- return -ENOBUFS;
- if (!cachefiles_cres_file(cres))
- return -ENOBUFS;
- }
-
- cachefiles_begin_secure(cache, &saved_cred);
- ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
- _start, _len, upper_len,
- no_space_allocated_yet);
- cachefiles_end_secure(cache, saved_cred);
- return ret;
-}
-
-static void cachefiles_prepare_write_subreq(struct netfs_io_subrequest *subreq)
-{
- struct netfs_io_request *wreq = subreq->rreq;
- struct netfs_cache_resources *cres = &wreq->cache_resources;
- struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
-
- _enter("W=%x[%x] %llx", wreq->debug_id, subreq->debug_index, subreq->start);
-
- stream->sreq_max_len = MAX_RW_COUNT;
- stream->sreq_max_segs = BIO_MAX_VECS;
-
- if (!cachefiles_cres_file(cres)) {
- if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
- return netfs_prepare_write_failed(subreq);
- if (!cachefiles_cres_file(cres))
- return netfs_prepare_write_failed(subreq);
- }
+ estimate->issue_at = stream->issue_from + MAX_RW_COUNT;
+ estimate->max_segs = BIO_MAX_VECS;
+ return 0;
}
static void cachefiles_issue_write(struct netfs_io_subrequest *subreq)
@@ -659,55 +748,55 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq)
struct netfs_cache_resources *cres = &wreq->cache_resources;
struct cachefiles_object *object = cachefiles_cres_object(cres);
struct cachefiles_cache *cache = object->volume->cache;
+ struct iov_iter iter;
const struct cred *saved_cred;
- size_t off, pre, post, len = subreq->len;
loff_t start = subreq->start;
- int ret;
+ size_t len = subreq->len;
+ int ret = -EINVAL;
_enter("W=%x[%x] %llx-%llx",
wreq->debug_id, subreq->debug_index, start, start + len - 1);
- /* We need to start on the cache granularity boundary */
- off = start & (cache->bsize - 1);
- if (off) {
- pre = cache->bsize - off;
- if (pre >= len) {
- fscache_count_dio_misfit();
- netfs_write_subrequest_terminated(subreq, len);
- return;
- }
- subreq->transferred += pre;
- start += pre;
- len -= pre;
- iov_iter_advance(&subreq->io_iter, pre);
- }
-
- /* We also need to end on the cache granularity boundary */
- post = len & (cache->bsize - 1);
- if (post) {
- len -= post;
- if (len == 0) {
- fscache_count_dio_misfit();
- netfs_write_subrequest_terminated(subreq, post);
- return;
- }
- iov_iter_truncate(&subreq->io_iter, len);
+ if (!cachefiles_cres_file(cres)) {
+ if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
+ goto failed;
+ if (!cachefiles_cres_file(cres))
+ goto failed;
+ }
+
+ ret = netfs_prepare_write_buffer(subreq, BIO_MAX_VECS);
+ if (ret < 0)
+ goto failed;
+
+ /* The buffer extraction func may round out start and end. */
+ start = subreq->start;
+ len = subreq->len;
+
+ /* We need to start and end on cache granularity boundaries. */
+ if (WARN_ON_ONCE(start & (cache->bsize - 1)) ||
+ WARN_ON_ONCE(len & (cache->bsize - 1))) {
+ fscache_count_dio_misfit();
+ ret = -EIO;
+ goto failed;
}
+ iov_iter_bvec_queue(&iter, ITER_SOURCE, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset, len);
+
trace_netfs_sreq(subreq, netfs_sreq_trace_cache_prepare);
cachefiles_begin_secure(cache, &saved_cred);
ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
&start, &len, len, true);
cachefiles_end_secure(cache, saved_cred);
- if (ret < 0) {
- netfs_write_subrequest_terminated(subreq, ret);
- return;
- }
+ if (ret < 0)
+ goto failed;
trace_netfs_sreq(subreq, netfs_sreq_trace_cache_write);
- cachefiles_write(&subreq->rreq->cache_resources,
- subreq->start, &subreq->io_iter,
+ cachefiles_write(&subreq->rreq->cache_resources, subreq->start, &iter,
netfs_write_subrequest_terminated, subreq);
+ return;
+failed:
+ return netfs_write_subrequest_terminated(subreq, ret);
}
/*
@@ -881,9 +970,9 @@ static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
.end_operation = cachefiles_end_operation,
.read = cachefiles_read,
.write = cachefiles_write,
+ .issue_read = cachefiles_issue_read,
.issue_write = cachefiles_issue_write,
- .prepare_write = cachefiles_prepare_write,
- .prepare_write_subreq = cachefiles_prepare_write_subreq,
+ .estimate_write = cachefiles_estimate_write,
.prepare_ondemand_read = cachefiles_prepare_ondemand_read,
.query_occupancy = cachefiles_query_occupancy,
.collect_write = cachefiles_collect_write,
diff --git a/fs/ceph/Kconfig b/fs/ceph/Kconfig
index 3d64a316ca31..aa6ccd7794d2 100644
--- a/fs/ceph/Kconfig
+++ b/fs/ceph/Kconfig
@@ -4,6 +4,7 @@ config CEPH_FS
depends on INET
select CEPH_LIB
select NETFS_SUPPORT
+ select NETFS_PGPRIV2
select FS_ENCRYPTION_ALGS if FS_ENCRYPTION
default n
help
diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index 0a86f672cc09..9f22d0a894a2 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -274,7 +274,7 @@ static void finish_netfs_read(struct ceph_osd_request *req)
ceph_dec_osd_stopping_blocker(fsc->mdsc);
}
-static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
+static int ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
{
struct netfs_io_request *rreq = subreq->rreq;
struct inode *inode = rreq->inode;
@@ -283,7 +283,8 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
struct ceph_mds_request *req;
struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
struct ceph_inode_info *ci = ceph_inode(inode);
- ssize_t err = 0;
+ struct iov_iter iter;
+ ssize_t err;
size_t len;
int mode;
@@ -292,21 +293,32 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
__clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
- if (subreq->start >= inode->i_size)
+ if (subreq->start >= inode->i_size) {
+ __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
+ err = 0;
goto out;
+ }
+
+ err = netfs_prepare_read_buffer(subreq, INT_MAX);
+ if (err < 0)
+ return err;
+
+ iov_iter_bvec_queue(&iter, ITER_DEST, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset,
+ subreq->len);
/* We need to fetch the inline data. */
mode = ceph_try_to_choose_auth_mds(inode, CEPH_STAT_CAP_INLINE_DATA);
req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_GETATTR, mode);
- if (IS_ERR(req)) {
- err = PTR_ERR(req);
- goto out;
- }
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+
req->r_ino1 = ci->i_vino;
req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA);
req->r_num_caps = 2;
trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
+
err = ceph_mdsc_do_request(mdsc, NULL, req);
if (err < 0)
goto out;
@@ -316,11 +328,11 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
if (iinfo->inline_version == CEPH_INLINE_NONE) {
/* The data got uninlined */
ceph_mdsc_put_request(req);
- return false;
+ return 1;
}
len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len);
- err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->io_iter);
+ err = copy_to_iter(iinfo->inline_data + subreq->start, len, &iter);
if (err == 0) {
err = -EFAULT;
} else {
@@ -333,23 +345,7 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
subreq->error = err;
trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress);
netfs_read_subreq_terminated(subreq);
- return true;
-}
-
-static int ceph_netfs_prepare_read(struct netfs_io_subrequest *subreq)
-{
- struct netfs_io_request *rreq = subreq->rreq;
- struct inode *inode = rreq->inode;
- struct ceph_inode_info *ci = ceph_inode(inode);
- struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
- u64 objno, objoff;
- u32 xlen;
-
- /* Truncate the extent at the end of the current block */
- ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len,
- &objno, &objoff, &xlen);
- rreq->io_streams[0].sreq_max_len = umin(xlen, fsc->mount_options->rsize);
- return 0;
+ return -EIOCBQUEUED;
}
static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
@@ -361,26 +357,34 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
struct ceph_client *cl = fsc->client;
struct ceph_osd_request *req = NULL;
struct ceph_vino vino = ceph_vino(inode);
- int err;
- u64 len;
+ struct iov_iter iter;
+ u64 objno, objoff, len, off = subreq->start;
+ u32 maxlen;
+ int err = -EIO;
bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD);
- u64 off = subreq->start;
int extent_cnt;
- if (ceph_inode_is_shutdown(inode)) {
- err = -EIO;
- goto out;
+ if (ceph_inode_is_shutdown(inode))
+ goto failed_noput;
+
+ if (ceph_has_inline_data(ci)) {
+ err = ceph_netfs_issue_op_inline(subreq);
+ if (err != 1)
+ goto failed_noput;
}
- if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq))
- return;
+ /* Truncate the extent at the end of the current block */
+ ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len,
+ &objno, &objoff, &maxlen);
+ maxlen = umin(maxlen, fsc->mount_options->rsize);
+ len = umin(subreq->len, maxlen);
+ subreq->len = len;
// TODO: This rounding here is slightly dodgy. It *should* work, for
// now, as the cache only deals in blocks that are a multiple of
// PAGE_SIZE and fscrypt blocks are at most PAGE_SIZE. What needs to
// happen is for the fscrypt driving to be moved into netfslib and the
// data in the cache also to be stored encrypted.
- len = subreq->len;
ceph_fscrypt_adjust_off_and_len(inode, &off, &len);
req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino,
@@ -389,20 +393,27 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
ci->i_truncate_size, false);
if (IS_ERR(req)) {
err = PTR_ERR(req);
- req = NULL;
- goto out;
+ goto failed_noput;
}
if (sparse) {
extent_cnt = __ceph_sparse_read_ext_count(inode, len);
err = ceph_alloc_sparse_ext_map(&req->r_ops[0], extent_cnt);
if (err)
- goto out;
+ goto failed;
}
doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n",
ceph_vinop(inode), subreq->start, subreq->len, len);
+ err = netfs_prepare_read_buffer(subreq, INT_MAX);
+ if (err < 0)
+ goto failed;
+
+ iov_iter_bvec_queue(&iter, ITER_DEST, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset,
+ subreq->len);
+
/*
* FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for
* encrypted inodes. We'd need infrastructure that handles an iov_iter
@@ -421,13 +432,11 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
* ceph_msg_data_cursor_init() triggers BUG_ON() in the case
* if msg->sparse_read_total > msg->data_length.
*/
- subreq->io_iter.count = len;
-
- err = iov_iter_get_pages_alloc2(&subreq->io_iter, &pages, len, &page_off);
+ err = iov_iter_get_pages_alloc2(&iter, &pages, len, &page_off);
if (err < 0) {
doutc(cl, "%llx.%llx failed to allocate pages, %d\n",
ceph_vinop(inode), err);
- goto out;
+ goto eio;
}
/* should always give us a page-aligned read */
@@ -438,12 +447,10 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false,
false);
} else {
- osd_req_op_extent_osd_iter(req, 0, &subreq->io_iter);
- }
- if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) {
- err = -EIO;
- goto out;
+ osd_req_op_extent_osd_iter(req, 0, &iter);
}
+ if (!ceph_inc_osd_stopping_blocker(fsc->mdsc))
+ goto eio;
req->r_callback = finish_netfs_read;
req->r_priv = subreq;
req->r_inode = inode;
@@ -451,19 +458,21 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
ceph_osdc_start_request(req->r_osdc, req);
-out:
ceph_osdc_put_request(req);
- if (err) {
- subreq->error = err;
- netfs_read_subreq_terminated(subreq);
- }
- doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err);
+ doutc(cl, "%llx.%llx result -EIOCBQUEUED\n", ceph_vinop(inode));
+ return;
+eio:
+ err = -EIO;
+failed:
+ ceph_osdc_put_request(req);
+failed_noput:
+ subreq->error = err;
+ return netfs_read_subreq_terminated(subreq);
}
static int ceph_init_request(struct netfs_io_request *rreq, struct file *file)
{
struct inode *inode = rreq->inode;
- struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
struct ceph_client *cl = ceph_inode_to_client(inode);
int got = 0, want = CEPH_CAP_FILE_CACHE;
struct ceph_netfs_request_data *priv;
@@ -515,7 +524,6 @@ static int ceph_init_request(struct netfs_io_request *rreq, struct file *file)
priv->caps = got;
rreq->netfs_priv = priv;
- rreq->io_streams[0].sreq_max_len = fsc->mount_options->rsize;
out:
if (ret < 0) {
@@ -543,7 +551,6 @@ static void ceph_netfs_free_request(struct netfs_io_request *rreq)
const struct netfs_request_ops ceph_netfs_ops = {
.init_request = ceph_init_request,
.free_request = ceph_netfs_free_request,
- .prepare_read = ceph_netfs_prepare_read,
.issue_read = ceph_netfs_issue_read,
.expand_readahead = ceph_netfs_expand_readahead,
.check_write_begin = ceph_netfs_check_write_begin,
diff --git a/fs/netfs/Kconfig b/fs/netfs/Kconfig
index 7701c037c328..d0e7b0971fa3 100644
--- a/fs/netfs/Kconfig
+++ b/fs/netfs/Kconfig
@@ -22,6 +22,9 @@ config NETFS_STATS
between CPUs. On the other hand, the stats are very useful for
debugging purposes. Saying 'Y' here is recommended.
+config NETFS_PGPRIV2
+ bool
+
config NETFS_DEBUG
bool "Enable dynamic debugging netfslib and FS-Cache"
depends on NETFS_SUPPORT
diff --git a/fs/netfs/Makefile b/fs/netfs/Makefile
index 0621e6870cbd..421dd0be413b 100644
--- a/fs/netfs/Makefile
+++ b/fs/netfs/Makefile
@@ -12,13 +12,13 @@ netfs-y := \
misc.o \
objects.o \
read_collect.o \
- read_pgpriv2.o \
read_retry.o \
read_single.o \
write_collect.o \
write_issue.o \
write_retry.o
+netfs-$(CONFIG_NETFS_PGPRIV2) += read_pgpriv2.o
netfs-$(CONFIG_NETFS_STATS) += stats.o
netfs-$(CONFIG_FSCACHE) += \
diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c
index 92716a6c9133..b47b3760fe0d 100644
--- a/fs/netfs/buffered_read.c
+++ b/fs/netfs/buffered_read.c
@@ -100,70 +100,98 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in
}
/*
- * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O
- * @subreq: The subrequest to be set up
- *
- * Prepare the I/O iterator representing the read buffer on a subrequest for
- * the filesystem to use for I/O (it can be passed directly to a socket). This
- * is intended to be called from the ->issue_read() method once the filesystem
- * has trimmed the request to the size it wants.
- *
- * Returns the limited size if successful and -ENOMEM if insufficient memory
- * available.
+ * Prepare the I/O buffer on a buffered read subrequest for the filesystem to
+ * use as a bvec queue.
*/
-static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq)
+static int netfs_prepare_buffered_read_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
{
struct netfs_io_request *rreq = subreq->rreq;
struct netfs_io_stream *stream = &rreq->io_streams[0];
ssize_t extracted;
- size_t rsize = subreq->len;
- if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER)
- rsize = umin(rsize, stream->sreq_max_len);
+ _enter("R=%08x[%x] l=%zx s=%u",
+ rreq->debug_id, subreq->debug_index, subreq->len, max_segs);
- bvecq_pos_set(&subreq->dispatch_pos, &rreq->dispatch_cursor);
- extracted = bvecq_slice(&rreq->dispatch_cursor, subreq->len,
- stream->sreq_max_segs, &subreq->nr_segs);
- if (extracted < rsize) {
+ bvecq_pos_set(&subreq->dispatch_pos, &stream->dispatch_cursor);
+ bvecq_pos_set(&subreq->content, &subreq->dispatch_pos);
+ extracted = bvecq_slice(&stream->dispatch_cursor, subreq->len,
+ max_segs, &subreq->nr_segs);
+
+ if (extracted < subreq->len) {
subreq->len = extracted;
trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
}
+ stream->buffered -= extracted;
+ stream->issue_from = subreq->start + subreq->len;
+ rreq->submitted = stream->issue_from;
- return subreq->len;
+ if (!stream->buffered)
+ netfs_all_subreqs_queued(rreq);
+ return 0;
}
-/*
- * Issue a read against the cache.
- * - Eats the caller's ref on subreq.
+/**
+ * netfs_prepare_read_buffer - Get the buffer for a subrequest
+ * @subreq: The subrequest to get the buffer for
+ * @max_segs: Maximum number of segments in buffer (or INT_MAX)
+ *
+ * Extract a slice of buffer from the stream and attach it to the subrequest as
+ * a bio_vec queue. The maximum amount of data attached is set by
+ * @subreq->len, but this may be shortened if @max_segs would be exceeded.
+ *
+ * [!] NOTE: This must be run in the same thread as ->issue_read() was called
+ * in as we access the readahead_control struct if there is one.
*/
-static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq,
- struct netfs_io_subrequest *subreq)
+int netfs_prepare_read_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
{
- struct netfs_cache_resources *cres = &rreq->cache_resources;
-
- netfs_stat(&netfs_n_rh_read);
- cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_IGNORE,
- netfs_cache_read_terminated, subreq);
+ switch (subreq->rreq->origin) {
+ case NETFS_READAHEAD:
+ case NETFS_READPAGE:
+ case NETFS_READ_FOR_WRITE:
+ if (subreq->retry_count)
+ return netfs_prepare_buffered_read_retry_buffer(subreq, max_segs);
+ return netfs_prepare_buffered_read_buffer(subreq, max_segs);
+
+ case NETFS_UNBUFFERED_READ:
+ case NETFS_DIO_READ:
+ case NETFS_READ_GAPS:
+ return netfs_prepare_unbuffered_read_buffer(subreq, max_segs);
+ case NETFS_READ_SINGLE:
+ return netfs_prepare_read_single_buffer(subreq, max_segs);
+ default:
+ WARN_ON_ONCE(1);
+ return -EIO;
+ }
}
+EXPORT_SYMBOL(netfs_prepare_read_buffer);
-int netfs_read_query_cache(struct netfs_io_request *rreq, struct fscache_occupancy *occ)
+void netfs_read_query_cache(struct netfs_io_request *rreq, struct fscache_occupancy *occ)
{
struct netfs_cache_resources *cres = &rreq->cache_resources;
occ->granularity = PAGE_SIZE;
if (occ->query_from >= occ->query_to)
- return 0;
+ return;
if (!cres->ops)
- return 0;
+ return;
occ->query_from = round_up(occ->query_from, occ->granularity);
- return cres->ops->query_occupancy(cres, occ);
+ cres->ops->query_occupancy(cres, occ);
}
-void netfs_queue_read(struct netfs_io_request *rreq,
- struct netfs_io_subrequest *subreq)
+/*
+ * Allocate and prepare a read subrequest.
+ */
+struct netfs_io_subrequest *netfs_alloc_read_subrequest(struct netfs_io_request *rreq)
{
+ struct netfs_io_subrequest *subreq;
struct netfs_io_stream *stream = &rreq->io_streams[0];
+ subreq = netfs_alloc_subrequest(rreq);
+ if (!subreq)
+ return subreq;
+
__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
/* We add to the end of the list whilst the collector may be walking
@@ -182,30 +210,37 @@ void netfs_queue_read(struct netfs_io_request *rreq,
}
spin_unlock(&rreq->lock);
+ return subreq;
}
static void netfs_issue_read(struct netfs_io_request *rreq,
- struct netfs_io_subrequest *subreq)
+ struct netfs_io_subrequest *subreq)
{
- bvecq_pos_set(&subreq->content, &subreq->dispatch_pos);
- iov_iter_bvec_queue(&subreq->io_iter, ITER_DEST, subreq->content.bvecq,
- subreq->content.slot, subreq->content.offset, subreq->len);
+ struct netfs_io_stream *stream = &rreq->io_streams[0];
+
+ _enter("R=%08x[%x]", rreq->debug_id, subreq->debug_index);
switch (subreq->source) {
case NETFS_DOWNLOAD_FROM_SERVER:
- rreq->netfs_ops->issue_read(subreq);
- break;
- case NETFS_READ_FROM_CACHE:
- netfs_read_cache_to_pagecache(rreq, subreq);
- break;
+ return rreq->netfs_ops->issue_read(subreq);
+ case NETFS_READ_FROM_CACHE: {
+ struct netfs_cache_resources *cres = &rreq->cache_resources;
+
+ netfs_stat(&netfs_n_rh_read);
+ return cres->ops->issue_read(subreq);
+ }
default:
- bvecq_zero(&rreq->dispatch_cursor, subreq->len);
+ WARN_ON_ONCE(1);
+ fallthrough;
+ case NETFS_FILL_WITH_ZEROES:
+ stream->issue_from = subreq->start + subreq->len;
+ stream->buffered = 0;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
+ netfs_all_subreqs_queued(rreq);
+ bvecq_zero(&stream->dispatch_cursor, subreq->len);
subreq->transferred = subreq->len;
subreq->error = 0;
- iov_iter_zero(subreq->len, &subreq->io_iter);
- subreq->transferred = subreq->len;
- netfs_read_subreq_terminated(subreq);
- break;
+ return netfs_read_subreq_terminated(subreq);
}
}
@@ -225,20 +260,17 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
.cached_to[1] = ULLONG_MAX,
};
struct fscache_occupancy *occ = &_occ;
- unsigned long long start = rreq->start;
- ssize_t size = rreq->len;
+ struct netfs_io_stream *stream = &rreq->io_streams[0];
int ret = 0;
_enter("R=%08x", rreq->debug_id);
- bvecq_pos_set(&rreq->dispatch_cursor, &rreq->load_cursor);
- bvecq_pos_set(&rreq->collect_cursor, &rreq->dispatch_cursor);
+ bvecq_pos_set(&stream->dispatch_cursor, &rreq->load_cursor);
+ bvecq_pos_set(&rreq->collect_cursor, &rreq->load_cursor);
do {
- int (*prepare_read)(struct netfs_io_subrequest *subreq) = NULL;
struct netfs_io_subrequest *subreq;
- unsigned long long hole_to, cache_to;
- ssize_t slice;
+ unsigned long long hole_to, cache_to, stop;
/* If we don't have any, find out the next couple of data
* extents from the cache, containing of following the
@@ -247,7 +279,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
*/
hole_to = occ->cached_from[0];
cache_to = occ->cached_to[0];
- if (start >= cache_to) {
+ if (stream->issue_from >= cache_to) {
/* Extent exhausted; shuffle down. */
int i;
@@ -263,51 +295,45 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
continue;
/* Get new extents */
- ret = netfs_read_query_cache(rreq, occ);
- if (ret < 0)
- break;
+ netfs_read_query_cache(rreq, occ);
continue;
}
- subreq = netfs_alloc_subrequest(rreq);
+ subreq = netfs_alloc_read_subrequest(rreq);
if (!subreq) {
ret = -ENOMEM;
break;
}
- subreq->start = start;
- subreq->len = size;
-
- netfs_queue_read(rreq, subreq);
+ subreq->start = stream->issue_from;
+ stop = stream->issue_from + stream->buffered;
unsigned long long zero_point = netfs_read_zero_point(rreq->inode);
unsigned long long zlimit = umin(zero_point, rreq->i_size);
_debug("rsub %llx %llx-%llx", subreq->start, hole_to, cache_to);
- if (start >= hole_to && start < cache_to) {
+ if (stream->issue_from >= hole_to && stream->issue_from < cache_to) {
/* Overlap with a cached region, where the cache may
* record a block of zeroes.
*/
- _debug("cached s=%llx c=%llx l=%zx", start, cache_to, size);
- subreq->len = umin(cache_to - start, size);
+ _debug("cached s=%llx c=%llx l=%zx",
+ stream->issue_from, cache_to, stream->buffered);
+ subreq->len = umin(cache_to - stream->issue_from, stream->buffered);
subreq->len = round_up(subreq->len, occ->granularity);
if (occ->cached_type[0] == FSCACHE_EXTENT_ZERO) {
subreq->source = NETFS_FILL_WITH_ZEROES;
netfs_stat(&netfs_n_rh_zero);
} else {
subreq->source = NETFS_READ_FROM_CACHE;
- prepare_read = rreq->cache_resources.ops->prepare_read;
}
-
- trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
-
- } else if (subreq->start >= zlimit && size > 0) {
+ } else if (subreq->start >= zlimit &&
+ subreq->start < stop) {
/* If this range lies beyond the zero-point, that part
* can just be cleared locally.
*/
- _debug("zero %llx-%llx", start, start + size);
- subreq->len = size;
+ _debug("zero %llx-%llx", subreq->start, stop);
+ subreq->len = stream->buffered;
subreq->source = NETFS_FILL_WITH_ZEROES;
if (rreq->cache_resources.ops)
__set_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
@@ -317,10 +343,10 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
* this range lies beyond the zero-point or the EOF,
* that part can just be cleared locally.
*/
- unsigned long long limit = min3(zlimit, start + size, hole_to);
+ unsigned long long limit = min3(zlimit, stop, hole_to);
_debug("limit %llx %llx", rreq->i_size, zero_point);
- _debug("download %llx-%llx", start, start + size);
+ _debug("download %llx-%llx", subreq->start, stop);
subreq->len = umin(limit - subreq->start, ULONG_MAX);
subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
if (rreq->cache_resources.ops)
@@ -328,41 +354,15 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
netfs_stat(&netfs_n_rh_download);
}
- if (size == 0) {
+ if (subreq->len == 0) {
pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx",
rreq->debug_id, subreq->debug_index,
- subreq->len, size,
+ subreq->len, stream->buffered,
subreq->start, zero_point, rreq->i_size);
netfs_cancel_read(subreq, ret);
break;
}
- rreq->io_streams[0].sreq_max_len = MAX_RW_COUNT;
- rreq->io_streams[0].sreq_max_segs = INT_MAX;
-
- if (prepare_read) {
- ret = prepare_read(subreq);
- if (ret < 0) {
- netfs_cancel_read(subreq, ret);
- break;
- }
- trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
- }
-
- slice = netfs_prepare_read_iterator(subreq);
- if (slice < 0) {
- ret = slice;
- netfs_cancel_read(subreq, ret);
- break;
- }
- start += slice;
- size -= slice;
- if (size <= 0) {
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
- }
-
- trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
netfs_issue_read(rreq, subreq);
netfs_maybe_bulk_drop_ra_refs(rreq);
@@ -371,19 +371,19 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
if (test_bit(NETFS_RREQ_FAILED, &rreq->flags))
break;
cond_resched();
- } while (size > 0);
+ } while (stream->buffered > 0);
- if (unlikely(size > 0)) {
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+ if (unlikely(!netfs_are_all_subreqs_queued(rreq))) {
+ netfs_all_subreqs_queued(rreq);
netfs_wake_collector(rreq);
}
/* Defer error return as we may need to wait for outstanding I/O. */
- cmpxchg(&rreq->error, 0, ret);
+ if (ret < 0)
+ cmpxchg(&rreq->error, 0, ret);
bvecq_pos_unset(&rreq->load_cursor);
- bvecq_pos_unset(&rreq->dispatch_cursor);
+ bvecq_pos_unset(&stream->dispatch_cursor);
}
/**
@@ -404,17 +404,22 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
void netfs_readahead(struct readahead_control *ractl)
{
struct netfs_io_request *rreq;
+ struct netfs_io_stream *stream;
struct netfs_inode *ictx = netfs_inode(ractl->mapping->host);
unsigned long long start = readahead_pos(ractl);
ssize_t added;
size_t size = readahead_length(ractl);
int ret;
+ _enter("");
+
rreq = netfs_alloc_request(ractl->mapping, ractl->file, start, size,
NETFS_READAHEAD);
if (IS_ERR(rreq))
return;
+ stream = &rreq->io_streams[0];
+
__set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags);
ret = netfs_begin_cache_read(rreq, ictx);
@@ -441,6 +446,8 @@ void netfs_readahead(struct readahead_control *ractl)
rreq->submitted = rreq->start + added;
rreq->cleaned_to = rreq->start;
rreq->front_folio_order = get_order(rreq->load_cursor.bvecq->bv[0].bv_len);
+ stream->issue_from = rreq->start;
+ stream->buffered = added;
netfs_read_to_pagecache(rreq);
netfs_maybe_bulk_drop_ra_refs(rreq);
@@ -456,6 +463,7 @@ EXPORT_SYMBOL(netfs_readahead);
*/
static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio)
{
+ struct netfs_io_stream *stream = &rreq->io_streams[0];
struct bvecq *bq;
size_t fsize = folio_size(folio);
@@ -466,6 +474,8 @@ static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct fo
bvec_set_folio(&bq->bv[0], folio, fsize, 0);
bvecq_filled_to(bq, 1);
rreq->submitted = rreq->start + fsize;
+ stream->issue_from = rreq->start;
+ stream->buffered = fsize;
return 0;
}
@@ -475,6 +485,7 @@ static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct fo
static int netfs_read_gaps(struct file *file, struct folio *folio)
{
struct netfs_io_request *rreq;
+ struct netfs_io_stream *stream;
struct address_space *mapping = folio->mapping;
struct netfs_group *group = netfs_folio_group(folio);
struct netfs_folio *finfo = netfs_folio_info(folio);
@@ -496,6 +507,7 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
ret = PTR_ERR(rreq);
goto alloc_error;
}
+ stream = &rreq->io_streams[0];
ret = netfs_begin_cache_read(rreq, ctx);
if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
@@ -544,6 +556,8 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
bvecq_filled_to(bq, slot);
rreq->submitted = rreq->start + flen;
+ stream->issue_from = rreq->start;
+ stream->buffered = flen;
netfs_read_to_pagecache(rreq);
@@ -622,6 +636,7 @@ int netfs_read_folio(struct file *file, struct folio *folio)
goto discard;
netfs_read_to_pagecache(rreq);
+
ret = netfs_wait_for_read(rreq);
netfs_put_request(rreq, netfs_rreq_trace_put_return);
return ret < 0 ? ret : 0;
diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c
index 350eb43ecc00..1369c28e9b30 100644
--- a/fs/netfs/buffered_write.c
+++ b/fs/netfs/buffered_write.c
@@ -95,8 +95,8 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
.range_start = iocb->ki_pos,
.range_end = iocb->ki_pos + iter->count,
};
- struct netfs_io_request *wreq = NULL;
- struct folio *folio = NULL, *writethrough = NULL;
+ struct netfs_writethrough *writethrough = NULL;
+ struct folio *folio = NULL;
unsigned int bdp_flags = (iocb->ki_flags & IOCB_NOWAIT) ? BDP_ASYNC : 0;
ssize_t written = 0, ret, ret2;
loff_t pos = iocb->ki_pos;
@@ -113,15 +113,13 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
goto out;
}
- wreq = netfs_begin_writethrough(iocb, iter->count);
- if (IS_ERR(wreq)) {
+ writethrough = netfs_begin_writethrough(iocb, iter->count);
+ if (IS_ERR(writethrough)) {
wbc_detach_inode(&wbc);
- ret = PTR_ERR(wreq);
- wreq = NULL;
+ ret = PTR_ERR(writethrough);
+ writethrough = NULL;
goto out;
}
- if (!is_sync_kiocb(iocb))
- wreq->iocb = iocb;
netfs_stat(&netfs_n_wh_writethrough);
} else {
netfs_stat(&netfs_n_wh_buffered_write);
@@ -387,14 +385,15 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
pos += copied;
written += copied;
- if (likely(!wreq)) {
+ if (likely(!writethrough)) {
folio_mark_dirty(folio);
folio_unlock(folio);
} else {
- netfs_advance_writethrough(wreq, &wbc, folio, copied,
- offset + copied == flen,
- &writethrough);
+ ret = netfs_advance_writethrough(writethrough, &wbc, folio, copied,
+ offset + copied == flen);
/* Folio unlocked */
+ if (ret < 0)
+ break;
}
retry:
folio_put(folio);
@@ -417,8 +416,8 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
ctx->ops->post_modify(inode);
}
- if (unlikely(wreq)) {
- ret2 = netfs_end_writethrough(wreq, &wbc, writethrough);
+ if (unlikely(writethrough)) {
+ ret2 = netfs_end_writethrough(writethrough, &wbc);
wbc_detach_inode(&wbc);
if (ret2 == -EIOCBQUEUED)
return ret2;
diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c
index 3c52c7584489..d2675e981405 100644
--- a/fs/netfs/direct_read.c
+++ b/fs/netfs/direct_read.c
@@ -16,6 +16,32 @@
#include <linux/netfs.h>
#include "internal.h"
+int netfs_prepare_unbuffered_read_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
+{
+ struct netfs_io_request *rreq = subreq->rreq;
+ struct netfs_io_stream *stream = &rreq->io_streams[0];
+ size_t len;
+
+ bvecq_pos_set(&subreq->dispatch_pos, &stream->dispatch_cursor);
+ bvecq_pos_set(&subreq->content, &stream->dispatch_cursor);
+ len = bvecq_slice(&stream->dispatch_cursor, subreq->len, max_segs,
+ &subreq->nr_segs);
+
+ if (len < subreq->len) {
+ subreq->len = len;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
+ }
+
+ stream->buffered -= subreq->len;
+ stream->issue_from += subreq->len;
+ rreq->submitted = stream->issue_from;
+
+ if (stream->buffered == 0)
+ netfs_all_subreqs_queued(rreq);
+ return 0;
+}
+
/*
* Perform a read to a buffer from the server, slicing up the region to be read
* according to the network rsize.
@@ -23,16 +49,13 @@
static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
{
struct netfs_io_stream *stream = &rreq->io_streams[0];
- unsigned long long start = rreq->start;
- ssize_t size = rreq->len;
- int ret;
- bvecq_pos_set(&rreq->dispatch_cursor, &rreq->load_cursor);
+ bvecq_pos_transfer(&stream->dispatch_cursor, &rreq->load_cursor);
do {
struct netfs_io_subrequest *subreq;
- subreq = netfs_alloc_subrequest(rreq);
+ subreq = netfs_alloc_read_subrequest(rreq);
if (!subreq) {
/* Stash the error in the request if there's not
* already an error set.
@@ -42,37 +65,10 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
}
subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
- subreq->start = start;
- subreq->len = size;
-
- netfs_queue_read(rreq, subreq);
+ subreq->start = stream->issue_from;
+ subreq->len = stream->buffered;
netfs_stat(&netfs_n_rh_download);
- if (rreq->netfs_ops->prepare_read) {
- ret = rreq->netfs_ops->prepare_read(subreq);
- if (ret < 0) {
- netfs_cancel_read(subreq, ret);
- break;
- }
- }
-
- bvecq_pos_set(&subreq->dispatch_pos, &rreq->dispatch_cursor);
- bvecq_pos_set(&subreq->content, &rreq->dispatch_cursor);
- subreq->len = bvecq_slice(&rreq->dispatch_cursor,
- umin(size, stream->sreq_max_len),
- stream->sreq_max_segs,
- &subreq->nr_segs);
-
- size -= subreq->len;
- start += subreq->len;
- rreq->submitted += subreq->len;
- if (size <= 0) {
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
- }
-
- iov_iter_bvec_queue(&subreq->io_iter, ITER_DEST, subreq->content.bvecq,
- subreq->content.slot, subreq->content.offset, subreq->len);
rreq->netfs_ops->issue_read(subreq);
@@ -81,15 +77,14 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
if (test_bit(NETFS_RREQ_FAILED, &rreq->flags))
break;
cond_resched();
- } while (size > 0);
+ } while (stream->buffered > 0);
- if (unlikely(size > 0)) {
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+ if (unlikely(stream->buffered > 0)) {
+ netfs_all_subreqs_queued(rreq);
netfs_wake_collector(rreq);
}
- bvecq_pos_unset(&rreq->dispatch_cursor);
+ bvecq_pos_unset(&stream->dispatch_cursor);
}
/*
@@ -140,6 +135,7 @@ static ssize_t netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync)
ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *iter)
{
struct netfs_io_request *rreq;
+ struct netfs_io_stream *stream;
ssize_t ret;
size_t orig_count = iov_iter_count(iter);
bool sync = is_sync_kiocb(iocb);
@@ -164,6 +160,8 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i
netfs_stat(&netfs_n_rh_dio_read);
trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_dio_read);
+ stream = &rreq->io_streams[0];
+
/* If this is an async op, we have to keep track of the destination
* buffer for ourselves as the caller's iterator will be trashed when
* we return.
@@ -179,6 +177,8 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i
goto error_put;
rreq->len = ret;
+ stream->buffered = ret;
+ stream->issue_from = rreq->start;
// TODO: Set up bounce buffer if needed
diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c
index 0309dd3c37d2..c51f3cbacd40 100644
--- a/fs/netfs/direct_write.c
+++ b/fs/netfs/direct_write.c
@@ -9,6 +9,34 @@
#include <linux/uio.h>
#include "internal.h"
+/*
+ * Prepare the buffer for an unbuffered/DIO write.
+ */
+int netfs_prepare_unbuffered_write_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
+{
+ struct netfs_io_stream *stream = &subreq->rreq->io_streams[subreq->stream_nr];
+ size_t len;
+
+ bvecq_pos_set(&subreq->dispatch_pos, &stream->dispatch_cursor);
+ bvecq_pos_set(&subreq->content, &stream->dispatch_cursor);
+ len = bvecq_slice(&stream->dispatch_cursor, subreq->len, max_segs,
+ &subreq->nr_segs);
+
+ if (len < subreq->len) {
+ subreq->len = len;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
+ }
+
+ // TODO: Wait here for completion of prev subreq
+
+ stream->issue_from += subreq->len;
+ stream->buffered -= subreq->len;
+ if (stream->buffered == 0)
+ netfs_all_subreqs_queued(subreq->rreq);
+ return 0;
+}
+
/*
* Perform the cleanup rituals after an unbuffered write is complete.
*/
@@ -74,9 +102,9 @@ static void netfs_unbuffered_write_collect(struct netfs_io_request *wreq,
wreq->transferred += subreq->transferred;
if (subreq->transferred < subreq->len) {
- bvecq_pos_unset(&wreq->dispatch_cursor);
- bvecq_pos_transfer(&wreq->dispatch_cursor, &subreq->dispatch_pos);
- bvecq_pos_advance(&wreq->dispatch_cursor, subreq->transferred);
+ bvecq_pos_unset(&stream->dispatch_cursor);
+ bvecq_pos_transfer(&stream->dispatch_cursor, &subreq->dispatch_pos);
+ bvecq_pos_advance(&stream->dispatch_cursor, subreq->transferred);
}
stream->collected_to = subreq->start + subreq->transferred;
@@ -85,6 +113,7 @@ static void netfs_unbuffered_write_collect(struct netfs_io_request *wreq,
trace_netfs_collect_stream(wreq, stream);
trace_netfs_collect_state(wreq, wreq->collected_to, 0);
+ /* TODO: Progressively clean up wreq->direct_bq */
}
/*
@@ -103,60 +132,36 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
_enter("%llx", wreq->len);
- bvecq_pos_set(&wreq->dispatch_cursor, &wreq->load_cursor);
- bvecq_pos_set(&wreq->collect_cursor, &wreq->dispatch_cursor);
+ stream->issue_from = wreq->start;
+ stream->buffered = wreq->len;
+ bvecq_pos_set(&stream->dispatch_cursor, &wreq->load_cursor);
+ bvecq_pos_set(&wreq->collect_cursor, &stream->dispatch_cursor);
if (wreq->origin == NETFS_DIO_WRITE)
inode_dio_begin(wreq->inode);
- stream->collected_to = wreq->start;
-
for (;;) {
bool retry = false;
if (!subreq) {
- netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred);
- subreq = stream->construct;
- stream->construct = NULL;
- } else {
- bvecq_pos_set(&subreq->dispatch_pos, &wreq->dispatch_cursor);
- }
-
- /* Check if (re-)preparation failed. */
- if (unlikely(test_bit(NETFS_SREQ_FAILED, &subreq->flags))) {
- netfs_write_subrequest_terminated(subreq, subreq->error);
- wreq->error = subreq->error;
- break;
+ subreq = netfs_alloc_write_subreq(wreq, stream);
+ if (!subreq)
+ return -ENOMEM;
}
- subreq->len = bvecq_slice(&wreq->dispatch_cursor, stream->sreq_max_len,
- stream->sreq_max_segs, &subreq->nr_segs);
- bvecq_pos_set(&subreq->content, &subreq->dispatch_pos);
-
- iov_iter_bvec_queue(&subreq->io_iter, ITER_SOURCE,
- subreq->content.bvecq, subreq->content.slot,
- subreq->content.offset,
- subreq->len);
-
- if (!iov_iter_count(&subreq->io_iter))
- break;
-
- trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
stream->issue_write(subreq);
- /* Async, need to wait. */
- netfs_wait_for_in_progress_stream(wreq, stream);
-
- if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
+ ret = netfs_wait_for_in_progress_subreq(wreq, subreq);
+ if (ret < 0) {
+ if (ret != -EAGAIN) {
+ list_del_init(&subreq->rreq_link);
+ ret = subreq->error;
+ netfs_put_subrequest(subreq, netfs_sreq_trace_put_failed);
+ subreq = NULL;
+ goto failed;
+ }
retry = true;
- } else if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) {
- ret = subreq->error;
- wreq->error = ret;
- netfs_see_subrequest(subreq, netfs_sreq_trace_see_failed);
- subreq = NULL;
- break;
}
- ret = 0;
if (!retry) {
netfs_unbuffered_write_collect(wreq, stream, subreq);
@@ -171,20 +176,21 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
continue;
}
- /* We need to retry the last subrequest, so first reset the
- * iterator, taking into account what, if anything, we managed
- * to transfer.
+ /* We need to retry the last subrequest, so first wind back the
+ * buffer position.
*/
subreq->error = -EAGAIN;
trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
bvecq_pos_unset(&subreq->content);
- bvecq_pos_unset(&wreq->dispatch_cursor);
- bvecq_pos_transfer(&wreq->dispatch_cursor, &subreq->dispatch_pos);
+ bvecq_pos_unset(&stream->dispatch_cursor);
+ bvecq_pos_transfer(&stream->dispatch_cursor, &subreq->dispatch_pos);
+ stream->issue_from -= subreq->len - subreq->transferred;
+ stream->buffered += subreq->len - subreq->transferred;
if (subreq->transferred > 0) {
- wreq->transferred += subreq->transferred;
- bvecq_pos_advance(&wreq->dispatch_cursor, subreq->transferred);
+ wreq->transferred += subreq->transferred;
+ bvecq_pos_advance(&stream->dispatch_cursor, subreq->transferred);
}
if (stream->source == NETFS_UPLOAD_TO_SERVER &&
@@ -192,25 +198,21 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
wreq->netfs_ops->retry_request(wreq, stream);
__clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
- __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
__clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
- subreq->start = wreq->start + wreq->transferred;
- subreq->len = wreq->len - wreq->transferred;
+ __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
+ subreq->start = stream->issue_from;
+ subreq->len = stream->buffered;
subreq->transferred = 0;
subreq->retry_count += 1;
- stream->sreq_max_len = UINT_MAX;
- stream->sreq_max_segs = INT_MAX;
netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
- if (stream->prepare_write)
- stream->prepare_write(subreq);
__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
netfs_stat(&netfs_n_wh_retry_write_subreq);
}
- bvecq_pos_unset(&wreq->dispatch_cursor);
- bvecq_pos_unset(&wreq->load_cursor);
+failed:
+ bvecq_pos_unset(&stream->dispatch_cursor);
netfs_unbuffered_write_done(wreq);
_leave(" = %d", ret);
return ret;
@@ -254,6 +256,7 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *
if (IS_ERR(wreq))
return PTR_ERR(wreq);
+ wreq->len = iov_iter_count(iter);
wreq->io_streams[0].avail = true;
trace_netfs_write(wreq, (iocb->ki_flags & IOCB_DIRECT ?
netfs_write_trace_dio_write :
@@ -264,9 +267,7 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *
* we have to save the source buffer as the iterator is only
* good until we return. In such a case, extract an iterator
* to represent as much of the the output buffer as we can
- * manage. Note that the extraction might not be able to
- * allocate a sufficiently large bvec array and may shorten the
- * request.
+ * manage. Note that the extraction may shorten the request.
*/
ssize_t n = netfs_extract_iter(iter, len, INT_MAX, iocb->ki_pos,
&wreq->load_cursor.bvecq, 0);
@@ -281,8 +282,6 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *
wreq->load_cursor.bvecq->max_slots);
}
- __set_bit(NETFS_RREQ_USE_IO_ITER, &wreq->flags);
-
/* Copy the data into the bounce buffer and encrypt it. */
// TODO
diff --git a/fs/netfs/fscache_io.c b/fs/netfs/fscache_io.c
index fafa8c6bec57..e4b7888fe757 100644
--- a/fs/netfs/fscache_io.c
+++ b/fs/netfs/fscache_io.c
@@ -239,10 +239,6 @@ void __fscache_write_to_cache(struct fscache_cookie *cookie,
fscache_access_io_write) < 0)
goto abandon_free;
- ret = cres->ops->prepare_write(cres, &start, &len, len, i_size, false);
- if (ret < 0)
- goto abandon_end;
-
/* TODO: Consider clearing page bits now for space the write isn't
* covering. This is more complicated than it appears when THPs are
* taken into account.
@@ -252,8 +248,6 @@ void __fscache_write_to_cache(struct fscache_cookie *cookie,
fscache_write(cres, start, &iter, fscache_wreq_done, wreq);
return;
-abandon_end:
- return fscache_wreq_done(wreq, ret);
abandon_free:
kfree(wreq);
abandon:
diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h
index a19d614fd8d4..21a21cdfc92f 100644
--- a/fs/netfs/internal.h
+++ b/fs/netfs/internal.h
@@ -22,10 +22,9 @@
/*
* buffered_read.c
*/
-int netfs_read_query_cache(struct netfs_io_request *rreq,
- struct fscache_occupancy *occ);
-void netfs_queue_read(struct netfs_io_request *rreq,
- struct netfs_io_subrequest *subreq);
+void netfs_read_query_cache(struct netfs_io_request *rreq,
+ struct fscache_occupancy *occ);
+struct netfs_io_subrequest *netfs_alloc_read_subrequest(struct netfs_io_request *rreq);
void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
int netfs_prefetch_for_write(struct file *file, struct folio *folio,
size_t offset, size_t len);
@@ -36,6 +35,18 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio,
void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
loff_t pos, size_t copied);
+/*
+ * direct_read.c
+ */
+int netfs_prepare_unbuffered_read_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs);
+
+/*
+ * direct_write.c
+ */
+int netfs_prepare_unbuffered_write_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs);
+
/*
* main.c
*/
@@ -72,6 +83,8 @@ struct bvecq *netfs_buffer_make_space(struct netfs_io_request *rreq,
enum netfs_bvecq_trace trace);
void netfs_wake_collector(struct netfs_io_request *rreq);
void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq);
+int netfs_wait_for_in_progress_subreq(struct netfs_io_request *rreq,
+ struct netfs_io_subrequest *subreq);
void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
struct netfs_io_stream *stream);
ssize_t netfs_wait_for_read(struct netfs_io_request *rreq);
@@ -117,16 +130,53 @@ void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
/*
* read_pgpriv2.c
*/
+#ifdef CONFIG_NETFS_PGPRIV2
+int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs);
void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq);
bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq);
+static inline bool netfs_using_pgpriv2(const struct netfs_io_request *rreq)
+{
+ return test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags);
+}
+#else
+static inline int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
+{
+ return -EIO;
+}
+static inline void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio)
+{
+}
+static inline void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq)
+{
+}
+static inline bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq)
+{
+ return true;
+}
+static inline bool netfs_using_pgpriv2(const struct netfs_io_request *rreq)
+{
+ return false;
+}
+#endif
/*
* read_retry.c
*/
+int netfs_prepare_buffered_read_retry_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs);
+int netfs_reset_for_read_retry(struct netfs_io_subrequest *subreq);
void netfs_retry_reads(struct netfs_io_request *rreq);
void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq);
+/*
+ * read_single.c
+ */
+int netfs_prepare_read_single_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs);
+
/*
* stats.c
*/
@@ -198,30 +248,25 @@ void netfs_write_collection_worker(struct work_struct *work);
/*
* write_issue.c
*/
+struct netfs_writethrough;
struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
struct file *file,
loff_t start,
enum netfs_io_origin origin);
-void netfs_prepare_write(struct netfs_io_request *wreq,
- struct netfs_io_stream *stream,
- loff_t start);
-void netfs_reissue_write(struct netfs_io_stream *stream,
- struct netfs_io_subrequest *subreq);
-void netfs_issue_write(struct netfs_io_request *wreq,
- struct netfs_io_stream *stream);
-size_t netfs_advance_write(struct netfs_io_request *wreq,
- struct netfs_io_stream *stream,
- loff_t start, size_t len, bool to_eof);
-struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len);
-int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
- struct folio *folio, size_t copied, bool to_page_end,
- struct folio **writethrough_cache);
-ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
- struct folio *writethrough_cache);
+struct netfs_io_subrequest *netfs_alloc_write_subreq(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream);
+struct netfs_writethrough *netfs_begin_writethrough(struct kiocb *iocb, size_t len);
+int netfs_advance_writethrough(struct netfs_writethrough *wthru,
+ struct writeback_control *wbc,
+ struct folio *folio, size_t copied, bool to_page_end);
+ssize_t netfs_end_writethrough(struct netfs_writethrough *wthru,
+ struct writeback_control *wbc);
/*
* write_retry.c
*/
+int netfs_prepare_write_retry_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs);
void netfs_retry_writes(struct netfs_io_request *wreq);
/*
@@ -316,6 +361,25 @@ static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subreque
return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
}
+/*
+ * Indicate that we've generated and queued all the subrequests we're going to.
+ */
+static inline void netfs_all_subreqs_queued(struct netfs_io_request *rreq)
+{
+ smp_wmb(); /* Write lists before ALL_QUEUED. */
+ set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+ trace_netfs_rreq(rreq, netfs_rreq_trace_all_queued);
+}
+
+/*
+ * Query if all subrequests are queued.
+ */
+static inline bool netfs_are_all_subreqs_queued(const struct netfs_io_request *rreq)
+{
+ /* Read lists after ALL_QUEUED. */
+ return test_bit_acquire(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+}
+
/*
* fscache-cache.c
*/
diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c
index 3040be52c293..e29aad1da0b3 100644
--- a/fs/netfs/iterator.c
+++ b/fs/netfs/iterator.c
@@ -102,14 +102,14 @@ ssize_t netfs_extract_iter(struct iov_iter *orig, size_t max_len, size_t max_pag
}
if (got == 0) {
- pr_err("extract_pages gave nothing from %zu, %zu\n",
+ pr_err("extract_pages gave nothing from %zx, %zx\n",
extracted, max_len);
ret = -EIO;
goto out;
}
if (WARN(got > max_len,
- "%s: extract_pages overrun %zd > %zu bytes\n",
+ "%s: extract_pages overrun %zx > %zx bytes\n",
__func__, got, max_len)) {
ret = -EIO;
break;
diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c
index 8fc4e5ef2152..0af45204fabc 100644
--- a/fs/netfs/misc.c
+++ b/fs/netfs/misc.c
@@ -250,6 +250,37 @@ void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq)
netfs_wake_collector(rreq);
}
+/*
+ * Wait for a subrequest to come to completion.
+ */
+int netfs_wait_for_in_progress_subreq(struct netfs_io_request *rreq,
+ struct netfs_io_subrequest *subreq)
+{
+ if (netfs_check_subreq_in_progress(subreq)) {
+ DEFINE_WAIT(myself);
+
+ trace_netfs_rreq(rreq, netfs_rreq_trace_wait_quiesce);
+ for (;;) {
+ prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
+
+ if (!netfs_check_subreq_in_progress(subreq))
+ break;
+
+ trace_netfs_sreq(subreq, netfs_sreq_trace_wait_for);
+ schedule();
+ }
+
+ trace_netfs_rreq(rreq, netfs_rreq_trace_waited_quiesce);
+ finish_wait(&rreq->waitq, &myself);
+ }
+
+ if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
+ return -EAGAIN;
+ if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
+ return subreq->error;
+ return 0;
+}
+
/*
* Wait for all outstanding I/O in a stream to quiesce.
*/
@@ -310,7 +341,7 @@ static int netfs_collect_in_app(struct netfs_io_request *rreq,
need_collect = true;
break;
}
- if (subreq || !test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags))
+ if (subreq || !netfs_are_all_subreqs_queued(rreq))
done = false;
}
@@ -380,7 +411,7 @@ static ssize_t netfs_wait_for_in_progress(struct netfs_io_request *rreq,
case NETFS_UNBUFFERED_WRITE:
break;
default:
- if (rreq->submitted < rreq->len) {
+ if (rreq->transferred < rreq->len) {
trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
ret = -EIO;
}
diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c
index 7f5187c64ae9..d4a95a462576 100644
--- a/fs/netfs/objects.c
+++ b/fs/netfs/objects.c
@@ -46,8 +46,6 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
rreq->i_size = i_size_read(inode);
rreq->debug_id = atomic_inc_return(&debug_ids);
rreq->wsize = INT_MAX;
- rreq->io_streams[0].sreq_max_len = ULONG_MAX;
- rreq->io_streams[0].sreq_max_segs = 0;
spin_lock_init(&rreq->lock);
INIT_LIST_HEAD(&rreq->io_streams[0].subrequests);
INIT_LIST_HEAD(&rreq->io_streams[1].subrequests);
@@ -134,9 +132,11 @@ static void netfs_deinit_request(struct netfs_io_request *rreq)
if (rreq->cache_resources.ops)
rreq->cache_resources.ops->end_operation(&rreq->cache_resources);
bvecq_pos_unset(&rreq->load_cursor);
- bvecq_pos_unset(&rreq->dispatch_cursor);
bvecq_pos_unset(&rreq->collect_cursor);
+ bvecq_pos_unset(&rreq->retry_cursor);
bvecq_put(rreq->spare);
+ for (int i = 0; i < NR_IO_STREAMS; i++)
+ bvecq_pos_unset(&rreq->io_streams[i].dispatch_cursor);
if (atomic_dec_and_test(&ictx->io_count))
wake_up_var(&ictx->io_count);
@@ -227,6 +227,7 @@ static void netfs_free_subrequest(struct netfs_io_subrequest *subreq)
struct netfs_io_request *rreq = subreq->rreq;
trace_netfs_sreq(subreq, netfs_sreq_trace_free);
+ WARN_ON_ONCE(!list_empty(&subreq->rreq_link));
if (rreq->netfs_ops->free_subrequest)
rreq->netfs_ops->free_subrequest(subreq);
bvecq_pos_unset(&subreq->dispatch_pos);
diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c
index fccc6c2d891e..aa7e206fccf2 100644
--- a/fs/netfs/read_collect.c
+++ b/fs/netfs/read_collect.c
@@ -36,6 +36,7 @@ static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
if (subreq->start + subreq->transferred >= subreq->rreq->i_size)
__set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
+ trace_netfs_rreq(subreq->rreq, netfs_rreq_trace_zero_unread);
}
/*
@@ -58,7 +59,7 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq,
flush_dcache_folio(folio);
folio_mark_uptodate(folio);
- if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
+ if (!netfs_using_pgpriv2(rreq)) {
finfo = netfs_folio_info(folio);
if (finfo) {
trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
@@ -258,8 +259,7 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq)
transferred = front->len;
trace_netfs_rreq(rreq, netfs_rreq_trace_set_abandon);
}
- if (front->start + transferred >= rreq->cleaned_to + fsize ||
- test_bit(NETFS_SREQ_HIT_EOF, &front->flags))
+ if (front->start + transferred >= rreq->cleaned_to + fsize)
netfs_read_unlock_folios(rreq, ¬es);
} else {
stream->collected_to = front->start + transferred;
@@ -378,31 +378,6 @@ static void netfs_rreq_assess_dio(struct netfs_io_request *rreq)
inode_dio_end(rreq->inode);
}
-/*
- * Do processing after reading a monolithic single object.
- */
-static void netfs_rreq_assess_single(struct netfs_io_request *rreq)
-{
- struct netfs_io_stream *stream = &rreq->io_streams[0];
-
- if (!rreq->error && stream->source == NETFS_DOWNLOAD_FROM_SERVER &&
- fscache_resources_valid(&rreq->cache_resources)) {
- trace_netfs_rreq(rreq, netfs_rreq_trace_dirty);
- netfs_single_mark_inode_dirty(rreq->inode);
- }
-
- if (rreq->iocb) {
- rreq->iocb->ki_pos += rreq->transferred;
- if (rreq->iocb->ki_complete) {
- trace_netfs_rreq(rreq, netfs_rreq_trace_ki_complete);
- rreq->iocb->ki_complete(
- rreq->iocb, rreq->error ? rreq->error : rreq->transferred);
- }
- }
- if (rreq->netfs_ops->done)
- rreq->netfs_ops->done(rreq);
-}
-
/*
* Perform the collection of subrequests and folios.
*
@@ -418,9 +393,8 @@ bool netfs_read_collection(struct netfs_io_request *rreq)
/* We're done when the app thread has finished posting subreqs and the
* queue is empty.
*/
- if (!test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags))
+ if (!netfs_are_all_subreqs_queued(rreq))
return false;
- smp_rmb(); /* Read ALL_QUEUED before subreq lists. */
if (!list_empty(&stream->subrequests))
return false;
@@ -438,7 +412,7 @@ bool netfs_read_collection(struct netfs_io_request *rreq)
netfs_rreq_assess_dio(rreq);
break;
case NETFS_READ_SINGLE:
- netfs_rreq_assess_single(rreq);
+ WARN_ON_ONCE(1);
break;
default:
break;
@@ -563,6 +537,11 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq)
} else if (test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags)) {
__set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
trace_netfs_sreq(subreq, netfs_sreq_trace_partial_read);
+ } else if (subreq->source == NETFS_READ_FROM_CACHE) {
+ netfs_stat(&netfs_n_rh_read_failed);
+ __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
+ subreq->error = -ENODATA;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_short);
} else {
__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
subreq->error = -ENODATA;
@@ -581,6 +560,8 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq)
if (unlikely(subreq->error < 0)) {
trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read);
+ if (subreq->error == -ENOMEM)
+ set_bit(NETFS_RREQ_SAW_ENOMEM, &rreq->flags);
if (subreq->source == NETFS_READ_FROM_CACHE) {
netfs_stat(&netfs_n_rh_read_failed);
__set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
diff --git a/fs/netfs/read_pgpriv2.c b/fs/netfs/read_pgpriv2.c
index f9a0fb3e89e3..e6a60cf67e4a 100644
--- a/fs/netfs/read_pgpriv2.c
+++ b/fs/netfs/read_pgpriv2.c
@@ -13,8 +13,37 @@
#include <linux/task_io_accounting_ops.h>
#include "internal.h"
+int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
+{
+ struct netfs_io_request *creq = subreq->rreq;
+ struct netfs_io_stream *stream = &creq->io_streams[1];
+ size_t len;
+
+ bvecq_pos_set(&subreq->dispatch_pos, &stream->dispatch_cursor);
+ bvecq_pos_set(&subreq->content, &stream->dispatch_cursor);
+ len = bvecq_slice(&stream->dispatch_cursor, subreq->len, max_segs,
+ &subreq->nr_segs);
+
+ if (len < subreq->len) {
+ subreq->len = len;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
+ }
+
+ // TODO: Wait here for completion of prev subreq
+
+ stream->issue_from += subreq->len;
+ stream->buffered -= subreq->len;
+ if (stream->buffered == 0)
+ netfs_all_subreqs_queued(creq);
+ return 0;
+}
+
/*
- * [DEPRECATED] Copy a folio to the cache with PG_private_2 set.
+ * [DEPRECATED] Copy a folio to the cache with PG_private_2 set. Note that the
+ * folio won't necessarily be contiguous with the previous one as there might
+ * be a mixture of folios read from the cache and downloaded from the server
+ * (or just zeroed).
*/
static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio *folio)
{
@@ -24,7 +53,6 @@ static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio
size_t dio_size = PAGE_SIZE;
size_t fsize = folio_size(folio), flen = fsize;
loff_t fpos = folio_pos(folio), i_size;
- bool to_eof = false;
_enter("");
@@ -44,12 +72,8 @@ static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio
if (fpos + fsize > creq->i_size)
creq->i_size = i_size;
- if (flen > i_size - fpos) {
+ if (flen > i_size - fpos)
flen = i_size - fpos;
- to_eof = true;
- } else if (flen == i_size - fpos) {
- to_eof = true;
- }
flen = round_up(flen, dio_size);
@@ -81,37 +105,9 @@ static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio
bvecq_filled_to(queue, slot);
creq->load_cursor.slot = slot;
creq->load_cursor.offset = 0;
+ trace_netfs_wback(creq, folio, 0);
- bvecq_pos_nudge(&creq->dispatch_cursor);
-
- cache->submit_off = 0;
- cache->submit_len = flen;
-
- /* Attach the folio to one or more subrequests. For a big folio, we
- * could end up with thousands of subrequests if the wsize is small -
- * but we might need to wait during the creation of subrequests for
- * network resources (eg. SMB credits).
- */
- do {
- ssize_t part;
-
- creq->dispatch_cursor.offset = cache->submit_off;
-
- atomic64_set(&creq->issued_to, fpos + cache->submit_off);
- part = netfs_advance_write(creq, cache, fpos + cache->submit_off,
- cache->submit_len, to_eof);
- cache->submit_off += part;
- if (part > cache->submit_len)
- cache->submit_len = 0;
- else
- cache->submit_len -= part;
- } while (cache->submit_len > 0);
-
- bvecq_pos_step(&creq->dispatch_cursor);
- atomic64_set(&creq->issued_to, fpos + fsize);
-
- if (flen < fsize)
- netfs_issue_write(creq, cache);
+ cache->buffered += flen;
}
/*
@@ -121,6 +117,7 @@ static struct netfs_io_request *netfs_pgpriv2_begin_copy_to_cache(
struct netfs_io_request *rreq, struct folio *folio)
{
struct netfs_io_request *creq;
+ struct netfs_io_stream *cache;
if (!fscache_resources_valid(&rreq->cache_resources))
goto cancel;
@@ -130,12 +127,15 @@ static struct netfs_io_request *netfs_pgpriv2_begin_copy_to_cache(
if (IS_ERR(creq))
goto cancel;
- if (!creq->io_streams[1].avail)
+ cache = &creq->io_streams[1];
+ if (!cache->avail)
goto cancel_put;
- bvecq_buffer_init(&creq->load_cursor, GFP_KERNEL);
- bvecq_pos_set(&creq->dispatch_cursor, &creq->load_cursor);
- bvecq_pos_set(&creq->collect_cursor, &creq->dispatch_cursor);
+ if (bvecq_buffer_init(&creq->load_cursor, GFP_KERNEL) < 0)
+ goto cancel_put;
+
+ bvecq_pos_set(&cache->dispatch_cursor, &creq->load_cursor);
+ bvecq_pos_set(&creq->collect_cursor, &creq->load_cursor);
__set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &creq->flags);
trace_netfs_copy2cache(rreq, creq);
@@ -178,19 +178,44 @@ void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *fo
netfs_pgpriv2_copy_folio(creq, folio);
}
+/*
+ * Issue all pending writes on the cache stream.
+ */
+static int netfs_pgpriv2_issue_stream(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream)
+{
+ int ret;
+
+ atomic64_set_release(&stream->issued_to, wreq->start);
+
+ do {
+ struct netfs_io_subrequest *subreq;
+
+ subreq = netfs_alloc_write_subreq(wreq, stream);
+ if (!subreq)
+ return -ENOMEM;
+
+ stream->issue_write(subreq);
+ if (test_bit(NETFS_RREQ_SAW_ENOMEM, &wreq->flags))
+ return -ENOMEM;
+
+ } while (stream->buffered > 0);
+
+ return ret;
+}
+
/*
* [DEPRECATED] End writing to the cache, flushing out any outstanding writes.
*/
void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq)
{
struct netfs_io_request *creq = rreq->copy_to_cache;
+ struct netfs_io_stream *stream = &creq->io_streams[1];
if (IS_ERR_OR_NULL(creq))
return;
- netfs_issue_write(creq, &creq->io_streams[1]);
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &creq->flags);
+ netfs_pgpriv2_issue_stream(creq, stream);
trace_netfs_rreq(rreq, netfs_rreq_trace_end_copy_to_cache);
if (list_empty_careful(&creq->io_streams[1].subrequests))
netfs_wake_collector(creq);
diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c
index c45aef8dc03c..a5cd6e20cae1 100644
--- a/fs/netfs/read_retry.c
+++ b/fs/netfs/read_retry.c
@@ -9,20 +9,55 @@
#include <linux/slab.h>
#include "internal.h"
-static void netfs_reissue_read(struct netfs_io_request *rreq,
- struct netfs_io_subrequest *subreq)
+/*
+ * Prepare the I/O buffer on a buffered read subrequest for the filesystem to
+ * use as a bvec queue.
+ */
+int netfs_prepare_buffered_read_retry_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
{
- bvecq_pos_unset(&subreq->content);
+ struct netfs_io_request *rreq = subreq->rreq;
+ size_t len;
+
+ bvecq_pos_set(&subreq->dispatch_pos, &rreq->retry_cursor);
bvecq_pos_set(&subreq->content, &subreq->dispatch_pos);
- iov_iter_bvec_queue(&subreq->io_iter, ITER_DEST, subreq->content.bvecq,
- subreq->content.slot, subreq->content.offset, subreq->len);
- iov_iter_advance(&subreq->io_iter, subreq->transferred);
+ len = bvecq_slice(&rreq->retry_cursor, subreq->len, max_segs,
+ &subreq->nr_segs);
+ if (len < subreq->len) {
+ subreq->len = len;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
+ }
+ rreq->retry_buffered -= subreq->len;
+ rreq->retry_start += subreq->len;
+ return 0;
+}
- subreq->error = 0;
+/*
+ * Reset the state of the subrequest and discard any buffering so that we can
+ * retry (where this may include sending it to the server instead of the
+ * cache).
+ */
+int netfs_reset_for_read_retry(struct netfs_io_subrequest *subreq)
+{
+ trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
+
+ if (subreq->retry_count > 3) {
+ trace_netfs_sreq(subreq, netfs_sreq_trace_too_many_retries);
+ return subreq->error;
+ }
+
+ subreq->retry_count++;
__clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
+ __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
+ __clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
- netfs_stat(&netfs_n_rh_retry_read_subreq);
- subreq->rreq->netfs_ops->issue_read(subreq);
+ bvecq_pos_unset(&subreq->content);
+ bvecq_pos_unset(&subreq->dispatch_pos);
+ subreq->error = 0;
+ subreq->transferred = 0;
+ netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
+ netfs_stat(&netfs_n_wh_retry_write_subreq);
+ return 0;
}
/*
@@ -33,8 +68,8 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
{
struct netfs_io_subrequest *subreq;
struct netfs_io_stream *stream = &rreq->io_streams[0];
- struct bvecq_pos dispatch_cursor = {};
struct list_head *next;
+ int ret;
_enter("R=%x", rreq->debug_id);
@@ -44,46 +79,18 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
if (rreq->netfs_ops->retry_request)
rreq->netfs_ops->retry_request(rreq, NULL);
- /* If there's no renegotiation to do, just resend each retryable subreq
- * up to the first permanently failed one.
- */
- if (!rreq->netfs_ops->prepare_read &&
- !rreq->cache_resources.ops) {
- list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
- if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
- break;
- if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
- subreq->retry_count++;
- netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
- netfs_reissue_read(rreq, subreq);
- }
- }
- return;
- }
-
- /* Okay, we need to renegotiate all the download requests and flip any
- * failed cache reads over to being download requests and negotiate
- * those also. All fully successful subreqs have been removed from the
- * list and any spare data from those has been donated.
- *
- * What we do is decant the list and rebuild it one subreq at a time so
- * that we don't end up with donations jumping over a gap we're busy
- * populating with smaller subrequests. In the event that the subreq
- * we just launched finishes before we insert the next subreq, it'll
- * fill in rreq->prev_donated instead.
- *
- * Note: Alternatively, we could split the tail subrequest right before
- * we reissue it and fix up the donations under lock.
+ /* Renegotiate all the download requests and flip any failed cache
+ * reads over to being download requests and negotiate those also.
*/
next = stream->subrequests.next;
do {
struct netfs_io_subrequest *from, *to, *tmp;
- unsigned long long start, len;
- size_t part;
- bool boundary = false, subreq_superfluous = false;
+ unsigned long long start;
+ size_t len;
+ bool subreq_superfluous = false;
- bvecq_pos_unset(&dispatch_cursor);
+ bvecq_pos_unset(&rreq->retry_cursor);
/* Go through the subreqs and find the next span of contiguous
* buffer that we then rejig (cifs, for example, needs the
@@ -98,8 +105,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
rreq->debug_id, from->debug_index,
from->start, from->transferred, from->len);
- if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
- !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) {
+ if (!test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) {
subreq = from;
goto abandon;
}
@@ -108,20 +114,21 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
subreq = list_entry(next, struct netfs_io_subrequest, rreq_link);
if (subreq->start != start + len ||
subreq->transferred > 0 ||
- test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) ||
!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
break;
to = subreq;
len += to->len;
}
- _debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
+ _debug(" - range: %llx-%llx %zx", start, start + len - 1, len);
/* Determine the set of buffers we're going to use. Each
- * subreq gets a subset of a single overall contiguous buffer.
+ * subreq takes a subset of a single overall contiguous buffer.
*/
- bvecq_pos_transfer(&dispatch_cursor, &from->dispatch_pos);
- bvecq_pos_advance(&dispatch_cursor, from->transferred);
+ bvecq_pos_transfer(&rreq->retry_cursor, &from->dispatch_pos);
+ bvecq_pos_advance(&rreq->retry_cursor, from->transferred);
+ rreq->retry_start = start;
+ rreq->retry_buffered = len;
from->transferred = 0;
/* Work through the sublist. The chain of buffers we're going
@@ -130,51 +137,25 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
*/
subreq = from;
list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
- if (!len) {
+ if (rreq->retry_buffered == 0) {
subreq_superfluous = true;
break;
}
subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
- subreq->start = start;
- subreq->len = len;
- __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
- __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
- subreq->retry_count++;
- subreq->transferred = 0;
+ subreq->start = rreq->retry_start;
+ subreq->len = rreq->retry_buffered;
- bvecq_pos_unset(&subreq->content);
- bvecq_pos_unset(&subreq->dispatch_pos);
- bvecq_pos_set(&subreq->dispatch_pos, &dispatch_cursor);
-
- trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
-
- /* Renegotiate max_len (rsize) */
- stream->sreq_max_len = len;
- stream->sreq_max_segs = INT_MAX;
- if (rreq->netfs_ops->prepare_read &&
- rreq->netfs_ops->prepare_read(subreq) < 0) {
- trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
+ ret = netfs_reset_for_read_retry(subreq);
+ if (ret < 0) {
__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
+ rreq->error = ret;
goto abandon;
}
- part = bvecq_slice(&dispatch_cursor,
- umin(len, stream->sreq_max_len),
- stream->sreq_max_segs,
- &subreq->nr_segs);
- subreq->len = part;
-
- len -= part;
- start += part;
- if (!len) {
- if (boundary)
- __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
- } else {
- __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
- }
-
- netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
- netfs_reissue_read(rreq, subreq);
+ netfs_stat(&netfs_n_rh_download);
+ rreq->netfs_ops->issue_read(subreq);
+ if (test_bit(NETFS_RREQ_SAW_ENOMEM, &rreq->flags))
+ goto abandon_after;
if (subreq == to) {
subreq_superfluous = false;
break;
@@ -184,7 +165,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
/* If we managed to use fewer subreqs, we can discard the
* excess; if we used the same number, then we're done.
*/
- if (!len) {
+ if (rreq->retry_buffered == 0) {
if (!subreq_superfluous)
continue;
list_for_each_entry_safe_from(subreq, tmp,
@@ -202,7 +183,8 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
}
/* We ran out of subrequests, so we need to allocate some more
- * and insert them after.
+ * and insert them after. They must start with being marked
+ * for retry to switch to the retry cursor.
*/
do {
subreq = netfs_alloc_subrequest(rreq);
@@ -211,8 +193,8 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
goto abandon_after;
}
subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
- subreq->start = start;
- subreq->len = len;
+ subreq->start = rreq->retry_start;
+ subreq->len = rreq->retry_buffered;
subreq->stream_nr = stream->stream_nr;
subreq->retry_count = 1;
@@ -220,43 +202,27 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
refcount_read(&subreq->ref),
netfs_sreq_trace_new);
+ __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
+
spin_lock(&rreq->lock);
+ /* Write IN_PROGRESS before pointer to new subreq */
+ smp_wmb();
list_add(&subreq->rreq_link, &to->rreq_link);
spin_unlock(&rreq->lock);
to = subreq;
trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
- stream->sreq_max_len = umin(len, rreq->rsize);
- stream->sreq_max_segs = INT_MAX;
-
netfs_stat(&netfs_n_rh_download);
- if (rreq->netfs_ops->prepare_read(subreq) < 0) {
- trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
- __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
- goto abandon;
- }
-
- bvecq_pos_set(&subreq->dispatch_pos, &dispatch_cursor);
- part = bvecq_slice(&dispatch_cursor,
- umin(len, stream->sreq_max_len),
- stream->sreq_max_segs,
- &subreq->nr_segs);
- subreq->len = part;
-
- len -= part;
- start += part;
- if (!len && boundary) {
- __set_bit(NETFS_SREQ_BOUNDARY, &to->flags);
- boundary = false;
- }
+ rreq->netfs_ops->issue_read(subreq);
+ if (test_bit(NETFS_RREQ_SAW_ENOMEM, &rreq->flags))
+ goto abandon_after;
- netfs_reissue_read(rreq, subreq);
- } while (len);
+ } while (rreq->retry_buffered > 0);
} while (!list_is_head(next, &stream->subrequests));
out:
- bvecq_pos_unset(&dispatch_cursor);
+ bvecq_pos_unset(&rreq->retry_cursor);
return;
/* If we hit an error, fail all remaining incomplete subrequests */
@@ -322,6 +288,7 @@ void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
}
trace_netfs_folio(folio, netfs_folio_trace_abandon);
folio_unlock(folio);
+ p->bv[slot].bv_page = NULL;
}
}
}
diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c
index 98938a54810e..8237894c8fd8 100644
--- a/fs/netfs/read_single.c
+++ b/fs/netfs/read_single.c
@@ -16,6 +16,22 @@
#include <linux/netfs.h>
#include "internal.h"
+int netfs_prepare_read_single_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
+{
+ struct netfs_io_request *rreq = subreq->rreq;
+ struct netfs_io_stream *stream = &rreq->io_streams[0];
+
+ bvecq_pos_set(&subreq->dispatch_pos, &rreq->load_cursor);
+ bvecq_pos_set(&subreq->content, &subreq->dispatch_pos);
+
+ stream->buffered = 0;
+ stream->issue_from += subreq->len;
+ rreq->submitted = stream->issue_from;
+ netfs_all_subreqs_queued(rreq);
+ return 0;
+}
+
/**
* netfs_single_mark_inode_dirty - Mark a single, monolithic object inode dirty
* @inode: The inode to mark
@@ -58,17 +74,6 @@ static int netfs_single_begin_cache_read(struct netfs_io_request *rreq, struct n
return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx));
}
-static void netfs_single_read_cache(struct netfs_io_request *rreq,
- struct netfs_io_subrequest *subreq)
-{
- struct netfs_cache_resources *cres = &rreq->cache_resources;
-
- _enter("R=%08x[%x]", rreq->debug_id, subreq->debug_index);
- netfs_stat(&netfs_n_rh_read);
- cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_FAIL,
- netfs_cache_read_terminated, subreq);
-}
-
/*
* Perform a read to a buffer from the cache or the server. Only a single
* subreq is permitted as the object must be fetched in a single transaction.
@@ -84,73 +89,74 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
.cached_to[1] = ULLONG_MAX,
};
struct netfs_io_subrequest *subreq;
- int ret = 0;
+ int ret;
+
+ netfs_read_query_cache(rreq, &occ);
- subreq = netfs_alloc_subrequest(rreq);
+ subreq = netfs_alloc_read_subrequest(rreq);
if (!subreq)
return -ENOMEM;
- subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
subreq->start = 0;
subreq->len = rreq->len;
- bvecq_pos_set(&subreq->dispatch_pos, &rreq->dispatch_cursor);
- bvecq_pos_set(&subreq->content, &rreq->dispatch_cursor);
-
- iov_iter_bvec_queue(&subreq->io_iter, ITER_DEST, subreq->content.bvecq,
- subreq->content.slot, subreq->content.offset, subreq->len);
-
- netfs_queue_read(rreq, subreq);
+ trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
/* Try to use the cache if the cache content matches the size of the
* remote file.
*/
- netfs_read_query_cache(rreq, &occ);
if (occ.cached_from[0] == 0 &&
- occ.cached_to[0] == rreq->len)
+ occ.cached_to[0] == rreq->len) {
+ struct netfs_cache_resources *cres = &rreq->cache_resources;
+
subreq->source = NETFS_READ_FROM_CACHE;
+ netfs_stat(&netfs_n_rh_read);
+ cres->ops->issue_read(subreq);
+ ret = netfs_wait_for_in_progress_subreq(rreq, subreq);
+ if (ret == -ENOMEM)
+ goto cancel;
+ if (ret == 0)
+ goto success;
+
+ /* Didn't manage to retrieve from the cache, so toss it to the
+ * server instead.
+ */
+ if (netfs_reset_for_read_retry(subreq) < 0)
+ goto cancel;
+ }
- switch (subreq->source) {
- case NETFS_DOWNLOAD_FROM_SERVER:
- netfs_stat(&netfs_n_rh_download);
- if (rreq->netfs_ops->prepare_read) {
- ret = rreq->netfs_ops->prepare_read(subreq);
- if (ret < 0)
- goto cancel;
- }
+ __set_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags);
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+ /* Try to send it to the cache. */
+ for (;;) {
+ subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
+ netfs_stat(&netfs_n_rh_download);
rreq->netfs_ops->issue_read(subreq);
- rreq->submitted += subreq->len;
- break;
- case NETFS_READ_FROM_CACHE:
- if (rreq->cache_resources.ops->prepare_read) {
- ret = rreq->cache_resources.ops->prepare_read(subreq);
- if (ret < 0)
- goto cancel;
- }
-
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
- trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
- netfs_single_read_cache(rreq, subreq);
- rreq->submitted += subreq->len;
- ret = 0;
- break;
- default:
- pr_warn("Unexpected single-read source %u\n", subreq->source);
- WARN_ON_ONCE(true);
- ret = -EIO;
- goto cancel;
+ ret = netfs_wait_for_in_progress_subreq(rreq, subreq);
+ if (ret == 0)
+ goto success;
+ if (ret == -ENOMEM)
+ goto cancel;
+ if (ret != -EAGAIN)
+ goto failed;
+ if (netfs_reset_for_read_retry(subreq) < 0)
+ goto cancel;
}
- return ret;
+success:
+ rreq->transferred = subreq->transferred;
+ list_del_init(&subreq->rreq_link);
+ netfs_put_subrequest(subreq, netfs_sreq_trace_put_consumed);
+ return 0;
cancel:
- netfs_cancel_read(subreq, ret);
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
- netfs_wake_collector(rreq);
+ rreq->error = ret;
+ list_del_init(&subreq->rreq_link);
+ netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
+ return ret;
+failed:
+ rreq->error = ret;
+ list_del_init(&subreq->rreq_link);
+ netfs_put_subrequest(subreq, netfs_sreq_trace_put_failed);
return ret;
}
@@ -182,7 +188,7 @@ ssize_t netfs_read_single(struct inode *inode, struct file *file, struct iov_ite
if (IS_ERR(rreq))
return PTR_ERR(rreq);
- ret = netfs_extract_iter(iter, rreq->len, INT_MAX, 0, &rreq->dispatch_cursor.bvecq, 0);
+ ret = netfs_extract_iter(iter, rreq->len, INT_MAX, 0, &rreq->load_cursor.bvecq, 0);
if (ret < 0)
goto cleanup_free;
@@ -193,9 +199,29 @@ ssize_t netfs_read_single(struct inode *inode, struct file *file, struct iov_ite
netfs_stat(&netfs_n_rh_read_single);
trace_netfs_read(rreq, 0, rreq->len, netfs_read_trace_read_single);
- netfs_single_dispatch_read(rreq);
+ ret = netfs_single_dispatch_read(rreq);
+
+ trace_netfs_rreq(rreq, netfs_rreq_trace_complete);
+ if (ret == 0) {
+ task_io_account_read(rreq->transferred);
+
+ if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags) &&
+ fscache_resources_valid(&rreq->cache_resources)) {
+ trace_netfs_rreq(rreq, netfs_rreq_trace_dirty);
+ netfs_single_mark_inode_dirty(rreq->inode);
+ }
+ ret = rreq->transferred;
+ }
+
+ if (rreq->netfs_ops->done)
+ rreq->netfs_ops->done(rreq);
+
+ netfs_wake_rreq_flag(rreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip);
+ /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */
+ netfs_put_request(rreq, netfs_rreq_trace_put_work_ip);
+
+ trace_netfs_rreq(rreq, netfs_rreq_trace_done);
- ret = netfs_wait_for_read(rreq);
netfs_put_request(rreq, netfs_rreq_trace_put_return);
return ret;
diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c
index a91b34cf01f5..6dc656fdecd1 100644
--- a/fs/netfs/write_collect.c
+++ b/fs/netfs/write_collect.c
@@ -28,8 +28,8 @@ static void netfs_dump_request(const struct netfs_io_request *rreq)
rreq->origin, rreq->error);
pr_err(" st=%llx tsl=%zx/%llx/%llx\n",
rreq->start, rreq->transferred, rreq->submitted, rreq->len);
- pr_err(" cci=%llx/%llx/%llx\n",
- rreq->cleaned_to, rreq->collected_to, atomic64_read(&rreq->issued_to));
+ pr_err(" cci=%llx/%llx\n",
+ rreq->cleaned_to, rreq->collected_to);
pr_err(" iw=%pSR\n", rreq->netfs_ops->issue_write);
for (int i = 0; i < NR_IO_STREAMS; i++) {
const struct netfs_io_subrequest *sreq;
@@ -38,8 +38,9 @@ static void netfs_dump_request(const struct netfs_io_request *rreq)
pr_err(" str[%x] s=%x e=%d acnf=%u,%u,%u,%u\n",
s->stream_nr, s->source, s->error,
s->avail, s->active, s->need_retry, s->failed);
- pr_err(" str[%x] ct=%llx t=%zx\n",
- s->stream_nr, s->collected_to, s->transferred);
+ pr_err(" str[%x] it=%llx ct=%llx t=%zx\n",
+ s->stream_nr, atomic64_read(&s->issued_to),
+ s->collected_to, s->transferred);
list_for_each_entry(sreq, &s->subrequests, rreq_link) {
pr_err(" sreq[%x:%x] sc=%u s=%llx t=%zx/%zx r=%d f=%lx\n",
sreq->stream_nr, sreq->debug_index, sreq->source,
@@ -56,7 +57,7 @@ static void netfs_dump_request(const struct netfs_io_request *rreq)
*/
int netfs_folio_written_back(struct folio *folio)
{
- enum netfs_folio_trace why = netfs_folio_trace_clear;
+ enum netfs_folio_trace why = netfs_folio_trace_endwb;
struct inode *inode = folio_inode(folio);
struct netfs_inode *ictx = netfs_inode(inode);
struct netfs_folio *finfo;
@@ -79,13 +80,13 @@ int netfs_folio_written_back(struct folio *folio)
group = finfo->netfs_group;
gcount++;
kfree(finfo);
- why = netfs_folio_trace_clear_s;
+ why = netfs_folio_trace_endwb_s;
goto end_wb;
}
if ((group = netfs_folio_group(folio))) {
if (group == NETFS_FOLIO_COPY_TO_CACHE) {
- why = netfs_folio_trace_clear_cc;
+ why = netfs_folio_trace_endwb_cc;
folio_detach_private(folio);
goto end_wb;
}
@@ -98,7 +99,7 @@ int netfs_folio_written_back(struct folio *folio)
if (!folio_test_dirty(folio)) {
folio_detach_private(folio);
gcount++;
- why = netfs_folio_trace_clear_g;
+ why = netfs_folio_trace_endwb_g;
}
}
@@ -150,6 +151,12 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq,
slot = wreq->collect_cursor.slot;
}
+ if (!bvecq->bv[slot].bv_page) {
+ WARN_ONCE(1, "R=%08x slot already cleared?\n", wreq->debug_id);
+ fsize = bvecq->bv[slot].bv_len;
+ goto skip;
+ }
+
folio = page_folio(bvecq->bv[slot].bv_page);
if (WARN_ONCE(!folio_test_writeback(folio),
"R=%08x: folio %lx is not under writeback\n",
@@ -174,6 +181,7 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq,
*notes |= MADE_PROGRESS;
bvecq->bv[slot].bv_page = NULL;
+ skip:
slot++;
if (fpos + fsize >= collected_to)
break;
@@ -224,9 +232,7 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
trace_netfs_rreq(wreq, netfs_rreq_trace_collect);
reassess_streams:
- /* Order reading the issued_to point before reading the queue it refers to. */
- issued_to = atomic64_read_acquire(&wreq->issued_to);
- smp_rmb();
+ issued_to = ULLONG_MAX;
collected_to = ULLONG_MAX;
if (wreq->origin == NETFS_WRITEBACK ||
wreq->origin == NETFS_WRITETHROUGH ||
@@ -241,17 +247,30 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
* to the tail whilst we're doing this.
*/
for (s = 0; s < NR_IO_STREAMS; s++) {
+ unsigned long long s_issued_to;
+
stream = &wreq->io_streams[s];
- /* Read active flag before list pointers */
+ /* Read active flag before issued_to */
if (!smp_load_acquire(&stream->active))
continue;
- front = list_first_entry_or_null_acquire(&stream->subrequests,
- struct netfs_io_subrequest, rreq_link);
- /* Read first subreq pointer before IN_PROGRESS flag. */
-
- while (front) {
+ for (;;) {
bool cancelled;
+
+ /* Order reading the issued_to point before reading the
+ * queue it refers to.
+ */
+ s_issued_to = atomic64_read_acquire(&stream->issued_to);
+ if (s_issued_to < issued_to)
+ issued_to = s_issued_to;
+
+ front = list_first_entry_or_null_acquire(&stream->subrequests,
+ struct netfs_io_subrequest,
+ rreq_link);
+ /* Read first subreq pointer before IN_PROGRESS flag. */
+ if (!front)
+ break;
+
trace_netfs_collect_sreq(wreq, front);
//_debug("sreq [%x] %llx %zx/%zx",
// front->debug_index, front->start, front->transferred, front->len);
@@ -420,9 +439,8 @@ bool netfs_write_collection(struct netfs_io_request *wreq)
/* We're done when the app thread has finished posting subreqs and all
* the queues in all the streams are empty.
*/
- if (!test_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags))
+ if (!netfs_are_all_subreqs_queued(wreq))
return false;
- smp_rmb(); /* Read ALL_QUEUED before lists. */
transferred = LONG_MAX;
for (s = 0; s < NR_IO_STREAMS; s++) {
@@ -533,6 +551,8 @@ void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error)
if (IS_ERR_VALUE(transferred_or_error)) {
subreq->error = transferred_or_error;
+ if (transferred_or_error == -ENOMEM)
+ set_bit(NETFS_RREQ_SAW_ENOMEM, &wreq->flags);
switch (subreq->source) {
case NETFS_WRITE_TO_CACHE:
diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c
index 986a578fd0da..66f5daf9d8cf 100644
--- a/fs/netfs/write_issue.c
+++ b/fs/netfs/write_issue.c
@@ -36,6 +36,39 @@
#include <linux/pagemap.h>
#include "internal.h"
+#define NOTE_UPLOAD_AVAIL 0x001 /* Upload is available */
+#define NOTE_CACHE_AVAIL 0x002 /* Local cache is available */
+#define NOTE_CACHE_COPY 0x004 /* Copy folio to cache */
+#define NOTE_UPLOAD 0x008 /* Upload folio to server */
+#define NOTE_UPLOAD_STARTED 0x010 /* Upload started */
+#define NOTE_STREAMW 0x020 /* Folio is from a streaming write */
+#define NOTE_DISCONTIG_BEFORE 0x040 /* Folio discontiguous with the previous folio */
+#define NOTE_DISCONTIG_AFTER 0x080 /* Folio discontiguous with the next folio */
+#define NOTE_TO_EOF 0x100 /* Data in folio ends at EOF */
+#define NOTE_FLUSH_ANYWAY 0x200 /* Flush data, even if not hit estimated limit */
+
+#define NOTES__KEEP_MASK (NOTE_UPLOAD_AVAIL | NOTE_CACHE_AVAIL | NOTE_UPLOAD_STARTED)
+
+struct netfs_wb_params {
+ unsigned long long last_end; /* End file pos of previous folio */
+ unsigned long long folio_start; /* File pos of folio */
+ unsigned int folio_len; /* Length of folio */
+ unsigned int dirty_offset; /* Offset of dirty region in folio */
+ unsigned int dirty_len; /* Length of dirty region in folio */
+ unsigned int notes; /* Notes on applicability */
+ struct bvecq_pos dispatch_cursor; /* Folio queue anchor for issue_at */
+ struct netfs_write_estimate estimates[2];
+};
+
+struct netfs_writethrough {
+ struct netfs_wb_params params;
+ struct netfs_io_request *wreq;
+ struct folio *in_progress;
+};
+
+static int netfs_prepare_write_single_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs);
+
/*
* Kill all dirty folios in the event of an unrecoverable error, starting with
* a locked folio we've already obtained from writeback_iter().
@@ -115,65 +148,48 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
wreq->io_streams[0].stream_nr = 0;
wreq->io_streams[0].source = NETFS_UPLOAD_TO_SERVER;
- wreq->io_streams[0].prepare_write = ictx->ops->prepare_write;
+ wreq->io_streams[0].applicable = NOTE_UPLOAD;
+ wreq->io_streams[0].estimate_write = ictx->ops->estimate_write;
wreq->io_streams[0].issue_write = ictx->ops->issue_write;
wreq->io_streams[0].collected_to = start;
wreq->io_streams[0].transferred = 0;
wreq->io_streams[1].stream_nr = 1;
wreq->io_streams[1].source = NETFS_WRITE_TO_CACHE;
+ wreq->io_streams[1].applicable = NOTE_CACHE_COPY;
wreq->io_streams[1].collected_to = start;
wreq->io_streams[1].transferred = 0;
if (fscache_resources_valid(&wreq->cache_resources)) {
wreq->io_streams[1].avail = true;
wreq->io_streams[1].active = true;
- wreq->io_streams[1].prepare_write = wreq->cache_resources.ops->prepare_write_subreq;
+ wreq->io_streams[1].estimate_write = wreq->cache_resources.ops->estimate_write;
wreq->io_streams[1].issue_write = wreq->cache_resources.ops->issue_write;
}
return wreq;
}
-/**
- * netfs_prepare_write_failed - Note write preparation failed
- * @subreq: The subrequest to mark
- *
- * Mark a subrequest to note that preparation for write failed.
- */
-void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq)
-{
- __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
- trace_netfs_sreq(subreq, netfs_sreq_trace_prep_failed);
-}
-EXPORT_SYMBOL(netfs_prepare_write_failed);
-
/*
- * Prepare a write subrequest. We need to allocate a new subrequest
- * if we don't have one.
+ * Allocate and prepare a write subrequest.
*/
-void netfs_prepare_write(struct netfs_io_request *wreq,
- struct netfs_io_stream *stream,
- loff_t start)
+struct netfs_io_subrequest *netfs_alloc_write_subreq(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream)
{
struct netfs_io_subrequest *subreq;
subreq = netfs_alloc_subrequest(wreq);
subreq->source = stream->source;
- subreq->start = start;
+ subreq->start = stream->issue_from;
+ subreq->len = stream->buffered;
subreq->stream_nr = stream->stream_nr;
- bvecq_pos_set(&subreq->dispatch_pos, &wreq->dispatch_cursor);
-
_enter("R=%x[%x]", wreq->debug_id, subreq->debug_index);
trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
- stream->sreq_max_len = UINT_MAX;
- stream->sreq_max_segs = INT_MAX;
switch (stream->source) {
case NETFS_UPLOAD_TO_SERVER:
netfs_stat(&netfs_n_wh_upload);
- stream->sreq_max_len = wreq->wsize;
break;
case NETFS_WRITE_TO_CACHE:
netfs_stat(&netfs_n_wh_write);
@@ -183,9 +199,6 @@ void netfs_prepare_write(struct netfs_io_request *wreq,
break;
}
- if (stream->prepare_write)
- stream->prepare_write(subreq);
-
__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
/* We add to the end of the list whilst the collector may be walking
@@ -195,99 +208,46 @@ void netfs_prepare_write(struct netfs_io_request *wreq,
spin_lock(&wreq->lock);
/* Write IN_PROGRESS before pointer to new subreq */
list_add_tail_release(&subreq->rreq_link, &stream->subrequests);
- if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
- if (!stream->active) {
- stream->collected_to = subreq->start;
- /* Write list pointers before active flag */
- smp_store_release(&stream->active, true);
- }
- }
+ if (list_is_first(&subreq->rreq_link, &stream->subrequests) &&
+ stream->collected_to == 0)
+ stream->collected_to = subreq->start;
spin_unlock(&wreq->lock);
-
- stream->construct = subreq;
+ return subreq;
}
/*
- * Set the I/O iterator for the filesystem/cache to use and dispatch the I/O
- * operation. The operation may be asynchronous and should call
- * netfs_write_subrequest_terminated() when complete.
+ * Prepare the buffer for a buffered write.
*/
-static void netfs_do_issue_write(struct netfs_io_stream *stream,
- struct netfs_io_subrequest *subreq)
+static int netfs_prepare_buffered_write_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
{
struct netfs_io_request *wreq = subreq->rreq;
+ struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
+ ssize_t len;
- _enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len);
+ _enter("%zx,{,%u,%u},%u",
+ subreq->len, stream->dispatch_cursor.slot, stream->dispatch_cursor.offset, max_segs);
- if (stream->source == NETFS_WRITE_TO_CACHE &&
- unlikely(test_bit(NETFS_RREQ_CACHE_STOP, &wreq->flags))) {
- size_t dio_size = wreq->cache_resources.dio_size;
- size_t len, disp;
-
- disp = subreq->start & (dio_size - 1);
- len = round_up(subreq->len + disp, dio_size);
-
- subreq->start -= disp;
- subreq->len = len;
-
- __set_bit(NETFS_SREQ_CANCELLED, &subreq->flags);
- return netfs_write_subrequest_terminated(subreq, subreq->len);
- }
-
- if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
- return netfs_write_subrequest_terminated(subreq, subreq->error);
-
- trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
- stream->issue_write(subreq);
-}
-
-void netfs_reissue_write(struct netfs_io_stream *stream,
- struct netfs_io_subrequest *subreq)
-{
- // TODO: Use encrypted buffer
- bvecq_pos_set(&subreq->content, &subreq->dispatch_pos);
- iov_iter_bvec_queue(&subreq->io_iter, ITER_SOURCE,
- subreq->content.bvecq, subreq->content.slot,
- subreq->content.offset,
- subreq->len);
- iov_iter_advance(&subreq->io_iter, subreq->transferred);
-
- subreq->retry_count++;
- subreq->error = 0;
- __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
- __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
- netfs_stat(&netfs_n_wh_retry_write_subreq);
- netfs_do_issue_write(stream, subreq);
-}
-
-void netfs_issue_write(struct netfs_io_request *wreq,
- struct netfs_io_stream *stream)
-{
- struct netfs_io_subrequest *subreq = stream->construct;
-
- if (!subreq)
- return;
+ bvecq_pos_set(&subreq->dispatch_pos, &stream->dispatch_cursor);
/* If we have a write to the cache, we need to round out the first and
* last entries (only those as the data will be on virtually contiguous
* folios) to cache DIO boundaries.
*/
if (subreq->source == NETFS_WRITE_TO_CACHE) {
- struct bvecq_pos tmp_pos;
struct bio_vec *bv;
struct bvecq *bq;
size_t dio_size = wreq->cache_resources.dio_size;
- size_t disp, len;
- int ret;
+ size_t disp, dlen;
- bvecq_pos_set(&tmp_pos, &subreq->dispatch_pos);
- ret = bvecq_extract(&tmp_pos, subreq->len, INT_MAX, &subreq->content.bvecq);
- bvecq_pos_unset(&tmp_pos);
- if (ret < 0) {
- netfs_write_subrequest_terminated(subreq, -ENOMEM);
- return;
- }
+ len = bvecq_extract(&stream->dispatch_cursor, subreq->len, max_segs,
+ &subreq->content.bvecq);
+ if (len < 0)
+ return -ENOMEM;
+
+ _debug("extract %zx/%zx", len, subreq->len);
+ subreq->len = len;
/* Round the first entry down. We should be able to get away
* with this as this path only happens for buffered reads and
@@ -315,88 +275,292 @@ void netfs_issue_write(struct netfs_io_request *wreq,
while (bq->next)
bq = bq->next;
bv = &bq->bv[bq->nr_slots - 1];
- len = round_up(bv->bv_len, dio_size);
- if (len > bv->bv_len) {
- subreq->len += len - bv->bv_len;
- bv->bv_len = len;
+ dlen = round_up(bv->bv_len, dio_size);
+ if (dlen > bv->bv_len) {
+ subreq->len += dlen - bv->bv_len;
+ bv->bv_len = dlen;
}
} else {
- bvecq_pos_set(&subreq->content, &subreq->dispatch_pos);
+ bvecq_pos_set(&subreq->content, &stream->dispatch_cursor);
+ len = bvecq_slice(&stream->dispatch_cursor, subreq->len, max_segs,
+ &subreq->nr_segs);
+
+ if (len < subreq->len) {
+ subreq->len = len;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
+ }
+ }
+
+ stream->issue_from += len;
+ stream->buffered -= len;
+ if (stream->buffered == 0) {
+ stream->buffering = false;
+ bvecq_pos_unset(&stream->dispatch_cursor);
+ }
+ /* Order loading the queue before updating the issue_to point */
+ atomic64_set_release(&stream->issued_to, stream->issue_from);
+ return 0;
+}
+
+/**
+ * netfs_prepare_write_buffer - Get the buffer for a subrequest
+ * @subreq: The subrequest to get the buffer for
+ * @max_segs: Maximum number of segments in buffer (or INT_MAX)
+ *
+ * Extract a slice of buffer from the stream and attach it to the subrequest as
+ * a bio_vec queue. The maximum amount of data attached is set by
+ * @subreq->len, but this may be shortened if @max_segs would be exceeded.
+ */
+int netfs_prepare_write_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
+{
+ struct netfs_io_request *rreq = subreq->rreq;
+
+ switch (rreq->origin) {
+ case NETFS_WRITEBACK:
+ case NETFS_WRITETHROUGH:
+ if (test_bit(NETFS_RREQ_RETRYING, &rreq->flags))
+ return netfs_prepare_write_retry_buffer(subreq, max_segs);
+ return netfs_prepare_buffered_write_buffer(subreq, max_segs);
+
+ case NETFS_UNBUFFERED_WRITE:
+ case NETFS_DIO_WRITE:
+ return netfs_prepare_unbuffered_write_buffer(subreq, max_segs);
+
+ case NETFS_WRITEBACK_SINGLE:
+ return netfs_prepare_write_single_buffer(subreq, max_segs);
+
+ case NETFS_PGPRIV2_COPY_TO_CACHE:
+ return netfs_prepare_pgpriv2_write_buffer(subreq, max_segs);
+
+ default:
+ WARN_ON_ONCE(1);
+ return -EIO;
}
+}
+EXPORT_SYMBOL(netfs_prepare_write_buffer);
+
+/*
+ * Issue writes for a stream.
+ */
+static int netfs_issue_writes(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_wb_params *params)
+{
+ struct netfs_write_estimate *estimate = ¶ms->estimates[stream->stream_nr];
+
+ for (;;) {
+ struct netfs_io_subrequest *subreq;
+
+ if (test_bit(NETFS_RREQ_PAUSE, &wreq->flags))
+ netfs_wait_for_paused_write(wreq);
+
+ subreq = netfs_alloc_write_subreq(wreq, stream);
+ if (!subreq)
+ return -ENOMEM;
+
+ if (stream->source == NETFS_WRITE_TO_CACHE &&
+ unlikely(test_bit(NETFS_RREQ_CACHE_STOP, &wreq->flags))) {
+ size_t dio_size = wreq->cache_resources.dio_size;
+ size_t len, disp;
+
+ disp = subreq->start & (dio_size - 1);
+ len = round_up(subreq->len + disp, dio_size);
+
+ subreq->start -= disp;
+ subreq->len = len;
+
+ stream->issue_from = subreq->start + len;
+ stream->buffered = 0;
+ stream->buffering = false;
+ bvecq_pos_unset(&stream->dispatch_cursor);
+ estimate->issue_at = subreq->start + len + 16 * 1024 * 1024;
+ estimate->max_segs = INT_MAX;
+ __set_bit(NETFS_SREQ_CANCELLED, &subreq->flags);
+ netfs_write_subrequest_terminated(subreq, len);
+ return 0;
+ }
+
+ stream->issue_write(subreq);
+ if (test_bit(NETFS_RREQ_SAW_ENOMEM, &wreq->flags))
+ return -ENOMEM;
- iov_iter_bvec_queue(&subreq->io_iter, ITER_SOURCE,
- subreq->content.bvecq, subreq->content.slot,
- subreq->content.offset,
- subreq->len);
+ if (stream->buffered == 0) {
+ if (stream->stream_nr == 0)
+ params->notes &= ~NOTE_UPLOAD_STARTED;
+ return 0;
+ }
- stream->construct = NULL;
- netfs_do_issue_write(stream, subreq);
+ if (!(params->notes & NOTE_FLUSH_ANYWAY)) {
+ estimate->issue_at = ULLONG_MAX;
+ estimate->max_segs = INT_MAX;
+ stream->estimate_write(wreq, stream, estimate);
+ if (stream->issue_from + stream->buffered < estimate->issue_at &&
+ estimate->max_segs > 0)
+ return 0;
+ }
+ }
}
/*
- * Add data to the write subrequest, dispatching each as we fill it up or if it
- * is discontiguous with the previous. We only fill one part at a time so that
- * we can avoid overrunning the credits obtained (cifs) and try to parallelise
- * content-crypto preparation with network writes.
+ * Issue pending writes on a stream.
*/
-size_t netfs_advance_write(struct netfs_io_request *wreq,
- struct netfs_io_stream *stream,
- loff_t start, size_t len, bool to_eof)
+static int netfs_issue_stream(struct netfs_io_request *wreq,
+ struct netfs_wb_params *params, int s)
{
- struct netfs_io_subrequest *subreq = stream->construct;
- size_t part;
+ struct netfs_write_estimate *estimate = ¶ms->estimates[s];
+ struct netfs_io_stream *stream = &wreq->io_streams[s];
+ unsigned long long dirty_start;
+ bool discontig_before = params->notes & NOTE_DISCONTIG_BEFORE;
+ int ret;
+
+ _enter("%x", params->notes);
- if (!stream->avail) {
- _leave("no write");
- return len;
+ /* If the current folio doesn't contribute to this stream, see if we
+ * need to flush it.
+ */
+ if (!(params->notes & stream->applicable)) {
+ if (!stream->buffering) {
+ atomic64_set_release(&stream->issued_to,
+ params->folio_start + params->folio_len);
+ return 0;
+ }
+ discontig_before = true;
}
- _enter("R=%x[%x]", wreq->debug_id, subreq ? subreq->debug_index : 0);
+ /* Issue writes if we meet a discontiguity before the current folio.
+ * Even if the filesystem can do sparse/vectored writes, we still
+ * generate a subreq per contiguous region rather than generating
+ * separate extent lists.
+ */
+ if (stream->buffering && discontig_before) {
+ params->notes |= NOTE_FLUSH_ANYWAY;
+ ret = netfs_issue_writes(wreq, stream, params);
+ if (ret < 0)
+ return ret;
+ stream->buffering = false;
+ params->notes &= ~NOTE_FLUSH_ANYWAY;
+ }
- if (subreq && start != subreq->start + subreq->len) {
- netfs_issue_write(wreq, stream);
- subreq = NULL;
+ if (!(params->notes & stream->applicable)) {
+ atomic64_set_release(&stream->issued_to,
+ params->folio_start + params->folio_len);
+ return 0;
}
- if (!stream->construct)
- netfs_prepare_write(wreq, stream, start);
- subreq = stream->construct;
+ /* If we're not currently buffering on this stream, we need to get an
+ * estimate of when we need to issue a write. It might be within the
+ * starting folio.
+ */
+ dirty_start = params->folio_start + params->dirty_offset;
+ if (!stream->buffering) {
+ stream->buffering = true;
+ stream->issue_from = dirty_start;
+ bvecq_pos_set(&stream->dispatch_cursor, ¶ms->dispatch_cursor);
+ estimate->issue_at = ULLONG_MAX;
+ estimate->max_segs = INT_MAX;
+ stream->estimate_write(wreq, stream, estimate);
+ }
- part = umin(stream->sreq_max_len - subreq->len, len);
- _debug("part %zx/%zx %zx/%zx", subreq->len, stream->sreq_max_len, part, len);
- subreq->len += part;
- subreq->nr_segs++;
+ stream->buffered += params->dirty_len;
+ estimate->max_segs--;
- if (subreq->len >= stream->sreq_max_len ||
- subreq->nr_segs >= stream->sreq_max_segs ||
- to_eof) {
- netfs_issue_write(wreq, stream);
- subreq = NULL;
+ /* Poke the filesystem to issue writes when we hit the limit it set or
+ * if the data ends before the end of the page.
+ */
+ if (params->notes & NOTE_DISCONTIG_AFTER)
+ params->notes |= NOTE_FLUSH_ANYWAY;
+ _debug("[%u] %llx + %zx >= %llx, %u %x",
+ s, stream->issue_from, stream->buffered, estimate->issue_at,
+ estimate->max_segs, params->notes);
+ if (stream->issue_from + stream->buffered >= estimate->issue_at ||
+ estimate->max_segs <= 0 ||
+ (params->notes & NOTE_FLUSH_ANYWAY)) {
+ ret = netfs_issue_writes(wreq, stream, params);
+ if (ret < 0)
+ return ret;
}
- return part;
+ return 0;
}
/*
- * Write some of a pending folio data back to the server.
+ * See which streams need writes issuing and issue them.
*/
-static int netfs_write_folio(struct netfs_io_request *wreq,
- struct writeback_control *wbc,
- struct folio *folio)
+static int netfs_issue_streams(struct netfs_io_request *wreq,
+ struct netfs_wb_params *params)
+{
+ int ret = 0, ret2;
+
+ _enter("%x", params->notes);
+
+ for (int s = 0; s < NR_IO_STREAMS; s++) {
+ ret2 = netfs_issue_stream(wreq, params, s);
+ if (ret2 < 0)
+ ret = ret2;
+ }
+ return ret;
+}
+
+/*
+ * End the issuing of writes, let the collector know we're done.
+ */
+static void netfs_end_issue_write(struct netfs_io_request *wreq,
+ struct netfs_wb_params *params)
+{
+ bool needs_poke = true;
+
+ params->notes |= NOTE_FLUSH_ANYWAY;
+
+ for (int s = 0; s < NR_IO_STREAMS; s++) {
+ struct netfs_io_stream *stream = &wreq->io_streams[s];
+ int ret;
+
+ if (stream->buffering) {
+ ret = netfs_issue_writes(wreq, stream, params);
+ if (ret < 0 && stream->source != NETFS_WRITE_TO_CACHE) {
+ /* Leave the error somewhere the completion
+ * path can pick it up if there isn't already
+ * another error logged.
+ */
+ cmpxchg(&wreq->error, 0, ret);
+ }
+ stream->buffering = false;
+ }
+ }
+
+ netfs_all_subreqs_queued(wreq);
+
+ for (int s = 0; s < NR_IO_STREAMS; s++) {
+ struct netfs_io_stream *stream = &wreq->io_streams[s];
+
+ if (!stream->active)
+ continue;
+ if (!list_empty(&stream->subrequests))
+ needs_poke = false;
+ }
+
+ if (needs_poke)
+ netfs_wake_collector(wreq);
+}
+
+/*
+ * Queue a folio for writeback.
+ */
+static int netfs_queue_wb_folio(struct netfs_io_request *wreq,
+ struct writeback_control *wbc,
+ struct folio *folio,
+ struct netfs_wb_params *params)
{
- struct netfs_io_stream *upload = &wreq->io_streams[0];
- struct netfs_io_stream *cache = &wreq->io_streams[1];
- struct netfs_io_stream *stream;
struct netfs_group *fgroup; /* TODO: Use this with ceph */
struct netfs_folio *finfo;
struct bvecq *queue = wreq->load_cursor.bvecq;
unsigned int slot;
size_t fsize = folio_size(folio), flen = fsize, foff = 0;
loff_t fpos = folio_pos(folio), i_size;
- bool to_eof = false, streamw = false;
- bool debug = false;
+ int ret;
- _enter("");
+ _enter("%x", params->notes);
if (!wreq->spare) {
wreq->spare = bvecq_alloc_one(BVECQ_STD_SLOTS, GFP_NOFS);
@@ -431,23 +595,36 @@ static int netfs_write_folio(struct netfs_io_request *wreq,
if (finfo) {
foff = finfo->dirty_offset;
flen = foff + finfo->dirty_len;
- streamw = true;
+ params->notes |= NOTE_STREAMW;
+ if (foff > 0)
+ params->notes |= NOTE_DISCONTIG_BEFORE;
+ if (flen < fsize)
+ params->notes |= NOTE_DISCONTIG_AFTER;
}
+ if (params->last_end && fpos != params->last_end)
+ params->notes |= NOTE_DISCONTIG_BEFORE;
+ params->last_end = fpos + fsize;
+
if (wreq->origin == NETFS_WRITETHROUGH) {
- to_eof = false;
if (flen > i_size - fpos)
flen = i_size - fpos;
+ /* EOF may be changing. */
} else if (flen > i_size - fpos) {
flen = i_size - fpos;
- if (!streamw)
+ if (!(params->notes & NOTE_STREAMW))
folio_zero_segment(folio, flen, fsize);
- to_eof = true;
+ params->notes |= NOTE_TO_EOF;
} else if (flen == i_size - fpos) {
- to_eof = true;
+ params->notes |= NOTE_TO_EOF;
}
flen -= foff;
+ params->folio_start = fpos;
+ params->folio_len = fsize;
+ params->dirty_offset = foff;
+ params->dirty_len = flen;
+
_debug("folio %zx %zx %zx", foff, flen, fsize);
/* Deal with discontinuities in the stream of dirty pages. These can
@@ -467,168 +644,84 @@ static int netfs_write_folio(struct netfs_io_request *wreq,
* write-back group.
*/
if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
- netfs_issue_write(wreq, upload);
+ if (!(params->notes & NOTE_CACHE_AVAIL)) {
+ trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
+ goto cancel_folio;
+ }
+ params->notes |= NOTE_CACHE_COPY;
+ trace_netfs_folio(folio, netfs_folio_trace_store_copy);
} else if (fgroup != wreq->group) {
/* We can't write this page to the server yet. */
kdebug("wrong group");
- folio_redirty_for_writepage(wbc, folio);
- folio_unlock(folio);
- netfs_issue_write(wreq, upload);
- netfs_issue_write(wreq, cache);
- return 0;
+ goto skip_folio;
+ } else if (!(params->notes & (NOTE_UPLOAD_AVAIL | NOTE_CACHE_AVAIL))) {
+ trace_netfs_folio(folio, netfs_folio_trace_cancel_store);
+ goto cancel_folio_discard;
+ } else {
+ if (params->notes & NOTE_UPLOAD_STARTED) {
+ params->notes |= NOTE_UPLOAD;
+ trace_netfs_folio(folio, netfs_folio_trace_store_plus);
+ } else {
+ params->notes |= NOTE_UPLOAD | NOTE_UPLOAD_STARTED;
+ trace_netfs_folio(folio, netfs_folio_trace_store);
+ }
+ if ((params->notes & NOTE_CACHE_AVAIL) &&
+ !(params->notes & NOTE_STREAMW))
+ params->notes |= NOTE_CACHE_COPY;
}
- if (foff > 0)
- netfs_issue_write(wreq, upload);
- if (streamw)
- netfs_issue_write(wreq, cache);
-
folio_start_writeback(folio);
folio_unlock(folio);
- if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
- if (!cache->avail) {
- trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
- netfs_issue_write(wreq, upload);
- netfs_folio_written_back(folio);
- return 0;
- }
- trace_netfs_folio(folio, netfs_folio_trace_store_copy);
- } else if (!upload->avail && !cache->avail) {
- trace_netfs_folio(folio, netfs_folio_trace_cancel_store);
- netfs_folio_written_back(folio);
- return 0;
- } else if (!upload->construct) {
- trace_netfs_folio(folio, netfs_folio_trace_store);
- } else {
- trace_netfs_folio(folio, netfs_folio_trace_store_plus);
- }
-
/* Institute a new bvec queue segment if the current one is full or if
* we encounter a discontiguity. The discontiguity break is important
* when it comes to bulk unlocking folios by file range.
*/
if (bvecq_is_full(queue) ||
- (fpos != wreq->last_end && wreq->last_end > 0)) {
+ ((params->notes & NOTE_DISCONTIG_BEFORE) && queue->nr_slots > 0)) {
bvecq_buffer_append(&wreq->load_cursor, wreq->spare);
wreq->spare = NULL;
queue = wreq->load_cursor.bvecq;
queue->fpos = fpos;
- if (fpos != wreq->last_end)
+ if (params->notes & NOTE_DISCONTIG_BEFORE)
queue->discontig = true;
- bvecq_pos_move(&wreq->dispatch_cursor, queue);
- wreq->dispatch_cursor.slot = 0;
+ bvecq_pos_move(¶ms->dispatch_cursor, queue);
+ params->dispatch_cursor.slot = 0;
}
/* Attach the folio to the rolling buffer. */
slot = queue->nr_slots;
- bvec_set_folio(&queue->bv[slot], folio, flen, 0);
+ bvec_set_folio(&queue->bv[slot], folio, flen, foff);
trace_netfs_bv_slot(queue, slot);
slot++;
bvecq_filled_to(queue, slot);
wreq->load_cursor.slot = slot;
wreq->load_cursor.offset = 0;
- wreq->last_end = fpos + foff + flen;
+ trace_netfs_wback(wreq, folio, params->notes);
- /* Move the submission point forward to allow for write-streaming data
- * not starting at the front of the page. We don't do write-streaming
- * with the cache as the cache requires DIO alignment.
- *
- * Also skip uploading for data that's been read and just needs copying
- * to the cache.
- */
- bvecq_pos_nudge(&wreq->dispatch_cursor);
-
- for (int s = 0; s < NR_IO_STREAMS; s++) {
- stream = &wreq->io_streams[s];
- stream->submit_off = 0;
- stream->submit_len = flen;
- if (!stream->avail ||
- (stream->source == NETFS_WRITE_TO_CACHE && streamw) ||
- (stream->source == NETFS_UPLOAD_TO_SERVER &&
- fgroup == NETFS_FOLIO_COPY_TO_CACHE)) {
- stream->submit_off = UINT_MAX;
- stream->submit_len = 0;
- }
- }
-
- /* Attach the folio to one or more subrequests. For a big folio, we
- * could end up with thousands of subrequests if the wsize is small -
- * but we might need to wait during the creation of subrequests for
- * network resources (eg. SMB credits).
- */
- for (;;) {
- ssize_t part;
- size_t lowest_off = ULONG_MAX;
- int choose_s = -1;
-
- /* Always add to the lowest-submitted stream first. */
- for (int s = 0; s < NR_IO_STREAMS; s++) {
- stream = &wreq->io_streams[s];
- if (stream->submit_len > 0 &&
- stream->submit_off < lowest_off) {
- lowest_off = stream->submit_off;
- choose_s = s;
- }
- }
-
- if (choose_s < 0)
- break;
- stream = &wreq->io_streams[choose_s];
-
- /* Advance the cursor. */
- wreq->dispatch_cursor.offset = stream->submit_off;
-
- atomic64_set(&wreq->issued_to, fpos + foff + stream->submit_off);
- part = netfs_advance_write(wreq, stream, fpos + foff + stream->submit_off,
- stream->submit_len, to_eof);
- stream->submit_off += part;
- if (part > stream->submit_len)
- stream->submit_len = 0;
- else
- stream->submit_len -= part;
- if (part > 0)
- debug = true;
- }
-
- bvecq_pos_step(&wreq->dispatch_cursor);
- /* Order loading the queue before updating the issue_to point */
- atomic64_set_release(&wreq->issued_to, fpos + fsize);
-
- if (!debug)
- kdebug("R=%x: No submit", wreq->debug_id);
-
- if (foff + flen < fsize)
- for (int s = 0; s < NR_IO_STREAMS; s++)
- netfs_issue_write(wreq, &wreq->io_streams[s]);
-
- _leave(" = 0");
+out:
+ _leave(" = %x", params->notes);
return 0;
-}
-/*
- * End the issuing of writes, letting the collector know we're done.
- */
-static void netfs_end_issue_write(struct netfs_io_request *wreq)
-{
- bool needs_poke = true;
-
- smp_wmb(); /* Write subreq lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
-
- for (int s = 0; s < NR_IO_STREAMS; s++) {
- struct netfs_io_stream *stream = &wreq->io_streams[s];
-
- if (!stream->active)
- continue;
- if (!list_empty(&stream->subrequests))
- needs_poke = false;
- netfs_issue_write(wreq, stream);
- }
-
- if (needs_poke)
- netfs_wake_collector(wreq);
+skip_folio:
+ ret = folio_redirty_for_writepage(wbc, folio);
+ folio_unlock(folio);
+ if (ret < 0)
+ return ret;
+ params->notes |= NOTE_DISCONTIG_BEFORE;
+ goto out;
+cancel_folio_discard:
+ netfs_put_group(fgroup);
+cancel_folio:
+ folio_detach_private(folio);
+ kfree(finfo);
+ folio_unlock(folio);
+ folio_cancel_dirty(folio);
+ if (wreq->origin == NETFS_WRITETHROUGH)
+ folio_end_writeback(folio);
+ params->notes |= NOTE_DISCONTIG_BEFORE;
+ goto out;
}
/*
@@ -639,6 +732,7 @@ int netfs_writepages(struct address_space *mapping,
{
struct netfs_inode *ictx = netfs_inode(mapping->host);
struct netfs_io_request *wreq = NULL;
+ struct netfs_wb_params params = {};
struct folio *folio;
int error = 0;
@@ -664,35 +758,48 @@ int netfs_writepages(struct address_space *mapping,
if (bvecq_buffer_init(&wreq->load_cursor, GFP_NOFS) < 0)
goto nomem;
- bvecq_pos_set(&wreq->dispatch_cursor, &wreq->load_cursor);
- bvecq_pos_set(&wreq->collect_cursor, &wreq->dispatch_cursor);
+ bvecq_pos_set(¶ms.dispatch_cursor, &wreq->load_cursor);
+ bvecq_pos_set(&wreq->collect_cursor, &wreq->load_cursor);
__set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
trace_netfs_write(wreq, netfs_write_trace_writeback);
netfs_stat(&netfs_n_wh_writepages);
- do {
- _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to));
+ if (wreq->io_streams[1].avail)
+ params.notes |= NOTE_CACHE_AVAIL;
- /* It appears we don't have to handle cyclic writeback wrapping. */
- WARN_ON_ONCE(wreq && folio_pos(folio) < atomic64_read(&wreq->issued_to));
+ do {
+ _debug("wbiter %lx", folio->index);
if (netfs_folio_group(folio) != NETFS_FOLIO_COPY_TO_CACHE &&
unlikely(!test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))) {
set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
wreq->netfs_ops->begin_writeback(wreq);
+ if (wreq->io_streams[0].avail) {
+ params.notes |= NOTE_UPLOAD_AVAIL;
+ /* Order setting the active flag after other fields. */
+ smp_store_release(&wreq->io_streams[0].active, true);
+ }
}
- error = netfs_write_folio(wreq, wbc, folio);
+ params.notes &= NOTES__KEEP_MASK;
+ error = netfs_queue_wb_folio(wreq, wbc, folio, ¶ms);
+ if (error < 0)
+ break;
+ error = netfs_issue_streams(wreq, ¶ms);
if (error < 0)
break;
+
+ bvecq_pos_step(¶ms.dispatch_cursor);
} while ((folio = writeback_iter(mapping, wbc, folio, &error)));
- netfs_end_issue_write(wreq);
+ netfs_end_issue_write(wreq, ¶ms);
mutex_unlock(&ictx->wb_lock);
bvecq_pos_unset(&wreq->load_cursor);
- bvecq_pos_unset(&wreq->dispatch_cursor);
+ bvecq_pos_unset(¶ms.dispatch_cursor);
+ for (int i = 0; i < NR_IO_STREAMS; i++)
+ bvecq_pos_unset(&wreq->io_streams[i].dispatch_cursor);
netfs_wake_collector(wreq);
netfs_put_request(wreq, netfs_rreq_trace_put_return);
@@ -714,32 +821,60 @@ EXPORT_SYMBOL(netfs_writepages);
/*
* Begin a write operation for writing through the pagecache.
*/
-struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len)
+struct netfs_writethrough *netfs_begin_writethrough(struct kiocb *iocb, size_t len)
{
+ struct netfs_writethrough *wthru = NULL;
struct netfs_io_request *wreq = NULL;
struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp));
+ wthru = kzalloc_obj(struct netfs_writethrough);
+ if (!wthru)
+ return ERR_PTR(-ENOMEM);
+
mutex_lock(&ictx->wb_lock);
wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp,
iocb->ki_pos, NETFS_WRITETHROUGH);
if (IS_ERR(wreq)) {
mutex_unlock(&ictx->wb_lock);
- return wreq;
+ kfree(wthru);
+ return ERR_CAST(wreq);
}
+ wthru->wreq = wreq;
- if (bvecq_buffer_init(&wreq->load_cursor, GFP_NOFS) < 0) {
- netfs_put_failed_request(wreq);
- mutex_unlock(&ictx->wb_lock);
- return ERR_PTR(-ENOMEM);
- }
+ wreq->spare = bvecq_alloc_one(BVECQ_STD_SLOTS, GFP_NOFS);
+ if (!wreq->spare)
+ goto nomem_unlock;
- bvecq_pos_set(&wreq->dispatch_cursor, &wreq->load_cursor);
- bvecq_pos_set(&wreq->collect_cursor, &wreq->dispatch_cursor);
+ if (bvecq_buffer_init(&wreq->load_cursor, GFP_NOFS) < 0)
+ goto nomem_unlock;
+
+ bvecq_pos_set(&wthru->params.dispatch_cursor, &wreq->load_cursor);
+ bvecq_pos_set(&wreq->collect_cursor, &wreq->load_cursor);
+
+ if (wreq->io_streams[1].avail)
+ wthru->params.notes |= NOTE_CACHE_AVAIL;
wreq->io_streams[0].avail = true;
trace_netfs_write(wreq, netfs_write_trace_writethrough);
- return wreq;
+ if (!is_sync_kiocb(iocb))
+ wreq->iocb = iocb;
+
+ if (unlikely(!test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))) {
+ set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
+ /* Don't call ->begin_writeback() as ->init_request() gets file*. */
+ if (wreq->io_streams[0].avail) {
+ wthru->params.notes |= NOTE_UPLOAD_AVAIL;
+ /* Order setting the active flag after other fields. */
+ smp_store_release(&wreq->io_streams[0].active, true);
+ }
+ }
+ return wthru;
+nomem_unlock:
+ netfs_put_failed_request(wreq);
+ mutex_unlock(&ictx->wb_lock);
+ kfree(wthru);
+ return ERR_PTR(-ENOMEM);
}
/*
@@ -748,10 +883,11 @@ struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len
* to the request. If we've added more than wsize then we need to create a new
* subrequest.
*/
-int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
- struct folio *folio, size_t copied, bool to_page_end,
- struct folio **writethrough_cache)
+int netfs_advance_writethrough(struct netfs_writethrough *wthru,
+ struct writeback_control *wbc,
+ struct folio *folio, size_t copied, bool to_page_end)
{
+ struct netfs_io_request *wreq = wthru->wreq;
int ret;
_enter("R=%x ws=%u cp=%zu tp=%u",
@@ -759,18 +895,18 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c
/* The folio is locked. */
- if (*writethrough_cache != folio) {
- if (*writethrough_cache) {
+ if (wthru->in_progress != folio) {
+ if (wthru->in_progress) {
/* Did the folio get moved? */
- folio_put(*writethrough_cache);
- *writethrough_cache = NULL;
+ folio_put(wthru->in_progress);
+ wthru->in_progress = NULL;
}
/* We can make multiple writes to the folio... */
if (wreq->len == 0)
trace_netfs_folio(folio, netfs_folio_trace_wthru);
else
trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
- *writethrough_cache = folio;
+ wthru->in_progress = folio;
folio_get(folio);
}
@@ -782,9 +918,20 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c
return 0;
}
- ret = netfs_write_folio(wreq, wbc, folio);
- folio_put(*writethrough_cache);
- *writethrough_cache = NULL;
+ wthru->params.notes &= NOTES__KEEP_MASK;
+ ret = netfs_queue_wb_folio(wreq, wbc, folio, &wthru->params);
+ if (ret < 0)
+ return ret;
+
+ if (!wreq->spare) {
+ wreq->spare = bvecq_alloc_one(BVECQ_STD_SLOTS, GFP_NOFS);
+ if (!wreq->spare)
+ return -ENOMEM;
+ }
+
+ ret = netfs_issue_streams(wreq, &wthru->params);
+ folio_put(wthru->in_progress);
+ wthru->in_progress = NULL;
wreq->submitted = wreq->len;
return ret;
}
@@ -792,41 +939,85 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c
/*
* End a write operation used when writing through the pagecache.
*/
-ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
- struct folio *writethrough_cache)
+ssize_t netfs_end_writethrough(struct netfs_writethrough *wthru,
+ struct writeback_control *wbc)
{
+ struct netfs_io_request *wreq = wthru->wreq;
struct netfs_inode *ictx = netfs_inode(wreq->inode);
+ struct folio *folio = wthru->in_progress;
ssize_t ret;
_enter("R=%x", wreq->debug_id);
- if (writethrough_cache) {
- folio_lock(writethrough_cache);
- netfs_write_folio(wreq, wbc, writethrough_cache);
- folio_put(writethrough_cache);
+ if (folio) {
+ folio_lock(folio);
+ wthru->params.notes &= NOTES__KEEP_MASK;
+ ret = netfs_queue_wb_folio(wreq, wbc, folio, &wthru->params);
+ if (ret == 0)
+ ret = netfs_issue_streams(wreq, &wthru->params);
+ folio_put(folio);
+ wthru->in_progress = NULL;
wreq->submitted = wreq->len;
}
- netfs_end_issue_write(wreq);
+ netfs_end_issue_write(wreq, &wthru->params);
mutex_unlock(&ictx->wb_lock);
bvecq_pos_unset(&wreq->load_cursor);
- bvecq_pos_unset(&wreq->dispatch_cursor);
+ bvecq_pos_unset(&wthru->params.dispatch_cursor);
+ for (int i = 0; i < NR_IO_STREAMS; i++)
+ bvecq_pos_unset(&wreq->io_streams[i].dispatch_cursor);
if (wreq->iocb)
ret = -EIOCBQUEUED;
else
ret = netfs_wait_for_write(wreq);
netfs_put_request(wreq, netfs_rreq_trace_put_return);
+ kfree(wthru);
return ret;
}
+/*
+ * Prepare a buffer for a single monolithic write.
+ */
+static int netfs_prepare_write_single_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
+{
+ struct netfs_io_request *wreq = subreq->rreq;
+ struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
+ struct bio_vec *bv;
+ struct bvecq *bq;
+ size_t dio_size = wreq->cache_resources.dio_size;
+ size_t dlen;
+
+ bvecq_pos_set(&subreq->dispatch_pos, &stream->dispatch_cursor);
+ bvecq_pos_set(&subreq->content, &subreq->dispatch_pos);
+
+ /* Round the end of the last entry up. */
+ bq = subreq->content.bvecq;
+ while (bq->next)
+ bq = bq->next;
+ bv = &bq->bv[bq->nr_slots - 1];
+ dlen = round_up(bv->bv_len, dio_size);
+ if (dlen > bv->bv_len) {
+ subreq->len += dlen - bv->bv_len;
+ bv->bv_len = dlen;
+ }
+
+ stream->buffered = 0;
+ stream->issue_from = subreq->len;
+ wreq->submitted = subreq->len;
+ netfs_all_subreqs_queued(wreq);
+ return 0;
+}
+
/**
* netfs_writeback_single - Write back a monolithic payload
* @mapping: The mapping to write from
* @wbc: Hints from the VM
- * @iter: Data to write.
+ * @iter: Data to write
+ * @len: Amount of data to write
*
* Write a monolithic, non-pagecache object back to the server and/or
* the cache. There's a maximum of one subrequest per stream.
@@ -837,12 +1028,15 @@ ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_c
*/
int netfs_writeback_single(struct address_space *mapping,
struct writeback_control *wbc,
- struct iov_iter *iter)
+ struct iov_iter *iter,
+ size_t len)
{
struct netfs_io_request *wreq;
struct netfs_inode *ictx = netfs_inode(mapping->host);
int ret;
+ _enter("%zx,%zx", iov_iter_count(iter), len);
+
if (!mutex_trylock(&ictx->wb_lock)) {
if (wbc->sync_mode == WB_SYNC_NONE) {
/* The VFS will have undirtied the inode. */
@@ -859,23 +1053,24 @@ int netfs_writeback_single(struct address_space *mapping,
ret = PTR_ERR(wreq);
goto couldnt_start;
}
- wreq->len = iov_iter_count(iter);
- ret = netfs_extract_iter(iter, wreq->len, INT_MAX, 0, &wreq->dispatch_cursor.bvecq, 0);
+ wreq->len = len;
+
+ ret = netfs_extract_iter(iter, len, INT_MAX, 0, &wreq->load_cursor.bvecq, 0);
if (ret < 0)
goto cleanup_free;
- if (ret < wreq->len) {
+ if (ret < len) {
ret = -EIO;
goto cleanup_free;
}
- bvecq_pos_set(&wreq->collect_cursor, &wreq->dispatch_cursor);
+ bvecq_pos_set(&wreq->collect_cursor, &wreq->load_cursor);
__set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
trace_netfs_write(wreq, netfs_write_trace_writeback_single);
netfs_stat(&netfs_n_wh_writepages);
- if (__test_and_set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))
+ if (test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))
wreq->netfs_ops->begin_writeback(wreq);
for (int s = 0; s < NR_IO_STREAMS; s++) {
@@ -885,21 +1080,29 @@ int netfs_writeback_single(struct address_space *mapping,
if (!stream->avail)
continue;
- netfs_prepare_write(wreq, stream, 0);
+ stream->issue_from = 0;
+ stream->buffered = len;
- subreq = stream->construct;
- subreq->len = wreq->len;
- stream->submit_len = subreq->len;
+ subreq = netfs_alloc_write_subreq(wreq, stream);
+ if (!subreq) {
+ ret = -ENOMEM;
+ break;
+ }
- netfs_issue_write(wreq, stream);
+ bvecq_pos_set(&stream->dispatch_cursor, &wreq->load_cursor);
+
+ stream->issue_write(subreq);
+
+ bvecq_pos_unset(&stream->dispatch_cursor);
}
wreq->submitted = wreq->len;
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
-
mutex_unlock(&ictx->wb_lock);
- netfs_wake_collector(wreq);
+
+ if (unlikely(!netfs_are_all_subreqs_queued(wreq))) {
+ netfs_all_subreqs_queued(wreq);
+ netfs_wake_collector(wreq);
+ }
/* TODO: Might want to be async here if WB_SYNC_NONE, but then need to
* wait before modifying.
diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c
index de2f9b196fa5..fb73a37ecc91 100644
--- a/fs/netfs/write_retry.c
+++ b/fs/netfs/write_retry.c
@@ -12,12 +12,43 @@
#include "internal.h"
/*
- * Perform retries on the streams that need it.
+ * Prepare the write buffer for a retry. We can't necessarily reuse the write
+ * buffer from the previous run of a subrequest because the filesystem is
+ * permitted to modify it (add headers/trailers, encrypt it). Further, the
+ * subrequest may now be a different size (e.g. cifs has to negotiate for
+ * maximum transfer size). Also, we can't look at *stream as that may still
+ * refer to the source material being broken up into original subrequests.
+ */
+int netfs_prepare_write_retry_buffer(struct netfs_io_subrequest *subreq,
+ unsigned int max_segs)
+{
+ struct netfs_io_request *wreq = subreq->rreq;
+ struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
+ size_t len;
+
+ bvecq_pos_set(&subreq->dispatch_pos, &wreq->retry_cursor);
+ bvecq_pos_set(&subreq->content, &wreq->retry_cursor);
+ len = bvecq_slice(&wreq->retry_cursor, subreq->len, max_segs, &subreq->nr_segs);
+
+ if (len < subreq->len) {
+ subreq->len = len;
+ trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
+ }
+
+ stream->issue_from += len;
+ stream->buffered -= len;
+ if (stream->buffered == 0)
+ bvecq_pos_unset(&wreq->retry_cursor);
+ return 0;
+}
+
+/*
+ * Perform retries on the streams that need it. This only has to deal with
+ * buffered writes; unbuffered write retry is handled in direct_write.c.
*/
static void netfs_retry_write_stream(struct netfs_io_request *wreq,
struct netfs_io_stream *stream)
{
- struct bvecq_pos dispatch_cursor = {};
struct list_head *next;
_enter("R=%x[%x:]", wreq->debug_id, stream->stream_nr);
@@ -32,30 +63,13 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
if (unlikely(stream->failed))
return;
- /* If there's no renegotiation to do, just resend each failed subreq. */
- if (!stream->prepare_write) {
- struct netfs_io_subrequest *subreq;
-
- list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
- if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
- break;
- if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
- netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
- netfs_reissue_write(stream, subreq);
- }
- }
- return;
- }
-
next = stream->subrequests.next;
do {
struct netfs_io_subrequest *subreq = NULL, *from, *to, *tmp;
unsigned long long start, len;
- size_t part;
- bool boundary = false;
- bvecq_pos_unset(&dispatch_cursor);
+ bvecq_pos_unset(&wreq->retry_cursor);
/* Go through the stream and find the next span of contiguous
* data that we then rejig (cifs, for example, needs the wsize
@@ -74,7 +88,6 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
subreq = list_entry(next, struct netfs_io_subrequest, rreq_link);
if (subreq->start != start + len ||
subreq->transferred > 0 ||
- test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) ||
!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
break;
to = subreq;
@@ -84,8 +97,10 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
/* Determine the set of buffers we're going to use. Each
* subreq gets a subset of a single overall contiguous buffer.
*/
- bvecq_pos_transfer(&dispatch_cursor, &from->dispatch_pos);
- bvecq_pos_advance(&dispatch_cursor, from->transferred);
+ bvecq_pos_transfer(&wreq->retry_cursor, &from->dispatch_pos);
+ bvecq_pos_advance(&wreq->retry_cursor, from->transferred);
+ wreq->retry_start = start;
+ wreq->retry_buffered = len;
/* Work through the sublist. The chain of buffers we're going
* to fill is attached to dispatch_cursor and we need to read
@@ -93,38 +108,29 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
*/
subreq = from;
list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
- if (!len)
+ if (!wreq->retry_buffered)
break;
- subreq->start = start;
- subreq->len = len;
- __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
- trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
- subreq->transferred = 0;
-
bvecq_pos_unset(&subreq->dispatch_pos);
bvecq_pos_unset(&subreq->content);
+ subreq->content.bvecq = NULL;
+ subreq->content.slot = 0;
+ subreq->content.offset = 0;
- /* Renegotiate max_len (wsize) */
- stream->sreq_max_len = len;
- stream->sreq_max_segs = INT_MAX;
- stream->prepare_write(subreq);
-
- bvecq_pos_set(&subreq->dispatch_pos, &dispatch_cursor);
- part = bvecq_slice(&dispatch_cursor,
- umin(len, stream->sreq_max_len),
- stream->sreq_max_segs,
- &subreq->nr_segs);
- subreq->len = part;
-
- len -= part;
- start += part;
- if (len && subreq == to &&
- __test_and_clear_bit(NETFS_SREQ_BOUNDARY, &to->flags))
- boundary = true;
-
+ __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
+ __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
+ __clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
+ __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
+ subreq->start = wreq->retry_start;
+ subreq->len = wreq->retry_buffered;
+ subreq->transferred = 0;
+ subreq->retry_count += 1;
+ subreq->error = 0;
+
+ netfs_stat(&netfs_n_wh_retry_write_subreq);
+ trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
- netfs_reissue_write(stream, subreq);
+ stream->issue_write(subreq);
if (subreq == to)
break;
}
@@ -155,6 +161,7 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
subreq = netfs_alloc_subrequest(wreq);
subreq->source = to->source;
subreq->start = start;
+ subreq->len = len;
subreq->stream_nr = to->stream_nr;
subreq->retry_count = 1;
@@ -163,18 +170,17 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
netfs_sreq_trace_new);
trace_netfs_sreq(subreq, netfs_sreq_trace_split);
+ __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
spin_lock(&wreq->lock);
+ /* Write IN_PROGRESS before pointer to new subreq */
+ smp_wmb();
list_add(&subreq->rreq_link, &to->rreq_link);
spin_unlock(&wreq->lock);
to = subreq;
- trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
- stream->sreq_max_len = len;
- stream->sreq_max_segs = INT_MAX;
switch (stream->source) {
case NETFS_UPLOAD_TO_SERVER:
netfs_stat(&netfs_n_wh_upload);
- stream->sreq_max_len = umin(len, wreq->wsize);
break;
case NETFS_WRITE_TO_CACHE:
netfs_stat(&netfs_n_wh_write);
@@ -183,32 +189,14 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
WARN_ON_ONCE(1);
}
- stream->prepare_write(subreq);
-
- bvecq_pos_set(&subreq->dispatch_pos, &dispatch_cursor);
- part = bvecq_slice(&dispatch_cursor,
- umin(len, stream->sreq_max_len),
- stream->sreq_max_segs,
- &subreq->nr_segs);
- subreq->len = subreq->transferred + part;
-
- len -= part;
- start += part;
- if (!len && boundary) {
- __set_bit(NETFS_SREQ_BOUNDARY, &to->flags);
- boundary = false;
- }
-
- netfs_reissue_write(stream, subreq);
- if (!len)
- break;
-
+ trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
+ stream->issue_write(subreq);
} while (len);
} while (!list_is_head(next, &stream->subrequests));
out:
- bvecq_pos_unset(&dispatch_cursor);
+ bvecq_pos_unset(&wreq->retry_cursor);
}
/*
diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig
index 6bb30543eff0..e7862f35b72c 100644
--- a/fs/nfs/Kconfig
+++ b/fs/nfs/Kconfig
@@ -174,6 +174,7 @@ config NFS_FSCACHE
bool "Provide NFS client caching support"
depends on NFS_FS
select NETFS_SUPPORT
+ select NETFS_PGPRIV2
select FSCACHE
help
Say Y here if you want NFS data to be cached locally on disc through
diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c
index 9b7fdad4a920..cf750faaec6a 100644
--- a/fs/nfs/fscache.c
+++ b/fs/nfs/fscache.c
@@ -23,6 +23,7 @@
#include "iostat.h"
#include "fscache.h"
#include "nfstrace.h"
+#include <trace/events/netfs.h>
#define NFS_MAX_KEY_LEN 1000
@@ -273,8 +274,6 @@ static int nfs_netfs_init_request(struct netfs_io_request *rreq, struct file *fi
rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id);
/* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */
__set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags);
- rreq->io_streams[0].sreq_max_len = NFS_SB(rreq->inode->i_sb)->rsize;
-
return 0;
}
@@ -298,6 +297,7 @@ static struct nfs_netfs_io_data *nfs_netfs_alloc(struct netfs_io_subrequest *sre
static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq)
{
+ struct netfs_io_request *rreq = sreq->rreq;
struct nfs_netfs_io_data *netfs;
struct nfs_pageio_descriptor pgio;
struct inode *inode = sreq->rreq->inode;
@@ -307,6 +307,15 @@ static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq)
pgoff_t start, last;
int err;
+ if (sreq->len > NFS_SB(rreq->inode->i_sb)->rsize)
+ sreq->len = NFS_SB(rreq->inode->i_sb)->rsize;
+
+ err = netfs_prepare_read_buffer(sreq, INT_MAX);
+ if (err < 0) {
+ sreq->error = err;
+ goto term;
+ }
+
start = (sreq->start + sreq->transferred) >> PAGE_SHIFT;
last = ((sreq->start + sreq->len - sreq->transferred - 1) >> PAGE_SHIFT);
@@ -315,13 +324,15 @@ static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq)
netfs = nfs_netfs_alloc(sreq);
if (!netfs) {
- sreq->error = -ENOMEM;
- return netfs_read_subreq_terminated(sreq);
+ sreq->error = err;
+ goto term;
}
+ trace_netfs_sreq(sreq, netfs_sreq_trace_submit);
+
pgio.pg_netfs = netfs; /* used in completion */
- xa_for_each_range(&sreq->rreq->mapping->i_pages, idx, page, start, last) {
+ xa_for_each_range(&rreq->mapping->i_pages, idx, page, start, last) {
/* nfs_read_add_folio() may schedule() due to pNFS layout and other RPCs */
err = nfs_read_add_folio(&pgio, ctx, page_folio(page));
if (err < 0) {
@@ -332,6 +343,8 @@ static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq)
out:
nfs_pageio_complete_read(&pgio);
nfs_netfs_put(netfs);
+term:
+ return netfs_read_subreq_terminated(sreq);
}
void nfs_netfs_initiate_read(struct nfs_pgio_header *hdr)
diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c
index 9e27bfa7376b..0b1051a17ca8 100644
--- a/fs/smb/client/cifssmb.c
+++ b/fs/smb/client/cifssmb.c
@@ -1467,8 +1467,7 @@ cifs_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink);
struct inode *inode = &ictx->inode;
struct smb_rqst rqst = { .rq_iov = rdata->iov,
- .rq_nvec = 1,
- .rq_iter = rdata->subreq.io_iter };
+ .rq_nvec = 1};
struct cifs_credits credits = {
.value = 1,
.instance = 0,
@@ -1482,6 +1481,11 @@ cifs_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
__func__, mid->mid, mid->mid_state, rdata->result,
rdata->subreq.len);
+ if (rdata->got_bytes)
+ iov_iter_bvec_queue(&rqst.rq_iter, ITER_DEST,
+ rdata->subreq.content.bvecq, rdata->subreq.content.slot,
+ rdata->subreq.content.offset, rdata->subreq.len);
+
switch (mid->mid_state) {
case MID_RESPONSE_RECEIVED:
/* result already set, check signature */
@@ -2003,7 +2007,10 @@ cifs_async_writev(struct cifs_io_subrequest *wdata)
rqst.rq_iov = iov;
rqst.rq_nvec = 1;
- rqst.rq_iter = wdata->subreq.io_iter;
+
+ iov_iter_bvec_queue(&rqst.rq_iter, ITER_SOURCE,
+ wdata->subreq.content.bvecq, wdata->subreq.content.slot,
+ wdata->subreq.content.offset, wdata->subreq.len);
cifs_dbg(FYI, "async write at %llu %zu bytes\n",
wdata->subreq.start, wdata->subreq.len);
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index b60344125f27..d3a9041786ac 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -43,18 +43,36 @@ static int cifs_reopen_file(struct cifsFileInfo *cfile, bool can_flush);
* Prepare a subrequest to upload to the server. We need to allocate credits
* so that we know the maximum amount of data that we can include in it.
*/
-static void cifs_prepare_write(struct netfs_io_subrequest *subreq)
+static int cifs_estimate_write(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_write_estimate *estimate)
+{
+ struct cifs_sb_info *cifs_sb = CIFS_SB(wreq->inode->i_sb);
+
+ estimate->issue_at = stream->issue_from + cifs_sb->ctx->wsize;
+ return 0;
+}
+
+/*
+ * Issue a subrequest to upload to the server.
+ */
+static void cifs_issue_write(struct netfs_io_subrequest *subreq)
{
struct cifs_io_subrequest *wdata =
container_of(subreq, struct cifs_io_subrequest, subreq);
struct cifs_io_request *req = wdata->req;
- struct netfs_io_stream *stream = &req->rreq.io_streams[subreq->stream_nr];
struct TCP_Server_Info *server;
struct cifsFileInfo *open_file = req->cfile;
- struct cifs_sb_info *cifs_sb = CIFS_SB(wdata->rreq->inode->i_sb);
- size_t wsize = req->rreq.wsize;
+ struct cifs_sb_info *cifs_sb = CIFS_SB(subreq->rreq->inode->i_sb);
+ unsigned int max_segs = INT_MAX;
+ size_t len;
int rc;
+ if (cifs_forced_shutdown(cifs_sb)) {
+ rc = smb_EIO(smb_eio_trace_forced_shutdown);
+ goto fail;
+ }
+
if (!wdata->have_xid) {
wdata->xid = get_xid();
wdata->have_xid = true;
@@ -73,18 +91,16 @@ static void cifs_prepare_write(struct netfs_io_subrequest *subreq)
if (rc < 0) {
if (rc == -EAGAIN)
goto retry;
- subreq->error = rc;
- return netfs_prepare_write_failed(subreq);
+ goto fail;
}
}
- rc = server->ops->wait_mtu_credits(server, wsize, &stream->sreq_max_len,
- &wdata->credits);
- if (rc < 0) {
- subreq->error = rc;
- return netfs_prepare_write_failed(subreq);
- }
+ len = umin(subreq->len, cifs_sb->ctx->wsize);
+ rc = server->ops->wait_mtu_credits(server, len, &len, &wdata->credits);
+ if (rc < 0)
+ goto fail;
+ subreq->len = len;
wdata->credits.rreq_debug_id = subreq->rreq->debug_id;
wdata->credits.rreq_debug_index = subreq->debug_index;
wdata->credits.in_flight_check = 1;
@@ -100,44 +116,33 @@ static void cifs_prepare_write(struct netfs_io_subrequest *subreq)
const struct smbdirect_socket_parameters *sp =
smbd_get_parameters(server->smbd_conn);
- stream->sreq_max_segs = sp->max_frmr_depth;
+ max_segs = sp->max_frmr_depth;
}
#endif
-}
-
-/*
- * Issue a subrequest to upload to the server.
- */
-static void cifs_issue_write(struct netfs_io_subrequest *subreq)
-{
- struct cifs_io_subrequest *wdata =
- container_of(subreq, struct cifs_io_subrequest, subreq);
- struct cifs_sb_info *sbi = CIFS_SB(subreq->rreq->inode->i_sb);
- int rc;
- if (cifs_forced_shutdown(sbi)) {
- rc = smb_EIO(smb_eio_trace_forced_shutdown);
- goto fail;
- }
+ rc = netfs_prepare_write_buffer(subreq, max_segs);
+ if (rc < 0)
+ goto fail_with_credits;
- rc = adjust_credits(wdata->server, wdata, cifs_trace_rw_credits_issue_write_adjust);
+ rc = adjust_credits(server, wdata, cifs_trace_rw_credits_issue_write_adjust);
if (rc)
- goto fail;
+ goto fail_with_credits;
rc = -EAGAIN;
if (wdata->req->cfile->invalidHandle)
- goto fail;
+ goto fail_with_credits;
wdata->server->ops->async_writev(wdata);
out:
return;
-fail:
+fail_with_credits:
if (rc == -EAGAIN)
trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
else
trace_netfs_sreq(subreq, netfs_sreq_trace_fail);
add_credits_and_wake_if(wdata->server, &wdata->credits, 0);
+fail:
cifs_write_subrequest_terminated(wdata, rc);
goto out;
}
@@ -148,17 +153,25 @@ static void cifs_netfs_invalidate_cache(struct netfs_io_request *wreq)
}
/*
- * Negotiate the size of a read operation on behalf of the netfs library.
+ * Issue a read operation on behalf of the netfs helper functions. We're asked
+ * to make a read of a certain size at a point in the file. We are permitted
+ * to only read a portion of that, but as long as we read something, the netfs
+ * helper will call us again so that we can issue another read.
*/
-static int cifs_prepare_read(struct netfs_io_subrequest *subreq)
+static void cifs_issue_read(struct netfs_io_subrequest *subreq)
{
struct netfs_io_request *rreq = subreq->rreq;
struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq);
struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq);
- struct TCP_Server_Info *server;
+ struct TCP_Server_Info *server = rdata->server;
struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb);
- size_t size;
- int rc = 0;
+ unsigned int max_segs = INT_MAX;
+ size_t len;
+ int rc;
+
+ cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n",
+ __func__, rreq->debug_id, subreq->debug_index, rreq->mapping,
+ subreq->transferred, subreq->len);
if (!rdata->have_xid) {
rdata->xid = get_xid();
@@ -172,17 +185,15 @@ static int cifs_prepare_read(struct netfs_io_subrequest *subreq)
cifs_negotiate_rsize(server, cifs_sb->ctx,
tlink_tcon(req->cfile->tlink));
- rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize,
- &size, &rdata->credits);
+ len = umin(subreq->len, cifs_sb->ctx->rsize);
+ rc = server->ops->wait_mtu_credits(server, len, &len, &rdata->credits);
if (rc)
- return rc;
-
- rreq->io_streams[0].sreq_max_len = size;
+ goto failed;
- rdata->credits.in_flight_check = 1;
+ subreq->len = len;
rdata->credits.rreq_debug_id = rreq->debug_id;
rdata->credits.rreq_debug_index = subreq->debug_index;
-
+ rdata->credits.in_flight_check = 1;
trace_smb3_rw_credits(rdata->rreq->debug_id,
rdata->subreq.debug_index,
rdata->credits.value,
@@ -194,33 +205,17 @@ static int cifs_prepare_read(struct netfs_io_subrequest *subreq)
const struct smbdirect_socket_parameters *sp =
smbd_get_parameters(server->smbd_conn);
- rreq->io_streams[0].sreq_max_segs = sp->max_frmr_depth;
+ max_segs = sp->max_frmr_depth;
}
#endif
- return 0;
-}
-/*
- * Issue a read operation on behalf of the netfs helper functions. We're asked
- * to make a read of a certain size at a point in the file. We are permitted
- * to only read a portion of that, but as long as we read something, the netfs
- * helper will call us again so that we can issue another read.
- */
-static void cifs_issue_read(struct netfs_io_subrequest *subreq)
-{
- struct netfs_io_request *rreq = subreq->rreq;
- struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq);
- struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq);
- struct TCP_Server_Info *server = rdata->server;
- int rc = 0;
-
- cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n",
- __func__, rreq->debug_id, subreq->debug_index, rreq->mapping,
- subreq->transferred, subreq->len);
+ rc = netfs_prepare_read_buffer(subreq, max_segs);
+ if (rc < 0)
+ goto fail_with_credits;
rc = adjust_credits(server, rdata, cifs_trace_rw_credits_issue_read_adjust);
if (rc)
- goto failed;
+ goto fail_with_credits;
if (req->cfile->invalidHandle) {
do {
@@ -235,14 +230,21 @@ static void cifs_issue_read(struct netfs_io_subrequest *subreq)
__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
+
rc = rdata->server->ops->async_readv(rdata);
if (rc)
goto failed;
return;
+fail_with_credits:
+ if (rc == -EAGAIN)
+ trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
+ else
+ trace_netfs_sreq(subreq, netfs_sreq_trace_fail);
+ add_credits_and_wake_if(rdata->server, &rdata->credits, 0);
failed:
subreq->error = rc;
- netfs_read_subreq_terminated(subreq);
+ return netfs_read_subreq_terminated(subreq);
}
/*
@@ -352,11 +354,10 @@ const struct netfs_request_ops cifs_req_ops = {
.init_request = cifs_init_request,
.free_request = cifs_free_request,
.free_subrequest = cifs_free_subrequest,
- .prepare_read = cifs_prepare_read,
.issue_read = cifs_issue_read,
.done = cifs_rreq_done,
.begin_writeback = cifs_begin_writeback,
- .prepare_write = cifs_prepare_write,
+ .estimate_write = cifs_estimate_write,
.issue_write = cifs_issue_write,
.invalidate_cache = cifs_netfs_invalidate_cache,
};
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index c14ae1a61a43..799acd200d0c 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -4743,6 +4743,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
unsigned int cur_page_idx;
unsigned int pad_len;
struct cifs_io_subrequest *rdata = mid->callback_data;
+ struct iov_iter iter;
struct smb2_hdr *shdr = (struct smb2_hdr *)buf;
size_t copied;
bool use_rdma_mr = false;
@@ -4815,6 +4816,10 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
pad_len = data_offset - server->vals->read_rsp_size;
+ iov_iter_bvec_queue(&iter, ITER_DEST,
+ rdata->subreq.content.bvecq, rdata->subreq.content.slot,
+ rdata->subreq.content.offset, rdata->subreq.len);
+
if (buf_len <= data_offset) {
/* read response payload is in pages */
cur_page_idx = pad_len / PAGE_SIZE;
@@ -4844,7 +4849,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
/* Copy the data to the output I/O iterator. */
rdata->result = cifs_copy_bvecq_to_iter(buffer, data_len,
- cur_off, &rdata->subreq.io_iter);
+ cur_off, &iter);
if (rdata->result != 0) {
if (is_offloaded)
mid->mid_state = MID_RESPONSE_MALFORMED;
@@ -4858,7 +4863,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
buf_len >= end_off) {
/* read response payload is in buf */
WARN_ONCE(buffer, "read data can be either in buf or in buffer");
- copied = copy_to_iter(buf + data_offset, data_len, &rdata->subreq.io_iter);
+ copied = copy_to_iter(buf + data_offset, data_len, &iter);
if (copied == 0)
return smb_EIO2(smb_eio_trace_rx_copy_to_iter, copied, data_len);
rdata->got_bytes = copied;
diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c
index fbeb2156ddb6..8a4ac54f247e 100644
--- a/fs/smb/client/smb2pdu.c
+++ b/fs/smb/client/smb2pdu.c
@@ -4553,9 +4553,13 @@ smb2_new_read_req(void **buf, unsigned int *total_len,
*/
if (rdata && smb3_use_rdma_offload(io_parms)) {
struct smbdirect_buffer_descriptor_v1 *v1;
+ struct iov_iter iter;
bool need_invalidate = server->dialect == SMB30_PROT_ID;
- rdata->mr = smbd_register_mr(server->smbd_conn, &rdata->subreq.io_iter,
+ iov_iter_bvec_queue(&iter, ITER_DEST,
+ rdata->subreq.content.bvecq, rdata->subreq.content.slot,
+ rdata->subreq.content.offset, rdata->subreq.len);
+ rdata->mr = smbd_register_mr(server->smbd_conn, &iter,
true, need_invalidate);
if (!rdata->mr)
return -EAGAIN;
@@ -4619,9 +4623,10 @@ smb2_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
unsigned int rreq_debug_id = rdata->rreq->debug_id;
unsigned int subreq_debug_index = rdata->subreq.debug_index;
- if (rdata->got_bytes) {
- rqst.rq_iter = rdata->subreq.io_iter;
- }
+ if (rdata->got_bytes)
+ iov_iter_bvec_queue(&rqst.rq_iter, ITER_DEST,
+ rdata->subreq.content.bvecq, rdata->subreq.content.slot,
+ rdata->subreq.content.offset, rdata->subreq.len);
WARN_ONCE(rdata->server != server,
"rdata server %p != mid server %p",
@@ -5109,7 +5114,9 @@ smb2_async_writev(struct cifs_io_subrequest *wdata)
goto out;
rqst.rq_iov = iov;
- rqst.rq_iter = wdata->subreq.io_iter;
+ iov_iter_bvec_queue(&rqst.rq_iter, ITER_SOURCE,
+ wdata->subreq.content.bvecq, wdata->subreq.content.slot,
+ wdata->subreq.content.offset, wdata->subreq.len);
rqst.rq_iov[0].iov_len = total_len - 1;
rqst.rq_iov[0].iov_base = (char *)req;
@@ -5148,9 +5155,14 @@ smb2_async_writev(struct cifs_io_subrequest *wdata)
*/
if (smb3_use_rdma_offload(io_parms)) {
struct smbdirect_buffer_descriptor_v1 *v1;
+ struct iov_iter iter;
bool need_invalidate = server->dialect == SMB30_PROT_ID;
- wdata->mr = smbd_register_mr(server->smbd_conn, &wdata->subreq.io_iter,
+ iov_iter_bvec_queue(&iter, ITER_SOURCE,
+ wdata->subreq.content.bvecq, wdata->subreq.content.slot,
+ wdata->subreq.content.offset, wdata->subreq.len);
+
+ wdata->mr = smbd_register_mr(server->smbd_conn, &iter,
false, need_invalidate);
if (!wdata->mr) {
rc = -EAGAIN;
@@ -5187,8 +5199,8 @@ smb2_async_writev(struct cifs_io_subrequest *wdata)
smb2_set_replay(server, &rqst);
}
- cifs_dbg(FYI, "async write at %llu %u bytes iter=%zx\n",
- io_parms->offset, io_parms->length, iov_iter_count(&wdata->subreq.io_iter));
+ cifs_dbg(FYI, "async write at %llu %u bytes len=%zx\n",
+ io_parms->offset, io_parms->length, wdata->subreq.len);
if (wdata->credits.value > 0) {
shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(wdata->subreq.len,
diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c
index fdf4e50c27ce..be2f6b909c34 100644
--- a/fs/smb/client/transport.c
+++ b/fs/smb/client/transport.c
@@ -1267,12 +1267,19 @@ cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid)
}
#ifdef CONFIG_CIFS_SMB_DIRECT
- if (rdata->mr)
+ if (rdata->mr) {
length = data_len; /* An RDMA read is already done. */
- else
+ } else {
+#endif
+ struct iov_iter iter;
+
+ iov_iter_bvec_queue(&iter, ITER_DEST, rdata->subreq.content.bvecq,
+ rdata->subreq.content.slot, rdata->subreq.content.offset,
+ data_len);
+ length = cifs_read_iter_from_socket(server, &iter, data_len);
+#ifdef CONFIG_CIFS_SMB_DIRECT
+ }
#endif
- length = cifs_read_iter_from_socket(server, &rdata->subreq.io_iter,
- data_len);
if (length > 0)
rdata->got_bytes += length;
server->total_read += length;
diff --git a/include/linux/netfs.h b/include/linux/netfs.h
index 7dca6a513509..86bef8fec14b 100644
--- a/include/linux/netfs.h
+++ b/include/linux/netfs.h
@@ -66,7 +66,7 @@ struct netfs_inode {
#endif
struct mutex wb_lock; /* Writeback serialisation */
loff_t _remote_i_size; /* Size of the remote file */
- loff_t _zero_point; /* Size after which we assume there's no data
+ unsigned long long _zero_point; /* Size after which we assume there's no data
* on the server */
atomic_t io_count; /* Number of outstanding reqs */
unsigned long flags;
@@ -126,25 +126,39 @@ static inline struct netfs_group *netfs_folio_group(struct folio *folio)
return priv;
}
+/*
+ * Estimate of maximum write subrequest for writeback. The filesystem is
+ * responsible for filling this in when called from ->estimate_write(), though
+ * netfslib will preset infinite defaults.
+ */
+struct netfs_write_estimate {
+ unsigned long long issue_at; /* Point at which we must submit */
+ int max_segs; /* Max number of segments in a single RPC */
+};
+
/*
* Stream of I/O subrequests going to a particular destination, such as the
* server or the local cache. This is mainly intended for writing where we may
* have to write to multiple destinations concurrently.
*/
struct netfs_io_stream {
- /* Submission tracking */
- struct netfs_io_subrequest *construct; /* Op being constructed */
- size_t sreq_max_len; /* Maximum size of a subrequest */
- unsigned int sreq_max_segs; /* 0 or max number of segments in an iterator */
- unsigned int submit_off; /* Folio offset we're submitting from */
- unsigned int submit_len; /* Amount of data left to submit */
- void (*prepare_write)(struct netfs_io_subrequest *subreq);
+ /* Submission tracking (main dispatch only; not retry) */
+ struct bvecq_pos dispatch_cursor; /* Point from which buffers are dispatched */
+ unsigned long long issue_from; /* Current issue point */
+ size_t buffered; /* Amount in buffer */
+ u8 applicable; /* What sources are applicable (NOTE_* mask) */
+ bool buffering; /* T if buffering on this stream */
+ int (*estimate_write)(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_write_estimate *estimate);
void (*issue_write)(struct netfs_io_subrequest *subreq);
+ atomic64_t issued_to; /* Point to which can be considered issued */
+
/* Collection tracking */
struct list_head subrequests; /* Contributory I/O operations */
unsigned long long collected_to; /* Position we've collected results to */
size_t transferred; /* The amount transferred from this stream */
- unsigned short error; /* Aggregate error for the stream */
+ short error; /* Aggregate error for the stream */
enum netfs_io_source source; /* Where to read from/write to */
unsigned char stream_nr; /* Index of stream in parent table */
bool avail; /* T if stream is available */
@@ -182,14 +196,13 @@ struct netfs_io_subrequest {
struct list_head rreq_link; /* Link in rreq->subrequests */
struct bvecq_pos dispatch_pos; /* Bookmark in the combined queue of the start */
struct bvecq_pos content; /* The (copied) content of the subrequest */
- struct iov_iter io_iter; /* Iterator for this subrequest */
unsigned long long start; /* Where to start the I/O */
size_t len; /* Size of the I/O */
size_t transferred; /* Amount of data transferred */
+ unsigned int nr_segs; /* Number of segments in content */
refcount_t ref;
short error; /* 0 or error that occurred */
unsigned short debug_index; /* Index in list (for debugging output) */
- unsigned int nr_segs; /* Number of segs in io_iter */
u8 retry_count; /* The number of retries (0 on initial pass) */
enum netfs_io_source source; /* Where to read from/write to */
unsigned char stream_nr; /* I/O stream this belongs to */
@@ -198,7 +211,6 @@ struct netfs_io_subrequest {
#define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */
#define NETFS_SREQ_MADE_PROGRESS 4 /* Set if we transferred at least some data */
#define NETFS_SREQ_ONDEMAND 5 /* Set if it's from on-demand read mode */
-#define NETFS_SREQ_BOUNDARY 6 /* Set if ends on hard boundary (eg. ceph object) */
#define NETFS_SREQ_HIT_EOF 7 /* Set if short due to EOF */
#define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */
#define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */
@@ -246,23 +258,26 @@ struct netfs_io_request {
struct netfs_group *group; /* Writeback group being written back */
struct bvecq *spare; /* Advance allocation of bvecq */
struct bvecq_pos load_cursor; /* Point at which new folios are loaded in */
- struct bvecq_pos dispatch_cursor; /* Point from which buffers are dispatched */
struct bvecq_pos collect_cursor; /* Clear-up point of I/O buffer */
+ struct bvecq_pos retry_cursor; /* Point from which retries are dispatched */
wait_queue_head_t waitq; /* Processor waiter */
void *netfs_priv; /* Private data for the netfs */
void *netfs_priv2; /* Private data for the netfs */
- unsigned long long last_end; /* End pos of last folio submitted */
unsigned long long submitted; /* Amount submitted for I/O so far */
unsigned long long len; /* Length of the request */
size_t transferred; /* Amount to be indicated as transferred */
long error; /* 0 or error that occurred */
unsigned long long i_size; /* Size of the file */
unsigned long long start; /* Start position */
- atomic64_t issued_to; /* Write issuer folio cursor */
unsigned long long collected_to; /* Point we've collected to */
unsigned long long cache_coll_to; /* Point the cache has collected to */
unsigned long long cleaned_to; /* Position we've cleaned folios to */
unsigned long long abandon_to; /* Position to abandon folios to */
+#ifdef CONFIG_NETFS_PGPRIV2
+ unsigned long long last_end; /* End of last folio added */
+#endif
+ unsigned long long retry_start; /* Position to retry from */
+ size_t retry_buffered; /* Amount of data to retry */
const struct folio *no_unlock_folio; /* Don't unlock this folio after read */
unsigned int debug_id;
unsigned int rsize; /* Maximum read size (0 for none) */
@@ -280,6 +295,7 @@ struct netfs_io_request {
#define NETFS_RREQ_FAILED 3 /* The request failed */
#define NETFS_RREQ_RETRYING 4 /* Set if we're in the retry path */
#define NETFS_RREQ_SHORT_TRANSFER 5 /* Set if we have a short transfer */
+#define NETFS_RREQ_SAW_ENOMEM 6 /* Set if we encounted ENOMEM */
#define NETFS_RREQ_CACHE_STOP 8 /* Set to stop caching (ENOBUFS or error) */
#define NETFS_RREQ_CACHE_ERROR 9 /* Set if we got an error from the cache */
#define NETFS_RREQ_OFFLOAD_COLLECTION 12 /* Offload collection to workqueue */
@@ -288,8 +304,10 @@ struct netfs_io_request {
#define NETFS_RREQ_UPLOAD_TO_SERVER 15 /* Need to write to the server */
#define NETFS_RREQ_USE_IO_ITER 16 /* Use ->io_iter rather than ->i_pages */
#define NETFS_RREQ_NEED_PUT_RA_REFS 17 /* Need to put the folio refs RA gave us */
+#ifdef CONFIG_NETFS_PGPRIV2
#define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark
* write to cache on read */
+#endif
const struct netfs_request_ops *netfs_ops;
};
@@ -305,7 +323,6 @@ struct netfs_request_ops {
/* Read request handling */
void (*expand_readahead)(struct netfs_io_request *rreq);
- int (*prepare_read)(struct netfs_io_subrequest *subreq);
void (*issue_read)(struct netfs_io_subrequest *subreq);
bool (*is_still_valid)(struct netfs_io_request *rreq);
int (*check_write_begin)(struct file *file, loff_t pos, unsigned len,
@@ -318,7 +335,9 @@ struct netfs_request_ops {
/* Write request handling */
void (*begin_writeback)(struct netfs_io_request *wreq);
- void (*prepare_write)(struct netfs_io_subrequest *subreq);
+ int (*estimate_write)(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_write_estimate *estimate);
void (*issue_write)(struct netfs_io_subrequest *subreq);
void (*retry_request)(struct netfs_io_request *wreq, struct netfs_io_stream *stream);
void (*invalidate_cache)(struct netfs_io_request *wreq);
@@ -360,6 +379,14 @@ struct netfs_cache_ops {
netfs_io_terminated_t term_func,
void *term_func_priv);
+ /* Estimate the amount of data that can be written in an op. */
+ int (*estimate_write)(struct netfs_io_request *wreq,
+ struct netfs_io_stream *stream,
+ struct netfs_write_estimate *estimate);
+
+ /* Read data from the cache for a netfs subrequest. */
+ void (*issue_read)(struct netfs_io_subrequest *subreq);
+
/* Write data to the cache from a netfs subrequest. */
void (*issue_write)(struct netfs_io_subrequest *subreq);
@@ -369,25 +396,6 @@ struct netfs_cache_ops {
unsigned long long *_len,
unsigned long long i_size);
- /* Prepare a read operation, shortening it to a cached/uncached
- * boundary as appropriate.
- */
- int (*prepare_read)(struct netfs_io_subrequest *subreq);
-
- /* Prepare a write subrequest, working out if we're allowed to do it
- * and finding out the maximum amount of data to gather before
- * attempting to submit. If we're not permitted to do it, the
- * subrequest should be marked failed.
- */
- void (*prepare_write_subreq)(struct netfs_io_subrequest *subreq);
-
- /* Prepare a write operation, working out what part of the write we can
- * actually do.
- */
- int (*prepare_write)(struct netfs_cache_resources *cres,
- loff_t *_start, size_t *_len, size_t upper_len,
- loff_t i_size, bool no_space_allocated_yet);
-
/* Prepare an on-demand read operation, shortening it to a cached/uncached
* boundary as appropriate.
*/
@@ -399,8 +407,8 @@ struct netfs_cache_ops {
/* Query the occupancy of the cache in a region, returning where the
* next chunk of data starts and how long it is.
*/
- int (*query_occupancy)(struct netfs_cache_resources *cres,
- struct fscache_occupancy *occ);
+ void (*query_occupancy)(struct netfs_cache_resources *cres,
+ struct fscache_occupancy *occ);
/* Collect the result of buffered writeback to the cache. This
* includes copying a read to the cache. block_type is one of:
@@ -434,10 +442,9 @@ void netfs_single_mark_inode_dirty(struct inode *inode);
ssize_t netfs_read_single(struct inode *inode, struct file *file, struct iov_iter *iter);
int netfs_writeback_single(struct address_space *mapping,
struct writeback_control *wbc,
- struct iov_iter *iter);
+ struct iov_iter *iter, size_t len);
/* Address operations API */
-struct readahead_control;
void netfs_readahead(struct readahead_control *);
int netfs_read_folio(struct file *, struct folio *);
int netfs_write_begin(struct netfs_inode *, struct file *,
@@ -464,7 +471,8 @@ void netfs_put_subrequest(struct netfs_io_subrequest *subreq,
ssize_t netfs_extract_iter(struct iov_iter *orig, size_t max_len, size_t max_pages,
unsigned long long fpos, struct bvecq **_bvecq_head,
iov_iter_extraction_t extraction_flags);
-void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq);
+int netfs_prepare_read_buffer(struct netfs_io_subrequest *subreq, unsigned int max_segs);
+int netfs_prepare_write_buffer(struct netfs_io_subrequest *subreq, unsigned int max_segs);
void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error);
int netfs_start_io_read(struct inode *inode);
diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h
index cc29582f6245..fbd000399b26 100644
--- a/include/trace/events/netfs.h
+++ b/include/trace/events/netfs.h
@@ -49,6 +49,7 @@
E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C")
#define netfs_rreq_traces \
+ EM(netfs_rreq_trace_all_queued, "ALL-Q ") \
EM(netfs_rreq_trace_assess, "ASSESS ") \
EM(netfs_rreq_trace_cache_cancelled, "CA-CNCL") \
EM(netfs_rreq_trace_cache_failed, "CA-FAIL") \
@@ -86,7 +87,8 @@
EM(netfs_rreq_trace_waited_quiesce, "DONE-QUIESCE") \
EM(netfs_rreq_trace_wake_ip, "WAKE-IP") \
EM(netfs_rreq_trace_wake_queue, "WAKE-Q ") \
- E_(netfs_rreq_trace_write_done, "WR-DONE")
+ EM(netfs_rreq_trace_write_done, "WR-DONE") \
+ E_(netfs_rreq_trace_zero_unread, "ZERO-UR")
#define netfs_sreq_sources \
EM(NETFS_SOURCE_UNKNOWN, "----") \
@@ -135,6 +137,7 @@
EM(netfs_sreq_trace_superfluous, "SPRFL") \
EM(netfs_sreq_trace_terminated, "TERM ") \
EM(netfs_sreq_trace_too_much, "!TOOM") \
+ EM(netfs_sreq_trace_too_many_retries, "!RETR") \
EM(netfs_sreq_trace_wait_for, "_WAIT") \
EM(netfs_sreq_trace_write, "WRITE") \
EM(netfs_sreq_trace_write_skip, "SKIP ") \
@@ -202,12 +205,12 @@
EM(netfs_folio_trace_alloc_buffer, "alloc-buf") \
EM(netfs_folio_trace_cancel_copy, "cancel-copy") \
EM(netfs_folio_trace_cancel_store, "cancel-store") \
- EM(netfs_folio_trace_clear, "clear") \
- EM(netfs_folio_trace_clear_cc, "clear-cc") \
- EM(netfs_folio_trace_clear_g, "clear-g") \
- EM(netfs_folio_trace_clear_s, "clear-s") \
EM(netfs_folio_trace_copy_to_cache, "mark-copy") \
EM(netfs_folio_trace_end_copy, "end-copy") \
+ EM(netfs_folio_trace_endwb, "endwb") \
+ EM(netfs_folio_trace_endwb_cc, "endwb-cc") \
+ EM(netfs_folio_trace_endwb_g, "endwb-g") \
+ EM(netfs_folio_trace_endwb_s, "endwb-s") \
EM(netfs_folio_trace_filled_gaps, "filled-gaps") \
EM(netfs_folio_trace_invalidate_all, "inval-all") \
EM(netfs_folio_trace_invalidate_front, "inval-front") \
@@ -400,10 +403,10 @@ TRACE_EVENT(netfs_sreq,
__entry->len = sreq->len;
__entry->transferred = sreq->transferred;
__entry->start = sreq->start;
- __entry->slot = sreq->dispatch_pos.slot;
+ __entry->slot = sreq->content.slot;
),
- TP_printk("R=%08x[%x] %s %s f=%03x s=%llx %zx/%zx qs=%u e=%d",
+ TP_printk("R=%08x[%x] %s %s f=%03x s=%llx %zx/%zx bv=%u e=%d",
__entry->rreq, __entry->index,
__print_symbolic(__entry->source, netfs_sreq_sources),
__print_symbolic(__entry->what, netfs_sreq_traces),
@@ -511,6 +514,7 @@ TRACE_EVENT(netfs_folio,
TP_STRUCT__entry(
__field(u64, ino)
__field(pgoff_t, index)
+ __field(unsigned long, pfn)
__field(unsigned int, nr)
__field(enum netfs_folio_trace, why)
),
@@ -521,13 +525,40 @@ TRACE_EVENT(netfs_folio,
__entry->why = why;
__entry->index = folio->index;
__entry->nr = folio_nr_pages(folio);
+ __entry->pfn = folio_pfn(folio);
),
- TP_printk("i=%05llx ix=%05lx-%05lx %s",
+ TP_printk("p=%lx i=%05llx ix=%05lx-%05lx %s",
+ __entry->pfn,
__entry->ino, __entry->index, __entry->index + __entry->nr - 1,
__print_symbolic(__entry->why, netfs_folio_traces))
);
+TRACE_EVENT(netfs_wback,
+ TP_PROTO(struct netfs_io_request *wreq, struct folio *folio, unsigned int notes),
+
+ TP_ARGS(wreq, folio, notes),
+
+ TP_STRUCT__entry(
+ __field(pgoff_t, index)
+ __field(unsigned int, wreq)
+ __field(unsigned int, nr)
+ __field(unsigned int, notes)
+ ),
+
+ TP_fast_assign(
+ __entry->wreq = wreq->debug_id;
+ __entry->notes = notes;
+ __entry->index = folio->index;
+ __entry->nr = folio_nr_pages(folio);
+ ),
+
+ TP_printk("R=%08x ix=%05lx-%05lx n=%02x",
+ __entry->wreq,
+ __entry->index, __entry->index + __entry->nr - 1,
+ __entry->notes)
+ );
+
TRACE_EVENT(netfs_write_iter,
TP_PROTO(const struct kiocb *iocb, const struct iov_iter *from),
@@ -771,7 +802,7 @@ TRACE_EVENT(netfs_collect_stream,
__entry->wreq = wreq->debug_id;
__entry->stream = stream->stream_nr;
__entry->collected_to = stream->collected_to;
- __entry->issued_to = atomic64_read(&wreq->issued_to);
+ __entry->issued_to = atomic64_read(&stream->issued_to);
),
TP_printk("R=%08x[%x:] cto=%llx ito=%llx",
@@ -795,7 +826,7 @@ TRACE_EVENT(netfs_bvecq,
__entry->trace = trace;
),
- TP_printk("fq=%x %s",
+ TP_printk("bq=%x %s",
__entry->id,
__print_symbolic(__entry->trace, netfs_bvecq_traces))
);
diff --git a/net/9p/client.c b/net/9p/client.c
index f0dcf252af7e..8d365c000553 100644
--- a/net/9p/client.c
+++ b/net/9p/client.c
@@ -1561,6 +1561,7 @@ void
p9_client_write_subreq(struct netfs_io_subrequest *subreq)
{
struct netfs_io_request *wreq = subreq->rreq;
+ struct iov_iter iter;
struct p9_fid *fid = wreq->netfs_priv;
struct p9_client *clnt = fid->clnt;
struct p9_req_t *req;
@@ -1571,14 +1572,17 @@ p9_client_write_subreq(struct netfs_io_subrequest *subreq)
p9_debug(P9_DEBUG_9P, ">>> TWRITE fid %d offset %llu len %d\n",
fid->fid, start, len);
+ iov_iter_bvec_queue(&iter, ITER_SOURCE, subreq->content.bvecq,
+ subreq->content.slot, subreq->content.offset, subreq->len);
+
/* Don't bother zerocopy for small IO (< 1024) */
if (clnt->trans_mod->zc_request && len > 1024) {
- req = p9_client_zc_rpc(clnt, P9_TWRITE, NULL, &subreq->io_iter,
+ req = p9_client_zc_rpc(clnt, P9_TWRITE, NULL, &iter,
0, wreq->len, P9_ZC_HDR_SZ, "dqd",
fid->fid, start, len);
} else {
req = p9_client_rpc(clnt, P9_TWRITE, "dqV", fid->fid,
- start, len, &subreq->io_iter);
+ start, len, &iter);
}
if (IS_ERR(req)) {
netfs_write_subrequest_terminated(subreq, PTR_ERR(req));
^ permalink raw reply related
* [PATCH v3 21/22] netfs: Limit the minimum trigger for progress reporting
From: David Howells @ 2026-06-08 14:54 UTC (permalink / raw)
To: Christian Brauner, Matthew Wilcox, Christoph Hellwig
Cc: David Howells, Paulo Alcantara, Jens Axboe, Leon Romanovsky,
Steve French, ChenXiaoSong, Marc Dionne, Eric Van Hensbergen,
Dominique Martinet, Ilya Dryomov, Trond Myklebust, netfs,
linux-afs, linux-cifs, linux-nfs, ceph-devel, v9fs, linux-erofs,
linux-fsdevel, linux-kernel
In-Reply-To: <20260608145432.681865-1-dhowells@redhat.com>
For really big read RPC ops that span multiple folios, netfslib allows the
filesystem to give progress notifications to wake up the collector thread
to do a collection of folios that have now been fetched, even if the RPC is
still ongoing, thereby allowing the application to make progress.
The trigger for this is that at least one folio has been downloaded since
the clean point. If, however, the folios are small, this means the
collector thread is constantly being woken up - which has a negative
performance impact on the system.
Set a minimum trigger of 256KiB or the size of the folio at the front of
the queue, whichever is larger.
Also, fix the base to be the stream collection point, not the point at
which the collector has cleaned up to (which is currently 0 until something
has been collected).
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Paulo Alcantara <pc@manguebit.org>
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
---
fs/netfs/read_collect.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c
index fc62eaef6107..fccc6c2d891e 100644
--- a/fs/netfs/read_collect.c
+++ b/fs/netfs/read_collect.c
@@ -491,15 +491,15 @@ void netfs_read_collection_worker(struct work_struct *work)
void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq)
{
struct netfs_io_request *rreq = subreq->rreq;
- struct netfs_io_stream *stream = &rreq->io_streams[0];
- size_t fsize = PAGE_SIZE << rreq->front_folio_order;
+ struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr];
+ size_t fsize = umax(PAGE_SIZE << rreq->front_folio_order, 256 * 1024);
trace_netfs_sreq(subreq, netfs_sreq_trace_progress);
/* If we are at the head of the queue, wake up the collector,
* getting a ref to it if we were the ones to do so.
*/
- if (subreq->start + subreq->transferred > rreq->cleaned_to + fsize &&
+ if (subreq->start + subreq->transferred >= stream->collected_to + fsize &&
(rreq->origin == NETFS_READAHEAD ||
rreq->origin == NETFS_READPAGE ||
rreq->origin == NETFS_READ_FOR_WRITE) &&
^ permalink raw reply related
* [PATCH v3 20/22] netfs: Check for too much data being read
From: David Howells @ 2026-06-08 14:54 UTC (permalink / raw)
To: Christian Brauner, Matthew Wilcox, Christoph Hellwig
Cc: David Howells, Paulo Alcantara, Jens Axboe, Leon Romanovsky,
Steve French, ChenXiaoSong, Marc Dionne, Eric Van Hensbergen,
Dominique Martinet, Ilya Dryomov, Trond Myklebust, netfs,
linux-afs, linux-cifs, linux-nfs, ceph-devel, v9fs, linux-erofs,
linux-fsdevel, linux-kernel
In-Reply-To: <20260608145432.681865-1-dhowells@redhat.com>
Put in a check in read subreq termination to detect more data being read
for a subrequest than was requested.
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Paulo Alcantara <pc@manguebit.org>
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
---
fs/netfs/read_collect.c | 8 ++++++++
include/trace/events/netfs.h | 1 +
2 files changed, 9 insertions(+)
diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c
index 977b69ac8725..fc62eaef6107 100644
--- a/fs/netfs/read_collect.c
+++ b/fs/netfs/read_collect.c
@@ -542,6 +542,14 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq)
break;
}
+ if (subreq->transferred > subreq->len) {
+ subreq->transferred = 0;
+ __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
+ __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
+ trace_netfs_sreq(subreq, netfs_sreq_trace_too_much);
+ subreq->error = -EIO;
+ }
+
/* Deal with retry requests, short reads and errors. If we retry
* but don't make progress, we abandon the attempt.
*/
diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h
index 59f330003d02..cc29582f6245 100644
--- a/include/trace/events/netfs.h
+++ b/include/trace/events/netfs.h
@@ -134,6 +134,7 @@
EM(netfs_sreq_trace_submit, "SUBMT") \
EM(netfs_sreq_trace_superfluous, "SPRFL") \
EM(netfs_sreq_trace_terminated, "TERM ") \
+ EM(netfs_sreq_trace_too_much, "!TOOM") \
EM(netfs_sreq_trace_wait_for, "_WAIT") \
EM(netfs_sreq_trace_write, "WRITE") \
EM(netfs_sreq_trace_write_skip, "SKIP ") \
^ permalink raw reply related
* [PATCH v3 19/22] netfs: Remove folio_queue and rolling_buffer
From: David Howells @ 2026-06-08 14:54 UTC (permalink / raw)
To: Christian Brauner, Matthew Wilcox, Christoph Hellwig
Cc: David Howells, Paulo Alcantara, Jens Axboe, Leon Romanovsky,
Steve French, ChenXiaoSong, Marc Dionne, Eric Van Hensbergen,
Dominique Martinet, Ilya Dryomov, Trond Myklebust, netfs,
linux-afs, linux-cifs, linux-nfs, ceph-devel, v9fs, linux-erofs,
linux-fsdevel, linux-kernel
In-Reply-To: <20260608145432.681865-1-dhowells@redhat.com>
Remove folio_queue and rolling_buffer as they're no longer used.
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Paulo Alcantara <pc@manguebit.org>
cc: Matthew Wilcox <willy@infradead.org>
cc: Christoph Hellwig <hch@infradead.org>
cc: Steve French <sfrench@samba.org>
cc: linux-cifs@vger.kernel.org
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
---
Documentation/core-api/folio_queue.rst | 209 --------------
Documentation/core-api/index.rst | 1 -
Documentation/filesystems/netfs_library.rst | 2 +-
fs/netfs/iterator.c | 192 -------------
fs/netfs/rolling_buffer.c | 297 --------------------
include/linux/folio_queue.h | 282 -------------------
include/linux/netfs.h | 2 -
include/linux/rolling_buffer.h | 64 -----
kernel/bpf/btf.c | 2 -
9 files changed, 1 insertion(+), 1050 deletions(-)
delete mode 100644 Documentation/core-api/folio_queue.rst
delete mode 100644 fs/netfs/rolling_buffer.c
delete mode 100644 include/linux/folio_queue.h
delete mode 100644 include/linux/rolling_buffer.h
diff --git a/Documentation/core-api/folio_queue.rst b/Documentation/core-api/folio_queue.rst
deleted file mode 100644
index b7628896d2b6..000000000000
--- a/Documentation/core-api/folio_queue.rst
+++ /dev/null
@@ -1,209 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0+
-
-===========
-Folio Queue
-===========
-
-:Author: David Howells <dhowells@redhat.com>
-
-.. Contents:
-
- * Overview
- * Initialisation
- * Adding and removing folios
- * Querying information about a folio
- * Querying information about a folio_queue
- * Folio queue iteration
- * Folio marks
- * Lockless simultaneous production/consumption issues
-
-
-Overview
-========
-
-The folio_queue struct forms a single segment in a segmented list of folios
-that can be used to form an I/O buffer. As such, the list can be iterated over
-using the ITER_FOLIOQ iov_iter type.
-
-The publicly accessible members of the structure are::
-
- struct folio_queue {
- struct folio_queue *next;
- struct folio_queue *prev;
- ...
- };
-
-A pair of pointers are provided, ``next`` and ``prev``, that point to the
-segments on either side of the segment being accessed. Whilst this is a
-doubly-linked list, it is intentionally not a circular list; the outward
-sibling pointers in terminal segments should be NULL.
-
-Each segment in the list also stores:
-
- * an ordered sequence of folio pointers,
- * the size of each folio and
- * three 1-bit marks per folio,
-
-but these should not be accessed directly as the underlying data structure may
-change, but rather the access functions outlined below should be used.
-
-The facility can be made accessible by::
-
- #include <linux/folio_queue.h>
-
-and to use the iterator::
-
- #include <linux/uio.h>
-
-
-Initialisation
-==============
-
-A segment should be initialised by calling::
-
- void folioq_init(struct folio_queue *folioq);
-
-with a pointer to the segment to be initialised. Note that this will not
-necessarily initialise all the folio pointers, so care must be taken to check
-the number of folios added.
-
-
-Adding and removing folios
-==========================
-
-Folios can be set in the next unused slot in a segment struct by calling one
-of::
-
- unsigned int folioq_append(struct folio_queue *folioq,
- struct folio *folio);
-
- unsigned int folioq_append_mark(struct folio_queue *folioq,
- struct folio *folio);
-
-Both functions update the stored folio count, store the folio and note its
-size. The second function also sets the first mark for the folio added. Both
-functions return the number of the slot used. [!] Note that no attempt is made
-to check that the capacity wasn't overrun and the list will not be extended
-automatically.
-
-A folio can be excised by calling::
-
- void folioq_clear(struct folio_queue *folioq, unsigned int slot);
-
-This clears the slot in the array and also clears all the marks for that folio,
-but doesn't change the folio count - so future accesses of that slot must check
-if the slot is occupied.
-
-
-Querying information about a folio
-==================================
-
-Information about the folio in a particular slot may be queried by the
-following function::
-
- struct folio *folioq_folio(const struct folio_queue *folioq,
- unsigned int slot);
-
-If a folio has not yet been set in that slot, this may yield an undefined
-pointer. The size of the folio in a slot may be queried with either of::
-
- unsigned int folioq_folio_order(const struct folio_queue *folioq,
- unsigned int slot);
-
- size_t folioq_folio_size(const struct folio_queue *folioq,
- unsigned int slot);
-
-The first function returns the size as an order and the second as a number of
-bytes.
-
-
-Querying information about a folio_queue
-========================================
-
-Information may be retrieved about a particular segment with the following
-functions::
-
- unsigned int folioq_nr_slots(const struct folio_queue *folioq);
-
- unsigned int folioq_count(struct folio_queue *folioq);
-
- bool folioq_full(struct folio_queue *folioq);
-
-The first function returns the maximum capacity of a segment. It must not be
-assumed that this won't vary between segments. The second returns the number
-of folios added to a segments and the third is a shorthand to indicate if the
-segment has been filled to capacity.
-
-Not that the count and fullness are not affected by clearing folios from the
-segment. These are more about indicating how many slots in the array have been
-initialised, and it assumed that slots won't get reused, but rather the segment
-will get discarded as the queue is consumed.
-
-
-Folio marks
-===========
-
-Folios within a queue can also have marks assigned to them. These marks can be
-used to note information such as if a folio needs folio_put() calling upon it.
-There are three marks available to be set for each folio.
-
-The marks can be set by::
-
- void folioq_mark(struct folio_queue *folioq, unsigned int slot);
- void folioq_mark2(struct folio_queue *folioq, unsigned int slot);
-
-Cleared by::
-
- void folioq_unmark(struct folio_queue *folioq, unsigned int slot);
- void folioq_unmark2(struct folio_queue *folioq, unsigned int slot);
-
-And the marks can be queried by::
-
- bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot);
- bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot);
-
-The marks can be used for any purpose and are not interpreted by this API.
-
-
-Folio queue iteration
-=====================
-
-A list of segments may be iterated over using the I/O iterator facility using
-an ``iov_iter`` iterator of ``ITER_FOLIOQ`` type. The iterator may be
-initialised with::
-
- void iov_iter_folio_queue(struct iov_iter *i, unsigned int direction,
- const struct folio_queue *folioq,
- unsigned int first_slot, unsigned int offset,
- size_t count);
-
-This may be told to start at a particular segment, slot and offset within a
-queue. The iov iterator functions will follow the next pointers when advancing
-and prev pointers when reverting when needed.
-
-
-Lockless simultaneous production/consumption issues
-===================================================
-
-If properly managed, the list can be extended by the producer at the head end
-and shortened by the consumer at the tail end simultaneously without the need
-to take locks. The ITER_FOLIOQ iterator inserts appropriate barriers to aid
-with this.
-
-Care must be taken when simultaneously producing and consuming a list. If the
-last segment is reached and the folios it refers to are entirely consumed by
-the IOV iterators, an iov_iter struct will be left pointing to the last segment
-with a slot number equal to the capacity of that segment. The iterator will
-try to continue on from this if there's another segment available when it is
-used again, but care must be taken lest the segment got removed and freed by
-the consumer before the iterator was advanced.
-
-It is recommended that the queue always contain at least one segment, even if
-that segment has never been filled or is entirely spent. This prevents the
-head and tail pointers from collapsing.
-
-
-API Function Reference
-======================
-
-.. kernel-doc:: include/linux/folio_queue.h
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
index 13769d5c40bf..16c529a33ac4 100644
--- a/Documentation/core-api/index.rst
+++ b/Documentation/core-api/index.rst
@@ -39,7 +39,6 @@ Library functionality that is used throughout the kernel.
kref
cleanup
assoc_array
- folio_queue
xarray
maple_tree
idr
diff --git a/Documentation/filesystems/netfs_library.rst b/Documentation/filesystems/netfs_library.rst
index ddd799df6ce3..18e3c3aae57c 100644
--- a/Documentation/filesystems/netfs_library.rst
+++ b/Documentation/filesystems/netfs_library.rst
@@ -449,7 +449,7 @@ be called from the writeback code to write the data to the cache, if there is
one.
The inode should be marked ``NETFS_ICTX_SINGLE_NO_UPLOAD`` if this API is to be
-used. The writeback function requires the buffer to be of ITER_FOLIOQ type.
+used.
High-Level VM API
==================
diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c
index 566693ac47ef..3040be52c293 100644
--- a/fs/netfs/iterator.c
+++ b/fs/netfs/iterator.c
@@ -139,195 +139,3 @@ ssize_t netfs_extract_iter(struct iov_iter *orig, size_t max_len, size_t max_pag
return extracted ?: ret;
}
EXPORT_SYMBOL_GPL(netfs_extract_iter);
-
-#if 0
-/*
- * Select the span of a bvec iterator we're going to use. Limit it by both maximum
- * size and maximum number of segments. Returns the size of the span in bytes.
- */
-static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset,
- size_t max_size, size_t max_segs)
-{
- const struct bio_vec *bvecs = iter->bvec;
- unsigned int nbv = iter->nr_segs, ix = 0, nsegs = 0;
- size_t len, span = 0, n = iter->count;
- size_t skip = iter->iov_offset + start_offset;
-
- if (WARN_ON(!iov_iter_is_bvec(iter)) ||
- WARN_ON(start_offset > n) ||
- n == 0)
- return 0;
-
- while (n && ix < nbv && skip) {
- len = bvecs[ix].bv_len;
- if (skip < len)
- break;
- skip -= len;
- n -= len;
- ix++;
- }
-
- while (n && ix < nbv) {
- len = min3(n, bvecs[ix].bv_len - skip, max_size);
- span += len;
- nsegs++;
- ix++;
- if (span >= max_size || nsegs >= max_segs)
- break;
- skip = 0;
- n -= len;
- }
-
- return min(span, max_size);
-}
-
-/*
- * Select the span of a kvec iterator we're going to use. Limit it by both
- * maximum size and maximum number of segments. Returns the size of the span
- * in bytes.
- */
-static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset,
- size_t max_size, size_t max_segs)
-{
- const struct kvec *kvecs = iter->kvec;
- unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0;
- size_t len, span = 0, n = iter->count;
- size_t skip = iter->iov_offset + start_offset;
-
- if (WARN_ON(!iov_iter_is_kvec(iter)) ||
- WARN_ON(start_offset > n) ||
- n == 0)
- return 0;
-
- while (n && ix < nkv && skip) {
- len = kvecs[ix].iov_len;
- if (skip < len)
- break;
- skip -= len;
- n -= len;
- ix++;
- }
-
- while (n && ix < nkv) {
- len = min3(n, kvecs[ix].iov_len - skip, max_size);
- span += len;
- nsegs++;
- ix++;
- if (span >= max_size || nsegs >= max_segs)
- break;
- skip = 0;
- n -= len;
- }
-
- return min(span, max_size);
-}
-
-/*
- * Select the span of an xarray iterator we're going to use. Limit it by both
- * maximum size and maximum number of segments. It is assumed that segments
- * can be larger than a page in size, provided they're physically contiguous.
- * Returns the size of the span in bytes.
- */
-static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offset,
- size_t max_size, size_t max_segs)
-{
- struct folio *folio;
- unsigned int nsegs = 0;
- loff_t pos = iter->xarray_start + iter->iov_offset;
- pgoff_t index = pos / PAGE_SIZE;
- size_t span = 0, n = iter->count;
-
- XA_STATE(xas, iter->xarray, index);
-
- if (WARN_ON(!iov_iter_is_xarray(iter)) ||
- WARN_ON(start_offset > n) ||
- n == 0)
- return 0;
- max_size = min(max_size, n - start_offset);
-
- rcu_read_lock();
- xas_for_each(&xas, folio, ULONG_MAX) {
- size_t offset, flen, len;
- if (xas_retry(&xas, folio))
- continue;
- if (WARN_ON(xa_is_value(folio)))
- break;
- if (WARN_ON(folio_test_hugetlb(folio)))
- break;
-
- flen = folio_size(folio);
- offset = offset_in_folio(folio, pos);
- len = min(max_size, flen - offset);
- span += len;
- nsegs++;
- if (span >= max_size || nsegs >= max_segs)
- break;
- }
-
- rcu_read_unlock();
- return min(span, max_size);
-}
-
-/*
- * Select the span of a folio queue iterator we're going to use. Limit it by
- * both maximum size and maximum number of segments. Returns the size of the
- * span in bytes.
- */
-static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset,
- size_t max_size, size_t max_segs)
-{
- const struct folio_queue *folioq = iter->folioq;
- unsigned int nsegs = 0;
- unsigned int slot = iter->folioq_slot;
- size_t span = 0, n = iter->count;
-
- if (WARN_ON(!iov_iter_is_folioq(iter)) ||
- WARN_ON(start_offset > n) ||
- n == 0)
- return 0;
- max_size = umin(max_size, n - start_offset);
-
- if (slot >= folioq_nr_slots(folioq)) {
- folioq = folioq->next;
- slot = 0;
- }
-
- start_offset += iter->iov_offset;
- do {
- size_t flen = folioq_folio_size(folioq, slot);
-
- if (start_offset < flen) {
- span += flen - start_offset;
- nsegs++;
- start_offset = 0;
- } else {
- start_offset -= flen;
- }
- if (span >= max_size || nsegs >= max_segs)
- break;
-
- slot++;
- if (slot >= folioq_nr_slots(folioq)) {
- folioq = folioq->next;
- slot = 0;
- }
- } while (folioq);
-
- return umin(span, max_size);
-}
-
-size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
- size_t max_size, size_t max_segs)
-{
- if (iov_iter_is_folioq(iter))
- return netfs_limit_folioq(iter, start_offset, max_size, max_segs);
- if (iov_iter_is_bvec(iter))
- return netfs_limit_bvec(iter, start_offset, max_size, max_segs);
- if (iov_iter_is_xarray(iter))
- return netfs_limit_xarray(iter, start_offset, max_size, max_segs);
- if (iov_iter_is_kvec(iter))
- return netfs_limit_kvec(iter, start_offset, max_size, max_segs);
- BUG();
-}
-EXPORT_SYMBOL(netfs_limit_iter);
-#endif
diff --git a/fs/netfs/rolling_buffer.c b/fs/netfs/rolling_buffer.c
deleted file mode 100644
index 576b425a227d..000000000000
--- a/fs/netfs/rolling_buffer.c
+++ /dev/null
@@ -1,297 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* Rolling buffer helpers
- *
- * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- */
-
-#include <linux/bitops.h>
-#include <linux/pagemap.h>
-#include <linux/rolling_buffer.h>
-#include <linux/slab.h>
-#include "internal.h"
-
-static atomic_t debug_ids;
-
-/**
- * netfs_folioq_alloc - Allocate a folio_queue struct
- * @rreq_id: Associated debugging ID for tracing purposes
- * @gfp: Allocation constraints
- * @trace: Trace tag to indicate the purpose of the allocation
- *
- * Allocate, initialise and account the folio_queue struct and log a trace line
- * to mark the allocation.
- */
-struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp,
- unsigned int /*enum netfs_folioq_trace*/ trace)
-{
- struct folio_queue *fq;
-
- fq = kmalloc_obj(*fq, gfp);
- if (fq) {
- netfs_stat(&netfs_n_folioq);
- folioq_init(fq, rreq_id);
- fq->debug_id = atomic_inc_return(&debug_ids);
- trace_netfs_folioq(fq, trace);
- }
- return fq;
-}
-EXPORT_SYMBOL(netfs_folioq_alloc);
-
-/**
- * netfs_folioq_free - Free a folio_queue struct
- * @folioq: The object to free
- * @trace: Trace tag to indicate which free
- *
- * Free and unaccount the folio_queue struct.
- */
-void netfs_folioq_free(struct folio_queue *folioq,
- unsigned int /*enum netfs_trace_folioq*/ trace)
-{
- trace_netfs_folioq(folioq, trace);
- netfs_stat_d(&netfs_n_folioq);
- kfree(folioq);
-}
-EXPORT_SYMBOL(netfs_folioq_free);
-
-/*
- * Initialise a rolling buffer. We allocate an empty folio queue struct to so
- * that the pointers can be independently driven by the producer and the
- * consumer.
- */
-int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id,
- unsigned int direction)
-{
- struct folio_queue *fq;
-
- fq = netfs_folioq_alloc(rreq_id, GFP_NOFS, netfs_trace_folioq_rollbuf_init);
- if (!fq)
- return -ENOMEM;
-
- roll->head = fq;
- roll->tail = fq;
- iov_iter_folio_queue(&roll->iter, direction, fq, 0, 0, 0);
- return 0;
-}
-
-/*
- * Add another folio_queue to a rolling buffer if there's no space left.
- */
-int rolling_buffer_make_space(struct rolling_buffer *roll)
-{
- struct folio_queue *fq, *head = roll->head;
-
- if (!folioq_full(head))
- return 0;
-
- fq = netfs_folioq_alloc(head->rreq_id, GFP_NOFS, netfs_trace_folioq_make_space);
- if (!fq)
- return -ENOMEM;
- fq->prev = head;
-
- roll->head = fq;
- if (folioq_full(head)) {
- /* Make sure we don't leave the master iterator pointing to a
- * block that might get immediately consumed.
- */
- if (roll->iter.folioq == head &&
- roll->iter.folioq_slot == folioq_nr_slots(head)) {
- roll->iter.folioq = fq;
- roll->iter.folioq_slot = 0;
- }
- }
-
- /* Make sure the initialisation is stored before the next pointer.
- *
- * [!] NOTE: After we set head->next, the consumer is at liberty to
- * immediately delete the old head.
- */
- smp_store_release(&head->next, fq);
- return 0;
-}
-
-/*
- * Decant the list of folios to read into a rolling buffer.
- */
-ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll,
- struct readahead_control *ractl,
- struct folio_batch *put_batch)
-{
- struct folio_queue *fq;
- struct page **vec;
- int nr, ix, to;
- ssize_t size = 0;
-
- if (rolling_buffer_make_space(roll) < 0)
- return -ENOMEM;
-
- fq = roll->head;
- vec = (struct page **)fq->vec.folios;
- nr = __readahead_batch(ractl, vec + folio_batch_count(&fq->vec),
- folio_batch_space(&fq->vec));
- ix = fq->vec.nr;
- to = ix + nr;
- fq->vec.nr = to;
- for (; ix < to; ix++) {
- struct folio *folio = folioq_folio(fq, ix);
- unsigned int order = folio_order(folio);
-
- fq->orders[ix] = order;
- size += PAGE_SIZE << order;
- trace_netfs_folio(folio, netfs_folio_trace_read);
- if (!folio_batch_add(put_batch, folio))
- folio_batch_release(put_batch);
- }
- WRITE_ONCE(roll->iter.count, roll->iter.count + size);
-
- /* Store the counter after setting the slot. */
- smp_store_release(&roll->next_head_slot, to);
- return size;
-}
-
-/*
- * Decant the entire list of folios to read into a rolling buffer.
- */
-ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll,
- struct readahead_control *ractl,
- unsigned int rreq_id)
-{
- XA_STATE(xas, &ractl->mapping->i_pages, ractl->_index);
- struct folio_queue *fq;
- struct folio *folio;
- ssize_t loaded = 0;
- int nr, slot = 0, npages = 0;
-
- /* First allocate all the folioqs we're going to need to avoid having
- * to deal with ENOMEM later.
- */
- nr = ractl->_nr_folios;
- do {
- fq = netfs_folioq_alloc(rreq_id, GFP_KERNEL,
- netfs_trace_folioq_make_space);
- if (!fq) {
- rolling_buffer_clear(roll);
- return -ENOMEM;
- }
- fq->prev = roll->head;
- if (!roll->tail)
- roll->tail = fq;
- else
- roll->head->next = fq;
- roll->head = fq;
-
- nr -= folioq_nr_slots(fq);
- } while (nr > 0);
-
- rcu_read_lock();
-
- fq = roll->tail;
- xas_for_each(&xas, folio, ractl->_index + ractl->_nr_pages - 1) {
- unsigned int order;
-
- if (xas_retry(&xas, folio))
- continue;
- VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
-
- order = folio_order(folio);
- fq->orders[slot] = order;
- fq->vec.folios[slot] = folio;
- loaded += PAGE_SIZE << order;
- npages += 1 << order;
- trace_netfs_folio(folio, netfs_folio_trace_read);
-
- slot++;
- if (slot >= folioq_nr_slots(fq)) {
- fq->vec.nr = slot;
- fq = fq->next;
- if (!fq) {
- WARN_ON_ONCE(npages < readahead_count(ractl));
- break;
- }
- slot = 0;
- }
- }
-
- rcu_read_unlock();
-
- if (fq)
- fq->vec.nr = slot;
-
- WRITE_ONCE(roll->iter.count, loaded);
- iov_iter_folio_queue(&roll->iter, ITER_DEST, roll->tail, 0, 0, loaded);
- ractl->_index += npages;
- ractl->_nr_pages -= npages;
- return loaded;
-}
-
-/*
- * Append a folio to the rolling buffer.
- */
-ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio,
- unsigned int flags)
-{
- ssize_t size = folio_size(folio);
- int slot;
-
- if (rolling_buffer_make_space(roll) < 0)
- return -ENOMEM;
-
- slot = folioq_append(roll->head, folio);
- if (flags & ROLLBUF_MARK_1)
- folioq_mark(roll->head, slot);
- if (flags & ROLLBUF_MARK_2)
- folioq_mark2(roll->head, slot);
-
- WRITE_ONCE(roll->iter.count, roll->iter.count + size);
-
- /* Store the counter after setting the slot. */
- smp_store_release(&roll->next_head_slot, slot);
- return size;
-}
-
-/*
- * Delete a spent buffer from a rolling queue and return the next in line. We
- * don't return the last buffer to keep the pointers independent, but return
- * NULL instead.
- */
-struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll)
-{
- struct folio_queue *spent = roll->tail, *next = READ_ONCE(spent->next);
-
- if (!next)
- return NULL;
- next->prev = NULL;
- netfs_folioq_free(spent, netfs_trace_folioq_delete);
- roll->tail = next;
- return next;
-}
-
-/*
- * Clear out a rolling queue. Folios that have mark 1 set are put.
- */
-void rolling_buffer_clear(struct rolling_buffer *roll)
-{
- struct folio_batch fbatch;
- struct folio_queue *p;
-
- folio_batch_init(&fbatch);
-
- while ((p = roll->tail)) {
- roll->tail = p->next;
- for (int slot = 0; slot < folioq_count(p); slot++) {
- struct folio *folio = folioq_folio(p, slot);
-
- if (!folio)
- continue;
- if (folioq_is_marked(p, slot)) {
- trace_netfs_folio(folio, netfs_folio_trace_put);
- if (!folio_batch_add(&fbatch, folio))
- folio_batch_release(&fbatch);
- }
- }
-
- netfs_folioq_free(p, netfs_trace_folioq_clear);
- }
-
- folio_batch_release(&fbatch);
-}
diff --git a/include/linux/folio_queue.h b/include/linux/folio_queue.h
deleted file mode 100644
index f6d5f1f127c9..000000000000
--- a/include/linux/folio_queue.h
+++ /dev/null
@@ -1,282 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Queue of folios definitions
- *
- * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- *
- * See:
- *
- * Documentation/core-api/folio_queue.rst
- *
- * for a description of the API.
- */
-
-#ifndef _LINUX_FOLIO_QUEUE_H
-#define _LINUX_FOLIO_QUEUE_H
-
-#include <linux/folio_batch.h>
-#include <linux/mm.h>
-
-/*
- * Segment in a queue of running buffers. Each segment can hold a number of
- * folios and a portion of the queue can be referenced with the ITER_FOLIOQ
- * iterator. The possibility exists of inserting non-folio elements into the
- * queue (such as gaps).
- *
- * Explicit prev and next pointers are used instead of a list_head to make it
- * easier to add segments to tail and remove them from the head without the
- * need for a lock.
- */
-struct folio_queue {
- struct folio_batch vec; /* Folios in the queue segment */
- u8 orders[FOLIO_BATCH_SIZE]; /* Order of each folio */
- struct folio_queue *next; /* Next queue segment or NULL */
- struct folio_queue *prev; /* Previous queue segment of NULL */
- unsigned long marks; /* 1-bit mark per folio */
- unsigned long marks2; /* Second 1-bit mark per folio */
-#if FOLIO_BATCH_SIZE > BITS_PER_LONG
-#error marks is not big enough
-#endif
- unsigned int rreq_id;
- unsigned int debug_id;
-};
-
-/**
- * folioq_init - Initialise a folio queue segment
- * @folioq: The segment to initialise
- * @rreq_id: The request identifier to use in tracelines.
- *
- * Initialise a folio queue segment and set an identifier to be used in traces.
- *
- * Note that the folio pointers are left uninitialised.
- */
-static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id)
-{
- folio_batch_init(&folioq->vec);
- folioq->next = NULL;
- folioq->prev = NULL;
- folioq->marks = 0;
- folioq->marks2 = 0;
- folioq->rreq_id = rreq_id;
- folioq->debug_id = 0;
-}
-
-/**
- * folioq_nr_slots: Query the capacity of a folio queue segment
- * @folioq: The segment to query
- *
- * Query the number of folios that a particular folio queue segment might hold.
- * [!] NOTE: This must not be assumed to be the same for every segment!
- */
-static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
-{
- return FOLIO_BATCH_SIZE;
-}
-
-/**
- * folioq_count: Query the occupancy of a folio queue segment
- * @folioq: The segment to query
- *
- * Query the number of folios that have been added to a folio queue segment.
- * Note that this is not decreased as folios are removed from a segment.
- */
-static inline unsigned int folioq_count(struct folio_queue *folioq)
-{
- return folio_batch_count(&folioq->vec);
-}
-
-/**
- * folioq_full: Query if a folio queue segment is full
- * @folioq: The segment to query
- *
- * Query if a folio queue segment is fully occupied. Note that this does not
- * change if folios are removed from a segment.
- */
-static inline bool folioq_full(struct folio_queue *folioq)
-{
- //return !folio_batch_space(&folioq->vec);
- return folioq_count(folioq) >= folioq_nr_slots(folioq);
-}
-
-/**
- * folioq_is_marked: Check first folio mark in a folio queue segment
- * @folioq: The segment to query
- * @slot: The slot number of the folio to query
- *
- * Determine if the first mark is set for the folio in the specified slot in a
- * folio queue segment.
- */
-static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot)
-{
- return test_bit(slot, &folioq->marks);
-}
-
-/**
- * folioq_mark: Set the first mark on a folio in a folio queue segment
- * @folioq: The segment to modify
- * @slot: The slot number of the folio to modify
- *
- * Set the first mark for the folio in the specified slot in a folio queue
- * segment.
- */
-static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot)
-{
- set_bit(slot, &folioq->marks);
-}
-
-/**
- * folioq_unmark: Clear the first mark on a folio in a folio queue segment
- * @folioq: The segment to modify
- * @slot: The slot number of the folio to modify
- *
- * Clear the first mark for the folio in the specified slot in a folio queue
- * segment.
- */
-static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot)
-{
- clear_bit(slot, &folioq->marks);
-}
-
-/**
- * folioq_is_marked2: Check second folio mark in a folio queue segment
- * @folioq: The segment to query
- * @slot: The slot number of the folio to query
- *
- * Determine if the second mark is set for the folio in the specified slot in a
- * folio queue segment.
- */
-static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot)
-{
- return test_bit(slot, &folioq->marks2);
-}
-
-/**
- * folioq_mark2: Set the second mark on a folio in a folio queue segment
- * @folioq: The segment to modify
- * @slot: The slot number of the folio to modify
- *
- * Set the second mark for the folio in the specified slot in a folio queue
- * segment.
- */
-static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot)
-{
- set_bit(slot, &folioq->marks2);
-}
-
-/**
- * folioq_unmark2: Clear the second mark on a folio in a folio queue segment
- * @folioq: The segment to modify
- * @slot: The slot number of the folio to modify
- *
- * Clear the second mark for the folio in the specified slot in a folio queue
- * segment.
- */
-static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot)
-{
- clear_bit(slot, &folioq->marks2);
-}
-
-/**
- * folioq_append: Add a folio to a folio queue segment
- * @folioq: The segment to add to
- * @folio: The folio to add
- *
- * Add a folio to the tail of the sequence in a folio queue segment, increasing
- * the occupancy count and returning the slot number for the folio just added.
- * The folio size is extracted and stored in the queue and the marks are left
- * unmodified.
- *
- * Note that it's left up to the caller to check that the segment capacity will
- * not be exceeded and to extend the queue.
- */
-static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio)
-{
- unsigned int slot = folioq->vec.nr++;
-
- folioq->vec.folios[slot] = folio;
- folioq->orders[slot] = folio_order(folio);
- return slot;
-}
-
-/**
- * folioq_append_mark: Add a folio to a folio queue segment
- * @folioq: The segment to add to
- * @folio: The folio to add
- *
- * Add a folio to the tail of the sequence in a folio queue segment, increasing
- * the occupancy count and returning the slot number for the folio just added.
- * The folio size is extracted and stored in the queue, the first mark is set
- * and and the second and third marks are left unmodified.
- *
- * Note that it's left up to the caller to check that the segment capacity will
- * not be exceeded and to extend the queue.
- */
-static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio)
-{
- unsigned int slot = folioq->vec.nr++;
-
- folioq->vec.folios[slot] = folio;
- folioq->orders[slot] = folio_order(folio);
- folioq_mark(folioq, slot);
- return slot;
-}
-
-/**
- * folioq_folio: Get a folio from a folio queue segment
- * @folioq: The segment to access
- * @slot: The folio slot to access
- *
- * Retrieve the folio in the specified slot from a folio queue segment. Note
- * that no bounds check is made and if the slot hasn't been added into yet, the
- * pointer will be undefined. If the slot has been cleared, NULL will be
- * returned.
- */
-static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot)
-{
- return folioq->vec.folios[slot];
-}
-
-/**
- * folioq_folio_order: Get the order of a folio from a folio queue segment
- * @folioq: The segment to access
- * @slot: The folio slot to access
- *
- * Retrieve the order of the folio in the specified slot from a folio queue
- * segment. Note that no bounds check is made and if the slot hasn't been
- * added into yet, the order returned will be 0.
- */
-static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot)
-{
- return folioq->orders[slot];
-}
-
-/**
- * folioq_folio_size: Get the size of a folio from a folio queue segment
- * @folioq: The segment to access
- * @slot: The folio slot to access
- *
- * Retrieve the size of the folio in the specified slot from a folio queue
- * segment. Note that no bounds check is made and if the slot hasn't been
- * added into yet, the size returned will be PAGE_SIZE.
- */
-static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot)
-{
- return PAGE_SIZE << folioq_folio_order(folioq, slot);
-}
-
-/**
- * folioq_clear: Clear a folio from a folio queue segment
- * @folioq: The segment to clear
- * @slot: The folio slot to clear
- *
- * Clear a folio from a sequence in a folio queue segment and clear its marks.
- * The occupancy count is left unchanged.
- */
-static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot)
-{
- folioq->vec.folios[slot] = NULL;
- folioq_unmark(folioq, slot);
- folioq_unmark2(folioq, slot);
-}
-
-#endif /* _LINUX_FOLIO_QUEUE_H */
diff --git a/include/linux/netfs.h b/include/linux/netfs.h
index d0b1408bd02f..7dca6a513509 100644
--- a/include/linux/netfs.h
+++ b/include/linux/netfs.h
@@ -464,8 +464,6 @@ void netfs_put_subrequest(struct netfs_io_subrequest *subreq,
ssize_t netfs_extract_iter(struct iov_iter *orig, size_t max_len, size_t max_pages,
unsigned long long fpos, struct bvecq **_bvecq_head,
iov_iter_extraction_t extraction_flags);
-size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
- size_t max_size, size_t max_segs);
void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq);
void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error);
diff --git a/include/linux/rolling_buffer.h b/include/linux/rolling_buffer.h
deleted file mode 100644
index b35ef43f325f..000000000000
--- a/include/linux/rolling_buffer.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Rolling buffer of folios
- *
- * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- */
-
-#ifndef _ROLLING_BUFFER_H
-#define _ROLLING_BUFFER_H
-
-#include <linux/folio_queue.h>
-#include <linux/uio.h>
-
-/*
- * Rolling buffer. Whilst the buffer is live and in use, folios and folio
- * queue segments can be added to one end by one thread and removed from the
- * other end by another thread. The buffer isn't allowed to be empty; it must
- * always have at least one folio_queue in it so that neither side has to
- * modify both queue pointers.
- *
- * The iterator in the buffer is extended as buffers are inserted. It can be
- * snapshotted to use a segment of the buffer.
- */
-struct rolling_buffer {
- struct folio_queue *head; /* Producer's insertion point */
- struct folio_queue *tail; /* Consumer's removal point */
- struct iov_iter iter; /* Iterator tracking what's left in the buffer */
- u8 next_head_slot; /* Next slot in ->head */
- u8 first_tail_slot; /* First slot in ->tail */
-};
-
-/*
- * Snapshot of a rolling buffer.
- */
-struct rolling_buffer_snapshot {
- struct folio_queue *curr_folioq; /* Queue segment in which current folio resides */
- unsigned char curr_slot; /* Folio currently being read */
- unsigned char curr_order; /* Order of folio */
-};
-
-/* Marks to store per-folio in the internal folio_queue structs. */
-#define ROLLBUF_MARK_1 BIT(0)
-#define ROLLBUF_MARK_2 BIT(1)
-
-int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id,
- unsigned int direction);
-int rolling_buffer_make_space(struct rolling_buffer *roll);
-ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll,
- struct readahead_control *ractl,
- struct folio_batch *put_batch);
-ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll,
- struct readahead_control *ractl,
- unsigned int rreq_id);
-ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio,
- unsigned int flags);
-struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll);
-void rolling_buffer_clear(struct rolling_buffer *roll);
-
-static inline void rolling_buffer_advance(struct rolling_buffer *roll, size_t amount)
-{
- iov_iter_advance(&roll->iter, amount);
-}
-
-#endif /* _ROLLING_BUFFER_H */
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index a62d78581207..dface7d06b98 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -6734,8 +6734,6 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
/* amdgpu */
{ "amdgpu_vm_bo_map", 0x1 },
{ "amdgpu_vm_bo_unmap", 0x1 },
- /* netfs */
- { "netfs_folioq", 0x1 },
/* xfs from xfs_defer_pending_class */
{ "xfs_defer_create_intent", 0x1 },
{ "xfs_defer_cancel_list", 0x1 },
^ permalink raw reply related
* [PATCH v3 18/22] iov_iter: Remove ITER_FOLIOQ
From: David Howells @ 2026-06-08 14:54 UTC (permalink / raw)
To: Christian Brauner, Matthew Wilcox, Christoph Hellwig
Cc: David Howells, Paulo Alcantara, Jens Axboe, Leon Romanovsky,
Steve French, ChenXiaoSong, Marc Dionne, Eric Van Hensbergen,
Dominique Martinet, Ilya Dryomov, Trond Myklebust, netfs,
linux-afs, linux-cifs, linux-nfs, ceph-devel, v9fs, linux-erofs,
linux-fsdevel, linux-kernel
In-Reply-To: <20260608145432.681865-1-dhowells@redhat.com>
Remove ITER_FOLIOQ as it's no longer used.
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Paulo Alcantara <pc@manguebit.org>
cc: Matthew Wilcox <willy@infradead.org>
cc: Christoph Hellwig <hch@infradead.org>
cc: Steve French <sfrench@samba.org>
cc: linux-cifs@vger.kernel.org
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
---
include/linux/iov_iter.h | 65 +--------
include/linux/uio.h | 12 --
lib/iov_iter.c | 175 +-----------------------
lib/scatterlist.c | 69 +---------
lib/tests/kunit_iov_iter.c | 271 -------------------------------------
5 files changed, 7 insertions(+), 585 deletions(-)
diff --git a/include/linux/iov_iter.h b/include/linux/iov_iter.h
index c19a4c561ab4..c4ed8dafa92f 100644
--- a/include/linux/iov_iter.h
+++ b/include/linux/iov_iter.h
@@ -11,7 +11,6 @@
#include <linux/uio.h>
#include <linux/bvec.h>
#include <linux/bvecq.h>
-#include <linux/folio_queue.h>
typedef size_t (*iov_step_f)(void *iter_base, size_t progress, size_t len,
void *priv, void *priv2);
@@ -202,62 +201,6 @@ size_t iterate_bvecq(struct iov_iter *iter, size_t len, void *priv, void *priv2,
return progress;
}
-/*
- * Handle ITER_FOLIOQ.
- */
-static __always_inline
-size_t iterate_folioq(struct iov_iter *iter, size_t len, void *priv, void *priv2,
- iov_step_f step)
-{
- const struct folio_queue *folioq = iter->folioq;
- unsigned int slot = iter->folioq_slot;
- size_t progress = 0, skip = iter->iov_offset;
-
- if (slot == folioq_nr_slots(folioq)) {
- /* The iterator may have been extended. */
- folioq = folioq->next;
- slot = 0;
- }
-
- do {
- struct folio *folio = folioq_folio(folioq, slot);
- size_t part, remain = 0, consumed;
- size_t fsize;
- void *base;
-
- if (!folio)
- break;
-
- fsize = folioq_folio_size(folioq, slot);
- if (skip < fsize) {
- base = kmap_local_folio(folio, skip);
- part = umin(len, PAGE_SIZE - skip % PAGE_SIZE);
- remain = step(base, progress, part, priv, priv2);
- kunmap_local(base);
- consumed = part - remain;
- len -= consumed;
- progress += consumed;
- skip += consumed;
- }
- if (skip >= fsize) {
- skip = 0;
- slot++;
- if (slot == folioq_nr_slots(folioq) && folioq->next) {
- folioq = folioq->next;
- slot = 0;
- }
- }
- if (remain)
- break;
- } while (len);
-
- iter->folioq_slot = slot;
- iter->folioq = folioq;
- iter->iov_offset = skip;
- iter->count -= progress;
- return progress;
-}
-
/*
* Handle ITER_XARRAY.
*/
@@ -369,8 +312,6 @@ size_t iterate_and_advance2(struct iov_iter *iter, size_t len, void *priv,
return iterate_kvec(iter, len, priv, priv2, step);
if (iov_iter_is_bvecq(iter))
return iterate_bvecq(iter, len, priv, priv2, step);
- if (iov_iter_is_folioq(iter))
- return iterate_folioq(iter, len, priv, priv2, step);
if (iov_iter_is_xarray(iter))
return iterate_xarray(iter, len, priv, priv2, step);
return iterate_discard(iter, len, priv, priv2, step);
@@ -405,8 +346,8 @@ size_t iterate_and_advance(struct iov_iter *iter, size_t len, void *priv,
* buffer is presented in segments, which for kernel iteration are broken up by
* physical pages and mapped, with the mapped address being presented.
*
- * [!] Note This will only handle BVEC, KVEC, BVECQ, FOLIOQ, XARRAY and
- * DISCARD-type iterators; it will not handle UBUF or IOVEC-type iterators.
+ * [!] Note This will only handle BVEC, KVEC, BVECQ, XARRAY and DISCARD-type
+ * iterators; it will not handle UBUF or IOVEC-type iterators.
*
* A step functions, @step, must be provided, one for handling mapped kernel
* addresses and the other is given user addresses which have the potential to
@@ -435,8 +376,6 @@ size_t iterate_and_advance_kernel(struct iov_iter *iter, size_t len, void *priv,
return iterate_kvec(iter, len, priv, priv2, step);
if (iov_iter_is_bvecq(iter))
return iterate_bvecq(iter, len, priv, priv2, step);
- if (iov_iter_is_folioq(iter))
- return iterate_folioq(iter, len, priv, priv2, step);
if (iov_iter_is_xarray(iter))
return iterate_xarray(iter, len, priv, priv2, step);
return iterate_discard(iter, len, priv, priv2, step);
diff --git a/include/linux/uio.h b/include/linux/uio.h
index f7cfa6ea8213..e84a0c4f28c6 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -11,7 +11,6 @@
#include <uapi/linux/uio.h>
struct page;
-struct folio_queue;
typedef unsigned int __bitwise iov_iter_extraction_t;
@@ -27,7 +26,6 @@ enum iter_type {
ITER_BVEC,
ITER_KVEC,
ITER_BVECQ,
- ITER_FOLIOQ,
ITER_XARRAY,
ITER_DISCARD,
};
@@ -70,7 +68,6 @@ struct iov_iter {
const struct kvec *kvec;
const struct bio_vec *bvec;
const struct bvecq *bvecq;
- const struct folio_queue *folioq;
struct xarray *xarray;
void __user *ubuf;
};
@@ -80,7 +77,6 @@ struct iov_iter {
union {
unsigned long nr_segs;
u16 bvecq_slot;
- u8 folioq_slot;
loff_t xarray_start;
};
};
@@ -153,11 +149,6 @@ static inline bool iov_iter_is_bvecq(const struct iov_iter *i)
return iov_iter_type(i) == ITER_BVECQ;
}
-static inline bool iov_iter_is_folioq(const struct iov_iter *i)
-{
- return iov_iter_type(i) == ITER_FOLIOQ;
-}
-
static inline bool iov_iter_is_xarray(const struct iov_iter *i)
{
return iov_iter_type(i) == ITER_XARRAY;
@@ -306,9 +297,6 @@ void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count);
void iov_iter_bvec_queue(struct iov_iter *i, unsigned int direction,
const struct bvecq *bvecq,
unsigned int first_slot, unsigned int offset, size_t count);
-void iov_iter_folio_queue(struct iov_iter *i, unsigned int direction,
- const struct folio_queue *folioq,
- unsigned int first_slot, unsigned int offset, size_t count);
void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray,
loff_t start, size_t count);
ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 63fc75c2bc48..f3626a640a4c 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -571,39 +571,6 @@ static void iov_iter_bvecq_advance(struct iov_iter *i, size_t by)
i->bvecq = bq;
}
-static void iov_iter_folioq_advance(struct iov_iter *i, size_t size)
-{
- const struct folio_queue *folioq = i->folioq;
- unsigned int slot = i->folioq_slot;
-
- if (!i->count)
- return;
- i->count -= size;
-
- if (slot >= folioq_nr_slots(folioq)) {
- folioq = folioq->next;
- slot = 0;
- }
-
- size += i->iov_offset; /* From beginning of current segment. */
- do {
- size_t fsize = folioq_folio_size(folioq, slot);
-
- if (likely(size < fsize))
- break;
- size -= fsize;
- slot++;
- if (slot >= folioq_nr_slots(folioq) && folioq->next) {
- folioq = folioq->next;
- slot = 0;
- }
- } while (size);
-
- i->iov_offset = size;
- i->folioq_slot = slot;
- i->folioq = folioq;
-}
-
void iov_iter_advance(struct iov_iter *i, size_t size)
{
if (unlikely(i->count < size))
@@ -618,8 +585,6 @@ void iov_iter_advance(struct iov_iter *i, size_t size)
iov_iter_bvec_advance(i, size);
} else if (iov_iter_is_bvecq(i)) {
iov_iter_bvecq_advance(i, size);
- } else if (iov_iter_is_folioq(i)) {
- iov_iter_folioq_advance(i, size);
} else if (iov_iter_is_discard(i)) {
i->count -= size;
}
@@ -652,32 +617,6 @@ static void iov_iter_bvecq_revert(struct iov_iter *i, size_t unroll)
i->bvecq = bq;
}
-static void iov_iter_folioq_revert(struct iov_iter *i, size_t unroll)
-{
- const struct folio_queue *folioq = i->folioq;
- unsigned int slot = i->folioq_slot;
-
- for (;;) {
- size_t fsize;
-
- if (slot == 0) {
- folioq = folioq->prev;
- slot = folioq_nr_slots(folioq);
- }
- slot--;
-
- fsize = folioq_folio_size(folioq, slot);
- if (unroll <= fsize) {
- i->iov_offset = fsize - unroll;
- break;
- }
- unroll -= fsize;
- }
-
- i->folioq_slot = slot;
- i->folioq = folioq;
-}
-
void iov_iter_revert(struct iov_iter *i, size_t unroll)
{
if (!unroll)
@@ -712,9 +651,6 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll)
} else if (iov_iter_is_bvecq(i)) {
i->iov_offset = 0;
iov_iter_bvecq_revert(i, unroll);
- } else if (iov_iter_is_folioq(i)) {
- i->iov_offset = 0;
- iov_iter_folioq_revert(i, unroll);
} else { /* same logics for iovec and kvec */
const struct iovec *iov = iter_iov(i);
while (1) {
@@ -758,8 +694,6 @@ size_t iov_iter_single_seg_count(const struct iov_iter *i)
}
return umin(i->count, bq->bv[slot].bv_len - offset);
}
- if (unlikely(iov_iter_is_folioq(i)))
- return umin(folioq_folio_size(i->folioq, i->folioq_slot), i->count);
return i->count;
}
EXPORT_SYMBOL(iov_iter_single_seg_count);
@@ -825,36 +759,6 @@ void iov_iter_bvec_queue(struct iov_iter *i, unsigned int direction,
}
EXPORT_SYMBOL(iov_iter_bvec_queue);
-/**
- * iov_iter_folio_queue - Initialise an I/O iterator to use the folios in a folio queue
- * @i: The iterator to initialise.
- * @direction: The direction of the transfer.
- * @folioq: The starting point in the folio queue.
- * @first_slot: The first slot in the folio queue to use
- * @offset: The offset into the folio in the first slot to start at
- * @count: The size of the I/O buffer in bytes.
- *
- * Set up an I/O iterator to either draw data out of the pages attached to an
- * inode or to inject data into those pages. The pages *must* be prevented
- * from evaporation, either by taking a ref on them or locking them by the
- * caller.
- */
-void iov_iter_folio_queue(struct iov_iter *i, unsigned int direction,
- const struct folio_queue *folioq, unsigned int first_slot,
- unsigned int offset, size_t count)
-{
- BUG_ON(direction & ~1);
- *i = (struct iov_iter) {
- .iter_type = ITER_FOLIOQ,
- .data_source = direction,
- .folioq = folioq,
- .folioq_slot = first_slot,
- .count = count,
- .iov_offset = offset,
- };
-}
-EXPORT_SYMBOL(iov_iter_folio_queue);
-
/**
* iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
* @i: The iterator to initialise.
@@ -996,9 +900,7 @@ unsigned long iov_iter_alignment(const struct iov_iter *i)
if (iov_iter_is_bvecq(i))
return iov_iter_alignment_bvecq(i);
- /* With both xarray and folioq types, we're dealing with whole folios. */
- if (iov_iter_is_folioq(i))
- return i->iov_offset | i->count;
+ /* With the xarray type, we're dealing with whole folios. */
if (iov_iter_is_xarray(i))
return (i->xarray_start + i->iov_offset) | i->count;
@@ -1253,11 +1155,6 @@ int iov_iter_npages(const struct iov_iter *i, int maxpages)
return bvec_npages(i, maxpages);
if (iov_iter_is_bvecq(i))
return iov_npages_bvecq(i, maxpages);
- if (iov_iter_is_folioq(i)) {
- unsigned offset = i->iov_offset % PAGE_SIZE;
- int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE);
- return min(npages, maxpages);
- }
if (iov_iter_is_xarray(i)) {
unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE;
int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE);
@@ -1680,68 +1577,6 @@ static ssize_t iov_iter_extract_bvecq_pages(struct iov_iter *iter,
return extracted;
}
-/*
- * Extract a list of contiguous pages from an ITER_FOLIOQ iterator. This does
- * not get references on the pages, nor does it get a pin on them.
- */
-static ssize_t iov_iter_extract_folioq_pages(struct iov_iter *i,
- struct page ***pages, size_t maxsize,
- unsigned int maxpages,
- iov_iter_extraction_t extraction_flags,
- size_t *offset0)
-{
- const struct folio_queue *folioq = i->folioq;
- struct page **p;
- unsigned int nr = 0;
- size_t extracted = 0, offset, slot = i->folioq_slot;
-
- if (slot >= folioq_nr_slots(folioq)) {
- folioq = folioq->next;
- slot = 0;
- if (WARN_ON(i->iov_offset != 0))
- return -EIO;
- }
-
- offset = i->iov_offset & ~PAGE_MASK;
- *offset0 = offset;
-
- maxpages = want_pages_array(pages, maxsize, offset, maxpages);
- if (!maxpages)
- return -ENOMEM;
- p = *pages;
-
- for (;;) {
- struct folio *folio = folioq_folio(folioq, slot);
- size_t offset = i->iov_offset, fsize = folioq_folio_size(folioq, slot);
- size_t part = PAGE_SIZE - offset % PAGE_SIZE;
-
- if (offset < fsize) {
- part = umin(part, umin(maxsize - extracted, fsize - offset));
- i->count -= part;
- i->iov_offset += part;
- extracted += part;
-
- p[nr++] = folio_page(folio, offset / PAGE_SIZE);
- }
-
- if (nr >= maxpages || extracted >= maxsize)
- break;
-
- if (i->iov_offset >= fsize) {
- i->iov_offset = 0;
- slot++;
- if (slot == folioq_nr_slots(folioq) && folioq->next) {
- folioq = folioq->next;
- slot = 0;
- }
- }
- }
-
- i->folioq = folioq;
- i->folioq_slot = slot;
- return extracted;
-}
-
/*
* Extract a list of contiguous pages from an ITER_XARRAY iterator. This does not
* get references on the pages, nor does it get a pin on them.
@@ -1986,8 +1821,8 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i,
* added to the pages, but refs will not be taken.
* iov_iter_extract_will_pin() will return true.
*
- * (*) If the iterator is ITER_KVEC, ITER_BVEC, ITER_FOLIOQ or ITER_XARRAY, the
- * pages are merely listed; no extra refs or pins are obtained.
+ * (*) If the iterator is ITER_KVEC, ITER_BVEC, ITER_XARRAY, the pages are
+ * merely listed; no extra refs or pins are obtained.
* iov_iter_extract_will_pin() will return 0.
*
* Note also:
@@ -2026,10 +1861,6 @@ ssize_t iov_iter_extract_pages(struct iov_iter *i,
return iov_iter_extract_bvecq_pages(i, pages, maxsize,
maxpages, extraction_flags,
offset0);
- if (iov_iter_is_folioq(i))
- return iov_iter_extract_folioq_pages(i, pages, maxsize,
- maxpages, extraction_flags,
- offset0);
if (iov_iter_is_xarray(i))
return iov_iter_extract_xarray_pages(i, pages, maxsize,
maxpages, extraction_flags,
diff --git a/lib/scatterlist.c b/lib/scatterlist.c
index b92144659543..11b6a890cf60 100644
--- a/lib/scatterlist.c
+++ b/lib/scatterlist.c
@@ -12,7 +12,6 @@
#include <linux/bvec.h>
#include <linux/bvecq.h>
#include <linux/uio.h>
-#include <linux/folio_queue.h>
/**
* sg_nents - return total count of entries in scatterlist
@@ -1330,67 +1329,6 @@ static ssize_t extract_bvecq_to_sg(struct iov_iter *iter,
return ret;
}
-/*
- * Extract up to sg_max folios from an FOLIOQ-type iterator and add them to
- * the scatterlist. The pages are not pinned.
- */
-static ssize_t extract_folioq_to_sg(struct iov_iter *iter,
- ssize_t maxsize,
- struct sg_table *sgtable,
- unsigned int sg_max,
- iov_iter_extraction_t extraction_flags)
-{
- const struct folio_queue *folioq = iter->folioq;
- struct scatterlist *sg = sgtable->sgl + sgtable->nents;
- unsigned int slot = iter->folioq_slot;
- ssize_t ret = 0;
- size_t offset = iter->iov_offset;
-
- BUG_ON(!folioq);
-
- if (slot >= folioq_nr_slots(folioq)) {
- folioq = folioq->next;
- if (WARN_ON_ONCE(!folioq))
- return 0;
- slot = 0;
- }
-
- do {
- struct folio *folio = folioq_folio(folioq, slot);
- size_t fsize = folioq_folio_size(folioq, slot);
-
- if (offset < fsize) {
- size_t part = umin(maxsize - ret, fsize - offset);
-
- sg_set_page(sg, folio_page(folio, 0), part, offset);
- sgtable->nents++;
- sg++;
- sg_max--;
- offset += part;
- ret += part;
- }
-
- if (offset >= fsize) {
- offset = 0;
- slot++;
- if (slot >= folioq_nr_slots(folioq)) {
- if (!folioq->next) {
- WARN_ON_ONCE(ret < iter->count);
- break;
- }
- folioq = folioq->next;
- slot = 0;
- }
- }
- } while (sg_max > 0 && ret < maxsize);
-
- iter->folioq = folioq;
- iter->folioq_slot = slot;
- iter->iov_offset = offset;
- iter->count -= ret;
- return ret;
-}
-
/*
* Extract up to sg_max folios from an XARRAY-type iterator and add them to
* the scatterlist. The pages are not pinned.
@@ -1453,8 +1391,8 @@ static ssize_t extract_xarray_to_sg(struct iov_iter *iter,
* addition of @sg_max elements.
*
* The pages referred to by UBUF- and IOVEC-type iterators are extracted and
- * pinned; BVEC-, BVECQ-, KVEC-, FOLIOQ- and XARRAY-type are extracted but
- * aren't pinned; DISCARD-type is not supported.
+ * pinned; BVEC-, BVECQ-, KVEC-, XARRAY-type are extracted but aren't pinned;
+ * DISCARD-type is not supported.
*
* No end mark is placed on the scatterlist; that's left to the caller.
*
@@ -1489,9 +1427,6 @@ ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t maxsize,
case ITER_BVECQ:
return extract_bvecq_to_sg(iter, maxsize, sgtable, sg_max,
extraction_flags);
- case ITER_FOLIOQ:
- return extract_folioq_to_sg(iter, maxsize, sgtable, sg_max,
- extraction_flags);
case ITER_XARRAY:
return extract_xarray_to_sg(iter, maxsize, sgtable, sg_max,
extraction_flags);
diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c
index d3e8e22ca9ca..ad91582b369e 100644
--- a/lib/tests/kunit_iov_iter.c
+++ b/lib/tests/kunit_iov_iter.c
@@ -13,7 +13,6 @@
#include <linux/uio.h>
#include <linux/bvec.h>
#include <linux/bvecq.h>
-#include <linux/folio_queue.h>
#include <linux/scatterlist.h>
#include <linux/minmax.h>
#include <linux/mman.h>
@@ -376,176 +375,6 @@ static void __init iov_kunit_copy_from_bvec(struct kunit *test)
KUNIT_SUCCEED(test);
}
-static void iov_kunit_destroy_folioq(void *data)
-{
- struct folio_queue *folioq, *next;
-
- for (folioq = data; folioq; folioq = next) {
- next = folioq->next;
- kfree(folioq);
- }
-}
-
-static void __init iov_kunit_load_folioq(struct kunit *test,
- struct iov_iter *iter, int dir,
- struct folio_queue *folioq,
- struct page **pages, size_t npages)
-{
- struct folio_queue *p = folioq;
- size_t size = 0;
- int i;
-
- for (i = 0; i < npages; i++) {
- if (folioq_full(p)) {
- p->next = kzalloc_obj(struct folio_queue);
- KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p->next);
- folioq_init(p->next, 0);
- p->next->prev = p;
- p = p->next;
- }
- folioq_append(p, page_folio(pages[i]));
- size += PAGE_SIZE;
- }
- iov_iter_folio_queue(iter, dir, folioq, 0, 0, size);
-}
-
-static struct folio_queue *iov_kunit_create_folioq(struct kunit *test)
-{
- struct folio_queue *folioq;
-
- folioq = kzalloc_obj(struct folio_queue);
- KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folioq);
- kunit_add_action_or_reset(test, iov_kunit_destroy_folioq, folioq);
- folioq_init(folioq, 0);
- return folioq;
-}
-
-/*
- * Test copying to a ITER_FOLIOQ-type iterator.
- */
-static void __init iov_kunit_copy_to_folioq(struct kunit *test)
-{
- const struct kvec_test_range *pr;
- struct iov_iter iter;
- struct folio_queue *folioq;
- struct page **spages, **bpages;
- u8 *scratch, *buffer;
- size_t bufsize, npages, size, copied;
- int i, patt;
-
- bufsize = 0x100000;
- npages = bufsize / PAGE_SIZE;
-
- folioq = iov_kunit_create_folioq(test);
-
- scratch = iov_kunit_create_buffer(test, &spages, npages);
- for (i = 0; i < bufsize; i++)
- scratch[i] = pattern(i);
-
- buffer = iov_kunit_create_buffer(test, &bpages, npages);
- memset(buffer, 0, bufsize);
-
- iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages);
-
- i = 0;
- for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
- size = pr->to - pr->from;
- KUNIT_ASSERT_LE(test, pr->to, bufsize);
-
- iov_iter_folio_queue(&iter, READ, folioq, 0, 0, pr->to);
- iov_iter_advance(&iter, pr->from);
- copied = copy_to_iter(scratch + i, size, &iter);
-
- KUNIT_EXPECT_EQ(test, copied, size);
- KUNIT_EXPECT_EQ(test, iter.count, 0);
- KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to % PAGE_SIZE);
- i += size;
- if (test->status == KUNIT_FAILURE)
- goto stop;
- }
-
- /* Build the expected image in the scratch buffer. */
- patt = 0;
- memset(scratch, 0, bufsize);
- for (pr = kvec_test_ranges; pr->from >= 0; pr++)
- for (i = pr->from; i < pr->to; i++)
- scratch[i] = pattern(patt++);
-
- /* Compare the images */
- for (i = 0; i < bufsize; i++) {
- KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i);
- if (buffer[i] != scratch[i])
- return;
- }
-
-stop:
- KUNIT_SUCCEED(test);
-}
-
-/*
- * Test copying from a ITER_FOLIOQ-type iterator.
- */
-static void __init iov_kunit_copy_from_folioq(struct kunit *test)
-{
- const struct kvec_test_range *pr;
- struct iov_iter iter;
- struct folio_queue *folioq;
- struct page **spages, **bpages;
- u8 *scratch, *buffer;
- size_t bufsize, npages, size, copied;
- int i, j;
-
- bufsize = 0x100000;
- npages = bufsize / PAGE_SIZE;
-
- folioq = iov_kunit_create_folioq(test);
-
- buffer = iov_kunit_create_buffer(test, &bpages, npages);
- for (i = 0; i < bufsize; i++)
- buffer[i] = pattern(i);
-
- scratch = iov_kunit_create_buffer(test, &spages, npages);
- memset(scratch, 0, bufsize);
-
- iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages);
-
- i = 0;
- for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
- size = pr->to - pr->from;
- KUNIT_ASSERT_LE(test, pr->to, bufsize);
-
- iov_iter_folio_queue(&iter, WRITE, folioq, 0, 0, pr->to);
- iov_iter_advance(&iter, pr->from);
- copied = copy_from_iter(scratch + i, size, &iter);
-
- KUNIT_EXPECT_EQ(test, copied, size);
- KUNIT_EXPECT_EQ(test, iter.count, 0);
- KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to % PAGE_SIZE);
- i += size;
- }
-
- /* Build the expected image in the main buffer. */
- i = 0;
- memset(buffer, 0, bufsize);
- for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
- for (j = pr->from; j < pr->to; j++) {
- buffer[i++] = pattern(j);
- if (i >= bufsize)
- goto stop;
- }
- }
-stop:
-
- /* Compare the images */
- for (i = 0; i < bufsize; i++) {
- KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i);
- if (scratch[i] != buffer[i])
- return;
- }
-
- KUNIT_SUCCEED(test);
-}
-
static void iov_kunit_destroy_bvecq(void *data)
{
struct bvecq *bq, *next;
@@ -1119,85 +948,6 @@ static void __init iov_kunit_extract_pages_bvecq(struct kunit *test)
KUNIT_SUCCEED(test);
}
-/*
- * Test the extraction of ITER_FOLIOQ-type iterators.
- */
-static void __init iov_kunit_extract_pages_folioq(struct kunit *test)
-{
- const struct kvec_test_range *pr;
- struct folio_queue *folioq;
- struct iov_iter iter;
- struct page **bpages, *pagelist[8], **pages = pagelist;
- ssize_t len;
- size_t bufsize, size = 0, npages;
- int i, from;
-
- bufsize = 0x100000;
- npages = bufsize / PAGE_SIZE;
-
- folioq = iov_kunit_create_folioq(test);
-
- iov_kunit_create_buffer(test, &bpages, npages);
- iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages);
-
- for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
- from = pr->from;
- size = pr->to - from;
- KUNIT_ASSERT_LE(test, pr->to, bufsize);
-
- iov_iter_folio_queue(&iter, WRITE, folioq, 0, 0, pr->to);
- iov_iter_advance(&iter, from);
-
- do {
- size_t offset0 = LONG_MAX;
-
- for (i = 0; i < ARRAY_SIZE(pagelist); i++)
- pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL;
-
- len = iov_iter_extract_pages(&iter, &pages, 100 * 1024,
- ARRAY_SIZE(pagelist), 0, &offset0);
- KUNIT_EXPECT_GE(test, len, 0);
- if (len < 0)
- break;
- KUNIT_EXPECT_LE(test, len, size);
- KUNIT_EXPECT_EQ(test, iter.count, size - len);
- if (len == 0)
- break;
- size -= len;
- KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0);
- KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE);
-
- for (i = 0; i < ARRAY_SIZE(pagelist); i++) {
- struct page *p;
- ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0);
- int ix;
-
- KUNIT_ASSERT_GE(test, part, 0);
- ix = from / PAGE_SIZE;
- KUNIT_ASSERT_LT(test, ix, npages);
- p = bpages[ix];
- KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p);
- KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE);
- from += part;
- len -= part;
- KUNIT_ASSERT_GE(test, len, 0);
- if (len == 0)
- break;
- offset0 = 0;
- }
-
- if (test->status == KUNIT_FAILURE)
- goto stop;
- } while (iov_iter_count(&iter) > 0);
-
- KUNIT_EXPECT_EQ(test, size, 0);
- KUNIT_EXPECT_EQ(test, iter.count, 0);
- }
-
-stop:
- KUNIT_SUCCEED(test);
-}
-
/*
* Test the extraction of ITER_XARRAY-type iterators.
*/
@@ -1425,23 +1175,6 @@ static void __init iov_kunit_iter_to_sg_bvec(struct kunit *test)
iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data);
}
-static void __init iov_kunit_iter_to_sg_folioq(struct kunit *test)
-{
- struct iov_kunit_iter_to_sg_data data;
- struct folio_queue *folioq;
- struct iov_iter iter;
- size_t bufsize;
-
- bufsize = 0x200000;
- iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
-
- folioq = iov_kunit_create_folioq(test);
- iov_kunit_load_folioq(test, &iter, READ, folioq, data.pages,
- data.npages);
-
- iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data);
-}
-
static void __init iov_kunit_iter_to_sg_xarray(struct kunit *test)
{
struct iov_kunit_iter_to_sg_data data;
@@ -1480,18 +1213,14 @@ static struct kunit_case __refdata iov_kunit_cases[] = {
KUNIT_CASE(iov_kunit_copy_from_bvec),
KUNIT_CASE(iov_kunit_copy_to_bvecq),
KUNIT_CASE(iov_kunit_copy_from_bvecq),
- KUNIT_CASE(iov_kunit_copy_to_folioq),
- KUNIT_CASE(iov_kunit_copy_from_folioq),
KUNIT_CASE(iov_kunit_copy_to_xarray),
KUNIT_CASE(iov_kunit_copy_from_xarray),
KUNIT_CASE(iov_kunit_extract_pages_kvec),
KUNIT_CASE(iov_kunit_extract_pages_bvec),
KUNIT_CASE(iov_kunit_extract_pages_bvecq),
- KUNIT_CASE(iov_kunit_extract_pages_folioq),
KUNIT_CASE(iov_kunit_extract_pages_xarray),
KUNIT_CASE(iov_kunit_iter_to_sg_kvec),
KUNIT_CASE(iov_kunit_iter_to_sg_bvec),
- KUNIT_CASE(iov_kunit_iter_to_sg_folioq),
KUNIT_CASE(iov_kunit_iter_to_sg_xarray),
KUNIT_CASE(iov_kunit_iter_to_sg_ubuf),
{}
^ permalink raw reply related
* [PATCH v3 17/22] netfs: Remove netfs_extract_user_iter()
From: David Howells @ 2026-06-08 14:54 UTC (permalink / raw)
To: Christian Brauner, Matthew Wilcox, Christoph Hellwig
Cc: David Howells, Paulo Alcantara, Jens Axboe, Leon Romanovsky,
Steve French, ChenXiaoSong, Marc Dionne, Eric Van Hensbergen,
Dominique Martinet, Ilya Dryomov, Trond Myklebust, netfs,
linux-afs, linux-cifs, linux-nfs, ceph-devel, v9fs, linux-erofs,
linux-fsdevel, linux-kernel
In-Reply-To: <20260608145432.681865-1-dhowells@redhat.com>
Remove netfs_extract_user_iter() as it has been replaced with
netfs_extract_iter().
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Paulo Alcantara <pc@manguebit.org>
cc: Matthew Wilcox <willy@infradead.org>
cc: Christoph Hellwig <hch@infradead.org>
cc: Steve French <sfrench@samba.org>
cc: linux-cifs@vger.kernel.org
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
---
fs/netfs/iterator.c | 104 ------------------------------------------
include/linux/netfs.h | 3 --
2 files changed, 107 deletions(-)
diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c
index 10a25a618712..566693ac47ef 100644
--- a/fs/netfs/iterator.c
+++ b/fs/netfs/iterator.c
@@ -141,110 +141,6 @@ ssize_t netfs_extract_iter(struct iov_iter *orig, size_t max_len, size_t max_pag
EXPORT_SYMBOL_GPL(netfs_extract_iter);
#if 0
-/**
- * netfs_extract_user_iter - Extract the pages from a user iterator into a bvec
- * @orig: The original iterator
- * @orig_len: The amount of iterator to copy
- * @new: The iterator to be set up
- * @extraction_flags: Flags to qualify the request
- *
- * Extract the page fragments from the given amount of the source iterator and
- * build up a second iterator that refers to all of those bits. This allows
- * the original iterator to be disposed of.
- *
- * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be
- * allowed on the pages extracted.
- *
- * On success, the number of elements in the bvec is returned, the original
- * iterator will have been advanced by the amount extracted.
- *
- * The iov_iter_extract_mode() function should be used to query how cleanup
- * should be performed.
- */
-ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
- struct iov_iter *new,
- iov_iter_extraction_t extraction_flags)
-{
- struct bio_vec *bv = NULL;
- struct page **pages;
- unsigned int cur_npages;
- unsigned int max_pages;
- unsigned int npages = 0;
- unsigned int i;
- ssize_t ret = 0;
- size_t count = orig_len, offset, len;
- size_t bv_size, pg_size;
-
- if (WARN_ON_ONCE(!iter_is_ubuf(orig) && !iter_is_iovec(orig)))
- return -EIO;
-
- max_pages = iov_iter_npages(orig, INT_MAX);
- bv_size = array_size(max_pages, sizeof(*bv));
- bv = kvmalloc(bv_size, GFP_KERNEL);
- if (!bv)
- return -ENOMEM;
-
- /* Put the page list at the end of the bvec list storage. bvec
- * elements are larger than page pointers, so as long as we work
- * 0->last, we should be fine.
- */
- pg_size = array_size(max_pages, sizeof(*pages));
- pages = (void *)bv + bv_size - pg_size;
-
- while (count && npages < max_pages) {
- ret = iov_iter_extract_pages(orig, &pages, count,
- max_pages - npages, extraction_flags,
- &offset);
- if (unlikely(ret <= 0)) {
- ret = ret ?: -EIO;
- break;
- }
-
- if (WARN(ret > count,
- "%s: extract_pages overrun %zd > %zu bytes\n",
- __func__, ret, count)) {
- ret = -EIO;
- break;
- }
-
- cur_npages = DIV_ROUND_UP(offset + ret, PAGE_SIZE);
- if (WARN(cur_npages > max_pages - npages,
- "%s: extract_pages overrun %u > %u pages\n",
- __func__, npages + cur_npages, max_pages)) {
- ret = -EIO;
- break;
- }
-
- count -= ret;
- ret += offset;
-
- for (i = 0; i < cur_npages; i++) {
- len = ret > PAGE_SIZE ? PAGE_SIZE : ret;
- bvec_set_page(bv + npages + i, *pages++, len - offset, offset);
- ret -= len;
- offset = 0;
- }
-
- npages += cur_npages;
- }
-
- /* Note: Don't try to clean up after EIO. Either we got no pages, so
- * nothing to clean up, or we got a buffer overrun, memory corruption
- * and can't trust the stuff in the buffer (a WARN was emitted).
- */
-
- if (ret < 0 && (ret == -ENOMEM || npages == 0)) {
- for (i = 0; i < npages; i++)
- unpin_user_page(bv[i].bv_page);
- kvfree(bv);
- return ret;
- }
-
- iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count);
- return npages;
-}
-EXPORT_SYMBOL_GPL(netfs_extract_user_iter);
-
/*
* Select the span of a bvec iterator we're going to use. Limit it by both maximum
* size and maximum number of segments. Returns the size of the span in bytes.
diff --git a/include/linux/netfs.h b/include/linux/netfs.h
index 9e551e09054f..d0b1408bd02f 100644
--- a/include/linux/netfs.h
+++ b/include/linux/netfs.h
@@ -464,9 +464,6 @@ void netfs_put_subrequest(struct netfs_io_subrequest *subreq,
ssize_t netfs_extract_iter(struct iov_iter *orig, size_t max_len, size_t max_pages,
unsigned long long fpos, struct bvecq **_bvecq_head,
iov_iter_extraction_t extraction_flags);
-ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
- struct iov_iter *new,
- iov_iter_extraction_t extraction_flags);
size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
size_t max_size, size_t max_segs);
void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq);
^ permalink raw reply related
* [PATCH v3 16/22] netfs: Remove netfs_alloc/free_folioq_buffer()
From: David Howells @ 2026-06-08 14:54 UTC (permalink / raw)
To: Christian Brauner, Matthew Wilcox, Christoph Hellwig
Cc: David Howells, Paulo Alcantara, Jens Axboe, Leon Romanovsky,
Steve French, ChenXiaoSong, Marc Dionne, Eric Van Hensbergen,
Dominique Martinet, Ilya Dryomov, Trond Myklebust, netfs,
linux-afs, linux-cifs, linux-nfs, ceph-devel, v9fs, linux-erofs,
linux-fsdevel, linux-kernel
In-Reply-To: <20260608145432.681865-1-dhowells@redhat.com>
Remove netfs_alloc/free_folioq_buffer() as these have been replaced with
netfs_alloc/free_bvecq_buffer().
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Paulo Alcantara <pc@manguebit.org>
cc: Matthew Wilcox <willy@infradead.org>
cc: Christoph Hellwig <hch@infradead.org>
cc: Steve French <sfrench@samba.org>
cc: linux-cifs@vger.kernel.org
cc: netfs@lists.linux.dev
cc: linux-fsdevel@vger.kernel.org
---
fs/afs/dir_edit.c | 1 -
fs/netfs/misc.c | 98 -----------------------------------------
fs/smb/client/smb2ops.c | 1 -
include/linux/netfs.h | 6 ---
4 files changed, 106 deletions(-)
diff --git a/fs/afs/dir_edit.c b/fs/afs/dir_edit.c
index b3e80c5c434f..b92d7aa6eeac 100644
--- a/fs/afs/dir_edit.c
+++ b/fs/afs/dir_edit.c
@@ -10,7 +10,6 @@
#include <linux/namei.h>
#include <linux/pagemap.h>
#include <linux/iversion.h>
-#include <linux/folio_queue.h>
#include "internal.h"
#include "xdr_fs.h"
diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c
index ee67a0681784..8fc4e5ef2152 100644
--- a/fs/netfs/misc.c
+++ b/fs/netfs/misc.c
@@ -8,104 +8,6 @@
#include <linux/swap.h>
#include "internal.h"
-#if 0
-/**
- * netfs_alloc_folioq_buffer - Allocate buffer space into a folio queue
- * @mapping: Address space to set on the folio (or NULL).
- * @_buffer: Pointer to the folio queue to add to (may point to a NULL; updated).
- * @_cur_size: Current size of the buffer (updated).
- * @size: Target size of the buffer.
- * @gfp: The allocation constraints.
- */
-int netfs_alloc_folioq_buffer(struct address_space *mapping,
- struct folio_queue **_buffer,
- size_t *_cur_size, ssize_t size, gfp_t gfp)
-{
- struct folio_queue *tail = *_buffer, *p;
-
- size = round_up(size, PAGE_SIZE);
- if (*_cur_size >= size)
- return 0;
-
- if (tail)
- while (tail->next)
- tail = tail->next;
-
- do {
- struct folio *folio;
- int order = 0, slot;
-
- if (!tail || folioq_full(tail)) {
- p = netfs_folioq_alloc(0, GFP_NOFS, netfs_trace_folioq_alloc_buffer);
- if (!p)
- return -ENOMEM;
- if (tail) {
- tail->next = p;
- p->prev = tail;
- } else {
- *_buffer = p;
- }
- tail = p;
- }
-
- if (size - *_cur_size > PAGE_SIZE)
- order = umin(ilog2(size - *_cur_size) - PAGE_SHIFT,
- MAX_PAGECACHE_ORDER);
-
- folio = folio_alloc(gfp, order);
- if (!folio && order > 0)
- folio = folio_alloc(gfp, 0);
- if (!folio)
- return -ENOMEM;
-
- folio->mapping = mapping;
- folio->index = *_cur_size / PAGE_SIZE;
- trace_netfs_folio(folio, netfs_folio_trace_alloc_buffer);
- slot = folioq_append_mark(tail, folio);
- *_cur_size += folioq_folio_size(tail, slot);
- } while (*_cur_size < size);
-
- return 0;
-}
-EXPORT_SYMBOL(netfs_alloc_folioq_buffer);
-
-/**
- * netfs_free_folioq_buffer - Free a folio queue.
- * @fq: The start of the folio queue to free
- *
- * Free up a chain of folio_queues and, if marked, the marked folios they point
- * to.
- */
-void netfs_free_folioq_buffer(struct folio_queue *fq)
-{
- struct folio_queue *next;
- struct folio_batch fbatch;
-
- folio_batch_init(&fbatch);
-
- for (; fq; fq = next) {
- for (int slot = 0; slot < folioq_count(fq); slot++) {
- struct folio *folio = folioq_folio(fq, slot);
-
- if (!folio ||
- !folioq_is_marked(fq, slot))
- continue;
-
- trace_netfs_folio(folio, netfs_folio_trace_put);
- if (folio_batch_add(&fbatch, folio))
- folio_batch_release(&fbatch);
- }
-
- netfs_stat_d(&netfs_n_folioq);
- next = fq->next;
- kfree(fq);
- }
-
- folio_batch_release(&fbatch);
-}
-EXPORT_SYMBOL(netfs_free_folioq_buffer);
-#endif
-
/**
* netfs_dirty_folio - Mark folio dirty and pin a cache object for writeback
* @mapping: The mapping the folio belongs to.
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index 6e3d43c4643a..c14ae1a61a43 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -13,7 +13,6 @@
#include <linux/sort.h>
#include <crypto/aead.h>
#include <linux/fiemap.h>
-#include <linux/folio_queue.h>
#include <uapi/linux/magic.h>
#include "cifsfs.h"
#include "cifsglob.h"
diff --git a/include/linux/netfs.h b/include/linux/netfs.h
index 15a1c3026733..9e551e09054f 100644
--- a/include/linux/netfs.h
+++ b/include/linux/netfs.h
@@ -479,12 +479,6 @@ void netfs_end_io_write(struct inode *inode);
int netfs_start_io_direct(struct inode *inode);
void netfs_end_io_direct(struct inode *inode);
-/* Buffer wrangling helpers API. */
-int netfs_alloc_folioq_buffer(struct address_space *mapping,
- struct folio_queue **_buffer,
- size_t *_cur_size, ssize_t size, gfp_t gfp);
-void netfs_free_folioq_buffer(struct folio_queue *fq);
-
/**
* netfs_inode - Get the netfs inode context from the inode
* @inode: The inode to query
^ permalink raw reply related
page: next (older)
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox