Re: [PATCH v5 16/19] nfsd: use SRCU to dereference nn->nfsd_serv

[Mike Snitzer <snitzer@kernel.org>]

On Wed, Jun 19, 2024 at 08:39:46AM -0400, Jeff Layton wrote:
> On Tue, 2024-06-18 at 16:19 -0400, Mike Snitzer wrote:
> > Introduce nfsd_serv_get, nfsd_serv_put and nfsd_serv_sync and update
> > the nfsd code to prevent nfsd_destroy_serv from destroying
> > nn->nfsd_serv until all nfsd code is done with it (particularly the
> > localio code that doesn't run in the context of nfsd's svc threads,
> > nor does it take the nfsd_mutex).
> > 
> > Commit 83d5e5b0af90 ("dm: optimize use SRCU and RCU") provided a
> > familiar well-worn pattern for how implement.
> > 
> > Suggested-by: NeilBrown <neilb@suse.de>
> > Signed-off-by: Mike Snitzer <snitzer@kernel.org>
> > ---
> >  fs/nfsd/filecache.c | 13 ++++++++---
> >  fs/nfsd/netns.h     | 12 ++++++++--
> >  fs/nfsd/nfs4state.c | 25 ++++++++++++++-------
> >  fs/nfsd/nfsctl.c    |  7 ++++--
> >  fs/nfsd/nfssvc.c    | 55 ++++++++++++++++++++++++++++++++++++---------
> >  5 files changed, 87 insertions(+), 25 deletions(-)
> > 
> > diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c
> > index 99631fa56662..474b3a3af3fb 100644
> > --- a/fs/nfsd/filecache.c
> > +++ b/fs/nfsd/filecache.c
> > @@ -413,12 +413,15 @@ nfsd_file_dispose_list_delayed(struct list_head *dispose)
> >  		struct nfsd_file *nf = list_first_entry(dispose,
> >  						struct nfsd_file, nf_lru);
> >  		struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);
> > +		int srcu_idx;
> > +		struct svc_serv *serv = nfsd_serv_get(nn, &srcu_idx);
> >  		struct nfsd_fcache_disposal *l = nn->fcache_disposal;
> >  
> >  		spin_lock(&l->lock);
> >  		list_move_tail(&nf->nf_lru, &l->freeme);
> >  		spin_unlock(&l->lock);
> > -		svc_wake_up(nn->nfsd_serv);
> > +		svc_wake_up(serv);
> > +		nfsd_serv_put(nn, srcu_idx);
> >  	}
> >  }
> >  
> > @@ -443,11 +446,15 @@ void nfsd_file_net_dispose(struct nfsd_net *nn)
> >  		for (i = 0; i < 8 && !list_empty(&l->freeme); i++)
> >  			list_move(l->freeme.next, &dispose);
> >  		spin_unlock(&l->lock);
> > -		if (!list_empty(&l->freeme))
> > +		if (!list_empty(&l->freeme)) {
> > +			int srcu_idx;
> > +			struct svc_serv *serv = nfsd_serv_get(nn, &srcu_idx);
> >  			/* Wake up another thread to share the work
> >  			 * *before* doing any actual disposing.
> >  			 */
> > -			svc_wake_up(nn->nfsd_serv);
> > +			svc_wake_up(serv);
> > +			nfsd_serv_put(nn, srcu_idx);
> > +		}
> >  		nfsd_file_dispose_list(&dispose);
> >  	}
> >  }
> > diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
> > index 0c5a1d97e4ac..0eebcc03bcd3 100644
> > --- a/fs/nfsd/netns.h
> > +++ b/fs/nfsd/netns.h
> > @@ -139,8 +139,12 @@ struct nfsd_net {
> >  	u32 clverifier_counter;
> >  
> >  	struct svc_info nfsd_info;
> > -#define nfsd_serv nfsd_info.serv
> > -
> > +	/*
> > +	 * The current 'nfsd_serv' at nfsd_info.serv
> > +	 * Use nfsd_serv_get() or take nfsd_mutex to dereference.
> > +	 */
> > +	void __rcu *nfsd_serv;
> 
> I don't understand why you need a void pointer here. This should only
> ever hold a pointer to the serv or NULL. It seems like this work just
> as well: 
> 
> 	struct svc_serv __rcu *nfsd_serv;
> 

It is defensive, future-proofs us from some new code being introduced
that dereferences nn->nfsd_serv without proper use of nfsd_serv_get().


> > @@ -589,9 +615,12 @@ void nfsd_destroy_serv(struct net *net)
> >  	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
> >  	struct svc_serv *serv = nn->nfsd_serv;
> >  
> > +	lockdep_assert_held(&nfsd_mutex);
> > +
> >  	spin_lock(&nfsd_notifier_lock);
> > -	nn->nfsd_serv = NULL;
> > +	rcu_assign_pointer(nn->nfsd_serv, NULL);
> >  	spin_unlock(&nfsd_notifier_lock);
> > +	nfsd_serv_sync(nn);
> >  
> >  	/* check if the notifier still has clients */
> >  	if (atomic_dec_return(&nfsd_notifier_refcount) == 0) {
> > @@ -711,6 +740,10 @@ int nfsd_create_serv(struct net *net)
> >  	if (nn->nfsd_serv)
> >  		return 0;
> >  
> > +	error = init_srcu_struct(&nn->nfsd_serv_srcu);
> > +	if (error)
> > +		return error;
> > +
> >  	if (nfsd_max_blksize == 0)
> >  		nfsd_max_blksize = nfsd_get_default_max_blksize();
> >  	nfsd_reset_versions(nn);
> > @@ -727,8 +760,10 @@ int nfsd_create_serv(struct net *net)
> >  	}
> >  	spin_lock(&nfsd_notifier_lock);
> >  	nn->nfsd_info.mutex = &nfsd_mutex;
> > -	nn->nfsd_serv = serv;
> > +	nn->nfsd_info.serv = serv;
> > +	rcu_assign_pointer(nn->nfsd_serv, nn->nfsd_info.serv);
> >  	spin_unlock(&nfsd_notifier_lock);
> > +	nfsd_serv_sync(nn);
> 
> I get why you're doing the synchronize on destroy, but why on the
> create? You're not tearing anything down here, so I don't see the need
> to ensure synchronization.

Yeah, it isn't needed.  Fixed, thanks.

Mike