Re: [PATCH v8 2/6] NFSD: convert write_threads to netlink command

[Jeff Layton <jlayton@kernel.org>]

On Tue, 2024-04-16 at 18:39 -0400, Jeff Layton wrote:
> On Wed, 2024-04-17 at 07:55 +1000, NeilBrown wrote:
> > On Tue, 16 Apr 2024, Lorenzo Bianconi wrote:
> > > Introduce write_threads netlink command similar to the one available
> > > through the procfs.
> > 
> > I think this should support write_pool_threads too.
> > i.e.  the number of threads should be an array.  If it is a singleton,
> > the it does write_threads.   If larger it does write_pool_threads.
> > I don't think we want to add a separate command later for pool_threads.
> > 
> > NeilBrown
> > 
> 
> That is a very good point. We'll take a closer look at the current
> pool_threads interface and see how we can extend this one to cover that
> use case. Making it an array sounds reasonable at first blush.
> 

Lorenzo and I had a look this morning:

Doing this properly is going to take some refactoring, I think. The
existing pool code has a weird restriction where you can't bring the
server up and down exclusively using the pool_threads interface. There
are no userland tools for it either, so it's out of scope since it's not
really part of the how rpc.nfsd works today.

I think the way forward here is to go ahead and make this interface use
an array of integers (like Neil suggests), but for now if someone tries
to send down a THREAD_SET operation and the pool_mode isn't global,
we'll just return an error of some sort (EINVAL, EOPNOTSUPP, ?).

One the new interfaces are in I'll take a look at adding support for
managing other pool_modes properly.

Does that sound ok?


> > 
> > > 
> > > Tested-by: Jeff Layton <jlayton@kernel.org>
> > > Reviewed-by: Jeff Layton <jlayton@kernel.org>
> > > Co-developed-by: Jeff Layton <jlayton@kernel.org>
> > > Signed-off-by: Jeff Layton <jlayton@kernel.org>
> > > Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
> > > ---
> > >  Documentation/netlink/specs/nfsd.yaml |  33 ++++++++
> > >  fs/nfsd/netlink.c                     |  19 +++++
> > >  fs/nfsd/netlink.h                     |   2 +
> > >  fs/nfsd/nfsctl.c                      | 104 ++++++++++++++++++++++++++
> > >  include/uapi/linux/nfsd_netlink.h     |  11 +++
> > >  5 files changed, 169 insertions(+)
> > > 
> > > diff --git a/Documentation/netlink/specs/nfsd.yaml b/Documentation/netlink/specs/nfsd.yaml
> > > index 05acc73e2e33..cbe6c5fd6c4d 100644
> > > --- a/Documentation/netlink/specs/nfsd.yaml
> > > +++ b/Documentation/netlink/specs/nfsd.yaml
> > > @@ -62,6 +62,18 @@ attribute-sets:
> > >          name: compound-ops
> > >          type: u32
> > >          multi-attr: true
> > > +  -
> > > +    name: server-worker
> > > +    attributes:
> > > +      -
> > > +        name: threads
> > > +        type: u32
> > > +      -
> > > +        name: gracetime
> > > +        type: u32
> > > +      -
> > > +        name: leasetime
> > > +        type: u32
> > >  
> > >  operations:
> > >    list:
> > > @@ -87,3 +99,24 @@ operations:
> > >              - sport
> > >              - dport
> > >              - compound-ops
> > > +    -
> > > +      name: threads-set
> > > +      doc: set the number of running threads
> > > +      attribute-set: server-worker
> > > +      flags: [ admin-perm ]
> > > +      do:
> > > +        request:
> > > +          attributes:
> > > +            - threads
> > > +            - gracetime
> > > +            - leasetime
> > > +    -
> > > +      name: threads-get
> > > +      doc: get the number of running threads
> > > +      attribute-set: server-worker
> > > +      do:
> > > +        reply:
> > > +          attributes:
> > > +            - threads
> > > +            - gracetime
> > > +            - leasetime
> > > diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c
> > > index 0e1d635ec5f9..20a646af0324 100644
> > > --- a/fs/nfsd/netlink.c
> > > +++ b/fs/nfsd/netlink.c
> > > @@ -10,6 +10,13 @@
> > >  
> > >  #include <uapi/linux/nfsd_netlink.h>
> > >  
> > > +/* NFSD_CMD_THREADS_SET - do */
> > > +static const struct nla_policy nfsd_threads_set_nl_policy[NFSD_A_SERVER_WORKER_LEASETIME + 1] = {
> > > +	[NFSD_A_SERVER_WORKER_THREADS] = { .type = NLA_U32, },
> > > +	[NFSD_A_SERVER_WORKER_GRACETIME] = { .type = NLA_U32, },
> > > +	[NFSD_A_SERVER_WORKER_LEASETIME] = { .type = NLA_U32, },
> > > +};
> > > +
> > >  /* Ops table for nfsd */
> > >  static const struct genl_split_ops nfsd_nl_ops[] = {
> > >  	{
> > > @@ -19,6 +26,18 @@ static const struct genl_split_ops nfsd_nl_ops[] = {
> > >  		.done	= nfsd_nl_rpc_status_get_done,
> > >  		.flags	= GENL_CMD_CAP_DUMP,
> > >  	},
> > > +	{
> > > +		.cmd		= NFSD_CMD_THREADS_SET,
> > > +		.doit		= nfsd_nl_threads_set_doit,
> > > +		.policy		= nfsd_threads_set_nl_policy,
> > > +		.maxattr	= NFSD_A_SERVER_WORKER_LEASETIME,
> > > +		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
> > > +	},
> > > +	{
> > > +		.cmd	= NFSD_CMD_THREADS_GET,
> > > +		.doit	= nfsd_nl_threads_get_doit,
> > > +		.flags	= GENL_CMD_CAP_DO,
> > > +	},
> > >  };
> > >  
> > >  struct genl_family nfsd_nl_family __ro_after_init = {
> > > diff --git a/fs/nfsd/netlink.h b/fs/nfsd/netlink.h
> > > index d83dd6bdee92..4137fac477e4 100644
> > > --- a/fs/nfsd/netlink.h
> > > +++ b/fs/nfsd/netlink.h
> > > @@ -16,6 +16,8 @@ int nfsd_nl_rpc_status_get_done(struct netlink_callback *cb);
> > >  
> > >  int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb,
> > >  				  struct netlink_callback *cb);
> > > +int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info);
> > > +int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info);
> > >  
> > >  extern struct genl_family nfsd_nl_family;
> > >  
> > > diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
> > > index f2e442d7fe16..38a5df03981b 100644
> > > --- a/fs/nfsd/nfsctl.c
> > > +++ b/fs/nfsd/nfsctl.c
> > > @@ -1653,6 +1653,110 @@ int nfsd_nl_rpc_status_get_done(struct netlink_callback *cb)
> > >  	return 0;
> > >  }
> > >  
> > > +/**
> > > + * nfsd_nl_threads_set_doit - set the number of running threads
> > > + * @skb: reply buffer
> > > + * @info: netlink metadata and command arguments
> > > + *
> > > + * Return 0 on success or a negative errno.
> > > + */
> > > +int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info)
> > > +{
> > > +	struct net *net = genl_info_net(info);
> > > +	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
> > > +	int ret = -EBUSY;
> > > +	u32 nthreads;
> > > +
> > > +	if (GENL_REQ_ATTR_CHECK(info, NFSD_A_SERVER_WORKER_THREADS))
> > > +		return -EINVAL;
> > > +
> > > +	nthreads = nla_get_u32(info->attrs[NFSD_A_SERVER_WORKER_THREADS]);
> > > +
> > > +	mutex_lock(&nfsd_mutex);
> > > +	if (info->attrs[NFSD_A_SERVER_WORKER_GRACETIME] ||
> > > +	    info->attrs[NFSD_A_SERVER_WORKER_LEASETIME]) {
> > > +		const struct nlattr *attr;
> > > +
> > > +		if (nn->nfsd_serv && nn->nfsd_serv->sv_nrthreads)
> > > +			goto out_unlock;
> > > +
> > > +		ret = -EINVAL;
> > > +		attr = info->attrs[NFSD_A_SERVER_WORKER_GRACETIME];
> > > +		if (attr) {
> > > +			u32 gracetime = nla_get_u32(attr);
> > > +
> > > +			if (gracetime < 10 || gracetime > 3600)
> > > +				goto out_unlock;
> > > +
> > > +			nn->nfsd4_grace = gracetime;
> > > +		}
> > > +
> > > +		attr = info->attrs[NFSD_A_SERVER_WORKER_LEASETIME];
> > > +		if (attr) {
> > > +			u32 leasetime = nla_get_u32(attr);
> > > +
> > > +			if (leasetime < 10 || leasetime > 3600)
> > > +				goto out_unlock;
> > > +
> > > +			nn->nfsd4_lease = leasetime;
> > > +		}
> > > +	}
> > > +
> > > +	ret = nfsd_svc(nthreads, net, get_current_cred());
> > > +out_unlock:
> > > +	mutex_unlock(&nfsd_mutex);
> > > +
> > > +	return ret == nthreads ? 0 : ret;
> > > +}
> > > +
> > > +/**
> > > + * nfsd_nl_threads_get_doit - get the number of running threads
> > > + * @skb: reply buffer
> > > + * @info: netlink metadata and command arguments
> > > + *
> > > + * Return 0 on success or a negative errno.
> > > + */
> > > +int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info)
> > > +{
> > > +	struct net *net = genl_info_net(info);
> > > +	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
> > > +	void *hdr;
> > > +	int err;
> > > +
> > > +	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
> > > +	if (!skb)
> > > +		return -ENOMEM;
> > > +
> > > +	hdr = genlmsg_iput(skb, info);
> > > +	if (!hdr) {
> > > +		err = -EMSGSIZE;
> > > +		goto err_free_msg;
> > > +	}
> > > +
> > > +	mutex_lock(&nfsd_mutex);
> > > +	err = nla_put_u32(skb, NFSD_A_SERVER_WORKER_GRACETIME,
> > > +			  nn->nfsd4_grace) ||
> > > +	      nla_put_u32(skb, NFSD_A_SERVER_WORKER_LEASETIME,
> > > +			  nn->nfsd4_lease) ||
> > > +	      nla_put_u32(skb, NFSD_A_SERVER_WORKER_THREADS,
> > > +			  nn->nfsd_serv ? nn->nfsd_serv->sv_nrthreads : 0);
> > > +	mutex_unlock(&nfsd_mutex);
> > > +
> > > +	if (err) {
> > > +		err = -EINVAL;
> > > +		goto err_free_msg;
> > > +	}
> > > +
> > > +	genlmsg_end(skb, hdr);
> > > +
> > > +	return genlmsg_reply(skb, info);
> > > +
> > > +err_free_msg:
> > > +	nlmsg_free(skb);
> > > +
> > > +	return err;
> > > +}
> > > +
> > >  /**
> > >   * nfsd_net_init - Prepare the nfsd_net portion of a new net namespace
> > >   * @net: a freshly-created network namespace
> > > diff --git a/include/uapi/linux/nfsd_netlink.h b/include/uapi/linux/nfsd_netlink.h
> > > index 3cd044edee5d..ccc78a5ee650 100644
> > > --- a/include/uapi/linux/nfsd_netlink.h
> > > +++ b/include/uapi/linux/nfsd_netlink.h
> > > @@ -29,8 +29,19 @@ enum {
> > >  	NFSD_A_RPC_STATUS_MAX = (__NFSD_A_RPC_STATUS_MAX - 1)
> > >  };
> > >  
> > > +enum {
> > > +	NFSD_A_SERVER_WORKER_THREADS = 1,
> > > +	NFSD_A_SERVER_WORKER_GRACETIME,
> > > +	NFSD_A_SERVER_WORKER_LEASETIME,
> > > +
> > > +	__NFSD_A_SERVER_WORKER_MAX,
> > > +	NFSD_A_SERVER_WORKER_MAX = (__NFSD_A_SERVER_WORKER_MAX - 1)
> > > +};
> > > +
> > >  enum {
> > >  	NFSD_CMD_RPC_STATUS_GET = 1,
> > > +	NFSD_CMD_THREADS_SET,
> > > +	NFSD_CMD_THREADS_GET,
> > >  
> > >  	__NFSD_CMD_MAX,
> > >  	NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1)
> > > -- 
> > > 2.44.0
> > > 
> > > 
> > 
> 

-- 
Jeff Layton <jlayton@kernel.org>