Re: [PATCH v12 09/24] nfs_common: add NFS LOCALIO auxiliary protocol enablement

[Jeff Layton <jlayton@kernel.org>]

On Mon, 2024-08-19 at 14:17 -0400, Mike Snitzer wrote:
> fs/nfs_common/nfslocalio.c provides interfaces that enable an NFS client
> to generate a nonce (single-use UUID) and associated short-lived
> nfs_uuid_t struct, register it with nfs_common for subsequent lookup and
> verification by the NFS server and if matched the NFS server populates
> members in the nfs_uuid_t struct.
> 
> nfs_common's nfs_uuids list is the basis for localio enablement, as such
> it has members that point to nfsd memory for direct use by the client
> (e.g. 'net' is the server's network namespace, through it the client can
> access nn->nfsd_serv with proper rcu read access).
> 
> This commit adds all the nfs_client members required to implement
> the entire localio feature (which depends on the LOCALIO protocol).
> 
> Signed-off-by: Mike Snitzer <snitzer@kernel.org>
> ---
>  fs/nfs/client.c            |  9 ++++
>  fs/nfs_common/Makefile     |  3 ++
>  fs/nfs_common/nfslocalio.c | 97 ++++++++++++++++++++++++++++++++++++++
>  include/linux/nfs_fs_sb.h  | 10 ++++
>  include/linux/nfslocalio.h | 37 +++++++++++++++
>  5 files changed, 156 insertions(+)
>  create mode 100644 fs/nfs_common/nfslocalio.c
>  create mode 100644 include/linux/nfslocalio.h
> 
> diff --git a/fs/nfs/client.c b/fs/nfs/client.c
> index 8286edd6062d..1b65a5d7af49 100644
> --- a/fs/nfs/client.c
> +++ b/fs/nfs/client.c
> @@ -178,6 +178,15 @@ struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_init)
>  	clp->cl_max_connect = cl_init->max_connect ? cl_init->max_connect : 1;
>  	clp->cl_net = get_net(cl_init->net);
>  
> +#if IS_ENABLED(CONFIG_NFS_LOCALIO)
> +	seqlock_init(&clp->cl_boot_lock);
> +	ktime_get_real_ts64(&clp->cl_nfssvc_boot);
> +	clp->cl_rpcclient_localio = ERR_PTR(-EINVAL);
> +	clp->nfsd_open_local_fh = NULL;
> +	clp->cl_nfssvc_net = NULL;
> +	clp->cl_nfssvc_dom = NULL;
> +#endif /* CONFIG_NFS_LOCALIO */
> +
>  	clp->cl_principal = "*";
>  	clp->cl_xprtsec = cl_init->xprtsec;
>  	return clp;
> diff --git a/fs/nfs_common/Makefile b/fs/nfs_common/Makefile
> index e58b01bb8dda..a5e54809701e 100644
> --- a/fs/nfs_common/Makefile
> +++ b/fs/nfs_common/Makefile
> @@ -6,6 +6,9 @@
>  obj-$(CONFIG_NFS_ACL_SUPPORT) += nfs_acl.o
>  nfs_acl-objs := nfsacl.o
>  
> +obj-$(CONFIG_NFS_COMMON_LOCALIO_SUPPORT) += nfs_localio.o
> +nfs_localio-objs := nfslocalio.o
> +
>  obj-$(CONFIG_GRACE_PERIOD) += grace.o
>  obj-$(CONFIG_NFS_V4_2_SSC_HELPER) += nfs_ssc.o
>  
> diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c
> new file mode 100644
> index 000000000000..a20ff7607707
> --- /dev/null
> +++ b/fs/nfs_common/nfslocalio.c
> @@ -0,0 +1,97 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (C) 2024 Mike Snitzer <snitzer@hammerspace.com>
> + */
> +
> +#include <linux/module.h>
> +#include <linux/rculist.h>
> +#include <linux/nfslocalio.h>
> +
> +MODULE_LICENSE("GPL");
> +MODULE_DESCRIPTION("NFS localio protocol bypass support");
> +
> +DEFINE_MUTEX(nfs_uuid_mutex);
> +
> +/*
> + * Global list of nfs_uuid_t instances, add/remove
> + * is protected by nfs_uuid_mutex.
> + * Reads are protected by RCU read lock (see below).
> + */
> +LIST_HEAD(nfs_uuids);
> +
> +void nfs_uuid_begin(nfs_uuid_t *nfs_uuid)
> +{
> +	nfs_uuid->net = NULL;
> +	nfs_uuid->dom = NULL;
> +	uuid_gen(&nfs_uuid->uuid);
> +
> +	mutex_lock(&nfs_uuid_mutex);
> +	list_add_tail_rcu(&nfs_uuid->list, &nfs_uuids);
> +	mutex_unlock(&nfs_uuid_mutex);
> +}
> +EXPORT_SYMBOL_GPL(nfs_uuid_begin);
> +
> +void nfs_uuid_end(nfs_uuid_t *nfs_uuid)
> +{
> +	mutex_lock(&nfs_uuid_mutex);
> +	list_del_rcu(&nfs_uuid->list);
> +	mutex_unlock(&nfs_uuid_mutex);
> +}
> +EXPORT_SYMBOL_GPL(nfs_uuid_end);
> +
> +/* Must be called with RCU read lock held. */
> +static nfs_uuid_t * nfs_uuid_lookup(const uuid_t *uuid)
> +{
> +	nfs_uuid_t *nfs_uuid;
> +
> +	list_for_each_entry_rcu(nfs_uuid, &nfs_uuids, list)
> +		if (uuid_equal(&nfs_uuid->uuid, uuid))
> +			return nfs_uuid;
> +
> +	return NULL;
> +}
> +
> +bool nfs_uuid_is_local(const uuid_t *uuid, struct net *net, struct auth_domain *dom)
> +{
> +	bool is_local = false;
> +	nfs_uuid_t *nfs_uuid;
> +
> +	rcu_read_lock();
> +	nfs_uuid = nfs_uuid_lookup(uuid);
> +	if (nfs_uuid) {
> +		is_local = true;
> +		nfs_uuid->net = net;
> +		kref_get(&dom->ref);
> +		nfs_uuid->dom = dom;
> +	}
> +	rcu_read_unlock();
> +
> +	return is_local;
> +}
> +EXPORT_SYMBOL_GPL(nfs_uuid_is_local);
> +
> +/*
> + * The nfs localio code needs to call into nfsd to do the filehandle -> struct path
> + * mapping, but cannot be statically linked, because that will make the nfs module
> + * depend on the nfsd module.
> + *
> + * Instead, do dynamic linking to the nfsd module (via nfs_common module). The
> + * nfs_common module will only hold a reference on nfsd when localio is in use.
> + * This allows some sanity checking, like giving up on localio if nfsd isn't loaded.
> + */
> +
> +extern int nfsd_open_local_fh(struct net *, struct auth_domain *, struct rpc_clnt *,
> +			const struct cred *, const struct nfs_fh *,
> +			const fmode_t, struct file **);
> +

BTW: this adds the above prototype, but the function itself isn't added
until patch #10.

> +nfs_to_nfsd_open_t get_nfsd_open_local_fh(void)
> +{
> +	return symbol_request(nfsd_open_local_fh);
> +}
> +EXPORT_SYMBOL_GPL(get_nfsd_open_local_fh);
> +
> +void put_nfsd_open_local_fh(void)
> +{
> +	symbol_put(nfsd_open_local_fh);
> +}
> +EXPORT_SYMBOL_GPL(put_nfsd_open_local_fh);
> diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
> index 1df86ab98c77..3849cc2832f0 100644
> --- a/include/linux/nfs_fs_sb.h
> +++ b/include/linux/nfs_fs_sb.h
> @@ -8,6 +8,7 @@
>  #include <linux/wait.h>
>  #include <linux/nfs_xdr.h>
>  #include <linux/sunrpc/xprt.h>
> +#include <linux/nfslocalio.h>
>  
>  #include <linux/atomic.h>
>  #include <linux/refcount.h>
> @@ -125,6 +126,15 @@ struct nfs_client {
>  	struct net		*cl_net;
>  	struct list_head	pending_cb_stateids;
>  	struct rcu_head		rcu;
> +
> +#if IS_ENABLED(CONFIG_NFS_LOCALIO)
> +	struct timespec64	cl_nfssvc_boot;
> +	seqlock_t		cl_boot_lock;
> +	struct rpc_clnt *	cl_rpcclient_localio;
> +	struct net *	        cl_nfssvc_net;
> +	struct auth_domain *	cl_nfssvc_dom;
> +	nfs_to_nfsd_open_t	nfsd_open_local_fh;
> +#endif /* CONFIG_NFS_LOCALIO */
>  };
>  
>  /*
> diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h
> new file mode 100644
> index 000000000000..109cb8534e3f
> --- /dev/null
> +++ b/include/linux/nfslocalio.h
> @@ -0,0 +1,37 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2024 Mike Snitzer <snitzer@hammerspace.com>
> + */
> +#ifndef __LINUX_NFSLOCALIO_H
> +#define __LINUX_NFSLOCALIO_H
> +
> +#include <linux/list.h>
> +#include <linux/uuid.h>
> +#include <linux/sunrpc/clnt.h>
> +#include <linux/sunrpc/svcauth.h>
> +#include <linux/nfs.h>
> +#include <net/net_namespace.h>
> +
> +/*
> + * Useful to allow a client to negotiate if localio
> + * possible with its server.
> + */
> +typedef struct {
> +	uuid_t uuid;
> +	struct list_head list;
> +	struct net *net; /* nfsd's network namespace */
> +	struct auth_domain *dom; /* auth_domain for localio */
> +} nfs_uuid_t;
> +
> +void nfs_uuid_begin(nfs_uuid_t *);
> +void nfs_uuid_end(nfs_uuid_t *);
> +bool nfs_uuid_is_local(const uuid_t *, struct net *, struct auth_domain *);
> +
> +typedef int (*nfs_to_nfsd_open_t)(struct net *, struct auth_domain *, struct rpc_clnt *,
> +				const struct cred *, const struct nfs_fh *,
> +				const fmode_t, struct file **);
> +
> +nfs_to_nfsd_open_t get_nfsd_open_local_fh(void);
> +void put_nfsd_open_local_fh(void);
> +
> +#endif  /* __LINUX_NFSLOCALIO_H */

-- 
Jeff Layton <jlayton@kernel.org>