Re: any idea about auto export multiple btrfs snapshots?

[Wang Yugui <wangyugui@e16-tech.com>]

[-- Attachment #1: Type: text/plain, Size: 4595 bytes --]

Hi,


> On Mon, 21 Jun 2021, Wang Yugui wrote:
> > Hi,
> > 
> > > > > It seems more fixes are needed.
> > > > 
> > > > I think the problem is that the submount doesn't appear in /proc/mounts.
> > > > "nfsd_fh()" in nfs-utils needs to be able to map from the uuid for a
> > > > filesystem to the mount point.  To do this it walks through /proc/mounts
> > > > checking the uuid of each filesystem.  If a filesystem isn't listed
> > > > there, it obviously fails.
> > > > 
> > > > I guess you could add code to nfs-utils to do whatever "btrfs subvol
> > > > list" does to make up for the fact that btrfs doesn't register in
> > > > /proc/mounts.
> > > 
> > > Another approach might be to just change svcxdr_encode_fattr3() and
> > > nfsd4_encode_fattr() in the 'FSIDSOJURCE_UUID' case to check if
> > > dentry->d_inode has a different btrfs volume id to
> > > exp->ex_path.dentry->d_inode.
> > > If it does, then mix the volume id into the fsid somehow.
> > > 
> > > With that, you wouldn't want the first change I suggested.
> > 
> > This is what I have done. and it is based on linux 5.10.44
> > 
> > but it still not work, so still more jobs needed.
> > 
> 
> The following is more what I had in mind.  It doesn't quite work and I
> cannot work out why.
> 
> If you 'stat' a file inside the subvol, then 'find' will not complete.
> If you don't, then it will.
> 
> Doing that 'stat' changes the st_dev number of the main filesystem,
> which seems really weird.
> I'm probably missing something obvious.  Maybe a more careful analysis
> of what is changing when will help.

we compare the trace output between crossmnt and btrfs subvol with some
trace, we found out that we need to add the subvol support to
follow_down().

btrfs subvol should be treated as virtual 'mount point' for nfsd in follow_down().

Best Regards
Wang Yugui (wangyugui@e16-tech.com)
2021/06/22


> NeilBrown
> 
> 
> diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c
> index 9421dae22737..790a3357525d 100644
> --- a/fs/nfsd/export.c
> +++ b/fs/nfsd/export.c
> @@ -15,6 +15,7 @@
>  #include <linux/slab.h>
>  #include <linux/namei.h>
>  #include <linux/module.h>
> +#include <linux/statfs.h>
>  #include <linux/exportfs.h>
>  #include <linux/sunrpc/svc_xprt.h>
>  
> @@ -575,6 +576,7 @@ static int svc_export_parse(struct cache_detail *cd, char *mesg, int mlen)
>  	int err;
>  	struct auth_domain *dom = NULL;
>  	struct svc_export exp = {}, *expp;
> +	struct kstatfs statfs;
>  	int an_int;
>  
>  	if (mesg[mlen-1] != '\n')
> @@ -604,6 +606,10 @@ static int svc_export_parse(struct cache_detail *cd, char *mesg, int mlen)
>  	err = kern_path(buf, 0, &exp.ex_path);
>  	if (err)
>  		goto out1;
> +	err = vfs_statfs(&exp.ex_path, &statfs);
> +	if (err)
> +		goto out3;
> +	exp.ex_fsid64 = statfs.f_fsid;
>  
>  	exp.ex_client = dom;
>  	exp.cd = cd;
> @@ -809,6 +815,7 @@ static void export_update(struct cache_head *cnew, struct cache_head *citem)
>  	new->ex_anon_uid = item->ex_anon_uid;
>  	new->ex_anon_gid = item->ex_anon_gid;
>  	new->ex_fsid = item->ex_fsid;
> +	new->ex_fsid64 = item->ex_fsid64;
>  	new->ex_devid_map = item->ex_devid_map;
>  	item->ex_devid_map = NULL;
>  	new->ex_uuid = item->ex_uuid;
> diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h
> index ee0e3aba4a6e..d3eb9a599918 100644
> --- a/fs/nfsd/export.h
> +++ b/fs/nfsd/export.h
> @@ -68,6 +68,7 @@ struct svc_export {
>  	kuid_t			ex_anon_uid;
>  	kgid_t			ex_anon_gid;
>  	int			ex_fsid;
> +	__kernel_fsid_t		ex_fsid64;
>  	unsigned char *		ex_uuid; /* 16 byte fsid */
>  	struct nfsd4_fs_locations ex_fslocs;
>  	uint32_t		ex_nflavors;
> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
> index 7abeccb975b2..8144e6037eae 100644
> --- a/fs/nfsd/nfs4xdr.c
> +++ b/fs/nfsd/nfs4xdr.c
> @@ -2869,6 +2869,7 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
>  	if (err)
>  		goto out_nfserr;
>  	if ((bmval0 & (FATTR4_WORD0_FILES_AVAIL | FATTR4_WORD0_FILES_FREE |
> +		       FATTR4_WORD0_FSID |
>  			FATTR4_WORD0_FILES_TOTAL | FATTR4_WORD0_MAXNAME)) ||
>  	    (bmval1 & (FATTR4_WORD1_SPACE_AVAIL | FATTR4_WORD1_SPACE_FREE |
>  		       FATTR4_WORD1_SPACE_TOTAL))) {
> @@ -3024,6 +3025,12 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
>  		case FSIDSOURCE_UUID:
>  			p = xdr_encode_opaque_fixed(p, exp->ex_uuid,
>  								EX_UUID_LEN);
> +			if (statfs.f_fsid.val[0] != exp->ex_fsid64.val[0] ||
> +			    statfs.f_fsid.val[1] != exp->ex_fsid64.val[1]) {
> +				/* looks like a btrfs subvol */
> +				p[-2] ^= statfs.f_fsid.val[0];
> +				p[-1] ^= statfs.f_fsid.val[1];
> +			}
>  			break;
>  		}
>  	}


[-- Attachment #2: 0001-nfsd-btrfs-subvol-support.txt --]
[-- Type: application/octet-stream, Size: 5521 bytes --]

From 57e6b3cec9b8ac396b661c190511af80839ddbe5 Mon Sep 17 00:00:00 2001
From: wangyugui <wangyugui@e16-tech.com>
Date: Thu, 17 Jun 2021 08:33:06 +0800
Subject: [PATCH] nfsd: btrfs subvol support

(struct statfs).f_fsid: 	uniq between btrfs subvols
(struct stat).st_dev: 		uniq between btrfs subvols
(struct statx).stx_mnt_id:	NOT uniq between btrfs subvols, but yet not used in nfs/nfsd
	kernel samples/vfs/test-statx.c
		stx_rdev_major/stx_rdev_minor seems be truncated by something
		like old_encode_dev()/old_decode_dev()?

TODO: (struct nfs_fattr).fsid
TODO: FSIDSOURCE_FSID in nfs3xdr.c/nfsxdr.c
---
 fs/nfsd/nfs3xdr.c |  2 +-
 fs/nfsd/nfs4xdr.c | 16 ++++++++++++----
 fs/nfsd/nfsd.h    | 27 +++++++++++++++++++++++++++
 fs/nfsd/vfs.c     | 10 ++++++++--
 4 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c
index 716566d..0de2953 100644
--- a/fs/nfsd/nfs3xdr.c
+++ b/fs/nfsd/nfs3xdr.c
@@ -877,7 +877,7 @@ compose_entry_fh(struct nfsd3_readdirres *cd, struct svc_fh *fhp,
 		dchild = lookup_positive_unlocked(name, dparent, namlen);
 	if (IS_ERR(dchild))
 		return rv;
-	if (d_mountpoint(dchild))
+	if (d_mountpoint(dchild) || unlikely(d_is_btrfs_subvol(dchild)))
 		goto out;
 	if (dchild->d_inode->i_ino != ino)
 		goto out;
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 5f5169b..ee335fc 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -2457,7 +2457,7 @@ static __be32 nfsd4_encode_path(struct xdr_stream *xdr,
 		if (path_equal(&cur, root))
 			break;
 		if (cur.dentry == cur.mnt->mnt_root) {
-			if (follow_up(&cur))
+			if (nfsd_follow_up(&cur))
 				continue;
 			goto out_free;
 		}
@@ -2648,7 +2648,7 @@ static int get_parent_attributes(struct svc_export *exp, struct kstat *stat)
 	int err;
 
 	path_get(&path);
-	while (follow_up(&path)) {
+	while (nfsd_follow_up(&path)) {
 		if (path.dentry != path.mnt->mnt_root)
 			break;
 	}
@@ -2728,6 +2728,7 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
 		.dentry	= dentry,
 	};
 	struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+	bool is_btrfs_subvol= d_is_btrfs_subvol(dentry);
 
 	BUG_ON(bmval1 & NFSD_WRITEONLY_ATTRS_WORD1);
 	BUG_ON(!nfsd_attrs_supported(minorversion, bmval));
@@ -2744,7 +2745,8 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
 	if ((bmval0 & (FATTR4_WORD0_FILES_AVAIL | FATTR4_WORD0_FILES_FREE |
 			FATTR4_WORD0_FILES_TOTAL | FATTR4_WORD0_MAXNAME)) ||
 	    (bmval1 & (FATTR4_WORD1_SPACE_AVAIL | FATTR4_WORD1_SPACE_FREE |
-		       FATTR4_WORD1_SPACE_TOTAL))) {
+		       FATTR4_WORD1_SPACE_TOTAL)) ||
+		unlikely(is_btrfs_subvol)) {
 		err = vfs_statfs(&path, &statfs);
 		if (err)
 			goto out_nfserr;
@@ -2895,7 +2897,13 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
 			*p++ = cpu_to_be32(MINOR(stat.dev));
 			break;
 		case FSIDSOURCE_UUID:
-			p = xdr_encode_opaque_fixed(p, exp->ex_uuid,
+			if (unlikely(is_btrfs_subvol)){
+				*p++ = cpu_to_be32(statfs.f_fsid.val[0]);
+				*p++ = cpu_to_be32(statfs.f_fsid.val[1]);
+				*p++ = cpu_to_be32(0);
+				*p++ = cpu_to_be32(0);
+			} else
+				p = xdr_encode_opaque_fixed(p, exp->ex_uuid,
 								EX_UUID_LEN);
 			break;
 		}
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index cb742e1..27baabb 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -487,4 +487,31 @@ static inline int nfsd4_is_junction(struct dentry *dentry)
 
 #endif /* CONFIG_NFSD_V4 */
 
+/* btrfs subvol support */
+/*
+ * same logical as fs/btrfs is_subvolume_inode(struct inode *inode)
+ * #define BTRFS_FIRST_FREE_OBJECTID 256ULL
+ * #define BTRFS_SUPER_MAGIC       0x9123683E
+ */
+static inline bool d_is_btrfs_subvol(const struct dentry *dentry)
+{
+    bool ret = dentry->d_inode && unlikely(dentry->d_inode->i_ino == 256ULL) &&
+		dentry->d_sb && dentry->d_sb->s_magic == BTRFS_SUPER_MAGIC;
+	//printk(KERN_INFO "d_is_btrfs_subvol(%s)=%d\n", dentry->d_name.name, ret);
+	return ret;
+}
+#include <linux/namei.h>
+/* add btrfs subvol support that only used in nfsd */
+static inline int nfsd_follow_down(struct path *path)
+{
+	return follow_down(path);
+}
+/* add btrfs subvol support that only used in nfsd */
+static inline int nfsd_follow_up(struct path *path)
+{
+	if(unlikely(d_is_btrfs_subvol(path->dentry)))
+		return 0;
+	return follow_up(path);
+}
+
 #endif /* LINUX_NFSD_NFSD_H */
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index 1ecacee..3ab9b7f 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -65,9 +65,13 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
 			    .dentry = dget(dentry)};
 	int err = 0;
 
-	err = follow_down(&path);
+	err = nfsd_follow_down(&path);
 	if (err < 0)
 		goto out;
+	if (unlikely(d_is_btrfs_subvol(dentry))){
+		path_put(&path);
+		goto out;
+	} else
 	if (path.mnt == exp->ex_path.mnt && path.dentry == dentry &&
 	    nfsd_mountpoint(dentry, exp) == 2) {
 		/* This is only a mountpoint in some other namespace */
@@ -114,7 +118,7 @@ static void follow_to_parent(struct path *path)
 {
 	struct dentry *dp;
 
-	while (path->dentry == path->mnt->mnt_root && follow_up(path))
+	while (path->dentry == path->mnt->mnt_root && nfsd_follow_up(path))
 		;
 	dp = dget_parent(path->dentry);
 	dput(path->dentry);
@@ -160,6 +164,8 @@ int nfsd_mountpoint(struct dentry *dentry, struct svc_export *exp)
 		return 1;
 	if (nfsd4_is_junction(dentry))
 		return 1;
+	if (d_is_btrfs_subvol(dentry))
+		return 1;
 	if (d_mountpoint(dentry))
 		/*
 		 * Might only be a mountpoint in a different namespace,
-- 
2.30.2


[-- Attachment #3: 0002-trace-nfsd-btrfs-subvol-support.txt --]
[-- Type: application/octet-stream, Size: 5897 bytes --]

From 639489a60b84f9d16955143f52fc6316205ac57a Mon Sep 17 00:00:00 2001
From: wangyugui <wangyugui@e16-tech.com>
Date: Thu, 17 Jun 2021 08:33:06 +0800
Subject: [PATCH] trace nfsd: btrfs subvol support

[  235.831136] set_version_and_fsid_type fsid_type=7
[  235.842483] nfsd_cross_mnt(test)=0
[  235.845882] nfsd: nfsd_lookup(fh 28: 00070001 00440001 00000000 73fb4b0a 31596b2e 7be9789b, test)=/
[  235.854902] set_version_and_fsid_type fsid_type=6
[  235.859686] nfs_d_automount(test)
[  235.863069] nfsd_cross_mnt(test)=0
[  235.866478] nfsd: nfsd_lookup(fh 28: 00070001 00440001 00000000 73fb4b0a 31596b2e 7be9789b, test)=/
[  235.875500] set_version_and_fsid_type fsid_type=6
[  239.204677] lookup_positive_unlocked(name=xfs2) dentry=xfs2
[  239.210311] nfsd_cross_mnt(xfs2)=0
[  239.213708] set_version_and_fsid_type fsid_type=6
[  239.218406] nfsd4_encode_dirent_fattr(/) FATTR4_WORD0_FSID=1  FATTR4_WORD1_MOUNTED_ON_FILEID=1
	why /?
[  239.227078] nfs_d_automount(xfs2)
	why?
[  239.230437] nfsd_cross_mnt(xfs2)=0
[  239.233838] nfsd: nfsd_lookup(fh 20: 00060001 2b031f7d c249fdd0 1aa84b8e 045d774a 00000000, xfs2)=/
[  239.242854] set_version_and_fsid_type fsid_type=6

[  373.332124] set_version_and_fsid_type fsid_type=7
[  373.337639] nfsd_cross_mnt(test)=0
[  373.341035] nfsd: nfsd_lookup(fh 28: 00070001 00440001 00000000 73fb4b0a 31596b2e 7be9789b, test)=/
[  373.350047] set_version_and_fsid_type fsid_type=6
[  373.354781] nfs_d_automount(test)
[  373.358125] nfsd_cross_mnt(test)=0
[  373.361524] nfsd: nfsd_lookup(fh 28: 00070001 00440001 00000000 73fb4b0a 31596b2e 7be9789b, test)=/
[  373.370537] set_version_and_fsid_type fsid_type=6
[  377.521908] lookup_positive_unlocked(name=sub1) dentry=sub1
[  377.527477] nfsd_cross_mnt(sub1)=0
[  377.530879] set_version_and_fsid_type fsid_type=6
[  377.535572] nfsd4_encode_dirent_fattr(sub1) FATTR4_WORD0_FSID=1  FATTR4_WORD1_MOUNTED_ON_FILEID=1
[  377.544420] lookup_positive_unlocked(name=.snapshot) dentry=.snapshot

btrfs subvols =>force crossmnt
	subvol nfs/umount: os shutdonw or manual nfs/umount?
		special status(BTRFS_LAST_FREE_OBJECTID,only return to nfs)?
		#define BTRFS_LAST_FREE_OBJECTID -256ULL
		(struct file )->(struct inode *f_inode)->(struct super_block *i_sb;)->(unsigned long s_magic)
btrfs->xfs	=>still need crossmnt
xfs->btrfs	=>still need crossmnt

NFSEXP_CROSSMOUNT
NFSD_JUNCTION_XATTR_NAME
AT_NO_AUTOMOUNT
NFS_ATTR_FATTR_MOUNTPOINT
S_AUTOMOUNT
---
 fs/nfs/dir.c       | 2 ++
 fs/nfs/namespace.c | 1 +
 fs/nfsd/nfs4xdr.c  | 5 +++++
 fs/nfsd/nfsfh.c    | 1 +
 fs/nfsd/vfs.c      | 5 +++++
 5 files changed, 14 insertions(+)

diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index c837675..975440d 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -1799,6 +1799,8 @@ nfs4_do_lookup_revalidate(struct inode *dir, struct dentry *dentry,
 
 	if (!(flags & LOOKUP_OPEN) || (flags & LOOKUP_DIRECTORY))
 		goto full_reval;
+	if (dentry->d_inode && dentry->d_inode->i_ino == 256ULL && dentry->d_sb)
+		printk(KERN_INFO "nfs4_do_lookup_revalidate(%s)=%lx\n", dentry->d_name.name, dentry->d_sb->s_magic);
 	if (d_mountpoint(dentry))
 		goto full_reval;
 
diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c
index 2bcbe38..f69715c 100644
--- a/fs/nfs/namespace.c
+++ b/fs/nfs/namespace.c
@@ -152,6 +152,7 @@ struct vfsmount *nfs_d_automount(struct path *path)
 	int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout);
 	int ret;
 
+	printk(KERN_INFO "nfs_d_automount(%s)\n", path->dentry->d_name.name);
 	if (IS_ROOT(path->dentry))
 		return ERR_PTR(-ESTALE);
 
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 6255b06..257ee17 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3307,6 +3307,7 @@ nfsd4_encode_dirent_fattr(struct xdr_stream *xdr, struct nfsd4_readdir *cd,
 	dentry = lookup_positive_unlocked(name, cd->rd_fhp->fh_dentry, namlen);
 	if (IS_ERR(dentry))
 		return nfserrno(PTR_ERR(dentry));
+	printk(KERN_INFO "lookup_positive_unlocked(name=%s) dentry=%s\n", name, dentry->d_name.name);
 
 	exp_get(exp);
 	/*
@@ -3345,6 +3346,10 @@ nfsd4_encode_dirent_fattr(struct xdr_stream *xdr, struct nfsd4_readdir *cd,
 out_put:
 	dput(dentry);
 	exp_put(exp);
+	printk(KERN_INFO "nfsd4_encode_dirent_fattr(%s) FATTR4_WORD0_FSID=%d  FATTR4_WORD1_MOUNTED_ON_FILEID=%d\n",
+		 dentry->d_name.name,
+		 !!(cd->rd_bmval[0]&FATTR4_WORD0_FSID),
+		!!(cd->rd_bmval[1]&FATTR4_WORD1_MOUNTED_ON_FILEID));
 	return nfserr;
 }
 
diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c
index c81dbba..28eaea3 100644
--- a/fs/nfsd/nfsfh.c
+++ b/fs/nfsd/nfsfh.c
@@ -530,6 +530,7 @@ static void set_version_and_fsid_type(struct svc_fh *fhp, struct svc_export *exp
 	fhp->fh_handle.fh_version = version;
 	if (version)
 		fhp->fh_handle.fh_fsid_type = fsid_type;
+	printk(KERN_INFO "set_version_and_fsid_type fsid_type=%d\n", fsid_type);
 }
 
 __be32
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index ae34ffc..6c55010 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -66,6 +66,8 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
 	int err = 0;
 
 	err = nfsd_follow_down(&path);
+	printk(KERN_INFO "follow_down()=%d path.mnt=%s path.dentry=%s\n", err,
+		path.mnt->mnt_root->d_name.name, path.dentry->d_name.name);
 	if (err < 0)
 		goto out;
 	if (unlikely(d_is_btrfs_subvol(dentry))){
@@ -111,6 +113,7 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
 	path_put(&path);
 	exp_put(exp2);
 out:
+	printk(KERN_INFO "nfsd_cross_mnt(%s)=%d\n", dentry->d_name.name, err);
 	return err;
 }
 
@@ -233,9 +236,11 @@ nfsd_lookup_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	}
 	*dentry_ret = dentry;
 	*exp_ret = exp;
+	// printk(KERN_INFO "nfsd: nfsd_lookup(fh %s, %.*s)=%s\n", SVCFH_fmt(fhp), len, name, dentry->d_name.name);
 	return 0;
 
 out_nfserr:
+	// printk(KERN_INFO "nfsd: nfsd_lookup(fh %s, %.*s) error\n", SVCFH_fmt(fhp), len, name);
 	exp_put(exp);
 	return nfserrno(host_err);
 }
-- 
2.30.2