Re: any idea about auto export multiple btrfs snapshots?

Wang Yugui
2021-06-22T07:14:11Z

[-- Attachment #1: Type: text/plain, Size: 2210 bytes --]

Hi,

> > >
> > > > > > It seems more fixes are needed.
> > > > >
> > > > > I think the problem is that the submount doesn't appear in /proc/mounts.
> > > > > "nfsd_fh()" in nfs-utils needs to be able to map from the uuid for a
> > > > > filesystem to the mount point. To do this it walks through /proc/mounts
> > > > > checking the uuid of each filesystem. If a filesystem isn't listed
> > > > > there, it obviously fails.
> > > > >
> > > > > I guess you could add code to nfs-utils to do whatever "btrfs subvol
> > > > > list" does to make up for the fact that btrfs doesn't register in
> > > > > /proc/mounts.
> > > >
> > > > Another approach might be to just change svcxdr_encode_fattr3() and
> > > > nfsd4_encode_fattr() in the 'FSIDSOJURCE_UUID' case to check if
> > > > dentry->d_inode has a different btrfs volume id to
> > > > exp->ex_path.dentry->d_inode.
> > > > If it does, then mix the volume id into the fsid somehow.
> > > >
> > > > With that, you wouldn't want the first change I suggested.
> > >
> > > This is what I have done. and it is based on linux 5.10.44
> > >
> > > but it still not work, so still more jobs needed.
> > >
> >
> > The following is more what I had in mind. It doesn't quite work and I
> > cannot work out why.
> >
> > If you 'stat' a file inside the subvol, then 'find' will not complete.
> > If you don't, then it will.
> >
> > Doing that 'stat' changes the st_dev number of the main filesystem,
> > which seems really weird.
> > I'm probably missing something obvious. Maybe a more careful analysis
> > of what is changing when will help.
>
> we compare the trace output between crossmnt and btrfs subvol with some
> trace, we found out that we need to add the subvol support to
> follow_down().
>
> btrfs subvol should be treated as virtual 'mount point' for nfsd in follow_down().

btrfs subvol crossmnt begin to work, although buggy.

some subvol is crossmnt-ed, some subvol is yet not, and some dir is
wrongly crossmnt-ed

'stat /nfs/test /nfs/test/sub1' will cause btrfs subvol crossmnt begin
to happen.

This is the current patch based on 5.10.44.
At least nfsd_follow_up() is buggy.

Best Regards
Wang Yugui (wangyugui@e16-tech.com)
2021/06/22

[-- Attachment #2: 0001-nfsd-btrfs-subvol-support.txt --]
[-- Type: application/octet-stream, Size: 6044 bytes --]

From 57e6b3cec9b8ac396b661c190511af80839ddbe5 Mon Sep 17 00:00:00 2001
From: wangyugui <wangyugui@e16-tech.com>
Date: Thu, 17 Jun 2021 08:33:06 +0800
Subject: [PATCH] nfsd: btrfs subvol support

(struct statfs).f_fsid: uniq between btrfs subvols
(struct stat).st_dev: uniq between btrfs subvols
(struct statx).stx_mnt_id: NOT uniq between btrfs subvols, but yet not used in nfs/nfsd
kernel samples/vfs/test-statx.c
stx_rdev_major/stx_rdev_minor seems be truncated by something
like old_encode_dev()/old_decode_dev()?

TODO: (struct nfs_fattr).fsid
TODO: FSIDSOURCE_FSID in nfs3xdr.c/nfsxdr.c
---
fs/nfsd/nfs3xdr.c | 2 +-
fs/nfsd/nfs4xdr.c | 16 ++++++++++++----
fs/nfsd/nfsd.h | 42 ++++++++++++++++++++++++++++++++++++++++++
fs/nfsd/vfs.c | 10 ++++++++--
4 files changed, 63 insertions(+), 7 deletions(-)

diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c
index 716566d..0de2953 100644
--- a/fs/nfsd/nfs3xdr.c
+++ b/fs/nfsd/nfs3xdr.c
@@ -877,7 +877,7 @@ compose_entry_fh(struct nfsd3_readdirres *cd, struct svc_fh *fhp,
dchild = lookup_positive_unlocked(name, dparent, namlen);
if (IS_ERR(dchild))
return rv;
- if (d_mountpoint(dchild))
+ if (d_mountpoint(dchild) || unlikely(d_is_btrfs_subvol(dchild)))
goto out;
if (dchild->d_inode->i_ino != ino)
goto out;
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 5f5169b..ee335fc 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -2457,7 +2457,7 @@ static __be32 nfsd4_encode_path(struct xdr_stream *xdr,
if (path_equal(&cur, root))
break;
if (cur.dentry == cur.mnt->mnt_root) {
- if (follow_up(&cur))
+ if (nfsd_follow_up(&cur))
continue;
goto out_free;
}
@@ -2648,7 +2648,7 @@ static int get_parent_attributes(struct svc_export *exp, struct kstat *stat)
int err;

path_get(&path);
- while (follow_up(&path)) {
+ while (nfsd_follow_up(&path)) {
if (path.dentry != path.mnt->mnt_root)
break;
}
@@ -2728,6 +2728,7 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
.dentry = dentry,
};
struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+ bool is_btrfs_subvol= d_is_btrfs_subvol(dentry);

BUG_ON(bmval1 & NFSD_WRITEONLY_ATTRS_WORD1);
BUG_ON(!nfsd_attrs_supported(minorversion, bmval));
@@ -2744,7 +2745,8 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
if ((bmval0 & (FATTR4_WORD0_FILES_AVAIL | FATTR4_WORD0_FILES_FREE |
FATTR4_WORD0_FILES_TOTAL | FATTR4_WORD0_MAXNAME)) ||
(bmval1 & (FATTR4_WORD1_SPACE_AVAIL | FATTR4_WORD1_SPACE_FREE |
- FATTR4_WORD1_SPACE_TOTAL))) {
+ FATTR4_WORD1_SPACE_TOTAL)) ||
+ unlikely(is_btrfs_subvol)) {
err = vfs_statfs(&path, &statfs);
if (err)
goto out_nfserr;
@@ -2895,7 +2897,13 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
*p++ = cpu_to_be32(MINOR(stat.dev));
break;
case FSIDSOURCE_UUID:
- p = xdr_encode_opaque_fixed(p, exp->ex_uuid,
+ if (unlikely(is_btrfs_subvol)){
+ *p++ = cpu_to_be32(statfs.f_fsid.val[0]);
+ *p++ = cpu_to_be32(statfs.f_fsid.val[1]);
+ *p++ = cpu_to_be32(0);
+ *p++ = cpu_to_be32(0);
+ } else
+ p = xdr_encode_opaque_fixed(p, exp->ex_uuid,
EX_UUID_LEN);
break;
}
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index cb742e1..42e14d6 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -487,4 +487,47 @@ static inline int nfsd4_is_junction(struct dentry *dentry)

#endif /* CONFIG_NFSD_V4 */

+/* btrfs subvol support */
+/*
+ * same logical as fs/btrfs is_subvolume_inode(struct inode *inode)
+ * #define BTRFS_FIRST_FREE_OBJECTID 256ULL
+ * #define BTRFS_SUPER_MAGIC 0x9123683E
+ */
+static inline bool d_is_btrfs_subvol(const struct dentry *dentry)
+{
+ bool ret = dentry->d_inode && unlikely(dentry->d_inode->i_ino == 256ULL) &&
+ dentry->d_sb && dentry->d_sb->s_magic == BTRFS_SUPER_MAGIC;
+ //printk(KERN_INFO "d_is_btrfs_subvol(%s)=%d\n", dentry->d_name.name, ret);
+ return ret;
+}
+#include <linux/namei.h>
+/* add btrfs subvol support that only used in nfsd */
+/* FIXME: free clone_private_mount()? */
+static inline int nfsd_follow_down(struct path *path)
+{
+ if(unlikely(d_is_btrfs_subvol(path->dentry))){
+ //struct dentry *mnt_root=path->dentry;
+ struct vfsmount *mounted = clone_private_mount(path);
+ if (mounted) {
+ //mounted->mnt_root=mnt_root;
+ //? dput(path->dentry);
+ //? mntput(path->mnt);
+ path->mnt = mounted;
+ path->dentry = dget(mounted->mnt_root);
+ return 0;
+ }
+ }
+ return follow_down(path);
+}
+/* add btrfs subvol support that only used in nfsd */
+/* FIXME: free clone_private_mount()? */
+static inline int nfsd_follow_up(struct path *path)
+{
+ printk(KERN_INFO "nfsd_follow_up(%s)\n", path->dentry->d_name.name);
+ if(unlikely(d_is_btrfs_subvol(path->dentry))){
+ return 0;
+ }
+ return follow_up(path);
+}
+
#endif /* LINUX_NFSD_NFSD_H */
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index 1ecacee..3ab9b7f 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -65,9 +65,13 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
.dentry = dget(dentry)};
int err = 0;

- err = follow_down(&path);
+ err = nfsd_follow_down(&path);
if (err < 0)
goto out;
+ if (unlikely(d_is_btrfs_subvol(dentry))){
+ path_put(&path);
+ goto out;
+ } else
if (path.mnt == exp->ex_path.mnt && path.dentry == dentry &&
nfsd_mountpoint(dentry, exp) == 2) {
/* This is only a mountpoint in some other namespace */
@@ -114,7 +118,7 @@ static void follow_to_parent(struct path *path)
{
struct dentry *dp;

- while (path->dentry == path->mnt->mnt_root && follow_up(path))
+ while (path->dentry == path->mnt->mnt_root && nfsd_follow_up(path))
;
dp = dget_parent(path->dentry);
dput(path->dentry);
@@ -160,6 +164,8 @@ int nfsd_mountpoint(struct dentry *dentry, struct svc_export *exp)
return 1;
if (nfsd4_is_junction(dentry))
return 1;
+ if (d_is_btrfs_subvol(dentry))
+ return 1;
if (d_mountpoint(dentry))
/*
* Might only be a mountpoint in a different namespace,
--
2.30.2

[-- Attachment #3: 0002-trace-nfsd-btrfs-subvol-support.txt --]
[-- Type: application/octet-stream, Size: 5897 bytes --]

From 639489a60b84f9d16955143f52fc6316205ac57a Mon Sep 17 00:00:00 2001
From: wangyugui <wangyugui@e16-tech.com>
Date: Thu, 17 Jun 2021 08:33:06 +0800
Subject: [PATCH] trace nfsd: btrfs subvol support

[ 235.831136] set_version_and_fsid_type fsid_type=7
[ 235.842483] nfsd_cross_mnt(test)=0
[ 235.845882] nfsd: nfsd_lookup(fh 28: 00070001 00440001 00000000 73fb4b0a 31596b2e 7be9789b, test)=/
[ 235.854902] set_version_and_fsid_type fsid_type=6
[ 235.859686] nfs_d_automount(test)
[ 235.863069] nfsd_cross_mnt(test)=0
[ 235.866478] nfsd: nfsd_lookup(fh 28: 00070001 00440001 00000000 73fb4b0a 31596b2e 7be9789b, test)=/
[ 235.875500] set_version_and_fsid_type fsid_type=6
[ 239.204677] lookup_positive_unlocked(name=xfs2) dentry=xfs2
[ 239.210311] nfsd_cross_mnt(xfs2)=0
[ 239.213708] set_version_and_fsid_type fsid_type=6
[ 239.218406] nfsd4_encode_dirent_fattr(/) FATTR4_WORD0_FSID=1 FATTR4_WORD1_MOUNTED_ON_FILEID=1
why /?
[ 239.227078] nfs_d_automount(xfs2)
why?
[ 239.230437] nfsd_cross_mnt(xfs2)=0
[ 239.233838] nfsd: nfsd_lookup(fh 20: 00060001 2b031f7d c249fdd0 1aa84b8e 045d774a 00000000, xfs2)=/
[ 239.242854] set_version_and_fsid_type fsid_type=6

[ 373.332124] set_version_and_fsid_type fsid_type=7
[ 373.337639] nfsd_cross_mnt(test)=0
[ 373.341035] nfsd: nfsd_lookup(fh 28: 00070001 00440001 00000000 73fb4b0a 31596b2e 7be9789b, test)=/
[ 373.350047] set_version_and_fsid_type fsid_type=6
[ 373.354781] nfs_d_automount(test)
[ 373.358125] nfsd_cross_mnt(test)=0
[ 373.361524] nfsd: nfsd_lookup(fh 28: 00070001 00440001 00000000 73fb4b0a 31596b2e 7be9789b, test)=/
[ 373.370537] set_version_and_fsid_type fsid_type=6
[ 377.521908] lookup_positive_unlocked(name=sub1) dentry=sub1
[ 377.527477] nfsd_cross_mnt(sub1)=0
[ 377.530879] set_version_and_fsid_type fsid_type=6
[ 377.535572] nfsd4_encode_dirent_fattr(sub1) FATTR4_WORD0_FSID=1 FATTR4_WORD1_MOUNTED_ON_FILEID=1
[ 377.544420] lookup_positive_unlocked(name=.snapshot) dentry=.snapshot

btrfs subvols =>force crossmnt
subvol nfs/umount: os shutdonw or manual nfs/umount?
special status(BTRFS_LAST_FREE_OBJECTID,only return to nfs)?
#define BTRFS_LAST_FREE_OBJECTID -256ULL
(struct file )->(struct inode *f_inode)->(struct super_block *i_sb;)->(unsigned long s_magic)
btrfs->xfs =>still need crossmnt
xfs->btrfs =>still need crossmnt

NFSEXP_CROSSMOUNT
NFSD_JUNCTION_XATTR_NAME
AT_NO_AUTOMOUNT
NFS_ATTR_FATTR_MOUNTPOINT
S_AUTOMOUNT
---
fs/nfs/dir.c | 2 ++
fs/nfs/namespace.c | 1 +
fs/nfsd/nfs4xdr.c | 5 +++++
fs/nfsd/nfsfh.c | 1 +
fs/nfsd/vfs.c | 5 +++++
5 files changed, 14 insertions(+)

diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index c837675..975440d 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -1799,6 +1799,8 @@ nfs4_do_lookup_revalidate(struct inode *dir, struct dentry *dentry,

if (!(flags & LOOKUP_OPEN) || (flags & LOOKUP_DIRECTORY))
goto full_reval;
+ if (dentry->d_inode && dentry->d_inode->i_ino == 256ULL && dentry->d_sb)
+ printk(KERN_INFO "nfs4_do_lookup_revalidate(%s)=%lx\n", dentry->d_name.name, dentry->d_sb->s_magic);
if (d_mountpoint(dentry))
goto full_reval;

diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c
index 2bcbe38..f69715c 100644
--- a/fs/nfs/namespace.c
+++ b/fs/nfs/namespace.c
@@ -152,6 +152,7 @@ struct vfsmount *nfs_d_automount(struct path *path)
int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout);
int ret;

+ printk(KERN_INFO "nfs_d_automount(%s)\n", path->dentry->d_name.name);
if (IS_ROOT(path->dentry))
return ERR_PTR(-ESTALE);

diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 6255b06..257ee17 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3307,6 +3307,7 @@ nfsd4_encode_dirent_fattr(struct xdr_stream *xdr, struct nfsd4_readdir *cd,
dentry = lookup_positive_unlocked(name, cd->rd_fhp->fh_dentry, namlen);
if (IS_ERR(dentry))
return nfserrno(PTR_ERR(dentry));
+ printk(KERN_INFO "lookup_positive_unlocked(name=%s) dentry=%s\n", name, dentry->d_name.name);

exp_get(exp);
/*
@@ -3345,6 +3346,10 @@ nfsd4_encode_dirent_fattr(struct xdr_stream *xdr, struct nfsd4_readdir *cd,
out_put:
dput(dentry);
exp_put(exp);
+ printk(KERN_INFO "nfsd4_encode_dirent_fattr(%s) FATTR4_WORD0_FSID=%d FATTR4_WORD1_MOUNTED_ON_FILEID=%d\n",
+ dentry->d_name.name,
+ !!(cd->rd_bmval[0]&FATTR4_WORD0_FSID),
+ !!(cd->rd_bmval[1]&FATTR4_WORD1_MOUNTED_ON_FILEID));
return nfserr;
}

diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c
index c81dbba..28eaea3 100644
--- a/fs/nfsd/nfsfh.c
+++ b/fs/nfsd/nfsfh.c
@@ -530,6 +530,7 @@ static void set_version_and_fsid_type(struct svc_fh *fhp, struct svc_export *exp
fhp->fh_handle.fh_version = version;
if (version)
fhp->fh_handle.fh_fsid_type = fsid_type;
+ printk(KERN_INFO "set_version_and_fsid_type fsid_type=%d\n", fsid_type);
}

__be32
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index ae34ffc..6c55010 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -66,6 +66,8 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
int err = 0;

err = nfsd_follow_down(&path);
+ printk(KERN_INFO "follow_down()=%d path.mnt=%s path.dentry=%s\n", err,
+ path.mnt->mnt_root->d_name.name, path.dentry->d_name.name);
if (err < 0)
goto out;
if (unlikely(d_is_btrfs_subvol(dentry))){
@@ -111,6 +113,7 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
path_put(&path);
exp_put(exp2);
out:
+ printk(KERN_INFO "nfsd_cross_mnt(%s)=%d\n", dentry->d_name.name, err);
return err;
}

@@ -233,9 +236,11 @@ nfsd_lookup_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp,
}
*dentry_ret = dentry;
*exp_ret = exp;
+ // printk(KERN_INFO "nfsd: nfsd_lookup(fh %s, %.*s)=%s\n", SVCFH_fmt(fhp), len, name, dentry->d_name.name);
return 0;

out_nfserr:
+ // printk(KERN_INFO "nfsd: nfsd_lookup(fh %s, %.*s) error\n", SVCFH_fmt(fhp), len, name);
exp_put(exp);
return nfserrno(host_err);
}
--
2.30.2