[PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode

[PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode

[Jingbo Xu]

v8:
- patch 7: fix the missing call for erofs_mkfs_dump_blobs() in rebuild
  mode by also calling erofs_mkfs_dump_blobs() when sbi->extra_devices
  is not zero; otherwise the content of the on-disk device slot is all 0
- patch 8: strip overlay.opaque xattr when --ovlfs-strip=1; update the
  commit message accordingly

v1: https://lore.kernel.org/all/20230814034239.54660-1-jefflexu@linux.alibaba.com/
v2: https://lore.kernel.org/all/20230816021347.126886-1-jefflexu@linux.alibaba.com/
v3: https://lore.kernel.org/all/20230822092457.114686-1-jefflexu@linux.alibaba.com/
v4: https://lore.kernel.org/all/20230823071517.12303-1-jefflexu@linux.alibaba.com/
v5: https://lore.kernel.org/all/20230901094706.27539-1-jefflexu@linux.alibaba.com/
v6: https://lore.kernel.org/all/20230905100227.1072-1-jefflexu@linux.alibaba.com/
v7: https://lore.kernel.org/all/20230909163240.42057-2-hsiangkao@linux.alibaba.com/

-------------------------

Introduce a new rebuild mode merging multiple erofs images generated
from either tarerfs index mode (--tar=i):

	mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layer0.erofs ... layerN.erofs

or tarerofs non-index mode (--tar=f):

	mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layer0.erofs ... layerN.erofs


Jingbo Xu (8):
  erofs-utils: lib: add list_splice_tail() helper
  erofs-utils: lib: make erofs_get_unhashed_chunk() global
  erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
  erofs-utils: lib: add erofs_insert_ihash() helper
  erofs-utils: lib: add erofs_rebuild_get_dentry() helper
  erofs-utils: lib: add erofs_rebuild_load_tree() helper
  erofs-utils: mkfs: introduce rebuild mode
  erofs-utils: mkfs: add `--ovlfs-strip` option

 include/erofs/blobchunk.h |   2 +
 include/erofs/config.h    |   1 +
 include/erofs/inode.h     |   5 +-
 include/erofs/internal.h  |   7 +-
 include/erofs/list.h      |  20 ++
 include/erofs/rebuild.h   |  21 ++
 include/erofs/xattr.h     |   2 +
 lib/Makefile.am           |   3 +-
 lib/blobchunk.c           |   2 +-
 lib/inode.c               |  31 ++-
 lib/rebuild.c             | 404 ++++++++++++++++++++++++++++++++++++++
 lib/tar.c                 | 109 +---------
 lib/xattr.c               |  87 ++++++++
 mkfs/main.c               | 227 ++++++++++++++++-----
 14 files changed, 751 insertions(+), 170 deletions(-)
 create mode 100644 include/erofs/rebuild.h
 create mode 100644 lib/rebuild.c

-- 
2.19.1.6.gb485710b

[PATCH v8 1/8] erofs-utils: lib: add list_splice_tail() helper

[Jingbo Xu]

Add list_splice_tail() helper.

Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 include/erofs/list.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/include/erofs/list.h b/include/erofs/list.h
index 3f5da1a..d7a9fee 100644
--- a/include/erofs/list.h
+++ b/include/erofs/list.h
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)
 	return head->next == head;
 }
 
+static inline void __list_splice(struct list_head *list,
+		struct list_head *prev, struct list_head *next)
+{
+	struct list_head *first = list->next;
+	struct list_head *last = list->prev;
+
+	first->prev = prev;
+	prev->next = first;
+
+	last->next = next;
+	next->prev = last;
+}
+
+static inline void list_splice_tail(struct list_head *list,
+				    struct list_head *head)
+{
+	if (!list_empty(list))
+		__list_splice(list, head->prev, head);
+}
+
 #define list_entry(ptr, type, member) container_of(ptr, type, member)
 
 #define list_first_entry(ptr, type, member)                                    \
-- 
2.19.1.6.gb485710b

[PATCH v8 2/8] erofs-utils: lib: make erofs_get_unhashed_chunk() global

[Jingbo Xu]

... so that it could be called from outside blobchunk.c later.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 include/erofs/blobchunk.h | 2 ++
 lib/blobchunk.c           | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
index 010aee1..fb85d8e 100644
--- a/include/erofs/blobchunk.h
+++ b/include/erofs/blobchunk.h
@@ -14,6 +14,8 @@ extern "C"
 
 #include "erofs/internal.h"
 
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+		erofs_blk_t blkaddr, erofs_off_t sourceoffset);
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
 int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
 int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset);
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
index 86b29c1..71fb2ff 100644
--- a/lib/blobchunk.c
+++ b/lib/blobchunk.c
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = {
 };
 static LIST_HEAD(unhashed_blobchunks);
 
-static struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
 		erofs_blk_t blkaddr, erofs_off_t sourceoffset)
 {
 	struct erofs_blobchunk *chunk;
-- 
2.19.1.6.gb485710b

[PATCH v8 3/8] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper

[Jingbo Xu]

Add erofs_read_xattrs_from_disk() helper to reading extended
attributes from disk.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 include/erofs/internal.h |  1 +
 include/erofs/xattr.h    |  1 +
 lib/xattr.c              | 69 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 71 insertions(+)

diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index c36bb24..3b866c9 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -196,6 +196,7 @@ struct erofs_inode {
 	bool compressed_idata;
 	bool lazy_tailblock;
 	bool with_tmpfile;
+	bool opaque;
 	/* OVL: non-merge dir that may contain whiteout entries */
 	bool whiteouts;
 
diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index 2ecb18e..0364f24 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -58,6 +58,7 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
 		   const void *value, size_t size);
 int erofs_set_opaque_xattr(struct erofs_inode *inode);
 int erofs_set_origin_xattr(struct erofs_inode *inode);
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
 
 #ifdef __cplusplus
 }
diff --git a/lib/xattr.c b/lib/xattr.c
index bf63a81..e3a1b44 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -566,6 +566,75 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)
 	return erofs_droid_xattr_set_caps(inode);
 }
 
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
+{
+	ssize_t kllen;
+	char *keylst, *key;
+	int ret;
+
+	init_list_head(&inode->i_xattrs);
+	kllen = erofs_listxattr(inode, NULL, 0);
+	if (kllen < 0)
+		return kllen;
+	if (kllen <= 1)
+		return 0;
+
+	keylst = malloc(kllen);
+	if (!keylst)
+		return -ENOMEM;
+
+	ret = erofs_listxattr(inode, keylst, kllen);
+	if (ret < 0)
+		goto out;
+
+	for (key = keylst; key < keylst + kllen; key += strlen(key) + 1) {
+		void *value = NULL;
+		size_t size = 0;
+
+		if (!strcmp(key, OVL_XATTR_OPAQUE)) {
+			if (!S_ISDIR(inode->i_mode)) {
+				erofs_dbg("file %s: opaque xattr on non-dir",
+					  inode->i_srcpath);
+				ret = -EINVAL;
+				goto out;
+			}
+			inode->opaque = true;
+		}
+
+		ret = erofs_getxattr(inode, key, NULL, 0);
+		if (ret < 0)
+			goto out;
+		if (ret) {
+			size = ret;
+			value = malloc(size);
+			if (!value) {
+				ret = -ENOMEM;
+				goto out;
+			}
+
+			ret = erofs_getxattr(inode, key, value, size);
+			if (ret < 0) {
+				free(value);
+				goto out;
+			}
+			DBG_BUGON(ret != size);
+		} else if (S_ISDIR(inode->i_mode) &&
+			   !strcmp(key, OVL_XATTR_ORIGIN)) {
+			ret = 0;
+			inode->whiteouts = true;
+			continue;
+		}
+
+		ret = erofs_setxattr(inode, key, value, size);
+		free(value);
+		if (ret)
+			break;
+	}
+out:
+	free(keylst);
+	return ret;
+}
+
 static inline unsigned int erofs_next_xattr_align(unsigned int pos,
 						  struct xattr_item *item)
 {
-- 
2.19.1.6.gb485710b

[PATCH v8 4/8] erofs-utils: lib: add erofs_insert_ihash() helper

[Jingbo Xu]

Add erofs_insert_ihash() helper inserting inode into inode hash table.

Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the
header file.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 include/erofs/inode.h |  3 +++
 lib/inode.c           | 10 +++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index e8a5670..1c602a8 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);
 unsigned char erofs_mode_to_ftype(umode_t mode);
 unsigned char erofs_ftype_to_dtype(unsigned int filetype);
 void erofs_inode_manager_init(void);
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget(dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget_by_nid(erofs_nid_t nid);
 unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
diff --git a/lib/inode.c b/lib/inode.c
index 92796c9..79e1795 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)
 		init_list_head(&inode_hashtable[i]);
 }
 
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino)
+{
+	list_add(&inode->i_hash,
+		 &inode_hashtable[(ino ^ dev) % NR_INODE_HASHTABLE]);
+}
+
 /* get the inode from the (source) inode # */
 struct erofs_inode *erofs_iget(dev_t dev, ino_t ino)
 {
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st,
 		inode->inode_isize = sizeof(struct erofs_inode_compact);
 	}
 
-	list_add(&inode->i_hash,
-		 &inode_hashtable[(st->st_ino ^ st->st_dev) %
-				  NR_INODE_HASHTABLE]);
+	erofs_insert_ihash(inode, st->st_dev, st->st_ino);
 	return 0;
 }
 
-- 
2.19.1.6.gb485710b

[PATCH v8 5/8] erofs-utils: lib: add erofs_rebuild_get_dentry() helper

[Jingbo Xu]

Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry() and
move it into lib/rebuild.c.

No logic changes.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 include/erofs/rebuild.h |  19 +++++++
 lib/Makefile.am         |   3 +-
 lib/rebuild.c           | 116 ++++++++++++++++++++++++++++++++++++++++
 lib/tar.c               | 109 ++-----------------------------------
 4 files changed, 140 insertions(+), 107 deletions(-)
 create mode 100644 include/erofs/rebuild.h
 create mode 100644 lib/rebuild.c

diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
new file mode 100644
index 0000000..92873c9
--- /dev/null
+++ b/include/erofs/rebuild.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_REBUILD_H
+#define __EROFS_REBUILD_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "internal.h"
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+		char *path, bool aufs, bool *whout, bool *opq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 3e09516..8a45bd6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/compress_hints.h \
       $(top_srcdir)/include/erofs/fragments.h \
       $(top_srcdir)/include/erofs/xxhash.h \
+      $(top_srcdir)/include/erofs/rebuild.h \
       $(top_srcdir)/lib/liberofs_private.h
 
 noinst_HEADERS += compressor.h
@@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
 		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
-		      block_list.c xxhash.c
+		      block_list.c xxhash.c rebuild.c
 
 liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
 if ENABLE_LZ4
diff --git a/lib/rebuild.c b/lib/rebuild.c
new file mode 100644
index 0000000..2398ca6
--- /dev/null
+++ b/lib/rebuild.c
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include "erofs/print.h"
+#include "erofs/inode.h"
+#include "erofs/rebuild.h"
+#include "erofs/internal.h"
+
+#ifdef HAVE_LINUX_AUFS_TYPE_H
+#include <linux/aufs_type.h>
+#else
+#define AUFS_WH_PFX		".wh."
+#define AUFS_DIROPQ_NAME	AUFS_WH_PFX ".opq"
+#define AUFS_WH_DIROPQ		AUFS_WH_PFX AUFS_DIROPQ_NAME
+#endif
+
+static struct erofs_dentry *erofs_rebuild_mkdir(struct erofs_inode *dir,
+						const char *s)
+{
+	struct erofs_inode *inode;
+	struct erofs_dentry *d;
+
+	inode = erofs_new_inode();
+	if (IS_ERR(inode))
+		return ERR_CAST(inode);
+
+	inode->i_mode = S_IFDIR | 0755;
+	inode->i_parent = dir;
+	inode->i_uid = getuid();
+	inode->i_gid = getgid();
+	inode->i_mtime = inode->sbi->build_time;
+	inode->i_mtime_nsec = inode->sbi->build_time_nsec;
+	erofs_init_empty_dir(inode);
+
+	d = erofs_d_alloc(dir, s);
+	if (!IS_ERR(d)) {
+		d->type = EROFS_FT_DIR;
+		d->inode = inode;
+	}
+	return d;
+}
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+		char *path, bool aufs, bool *whout, bool *opq)
+{
+	struct erofs_dentry *d = NULL;
+	unsigned int len = strlen(path);
+	char *s = path;
+
+	*whout = false;
+	*opq = false;
+
+	while (s < path + len) {
+		char *slash = memchr(s, '/', path + len - s);
+
+		if (slash) {
+			if (s == slash) {
+				while (*++s == '/');	/* skip '//...' */
+				continue;
+			}
+			*slash = '\0';
+		}
+
+		if (!memcmp(s, ".", 2)) {
+			/* null */
+		} else if (!memcmp(s, "..", 3)) {
+			pwd = pwd->i_parent;
+		} else {
+			struct erofs_inode *inode = NULL;
+
+			if (aufs && !slash) {
+				if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
+					*opq = true;
+					break;
+				}
+				if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
+					s += sizeof(AUFS_WH_PFX) - 1;
+					*whout = true;
+				}
+			}
+
+			list_for_each_entry(d, &pwd->i_subdirs, d_child) {
+				if (!strcmp(d->name, s)) {
+					if (d->type != EROFS_FT_DIR && slash)
+						return ERR_PTR(-EIO);
+					inode = d->inode;
+					break;
+				}
+			}
+
+			if (inode) {
+				pwd = inode;
+			} else if (!slash) {
+				d = erofs_d_alloc(pwd, s);
+				if (IS_ERR(d))
+					return d;
+				d->type = EROFS_FT_UNKNOWN;
+				d->inode = pwd;
+			} else {
+				d = erofs_rebuild_mkdir(pwd, s);
+				if (IS_ERR(d))
+					return d;
+				pwd = d->inode;
+			}
+		}
+		if (slash) {
+			*slash = '/';
+			s = slash + 1;
+		} else {
+			break;
+		}
+	}
+	return d;
+}
diff --git a/lib/tar.c b/lib/tar.c
index b58bfd5..0b96cae 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -3,13 +3,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
-#ifdef HAVE_LINUX_AUFS_TYPE_H
-#include <linux/aufs_type.h>
-#else
-#define AUFS_WH_PFX		".wh."
-#define AUFS_DIROPQ_NAME	AUFS_WH_PFX ".opq"
-#define AUFS_WH_DIROPQ		AUFS_WH_PFX AUFS_DIROPQ_NAME
-#endif
 #include "erofs/print.h"
 #include "erofs/cache.h"
 #include "erofs/inode.h"
@@ -18,6 +11,7 @@
 #include "erofs/io.h"
 #include "erofs/xattr.h"
 #include "erofs/blobchunk.h"
+#include "erofs/rebuild.h"
 
 static char erofs_libbuf[16384];
 
@@ -131,103 +125,6 @@ static long long tarerofs_parsenum(const char *ptr, int len)
 	return tarerofs_otoi(ptr, len);
 }
 
-static struct erofs_dentry *tarerofs_mkdir(struct erofs_inode *dir, const char *s)
-{
-	struct erofs_inode *inode;
-	struct erofs_dentry *d;
-
-	inode = erofs_new_inode();
-	if (IS_ERR(inode))
-		return ERR_CAST(inode);
-
-	inode->i_mode = S_IFDIR | 0755;
-	inode->i_parent = dir;
-	inode->i_uid = getuid();
-	inode->i_gid = getgid();
-	inode->i_mtime = inode->sbi->build_time;
-	inode->i_mtime_nsec = inode->sbi->build_time_nsec;
-	erofs_init_empty_dir(inode);
-
-	d = erofs_d_alloc(dir, s);
-	if (!IS_ERR(d)) {
-		d->type = EROFS_FT_DIR;
-		d->inode = inode;
-	}
-	return d;
-}
-
-static struct erofs_dentry *tarerofs_get_dentry(struct erofs_inode *pwd, char *path,
-					        bool aufs, bool *whout, bool *opq)
-{
-	struct erofs_dentry *d = NULL;
-	unsigned int len = strlen(path);
-	char *s = path;
-
-	*whout = false;
-	*opq = false;
-
-	while (s < path + len) {
-		char *slash = memchr(s, '/', path + len - s);
-		if (slash) {
-			if (s == slash) {
-				while (*++s == '/');	/* skip '//...' */
-				continue;
-			}
-			*slash = '\0';
-		}
-
-		if (!memcmp(s, ".", 2)) {
-			/* null */
-		} else if (!memcmp(s, "..", 3)) {
-			pwd = pwd->i_parent;
-		} else {
-			struct erofs_inode *inode = NULL;
-
-			if (aufs && !slash) {
-				if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
-					*opq = true;
-					break;
-				}
-				if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
-					s += sizeof(AUFS_WH_PFX) - 1;
-					*whout = true;
-				}
-			}
-
-			list_for_each_entry(d, &pwd->i_subdirs, d_child) {
-				if (!strcmp(d->name, s)) {
-					if (d->type != EROFS_FT_DIR && slash)
-						return ERR_PTR(-EIO);
-					inode = d->inode;
-					break;
-				}
-			}
-
-			if (inode) {
-				pwd = inode;
-			} else if (!slash) {
-				d = erofs_d_alloc(pwd, s);
-				if (IS_ERR(d))
-					return d;
-				d->type = EROFS_FT_UNKNOWN;
-				d->inode = pwd;
-			} else {
-				d = tarerofs_mkdir(pwd, s);
-				if (IS_ERR(d))
-					return d;
-				pwd = d->inode;
-			}
-		}
-		if (slash) {
-			*slash = '/';
-			s = slash + 1;
-		} else {
-			break;
-		}
-	}
-	return d;
-}
-
 struct tarerofs_xattr_item {
 	struct list_head list;
 	char *kv;
@@ -765,7 +662,7 @@ restart:
 
 	erofs_dbg("parsing %s (mode %05o)", eh.path, st.st_mode);
 
-	d = tarerofs_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
+	d = erofs_rebuild_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
 	if (IS_ERR(d)) {
 		ret = PTR_ERR(d);
 		goto out;
@@ -798,7 +695,7 @@ restart:
 		}
 		d->inode = NULL;
 
-		d2 = tarerofs_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
+		d2 = erofs_rebuild_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
 		if (IS_ERR(d2)) {
 			ret = PTR_ERR(d2);
 			goto out;
-- 
2.19.1.6.gb485710b

[PATCH v8 6/8] erofs-utils: lib: add erofs_rebuild_load_tree() helper

[Jingbo Xu]

Add erofs_rebuild_load_tree() helper to load the inode tree from a
given erofs image, and make it merged acting as an overlayfs-like
model.

Since the content of the symlink file needs to be read when loading
tree, let's add a dependency on zlib_LIBS for mkfs.erofs.

Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree() since
it is also called in the rebuild mode.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 include/erofs/inode.h    |   2 +-
 include/erofs/internal.h |   6 +-
 include/erofs/rebuild.h  |   2 +
 lib/inode.c              |   4 +-
 lib/rebuild.c            | 288 +++++++++++++++++++++++++++++++++++++++
 mkfs/main.c              |   2 +-
 6 files changed, 298 insertions(+), 6 deletions(-)

diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index 1c602a8..fe9dda2 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
 				   const char *name);
-int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_rebuild_dump_tree(struct erofs_inode *dir);
 int erofs_init_empty_dir(struct erofs_inode *dir);
 struct erofs_inode *erofs_new_inode(void);
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 3b866c9..19b912b 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -109,6 +109,7 @@ struct erofs_sb_info {
 
 	int devfd;
 	u64 devsz;
+	dev_t dev;
 	unsigned int nblobs;
 	unsigned int blobfd[256];
 
@@ -155,8 +156,6 @@ struct erofs_inode {
 	union {
 		/* (erofsfuse) runtime flags */
 		unsigned int flags;
-		/* (mkfs.erofs) device ID containing source file */
-		u32 dev;
 		/* (mkfs.erofs) queued sub-directories blocking dump */
 		u32 subdirs_queued;
 	};
@@ -164,6 +163,9 @@ struct erofs_inode {
 	struct erofs_sb_info *sbi;
 	struct erofs_inode *i_parent;
 
+	/* (mkfs.erofs) device ID containing source file */
+	u32 dev;
+
 	umode_t i_mode;
 	erofs_off_t i_size;
 
diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
index 92873c9..3ac074c 100644
--- a/include/erofs/rebuild.h
+++ b/include/erofs/rebuild.h
@@ -12,6 +12,8 @@ extern "C"
 struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
 		char *path, bool aufs, bool *whout, bool *opq);
 
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/inode.c b/lib/inode.c
index 79e1795..93e6b23 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1324,7 +1324,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
 	return inode;
 }
 
-int tarerofs_dump_tree(struct erofs_inode *dir)
+int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 {
 	struct erofs_dentry *d;
 	unsigned int nr_subdirs;
@@ -1395,7 +1395,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
 			continue;
 
 		inode = erofs_igrab(d->inode);
-		ret = tarerofs_dump_tree(inode);
+		ret = erofs_rebuild_dump_tree(inode);
 		dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
 		erofs_iput(inode);
 		if (ret)
diff --git a/lib/rebuild.c b/lib/rebuild.c
index 2398ca6..27a1df4 100644
--- a/lib/rebuild.c
+++ b/lib/rebuild.c
@@ -3,9 +3,18 @@
 #include <unistd.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
+#include <config.h>
+#if defined(HAVE_SYS_SYSMACROS_H)
+#include <sys/sysmacros.h>
+#endif
 #include "erofs/print.h"
 #include "erofs/inode.h"
 #include "erofs/rebuild.h"
+#include "erofs/io.h"
+#include "erofs/dir.h"
+#include "erofs/xattr.h"
+#include "erofs/blobchunk.h"
 #include "erofs/internal.h"
 
 #ifdef HAVE_LINUX_AUFS_TYPE_H
@@ -114,3 +123,282 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
 	}
 	return d;
 }
+
+static int erofs_rebuild_fixup_inode_index(struct erofs_inode *inode)
+{
+	int ret;
+	unsigned int count, unit, chunkbits, i;
+	struct erofs_inode_chunk_index *idx;
+	erofs_off_t chunksize;
+	erofs_blk_t blkaddr;
+
+	/* TODO: fill data map in other layouts */
+	if (inode->datalayout != EROFS_INODE_CHUNK_BASED &&
+	    inode->datalayout != EROFS_INODE_FLAT_PLAIN) {
+		erofs_err("%s: unsupported datalayout %d", inode->i_srcpath, inode->datalayout);
+		return -EOPNOTSUPP;
+	}
+
+	if (inode->sbi->extra_devices) {
+		chunkbits = inode->u.chunkbits;
+		if (chunkbits < sbi.blkszbits) {
+			erofs_err("%s: chunk size %u is too small to fit the target block size %u",
+				  inode->i_srcpath, 1U << chunkbits, 1U << sbi.blkszbits);
+			return -EINVAL;
+		}
+	} else {
+		chunkbits = ilog2(inode->i_size - 1) + 1;
+		if (chunkbits < sbi.blkszbits)
+			chunkbits = sbi.blkszbits;
+		if (chunkbits - sbi.blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
+			chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi.blkszbits;
+	}
+	chunksize = 1ULL << chunkbits;
+	count = DIV_ROUND_UP(inode->i_size, chunksize);
+
+	unit = sizeof(struct erofs_inode_chunk_index);
+	inode->extent_isize = count * unit;
+	idx = malloc(max(sizeof(*idx), sizeof(void *)));
+	if (!idx)
+		return -ENOMEM;
+	inode->chunkindexes = idx;
+
+	for (i = 0; i < count; i++) {
+		struct erofs_blobchunk *chunk;
+		struct erofs_map_blocks map = {
+			.index = UINT_MAX,
+		};
+
+		map.m_la = i << chunkbits;
+		ret = erofs_map_blocks(inode, &map, 0);
+		if (ret)
+			goto err;
+
+		blkaddr = erofs_blknr(&sbi, map.m_pa);
+		chunk = erofs_get_unhashed_chunk(inode->dev, blkaddr, 0);
+		if (IS_ERR(chunk)) {
+			ret = PTR_ERR(chunk);
+			goto err;
+		}
+		*(void **)idx++ = chunk;
+
+	}
+	inode->datalayout = EROFS_INODE_CHUNK_BASED;
+	inode->u.chunkformat = EROFS_CHUNK_FORMAT_INDEXES;
+	inode->u.chunkformat |= chunkbits - sbi.blkszbits;
+	return 0;
+err:
+	free(idx);
+	inode->chunkindexes = NULL;
+	return ret;
+}
+
+static int erofs_rebuild_fill_inode(struct erofs_inode *inode)
+{
+	switch (inode->i_mode & S_IFMT) {
+	case S_IFCHR:
+		if (erofs_inode_is_whiteout(inode))
+			inode->i_parent->whiteouts = true;
+		/* fallthrough */
+	case S_IFBLK:
+	case S_IFIFO:
+	case S_IFSOCK:
+		inode->i_size = 0;
+		erofs_dbg("\tdev: %d %d", major(inode->u.i_rdev),
+			  minor(inode->u.i_rdev));
+		inode->u.i_rdev = erofs_new_encode_dev(inode->u.i_rdev);
+		return 0;
+	case S_IFDIR:
+		return erofs_init_empty_dir(inode);
+	case S_IFLNK: {
+		int ret;
+
+		inode->i_link = malloc(inode->i_size + 1);
+		if (!inode->i_link)
+			return -ENOMEM;
+		ret = erofs_pread(inode, inode->i_link, inode->i_size, 0);
+		erofs_dbg("\tsymlink: %s -> %s", inode->i_srcpath, inode->i_link);
+		return ret;
+	}
+	case S_IFREG:
+		if (inode->i_size)
+			return erofs_rebuild_fixup_inode_index(inode);
+		return 0;
+	default:
+		break;
+	}
+	return -EINVAL;
+}
+
+/*
+ * @parent:  parent directory in inode tree
+ * @ctx.dir: parent directory when itering erofs_iterate_dir()
+ */
+struct erofs_rebuild_dir_context {
+	struct erofs_dir_context ctx;
+	struct erofs_inode *parent;
+};
+
+static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx)
+{
+	struct erofs_rebuild_dir_context *rctx = (void *)ctx;
+	struct erofs_inode *parent = rctx->parent;
+	struct erofs_inode *dir = ctx->dir;
+	struct erofs_inode *inode, *candidate;
+	struct erofs_inode src;
+	struct erofs_dentry *d;
+	char *path, *dname;
+	bool dumb;
+	int ret;
+
+	if (ctx->dot_dotdot)
+		return 0;
+
+	ret = asprintf(&path, "%s/%.*s", rctx->parent->i_srcpath,
+		       ctx->de_namelen, ctx->dname);
+	if (ret < 0)
+		return ret;
+
+	erofs_dbg("parsing %s", path);
+	dname = path + strlen(parent->i_srcpath) + 1;
+
+	d = erofs_rebuild_get_dentry(parent, dname, false, &dumb, &dumb);
+	if (IS_ERR(d)) {
+		ret = PTR_ERR(d);
+		goto out;
+	}
+
+	ret = 0;
+	if (d->type != EROFS_FT_UNKNOWN) {
+		/*
+		 * bail out if the file exists in the upper layers.  (Note that
+		 * extended attributes won't be merged too even for dirs.)
+		 */
+		if (!S_ISDIR(d->inode->i_mode) || d->inode->opaque)
+			goto out;
+
+		/* merge directory entries */
+		src = (struct erofs_inode) {
+			.sbi = dir->sbi,
+			.nid = ctx->de_nid
+		};
+		ret = erofs_read_inode_from_disk(&src);
+		if (ret || !S_ISDIR(src.i_mode))
+			goto out;
+		parent = d->inode;
+		inode = dir = &src;
+	} else {
+		u64 nid;
+
+		DBG_BUGON(parent != d->inode);
+		inode = erofs_new_inode();
+		if (IS_ERR(inode)) {
+			ret = PTR_ERR(inode);
+			goto out;
+		}
+
+		/* reuse i_ino[0] to read nid in source fs */
+		nid = inode->i_ino[0];
+		inode->sbi = dir->sbi;
+		inode->nid = ctx->de_nid;
+		ret = erofs_read_inode_from_disk(inode);
+		if (ret)
+			goto out;
+
+		/* restore nid in new generated fs */
+		inode->i_ino[1] = inode->i_ino[0];
+		inode->i_ino[0] = nid;
+		inode->dev = inode->sbi->dev;
+
+		if (S_ISREG(inode->i_mode) && inode->i_nlink > 1 &&
+		    (candidate = erofs_iget(inode->dev, ctx->de_nid))) {
+			/* hardlink file */
+			erofs_iput(inode);
+			inode = candidate;
+			if (S_ISDIR(inode->i_mode)) {
+				erofs_err("hardlink directory not supported");
+				ret = -EISDIR;
+				goto out;
+			}
+			inode->i_nlink++;
+			erofs_dbg("\thardlink: %s -> %s", path, inode->i_srcpath);
+		} else {
+			ret = erofs_read_xattrs_from_disk(inode);
+			if (ret) {
+				erofs_iput(inode);
+				goto out;
+			}
+
+			inode->i_parent = d->inode;
+			inode->i_srcpath = path;
+			path = NULL;
+			inode->i_ino[1] = inode->nid;
+			inode->i_nlink = 1;
+
+			ret = erofs_rebuild_fill_inode(inode);
+			if (ret) {
+				erofs_iput(inode);
+				goto out;
+			}
+
+			erofs_insert_ihash(inode, inode->dev, inode->i_ino[1]);
+			parent = dir = inode;
+		}
+
+		d->inode = inode;
+		d->type = erofs_mode_to_ftype(inode->i_mode);
+	}
+
+	if (S_ISDIR(inode->i_mode)) {
+		struct erofs_rebuild_dir_context nctx = *rctx;
+
+		nctx.parent = parent;
+		nctx.ctx.dir = dir;
+		ret = erofs_iterate_dir(&nctx.ctx, false);
+		if (ret)
+			goto out;
+	}
+
+	/* reset sbi, nid after subdirs are all loaded for the final dump */
+	inode->sbi = &sbi;
+	inode->nid = 0;
+out:
+	free(path);
+	return ret;
+}
+
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi)
+{
+	struct erofs_inode inode = {};
+	struct erofs_rebuild_dir_context ctx;
+	int ret;
+
+	if (!sbi->devname) {
+		erofs_err("failed to find a device for rebuilding");
+		return -EINVAL;
+	}
+
+	ret = erofs_read_superblock(sbi);
+	if (ret) {
+		erofs_err("failed to read superblock of %s", sbi->devname);
+		return ret;
+	}
+
+	inode.nid = sbi->root_nid;
+	inode.sbi = sbi;
+	ret = erofs_read_inode_from_disk(&inode);
+	if (ret) {
+		erofs_err("failed to read root inode of %s", sbi->devname);
+		return ret;
+	}
+	inode.i_srcpath = strdup("/");
+
+	ctx = (struct erofs_rebuild_dir_context) {
+		.ctx.dir = &inode,
+		.ctx.cb = erofs_rebuild_dirent_iter,
+		.parent = root,
+	};
+	ret = erofs_iterate_dir(&ctx.ctx, false);
+	free(inode.i_srcpath);
+	return ret;
+}
diff --git a/mkfs/main.c b/mkfs/main.c
index c7f4047..49b91c5 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -965,7 +965,7 @@ int main(int argc, char **argv)
 		if (err < 0)
 			goto exit;
 
-		err = tarerofs_dump_tree(root_inode);
+		err = erofs_rebuild_dump_tree(root_inode);
 		if (err < 0)
 			goto exit;
 	}
-- 
2.19.1.6.gb485710b

[PATCH v8 7/8] erofs-utils: mkfs: introduce rebuild mode

[Jingbo Xu]

Introduce a new EXPERIMENTAL rebuild mode, which can be used to
generate a meta-only multidev manifest image with an overlayfs-like
merged tree from multiple specific EROFS images either of

tarerofs index mode (--tar=i):

  mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
  ...
  mkfs.erofs --tar=i --aufs layerN.erofs layerN-1.tar

or mkfs.erofs uncompressed mode without inline data:

  mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
  ...
  mkfs.erofs --tar=f -Enoinline_data --aufs layerN-1.erofs layerN-1.tar

To merge these layers, just type:
  mkfs.erofs merged.erofs layer0.erofs ... layerN-1.erofs

It doesn't support compression and/or flat inline datalayout yet.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 mkfs/main.c | 219 ++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 168 insertions(+), 51 deletions(-)

diff --git a/mkfs/main.c b/mkfs/main.c
index 49b91c5..ce2b0e2 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -26,6 +26,7 @@
 #include "erofs/compress_hints.h"
 #include "erofs/blobchunk.h"
 #include "erofs/fragments.h"
+#include "erofs/rebuild.h"
 #include "../lib/liberofs_private.h"
 #include "../lib/liberofs_uuid.h"
 
@@ -83,8 +84,8 @@ static void print_available_compressors(FILE *f, const char *delim)
 
 static void usage(void)
 {
-	fputs("usage: [options] FILE DIRECTORY\n\n"
-	      "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
+	fputs("usage: [options] FILE SOURCE(s)\n"
+	      "Generate EROFS image (FILE) from DIRECTORY, TARBALL and/or EROFS images.  And [options] are:\n"
 	      " -b#                   set block size to # (# = page size by default)\n"
 	      " -d#                   set output message level to # (maximum 9)\n"
 	      " -x#                   set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
@@ -135,7 +136,10 @@ static unsigned int pclustersize_packed, pclustersize_max;
 static struct erofs_tarfile erofstar = {
 	.global.xattrs = LIST_HEAD_INIT(erofstar.global.xattrs)
 };
-static bool tar_mode;
+static bool tar_mode, rebuild_mode;
+
+static unsigned int rebuild_src_count;
+static LIST_HEAD(rebuild_src_list);
 
 static int parse_extended_opts(const char *opts)
 {
@@ -277,10 +281,23 @@ static int mkfs_parse_compress_algs(char *algs)
 	return 0;
 }
 
+static void erofs_rebuild_cleanup(void)
+{
+	struct erofs_sb_info *src, *n;
+
+	list_for_each_entry_safe(src, n, &rebuild_src_list, list) {
+		list_del(&src->list);
+		erofs_put_super(src);
+		dev_close(src);
+		free(src);
+	}
+	rebuild_src_count = 0;
+}
+
 static int mkfs_parse_options_cfg(int argc, char *argv[])
 {
 	char *endptr;
-	int opt, i;
+	int opt, i, err;
 	bool quiet = false;
 
 	while ((opt = getopt_long(argc, argv, "C:E:L:T:U:b:d:x:z:",
@@ -531,12 +548,14 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 
 	if (optind >= argc) {
 		if (!tar_mode) {
-			erofs_err("missing argument: DIRECTORY");
+			erofs_err("missing argument: SOURCE(s)");
 			return -EINVAL;
 		} else {
 			erofstar.fd = STDIN_FILENO;
 		}
-	}else {
+	} else {
+		struct stat st;
+
 		cfg.c_src_path = realpath(argv[optind++], NULL);
 		if (!cfg.c_src_path) {
 			erofs_err("failed to parse source directory: %s",
@@ -544,7 +563,47 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			return -ENOENT;
 		}
 
-		if (optind < argc) {
+		if (tar_mode) {
+			erofstar.fd = open(cfg.c_src_path, O_RDONLY);
+			if (erofstar.fd < 0) {
+				erofs_err("failed to open file: %s", cfg.c_src_path);
+				usage();
+				return -errno;
+			}
+		} else {
+			err = lstat(cfg.c_src_path, &st);
+			if (err)
+				return -errno;
+			if (S_ISDIR(st.st_mode))
+				erofs_set_fs_root(cfg.c_src_path);
+			else
+				rebuild_mode = true;
+		}
+
+		if (rebuild_mode) {
+			char *srcpath = cfg.c_src_path;
+
+			do {
+				struct erofs_sb_info *src;
+
+				src = calloc(1, sizeof(struct erofs_sb_info));
+				if (!src) {
+					erofs_rebuild_cleanup();
+					return -ENOMEM;
+				}
+
+				err = dev_open_ro(src, srcpath);
+				if (err) {
+					free(src);
+					erofs_rebuild_cleanup();
+					return err;
+				}
+
+				/* extra device index starts from 1 */
+				src->dev = ++rebuild_src_count;
+				list_add(&src->list, &rebuild_src_list);
+			} while (optind < argc && (srcpath = argv[optind++]));
+		} else if (optind < argc) {
 			erofs_err("unexpected argument: %s\n", argv[optind]);
 			return -EINVAL;
 		}
@@ -735,6 +794,61 @@ void erofs_show_progs(int argc, char *argv[])
 	if (cfg.c_dbg_lvl >= EROFS_WARN)
 		printf("%s %s\n", basename(argv[0]), cfg.c_version);
 }
+static struct erofs_inode *erofs_alloc_root_inode(void)
+{
+	struct erofs_inode *root;
+
+	root = erofs_new_inode();
+	if (IS_ERR(root))
+		return root;
+	root->i_srcpath = strdup("/");
+	root->i_mode = S_IFDIR | 0777;
+	root->i_parent = root;
+	root->i_mtime = root->sbi->build_time;
+	root->i_mtime_nsec = root->sbi->build_time_nsec;
+	erofs_init_empty_dir(root);
+	return root;
+}
+
+static int erofs_rebuild_load_trees(struct erofs_inode *root)
+{
+	struct erofs_sb_info *src;
+	unsigned int extra_devices = 0;
+	erofs_blk_t nblocks;
+	int ret, i = 0;
+
+	list_for_each_entry(src, &rebuild_src_list, list) {
+		ret = erofs_rebuild_load_tree(root, src);
+		if (ret) {
+			erofs_err("failed to load %s", src->devname);
+			return ret;
+		}
+		if (src->extra_devices > 1) {
+			erofs_err("%s: unsupported number of extra devices",
+				  src->devname, src->extra_devices);
+			return -EOPNOTSUPP;
+		}
+		extra_devices += src->extra_devices;
+	}
+
+	if (extra_devices && extra_devices != rebuild_src_count) {
+		erofs_err("unsupported mix of source images");
+		return -EOPNOTSUPP;
+	}
+
+	ret = erofs_mkfs_init_devices(&sbi, rebuild_src_count);
+	if (ret)
+		return ret;
+
+	list_for_each_entry(src, &rebuild_src_list, list) {
+		if (extra_devices)
+			nblocks = src->devs[0].blocks;
+		else
+			nblocks = src->primarydevice_blocks;
+		sbi.devs[i++].blocks = nblocks;
+	}
+	return 0;
+}
 
 static void erofs_mkfs_showsummaries(erofs_blk_t nblocks)
 {
@@ -761,7 +875,6 @@ int main(int argc, char **argv)
 	struct erofs_buffer_head *sb_bh;
 	struct erofs_inode *root_inode, *packed_inode;
 	erofs_nid_t root_nid, packed_nid;
-	struct stat st;
 	erofs_blk_t nblocks;
 	struct timeval t;
 	FILE *packedfile = NULL;
@@ -783,26 +896,6 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
-	if (!tar_mode) {
-		err = lstat(cfg.c_src_path, &st);
-		if (err)
-			return 1;
-		if (!S_ISDIR(st.st_mode)) {
-			erofs_err("root of the filesystem is not a directory - %s",
-				  cfg.c_src_path);
-			usage();
-			return 1;
-		}
-		erofs_set_fs_root(cfg.c_src_path);
-	} else if (cfg.c_src_path) {
-		erofstar.fd = open(cfg.c_src_path, O_RDONLY);
-		if (erofstar.fd < 0) {
-			erofs_err("failed to open file: %s", cfg.c_src_path);
-			usage();
-			return 1;
-		}
-	}
-
 	if (cfg.c_unix_timestamp != -1) {
 		sbi.build_time      = cfg.c_unix_timestamp;
 		sbi.build_time_nsec = 0;
@@ -866,6 +959,22 @@ int main(int argc, char **argv)
 		}
 	}
 
+	if (rebuild_mode) {
+		struct erofs_sb_info *src;
+
+		erofs_warn("EXPERIMENTAL rebuild mode in use. Use at your own risk!");
+
+		src = list_first_entry(&rebuild_src_list, struct erofs_sb_info, list);
+		if (!src)
+			goto exit;
+		err = erofs_read_superblock(src);
+		if (err) {
+			erofs_err("failed to read superblock of %s", src->devname);
+			goto exit;
+		}
+		sbi.blkszbits = src->blkszbits;
+	}
+
 	sb_bh = erofs_buffer_init();
 	if (IS_ERR(sb_bh)) {
 		err = PTR_ERR(sb_bh);
@@ -931,7 +1040,35 @@ int main(int argc, char **argv)
 
 	erofs_inode_manager_init();
 
-	if (!tar_mode) {
+	if (tar_mode) {
+		root_inode = erofs_alloc_root_inode();
+		if (IS_ERR(root_inode)) {
+			err = PTR_ERR(root_inode);
+			goto exit;
+		}
+
+		while (!(err = tarerofs_parse_tar(root_inode, &erofstar)));
+
+		if (err < 0)
+			goto exit;
+
+		err = erofs_rebuild_dump_tree(root_inode);
+		if (err < 0)
+			goto exit;
+	} else if (rebuild_mode) {
+		root_inode = erofs_alloc_root_inode();
+		if (IS_ERR(root_inode)) {
+			err = PTR_ERR(root_inode);
+			goto exit;
+		}
+
+		err = erofs_rebuild_load_trees(root_inode);
+		if (err)
+			goto exit;
+		err = erofs_rebuild_dump_tree(root_inode);
+		if (err)
+			goto exit;
+	} else {
 		err = erofs_build_shared_xattrs_from_path(&sbi, cfg.c_src_path);
 		if (err) {
 			erofs_err("failed to build shared xattrs: %s",
@@ -947,32 +1084,11 @@ int main(int argc, char **argv)
 			err = PTR_ERR(root_inode);
 			goto exit;
 		}
-	} else {
-		root_inode = erofs_new_inode();
-		if (IS_ERR(root_inode)) {
-			err = PTR_ERR(root_inode);
-			goto exit;
-		}
-		root_inode->i_srcpath = strdup("/");
-		root_inode->i_mode = S_IFDIR | 0777;
-		root_inode->i_parent = root_inode;
-		root_inode->i_mtime = sbi.build_time;
-		root_inode->i_mtime_nsec = sbi.build_time_nsec;
-		erofs_init_empty_dir(root_inode);
-
-		while (!(err = tarerofs_parse_tar(root_inode, &erofstar)));
-
-		if (err < 0)
-			goto exit;
-
-		err = erofs_rebuild_dump_tree(root_inode);
-		if (err < 0)
-			goto exit;
 	}
 	root_nid = erofs_lookupnid(root_inode);
 	erofs_iput(root_inode);
 
-	if (erofstar.index_mode || cfg.c_chunkbits) {
+	if (erofstar.index_mode || cfg.c_chunkbits || sbi.extra_devices) {
 		if (erofstar.index_mode && !erofstar.mapfile)
 			sbi.devs[0].blocks =
 				BLK_ROUND_UP(&sbi, erofstar.offset);
@@ -1026,6 +1142,7 @@ exit:
 		z_erofs_fragments_exit();
 	erofs_packedfile_exit();
 	erofs_xattr_cleanup_name_prefixes();
+	erofs_rebuild_cleanup();
 	erofs_exit_configure();
 
 	if (err) {
-- 
2.19.1.6.gb485710b

[PATCH v8 8/8] erofs-utils: mkfs: add `--ovlfs-strip` option

[Jingbo Xu]

Add `--ovlfs-strip=[0|1]` option for tarfs and rebuild mode for now
in order to control whether some overlayfs related stuffs (e.g.
whiteout files, OVL_XATTR_OPAQUE and OVL_XATTR_ORIGIN xattrs) are
finally stripped.

This option is disabled by default for mkfs, that is, the overlayfs
related stuffs described above are kept in the image by default.

Specify `--ovlfs-strip=1` explicitly to strip these stuffs.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 include/erofs/config.h |  1 +
 include/erofs/xattr.h  |  1 +
 lib/inode.c            | 17 ++++++++++++++---
 lib/xattr.c            | 18 ++++++++++++++++++
 mkfs/main.c            |  8 ++++++++
 5 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/include/erofs/config.h b/include/erofs/config.h
index 5d3bfba..e342722 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -54,6 +54,7 @@ struct erofs_configure {
 	bool c_showprogress;
 	bool c_extra_ea_name_prefixes;
 	bool c_xattr_name_filter;
+	bool c_ovlfs_strip;
 
 #ifdef HAVE_LIBSELINUX
 	struct selabel_handle *sehnd;
diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index 0364f24..0f76037 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -57,6 +57,7 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
 int erofs_setxattr(struct erofs_inode *inode, char *key,
 		   const void *value, size_t size);
 int erofs_set_opaque_xattr(struct erofs_inode *inode);
+void erofs_clear_opaque_xattr(struct erofs_inode *inode);
 int erofs_set_origin_xattr(struct erofs_inode *inode);
 int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
 
diff --git a/lib/inode.c b/lib/inode.c
index 93e6b23..37aa79e 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1326,7 +1326,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
 
 int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 {
-	struct erofs_dentry *d;
+	struct erofs_dentry *d, *n;
 	unsigned int nr_subdirs;
 	int ret;
 
@@ -1341,7 +1341,10 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 		dir->inode_isize = sizeof(struct erofs_inode_compact);
 	}
 
-	if (dir->whiteouts)
+	/* strip all unnecessary overlayfs xattrs when ovlfs_strip is enabled */
+	if (cfg.c_ovlfs_strip)
+		erofs_clear_opaque_xattr(dir);
+	else if (dir->whiteouts)
 		erofs_set_origin_xattr(dir);
 
 	ret = erofs_prepare_xattr_ibody(dir);
@@ -1373,8 +1376,16 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 	}
 
 	nr_subdirs = 0;
-	list_for_each_entry(d, &dir->i_subdirs, d_child)
+	list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) {
+		if (cfg.c_ovlfs_strip && erofs_inode_is_whiteout(d->inode)) {
+			erofs_dbg("remove whiteout %s", d->inode->i_srcpath);
+			list_del(&d->d_child);
+			erofs_d_invalidate(d);
+			free(d);
+			continue;
+		}
 		++nr_subdirs;
+	}
 
 	ret = erofs_prepare_dir_layout(dir, nr_subdirs);
 	if (ret)
diff --git a/lib/xattr.c b/lib/xattr.c
index e3a1b44..fffccb4 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -499,11 +499,29 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
 	return erofs_xattr_add(&inode->i_xattrs, item);
 }
 
+static void erofs_clearxattr(struct erofs_inode *inode, const char *key)
+{
+	struct inode_xattr_node *node, *n;
+
+	list_for_each_entry_safe(node, n, &inode->i_xattrs, list) {
+		if (!strcmp(node->item->kvbuf, key)) {
+			list_del(&node->list);
+			put_xattritem(node->item);
+			free(node);
+		}
+	}
+}
+
 int erofs_set_opaque_xattr(struct erofs_inode *inode)
 {
 	return erofs_setxattr(inode, OVL_XATTR_OPAQUE, "y", 1);
 }
 
+void erofs_clear_opaque_xattr(struct erofs_inode *inode)
+{
+	erofs_clearxattr(inode, OVL_XATTR_OPAQUE);
+}
+
 int erofs_set_origin_xattr(struct erofs_inode *inode)
 {
 	return erofs_setxattr(inode, OVL_XATTR_ORIGIN, NULL, 0);
diff --git a/mkfs/main.c b/mkfs/main.c
index ce2b0e2..389bb1b 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -64,6 +64,7 @@ static struct option long_options[] = {
 	{"fs-config-file", required_argument, NULL, 514},
 	{"block-list-file", required_argument, NULL, 515},
 #endif
+	{"ovlfs-strip", optional_argument, NULL, 516},
 	{0, 0, 0, 0},
 };
 
@@ -115,6 +116,7 @@ static void usage(void)
 	      " --preserve-mtime      keep per-file modification time strictly\n"
 	      " --aufs                replace aufs special files with overlayfs metadata\n"
 	      " --tar=[fi]            generate an image from tarball(s)\n"
+	      " --ovlfs-strip=[01]    strip overlayfs metadata in the target image (e.g. whiteouts)\n"
 	      " --quiet               quiet execution (do not write anything to standard output.)\n"
 #ifndef NDEBUG
 	      " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
@@ -516,6 +518,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 21:
 			erofstar.aufs = true;
 			break;
+		case 516:
+			if (!optarg || !strcmp(optarg, "1"))
+				cfg.c_ovlfs_strip = true;
+			else
+				cfg.c_ovlfs_strip = false;
+			break;
 		case 1:
 			usage();
 			exit(0);
-- 
2.19.1.6.gb485710b

[PATCH v3] erofs-utils: lib: refactor extended attribute name prefixes

[Jingbo Xu]

Previously, the extended attribute name in xattr_item was actually the
part with the matched prefix stripped, which makes it clumsy to strip
or delete a specific attribute from one file.

To fix this, make the complete attribute name stored in xattr_item.
One thing worth noting is that the attribute name in xattr_item has a
trailing '\0' for ease of name comparison, while the trailing '\0' will
get stripped when writing to on-disk erofs_xattr_entry.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
v3:
- introduce EROFS_XATTR_KSIZE() and EROFS_XATTR_KVSIZE() macro
- make the full long name prefix (instead of the infix in v2) stored in
  ea_type_node; and thus compare with the full long name prefix in
  get_xattritem() directly
- convert the data type of in-memory prefix index and prefix_len from
  u8/u16 to unsigned int
---
 lib/xattr.c | 379 ++++++++++++++++++++++------------------------------
 1 file changed, 161 insertions(+), 218 deletions(-)

diff --git a/lib/xattr.c b/lib/xattr.c
index 54a6ae2..8119013 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -51,6 +51,12 @@
 #ifndef XATTR_NAME_POSIX_ACL_DEFAULT
 #define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
 #endif
+#ifndef XATTR_NAME_SECURITY_SELINUX
+#define XATTR_NAME_SECURITY_SELINUX "security.selinux"
+#endif
+#ifndef XATTR_NAME_SECURITY_CAPABILITY
+#define XATTR_NAME_SECURITY_CAPABILITY "security.capability"
+#endif
 #ifndef OVL_XATTR_NAMESPACE
 #define OVL_XATTR_NAMESPACE "overlay."
 #endif
@@ -72,12 +78,23 @@
 
 #define EA_HASHTABLE_BITS 16
 
+/* extra one byte for the trailing `\0` of attribute name */
+#define EROFS_XATTR_KSIZE(kvlen)	(kvlen[0] + 1)
+#define EROFS_XATTR_KVSIZE(kvlen)	(EROFS_XATTR_KSIZE(kvlen) + kvlen[1])
+
+/*
+ * @base_index:	the index of matched predefined short prefix
+ * @prefix:	the index of matched long prefix if any;
+ *		same as base_index otherwise
+ * @prefix_len:	the length of matched long prefix if any;
+ *		the length of matched predefined short prefix otherwise
+ */
 struct xattr_item {
 	struct xattr_item *next_shared_xattr;
 	const char *kvbuf;
 	unsigned int hash[2], len[2], count;
 	int shared_xattr_id;
-	u8 prefix;
+	unsigned int prefix, base_index, prefix_len;
 	struct hlist_node node;
 };
 
@@ -93,7 +110,7 @@ static unsigned int shared_xattrs_count;
 
 static struct xattr_prefix {
 	const char *prefix;
-	u8 prefix_len;
+	unsigned int prefix_len;
 } xattr_types[] = {
 	[EROFS_XATTR_INDEX_USER] = {
 		XATTR_USER_PREFIX,
@@ -116,11 +133,27 @@ static struct xattr_prefix {
 struct ea_type_node {
 	struct list_head list;
 	struct xattr_prefix type;
-	u8 index;
+	unsigned int index, base_index, base_len;
 };
+
 static LIST_HEAD(ea_name_prefixes);
 static unsigned int ea_prefix_count;
 
+static bool match_prefix(const char *key, unsigned int *index,
+			 unsigned int *len)
+{
+	struct xattr_prefix *p;
+
+	for (p = xattr_types; p < xattr_types + ARRAY_SIZE(xattr_types); ++p) {
+		if (p->prefix && !strncmp(p->prefix, key, p->prefix_len)) {
+			*len = p->prefix_len;
+			*index = p - xattr_types;
+			return true;
+		}
+	}
+	return false;
+}
+
 static unsigned int BKDRHash(char *str, unsigned int len)
 {
 	const unsigned int seed = 131313;
@@ -133,13 +166,12 @@ static unsigned int BKDRHash(char *str, unsigned int len)
 	return hash;
 }
 
-static unsigned int xattr_item_hash(u8 prefix, char *buf,
-				    unsigned int len[2], unsigned int hash[2])
+static unsigned int xattr_item_hash(char *buf, unsigned int len[2],
+				    unsigned int hash[2])
 {
 	hash[0] = BKDRHash(buf, len[0]);	/* key */
 	hash[1] = BKDRHash(buf + len[0], len[1]);	/* value */
-
-	return prefix ^ hash[0] ^ hash[1];
+	return hash[0] ^ hash[1];
 }
 
 static unsigned int put_xattritem(struct xattr_item *item)
@@ -150,17 +182,14 @@ static unsigned int put_xattritem(struct xattr_item *item)
 	return 0;
 }
 
-static struct xattr_item *get_xattritem(u8 prefix, char *kvbuf,
-					unsigned int len[2])
+static struct xattr_item *get_xattritem(char *kvbuf, unsigned int len[2])
 {
 	struct xattr_item *item;
 	unsigned int hash[2], hkey;
 
-	hkey = xattr_item_hash(prefix, kvbuf, len, hash);
-
+	hkey = xattr_item_hash(kvbuf, len, hash);
 	hash_for_each_possible(ea_hashtable, item, node, hkey) {
-		if (prefix == item->prefix &&
-		    item->len[0] == len[0] && item->len[1] == len[1] &&
+		if (item->len[0] == len[0] && item->len[1] == len[1] &&
 		    item->hash[0] == hash[0] && item->hash[1] == hash[1] &&
 		    !memcmp(kvbuf, item->kvbuf, len[0] + len[1])) {
 			free(kvbuf);
@@ -174,6 +203,14 @@ static struct xattr_item *get_xattritem(u8 prefix, char *kvbuf,
 		free(kvbuf);
 		return ERR_PTR(-ENOMEM);
 	}
+
+	if (!match_prefix(kvbuf, &item->base_index, &item->prefix_len)) {
+		free(item);
+		free(kvbuf);
+		return ERR_PTR(-ENODATA);
+	}
+	DBG_BUGON(len[0] < item->prefix_len);
+
 	INIT_HLIST_NODE(&item->node);
 	item->count = 1;
 	item->kvbuf = kvbuf;
@@ -182,56 +219,37 @@ static struct xattr_item *get_xattritem(u8 prefix, char *kvbuf,
 	item->hash[0] = hash[0];
 	item->hash[1] = hash[1];
 	item->shared_xattr_id = -1;
-	item->prefix = prefix;
-	hash_add(ea_hashtable, &item->node, hkey);
-	return item;
-}
+	item->prefix = item->base_index;
 
-static bool match_base_prefix(const char *key, u8 *index, u16 *len)
-{
-	struct xattr_prefix *p;
+	if (cfg.c_extra_ea_name_prefixes) {
+		struct ea_type_node *tnode;
 
-	for (p = xattr_types; p < xattr_types + ARRAY_SIZE(xattr_types); ++p) {
-		if (p->prefix && !strncmp(p->prefix, key, p->prefix_len)) {
-			*len = p->prefix_len;
-			*index = p - xattr_types;
-			return true;
+		list_for_each_entry(tnode, &ea_name_prefixes, list) {
+			if (item->base_index == tnode->base_index &&
+			    !strncmp(tnode->type.prefix, kvbuf,
+				     tnode->type.prefix_len)) {
+				item->prefix = tnode->index;
+				item->prefix_len = tnode->type.prefix_len;
+				break;
+			}
 		}
 	}
-	return false;
-}
 
-static bool match_prefix(const char *key, u8 *index, u16 *len)
-{
-	struct xattr_prefix *p;
-	struct ea_type_node *tnode;
-
-	list_for_each_entry(tnode, &ea_name_prefixes, list) {
-		p = &tnode->type;
-		if (p->prefix && !strncmp(p->prefix, key, p->prefix_len)) {
-			*len = p->prefix_len;
-			*index = tnode->index;
-			return true;
-		}
-	}
-	return match_base_prefix(key, index, len);
+	hash_add(ea_hashtable, &item->node, hkey);
+	return item;
 }
 
 static struct xattr_item *parse_one_xattr(const char *path, const char *key,
 					  unsigned int keylen)
 {
 	ssize_t ret;
-	u8 prefix;
-	u16 prefixlen;
 	unsigned int len[2];
 	char *kvbuf;
 
 	erofs_dbg("parse xattr [%s] of %s", path, key);
 
-	if (!match_prefix(key, &prefix, &prefixlen))
-		return ERR_PTR(-ENODATA);
-
-	DBG_BUGON(keylen < prefixlen);
+	/* length of the key */
+	len[0] = keylen;
 
 	/* determine length of the value */
 #ifdef HAVE_LGETXATTR
@@ -246,19 +264,18 @@ static struct xattr_item *parse_one_xattr(const char *path, const char *key,
 	len[1] = ret;
 
 	/* allocate key-value buffer */
-	len[0] = keylen - prefixlen;
-
-	kvbuf = malloc(len[0] + len[1]);
+	kvbuf = malloc(EROFS_XATTR_KVSIZE(len));
 	if (!kvbuf)
 		return ERR_PTR(-ENOMEM);
-	memcpy(kvbuf, key + prefixlen, len[0]);
+	sprintf(kvbuf, "%s", key);
 	if (len[1]) {
 		/* copy value to buffer */
 #ifdef HAVE_LGETXATTR
-		ret = lgetxattr(path, key, kvbuf + len[0], len[1]);
+		ret = lgetxattr(path, key, kvbuf + EROFS_XATTR_KSIZE(len),
+				len[1]);
 #elif defined(__APPLE__)
-		ret = getxattr(path, key, kvbuf + len[0], len[1], 0,
-			       XATTR_NOFOLLOW);
+		ret = getxattr(path, key, kvbuf + EROFS_XATTR_KSIZE(len),
+			       len[1], 0, XATTR_NOFOLLOW);
 #else
 		free(kvbuf);
 		return ERR_PTR(-EOPNOTSUPP);
@@ -273,7 +290,7 @@ static struct xattr_item *parse_one_xattr(const char *path, const char *key,
 			len[1] = ret;
 		}
 	}
-	return get_xattritem(prefix, kvbuf, len);
+	return get_xattritem(kvbuf, len);
 }
 
 static struct xattr_item *erofs_get_selabel_xattr(const char *srcpath,
@@ -308,16 +325,17 @@ static struct xattr_item *erofs_get_selabel_xattr(const char *srcpath,
 			return NULL;
 		}
 
-		len[0] = sizeof("selinux") - 1;
+		len[0] = sizeof(XATTR_NAME_SECURITY_SELINUX) - 1;
 		len[1] = strlen(secontext);
-		kvbuf = malloc(len[0] + len[1] + 1);
+		kvbuf = malloc(EROFS_XATTR_KVSIZE(len));
 		if (!kvbuf) {
 			freecon(secontext);
 			return ERR_PTR(-ENOMEM);
 		}
-		sprintf(kvbuf, "selinux%s", secontext);
+		sprintf(kvbuf, "%s", XATTR_NAME_SECURITY_SELINUX);
+		memcpy(kvbuf + EROFS_XATTR_KSIZE(len), secontext, len[1]);
 		freecon(secontext);
-		return get_xattritem(EROFS_XATTR_INDEX_SECURITY, kvbuf, len);
+		return get_xattritem(kvbuf, len);
 	}
 #endif
 	return NULL;
@@ -466,24 +484,18 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
 	char *kvbuf;
 	unsigned int len[2];
 	struct xattr_item *item;
-	u8 prefix;
-	u16 prefixlen;
-
-	if (!match_prefix(key, &prefix, &prefixlen))
-		return -ENODATA;
 
+	len[0] = strlen(key);
 	len[1] = size;
-	/* allocate key-value buffer */
-	len[0] = strlen(key) - prefixlen;
 
-	kvbuf = malloc(len[0] + len[1]);
+	kvbuf = malloc(EROFS_XATTR_KVSIZE(len));
 	if (!kvbuf)
 		return -ENOMEM;
 
-	memcpy(kvbuf, key + prefixlen, len[0]);
-	memcpy(kvbuf + len[0], value, size);
+	sprintf(kvbuf, "%s", key);
+	memcpy(kvbuf + EROFS_XATTR_KSIZE(len), value, size);
 
-	item = get_xattritem(prefix, kvbuf, len);
+	item = get_xattritem(kvbuf, len);
 	if (IS_ERR(item))
 		return PTR_ERR(item);
 	DBG_BUGON(!item);
@@ -513,22 +525,22 @@ static int erofs_droid_xattr_set_caps(struct erofs_inode *inode)
 	if (!capabilities)
 		return 0;
 
-	len[0] = sizeof("capability") - 1;
+	len[0] = sizeof(XATTR_NAME_SECURITY_CAPABILITY) - 1;
 	len[1] = sizeof(caps);
 
-	kvbuf = malloc(len[0] + len[1]);
+	kvbuf = malloc(EROFS_XATTR_KVSIZE(len));
 	if (!kvbuf)
 		return -ENOMEM;
 
-	memcpy(kvbuf, "capability", len[0]);
+	sprintf(kvbuf, "%s", XATTR_NAME_SECURITY_CAPABILITY);
 	caps.magic_etc = VFS_CAP_REVISION_2 | VFS_CAP_FLAGS_EFFECTIVE;
 	caps.data[0].permitted = (u32) capabilities;
 	caps.data[0].inheritable = 0;
 	caps.data[1].permitted = (u32) (capabilities >> 32);
 	caps.data[1].inheritable = 0;
-	memcpy(kvbuf + len[0], &caps, len[1]);
+	memcpy(kvbuf + EROFS_XATTR_KSIZE(len), &caps, len[1]);
 
-	item = get_xattritem(EROFS_XATTR_INDEX_SECURITY, kvbuf, len);
+	item = get_xattritem(kvbuf, len);
 	if (IS_ERR(item))
 		return PTR_ERR(item);
 	DBG_BUGON(!item);
@@ -558,6 +570,13 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)
 	return erofs_droid_xattr_set_caps(inode);
 }
 
+static inline unsigned int erofs_next_xattr_align(unsigned int pos,
+						  struct xattr_item *item)
+{
+	return EROFS_XATTR_ALIGN(pos + sizeof(struct erofs_xattr_entry) +
+			item->len[0] + item->len[1] - item->prefix_len);
+}
+
 int erofs_prepare_xattr_ibody(struct erofs_inode *inode)
 {
 	int ret;
@@ -572,14 +591,13 @@ int erofs_prepare_xattr_ibody(struct erofs_inode *inode)
 	/* get xattr ibody size */
 	ret = sizeof(struct erofs_xattr_ibody_header);
 	list_for_each_entry(node, ixattrs, list) {
-		const struct xattr_item *item = node->item;
+		struct xattr_item *item = node->item;
 
 		if (item->shared_xattr_id >= 0) {
 			ret += sizeof(__le32);
 			continue;
 		}
-		ret += sizeof(struct erofs_xattr_entry);
-		ret = EROFS_XATTR_ALIGN(ret + item->len[0] + item->len[1]);
+		ret = erofs_next_xattr_align(ret, item);
 	}
 	inode->xattr_isize = ret;
 	return ret;
@@ -688,25 +706,9 @@ static int comp_shared_xattr_item(const void *a, const void *b)
 	return la > lb;
 }
 
-static inline int erofs_xattr_index_by_prefix(const char *prefix, int *len)
-{
-	if (!strncmp(prefix, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)){
-		*len = XATTR_USER_PREFIX_LEN;
-		return EROFS_XATTR_INDEX_USER;
-	} else if (!strncmp(prefix, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN)) {
-		*len = XATTR_TRUSTED_PREFIX_LEN;
-		return EROFS_XATTR_INDEX_TRUSTED;
-	} else if (!strncmp(prefix, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN)) {
-		*len = XATTR_SECURITY_PREFIX_LEN;
-		return EROFS_XATTR_INDEX_SECURITY;
-	}
-	return -ENODATA;
-}
-
 int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f)
 {
 	struct ea_type_node *tnode;
-	struct xattr_prefix *p;
 	off_t offset;
 
 	if (!ea_prefix_count)
@@ -732,16 +734,13 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f)
 			u8 data[EROFS_NAME_LEN + 2 +
 				sizeof(struct erofs_xattr_long_prefix)];
 		} u;
-		int ret, len;
+		int len, infix_len;
 
-		p = &tnode->type;
-		ret = erofs_xattr_index_by_prefix(p->prefix, &len);
-		if (ret < 0)
-			return ret;
-		u.s.prefix.base_index = ret;
-		memcpy(u.s.prefix.infix, p->prefix + len, p->prefix_len - len);
-		len = sizeof(struct erofs_xattr_long_prefix) +
-			p->prefix_len - len;
+		u.s.prefix.base_index = tnode->base_index;
+		infix_len = tnode->type.prefix_len - tnode->base_len;
+		memcpy(u.s.prefix.infix, tnode->type.prefix + tnode->base_len,
+		       infix_len);
+		len = sizeof(struct erofs_xattr_long_prefix) + infix_len;
 		u.s.size = cpu_to_le16(len);
 		if (fwrite(&u.s, sizeof(__le16) + len, 1, f) != 1)
 			return -EIO;
@@ -754,11 +753,30 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f)
 	return 0;
 }
 
+static void erofs_write_xattr_entry(char *buf, struct xattr_item *item)
+{
+	struct erofs_xattr_entry entry = {
+		.e_name_index = item->prefix,
+		.e_name_len = item->len[0] - item->prefix_len,
+		.e_value_size = cpu_to_le16(item->len[1]),
+	};
+
+	memcpy(buf, &entry, sizeof(entry));
+	buf += sizeof(struct erofs_xattr_entry);
+	memcpy(buf, item->kvbuf + item->prefix_len,
+	       item->len[0] - item->prefix_len);
+	buf += item->len[0] - item->prefix_len;
+	memcpy(buf, item->kvbuf + item->len[0] + 1, item->len[1]);
+
+	erofs_dbg("writing xattr %d %s (%d %s)", item->base_index, item->kvbuf,
+			item->prefix, item->kvbuf + item->prefix_len);
+}
+
 int erofs_build_shared_xattrs_from_path(struct erofs_sb_info *sbi, const char *path)
 {
 	int ret;
 	struct erofs_buffer_head *bh;
-	struct xattr_item *n, **sorted_n;
+	struct xattr_item *item, *n, **sorted_n;
 	char *buf;
 	unsigned int p, i;
 	erofs_off_t off;
@@ -787,13 +805,11 @@ int erofs_build_shared_xattrs_from_path(struct erofs_sb_info *sbi, const char *p
 
 	i = 0;
 	while (shared_xattrs_list) {
-		struct xattr_item *item = shared_xattrs_list;
-
+		item = shared_xattrs_list;
 		sorted_n[i++] = item;
 		shared_xattrs_list = item->next_shared_xattr;
-		shared_xattrs_size += sizeof(struct erofs_xattr_entry);
-		shared_xattrs_size = EROFS_XATTR_ALIGN(shared_xattrs_size +
-				item->len[0] + item->len[1]);
+		shared_xattrs_size = erofs_next_xattr_align(shared_xattrs_size,
+							    item);
 	}
 	DBG_BUGON(i != shared_xattrs_count);
 	sorted_n[i] = NULL;
@@ -820,20 +836,11 @@ int erofs_build_shared_xattrs_from_path(struct erofs_sb_info *sbi, const char *p
 	off %= erofs_blksiz(sbi);
 	p = 0;
 	for (i = 0; i < shared_xattrs_count; i++) {
-		struct xattr_item *item = sorted_n[i];
-		const struct erofs_xattr_entry entry = {
-			.e_name_index = item->prefix,
-			.e_name_len = item->len[0],
-			.e_value_size = cpu_to_le16(item->len[1])
-		};
-
+		item = sorted_n[i];
+		erofs_write_xattr_entry(buf + p, item);
 		item->next_shared_xattr = sorted_n[i + 1];
 		item->shared_xattr_id = (off + p) / sizeof(__le32);
-
-		memcpy(buf + p, &entry, sizeof(entry));
-		p += sizeof(struct erofs_xattr_entry);
-		memcpy(buf + p, item->kvbuf, item->len[0] + item->len[1]);
-		p = EROFS_XATTR_ALIGN(p + item->len[0] + item->len[1]);
+		p = erofs_next_xattr_align(p, item);
 	}
 	shared_xattrs_list = sorted_n[0];
 	free(sorted_n);
@@ -846,70 +853,12 @@ out:
 	return ret;
 }
 
-static int erofs_xattr_filter_hashbit(struct xattr_item *item)
-{
-	u8 prefix = item->prefix;
-	const char *key = item->kvbuf;
-	unsigned int len = item->len[0];
-	char *name = NULL;
-	uint32_t hashbit;
-
-	if (prefix & EROFS_XATTR_LONG_PREFIX) {
-		struct ea_type_node *tnode;
-		u16 prefix_len;
-		int ret;
-
-		list_for_each_entry(tnode, &ea_name_prefixes, list) {
-			if (tnode->index == item->prefix) {
-				ret = asprintf(&name, "%s%.*s",
-					       tnode->type.prefix, len, key);
-				if (ret < 0)
-					return -ENOMEM;
-				break;
-			}
-		}
-		if (!name)
-			return -ENOENT;
-
-		if (!match_base_prefix(name, &prefix, &prefix_len)) {
-			free(name);
-			return -ENOENT;
-		}
-		key = name + prefix_len;
-		len = strlen(key);
-	}
-
-	hashbit = xxh32(key, len, EROFS_XATTR_FILTER_SEED + prefix) &
-		  (EROFS_XATTR_FILTER_BITS - 1);
-	if (name)
-		free(name);
-	return hashbit;
-}
-
-static u32 erofs_xattr_filter_map(struct list_head *ixattrs)
-{
-	struct inode_xattr_node *node, *n;
-	u32 name_filter;
-	int hashbit;
-
-	name_filter = 0;
-	list_for_each_entry_safe(node, n, ixattrs, list) {
-		hashbit = erofs_xattr_filter_hashbit(node->item);
-		if (hashbit < 0) {
-			erofs_warn("failed to generate xattr name filter: %s",
-				   strerror(-hashbit));
-			return 0;
-		}
-		name_filter |= (1UL << hashbit);
-	}
-	return EROFS_XATTR_FILTER_DEFAULT & ~name_filter;
-}
-
 char *erofs_export_xattr_ibody(struct erofs_inode *inode)
 {
 	struct list_head *ixattrs = &inode->i_xattrs;
 	unsigned int size = inode->xattr_isize;
 	struct inode_xattr_node *node, *n;
+	struct xattr_item *item;
 	struct erofs_xattr_ibody_header *header;
 	LIST_HEAD(ilst);
 	unsigned int p;
@@ -922,16 +871,29 @@ char *erofs_export_xattr_ibody(struct erofs_inode *inode)
 	header->h_shared_count = 0;
 
 	if (cfg.c_xattr_name_filter) {
-		header->h_name_filter =
-			cpu_to_le32(erofs_xattr_filter_map(ixattrs));
+		u32 name_filter = 0;
+		int hashbit;
+		unsigned int base_len;
+
+		list_for_each_entry(node, ixattrs, list) {
+			item = node->item;
+			base_len = xattr_types[item->base_index].prefix_len;
+			hashbit = xxh32(item->kvbuf + base_len,
+					item->len[0] - base_len,
+					EROFS_XATTR_FILTER_SEED + item->base_index) &
+				  (EROFS_XATTR_FILTER_BITS - 1);
+			name_filter |= (1UL << hashbit);
+		}
+		name_filter = EROFS_XATTR_FILTER_DEFAULT & ~name_filter;
+
+		header->h_name_filter = cpu_to_le32(name_filter);
 		if (header->h_name_filter)
 			erofs_sb_set_xattr_filter(inode->sbi);
 	}
 
 	p = sizeof(struct erofs_xattr_ibody_header);
 	list_for_each_entry_safe(node, n, ixattrs, list) {
-		struct xattr_item *const item = node->item;
-
+		item = node->item;
 		list_del(&node->list);
 
 		/* move inline xattrs to the onstack list */
@@ -948,18 +910,9 @@ char *erofs_export_xattr_ibody(struct erofs_inode *inode)
 	}
 
 	list_for_each_entry_safe(node, n, &ilst, list) {
-		struct xattr_item *const item = node->item;
-		const struct erofs_xattr_entry entry = {
-			.e_name_index = item->prefix,
-			.e_name_len = item->len[0],
-			.e_value_size = cpu_to_le16(item->len[1])
-		};
-
-		memcpy(buf + p, &entry, sizeof(entry));
-		p += sizeof(struct erofs_xattr_entry);
-		memcpy(buf + p, item->kvbuf, item->len[0] + item->len[1]);
-		p = EROFS_XATTR_ALIGN(p + item->len[0] + item->len[1]);
-
+		item = node->item;
+		erofs_write_xattr_entry(buf + p, item);
+		p = erofs_next_xattr_align(p, item);
 		list_del(&node->list);
 		free(node);
 		put_xattritem(item);
@@ -1354,8 +1307,7 @@ int erofs_getxattr(struct erofs_inode *vi, const char *name, char *buffer,
 		   size_t buffer_size)
 {
 	int ret;
-	u8 prefix;
-	u16 prefixlen;
+	unsigned int prefix, prefixlen;
 	struct getxattr_iter it;
 
 	if (!name)
@@ -1365,7 +1317,7 @@ int erofs_getxattr(struct erofs_inode *vi, const char *name, char *buffer,
 	if (ret)
 		return ret;
 
-	if (!match_base_prefix(name, &prefix, &prefixlen))
+	if (!match_prefix(name, &prefix, &prefixlen))
 		return -ENODATA;
 
 	it.it.sbi = vi->sbi;
@@ -1532,34 +1484,25 @@ int erofs_listxattr(struct erofs_inode *vi, char *buffer, size_t buffer_size)
 int erofs_xattr_insert_name_prefix(const char *prefix)
 {
 	struct ea_type_node *tnode;
-	struct xattr_prefix *p;
-	bool matched = false;
-	char *s;
 
 	if (ea_prefix_count >= 0x80 || strlen(prefix) > UINT8_MAX)
 		return -EOVERFLOW;
 
-	for (p = xattr_types; p < xattr_types + ARRAY_SIZE(xattr_types); ++p) {
-		if (!strncmp(p->prefix, prefix, p->prefix_len)) {
-			matched = true;
-			break;
-		}
-	}
-	if (!matched)
-		return -ENODATA;
-
-	s = strdup(prefix);
-	if (!s)
+	tnode = calloc(1, sizeof(*tnode));
+	if (!tnode)
 		return -ENOMEM;
 
-	tnode = malloc(sizeof(*tnode));
-	if (!tnode) {
-		free(s);
-		return -ENOMEM;
+	if (!match_prefix(prefix, &tnode->base_index, &tnode->base_len)) {
+		free(tnode);
+		return -ENODATA;
 	}
 
-	tnode->type.prefix = s;
 	tnode->type.prefix_len = strlen(prefix);
+	tnode->type.prefix = strdup(prefix);
+	if (!tnode->type.prefix) {
+		free(tnode);
+		return -ENOMEM;
+	}
 
 	tnode->index = EROFS_XATTR_LONG_PREFIX | ea_prefix_count;
 	ea_prefix_count++;
-- 
2.19.1.6.gb485710b

Re: [PATCH v3] erofs-utils: lib: refactor extended attribute name prefixes

[Jingbo Xu]

Sorry I send this by mistake.  It has no difference with the previously
sent v3 here[*]

Sorry for the noise.

[*]
https://lore.kernel.org/all/20230913031804.84819-1-jefflexu@linux.alibaba.com/

-- 
Thanks,
Jingbo

Re: [PATCH v8 8/8] erofs-utils: mkfs: add `--ovlfs-strip` option

[Gao Xiang]

On Wed, Sep 13, 2023 at 08:03:03PM +0800, Jingbo Xu wrote:
> Add `--ovlfs-strip=[0|1]` option for tarfs and rebuild mode for now
> in order to control whether some overlayfs related stuffs (e.g.
> whiteout files, OVL_XATTR_OPAQUE and OVL_XATTR_ORIGIN xattrs) are
> finally stripped.
> 
> This option is disabled by default for mkfs, that is, the overlayfs
> related stuffs described above are kept in the image by default.
> 
> Specify `--ovlfs-strip=1` explicitly to strip these stuffs.
> 
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
> ---

...

> diff --git a/lib/xattr.c b/lib/xattr.c
> index e3a1b44..fffccb4 100644
> --- a/lib/xattr.c
> +++ b/lib/xattr.c
> @@ -499,11 +499,29 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
>  	return erofs_xattr_add(&inode->i_xattrs, item);
>  }
>  
> +static void erofs_clearxattr(struct erofs_inode *inode, const char *key)

I rename it as erofs_removexattr().

Thanks,
Gao Xiang

Re: [PATCH v8 7/8] erofs-utils: mkfs: introduce rebuild mode

[Gao Xiang]

On 2023/9/13 20:03, Jingbo Xu wrote:
> Introduce a new EXPERIMENTAL rebuild mode, which can be used to
> generate a meta-only multidev manifest image with an overlayfs-like
> merged tree from multiple specific EROFS images either of
> 
> tarerofs index mode (--tar=i):
> 
>    mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
>    ...
>    mkfs.erofs --tar=i --aufs layerN.erofs layerN-1.tar
> 
> or mkfs.erofs uncompressed mode without inline data:
> 
>    mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
>    ...
>    mkfs.erofs --tar=f -Enoinline_data --aufs layerN-1.erofs layerN-1.tar
> 
> To merge these layers, just type:
>    mkfs.erofs merged.erofs layer0.erofs ... layerN-1.erofs
> 
> It doesn't support compression and/or flat inline datalayout yet.
> 
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>

As Jingbo reported, applying the following diff to fix the device table:

diff --git a/mkfs/main.c b/mkfs/main.c
index 389bb1b..c5dda66 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -590,10 +590,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
  
  		if (rebuild_mode) {
  			char *srcpath = cfg.c_src_path;
+			struct erofs_sb_info *src;
  
  			do {
-				struct erofs_sb_info *src;
-
  				src = calloc(1, sizeof(struct erofs_sb_info));
  				if (!src) {
  					erofs_rebuild_cleanup();
@@ -823,7 +822,7 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)
  	struct erofs_sb_info *src;
  	unsigned int extra_devices = 0;
  	erofs_blk_t nblocks;
-	int ret, i = 0;
+	int ret;
  
  	list_for_each_entry(src, &rebuild_src_list, list) {
  		ret = erofs_rebuild_load_tree(root, src);
@@ -853,7 +852,7 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)
  			nblocks = src->devs[0].blocks;
  		else
  			nblocks = src->primarydevice_blocks;
-		sbi.devs[i++].blocks = nblocks;
+		sbi.devs[src->dev - 1].blocks = nblocks;
  	}
  	return 0;
  }
-- 
2.19.1.6.gb485710b