[PATCH v6 00/11] erofs-utils: mkfs: introduce rebuild mode

[PATCH v6 00/11] erofs-utils: mkfs: introduce rebuild mode

[Jingbo Xu]

v6:
- newly add cleanup patches (patch 1 and 9)
- patch 8: improve the code of parsing multiple sources with "do {}
  while" loop
- patch 10: newly added to work around the potential issue of exposing
  whiteouts in the overlayfs mount by setting origin xattr on parent
  directory of whiteouts
- patch 11: the former "--keep-whiteout" option is renamed to
  "--overlay-strip" option; apart from whiteouts, origin xattrs will
  also be stripped when the option is enabled

[1] https://lore.kernel.org/all/20230822092457.114686-2-jefflexu@linux.alibaba.com/
[2] https://lore.kernel.org/all/20230822092457.114686-3-jefflexu@linux.alibaba.com/

v1: https://lore.kernel.org/all/20230814034239.54660-1-jefflexu@linux.alibaba.com/
v2: https://lore.kernel.org/all/20230816021347.126886-1-jefflexu@linux.alibaba.com/
v3: https://lore.kernel.org/all/20230822092457.114686-1-jefflexu@linux.alibaba.com/
v4: https://lore.kernel.org/all/20230823071517.12303-1-jefflexu@linux.alibaba.com/
v5: https://lore.kernel.org/all/20230901094706.27539-1-jefflexu@linux.alibaba.com/

-------------------------

Introduce a new rebuild mode merging multiple erofs images generated
from either tarerfs index mode (--tar=i):

	mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

or tarerofs non-index mode (--tar=f):

	mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs


Jingbo Xu (11):
  erofs-utils: lib: remove unneeded NULL checking for returned item
  erofs-utils: lib: add list_splice_tail() helper
  erofs-utils: lib: make erofs_get_unhashed_chunk() global
  erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
  erofs-utils: lib: add erofs_insert_ihash() helper
  erofs-utils: lib: add erofs_rebuild_get_dentry() helper
  erofs-utils: lib: add erofs_rebuild_load_tree() helper
  erofs-utils: mkfs: introduce rebuild mode
  erofs-utils: lib: add erofs_inode_is_whiteout() helper
  erofs-utils: lib: set origin xattr on parent directory of whiteout
  erofs-utils: mkfs: add --overlay-strip option for tarfs and rebuild
    mode

 include/erofs/blobchunk.h |   2 +
 include/erofs/config.h    |   1 +
 include/erofs/inode.h     |   5 +-
 include/erofs/internal.h  |  11 +
 include/erofs/list.h      |  20 ++
 include/erofs/rebuild.h   |  21 ++
 include/erofs/xattr.h     |  36 +---
 lib/Makefile.am           |   3 +-
 lib/blobchunk.c           |   2 +-
 lib/inode.c               |  26 ++-
 lib/rebuild.c             | 409 ++++++++++++++++++++++++++++++++++++++
 lib/tar.c                 | 129 ++----------
 lib/xattr.c               | 144 +++++++++++++-
 mkfs/main.c               | 228 ++++++++++++++++-----
 14 files changed, 827 insertions(+), 210 deletions(-)
 create mode 100644 include/erofs/rebuild.h
 create mode 100644 lib/rebuild.c

-- 
2.19.1.6.gb485710b

[PATCH v6 01/11] erofs-utils: lib: remove unneeded NULL checking for returned item

[Jingbo Xu]

get_xattritem() will in no way return NULL.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 lib/xattr.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lib/xattr.c b/lib/xattr.c
index 0cab29f..5acc1ae 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -437,8 +437,6 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
 	item = get_xattritem(prefix, kvbuf, len);
 	if (IS_ERR(item))
 		return PTR_ERR(item);
-	if (!item)
-		return 0;
 
 	return erofs_xattr_add(&inode->i_xattrs, item);
 }
@@ -473,8 +471,6 @@ static int erofs_droid_xattr_set_caps(struct erofs_inode *inode)
 	item = get_xattritem(EROFS_XATTR_INDEX_SECURITY, kvbuf, len);
 	if (IS_ERR(item))
 		return PTR_ERR(item);
-	if (!item)
-		return 0;
 
 	return erofs_xattr_add(&inode->i_xattrs, item);
 }
-- 
2.19.1.6.gb485710b

[PATCH v6 02/11] erofs-utils: lib: add list_splice_tail() helper

[Jingbo Xu]

Add list_splice_tail() helper.

Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/list.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/include/erofs/list.h b/include/erofs/list.h
index 3f5da1a..d7a9fee 100644
--- a/include/erofs/list.h
+++ b/include/erofs/list.h
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)
 	return head->next == head;
 }
 
+static inline void __list_splice(struct list_head *list,
+		struct list_head *prev, struct list_head *next)
+{
+	struct list_head *first = list->next;
+	struct list_head *last = list->prev;
+
+	first->prev = prev;
+	prev->next = first;
+
+	last->next = next;
+	next->prev = last;
+}
+
+static inline void list_splice_tail(struct list_head *list,
+				    struct list_head *head)
+{
+	if (!list_empty(list))
+		__list_splice(list, head->prev, head);
+}
+
 #define list_entry(ptr, type, member) container_of(ptr, type, member)
 
 #define list_first_entry(ptr, type, member)                                    \
-- 
2.19.1.6.gb485710b

[PATCH v6 03/11] erofs-utils: lib: make erofs_get_unhashed_chunk() global

[Jingbo Xu]

... so that it could be called from outside blobchunk.c later.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/blobchunk.h | 2 ++
 lib/blobchunk.c           | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
index 010aee1..fb85d8e 100644
--- a/include/erofs/blobchunk.h
+++ b/include/erofs/blobchunk.h
@@ -14,6 +14,8 @@ extern "C"
 
 #include "erofs/internal.h"
 
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+		erofs_blk_t blkaddr, erofs_off_t sourceoffset);
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
 int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
 int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset);
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
index cada5bb..07f18bd 100644
--- a/lib/blobchunk.c
+++ b/lib/blobchunk.c
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = {
 };
 static LIST_HEAD(unhashed_blobchunks);
 
-static struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
 		erofs_blk_t blkaddr, erofs_off_t sourceoffset)
 {
 	struct erofs_blobchunk *chunk;
-- 
2.19.1.6.gb485710b

[PATCH v6 04/11] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper

[Jingbo Xu]

Add erofs_read_xattrs_from_disk() helper reading extended attributes
from disk, add checking if it's an opaque directory.

Move all xattr name related macros to xattr.c and introduce
erofs_set_opaque_xattr() helper to hide all these details.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/internal.h |   1 +
 include/erofs/xattr.h    |  33 +-----------
 lib/tar.c                |   7 +--
 lib/xattr.c              | 112 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 116 insertions(+), 37 deletions(-)

diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 382024a..6020553 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -191,6 +191,7 @@ struct erofs_inode {
 	bool compressed_idata;
 	bool lazy_tailblock;
 	bool with_tmpfile;
+	bool opaque;
 
 	unsigned int xattr_isize;
 	unsigned int extent_isize;
diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index cf02257..9fa1ad1 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -43,37 +43,6 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi,
 	(_size - sizeof(struct erofs_xattr_ibody_header)) / \
 	sizeof(struct erofs_xattr_entry) + 1; })
 
-#ifndef XATTR_SYSTEM_PREFIX
-#define XATTR_SYSTEM_PREFIX	"system."
-#endif
-#ifndef XATTR_SYSTEM_PREFIX_LEN
-#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1)
-#endif
-#ifndef XATTR_USER_PREFIX
-#define XATTR_USER_PREFIX	"user."
-#endif
-#ifndef XATTR_USER_PREFIX_LEN
-#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1)
-#endif
-#ifndef XATTR_SECURITY_PREFIX
-#define XATTR_SECURITY_PREFIX	"security."
-#endif
-#ifndef XATTR_SECURITY_PREFIX_LEN
-#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1)
-#endif
-#ifndef XATTR_TRUSTED_PREFIX
-#define XATTR_TRUSTED_PREFIX	"trusted."
-#endif
-#ifndef XATTR_TRUSTED_PREFIX_LEN
-#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1)
-#endif
-#ifndef XATTR_NAME_POSIX_ACL_ACCESS
-#define XATTR_NAME_POSIX_ACL_ACCESS "system.posix_acl_access"
-#endif
-#ifndef XATTR_NAME_POSIX_ACL_DEFAULT
-#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
-#endif
-
 int erofs_scan_file_xattrs(struct erofs_inode *inode);
 int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
 char *erofs_export_xattr_ibody(struct erofs_inode *inode);
@@ -87,6 +56,8 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
 
 int erofs_setxattr(struct erofs_inode *inode, char *key,
 		   const void *value, size_t size);
+int erofs_set_opaque_xattr(struct erofs_inode *inode);
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
 
 #ifdef __cplusplus
 }
diff --git a/lib/tar.c b/lib/tar.c
index 53196c1..5b63cf5 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -19,11 +19,6 @@
 #include "erofs/xattr.h"
 #include "erofs/blobchunk.h"
 
-#define OVL_XATTR_NAMESPACE "overlay."
-#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
-#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
-#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
-
 #define EROFS_WHITEOUT_DEV	0
 
 static char erofs_libbuf[16384];
@@ -788,7 +783,7 @@ restart:
 	} else if (opq) {
 		DBG_BUGON(d->type == EROFS_FT_UNKNOWN);
 		DBG_BUGON(!d->inode);
-		ret = erofs_setxattr(d->inode, OVL_XATTR_OPAQUE, "y", 1);
+		ret = erofs_set_opaque_xattr(d->inode);
 		goto out;
 	} else if (th.typeflag == '1') {	/* hard link cases */
 		struct erofs_dentry *d2;
diff --git a/lib/xattr.c b/lib/xattr.c
index 5acc1ae..cc6a393 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -21,6 +21,49 @@
 #include "erofs/xxhash.h"
 #include "liberofs_private.h"
 
+#ifndef XATTR_SYSTEM_PREFIX
+#define XATTR_SYSTEM_PREFIX	"system."
+#endif
+#ifndef XATTR_SYSTEM_PREFIX_LEN
+#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1)
+#endif
+#ifndef XATTR_USER_PREFIX
+#define XATTR_USER_PREFIX	"user."
+#endif
+#ifndef XATTR_USER_PREFIX_LEN
+#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1)
+#endif
+#ifndef XATTR_SECURITY_PREFIX
+#define XATTR_SECURITY_PREFIX	"security."
+#endif
+#ifndef XATTR_SECURITY_PREFIX_LEN
+#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1)
+#endif
+#ifndef XATTR_TRUSTED_PREFIX
+#define XATTR_TRUSTED_PREFIX	"trusted."
+#endif
+#ifndef XATTR_TRUSTED_PREFIX_LEN
+#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1)
+#endif
+#ifndef XATTR_NAME_POSIX_ACL_ACCESS
+#define XATTR_NAME_POSIX_ACL_ACCESS "system.posix_acl_access"
+#endif
+#ifndef XATTR_NAME_POSIX_ACL_DEFAULT
+#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
+#endif
+#ifndef OVL_XATTR_NAMESPACE
+#define OVL_XATTR_NAMESPACE "overlay."
+#endif
+#ifndef OVL_XATTR_OPAQUE_POSTFIX
+#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
+#endif
+#ifndef OVL_XATTR_TRUSTED_PREFIX
+#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
+#endif
+#ifndef OVL_XATTR_OPAQUE
+#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
+#endif
+
 #define EA_HASHTABLE_BITS 16
 
 struct xattr_item {
@@ -441,6 +484,11 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
 	return erofs_xattr_add(&inode->i_xattrs, item);
 }
 
+int erofs_set_opaque_xattr(struct erofs_inode *inode)
+{
+	return erofs_setxattr(inode, OVL_XATTR_OPAQUE, "y", 1);
+}
+
 #ifdef WITH_ANDROID
 static int erofs_droid_xattr_set_caps(struct erofs_inode *inode)
 {
@@ -497,6 +545,70 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)
 	return erofs_droid_xattr_set_caps(inode);
 }
 
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
+{
+	ssize_t kllen;
+	char *keylst, *key;
+	int ret;
+
+	init_list_head(&inode->i_xattrs);
+	kllen = erofs_listxattr(inode, NULL, 0);
+	if (kllen < 0)
+		return kllen;
+	if (kllen <= 1)
+		return 0;
+
+	keylst = malloc(kllen);
+	if (!keylst)
+		return -ENOMEM;
+
+	ret = erofs_listxattr(inode, keylst, kllen);
+	if (ret < 0)
+		goto out;
+
+	for (key = keylst; key < keylst + kllen; key += strlen(key) + 1) {
+		void *value = NULL;
+		size_t size = 0;
+
+		if (!strcmp(key, OVL_XATTR_OPAQUE)) {
+			if (!S_ISDIR(inode->i_mode)) {
+				erofs_dbg("file %s: opaque xattr on non-dir",
+					  inode->i_srcpath);
+				ret = -EINVAL;
+				goto out;
+			}
+			inode->opaque = true;
+		}
+
+		ret = erofs_getxattr(inode, key, NULL, 0);
+		if (ret < 0)
+			goto out;
+		if (ret) {
+			size = ret;
+			value = malloc(size);
+			if (!value) {
+				ret = -ENOMEM;
+				goto out;
+			}
+
+			ret = erofs_getxattr(inode, key, value, size);
+			if (ret < 0) {
+				free(value);
+				goto out;
+			}
+			DBG_BUGON(ret != size);
+		}
+
+		ret = erofs_setxattr(inode, key, value, size);
+		free(value);
+		if (ret)
+			break;
+	}
+out:
+	free(keylst);
+	return ret;
+}
+
 int erofs_prepare_xattr_ibody(struct erofs_inode *inode)
 {
 	int ret;
-- 
2.19.1.6.gb485710b

[PATCH v6 05/11] erofs-utils: lib: add erofs_insert_ihash() helper

[Jingbo Xu]

Add erofs_insert_ihash() helper inserting inode into inode hash table.

Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the
header file.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/inode.h |  3 +++
 lib/inode.c           | 10 +++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index e8a5670..1c602a8 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);
 unsigned char erofs_mode_to_ftype(umode_t mode);
 unsigned char erofs_ftype_to_dtype(unsigned int filetype);
 void erofs_inode_manager_init(void);
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget(dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget_by_nid(erofs_nid_t nid);
 unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
diff --git a/lib/inode.c b/lib/inode.c
index a3a643f..8fab964 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)
 		init_list_head(&inode_hashtable[i]);
 }
 
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino)
+{
+	list_add(&inode->i_hash,
+		 &inode_hashtable[(ino ^ dev) % NR_INODE_HASHTABLE]);
+}
+
 /* get the inode from the (source) inode # */
 struct erofs_inode *erofs_iget(dev_t dev, ino_t ino)
 {
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st,
 		inode->inode_isize = sizeof(struct erofs_inode_compact);
 	}
 
-	list_add(&inode->i_hash,
-		 &inode_hashtable[(st->st_ino ^ st->st_dev) %
-				  NR_INODE_HASHTABLE]);
+	erofs_insert_ihash(inode, st->st_dev, st->st_ino);
 	return 0;
 }
 
-- 
2.19.1.6.gb485710b

[PATCH v6 06/11] erofs-utils: lib: add erofs_rebuild_get_dentry() helper

[Jingbo Xu]

Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry().

Also make `whout` and 'opq' parameter optional when `aufs` is false.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/rebuild.h |  19 +++++++
 lib/Makefile.am         |   3 +-
 lib/rebuild.c           | 117 ++++++++++++++++++++++++++++++++++++++++
 lib/tar.c               | 109 ++-----------------------------------
 4 files changed, 141 insertions(+), 107 deletions(-)
 create mode 100644 include/erofs/rebuild.h
 create mode 100644 lib/rebuild.c

diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
new file mode 100644
index 0000000..92873c9
--- /dev/null
+++ b/include/erofs/rebuild.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_REBUILD_H
+#define __EROFS_REBUILD_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "internal.h"
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+		char *path, bool aufs, bool *whout, bool *opq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 3e09516..8a45bd6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/compress_hints.h \
       $(top_srcdir)/include/erofs/fragments.h \
       $(top_srcdir)/include/erofs/xxhash.h \
+      $(top_srcdir)/include/erofs/rebuild.h \
       $(top_srcdir)/lib/liberofs_private.h
 
 noinst_HEADERS += compressor.h
@@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
 		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
-		      block_list.c xxhash.c
+		      block_list.c xxhash.c rebuild.c
 
 liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
 if ENABLE_LZ4
diff --git a/lib/rebuild.c b/lib/rebuild.c
new file mode 100644
index 0000000..7aaa071
--- /dev/null
+++ b/lib/rebuild.c
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include "erofs/print.h"
+#include "erofs/inode.h"
+#include "erofs/rebuild.h"
+#include "erofs/internal.h"
+
+#ifdef HAVE_LINUX_AUFS_TYPE_H
+#include <linux/aufs_type.h>
+#else
+#define AUFS_WH_PFX		".wh."
+#define AUFS_DIROPQ_NAME	AUFS_WH_PFX ".opq"
+#define AUFS_WH_DIROPQ		AUFS_WH_PFX AUFS_DIROPQ_NAME
+#endif
+
+static struct erofs_dentry *erofs_rebuild_mkdir(struct erofs_inode *dir,
+						const char *s)
+{
+	struct erofs_inode *inode;
+	struct erofs_dentry *d;
+
+	inode = erofs_new_inode();
+	if (IS_ERR(inode))
+		return ERR_CAST(inode);
+
+	inode->i_mode = S_IFDIR | 0755;
+	inode->i_parent = dir;
+	inode->i_uid = getuid();
+	inode->i_gid = getgid();
+	inode->i_mtime = inode->sbi->build_time;
+	inode->i_mtime_nsec = inode->sbi->build_time_nsec;
+	erofs_init_empty_dir(inode);
+
+	d = erofs_d_alloc(dir, s);
+	if (!IS_ERR(d)) {
+		d->type = EROFS_FT_DIR;
+		d->inode = inode;
+	}
+	return d;
+}
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+		char *path, bool aufs, bool *whout, bool *opq)
+{
+	struct erofs_dentry *d = NULL;
+	unsigned int len = strlen(path);
+	char *s = path;
+
+	if (aufs) {
+		*whout = false;
+		*opq = false;
+	}
+
+	while (s < path + len) {
+		char *slash = memchr(s, '/', path + len - s);
+		if (slash) {
+			if (s == slash) {
+				while (*++s == '/');	/* skip '//...' */
+				continue;
+			}
+			*slash = '\0';
+		}
+
+		if (!memcmp(s, ".", 2)) {
+			/* null */
+		} else if (!memcmp(s, "..", 3)) {
+			pwd = pwd->i_parent;
+		} else {
+			struct erofs_inode *inode = NULL;
+
+			if (aufs && !slash) {
+				if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
+					*opq = true;
+					break;
+				}
+				if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
+					s += sizeof(AUFS_WH_PFX) - 1;
+					*whout = true;
+				}
+			}
+
+			list_for_each_entry(d, &pwd->i_subdirs, d_child) {
+				if (!strcmp(d->name, s)) {
+					if (d->type != EROFS_FT_DIR && slash)
+						return ERR_PTR(-EIO);
+					inode = d->inode;
+					break;
+				}
+			}
+
+			if (inode) {
+				pwd = inode;
+			} else if (!slash) {
+				d = erofs_d_alloc(pwd, s);
+				if (IS_ERR(d))
+					return d;
+				d->type = EROFS_FT_UNKNOWN;
+				d->inode = pwd;
+			} else {
+				d = erofs_rebuild_mkdir(pwd, s);
+				if (IS_ERR(d))
+					return d;
+				pwd = d->inode;
+			}
+		}
+		if (slash) {
+			*slash = '/';
+			s = slash + 1;
+		} else {
+			break;
+		}
+	}
+	return d;
+}
diff --git a/lib/tar.c b/lib/tar.c
index 5b63cf5..e0fe2a2 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -3,13 +3,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
-#ifdef HAVE_LINUX_AUFS_TYPE_H
-#include <linux/aufs_type.h>
-#else
-#define AUFS_WH_PFX		".wh."
-#define AUFS_DIROPQ_NAME	AUFS_WH_PFX ".opq"
-#define AUFS_WH_DIROPQ		AUFS_WH_PFX AUFS_DIROPQ_NAME
-#endif
 #include "erofs/print.h"
 #include "erofs/cache.h"
 #include "erofs/inode.h"
@@ -18,6 +11,7 @@
 #include "erofs/io.h"
 #include "erofs/xattr.h"
 #include "erofs/blobchunk.h"
+#include "erofs/rebuild.h"
 
 #define EROFS_WHITEOUT_DEV	0
 
@@ -133,103 +127,6 @@ static long long tarerofs_parsenum(const char *ptr, int len)
 	return tarerofs_otoi(ptr, len);
 }
 
-static struct erofs_dentry *tarerofs_mkdir(struct erofs_inode *dir, const char *s)
-{
-	struct erofs_inode *inode;
-	struct erofs_dentry *d;
-
-	inode = erofs_new_inode();
-	if (IS_ERR(inode))
-		return ERR_CAST(inode);
-
-	inode->i_mode = S_IFDIR | 0755;
-	inode->i_parent = dir;
-	inode->i_uid = getuid();
-	inode->i_gid = getgid();
-	inode->i_mtime = inode->sbi->build_time;
-	inode->i_mtime_nsec = inode->sbi->build_time_nsec;
-	erofs_init_empty_dir(inode);
-
-	d = erofs_d_alloc(dir, s);
-	if (!IS_ERR(d)) {
-		d->type = EROFS_FT_DIR;
-		d->inode = inode;
-	}
-	return d;
-}
-
-static struct erofs_dentry *tarerofs_get_dentry(struct erofs_inode *pwd, char *path,
-					        bool aufs, bool *whout, bool *opq)
-{
-	struct erofs_dentry *d = NULL;
-	unsigned int len = strlen(path);
-	char *s = path;
-
-	*whout = false;
-	*opq = false;
-
-	while (s < path + len) {
-		char *slash = memchr(s, '/', path + len - s);
-		if (slash) {
-			if (s == slash) {
-				while (*++s == '/');	/* skip '//...' */
-				continue;
-			}
-			*slash = '\0';
-		}
-
-		if (!memcmp(s, ".", 2)) {
-			/* null */
-		} else if (!memcmp(s, "..", 3)) {
-			pwd = pwd->i_parent;
-		} else {
-			struct erofs_inode *inode = NULL;
-
-			if (aufs && !slash) {
-				if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
-					*opq = true;
-					break;
-				}
-				if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
-					s += sizeof(AUFS_WH_PFX) - 1;
-					*whout = true;
-				}
-			}
-
-			list_for_each_entry(d, &pwd->i_subdirs, d_child) {
-				if (!strcmp(d->name, s)) {
-					if (d->type != EROFS_FT_DIR && slash)
-						return ERR_PTR(-EIO);
-					inode = d->inode;
-					break;
-				}
-			}
-
-			if (inode) {
-				pwd = inode;
-			} else if (!slash) {
-				d = erofs_d_alloc(pwd, s);
-				if (IS_ERR(d))
-					return d;
-				d->type = EROFS_FT_UNKNOWN;
-				d->inode = pwd;
-			} else {
-				d = tarerofs_mkdir(pwd, s);
-				if (IS_ERR(d))
-					return d;
-				pwd = d->inode;
-			}
-		}
-		if (slash) {
-			*slash = '/';
-			s = slash + 1;
-		} else {
-			break;
-		}
-	}
-	return d;
-}
-
 struct tarerofs_xattr_item {
 	struct list_head list;
 	char *kv;
@@ -767,7 +664,7 @@ restart:
 
 	erofs_dbg("parsing %s (mode %05o)", eh.path, st.st_mode);
 
-	d = tarerofs_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
+	d = erofs_rebuild_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
 	if (IS_ERR(d)) {
 		ret = PTR_ERR(d);
 		goto out;
@@ -800,7 +697,7 @@ restart:
 		}
 		d->inode = NULL;
 
-		d2 = tarerofs_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
+		d2 = erofs_rebuild_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
 		if (IS_ERR(d2)) {
 			ret = PTR_ERR(d2);
 			goto out;
-- 
2.19.1.6.gb485710b

[PATCH v6 07/11] erofs-utils: lib: add erofs_rebuild_load_tree() helper

[Jingbo Xu]

Add erofs_rebuild_load_tree() helper loading inode tree from given erofs
image, and making it merged into a given inode tree in an overlayfs like
model.

Since we need to read the content of the symlink file from disk when
loading tree, add dependency on zlib_LIBS for mkfs.erofs.

Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree(), so that
it could also be called from rebuild mode.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/inode.h    |   2 +-
 include/erofs/internal.h |   1 +
 include/erofs/rebuild.h  |   2 +
 lib/inode.c              |   4 +-
 lib/rebuild.c            | 284 +++++++++++++++++++++++++++++++++++++++
 mkfs/main.c              |   2 +-
 6 files changed, 291 insertions(+), 4 deletions(-)

diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index 1c602a8..fe9dda2 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
 				   const char *name);
-int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_rebuild_dump_tree(struct erofs_inode *dir);
 int erofs_init_empty_dir(struct erofs_inode *dir);
 struct erofs_inode *erofs_new_inode(void);
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 6020553..3d5c337 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -108,6 +108,7 @@ struct erofs_sb_info {
 
 	int devfd;
 	u64 devsz;
+	dev_t dev;
 	unsigned int nblobs;
 	unsigned int blobfd[256];
 };
diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
index 92873c9..3ac074c 100644
--- a/include/erofs/rebuild.h
+++ b/include/erofs/rebuild.h
@@ -12,6 +12,8 @@ extern "C"
 struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
 		char *path, bool aufs, bool *whout, bool *opq);
 
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/inode.c b/lib/inode.c
index 8fab964..7e5d464 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1324,7 +1324,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
 	return inode;
 }
 
-int tarerofs_dump_tree(struct erofs_inode *dir)
+int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 {
 	struct erofs_dentry *d;
 	unsigned int nr_subdirs;
@@ -1392,7 +1392,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
 			continue;
 
 		inode = erofs_igrab(d->inode);
-		ret = tarerofs_dump_tree(inode);
+		ret = erofs_rebuild_dump_tree(inode);
 		dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
 		erofs_iput(inode);
 		if (ret)
diff --git a/lib/rebuild.c b/lib/rebuild.c
index 7aaa071..477d68d 100644
--- a/lib/rebuild.c
+++ b/lib/rebuild.c
@@ -3,9 +3,18 @@
 #include <unistd.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
+#include <config.h>
+#if defined(HAVE_SYS_SYSMACROS_H)
+#include <sys/sysmacros.h>
+#endif
 #include "erofs/print.h"
 #include "erofs/inode.h"
 #include "erofs/rebuild.h"
+#include "erofs/io.h"
+#include "erofs/dir.h"
+#include "erofs/xattr.h"
+#include "erofs/blobchunk.h"
 #include "erofs/internal.h"
 
 #ifdef HAVE_LINUX_AUFS_TYPE_H
@@ -115,3 +124,278 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
 	}
 	return d;
 }
+
+static int erofs_rebuild_fixup_inode_index(struct erofs_inode *inode)
+{
+	int ret;
+	unsigned int count, unit, chunkbits, i;
+	unsigned int deviceid = inode->dev;
+	struct erofs_inode_chunk_index *idx;
+	erofs_off_t chunksize;
+	erofs_blk_t blkaddr;
+
+	/* TODO: fill data map in other layouts */
+	if (inode->datalayout != EROFS_INODE_CHUNK_BASED &&
+	    inode->datalayout != EROFS_INODE_FLAT_PLAIN) {
+		erofs_err("unsupported datalayout %d", inode->datalayout);
+		return -EOPNOTSUPP;
+	}
+
+	if (inode->sbi->extra_devices) {
+		chunkbits = inode->u.chunkbits;
+		if (chunkbits < sbi.blkszbits) {
+			erofs_err("blkszbits no greater than source blkszbits");
+			return -EINVAL;
+		}
+	} else {
+		chunkbits = ilog2(inode->i_size - 1) + 1;
+		if (chunkbits < sbi.blkszbits)
+			chunkbits = sbi.blkszbits;
+		if (chunkbits - sbi.blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
+			chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi.blkszbits;
+	}
+	chunksize = 1ULL << chunkbits;
+	count = DIV_ROUND_UP(inode->i_size, chunksize);
+
+	unit = sizeof(struct erofs_inode_chunk_index);
+	inode->extent_isize = count * unit;
+	idx = malloc(max(sizeof(*idx), sizeof(void *)));
+	if (!idx)
+		return -ENOMEM;
+	inode->chunkindexes = idx;
+
+	for (i = 0; i < count; i++) {
+		struct erofs_blobchunk *chunk;
+		struct erofs_map_blocks map = {
+			.index = UINT_MAX,
+		};
+
+		map.m_la = i << chunkbits;
+		ret = erofs_map_blocks(inode, &map, 0);
+		if (ret)
+			goto err;
+
+		blkaddr = erofs_blknr(&sbi, map.m_pa);
+		chunk = erofs_get_unhashed_chunk(deviceid, blkaddr, 0);
+		if (IS_ERR(chunk)) {
+			ret = PTR_ERR(chunk);
+			goto err;
+		}
+		*(void **)idx++ = chunk;
+
+	}
+	inode->datalayout = EROFS_INODE_CHUNK_BASED;
+	inode->u.chunkformat = EROFS_CHUNK_FORMAT_INDEXES;
+	inode->u.chunkformat |= chunkbits - sbi.blkszbits;
+	return 0;
+err:
+	free(idx);
+	inode->chunkindexes = NULL;
+	return ret;
+}
+
+static int erofs_rebuild_fill_inode(struct erofs_inode *inode)
+{
+	int ret = 0;
+
+	switch (inode->i_mode & S_IFMT) {
+	case S_IFCHR:
+	case S_IFBLK:
+	case S_IFIFO:
+	case S_IFSOCK:
+		inode->i_size = 0;
+		erofs_dbg("\tdev: %d %d", major(inode->u.i_rdev),
+			  minor(inode->u.i_rdev));
+		inode->u.i_rdev = erofs_new_encode_dev(inode->u.i_rdev);
+		break;
+	case S_IFDIR:
+		ret = erofs_init_empty_dir(inode);
+		break;
+	case S_IFLNK:
+		inode->i_link = malloc(inode->i_size + 1);
+		if (!inode->i_link)
+			return -ENOMEM;
+		ret = erofs_pread(inode, inode->i_link, inode->i_size, 0);
+		erofs_dbg("\tsymlink: %s -> %s", inode->i_srcpath, inode->i_link);
+		break;
+	case S_IFREG:
+		if (inode->i_size)
+			ret = erofs_rebuild_fixup_inode_index(inode);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * @parent:  parent directory in inode tree
+ * @ctx.dir: parent directory when itering erofs_iterate_dir()
+ */
+struct erofs_rebuild_dir_context {
+	struct erofs_dir_context ctx;
+	struct erofs_inode *parent;
+	dev_t dev;
+};
+
+static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx)
+{
+	struct erofs_rebuild_dir_context *rctx = (void *)ctx;
+	struct erofs_inode *parent = rctx->parent;
+	struct erofs_inode *dir = ctx->dir;
+	struct erofs_inode *inode, *candidate;
+	struct erofs_inode src;
+	struct erofs_dentry *d;
+	char *path, *dname;
+	int ret;
+
+	if (ctx->dot_dotdot)
+		return 0;
+
+	ret = asprintf(&path, "%s/%.*s", rctx->parent->i_srcpath,
+		       ctx->de_namelen, ctx->dname);
+	if (ret < 0)
+		return ret;
+
+	erofs_dbg("parsing path %s", path);
+	dname = path + strlen(parent->i_srcpath) + 1;
+
+	d = erofs_rebuild_get_dentry(parent, dname, false, NULL, NULL);
+	if (IS_ERR(d)) {
+		ret = PTR_ERR(d);
+		goto out;
+	}
+	DBG_BUGON(parent != d->inode);
+
+	ret = 0;
+	if (d->type != EROFS_FT_UNKNOWN) {
+		/* file exists in upper layer */
+		if (!S_ISDIR(d->inode->i_mode) || d->inode->opaque)
+			goto out;
+
+		/* merging directories */
+		src = (struct erofs_inode) {
+			.sbi = dir->sbi,
+			.nid = ctx->de_nid
+		};
+		ret = erofs_read_inode_from_disk(&src);
+		if (ret || !S_ISDIR(src.i_mode))
+			goto out;
+		parent = d->inode;
+		inode = dir = &src;
+	} else {
+		u64 nid;
+
+		inode = erofs_new_inode();
+		if (IS_ERR(inode)) {
+			ret = PTR_ERR(inode);
+			goto out;
+		}
+
+		/* reuse i_ino[0] to read nid in source fs */
+		nid = inode->i_ino[0];
+		inode->sbi = dir->sbi;
+		inode->nid = ctx->de_nid;
+		ret = erofs_read_inode_from_disk(inode);
+		if (ret)
+			goto out;
+
+		/* restore nid in new generated fs */
+		inode->i_ino[1] = inode->i_ino[0];
+		inode->i_ino[0] = nid;
+
+		if (S_ISREG(inode->i_mode) && inode->i_nlink > 1 &&
+		    (candidate = erofs_iget(rctx->dev, ctx->de_nid))) {
+			/* hardlink file */
+			erofs_iput(inode);
+			inode = candidate;
+			if (S_ISDIR(inode->i_mode)) {
+				erofs_err("hardlink directory not supported");
+				ret = -EISDIR;
+				goto out;
+			}
+			inode->i_nlink++;
+			erofs_dbg("\thardlink: %s -> %s", path, inode->i_srcpath);
+		} else {
+			ret = erofs_read_xattrs_from_disk(inode);
+			if (ret) {
+				erofs_iput(inode);
+				goto out;
+			}
+
+			inode->i_parent = d->inode;
+			inode->dev = rctx->dev;
+			inode->i_srcpath = path;
+			path = NULL;
+			inode->i_ino[1] = inode->nid;
+			inode->i_nlink = 1;
+
+			ret = erofs_rebuild_fill_inode(inode);
+			if (ret) {
+				erofs_iput(inode);
+				goto out;
+			}
+
+			erofs_insert_ihash(inode, rctx->dev, inode->i_ino[1]);
+			parent = dir = inode;
+		}
+
+		d->inode = inode;
+		d->type = erofs_mode_to_ftype(inode->i_mode);
+	}
+
+	ret = 0;
+	if (S_ISDIR(inode->i_mode)) {
+		struct erofs_rebuild_dir_context nctx = *rctx;
+
+		nctx.parent = parent;
+		nctx.ctx.dir = dir;
+		ret = erofs_iterate_dir(&nctx.ctx, false);
+	}
+
+	/* don't restore inode->sbi until all the read work has finished */
+	inode->sbi = &sbi;
+	inode->nid = 0;
+out:
+	if (path)
+		free(path);
+	return ret;
+}
+
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi)
+{
+	struct erofs_inode inode = {};
+	struct erofs_rebuild_dir_context ctx;
+	int ret;
+
+	if (!sbi->devname) {
+		erofs_err("failed to find a device for rebuilding");
+		return -EINVAL;
+	}
+
+	ret = erofs_read_superblock(sbi);
+	if (ret) {
+		erofs_err("failed to read superblock of %s", sbi->devname);
+		return ret;
+	}
+
+	inode.nid = sbi->root_nid;
+	inode.sbi = sbi;
+	ret = erofs_read_inode_from_disk(&inode);
+	if (ret) {
+		erofs_err("failed to read root inode of %s", sbi->devname);
+		return ret;
+	}
+	inode.i_srcpath = strdup("/");
+
+	ctx = (struct erofs_rebuild_dir_context) {
+		.ctx.dir = &inode,
+		.ctx.cb = erofs_rebuild_dirent_iter,
+		.parent = root,
+		.dev = sbi->dev,
+	};
+	ret = erofs_iterate_dir(&ctx.ctx, false);
+	free(inode.i_srcpath);
+	return ret;
+}
diff --git a/mkfs/main.c b/mkfs/main.c
index 607c883..94f6407 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -962,7 +962,7 @@ int main(int argc, char **argv)
 		if (err < 0)
 			goto exit;
 
-		err = tarerofs_dump_tree(root_inode);
+		err = erofs_rebuild_dump_tree(root_inode);
 		if (err < 0)
 			goto exit;
 	}
-- 
2.19.1.6.gb485710b

[PATCH v6 08/11] erofs-utils: mkfs: introduce rebuild mode

[Jingbo Xu]

Introduce a new rebuild mode merging multiple erofs images generated
from either tarerfs index mode (--tar=i):

	mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

or tarerofs non-index mode (--tar=f):

	mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
	...
	mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar

	mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs

When calling rebuild mode, the top most layer must be specified firstly
in the cmdline, while the bottom most layer i.e. layer0 the lastly.

The rebuild mode doesn't support flat inline datalayout yet.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/internal.h |   2 +
 mkfs/main.c              | 216 ++++++++++++++++++++++++++++++---------
 2 files changed, 170 insertions(+), 48 deletions(-)

diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 3d5c337..fcc0710 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -111,6 +111,8 @@ struct erofs_sb_info {
 	dev_t dev;
 	unsigned int nblobs;
 	unsigned int blobfd[256];
+
+	struct list_head list;
 };
 
 /* make sure that any user of the erofs headers has atleast 64bit off_t type */
diff --git a/mkfs/main.c b/mkfs/main.c
index 94f6407..b51ef37 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -26,6 +26,7 @@
 #include "erofs/compress_hints.h"
 #include "erofs/blobchunk.h"
 #include "erofs/fragments.h"
+#include "erofs/rebuild.h"
 #include "../lib/liberofs_private.h"
 #include "../lib/liberofs_uuid.h"
 
@@ -83,8 +84,14 @@ static void print_available_compressors(FILE *f, const char *delim)
 
 static void usage(void)
 {
-	fputs("usage: [options] FILE DIRECTORY\n\n"
-	      "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
+	fputs("usage:\n"
+	      "[options] FILE DIRECTORY\t"
+	      "Generate erofs image (FILE) from DIRECTORY.\n"
+	      "[options] FILE TARBALL\t\t"
+	      "Generate erofs image (FILE) from TARBALL.\n"
+	      "[options] FILE SOURCE(s)\t"
+	      "Generate erofs image (FILE) by merging previously generated images.\n\n"
+	      "[options] are:\n"
 	      " -b#                   set block size to # (# = page size by default)\n"
 	      " -d#                   set output message level to # (maximum 9)\n"
 	      " -x#                   set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
@@ -136,6 +143,10 @@ static struct erofs_tarfile erofstar = {
 	.global.xattrs = LIST_HEAD_INIT(erofstar.global.xattrs)
 };
 static bool tar_mode;
+static bool rebuild_mode;
+
+static unsigned int rebuild_src_count;
+static LIST_HEAD(rebuild_src_list);
 
 static int parse_extended_opts(const char *opts)
 {
@@ -291,10 +302,24 @@ static int mkfs_parse_compress_algs(char *algs)
 	return 0;
 }
 
+static void erofs_rebuild_cleanup(void)
+{
+	struct erofs_sb_info *src, *n;
+
+	list_for_each_entry_safe(src, n, &rebuild_src_list, list) {
+		list_del(&src->list);
+		erofs_put_super(src);
+		dev_close(src);
+		free(src);
+	}
+	rebuild_src_count = 0;
+}
+
 static int mkfs_parse_options_cfg(int argc, char *argv[])
 {
 	char *endptr;
-	int opt, i;
+	struct stat st;
+	int opt, i, err;
 	bool quiet = false;
 
 	while ((opt = getopt_long(argc, argv, "C:E:L:T:U:b:d:x:z:",
@@ -550,7 +575,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		} else {
 			erofstar.fd = STDIN_FILENO;
 		}
-	}else {
+	} else {
 		cfg.c_src_path = realpath(argv[optind++], NULL);
 		if (!cfg.c_src_path) {
 			erofs_err("failed to parse source directory: %s",
@@ -558,7 +583,46 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			return -ENOENT;
 		}
 
-		if (optind < argc) {
+		if (tar_mode) {
+			erofstar.fd = open(cfg.c_src_path, O_RDONLY);
+			if (erofstar.fd < 0) {
+				erofs_err("failed to open file: %s", cfg.c_src_path);
+				usage();
+				return -errno;
+			}
+		} else {
+			err = lstat(cfg.c_src_path, &st);
+			if (err)
+				return -errno;
+			if (S_ISDIR(st.st_mode))
+				erofs_set_fs_root(cfg.c_src_path);
+			else
+				rebuild_mode = true;
+		}
+
+		if (rebuild_mode) {
+			char *srcpath = cfg.c_src_path;
+			struct erofs_sb_info *src;
+
+			do {
+				src = malloc(sizeof(struct erofs_sb_info));
+				if (!src) {
+					erofs_rebuild_cleanup();
+					return -ENOMEM;
+				}
+
+				err = dev_open_ro(src, srcpath);
+				if (err) {
+					free(src);
+					erofs_rebuild_cleanup();
+					return err;
+				}
+
+				/* extra device index starts from 1 */
+				src->dev = ++rebuild_src_count;
+				list_add_tail(&src->list, &rebuild_src_list);
+			} while (optind < argc && (srcpath = argv[optind++]));
+		} else if (optind < argc) {
 			erofs_err("unexpected argument: %s\n", argv[optind]);
 			return -EINVAL;
 		}
@@ -748,6 +812,61 @@ void erofs_show_progs(int argc, char *argv[])
 	if (cfg.c_dbg_lvl >= EROFS_WARN)
 		printf("%s %s\n", basename(argv[0]), cfg.c_version);
 }
+static struct erofs_inode *erofs_alloc_root_inode(void)
+{
+	struct erofs_inode *root;
+
+	root = erofs_new_inode();
+	if (IS_ERR(root))
+		return root;
+	root->i_srcpath = strdup("/");
+	root->i_mode = S_IFDIR | 0777;
+	root->i_parent = root;
+	root->i_mtime = root->sbi->build_time;
+	root->i_mtime_nsec = root->sbi->build_time_nsec;
+	erofs_init_empty_dir(root);
+	return root;
+}
+
+static int erofs_rebuild_load_trees(struct erofs_inode *root)
+{
+	struct erofs_sb_info *src;
+	unsigned int extra_devices = 0;
+	erofs_blk_t nblocks;
+	int ret, i = 0;
+
+	list_for_each_entry(src, &rebuild_src_list, list) {
+		ret = erofs_rebuild_load_tree(root, src);
+		if (ret) {
+			erofs_err("failed to load %s", src->devname);
+			return ret;
+		}
+		if (src->extra_devices > 1) {
+			erofs_err("%s: unsupported number of extra devices",
+				  src->devname, src->extra_devices);
+			return -EOPNOTSUPP;
+		}
+		extra_devices += src->extra_devices;
+	}
+
+	if (extra_devices && extra_devices != rebuild_src_count) {
+		erofs_err("unsupported mix of source images");
+		return -EOPNOTSUPP;
+	}
+
+	ret = erofs_mkfs_init_devices(&sbi, rebuild_src_count);
+	if (ret)
+		return ret;
+
+	list_for_each_entry(src, &rebuild_src_list, list) {
+		if (extra_devices)
+			nblocks = src->devs[0].blocks;
+		else
+			nblocks = src->primarydevice_blocks;
+		sbi.devs[i++].blocks = nblocks;
+	}
+	return 0;
+}
 
 int main(int argc, char **argv)
 {
@@ -755,7 +874,6 @@ int main(int argc, char **argv)
 	struct erofs_buffer_head *sb_bh;
 	struct erofs_inode *root_inode, *packed_inode;
 	erofs_nid_t root_nid, packed_nid;
-	struct stat st;
 	erofs_blk_t nblocks;
 	struct timeval t;
 	char uuid_str[37];
@@ -778,26 +896,6 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
-	if (!tar_mode) {
-		err = lstat(cfg.c_src_path, &st);
-		if (err)
-			return 1;
-		if (!S_ISDIR(st.st_mode)) {
-			erofs_err("root of the filesystem is not a directory - %s",
-				  cfg.c_src_path);
-			usage();
-			return 1;
-		}
-		erofs_set_fs_root(cfg.c_src_path);
-	} else if (cfg.c_src_path) {
-		erofstar.fd = open(cfg.c_src_path, O_RDONLY);
-		if (erofstar.fd < 0) {
-			erofs_err("failed to open file: %s", cfg.c_src_path);
-			usage();
-			return 1;
-		}
-	}
-
 	if (cfg.c_unix_timestamp != -1) {
 		sbi.build_time      = cfg.c_unix_timestamp;
 		sbi.build_time_nsec = 0;
@@ -861,6 +959,20 @@ int main(int argc, char **argv)
 		}
 	}
 
+	if (rebuild_mode) {
+		struct erofs_sb_info *src;
+
+		src = list_first_entry(&rebuild_src_list, struct erofs_sb_info, list);
+		if (!src)
+			goto exit;
+		err = erofs_read_superblock(src);
+		if (err) {
+			erofs_err("failed to read superblock of %s", src->devname);
+			goto exit;
+		}
+		sbi.blkszbits = src->blkszbits;
+	}
+
 	sb_bh = erofs_buffer_init();
 	if (IS_ERR(sb_bh)) {
 		err = PTR_ERR(sb_bh);
@@ -928,7 +1040,35 @@ int main(int argc, char **argv)
 
 	erofs_inode_manager_init();
 
-	if (!tar_mode) {
+	if (tar_mode) {
+		root_inode = erofs_alloc_root_inode();
+		if (IS_ERR(root_inode)) {
+			err = PTR_ERR(root_inode);
+			goto exit;
+		}
+
+		while (!(err = tarerofs_parse_tar(root_inode, &erofstar)));
+
+		if (err < 0)
+			goto exit;
+
+		err = erofs_rebuild_dump_tree(root_inode);
+		if (err < 0)
+			goto exit;
+	} else if (rebuild_mode) {
+		root_inode = erofs_alloc_root_inode();
+		if (IS_ERR(root_inode)) {
+			err = PTR_ERR(root_inode);
+			goto exit;
+		}
+
+		err = erofs_rebuild_load_trees(root_inode);
+		if (err)
+			goto exit;
+		err = erofs_rebuild_dump_tree(root_inode);
+		if (err)
+			goto exit;
+	} else {
 		err = erofs_build_shared_xattrs_from_path(&sbi, cfg.c_src_path);
 		if (err) {
 			erofs_err("failed to build shared xattrs: %s",
@@ -944,27 +1084,6 @@ int main(int argc, char **argv)
 			err = PTR_ERR(root_inode);
 			goto exit;
 		}
-	} else {
-		root_inode = erofs_new_inode();
-		if (IS_ERR(root_inode)) {
-			err = PTR_ERR(root_inode);
-			goto exit;
-		}
-		root_inode->i_srcpath = strdup("/");
-		root_inode->i_mode = S_IFDIR | 0777;
-		root_inode->i_parent = root_inode;
-		root_inode->i_mtime = sbi.build_time;
-		root_inode->i_mtime_nsec = sbi.build_time_nsec;
-		erofs_init_empty_dir(root_inode);
-
-		while (!(err = tarerofs_parse_tar(root_inode, &erofstar)));
-
-		if (err < 0)
-			goto exit;
-
-		err = erofs_rebuild_dump_tree(root_inode);
-		if (err < 0)
-			goto exit;
 	}
 	root_nid = erofs_lookupnid(root_inode);
 	erofs_iput(root_inode);
@@ -1024,6 +1143,7 @@ exit:
 		z_erofs_fragments_exit();
 	erofs_packedfile_exit();
 	erofs_xattr_cleanup_name_prefixes();
+	erofs_rebuild_cleanup();
 	erofs_exit_configure();
 
 	if (err) {
-- 
2.19.1.6.gb485710b

[PATCH v6 09/11] erofs-utils: lib: add erofs_inode_is_whiteout() helper

[Jingbo Xu]

Add erofs_inode_is_whiteout() helper checking if given inode is a
whiteout.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/internal.h | 7 +++++++
 lib/tar.c                | 2 --
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index fcc0710..5df689c 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -20,6 +20,7 @@ typedef unsigned short umode_t;
 #include "erofs_fs.h"
 #include <fcntl.h>
 #include <sys/types.h> /* for off_t definition */
+#include <sys/stat.h> /* for S_ISCHR definition */
 #include <stdio.h>
 
 #ifndef PATH_MAX
@@ -427,6 +428,12 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len)
 	return crc;
 }
 
+#define EROFS_WHITEOUT_DEV	0
+static inline bool erofs_inode_is_whiteout(struct erofs_inode *inode)
+{
+	return S_ISCHR(inode->i_mode) && inode->u.i_rdev == EROFS_WHITEOUT_DEV;
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/tar.c b/lib/tar.c
index e0fe2a2..8cb392c 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -13,8 +13,6 @@
 #include "erofs/blobchunk.h"
 #include "erofs/rebuild.h"
 
-#define EROFS_WHITEOUT_DEV	0
-
 static char erofs_libbuf[16384];
 
 struct tar_header {
-- 
2.19.1.6.gb485710b

[PATCH v6 10/11] erofs-utils: lib: set origin xattr on parent directory of whiteout

[Jingbo Xu]

Whiteouts will be exposed in the overlayfs mount if the whiteouts are
from non-merged directory without origin xattr set on the directory.

To fix this, set origin xattr (with empty value) on parent directory of
whiteouts, marking the parent directory as copied-up to avoid exposing
whiteouts in the overlayfs mount.

To avoid setting multiple origin xattrs on the same directory when
there are multiple whiteouts under the directory, enhance
erofs_setxattr() so that it could skip duplicate xattrs.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/xattr.h |  3 ++-
 lib/rebuild.c         |  6 ++++++
 lib/tar.c             |  9 ++++++++-
 lib/xattr.c           | 27 ++++++++++++++++++++++++---
 4 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index 9fa1ad1..62d0b8b 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -55,8 +55,9 @@ void erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi);
 int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
 
 int erofs_setxattr(struct erofs_inode *inode, char *key,
-		   const void *value, size_t size);
+		   const void *value, size_t size, bool nodup);
 int erofs_set_opaque_xattr(struct erofs_inode *inode);
+int erofs_set_origin_xattr(struct erofs_inode *inode);
 int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
 
 #ifdef __cplusplus
diff --git a/lib/rebuild.c b/lib/rebuild.c
index 477d68d..b254dea 100644
--- a/lib/rebuild.c
+++ b/lib/rebuild.c
@@ -200,6 +200,12 @@ static int erofs_rebuild_fill_inode(struct erofs_inode *inode)
 
 	switch (inode->i_mode & S_IFMT) {
 	case S_IFCHR:
+		if (erofs_inode_is_whiteout(inode)) {
+			ret = erofs_set_origin_xattr(inode->i_parent);
+			if (ret)
+				break;
+		}
+		/* fallthrough */
 	case S_IFBLK:
 	case S_IFIFO:
 	case S_IFSOCK:
diff --git a/lib/tar.c b/lib/tar.c
index 8cb392c..b75b723 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -206,7 +206,7 @@ int tarerofs_apply_xattrs(struct erofs_inode *inode, struct list_head *xattrs)
 		item->kv[item->namelen] = '\0';
 		erofs_dbg("Recording xattr(%s)=\"%s\" (of %u bytes) to file %s",
 			  item->kv, v, vsz, inode->i_srcpath);
-		ret = erofs_setxattr(inode, item->kv, v, vsz);
+		ret = erofs_setxattr(inode, item->kv, v, vsz, false);
 		if (ret == -ENODATA)
 			erofs_err("Failed to set xattr(%s)=%s to file %s",
 				  item->kv, v, inode->i_srcpath);
@@ -740,6 +740,13 @@ new_inode:
 		inode->i_mode = (inode->i_mode & ~S_IFMT) | S_IFCHR;
 		inode->u.i_rdev = EROFS_WHITEOUT_DEV;
 		d->type = EROFS_FT_CHRDEV;
+		/*
+		 * Mark parent directory as copied-up to avoid exposing
+		 * whiteouts in the overlayfs mount.
+		 */
+		ret = erofs_set_origin_xattr(inode->i_parent);
+		if (ret)
+			goto out;
 	} else {
 		inode->i_mode = st.st_mode;
 		if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
diff --git a/lib/xattr.c b/lib/xattr.c
index cc6a393..feafe96 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -57,12 +57,18 @@
 #ifndef OVL_XATTR_OPAQUE_POSTFIX
 #define OVL_XATTR_OPAQUE_POSTFIX "opaque"
 #endif
+#ifndef OVL_XATTR_ORIGIN_POSTFIX
+#define OVL_XATTR_ORIGIN_POSTFIX "origin"
+#endif
 #ifndef OVL_XATTR_TRUSTED_PREFIX
 #define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
 #endif
 #ifndef OVL_XATTR_OPAQUE
 #define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
 #endif
+#ifndef OVL_XATTR_ORIGIN
+#define OVL_XATTR_ORIGIN OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_ORIGIN_POSTFIX
+#endif
 
 #define EA_HASHTABLE_BITS 16
 
@@ -455,11 +461,12 @@ err:
 }
 
 int erofs_setxattr(struct erofs_inode *inode, char *key,
-		   const void *value, size_t size)
+		   const void *value, size_t size, bool nodup)
 {
 	char *kvbuf;
 	unsigned int len[2];
 	struct xattr_item *item;
+	struct inode_xattr_node *node;
 	u8 prefix;
 	u16 prefixlen;
 
@@ -481,12 +488,26 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
 	if (IS_ERR(item))
 		return PTR_ERR(item);
 
+	if (nodup) {
+		list_for_each_entry(node, &inode->i_xattrs, list) {
+			if (node->item == item) {
+				put_xattritem(item);
+				return 0;
+			}
+		}
+	}
+
 	return erofs_xattr_add(&inode->i_xattrs, item);
 }
 
 int erofs_set_opaque_xattr(struct erofs_inode *inode)
 {
-	return erofs_setxattr(inode, OVL_XATTR_OPAQUE, "y", 1);
+	return erofs_setxattr(inode, OVL_XATTR_OPAQUE, "y", 1, false);
+}
+
+int erofs_set_origin_xattr(struct erofs_inode *inode)
+{
+	return erofs_setxattr(inode, OVL_XATTR_ORIGIN, NULL, 0, true);
 }
 
 #ifdef WITH_ANDROID
@@ -599,7 +620,7 @@ int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
 			DBG_BUGON(ret != size);
 		}
 
-		ret = erofs_setxattr(inode, key, value, size);
+		ret = erofs_setxattr(inode, key, value, size, false);
 		free(value);
 		if (ret)
 			break;
-- 
2.19.1.6.gb485710b

[PATCH v6 11/11] erofs-utils: mkfs: add --overlay-strip option for tarfs and rebuild mode

[Jingbo Xu]

Add "--overlay-strip=[0|1]" option for tarfs and rebuild mode,
controlling whether overlayfs related stuff, e.g. whiteout files and
origin xattrs, are stripped from the generated image.

This option is disabled by default for both tarfs and rebuild mode, that
is, whiteout files and origin xattrs are kept in the image by default.
Specify "--overlay-strip=1" explicitly to strip these stuff.

Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
 include/erofs/config.h |  1 +
 lib/inode.c            | 12 ++++++++++--
 lib/rebuild.c          | 10 ++++++----
 lib/tar.c              | 16 +++++++++-------
 lib/xattr.c            |  5 +++++
 mkfs/main.c            | 12 ++++++++++++
 6 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/include/erofs/config.h b/include/erofs/config.h
index c51f0cd..f244607 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -54,6 +54,7 @@ struct erofs_configure {
 	bool c_showprogress;
 	bool c_extra_ea_name_prefixes;
 	bool c_xattr_name_filter;
+	bool c_overlay_strip;
 
 #ifdef HAVE_LIBSELINUX
 	struct selabel_handle *sehnd;
diff --git a/lib/inode.c b/lib/inode.c
index 7e5d464..42a4c2f 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1326,7 +1326,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
 
 int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 {
-	struct erofs_dentry *d;
+	struct erofs_dentry *d, *n;
 	unsigned int nr_subdirs;
 	int ret;
 
@@ -1370,8 +1370,16 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir)
 	}
 
 	nr_subdirs = 0;
-	list_for_each_entry(d, &dir->i_subdirs, d_child)
+	list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) {
+		if (cfg.c_overlay_strip && erofs_inode_is_whiteout(d->inode)) {
+			erofs_dbg("strip whiteout %s", d->inode->i_srcpath);
+			list_del(&d->d_child);
+			erofs_d_invalidate(d);
+			free(d);
+			continue;
+		}
 		++nr_subdirs;
+	}
 
 	ret = erofs_prepare_dir_layout(dir, nr_subdirs);
 	if (ret)
diff --git a/lib/rebuild.c b/lib/rebuild.c
index b254dea..1d8c7bc 100644
--- a/lib/rebuild.c
+++ b/lib/rebuild.c
@@ -200,10 +200,12 @@ static int erofs_rebuild_fill_inode(struct erofs_inode *inode)
 
 	switch (inode->i_mode & S_IFMT) {
 	case S_IFCHR:
-		if (erofs_inode_is_whiteout(inode)) {
-			ret = erofs_set_origin_xattr(inode->i_parent);
-			if (ret)
-				break;
+		if (!cfg.c_overlay_strip) {
+			if (erofs_inode_is_whiteout(inode)) {
+				ret = erofs_set_origin_xattr(inode->i_parent);
+				if (ret)
+					break;
+			}
 		}
 		/* fallthrough */
 	case S_IFBLK:
diff --git a/lib/tar.c b/lib/tar.c
index b75b723..3389bf6 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -740,13 +740,15 @@ new_inode:
 		inode->i_mode = (inode->i_mode & ~S_IFMT) | S_IFCHR;
 		inode->u.i_rdev = EROFS_WHITEOUT_DEV;
 		d->type = EROFS_FT_CHRDEV;
-		/*
-		 * Mark parent directory as copied-up to avoid exposing
-		 * whiteouts in the overlayfs mount.
-		 */
-		ret = erofs_set_origin_xattr(inode->i_parent);
-		if (ret)
-			goto out;
+		if (!cfg.c_overlay_strip) {
+			/*
+			 * Mark parent directory as copied-up to avoid exposing
+			 * whiteouts in the overlayfs mount.
+			 */
+			ret = erofs_set_origin_xattr(inode->i_parent);
+			if (ret)
+				goto out;
+		}
 	} else {
 		inode->i_mode = st.st_mode;
 		if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
diff --git a/lib/xattr.c b/lib/xattr.c
index feafe96..d7f4e3a 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -601,6 +601,11 @@ int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
 			inode->opaque = true;
 		}
 
+		if (cfg.c_overlay_strip && !strcmp(key, OVL_XATTR_ORIGIN)) {
+			ret = 0;
+			continue;
+		}
+
 		ret = erofs_getxattr(inode, key, NULL, 0);
 		if (ret < 0)
 			goto out;
diff --git a/mkfs/main.c b/mkfs/main.c
index b51ef37..4af07f2 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -57,6 +57,7 @@ static struct option long_options[] = {
 	{"gid-offset", required_argument, NULL, 17},
 	{"tar", optional_argument, NULL, 20},
 	{"aufs", no_argument, NULL, 21},
+	{"overlay-strip", required_argument, NULL, 22},
 	{"mount-point", required_argument, NULL, 512},
 	{"xattr-prefix", required_argument, NULL, 19},
 #ifdef WITH_ANDROID
@@ -121,6 +122,7 @@ static void usage(void)
 	      " --preserve-mtime      keep per-file modification time strictly\n"
 	      " --aufs                replace aufs special files with overlayfs metadata\n"
 	      " --tar=[fi]            generate an image from tarball(s)\n"
+	      " --overlay-strip=[01]  strip overlayfs stuff e.g. whiteout in tarfs or rebuild mode\n"
 	      " --quiet               quiet execution (do not write anything to standard output.)\n"
 #ifndef NDEBUG
 	      " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
@@ -538,6 +540,16 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 21:
 			erofstar.aufs = true;
 			break;
+		case 22:
+			if (!strcmp(optarg, "0"))
+				cfg.c_overlay_strip = false;
+			else if (!strcmp(optarg, "1"))
+				cfg.c_overlay_strip = true;
+			else {
+				erofs_err("invalid overlay-strip %s", optarg);
+				return -EINVAL;
+			}
+			break;
 		case 1:
 			usage();
 			exit(0);
-- 
2.19.1.6.gb485710b

Re: [PATCH v6 04/11] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper

[Gao Xiang]

On 2023/9/5 18:02, Jingbo Xu wrote:
> Add erofs_read_xattrs_from_disk() helper reading extended attributes
> from disk, add checking if it's an opaque directory.
> 
> Move all xattr name related macros to xattr.c and introduce
> erofs_set_opaque_xattr() helper to hide all these details.
> 
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>

Add erofs_read_xattrs_from_disk() helper to read extended attributes
from disk as well as a helper to check if it's an opaque directory.

Move all xattr name related macros to xattr.c and introduce
erofs_set_opaque_xattr() helper to hide all these details.


I will update this manually, otherwise it looks good to me,

Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>

Thanks,
Gao Xiang

Re: [PATCH v6 05/11] erofs-utils: lib: add erofs_insert_ihash() helper

[Gao Xiang]

On 2023/9/5 18:02, Jingbo Xu wrote:
> Add erofs_insert_ihash() helper inserting inode into inode hash table.
> 
> Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the
> header file.
> 
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>

Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>

Thanks,
Gao Xiang

Re: [PATCH v6 06/11] erofs-utils: lib: add erofs_rebuild_get_dentry() helper

[Gao Xiang]

On 2023/9/5 18:02, Jingbo Xu wrote:
> Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry().
> 
> Also make `whout` and 'opq' parameter optional when `aufs` is false.
> 
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> ---
>   include/erofs/rebuild.h |  19 +++++++
>   lib/Makefile.am         |   3 +-
>   lib/rebuild.c           | 117 ++++++++++++++++++++++++++++++++++++++++
>   lib/tar.c               | 109 ++-----------------------------------
>   4 files changed, 141 insertions(+), 107 deletions(-)
>   create mode 100644 include/erofs/rebuild.h
>   create mode 100644 lib/rebuild.c
> 
> diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
> new file mode 100644
> index 0000000..92873c9
> --- /dev/null
> +++ b/include/erofs/rebuild.h
> @@ -0,0 +1,19 @@
> +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
> +#ifndef __EROFS_REBUILD_H
> +#define __EROFS_REBUILD_H
> +
> +#ifdef __cplusplus
> +extern "C"
> +{
> +#endif
> +
> +#include "internal.h"
> +
> +struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
> +		char *path, bool aufs, bool *whout, bool *opq);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif
> diff --git a/lib/Makefile.am b/lib/Makefile.am
> index 3e09516..8a45bd6 100644
> --- a/lib/Makefile.am
> +++ b/lib/Makefile.am
> @@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
>         $(top_srcdir)/include/erofs/compress_hints.h \
>         $(top_srcdir)/include/erofs/fragments.h \
>         $(top_srcdir)/include/erofs/xxhash.h \
> +      $(top_srcdir)/include/erofs/rebuild.h \
>         $(top_srcdir)/lib/liberofs_private.h
>   
>   noinst_HEADERS += compressor.h
> @@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
>   		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
>   		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
>   		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
> -		      block_list.c xxhash.c
> +		      block_list.c xxhash.c rebuild.c
>   
>   liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
>   if ENABLE_LZ4
> diff --git a/lib/rebuild.c b/lib/rebuild.c
> new file mode 100644
> index 0000000..7aaa071
> --- /dev/null
> +++ b/lib/rebuild.c
> @@ -0,0 +1,117 @@
> +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
> +#define _GNU_SOURCE
> +#include <unistd.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include "erofs/print.h"
> +#include "erofs/inode.h"
> +#include "erofs/rebuild.h"
> +#include "erofs/internal.h"
> +
> +#ifdef HAVE_LINUX_AUFS_TYPE_H
> +#include <linux/aufs_type.h>
> +#else
> +#define AUFS_WH_PFX		".wh."
> +#define AUFS_DIROPQ_NAME	AUFS_WH_PFX ".opq"
> +#define AUFS_WH_DIROPQ		AUFS_WH_PFX AUFS_DIROPQ_NAME
> +#endif
> +
> +static struct erofs_dentry *erofs_rebuild_mkdir(struct erofs_inode *dir,
> +						const char *s)
> +{
> +	struct erofs_inode *inode;
> +	struct erofs_dentry *d;
> +
> +	inode = erofs_new_inode();
> +	if (IS_ERR(inode))
> +		return ERR_CAST(inode);
> +
> +	inode->i_mode = S_IFDIR | 0755;
> +	inode->i_parent = dir;
> +	inode->i_uid = getuid();
> +	inode->i_gid = getgid();
> +	inode->i_mtime = inode->sbi->build_time;
> +	inode->i_mtime_nsec = inode->sbi->build_time_nsec;
> +	erofs_init_empty_dir(inode);
> +
> +	d = erofs_d_alloc(dir, s);
> +	if (!IS_ERR(d)) {
> +		d->type = EROFS_FT_DIR;
> +		d->inode = inode;
> +	}
> +	return d;
> +}
> +
> +struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
> +		char *path, bool aufs, bool *whout, bool *opq)
> +{
> +	struct erofs_dentry *d = NULL;
> +	unsigned int len = strlen(path);
> +	char *s = path;
> +
> +	if (aufs) {
> +		*whout = false;
> +		*opq = false;
> +	}

I'd suggest adding a boolean dumb instead here, otherwise
it looks good to me.

Thanks,
Gao Xiang