[PATCH v3 4/4] erofs-utils: mkfs: support rebuild mode for compressed inodes

[Gao Xiang <xiang@kernel.org>]

Add support for generating a meta-only multidev manifest image with
an overlayfs-like merged tree from multiple compressed EROFS images:

 $ mkfs.erofs -zlz4 -Enoinline_data layerA.erofs layerA/
 $ mkfs.erofs -zlz4 -Enoinline_data layerB.erofs layerB/
 $ mkfs.erofs -zlz4 fsmeta.erofs layerA.erofs layerB.erofs
 # mount -t erofs \
	-odevice=layerA.erofs,device=layerB.erofs
	fsmeta.erofs mnt/

Note that `-Efragments` is not yet supported: it needs further
userspace development.

Cc: Jonathan Calmels <jcalmels@nvidia.com>
Signed-off-by: Gao Xiang <xiang@kernel.org>
---
 lib/compress.c | 145 ++++++++++++++++++++++++++++++++++++++++++++++++-
 lib/inode.c    |  18 ++++--
 lib/rebuild.c  |  29 +++++++++-
 3 files changed, 186 insertions(+), 6 deletions(-)

diff --git a/lib/compress.c b/lib/compress.c
index cda138f..8305543 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1269,11 +1269,36 @@ int z_erofs_drop_inline_pcluster(struct erofs_inode *inode)
 	int err;
 
 	inode->z_advise &= ~Z_EROFS_ADVISE_INLINE_PCLUSTER;
+	ei = list_last_entry(&ctx->extents, struct z_erofs_extent_item, list);
+	if (!inode->bh_data) {
+		struct erofs_buffer_head *bh;
+
+		bh = erofs_balloc(sbi->bmgr, DATA, 0, 0);
+		if (IS_ERR(bh))
+			return PTR_ERR(bh);
+
+		bh->op = &erofs_skip_write_bhops;
+		inode->bh_data = bh;
+		err = erofs_mapbh(NULL, bh->block);
+		if (err < 0) {
+			DBG_BUGON(1);
+			return err;
+		}
+		inode->lazy_tailblock = false;
+		inode->bh_data = bh;
+		ei->e.device_id = 0;
+		ei->e.pstart = erofs_btell(bh, false);
+		if (inode->datalayout == EROFS_INODE_COMPRESSED_COMPACT) {
+			inode->datalayout = EROFS_INODE_COMPRESSED_FULL;
+			inode->z_advise &= ~Z_EROFS_ADVISE_COMPACTED_2B;
+			inode->extent_isize = EROFS_FULL_INDEXES_SZ(inode);
+		}
+	}
+
 	if (inode->idata_type == EROFS_IDATA_TYPE_RAW ||
 	    (inode->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER))
 		return 0;
 
-	ei = list_last_entry(&ctx->extents, struct z_erofs_extent_item, list);
 	DBG_BUGON(ei->e.raw);
 
 	rawsz = ei->e.length;
@@ -2455,3 +2480,121 @@ int z_erofs_compress_exit(struct erofs_sb_info *sbi)
 	free(sbi->zmgr);
 	return 0;
 }
+
+int z_erofs_rebuild_load_metadata(struct erofs_sb_info *dst_sbi,
+				  struct erofs_inode *inode)
+{
+	struct erofs_sb_info *sbi = inode->sbi;
+	struct erofs_map_blocks map = {
+		.buf = __EROFS_BUF_INITIALIZER,
+		.m_la = 0,
+	};
+	struct z_erofs_extent_item *ei, *n;
+	erofs_off_t pend = EROFS_NULL_ADDR;
+	bool consecutive = true;
+	LIST_HEAD(extents);
+	int err, device_id = -1;
+
+	if (sbi->blkszbits != dst_sbi->blkszbits) {
+		erofs_err("filesystem block size mismatch: %d for image %d, target %d",
+			  erofs_blksiz(sbi), sbi->dev, erofs_blksiz(dst_sbi));
+		return -EOPNOTSUPP;
+	}
+
+	while (map.m_la < inode->i_size) {
+		struct erofs_map_dev mdev;
+		bool raw;
+
+		err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_FIEMAP);
+		if (err)
+			goto err_out;
+
+		mdev = (struct erofs_map_dev) {
+			.m_deviceid = map.m_deviceid,
+			.m_pa = map.m_pa,
+		};
+		err = erofs_map_dev(sbi, &mdev);
+		if (err)
+			goto err_out;
+
+		raw = (map.m_algorithmformat >= Z_EROFS_COMPRESSION_MAX);
+
+		ei = malloc(sizeof(*ei));
+		if (!ei) {
+			err = -ENOMEM;
+			goto err_out;
+		}
+
+		if (map.m_flags & __EROFS_MAP_FRAGMENT) {
+			DBG_BUGON(!(inode->z_advise &
+					Z_EROFS_ADVISE_FRAGMENT_PCLUSTER));
+			err = -EOPNOTSUPP;
+			goto err_free_ei;
+//			inode->fragment_size = map.m_llen;
+//			DBG_BUGON(inode->fragmentoff != map.m_pa);
+		} else if (map.m_flags & EROFS_MAP_META) {
+			DBG_BUGON(inode->idata_size != map.m_plen);
+			inode->idata = malloc(inode->idata_size);
+			if (!inode->idata) {
+				err = -ENOMEM;
+				goto err_free_ei;
+			}
+			err = erofs_dev_read(sbi, 0, inode->idata, mdev.m_pa,
+					     inode->idata_size);
+			if (err)
+				goto err_free_ei;
+			ei->e = (struct z_erofs_inmem_extent) {
+				.length = map.m_llen,
+				.plen = erofs_blksiz(sbi),
+				.pstart = (pend != EROFS_NULL_ADDR ? pend : 0),
+				.device_id = inode->dev,	/* FIXME! */
+				.raw = raw,
+				.inlined = true,
+			};
+
+			inode->idata_type = (raw ? EROFS_IDATA_TYPE_RAW :
+				EROFS_IDATA_TYPE_COMPRESSED_DEFAULT);
+			DBG_BUGON(map.m_la + map.m_llen != inode->i_size);
+		} else {
+			ei->e = (struct z_erofs_inmem_extent) {
+				.length = map.m_llen,
+				.plen = map.m_plen,
+				.pstart = mdev.m_pa,
+				.device_id = inode->dev,	/* FIXME! */
+				.partial = map.m_flags & EROFS_MAP_PARTIAL_REF,
+				.raw = raw,
+				.inlined = false,
+			};
+			if (pend != EROFS_NULL_ADDR && (pend != ei->e.pstart ||
+						device_id != ei->e.device_id))
+				consecutive = false;
+			pend = ei->e.pstart + ei->e.plen;
+			device_id = ei->e.device_id;
+		}
+
+		if (!raw) {
+			ei->e.algofmt = map.m_algorithmformat;
+			if (!(dst_sbi->available_compr_algs & (1 << ei->e.algofmt))) {
+				err = -EOPNOTSUPP;
+				goto err_free_ei;
+			}
+		}
+		init_list_head(&ei->list);
+		list_add_tail(&ei->list, &extents);
+		map.m_la += map.m_llen;
+	}
+	inode->z_lclusterbits = sbi->blkszbits;
+	err = z_erofs_prepare_layout(inode, &extents, consecutive, false);
+	if (err)
+		goto err_out;
+	return 0;
+
+err_free_ei:
+	free(ei);
+err_out:
+	list_for_each_entry_safe(ei, n, &extents, list) {
+		list_del(&ei->list);
+		free(ei);
+	}
+	return err;
+}
diff --git a/lib/inode.c b/lib/inode.c
index 1e497a0..ec96556 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -998,6 +998,16 @@ static bool erofs_inode_need_48bit(struct erofs_inode *inode)
 	return false;
 }
 
+static unsigned int erofs_inode_meta_size(struct erofs_inode *inode)
+{
+	unsigned int inodesize;
+
+	inodesize = inode->inode_isize + inode->xattr_isize;
+	if (inode->extent_isize)
+		inodesize = roundup(inodesize, 8) + inode->extent_isize;
+	return inodesize;
+}
+
 static int erofs_prepare_inode_buffer(struct erofs_importer *im,
 				      struct erofs_inode *inode)
 {
@@ -1017,9 +1027,7 @@ static int erofs_prepare_inode_buffer(struct erofs_importer *im,
 			inode->inode_isize =
 				sizeof(struct erofs_inode_extended);
 	}
-	inodesize = inode->inode_isize + inode->xattr_isize;
-	if (inode->extent_isize)
-		inodesize = roundup(inodesize, 8) + inode->extent_isize;
+	inodesize = erofs_inode_meta_size(inode);
 
 	if (!erofs_is_special_identifier(inode->i_srcpath) && sbi->mxgr)
 		inode->in_metabox = true;
@@ -1053,6 +1061,7 @@ static int erofs_prepare_inode_buffer(struct erofs_importer *im,
 			ret = z_erofs_drop_inline_pcluster(inode);
 			if (ret)
 				return ret;
+			inodesize = erofs_inode_meta_size(inode);
 		} else {
 			inode->datalayout = EROFS_INODE_FLAT_PLAIN;
 		}
@@ -1069,7 +1078,8 @@ noinline:
 		return PTR_ERR(bh);
 	} else if (inode->idata_size) {
 		if (is_inode_layout_compression(inode)) {
-			DBG_BUGON(!params->ztailpacking);
+			DBG_BUGON(!erofs_sb_has_ztailpacking(sbi) &&
+				  !params->ztailpacking);
 			erofs_dbg("Inline %scompressed data (%u bytes) to %s",
 				  inode->idata_type == EROFS_IDATA_TYPE_RAW ? "un": "",
 				  inode->idata_size, inode->i_srcpath);
diff --git a/lib/rebuild.c b/lib/rebuild.c
index 38ced3c..ebac1c5 100644
--- a/lib/rebuild.c
+++ b/lib/rebuild.c
@@ -152,7 +152,7 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
 	return d;
 }
 
-static int erofs_rebuild_write_blob_index(struct erofs_sb_info *dst_sb,
+static int erofs_rebuild_load_chunk_index(struct erofs_sb_info *dst_sb,
 					  struct erofs_inode *inode)
 {
 	int ret;
@@ -222,6 +222,17 @@ err:
 	return ret;
 }
 
+static int erofs_rebuild_write_blob_index(struct erofs_sb_info *dst_sb,
+					  struct erofs_inode *inode)
+{
+	int z_erofs_rebuild_load_metadata(struct erofs_sb_info *dst_sb,
+					  struct erofs_inode *inode);
+
+	if (is_inode_layout_compression(inode))
+		return z_erofs_rebuild_load_metadata(dst_sb, inode);
+	return erofs_rebuild_load_chunk_index(dst_sb, inode);
+}
+
 static int erofs_rebuild_write_full_data(struct erofs_inode *inode)
 {
 	struct erofs_sb_info *src_sbi = inode->sbi;
@@ -497,6 +508,7 @@ out:
 int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi,
 			    enum erofs_rebuild_datamode mode)
 {
+	struct erofs_sb_info *dst_sbi = root->sbi;
 	struct erofs_inode inode = {};
 	struct erofs_rebuild_dir_context ctx;
 	char uuid_str[37];
@@ -514,6 +526,21 @@ int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi,
 		return ret;
 	}
 
+	if (mode == EROFS_REBUILD_DATA_BLOB_INDEX) {
+		if (erofs_sb_has_ztailpacking(sbi))
+			erofs_sb_set_ztailpacking(dst_sbi);
+		if (erofs_sb_has_dedupe(sbi))
+			erofs_sb_set_dedupe(dst_sbi);
+		if (erofs_sb_has_big_pcluster(sbi) &&
+		    !erofs_sb_has_big_pcluster(dst_sbi)) {
+			erofs_err("failed to load tree from image %d with big pclusters",
+				  sbi->dev);
+			return -EOPNOTSUPP;
+		}
+		if (erofs_sb_has_48bit(sbi))
+			erofs_sb_set_48bit(dst_sbi);
+	}
+
 	inode.nid = sbi->root_nid;
 	inode.sbi = sbi;
 	ret = erofs_read_inode_from_disk(&inode);
-- 
2.47.3