[RFC 4/7] erofs-utils: mkfs: add manifest input mode

[Wenyuan Xu <sereno.xu@deepseek.com>]

Add --manifest=FILE to build an inode tree from explicit path, payload,
metadata and xattr records without scanning the source directory.
Regular file payloads are resolved relative to the source directory.

Support escaped strings, binary xattr values, hard links and special
files. Reject duplicate paths and xattr keys, require parents before
children, and report malformed records with their line and field.

Build shared xattrs from the loaded tree and mark directories containing
whiteouts before passing the tree to the importer.

Assisted-by: LLM
Signed-off-by: Wenyuan Xu <sereno.xu@deepseek.com>
---
 include/erofs/manifest.h |  17 ++
 lib/Makefile.am          |   3 +-
 lib/manifest.c           | 443 +++++++++++++++++++++++++++++++++++++++
 mkfs/main.c              |  35 ++++
 4 files changed, 497 insertions(+), 1 deletion(-)
 create mode 100644 include/erofs/manifest.h
 create mode 100644 lib/manifest.c

diff --git a/include/erofs/manifest.h b/include/erofs/manifest.h
new file mode 100644
index 0000000..dce87d3
--- /dev/null
+++ b/include/erofs/manifest.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR MIT */
+#ifndef __EROFS_MANIFEST_H
+#define __EROFS_MANIFEST_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+struct erofs_importer;
+int erofs_manifest_load(struct erofs_importer *im, const char *filename);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 0a82915..eb44d67 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -20,6 +20,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/xattr.h \
       $(top_srcdir)/include/erofs/compress_hints.h \
       $(top_srcdir)/include/erofs/importer.h \
+      $(top_srcdir)/include/erofs/manifest.h \
       $(top_srcdir)/lib/liberofs_base64.h \
       $(top_srcdir)/lib/liberofs_cache.h \
       $(top_srcdir)/lib/liberofs_chunk.h \
@@ -44,7 +45,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      compress_hints.c sha256.c blobchunk.c dir.c \
 		      fragments.c dedupe.c uuid_unparse.c uuid.c tar.c \
 		      block_list.c rebuild.c diskbuf.c bitops.c dedupe_ext.c \
-		      vmdk.c metabox.c global.c importer.c base64.c
+		      vmdk.c metabox.c global.c importer.c base64.c manifest.c
 
 liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
 liberofs_la_LIBADD = $(LIBEROFS_LIBS)
diff --git a/lib/manifest.c b/lib/manifest.c
new file mode 100644
index 0000000..bf8eb8c
--- /dev/null
+++ b/lib/manifest.c
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+#define _GNU_SOURCE
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#ifdef HAVE_SYS_SYSMACROS_H
+#include <sys/sysmacros.h>
+#endif
+#include "erofs/importer.h"
+#include "erofs/inode.h"
+#include "erofs/xattr.h"
+#include "erofs/print.h"
+#include "erofs/manifest.h"
+
+static int erofs_manifest_hex(unsigned char c)
+{
+	if (c >= '0' && c <= '9')
+		return c - '0';
+	if (c >= 'a' && c <= 'f')
+		return c - 'a' + 10;
+	if (c >= 'A' && c <= 'F')
+		return c - 'A' + 10;
+	return -1;
+}
+
+/* Decode in place, retaining the length for binary xattr values. */
+static int erofs_manifest_decode(char *s, bool binary, size_t *len)
+{
+	char *p = s, *q = s;
+	int a, b;
+
+	while (*p) {
+		unsigned char c = *p++;
+
+		if (c == '\\') {
+			/*
+			 * A space terminates the token before decoding, so
+			 * \x or \xX immediately followed by a space is invalid.
+			 */
+			if (p[0] != 'x' || !p[1] || !p[2])
+				return -EINVAL;
+			a = erofs_manifest_hex(p[1]);
+			b = erofs_manifest_hex(p[2]);
+			if (a < 0 || b < 0)
+				return -EINVAL;
+			c = (a << 4) | b;
+			p += 3;
+		} else if (c <= ' ' || c == 127) {
+			return -EINVAL;
+		}
+		/* \x00 is invalid except in binary xattr values. */
+		if (!c && !binary)
+			return -EINVAL;
+		*q++ = c;
+	}
+	*len = q - s;
+	*q = '\0';
+	return 0;
+}
+
+static int erofs_manifest_number(const char *s, unsigned int base,
+				 u64 max, u64 *v)
+{
+	u64 n = 0;
+
+	if (!*s)
+		return -EINVAL;
+	for (; *s; ++s) {
+		unsigned int d = (unsigned char)*s - '0';
+
+		if (d >= base || n > (max - d) / base)
+			return -EINVAL;
+		n = n * base + d;
+	}
+	*v = n;
+	return 0;
+}
+
+/*
+ * Validate and resolve each component once, without modifying the path.
+ * Return NULL for a missing final entry, or an error for a missing parent.
+ * The empty component under sentinel represents the root directory.
+ */
+static struct erofs_inode *erofs_manifest_lookup(struct erofs_inode *sentinel,
+		const char *path, struct erofs_inode **parent, char *name)
+{
+	struct erofs_inode *dir = sentinel;
+	const char *p = path, *component = "";
+	size_t len = 0;
+
+	if (*p != '/')
+		return ERR_PTR(-EINVAL);
+	while (*p == '/')
+		++p;
+	for (;;) {
+		struct erofs_dentry *d;
+		struct erofs_inode *next = NULL;
+
+		list_for_each_entry(d, &dir->i_subdirs, d_child) {
+			if (d->namelen == len && !memcmp(d->name, component, len)) {
+				next = d->inode;
+				break;
+			}
+		}
+		if (!*p) {
+			if (parent)
+				*parent = dir;
+			if (name) {
+				memcpy(name, component, len);
+				name[len] = '\0';
+			}
+			return next;
+		}
+		if (!next)
+			return ERR_PTR(-ENOENT);
+		if (!S_ISDIR(next->i_mode))
+			return ERR_PTR(-ENOTDIR);
+		dir = next;
+		component = p;
+		while (*p && *p != '/')
+			++p;
+		len = p - component;
+		if (len > EROFS_NAME_LEN ||
+		    (len == 1 && component[0] == '.') ||
+		    (len == 2 && component[0] == '.' && component[1] == '.'))
+			return ERR_PTR(-EINVAL);
+		while (*p == '/')
+			++p;
+	}
+}
+
+static int erofs_manifest_xattrs(struct erofs_inode *inode, char *rest)
+{
+	char *token;
+	int ret;
+
+	while ((token = strsep(&rest, " "))) {
+		char *eq;
+		size_t len, size;
+		unsigned int index, prefixlen;
+
+		if (!*token)
+			continue;
+		eq = strchr(token, '=');
+		if (!eq || strchr(eq + 1, '='))
+			return -EINVAL;
+		*eq++ = '\0';
+		if (erofs_manifest_decode(token, false, &len) || !len || len > 255 ||
+		    erofs_manifest_decode(eq, true, &size) || size > UINT16_MAX)
+			return -EINVAL;
+		if (erofs_inode_has_xattr(inode, token))
+			return -EEXIST;
+		if (!erofs_xattr_prefix_matches(token, &index, &prefixlen))
+			return -EOPNOTSUPP;
+		ret = erofs_vfs_setxattr(inode, token, eq, size);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+enum erofs_manifest_field {
+	EROFS_MANIFEST_PATH,
+	EROFS_MANIFEST_PAYLOAD,
+	EROFS_MANIFEST_SIZE,
+	EROFS_MANIFEST_MODE,
+	EROFS_MANIFEST_UID,
+	EROFS_MANIFEST_GID,
+	EROFS_MANIFEST_RDEV,
+	EROFS_MANIFEST_MTIME,
+	EROFS_MANIFEST_FIELD_COUNT,
+};
+
+static int erofs_manifest_parse_record(struct erofs_importer *im,
+				 struct erofs_inode *sentinel, char *line)
+{
+	const char *basedir = im->params->source;
+	struct erofs_inode *inode = NULL, *parent = NULL;
+	struct erofs_dentry *d;
+	char *f[EROFS_MANIFEST_FIELD_COUNT], *token, *dot;
+	char name[EROFS_NAME_LEN + 1];
+	bool hardlink, is_root;
+	u64 mode, size = 0, uid, gid, rdev = 0, sec, nsec;
+	size_t len;
+	int i, ret = -EINVAL;
+
+	for (i = 0; i < EROFS_MANIFEST_FIELD_COUNT; ++i) {
+		/*
+		 * strsep() yields empty tokens for leading or repeated spaces.
+		 * skip in this case.
+		 */
+		do {
+			token = strsep(&line, " ");
+		} while (token && !*token);
+		if (!token)
+			return -EINVAL;
+		f[i] = token;
+	}
+
+	/* PATH */
+	if (erofs_manifest_decode(f[EROFS_MANIFEST_PATH], false, &len))
+		return -EINVAL;
+	inode = erofs_manifest_lookup(sentinel, f[EROFS_MANIFEST_PATH],
+				      &parent, name);
+	if (IS_ERR(inode))
+		return PTR_ERR(inode);
+	if (inode)
+		return -EEXIST;
+	is_root = parent == sentinel;
+
+	/* MODE */
+	hardlink = f[EROFS_MANIFEST_MODE][0] == '@';
+	if (hardlink) {
+		if (is_root)
+			return -EINVAL;
+
+		/* PAYLOAD */
+		if (erofs_manifest_decode(f[EROFS_MANIFEST_PAYLOAD], false, &len))
+			return -EINVAL;
+
+		inode = erofs_manifest_lookup(sentinel, f[EROFS_MANIFEST_PAYLOAD],
+					      NULL, NULL);
+		if (IS_ERR(inode))
+			return PTR_ERR(inode);
+		if (!inode || S_ISDIR(inode->i_mode))
+			return -EINVAL;
+		if (inode->i_nlink == UINT_MAX)
+			return -EOVERFLOW;
+
+		inode = erofs_igrab(inode);
+		++inode->i_nlink;
+		goto link;
+	}
+
+	/* MODE */
+	if (erofs_manifest_number(f[EROFS_MANIFEST_MODE], 8, UINT16_MAX, &mode))
+		return -EINVAL;
+	if (is_root && !S_ISDIR(mode))
+		return -EINVAL;
+	switch (mode & S_IFMT) {
+	case S_IFREG:
+	case S_IFLNK:
+		/* SIZE */
+		if (erofs_manifest_number(f[EROFS_MANIFEST_SIZE], 10,
+					  INT64_MAX, &size))
+			return -EINVAL;
+		break;
+	case S_IFCHR:
+	case S_IFBLK:
+	case S_IFIFO:
+	case S_IFSOCK:
+		/* RDEV */
+		if (erofs_manifest_number(f[EROFS_MANIFEST_RDEV], 10,
+					  UINT64_MAX, &rdev) ||
+		    (u64)(dev_t)rdev != rdev ||
+		    major((dev_t)rdev) > 0xfff || minor((dev_t)rdev) > 0xfffff)
+			return -EINVAL;
+		break;
+	case S_IFDIR:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* UID */
+	if (erofs_manifest_number(f[EROFS_MANIFEST_UID], 10, UINT32_MAX, &uid))
+		return -EINVAL;
+
+	/* GID */
+	if (erofs_manifest_number(f[EROFS_MANIFEST_GID], 10, UINT32_MAX, &gid))
+		return -EINVAL;
+
+	/* MTIME */
+	dot = strchr(f[EROFS_MANIFEST_MTIME], '.');
+	if (!dot)
+		return -EINVAL;
+	*dot++ = '\0';
+	if (erofs_manifest_number(f[EROFS_MANIFEST_MTIME], 10, INT64_MAX, &sec) ||
+	    erofs_manifest_number(dot, 10, 999999999, &nsec))
+		return -EINVAL;
+
+	inode = erofs_new_inode(im->sbi);
+	if (IS_ERR(inode))
+		return PTR_ERR(inode);
+	inode->dev = sentinel->dev;
+	inode->i_mode = mode;
+	inode->i_uid = uid;
+	inode->i_gid = gid;
+	inode->i_size = size;
+	inode->i_mtime = sec;
+	inode->i_mtime_nsec = nsec;
+	inode->i_nlink = S_ISDIR(mode) ? 2 : 1;
+	inode->i_parent = parent;
+	if (S_ISCHR(mode) || S_ISBLK(mode) || S_ISFIFO(mode) || S_ISSOCK(mode))
+		inode->u.i_rdev = erofs_new_encode_dev(rdev);
+
+	/* PAYLOAD */
+	if (S_ISREG(mode)) {
+		struct stat st;
+		int fd;
+
+		if (erofs_manifest_decode(f[EROFS_MANIFEST_PAYLOAD], false, &len))
+			goto out;
+		if (f[EROFS_MANIFEST_PAYLOAD][0] != '/')
+			goto out;
+		if (asprintf(&inode->i_srcpath, "%s%s", basedir,
+			     f[EROFS_MANIFEST_PAYLOAD]) < 0) {
+			ret = -ENOMEM;
+			goto out;
+		}
+		fd = open(inode->i_srcpath, O_RDONLY | O_NONBLOCK | O_BINARY);
+		if (fd < 0) {
+			ret = -errno;
+			goto out;
+		}
+		ret = fstat(fd, &st) ? -errno : 0;
+		close(fd);
+		if (ret)
+			goto out;
+		ret = -EINVAL;
+		if (!S_ISREG(st.st_mode))
+			goto out;
+		if (st.st_size != size)
+			goto out;
+		inode->datasource = EROFS_INODE_DATA_SOURCE_LOCALPATH;
+	} else {
+		/* Use the image path for progress reporting and diagnostics. */
+		if (asprintf(&inode->i_srcpath, "%s%s", basedir,
+			     f[EROFS_MANIFEST_PATH]) < 0) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		if (S_ISLNK(mode)) {
+			if (erofs_manifest_decode(f[EROFS_MANIFEST_PAYLOAD], false, &len))
+				goto out;
+			if (len != size)
+				goto out;
+			inode->i_link = strdup(f[EROFS_MANIFEST_PAYLOAD]);
+			if (!inode->i_link) {
+				ret = -ENOMEM;
+				goto out;
+			}
+		}
+	}
+
+	/* XATTR */
+	ret = erofs_manifest_xattrs(inode, line);
+	if (ret)
+		goto out;
+link:
+	/* PATH */
+	d = erofs_d_alloc(parent, name);
+	if (IS_ERR(d)) {
+		ret = PTR_ERR(d);
+		goto out;
+	}
+	d->inode = inode;
+	d->type = erofs_mode_to_ftype(inode->i_mode);
+	if (erofs_inode_is_whiteout(inode))
+		parent->whiteouts = true;
+	return 0;
+out:
+	erofs_iput(inode);
+	return ret;
+}
+
+/* The importer has not taken ownership of the tree on a parse error. */
+static void erofs_manifest_drop_children(struct erofs_inode *dir)
+{
+	struct erofs_dentry *d, *next;
+
+	list_for_each_entry_safe(d, next, &dir->i_subdirs, d_child) {
+		if (S_ISDIR(d->inode->i_mode))
+			erofs_manifest_drop_children(d->inode);
+		erofs_iput(d->inode);
+		list_del(&d->d_child);
+		free(d);
+	}
+}
+
+int erofs_manifest_load(struct erofs_importer *im, const char *filename)
+{
+	struct erofs_inode sentinel = { .i_mode = S_IFDIR };
+	struct erofs_dentry *d;
+	struct stat st;
+	FILE *fp = fopen(filename, "r");
+	char *line = NULL;
+	size_t capacity = 0;
+	unsigned long lineno = 0;
+	ssize_t len;
+	int ret = 0;
+
+	if (!fp) {
+		ret = -errno;
+		erofs_err("%s: cannot open manifest: %s", filename, strerror(errno));
+		return ret;
+	}
+	if (stat(im->params->source, &st)) {
+		ret = -errno;
+		erofs_err("%s: cannot stat base directory: %s",
+			  im->params->source, erofs_strerror(ret));
+		fclose(fp);
+		return ret;
+	}
+	sentinel.dev = st.st_dev;
+	init_list_head(&sentinel.i_subdirs);
+
+	while ((len = getline(&line, &capacity, fp)) >= 0) {
+		++lineno;
+		if (memchr(line, '\0', len)) {
+			ret = -EINVAL;
+			break;
+		}
+		if (len && line[len - 1] == '\n')
+			line[--len] = '\0';
+		ret = erofs_manifest_parse_record(im, &sentinel, line);
+		if (ret)
+			break;
+	}
+	if (!ret && ferror(fp))
+		ret = -EIO;
+	if (!ret && !lineno)
+		ret = -EINVAL;
+	if (ret) {
+		erofs_manifest_drop_children(&sentinel);
+		erofs_err("%s:%lu: %s", filename, lineno, erofs_strerror(ret));
+	} else {
+		/* Transfer the root out of sentinel before it leaves scope. */
+		d = list_first_entry(&sentinel.i_subdirs, struct erofs_dentry, d_child);
+		im->root = d->inode;
+		im->root->i_parent = im->root;
+		list_del(&d->d_child);
+		free(d);
+	}
+	free(line);
+	fclose(fp);
+	return ret;
+}
diff --git a/mkfs/main.c b/mkfs/main.c
index d8706d7..6686c5a 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -24,6 +24,7 @@
 #include "erofs/exclude.h"
 #include "erofs/block_list.h"
 #include "erofs/compress_hints.h"
+#include "erofs/manifest.h"
 #include "../lib/compressor.h"
 #include "../lib/liberofs_chunk.h"
 #include "../lib/liberofs_gzran.h"
@@ -105,6 +106,7 @@ static struct option long_options[] = {
 	{"MZ", optional_argument, NULL, 537},
 	{"xattr-prefix", required_argument, NULL, 538},
 	{"xattr-inode-digest", optional_argument, NULL, 539},
+	{"manifest", required_argument, NULL, 540},
 	{0, 0, 0, 0},
 };
 
@@ -212,6 +214,7 @@ static void usage(int argc, char **argv)
 		" --hard-dereference     dereference hardlinks, add links as separate inodes\n"
 		" --ignore-mtime         use build time instead of strict per-file modification time\n"
 		" --max-extent-bytes=#   set maximum decompressed extent size # in bytes\n"
+		" --manifest=FILE        build an uncompressed image from a manifest and basedir\n"
 		" --mount-point=X        X=prefix of target fs path (default: /)\n"
 		" --preserve-mtime       keep per-file modification time strictly\n"
 		" --offset=#             skip # bytes at the beginning of IMAGE.\n"
@@ -324,8 +327,10 @@ static enum {
 	EROFS_MKFS_SOURCE_S3,
 	EROFS_MKFS_SOURCE_OCI,
 	EROFS_MKFS_SOURCE_REBUILD,
+	EROFS_MKFS_SOURCE_MANIFEST,
 } source_mode;
 
+static const char *manifest_file;
 static unsigned int rebuild_src_count;
 static LIST_HEAD(rebuild_src_list);
 static u8 fixeduuid[16];
@@ -971,6 +976,7 @@ static int mkfs_parse_sources(int argc, char *argv[], int optind)
 	char *s;
 
 	switch (source_mode) {
+	case EROFS_MKFS_SOURCE_MANIFEST:
 	case EROFS_MKFS_SOURCE_LOCALDIR:
 		err = lstat((s = argv[optind++]), &st);
 		if (err) {
@@ -986,6 +992,8 @@ static int mkfs_parse_sources(int argc, char *argv[], int optind)
 				return -ENOENT;
 			}
 			erofs_set_fs_root(cfg.c_src_path);
+		} else if (source_mode == EROFS_MKFS_SOURCE_MANIFEST) {
+				return -ENOTDIR;
 		} else {
 			cfg.c_src_path = strdup(s);
 			if (!cfg.c_src_path)
@@ -1497,6 +1505,14 @@ static int mkfs_parse_options_cfg(struct erofs_importer_params *params,
 				return err;
 			}
 			break;
+		case 540:
+			if (manifest_file) {
+				erofs_err("duplicate manifest file");
+				return -EINVAL;
+			}
+			manifest_file = optarg;
+			source_mode = EROFS_MKFS_SOURCE_MANIFEST;
+			break;
 		case 'V':
 			version();
 			exit(0);
@@ -1970,6 +1986,14 @@ int main(int argc, char **argv)
 			err = PTR_ERR(root);
 			goto exit;
 		}
+	} else if (source_mode == EROFS_MKFS_SOURCE_MANIFEST)  {
+		err = erofs_xattr_flush_name_prefixes(&importer,
+						      mkfs_plain_xattr_pfx);
+		if (err) {
+			erofs_err("failed to flush long xattr prefixes: %s",
+				  erofs_strerror(err));
+			goto exit;
+		}
 	} else {
 		err = erofs_xattr_flush_name_prefixes(&importer,
 						      mkfs_plain_xattr_pfx);
@@ -1993,6 +2017,17 @@ int main(int argc, char **argv)
 		err = lstat(cfg.c_src_path, &st);
 		if (!err)
 			err = erofs_fill_inode(&importer, importer.root, &st, cfg.c_src_path);
+	} else if (source_mode == EROFS_MKFS_SOURCE_MANIFEST) {
+		err = erofs_manifest_load(&importer, manifest_file);
+		if (err)
+			goto exit;
+		root = importer.root;
+
+		err = erofs_build_shared_xattrs(&g_sbi, mkfscfg.inlinexattr_tolerance);
+		if (err) {
+			erofs_err("failed to build shared xattrs: %s", erofs_strerror(err));
+			goto exit;
+		}
 	} else if (source_mode == EROFS_MKFS_SOURCE_TAR) {
 		while (!(err = tarerofs_parse_tar(&importer, &erofstar)))
 			;
-- 
2.43.0