[PATCH v3 1/2] erofs-utils: lib: don't abort on compression fallback

[Gao Xiang <hsiangkao@linux.alibaba.com>]

From: Yifan Zhao <zhaoyifan28@huawei.com>

File-level compression fallback is control flow, not a real error.
Return an erofs-specific status code for it instead of overloading
-ENOSPC, which can also report real space failures.

Keep the global compression context reusable for that fallback while
preserving the fatal state for real errors.

Fixes: a729584ef975 ("erofs-utils: mkfs: avoid hanging if fragment is on and tmpdir is full")
Reported-by: Bastian Schmitz <bastian.schmitz@vorwerk.de>
Closes: https://github.com/erofs/erofs-utils/issues/50
Assisted-by: Codex:GPT-5.5
Signed-off-by: Yifan Zhao <zhaoyifan28@huawei.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
 include/erofs/err.h |  3 +++
 lib/compress.c      | 10 +++++++---
 lib/inode.c         |  6 +++---
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/include/erofs/err.h b/include/erofs/err.h
index 7dacc917a4c1..bf5a4e1cf9b7 100644
--- a/include/erofs/err.h
+++ b/include/erofs/err.h
@@ -53,6 +53,9 @@ static inline void * ERR_CAST(const void *ptr)
 	return (void *) ptr;
 }
 
+/* EROFS-specific error codes */
+#define EROFS_RETCODE_FALLBACK		MAX_ERRNO
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/compress.c b/lib/compress.c
index ea07409defef..0f448e400a2d 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1375,7 +1375,7 @@ int erofs_commit_compressed_file(struct z_erofs_compress_ictx *ictx,
 	    legacymetasize >= inode->i_size) {
 		z_erofs_dedupe_ext_commit(true);
 		z_erofs_dedupe_commit(true);
-		ret = -ENOSPC;
+		ret = EROFS_RETCODE_FALLBACK;
 		goto err_free_meta;
 	}
 	z_erofs_dedupe_ext_commit(false);
@@ -2031,7 +2031,11 @@ err_free_idata:
 out:
 #ifdef EROFS_MT_ENABLED
 	pthread_mutex_lock(&ictx->mutex);
-	ictx->seg_num = ret < 0 ? INT_MAX : 0;
+	if (ret < 0 && ret != EROFS_RETCODE_FALLBACK)
+		/* mark as failed to avoid further processing */
+		ictx->seg_num = INT_MAX;
+	else
+		ictx->seg_num = 0;
 	pthread_cond_signal(&ictx->cond);
 	pthread_mutex_unlock(&ictx->mutex);
 #endif
@@ -2044,7 +2048,7 @@ int erofs_begin_compress_dir(struct erofs_importer *im,
 {
 	if (!im->params->compress_dir ||
 	    inode->i_size < Z_EROFS_LEGACY_MAP_HEADER_SIZE)
-		return -ENOSPC;
+		return EROFS_RETCODE_FALLBACK;
 
 	inode->z_advise |= Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
 	erofs_sb_set_fragments(inode->sbi);
diff --git a/lib/inode.c b/lib/inode.c
index c225faa121e7..4c2d094bac7e 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1507,7 +1507,7 @@ static int erofs_mkfs_job_write_file(struct erofs_mkfs_job_ndir_ctx *ctx)
 
 	if (ctx->ictx) {
 		ret = erofs_write_compressed_file(ctx->ictx);
-		if (ret != -ENOSPC)
+		if (ret != EROFS_RETCODE_FALLBACK)
 			goto out;
 		if (lseek(ctx->fd, ctx->fpos, SEEK_SET) < 0) {
 			ret = -errno;
@@ -1594,7 +1594,7 @@ static int erofs_mkfs_create_directory(const struct erofs_mkfs_btctx *ctx,
 		inode->datalayout = EROFS_INODE_FLAT_INLINE;
 
 		ret = erofs_begin_compress_dir(ctx->im, inode);
-		if (ret && ret != -ENOSPC)
+		if (ret && ret != EROFS_RETCODE_FALLBACK)
 			return ret;
 	} else {
 		DBG_BUGON(inode->datalayout != EROFS_INODE_FLAT_PLAIN);
@@ -2391,7 +2391,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(struct erofs_importer *im,
 		ret = erofs_write_compressed_file(ictx);
 		if (!ret)
 			goto out;
-		if (ret != -ENOSPC)
+		if (ret != EROFS_RETCODE_FALLBACK)
 			 return ERR_PTR(ret);
 
 		ret = lseek(fd, 0, SEEK_SET);
-- 
2.43.5