* [PATCH 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
@ 2024-01-06 18:11 ` Yifan Zhao
2024-01-17 10:56 ` Gao Xiang
2024-01-06 18:11 ` [PATCH 3/3] erofs-utils: mkfs: reorganize logic in erofs_compressor_init() Yifan Zhao
` (9 subsequent siblings)
10 siblings, 1 reply; 15+ messages in thread
From: Yifan Zhao @ 2024-01-06 18:11 UTC (permalink / raw)
To: linux-erofs
Currently, the dictionary size for compression algorithms is fixed. This
patch allows to specify it with -zX,dictsize=<dictsize> options in
mkfs command line.
This patch also changes the way to specify compression levels. Now, the
compression level is specified with -zX,level=<level> options and could
be specified together with dictsize. The old -zX,<level> way is still
supported for compatibility.
Suggested-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
include/erofs/config.h | 10 ++--
include/erofs_fs.h | 4 ++
lib/compress.c | 45 +++++++++++++----
lib/compress_hints.c | 2 +-
lib/compressor.c | 17 ++++++-
lib/compressor.h | 6 ++-
lib/compressor_deflate.c | 26 ++++++++--
lib/compressor_liblzma.c | 33 +++++++-----
lib/config.c | 4 +-
lib/inode.c | 2 +-
lib/kite_deflate.c | 3 +-
mkfs/main.c | 106 ++++++++++++++++++++++++++++++++-------
12 files changed, 203 insertions(+), 55 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 89fe522..7dbfd1e 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -34,6 +34,12 @@ enum {
#define EROFS_MAX_COMPR_CFGS 64
+struct erofs_compr_cfg {
+ char *alg;
+ int level;
+ u32 dict_size;
+};
+
struct erofs_configure {
const char *c_version;
int c_dbg_lvl;
@@ -64,8 +70,7 @@ struct erofs_configure {
char *c_src_path;
char *c_blobdev_path;
char *c_compress_hints_file;
- char *c_compr_alg[EROFS_MAX_COMPR_CFGS];
- int c_compr_level[EROFS_MAX_COMPR_CFGS];
+ struct erofs_compr_cfg c_compr_opts[EROFS_MAX_COMPR_CFGS];
char c_force_inodeversion;
char c_force_chunkformat;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
@@ -73,7 +78,6 @@ struct erofs_configure {
u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;
u32 c_max_decompressed_extent_bytes;
- u32 c_dict_size;
u64 c_unix_timestamp;
u32 c_uid, c_gid;
const char *mount_point;
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index eba6c26..72f0ca6 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -322,6 +322,7 @@ struct z_erofs_lzma_cfgs {
u8 reserved[8];
} __packed;
+#define Z_EROFS_LZMA_DEFAULT_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
/* 6 bytes (+ length field = 8 bytes) */
@@ -330,6 +331,9 @@ struct z_erofs_deflate_cfgs {
u8 reserved[5];
} __packed;
+#define Z_EROFS_DEFLATE_DEFULT_DICT_SIZE (1U << 15)
+#define Z_EROFS_DEFLATE_MAX_DICT_SIZE (1U << 15)
+
/*
* bit 0 : COMPACTED_2B indexes (0 - off; 1 - on)
* e.g. for 4k logical cluster size, 4B if compacted 2B is off;
diff --git a/lib/compress.c b/lib/compress.c
index 216aeb4..6f074fd 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1116,10 +1116,12 @@ err_free_meta:
}
static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
- struct erofs_buffer_head *sb_bh)
+ struct erofs_buffer_head *sb_bh,
+ struct erofs_compress_cfg *ccfg)
{
struct erofs_buffer_head *bh = sb_bh;
- int ret = 0;
+ int i, ret = 0;
+ u32 dict_size = 0;
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZ4)) {
struct {
@@ -1146,13 +1148,22 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
}
#ifdef HAVE_LIBLZMA
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZMA)) {
+ for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
+ if (ccfg[i].enable &&
+ ccfg[i].algorithmtype == Z_EROFS_COMPRESSION_LZMA) {
+ dict_size = ccfg[i].handle.dict_size;
+ break;
+ }
+ }
+ DBG_BUGON(!dict_size);
+
struct {
__le16 size;
struct z_erofs_lzma_cfgs lzma;
} __packed lzmaalg = {
.size = cpu_to_le16(sizeof(struct z_erofs_lzma_cfgs)),
.lzma = {
- .dict_size = cpu_to_le32(cfg.c_dict_size),
+ .dict_size = cpu_to_le32(dict_size),
}
};
@@ -1168,6 +1179,15 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
}
#endif
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_DEFLATE)) {
+ for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
+ if (ccfg[i].enable &&
+ ccfg[i].algorithmtype == Z_EROFS_COMPRESSION_DEFLATE) {
+ dict_size = ccfg[i].handle.dict_size;
+ break;
+ }
+ }
+ DBG_BUGON(!dict_size);
+
struct {
__le16 size;
struct z_erofs_deflate_cfgs z;
@@ -1175,7 +1195,7 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
.size = cpu_to_le16(sizeof(struct z_erofs_deflate_cfgs)),
.z = {
.windowbits =
- cpu_to_le32(ilog2(cfg.c_dict_size)),
+ cpu_to_le32(ilog2(dict_size)),
}
};
@@ -1196,10 +1216,12 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
{
int i, ret;
- for (i = 0; cfg.c_compr_alg[i]; ++i) {
+ for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
struct erofs_compress *c = &erofs_ccfg[i].handle;
- ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i], cfg.c_compr_level[i]);
+ ret = erofs_compressor_init(sbi, c, cfg.c_compr_opts[i].alg,
+ cfg.c_compr_opts[i].level,
+ cfg.c_compr_opts[i].dict_size);
if (ret)
return ret;
@@ -1215,11 +1237,12 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
* if primary algorithm is empty (e.g. compression off),
* clear 0PADDING feature for old kernel compatibility.
*/
- if (!cfg.c_compr_alg[0] ||
- (cfg.c_legacy_compress && !strncmp(cfg.c_compr_alg[0], "lz4", 3)))
+ if (!cfg.c_compr_opts[0].alg ||
+ (cfg.c_legacy_compress &&
+ !strncmp(cfg.c_compr_opts[0].alg, "lz4", 3)))
erofs_sb_clear_lz4_0padding(sbi);
- if (!cfg.c_compr_alg[0])
+ if (!cfg.c_compr_opts[0].alg)
return 0;
/*
@@ -1241,7 +1264,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
}
if (erofs_sb_has_compr_cfgs(sbi))
- return z_erofs_build_compr_cfgs(sbi, sb_bh);
+ return z_erofs_build_compr_cfgs(sbi, sb_bh, erofs_ccfg);
return 0;
}
@@ -1249,7 +1272,7 @@ int z_erofs_compress_exit(void)
{
int i, ret;
- for (i = 0; cfg.c_compr_alg[i]; ++i) {
+ for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
ret = erofs_compressor_exit(&erofs_ccfg[i].handle);
if (ret)
return ret;
diff --git a/lib/compress_hints.c b/lib/compress_hints.c
index afc9f8f..8b78f80 100644
--- a/lib/compress_hints.c
+++ b/lib/compress_hints.c
@@ -125,7 +125,7 @@ int erofs_load_compress_hints(struct erofs_sb_info *sbi)
} else {
ccfg = atoi(alg);
if (ccfg >= EROFS_MAX_COMPR_CFGS ||
- !cfg.c_compr_alg[ccfg]) {
+ !cfg.c_compr_opts[ccfg].alg) {
erofs_err("invalid compressing configuration \"%s\" at line %u",
alg, line);
ret = -EINVAL;
diff --git a/lib/compressor.c b/lib/compressor.c
index 7ec51c2..5061417 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -78,7 +78,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
}
int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
- char *alg_name, int compression_level)
+ char *alg_name, int compression_level, u32 dict_size)
{
int ret, i;
@@ -106,6 +106,8 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
if (erofs_algs[i].c->setlevel) {
ret = erofs_algs[i].c->setlevel(c, compression_level);
+ if (ret)
+ return ret;
} else {
if (compression_level >= 0)
erofs_warn(
@@ -113,6 +115,19 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
compression_level, alg_name);
c->compression_level = 0;
}
+
+ if (erofs_algs[i].c->setdictsize) {
+ ret = erofs_algs[i].c->setdictsize(c, dict_size);
+ if (ret)
+ return ret;
+ } else {
+ if (dict_size)
+ erofs_warn(
+ "dictionary size %u is ignored for %s",
+ dict_size, alg_name);
+ c->dict_size = 0;
+ }
+
if (!ret) {
c->alg = &erofs_algs[i];
return 0;
diff --git a/lib/compressor.h b/lib/compressor.h
index ec5485d..d8ccf2e 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -14,10 +14,13 @@ struct erofs_compress;
struct erofs_compressor {
int default_level;
int best_level;
+ u32 default_dictsize;
+ u32 max_dictsize;
int (*init)(struct erofs_compress *c);
int (*exit)(struct erofs_compress *c);
int (*setlevel)(struct erofs_compress *c, int compression_level);
+ int (*setdictsize)(struct erofs_compress *c, u32 dict_size);
int (*compress_destsize)(const struct erofs_compress *c,
const void *src, unsigned int *srcsize,
@@ -39,6 +42,7 @@ struct erofs_compress {
unsigned int compress_threshold;
unsigned int compression_level;
+ unsigned int dict_size;
void *private_data;
};
@@ -56,7 +60,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
void *dst, unsigned int dstsize);
int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
- char *alg_name, int compression_level);
+ char *alg_name, int compression_level, u32 dict_size);
int erofs_compressor_exit(struct erofs_compress *c);
#endif
diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
index 4e5902e..c95e53e 100644
--- a/lib/compressor_deflate.c
+++ b/lib/compressor_deflate.c
@@ -46,6 +46,16 @@ static int compressor_deflate_init(struct erofs_compress *c)
static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
int compression_level)
+{
+ if (compression_level < 0)
+ compression_level = erofs_compressor_deflate.default_level;
+
+ c->compression_level = compression_level;
+ return 0;
+}
+
+static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
+ u32 dict_size)
{
void *s;
@@ -54,23 +64,31 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
c->private_data = NULL;
}
- if (compression_level < 0)
- compression_level = erofs_compressor_deflate.default_level;
+ if (dict_size > erofs_compressor_deflate.max_dictsize) {
+ erofs_err("dict size %u is too large", dict_size);
+ return -EINVAL;
+ }
+
+ if (dict_size == 0)
+ dict_size = erofs_compressor_deflate.default_dictsize;
- s = kite_deflate_init(compression_level, cfg.c_dict_size);
+ s = kite_deflate_init(c->compression_level, dict_size);
if (IS_ERR(s))
return PTR_ERR(s);
c->private_data = s;
- c->compression_level = compression_level;
+ c->dict_size = dict_size;
return 0;
}
const struct erofs_compressor erofs_compressor_deflate = {
.default_level = 1,
.best_level = 9,
+ .default_dictsize = Z_EROFS_DEFLATE_DEFULT_DICT_SIZE,
+ .max_dictsize = Z_EROFS_DEFLATE_MAX_DICT_SIZE,
.init = compressor_deflate_init,
.exit = compressor_deflate_exit,
.setlevel = erofs_compressor_deflate_setlevel,
+ .setdictsize = erofs_compressor_deflate_setdictsize,
.compress_destsize = deflate_compress_destsize,
};
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
index 0ed6f23..5c7f830 100644
--- a/lib/compressor_liblzma.c
+++ b/lib/compressor_liblzma.c
@@ -68,22 +68,28 @@ static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
if (lzma_lzma_preset(&ctx->opt, preset))
return -EINVAL;
- /* XXX: temporary hack */
- if (cfg.c_dict_size) {
- if (cfg.c_dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) {
- erofs_err("dict size %u is too large", cfg.c_dict_size);
- return -EINVAL;
- }
- ctx->opt.dict_size = cfg.c_dict_size;
- } else {
- if (ctx->opt.dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE)
- ctx->opt.dict_size = Z_EROFS_LZMA_MAX_DICT_SIZE;
- cfg.c_dict_size = ctx->opt.dict_size;
- }
c->compression_level = compression_level;
return 0;
}
+static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
+ u32 dict_size)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+
+ if (dict_size > erofs_compressor_lzma.max_dictsize) {
+ erofs_err("dict size %u is too large", dict_size);
+ return -EINVAL;
+ }
+
+ if (dict_size == 0)
+ dict_size = erofs_compressor_lzma.default_dictsize;
+
+ ctx->opt.dict_size = dict_size;
+ c->dict_size = dict_size;
+ return 0;
+}
+
static int erofs_compressor_liblzma_init(struct erofs_compress *c)
{
struct erofs_liblzma_context *ctx;
@@ -101,9 +107,12 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c)
const struct erofs_compressor erofs_compressor_lzma = {
.default_level = LZMA_PRESET_DEFAULT,
.best_level = 109,
+ .default_dictsize = Z_EROFS_LZMA_DEFAULT_DICT_SIZE,
+ .max_dictsize = Z_EROFS_LZMA_MAX_DICT_SIZE,
.init = erofs_compressor_liblzma_init,
.exit = erofs_compressor_liblzma_exit,
.setlevel = erofs_compressor_liblzma_setlevel,
+ .setdictsize = erofs_compressor_liblzma_setdictsize,
.compress_destsize = erofs_liblzma_compress_destsize,
};
#endif
diff --git a/lib/config.c b/lib/config.c
index 0cddc95..d3e8e53 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -58,8 +58,8 @@ void erofs_exit_configure(void)
free(cfg.c_img_path);
if (cfg.c_src_path)
free(cfg.c_src_path);
- for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_alg[i]; i++)
- free(cfg.c_compr_alg[i]);
+ for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_opts[i].alg; i++)
+ free(cfg.c_compr_opts[i].alg);
}
static unsigned int fullpath_prefix; /* root directory prefix length */
diff --git a/lib/inode.c b/lib/inode.c
index bcdb4b8..c6424c0 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -492,7 +492,7 @@ int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
return erofs_blob_write_chunked_file(inode, fd, fpos);
}
- if (cfg.c_compr_alg[0] && erofs_file_is_compressible(inode)) {
+ if (cfg.c_compr_opts[0].alg && erofs_file_is_compressible(inode)) {
ret = erofs_write_compressed_file(inode, fd);
if (!ret || ret != -ENOSPC)
return ret;
diff --git a/lib/kite_deflate.c b/lib/kite_deflate.c
index 8667954..2357f76 100644
--- a/lib/kite_deflate.c
+++ b/lib/kite_deflate.c
@@ -5,6 +5,7 @@
* Copyright (C) 2023, Alibaba Cloud
* Copyright (C) 2023, Gao Xiang <xiang@kernel.org>
*/
+#include "erofs/internal.h"
#include "erofs/defs.h"
#include "erofs/print.h"
#include <errno.h>
@@ -22,7 +23,7 @@ unsigned long erofs_memcmp2(const u8 *s1, const u8 *s2,
#define kite_dbg(x, ...)
#endif
-#define kHistorySize32 (1U << 15)
+#define kHistorySize32 Z_EROFS_DEFLATE_DEFULT_DICT_SIZE
#define kNumLenSymbols32 256
#define kNumLenSymbolsMax kNumLenSymbols32
diff --git a/mkfs/main.c b/mkfs/main.c
index 0517849..702eb4b 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -5,6 +5,7 @@
* Created by Li Guifu <bluce.liguifu@huawei.com>
*/
#define _GNU_SOURCE
+#include <ctype.h>
#include <time.h>
#include <sys/time.h>
#include <stdlib.h>
@@ -108,24 +109,29 @@ static void usage(int argc, char **argv)
" -b# set block size to # (# = page size by default)\n"
" -d<0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n"
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
- " -zX[,Y][:...] X=compressor (Y=compression level, optional)\n"
- " alternative compressors can be separated by colons(:)\n"
- " supported compressors and their level ranges are:\n",
+ " -zX[,level=Y] X=compressor (Y=compression level, Z=dictionary size, optional)\n"
+ " [,dictsize=Z] alternative compressors can be separated by colons(:)\n"
+ " [:...] supported compressors and their option ranges are:\n",
argv[0], EROFS_WARN);
while ((s = z_erofs_list_available_compressors(&i)) != NULL) {
- printf(" %s", s->name);
+ char * const spaces = " ";
+
+ printf("%s%s\n", spaces, s->name);
if (s->c->setlevel) {
if (!strcmp(s->name, "lzma"))
/* A little kludge to show the range as disjointed
* "0-9,100-109" instead of a continuous "0-109", and to
* state what those two subranges respectively mean. */
- printf("[<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)",
- s->c->default_level);
+ printf("%s [,level=<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)\n",
+ spaces, s->c->default_level);
else
- printf("[,<0-%i>]\t(default=%i)",
- s->c->best_level, s->c->default_level);
+ printf("%s [,level=<0-%i>]\t\t(default=%i)\n",
+ spaces, s->c->best_level, s->c->default_level);
+ }
+ if (s->c->setdictsize) {
+ printf("%s [,dictsize=<dictsize>]\t(default=%u, max=%u)\n",
+ spaces, s->c->default_dictsize, s->c->max_dictsize);
}
- putchar('\n');
}
printf(
" -C# specify the size of compress physical cluster in bytes\n"
@@ -310,20 +316,84 @@ static int mkfs_parse_compress_algs(char *algs)
char *s;
for (s = strtok(algs, ":"), i = 0; s; s = strtok(NULL, ":"), ++i) {
- const char *lv;
+ char *p, *q, *opt, *endptr;
+ struct erofs_compr_cfg *ccfg;
if (i >= EROFS_MAX_COMPR_CFGS - 1) {
erofs_err("too many algorithm types");
return -EINVAL;
}
- lv = strchr(s, ',');
- if (lv) {
- cfg.c_compr_level[i] = atoi(lv + 1);
- cfg.c_compr_alg[i] = strndup(s, lv - s);
+ ccfg = cfg.c_compr_opts + i;
+
+ ccfg->level = -1;
+ ccfg->dict_size = 0;
+
+ p = strchr(s, ',');
+ if (p) {
+ ccfg->alg = strndup(s, p - s);
+
+ /* backward compatibility */
+ if (isdigit(*(p + 1))) {
+ ccfg->level = strtol(p + 1, &endptr, 10);
+ if (*endptr && *endptr != ',') {
+ erofs_err(
+ "invalid compression level %s",
+ p + 1);
+ return -EINVAL;
+ }
+ continue;
+ }
} else {
- cfg.c_compr_level[i] = -1;
- cfg.c_compr_alg[i] = strdup(s);
+ ccfg->alg = strdup(s);
+ continue;
+ }
+
+ opt = p + 1;
+ while (opt) {
+ q = strchr(opt, ',');
+ if (q)
+ *q = '\0';
+
+ if ((p = strstr(opt, "level="))) {
+ p += strlen("level=");
+ ccfg->level = strtol(p, &endptr, 10);
+ if (*endptr && *endptr != ',') {
+ erofs_err(
+ "invalid compression level %s",
+ p);
+ return -EINVAL;
+ }
+ } else if ((p = strstr(opt, "dictsize="))) {
+ p += strlen("dictsize=");
+ ccfg->dict_size = strtoul(p, &endptr, 10);
+ switch (*endptr) {
+ case ',':
+ case '\0':
+ break;
+ case 'K':
+ case 'k':
+ ccfg->dict_size <<= 10;
+ break;
+ case 'M':
+ case 'm':
+ ccfg->dict_size <<= 20;
+ break;
+ default:
+ erofs_err(
+ "invalid compression dictsize %s",
+ p);
+ return -EINVAL;
+ }
+ } else {
+ erofs_err("invalid compression option %s", opt);
+ return -EINVAL;
+ }
+
+ if (q)
+ opt = q + 1;
+ else
+ opt = NULL;
}
}
return 0;
@@ -692,7 +762,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
cfg.c_showprogress = false;
}
- if (cfg.c_compr_alg[0] && erofs_blksiz(&sbi) != getpagesize())
+ if (cfg.c_compr_opts[0].alg && erofs_blksiz(&sbi) != getpagesize())
erofs_warn("Please note that subpage blocksize with compression isn't yet supported in kernel. "
"This compressed image will only work with bs = ps = %u bytes",
erofs_blksiz(&sbi));
@@ -1119,7 +1189,7 @@ int main(int argc, char **argv)
}
if (cfg.c_dedupe) {
- if (!cfg.c_compr_alg[0]) {
+ if (!cfg.c_compr_opts[0].alg) {
erofs_err("Compression is not enabled. Turn on chunk-based data deduplication instead.");
cfg.c_chunkbits = sbi.blkszbits;
} else {
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms
2024-01-06 18:11 ` [PATCH 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
@ 2024-01-17 10:56 ` Gao Xiang
0 siblings, 0 replies; 15+ messages in thread
From: Gao Xiang @ 2024-01-17 10:56 UTC (permalink / raw)
To: Yifan Zhao, linux-erofs
On 2024/1/7 02:11, Yifan Zhao wrote:
> Currently, the dictionary size for compression algorithms is fixed. This
> patch allows to specify it with -zX,dictsize=<dictsize> options in
> mkfs command line.
..patch allows to specify different ones with new -zX,dictsize=<dictsize>
options.
>
> This patch also changes the way to specify compression levels. Now, the
> compression level is specified with -zX,level=<level> options and could
> be specified together with dictsize. The old -zX,<level> way is still
The old -zX,<level> form is still...
> supported for compatibility.
>
> Suggested-by: Gao Xiang <hsiangkao@linux.alibaba.com>
> Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
> ---
> include/erofs/config.h | 10 ++--
> include/erofs_fs.h | 4 ++
> lib/compress.c | 45 +++++++++++++----
> lib/compress_hints.c | 2 +-
> lib/compressor.c | 17 ++++++-
> lib/compressor.h | 6 ++-
> lib/compressor_deflate.c | 26 ++++++++--
> lib/compressor_liblzma.c | 33 +++++++-----
> lib/config.c | 4 +-
> lib/inode.c | 2 +-
> lib/kite_deflate.c | 3 +-
> mkfs/main.c | 106 ++++++++++++++++++++++++++++++++-------
> 12 files changed, 203 insertions(+), 55 deletions(-)
>
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index 89fe522..7dbfd1e 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -34,6 +34,12 @@ enum {
>
> #define EROFS_MAX_COMPR_CFGS 64
>
> +struct erofs_compr_cfg {
better to rename as `struct erofs_compr_opts`
> + char *alg;
> + int level;
> + u32 dict_size;
> +};
> +
> struct erofs_configure {
> const char *c_version;
> int c_dbg_lvl;
> @@ -64,8 +70,7 @@ struct erofs_configure {
> char *c_src_path;
> char *c_blobdev_path;
> char *c_compress_hints_file;
> - char *c_compr_alg[EROFS_MAX_COMPR_CFGS];
> - int c_compr_level[EROFS_MAX_COMPR_CFGS];
> + struct erofs_compr_cfg c_compr_opts[EROFS_MAX_COMPR_CFGS];
> char c_force_inodeversion;
> char c_force_chunkformat;
> /* < 0, xattr disabled and INT_MAX, always use inline xattrs */
> @@ -73,7 +78,6 @@ struct erofs_configure {
>
> u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;
> u32 c_max_decompressed_extent_bytes;
> - u32 c_dict_size;
> u64 c_unix_timestamp;
> u32 c_uid, c_gid;
> const char *mount_point;
> diff --git a/include/erofs_fs.h b/include/erofs_fs.h
> index eba6c26..72f0ca6 100644
> --- a/include/erofs_fs.h
> +++ b/include/erofs_fs.h
> @@ -322,6 +322,7 @@ struct z_erofs_lzma_cfgs {
> u8 reserved[8];
> } __packed;
>
> +#define Z_EROFS_LZMA_DEFAULT_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
No need to introduce this since it's not a part of
on-disk format.
It's enough to just update .default_dictsize.
> #define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
>
> /* 6 bytes (+ length field = 8 bytes) */
> @@ -330,6 +331,9 @@ struct z_erofs_deflate_cfgs {
> u8 reserved[5];
> } __packed;
>
> +#define Z_EROFS_DEFLATE_DEFULT_DICT_SIZE (1U << 15)
Same here.
> +#define Z_EROFS_DEFLATE_MAX_DICT_SIZE (1U << 15)
According to DEFLATE specification, it is a hardcoded number
too, so avoid introducing another
Z_EROFS_DEFLATE_MAX_DICT_SIZE here.
> +
> /*
> * bit 0 : COMPACTED_2B indexes (0 - off; 1 - on)
> * e.g. for 4k logical cluster size, 4B if compacted 2B is off;
> diff --git a/lib/compress.c b/lib/compress.c
> index 216aeb4..6f074fd 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -1116,10 +1116,12 @@ err_free_meta:
> }
>
> static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
> - struct erofs_buffer_head *sb_bh)
> + struct erofs_buffer_head *sb_bh,
> + struct erofs_compress_cfg *ccfg)
> {
> struct erofs_buffer_head *bh = sb_bh;
> - int ret = 0;
> + int i, ret = 0;
> + u32 dict_size = 0;
>
> if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZ4)) {
> struct {
> @@ -1146,13 +1148,22 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
> }
> #ifdef HAVE_LIBLZMA
> if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZMA)) {
> + for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
> + if (ccfg[i].enable &&
> + ccfg[i].algorithmtype == Z_EROFS_COMPRESSION_LZMA) {
> + dict_size = ccfg[i].handle.dict_size;
> + break;
> + }
> + }
That is... too hacky to treat like this.
Perhaps just
u32 max_dict_size[Z_EROFS_COMPRESSION_MAX];
for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
if (ccfg[i].enable &&
max_dict_size[ccfg[i].algorithmtype] < ccfg[i].dict_size)
max_dict_size[ccfg[i].algorithmtype] = ccfg[i].dict_size;
}
I don't know though, since different ccfg[i] can have
different levels.
But we need the max dict size though.
> + DBG_BUGON(!dict_size);
> +
> struct {
> __le16 size;
> struct z_erofs_lzma_cfgs lzma;
> } __packed lzmaalg = {
> .size = cpu_to_le16(sizeof(struct z_erofs_lzma_cfgs)),
> .lzma = {
> - .dict_size = cpu_to_le32(cfg.c_dict_size),
> + .dict_size = cpu_to_le32(dict_size),
> }
> };
>
> @@ -1168,6 +1179,15 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
> }
> #endif
> if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_DEFLATE)) {
> + for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
> + if (ccfg[i].enable &&
> + ccfg[i].algorithmtype == Z_EROFS_COMPRESSION_DEFLATE) {
> + dict_size = ccfg[i].handle.dict_size;
> + break;
> + }
> + }
Same here.
> + DBG_BUGON(!dict_size);
> +
> struct {
> __le16 size;
> struct z_erofs_deflate_cfgs z;
> @@ -1175,7 +1195,7 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
> .size = cpu_to_le16(sizeof(struct z_erofs_deflate_cfgs)),
> .z = {
> .windowbits =
> - cpu_to_le32(ilog2(cfg.c_dict_size)),
> + cpu_to_le32(ilog2(dict_size)),
> }
> };
>
> @@ -1196,10 +1216,12 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
> {
> int i, ret;
>
> - for (i = 0; cfg.c_compr_alg[i]; ++i) {
> + for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
> struct erofs_compress *c = &erofs_ccfg[i].handle;
>
> - ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i], cfg.c_compr_level[i]);
> + ret = erofs_compressor_init(sbi, c, cfg.c_compr_opts[i].alg,
> + cfg.c_compr_opts[i].level,
> + cfg.c_compr_opts[i].dict_size);
> if (ret)
> return ret;
>
> @@ -1215,11 +1237,12 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
> * if primary algorithm is empty (e.g. compression off),
> * clear 0PADDING feature for old kernel compatibility.
> */
> - if (!cfg.c_compr_alg[0] ||
> - (cfg.c_legacy_compress && !strncmp(cfg.c_compr_alg[0], "lz4", 3)))
> + if (!cfg.c_compr_opts[0].alg ||
> + (cfg.c_legacy_compress &&
> + !strncmp(cfg.c_compr_opts[0].alg, "lz4", 3)))
> erofs_sb_clear_lz4_0padding(sbi);
>
> - if (!cfg.c_compr_alg[0])
> + if (!cfg.c_compr_opts[0].alg)
> return 0;
>
> /*
> @@ -1241,7 +1264,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
> }
>
> if (erofs_sb_has_compr_cfgs(sbi))
> - return z_erofs_build_compr_cfgs(sbi, sb_bh);
> + return z_erofs_build_compr_cfgs(sbi, sb_bh, erofs_ccfg);
> return 0;
> }
>
> @@ -1249,7 +1272,7 @@ int z_erofs_compress_exit(void)
> {
> int i, ret;
>
> - for (i = 0; cfg.c_compr_alg[i]; ++i) {
> + for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
> ret = erofs_compressor_exit(&erofs_ccfg[i].handle);
> if (ret)
> return ret;
> diff --git a/lib/compress_hints.c b/lib/compress_hints.c
> index afc9f8f..8b78f80 100644
> --- a/lib/compress_hints.c
> +++ b/lib/compress_hints.c
> @@ -125,7 +125,7 @@ int erofs_load_compress_hints(struct erofs_sb_info *sbi)
> } else {
> ccfg = atoi(alg);
> if (ccfg >= EROFS_MAX_COMPR_CFGS ||
> - !cfg.c_compr_alg[ccfg]) {
> + !cfg.c_compr_opts[ccfg].alg) {
> erofs_err("invalid compressing configuration \"%s\" at line %u",
> alg, line);
> ret = -EINVAL;
> diff --git a/lib/compressor.c b/lib/compressor.c
> index 7ec51c2..5061417 100644
> --- a/lib/compressor.c
> +++ b/lib/compressor.c
> @@ -78,7 +78,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
> }
>
> int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
> - char *alg_name, int compression_level)
> + char *alg_name, int compression_level, u32 dict_size)
> {
> int ret, i;
>
> @@ -106,6 +106,8 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
>
> if (erofs_algs[i].c->setlevel) {
> ret = erofs_algs[i].c->setlevel(c, compression_level);
> + if (ret)
> + return ret;
> } else {
> if (compression_level >= 0)
> erofs_warn(
> @@ -113,6 +115,19 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
> compression_level, alg_name);
> c->compression_level = 0;
> }
> +
> + if (erofs_algs[i].c->setdictsize) {
> + ret = erofs_algs[i].c->setdictsize(c, dict_size);
> + if (ret)
> + return ret;
> + } else {
> + if (dict_size)
> + erofs_warn(
Avoid a seperate line if it's just the message exceeds 80-char, I mean
erofs_warn("specifing a dictionary size %u for %s is unsupported, ignored"
dict_size, alg_name);
...
> + "dictionary size %u is ignored for %s",
> + dict_size, alg_name);
> + c->dict_size = 0;
> + }
> +
> if (!ret) {
> c->alg = &erofs_algs[i];
> return 0;
> diff --git a/lib/compressor.h b/lib/compressor.h
> index ec5485d..d8ccf2e 100644
> --- a/lib/compressor.h
> +++ b/lib/compressor.h
> @@ -14,10 +14,13 @@ struct erofs_compress;
> struct erofs_compressor {
> int default_level;
> int best_level;
> + u32 default_dictsize;
> + u32 max_dictsize;
>
> int (*init)(struct erofs_compress *c);
> int (*exit)(struct erofs_compress *c);
> int (*setlevel)(struct erofs_compress *c, int compression_level);
> + int (*setdictsize)(struct erofs_compress *c, u32 dict_size);
>
> int (*compress_destsize)(const struct erofs_compress *c,
> const void *src, unsigned int *srcsize,
> @@ -39,6 +42,7 @@ struct erofs_compress {
>
> unsigned int compress_threshold;
> unsigned int compression_level;
> + unsigned int dict_size;
>
> void *private_data;
> };
> @@ -56,7 +60,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
> void *dst, unsigned int dstsize);
>
> int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
> - char *alg_name, int compression_level);
> + char *alg_name, int compression_level, u32 dict_size);
> int erofs_compressor_exit(struct erofs_compress *c);
>
> #endif
> diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
> index 4e5902e..c95e53e 100644
> --- a/lib/compressor_deflate.c
> +++ b/lib/compressor_deflate.c
> @@ -46,6 +46,16 @@ static int compressor_deflate_init(struct erofs_compress *c)
>
> static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
> int compression_level)
> +{
> + if (compression_level < 0)
> + compression_level = erofs_compressor_deflate.default_level;
> +
> + c->compression_level = compression_level;
> + return 0;
> +}
> +
> +static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
> + u32 dict_size)
> {
> void *s;
>
> @@ -54,23 +64,31 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
> c->private_data = NULL;
> }
>
> - if (compression_level < 0)
> - compression_level = erofs_compressor_deflate.default_level;
> + if (dict_size > erofs_compressor_deflate.max_dictsize) {
> + erofs_err("dict size %u is too large", dict_size);
> + return -EINVAL;
> + }
> +
> + if (dict_size == 0)
> + dict_size = erofs_compressor_deflate.default_dictsize;
>
> - s = kite_deflate_init(compression_level, cfg.c_dict_size);
> + s = kite_deflate_init(c->compression_level, dict_size);
> if (IS_ERR(s))
> return PTR_ERR(s);
>
> c->private_data = s;
> - c->compression_level = compression_level;
> + c->dict_size = dict_size;
> return 0;
> }
>
> const struct erofs_compressor erofs_compressor_deflate = {
> .default_level = 1,
> .best_level = 9,
> + .default_dictsize = Z_EROFS_DEFLATE_DEFULT_DICT_SIZE,
> + .max_dictsize = Z_EROFS_DEFLATE_MAX_DICT_SIZE,
> .init = compressor_deflate_init,
> .exit = compressor_deflate_exit,
> .setlevel = erofs_compressor_deflate_setlevel,
> + .setdictsize = erofs_compressor_deflate_setdictsize,
> .compress_destsize = deflate_compress_destsize,
> };
> diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
> index 0ed6f23..5c7f830 100644
> --- a/lib/compressor_liblzma.c
> +++ b/lib/compressor_liblzma.c
> @@ -68,22 +68,28 @@ static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
> if (lzma_lzma_preset(&ctx->opt, preset))
> return -EINVAL;
>
> - /* XXX: temporary hack */
> - if (cfg.c_dict_size) {
> - if (cfg.c_dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) {
> - erofs_err("dict size %u is too large", cfg.c_dict_size);
> - return -EINVAL;
> - }
> - ctx->opt.dict_size = cfg.c_dict_size;
> - } else {
> - if (ctx->opt.dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE)
> - ctx->opt.dict_size = Z_EROFS_LZMA_MAX_DICT_SIZE;
> - cfg.c_dict_size = ctx->opt.dict_size;
> - }
> c->compression_level = compression_level;
> return 0;
> }
>
> +static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
> + u32 dict_size)
> +{
> + struct erofs_liblzma_context *ctx = c->private_data;
> +
> + if (dict_size > erofs_compressor_lzma.max_dictsize) {
> + erofs_err("dict size %u is too large", dict_size);
> + return -EINVAL;
> + }
> +
> + if (dict_size == 0)
> + dict_size = erofs_compressor_lzma.default_dictsize;
> +
> + ctx->opt.dict_size = dict_size;
> + c->dict_size = dict_size;
> + return 0;
> +}
> +
> static int erofs_compressor_liblzma_init(struct erofs_compress *c)
> {
> struct erofs_liblzma_context *ctx;
> @@ -101,9 +107,12 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c)
> const struct erofs_compressor erofs_compressor_lzma = {
> .default_level = LZMA_PRESET_DEFAULT,
> .best_level = 109,
> + .default_dictsize = Z_EROFS_LZMA_DEFAULT_DICT_SIZE,
> + .max_dictsize = Z_EROFS_LZMA_MAX_DICT_SIZE,
> .init = erofs_compressor_liblzma_init,
> .exit = erofs_compressor_liblzma_exit,
> .setlevel = erofs_compressor_liblzma_setlevel,
> + .setdictsize = erofs_compressor_liblzma_setdictsize,
> .compress_destsize = erofs_liblzma_compress_destsize,
> };
> #endif
> diff --git a/lib/config.c b/lib/config.c
> index 0cddc95..d3e8e53 100644
> --- a/lib/config.c
> +++ b/lib/config.c
> @@ -58,8 +58,8 @@ void erofs_exit_configure(void)
> free(cfg.c_img_path);
> if (cfg.c_src_path)
> free(cfg.c_src_path);
> - for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_alg[i]; i++)
> - free(cfg.c_compr_alg[i]);
> + for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_opts[i].alg; i++)
> + free(cfg.c_compr_opts[i].alg);
> }
>
> static unsigned int fullpath_prefix; /* root directory prefix length */
> diff --git a/lib/inode.c b/lib/inode.c
> index bcdb4b8..c6424c0 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -492,7 +492,7 @@ int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
> return erofs_blob_write_chunked_file(inode, fd, fpos);
> }
>
> - if (cfg.c_compr_alg[0] && erofs_file_is_compressible(inode)) {
> + if (cfg.c_compr_opts[0].alg && erofs_file_is_compressible(inode)) {
> ret = erofs_write_compressed_file(inode, fd);
> if (!ret || ret != -ENOSPC)
> return ret;
> diff --git a/lib/kite_deflate.c b/lib/kite_deflate.c
> index 8667954..2357f76 100644
> --- a/lib/kite_deflate.c
> +++ b/lib/kite_deflate.c
> @@ -5,6 +5,7 @@
> * Copyright (C) 2023, Alibaba Cloud
> * Copyright (C) 2023, Gao Xiang <xiang@kernel.org>
> */
> +#include "erofs/internal.h"
> #include "erofs/defs.h"
> #include "erofs/print.h"
> #include <errno.h>
> @@ -22,7 +23,7 @@ unsigned long erofs_memcmp2(const u8 *s1, const u8 *s2,
> #define kite_dbg(x, ...)
> #endif
>
> -#define kHistorySize32 (1U << 15)
> +#define kHistorySize32 Z_EROFS_DEFLATE_DEFULT_DICT_SIZE
No, please don't touch this file (kite_deflate.c).
>
> #define kNumLenSymbols32 256
> #define kNumLenSymbolsMax kNumLenSymbols32
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 0517849..702eb4b 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -5,6 +5,7 @@
> * Created by Li Guifu <bluce.liguifu@huawei.com>
> */
> #define _GNU_SOURCE
> +#include <ctype.h>
> #include <time.h>
> #include <sys/time.h>
> #include <stdlib.h>
> @@ -108,24 +109,29 @@ static void usage(int argc, char **argv)
> " -b# set block size to # (# = page size by default)\n"
> " -d<0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n"
> " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> - " -zX[,Y][:...] X=compressor (Y=compression level, optional)\n"
> - " alternative compressors can be separated by colons(:)\n"
> - " supported compressors and their level ranges are:\n",
> + " -zX[,level=Y] X=compressor (Y=compression level, Z=dictionary size, optional)\n"
> + " [,dictsize=Z] alternative compressors can be separated by colons(:)\n"
> + " [:...] supported compressors and their option ranges are:\n",
> argv[0], EROFS_WARN);
> while ((s = z_erofs_list_available_compressors(&i)) != NULL) {
> - printf(" %s", s->name);
> + char * const spaces = " ";
> +
> + printf("%s%s\n", spaces, s->name);
> if (s->c->setlevel) {
> if (!strcmp(s->name, "lzma"))
> /* A little kludge to show the range as disjointed
> * "0-9,100-109" instead of a continuous "0-109", and to
> * state what those two subranges respectively mean. */
> - printf("[<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)",
> - s->c->default_level);
> + printf("%s [,level=<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)\n",
> + spaces, s->c->default_level);
> else
> - printf("[,<0-%i>]\t(default=%i)",
> - s->c->best_level, s->c->default_level);
> + printf("%s [,level=<0-%i>]\t\t(default=%i)\n",
> + spaces, s->c->best_level, s->c->default_level);
> + }
> + if (s->c->setdictsize) {
> + printf("%s [,dictsize=<dictsize>]\t(default=%u, max=%u)\n",
> + spaces, s->c->default_dictsize, s->c->max_dictsize);
> }
> - putchar('\n');
> }
> printf(
> " -C# specify the size of compress physical cluster in bytes\n"
> @@ -310,20 +316,84 @@ static int mkfs_parse_compress_algs(char *algs)
> char *s;
>
> for (s = strtok(algs, ":"), i = 0; s; s = strtok(NULL, ":"), ++i) {
> - const char *lv;
> + char *p, *q, *opt, *endptr;
> + struct erofs_compr_cfg *ccfg;
>
> if (i >= EROFS_MAX_COMPR_CFGS - 1) {
> erofs_err("too many algorithm types");
> return -EINVAL;
> }
>
> - lv = strchr(s, ',');
> - if (lv) {
> - cfg.c_compr_level[i] = atoi(lv + 1);
> - cfg.c_compr_alg[i] = strndup(s, lv - s);
Let's introduce another helper function to deal
with the following logic.
Thanks,
Gao Xiang
> + ccfg = cfg.c_compr_opts + i;
> +
> + ccfg->level = -1;
> + ccfg->dict_size = 0;
> +
> + p = strchr(s, ',');
> + if (p) {
> + ccfg->alg = strndup(s, p - s);
> +
> + /* backward compatibility */
> + if (isdigit(*(p + 1))) {
> + ccfg->level = strtol(p + 1, &endptr, 10);
> + if (*endptr && *endptr != ',') {
> + erofs_err(
> + "invalid compression level %s",
> + p + 1);
> + return -EINVAL;
> + }
> + continue;
> + }
> } else {
> - cfg.c_compr_level[i] = -1;
> - cfg.c_compr_alg[i] = strdup(s);
> + ccfg->alg = strdup(s);
> + continue;
> + }
> +
> + opt = p + 1;
> + while (opt) {
> + q = strchr(opt, ',');
> + if (q)
> + *q = '\0';
> +
> + if ((p = strstr(opt, "level="))) {
> + p += strlen("level=");
> + ccfg->level = strtol(p, &endptr, 10);
> + if (*endptr && *endptr != ',') {
> + erofs_err(
> + "invalid compression level %s",
> + p);
> + return -EINVAL;
> + }
> + } else if ((p = strstr(opt, "dictsize="))) {
> + p += strlen("dictsize=");
> + ccfg->dict_size = strtoul(p, &endptr, 10);
> + switch (*endptr) {
> + case ',':
> + case '\0':
> + break;
> + case 'K':
> + case 'k':
> + ccfg->dict_size <<= 10;
> + break;
> + case 'M':
> + case 'm':
> + ccfg->dict_size <<= 20;
> + break;
> + default:
> + erofs_err(
> + "invalid compression dictsize %s",
> + p);
> + return -EINVAL;
> + }
> + } else {
> + erofs_err("invalid compression option %s", opt);
> + return -EINVAL;
> + }
> +
> + if (q)
> + opt = q + 1;
> + else
> + opt = NULL;
> }
> }
> return 0;
> @@ -692,7 +762,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> cfg.c_showprogress = false;
> }
>
> - if (cfg.c_compr_alg[0] && erofs_blksiz(&sbi) != getpagesize())
> + if (cfg.c_compr_opts[0].alg && erofs_blksiz(&sbi) != getpagesize())
> erofs_warn("Please note that subpage blocksize with compression isn't yet supported in kernel. "
> "This compressed image will only work with bs = ps = %u bytes",
> erofs_blksiz(&sbi));
> @@ -1119,7 +1189,7 @@ int main(int argc, char **argv)
> }
>
> if (cfg.c_dedupe) {
> - if (!cfg.c_compr_alg[0]) {
> + if (!cfg.c_compr_opts[0].alg) {
> erofs_err("Compression is not enabled. Turn on chunk-based data deduplication instead.");
> cfg.c_chunkbits = sbi.blkszbits;
> } else {
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 3/3] erofs-utils: mkfs: reorganize logic in erofs_compressor_init()
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
2024-01-06 18:11 ` [PATCH 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
@ 2024-01-06 18:11 ` Yifan Zhao
2024-01-17 10:58 ` Gao Xiang
2024-01-08 6:00 ` [PATCH v2 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
` (8 subsequent siblings)
10 siblings, 1 reply; 15+ messages in thread
From: Yifan Zhao @ 2024-01-06 18:11 UTC (permalink / raw)
To: linux-erofs
Currently, the initialization of compressors is weird: init() is called
first, followed by setlevel(), then setdictsize(). The initialization
process occurs in the last called setdictsize() for lzma, deflate, and
libdeflate.
This patch reorders the three functions, with init() now being invoked
last, allowing it to use the compression level and dictsize already set
so that the behavior of the functions matches their names.
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
lib/compressor.c | 8 ++++----
lib/compressor_deflate.c | 32 ++++++++++++++++----------------
lib/compressor_libdeflate.c | 14 +++++++++-----
lib/compressor_liblzma.c | 34 +++++++++++++++++++---------------
lib/compressor_lz4hc.c | 5 ++++-
5 files changed, 52 insertions(+), 41 deletions(-)
diff --git a/lib/compressor.c b/lib/compressor.c
index 5061417..27b4077 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -100,10 +100,6 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
if (!erofs_algs[i].c)
continue;
- ret = erofs_algs[i].c->init(c);
- if (ret)
- return ret;
-
if (erofs_algs[i].c->setlevel) {
ret = erofs_algs[i].c->setlevel(c, compression_level);
if (ret)
@@ -128,6 +124,10 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
c->dict_size = 0;
}
+ ret = erofs_algs[i].c->init(c);
+ if (ret)
+ return ret;
+
if (!ret) {
c->alg = &erofs_algs[i];
return 0;
diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
index c95e53e..d9f8a91 100644
--- a/lib/compressor_deflate.c
+++ b/lib/compressor_deflate.c
@@ -36,7 +36,14 @@ static int compressor_deflate_exit(struct erofs_compress *c)
static int compressor_deflate_init(struct erofs_compress *c)
{
- c->private_data = NULL;
+ if (c->private_data) {
+ kite_deflate_end(c->private_data);
+ c->private_data = NULL;
+ }
+
+ c->private_data = kite_deflate_init(c->compression_level, c->dict_size);
+ if (IS_ERR_VALUE(c->private_data))
+ return PTR_ERR(c->private_data);
erofs_warn("EXPERIMENTAL DEFLATE algorithm in use. Use at your own risk!");
erofs_warn("*Carefully* check filesystem data correctness to avoid corruption!");
@@ -50,6 +57,11 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
if (compression_level < 0)
compression_level = erofs_compressor_deflate.default_level;
+ if (compression_level > erofs_compressor_deflate.best_level) {
+ erofs_err("invalid compression level %d", compression_level);
+ return -EINVAL;
+ }
+
c->compression_level = compression_level;
return 0;
}
@@ -57,26 +69,14 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
u32 dict_size)
{
- void *s;
-
- if (c->private_data) {
- kite_deflate_end(c->private_data);
- c->private_data = NULL;
- }
+ if (dict_size == 0)
+ dict_size = erofs_compressor_deflate.default_dictsize;
if (dict_size > erofs_compressor_deflate.max_dictsize) {
- erofs_err("dict size %u is too large", dict_size);
+ erofs_err("invalid dict size %u", dict_size);
return -EINVAL;
}
- if (dict_size == 0)
- dict_size = erofs_compressor_deflate.default_dictsize;
-
- s = kite_deflate_init(c->compression_level, dict_size);
- if (IS_ERR(s))
- return PTR_ERR(s);
-
- c->private_data = s;
c->dict_size = dict_size;
return 0;
}
diff --git a/lib/compressor_libdeflate.c b/lib/compressor_libdeflate.c
index c0b019a..ad5eeec 100644
--- a/lib/compressor_libdeflate.c
+++ b/lib/compressor_libdeflate.c
@@ -82,7 +82,10 @@ static int compressor_libdeflate_exit(struct erofs_compress *c)
static int compressor_libdeflate_init(struct erofs_compress *c)
{
- c->private_data = NULL;
+ libdeflate_free_compressor(c->private_data);
+ c->private_data = libdeflate_alloc_compressor(c->compression_level);
+ if (!c->private_data)
+ return -ENOMEM;
erofs_warn("EXPERIMENTAL libdeflate compressor in use. Use at your own risk!");
return 0;
@@ -94,10 +97,11 @@ static int erofs_compressor_libdeflate_setlevel(struct erofs_compress *c,
if (compression_level < 0)
compression_level = erofs_compressor_deflate.default_level;
- libdeflate_free_compressor(c->private_data);
- c->private_data = libdeflate_alloc_compressor(compression_level);
- if (!c->private_data)
- return -ENOMEM;
+ if (compression_level > erofs_compressor_deflate.best_level) {
+ erofs_err("invalid compression level %d", compression_level);
+ return -EINVAL;
+ }
+
c->compression_level = compression_level;
return 0;
}
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
index 5c7f830..0203a5c 100644
--- a/lib/compressor_liblzma.c
+++ b/lib/compressor_liblzma.c
@@ -55,18 +55,13 @@ static int erofs_compressor_liblzma_exit(struct erofs_compress *c)
static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
int compression_level)
{
- struct erofs_liblzma_context *ctx = c->private_data;
- u32 preset;
-
if (compression_level < 0)
- preset = LZMA_PRESET_DEFAULT;
- else if (compression_level >= 100)
- preset = (compression_level - 100) | LZMA_PRESET_EXTREME;
- else
- preset = compression_level;
+ compression_level = erofs_compressor_lzma.default_level;
- if (lzma_lzma_preset(&ctx->opt, preset))
+ if (compression_level > erofs_compressor_lzma.best_level) {
+ erofs_err("invalid compression level %d", compression_level);
return -EINVAL;
+ }
c->compression_level = compression_level;
return 0;
@@ -75,17 +70,14 @@ static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
u32 dict_size)
{
- struct erofs_liblzma_context *ctx = c->private_data;
+ if (dict_size == 0)
+ dict_size = erofs_compressor_lzma.default_dictsize;
if (dict_size > erofs_compressor_lzma.max_dictsize) {
- erofs_err("dict size %u is too large", dict_size);
+ erofs_err("invalid dict size %u", dict_size);
return -EINVAL;
}
- if (dict_size == 0)
- dict_size = erofs_compressor_lzma.default_dictsize;
-
- ctx->opt.dict_size = dict_size;
c->dict_size = dict_size;
return 0;
}
@@ -93,11 +85,23 @@ static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
static int erofs_compressor_liblzma_init(struct erofs_compress *c)
{
struct erofs_liblzma_context *ctx;
+ u32 preset;
ctx = malloc(sizeof(*ctx));
if (!ctx)
return -ENOMEM;
+
ctx->strm = (lzma_stream)LZMA_STREAM_INIT;
+ if (c->compression_level < 0)
+ preset = LZMA_PRESET_DEFAULT;
+ else if (c->compression_level >= 100)
+ preset = (c->compression_level - 100) | LZMA_PRESET_EXTREME;
+ else
+ preset = c->compression_level;
+ if (lzma_lzma_preset(&ctx->opt, preset))
+ return -EINVAL;
+ ctx->opt.dict_size = c->dict_size;
+
c->private_data = ctx;
erofs_warn("EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!");
erofs_warn("Note that it may take more time since the compressor is still single-threaded for now.");
diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
index b410e15..6fc8847 100644
--- a/lib/compressor_lz4hc.c
+++ b/lib/compressor_lz4hc.c
@@ -7,6 +7,7 @@
#define LZ4_HC_STATIC_LINKING_ONLY (1)
#include <lz4hc.h>
#include "erofs/internal.h"
+#include "erofs/print.h"
#include "compressor.h"
#ifndef LZ4_DISTANCE_MAX /* history window size */
@@ -49,8 +50,10 @@ static int compressor_lz4hc_init(struct erofs_compress *c)
static int compressor_lz4hc_setlevel(struct erofs_compress *c,
int compression_level)
{
- if (compression_level > LZ4HC_CLEVEL_MAX)
+ if (compression_level > erofs_compressor_lz4hc.best_level) {
+ erofs_err("invalid compression level %d", compression_level);
return -EINVAL;
+ }
c->compression_level = compression_level < 0 ?
LZ4HC_CLEVEL_DEFAULT : compression_level;
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 3/3] erofs-utils: mkfs: reorganize logic in erofs_compressor_init()
2024-01-06 18:11 ` [PATCH 3/3] erofs-utils: mkfs: reorganize logic in erofs_compressor_init() Yifan Zhao
@ 2024-01-17 10:58 ` Gao Xiang
0 siblings, 0 replies; 15+ messages in thread
From: Gao Xiang @ 2024-01-17 10:58 UTC (permalink / raw)
To: Yifan Zhao, linux-erofs
Hi Yifan,
Note that I don't have more useful comments of this one.
Let's see how PATCHSET v2 works first.
Thanks,
Gao Xiang
On 2024/1/7 02:11, Yifan Zhao wrote:
> Currently, the initialization of compressors is weird: init() is called
> first, followed by setlevel(), then setdictsize(). The initialization
> process occurs in the last called setdictsize() for lzma, deflate, and
> libdeflate.
>
> This patch reorders the three functions, with init() now being invoked
> last, allowing it to use the compression level and dictsize already set
> so that the behavior of the functions matches their names.
>
> Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
> ---
> lib/compressor.c | 8 ++++----
> lib/compressor_deflate.c | 32 ++++++++++++++++----------------
> lib/compressor_libdeflate.c | 14 +++++++++-----
> lib/compressor_liblzma.c | 34 +++++++++++++++++++---------------
> lib/compressor_lz4hc.c | 5 ++++-
> 5 files changed, 52 insertions(+), 41 deletions(-)
>
> diff --git a/lib/compressor.c b/lib/compressor.c
> index 5061417..27b4077 100644
> --- a/lib/compressor.c
> +++ b/lib/compressor.c
> @@ -100,10 +100,6 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
> if (!erofs_algs[i].c)
> continue;
>
> - ret = erofs_algs[i].c->init(c);
> - if (ret)
> - return ret;
> -
> if (erofs_algs[i].c->setlevel) {
> ret = erofs_algs[i].c->setlevel(c, compression_level);
> if (ret)
> @@ -128,6 +124,10 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
> c->dict_size = 0;
> }
>
> + ret = erofs_algs[i].c->init(c);
> + if (ret)
> + return ret;
> +
> if (!ret) {
> c->alg = &erofs_algs[i];
> return 0;
> diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
> index c95e53e..d9f8a91 100644
> --- a/lib/compressor_deflate.c
> +++ b/lib/compressor_deflate.c
> @@ -36,7 +36,14 @@ static int compressor_deflate_exit(struct erofs_compress *c)
>
> static int compressor_deflate_init(struct erofs_compress *c)
> {
> - c->private_data = NULL;
> + if (c->private_data) {
> + kite_deflate_end(c->private_data);
> + c->private_data = NULL;
> + }
> +
> + c->private_data = kite_deflate_init(c->compression_level, c->dict_size);
> + if (IS_ERR_VALUE(c->private_data))
> + return PTR_ERR(c->private_data);
>
> erofs_warn("EXPERIMENTAL DEFLATE algorithm in use. Use at your own risk!");
> erofs_warn("*Carefully* check filesystem data correctness to avoid corruption!");
> @@ -50,6 +57,11 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
> if (compression_level < 0)
> compression_level = erofs_compressor_deflate.default_level;
>
> + if (compression_level > erofs_compressor_deflate.best_level) {
> + erofs_err("invalid compression level %d", compression_level);
> + return -EINVAL;
> + }
> +
> c->compression_level = compression_level;
> return 0;
> }
> @@ -57,26 +69,14 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
> static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
> u32 dict_size)
> {
> - void *s;
> -
> - if (c->private_data) {
> - kite_deflate_end(c->private_data);
> - c->private_data = NULL;
> - }
> + if (dict_size == 0)
> + dict_size = erofs_compressor_deflate.default_dictsize;
>
> if (dict_size > erofs_compressor_deflate.max_dictsize) {
> - erofs_err("dict size %u is too large", dict_size);
> + erofs_err("invalid dict size %u", dict_size);
> return -EINVAL;
> }
>
> - if (dict_size == 0)
> - dict_size = erofs_compressor_deflate.default_dictsize;
> -
> - s = kite_deflate_init(c->compression_level, dict_size);
> - if (IS_ERR(s))
> - return PTR_ERR(s);
> -
> - c->private_data = s;
> c->dict_size = dict_size;
> return 0;
> }
> diff --git a/lib/compressor_libdeflate.c b/lib/compressor_libdeflate.c
> index c0b019a..ad5eeec 100644
> --- a/lib/compressor_libdeflate.c
> +++ b/lib/compressor_libdeflate.c
> @@ -82,7 +82,10 @@ static int compressor_libdeflate_exit(struct erofs_compress *c)
>
> static int compressor_libdeflate_init(struct erofs_compress *c)
> {
> - c->private_data = NULL;
> + libdeflate_free_compressor(c->private_data);
> + c->private_data = libdeflate_alloc_compressor(c->compression_level);
> + if (!c->private_data)
> + return -ENOMEM;
>
> erofs_warn("EXPERIMENTAL libdeflate compressor in use. Use at your own risk!");
> return 0;
> @@ -94,10 +97,11 @@ static int erofs_compressor_libdeflate_setlevel(struct erofs_compress *c,
> if (compression_level < 0)
> compression_level = erofs_compressor_deflate.default_level;
>
> - libdeflate_free_compressor(c->private_data);
> - c->private_data = libdeflate_alloc_compressor(compression_level);
> - if (!c->private_data)
> - return -ENOMEM;
> + if (compression_level > erofs_compressor_deflate.best_level) {
> + erofs_err("invalid compression level %d", compression_level);
> + return -EINVAL;
> + }
> +
> c->compression_level = compression_level;
> return 0;
> }
> diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
> index 5c7f830..0203a5c 100644
> --- a/lib/compressor_liblzma.c
> +++ b/lib/compressor_liblzma.c
> @@ -55,18 +55,13 @@ static int erofs_compressor_liblzma_exit(struct erofs_compress *c)
> static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
> int compression_level)
> {
> - struct erofs_liblzma_context *ctx = c->private_data;
> - u32 preset;
> -
> if (compression_level < 0)
> - preset = LZMA_PRESET_DEFAULT;
> - else if (compression_level >= 100)
> - preset = (compression_level - 100) | LZMA_PRESET_EXTREME;
> - else
> - preset = compression_level;
> + compression_level = erofs_compressor_lzma.default_level;
>
> - if (lzma_lzma_preset(&ctx->opt, preset))
> + if (compression_level > erofs_compressor_lzma.best_level) {
> + erofs_err("invalid compression level %d", compression_level);
> return -EINVAL;
> + }
>
> c->compression_level = compression_level;
> return 0;
> @@ -75,17 +70,14 @@ static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
> static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
> u32 dict_size)
> {
> - struct erofs_liblzma_context *ctx = c->private_data;
> + if (dict_size == 0)
> + dict_size = erofs_compressor_lzma.default_dictsize;
>
> if (dict_size > erofs_compressor_lzma.max_dictsize) {
> - erofs_err("dict size %u is too large", dict_size);
> + erofs_err("invalid dict size %u", dict_size);
> return -EINVAL;
> }
>
> - if (dict_size == 0)
> - dict_size = erofs_compressor_lzma.default_dictsize;
> -
> - ctx->opt.dict_size = dict_size;
> c->dict_size = dict_size;
> return 0;
> }
> @@ -93,11 +85,23 @@ static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
> static int erofs_compressor_liblzma_init(struct erofs_compress *c)
> {
> struct erofs_liblzma_context *ctx;
> + u32 preset;
>
> ctx = malloc(sizeof(*ctx));
> if (!ctx)
> return -ENOMEM;
> +
> ctx->strm = (lzma_stream)LZMA_STREAM_INIT;
> + if (c->compression_level < 0)
> + preset = LZMA_PRESET_DEFAULT;
> + else if (c->compression_level >= 100)
> + preset = (c->compression_level - 100) | LZMA_PRESET_EXTREME;
> + else
> + preset = c->compression_level;
> + if (lzma_lzma_preset(&ctx->opt, preset))
> + return -EINVAL;
> + ctx->opt.dict_size = c->dict_size;
> +
> c->private_data = ctx;
> erofs_warn("EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!");
> erofs_warn("Note that it may take more time since the compressor is still single-threaded for now.");
> diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
> index b410e15..6fc8847 100644
> --- a/lib/compressor_lz4hc.c
> +++ b/lib/compressor_lz4hc.c
> @@ -7,6 +7,7 @@
> #define LZ4_HC_STATIC_LINKING_ONLY (1)
> #include <lz4hc.h>
> #include "erofs/internal.h"
> +#include "erofs/print.h"
> #include "compressor.h"
>
> #ifndef LZ4_DISTANCE_MAX /* history window size */
> @@ -49,8 +50,10 @@ static int compressor_lz4hc_init(struct erofs_compress *c)
> static int compressor_lz4hc_setlevel(struct erofs_compress *c,
> int compression_level)
> {
> - if (compression_level > LZ4HC_CLEVEL_MAX)
> + if (compression_level > erofs_compressor_lz4hc.best_level) {
> + erofs_err("invalid compression level %d", compression_level);
> return -EINVAL;
> + }
>
> c->compression_level = compression_level < 0 ?
> LZ4HC_CLEVEL_DEFAULT : compression_level;
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH v2 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
2024-01-06 18:11 ` [PATCH 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
2024-01-06 18:11 ` [PATCH 3/3] erofs-utils: mkfs: reorganize logic in erofs_compressor_init() Yifan Zhao
@ 2024-01-08 6:00 ` Yifan Zhao
2024-01-13 17:39 ` [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Gao Xiang
` (7 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Yifan Zhao @ 2024-01-08 6:00 UTC (permalink / raw)
To: linux-erofs
Currently, the dictionary size for compression algorithms is fixed. This
patch allows to specify it with -zX,dictsize=<dictsize> options in
mkfs command line.
This patch also changes the way to specify compression levels. Now, the
compression level is specified with -zX,level=<level> options and could
be specified together with dictsize. The old -zX,<level> way is still
supported for compatibility.
Suggested-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
change since v1:
- fix a bug in mkfs_parse_compress_algs() that "-zlzma,dictsize=" is not
correctly parsed as error.
include/erofs/config.h | 10 ++--
include/erofs_fs.h | 4 ++
lib/compress.c | 45 +++++++++++++-----
lib/compress_hints.c | 2 +-
lib/compressor.c | 17 ++++++-
lib/compressor.h | 6 ++-
lib/compressor_deflate.c | 26 ++++++++--
lib/compressor_liblzma.c | 33 ++++++++-----
lib/config.c | 4 +-
lib/inode.c | 2 +-
lib/kite_deflate.c | 3 +-
mkfs/main.c | 100 ++++++++++++++++++++++++++++++++-------
12 files changed, 197 insertions(+), 55 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 89fe522..7dbfd1e 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -34,6 +34,12 @@ enum {
#define EROFS_MAX_COMPR_CFGS 64
+struct erofs_compr_cfg {
+ char *alg;
+ int level;
+ u32 dict_size;
+};
+
struct erofs_configure {
const char *c_version;
int c_dbg_lvl;
@@ -64,8 +70,7 @@ struct erofs_configure {
char *c_src_path;
char *c_blobdev_path;
char *c_compress_hints_file;
- char *c_compr_alg[EROFS_MAX_COMPR_CFGS];
- int c_compr_level[EROFS_MAX_COMPR_CFGS];
+ struct erofs_compr_cfg c_compr_opts[EROFS_MAX_COMPR_CFGS];
char c_force_inodeversion;
char c_force_chunkformat;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
@@ -73,7 +78,6 @@ struct erofs_configure {
u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;
u32 c_max_decompressed_extent_bytes;
- u32 c_dict_size;
u64 c_unix_timestamp;
u32 c_uid, c_gid;
const char *mount_point;
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index eba6c26..72f0ca6 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -322,6 +322,7 @@ struct z_erofs_lzma_cfgs {
u8 reserved[8];
} __packed;
+#define Z_EROFS_LZMA_DEFAULT_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
/* 6 bytes (+ length field = 8 bytes) */
@@ -330,6 +331,9 @@ struct z_erofs_deflate_cfgs {
u8 reserved[5];
} __packed;
+#define Z_EROFS_DEFLATE_DEFULT_DICT_SIZE (1U << 15)
+#define Z_EROFS_DEFLATE_MAX_DICT_SIZE (1U << 15)
+
/*
* bit 0 : COMPACTED_2B indexes (0 - off; 1 - on)
* e.g. for 4k logical cluster size, 4B if compacted 2B is off;
diff --git a/lib/compress.c b/lib/compress.c
index 216aeb4..6f074fd 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1116,10 +1116,12 @@ err_free_meta:
}
static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
- struct erofs_buffer_head *sb_bh)
+ struct erofs_buffer_head *sb_bh,
+ struct erofs_compress_cfg *ccfg)
{
struct erofs_buffer_head *bh = sb_bh;
- int ret = 0;
+ int i, ret = 0;
+ u32 dict_size = 0;
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZ4)) {
struct {
@@ -1146,13 +1148,22 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
}
#ifdef HAVE_LIBLZMA
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZMA)) {
+ for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
+ if (ccfg[i].enable &&
+ ccfg[i].algorithmtype == Z_EROFS_COMPRESSION_LZMA) {
+ dict_size = ccfg[i].handle.dict_size;
+ break;
+ }
+ }
+ DBG_BUGON(!dict_size);
+
struct {
__le16 size;
struct z_erofs_lzma_cfgs lzma;
} __packed lzmaalg = {
.size = cpu_to_le16(sizeof(struct z_erofs_lzma_cfgs)),
.lzma = {
- .dict_size = cpu_to_le32(cfg.c_dict_size),
+ .dict_size = cpu_to_le32(dict_size),
}
};
@@ -1168,6 +1179,15 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
}
#endif
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_DEFLATE)) {
+ for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
+ if (ccfg[i].enable &&
+ ccfg[i].algorithmtype == Z_EROFS_COMPRESSION_DEFLATE) {
+ dict_size = ccfg[i].handle.dict_size;
+ break;
+ }
+ }
+ DBG_BUGON(!dict_size);
+
struct {
__le16 size;
struct z_erofs_deflate_cfgs z;
@@ -1175,7 +1195,7 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
.size = cpu_to_le16(sizeof(struct z_erofs_deflate_cfgs)),
.z = {
.windowbits =
- cpu_to_le32(ilog2(cfg.c_dict_size)),
+ cpu_to_le32(ilog2(dict_size)),
}
};
@@ -1196,10 +1216,12 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
{
int i, ret;
- for (i = 0; cfg.c_compr_alg[i]; ++i) {
+ for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
struct erofs_compress *c = &erofs_ccfg[i].handle;
- ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i], cfg.c_compr_level[i]);
+ ret = erofs_compressor_init(sbi, c, cfg.c_compr_opts[i].alg,
+ cfg.c_compr_opts[i].level,
+ cfg.c_compr_opts[i].dict_size);
if (ret)
return ret;
@@ -1215,11 +1237,12 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
* if primary algorithm is empty (e.g. compression off),
* clear 0PADDING feature for old kernel compatibility.
*/
- if (!cfg.c_compr_alg[0] ||
- (cfg.c_legacy_compress && !strncmp(cfg.c_compr_alg[0], "lz4", 3)))
+ if (!cfg.c_compr_opts[0].alg ||
+ (cfg.c_legacy_compress &&
+ !strncmp(cfg.c_compr_opts[0].alg, "lz4", 3)))
erofs_sb_clear_lz4_0padding(sbi);
- if (!cfg.c_compr_alg[0])
+ if (!cfg.c_compr_opts[0].alg)
return 0;
/*
@@ -1241,7 +1264,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
}
if (erofs_sb_has_compr_cfgs(sbi))
- return z_erofs_build_compr_cfgs(sbi, sb_bh);
+ return z_erofs_build_compr_cfgs(sbi, sb_bh, erofs_ccfg);
return 0;
}
@@ -1249,7 +1272,7 @@ int z_erofs_compress_exit(void)
{
int i, ret;
- for (i = 0; cfg.c_compr_alg[i]; ++i) {
+ for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
ret = erofs_compressor_exit(&erofs_ccfg[i].handle);
if (ret)
return ret;
diff --git a/lib/compress_hints.c b/lib/compress_hints.c
index afc9f8f..8b78f80 100644
--- a/lib/compress_hints.c
+++ b/lib/compress_hints.c
@@ -125,7 +125,7 @@ int erofs_load_compress_hints(struct erofs_sb_info *sbi)
} else {
ccfg = atoi(alg);
if (ccfg >= EROFS_MAX_COMPR_CFGS ||
- !cfg.c_compr_alg[ccfg]) {
+ !cfg.c_compr_opts[ccfg].alg) {
erofs_err("invalid compressing configuration \"%s\" at line %u",
alg, line);
ret = -EINVAL;
diff --git a/lib/compressor.c b/lib/compressor.c
index 7ec51c2..5061417 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -78,7 +78,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
}
int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
- char *alg_name, int compression_level)
+ char *alg_name, int compression_level, u32 dict_size)
{
int ret, i;
@@ -106,6 +106,8 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
if (erofs_algs[i].c->setlevel) {
ret = erofs_algs[i].c->setlevel(c, compression_level);
+ if (ret)
+ return ret;
} else {
if (compression_level >= 0)
erofs_warn(
@@ -113,6 +115,19 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
compression_level, alg_name);
c->compression_level = 0;
}
+
+ if (erofs_algs[i].c->setdictsize) {
+ ret = erofs_algs[i].c->setdictsize(c, dict_size);
+ if (ret)
+ return ret;
+ } else {
+ if (dict_size)
+ erofs_warn(
+ "dictionary size %u is ignored for %s",
+ dict_size, alg_name);
+ c->dict_size = 0;
+ }
+
if (!ret) {
c->alg = &erofs_algs[i];
return 0;
diff --git a/lib/compressor.h b/lib/compressor.h
index ec5485d..d8ccf2e 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -14,10 +14,13 @@ struct erofs_compress;
struct erofs_compressor {
int default_level;
int best_level;
+ u32 default_dictsize;
+ u32 max_dictsize;
int (*init)(struct erofs_compress *c);
int (*exit)(struct erofs_compress *c);
int (*setlevel)(struct erofs_compress *c, int compression_level);
+ int (*setdictsize)(struct erofs_compress *c, u32 dict_size);
int (*compress_destsize)(const struct erofs_compress *c,
const void *src, unsigned int *srcsize,
@@ -39,6 +42,7 @@ struct erofs_compress {
unsigned int compress_threshold;
unsigned int compression_level;
+ unsigned int dict_size;
void *private_data;
};
@@ -56,7 +60,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
void *dst, unsigned int dstsize);
int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
- char *alg_name, int compression_level);
+ char *alg_name, int compression_level, u32 dict_size);
int erofs_compressor_exit(struct erofs_compress *c);
#endif
diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
index 4e5902e..c95e53e 100644
--- a/lib/compressor_deflate.c
+++ b/lib/compressor_deflate.c
@@ -46,6 +46,16 @@ static int compressor_deflate_init(struct erofs_compress *c)
static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
int compression_level)
+{
+ if (compression_level < 0)
+ compression_level = erofs_compressor_deflate.default_level;
+
+ c->compression_level = compression_level;
+ return 0;
+}
+
+static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
+ u32 dict_size)
{
void *s;
@@ -54,23 +64,31 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
c->private_data = NULL;
}
- if (compression_level < 0)
- compression_level = erofs_compressor_deflate.default_level;
+ if (dict_size > erofs_compressor_deflate.max_dictsize) {
+ erofs_err("dict size %u is too large", dict_size);
+ return -EINVAL;
+ }
+
+ if (dict_size == 0)
+ dict_size = erofs_compressor_deflate.default_dictsize;
- s = kite_deflate_init(compression_level, cfg.c_dict_size);
+ s = kite_deflate_init(c->compression_level, dict_size);
if (IS_ERR(s))
return PTR_ERR(s);
c->private_data = s;
- c->compression_level = compression_level;
+ c->dict_size = dict_size;
return 0;
}
const struct erofs_compressor erofs_compressor_deflate = {
.default_level = 1,
.best_level = 9,
+ .default_dictsize = Z_EROFS_DEFLATE_DEFULT_DICT_SIZE,
+ .max_dictsize = Z_EROFS_DEFLATE_MAX_DICT_SIZE,
.init = compressor_deflate_init,
.exit = compressor_deflate_exit,
.setlevel = erofs_compressor_deflate_setlevel,
+ .setdictsize = erofs_compressor_deflate_setdictsize,
.compress_destsize = deflate_compress_destsize,
};
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
index 0ed6f23..5c7f830 100644
--- a/lib/compressor_liblzma.c
+++ b/lib/compressor_liblzma.c
@@ -68,22 +68,28 @@ static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
if (lzma_lzma_preset(&ctx->opt, preset))
return -EINVAL;
- /* XXX: temporary hack */
- if (cfg.c_dict_size) {
- if (cfg.c_dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) {
- erofs_err("dict size %u is too large", cfg.c_dict_size);
- return -EINVAL;
- }
- ctx->opt.dict_size = cfg.c_dict_size;
- } else {
- if (ctx->opt.dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE)
- ctx->opt.dict_size = Z_EROFS_LZMA_MAX_DICT_SIZE;
- cfg.c_dict_size = ctx->opt.dict_size;
- }
c->compression_level = compression_level;
return 0;
}
+static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
+ u32 dict_size)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+
+ if (dict_size > erofs_compressor_lzma.max_dictsize) {
+ erofs_err("dict size %u is too large", dict_size);
+ return -EINVAL;
+ }
+
+ if (dict_size == 0)
+ dict_size = erofs_compressor_lzma.default_dictsize;
+
+ ctx->opt.dict_size = dict_size;
+ c->dict_size = dict_size;
+ return 0;
+}
+
static int erofs_compressor_liblzma_init(struct erofs_compress *c)
{
struct erofs_liblzma_context *ctx;
@@ -101,9 +107,12 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c)
const struct erofs_compressor erofs_compressor_lzma = {
.default_level = LZMA_PRESET_DEFAULT,
.best_level = 109,
+ .default_dictsize = Z_EROFS_LZMA_DEFAULT_DICT_SIZE,
+ .max_dictsize = Z_EROFS_LZMA_MAX_DICT_SIZE,
.init = erofs_compressor_liblzma_init,
.exit = erofs_compressor_liblzma_exit,
.setlevel = erofs_compressor_liblzma_setlevel,
+ .setdictsize = erofs_compressor_liblzma_setdictsize,
.compress_destsize = erofs_liblzma_compress_destsize,
};
#endif
diff --git a/lib/config.c b/lib/config.c
index 0cddc95..d3e8e53 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -58,8 +58,8 @@ void erofs_exit_configure(void)
free(cfg.c_img_path);
if (cfg.c_src_path)
free(cfg.c_src_path);
- for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_alg[i]; i++)
- free(cfg.c_compr_alg[i]);
+ for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_opts[i].alg; i++)
+ free(cfg.c_compr_opts[i].alg);
}
static unsigned int fullpath_prefix; /* root directory prefix length */
diff --git a/lib/inode.c b/lib/inode.c
index bcdb4b8..c6424c0 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -492,7 +492,7 @@ int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
return erofs_blob_write_chunked_file(inode, fd, fpos);
}
- if (cfg.c_compr_alg[0] && erofs_file_is_compressible(inode)) {
+ if (cfg.c_compr_opts[0].alg && erofs_file_is_compressible(inode)) {
ret = erofs_write_compressed_file(inode, fd);
if (!ret || ret != -ENOSPC)
return ret;
diff --git a/lib/kite_deflate.c b/lib/kite_deflate.c
index 8667954..2357f76 100644
--- a/lib/kite_deflate.c
+++ b/lib/kite_deflate.c
@@ -5,6 +5,7 @@
* Copyright (C) 2023, Alibaba Cloud
* Copyright (C) 2023, Gao Xiang <xiang@kernel.org>
*/
+#include "erofs/internal.h"
#include "erofs/defs.h"
#include "erofs/print.h"
#include <errno.h>
@@ -22,7 +23,7 @@ unsigned long erofs_memcmp2(const u8 *s1, const u8 *s2,
#define kite_dbg(x, ...)
#endif
-#define kHistorySize32 (1U << 15)
+#define kHistorySize32 Z_EROFS_DEFLATE_DEFULT_DICT_SIZE
#define kNumLenSymbols32 256
#define kNumLenSymbolsMax kNumLenSymbols32
diff --git a/mkfs/main.c b/mkfs/main.c
index 0517849..b046c6f 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -5,6 +5,7 @@
* Created by Li Guifu <bluce.liguifu@huawei.com>
*/
#define _GNU_SOURCE
+#include <ctype.h>
#include <time.h>
#include <sys/time.h>
#include <stdlib.h>
@@ -108,24 +109,29 @@ static void usage(int argc, char **argv)
" -b# set block size to # (# = page size by default)\n"
" -d<0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n"
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
- " -zX[,Y][:...] X=compressor (Y=compression level, optional)\n"
- " alternative compressors can be separated by colons(:)\n"
- " supported compressors and their level ranges are:\n",
+ " -zX[,level=Y] X=compressor (Y=compression level, Z=dictionary size, optional)\n"
+ " [,dictsize=Z] alternative compressors can be separated by colons(:)\n"
+ " [:...] supported compressors and their option ranges are:\n",
argv[0], EROFS_WARN);
while ((s = z_erofs_list_available_compressors(&i)) != NULL) {
- printf(" %s", s->name);
+ char * const spaces = " ";
+
+ printf("%s%s\n", spaces, s->name);
if (s->c->setlevel) {
if (!strcmp(s->name, "lzma"))
/* A little kludge to show the range as disjointed
* "0-9,100-109" instead of a continuous "0-109", and to
* state what those two subranges respectively mean. */
- printf("[<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)",
- s->c->default_level);
+ printf("%s [,level=<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)\n",
+ spaces, s->c->default_level);
else
- printf("[,<0-%i>]\t(default=%i)",
- s->c->best_level, s->c->default_level);
+ printf("%s [,level=<0-%i>]\t\t(default=%i)\n",
+ spaces, s->c->best_level, s->c->default_level);
+ }
+ if (s->c->setdictsize) {
+ printf("%s [,dictsize=<dictsize>]\t(default=%u, max=%u)\n",
+ spaces, s->c->default_dictsize, s->c->max_dictsize);
}
- putchar('\n');
}
printf(
" -C# specify the size of compress physical cluster in bytes\n"
@@ -310,20 +316,78 @@ static int mkfs_parse_compress_algs(char *algs)
char *s;
for (s = strtok(algs, ":"), i = 0; s; s = strtok(NULL, ":"), ++i) {
- const char *lv;
+ char *p, *q, *opt, *endptr;
+ struct erofs_compr_cfg *ccfg;
if (i >= EROFS_MAX_COMPR_CFGS - 1) {
erofs_err("too many algorithm types");
return -EINVAL;
}
- lv = strchr(s, ',');
- if (lv) {
- cfg.c_compr_level[i] = atoi(lv + 1);
- cfg.c_compr_alg[i] = strndup(s, lv - s);
+ ccfg = cfg.c_compr_opts + i;
+
+ ccfg->level = -1;
+ ccfg->dict_size = 0;
+
+ p = strchr(s, ',');
+ if (p) {
+ ccfg->alg = strndup(s, p - s);
+
+ /* backward compatibility */
+ if (isdigit(*(p + 1))) {
+ ccfg->level = strtol(p + 1, &endptr, 10);
+ if (*endptr && *endptr != ',') {
+ erofs_err(
+ "invalid compression level %s",
+ p + 1);
+ return -EINVAL;
+ }
+ continue;
+ }
} else {
- cfg.c_compr_level[i] = -1;
- cfg.c_compr_alg[i] = strdup(s);
+ ccfg->alg = strdup(s);
+ continue;
+ }
+
+ opt = p + 1;
+ while (opt) {
+ q = strchr(opt, ',');
+ if (q)
+ *q = '\0';
+
+ if ((p = strstr(opt, "level="))) {
+ p += strlen("level=");
+ ccfg->level = strtol(p, &endptr, 10);
+ if ((endptr == p) ||
+ (*endptr && *endptr != ',')) {
+ erofs_err(
+ "invalid compression level %s",
+ p);
+ return -EINVAL;
+ }
+ } else if ((p = strstr(opt, "dictsize="))) {
+ p += strlen("dictsize=");
+ ccfg->dict_size = strtoul(p, &endptr, 10);
+ if (*endptr == 'k' || *endptr == 'K')
+ ccfg->dict_size <<= 10;
+ else if (*endptr == 'm' || *endptr == 'M')
+ ccfg->dict_size <<= 20;
+ else if ((endptr == p) ||
+ (*endptr && *endptr != ',')) {
+ erofs_err(
+ "invalid compression dictsize %s",
+ p);
+ return -EINVAL;
+ }
+ } else {
+ erofs_err("invalid compression option %s", opt);
+ return -EINVAL;
+ }
+
+ if (q)
+ opt = q + 1;
+ else
+ opt = NULL;
}
}
return 0;
@@ -692,7 +756,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
cfg.c_showprogress = false;
}
- if (cfg.c_compr_alg[0] && erofs_blksiz(&sbi) != getpagesize())
+ if (cfg.c_compr_opts[0].alg && erofs_blksiz(&sbi) != getpagesize())
erofs_warn("Please note that subpage blocksize with compression isn't yet supported in kernel. "
"This compressed image will only work with bs = ps = %u bytes",
erofs_blksiz(&sbi));
@@ -1119,7 +1183,7 @@ int main(int argc, char **argv)
}
if (cfg.c_dedupe) {
- if (!cfg.c_compr_alg[0]) {
+ if (!cfg.c_compr_opts[0].alg) {
erofs_err("Compression is not enabled. Turn on chunk-based data deduplication instead.");
cfg.c_chunkbits = sbi.blkszbits;
} else {
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init()
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (2 preceding siblings ...)
2024-01-08 6:00 ` [PATCH v2 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
@ 2024-01-13 17:39 ` Gao Xiang
2024-01-19 12:46 ` [PATCH v3 0/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
` (6 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Gao Xiang @ 2024-01-13 17:39 UTC (permalink / raw)
To: Yifan Zhao; +Cc: linux-erofs
Hi Yifan,
On Sun, Jan 07, 2024 at 02:10:40AM +0800, Yifan Zhao wrote:
> Currently erofs_compressor_setlevel() is only called once just after
> erofs_compressor_init() while initializing compressors. Let's just hide
> this interface and set the compression level in erofs_compressor_init().
>
> BTW, if the user gives a compression level to an algorithm which doesn't
> support it, let's report a warning rather than early aborting. Besides,
Really, we should error out invalid standard compression levels
instead of just ignoring.
Thanks,
Gao Xiang
> we do not need to assign the {default,best}_level for such an algorithm
> in erofs_compressor struct.
>
> Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
> ---
> lib/compress.c | 6 +-----
> lib/compressor.c | 28 ++++++++++++++--------------
> lib/compressor.h | 5 ++---
> lib/compressor_lz4.c | 2 --
> 4 files changed, 17 insertions(+), 24 deletions(-)
>
> diff --git a/lib/compress.c b/lib/compress.c
> index 8f61f92..216aeb4 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -1199,11 +1199,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
> for (i = 0; cfg.c_compr_alg[i]; ++i) {
> struct erofs_compress *c = &erofs_ccfg[i].handle;
>
> - ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i]);
> - if (ret)
> - return ret;
> -
> - ret = erofs_compressor_setlevel(c, cfg.c_compr_level[i]);
> + ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i], cfg.c_compr_level[i]);
> if (ret)
> return ret;
>
> diff --git a/lib/compressor.c b/lib/compressor.c
> index a71436a..7ec51c2 100644
> --- a/lib/compressor.c
> +++ b/lib/compressor.c
> @@ -77,20 +77,8 @@ int erofs_compress_destsize(const struct erofs_compress *c,
> return c->alg->c->compress_destsize(c, src, srcsize, dst, dstsize);
> }
>
> -int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level)
> -{
> - DBG_BUGON(!c->alg);
> - if (c->alg->c->setlevel)
> - return c->alg->c->setlevel(c, compression_level);
> -
> - if (compression_level >= 0)
> - return -EINVAL;
> - c->compression_level = 0;
> - return 0;
> -}
> -
> -int erofs_compressor_init(struct erofs_sb_info *sbi,
> - struct erofs_compress *c, char *alg_name)
> +int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
> + char *alg_name, int compression_level)
> {
> int ret, i;
>
> @@ -113,6 +101,18 @@ int erofs_compressor_init(struct erofs_sb_info *sbi,
> continue;
>
> ret = erofs_algs[i].c->init(c);
> + if (ret)
> + return ret;
> +
> + if (erofs_algs[i].c->setlevel) {
> + ret = erofs_algs[i].c->setlevel(c, compression_level);
> + } else {
> + if (compression_level >= 0)
> + erofs_warn(
> + "compression level %d is ignored for %s",
> + compression_level, alg_name);
> + c->compression_level = 0;
> + }
> if (!ret) {
> c->alg = &erofs_algs[i];
> return 0;
> diff --git a/lib/compressor.h b/lib/compressor.h
> index 6875cf1..ec5485d 100644
> --- a/lib/compressor.h
> +++ b/lib/compressor.h
> @@ -55,9 +55,8 @@ int erofs_compress_destsize(const struct erofs_compress *c,
> const void *src, unsigned int *srcsize,
> void *dst, unsigned int dstsize);
>
> -int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
> -int erofs_compressor_init(struct erofs_sb_info *sbi,
> - struct erofs_compress *c, char *alg_name);
> +int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
> + char *alg_name, int compression_level);
> int erofs_compressor_exit(struct erofs_compress *c);
>
> #endif
> diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
> index 6677693..f4e72c3 100644
> --- a/lib/compressor_lz4.c
> +++ b/lib/compressor_lz4.c
> @@ -37,8 +37,6 @@ static int compressor_lz4_init(struct erofs_compress *c)
> }
>
> const struct erofs_compressor erofs_compressor_lz4 = {
> - .default_level = 0,
> - .best_level = 0,
> .init = compressor_lz4_init,
> .exit = compressor_lz4_exit,
> .compress_destsize = lz4_compress_destsize,
> --
> 2.43.0
>
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v3 0/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (3 preceding siblings ...)
2024-01-13 17:39 ` [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Gao Xiang
@ 2024-01-19 12:46 ` Yifan Zhao
2024-01-19 12:46 ` [PATCH v3 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (5 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Yifan Zhao @ 2024-01-19 12:46 UTC (permalink / raw)
To: linux-erofs
This patchset allows to specify dictionary size for compression algorithms.
change since v2:
- does not touch kite_deflate.c
- does not add unneeded macro definitions unrelated to on-disk format
- modify the hacky way to get max_dict_size in z_erofs_build_compr_cfgs()
- some rename, reorganize and code style fix
Yifan Zhao (3):
erofs-utils: mkfs: merge erofs_compressor_setlevel() into
erofs_compressor_init()
erofs-utils: mkfs: allow to specify dictionary size for compression
algorithms
erofs-utils: mkfs: reorganize logic in erofs_compressor_init()
include/erofs/config.h | 10 ++--
lib/compress.c | 37 +++++++------
lib/compress_hints.c | 2 +-
lib/compressor.c | 45 +++++++++++-----
lib/compressor.h | 9 ++--
lib/compressor_deflate.c | 40 ++++++++++----
lib/compressor_libdeflate.c | 14 +++--
lib/compressor_liblzma.c | 54 ++++++++++++-------
lib/compressor_lz4.c | 2 -
lib/compressor_lz4hc.c | 5 +-
lib/config.c | 4 +-
lib/inode.c | 2 +-
mkfs/main.c | 104 ++++++++++++++++++++++++++++--------
13 files changed, 229 insertions(+), 99 deletions(-)
Interdiff against v2:
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 7dbfd1e..eecf575 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -34,7 +34,7 @@ enum {
#define EROFS_MAX_COMPR_CFGS 64
-struct erofs_compr_cfg {
+struct erofs_compr_opts {
char *alg;
int level;
u32 dict_size;
@@ -70,7 +70,7 @@ struct erofs_configure {
char *c_src_path;
char *c_blobdev_path;
char *c_compress_hints_file;
- struct erofs_compr_cfg c_compr_opts[EROFS_MAX_COMPR_CFGS];
+ struct erofs_compr_opts c_compr_opts[EROFS_MAX_COMPR_CFGS];
char c_force_inodeversion;
char c_force_chunkformat;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index 72f0ca6..eba6c26 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -322,7 +322,6 @@ struct z_erofs_lzma_cfgs {
u8 reserved[8];
} __packed;
-#define Z_EROFS_LZMA_DEFAULT_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
/* 6 bytes (+ length field = 8 bytes) */
@@ -331,9 +330,6 @@ struct z_erofs_deflate_cfgs {
u8 reserved[5];
} __packed;
-#define Z_EROFS_DEFLATE_DEFULT_DICT_SIZE (1U << 15)
-#define Z_EROFS_DEFLATE_MAX_DICT_SIZE (1U << 15)
-
/*
* bit 0 : COMPACTED_2B indexes (0 - off; 1 - on)
* e.g. for 4k logical cluster size, 4B if compacted 2B is off;
diff --git a/lib/compress.c b/lib/compress.c
index b7ee9ec..ea9d00d 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1124,11 +1124,10 @@ err_free_meta:
static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
struct erofs_buffer_head *sb_bh,
- struct erofs_compress_cfg *ccfg)
+ u32 *max_dict_size)
{
struct erofs_buffer_head *bh = sb_bh;
- int i, ret = 0;
- u32 dict_size = 0;
+ int ret = 0;
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZ4)) {
struct {
@@ -1155,22 +1154,15 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
}
#ifdef HAVE_LIBLZMA
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZMA)) {
- for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
- if (ccfg[i].enable &&
- ccfg[i].algorithmtype == Z_EROFS_COMPRESSION_LZMA) {
- dict_size = ccfg[i].handle.dict_size;
- break;
- }
- }
- DBG_BUGON(!dict_size);
-
struct {
__le16 size;
struct z_erofs_lzma_cfgs lzma;
} __packed lzmaalg = {
.size = cpu_to_le16(sizeof(struct z_erofs_lzma_cfgs)),
.lzma = {
- .dict_size = cpu_to_le32(dict_size),
+ .dict_size = cpu_to_le32(
+ max_dict_size
+ [Z_EROFS_COMPRESSION_LZMA]),
}
};
@@ -1186,23 +1178,15 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
}
#endif
if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_DEFLATE)) {
- for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++) {
- if (ccfg[i].enable &&
- ccfg[i].algorithmtype == Z_EROFS_COMPRESSION_DEFLATE) {
- dict_size = ccfg[i].handle.dict_size;
- break;
- }
- }
- DBG_BUGON(!dict_size);
-
struct {
__le16 size;
struct z_erofs_deflate_cfgs z;
} __packed zalg = {
.size = cpu_to_le16(sizeof(struct z_erofs_deflate_cfgs)),
.z = {
- .windowbits =
- cpu_to_le32(ilog2(dict_size)),
+ .windowbits = cpu_to_le32(ilog2(
+ max_dict_size
+ [Z_EROFS_COMPRESSION_DEFLATE])),
}
};
@@ -1222,6 +1206,7 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *sb_bh)
{
int i, ret;
+ u32 max_dict_size[Z_EROFS_COMPRESSION_MAX];
for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
struct erofs_compress *c = &erofs_ccfg[i].handle;
@@ -1238,6 +1223,9 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
sbi->available_compr_algs |= 1 << erofs_ccfg[i].algorithmtype;
if (erofs_ccfg[i].algorithmtype != Z_EROFS_COMPRESSION_LZ4)
erofs_sb_set_compr_cfgs(sbi);
+ max_dict_size[erofs_ccfg[i].algorithmtype] =
+ max(max_dict_size[erofs_ccfg[i].algorithmtype],
+ c->dict_size);
}
/*
@@ -1271,7 +1259,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
}
if (erofs_sb_has_compr_cfgs(sbi))
- return z_erofs_build_compr_cfgs(sbi, sb_bh, erofs_ccfg);
+ return z_erofs_build_compr_cfgs(sbi, sb_bh, max_dict_size);
return 0;
}
diff --git a/lib/compressor.c b/lib/compressor.c
index 27b4077..290746e 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -102,26 +102,28 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
if (erofs_algs[i].c->setlevel) {
ret = erofs_algs[i].c->setlevel(c, compression_level);
- if (ret)
- return ret;
- } else {
- if (compression_level >= 0)
- erofs_warn(
- "compression level %d is ignored for %s",
+ if (ret) {
+ erofs_err("failed to set compression level %d for %s",
compression_level, alg_name);
- c->compression_level = 0;
+ return ret;
+ }
+ } else if (compression_level >= 0) {
+ erofs_err("compression level is not supported for %s",
+ alg_name);
+ return -EINVAL;
}
if (erofs_algs[i].c->setdictsize) {
ret = erofs_algs[i].c->setdictsize(c, dict_size);
- if (ret)
- return ret;
- } else {
- if (dict_size)
- erofs_warn(
- "dictionary size %u is ignored for %s",
+ if (ret) {
+ erofs_err("failed to set dict size %u for %s",
dict_size, alg_name);
- c->dict_size = 0;
+ return ret;
+ }
+ } else if (dict_size) {
+ erofs_err("dict size is not supported for %s",
+ alg_name);
+ return -EINVAL;
}
ret = erofs_algs[i].c->init(c);
diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
index d9f8a91..479ad56 100644
--- a/lib/compressor_deflate.c
+++ b/lib/compressor_deflate.c
@@ -84,8 +84,8 @@ static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
const struct erofs_compressor erofs_compressor_deflate = {
.default_level = 1,
.best_level = 9,
- .default_dictsize = Z_EROFS_DEFLATE_DEFULT_DICT_SIZE,
- .max_dictsize = Z_EROFS_DEFLATE_MAX_DICT_SIZE,
+ .default_dictsize = 1 << 15,
+ .max_dictsize = 1 << 15,
.init = compressor_deflate_init,
.exit = compressor_deflate_exit,
.setlevel = erofs_compressor_deflate_setlevel,
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
index 0203a5c..b8ae29c 100644
--- a/lib/compressor_liblzma.c
+++ b/lib/compressor_liblzma.c
@@ -73,7 +73,8 @@ static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
if (dict_size == 0)
dict_size = erofs_compressor_lzma.default_dictsize;
- if (dict_size > erofs_compressor_lzma.max_dictsize) {
+ if (dict_size > erofs_compressor_lzma.max_dictsize ||
+ dict_size < 4096) {
erofs_err("invalid dict size %u", dict_size);
return -EINVAL;
}
@@ -111,7 +112,7 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c)
const struct erofs_compressor erofs_compressor_lzma = {
.default_level = LZMA_PRESET_DEFAULT,
.best_level = 109,
- .default_dictsize = Z_EROFS_LZMA_DEFAULT_DICT_SIZE,
+ .default_dictsize = 8 * Z_EROFS_PCLUSTER_MAX_SIZE,
.max_dictsize = Z_EROFS_LZMA_MAX_DICT_SIZE,
.init = erofs_compressor_liblzma_init,
.exit = erofs_compressor_liblzma_exit,
diff --git a/lib/kite_deflate.c b/lib/kite_deflate.c
index 2357f76..8667954 100644
--- a/lib/kite_deflate.c
+++ b/lib/kite_deflate.c
@@ -5,7 +5,6 @@
* Copyright (C) 2023, Alibaba Cloud
* Copyright (C) 2023, Gao Xiang <xiang@kernel.org>
*/
-#include "erofs/internal.h"
#include "erofs/defs.h"
#include "erofs/print.h"
#include <errno.h>
@@ -23,7 +22,7 @@ unsigned long erofs_memcmp2(const u8 *s1, const u8 *s2,
#define kite_dbg(x, ...)
#endif
-#define kHistorySize32 Z_EROFS_DEFLATE_DEFULT_DICT_SIZE
+#define kHistorySize32 (1U << 15)
#define kNumLenSymbols32 256
#define kNumLenSymbolsMax kNumLenSymbols32
diff --git a/mkfs/main.c b/mkfs/main.c
index 8a9c3cc..acb2108 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -310,43 +310,31 @@ handle_fragment:
return 0;
}
-static int mkfs_parse_compress_algs(char *algs)
+static int mkfs_parse_one_compress_alg(char *alg,
+ struct erofs_compr_opts *copts)
{
- unsigned int i;
- char *s;
-
- for (s = strtok(algs, ":"), i = 0; s; s = strtok(NULL, ":"), ++i) {
char *p, *q, *opt, *endptr;
- struct erofs_compr_cfg *ccfg;
- if (i >= EROFS_MAX_COMPR_CFGS - 1) {
- erofs_err("too many algorithm types");
- return -EINVAL;
- }
+ copts->level = -1;
+ copts->dict_size = 0;
- ccfg = cfg.c_compr_opts + i;
-
- ccfg->level = -1;
- ccfg->dict_size = 0;
-
- p = strchr(s, ',');
+ p = strchr(alg, ',');
if (p) {
- ccfg->alg = strndup(s, p - s);
+ copts->alg = strndup(alg, p - alg);
- /* backward compatibility */
+ /* support old '-zlzma,9' form */
if (isdigit(*(p + 1))) {
- ccfg->level = strtol(p + 1, &endptr, 10);
+ copts->level = strtol(p + 1, &endptr, 10);
if (*endptr && *endptr != ',') {
- erofs_err(
- "invalid compression level %s",
+ erofs_err("invalid compression level %s",
p + 1);
return -EINVAL;
}
- continue;
+ return 0;
}
} else {
- ccfg->alg = strdup(s);
- continue;
+ copts->alg = strdup(alg);
+ return 0;
}
opt = p + 1;
@@ -357,26 +345,20 @@ static int mkfs_parse_compress_algs(char *algs)
if ((p = strstr(opt, "level="))) {
p += strlen("level=");
- ccfg->level = strtol(p, &endptr, 10);
- if ((endptr == p) ||
- (*endptr && *endptr != ',')) {
- erofs_err(
- "invalid compression level %s",
- p);
+ copts->level = strtol(p, &endptr, 10);
+ if ((endptr == p) || (*endptr && *endptr != ',')) {
+ erofs_err("invalid compression level %s", p);
return -EINVAL;
}
} else if ((p = strstr(opt, "dictsize="))) {
p += strlen("dictsize=");
- ccfg->dict_size = strtoul(p, &endptr, 10);
+ copts->dict_size = strtoul(p, &endptr, 10);
if (*endptr == 'k' || *endptr == 'K')
- ccfg->dict_size <<= 10;
+ copts->dict_size <<= 10;
else if (*endptr == 'm' || *endptr == 'M')
- ccfg->dict_size <<= 20;
- else if ((endptr == p) ||
- (*endptr && *endptr != ',')) {
- erofs_err(
- "invalid compression dictsize %s",
- p);
+ copts->dict_size <<= 20;
+ else if ((endptr == p) || (*endptr && *endptr != ',')) {
+ erofs_err("invalid compression dictsize %s", p);
return -EINVAL;
}
} else {
@@ -384,11 +366,27 @@ static int mkfs_parse_compress_algs(char *algs)
return -EINVAL;
}
- if (q)
- opt = q + 1;
- else
- opt = NULL;
+ opt = q ? q + 1 : NULL;
}
+
+ return 0;
+}
+
+static int mkfs_parse_compress_algs(char *algs)
+{
+ unsigned int i;
+ char *s;
+ int ret;
+
+ for (s = strtok(algs, ":"), i = 0; s; s = strtok(NULL, ":"), ++i) {
+ if (i >= EROFS_MAX_COMPR_CFGS - 1) {
+ erofs_err("too many algorithm types");
+ return -EINVAL;
+ }
+
+ ret = mkfs_parse_one_compress_alg(s, &cfg.c_compr_opts[i]);
+ if (ret)
+ return ret;
}
return 0;
}
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init()
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (4 preceding siblings ...)
2024-01-19 12:46 ` [PATCH v3 0/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
@ 2024-01-19 12:46 ` Yifan Zhao
2024-01-19 12:46 ` [PATCH v3 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
` (4 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Yifan Zhao @ 2024-01-19 12:46 UTC (permalink / raw)
To: linux-erofs
Currently erofs_compressor_setlevel() is only called once just after
erofs_compressor_init() while initializing compressors. Let's just hide
this interface and set the compression level in erofs_compressor_init().
Besides, we do not need to assign the {default,best}_level for an
algorithm which does not support the compression level in its
erofs_compressor struct.
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
lib/compress.c | 6 +-----
lib/compressor.c | 31 +++++++++++++++++--------------
lib/compressor.h | 5 ++---
lib/compressor_lz4.c | 2 --
4 files changed, 20 insertions(+), 24 deletions(-)
diff --git a/lib/compress.c b/lib/compress.c
index 22782cb..3ea735c 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1206,11 +1206,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
for (i = 0; cfg.c_compr_alg[i]; ++i) {
struct erofs_compress *c = &erofs_ccfg[i].handle;
- ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i]);
- if (ret)
- return ret;
-
- ret = erofs_compressor_setlevel(c, cfg.c_compr_level[i]);
+ ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i], cfg.c_compr_level[i]);
if (ret)
return ret;
diff --git a/lib/compressor.c b/lib/compressor.c
index a71436a..295aa47 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -77,20 +77,8 @@ int erofs_compress_destsize(const struct erofs_compress *c,
return c->alg->c->compress_destsize(c, src, srcsize, dst, dstsize);
}
-int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level)
-{
- DBG_BUGON(!c->alg);
- if (c->alg->c->setlevel)
- return c->alg->c->setlevel(c, compression_level);
-
- if (compression_level >= 0)
- return -EINVAL;
- c->compression_level = 0;
- return 0;
-}
-
-int erofs_compressor_init(struct erofs_sb_info *sbi,
- struct erofs_compress *c, char *alg_name)
+int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
+ char *alg_name, int compression_level)
{
int ret, i;
@@ -113,6 +101,21 @@ int erofs_compressor_init(struct erofs_sb_info *sbi,
continue;
ret = erofs_algs[i].c->init(c);
+ if (ret)
+ return ret;
+
+ if (erofs_algs[i].c->setlevel) {
+ ret = erofs_algs[i].c->setlevel(c, compression_level);
+ if (ret) {
+ erofs_err("failed to set compression level %d for %s",
+ compression_level, alg_name);
+ return ret;
+ }
+ } else if (compression_level >= 0) {
+ erofs_err("compression level is not supported for %s",
+ alg_name);
+ return -EINVAL;
+ }
if (!ret) {
c->alg = &erofs_algs[i];
return 0;
diff --git a/lib/compressor.h b/lib/compressor.h
index 6875cf1..ec5485d 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -55,9 +55,8 @@ int erofs_compress_destsize(const struct erofs_compress *c,
const void *src, unsigned int *srcsize,
void *dst, unsigned int dstsize);
-int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
-int erofs_compressor_init(struct erofs_sb_info *sbi,
- struct erofs_compress *c, char *alg_name);
+int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
+ char *alg_name, int compression_level);
int erofs_compressor_exit(struct erofs_compress *c);
#endif
diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
index 6677693..f4e72c3 100644
--- a/lib/compressor_lz4.c
+++ b/lib/compressor_lz4.c
@@ -37,8 +37,6 @@ static int compressor_lz4_init(struct erofs_compress *c)
}
const struct erofs_compressor erofs_compressor_lz4 = {
- .default_level = 0,
- .best_level = 0,
.init = compressor_lz4_init,
.exit = compressor_lz4_exit,
.compress_destsize = lz4_compress_destsize,
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (5 preceding siblings ...)
2024-01-19 12:46 ` [PATCH v3 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
@ 2024-01-19 12:46 ` Yifan Zhao
2024-01-20 9:26 ` Gao Xiang
2024-01-19 12:47 ` [PATCH v3 3/3] erofs-utils: mkfs: reorganize logic in erofs_compressor_init() Yifan Zhao
` (3 subsequent siblings)
10 siblings, 1 reply; 15+ messages in thread
From: Yifan Zhao @ 2024-01-19 12:46 UTC (permalink / raw)
To: linux-erofs
Currently, the dictionary size for compression algorithms is fixed. This
patch allows to specify different ones with new -zX,dictsize=<dictsize>
options.
This patch also changes the way to specify compression levels. Now, the
compression level is specified with -zX,level=<level> options and could
be specified together with dictsize. The old -zX,<level> form is still
supported for compatibility.
Suggested-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
include/erofs/config.h | 10 ++--
lib/compress.c | 33 ++++++++-----
lib/compress_hints.c | 2 +-
lib/compressor.c | 16 +++++-
lib/compressor.h | 6 ++-
lib/compressor_deflate.c | 26 ++++++++--
lib/compressor_liblzma.c | 32 +++++++-----
lib/config.c | 4 +-
lib/inode.c | 2 +-
mkfs/main.c | 104 +++++++++++++++++++++++++++++++--------
10 files changed, 179 insertions(+), 56 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 89fe522..eecf575 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -34,6 +34,12 @@ enum {
#define EROFS_MAX_COMPR_CFGS 64
+struct erofs_compr_opts {
+ char *alg;
+ int level;
+ u32 dict_size;
+};
+
struct erofs_configure {
const char *c_version;
int c_dbg_lvl;
@@ -64,8 +70,7 @@ struct erofs_configure {
char *c_src_path;
char *c_blobdev_path;
char *c_compress_hints_file;
- char *c_compr_alg[EROFS_MAX_COMPR_CFGS];
- int c_compr_level[EROFS_MAX_COMPR_CFGS];
+ struct erofs_compr_opts c_compr_opts[EROFS_MAX_COMPR_CFGS];
char c_force_inodeversion;
char c_force_chunkformat;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
@@ -73,7 +78,6 @@ struct erofs_configure {
u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;
u32 c_max_decompressed_extent_bytes;
- u32 c_dict_size;
u64 c_unix_timestamp;
u32 c_uid, c_gid;
const char *mount_point;
diff --git a/lib/compress.c b/lib/compress.c
index 3ea735c..ea9d00d 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1123,7 +1123,8 @@ err_free_meta:
}
static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
- struct erofs_buffer_head *sb_bh)
+ struct erofs_buffer_head *sb_bh,
+ u32 *max_dict_size)
{
struct erofs_buffer_head *bh = sb_bh;
int ret = 0;
@@ -1159,7 +1160,9 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
} __packed lzmaalg = {
.size = cpu_to_le16(sizeof(struct z_erofs_lzma_cfgs)),
.lzma = {
- .dict_size = cpu_to_le32(cfg.c_dict_size),
+ .dict_size = cpu_to_le32(
+ max_dict_size
+ [Z_EROFS_COMPRESSION_LZMA]),
}
};
@@ -1181,8 +1184,9 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
} __packed zalg = {
.size = cpu_to_le16(sizeof(struct z_erofs_deflate_cfgs)),
.z = {
- .windowbits =
- cpu_to_le32(ilog2(cfg.c_dict_size)),
+ .windowbits = cpu_to_le32(ilog2(
+ max_dict_size
+ [Z_EROFS_COMPRESSION_DEFLATE])),
}
};
@@ -1202,11 +1206,14 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *sb_bh)
{
int i, ret;
+ u32 max_dict_size[Z_EROFS_COMPRESSION_MAX];
- for (i = 0; cfg.c_compr_alg[i]; ++i) {
+ for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
struct erofs_compress *c = &erofs_ccfg[i].handle;
- ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i], cfg.c_compr_level[i]);
+ ret = erofs_compressor_init(sbi, c, cfg.c_compr_opts[i].alg,
+ cfg.c_compr_opts[i].level,
+ cfg.c_compr_opts[i].dict_size);
if (ret)
return ret;
@@ -1216,17 +1223,21 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
sbi->available_compr_algs |= 1 << erofs_ccfg[i].algorithmtype;
if (erofs_ccfg[i].algorithmtype != Z_EROFS_COMPRESSION_LZ4)
erofs_sb_set_compr_cfgs(sbi);
+ max_dict_size[erofs_ccfg[i].algorithmtype] =
+ max(max_dict_size[erofs_ccfg[i].algorithmtype],
+ c->dict_size);
}
/*
* if primary algorithm is empty (e.g. compression off),
* clear 0PADDING feature for old kernel compatibility.
*/
- if (!cfg.c_compr_alg[0] ||
- (cfg.c_legacy_compress && !strncmp(cfg.c_compr_alg[0], "lz4", 3)))
+ if (!cfg.c_compr_opts[0].alg ||
+ (cfg.c_legacy_compress &&
+ !strncmp(cfg.c_compr_opts[0].alg, "lz4", 3)))
erofs_sb_clear_lz4_0padding(sbi);
- if (!cfg.c_compr_alg[0])
+ if (!cfg.c_compr_opts[0].alg)
return 0;
/*
@@ -1248,7 +1259,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
}
if (erofs_sb_has_compr_cfgs(sbi))
- return z_erofs_build_compr_cfgs(sbi, sb_bh);
+ return z_erofs_build_compr_cfgs(sbi, sb_bh, max_dict_size);
return 0;
}
@@ -1256,7 +1267,7 @@ int z_erofs_compress_exit(void)
{
int i, ret;
- for (i = 0; cfg.c_compr_alg[i]; ++i) {
+ for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
ret = erofs_compressor_exit(&erofs_ccfg[i].handle);
if (ret)
return ret;
diff --git a/lib/compress_hints.c b/lib/compress_hints.c
index afc9f8f..8b78f80 100644
--- a/lib/compress_hints.c
+++ b/lib/compress_hints.c
@@ -125,7 +125,7 @@ int erofs_load_compress_hints(struct erofs_sb_info *sbi)
} else {
ccfg = atoi(alg);
if (ccfg >= EROFS_MAX_COMPR_CFGS ||
- !cfg.c_compr_alg[ccfg]) {
+ !cfg.c_compr_opts[ccfg].alg) {
erofs_err("invalid compressing configuration \"%s\" at line %u",
alg, line);
ret = -EINVAL;
diff --git a/lib/compressor.c b/lib/compressor.c
index 295aa47..9f8d220 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -78,7 +78,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
}
int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
- char *alg_name, int compression_level)
+ char *alg_name, int compression_level, u32 dict_size)
{
int ret, i;
@@ -116,6 +116,20 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
alg_name);
return -EINVAL;
}
+
+ if (erofs_algs[i].c->setdictsize) {
+ ret = erofs_algs[i].c->setdictsize(c, dict_size);
+ if (ret) {
+ erofs_err("failed to set dict size %u for %s",
+ dict_size, alg_name);
+ return ret;
+ }
+ } else if (dict_size) {
+ erofs_err("dict size is not supported for %s",
+ alg_name);
+ return -EINVAL;
+ }
+
if (!ret) {
c->alg = &erofs_algs[i];
return 0;
diff --git a/lib/compressor.h b/lib/compressor.h
index ec5485d..d8ccf2e 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -14,10 +14,13 @@ struct erofs_compress;
struct erofs_compressor {
int default_level;
int best_level;
+ u32 default_dictsize;
+ u32 max_dictsize;
int (*init)(struct erofs_compress *c);
int (*exit)(struct erofs_compress *c);
int (*setlevel)(struct erofs_compress *c, int compression_level);
+ int (*setdictsize)(struct erofs_compress *c, u32 dict_size);
int (*compress_destsize)(const struct erofs_compress *c,
const void *src, unsigned int *srcsize,
@@ -39,6 +42,7 @@ struct erofs_compress {
unsigned int compress_threshold;
unsigned int compression_level;
+ unsigned int dict_size;
void *private_data;
};
@@ -56,7 +60,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
void *dst, unsigned int dstsize);
int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
- char *alg_name, int compression_level);
+ char *alg_name, int compression_level, u32 dict_size);
int erofs_compressor_exit(struct erofs_compress *c);
#endif
diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
index 4e5902e..aa2ff24 100644
--- a/lib/compressor_deflate.c
+++ b/lib/compressor_deflate.c
@@ -46,6 +46,16 @@ static int compressor_deflate_init(struct erofs_compress *c)
static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
int compression_level)
+{
+ if (compression_level < 0)
+ compression_level = erofs_compressor_deflate.default_level;
+
+ c->compression_level = compression_level;
+ return 0;
+}
+
+static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
+ u32 dict_size)
{
void *s;
@@ -54,23 +64,31 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
c->private_data = NULL;
}
- if (compression_level < 0)
- compression_level = erofs_compressor_deflate.default_level;
+ if (dict_size > erofs_compressor_deflate.max_dictsize) {
+ erofs_err("dict size %u is too large", dict_size);
+ return -EINVAL;
+ }
+
+ if (dict_size == 0)
+ dict_size = erofs_compressor_deflate.default_dictsize;
- s = kite_deflate_init(compression_level, cfg.c_dict_size);
+ s = kite_deflate_init(c->compression_level, dict_size);
if (IS_ERR(s))
return PTR_ERR(s);
c->private_data = s;
- c->compression_level = compression_level;
+ c->dict_size = dict_size;
return 0;
}
const struct erofs_compressor erofs_compressor_deflate = {
.default_level = 1,
.best_level = 9,
+ .default_dictsize = 1 << 15,
+ .max_dictsize = 1 << 15,
.init = compressor_deflate_init,
.exit = compressor_deflate_exit,
.setlevel = erofs_compressor_deflate_setlevel,
+ .setdictsize = erofs_compressor_deflate_setdictsize,
.compress_destsize = deflate_compress_destsize,
};
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
index 0ed6f23..a9551e2 100644
--- a/lib/compressor_liblzma.c
+++ b/lib/compressor_liblzma.c
@@ -68,19 +68,26 @@ static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
if (lzma_lzma_preset(&ctx->opt, preset))
return -EINVAL;
- /* XXX: temporary hack */
- if (cfg.c_dict_size) {
- if (cfg.c_dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) {
- erofs_err("dict size %u is too large", cfg.c_dict_size);
- return -EINVAL;
+ c->compression_level = compression_level;
+ return 0;
}
- ctx->opt.dict_size = cfg.c_dict_size;
- } else {
- if (ctx->opt.dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE)
- ctx->opt.dict_size = Z_EROFS_LZMA_MAX_DICT_SIZE;
- cfg.c_dict_size = ctx->opt.dict_size;
+
+static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
+ u32 dict_size)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+
+ if (dict_size > erofs_compressor_lzma.max_dictsize ||
+ dict_size < 4096) {
+ erofs_err("invalid dict size %u", dict_size);
+ return -EINVAL;
}
- c->compression_level = compression_level;
+
+ if (dict_size == 0)
+ dict_size = erofs_compressor_lzma.default_dictsize;
+
+ ctx->opt.dict_size = dict_size;
+ c->dict_size = dict_size;
return 0;
}
@@ -101,9 +108,12 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c)
const struct erofs_compressor erofs_compressor_lzma = {
.default_level = LZMA_PRESET_DEFAULT,
.best_level = 109,
+ .default_dictsize = 8 * Z_EROFS_PCLUSTER_MAX_SIZE,
+ .max_dictsize = Z_EROFS_LZMA_MAX_DICT_SIZE,
.init = erofs_compressor_liblzma_init,
.exit = erofs_compressor_liblzma_exit,
.setlevel = erofs_compressor_liblzma_setlevel,
+ .setdictsize = erofs_compressor_liblzma_setdictsize,
.compress_destsize = erofs_liblzma_compress_destsize,
};
#endif
diff --git a/lib/config.c b/lib/config.c
index aa3dd1f..1096cd1 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -62,8 +62,8 @@ void erofs_exit_configure(void)
free(cfg.c_img_path);
if (cfg.c_src_path)
free(cfg.c_src_path);
- for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_alg[i]; i++)
- free(cfg.c_compr_alg[i]);
+ for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_opts[i].alg; i++)
+ free(cfg.c_compr_opts[i].alg);
}
static unsigned int fullpath_prefix; /* root directory prefix length */
diff --git a/lib/inode.c b/lib/inode.c
index bcdb4b8..c6424c0 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -492,7 +492,7 @@ int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
return erofs_blob_write_chunked_file(inode, fd, fpos);
}
- if (cfg.c_compr_alg[0] && erofs_file_is_compressible(inode)) {
+ if (cfg.c_compr_opts[0].alg && erofs_file_is_compressible(inode)) {
ret = erofs_write_compressed_file(inode, fd);
if (!ret || ret != -ENOSPC)
return ret;
diff --git a/mkfs/main.c b/mkfs/main.c
index 13fea41..acb2108 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -5,6 +5,7 @@
* Created by Li Guifu <bluce.liguifu@huawei.com>
*/
#define _GNU_SOURCE
+#include <ctype.h>
#include <time.h>
#include <sys/time.h>
#include <stdlib.h>
@@ -108,24 +109,29 @@ static void usage(int argc, char **argv)
" -b# set block size to # (# = page size by default)\n"
" -d<0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n"
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
- " -zX[,Y][:...] X=compressor (Y=compression level, optional)\n"
- " alternative compressors can be separated by colons(:)\n"
- " supported compressors and their level ranges are:\n",
+ " -zX[,level=Y] X=compressor (Y=compression level, Z=dictionary size, optional)\n"
+ " [,dictsize=Z] alternative compressors can be separated by colons(:)\n"
+ " [:...] supported compressors and their option ranges are:\n",
argv[0], EROFS_WARN);
while ((s = z_erofs_list_available_compressors(&i)) != NULL) {
- printf(" %s", s->name);
+ char * const spaces = " ";
+
+ printf("%s%s\n", spaces, s->name);
if (s->c->setlevel) {
if (!strcmp(s->name, "lzma"))
/* A little kludge to show the range as disjointed
* "0-9,100-109" instead of a continuous "0-109", and to
* state what those two subranges respectively mean. */
- printf("[<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)",
- s->c->default_level);
+ printf("%s [,level=<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)\n",
+ spaces, s->c->default_level);
else
- printf("[,<0-%i>]\t(default=%i)",
- s->c->best_level, s->c->default_level);
+ printf("%s [,level=<0-%i>]\t\t(default=%i)\n",
+ spaces, s->c->best_level, s->c->default_level);
+ }
+ if (s->c->setdictsize) {
+ printf("%s [,dictsize=<dictsize>]\t(default=%u, max=%u)\n",
+ spaces, s->c->default_dictsize, s->c->max_dictsize);
}
- putchar('\n');
}
printf(
" -C# specify the size of compress physical cluster in bytes\n"
@@ -304,27 +310,83 @@ handle_fragment:
return 0;
}
+static int mkfs_parse_one_compress_alg(char *alg,
+ struct erofs_compr_opts *copts)
+{
+ char *p, *q, *opt, *endptr;
+
+ copts->level = -1;
+ copts->dict_size = 0;
+
+ p = strchr(alg, ',');
+ if (p) {
+ copts->alg = strndup(alg, p - alg);
+
+ /* support old '-zlzma,9' form */
+ if (isdigit(*(p + 1))) {
+ copts->level = strtol(p + 1, &endptr, 10);
+ if (*endptr && *endptr != ',') {
+ erofs_err("invalid compression level %s",
+ p + 1);
+ return -EINVAL;
+ }
+ return 0;
+ }
+ } else {
+ copts->alg = strdup(alg);
+ return 0;
+ }
+
+ opt = p + 1;
+ while (opt) {
+ q = strchr(opt, ',');
+ if (q)
+ *q = '\0';
+
+ if ((p = strstr(opt, "level="))) {
+ p += strlen("level=");
+ copts->level = strtol(p, &endptr, 10);
+ if ((endptr == p) || (*endptr && *endptr != ',')) {
+ erofs_err("invalid compression level %s", p);
+ return -EINVAL;
+ }
+ } else if ((p = strstr(opt, "dictsize="))) {
+ p += strlen("dictsize=");
+ copts->dict_size = strtoul(p, &endptr, 10);
+ if (*endptr == 'k' || *endptr == 'K')
+ copts->dict_size <<= 10;
+ else if (*endptr == 'm' || *endptr == 'M')
+ copts->dict_size <<= 20;
+ else if ((endptr == p) || (*endptr && *endptr != ',')) {
+ erofs_err("invalid compression dictsize %s", p);
+ return -EINVAL;
+ }
+ } else {
+ erofs_err("invalid compression option %s", opt);
+ return -EINVAL;
+ }
+
+ opt = q ? q + 1 : NULL;
+ }
+
+ return 0;
+}
+
static int mkfs_parse_compress_algs(char *algs)
{
unsigned int i;
char *s;
+ int ret;
for (s = strtok(algs, ":"), i = 0; s; s = strtok(NULL, ":"), ++i) {
- const char *lv;
-
if (i >= EROFS_MAX_COMPR_CFGS - 1) {
erofs_err("too many algorithm types");
return -EINVAL;
}
- lv = strchr(s, ',');
- if (lv) {
- cfg.c_compr_level[i] = atoi(lv + 1);
- cfg.c_compr_alg[i] = strndup(s, lv - s);
- } else {
- cfg.c_compr_level[i] = -1;
- cfg.c_compr_alg[i] = strdup(s);
- }
+ ret = mkfs_parse_one_compress_alg(s, &cfg.c_compr_opts[i]);
+ if (ret)
+ return ret;
}
return 0;
}
@@ -692,7 +754,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
cfg.c_showprogress = false;
}
- if (cfg.c_compr_alg[0] && erofs_blksiz(&sbi) != getpagesize())
+ if (cfg.c_compr_opts[0].alg && erofs_blksiz(&sbi) != getpagesize())
erofs_warn("Please note that subpage blocksize with compression isn't yet supported in kernel. "
"This compressed image will only work with bs = ps = %u bytes",
erofs_blksiz(&sbi));
@@ -1119,7 +1181,7 @@ int main(int argc, char **argv)
}
if (cfg.c_dedupe) {
- if (!cfg.c_compr_alg[0]) {
+ if (!cfg.c_compr_opts[0].alg) {
erofs_err("Compression is not enabled. Turn on chunk-based data deduplication instead.");
cfg.c_chunkbits = sbi.blkszbits;
} else {
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH v3 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms
2024-01-19 12:46 ` [PATCH v3 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
@ 2024-01-20 9:26 ` Gao Xiang
0 siblings, 0 replies; 15+ messages in thread
From: Gao Xiang @ 2024-01-20 9:26 UTC (permalink / raw)
To: Yifan Zhao; +Cc: linux-erofs
Hi Yifan,
On Fri, Jan 19, 2024 at 08:46:55PM +0800, Yifan Zhao wrote:
> Currently, the dictionary size for compression algorithms is fixed. This
> patch allows to specify different ones with new -zX,dictsize=<dictsize>
> options.
>
> This patch also changes the way to specify compression levels. Now, the
> compression level is specified with -zX,level=<level> options and could
> be specified together with dictsize. The old -zX,<level> form is still
> supported for compatibility.
>
> Suggested-by: Gao Xiang <hsiangkao@linux.alibaba.com>
> Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
I cannot apply this patch. The error message shows as below:
Applying: erofs-utils: mkfs: merge erofs_compressor_setlevel() into
erofs_compressor_init()
Applying: erofs-utils: mkfs: allow to specify dictionary size for
compression algorithms
Using index info to reconstruct a base tree...
error: patch failed: lib/compressor_liblzma.c:68
error: lib/compressor_liblzma.c: patch does not apply
error: Did you hand edit your patch?
It does not apply to blobs recorded in its index.
Patch failed at 0002 erofs-utils: mkfs: allow to specify dictionary size
for compression algorithms
hint: Use 'git am --show-current-patch=diff' to see the failed patch
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".
Did you use the proper branch (the latest -dev) to form patches?
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH v3 3/3] erofs-utils: mkfs: reorganize logic in erofs_compressor_init()
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (6 preceding siblings ...)
2024-01-19 12:46 ` [PATCH v3 2/3] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
@ 2024-01-19 12:47 ` Yifan Zhao
2024-01-20 11:53 ` [PATCH v4 0/2] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
` (2 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Yifan Zhao @ 2024-01-19 12:47 UTC (permalink / raw)
To: linux-erofs
Currently, the initialization of compressors is weird: init() is called
first, followed by setlevel(), then setdictsize(). The initialization
process occurs in the last called setdictsize() for lzma, deflate, and
libdeflate.
This patch reorders the three functions, with init() now being invoked
last, allowing it to use the compression level and dictsize already set
so that the behavior of the functions matches their names.
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
lib/compressor.c | 8 ++++----
lib/compressor_deflate.c | 30 +++++++++++++++---------------
lib/compressor_libdeflate.c | 14 +++++++++-----
lib/compressor_liblzma.c | 32 ++++++++++++++++++--------------
lib/compressor_lz4hc.c | 5 ++++-
5 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/lib/compressor.c b/lib/compressor.c
index 9f8d220..290746e 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -100,10 +100,6 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
if (!erofs_algs[i].c)
continue;
- ret = erofs_algs[i].c->init(c);
- if (ret)
- return ret;
-
if (erofs_algs[i].c->setlevel) {
ret = erofs_algs[i].c->setlevel(c, compression_level);
if (ret) {
@@ -130,6 +126,10 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
return -EINVAL;
}
+ ret = erofs_algs[i].c->init(c);
+ if (ret)
+ return ret;
+
if (!ret) {
c->alg = &erofs_algs[i];
return 0;
diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
index aa2ff24..479ad56 100644
--- a/lib/compressor_deflate.c
+++ b/lib/compressor_deflate.c
@@ -36,7 +36,14 @@ static int compressor_deflate_exit(struct erofs_compress *c)
static int compressor_deflate_init(struct erofs_compress *c)
{
+ if (c->private_data) {
+ kite_deflate_end(c->private_data);
c->private_data = NULL;
+ }
+
+ c->private_data = kite_deflate_init(c->compression_level, c->dict_size);
+ if (IS_ERR_VALUE(c->private_data))
+ return PTR_ERR(c->private_data);
erofs_warn("EXPERIMENTAL DEFLATE algorithm in use. Use at your own risk!");
erofs_warn("*Carefully* check filesystem data correctness to avoid corruption!");
@@ -50,6 +57,11 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
if (compression_level < 0)
compression_level = erofs_compressor_deflate.default_level;
+ if (compression_level > erofs_compressor_deflate.best_level) {
+ erofs_err("invalid compression level %d", compression_level);
+ return -EINVAL;
+ }
+
c->compression_level = compression_level;
return 0;
}
@@ -57,26 +69,14 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
u32 dict_size)
{
- void *s;
-
- if (c->private_data) {
- kite_deflate_end(c->private_data);
- c->private_data = NULL;
- }
+ if (dict_size == 0)
+ dict_size = erofs_compressor_deflate.default_dictsize;
if (dict_size > erofs_compressor_deflate.max_dictsize) {
- erofs_err("dict size %u is too large", dict_size);
+ erofs_err("invalid dict size %u", dict_size);
return -EINVAL;
}
- if (dict_size == 0)
- dict_size = erofs_compressor_deflate.default_dictsize;
-
- s = kite_deflate_init(c->compression_level, dict_size);
- if (IS_ERR(s))
- return PTR_ERR(s);
-
- c->private_data = s;
c->dict_size = dict_size;
return 0;
}
diff --git a/lib/compressor_libdeflate.c b/lib/compressor_libdeflate.c
index c0b019a..ad5eeec 100644
--- a/lib/compressor_libdeflate.c
+++ b/lib/compressor_libdeflate.c
@@ -82,7 +82,10 @@ static int compressor_libdeflate_exit(struct erofs_compress *c)
static int compressor_libdeflate_init(struct erofs_compress *c)
{
- c->private_data = NULL;
+ libdeflate_free_compressor(c->private_data);
+ c->private_data = libdeflate_alloc_compressor(c->compression_level);
+ if (!c->private_data)
+ return -ENOMEM;
erofs_warn("EXPERIMENTAL libdeflate compressor in use. Use at your own risk!");
return 0;
@@ -94,10 +97,11 @@ static int erofs_compressor_libdeflate_setlevel(struct erofs_compress *c,
if (compression_level < 0)
compression_level = erofs_compressor_deflate.default_level;
- libdeflate_free_compressor(c->private_data);
- c->private_data = libdeflate_alloc_compressor(compression_level);
- if (!c->private_data)
- return -ENOMEM;
+ if (compression_level > erofs_compressor_deflate.best_level) {
+ erofs_err("invalid compression level %d", compression_level);
+ return -EINVAL;
+ }
+
c->compression_level = compression_level;
return 0;
}
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
index a9551e2..b8ae29c 100644
--- a/lib/compressor_liblzma.c
+++ b/lib/compressor_liblzma.c
@@ -55,18 +55,13 @@ static int erofs_compressor_liblzma_exit(struct erofs_compress *c)
static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
int compression_level)
{
- struct erofs_liblzma_context *ctx = c->private_data;
- u32 preset;
-
if (compression_level < 0)
- preset = LZMA_PRESET_DEFAULT;
- else if (compression_level >= 100)
- preset = (compression_level - 100) | LZMA_PRESET_EXTREME;
- else
- preset = compression_level;
+ compression_level = erofs_compressor_lzma.default_level;
- if (lzma_lzma_preset(&ctx->opt, preset))
+ if (compression_level > erofs_compressor_lzma.best_level) {
+ erofs_err("invalid compression level %d", compression_level);
return -EINVAL;
+ }
c->compression_level = compression_level;
return 0;
@@ -75,7 +70,8 @@ static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
u32 dict_size)
{
- struct erofs_liblzma_context *ctx = c->private_data;
+ if (dict_size == 0)
+ dict_size = erofs_compressor_lzma.default_dictsize;
if (dict_size > erofs_compressor_lzma.max_dictsize ||
dict_size < 4096) {
@@ -83,10 +79,6 @@ static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
return -EINVAL;
}
- if (dict_size == 0)
- dict_size = erofs_compressor_lzma.default_dictsize;
-
- ctx->opt.dict_size = dict_size;
c->dict_size = dict_size;
return 0;
}
@@ -94,11 +86,23 @@ static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
static int erofs_compressor_liblzma_init(struct erofs_compress *c)
{
struct erofs_liblzma_context *ctx;
+ u32 preset;
ctx = malloc(sizeof(*ctx));
if (!ctx)
return -ENOMEM;
+
ctx->strm = (lzma_stream)LZMA_STREAM_INIT;
+ if (c->compression_level < 0)
+ preset = LZMA_PRESET_DEFAULT;
+ else if (c->compression_level >= 100)
+ preset = (c->compression_level - 100) | LZMA_PRESET_EXTREME;
+ else
+ preset = c->compression_level;
+ if (lzma_lzma_preset(&ctx->opt, preset))
+ return -EINVAL;
+ ctx->opt.dict_size = c->dict_size;
+
c->private_data = ctx;
erofs_warn("EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!");
erofs_warn("Note that it may take more time since the compressor is still single-threaded for now.");
diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
index b410e15..6fc8847 100644
--- a/lib/compressor_lz4hc.c
+++ b/lib/compressor_lz4hc.c
@@ -7,6 +7,7 @@
#define LZ4_HC_STATIC_LINKING_ONLY (1)
#include <lz4hc.h>
#include "erofs/internal.h"
+#include "erofs/print.h"
#include "compressor.h"
#ifndef LZ4_DISTANCE_MAX /* history window size */
@@ -49,8 +50,10 @@ static int compressor_lz4hc_init(struct erofs_compress *c)
static int compressor_lz4hc_setlevel(struct erofs_compress *c,
int compression_level)
{
- if (compression_level > LZ4HC_CLEVEL_MAX)
+ if (compression_level > erofs_compressor_lz4hc.best_level) {
+ erofs_err("invalid compression level %d", compression_level);
return -EINVAL;
+ }
c->compression_level = compression_level < 0 ?
LZ4HC_CLEVEL_DEFAULT : compression_level;
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v4 0/2] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (7 preceding siblings ...)
2024-01-19 12:47 ` [PATCH v3 3/3] erofs-utils: mkfs: reorganize logic in erofs_compressor_init() Yifan Zhao
@ 2024-01-20 11:53 ` Yifan Zhao
2024-01-20 11:53 ` [PATCH v4 1/2] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
2024-01-20 11:53 ` [PATCH v4 2/2] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
10 siblings, 0 replies; 15+ messages in thread
From: Yifan Zhao @ 2024-01-20 11:53 UTC (permalink / raw)
To: linux-erofs
This patchset allows to specify dictionary size for compression algorithms.
change since v3:
- remove the third patch as is unrelated to this patchset, will send as a
separate patch later
- Now patch 1 and 2 are able to be `git am` without any conflict
Yifan Zhao (2):
erofs-utils: mkfs: merge erofs_compressor_setlevel() into
erofs_compressor_init()
erofs-utils: mkfs: allow to specify dictionary size for compression
algorithms
include/erofs/config.h | 10 ++--
lib/compress.c | 37 ++++++++------
lib/compress_hints.c | 2 +-
lib/compressor.c | 45 +++++++++++------
lib/compressor.h | 9 ++--
lib/compressor_deflate.c | 26 ++++++++--
lib/compressor_liblzma.c | 34 ++++++++-----
lib/compressor_lz4.c | 2 -
lib/config.c | 4 +-
lib/inode.c | 2 +-
mkfs/main.c | 104 +++++++++++++++++++++++++++++++--------
11 files changed, 197 insertions(+), 78 deletions(-)
--
2.43.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v4 1/2] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init()
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (8 preceding siblings ...)
2024-01-20 11:53 ` [PATCH v4 0/2] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
@ 2024-01-20 11:53 ` Yifan Zhao
2024-01-20 11:53 ` [PATCH v4 2/2] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms Yifan Zhao
10 siblings, 0 replies; 15+ messages in thread
From: Yifan Zhao @ 2024-01-20 11:53 UTC (permalink / raw)
To: linux-erofs
Currently erofs_compressor_setlevel() is only called once just after
erofs_compressor_init() while initializing compressors. Let's just hide
this interface and set the compression level in erofs_compressor_init().
Besides, we do not need to assign the {default,best}_level for an
algorithm which does not support the compression level in its
erofs_compressor struct.
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
lib/compress.c | 6 +-----
lib/compressor.c | 31 +++++++++++++++++--------------
lib/compressor.h | 5 ++---
lib/compressor_lz4.c | 2 --
4 files changed, 20 insertions(+), 24 deletions(-)
diff --git a/lib/compress.c b/lib/compress.c
index 22782cb..3ea735c 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1206,11 +1206,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
for (i = 0; cfg.c_compr_alg[i]; ++i) {
struct erofs_compress *c = &erofs_ccfg[i].handle;
- ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i]);
- if (ret)
- return ret;
-
- ret = erofs_compressor_setlevel(c, cfg.c_compr_level[i]);
+ ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i], cfg.c_compr_level[i]);
if (ret)
return ret;
diff --git a/lib/compressor.c b/lib/compressor.c
index a71436a..295aa47 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -77,20 +77,8 @@ int erofs_compress_destsize(const struct erofs_compress *c,
return c->alg->c->compress_destsize(c, src, srcsize, dst, dstsize);
}
-int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level)
-{
- DBG_BUGON(!c->alg);
- if (c->alg->c->setlevel)
- return c->alg->c->setlevel(c, compression_level);
-
- if (compression_level >= 0)
- return -EINVAL;
- c->compression_level = 0;
- return 0;
-}
-
-int erofs_compressor_init(struct erofs_sb_info *sbi,
- struct erofs_compress *c, char *alg_name)
+int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
+ char *alg_name, int compression_level)
{
int ret, i;
@@ -113,6 +101,21 @@ int erofs_compressor_init(struct erofs_sb_info *sbi,
continue;
ret = erofs_algs[i].c->init(c);
+ if (ret)
+ return ret;
+
+ if (erofs_algs[i].c->setlevel) {
+ ret = erofs_algs[i].c->setlevel(c, compression_level);
+ if (ret) {
+ erofs_err("failed to set compression level %d for %s",
+ compression_level, alg_name);
+ return ret;
+ }
+ } else if (compression_level >= 0) {
+ erofs_err("compression level is not supported for %s",
+ alg_name);
+ return -EINVAL;
+ }
if (!ret) {
c->alg = &erofs_algs[i];
return 0;
diff --git a/lib/compressor.h b/lib/compressor.h
index 6875cf1..ec5485d 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -55,9 +55,8 @@ int erofs_compress_destsize(const struct erofs_compress *c,
const void *src, unsigned int *srcsize,
void *dst, unsigned int dstsize);
-int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
-int erofs_compressor_init(struct erofs_sb_info *sbi,
- struct erofs_compress *c, char *alg_name);
+int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
+ char *alg_name, int compression_level);
int erofs_compressor_exit(struct erofs_compress *c);
#endif
diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
index 6677693..f4e72c3 100644
--- a/lib/compressor_lz4.c
+++ b/lib/compressor_lz4.c
@@ -37,8 +37,6 @@ static int compressor_lz4_init(struct erofs_compress *c)
}
const struct erofs_compressor erofs_compressor_lz4 = {
- .default_level = 0,
- .best_level = 0,
.init = compressor_lz4_init,
.exit = compressor_lz4_exit,
.compress_destsize = lz4_compress_destsize,
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v4 2/2] erofs-utils: mkfs: allow to specify dictionary size for compression algorithms
2024-01-06 18:10 [PATCH 1/3] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
` (9 preceding siblings ...)
2024-01-20 11:53 ` [PATCH v4 1/2] erofs-utils: mkfs: merge erofs_compressor_setlevel() into erofs_compressor_init() Yifan Zhao
@ 2024-01-20 11:53 ` Yifan Zhao
10 siblings, 0 replies; 15+ messages in thread
From: Yifan Zhao @ 2024-01-20 11:53 UTC (permalink / raw)
To: linux-erofs
Currently, the dictionary size for compression algorithms is fixed. This
patch allows to specify different ones with new -zX,dictsize=<dictsize>
options.
This patch also changes the way to specify compression levels. Now, the
compression level is specified with -zX,level=<level> options and could
be specified together with dictsize. The old -zX,<level> form is still
supported for compatibility.
Suggested-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
include/erofs/config.h | 10 ++--
lib/compress.c | 33 ++++++++-----
lib/compress_hints.c | 2 +-
lib/compressor.c | 16 +++++-
lib/compressor.h | 6 ++-
lib/compressor_deflate.c | 26 ++++++++--
lib/compressor_liblzma.c | 34 ++++++++-----
lib/config.c | 4 +-
lib/inode.c | 2 +-
mkfs/main.c | 104 +++++++++++++++++++++++++++++++--------
10 files changed, 180 insertions(+), 57 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 89fe522..eecf575 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -34,6 +34,12 @@ enum {
#define EROFS_MAX_COMPR_CFGS 64
+struct erofs_compr_opts {
+ char *alg;
+ int level;
+ u32 dict_size;
+};
+
struct erofs_configure {
const char *c_version;
int c_dbg_lvl;
@@ -64,8 +70,7 @@ struct erofs_configure {
char *c_src_path;
char *c_blobdev_path;
char *c_compress_hints_file;
- char *c_compr_alg[EROFS_MAX_COMPR_CFGS];
- int c_compr_level[EROFS_MAX_COMPR_CFGS];
+ struct erofs_compr_opts c_compr_opts[EROFS_MAX_COMPR_CFGS];
char c_force_inodeversion;
char c_force_chunkformat;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
@@ -73,7 +78,6 @@ struct erofs_configure {
u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;
u32 c_max_decompressed_extent_bytes;
- u32 c_dict_size;
u64 c_unix_timestamp;
u32 c_uid, c_gid;
const char *mount_point;
diff --git a/lib/compress.c b/lib/compress.c
index 3ea735c..ea9d00d 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1123,7 +1123,8 @@ err_free_meta:
}
static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
- struct erofs_buffer_head *sb_bh)
+ struct erofs_buffer_head *sb_bh,
+ u32 *max_dict_size)
{
struct erofs_buffer_head *bh = sb_bh;
int ret = 0;
@@ -1159,7 +1160,9 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
} __packed lzmaalg = {
.size = cpu_to_le16(sizeof(struct z_erofs_lzma_cfgs)),
.lzma = {
- .dict_size = cpu_to_le32(cfg.c_dict_size),
+ .dict_size = cpu_to_le32(
+ max_dict_size
+ [Z_EROFS_COMPRESSION_LZMA]),
}
};
@@ -1181,8 +1184,9 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
} __packed zalg = {
.size = cpu_to_le16(sizeof(struct z_erofs_deflate_cfgs)),
.z = {
- .windowbits =
- cpu_to_le32(ilog2(cfg.c_dict_size)),
+ .windowbits = cpu_to_le32(ilog2(
+ max_dict_size
+ [Z_EROFS_COMPRESSION_DEFLATE])),
}
};
@@ -1202,11 +1206,14 @@ static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *sb_bh)
{
int i, ret;
+ u32 max_dict_size[Z_EROFS_COMPRESSION_MAX];
- for (i = 0; cfg.c_compr_alg[i]; ++i) {
+ for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
struct erofs_compress *c = &erofs_ccfg[i].handle;
- ret = erofs_compressor_init(sbi, c, cfg.c_compr_alg[i], cfg.c_compr_level[i]);
+ ret = erofs_compressor_init(sbi, c, cfg.c_compr_opts[i].alg,
+ cfg.c_compr_opts[i].level,
+ cfg.c_compr_opts[i].dict_size);
if (ret)
return ret;
@@ -1216,17 +1223,21 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
sbi->available_compr_algs |= 1 << erofs_ccfg[i].algorithmtype;
if (erofs_ccfg[i].algorithmtype != Z_EROFS_COMPRESSION_LZ4)
erofs_sb_set_compr_cfgs(sbi);
+ max_dict_size[erofs_ccfg[i].algorithmtype] =
+ max(max_dict_size[erofs_ccfg[i].algorithmtype],
+ c->dict_size);
}
/*
* if primary algorithm is empty (e.g. compression off),
* clear 0PADDING feature for old kernel compatibility.
*/
- if (!cfg.c_compr_alg[0] ||
- (cfg.c_legacy_compress && !strncmp(cfg.c_compr_alg[0], "lz4", 3)))
+ if (!cfg.c_compr_opts[0].alg ||
+ (cfg.c_legacy_compress &&
+ !strncmp(cfg.c_compr_opts[0].alg, "lz4", 3)))
erofs_sb_clear_lz4_0padding(sbi);
- if (!cfg.c_compr_alg[0])
+ if (!cfg.c_compr_opts[0].alg)
return 0;
/*
@@ -1248,7 +1259,7 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s
}
if (erofs_sb_has_compr_cfgs(sbi))
- return z_erofs_build_compr_cfgs(sbi, sb_bh);
+ return z_erofs_build_compr_cfgs(sbi, sb_bh, max_dict_size);
return 0;
}
@@ -1256,7 +1267,7 @@ int z_erofs_compress_exit(void)
{
int i, ret;
- for (i = 0; cfg.c_compr_alg[i]; ++i) {
+ for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
ret = erofs_compressor_exit(&erofs_ccfg[i].handle);
if (ret)
return ret;
diff --git a/lib/compress_hints.c b/lib/compress_hints.c
index afc9f8f..8b78f80 100644
--- a/lib/compress_hints.c
+++ b/lib/compress_hints.c
@@ -125,7 +125,7 @@ int erofs_load_compress_hints(struct erofs_sb_info *sbi)
} else {
ccfg = atoi(alg);
if (ccfg >= EROFS_MAX_COMPR_CFGS ||
- !cfg.c_compr_alg[ccfg]) {
+ !cfg.c_compr_opts[ccfg].alg) {
erofs_err("invalid compressing configuration \"%s\" at line %u",
alg, line);
ret = -EINVAL;
diff --git a/lib/compressor.c b/lib/compressor.c
index 295aa47..9f8d220 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -78,7 +78,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
}
int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
- char *alg_name, int compression_level)
+ char *alg_name, int compression_level, u32 dict_size)
{
int ret, i;
@@ -116,6 +116,20 @@ int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
alg_name);
return -EINVAL;
}
+
+ if (erofs_algs[i].c->setdictsize) {
+ ret = erofs_algs[i].c->setdictsize(c, dict_size);
+ if (ret) {
+ erofs_err("failed to set dict size %u for %s",
+ dict_size, alg_name);
+ return ret;
+ }
+ } else if (dict_size) {
+ erofs_err("dict size is not supported for %s",
+ alg_name);
+ return -EINVAL;
+ }
+
if (!ret) {
c->alg = &erofs_algs[i];
return 0;
diff --git a/lib/compressor.h b/lib/compressor.h
index ec5485d..d8ccf2e 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -14,10 +14,13 @@ struct erofs_compress;
struct erofs_compressor {
int default_level;
int best_level;
+ u32 default_dictsize;
+ u32 max_dictsize;
int (*init)(struct erofs_compress *c);
int (*exit)(struct erofs_compress *c);
int (*setlevel)(struct erofs_compress *c, int compression_level);
+ int (*setdictsize)(struct erofs_compress *c, u32 dict_size);
int (*compress_destsize)(const struct erofs_compress *c,
const void *src, unsigned int *srcsize,
@@ -39,6 +42,7 @@ struct erofs_compress {
unsigned int compress_threshold;
unsigned int compression_level;
+ unsigned int dict_size;
void *private_data;
};
@@ -56,7 +60,7 @@ int erofs_compress_destsize(const struct erofs_compress *c,
void *dst, unsigned int dstsize);
int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
- char *alg_name, int compression_level);
+ char *alg_name, int compression_level, u32 dict_size);
int erofs_compressor_exit(struct erofs_compress *c);
#endif
diff --git a/lib/compressor_deflate.c b/lib/compressor_deflate.c
index 4e5902e..aa2ff24 100644
--- a/lib/compressor_deflate.c
+++ b/lib/compressor_deflate.c
@@ -46,6 +46,16 @@ static int compressor_deflate_init(struct erofs_compress *c)
static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
int compression_level)
+{
+ if (compression_level < 0)
+ compression_level = erofs_compressor_deflate.default_level;
+
+ c->compression_level = compression_level;
+ return 0;
+}
+
+static int erofs_compressor_deflate_setdictsize(struct erofs_compress *c,
+ u32 dict_size)
{
void *s;
@@ -54,23 +64,31 @@ static int erofs_compressor_deflate_setlevel(struct erofs_compress *c,
c->private_data = NULL;
}
- if (compression_level < 0)
- compression_level = erofs_compressor_deflate.default_level;
+ if (dict_size > erofs_compressor_deflate.max_dictsize) {
+ erofs_err("dict size %u is too large", dict_size);
+ return -EINVAL;
+ }
+
+ if (dict_size == 0)
+ dict_size = erofs_compressor_deflate.default_dictsize;
- s = kite_deflate_init(compression_level, cfg.c_dict_size);
+ s = kite_deflate_init(c->compression_level, dict_size);
if (IS_ERR(s))
return PTR_ERR(s);
c->private_data = s;
- c->compression_level = compression_level;
+ c->dict_size = dict_size;
return 0;
}
const struct erofs_compressor erofs_compressor_deflate = {
.default_level = 1,
.best_level = 9,
+ .default_dictsize = 1 << 15,
+ .max_dictsize = 1 << 15,
.init = compressor_deflate_init,
.exit = compressor_deflate_exit,
.setlevel = erofs_compressor_deflate_setlevel,
+ .setdictsize = erofs_compressor_deflate_setdictsize,
.compress_destsize = deflate_compress_destsize,
};
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
index 0ed6f23..a9551e2 100644
--- a/lib/compressor_liblzma.c
+++ b/lib/compressor_liblzma.c
@@ -68,22 +68,29 @@ static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
if (lzma_lzma_preset(&ctx->opt, preset))
return -EINVAL;
- /* XXX: temporary hack */
- if (cfg.c_dict_size) {
- if (cfg.c_dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) {
- erofs_err("dict size %u is too large", cfg.c_dict_size);
- return -EINVAL;
- }
- ctx->opt.dict_size = cfg.c_dict_size;
- } else {
- if (ctx->opt.dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE)
- ctx->opt.dict_size = Z_EROFS_LZMA_MAX_DICT_SIZE;
- cfg.c_dict_size = ctx->opt.dict_size;
- }
c->compression_level = compression_level;
return 0;
}
+static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
+ u32 dict_size)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+
+ if (dict_size > erofs_compressor_lzma.max_dictsize ||
+ dict_size < 4096) {
+ erofs_err("invalid dict size %u", dict_size);
+ return -EINVAL;
+ }
+
+ if (dict_size == 0)
+ dict_size = erofs_compressor_lzma.default_dictsize;
+
+ ctx->opt.dict_size = dict_size;
+ c->dict_size = dict_size;
+ return 0;
+}
+
static int erofs_compressor_liblzma_init(struct erofs_compress *c)
{
struct erofs_liblzma_context *ctx;
@@ -101,9 +108,12 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c)
const struct erofs_compressor erofs_compressor_lzma = {
.default_level = LZMA_PRESET_DEFAULT,
.best_level = 109,
+ .default_dictsize = 8 * Z_EROFS_PCLUSTER_MAX_SIZE,
+ .max_dictsize = Z_EROFS_LZMA_MAX_DICT_SIZE,
.init = erofs_compressor_liblzma_init,
.exit = erofs_compressor_liblzma_exit,
.setlevel = erofs_compressor_liblzma_setlevel,
+ .setdictsize = erofs_compressor_liblzma_setdictsize,
.compress_destsize = erofs_liblzma_compress_destsize,
};
#endif
diff --git a/lib/config.c b/lib/config.c
index aa3dd1f..1096cd1 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -62,8 +62,8 @@ void erofs_exit_configure(void)
free(cfg.c_img_path);
if (cfg.c_src_path)
free(cfg.c_src_path);
- for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_alg[i]; i++)
- free(cfg.c_compr_alg[i]);
+ for (i = 0; i < EROFS_MAX_COMPR_CFGS && cfg.c_compr_opts[i].alg; i++)
+ free(cfg.c_compr_opts[i].alg);
}
static unsigned int fullpath_prefix; /* root directory prefix length */
diff --git a/lib/inode.c b/lib/inode.c
index bcdb4b8..c6424c0 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -492,7 +492,7 @@ int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
return erofs_blob_write_chunked_file(inode, fd, fpos);
}
- if (cfg.c_compr_alg[0] && erofs_file_is_compressible(inode)) {
+ if (cfg.c_compr_opts[0].alg && erofs_file_is_compressible(inode)) {
ret = erofs_write_compressed_file(inode, fd);
if (!ret || ret != -ENOSPC)
return ret;
diff --git a/mkfs/main.c b/mkfs/main.c
index 13fea41..acb2108 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -5,6 +5,7 @@
* Created by Li Guifu <bluce.liguifu@huawei.com>
*/
#define _GNU_SOURCE
+#include <ctype.h>
#include <time.h>
#include <sys/time.h>
#include <stdlib.h>
@@ -108,24 +109,29 @@ static void usage(int argc, char **argv)
" -b# set block size to # (# = page size by default)\n"
" -d<0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n"
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
- " -zX[,Y][:...] X=compressor (Y=compression level, optional)\n"
- " alternative compressors can be separated by colons(:)\n"
- " supported compressors and their level ranges are:\n",
+ " -zX[,level=Y] X=compressor (Y=compression level, Z=dictionary size, optional)\n"
+ " [,dictsize=Z] alternative compressors can be separated by colons(:)\n"
+ " [:...] supported compressors and their option ranges are:\n",
argv[0], EROFS_WARN);
while ((s = z_erofs_list_available_compressors(&i)) != NULL) {
- printf(" %s", s->name);
+ char * const spaces = " ";
+
+ printf("%s%s\n", spaces, s->name);
if (s->c->setlevel) {
if (!strcmp(s->name, "lzma"))
/* A little kludge to show the range as disjointed
* "0-9,100-109" instead of a continuous "0-109", and to
* state what those two subranges respectively mean. */
- printf("[<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)",
- s->c->default_level);
+ printf("%s [,level=<0-9,100-109>]\t0-9=normal, 100-109=extreme (default=%i)\n",
+ spaces, s->c->default_level);
else
- printf("[,<0-%i>]\t(default=%i)",
- s->c->best_level, s->c->default_level);
+ printf("%s [,level=<0-%i>]\t\t(default=%i)\n",
+ spaces, s->c->best_level, s->c->default_level);
+ }
+ if (s->c->setdictsize) {
+ printf("%s [,dictsize=<dictsize>]\t(default=%u, max=%u)\n",
+ spaces, s->c->default_dictsize, s->c->max_dictsize);
}
- putchar('\n');
}
printf(
" -C# specify the size of compress physical cluster in bytes\n"
@@ -304,27 +310,83 @@ handle_fragment:
return 0;
}
+static int mkfs_parse_one_compress_alg(char *alg,
+ struct erofs_compr_opts *copts)
+{
+ char *p, *q, *opt, *endptr;
+
+ copts->level = -1;
+ copts->dict_size = 0;
+
+ p = strchr(alg, ',');
+ if (p) {
+ copts->alg = strndup(alg, p - alg);
+
+ /* support old '-zlzma,9' form */
+ if (isdigit(*(p + 1))) {
+ copts->level = strtol(p + 1, &endptr, 10);
+ if (*endptr && *endptr != ',') {
+ erofs_err("invalid compression level %s",
+ p + 1);
+ return -EINVAL;
+ }
+ return 0;
+ }
+ } else {
+ copts->alg = strdup(alg);
+ return 0;
+ }
+
+ opt = p + 1;
+ while (opt) {
+ q = strchr(opt, ',');
+ if (q)
+ *q = '\0';
+
+ if ((p = strstr(opt, "level="))) {
+ p += strlen("level=");
+ copts->level = strtol(p, &endptr, 10);
+ if ((endptr == p) || (*endptr && *endptr != ',')) {
+ erofs_err("invalid compression level %s", p);
+ return -EINVAL;
+ }
+ } else if ((p = strstr(opt, "dictsize="))) {
+ p += strlen("dictsize=");
+ copts->dict_size = strtoul(p, &endptr, 10);
+ if (*endptr == 'k' || *endptr == 'K')
+ copts->dict_size <<= 10;
+ else if (*endptr == 'm' || *endptr == 'M')
+ copts->dict_size <<= 20;
+ else if ((endptr == p) || (*endptr && *endptr != ',')) {
+ erofs_err("invalid compression dictsize %s", p);
+ return -EINVAL;
+ }
+ } else {
+ erofs_err("invalid compression option %s", opt);
+ return -EINVAL;
+ }
+
+ opt = q ? q + 1 : NULL;
+ }
+
+ return 0;
+}
+
static int mkfs_parse_compress_algs(char *algs)
{
unsigned int i;
char *s;
+ int ret;
for (s = strtok(algs, ":"), i = 0; s; s = strtok(NULL, ":"), ++i) {
- const char *lv;
-
if (i >= EROFS_MAX_COMPR_CFGS - 1) {
erofs_err("too many algorithm types");
return -EINVAL;
}
- lv = strchr(s, ',');
- if (lv) {
- cfg.c_compr_level[i] = atoi(lv + 1);
- cfg.c_compr_alg[i] = strndup(s, lv - s);
- } else {
- cfg.c_compr_level[i] = -1;
- cfg.c_compr_alg[i] = strdup(s);
- }
+ ret = mkfs_parse_one_compress_alg(s, &cfg.c_compr_opts[i]);
+ if (ret)
+ return ret;
}
return 0;
}
@@ -692,7 +754,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
cfg.c_showprogress = false;
}
- if (cfg.c_compr_alg[0] && erofs_blksiz(&sbi) != getpagesize())
+ if (cfg.c_compr_opts[0].alg && erofs_blksiz(&sbi) != getpagesize())
erofs_warn("Please note that subpage blocksize with compression isn't yet supported in kernel. "
"This compressed image will only work with bs = ps = %u bytes",
erofs_blksiz(&sbi));
@@ -1119,7 +1181,7 @@ int main(int argc, char **argv)
}
if (cfg.c_dedupe) {
- if (!cfg.c_compr_alg[0]) {
+ if (!cfg.c_compr_opts[0].alg) {
erofs_err("Compression is not enabled. Turn on chunk-based data deduplication instead.");
cfg.c_chunkbits = sbi.blkszbits;
} else {
--
2.43.0
^ permalink raw reply related [flat|nested] 15+ messages in thread