[f2fs-dev] [PATCH v3 0/3] f2fs-tools: sload compression support

[f2fs-dev] [PATCH v3 0/3] f2fs-tools: sload compression support

[Robin Hsu]

From: Robin Hsu <robinhsu@google.com>

* 3 patch set:
  #1: added some #ifdef for easier support
  #2: main code change
  #3: automake changes

v2 fix (from v1): fixed a bug and a more elegant error handling flow.
v3 fix (from v2): ./configure (automake) automatically determine to
compile in lzo and/or lz4 compression support depending on the presence
of liblzo2-dev and/or liblz4-dev

Robin Hsu (3):
  f2fs-tools: Added #ifdef WITH_func
  f2fs-tools:sload.f2fs compression support
  f2fs-tools:sload.f2fs compress: Fixed automake

 configure.ac            |  12 +++
 fsck/Makefile.am        |   9 +-
 fsck/compress_wrapper.c | 102 ++++++++++++++++++++
 fsck/compress_wrapper.h |  22 +++++
 fsck/fsck.h             |  15 +++
 fsck/main.c             | 157 ++++++++++++++++++++++++++++++-
 fsck/segment.c          | 202 +++++++++++++++++++++++++++++++++++++---
 fsck/sload.c            |  67 +++++++++++++
 include/f2fs_fs.h       |  76 ++++++++++++++-
 lib/libf2fs_io.c        |  33 +++++++
 10 files changed, 678 insertions(+), 17 deletions(-)
 create mode 100644 fsck/compress_wrapper.c
 create mode 100644 fsck/compress_wrapper.h

-- 
2.29.2.576.ga3fc446d84-goog



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

[f2fs-dev] [PATCH v3 1/3] f2fs-tools: Added #ifdef WITH_func

[Robin Hsu]

From: Robin Hsu <robinhsu@google.com>

Add proprocessor defines (options) 'WITH_func',
where func = DUMP, DEFRAG, RESIZE, or SLOAD

Signed-off-by: Robin Hsu <robinhsu@google.com>
---
 fsck/main.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/fsck/main.c b/fsck/main.c
index e70048b..b20498f 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -345,6 +345,7 @@ void f2fs_parse_options(int argc, char *argv[])
 				break;
 		}
 	} else if (!strcmp("dump.f2fs", prog)) {
+#ifdef WITH_DUMP
 		const char *option_string = "d:i:n:s:Sa:b:V";
 		static struct dump_option dump_opt = {
 			.nid = 0,	/* default root ino */
@@ -426,7 +427,9 @@ void f2fs_parse_options(int argc, char *argv[])
 		}
 
 		c.private = &dump_opt;
+#endif
 	} else if (!strcmp("defrag.f2fs", prog)) {
+#ifdef WITH_DEFRAG
 		const char *option_string = "d:s:Sl:t:iV";
 
 		c.func = DEFRAG;
@@ -484,7 +487,9 @@ void f2fs_parse_options(int argc, char *argv[])
 			if (err != NOERROR)
 				break;
 		}
+#endif
 	} else if (!strcmp("resize.f2fs", prog)) {
+#ifdef WITH_RESIZE
 		const char *option_string = "d:st:iV";
 
 		c.func = RESIZE;
@@ -526,7 +531,9 @@ void f2fs_parse_options(int argc, char *argv[])
 			if (err != NOERROR)
 				break;
 		}
+#endif
 	} else if (!strcmp("sload.f2fs", prog)) {
+#ifdef WITH_SLOAD
 		const char *option_string = "C:d:f:p:s:St:T:V";
 #ifdef HAVE_LIBSELINUX
 		int max_nr_opt = (int)sizeof(c.seopt_file) /
@@ -595,6 +602,7 @@ void f2fs_parse_options(int argc, char *argv[])
 			if (err != NOERROR)
 				break;
 		}
+#endif /* WITH_SLOAD */
 	}
 
 	if (err == NOERROR) {
@@ -707,6 +715,7 @@ static int do_fsck(struct f2fs_sb_info *sbi)
 	return FSCK_ERRORS_LEFT_UNCORRECTED;
 }
 
+#ifdef WITH_DUMP
 static void do_dump(struct f2fs_sb_info *sbi)
 {
 	struct dump_option *opt = (struct dump_option *)c.private;
@@ -733,7 +742,9 @@ static void do_dump(struct f2fs_sb_info *sbi)
 	print_cp_state(flag);
 
 }
+#endif
 
+#ifdef WITH_DEFRAG
 static int do_defrag(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_super_block *sb = F2FS_RAW_SUPER(sbi);
@@ -782,7 +793,9 @@ out_range:
 				c.defrag_target);
 	return -1;
 }
+#endif
 
+#ifdef WITH_RESIZE
 static int do_resize(struct f2fs_sb_info *sbi)
 {
 	if (!c.target_sectors)
@@ -796,7 +809,9 @@ static int do_resize(struct f2fs_sb_info *sbi)
 
 	return f2fs_resize(sbi);
 }
+#endif
 
+#ifdef WITH_SLOAD
 static int do_sload(struct f2fs_sb_info *sbi)
 {
 	if (!c.from_dir) {
@@ -808,6 +823,7 @@ static int do_sload(struct f2fs_sb_info *sbi)
 
 	return f2fs_sload(sbi);
 }
+#endif
 
 #if defined(__APPLE__)
 static u64 get_boottime_ns()
-- 
2.29.2.576.ga3fc446d84-goog



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH v3 1/3] f2fs-tools: Added #ifdef WITH_func

[Chao Yu]

On 2020/12/8 16:15, Robin Hsu wrote:
> From: Robin Hsu <robinhsu@google.com>
> 
> Add proprocessor defines (options) 'WITH_func',
> where func = DUMP, DEFRAG, RESIZE, or SLOAD
> 
> Signed-off-by: Robin Hsu <robinhsu@google.com>

Reviewed-by: Chao Yu <yuchao0@huawei.com>

Thanks,


_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

[f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support

[Robin Hsu]

From: Robin Hsu <robinhsu@google.com>

Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny
  policy
* Support choice of compression algorithm, LZO (version 2) or LZ4
  (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1).
  A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) option

Signed-off-by: Robin Hsu <robinhsu@google.com>
---
 fsck/compress_wrapper.c | 102 ++++++++++++++++++++
 fsck/compress_wrapper.h |  22 +++++
 fsck/fsck.h             |  15 +++
 fsck/main.c             | 141 +++++++++++++++++++++++++++-
 fsck/segment.c          | 202 +++++++++++++++++++++++++++++++++++++---
 fsck/sload.c            |  67 +++++++++++++
 include/f2fs_fs.h       |  76 ++++++++++++++-
 lib/libf2fs_io.c        |  33 +++++++
 8 files changed, 644 insertions(+), 14 deletions(-)
 create mode 100644 fsck/compress_wrapper.c
 create mode 100644 fsck/compress_wrapper.h

diff --git a/fsck/compress_wrapper.c b/fsck/compress_wrapper.c
new file mode 100644
index 0000000..2cdc4fd
--- /dev/null
+++ b/fsck/compress_wrapper.c
@@ -0,0 +1,102 @@
+/**
+ * compress_wrapper.c
+ *
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *   : initial created, for sload compression support
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include "f2fs.h" /* for config.h for general environment (non-Android) */
+
+#include "compress_wrapper.h"
+#ifdef HAVE_LIBLZO2
+#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */
+#endif
+#ifdef HAVE_LIBLZ4
+#include <lz4.h> /* for LZ4_compress_fast_extState() */
+#endif
+
+/*
+ * macro/constants borrowed from kernel header (GPL-2.0):
+ * include/linux/lzo.h, and include/linux/lz4.h
+ */
+#ifdef HAVE_LIBLZO2
+#define lzo1x_worst_compress(x)		((x) + (x) / 16 + 64 + 3 + 2)
+#define LZO_WORK_SIZE			ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8)
+#endif
+#ifdef HAVE_LIBLZ4
+#define LZ4_MEMORY_USAGE		14
+#define LZ4_MAX_INPUT_SIZE		0x7E000000 /* 2 113 929 216 bytes */
+#define LZ4_STREAMSIZE			(LZ4_STREAMSIZE_U64 * sizeof(long long))
+#define LZ4_MEM_COMPRESS		LZ4_STREAMSIZE
+#define LZ4_ACCELERATION_DEFAULT	1
+#define LZ4_WORK_SIZE			ALIGN_UP(LZ4_MEM_COMPRESS, 8)
+#endif
+
+#ifdef HAVE_LIBLZO2
+static void lzo_compress_init(struct compress_ctx *cc)
+{
+	size_t size = cc->cluster_size * F2FS_BLKSIZE;
+	size_t alloc = size + lzo1x_worst_compress(size)
+			+ COMPRESS_HEADER_SIZE + LZO_WORK_SIZE;
+	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
+	cc->rbuf = (char *) cc->private + LZO_WORK_SIZE;
+	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
+}
+
+static int lzo_compress(struct compress_ctx *cc)
+{
+	int ret = lzo1x_1_15_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
+			(lzo_uintp)(&cc->clen), cc->private);
+	cc->cbuf->clen = cpu_to_le32(cc->clen);
+	return ret;
+}
+#endif
+
+#ifdef HAVE_LIBLZ4
+static void lz4_compress_init(struct compress_ctx *cc)
+{
+	size_t size = cc->cluster_size * F2FS_BLKSIZE;
+	size_t alloc = size + LZ4_COMPRESSBOUND(size)
+			+ COMPRESS_HEADER_SIZE + LZ4_WORK_SIZE;
+	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
+	cc->rbuf = (char *) cc->private + LZ4_WORK_SIZE;
+	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
+}
+
+static int lz4_compress(struct compress_ctx *cc)
+{
+	cc->clen = LZ4_compress_fast_extState(cc->private, cc->rbuf,
+			(char *)cc->cbuf->cdata, cc->rlen,
+			cc->rlen - F2FS_BLKSIZE * c.sldc_min_cbpc,
+			LZ4_ACCELERATION_DEFAULT);
+
+	if (!cc->clen)
+		return 1;
+
+	cc->cbuf->clen = cpu_to_le32(cc->clen);
+	return 0;
+}
+#endif
+
+const char *ca_names[] = {
+	"LZO",
+	"LZ4",
+	"", /* end of the name list */
+};
+
+compress_ops compr_ops[] = {
+#ifdef HAVE_LIBLZO2
+	{lzo_compress_init, lzo_compress},
+#else
+	{NULL, NULL},
+#endif
+#ifdef HAVE_LIBLZ4
+	{lz4_compress_init, lz4_compress},
+#else
+	{NULL, NULL},
+#endif
+};
diff --git a/fsck/compress_wrapper.h b/fsck/compress_wrapper.h
new file mode 100644
index 0000000..ec33d43
--- /dev/null
+++ b/fsck/compress_wrapper.h
@@ -0,0 +1,22 @@
+/**
+ * compress_wrapper.h
+ *
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *   : initial created, for sload compression support
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef COMPRESS_WRAPPER_H
+#define COMPRESS_WRAPPER_H
+
+#include "f2fs_fs.h"
+extern compress_ops compr_ops[]; /* [0]: LZO, [1]: LZ4, */
+
+#endif /* COMPRESS_WRAPPER_H */
diff --git a/fsck/fsck.h b/fsck/fsck.h
index c5e85fe..4e866ec 100644
--- a/fsck/fsck.h
+++ b/fsck/fsck.h
@@ -3,6 +3,9 @@
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *             http://www.samsung.com/
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);
 
 /* sload.c */
 int f2fs_sload(struct f2fs_sb_info *);
+void sldc_erase_bufs(struct compress_ctx *cc);
+void sload_countblk(void);
+extern struct ext_tbl_op ext_filter;
 
 /* segment.c */
 int reserve_new_block(struct f2fs_sb_info *, block_t *,
@@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,
 struct quota_file;
 u64 f2fs_quota_size(struct quota_file *);
 u64 f2fs_read(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+enum wr_addr_type {
+	WR_NORMAL = 1,
+	WR_COMPRESS_DATA = 2,
+	WR_NULL_ADDR = NULL_ADDR,  /* 0 */
+	WR_NEW_ADDR = NEW_ADDR, /* -1U */
+	WR_COMPRESS_ADDR = COMPRESS_ADDR, /* -2U */
+};
 u64 f2fs_write(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+u64 f2fs_write_compress_data(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+u64 f2fs_write_addrtag(struct f2fs_sb_info *, nid_t, pgoff_t, unsigned int);
 void f2fs_filesize_update(struct f2fs_sb_info *, nid_t, u64);
 
 int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,
diff --git a/fsck/main.c b/fsck/main.c
index b20498f..001eed0 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -13,6 +13,9 @@
  * Copyright (c) 2019 Google Inc.
  *   Robin Hsu <robinhsu@google.com>
  *  : add cache layer
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -25,6 +28,7 @@
 #include <getopt.h>
 #include <stdbool.h>
 #include "quotaio.h"
+#include "compress_wrapper.h"
 
 struct f2fs_fsck gfsck;
 
@@ -134,6 +138,17 @@ void sload_usage()
 	MSG(0, "  -S sparse_mode\n");
 	MSG(0, "  -t mount point [prefix of target fs path, default:/]\n");
 	MSG(0, "  -T timestamp\n");
+	MSG(0, "  -c enable compression (default allow policy)\n");
+	MSG(0, "    ------------ Compression sub-options -----------------\n");
+	MSG(0, "    -L <log-of-blocks-per-cluster>, default 2\n");
+	MSG(0, "    -a <algorithm> compression algorithm, default LZ4\n");
+	MSG(0, "    -x <ext> compress files except for these extensions.\n");
+	MSG(0, "    -i <ext> compress files with these extensions only.\n");
+	MSG(0, "    * -i or -x: use it many times for multiple extensions.\n");
+	MSG(0, "    * -i and -x cannot be used together..\n");
+	MSG(0, "    -m <num> min compressed blocks per cluster\n");
+	MSG(0, "    -r readonly (IMMUTABLE) for compressed files\n");
+	MSG(0, "    ------------------------------------------------------\n");
 	MSG(0, "  -d debug level [default:0]\n");
 	MSG(0, "  -V print the version number and exit\n");
 	exit(1);
@@ -534,7 +549,7 @@ void f2fs_parse_options(int argc, char *argv[])
 #endif
 	} else if (!strcmp("sload.f2fs", prog)) {
 #ifdef WITH_SLOAD
-		const char *option_string = "C:d:f:p:s:St:T:V";
+		const char *option_string = "cL:a:i:x:m:rC:d:f:p:s:St:T:V";
 #ifdef HAVE_LIBSELINUX
 		int max_nr_opt = (int)sizeof(c.seopt_file) /
 			sizeof(c.seopt_file[0]);
@@ -543,8 +558,82 @@ void f2fs_parse_options(int argc, char *argv[])
 		char *p;
 
 		c.func = SLOAD;
+		c.sldc_cc.log_cluster_size = 2;
+		c.sldc_ca = CA_LZ4;
+		c.sldc_min_cbpc = 1;
+		c.sldc_ef = &ext_filter;
 		while ((option = getopt(argc, argv, option_string)) != EOF) {
+			unsigned int i;
+			int val;
+
 			switch (option) {
+			case 'c': /* compression support */
+				c.sldc_en = true;
+				break;
+			case 'L': /* compression: log of blocks-per-cluster */
+				c.sldc_got_opt = true;
+				val = atoi(optarg);
+				if (val < MIN_COMPRESS_LOG_SIZE ||
+						val > MAX_COMPRESS_LOG_SIZE) {
+					MSG(0, "\tError: log of blocks per"
+						" cluster must be in the range"
+						" of %d .. %d.\n",
+						MIN_COMPRESS_LOG_SIZE,
+						MAX_COMPRESS_LOG_SIZE);
+					error_out(prog);
+				}
+				c.sldc_cc.log_cluster_size = val;
+				break;
+			case 'a': /* compression: choose algorithm */
+				c.sldc_got_opt = true;
+				c.sldc_ca = (u8)-1;
+				for (i = 0; ca_names[i][0] != 0; i++) {
+					if (!strcmp(ca_names[i], optarg)) {
+						c.sldc_ca = i;
+						break;
+					}
+				}
+				if (c.sldc_ca == (u8)-1) {
+					MSG(0, "\tError: Unknown compression"
+						" algorithm %s\n", optarg);
+					error_out(prog);
+				}
+				break;
+			case 'i': /* compress only these extensions */
+				c.sldc_got_opt = true;
+				if (c.sldc_policy == FP_ALLOW) {
+					MSG(0, "\tError: could not mix option"
+							" -i and -x\n");
+					error_out(prog);
+				}
+				c.sldc_policy = FP_DENY;
+				c.sldc_ef->add(optarg);
+				break;
+			case 'x': /* compress except for these extensions */
+				c.sldc_got_opt = true;
+				if (c.sldc_policy == FP_DENY) {
+					MSG(0, "\tError: could not mix option"
+							" -i and -x\n");
+					error_out(prog);
+				}
+				c.sldc_policy = FP_ALLOW;
+				c.sldc_ef->add(optarg);
+				break;
+			case 'm': /* minimum compressed blocks per cluster */
+				c.sldc_got_opt = true;
+				val = atoi(optarg);
+				if (val <= 0) {
+					MSG(0, "\tError: minimum compressed"
+						" blocks per cluster must be"
+						" positive.\n");
+					error_out(prog);
+				}
+				c.sldc_min_cbpc = val;
+				break;
+			case 'r': /* compress file to set IMMUTABLE */
+				c.sldc_got_opt = true;
+				c.sldc_immutable = true;
+				break;
 			case 'C':
 				c.fs_config_file = absolute_path(optarg);
 				break;
@@ -602,6 +691,27 @@ void f2fs_parse_options(int argc, char *argv[])
 			if (err != NOERROR)
 				break;
 		}
+		if (c.sldc_got_opt && !c.sldc_en) {
+			MSG(0, "\tError: compression sub-options are used"
+				" without the compression enable (-c) option\n"
+			);
+			error_out(prog);
+		}
+		if (err == NOERROR && c.sldc_en) {
+			c.sldc_cc.cluster_size = 1
+				<< c.sldc_cc.log_cluster_size;
+			if (c.sldc_policy == FP_UNASSIGNED)
+				c.sldc_policy = FP_ALLOW;
+			if (c.sldc_min_cbpc >= c.sldc_cc.cluster_size) {
+				MSG(0, "\tError: minimum reduced blocks by"
+					" compression per cluster must be at"
+					" most one less than blocks per"
+					" cluster, i.e. %d\n",
+					c.sldc_cc.cluster_size - 1);
+				error_out(prog);
+			}
+			qbuf_init();
+		}
 #endif /* WITH_SLOAD */
 	}
 
@@ -812,6 +922,27 @@ static int do_resize(struct f2fs_sb_info *sbi)
 #endif
 
 #ifdef WITH_SLOAD
+int init_compr(struct f2fs_sb_info *sbi)
+{
+	if (!(sbi->raw_super->feature
+			& cpu_to_le32(F2FS_FEATURE_COMPRESSION))) {
+		MSG(0, "Error: Compression (-c) was requested "
+			"but the file system is not created "
+			"with such feature.\n");
+		return -1;
+	}
+	if (compr_ops[c.sldc_ca].init == NULL) {
+		MSG(0, "Error: The selected compression algorithm is not"
+				" supported\n");
+		return -1;
+	}
+	c.sldc_compr = compr_ops + c.sldc_ca;
+	c.sldc_compr->init(&c.sldc_cc);
+	sldc_erase_bufs(&c.sldc_cc);
+	c.sldc_cc.rlen = c.sldc_cc.cluster_size * F2FS_BLKSIZE;
+	return 0;
+}
+
 static int do_sload(struct f2fs_sb_info *sbi)
 {
 	if (!c.from_dir) {
@@ -821,6 +952,11 @@ static int do_sload(struct f2fs_sb_info *sbi)
 	if (!c.mount_point)
 		c.mount_point = "/";
 
+	if (c.sldc_en) {
+		if (init_compr(sbi))
+			return -1;
+	}
+
 	return f2fs_sload(sbi);
 }
 #endif
@@ -971,6 +1107,9 @@ retry:
 		return ret2;
 	}
 
+	if (c.func == SLOAD)
+		c.sldc_ef->destroy();
+
 	printf("\nDone: %lf secs\n", (get_boottime_ns() - start) / 1000000000.0);
 	return ret;
 
diff --git a/fsck/segment.c b/fsck/segment.c
index 0487f41..e4c8cea 100644
--- a/fsck/segment.c
+++ b/fsck/segment.c
@@ -8,6 +8,9 @@
  *   Hou Pengyang <houpengyang@huawei.com>
  *   Liu Shuoran <liushuoran@huawei.com>
  *   Jaegeuk Kim <jaegeuk@kernel.org>
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -16,6 +19,7 @@
 #include "fsck.h"
 #include "node.h"
 #include "quotaio.h"
+#include "compress_wrapper.h"
 
 int reserve_new_block(struct f2fs_sb_info *sbi, block_t *to,
 			struct f2fs_summary *sum, int type, bool is_inode)
@@ -228,8 +232,14 @@ u64 f2fs_read(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 	return read_count;
 }
 
-u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
-					u64 count, pgoff_t offset)
+/*
+ * Do not call this function directly.  Instead, call one of the following:
+ *     u64 f2fs_write();
+ *     u64 f2fs_write_compress_data();
+ *     u64 f2fs_write_addrtag();
+ */
+static u64 f2fs_write_ex(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
+		u64 count, pgoff_t offset, enum wr_addr_type addr_type)
 {
 	struct dnode_of_data dn;
 	struct node_info ni;
@@ -243,6 +253,19 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 	void* index_node = NULL;
 	int idirty = 0;
 	int err;
+	bool has_data = (addr_type == WR_NORMAL
+			|| addr_type == WR_COMPRESS_DATA);
+
+	if (count == 0)
+		return 0;
+
+	/*
+	 * Enforce calling from f2fs_write(), f2fs_write_compress_data(),
+	 * and f2fs_write_addrtag().   Beside, check if is properly called.
+	 */
+	ASSERT((!has_data && buffer == NULL) || (has_data && buffer != NULL));
+	if (addr_type != WR_NORMAL)
+		ASSERT(offset % F2FS_BLKSIZE == 0); /* block boundary only */
 
 	/* Memory allocation for block buffer and inode. */
 	blk_buffer = calloc(BLOCK_SZ, 2);
@@ -265,15 +288,26 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 			if (err)
 				break;
 			idirty |= dn.idirty;
-			if (index_node)
-				free(index_node);
+			free(index_node);
 			index_node = (dn.node_blk == dn.inode_blk) ?
-							NULL : dn.node_blk;
+					NULL : dn.node_blk;
 			remained_blkentries = ADDRS_PER_PAGE(sbi,
-						dn.node_blk, dn.inode_blk);
+					dn.node_blk, dn.inode_blk) -
+					dn.ofs_in_node;
 		}
 		ASSERT(remained_blkentries > 0);
 
+		if (!has_data) {
+			dn.data_blkaddr = addr_type;
+			set_data_blkaddr(&dn);
+			idirty |= dn.idirty;
+			if (dn.ndirty)
+				ASSERT(dev_write_block(dn.node_blk,
+						dn.node_blkaddr) >= 0);
+			written_count = 0;
+			break;
+		}
+
 		blkaddr = datablock_addr(dn.node_blk, dn.ofs_in_node);
 		if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) {
 			err = new_data_block(sbi, blk_buffer,
@@ -281,6 +315,7 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 			if (err)
 				break;
 			blkaddr = dn.data_blkaddr;
+			idirty |= dn.idirty;
 		}
 
 		off_in_blk = offset % BLOCK_SZ;
@@ -305,9 +340,10 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 
 		dn.ofs_in_node++;
 		if ((--remained_blkentries == 0 || count == 0) && (dn.ndirty))
-			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr) >= 0);
+			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr)
+					>= 0);
 	}
-	if (offset > le64_to_cpu(inode->i.i_size)) {
+	if (addr_type == WR_NORMAL && offset > le64_to_cpu(inode->i.i_size)) {
 		inode->i.i_size = cpu_to_le64(offset);
 		idirty = 1;
 	}
@@ -315,13 +351,33 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 		ASSERT(inode == dn.inode_blk);
 		ASSERT(write_inode(inode, ni.blk_addr) >= 0);
 	}
-	if (index_node)
-		free(index_node);
+
+	free(index_node);
 	free(blk_buffer);
 
 	return written_count;
 }
 
+u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
+					u64 count, pgoff_t offset)
+{
+	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_NORMAL);
+}
+
+u64 f2fs_write_compress_data(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
+					u64 count, pgoff_t offset)
+{
+	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_COMPRESS_DATA);
+}
+
+u64 f2fs_write_addrtag(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
+		unsigned int addrtag)
+{
+	ASSERT(addrtag == COMPRESS_ADDR || addrtag == NEW_ADDR
+			|| addrtag == NULL_ADDR);
+	return f2fs_write_ex(sbi, ino, NULL, F2FS_BLKSIZE, offset, addrtag);
+}
+
 /* This function updates only inode->i.i_size */
 void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
 {
@@ -342,11 +398,59 @@ void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
 	free(inode);
 }
 
+#define MAX_BULKR_RETRY 5
+int bulkread(int fd, void *rbuf, size_t rsize, bool *eof)
+{
+	int n = 0;
+	int retry = MAX_BULKR_RETRY;
+	int cur;
+
+	if (!rsize)
+		return 0;
+
+	if (eof != NULL)
+		*eof = false;
+	while (rsize && (cur = read(fd, rbuf, rsize)) != 0) {
+		if (cur == -1) {
+			if (errno == EINTR && retry--)
+				continue;
+			return -1;
+		}
+		retry = MAX_BULKR_RETRY;
+
+		rsize -= cur;
+		n += cur;
+	}
+	if (eof != NULL)
+		*eof = (cur == 0);
+	return n;
+}
+
+u64 f2fs_fix_mutable(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
+		unsigned int compressed)
+{
+	unsigned int i;
+	u64 wlen;
+
+	if (c.sldc_immutable)
+		return 0;
+
+	for (i = 0; i < compressed - 1; i++) {
+		wlen = f2fs_write_addrtag(sbi, ino,
+				offset + (i << F2FS_BLKSIZE_BITS), NEW_ADDR);
+		if (wlen)
+			return wlen;
+	}
+	return 0;
+}
+
 int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
 {
 	int fd, n;
 	pgoff_t off = 0;
 	u8 buffer[BLOCK_SZ];
+	struct node_info ni;
+	struct f2fs_node *node_blk;
 
 	if (de->ino == 0)
 		return -1;
@@ -359,8 +463,6 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
 
 	/* inline_data support */
 	if (de->size <= DEF_MAX_INLINE_DATA) {
-		struct node_info ni;
-		struct f2fs_node *node_blk;
 		int ret;
 
 		get_node_info(sbi, de->ino, &ni);
@@ -385,6 +487,82 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
 		node_blk->i.i_size = cpu_to_le64(de->size);
 		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
 		free(node_blk);
+#ifdef WITH_SLOAD
+	} else if (c.func == SLOAD && c.sldc_en &&
+			c.sldc_ef->filter(de->full_path)) {
+		bool eof = false;
+		u8 *rbuf = c.sldc_cc.rbuf;
+		unsigned int cblocks = 0;
+
+		node_blk = calloc(BLOCK_SZ, 1);
+		ASSERT(node_blk);
+
+		/* read inode */
+		get_node_info(sbi, de->ino, &ni);
+		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
+		/* update inode meta */
+		node_blk->i.i_compress_algrithm = c.sldc_ca;
+		node_blk->i.i_log_cluster_size =
+				c.sldc_cc.log_cluster_size;
+		node_blk->i.i_flags = cpu_to_le32(
+				F2FS_COMPR_FL |
+				(c.sldc_immutable ? FS_IMMUTABLE_FL : 0));
+		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
+
+		while (!eof && (n = bulkread(fd, rbuf, c.sldc_cc.rlen,
+				&eof)) > 0) {
+			int ret = c.sldc_compr->compress(&c.sldc_cc);
+			u64 wlen;
+			u32 csize = ALIGN_UP(c.sldc_cc.clen +
+					COMPRESS_HEADER_SIZE, BLOCK_SZ);
+			unsigned int cur_cblk;
+
+			if (ret || n < (int)(csize + BLOCK_SZ *
+						c.sldc_min_cbpc)) {
+				wlen = f2fs_write(sbi, de->ino, rbuf, n, off);
+				ASSERT((int)wlen == n);
+			} else {
+				wlen = f2fs_write_addrtag(sbi, de->ino, off,
+						WR_COMPRESS_ADDR);
+				ASSERT(!wlen);
+				wlen = f2fs_write_compress_data(sbi, de->ino,
+						(u8 *)c.sldc_cc.cbuf,
+						csize, off + BLOCK_SZ);
+				ASSERT(wlen == csize);
+				sldc_erase_bufs(&c.sldc_cc);
+				cur_cblk = (c.sldc_cc.rlen - csize) / BLOCK_SZ;
+				cblocks += cur_cblk;
+				wlen = f2fs_fix_mutable(sbi, de->ino,
+						off + BLOCK_SZ + csize,
+						cur_cblk);
+				ASSERT(!wlen);
+			}
+			off += n;
+		}
+		if (n == -1) {
+			fprintf(stderr, "Load file '%s' failed: ",
+					de->full_path);
+			perror(NULL);
+		}
+		/* read inode */
+		get_node_info(sbi, de->ino, &ni);
+		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
+		/* update inode meta */
+		node_blk->i.i_size = cpu_to_le64(off);
+		if (!c.sldc_immutable)
+			node_blk->i.i_compr_blocks = cpu_to_le64(cblocks);
+		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
+		free(node_blk);
+
+		if (!c.sldc_immutable) {
+			sbi->total_valid_block_count += cblocks;
+			if (sbi->total_valid_block_count >=
+					sbi->user_block_count) {
+				ERR_MSG("Not enough space\n");
+				ASSERT(0);
+			}
+		}
+#endif
 	} else {
 		while ((n = read(fd, buffer, BLOCK_SZ)) > 0) {
 			f2fs_write(sbi, de->ino, buffer, n, off);
diff --git a/fsck/sload.c b/fsck/sload.c
index 14012fb..13e523a 100644
--- a/fsck/sload.c
+++ b/fsck/sload.c
@@ -6,6 +6,9 @@
  *   Hou Pengyang <houpengyang@huawei.com>
  *   Liu Shuoran <liushuoran@huawei.com>
  *   Jaegeuk Kim <jaegeuk@kernel.org>
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -317,6 +320,70 @@ skip:
 	return 0;
 }
 
+typedef struct _ext_tbl {
+	const char *ext;
+	struct _ext_tbl *next; /* linked list */
+} ext_tbl_t;
+static ext_tbl_t *ext_tbl;
+
+static bool ext_found(const char *ext)
+{
+	ext_tbl_t *p = ext_tbl;
+
+	while (p != NULL && strcmp(ext, p->ext))
+		p = p->next;
+	return (p != NULL);
+}
+
+static const char *get_ext(const char *path)
+{
+	char *p = strrchr(path, '.');
+	return p == NULL ? path + strlen(path) : p + 1;
+}
+
+static bool ext_do_filter(const char *path)
+{
+	return (ext_found(get_ext(path)) == true) ^ (c.sldc_policy == FP_ALLOW);
+}
+
+static void ext_filter_add(const char *ext)
+{
+	ext_tbl_t *node;
+
+	ASSERT(ext != NULL);
+	if (ext_found(ext))
+		return; /* ext was already registered */
+	node = malloc(sizeof(ext_tbl_t));
+	ASSERT(node != NULL);
+	node->ext = ext;
+	node->next = ext_tbl;
+	ext_tbl = node;
+}
+
+static void ext_filter_destroy(void)
+{
+	ext_tbl_t *p;
+
+	while (ext_tbl != NULL) {
+		p = ext_tbl;
+		ext_tbl = p->next;
+		free(p);
+	}
+}
+
+struct ext_tbl_op ext_filter = {
+	.add = ext_filter_add,
+	.destroy = ext_filter_destroy,
+	.filter = ext_do_filter,
+};
+
+void sldc_erase_bufs(struct compress_ctx *cc)
+{
+	memset(cc->rbuf, 0, cc->cluster_size * F2FS_BLKSIZE);
+	memset(cc->cbuf->cdata, 0, cc->cluster_size * F2FS_BLKSIZE
+			- F2FS_BLKSIZE);
+}
+
 int f2fs_sload(struct f2fs_sb_info *sbi)
 {
 	int ret = 0;
diff --git a/include/f2fs_fs.h b/include/f2fs_fs.h
index 1348e39..2a2dc15 100644
--- a/include/f2fs_fs.h
+++ b/include/f2fs_fs.h
@@ -5,6 +5,9 @@
  *             http://www.samsung.com/
  * Copyright (c) 2019 Google Inc.
  *             http://www.google.com/
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
  *
  * Dual licensed under the GPL or LGPL version 2 licenses.
  *
@@ -68,6 +71,10 @@ typedef uint16_t u_int16_t;
 typedef uint8_t u_int8_t;
 #endif
 
+/* codes from kernel's f2fs.h, GPL-v2.0 */
+#define MIN_COMPRESS_LOG_SIZE	2
+#define MAX_COMPRESS_LOG_SIZE	8
+
 typedef u_int64_t	u64;
 typedef u_int32_t	u32;
 typedef u_int16_t	u16;
@@ -93,6 +100,31 @@ typedef u32	__be32;
 typedef u64	__be64;
 #endif
 
+/*
+ * code borrowed from kernel f2fs dirver: f2fs.h, GPL-2.0
+ *  : definitions of COMPRESS_DATA_RESERVED_SIZE,
+ *    struct compress_data, COMPRESS_HEADER_SIZE,
+ *    and struct compress_ctx
+ */
+#define COMPRESS_DATA_RESERVED_SIZE		4
+struct compress_data {
+	__le32 clen;			/* compressed data size */
+	__le32 chksum;			/* checksum of compressed data */
+	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
+	u8 cdata[];			/* compressed data */
+};
+#define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
+/* compress context */
+struct compress_ctx {
+	unsigned int cluster_size;	/* page count in cluster */
+	unsigned int log_cluster_size;	/* log of cluster size */
+	void *rbuf;			/* compression input buffer */
+	struct compress_data *cbuf;	/* comprsssion output header + data */
+	size_t rlen;			/* valid data length in rbuf */
+	size_t clen;			/* valid data length in cbuf */
+	void *private;			/* work buf for compress algorithm */
+};
+
 #if HAVE_BYTESWAP_H
 #include <byteswap.h>
 #else
@@ -345,6 +377,25 @@ typedef struct {
 	bool dbg_en;
 } dev_cache_config_t;
 
+/* f2fs_configration: sldc_ca, the sload compress algorithm */
+enum {CA_LZO, CA_LZ4};
+extern const char *ca_names[];
+
+typedef struct  {
+	void (*init)(struct compress_ctx *cc);
+	int (*compress)(struct compress_ctx *cc);
+} compress_ops;
+
+#define ALIGN_UP(value, size) ((value) + ((value) % (size) > 0 ? \
+		(size) - (value) % (size) : 0))
+
+enum filter_policy {FP_UNASSIGNED = 0, FP_ALLOW, FP_DENY};
+struct ext_tbl_op {
+	void (*add)(const char *);
+	void (*destroy)(void);
+	bool (*filter)(const char *);
+};
+
 struct f2fs_configuration {
 	u_int32_t reserved_segments;
 	u_int32_t new_reserved_segments;
@@ -441,6 +492,24 @@ struct f2fs_configuration {
 
 	/* cache parameters */
 	dev_cache_config_t cache_config;
+
+	/* quick dynamic buffer */
+	bool qbuf_initialized;
+	size_t qbufsize;
+	void *qbuf;
+
+	/* sldc: sload compression support */
+	bool sldc_en;
+	bool sldc_use_allow_list;  /* default false to use the deny list */
+	struct compress_ctx sldc_cc;
+	u8 sldc_ca; /* compress algorithm: 0 = LZO, 1 = LZ4 */
+	compress_ops *sldc_compr;
+	enum filter_policy sldc_policy;
+	/* max_cppc can used to specify minimum compression rate */
+	unsigned int sldc_min_cbpc; /* min compressed pages per cluster */
+	bool sldc_got_opt;
+	bool sldc_immutable;
+	struct ext_tbl_op *sldc_ef; /* extension filter */
 };
 
 #ifdef CONFIG_64BIT
@@ -1226,6 +1295,11 @@ extern void f2fs_release_sparse_resource(void);
 extern int f2fs_finalize_device(void);
 extern int f2fs_fsync_device(void);
 
+/* quick (shared) buffer */
+extern void qbuf_free(void);
+extern void *qbuf_alloc(size_t size);
+extern void qbuf_init(void);
+
 extern void dcache_init(void);
 extern void dcache_release(void);
 
@@ -1377,7 +1451,7 @@ int f2fs_reset_zone(int, void *);
 extern int f2fs_reset_zones(int);
 extern uint32_t f2fs_get_usable_segments(struct f2fs_super_block *sb);
 
-#define SIZE_ALIGN(val, size)	((val) + (size) - 1) / (size)
+#define SIZE_ALIGN(val, size)	(((val) + (size) - 1) / (size))
 #define SEG_ALIGN(blks)		SIZE_ALIGN(blks, c.blks_per_seg)
 #define ZONE_ALIGN(blks)	SIZE_ALIGN(blks, c.blks_per_seg * \
 					c.segs_per_zone)
diff --git a/lib/libf2fs_io.c b/lib/libf2fs_io.c
index 138285d..0280896 100644
--- a/lib/libf2fs_io.c
+++ b/lib/libf2fs_io.c
@@ -5,6 +5,9 @@
  *             http://www.samsung.com/
  * Copyright (c) 2019 Google Inc.
  *             http://www.google.com/
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add quick-buffer for sload compression support
  *
  * Dual licensed under the GPL or LGPL version 2 licenses.
  */
@@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = {
 };
 static int dcache_relocate_offset[16];
 
+/* quick (shared) buffer */
+static bool qbuf_initialized;
+static void *qbuf;
+static size_t qbufsize;
+void qbuf_free(void)
+{
+	ASSERT(qbuf_initialized);
+	if (qbuf != NULL) {
+		free(qbuf);
+		qbuf = NULL;
+		qbufsize = 0;
+	}
+}
+void *qbuf_alloc(size_t size)
+{
+	ASSERT(qbuf_initialized);
+	if (size > qbufsize) {
+		qbuf_free();
+		qbuf = malloc(size);
+	}
+	return qbuf;
+}
+void qbuf_init(void)
+{
+	if (qbuf_initialized)
+		return;
+	atexit(qbuf_free);
+	qbuf_initialized = true;
+}
+
 static void dcache_print_statistics(void)
 {
 	long i;
-- 
2.29.2.576.ga3fc446d84-goog



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support

[Jaegeuk Kim]

Hi Robin,

I found some bugs and want to suggest a fix.
I'll apply this on your patch and let me know, if you want to modify more on top
of this.

--- a/fsck/segment.c
+++ b/fsck/segment.c
@@ -115,6 +115,8 @@ int new_data_block(struct f2fs_sb_info *sbi, void *block,

        get_node_info(sbi, dn->nid, &ni);
        set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
+
+       dn->data_blkaddr = blkaddr;
        ret = reserve_new_block(sbi, &dn->data_blkaddr, &sum, type, 0);
        if (ret) {
                c.alloc_failed = 1;
@@ -517,8 +519,8 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
                                        COMPRESS_HEADER_SIZE, BLOCK_SZ);
                        unsigned int cur_cblk;

-                       if (ret || n < (int)(csize + BLOCK_SZ *
-                                               c.sldc_min_cbpc)) {
+                       if (ret || n < (1 << c.sldc_cc.log_cluster_size) * BLOCK_SZ ||
+                               n < (int)(csize + BLOCK_SZ * c.sldc_min_cbpc)) {
                                wlen = f2fs_write(sbi, de->ino, rbuf, n, off);
                                ASSERT((int)wlen == n);
                        } else {
@@ -549,8 +551,10 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
                ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
                /* update inode meta */
                node_blk->i.i_size = cpu_to_le64(off);
-               if (!c.sldc_immutable)
+               if (!c.sldc_immutable) {
                        node_blk->i.i_compr_blocks = cpu_to_le64(cblocks);
+                       node_blk->i.i_blocks += cpu_to_le64(cblocks);
+               }
                ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
                free(node_blk);


On 12/08, Robin Hsu wrote:
> From: Robin Hsu <robinhsu@google.com>
> 
> Add F2FS compression support for sload
> * Support file extension filter, either default-accept or default-deny
>   policy
> * Support choice of compression algorithm, LZO (version 2) or LZ4
>   (default)
> * Support custom log of cluster size
> * Support minimum number of compressed blocks per cluster (default 1).
>   A cluster will not be compressed if the number can not be met.
> * suuport -r (read-only) option
> 
> Signed-off-by: Robin Hsu <robinhsu@google.com>
> ---
>  fsck/compress_wrapper.c | 102 ++++++++++++++++++++
>  fsck/compress_wrapper.h |  22 +++++
>  fsck/fsck.h             |  15 +++
>  fsck/main.c             | 141 +++++++++++++++++++++++++++-
>  fsck/segment.c          | 202 +++++++++++++++++++++++++++++++++++++---
>  fsck/sload.c            |  67 +++++++++++++
>  include/f2fs_fs.h       |  76 ++++++++++++++-
>  lib/libf2fs_io.c        |  33 +++++++
>  8 files changed, 644 insertions(+), 14 deletions(-)
>  create mode 100644 fsck/compress_wrapper.c
>  create mode 100644 fsck/compress_wrapper.h
> 
> diff --git a/fsck/compress_wrapper.c b/fsck/compress_wrapper.c
> new file mode 100644
> index 0000000..2cdc4fd
> --- /dev/null
> +++ b/fsck/compress_wrapper.c
> @@ -0,0 +1,102 @@
> +/**
> + * compress_wrapper.c
> + *
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *   : initial created, for sload compression support
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +#include "f2fs.h" /* for config.h for general environment (non-Android) */
> +
> +#include "compress_wrapper.h"
> +#ifdef HAVE_LIBLZO2
> +#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */
> +#endif
> +#ifdef HAVE_LIBLZ4
> +#include <lz4.h> /* for LZ4_compress_fast_extState() */
> +#endif
> +
> +/*
> + * macro/constants borrowed from kernel header (GPL-2.0):
> + * include/linux/lzo.h, and include/linux/lz4.h
> + */
> +#ifdef HAVE_LIBLZO2
> +#define lzo1x_worst_compress(x)		((x) + (x) / 16 + 64 + 3 + 2)
> +#define LZO_WORK_SIZE			ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8)
> +#endif
> +#ifdef HAVE_LIBLZ4
> +#define LZ4_MEMORY_USAGE		14
> +#define LZ4_MAX_INPUT_SIZE		0x7E000000 /* 2 113 929 216 bytes */
> +#define LZ4_STREAMSIZE			(LZ4_STREAMSIZE_U64 * sizeof(long long))
> +#define LZ4_MEM_COMPRESS		LZ4_STREAMSIZE
> +#define LZ4_ACCELERATION_DEFAULT	1
> +#define LZ4_WORK_SIZE			ALIGN_UP(LZ4_MEM_COMPRESS, 8)
> +#endif
> +
> +#ifdef HAVE_LIBLZO2
> +static void lzo_compress_init(struct compress_ctx *cc)
> +{
> +	size_t size = cc->cluster_size * F2FS_BLKSIZE;
> +	size_t alloc = size + lzo1x_worst_compress(size)
> +			+ COMPRESS_HEADER_SIZE + LZO_WORK_SIZE;
> +	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
> +	cc->rbuf = (char *) cc->private + LZO_WORK_SIZE;
> +	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
> +}
> +
> +static int lzo_compress(struct compress_ctx *cc)
> +{
> +	int ret = lzo1x_1_15_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
> +			(lzo_uintp)(&cc->clen), cc->private);
> +	cc->cbuf->clen = cpu_to_le32(cc->clen);
> +	return ret;
> +}
> +#endif
> +
> +#ifdef HAVE_LIBLZ4
> +static void lz4_compress_init(struct compress_ctx *cc)
> +{
> +	size_t size = cc->cluster_size * F2FS_BLKSIZE;
> +	size_t alloc = size + LZ4_COMPRESSBOUND(size)
> +			+ COMPRESS_HEADER_SIZE + LZ4_WORK_SIZE;
> +	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
> +	cc->rbuf = (char *) cc->private + LZ4_WORK_SIZE;
> +	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
> +}
> +
> +static int lz4_compress(struct compress_ctx *cc)
> +{
> +	cc->clen = LZ4_compress_fast_extState(cc->private, cc->rbuf,
> +			(char *)cc->cbuf->cdata, cc->rlen,
> +			cc->rlen - F2FS_BLKSIZE * c.sldc_min_cbpc,
> +			LZ4_ACCELERATION_DEFAULT);
> +
> +	if (!cc->clen)
> +		return 1;
> +
> +	cc->cbuf->clen = cpu_to_le32(cc->clen);
> +	return 0;
> +}
> +#endif
> +
> +const char *ca_names[] = {
> +	"LZO",
> +	"LZ4",
> +	"", /* end of the name list */
> +};
> +
> +compress_ops compr_ops[] = {
> +#ifdef HAVE_LIBLZO2
> +	{lzo_compress_init, lzo_compress},
> +#else
> +	{NULL, NULL},
> +#endif
> +#ifdef HAVE_LIBLZ4
> +	{lz4_compress_init, lz4_compress},
> +#else
> +	{NULL, NULL},
> +#endif
> +};
> diff --git a/fsck/compress_wrapper.h b/fsck/compress_wrapper.h
> new file mode 100644
> index 0000000..ec33d43
> --- /dev/null
> +++ b/fsck/compress_wrapper.h
> @@ -0,0 +1,22 @@
> +/**
> + * compress_wrapper.h
> + *
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *   : initial created, for sload compression support
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +
> +#ifndef COMPRESS_WRAPPER_H
> +#define COMPRESS_WRAPPER_H
> +
> +#include "f2fs_fs.h"
> +extern compress_ops compr_ops[]; /* [0]: LZO, [1]: LZ4, */
> +
> +#endif /* COMPRESS_WRAPPER_H */
> diff --git a/fsck/fsck.h b/fsck/fsck.h
> index c5e85fe..4e866ec 100644
> --- a/fsck/fsck.h
> +++ b/fsck/fsck.h
> @@ -3,6 +3,9 @@
>   *
>   * Copyright (c) 2013 Samsung Electronics Co., Ltd.
>   *             http://www.samsung.com/
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * This program is free software; you can redistribute it and/or modify
>   * it under the terms of the GNU General Public License version 2 as
> @@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);
>  
>  /* sload.c */
>  int f2fs_sload(struct f2fs_sb_info *);
> +void sldc_erase_bufs(struct compress_ctx *cc);
> +void sload_countblk(void);
> +extern struct ext_tbl_op ext_filter;
>  
>  /* segment.c */
>  int reserve_new_block(struct f2fs_sb_info *, block_t *,
> @@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,
>  struct quota_file;
>  u64 f2fs_quota_size(struct quota_file *);
>  u64 f2fs_read(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> +enum wr_addr_type {
> +	WR_NORMAL = 1,
> +	WR_COMPRESS_DATA = 2,
> +	WR_NULL_ADDR = NULL_ADDR,  /* 0 */
> +	WR_NEW_ADDR = NEW_ADDR, /* -1U */
> +	WR_COMPRESS_ADDR = COMPRESS_ADDR, /* -2U */
> +};
>  u64 f2fs_write(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> +u64 f2fs_write_compress_data(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> +u64 f2fs_write_addrtag(struct f2fs_sb_info *, nid_t, pgoff_t, unsigned int);
>  void f2fs_filesize_update(struct f2fs_sb_info *, nid_t, u64);
>  
>  int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,
> diff --git a/fsck/main.c b/fsck/main.c
> index b20498f..001eed0 100644
> --- a/fsck/main.c
> +++ b/fsck/main.c
> @@ -13,6 +13,9 @@
>   * Copyright (c) 2019 Google Inc.
>   *   Robin Hsu <robinhsu@google.com>
>   *  : add cache layer
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * This program is free software; you can redistribute it and/or modify
>   * it under the terms of the GNU General Public License version 2 as
> @@ -25,6 +28,7 @@
>  #include <getopt.h>
>  #include <stdbool.h>
>  #include "quotaio.h"
> +#include "compress_wrapper.h"
>  
>  struct f2fs_fsck gfsck;
>  
> @@ -134,6 +138,17 @@ void sload_usage()
>  	MSG(0, "  -S sparse_mode\n");
>  	MSG(0, "  -t mount point [prefix of target fs path, default:/]\n");
>  	MSG(0, "  -T timestamp\n");
> +	MSG(0, "  -c enable compression (default allow policy)\n");
> +	MSG(0, "    ------------ Compression sub-options -----------------\n");
> +	MSG(0, "    -L <log-of-blocks-per-cluster>, default 2\n");
> +	MSG(0, "    -a <algorithm> compression algorithm, default LZ4\n");
> +	MSG(0, "    -x <ext> compress files except for these extensions.\n");
> +	MSG(0, "    -i <ext> compress files with these extensions only.\n");
> +	MSG(0, "    * -i or -x: use it many times for multiple extensions.\n");
> +	MSG(0, "    * -i and -x cannot be used together..\n");
> +	MSG(0, "    -m <num> min compressed blocks per cluster\n");
> +	MSG(0, "    -r readonly (IMMUTABLE) for compressed files\n");
> +	MSG(0, "    ------------------------------------------------------\n");
>  	MSG(0, "  -d debug level [default:0]\n");
>  	MSG(0, "  -V print the version number and exit\n");
>  	exit(1);
> @@ -534,7 +549,7 @@ void f2fs_parse_options(int argc, char *argv[])
>  #endif
>  	} else if (!strcmp("sload.f2fs", prog)) {
>  #ifdef WITH_SLOAD
> -		const char *option_string = "C:d:f:p:s:St:T:V";
> +		const char *option_string = "cL:a:i:x:m:rC:d:f:p:s:St:T:V";
>  #ifdef HAVE_LIBSELINUX
>  		int max_nr_opt = (int)sizeof(c.seopt_file) /
>  			sizeof(c.seopt_file[0]);
> @@ -543,8 +558,82 @@ void f2fs_parse_options(int argc, char *argv[])
>  		char *p;
>  
>  		c.func = SLOAD;
> +		c.sldc_cc.log_cluster_size = 2;
> +		c.sldc_ca = CA_LZ4;
> +		c.sldc_min_cbpc = 1;
> +		c.sldc_ef = &ext_filter;
>  		while ((option = getopt(argc, argv, option_string)) != EOF) {
> +			unsigned int i;
> +			int val;
> +
>  			switch (option) {
> +			case 'c': /* compression support */
> +				c.sldc_en = true;
> +				break;
> +			case 'L': /* compression: log of blocks-per-cluster */
> +				c.sldc_got_opt = true;
> +				val = atoi(optarg);
> +				if (val < MIN_COMPRESS_LOG_SIZE ||
> +						val > MAX_COMPRESS_LOG_SIZE) {
> +					MSG(0, "\tError: log of blocks per"
> +						" cluster must be in the range"
> +						" of %d .. %d.\n",
> +						MIN_COMPRESS_LOG_SIZE,
> +						MAX_COMPRESS_LOG_SIZE);
> +					error_out(prog);
> +				}
> +				c.sldc_cc.log_cluster_size = val;
> +				break;
> +			case 'a': /* compression: choose algorithm */
> +				c.sldc_got_opt = true;
> +				c.sldc_ca = (u8)-1;
> +				for (i = 0; ca_names[i][0] != 0; i++) {
> +					if (!strcmp(ca_names[i], optarg)) {
> +						c.sldc_ca = i;
> +						break;
> +					}
> +				}
> +				if (c.sldc_ca == (u8)-1) {
> +					MSG(0, "\tError: Unknown compression"
> +						" algorithm %s\n", optarg);
> +					error_out(prog);
> +				}
> +				break;
> +			case 'i': /* compress only these extensions */
> +				c.sldc_got_opt = true;
> +				if (c.sldc_policy == FP_ALLOW) {
> +					MSG(0, "\tError: could not mix option"
> +							" -i and -x\n");
> +					error_out(prog);
> +				}
> +				c.sldc_policy = FP_DENY;
> +				c.sldc_ef->add(optarg);
> +				break;
> +			case 'x': /* compress except for these extensions */
> +				c.sldc_got_opt = true;
> +				if (c.sldc_policy == FP_DENY) {
> +					MSG(0, "\tError: could not mix option"
> +							" -i and -x\n");
> +					error_out(prog);
> +				}
> +				c.sldc_policy = FP_ALLOW;
> +				c.sldc_ef->add(optarg);
> +				break;
> +			case 'm': /* minimum compressed blocks per cluster */
> +				c.sldc_got_opt = true;
> +				val = atoi(optarg);
> +				if (val <= 0) {
> +					MSG(0, "\tError: minimum compressed"
> +						" blocks per cluster must be"
> +						" positive.\n");
> +					error_out(prog);
> +				}
> +				c.sldc_min_cbpc = val;
> +				break;
> +			case 'r': /* compress file to set IMMUTABLE */
> +				c.sldc_got_opt = true;
> +				c.sldc_immutable = true;
> +				break;
>  			case 'C':
>  				c.fs_config_file = absolute_path(optarg);
>  				break;
> @@ -602,6 +691,27 @@ void f2fs_parse_options(int argc, char *argv[])
>  			if (err != NOERROR)
>  				break;
>  		}
> +		if (c.sldc_got_opt && !c.sldc_en) {
> +			MSG(0, "\tError: compression sub-options are used"
> +				" without the compression enable (-c) option\n"
> +			);
> +			error_out(prog);
> +		}
> +		if (err == NOERROR && c.sldc_en) {
> +			c.sldc_cc.cluster_size = 1
> +				<< c.sldc_cc.log_cluster_size;
> +			if (c.sldc_policy == FP_UNASSIGNED)
> +				c.sldc_policy = FP_ALLOW;
> +			if (c.sldc_min_cbpc >= c.sldc_cc.cluster_size) {
> +				MSG(0, "\tError: minimum reduced blocks by"
> +					" compression per cluster must be at"
> +					" most one less than blocks per"
> +					" cluster, i.e. %d\n",
> +					c.sldc_cc.cluster_size - 1);
> +				error_out(prog);
> +			}
> +			qbuf_init();
> +		}
>  #endif /* WITH_SLOAD */
>  	}
>  
> @@ -812,6 +922,27 @@ static int do_resize(struct f2fs_sb_info *sbi)
>  #endif
>  
>  #ifdef WITH_SLOAD
> +int init_compr(struct f2fs_sb_info *sbi)
> +{
> +	if (!(sbi->raw_super->feature
> +			& cpu_to_le32(F2FS_FEATURE_COMPRESSION))) {
> +		MSG(0, "Error: Compression (-c) was requested "
> +			"but the file system is not created "
> +			"with such feature.\n");
> +		return -1;
> +	}
> +	if (compr_ops[c.sldc_ca].init == NULL) {
> +		MSG(0, "Error: The selected compression algorithm is not"
> +				" supported\n");
> +		return -1;
> +	}
> +	c.sldc_compr = compr_ops + c.sldc_ca;
> +	c.sldc_compr->init(&c.sldc_cc);
> +	sldc_erase_bufs(&c.sldc_cc);
> +	c.sldc_cc.rlen = c.sldc_cc.cluster_size * F2FS_BLKSIZE;
> +	return 0;
> +}
> +
>  static int do_sload(struct f2fs_sb_info *sbi)
>  {
>  	if (!c.from_dir) {
> @@ -821,6 +952,11 @@ static int do_sload(struct f2fs_sb_info *sbi)
>  	if (!c.mount_point)
>  		c.mount_point = "/";
>  
> +	if (c.sldc_en) {
> +		if (init_compr(sbi))
> +			return -1;
> +	}
> +
>  	return f2fs_sload(sbi);
>  }
>  #endif
> @@ -971,6 +1107,9 @@ retry:
>  		return ret2;
>  	}
>  
> +	if (c.func == SLOAD)
> +		c.sldc_ef->destroy();
> +
>  	printf("\nDone: %lf secs\n", (get_boottime_ns() - start) / 1000000000.0);
>  	return ret;
>  
> diff --git a/fsck/segment.c b/fsck/segment.c
> index 0487f41..e4c8cea 100644
> --- a/fsck/segment.c
> +++ b/fsck/segment.c
> @@ -8,6 +8,9 @@
>   *   Hou Pengyang <houpengyang@huawei.com>
>   *   Liu Shuoran <liushuoran@huawei.com>
>   *   Jaegeuk Kim <jaegeuk@kernel.org>
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * This program is free software; you can redistribute it and/or modify
>   * it under the terms of the GNU General Public License version 2 as
> @@ -16,6 +19,7 @@
>  #include "fsck.h"
>  #include "node.h"
>  #include "quotaio.h"
> +#include "compress_wrapper.h"
>  
>  int reserve_new_block(struct f2fs_sb_info *sbi, block_t *to,
>  			struct f2fs_summary *sum, int type, bool is_inode)
> @@ -228,8 +232,14 @@ u64 f2fs_read(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  	return read_count;
>  }
>  
> -u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> -					u64 count, pgoff_t offset)
> +/*
> + * Do not call this function directly.  Instead, call one of the following:
> + *     u64 f2fs_write();
> + *     u64 f2fs_write_compress_data();
> + *     u64 f2fs_write_addrtag();
> + */
> +static u64 f2fs_write_ex(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> +		u64 count, pgoff_t offset, enum wr_addr_type addr_type)
>  {
>  	struct dnode_of_data dn;
>  	struct node_info ni;
> @@ -243,6 +253,19 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  	void* index_node = NULL;
>  	int idirty = 0;
>  	int err;
> +	bool has_data = (addr_type == WR_NORMAL
> +			|| addr_type == WR_COMPRESS_DATA);
> +
> +	if (count == 0)
> +		return 0;
> +
> +	/*
> +	 * Enforce calling from f2fs_write(), f2fs_write_compress_data(),
> +	 * and f2fs_write_addrtag().   Beside, check if is properly called.
> +	 */
> +	ASSERT((!has_data && buffer == NULL) || (has_data && buffer != NULL));
> +	if (addr_type != WR_NORMAL)
> +		ASSERT(offset % F2FS_BLKSIZE == 0); /* block boundary only */
>  
>  	/* Memory allocation for block buffer and inode. */
>  	blk_buffer = calloc(BLOCK_SZ, 2);
> @@ -265,15 +288,26 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  			if (err)
>  				break;
>  			idirty |= dn.idirty;
> -			if (index_node)
> -				free(index_node);
> +			free(index_node);
>  			index_node = (dn.node_blk == dn.inode_blk) ?
> -							NULL : dn.node_blk;
> +					NULL : dn.node_blk;
>  			remained_blkentries = ADDRS_PER_PAGE(sbi,
> -						dn.node_blk, dn.inode_blk);
> +					dn.node_blk, dn.inode_blk) -
> +					dn.ofs_in_node;
>  		}
>  		ASSERT(remained_blkentries > 0);
>  
> +		if (!has_data) {
> +			dn.data_blkaddr = addr_type;
> +			set_data_blkaddr(&dn);
> +			idirty |= dn.idirty;
> +			if (dn.ndirty)
> +				ASSERT(dev_write_block(dn.node_blk,
> +						dn.node_blkaddr) >= 0);
> +			written_count = 0;
> +			break;
> +		}
> +
>  		blkaddr = datablock_addr(dn.node_blk, dn.ofs_in_node);
>  		if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) {
>  			err = new_data_block(sbi, blk_buffer,
> @@ -281,6 +315,7 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  			if (err)
>  				break;
>  			blkaddr = dn.data_blkaddr;
> +			idirty |= dn.idirty;
>  		}
>  
>  		off_in_blk = offset % BLOCK_SZ;
> @@ -305,9 +340,10 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  
>  		dn.ofs_in_node++;
>  		if ((--remained_blkentries == 0 || count == 0) && (dn.ndirty))
> -			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr) >= 0);
> +			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr)
> +					>= 0);
>  	}
> -	if (offset > le64_to_cpu(inode->i.i_size)) {
> +	if (addr_type == WR_NORMAL && offset > le64_to_cpu(inode->i.i_size)) {
>  		inode->i.i_size = cpu_to_le64(offset);
>  		idirty = 1;
>  	}
> @@ -315,13 +351,33 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  		ASSERT(inode == dn.inode_blk);
>  		ASSERT(write_inode(inode, ni.blk_addr) >= 0);
>  	}
> -	if (index_node)
> -		free(index_node);
> +
> +	free(index_node);
>  	free(blk_buffer);
>  
>  	return written_count;
>  }
>  
> +u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> +					u64 count, pgoff_t offset)
> +{
> +	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_NORMAL);
> +}
> +
> +u64 f2fs_write_compress_data(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> +					u64 count, pgoff_t offset)
> +{
> +	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_COMPRESS_DATA);
> +}
> +
> +u64 f2fs_write_addrtag(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
> +		unsigned int addrtag)
> +{
> +	ASSERT(addrtag == COMPRESS_ADDR || addrtag == NEW_ADDR
> +			|| addrtag == NULL_ADDR);
> +	return f2fs_write_ex(sbi, ino, NULL, F2FS_BLKSIZE, offset, addrtag);
> +}
> +
>  /* This function updates only inode->i.i_size */
>  void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
>  {
> @@ -342,11 +398,59 @@ void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
>  	free(inode);
>  }
>  
> +#define MAX_BULKR_RETRY 5
> +int bulkread(int fd, void *rbuf, size_t rsize, bool *eof)
> +{
> +	int n = 0;
> +	int retry = MAX_BULKR_RETRY;
> +	int cur;
> +
> +	if (!rsize)
> +		return 0;
> +
> +	if (eof != NULL)
> +		*eof = false;
> +	while (rsize && (cur = read(fd, rbuf, rsize)) != 0) {
> +		if (cur == -1) {
> +			if (errno == EINTR && retry--)
> +				continue;
> +			return -1;
> +		}
> +		retry = MAX_BULKR_RETRY;
> +
> +		rsize -= cur;
> +		n += cur;
> +	}
> +	if (eof != NULL)
> +		*eof = (cur == 0);
> +	return n;
> +}
> +
> +u64 f2fs_fix_mutable(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
> +		unsigned int compressed)
> +{
> +	unsigned int i;
> +	u64 wlen;
> +
> +	if (c.sldc_immutable)
> +		return 0;
> +
> +	for (i = 0; i < compressed - 1; i++) {
> +		wlen = f2fs_write_addrtag(sbi, ino,
> +				offset + (i << F2FS_BLKSIZE_BITS), NEW_ADDR);
> +		if (wlen)
> +			return wlen;
> +	}
> +	return 0;
> +}
> +
>  int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
>  {
>  	int fd, n;
>  	pgoff_t off = 0;
>  	u8 buffer[BLOCK_SZ];
> +	struct node_info ni;
> +	struct f2fs_node *node_blk;
>  
>  	if (de->ino == 0)
>  		return -1;
> @@ -359,8 +463,6 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
>  
>  	/* inline_data support */
>  	if (de->size <= DEF_MAX_INLINE_DATA) {
> -		struct node_info ni;
> -		struct f2fs_node *node_blk;
>  		int ret;
>  
>  		get_node_info(sbi, de->ino, &ni);
> @@ -385,6 +487,82 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
>  		node_blk->i.i_size = cpu_to_le64(de->size);
>  		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
>  		free(node_blk);
> +#ifdef WITH_SLOAD
> +	} else if (c.func == SLOAD && c.sldc_en &&
> +			c.sldc_ef->filter(de->full_path)) {
> +		bool eof = false;
> +		u8 *rbuf = c.sldc_cc.rbuf;
> +		unsigned int cblocks = 0;
> +
> +		node_blk = calloc(BLOCK_SZ, 1);
> +		ASSERT(node_blk);
> +
> +		/* read inode */
> +		get_node_info(sbi, de->ino, &ni);
> +		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
> +		/* update inode meta */
> +		node_blk->i.i_compress_algrithm = c.sldc_ca;
> +		node_blk->i.i_log_cluster_size =
> +				c.sldc_cc.log_cluster_size;
> +		node_blk->i.i_flags = cpu_to_le32(
> +				F2FS_COMPR_FL |
> +				(c.sldc_immutable ? FS_IMMUTABLE_FL : 0));
> +		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
> +
> +		while (!eof && (n = bulkread(fd, rbuf, c.sldc_cc.rlen,
> +				&eof)) > 0) {
> +			int ret = c.sldc_compr->compress(&c.sldc_cc);
> +			u64 wlen;
> +			u32 csize = ALIGN_UP(c.sldc_cc.clen +
> +					COMPRESS_HEADER_SIZE, BLOCK_SZ);
> +			unsigned int cur_cblk;
> +
> +			if (ret || n < (int)(csize + BLOCK_SZ *
> +						c.sldc_min_cbpc)) {
> +				wlen = f2fs_write(sbi, de->ino, rbuf, n, off);
> +				ASSERT((int)wlen == n);
> +			} else {
> +				wlen = f2fs_write_addrtag(sbi, de->ino, off,
> +						WR_COMPRESS_ADDR);
> +				ASSERT(!wlen);
> +				wlen = f2fs_write_compress_data(sbi, de->ino,
> +						(u8 *)c.sldc_cc.cbuf,
> +						csize, off + BLOCK_SZ);
> +				ASSERT(wlen == csize);
> +				sldc_erase_bufs(&c.sldc_cc);
> +				cur_cblk = (c.sldc_cc.rlen - csize) / BLOCK_SZ;
> +				cblocks += cur_cblk;
> +				wlen = f2fs_fix_mutable(sbi, de->ino,
> +						off + BLOCK_SZ + csize,
> +						cur_cblk);
> +				ASSERT(!wlen);
> +			}
> +			off += n;
> +		}
> +		if (n == -1) {
> +			fprintf(stderr, "Load file '%s' failed: ",
> +					de->full_path);
> +			perror(NULL);
> +		}
> +		/* read inode */
> +		get_node_info(sbi, de->ino, &ni);
> +		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
> +		/* update inode meta */
> +		node_blk->i.i_size = cpu_to_le64(off);
> +		if (!c.sldc_immutable)
> +			node_blk->i.i_compr_blocks = cpu_to_le64(cblocks);
> +		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
> +		free(node_blk);
> +
> +		if (!c.sldc_immutable) {
> +			sbi->total_valid_block_count += cblocks;
> +			if (sbi->total_valid_block_count >=
> +					sbi->user_block_count) {
> +				ERR_MSG("Not enough space\n");
> +				ASSERT(0);
> +			}
> +		}
> +#endif
>  	} else {
>  		while ((n = read(fd, buffer, BLOCK_SZ)) > 0) {
>  			f2fs_write(sbi, de->ino, buffer, n, off);
> diff --git a/fsck/sload.c b/fsck/sload.c
> index 14012fb..13e523a 100644
> --- a/fsck/sload.c
> +++ b/fsck/sload.c
> @@ -6,6 +6,9 @@
>   *   Hou Pengyang <houpengyang@huawei.com>
>   *   Liu Shuoran <liushuoran@huawei.com>
>   *   Jaegeuk Kim <jaegeuk@kernel.org>
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * This program is free software; you can redistribute it and/or modify
>   * it under the terms of the GNU General Public License version 2 as
> @@ -317,6 +320,70 @@ skip:
>  	return 0;
>  }
>  
> +typedef struct _ext_tbl {
> +	const char *ext;
> +	struct _ext_tbl *next; /* linked list */
> +} ext_tbl_t;
> +static ext_tbl_t *ext_tbl;
> +
> +static bool ext_found(const char *ext)
> +{
> +	ext_tbl_t *p = ext_tbl;
> +
> +	while (p != NULL && strcmp(ext, p->ext))
> +		p = p->next;
> +	return (p != NULL);
> +}
> +
> +static const char *get_ext(const char *path)
> +{
> +	char *p = strrchr(path, '.');
> +	return p == NULL ? path + strlen(path) : p + 1;
> +}
> +
> +static bool ext_do_filter(const char *path)
> +{
> +	return (ext_found(get_ext(path)) == true) ^ (c.sldc_policy == FP_ALLOW);
> +}
> +
> +static void ext_filter_add(const char *ext)
> +{
> +	ext_tbl_t *node;
> +
> +	ASSERT(ext != NULL);
> +	if (ext_found(ext))
> +		return; /* ext was already registered */
> +	node = malloc(sizeof(ext_tbl_t));
> +	ASSERT(node != NULL);
> +	node->ext = ext;
> +	node->next = ext_tbl;
> +	ext_tbl = node;
> +}
> +
> +static void ext_filter_destroy(void)
> +{
> +	ext_tbl_t *p;
> +
> +	while (ext_tbl != NULL) {
> +		p = ext_tbl;
> +		ext_tbl = p->next;
> +		free(p);
> +	}
> +}
> +
> +struct ext_tbl_op ext_filter = {
> +	.add = ext_filter_add,
> +	.destroy = ext_filter_destroy,
> +	.filter = ext_do_filter,
> +};
> +
> +void sldc_erase_bufs(struct compress_ctx *cc)
> +{
> +	memset(cc->rbuf, 0, cc->cluster_size * F2FS_BLKSIZE);
> +	memset(cc->cbuf->cdata, 0, cc->cluster_size * F2FS_BLKSIZE
> +			- F2FS_BLKSIZE);
> +}
> +
>  int f2fs_sload(struct f2fs_sb_info *sbi)
>  {
>  	int ret = 0;
> diff --git a/include/f2fs_fs.h b/include/f2fs_fs.h
> index 1348e39..2a2dc15 100644
> --- a/include/f2fs_fs.h
> +++ b/include/f2fs_fs.h
> @@ -5,6 +5,9 @@
>   *             http://www.samsung.com/
>   * Copyright (c) 2019 Google Inc.
>   *             http://www.google.com/
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * Dual licensed under the GPL or LGPL version 2 licenses.
>   *
> @@ -68,6 +71,10 @@ typedef uint16_t u_int16_t;
>  typedef uint8_t u_int8_t;
>  #endif
>  
> +/* codes from kernel's f2fs.h, GPL-v2.0 */
> +#define MIN_COMPRESS_LOG_SIZE	2
> +#define MAX_COMPRESS_LOG_SIZE	8
> +
>  typedef u_int64_t	u64;
>  typedef u_int32_t	u32;
>  typedef u_int16_t	u16;
> @@ -93,6 +100,31 @@ typedef u32	__be32;
>  typedef u64	__be64;
>  #endif
>  
> +/*
> + * code borrowed from kernel f2fs dirver: f2fs.h, GPL-2.0
> + *  : definitions of COMPRESS_DATA_RESERVED_SIZE,
> + *    struct compress_data, COMPRESS_HEADER_SIZE,
> + *    and struct compress_ctx
> + */
> +#define COMPRESS_DATA_RESERVED_SIZE		4
> +struct compress_data {
> +	__le32 clen;			/* compressed data size */
> +	__le32 chksum;			/* checksum of compressed data */
> +	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
> +	u8 cdata[];			/* compressed data */
> +};
> +#define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
> +/* compress context */
> +struct compress_ctx {
> +	unsigned int cluster_size;	/* page count in cluster */
> +	unsigned int log_cluster_size;	/* log of cluster size */
> +	void *rbuf;			/* compression input buffer */
> +	struct compress_data *cbuf;	/* comprsssion output header + data */
> +	size_t rlen;			/* valid data length in rbuf */
> +	size_t clen;			/* valid data length in cbuf */
> +	void *private;			/* work buf for compress algorithm */
> +};
> +
>  #if HAVE_BYTESWAP_H
>  #include <byteswap.h>
>  #else
> @@ -345,6 +377,25 @@ typedef struct {
>  	bool dbg_en;
>  } dev_cache_config_t;
>  
> +/* f2fs_configration: sldc_ca, the sload compress algorithm */
> +enum {CA_LZO, CA_LZ4};
> +extern const char *ca_names[];
> +
> +typedef struct  {
> +	void (*init)(struct compress_ctx *cc);
> +	int (*compress)(struct compress_ctx *cc);
> +} compress_ops;
> +
> +#define ALIGN_UP(value, size) ((value) + ((value) % (size) > 0 ? \
> +		(size) - (value) % (size) : 0))
> +
> +enum filter_policy {FP_UNASSIGNED = 0, FP_ALLOW, FP_DENY};
> +struct ext_tbl_op {
> +	void (*add)(const char *);
> +	void (*destroy)(void);
> +	bool (*filter)(const char *);
> +};
> +
>  struct f2fs_configuration {
>  	u_int32_t reserved_segments;
>  	u_int32_t new_reserved_segments;
> @@ -441,6 +492,24 @@ struct f2fs_configuration {
>  
>  	/* cache parameters */
>  	dev_cache_config_t cache_config;
> +
> +	/* quick dynamic buffer */
> +	bool qbuf_initialized;
> +	size_t qbufsize;
> +	void *qbuf;
> +
> +	/* sldc: sload compression support */
> +	bool sldc_en;
> +	bool sldc_use_allow_list;  /* default false to use the deny list */
> +	struct compress_ctx sldc_cc;
> +	u8 sldc_ca; /* compress algorithm: 0 = LZO, 1 = LZ4 */
> +	compress_ops *sldc_compr;
> +	enum filter_policy sldc_policy;
> +	/* max_cppc can used to specify minimum compression rate */
> +	unsigned int sldc_min_cbpc; /* min compressed pages per cluster */
> +	bool sldc_got_opt;
> +	bool sldc_immutable;
> +	struct ext_tbl_op *sldc_ef; /* extension filter */
>  };
>  
>  #ifdef CONFIG_64BIT
> @@ -1226,6 +1295,11 @@ extern void f2fs_release_sparse_resource(void);
>  extern int f2fs_finalize_device(void);
>  extern int f2fs_fsync_device(void);
>  
> +/* quick (shared) buffer */
> +extern void qbuf_free(void);
> +extern void *qbuf_alloc(size_t size);
> +extern void qbuf_init(void);
> +
>  extern void dcache_init(void);
>  extern void dcache_release(void);
>  
> @@ -1377,7 +1451,7 @@ int f2fs_reset_zone(int, void *);
>  extern int f2fs_reset_zones(int);
>  extern uint32_t f2fs_get_usable_segments(struct f2fs_super_block *sb);
>  
> -#define SIZE_ALIGN(val, size)	((val) + (size) - 1) / (size)
> +#define SIZE_ALIGN(val, size)	(((val) + (size) - 1) / (size))
>  #define SEG_ALIGN(blks)		SIZE_ALIGN(blks, c.blks_per_seg)
>  #define ZONE_ALIGN(blks)	SIZE_ALIGN(blks, c.blks_per_seg * \
>  					c.segs_per_zone)
> diff --git a/lib/libf2fs_io.c b/lib/libf2fs_io.c
> index 138285d..0280896 100644
> --- a/lib/libf2fs_io.c
> +++ b/lib/libf2fs_io.c
> @@ -5,6 +5,9 @@
>   *             http://www.samsung.com/
>   * Copyright (c) 2019 Google Inc.
>   *             http://www.google.com/
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add quick-buffer for sload compression support
>   *
>   * Dual licensed under the GPL or LGPL version 2 licenses.
>   */
> @@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = {
>  };
>  static int dcache_relocate_offset[16];
>  
> +/* quick (shared) buffer */
> +static bool qbuf_initialized;
> +static void *qbuf;
> +static size_t qbufsize;
> +void qbuf_free(void)
> +{
> +	ASSERT(qbuf_initialized);
> +	if (qbuf != NULL) {
> +		free(qbuf);
> +		qbuf = NULL;
> +		qbufsize = 0;
> +	}
> +}
> +void *qbuf_alloc(size_t size)
> +{
> +	ASSERT(qbuf_initialized);
> +	if (size > qbufsize) {
> +		qbuf_free();
> +		qbuf = malloc(size);
> +	}
> +	return qbuf;
> +}
> +void qbuf_init(void)
> +{
> +	if (qbuf_initialized)
> +		return;
> +	atexit(qbuf_free);
> +	qbuf_initialized = true;
> +}
> +
>  static void dcache_print_statistics(void)
>  {
>  	long i;
> -- 
> 2.29.2.576.ga3fc446d84-goog


_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support

[Chao Yu]

On 2020/12/8 16:15, Robin Hsu wrote:
> From: Robin Hsu <robinhsu@google.com>
> 
> Add F2FS compression support for sload
> * Support file extension filter, either default-accept or default-deny
>    policy
> * Support choice of compression algorithm, LZO (version 2) or LZ4
>    (default)
> * Support custom log of cluster size
> * Support minimum number of compressed blocks per cluster (default 1).
>    A cluster will not be compressed if the number can not be met.
> * suuport -r (read-only) option

Could you please update manual as well?

> +
> +	/* sldc: sload compression support */

Personally, I don't like the naming method of adding "sldc_" prefix... :(

> +	bool sldc_en;
> +	bool sldc_use_allow_list;  /* default false to use the deny list */
> +	struct compress_ctx sldc_cc;
> +	u8 sldc_ca; /* compress algorithm: 0 = LZO, 1 = LZ4 */
> +	compress_ops *sldc_compr;
> +	enum filter_policy sldc_policy;
> +	/* max_cppc can used to specify minimum compression rate */
> +	unsigned int sldc_min_cbpc; /* min compressed pages per cluster */
> +	bool sldc_got_opt;
> +	bool sldc_immutable;
> +	struct ext_tbl_op *sldc_ef; /* extension filter */

The variables name like sldc_en, sldc_ca, min_cbpc, sldc_ef makes developers
hard to understand w/o comments, and also there is no comments for several
variable like sldc_en, sldc_cc...

Could you please improve the naming like f2fs-tools style?

Thanks,


_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support

[Jaegeuk Kim]

On 12/10, 徐瑞斌 wrote:
> Hi, Jaegeuk,
> 
> I comment here the patch your provided (3 parts, since the patch contains 3
> fixes):
> 1.  +       dn->data_blkaddr = blkaddr;
>         ret = reserve_new_block(sbi, &dn->data_blkaddr, &sum, type, 0);
> 
> We cannot assign dn->data_blkaddr here.  The old one is to be used in
> reserve_new_block() function.  Also, reserve_new_block() function actually
> will set dn->data_blkaddr to blkaddr in the end.

This tries to avoid deleting the block address used in the previous offset.
Otherwise, we'll see wrong i_blocks.

> 
> 2.   Added condition "n < (1 << c.sldc_cc.log_cluster_size) * BLOCK_SZ"
> 
> The semantic meaning of the whole if statement is to say:
>    When the compression fail (ret  != 0) or the original read size is
> smaller than the compressed size plus (the minimum block saved (specified
> by the user) x block size), we will not do compression but just write the
> data as is.

This is missing the last block having < 4Kb.

> 
> The right hand side (RHS) of your added condition is exactly the read size,
> i.e. the cluster size.  That means the condition is always false except the
> read of the last part of the file, when the file size is not exactly the
> multiple of the cluster size.  That means we will never try to compress the
> last part of the file (when the last part is not a multiple of the cluster
> size)
> 
> IMHO, the original implementation should be correct.
> 
> 3.  node_blk->i.i_blocks += cpu_to_le64(cblocks);
> 
> I am not quite sure of the i_blocks count.  Did you mean that when the file
> is mutable,   meaning that the file reserves some blocks for future write,
> we will add count to i_blocks to mark the block as a used block by the
> file, right?  I thought we only need to increment the allocated count...

Should add it.

> 
> Regards,
> Robin Hsu 徐瑞斌
> 
> 
> On Thu, Dec 10, 2020 at 4:42 PM Chao Yu <yuchao0@huawei.com> wrote:
> 
> > On 2020/12/8 16:15, Robin Hsu wrote:
> > > From: Robin Hsu <robinhsu@google.com>
> > >
> > > Add F2FS compression support for sload
> > > * Support file extension filter, either default-accept or default-deny
> > >    policy
> > > * Support choice of compression algorithm, LZO (version 2) or LZ4
> > >    (default)
> > > * Support custom log of cluster size
> > > * Support minimum number of compressed blocks per cluster (default 1).
> > >    A cluster will not be compressed if the number can not be met.
> > > * suuport -r (read-only) option
> >
> > Could you please update manual as well?
> >
> > > +
> > > +     /* sldc: sload compression support */
> >
> > Personally, I don't like the naming method of adding "sldc_" prefix... :(
> >
> > > +     bool sldc_en;
> > > +     bool sldc_use_allow_list;  /* default false to use the deny list */
> > > +     struct compress_ctx sldc_cc;
> > > +     u8 sldc_ca; /* compress algorithm: 0 = LZO, 1 = LZ4 */
> > > +     compress_ops *sldc_compr;
> > > +     enum filter_policy sldc_policy;
> > > +     /* max_cppc can used to specify minimum compression rate */
> > > +     unsigned int sldc_min_cbpc; /* min compressed pages per cluster */
> > > +     bool sldc_got_opt;
> > > +     bool sldc_immutable;
> > > +     struct ext_tbl_op *sldc_ef; /* extension filter */
> >
> > The variables name like sldc_en, sldc_ca, min_cbpc, sldc_ef makes
> > developers
> > hard to understand w/o comments, and also there is no comments for several
> > variable like sldc_en, sldc_cc...
> >
> > Could you please improve the naming like f2fs-tools style?
> >
> > Thanks,
> >


_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support

[Jaegeuk Kim]

Chao and Robin,

I refactored the patch to modify the names and structures.
Please take a look at this.
https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs-tools.git/commit/?h=dev&id=5cd8e5fcc79611c0692f18c7e7e13d6b4742d3c0

On 12/10, Jaegeuk Kim wrote:
> On 12/10, 徐瑞斌 wrote:
> > Hi, Jaegeuk,
> > 
> > I comment here the patch your provided (3 parts, since the patch contains 3
> > fixes):
> > 1.  +       dn->data_blkaddr = blkaddr;
> >         ret = reserve_new_block(sbi, &dn->data_blkaddr, &sum, type, 0);
> > 
> > We cannot assign dn->data_blkaddr here.  The old one is to be used in
> > reserve_new_block() function.  Also, reserve_new_block() function actually
> > will set dn->data_blkaddr to blkaddr in the end.
> 
> This tries to avoid deleting the block address used in the previous offset.
> Otherwise, we'll see wrong i_blocks.
> 
> > 
> > 2.   Added condition "n < (1 << c.sldc_cc.log_cluster_size) * BLOCK_SZ"
> > 
> > The semantic meaning of the whole if statement is to say:
> >    When the compression fail (ret  != 0) or the original read size is
> > smaller than the compressed size plus (the minimum block saved (specified
> > by the user) x block size), we will not do compression but just write the
> > data as is.
> 
> This is missing the last block having < 4Kb.
> 
> > 
> > The right hand side (RHS) of your added condition is exactly the read size,
> > i.e. the cluster size.  That means the condition is always false except the
> > read of the last part of the file, when the file size is not exactly the
> > multiple of the cluster size.  That means we will never try to compress the
> > last part of the file (when the last part is not a multiple of the cluster
> > size)
> > 
> > IMHO, the original implementation should be correct.
> > 
> > 3.  node_blk->i.i_blocks += cpu_to_le64(cblocks);
> > 
> > I am not quite sure of the i_blocks count.  Did you mean that when the file
> > is mutable,   meaning that the file reserves some blocks for future write,
> > we will add count to i_blocks to mark the block as a used block by the
> > file, right?  I thought we only need to increment the allocated count...
> 
> Should add it.
> 
> > 
> > Regards,
> > Robin Hsu 徐瑞斌
> > 
> > 
> > On Thu, Dec 10, 2020 at 4:42 PM Chao Yu <yuchao0@huawei.com> wrote:
> > 
> > > On 2020/12/8 16:15, Robin Hsu wrote:
> > > > From: Robin Hsu <robinhsu@google.com>
> > > >
> > > > Add F2FS compression support for sload
> > > > * Support file extension filter, either default-accept or default-deny
> > > >    policy
> > > > * Support choice of compression algorithm, LZO (version 2) or LZ4
> > > >    (default)
> > > > * Support custom log of cluster size
> > > > * Support minimum number of compressed blocks per cluster (default 1).
> > > >    A cluster will not be compressed if the number can not be met.
> > > > * suuport -r (read-only) option
> > >
> > > Could you please update manual as well?
> > >
> > > > +
> > > > +     /* sldc: sload compression support */
> > >
> > > Personally, I don't like the naming method of adding "sldc_" prefix... :(
> > >
> > > > +     bool sldc_en;
> > > > +     bool sldc_use_allow_list;  /* default false to use the deny list */
> > > > +     struct compress_ctx sldc_cc;
> > > > +     u8 sldc_ca; /* compress algorithm: 0 = LZO, 1 = LZ4 */
> > > > +     compress_ops *sldc_compr;
> > > > +     enum filter_policy sldc_policy;
> > > > +     /* max_cppc can used to specify minimum compression rate */
> > > > +     unsigned int sldc_min_cbpc; /* min compressed pages per cluster */
> > > > +     bool sldc_got_opt;
> > > > +     bool sldc_immutable;
> > > > +     struct ext_tbl_op *sldc_ef; /* extension filter */
> > >
> > > The variables name like sldc_en, sldc_ca, min_cbpc, sldc_ef makes
> > > developers
> > > hard to understand w/o comments, and also there is no comments for several
> > > variable like sldc_en, sldc_cc...
> > >
> > > Could you please improve the naming like f2fs-tools style?
> > >
> > > Thanks,
> > >
> 
> 
> _______________________________________________
> Linux-f2fs-devel mailing list
> Linux-f2fs-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel


_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support

[Satya Tangirala via Linux-f2fs-devel]

On Tue, Dec 08, 2020 at 04:15:54PM +0800, Robin Hsu wrote:
> From: Robin Hsu <robinhsu@google.com>
> 
> Add F2FS compression support for sload
> * Support file extension filter, either default-accept or default-deny
>   policy
> * Support choice of compression algorithm, LZO (version 2) or LZ4
>   (default)
> * Support custom log of cluster size
> * Support minimum number of compressed blocks per cluster (default 1).
>   A cluster will not be compressed if the number can not be met.
> * suuport -r (read-only) option
> 
> Signed-off-by: Robin Hsu <robinhsu@google.com>
> ---
>  fsck/compress_wrapper.c | 102 ++++++++++++++++++++
>  fsck/compress_wrapper.h |  22 +++++
>  fsck/fsck.h             |  15 +++
>  fsck/main.c             | 141 +++++++++++++++++++++++++++-
>  fsck/segment.c          | 202 +++++++++++++++++++++++++++++++++++++---
>  fsck/sload.c            |  67 +++++++++++++
>  include/f2fs_fs.h       |  76 ++++++++++++++-
>  lib/libf2fs_io.c        |  33 +++++++
>  8 files changed, 644 insertions(+), 14 deletions(-)
>  create mode 100644 fsck/compress_wrapper.c
>  create mode 100644 fsck/compress_wrapper.h
> 
> diff --git a/fsck/compress_wrapper.c b/fsck/compress_wrapper.c
> new file mode 100644
> index 0000000..2cdc4fd
> --- /dev/null
> +++ b/fsck/compress_wrapper.c
> @@ -0,0 +1,102 @@
> +/**
> + * compress_wrapper.c
> + *
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *   : initial created, for sload compression support
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +#include "f2fs.h" /* for config.h for general environment (non-Android) */
> +
> +#include "compress_wrapper.h"
> +#ifdef HAVE_LIBLZO2
> +#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */
> +#endif
> +#ifdef HAVE_LIBLZ4
> +#include <lz4.h> /* for LZ4_compress_fast_extState() */
> +#endif
> +
> +/*
> + * macro/constants borrowed from kernel header (GPL-2.0):
> + * include/linux/lzo.h, and include/linux/lz4.h
> + */
> +#ifdef HAVE_LIBLZO2
> +#define lzo1x_worst_compress(x)		((x) + (x) / 16 + 64 + 3 + 2)
> +#define LZO_WORK_SIZE			ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8)
> +#endif
> +#ifdef HAVE_LIBLZ4
> +#define LZ4_MEMORY_USAGE		14
> +#define LZ4_MAX_INPUT_SIZE		0x7E000000 /* 2 113 929 216 bytes */
> +#define LZ4_STREAMSIZE			(LZ4_STREAMSIZE_U64 * sizeof(long long))
> +#define LZ4_MEM_COMPRESS		LZ4_STREAMSIZE
> +#define LZ4_ACCELERATION_DEFAULT	1
> +#define LZ4_WORK_SIZE			ALIGN_UP(LZ4_MEM_COMPRESS, 8)
> +#endif
> +
> +#ifdef HAVE_LIBLZO2
> +static void lzo_compress_init(struct compress_ctx *cc)
> +{
> +	size_t size = cc->cluster_size * F2FS_BLKSIZE;
> +	size_t alloc = size + lzo1x_worst_compress(size)
> +			+ COMPRESS_HEADER_SIZE + LZO_WORK_SIZE;
> +	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
> +	cc->rbuf = (char *) cc->private + LZO_WORK_SIZE;
> +	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
> +}
> +
> +static int lzo_compress(struct compress_ctx *cc)
> +{
> +	int ret = lzo1x_1_15_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
> +			(lzo_uintp)(&cc->clen), cc->private);
> +	cc->cbuf->clen = cpu_to_le32(cc->clen);
> +	return ret;
> +}
> +#endif
> +
> +#ifdef HAVE_LIBLZ4
> +static void lz4_compress_init(struct compress_ctx *cc)
> +{
> +	size_t size = cc->cluster_size * F2FS_BLKSIZE;
> +	size_t alloc = size + LZ4_COMPRESSBOUND(size)
> +			+ COMPRESS_HEADER_SIZE + LZ4_WORK_SIZE;
> +	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
> +	cc->rbuf = (char *) cc->private + LZ4_WORK_SIZE;
> +	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
> +}
> +
> +static int lz4_compress(struct compress_ctx *cc)
> +{
> +	cc->clen = LZ4_compress_fast_extState(cc->private, cc->rbuf,
> +			(char *)cc->cbuf->cdata, cc->rlen,
> +			cc->rlen - F2FS_BLKSIZE * c.sldc_min_cbpc,
> +			LZ4_ACCELERATION_DEFAULT);
> +
> +	if (!cc->clen)
> +		return 1;
> +
> +	cc->cbuf->clen = cpu_to_le32(cc->clen);
> +	return 0;
> +}
> +#endif
> +
> +const char *ca_names[] = {
> +	"LZO",
> +	"LZ4",
> +	"", /* end of the name list */
> +};
> +
> +compress_ops compr_ops[] = {
> +#ifdef HAVE_LIBLZO2
> +	{lzo_compress_init, lzo_compress},
> +#else
> +	{NULL, NULL},
> +#endif
> +#ifdef HAVE_LIBLZ4
> +	{lz4_compress_init, lz4_compress},
> +#else
> +	{NULL, NULL},
> +#endif
> +};
> diff --git a/fsck/compress_wrapper.h b/fsck/compress_wrapper.h
> new file mode 100644
> index 0000000..ec33d43
> --- /dev/null
> +++ b/fsck/compress_wrapper.h
> @@ -0,0 +1,22 @@
> +/**
> + * compress_wrapper.h
> + *
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *   : initial created, for sload compression support
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +
> +#ifndef COMPRESS_WRAPPER_H
> +#define COMPRESS_WRAPPER_H
> +
> +#include "f2fs_fs.h"
> +extern compress_ops compr_ops[]; /* [0]: LZO, [1]: LZ4, */
> +
> +#endif /* COMPRESS_WRAPPER_H */
> diff --git a/fsck/fsck.h b/fsck/fsck.h
> index c5e85fe..4e866ec 100644
> --- a/fsck/fsck.h
> +++ b/fsck/fsck.h
> @@ -3,6 +3,9 @@
>   *
>   * Copyright (c) 2013 Samsung Electronics Co., Ltd.
>   *             http://www.samsung.com/
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * This program is free software; you can redistribute it and/or modify
>   * it under the terms of the GNU General Public License version 2 as
> @@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);
>  
>  /* sload.c */
>  int f2fs_sload(struct f2fs_sb_info *);
> +void sldc_erase_bufs(struct compress_ctx *cc);
> +void sload_countblk(void);
> +extern struct ext_tbl_op ext_filter;
>  
>  /* segment.c */
>  int reserve_new_block(struct f2fs_sb_info *, block_t *,
> @@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,
>  struct quota_file;
>  u64 f2fs_quota_size(struct quota_file *);
>  u64 f2fs_read(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> +enum wr_addr_type {
> +	WR_NORMAL = 1,
> +	WR_COMPRESS_DATA = 2,
> +	WR_NULL_ADDR = NULL_ADDR,  /* 0 */
> +	WR_NEW_ADDR = NEW_ADDR, /* -1U */
> +	WR_COMPRESS_ADDR = COMPRESS_ADDR, /* -2U */
> +};
>  u64 f2fs_write(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> +u64 f2fs_write_compress_data(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> +u64 f2fs_write_addrtag(struct f2fs_sb_info *, nid_t, pgoff_t, unsigned int);
>  void f2fs_filesize_update(struct f2fs_sb_info *, nid_t, u64);
>  
>  int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,
> diff --git a/fsck/main.c b/fsck/main.c
> index b20498f..001eed0 100644
> --- a/fsck/main.c
> +++ b/fsck/main.c
> @@ -13,6 +13,9 @@
>   * Copyright (c) 2019 Google Inc.
>   *   Robin Hsu <robinhsu@google.com>
>   *  : add cache layer
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * This program is free software; you can redistribute it and/or modify
>   * it under the terms of the GNU General Public License version 2 as
> @@ -25,6 +28,7 @@
>  #include <getopt.h>
>  #include <stdbool.h>
>  #include "quotaio.h"
> +#include "compress_wrapper.h"
>  
>  struct f2fs_fsck gfsck;
>  
> @@ -134,6 +138,17 @@ void sload_usage()
>  	MSG(0, "  -S sparse_mode\n");
>  	MSG(0, "  -t mount point [prefix of target fs path, default:/]\n");
>  	MSG(0, "  -T timestamp\n");
> +	MSG(0, "  -c enable compression (default allow policy)\n");
> +	MSG(0, "    ------------ Compression sub-options -----------------\n");
> +	MSG(0, "    -L <log-of-blocks-per-cluster>, default 2\n");
> +	MSG(0, "    -a <algorithm> compression algorithm, default LZ4\n");
> +	MSG(0, "    -x <ext> compress files except for these extensions.\n");
> +	MSG(0, "    -i <ext> compress files with these extensions only.\n");
> +	MSG(0, "    * -i or -x: use it many times for multiple extensions.\n");
> +	MSG(0, "    * -i and -x cannot be used together..\n");
> +	MSG(0, "    -m <num> min compressed blocks per cluster\n");
> +	MSG(0, "    -r readonly (IMMUTABLE) for compressed files\n");
> +	MSG(0, "    ------------------------------------------------------\n");
>  	MSG(0, "  -d debug level [default:0]\n");
>  	MSG(0, "  -V print the version number and exit\n");
>  	exit(1);
> @@ -534,7 +549,7 @@ void f2fs_parse_options(int argc, char *argv[])
>  #endif
>  	} else if (!strcmp("sload.f2fs", prog)) {
>  #ifdef WITH_SLOAD
> -		const char *option_string = "C:d:f:p:s:St:T:V";
> +		const char *option_string = "cL:a:i:x:m:rC:d:f:p:s:St:T:V";
>  #ifdef HAVE_LIBSELINUX
>  		int max_nr_opt = (int)sizeof(c.seopt_file) /
>  			sizeof(c.seopt_file[0]);
> @@ -543,8 +558,82 @@ void f2fs_parse_options(int argc, char *argv[])
>  		char *p;
>  
>  		c.func = SLOAD;
> +		c.sldc_cc.log_cluster_size = 2;
> +		c.sldc_ca = CA_LZ4;
> +		c.sldc_min_cbpc = 1;
> +		c.sldc_ef = &ext_filter;
>  		while ((option = getopt(argc, argv, option_string)) != EOF) {
> +			unsigned int i;
> +			int val;
> +
>  			switch (option) {
> +			case 'c': /* compression support */
> +				c.sldc_en = true;
> +				break;
> +			case 'L': /* compression: log of blocks-per-cluster */
> +				c.sldc_got_opt = true;
> +				val = atoi(optarg);
> +				if (val < MIN_COMPRESS_LOG_SIZE ||
> +						val > MAX_COMPRESS_LOG_SIZE) {
> +					MSG(0, "\tError: log of blocks per"
> +						" cluster must be in the range"
> +						" of %d .. %d.\n",
> +						MIN_COMPRESS_LOG_SIZE,
> +						MAX_COMPRESS_LOG_SIZE);
> +					error_out(prog);
> +				}
> +				c.sldc_cc.log_cluster_size = val;
> +				break;
> +			case 'a': /* compression: choose algorithm */
> +				c.sldc_got_opt = true;
> +				c.sldc_ca = (u8)-1;
> +				for (i = 0; ca_names[i][0] != 0; i++) {
> +					if (!strcmp(ca_names[i], optarg)) {
> +						c.sldc_ca = i;
> +						break;
> +					}
> +				}
> +				if (c.sldc_ca == (u8)-1) {
> +					MSG(0, "\tError: Unknown compression"
> +						" algorithm %s\n", optarg);
> +					error_out(prog);
> +				}
> +				break;
> +			case 'i': /* compress only these extensions */
> +				c.sldc_got_opt = true;
> +				if (c.sldc_policy == FP_ALLOW) {
> +					MSG(0, "\tError: could not mix option"
> +							" -i and -x\n");
> +					error_out(prog);
> +				}
> +				c.sldc_policy = FP_DENY;
> +				c.sldc_ef->add(optarg);
> +				break;
> +			case 'x': /* compress except for these extensions */
> +				c.sldc_got_opt = true;
> +				if (c.sldc_policy == FP_DENY) {
> +					MSG(0, "\tError: could not mix option"
> +							" -i and -x\n");
> +					error_out(prog);
> +				}
> +				c.sldc_policy = FP_ALLOW;
> +				c.sldc_ef->add(optarg);
> +				break;
> +			case 'm': /* minimum compressed blocks per cluster */
> +				c.sldc_got_opt = true;
> +				val = atoi(optarg);
> +				if (val <= 0) {
> +					MSG(0, "\tError: minimum compressed"
> +						" blocks per cluster must be"
> +						" positive.\n");
> +					error_out(prog);
> +				}
> +				c.sldc_min_cbpc = val;
> +				break;
> +			case 'r': /* compress file to set IMMUTABLE */
> +				c.sldc_got_opt = true;
> +				c.sldc_immutable = true;
> +				break;
>  			case 'C':
>  				c.fs_config_file = absolute_path(optarg);
>  				break;
> @@ -602,6 +691,27 @@ void f2fs_parse_options(int argc, char *argv[])
>  			if (err != NOERROR)
>  				break;
>  		}
> +		if (c.sldc_got_opt && !c.sldc_en) {
> +			MSG(0, "\tError: compression sub-options are used"
> +				" without the compression enable (-c) option\n"
> +			);
> +			error_out(prog);
> +		}
> +		if (err == NOERROR && c.sldc_en) {
> +			c.sldc_cc.cluster_size = 1
> +				<< c.sldc_cc.log_cluster_size;
> +			if (c.sldc_policy == FP_UNASSIGNED)
> +				c.sldc_policy = FP_ALLOW;
> +			if (c.sldc_min_cbpc >= c.sldc_cc.cluster_size) {
> +				MSG(0, "\tError: minimum reduced blocks by"
> +					" compression per cluster must be at"
> +					" most one less than blocks per"
> +					" cluster, i.e. %d\n",
> +					c.sldc_cc.cluster_size - 1);
> +				error_out(prog);
> +			}
> +			qbuf_init();
> +		}
>  #endif /* WITH_SLOAD */
>  	}
>  
> @@ -812,6 +922,27 @@ static int do_resize(struct f2fs_sb_info *sbi)
>  #endif
>  
>  #ifdef WITH_SLOAD
> +int init_compr(struct f2fs_sb_info *sbi)
> +{
> +	if (!(sbi->raw_super->feature
> +			& cpu_to_le32(F2FS_FEATURE_COMPRESSION))) {
> +		MSG(0, "Error: Compression (-c) was requested "
> +			"but the file system is not created "
> +			"with such feature.\n");
> +		return -1;
> +	}
> +	if (compr_ops[c.sldc_ca].init == NULL) {
> +		MSG(0, "Error: The selected compression algorithm is not"
> +				" supported\n");
> +		return -1;
> +	}
> +	c.sldc_compr = compr_ops + c.sldc_ca;
> +	c.sldc_compr->init(&c.sldc_cc);
> +	sldc_erase_bufs(&c.sldc_cc);
> +	c.sldc_cc.rlen = c.sldc_cc.cluster_size * F2FS_BLKSIZE;
> +	return 0;
> +}
> +
>  static int do_sload(struct f2fs_sb_info *sbi)
>  {
>  	if (!c.from_dir) {
> @@ -821,6 +952,11 @@ static int do_sload(struct f2fs_sb_info *sbi)
>  	if (!c.mount_point)
>  		c.mount_point = "/";
>  
> +	if (c.sldc_en) {
> +		if (init_compr(sbi))
> +			return -1;
> +	}
> +
>  	return f2fs_sload(sbi);
>  }
>  #endif
> @@ -971,6 +1107,9 @@ retry:
>  		return ret2;
>  	}
>  
> +	if (c.func == SLOAD)
> +		c.sldc_ef->destroy();
> +
>  	printf("\nDone: %lf secs\n", (get_boottime_ns() - start) / 1000000000.0);
>  	return ret;
>  
> diff --git a/fsck/segment.c b/fsck/segment.c
> index 0487f41..e4c8cea 100644
> --- a/fsck/segment.c
> +++ b/fsck/segment.c
> @@ -8,6 +8,9 @@
>   *   Hou Pengyang <houpengyang@huawei.com>
>   *   Liu Shuoran <liushuoran@huawei.com>
>   *   Jaegeuk Kim <jaegeuk@kernel.org>
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * This program is free software; you can redistribute it and/or modify
>   * it under the terms of the GNU General Public License version 2 as
> @@ -16,6 +19,7 @@
>  #include "fsck.h"
>  #include "node.h"
>  #include "quotaio.h"
> +#include "compress_wrapper.h"
>  
>  int reserve_new_block(struct f2fs_sb_info *sbi, block_t *to,
>  			struct f2fs_summary *sum, int type, bool is_inode)
> @@ -228,8 +232,14 @@ u64 f2fs_read(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  	return read_count;
>  }
>  
> -u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> -					u64 count, pgoff_t offset)
> +/*
> + * Do not call this function directly.  Instead, call one of the following:
> + *     u64 f2fs_write();
> + *     u64 f2fs_write_compress_data();
> + *     u64 f2fs_write_addrtag();
> + */
> +static u64 f2fs_write_ex(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> +		u64 count, pgoff_t offset, enum wr_addr_type addr_type)
>  {
>  	struct dnode_of_data dn;
>  	struct node_info ni;
> @@ -243,6 +253,19 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  	void* index_node = NULL;
>  	int idirty = 0;
>  	int err;
> +	bool has_data = (addr_type == WR_NORMAL
> +			|| addr_type == WR_COMPRESS_DATA);
> +
> +	if (count == 0)
> +		return 0;
> +
> +	/*
> +	 * Enforce calling from f2fs_write(), f2fs_write_compress_data(),
> +	 * and f2fs_write_addrtag().   Beside, check if is properly called.
> +	 */
> +	ASSERT((!has_data && buffer == NULL) || (has_data && buffer != NULL));
> +	if (addr_type != WR_NORMAL)
> +		ASSERT(offset % F2FS_BLKSIZE == 0); /* block boundary only */
>  
>  	/* Memory allocation for block buffer and inode. */
>  	blk_buffer = calloc(BLOCK_SZ, 2);
> @@ -265,15 +288,26 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  			if (err)
>  				break;
>  			idirty |= dn.idirty;
> -			if (index_node)
> -				free(index_node);
> +			free(index_node);
>  			index_node = (dn.node_blk == dn.inode_blk) ?
> -							NULL : dn.node_blk;
> +					NULL : dn.node_blk;
>  			remained_blkentries = ADDRS_PER_PAGE(sbi,
> -						dn.node_blk, dn.inode_blk);
> +					dn.node_blk, dn.inode_blk) -
> +					dn.ofs_in_node;
>  		}
>  		ASSERT(remained_blkentries > 0);
>  
> +		if (!has_data) {
> +			dn.data_blkaddr = addr_type;
> +			set_data_blkaddr(&dn);
> +			idirty |= dn.idirty;
> +			if (dn.ndirty)
> +				ASSERT(dev_write_block(dn.node_blk,
> +						dn.node_blkaddr) >= 0);
> +			written_count = 0;
> +			break;
> +		}
> +
>  		blkaddr = datablock_addr(dn.node_blk, dn.ofs_in_node);
>  		if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) {
>  			err = new_data_block(sbi, blk_buffer,
> @@ -281,6 +315,7 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  			if (err)
>  				break;
>  			blkaddr = dn.data_blkaddr;
> +			idirty |= dn.idirty;
>  		}
>  
>  		off_in_blk = offset % BLOCK_SZ;
> @@ -305,9 +340,10 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  
>  		dn.ofs_in_node++;
>  		if ((--remained_blkentries == 0 || count == 0) && (dn.ndirty))
> -			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr) >= 0);
> +			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr)
> +					>= 0);
>  	}
> -	if (offset > le64_to_cpu(inode->i.i_size)) {
> +	if (addr_type == WR_NORMAL && offset > le64_to_cpu(inode->i.i_size)) {
>  		inode->i.i_size = cpu_to_le64(offset);
>  		idirty = 1;
>  	}
> @@ -315,13 +351,33 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
>  		ASSERT(inode == dn.inode_blk);
>  		ASSERT(write_inode(inode, ni.blk_addr) >= 0);
>  	}
> -	if (index_node)
> -		free(index_node);
> +
> +	free(index_node);
>  	free(blk_buffer);
>  
>  	return written_count;
>  }
>  
> +u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> +					u64 count, pgoff_t offset)
> +{
> +	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_NORMAL);
> +}
> +
> +u64 f2fs_write_compress_data(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> +					u64 count, pgoff_t offset)
> +{
> +	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_COMPRESS_DATA);
> +}
> +
> +u64 f2fs_write_addrtag(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
> +		unsigned int addrtag)
> +{
> +	ASSERT(addrtag == COMPRESS_ADDR || addrtag == NEW_ADDR
> +			|| addrtag == NULL_ADDR);
> +	return f2fs_write_ex(sbi, ino, NULL, F2FS_BLKSIZE, offset, addrtag);
> +}
> +
>  /* This function updates only inode->i.i_size */
>  void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
>  {
> @@ -342,11 +398,59 @@ void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
>  	free(inode);
>  }
>  
> +#define MAX_BULKR_RETRY 5
> +int bulkread(int fd, void *rbuf, size_t rsize, bool *eof)
> +{
> +	int n = 0;
> +	int retry = MAX_BULKR_RETRY;
> +	int cur;
> +
> +	if (!rsize)
> +		return 0;
> +
> +	if (eof != NULL)
> +		*eof = false;
> +	while (rsize && (cur = read(fd, rbuf, rsize)) != 0) {
> +		if (cur == -1) {
> +			if (errno == EINTR && retry--)
> +				continue;
> +			return -1;
> +		}
> +		retry = MAX_BULKR_RETRY;
> +
> +		rsize -= cur;
> +		n += cur;
> +	}
> +	if (eof != NULL)
> +		*eof = (cur == 0);
> +	return n;
> +}
> +
> +u64 f2fs_fix_mutable(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
> +		unsigned int compressed)
> +{
> +	unsigned int i;
> +	u64 wlen;
> +
> +	if (c.sldc_immutable)
> +		return 0;
> +
> +	for (i = 0; i < compressed - 1; i++) {
> +		wlen = f2fs_write_addrtag(sbi, ino,
> +				offset + (i << F2FS_BLKSIZE_BITS), NEW_ADDR);
> +		if (wlen)
> +			return wlen;
> +	}
> +	return 0;
> +}
> +
>  int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
>  {
>  	int fd, n;
>  	pgoff_t off = 0;
>  	u8 buffer[BLOCK_SZ];
> +	struct node_info ni;
> +	struct f2fs_node *node_blk;
>  
>  	if (de->ino == 0)
>  		return -1;
> @@ -359,8 +463,6 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
>  
>  	/* inline_data support */
>  	if (de->size <= DEF_MAX_INLINE_DATA) {
> -		struct node_info ni;
> -		struct f2fs_node *node_blk;
>  		int ret;
>  
>  		get_node_info(sbi, de->ino, &ni);
> @@ -385,6 +487,82 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
>  		node_blk->i.i_size = cpu_to_le64(de->size);
>  		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
>  		free(node_blk);
> +#ifdef WITH_SLOAD
> +	} else if (c.func == SLOAD && c.sldc_en &&
> +			c.sldc_ef->filter(de->full_path)) {
> +		bool eof = false;
> +		u8 *rbuf = c.sldc_cc.rbuf;
> +		unsigned int cblocks = 0;
> +
> +		node_blk = calloc(BLOCK_SZ, 1);
> +		ASSERT(node_blk);
> +
> +		/* read inode */
> +		get_node_info(sbi, de->ino, &ni);
> +		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
> +		/* update inode meta */
> +		node_blk->i.i_compress_algrithm = c.sldc_ca;
> +		node_blk->i.i_log_cluster_size =
> +				c.sldc_cc.log_cluster_size;
> +		node_blk->i.i_flags = cpu_to_le32(
> +				F2FS_COMPR_FL |
> +				(c.sldc_immutable ? FS_IMMUTABLE_FL : 0));
> +		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
> +
> +		while (!eof && (n = bulkread(fd, rbuf, c.sldc_cc.rlen,
> +				&eof)) > 0) {
> +			int ret = c.sldc_compr->compress(&c.sldc_cc);
> +			u64 wlen;
> +			u32 csize = ALIGN_UP(c.sldc_cc.clen +
> +					COMPRESS_HEADER_SIZE, BLOCK_SZ);
> +			unsigned int cur_cblk;
> +
> +			if (ret || n < (int)(csize + BLOCK_SZ *
> +						c.sldc_min_cbpc)) {
> +				wlen = f2fs_write(sbi, de->ino, rbuf, n, off);
> +				ASSERT((int)wlen == n);
> +			} else {
> +				wlen = f2fs_write_addrtag(sbi, de->ino, off,
> +						WR_COMPRESS_ADDR);
> +				ASSERT(!wlen);
> +				wlen = f2fs_write_compress_data(sbi, de->ino,
> +						(u8 *)c.sldc_cc.cbuf,
> +						csize, off + BLOCK_SZ);
> +				ASSERT(wlen == csize);
> +				sldc_erase_bufs(&c.sldc_cc);
> +				cur_cblk = (c.sldc_cc.rlen - csize) / BLOCK_SZ;
> +				cblocks += cur_cblk;
> +				wlen = f2fs_fix_mutable(sbi, de->ino,
> +						off + BLOCK_SZ + csize,
> +						cur_cblk);
> +				ASSERT(!wlen);
> +			}
> +			off += n;
> +		}
> +		if (n == -1) {
> +			fprintf(stderr, "Load file '%s' failed: ",
> +					de->full_path);
> +			perror(NULL);
> +		}
> +		/* read inode */
> +		get_node_info(sbi, de->ino, &ni);
> +		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
> +		/* update inode meta */
> +		node_blk->i.i_size = cpu_to_le64(off);
> +		if (!c.sldc_immutable)
> +			node_blk->i.i_compr_blocks = cpu_to_le64(cblocks);
> +		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
> +		free(node_blk);
> +
> +		if (!c.sldc_immutable) {
> +			sbi->total_valid_block_count += cblocks;
> +			if (sbi->total_valid_block_count >=
> +					sbi->user_block_count) {
> +				ERR_MSG("Not enough space\n");
> +				ASSERT(0);
> +			}
> +		}
> +#endif
>  	} else {
>  		while ((n = read(fd, buffer, BLOCK_SZ)) > 0) {
>  			f2fs_write(sbi, de->ino, buffer, n, off);
> diff --git a/fsck/sload.c b/fsck/sload.c
> index 14012fb..13e523a 100644
> --- a/fsck/sload.c
> +++ b/fsck/sload.c
> @@ -6,6 +6,9 @@
>   *   Hou Pengyang <houpengyang@huawei.com>
>   *   Liu Shuoran <liushuoran@huawei.com>
>   *   Jaegeuk Kim <jaegeuk@kernel.org>
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * This program is free software; you can redistribute it and/or modify
>   * it under the terms of the GNU General Public License version 2 as
> @@ -317,6 +320,70 @@ skip:
>  	return 0;
>  }
>  
> +typedef struct _ext_tbl {
> +	const char *ext;
> +	struct _ext_tbl *next; /* linked list */
> +} ext_tbl_t;
> +static ext_tbl_t *ext_tbl;
> +
> +static bool ext_found(const char *ext)
> +{
> +	ext_tbl_t *p = ext_tbl;
> +
> +	while (p != NULL && strcmp(ext, p->ext))
> +		p = p->next;
> +	return (p != NULL);
> +}
> +
> +static const char *get_ext(const char *path)
> +{
> +	char *p = strrchr(path, '.');
> +	return p == NULL ? path + strlen(path) : p + 1;
> +}
> +
> +static bool ext_do_filter(const char *path)
> +{
> +	return (ext_found(get_ext(path)) == true) ^ (c.sldc_policy == FP_ALLOW);
> +}
> +
> +static void ext_filter_add(const char *ext)
> +{
> +	ext_tbl_t *node;
> +
> +	ASSERT(ext != NULL);
> +	if (ext_found(ext))
> +		return; /* ext was already registered */
> +	node = malloc(sizeof(ext_tbl_t));
> +	ASSERT(node != NULL);
> +	node->ext = ext;
> +	node->next = ext_tbl;
> +	ext_tbl = node;
> +}
> +
> +static void ext_filter_destroy(void)
> +{
> +	ext_tbl_t *p;
> +
> +	while (ext_tbl != NULL) {
> +		p = ext_tbl;
> +		ext_tbl = p->next;
> +		free(p);
> +	}
> +}
> +
> +struct ext_tbl_op ext_filter = {
> +	.add = ext_filter_add,
> +	.destroy = ext_filter_destroy,
> +	.filter = ext_do_filter,
> +};
> +
> +void sldc_erase_bufs(struct compress_ctx *cc)
> +{
> +	memset(cc->rbuf, 0, cc->cluster_size * F2FS_BLKSIZE);
> +	memset(cc->cbuf->cdata, 0, cc->cluster_size * F2FS_BLKSIZE
> +			- F2FS_BLKSIZE);
> +}
> +
>  int f2fs_sload(struct f2fs_sb_info *sbi)
>  {
>  	int ret = 0;
> diff --git a/include/f2fs_fs.h b/include/f2fs_fs.h
> index 1348e39..2a2dc15 100644
> --- a/include/f2fs_fs.h
> +++ b/include/f2fs_fs.h
> @@ -5,6 +5,9 @@
>   *             http://www.samsung.com/
>   * Copyright (c) 2019 Google Inc.
>   *             http://www.google.com/
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add sload compression support
>   *
>   * Dual licensed under the GPL or LGPL version 2 licenses.
>   *
> @@ -68,6 +71,10 @@ typedef uint16_t u_int16_t;
>  typedef uint8_t u_int8_t;
>  #endif
>  
> +/* codes from kernel's f2fs.h, GPL-v2.0 */
> +#define MIN_COMPRESS_LOG_SIZE	2
> +#define MAX_COMPRESS_LOG_SIZE	8
> +
>  typedef u_int64_t	u64;
>  typedef u_int32_t	u32;
>  typedef u_int16_t	u16;
> @@ -93,6 +100,31 @@ typedef u32	__be32;
>  typedef u64	__be64;
>  #endif
>  
> +/*
> + * code borrowed from kernel f2fs dirver: f2fs.h, GPL-2.0
> + *  : definitions of COMPRESS_DATA_RESERVED_SIZE,
> + *    struct compress_data, COMPRESS_HEADER_SIZE,
> + *    and struct compress_ctx
> + */
> +#define COMPRESS_DATA_RESERVED_SIZE		4
> +struct compress_data {
> +	__le32 clen;			/* compressed data size */
> +	__le32 chksum;			/* checksum of compressed data */
> +	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
> +	u8 cdata[];			/* compressed data */
> +};
> +#define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
> +/* compress context */
> +struct compress_ctx {
> +	unsigned int cluster_size;	/* page count in cluster */
> +	unsigned int log_cluster_size;	/* log of cluster size */
> +	void *rbuf;			/* compression input buffer */
> +	struct compress_data *cbuf;	/* comprsssion output header + data */
> +	size_t rlen;			/* valid data length in rbuf */
> +	size_t clen;			/* valid data length in cbuf */
> +	void *private;			/* work buf for compress algorithm */
> +};
> +
>  #if HAVE_BYTESWAP_H
>  #include <byteswap.h>
>  #else
> @@ -345,6 +377,25 @@ typedef struct {
>  	bool dbg_en;
>  } dev_cache_config_t;
>  
> +/* f2fs_configration: sldc_ca, the sload compress algorithm */
> +enum {CA_LZO, CA_LZ4};
> +extern const char *ca_names[];
> +
> +typedef struct  {
> +	void (*init)(struct compress_ctx *cc);
> +	int (*compress)(struct compress_ctx *cc);
> +} compress_ops;
> +
> +#define ALIGN_UP(value, size) ((value) + ((value) % (size) > 0 ? \
> +		(size) - (value) % (size) : 0))
> +
> +enum filter_policy {FP_UNASSIGNED = 0, FP_ALLOW, FP_DENY};
> +struct ext_tbl_op {
> +	void (*add)(const char *);
> +	void (*destroy)(void);
> +	bool (*filter)(const char *);
> +};
> +
>  struct f2fs_configuration {
>  	u_int32_t reserved_segments;
>  	u_int32_t new_reserved_segments;
> @@ -441,6 +492,24 @@ struct f2fs_configuration {
>  
>  	/* cache parameters */
>  	dev_cache_config_t cache_config;
> +
> +	/* quick dynamic buffer */
> +	bool qbuf_initialized;
> +	size_t qbufsize;
> +	void *qbuf;
I don't think these qbuf* variables inside the f2fs_configuration are
used anywhere - shouldn't they be removed?
> +
> +	/* sldc: sload compression support */
> +	bool sldc_en;
> +	bool sldc_use_allow_list;  /* default false to use the deny list */
> +	struct compress_ctx sldc_cc;
> +	u8 sldc_ca; /* compress algorithm: 0 = LZO, 1 = LZ4 */
> +	compress_ops *sldc_compr;
> +	enum filter_policy sldc_policy;
> +	/* max_cppc can used to specify minimum compression rate */
> +	unsigned int sldc_min_cbpc; /* min compressed pages per cluster */
> +	bool sldc_got_opt;
> +	bool sldc_immutable;
> +	struct ext_tbl_op *sldc_ef; /* extension filter */
>  };
>  
>  #ifdef CONFIG_64BIT
> @@ -1226,6 +1295,11 @@ extern void f2fs_release_sparse_resource(void);
>  extern int f2fs_finalize_device(void);
>  extern int f2fs_fsync_device(void);
>  
> +/* quick (shared) buffer */
> +extern void qbuf_free(void);
> +extern void *qbuf_alloc(size_t size);
> +extern void qbuf_init(void);
> +
>  extern void dcache_init(void);
>  extern void dcache_release(void);
>  
> @@ -1377,7 +1451,7 @@ int f2fs_reset_zone(int, void *);
>  extern int f2fs_reset_zones(int);
>  extern uint32_t f2fs_get_usable_segments(struct f2fs_super_block *sb);
>  
> -#define SIZE_ALIGN(val, size)	((val) + (size) - 1) / (size)
> +#define SIZE_ALIGN(val, size)	(((val) + (size) - 1) / (size))
>  #define SEG_ALIGN(blks)		SIZE_ALIGN(blks, c.blks_per_seg)
>  #define ZONE_ALIGN(blks)	SIZE_ALIGN(blks, c.blks_per_seg * \
>  					c.segs_per_zone)
> diff --git a/lib/libf2fs_io.c b/lib/libf2fs_io.c
> index 138285d..0280896 100644
> --- a/lib/libf2fs_io.c
> +++ b/lib/libf2fs_io.c
> @@ -5,6 +5,9 @@
>   *             http://www.samsung.com/
>   * Copyright (c) 2019 Google Inc.
>   *             http://www.google.com/
> + * Copyright (c) 2020 Google Inc.
> + *   Robin Hsu <robinhsu@google.com>
> + *  : add quick-buffer for sload compression support
>   *
>   * Dual licensed under the GPL or LGPL version 2 licenses.
>   */
> @@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = {
>  };
>  static int dcache_relocate_offset[16];
>  
> +/* quick (shared) buffer */
> +static bool qbuf_initialized;
> +static void *qbuf;
> +static size_t qbufsize;
> +void qbuf_free(void)
> +{
> +	ASSERT(qbuf_initialized);
> +	if (qbuf != NULL) {
> +		free(qbuf);
> +		qbuf = NULL;
> +		qbufsize = 0;
> +	}
> +}
> +void *qbuf_alloc(size_t size)
> +{
> +	ASSERT(qbuf_initialized);
> +	if (size > qbufsize) {
> +		qbuf_free();
> +		qbuf = malloc(size);
qbufsize needs to be updated to size here.
> +	}
> +	return qbuf;
> +}
> +void qbuf_init(void)
> +{
> +	if (qbuf_initialized)
> +		return;
> +	atexit(qbuf_free);
> +	qbuf_initialized = true;
> +}
Is there really a need for all this qbuf code? As far as I can tell,
qbuf_alloc() is only ever called once during any invocation of
sload, and it'd be better/simpler to replace that with a malloc()...
> +
>  static void dcache_print_statistics(void)
>  {
>  	long i;
> -- 
> 2.29.2.576.ga3fc446d84-goog
> 
> 
> 
> _______________________________________________
> Linux-f2fs-devel mailing list
> Linux-f2fs-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel


_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH v3 2/3] f2fs-tools:sload.f2fs compression support

[Jaegeuk Kim]

On 12/18, Satya Tangirala wrote:
> On Tue, Dec 08, 2020 at 04:15:54PM +0800, Robin Hsu wrote:
> > From: Robin Hsu <robinhsu@google.com>
> > 
> > Add F2FS compression support for sload
> > * Support file extension filter, either default-accept or default-deny
> >   policy
> > * Support choice of compression algorithm, LZO (version 2) or LZ4
> >   (default)
> > * Support custom log of cluster size
> > * Support minimum number of compressed blocks per cluster (default 1).
> >   A cluster will not be compressed if the number can not be met.
> > * suuport -r (read-only) option
> > 
> > Signed-off-by: Robin Hsu <robinhsu@google.com>
> > ---
> >  fsck/compress_wrapper.c | 102 ++++++++++++++++++++
> >  fsck/compress_wrapper.h |  22 +++++
> >  fsck/fsck.h             |  15 +++
> >  fsck/main.c             | 141 +++++++++++++++++++++++++++-
> >  fsck/segment.c          | 202 +++++++++++++++++++++++++++++++++++++---
> >  fsck/sload.c            |  67 +++++++++++++
> >  include/f2fs_fs.h       |  76 ++++++++++++++-
> >  lib/libf2fs_io.c        |  33 +++++++
> >  8 files changed, 644 insertions(+), 14 deletions(-)
> >  create mode 100644 fsck/compress_wrapper.c
> >  create mode 100644 fsck/compress_wrapper.h
> > 
> > diff --git a/fsck/compress_wrapper.c b/fsck/compress_wrapper.c
> > new file mode 100644
> > index 0000000..2cdc4fd
> > --- /dev/null
> > +++ b/fsck/compress_wrapper.c
> > @@ -0,0 +1,102 @@
> > +/**
> > + * compress_wrapper.c
> > + *
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *   : initial created, for sload compression support
> > + *
> > + * This program is free software; you can redistribute it and/or modify
> > + * it under the terms of the GNU General Public License version 2 as
> > + * published by the Free Software Foundation.
> > + */
> > +#include "f2fs.h" /* for config.h for general environment (non-Android) */
> > +
> > +#include "compress_wrapper.h"
> > +#ifdef HAVE_LIBLZO2
> > +#include <lzo/lzo1x.h> /* for lzo1x_1_15_compress() */
> > +#endif
> > +#ifdef HAVE_LIBLZ4
> > +#include <lz4.h> /* for LZ4_compress_fast_extState() */
> > +#endif
> > +
> > +/*
> > + * macro/constants borrowed from kernel header (GPL-2.0):
> > + * include/linux/lzo.h, and include/linux/lz4.h
> > + */
> > +#ifdef HAVE_LIBLZO2
> > +#define lzo1x_worst_compress(x)		((x) + (x) / 16 + 64 + 3 + 2)
> > +#define LZO_WORK_SIZE			ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8)
> > +#endif
> > +#ifdef HAVE_LIBLZ4
> > +#define LZ4_MEMORY_USAGE		14
> > +#define LZ4_MAX_INPUT_SIZE		0x7E000000 /* 2 113 929 216 bytes */
> > +#define LZ4_STREAMSIZE			(LZ4_STREAMSIZE_U64 * sizeof(long long))
> > +#define LZ4_MEM_COMPRESS		LZ4_STREAMSIZE
> > +#define LZ4_ACCELERATION_DEFAULT	1
> > +#define LZ4_WORK_SIZE			ALIGN_UP(LZ4_MEM_COMPRESS, 8)
> > +#endif
> > +
> > +#ifdef HAVE_LIBLZO2
> > +static void lzo_compress_init(struct compress_ctx *cc)
> > +{
> > +	size_t size = cc->cluster_size * F2FS_BLKSIZE;
> > +	size_t alloc = size + lzo1x_worst_compress(size)
> > +			+ COMPRESS_HEADER_SIZE + LZO_WORK_SIZE;
> > +	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
> > +	cc->rbuf = (char *) cc->private + LZO_WORK_SIZE;
> > +	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
> > +}
> > +
> > +static int lzo_compress(struct compress_ctx *cc)
> > +{
> > +	int ret = lzo1x_1_15_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
> > +			(lzo_uintp)(&cc->clen), cc->private);
> > +	cc->cbuf->clen = cpu_to_le32(cc->clen);
> > +	return ret;
> > +}
> > +#endif
> > +
> > +#ifdef HAVE_LIBLZ4
> > +static void lz4_compress_init(struct compress_ctx *cc)
> > +{
> > +	size_t size = cc->cluster_size * F2FS_BLKSIZE;
> > +	size_t alloc = size + LZ4_COMPRESSBOUND(size)
> > +			+ COMPRESS_HEADER_SIZE + LZ4_WORK_SIZE;
> > +	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
> > +	cc->rbuf = (char *) cc->private + LZ4_WORK_SIZE;
> > +	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
> > +}
> > +
> > +static int lz4_compress(struct compress_ctx *cc)
> > +{
> > +	cc->clen = LZ4_compress_fast_extState(cc->private, cc->rbuf,
> > +			(char *)cc->cbuf->cdata, cc->rlen,
> > +			cc->rlen - F2FS_BLKSIZE * c.sldc_min_cbpc,
> > +			LZ4_ACCELERATION_DEFAULT);
> > +
> > +	if (!cc->clen)
> > +		return 1;
> > +
> > +	cc->cbuf->clen = cpu_to_le32(cc->clen);
> > +	return 0;
> > +}
> > +#endif
> > +
> > +const char *ca_names[] = {
> > +	"LZO",
> > +	"LZ4",
> > +	"", /* end of the name list */
> > +};
> > +
> > +compress_ops compr_ops[] = {
> > +#ifdef HAVE_LIBLZO2
> > +	{lzo_compress_init, lzo_compress},
> > +#else
> > +	{NULL, NULL},
> > +#endif
> > +#ifdef HAVE_LIBLZ4
> > +	{lz4_compress_init, lz4_compress},
> > +#else
> > +	{NULL, NULL},
> > +#endif
> > +};
> > diff --git a/fsck/compress_wrapper.h b/fsck/compress_wrapper.h
> > new file mode 100644
> > index 0000000..ec33d43
> > --- /dev/null
> > +++ b/fsck/compress_wrapper.h
> > @@ -0,0 +1,22 @@
> > +/**
> > + * compress_wrapper.h
> > + *
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *   : initial created, for sload compression support
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *  : add sload compression support
> > + *
> > + * This program is free software; you can redistribute it and/or modify
> > + * it under the terms of the GNU General Public License version 2 as
> > + * published by the Free Software Foundation.
> > + */
> > +
> > +#ifndef COMPRESS_WRAPPER_H
> > +#define COMPRESS_WRAPPER_H
> > +
> > +#include "f2fs_fs.h"
> > +extern compress_ops compr_ops[]; /* [0]: LZO, [1]: LZ4, */
> > +
> > +#endif /* COMPRESS_WRAPPER_H */
> > diff --git a/fsck/fsck.h b/fsck/fsck.h
> > index c5e85fe..4e866ec 100644
> > --- a/fsck/fsck.h
> > +++ b/fsck/fsck.h
> > @@ -3,6 +3,9 @@
> >   *
> >   * Copyright (c) 2013 Samsung Electronics Co., Ltd.
> >   *             http://www.samsung.com/
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *  : add sload compression support
> >   *
> >   * This program is free software; you can redistribute it and/or modify
> >   * it under the terms of the GNU General Public License version 2 as
> > @@ -266,6 +269,9 @@ int f2fs_resize(struct f2fs_sb_info *);
> >  
> >  /* sload.c */
> >  int f2fs_sload(struct f2fs_sb_info *);
> > +void sldc_erase_bufs(struct compress_ctx *cc);
> > +void sload_countblk(void);
> > +extern struct ext_tbl_op ext_filter;
> >  
> >  /* segment.c */
> >  int reserve_new_block(struct f2fs_sb_info *, block_t *,
> > @@ -282,7 +288,16 @@ block_t new_node_block(struct f2fs_sb_info *,
> >  struct quota_file;
> >  u64 f2fs_quota_size(struct quota_file *);
> >  u64 f2fs_read(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> > +enum wr_addr_type {
> > +	WR_NORMAL = 1,
> > +	WR_COMPRESS_DATA = 2,
> > +	WR_NULL_ADDR = NULL_ADDR,  /* 0 */
> > +	WR_NEW_ADDR = NEW_ADDR, /* -1U */
> > +	WR_COMPRESS_ADDR = COMPRESS_ADDR, /* -2U */
> > +};
> >  u64 f2fs_write(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> > +u64 f2fs_write_compress_data(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
> > +u64 f2fs_write_addrtag(struct f2fs_sb_info *, nid_t, pgoff_t, unsigned int);
> >  void f2fs_filesize_update(struct f2fs_sb_info *, nid_t, u64);
> >  
> >  int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,
> > diff --git a/fsck/main.c b/fsck/main.c
> > index b20498f..001eed0 100644
> > --- a/fsck/main.c
> > +++ b/fsck/main.c
> > @@ -13,6 +13,9 @@
> >   * Copyright (c) 2019 Google Inc.
> >   *   Robin Hsu <robinhsu@google.com>
> >   *  : add cache layer
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *  : add sload compression support
> >   *
> >   * This program is free software; you can redistribute it and/or modify
> >   * it under the terms of the GNU General Public License version 2 as
> > @@ -25,6 +28,7 @@
> >  #include <getopt.h>
> >  #include <stdbool.h>
> >  #include "quotaio.h"
> > +#include "compress_wrapper.h"
> >  
> >  struct f2fs_fsck gfsck;
> >  
> > @@ -134,6 +138,17 @@ void sload_usage()
> >  	MSG(0, "  -S sparse_mode\n");
> >  	MSG(0, "  -t mount point [prefix of target fs path, default:/]\n");
> >  	MSG(0, "  -T timestamp\n");
> > +	MSG(0, "  -c enable compression (default allow policy)\n");
> > +	MSG(0, "    ------------ Compression sub-options -----------------\n");
> > +	MSG(0, "    -L <log-of-blocks-per-cluster>, default 2\n");
> > +	MSG(0, "    -a <algorithm> compression algorithm, default LZ4\n");
> > +	MSG(0, "    -x <ext> compress files except for these extensions.\n");
> > +	MSG(0, "    -i <ext> compress files with these extensions only.\n");
> > +	MSG(0, "    * -i or -x: use it many times for multiple extensions.\n");
> > +	MSG(0, "    * -i and -x cannot be used together..\n");
> > +	MSG(0, "    -m <num> min compressed blocks per cluster\n");
> > +	MSG(0, "    -r readonly (IMMUTABLE) for compressed files\n");
> > +	MSG(0, "    ------------------------------------------------------\n");
> >  	MSG(0, "  -d debug level [default:0]\n");
> >  	MSG(0, "  -V print the version number and exit\n");
> >  	exit(1);
> > @@ -534,7 +549,7 @@ void f2fs_parse_options(int argc, char *argv[])
> >  #endif
> >  	} else if (!strcmp("sload.f2fs", prog)) {
> >  #ifdef WITH_SLOAD
> > -		const char *option_string = "C:d:f:p:s:St:T:V";
> > +		const char *option_string = "cL:a:i:x:m:rC:d:f:p:s:St:T:V";
> >  #ifdef HAVE_LIBSELINUX
> >  		int max_nr_opt = (int)sizeof(c.seopt_file) /
> >  			sizeof(c.seopt_file[0]);
> > @@ -543,8 +558,82 @@ void f2fs_parse_options(int argc, char *argv[])
> >  		char *p;
> >  
> >  		c.func = SLOAD;
> > +		c.sldc_cc.log_cluster_size = 2;
> > +		c.sldc_ca = CA_LZ4;
> > +		c.sldc_min_cbpc = 1;
> > +		c.sldc_ef = &ext_filter;
> >  		while ((option = getopt(argc, argv, option_string)) != EOF) {
> > +			unsigned int i;
> > +			int val;
> > +
> >  			switch (option) {
> > +			case 'c': /* compression support */
> > +				c.sldc_en = true;
> > +				break;
> > +			case 'L': /* compression: log of blocks-per-cluster */
> > +				c.sldc_got_opt = true;
> > +				val = atoi(optarg);
> > +				if (val < MIN_COMPRESS_LOG_SIZE ||
> > +						val > MAX_COMPRESS_LOG_SIZE) {
> > +					MSG(0, "\tError: log of blocks per"
> > +						" cluster must be in the range"
> > +						" of %d .. %d.\n",
> > +						MIN_COMPRESS_LOG_SIZE,
> > +						MAX_COMPRESS_LOG_SIZE);
> > +					error_out(prog);
> > +				}
> > +				c.sldc_cc.log_cluster_size = val;
> > +				break;
> > +			case 'a': /* compression: choose algorithm */
> > +				c.sldc_got_opt = true;
> > +				c.sldc_ca = (u8)-1;
> > +				for (i = 0; ca_names[i][0] != 0; i++) {
> > +					if (!strcmp(ca_names[i], optarg)) {
> > +						c.sldc_ca = i;
> > +						break;
> > +					}
> > +				}
> > +				if (c.sldc_ca == (u8)-1) {
> > +					MSG(0, "\tError: Unknown compression"
> > +						" algorithm %s\n", optarg);
> > +					error_out(prog);
> > +				}
> > +				break;
> > +			case 'i': /* compress only these extensions */
> > +				c.sldc_got_opt = true;
> > +				if (c.sldc_policy == FP_ALLOW) {
> > +					MSG(0, "\tError: could not mix option"
> > +							" -i and -x\n");
> > +					error_out(prog);
> > +				}
> > +				c.sldc_policy = FP_DENY;
> > +				c.sldc_ef->add(optarg);
> > +				break;
> > +			case 'x': /* compress except for these extensions */
> > +				c.sldc_got_opt = true;
> > +				if (c.sldc_policy == FP_DENY) {
> > +					MSG(0, "\tError: could not mix option"
> > +							" -i and -x\n");
> > +					error_out(prog);
> > +				}
> > +				c.sldc_policy = FP_ALLOW;
> > +				c.sldc_ef->add(optarg);
> > +				break;
> > +			case 'm': /* minimum compressed blocks per cluster */
> > +				c.sldc_got_opt = true;
> > +				val = atoi(optarg);
> > +				if (val <= 0) {
> > +					MSG(0, "\tError: minimum compressed"
> > +						" blocks per cluster must be"
> > +						" positive.\n");
> > +					error_out(prog);
> > +				}
> > +				c.sldc_min_cbpc = val;
> > +				break;
> > +			case 'r': /* compress file to set IMMUTABLE */
> > +				c.sldc_got_opt = true;
> > +				c.sldc_immutable = true;
> > +				break;
> >  			case 'C':
> >  				c.fs_config_file = absolute_path(optarg);
> >  				break;
> > @@ -602,6 +691,27 @@ void f2fs_parse_options(int argc, char *argv[])
> >  			if (err != NOERROR)
> >  				break;
> >  		}
> > +		if (c.sldc_got_opt && !c.sldc_en) {
> > +			MSG(0, "\tError: compression sub-options are used"
> > +				" without the compression enable (-c) option\n"
> > +			);
> > +			error_out(prog);
> > +		}
> > +		if (err == NOERROR && c.sldc_en) {
> > +			c.sldc_cc.cluster_size = 1
> > +				<< c.sldc_cc.log_cluster_size;
> > +			if (c.sldc_policy == FP_UNASSIGNED)
> > +				c.sldc_policy = FP_ALLOW;
> > +			if (c.sldc_min_cbpc >= c.sldc_cc.cluster_size) {
> > +				MSG(0, "\tError: minimum reduced blocks by"
> > +					" compression per cluster must be at"
> > +					" most one less than blocks per"
> > +					" cluster, i.e. %d\n",
> > +					c.sldc_cc.cluster_size - 1);
> > +				error_out(prog);
> > +			}
> > +			qbuf_init();
> > +		}
> >  #endif /* WITH_SLOAD */
> >  	}
> >  
> > @@ -812,6 +922,27 @@ static int do_resize(struct f2fs_sb_info *sbi)
> >  #endif
> >  
> >  #ifdef WITH_SLOAD
> > +int init_compr(struct f2fs_sb_info *sbi)
> > +{
> > +	if (!(sbi->raw_super->feature
> > +			& cpu_to_le32(F2FS_FEATURE_COMPRESSION))) {
> > +		MSG(0, "Error: Compression (-c) was requested "
> > +			"but the file system is not created "
> > +			"with such feature.\n");
> > +		return -1;
> > +	}
> > +	if (compr_ops[c.sldc_ca].init == NULL) {
> > +		MSG(0, "Error: The selected compression algorithm is not"
> > +				" supported\n");
> > +		return -1;
> > +	}
> > +	c.sldc_compr = compr_ops + c.sldc_ca;
> > +	c.sldc_compr->init(&c.sldc_cc);
> > +	sldc_erase_bufs(&c.sldc_cc);
> > +	c.sldc_cc.rlen = c.sldc_cc.cluster_size * F2FS_BLKSIZE;
> > +	return 0;
> > +}
> > +
> >  static int do_sload(struct f2fs_sb_info *sbi)
> >  {
> >  	if (!c.from_dir) {
> > @@ -821,6 +952,11 @@ static int do_sload(struct f2fs_sb_info *sbi)
> >  	if (!c.mount_point)
> >  		c.mount_point = "/";
> >  
> > +	if (c.sldc_en) {
> > +		if (init_compr(sbi))
> > +			return -1;
> > +	}
> > +
> >  	return f2fs_sload(sbi);
> >  }
> >  #endif
> > @@ -971,6 +1107,9 @@ retry:
> >  		return ret2;
> >  	}
> >  
> > +	if (c.func == SLOAD)
> > +		c.sldc_ef->destroy();
> > +
> >  	printf("\nDone: %lf secs\n", (get_boottime_ns() - start) / 1000000000.0);
> >  	return ret;
> >  
> > diff --git a/fsck/segment.c b/fsck/segment.c
> > index 0487f41..e4c8cea 100644
> > --- a/fsck/segment.c
> > +++ b/fsck/segment.c
> > @@ -8,6 +8,9 @@
> >   *   Hou Pengyang <houpengyang@huawei.com>
> >   *   Liu Shuoran <liushuoran@huawei.com>
> >   *   Jaegeuk Kim <jaegeuk@kernel.org>
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *  : add sload compression support
> >   *
> >   * This program is free software; you can redistribute it and/or modify
> >   * it under the terms of the GNU General Public License version 2 as
> > @@ -16,6 +19,7 @@
> >  #include "fsck.h"
> >  #include "node.h"
> >  #include "quotaio.h"
> > +#include "compress_wrapper.h"
> >  
> >  int reserve_new_block(struct f2fs_sb_info *sbi, block_t *to,
> >  			struct f2fs_summary *sum, int type, bool is_inode)
> > @@ -228,8 +232,14 @@ u64 f2fs_read(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> >  	return read_count;
> >  }
> >  
> > -u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> > -					u64 count, pgoff_t offset)
> > +/*
> > + * Do not call this function directly.  Instead, call one of the following:
> > + *     u64 f2fs_write();
> > + *     u64 f2fs_write_compress_data();
> > + *     u64 f2fs_write_addrtag();
> > + */
> > +static u64 f2fs_write_ex(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> > +		u64 count, pgoff_t offset, enum wr_addr_type addr_type)
> >  {
> >  	struct dnode_of_data dn;
> >  	struct node_info ni;
> > @@ -243,6 +253,19 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> >  	void* index_node = NULL;
> >  	int idirty = 0;
> >  	int err;
> > +	bool has_data = (addr_type == WR_NORMAL
> > +			|| addr_type == WR_COMPRESS_DATA);
> > +
> > +	if (count == 0)
> > +		return 0;
> > +
> > +	/*
> > +	 * Enforce calling from f2fs_write(), f2fs_write_compress_data(),
> > +	 * and f2fs_write_addrtag().   Beside, check if is properly called.
> > +	 */
> > +	ASSERT((!has_data && buffer == NULL) || (has_data && buffer != NULL));
> > +	if (addr_type != WR_NORMAL)
> > +		ASSERT(offset % F2FS_BLKSIZE == 0); /* block boundary only */
> >  
> >  	/* Memory allocation for block buffer and inode. */
> >  	blk_buffer = calloc(BLOCK_SZ, 2);
> > @@ -265,15 +288,26 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> >  			if (err)
> >  				break;
> >  			idirty |= dn.idirty;
> > -			if (index_node)
> > -				free(index_node);
> > +			free(index_node);
> >  			index_node = (dn.node_blk == dn.inode_blk) ?
> > -							NULL : dn.node_blk;
> > +					NULL : dn.node_blk;
> >  			remained_blkentries = ADDRS_PER_PAGE(sbi,
> > -						dn.node_blk, dn.inode_blk);
> > +					dn.node_blk, dn.inode_blk) -
> > +					dn.ofs_in_node;
> >  		}
> >  		ASSERT(remained_blkentries > 0);
> >  
> > +		if (!has_data) {
> > +			dn.data_blkaddr = addr_type;
> > +			set_data_blkaddr(&dn);
> > +			idirty |= dn.idirty;
> > +			if (dn.ndirty)
> > +				ASSERT(dev_write_block(dn.node_blk,
> > +						dn.node_blkaddr) >= 0);
> > +			written_count = 0;
> > +			break;
> > +		}
> > +
> >  		blkaddr = datablock_addr(dn.node_blk, dn.ofs_in_node);
> >  		if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) {
> >  			err = new_data_block(sbi, blk_buffer,
> > @@ -281,6 +315,7 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> >  			if (err)
> >  				break;
> >  			blkaddr = dn.data_blkaddr;
> > +			idirty |= dn.idirty;
> >  		}
> >  
> >  		off_in_blk = offset % BLOCK_SZ;
> > @@ -305,9 +340,10 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> >  
> >  		dn.ofs_in_node++;
> >  		if ((--remained_blkentries == 0 || count == 0) && (dn.ndirty))
> > -			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr) >= 0);
> > +			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr)
> > +					>= 0);
> >  	}
> > -	if (offset > le64_to_cpu(inode->i.i_size)) {
> > +	if (addr_type == WR_NORMAL && offset > le64_to_cpu(inode->i.i_size)) {
> >  		inode->i.i_size = cpu_to_le64(offset);
> >  		idirty = 1;
> >  	}
> > @@ -315,13 +351,33 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> >  		ASSERT(inode == dn.inode_blk);
> >  		ASSERT(write_inode(inode, ni.blk_addr) >= 0);
> >  	}
> > -	if (index_node)
> > -		free(index_node);
> > +
> > +	free(index_node);
> >  	free(blk_buffer);
> >  
> >  	return written_count;
> >  }
> >  
> > +u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> > +					u64 count, pgoff_t offset)
> > +{
> > +	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_NORMAL);
> > +}
> > +
> > +u64 f2fs_write_compress_data(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
> > +					u64 count, pgoff_t offset)
> > +{
> > +	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_COMPRESS_DATA);
> > +}
> > +
> > +u64 f2fs_write_addrtag(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
> > +		unsigned int addrtag)
> > +{
> > +	ASSERT(addrtag == COMPRESS_ADDR || addrtag == NEW_ADDR
> > +			|| addrtag == NULL_ADDR);
> > +	return f2fs_write_ex(sbi, ino, NULL, F2FS_BLKSIZE, offset, addrtag);
> > +}
> > +
> >  /* This function updates only inode->i.i_size */
> >  void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
> >  {
> > @@ -342,11 +398,59 @@ void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
> >  	free(inode);
> >  }
> >  
> > +#define MAX_BULKR_RETRY 5
> > +int bulkread(int fd, void *rbuf, size_t rsize, bool *eof)
> > +{
> > +	int n = 0;
> > +	int retry = MAX_BULKR_RETRY;
> > +	int cur;
> > +
> > +	if (!rsize)
> > +		return 0;
> > +
> > +	if (eof != NULL)
> > +		*eof = false;
> > +	while (rsize && (cur = read(fd, rbuf, rsize)) != 0) {
> > +		if (cur == -1) {
> > +			if (errno == EINTR && retry--)
> > +				continue;
> > +			return -1;
> > +		}
> > +		retry = MAX_BULKR_RETRY;
> > +
> > +		rsize -= cur;
> > +		n += cur;
> > +	}
> > +	if (eof != NULL)
> > +		*eof = (cur == 0);
> > +	return n;
> > +}
> > +
> > +u64 f2fs_fix_mutable(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
> > +		unsigned int compressed)
> > +{
> > +	unsigned int i;
> > +	u64 wlen;
> > +
> > +	if (c.sldc_immutable)
> > +		return 0;
> > +
> > +	for (i = 0; i < compressed - 1; i++) {
> > +		wlen = f2fs_write_addrtag(sbi, ino,
> > +				offset + (i << F2FS_BLKSIZE_BITS), NEW_ADDR);
> > +		if (wlen)
> > +			return wlen;
> > +	}
> > +	return 0;
> > +}
> > +
> >  int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
> >  {
> >  	int fd, n;
> >  	pgoff_t off = 0;
> >  	u8 buffer[BLOCK_SZ];
> > +	struct node_info ni;
> > +	struct f2fs_node *node_blk;
> >  
> >  	if (de->ino == 0)
> >  		return -1;
> > @@ -359,8 +463,6 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
> >  
> >  	/* inline_data support */
> >  	if (de->size <= DEF_MAX_INLINE_DATA) {
> > -		struct node_info ni;
> > -		struct f2fs_node *node_blk;
> >  		int ret;
> >  
> >  		get_node_info(sbi, de->ino, &ni);
> > @@ -385,6 +487,82 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
> >  		node_blk->i.i_size = cpu_to_le64(de->size);
> >  		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
> >  		free(node_blk);
> > +#ifdef WITH_SLOAD
> > +	} else if (c.func == SLOAD && c.sldc_en &&
> > +			c.sldc_ef->filter(de->full_path)) {
> > +		bool eof = false;
> > +		u8 *rbuf = c.sldc_cc.rbuf;
> > +		unsigned int cblocks = 0;
> > +
> > +		node_blk = calloc(BLOCK_SZ, 1);
> > +		ASSERT(node_blk);
> > +
> > +		/* read inode */
> > +		get_node_info(sbi, de->ino, &ni);
> > +		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
> > +		/* update inode meta */
> > +		node_blk->i.i_compress_algrithm = c.sldc_ca;
> > +		node_blk->i.i_log_cluster_size =
> > +				c.sldc_cc.log_cluster_size;
> > +		node_blk->i.i_flags = cpu_to_le32(
> > +				F2FS_COMPR_FL |
> > +				(c.sldc_immutable ? FS_IMMUTABLE_FL : 0));
> > +		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
> > +
> > +		while (!eof && (n = bulkread(fd, rbuf, c.sldc_cc.rlen,
> > +				&eof)) > 0) {
> > +			int ret = c.sldc_compr->compress(&c.sldc_cc);
> > +			u64 wlen;
> > +			u32 csize = ALIGN_UP(c.sldc_cc.clen +
> > +					COMPRESS_HEADER_SIZE, BLOCK_SZ);
> > +			unsigned int cur_cblk;
> > +
> > +			if (ret || n < (int)(csize + BLOCK_SZ *
> > +						c.sldc_min_cbpc)) {
> > +				wlen = f2fs_write(sbi, de->ino, rbuf, n, off);
> > +				ASSERT((int)wlen == n);
> > +			} else {
> > +				wlen = f2fs_write_addrtag(sbi, de->ino, off,
> > +						WR_COMPRESS_ADDR);
> > +				ASSERT(!wlen);
> > +				wlen = f2fs_write_compress_data(sbi, de->ino,
> > +						(u8 *)c.sldc_cc.cbuf,
> > +						csize, off + BLOCK_SZ);
> > +				ASSERT(wlen == csize);
> > +				sldc_erase_bufs(&c.sldc_cc);
> > +				cur_cblk = (c.sldc_cc.rlen - csize) / BLOCK_SZ;
> > +				cblocks += cur_cblk;
> > +				wlen = f2fs_fix_mutable(sbi, de->ino,
> > +						off + BLOCK_SZ + csize,
> > +						cur_cblk);
> > +				ASSERT(!wlen);
> > +			}
> > +			off += n;
> > +		}
> > +		if (n == -1) {
> > +			fprintf(stderr, "Load file '%s' failed: ",
> > +					de->full_path);
> > +			perror(NULL);
> > +		}
> > +		/* read inode */
> > +		get_node_info(sbi, de->ino, &ni);
> > +		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
> > +		/* update inode meta */
> > +		node_blk->i.i_size = cpu_to_le64(off);
> > +		if (!c.sldc_immutable)
> > +			node_blk->i.i_compr_blocks = cpu_to_le64(cblocks);
> > +		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
> > +		free(node_blk);
> > +
> > +		if (!c.sldc_immutable) {
> > +			sbi->total_valid_block_count += cblocks;
> > +			if (sbi->total_valid_block_count >=
> > +					sbi->user_block_count) {
> > +				ERR_MSG("Not enough space\n");
> > +				ASSERT(0);
> > +			}
> > +		}
> > +#endif
> >  	} else {
> >  		while ((n = read(fd, buffer, BLOCK_SZ)) > 0) {
> >  			f2fs_write(sbi, de->ino, buffer, n, off);
> > diff --git a/fsck/sload.c b/fsck/sload.c
> > index 14012fb..13e523a 100644
> > --- a/fsck/sload.c
> > +++ b/fsck/sload.c
> > @@ -6,6 +6,9 @@
> >   *   Hou Pengyang <houpengyang@huawei.com>
> >   *   Liu Shuoran <liushuoran@huawei.com>
> >   *   Jaegeuk Kim <jaegeuk@kernel.org>
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *  : add sload compression support
> >   *
> >   * This program is free software; you can redistribute it and/or modify
> >   * it under the terms of the GNU General Public License version 2 as
> > @@ -317,6 +320,70 @@ skip:
> >  	return 0;
> >  }
> >  
> > +typedef struct _ext_tbl {
> > +	const char *ext;
> > +	struct _ext_tbl *next; /* linked list */
> > +} ext_tbl_t;
> > +static ext_tbl_t *ext_tbl;
> > +
> > +static bool ext_found(const char *ext)
> > +{
> > +	ext_tbl_t *p = ext_tbl;
> > +
> > +	while (p != NULL && strcmp(ext, p->ext))
> > +		p = p->next;
> > +	return (p != NULL);
> > +}
> > +
> > +static const char *get_ext(const char *path)
> > +{
> > +	char *p = strrchr(path, '.');
> > +	return p == NULL ? path + strlen(path) : p + 1;
> > +}
> > +
> > +static bool ext_do_filter(const char *path)
> > +{
> > +	return (ext_found(get_ext(path)) == true) ^ (c.sldc_policy == FP_ALLOW);
> > +}
> > +
> > +static void ext_filter_add(const char *ext)
> > +{
> > +	ext_tbl_t *node;
> > +
> > +	ASSERT(ext != NULL);
> > +	if (ext_found(ext))
> > +		return; /* ext was already registered */
> > +	node = malloc(sizeof(ext_tbl_t));
> > +	ASSERT(node != NULL);
> > +	node->ext = ext;
> > +	node->next = ext_tbl;
> > +	ext_tbl = node;
> > +}
> > +
> > +static void ext_filter_destroy(void)
> > +{
> > +	ext_tbl_t *p;
> > +
> > +	while (ext_tbl != NULL) {
> > +		p = ext_tbl;
> > +		ext_tbl = p->next;
> > +		free(p);
> > +	}
> > +}
> > +
> > +struct ext_tbl_op ext_filter = {
> > +	.add = ext_filter_add,
> > +	.destroy = ext_filter_destroy,
> > +	.filter = ext_do_filter,
> > +};
> > +
> > +void sldc_erase_bufs(struct compress_ctx *cc)
> > +{
> > +	memset(cc->rbuf, 0, cc->cluster_size * F2FS_BLKSIZE);
> > +	memset(cc->cbuf->cdata, 0, cc->cluster_size * F2FS_BLKSIZE
> > +			- F2FS_BLKSIZE);
> > +}
> > +
> >  int f2fs_sload(struct f2fs_sb_info *sbi)
> >  {
> >  	int ret = 0;
> > diff --git a/include/f2fs_fs.h b/include/f2fs_fs.h
> > index 1348e39..2a2dc15 100644
> > --- a/include/f2fs_fs.h
> > +++ b/include/f2fs_fs.h
> > @@ -5,6 +5,9 @@
> >   *             http://www.samsung.com/
> >   * Copyright (c) 2019 Google Inc.
> >   *             http://www.google.com/
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *  : add sload compression support
> >   *
> >   * Dual licensed under the GPL or LGPL version 2 licenses.
> >   *
> > @@ -68,6 +71,10 @@ typedef uint16_t u_int16_t;
> >  typedef uint8_t u_int8_t;
> >  #endif
> >  
> > +/* codes from kernel's f2fs.h, GPL-v2.0 */
> > +#define MIN_COMPRESS_LOG_SIZE	2
> > +#define MAX_COMPRESS_LOG_SIZE	8
> > +
> >  typedef u_int64_t	u64;
> >  typedef u_int32_t	u32;
> >  typedef u_int16_t	u16;
> > @@ -93,6 +100,31 @@ typedef u32	__be32;
> >  typedef u64	__be64;
> >  #endif
> >  
> > +/*
> > + * code borrowed from kernel f2fs dirver: f2fs.h, GPL-2.0
> > + *  : definitions of COMPRESS_DATA_RESERVED_SIZE,
> > + *    struct compress_data, COMPRESS_HEADER_SIZE,
> > + *    and struct compress_ctx
> > + */
> > +#define COMPRESS_DATA_RESERVED_SIZE		4
> > +struct compress_data {
> > +	__le32 clen;			/* compressed data size */
> > +	__le32 chksum;			/* checksum of compressed data */
> > +	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
> > +	u8 cdata[];			/* compressed data */
> > +};
> > +#define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
> > +/* compress context */
> > +struct compress_ctx {
> > +	unsigned int cluster_size;	/* page count in cluster */
> > +	unsigned int log_cluster_size;	/* log of cluster size */
> > +	void *rbuf;			/* compression input buffer */
> > +	struct compress_data *cbuf;	/* comprsssion output header + data */
> > +	size_t rlen;			/* valid data length in rbuf */
> > +	size_t clen;			/* valid data length in cbuf */
> > +	void *private;			/* work buf for compress algorithm */
> > +};
> > +
> >  #if HAVE_BYTESWAP_H
> >  #include <byteswap.h>
> >  #else
> > @@ -345,6 +377,25 @@ typedef struct {
> >  	bool dbg_en;
> >  } dev_cache_config_t;
> >  
> > +/* f2fs_configration: sldc_ca, the sload compress algorithm */
> > +enum {CA_LZO, CA_LZ4};
> > +extern const char *ca_names[];
> > +
> > +typedef struct  {
> > +	void (*init)(struct compress_ctx *cc);
> > +	int (*compress)(struct compress_ctx *cc);
> > +} compress_ops;
> > +
> > +#define ALIGN_UP(value, size) ((value) + ((value) % (size) > 0 ? \
> > +		(size) - (value) % (size) : 0))
> > +
> > +enum filter_policy {FP_UNASSIGNED = 0, FP_ALLOW, FP_DENY};
> > +struct ext_tbl_op {
> > +	void (*add)(const char *);
> > +	void (*destroy)(void);
> > +	bool (*filter)(const char *);
> > +};
> > +
> >  struct f2fs_configuration {
> >  	u_int32_t reserved_segments;
> >  	u_int32_t new_reserved_segments;
> > @@ -441,6 +492,24 @@ struct f2fs_configuration {
> >  
> >  	/* cache parameters */
> >  	dev_cache_config_t cache_config;
> > +
> > +	/* quick dynamic buffer */
> > +	bool qbuf_initialized;
> > +	size_t qbufsize;
> > +	void *qbuf;
> I don't think these qbuf* variables inside the f2fs_configuration are
> used anywhere - shouldn't they be removed?
> > +
> > +	/* sldc: sload compression support */
> > +	bool sldc_en;
> > +	bool sldc_use_allow_list;  /* default false to use the deny list */
> > +	struct compress_ctx sldc_cc;
> > +	u8 sldc_ca; /* compress algorithm: 0 = LZO, 1 = LZ4 */
> > +	compress_ops *sldc_compr;
> > +	enum filter_policy sldc_policy;
> > +	/* max_cppc can used to specify minimum compression rate */
> > +	unsigned int sldc_min_cbpc; /* min compressed pages per cluster */
> > +	bool sldc_got_opt;
> > +	bool sldc_immutable;
> > +	struct ext_tbl_op *sldc_ef; /* extension filter */
> >  };
> >  
> >  #ifdef CONFIG_64BIT
> > @@ -1226,6 +1295,11 @@ extern void f2fs_release_sparse_resource(void);
> >  extern int f2fs_finalize_device(void);
> >  extern int f2fs_fsync_device(void);
> >  
> > +/* quick (shared) buffer */
> > +extern void qbuf_free(void);
> > +extern void *qbuf_alloc(size_t size);
> > +extern void qbuf_init(void);
> > +
> >  extern void dcache_init(void);
> >  extern void dcache_release(void);
> >  
> > @@ -1377,7 +1451,7 @@ int f2fs_reset_zone(int, void *);
> >  extern int f2fs_reset_zones(int);
> >  extern uint32_t f2fs_get_usable_segments(struct f2fs_super_block *sb);
> >  
> > -#define SIZE_ALIGN(val, size)	((val) + (size) - 1) / (size)
> > +#define SIZE_ALIGN(val, size)	(((val) + (size) - 1) / (size))
> >  #define SEG_ALIGN(blks)		SIZE_ALIGN(blks, c.blks_per_seg)
> >  #define ZONE_ALIGN(blks)	SIZE_ALIGN(blks, c.blks_per_seg * \
> >  					c.segs_per_zone)
> > diff --git a/lib/libf2fs_io.c b/lib/libf2fs_io.c
> > index 138285d..0280896 100644
> > --- a/lib/libf2fs_io.c
> > +++ b/lib/libf2fs_io.c
> > @@ -5,6 +5,9 @@
> >   *             http://www.samsung.com/
> >   * Copyright (c) 2019 Google Inc.
> >   *             http://www.google.com/
> > + * Copyright (c) 2020 Google Inc.
> > + *   Robin Hsu <robinhsu@google.com>
> > + *  : add quick-buffer for sload compression support
> >   *
> >   * Dual licensed under the GPL or LGPL version 2 licenses.
> >   */
> > @@ -106,6 +109,36 @@ static long dcache_relocate_offset0[] = {
> >  };
> >  static int dcache_relocate_offset[16];
> >  
> > +/* quick (shared) buffer */
> > +static bool qbuf_initialized;
> > +static void *qbuf;
> > +static size_t qbufsize;
> > +void qbuf_free(void)
> > +{
> > +	ASSERT(qbuf_initialized);
> > +	if (qbuf != NULL) {
> > +		free(qbuf);
> > +		qbuf = NULL;
> > +		qbufsize = 0;
> > +	}
> > +}
> > +void *qbuf_alloc(size_t size)
> > +{
> > +	ASSERT(qbuf_initialized);
> > +	if (size > qbufsize) {
> > +		qbuf_free();
> > +		qbuf = malloc(size);
> qbufsize needs to be updated to size here.
> > +	}
> > +	return qbuf;
> > +}
> > +void qbuf_init(void)
> > +{
> > +	if (qbuf_initialized)
> > +		return;
> > +	atexit(qbuf_free);
> > +	qbuf_initialized = true;
> > +}
> Is there really a need for all this qbuf code? As far as I can tell,
> qbuf_alloc() is only ever called once during any invocation of
> sload, and it'd be better/simpler to replace that with a malloc()...

Yeah, it seems we don't need qbuf. Let me apply this change.
Please check -dev branch.

Thanks,

---
 fsck/compress.c   |  6 ++++--
 fsck/main.c       |  1 -
 include/f2fs_fs.h | 10 ----------
 lib/libf2fs_io.c  | 30 ------------------------------
 4 files changed, 4 insertions(+), 43 deletions(-)

diff --git a/fsck/compress.c b/fsck/compress.c
index d4baa04e645c..620768d7de20 100644
--- a/fsck/compress.c
+++ b/fsck/compress.c
@@ -55,7 +55,8 @@ static void lzo_compress_init(struct compress_ctx *cc)
 	size_t size = cc->cluster_size * F2FS_BLKSIZE;
 	size_t alloc = size + lzo1x_worst_compress(size)
 			+ COMPRESS_HEADER_SIZE + LZO_WORK_SIZE;
-	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
+	cc->private = malloc(alloc);
+	ASSERT(cc->private);
 	cc->rbuf = (char *) cc->private + LZO_WORK_SIZE;
 	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
 }
@@ -75,7 +76,8 @@ static void lz4_compress_init(struct compress_ctx *cc)
 	size_t size = cc->cluster_size * F2FS_BLKSIZE;
 	size_t alloc = size + LZ4_COMPRESSBOUND(size)
 			+ COMPRESS_HEADER_SIZE + LZ4_WORK_SIZE;
-	ASSERT((cc->private = qbuf_alloc(alloc)) != NULL);
+	cc->private = malloc(alloc);
+	ASSERT(cc->private);
 	cc->rbuf = (char *) cc->private + LZ4_WORK_SIZE;
 	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
 }
diff --git a/fsck/main.c b/fsck/main.c
index e56fbed93f8b..a538c72dcc66 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -712,7 +712,6 @@ void f2fs_parse_options(int argc, char *argv[])
 					c.compress.cc.cluster_size - 1);
 				error_out(prog);
 			}
-			qbuf_init();
 		}
 #endif /* WITH_SLOAD */
 	}
diff --git a/include/f2fs_fs.h b/include/f2fs_fs.h
index 3812a4f7ae8f..b9dc0b6f480e 100644
--- a/include/f2fs_fs.h
+++ b/include/f2fs_fs.h
@@ -515,11 +515,6 @@ struct f2fs_configuration {
 	/* cache parameters */
 	dev_cache_config_t cache_config;
 
-	/* quick dynamic buffer */
-	bool qbuf_initialized;
-	size_t qbufsize;
-	void *qbuf;
-
 	/* compression support for sload.f2fs */
 	compress_config_t compress;
 };
@@ -1307,11 +1302,6 @@ extern void f2fs_release_sparse_resource(void);
 extern int f2fs_finalize_device(void);
 extern int f2fs_fsync_device(void);
 
-/* quick (shared) buffer */
-extern void qbuf_free(void);
-extern void *qbuf_alloc(size_t size);
-extern void qbuf_init(void);
-
 extern void dcache_init(void);
 extern void dcache_release(void);
 
diff --git a/lib/libf2fs_io.c b/lib/libf2fs_io.c
index 0280896de066..dcedc173a8d9 100644
--- a/lib/libf2fs_io.c
+++ b/lib/libf2fs_io.c
@@ -109,36 +109,6 @@ static long dcache_relocate_offset0[] = {
 };
 static int dcache_relocate_offset[16];
 
-/* quick (shared) buffer */
-static bool qbuf_initialized;
-static void *qbuf;
-static size_t qbufsize;
-void qbuf_free(void)
-{
-	ASSERT(qbuf_initialized);
-	if (qbuf != NULL) {
-		free(qbuf);
-		qbuf = NULL;
-		qbufsize = 0;
-	}
-}
-void *qbuf_alloc(size_t size)
-{
-	ASSERT(qbuf_initialized);
-	if (size > qbufsize) {
-		qbuf_free();
-		qbuf = malloc(size);
-	}
-	return qbuf;
-}
-void qbuf_init(void)
-{
-	if (qbuf_initialized)
-		return;
-	atexit(qbuf_free);
-	qbuf_initialized = true;
-}
-
 static void dcache_print_statistics(void)
 {
 	long i;
-- 
2.29.2.729.g45daf8777d-goog



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

[f2fs-dev] [PATCH v3 3/3] f2fs-tools:sload.f2fs compress: Fixed automake

[Robin Hsu]

From: Robin Hsu <robinhsu@google.com>

Fixed automake for sload.f2fs compression support

./configure automatcally compile in liblzo2 (for sload to support -a LZO)
and liblz4 (for sload to support -a LZ4), whhen the libraries present.

Signed-off-by: Robin Hsu <robinhsu@google.com>
---
 configure.ac     | 12 ++++++++++++
 fsck/Makefile.am |  9 ++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index 1e5619d..01d1a05 100644
--- a/configure.ac
+++ b/configure.ac
@@ -52,6 +52,18 @@ AC_PATH_PROG([LDCONFIG], [ldconfig],
        [$PATH:/sbin])
 
 # Checks for libraries.
+AC_CHECK_LIB([lzo2], [main],
+	[AC_SUBST([liblzo2_LIBS], ["-llzo2"])
+		AC_DEFINE([HAVE_LIBLZO2], [1],
+		[Define if you have liblzo2])
+	], [], [])
+
+AC_CHECK_LIB([lz4], [main],
+	[AC_SUBST([liblz4_LIBS], ["-llz4"])
+		AC_DEFINE([HAVE_LIBLZ4], [1],
+		[Define if you have liblz4])
+	], [], [])
+
 PKG_CHECK_MODULES([libuuid], [uuid])
 
 AS_IF([test "x$with_selinux" != "xno"],
diff --git a/fsck/Makefile.am b/fsck/Makefile.am
index 1fc7310..74bc4b8 100644
--- a/fsck/Makefile.am
+++ b/fsck/Makefile.am
@@ -3,12 +3,15 @@
 AM_CPPFLAGS = ${libuuid_CFLAGS} -I$(top_srcdir)/include
 AM_CFLAGS = -Wall
 sbin_PROGRAMS = fsck.f2fs
-noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h quotaio_tree.h quotaio_v2.h xattr.h
+noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h \
+		quotaio_tree.h quotaio_v2.h xattr.h compress_wrapper.h
 include_HEADERS = $(top_srcdir)/include/quota.h
 fsck_f2fs_SOURCES = main.c fsck.c dump.c mount.c defrag.c resize.c \
-		node.c segment.c dir.c sload.c xattr.c \
+		node.c segment.c dir.c sload.c xattr.c compress_wrapper.c \
 		dict.c mkquota.c quotaio.c quotaio_tree.c quotaio_v2.c
-fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} $(top_builddir)/lib/libf2fs.la
+fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} \
+	${liblzo2_LIBS} ${liblz4_LIBS} \
+	$(top_builddir)/lib/libf2fs.la
 
 install-data-hook:
 	ln -sf fsck.f2fs $(DESTDIR)/$(sbindir)/dump.f2fs
-- 
2.29.2.576.ga3fc446d84-goog



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel