testing UBIFS with zstd

testing UBIFS with zstd

[Michele Dionisio]

[-- Attachment #1: Type: text/plain, Size: 315 bytes --]

I'm testing ubifs with zstd compression. In my opinion it is a good
idea to start to insert this "new" compression algorithem because the
performance is paragonable with LZ4 and the compression is much
better.

I attach patches for mtd-utils and linux too

(remember to add also https://lkml.org/lkml/2017/9/8/824)

[-- Attachment #2: 0000-zstd.patch --]
[-- Type: text/x-patch, Size: 7112 bytes --]

From a492983c143ea7c12a5d29bf7126c181b9eadb41 Mon Sep 17 00:00:00 2001
From: Michele Dionisio <michele.dionisio@powersoft.it>
Date: Thu, 26 Apr 2018 11:13:06 +0200
Subject: [ADD] add support zstd for ubi


diff --git a/Makefile.am b/Makefile.am
index 5a6e77c..9b1c9c6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,6 +10,10 @@ if WITHOUT_LZO
 AM_CPPFLAGS += -DWITHOUT_LZO
 endif
 
+if WITHOUT_ZSTD
+AM_CPPFLAGS += -DWITHOUT_ZSTD
+endif
+
 sbin_PROGRAMS =
 sbin_SCRIPTS =
 check_PROGRAMS =
diff --git a/configure.ac b/configure.ac
index 1ac45c7..3432e3b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -57,6 +57,7 @@ need_pthread="no"
 need_uuid="no"
 need_zlib="no"
 need_lzo="no"
+need_zstd="no"
 need_xattr="no"
 need_cmocka="no"
 
@@ -119,6 +120,7 @@ AM_COND_IF([BUILD_UBIFS], [
 	need_xattr="yes"
 	need_zlib="yes"
 	need_lzo="yes"
+	need_zstd="yes"
 ])
 
 AM_COND_IF([BUILD_JFFSX], [
@@ -144,6 +146,14 @@ AC_ARG_WITH([lzo],
 	*) AC_MSG_ERROR([bad value ${withval} for --without-lzo]) ;;
 	esac])
 
+AC_ARG_WITH([zstd],
+	[AS_HELP_STRING([--without-zstd], [Disable support for zstd compression])],
+	[case "${withval}" in
+	yes) ;;
+	no) need_zstd="no" ;;
+	*) AC_MSG_ERROR([bad value ${withval} for --without-zstd]) ;;
+	esac])
+
 ##### search for dependencies #####
 
 clock_gettime_missing="no"
@@ -151,6 +161,7 @@ pthread_missing="no"
 uuid_missing="no"
 zlib_missing="no"
 lzo_missing="no"
+zstd_missing="no"
 xattr_missing="no"
 cmocka_missing="no"
 
@@ -181,6 +192,12 @@ if test "x$need_lzo" = "xyes"; then
 	)
 fi
 
+if test "x$need_zstd" = "xyes"; then
+	AC_ARG_VAR([ZSTD_CFLAGS], [C compiler flags for zstd])
+	AC_ARG_VAR([ZSTD_LIBS], [linker flags for zstd])
+	AC_CHECK_LIB([zstd], [ZSTD_compress], [ZSTD_LIBS="-lzstd"], [zstd_missing="yes"])
+fi
+
 if test "x$need_xattr" = "xyes"; then
 	AC_CHECK_HEADERS([sys/xattr.h], [], [xattr_missing="yes"])
 	AC_CHECK_HEADERS([sys/acl.h], [], [xattr_missing="yes"])
@@ -229,6 +246,13 @@ if test "x$lzo_missing" = "xyes"; then
 	dep_missing="yes"
 fi
 
+if test "x$zstd_missing" = "xyes"; then
+	AC_MSG_WARN([cannot find zstd library required for mkfs programs])
+	AC_MSG_NOTICE([mtd-utils can optionally be built without mkfs.ubifs])
+	AC_MSG_NOTICE([mtd-utils can optionally be built without zstd support])
+	dep_missing="yes"
+fi
+
 if test "x$xattr_missing" = "xyes"; then
 	AC_MSG_WARN([cannot find headers for extended attributes])
 	AC_MSG_WARN([disabling XATTR support])
@@ -248,6 +272,7 @@ fi
 ##### generate output #####
 
 AM_CONDITIONAL([WITHOUT_LZO], [test "x$need_lzo" != "xyes"])
+AM_CONDITIONAL([WITHOUT_ZSTD], [test "x$need_zstd" != "xyes"])
 AM_CONDITIONAL([WITHOUT_XATTR], [test "x$need_xattr" != "xyes"])
 
 AC_CHECK_SIZEOF([off_t])
diff --git a/include/mtd/ubifs-media.h b/include/mtd/ubifs-media.h
index a324e90..6575da1 100644
--- a/include/mtd/ubifs-media.h
+++ b/include/mtd/ubifs-media.h
@@ -305,6 +305,7 @@ enum {
 	UBIFS_COMPR_NONE,
 	UBIFS_COMPR_LZO,
 	UBIFS_COMPR_ZLIB,
+    UBIFS_COMPR_ZSTD,
 	UBIFS_COMPR_TYPES_CNT,
 };
 
diff --git a/ubifs-utils/Makemodule.am b/ubifs-utils/Makemodule.am
index 5862afb..e451b38 100644
--- a/ubifs-utils/Makemodule.am
+++ b/ubifs-utils/Makemodule.am
@@ -16,8 +16,8 @@ mkfs_ubifs_SOURCES = \
 	ubifs-utils/mkfs.ubifs/hashtable/hashtable.c \
 	ubifs-utils/mkfs.ubifs/hashtable/hashtable_itr.c \
 	ubifs-utils/mkfs.ubifs/devtable.c
-mkfs_ubifs_LDADD = libmtd.a libubi.a $(ZLIB_LIBS) $(LZO_LIBS) $(UUID_LIBS) -lm
-mkfs_ubifs_CPPFLAGS = $(AM_CPPFLAGS) $(ZLIB_CFLAGS) $(LZO_CFLAGS) $(UUID_CFLAGS) \
+mkfs_ubifs_LDADD = libmtd.a libubi.a $(ZLIB_LIBS) $(LZO_LIBS) $(ZSTD_LIBS) $(UUID_LIBS) -lm
+mkfs_ubifs_CPPFLAGS = $(AM_CPPFLAGS) $(ZLIB_CFLAGS) $(LZO_CFLAGS) $(ZSTD_CFLAGS) $(UUID_CFLAGS) \
 	-I$(top_srcdir)/ubi-utils/include -I$(top_srcdir)/ubifs-utils/mkfs.ubifs/
 
 UBIFS_BINS = \
diff --git a/ubifs-utils/mkfs.ubifs/compr.c b/ubifs-utils/mkfs.ubifs/compr.c
index 8eff186..a113d10 100644
--- a/ubifs-utils/mkfs.ubifs/compr.c
+++ b/ubifs-utils/mkfs.ubifs/compr.c
@@ -27,6 +27,9 @@
 #ifndef WITHOUT_LZO
 #include <lzo/lzo1x.h>
 #endif
+#ifndef WITHOUT_ZSTD
+#include <zstd.h>
+#endif
 #include <linux/types.h>
 
 #define crc32 __zlib_crc32
@@ -109,6 +112,22 @@ static int lzo_compress(void *in_buf, size_t in_len, void *out_buf,
 }
 #endif
 
+#ifndef WITHOUT_ZSTD
+static int zstd_compress(void *in_buf, size_t in_len, void *out_buf,
+			size_t *out_len)
+{
+	size_t ret;
+    ret = ZSTD_compress(out_buf, *out_len, in_buf, in_len, 19);
+    if (ZSTD_isError(ret))  {
+		errcnt += 1;
+		return -1;
+	}
+    *out_len = ret;
+
+	return 0;
+}
+#endif
+
 static int no_compress(void *in_buf, size_t in_len, void *out_buf,
 		       size_t *out_len)
 {
@@ -192,6 +211,11 @@ int compress_data(void *in_buf, size_t in_len, void *out_buf, size_t *out_len,
 		case MKFS_UBIFS_COMPR_ZLIB:
 			ret = zlib_deflate(in_buf, in_len, out_buf, out_len);
 			break;
+#ifndef WITHOUT_ZSTD
+        case MKFS_UBIFS_COMPR_ZSTD:
+			ret = zstd_compress(in_buf, in_len, out_buf, out_len);
+			break;
+#endif
 		case MKFS_UBIFS_COMPR_NONE:
 			ret = 1;
 			break;
diff --git a/ubifs-utils/mkfs.ubifs/compr.h b/ubifs-utils/mkfs.ubifs/compr.h
index e3dd95c..c21a682 100644
--- a/ubifs-utils/mkfs.ubifs/compr.h
+++ b/ubifs-utils/mkfs.ubifs/compr.h
@@ -36,6 +36,7 @@ enum compression_type
 	MKFS_UBIFS_COMPR_NONE,
 	MKFS_UBIFS_COMPR_LZO,
 	MKFS_UBIFS_COMPR_ZLIB,
+    MKFS_UBIFS_COMPR_ZSTD,
 };
 
 int compress_data(void *in_buf, size_t in_len, void *out_buf, size_t *out_len,
diff --git a/ubifs-utils/mkfs.ubifs/mkfs.ubifs.c b/ubifs-utils/mkfs.ubifs/mkfs.ubifs.c
index 6323dd4..ebc3ff6 100644
--- a/ubifs-utils/mkfs.ubifs/mkfs.ubifs.c
+++ b/ubifs-utils/mkfs.ubifs/mkfs.ubifs.c
@@ -184,7 +184,7 @@ static const char *helptext =
 "-o, --output=FILE        output to FILE\n"
 "-j, --jrn-size=SIZE      journal size\n"
 "-R, --reserved=SIZE      how much space should be reserved for the super-user\n"
-"-x, --compr=TYPE         compression type - \"lzo\", \"favor_lzo\", \"zlib\" or\n"
+"-x, --compr=TYPE         compression type - \"lzo\", \"favor_lzo\", \"zlib\", \"zstd\" or\n"
 "                         \"none\" (default: \"lzo\")\n"
 "-X, --favor-percent      may only be used with favor LZO compression and defines\n"
 "                         how many percent better zlib should compress to make\n"
@@ -605,6 +605,10 @@ static int get_options(int argc, char**argv)
 #ifndef WITHOUT_LZO
 			else if (strcmp(optarg, "favor_lzo") == 0)
 				c->favor_lzo = 1;
+#ifndef WITHOUT_ZSTD
+            else if (strcmp(optarg, "zstd") == 0)
+				c->default_compr = UBIFS_COMPR_ZSTD;
+#endif
 			else if (strcmp(optarg, "lzo") != 0)
 #else
 			else
diff --git a/ubifs-utils/mkfs.ubifs/mkfs.ubifs.h b/ubifs-utils/mkfs.ubifs/mkfs.ubifs.h
index 1321191..2097210 100644
--- a/ubifs-utils/mkfs.ubifs/mkfs.ubifs.h
+++ b/ubifs-utils/mkfs.ubifs/mkfs.ubifs.h
@@ -73,6 +73,9 @@
 #if MKFS_UBIFS_COMPR_ZLIB != UBIFS_COMPR_ZLIB
 #error MKFS_UBIFS_COMPR_ZLIB != UBIFS_COMPR_ZLIB
 #endif
+#if MKFS_UBIFS_COMPR_ZSTD != UBIFS_COMPR_ZSTD
+#error MKFS_UBIFS_COMPR_ZSTD != UBIFS_COMPR_ZSTD
+#endif
 
 extern int verbose;
 extern int debug_level;

[-- Attachment #3: ztsd.patch --]
[-- Type: text/x-patch, Size: 3607 bytes --]

From e4e484259296b1cfbaf14fe3f403df3b4f0b0849 Mon Sep 17 00:00:00 2001
From: Michele Dionisio <michele.dionisio@powersoft.com>
Date: Wed, 6 Jun 2018 14:53:50 +0200
Subject: Add zstd compression on ubifs

Signed-off-by: Michele Dionisio <michele.dionisio@gmail.com>

diff --git a/fs/ubifs/Kconfig b/fs/ubifs/Kconfig
index 83a961bf7280..1f44a6ff99cf 100644
--- a/fs/ubifs/Kconfig
+++ b/fs/ubifs/Kconfig
@@ -5,8 +5,10 @@ config UBIFS_FS
 	select CRYPTO if UBIFS_FS_ADVANCED_COMPR
 	select CRYPTO if UBIFS_FS_LZO
 	select CRYPTO if UBIFS_FS_ZLIB
+	select CRYPTO if UBIFS_FS_ZSTD
 	select CRYPTO_LZO if UBIFS_FS_LZO
 	select CRYPTO_DEFLATE if UBIFS_FS_ZLIB
+	select CRYPTO_ZSTD if UBIFS_FS_ZSTD
 	depends on MTD_UBI
 	help
 	  UBIFS is a file system for flash devices which works on top of UBI.
@@ -36,6 +38,14 @@ config UBIFS_FS_ZLIB
 	help
 	  Zlib compresses better than LZO but it is slower. Say 'Y' if unsure.
 
+config UBIFS_FS_ZSTD
+	bool "ZSTD compression support" if UBIFS_FS_ADVANCED_COMPR
+	depends on UBIFS_FS
+	default y
+	help
+	  ZSTD compresses is a big win in speed over Zlib and
+	  in compression ratio over LZO. Say 'Y' if unsure.
+
 config UBIFS_ATIME_SUPPORT
 	bool "Access time support" if UBIFS_FS
 	depends on UBIFS_FS
diff --git a/fs/ubifs/compress.c b/fs/ubifs/compress.c
index 565cb56d7225..89183aeeeb7a 100644
--- a/fs/ubifs/compress.c
+++ b/fs/ubifs/compress.c
@@ -71,6 +71,24 @@ static struct ubifs_compressor zlib_compr = {
 };
 #endif
 
+#ifdef CONFIG_UBIFS_FS_ZSTD
+static DEFINE_MUTEX(zstd_enc_mutex);
+static DEFINE_MUTEX(zstd_dec_mutex);
+
+static struct ubifs_compressor zstd_compr = {
+	.compr_type = UBIFS_COMPR_ZSTD,
+	.comp_mutex = &zstd_enc_mutex,
+	.decomp_mutex = &zstd_dec_mutex,
+	.name = "zstd",
+	.capi_name = "zstd",
+};
+#else
+static struct ubifs_compressor zstd_compr = {
+	.compr_type = UBIFS_COMPR_ZSTD,
+	.name = "zstd",
+};
+#endif
+
 /* All UBIFS compressors */
 struct ubifs_compressor *ubifs_compressors[UBIFS_COMPR_TYPES_CNT];
 
@@ -228,13 +246,19 @@ int __init ubifs_compressors_init(void)
 	if (err)
 		return err;
 
-	err = compr_init(&zlib_compr);
+	err = compr_init(&zstd_compr);
 	if (err)
 		goto out_lzo;
 
+	err = compr_init(&zlib_compr);
+	if (err)
+		goto out_zstd;
+
 	ubifs_compressors[UBIFS_COMPR_NONE] = &none_compr;
 	return 0;
 
+out_zstd:
+	compr_exit(&zstd_compr);
 out_lzo:
 	compr_exit(&lzo_compr);
 	return err;
@@ -247,4 +271,5 @@ void ubifs_compressors_exit(void)
 {
 	compr_exit(&lzo_compr);
 	compr_exit(&zlib_compr);
+	compr_exit(&zstd_compr);
 }
diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c
index 6c397a389105..2a725c8174b4 100644
--- a/fs/ubifs/super.c
+++ b/fs/ubifs/super.c
@@ -1035,6 +1035,8 @@ static int ubifs_parse_options(struct ubifs_info *c, char *options,
 				c->mount_opts.compr_type = UBIFS_COMPR_LZO;
 			else if (!strcmp(name, "zlib"))
 				c->mount_opts.compr_type = UBIFS_COMPR_ZLIB;
+			else if (!strcmp(name, "zstd"))
+				c->mount_opts.compr_type = UBIFS_COMPR_ZSTD;
 			else {
 				ubifs_err(c, "unknown compressor \"%s\"", name); //FIXME: is c ready?
 				kfree(name);
diff --git a/fs/ubifs/ubifs-media.h b/fs/ubifs/ubifs-media.h
index e8c23c9d4f4a..9f869c7c1532 100644
--- a/fs/ubifs/ubifs-media.h
+++ b/fs/ubifs/ubifs-media.h
@@ -341,12 +341,14 @@ enum {
  * UBIFS_COMPR_NONE: no compression
  * UBIFS_COMPR_LZO: LZO compression
  * UBIFS_COMPR_ZLIB: ZLIB compression
+ * UBIFS_COMPR_ZSTD: ZSTD compression
  * UBIFS_COMPR_TYPES_CNT: count of supported compression types
  */
 enum {
 	UBIFS_COMPR_NONE,
 	UBIFS_COMPR_LZO,
 	UBIFS_COMPR_ZLIB,
+	UBIFS_COMPR_ZSTD,
 	UBIFS_COMPR_TYPES_CNT,
 };
 

Re: testing UBIFS with zstd

[Richard Weinberger]

On Wed, Jun 6, 2018 at 2:55 PM, Michele Dionisio
<michele.dionisio@gmail.com> wrote:
> I'm testing ubifs with zstd compression. In my opinion it is a good
> idea to start to insert this "new" compression algorithem because the
> performance is paragonable with LZ4 and the compression is much
> better.

Numbers, please. :-)
Also see:
http://lists.infradead.org/pipermail/linux-mtd/2018-January/078742.html

-- 
Thanks,
//richard

Re: testing UBIFS with zstd

[Steve deRosier]

Hi Michele,

On Wed, Jun 6, 2018 at 5:56 AM Michele Dionisio
<michele.dionisio@gmail.com> wrote:
>
> I'm testing ubifs with zstd compression. In my opinion it is a good
> idea to start to insert this "new" compression algorithem because the
> performance is paragonable with LZ4 and the compression is much
> better.
>
> I attach patches for mtd-utils and linux too

When you send patches upstream, please send them in-line as plain text
in your email instead of as attachments. They can't be reviewed and
commented on as blobs.
https://www.kernel.org/doc/html/v4.17/process/submitting-patches.html#no-mime-no-links-no-compression-no-attachments-just-plain-text

Thanks,
- Steve

Re: testing UBIFS with zstd

[Michele Dionisio]

I have 2 partitions on same NAND. one with LZ4 one with ztsd. Mount option are:

root@flexaT:/# mount
rootfs on / type rootfs (rw)
ubi0:data on /opt/device/data type ubifs (rw,noatime,bulk_read)
ubi1:spare on /opt/device/real_spare type ubifs (rw,noatime,bulk_read)

I execute the following sequence of command for test

INFILE="/dev/zero"
rm -f /tmp/testfile
dd if=${INFILE} of=/tmp/testfile bs=16k count=1k
rm -f /opt/device/data/test
df /opt/device/data
sysctl -w vm.drop_caches=3
time dd if=/tmp/testfile of=/opt/device/data/test bs=64k count=1k
sync
sysctl -w vm.drop_caches=3
time dd if=/opt/device/data/test of=/dev/null bs=64k
df /opt/device/data
rm -f /opt/device/data/test
rm -f /opt/device/real_spare/test
df /opt/device/real_spare
sysctl -w vm.drop_caches=3
time dd if=/tmp/testfile of=/opt/device/real_spare/test bs=64k count=1k
sync
sysctl -w vm.drop_caches=3
time dd if=/opt/device/real_spare/test of=/dev/null bs=64k
df /opt/device/real_spare
rm -f /opt/device/real_spare/test
rm -f /tmp/testfile

I have that the test 2 time using INFILE="/dev/zero" and
INFILE="/dev/urandom". Both test is not realistic because zeros is too
much comprimable and random is not comprimibile. The result is:

----------------------------------------------------------------------------------------
COMPRESSION           |  time write    | time read   |   space used
LZO (/dev/zero)             |  2.39          | 1.40        |   624 blocks
ZSTD (/dev/zero)           |  2.63          | 1.43        |   596 blocks
LZO (/dev/urandom)    |  13.38         | 5.88        |   16024 blocks
ZSTD (/dev/urandom)   |  7.89          | 5.83        |   15896 blocks

to have an idea about compression I have try to use one sqlite3 db and
the size of the same file on both database is:
LZO : 2688 bloks
ZSTD: 1356 bloks


Il giorno mer 6 giu 2018 alle ore 15:06 Richard Weinberger
<richard.weinberger@gmail.com> ha scritto:
>
> On Wed, Jun 6, 2018 at 2:55 PM, Michele Dionisio
> <michele.dionisio@gmail.com> wrote:
> > I'm testing ubifs with zstd compression. In my opinion it is a good
> > idea to start to insert this "new" compression algorithem because the
> > performance is paragonable with LZ4 and the compression is much
> > better.
>
> Numbers, please. :-)
> Also see:
> http://lists.infradead.org/pipermail/linux-mtd/2018-January/078742.html
>
> --
> Thanks,
> //richard

Re: testing UBIFS with zstd

[Richard Weinberger]

Michele,

Am Mittwoch, 6. Juni 2018, 18:01:17 CEST schrieb Michele Dionisio:
> I have 2 partitions on same NAND. one with LZ4 one with ztsd. Mount option are:

LZ4 or LZO?
Below you write LZO...
 
> root@flexaT:/# mount
> rootfs on / type rootfs (rw)
> ubi0:data on /opt/device/data type ubifs (rw,noatime,bulk_read)
> ubi1:spare on /opt/device/real_spare type ubifs (rw,noatime,bulk_read)
> 
> I execute the following sequence of command for test
> 
> INFILE="/dev/zero"
> rm -f /tmp/testfile
> dd if=${INFILE} of=/tmp/testfile bs=16k count=1k
> rm -f /opt/device/data/test
> df /opt/device/data
> sysctl -w vm.drop_caches=3
> time dd if=/tmp/testfile of=/opt/device/data/test bs=64k count=1k
> sync
> sysctl -w vm.drop_caches=3
> time dd if=/opt/device/data/test of=/dev/null bs=64k
> df /opt/device/data
> rm -f /opt/device/data/test
> rm -f /opt/device/real_spare/test
> df /opt/device/real_spare
> sysctl -w vm.drop_caches=3
> time dd if=/tmp/testfile of=/opt/device/real_spare/test bs=64k count=1k
> sync
> sysctl -w vm.drop_caches=3
> time dd if=/opt/device/real_spare/test of=/dev/null bs=64k
> df /opt/device/real_spare
> rm -f /opt/device/real_spare/test
> rm -f /tmp/testfile
> 
> I have that the test 2 time using INFILE="/dev/zero" and
> INFILE="/dev/urandom". Both test is not realistic because zeros is too
> much comprimable and random is not comprimibile. The result is:
> 
> ----------------------------------------------------------------------------------------
> COMPRESSION           |  time write    | time read   |   space used
> LZO (/dev/zero)             |  2.39          | 1.40        |   624 blocks
> ZSTD (/dev/zero)           |  2.63          | 1.43        |   596 blocks
> LZO (/dev/urandom)    |  13.38         | 5.88        |   16024 blocks
> ZSTD (/dev/urandom)   |  7.89          | 5.83        |   15896 blocks

What scale is time? ms?
And what is block? LEBs?

Thanks,
//richard

-- 
sigma star gmbh - Eduard-Bodem-Gasse 6 - 6020 Innsbruck - Austria
ATU66964118 - FN 374287y