[PATCH 2/5] jffs2: Add LZO compression support to jffs2

[PATCH 2/5] jffs2: Add LZO compression support to jffs2

[Richard Purdie]

Add LZO1X compression/decompression support to jffs2.

LZO's interface doesn't entirely match that required by jffs2 so a 
buffer and memcpy is unavoidable.

Signed-off-by: Richard Purdie <rpurdie@openedhand.com>

---
 fs/Kconfig            |   10 ++++
 fs/jffs2/Makefile     |    1 
 fs/jffs2/compr.c      |    6 ++
 fs/jffs2/compr.h      |    3 -
 fs/jffs2/compr_lzo.c  |  120 ++++++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/jffs2.h |    1 
 6 files changed, 140 insertions(+), 1 deletion(-)

Index: linux/fs/Kconfig
===================================================================
--- linux.orig/fs/Kconfig	2007-02-28 18:12:17.000000000 +0000
+++ linux/fs/Kconfig	2007-02-28 18:13:10.000000000 +0000
@@ -1310,6 +1310,16 @@ config JFFS2_ZLIB
 
           Say 'Y' if unsure.
 
+config JFFS2_LZO
+	bool "JFFS2 LZO compression support" if JFFS2_COMPRESSION_OPTIONS
+	select LZO
+	depends on JFFS2_FS
+	default y
+        help
+          minilzo-based compression. Generally works better than Zlib.
+
+          Say 'Y' if unsure.
+
 config JFFS2_RTIME
 	bool "JFFS2 RTIME compression support" if JFFS2_COMPRESSION_OPTIONS
 	depends on JFFS2_FS
Index: linux/fs/jffs2/Makefile
===================================================================
--- linux.orig/fs/jffs2/Makefile	2007-02-28 18:12:17.000000000 +0000
+++ linux/fs/jffs2/Makefile	2007-02-28 18:12:31.000000000 +0000
@@ -18,4 +18,5 @@ jffs2-$(CONFIG_JFFS2_FS_POSIX_ACL)	+= ac
 jffs2-$(CONFIG_JFFS2_RUBIN)	+= compr_rubin.o
 jffs2-$(CONFIG_JFFS2_RTIME)	+= compr_rtime.o
 jffs2-$(CONFIG_JFFS2_ZLIB)	+= compr_zlib.o
+jffs2-$(CONFIG_JFFS2_LZO)	+= compr_lzo.o
 jffs2-$(CONFIG_JFFS2_SUMMARY)   += summary.o
Index: linux/fs/jffs2/compr.c
===================================================================
--- linux.orig/fs/jffs2/compr.c	2007-02-28 18:12:17.000000000 +0000
+++ linux/fs/jffs2/compr.c	2007-02-28 18:13:10.000000000 +0000
@@ -425,6 +425,9 @@ int __init jffs2_compressors_init(void)
         jffs2_rubinmips_init();
         jffs2_dynrubin_init();
 #endif
+#ifdef CONFIG_JFFS2_LZO
+        jffs2_lzo_init();
+#endif
 /* Setting default compression mode */
 #ifdef CONFIG_JFFS2_CMODE_NONE
         jffs2_compression_mode = JFFS2_COMPR_MODE_NONE;
@@ -443,6 +446,9 @@ int __init jffs2_compressors_init(void)
 int jffs2_compressors_exit(void)
 {
 /* Unregistering compressors */
+#ifdef CONFIG_JFFS2_LZO
+        jffs2_lzo_exit();
+#endif
 #ifdef CONFIG_JFFS2_RUBIN
         jffs2_dynrubin_exit();
         jffs2_rubinmips_exit();
Index: linux/fs/jffs2/compr_lzo.c
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/fs/jffs2/compr_lzo.c	2007-02-28 18:12:31.000000000 +0000
@@ -0,0 +1,120 @@
+/*
+ * JFFS2 LZO Compression Interface
+ *
+ * Copyright (C) 2007 Nokia Corporation. All rights reserved.
+ *
+ * Author: Richard Purdie <rpurdie@openedhand.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/init.h>
+#include <linux/lzo.h>
+#include "compr.h"
+
+static void *lzo_mem;
+static void *lzo_compress_buf;
+static DEFINE_MUTEX(deflate_mutex);
+
+static void free_workspace(void)
+{
+	vfree(lzo_mem);
+	vfree(lzo_compress_buf);
+}
+
+static int __init alloc_workspace(void)
+{
+	lzo_mem = vmalloc(LZO1X_MEM_COMPRESS);
+	lzo_compress_buf = vmalloc(lzo1x_worst_compress(PAGE_SIZE));
+
+	if (!lzo_mem || !lzo_compress_buf) {
+		printk(KERN_WARNING "Failed to allocate lzo deflate workspace\n");
+		free_workspace();
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int jffs2_lzo_compress(unsigned char *data_in, unsigned char *cpage_out,
+			      uint32_t *sourcelen, uint32_t *dstlen, void *model)
+{
+	unsigned long compress_size;
+	int ret;
+
+	mutex_lock(&deflate_mutex);
+	ret = lzo1x_1_compress(data_in, *sourcelen, lzo_compress_buf, &compress_size, lzo_mem);
+	mutex_unlock(&deflate_mutex);
+
+	if (ret != LZO_E_OK)
+		return -1;
+
+	if (compress_size > *dstlen)
+		return -1;
+
+	memcpy(cpage_out, lzo_compress_buf, compress_size);
+	*dstlen = compress_size;
+
+	return 0;
+}
+
+static int jffs2_lzo_decompress(unsigned char *data_in, unsigned char *cpage_out,
+				 uint32_t srclen, uint32_t destlen, void *model)
+{
+	unsigned long dl = destlen;
+	int ret;
+
+	ret = lzo1x_decompress_safe(data_in, srclen, cpage_out, &dl, NULL);
+
+	if (ret != LZO_E_OK || dl != destlen)
+		return -1;
+
+	return 0;
+}
+
+static struct jffs2_compressor jffs2_lzo_comp = {
+	.priority = JFFS2_LZO_PRIORITY,
+	.name = "lzo",
+	.compr = JFFS2_COMPR_LZO,
+	.compress = &jffs2_lzo_compress,
+	.decompress = &jffs2_lzo_decompress,
+	.disabled = 0,
+};
+
+int __init jffs2_lzo_init(void)
+{
+	int ret;
+
+	ret = alloc_workspace();
+	if (ret < 0)
+		return ret;
+
+	ret = jffs2_register_compressor(&jffs2_lzo_comp);
+	if (ret)
+		free_workspace();
+
+	return ret;
+}
+
+void jffs2_lzo_exit(void)
+{
+	jffs2_unregister_compressor(&jffs2_lzo_comp);
+	free_workspace();
+}
Index: linux/include/linux/jffs2.h
===================================================================
--- linux.orig/include/linux/jffs2.h	2007-02-28 18:12:17.000000000 +0000
+++ linux/include/linux/jffs2.h	2007-02-28 18:12:31.000000000 +0000
@@ -46,6 +46,7 @@
 #define JFFS2_COMPR_COPY	0x04
 #define JFFS2_COMPR_DYNRUBIN	0x05
 #define JFFS2_COMPR_ZLIB	0x06
+#define JFFS2_COMPR_LZO		0x07
 /* Compatibility flags. */
 #define JFFS2_COMPAT_MASK 0xc000      /* What do to if an unknown nodetype is found */
 #define JFFS2_NODE_ACCURATE 0x2000
Index: linux/fs/jffs2/compr.h
===================================================================
--- linux.orig/fs/jffs2/compr.h	2007-02-28 18:12:17.000000000 +0000
+++ linux/fs/jffs2/compr.h	2007-02-28 18:13:10.000000000 +0000
@@ -30,9 +30,10 @@
 #define JFFS2_RUBINMIPS_PRIORITY 10
 #define JFFS2_DYNRUBIN_PRIORITY  20
 #define JFFS2_LZARI_PRIORITY     30
-#define JFFS2_LZO_PRIORITY       40
 #define JFFS2_RTIME_PRIORITY     50
 #define JFFS2_ZLIB_PRIORITY      60
+#define JFFS2_LZO_PRIORITY       80
+
 
 #define JFFS2_RUBINMIPS_DISABLED /* RUBINs will be used only */
 #define JFFS2_DYNRUBIN_DISABLED  /*        for decompression */

Re: [PATCH 2/5] jffs2: Add LZO compression support to jffs2

[David Woodhouse]

On Wed, 2007-02-28 at 19:13 +0000, Richard Purdie wrote:
> Add LZO1X compression/decompression support to jffs2.
> 
> LZO's interface doesn't entirely match that required by jffs2 so a 
> buffer and memcpy is unavoidable.
> 
> Signed-off-by: Richard Purdie <rpurdie@openedhand.com>

> Index: linux/fs/jffs2/compr_lzo.c
> ===================================================================
> --- /dev/null	1970-01-01 00:00:00.000000000 +0000
> +++ linux/fs/jffs2/compr_lzo.c	2007-02-28 18:12:31.000000000 +0000
> @@ -0,0 +1,120 @@
> +/*
> + * JFFS2 LZO Compression Interface
> + *
> + * Copyright (C) 2007 Nokia Corporation. All rights reserved.
> + *
> + * Author: Richard Purdie <rpurdie@openedhand.com>
> + *
> + * This program is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU General Public License
> + * version 2 as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful, but
> + * WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
> + * 02110-1301 USA
> + *
> + */

Please use the same licence, and the same boilerplate, as the rest of
the JFFS2 code.

-- 
dwmw2

[PATCH 2/5] jffs2: Add LZO compression support to jffs2

[Richard Purdie]

Add LZO1X compression/decompression support to jffs2.

LZO's interface doesn't entirely match that required by jffs2 so a
buffer and memcpy is unavoidable.

Signed-off-by: Richard Purdie <rpurdie@openedhand.com>
---
 fs/Kconfig            |   10 ++++
 fs/jffs2/Makefile     |    1 +
 fs/jffs2/compr.c      |    6 +++
 fs/jffs2/compr.h      |    7 +++-
 fs/jffs2/compr_lzo.c  |  110 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/jffs2.h |    1 +
 6 files changed, 134 insertions(+), 1 deletions(-)

diff --git a/fs/Kconfig b/fs/Kconfig
index a42f767..1645dfa 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -1310,6 +1310,16 @@ config JFFS2_ZLIB
 
           Say 'Y' if unsure.
 
+config JFFS2_LZO
+	bool "JFFS2 LZO compression support" if JFFS2_COMPRESSION_OPTIONS
+	select LZO
+	depends on JFFS2_FS
+	default y
+        help
+          minilzo-based compression. Generally works better than Zlib.
+
+          Say 'Y' if unsure.
+
 config JFFS2_RTIME
 	bool "JFFS2 RTIME compression support" if JFFS2_COMPRESSION_OPTIONS
 	depends on JFFS2_FS
diff --git a/fs/jffs2/Makefile b/fs/jffs2/Makefile
index c32b241..60e5d49 100644
--- a/fs/jffs2/Makefile
+++ b/fs/jffs2/Makefile
@@ -17,4 +17,5 @@ jffs2-$(CONFIG_JFFS2_FS_POSIX_ACL)	+= acl.o
 jffs2-$(CONFIG_JFFS2_RUBIN)	+= compr_rubin.o
 jffs2-$(CONFIG_JFFS2_RTIME)	+= compr_rtime.o
 jffs2-$(CONFIG_JFFS2_ZLIB)	+= compr_zlib.o
+jffs2-$(CONFIG_JFFS2_LZO)	+= compr_lzo.o
 jffs2-$(CONFIG_JFFS2_SUMMARY)   += summary.o
diff --git a/fs/jffs2/compr.c b/fs/jffs2/compr.c
index 485d065..6a23408 100644
--- a/fs/jffs2/compr.c
+++ b/fs/jffs2/compr.c
@@ -285,6 +285,9 @@ int __init jffs2_compressors_init(void)
         jffs2_rubinmips_init();
         jffs2_dynrubin_init();
 #endif
+#ifdef CONFIG_JFFS2_LZO
+        jffs2_lzo_init();
+#endif
 /* Setting default compression mode */
 #ifdef CONFIG_JFFS2_CMODE_NONE
         jffs2_compression_mode = JFFS2_COMPR_MODE_NONE;
@@ -303,6 +306,9 @@ int __init jffs2_compressors_init(void)
 int jffs2_compressors_exit(void)
 {
 /* Unregistering compressors */
+#ifdef CONFIG_JFFS2_LZO
+        jffs2_lzo_exit();
+#endif
 #ifdef CONFIG_JFFS2_RUBIN
         jffs2_dynrubin_exit();
         jffs2_rubinmips_exit();
diff --git a/fs/jffs2/compr.h b/fs/jffs2/compr.h
index 68cc701..8c6b2af 100644
--- a/fs/jffs2/compr.h
+++ b/fs/jffs2/compr.h
@@ -27,9 +27,10 @@
 #define JFFS2_RUBINMIPS_PRIORITY 10
 #define JFFS2_DYNRUBIN_PRIORITY  20
 #define JFFS2_LZARI_PRIORITY     30
-#define JFFS2_LZO_PRIORITY       40
 #define JFFS2_RTIME_PRIORITY     50
 #define JFFS2_ZLIB_PRIORITY      60
+#define JFFS2_LZO_PRIORITY       80
+
 
 #define JFFS2_RUBINMIPS_DISABLED /* RUBINs will be used only */
 #define JFFS2_DYNRUBIN_DISABLED  /*        for decompression */
@@ -90,5 +91,9 @@ void jffs2_rtime_exit(void);
 int jffs2_zlib_init(void);
 void jffs2_zlib_exit(void);
 #endif
+#ifdef CONFIG_JFFS2_LZO
+int jffs2_lzo_init(void);
+void jffs2_lzo_exit(void);
+#endif
 
 #endif /* __JFFS2_COMPR_H__ */
diff --git a/fs/jffs2/compr_lzo.c b/fs/jffs2/compr_lzo.c
new file mode 100644
index 0000000..6396682
--- /dev/null
+++ b/fs/jffs2/compr_lzo.c
@@ -0,0 +1,110 @@
+/*
+ * JFFS2 -- Journalling Flash File System, Version 2.
+ *
+ * LZO Compression Interface
+ *
+ * Copyright © 2007 Nokia Corporation. All rights reserved.
+ *
+ * Created by Richard Purdie <rpurdie@openedhand.com>
+ *
+ * For licensing information, see the file 'LICENCE' in this directory.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/init.h>
+#include <linux/lzo.h>
+#include "compr.h"
+
+static void *lzo_mem;
+static void *lzo_compress_buf;
+static DEFINE_MUTEX(deflate_mutex);
+
+static void free_workspace(void)
+{
+	vfree(lzo_mem);
+	vfree(lzo_compress_buf);
+}
+
+static int __init alloc_workspace(void)
+{
+	lzo_mem = vmalloc(LZO1X_MEM_COMPRESS);
+	lzo_compress_buf = vmalloc(lzo1x_worst_compress(PAGE_SIZE));
+
+	if (!lzo_mem || !lzo_compress_buf) {
+		printk(KERN_WARNING "Failed to allocate lzo deflate workspace\n");
+		free_workspace();
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int jffs2_lzo_compress(unsigned char *data_in, unsigned char *cpage_out,
+			      uint32_t *sourcelen, uint32_t *dstlen, void *model)
+{
+	unsigned long compress_size;
+	int ret;
+
+	mutex_lock(&deflate_mutex);
+	ret = lzo1x_1_compress(data_in, *sourcelen, lzo_compress_buf, &compress_size, lzo_mem);
+	mutex_unlock(&deflate_mutex);
+
+	if (ret != LZO_E_OK)
+		return -1;
+
+	if (compress_size > *dstlen)
+		return -1;
+
+	memcpy(cpage_out, lzo_compress_buf, compress_size);
+	*dstlen = compress_size;
+
+	return 0;
+}
+
+static int jffs2_lzo_decompress(unsigned char *data_in, unsigned char *cpage_out,
+				 uint32_t srclen, uint32_t destlen, void *model)
+{
+	unsigned long dl = destlen;
+	int ret;
+
+	ret = lzo1x_decompress_safe(data_in, srclen, cpage_out, &dl, NULL);
+
+	if (ret != LZO_E_OK || dl != destlen)
+		return -1;
+
+	return 0;
+}
+
+static struct jffs2_compressor jffs2_lzo_comp = {
+	.priority = JFFS2_LZO_PRIORITY,
+	.name = "lzo",
+	.compr = JFFS2_COMPR_LZO,
+	.compress = &jffs2_lzo_compress,
+	.decompress = &jffs2_lzo_decompress,
+	.disabled = 0,
+};
+
+int __init jffs2_lzo_init(void)
+{
+	int ret;
+
+	ret = alloc_workspace();
+	if (ret < 0)
+		return ret;
+
+	ret = jffs2_register_compressor(&jffs2_lzo_comp);
+	if (ret)
+		free_workspace();
+
+	return ret;
+}
+
+void jffs2_lzo_exit(void)
+{
+	jffs2_unregister_compressor(&jffs2_lzo_comp);
+	free_workspace();
+}
diff --git a/include/linux/jffs2.h b/include/linux/jffs2.h
index 840631f..6b563ca 100644
--- a/include/linux/jffs2.h
+++ b/include/linux/jffs2.h
@@ -46,6 +46,7 @@
 #define JFFS2_COMPR_COPY	0x04
 #define JFFS2_COMPR_DYNRUBIN	0x05
 #define JFFS2_COMPR_ZLIB	0x06
+#define JFFS2_COMPR_LZO		0x07
 /* Compatibility flags. */
 #define JFFS2_COMPAT_MASK 0xc000      /* What do to if an unknown nodetype is found */
 #define JFFS2_NODE_ACCURATE 0x2000

Re: [PATCH 2/5] jffs2: Add LZO compression support to jffs2

[Randy Dunlap]

On Tue, 01 May 2007 15:47:03 +0100 Richard Purdie wrote:

> Add LZO1X compression/decompression support to jffs2.
> 
> LZO's interface doesn't entirely match that required by jffs2 so a
> buffer and memcpy is unavoidable.
> 
> Signed-off-by: Richard Purdie <rpurdie@openedhand.com>
> ---
> 
> diff --git a/fs/Kconfig b/fs/Kconfig
> index a42f767..1645dfa 100644
> --- a/fs/Kconfig
> +++ b/fs/Kconfig
> @@ -1310,6 +1310,16 @@ config JFFS2_ZLIB
>  
>            Say 'Y' if unsure.
>  
> +config JFFS2_LZO
> +	bool "JFFS2 LZO compression support" if JFFS2_COMPRESSION_OPTIONS
> +	select LZO
> +	depends on JFFS2_FS
> +	default y
> +        help
> +          minilzo-based compression. Generally works better than Zlib.
> +
> +          Say 'Y' if unsure.

Use <tab> on the "help" line (instead of spaces) and
use <tab><space><space> on the help text lines.

>  config JFFS2_RTIME
>  	bool "JFFS2 RTIME compression support" if JFFS2_COMPRESSION_OPTIONS
>  	depends on JFFS2_FS

---
~Randy
*** Remember to use Documentation/SubmitChecklist when testing your code ***

Re: [PATCH 2/5] jffs2: Add LZO compression support to jffs2

[Satyam Sharma]

Hi Richard,

On 5/1/07, Richard Purdie <rpurdie@openedhand.com> wrote:
> Add LZO1X compression/decompression support to jffs2.
>
> LZO's interface doesn't entirely match that required by jffs2 so a
> buffer and memcpy is unavoidable.
>
> Signed-off-by: Richard Purdie <rpurdie@openedhand.com>
> ---
> [...]
> +++ b/fs/jffs2/compr_lzo.c
> [...]
> +static void *lzo_mem;
> +static void *lzo_compress_buf;
> +static DEFINE_MUTEX(deflate_mutex);
> +
> +static void free_workspace(void)
> +{
> +       vfree(lzo_mem);
> +       vfree(lzo_compress_buf);
> +}
> +
> +static int __init alloc_workspace(void)
> +{
> +       lzo_mem = vmalloc(LZO1X_MEM_COMPRESS);
> +       lzo_compress_buf = vmalloc(lzo1x_worst_compress(PAGE_SIZE));
> +
> +       if (!lzo_mem || !lzo_compress_buf) {
> +               printk(KERN_WARNING "Failed to allocate lzo deflate workspace\n");
> +               free_workspace();
> +               return -ENOMEM;
> +       }
> +
> +       return 0;
> +}
> +
> +static int jffs2_lzo_compress(unsigned char *data_in, unsigned char *cpage_out,
> +                             uint32_t *sourcelen, uint32_t *dstlen, void *model)
> +{
> +       unsigned long compress_size;
> +       int ret;
> +
> +       mutex_lock(&deflate_mutex);
> +       ret = lzo1x_1_compress(data_in, *sourcelen, lzo_compress_buf, &compress_size, lzo_mem);
> +       mutex_unlock(&deflate_mutex);

Considering we do have to memcpy() the entire compressed result to the
destination output buffer later anyway (note that
fs/jffs2/compr_zlib.c doesn't need to do that), do we really gain much
by avoiding vmalloc() and vfree() in jffs2_lzo_compress() itself and
keeping the workspace buffers pre-allocated? I ask because I always
found these global static workspace buffers ugly, and all the
associated code + mutex could go away if we make them local to
jffs2_lzo_compress() -- as long as it doesn't hurt performance
terribly, of course.

Thanks,
Satyam

Re: [PATCH 2/5] jffs2: Add LZO compression support to jffs2

[Richard Purdie]

On Fri, 2007-05-04 at 14:36 +0530, Satyam Sharma wrote:
> On 5/1/07, Richard Purdie <rpurdie@openedhand.com> wrote:
> > +++ b/fs/jffs2/compr_lzo.c
> > [...]
> > +static void *lzo_mem;
> > +static void *lzo_compress_buf;
> > +static DEFINE_MUTEX(deflate_mutex);
> > +
> > +static void free_workspace(void)
> > +{
> > +       vfree(lzo_mem);
> > +       vfree(lzo_compress_buf);
> > +}
> > +
> > +static int __init alloc_workspace(void)
> > +{
> > +       lzo_mem = vmalloc(LZO1X_MEM_COMPRESS);
> > +       lzo_compress_buf = vmalloc(lzo1x_worst_compress(PAGE_SIZE));
> > +
> > +       if (!lzo_mem || !lzo_compress_buf) {
> > +               printk(KERN_WARNING "Failed to allocate lzo deflate workspace\n");
> > +               free_workspace();
> > +               return -ENOMEM;
> > +       }
> > +
> > +       return 0;
> > +}
> > +
> > +static int jffs2_lzo_compress(unsigned char *data_in, unsigned char *cpage_out,
> > +                             uint32_t *sourcelen, uint32_t *dstlen, void *model)
> > +{
> > +       unsigned long compress_size;
> > +       int ret;
> > +
> > +       mutex_lock(&deflate_mutex);
> > +       ret = lzo1x_1_compress(data_in, *sourcelen, lzo_compress_buf, &compress_size, lzo_mem);
> > +       mutex_unlock(&deflate_mutex);
> 
> Considering we do have to memcpy() the entire compressed result to the
> destination output buffer later anyway (note that
> fs/jffs2/compr_zlib.c doesn't need to do that), do we really gain much
> by avoiding vmalloc() and vfree() in jffs2_lzo_compress() itself and
> keeping the workspace buffers pre-allocated? I ask because I always
> found these global static workspace buffers ugly, and all the
> associated code + mutex could go away if we make them local to
> jffs2_lzo_compress() -- as long as it doesn't hurt performance
> terribly, of course.

memcpy is relatively fast and I'd expect continually allocing and
freeing buffers to have a significant overhead compared with that and
impact performance. The current approach means you don't get ENOMEM
errors in the uncompress/compress paths either.

Regards,

Richard

[PATCH 2/5] jffs2: Add LZO compression support to jffs2

[Richard Purdie]

Add LZO1X compression/decompression support to jffs2.

LZO's interface doesn't entirely match that required by jffs2 so a
buffer and memcpy is unavoidable.

Signed-off-by: Richard Purdie <rpurdie@openedhand.com>
---
 fs/Kconfig            |   11 +++++
 fs/jffs2/Makefile     |    1 +
 fs/jffs2/compr.c      |    6 +++
 fs/jffs2/compr.h      |    7 +++-
 fs/jffs2/compr_lzo.c  |  110 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/jffs2.h |    1 +
 6 files changed, 135 insertions(+), 1 deletions(-)

diff --git a/fs/Kconfig b/fs/Kconfig
index a42f767..421c821 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -1310,6 +1310,17 @@ config JFFS2_ZLIB
 
           Say 'Y' if unsure.
 
+config JFFS2_LZO
+	bool "JFFS2 LZO compression support" if JFFS2_COMPRESSION_OPTIONS
+	select LZO
+	depends on JFFS2_FS
+	default y
+	help
+	  LZO based compression. Generally works faster than Zlib with 
+	  a slightly lower compression ratio.
+
+	  Say 'Y' if unsure.
+
 config JFFS2_RTIME
 	bool "JFFS2 RTIME compression support" if JFFS2_COMPRESSION_OPTIONS
 	depends on JFFS2_FS
diff --git a/fs/jffs2/Makefile b/fs/jffs2/Makefile
index c32b241..60e5d49 100644
--- a/fs/jffs2/Makefile
+++ b/fs/jffs2/Makefile
@@ -17,4 +17,5 @@ jffs2-$(CONFIG_JFFS2_FS_POSIX_ACL)	+= acl.o
 jffs2-$(CONFIG_JFFS2_RUBIN)	+= compr_rubin.o
 jffs2-$(CONFIG_JFFS2_RTIME)	+= compr_rtime.o
 jffs2-$(CONFIG_JFFS2_ZLIB)	+= compr_zlib.o
+jffs2-$(CONFIG_JFFS2_LZO)	+= compr_lzo.o
 jffs2-$(CONFIG_JFFS2_SUMMARY)   += summary.o
diff --git a/fs/jffs2/compr.c b/fs/jffs2/compr.c
index 485d065..6a23408 100644
--- a/fs/jffs2/compr.c
+++ b/fs/jffs2/compr.c
@@ -285,6 +285,9 @@ int __init jffs2_compressors_init(void)
         jffs2_rubinmips_init();
         jffs2_dynrubin_init();
 #endif
+#ifdef CONFIG_JFFS2_LZO
+        jffs2_lzo_init();
+#endif
 /* Setting default compression mode */
 #ifdef CONFIG_JFFS2_CMODE_NONE
         jffs2_compression_mode = JFFS2_COMPR_MODE_NONE;
@@ -303,6 +306,9 @@ int __init jffs2_compressors_init(void)
 int jffs2_compressors_exit(void)
 {
 /* Unregistering compressors */
+#ifdef CONFIG_JFFS2_LZO
+        jffs2_lzo_exit();
+#endif
 #ifdef CONFIG_JFFS2_RUBIN
         jffs2_dynrubin_exit();
         jffs2_rubinmips_exit();
diff --git a/fs/jffs2/compr.h b/fs/jffs2/compr.h
index 68cc701..8c6b2af 100644
--- a/fs/jffs2/compr.h
+++ b/fs/jffs2/compr.h
@@ -27,9 +27,10 @@
 #define JFFS2_RUBINMIPS_PRIORITY 10
 #define JFFS2_DYNRUBIN_PRIORITY  20
 #define JFFS2_LZARI_PRIORITY     30
-#define JFFS2_LZO_PRIORITY       40
 #define JFFS2_RTIME_PRIORITY     50
 #define JFFS2_ZLIB_PRIORITY      60
+#define JFFS2_LZO_PRIORITY       80
+
 
 #define JFFS2_RUBINMIPS_DISABLED /* RUBINs will be used only */
 #define JFFS2_DYNRUBIN_DISABLED  /*        for decompression */
@@ -90,5 +91,9 @@ void jffs2_rtime_exit(void);
 int jffs2_zlib_init(void);
 void jffs2_zlib_exit(void);
 #endif
+#ifdef CONFIG_JFFS2_LZO
+int jffs2_lzo_init(void);
+void jffs2_lzo_exit(void);
+#endif
 
 #endif /* __JFFS2_COMPR_H__ */
diff --git a/fs/jffs2/compr_lzo.c b/fs/jffs2/compr_lzo.c
new file mode 100644
index 0000000..6396682
--- /dev/null
+++ b/fs/jffs2/compr_lzo.c
@@ -0,0 +1,110 @@
+/*
+ * JFFS2 -- Journalling Flash File System, Version 2.
+ *
+ * LZO Compression Interface
+ *
+ * Copyright © 2007 Nokia Corporation. All rights reserved.
+ *
+ * Created by Richard Purdie <rpurdie@openedhand.com>
+ *
+ * For licensing information, see the file 'LICENCE' in this directory.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/init.h>
+#include <linux/lzo.h>
+#include "compr.h"
+
+static void *lzo_mem;
+static void *lzo_compress_buf;
+static DEFINE_MUTEX(deflate_mutex);
+
+static void free_workspace(void)
+{
+	vfree(lzo_mem);
+	vfree(lzo_compress_buf);
+}
+
+static int __init alloc_workspace(void)
+{
+	lzo_mem = vmalloc(LZO1X_MEM_COMPRESS);
+	lzo_compress_buf = vmalloc(lzo1x_worst_compress(PAGE_SIZE));
+
+	if (!lzo_mem || !lzo_compress_buf) {
+		printk(KERN_WARNING "Failed to allocate lzo deflate workspace\n");
+		free_workspace();
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int jffs2_lzo_compress(unsigned char *data_in, unsigned char *cpage_out,
+			      uint32_t *sourcelen, uint32_t *dstlen, void *model)
+{
+	unsigned long compress_size;
+	int ret;
+
+	mutex_lock(&deflate_mutex);
+	ret = lzo1x_1_compress(data_in, *sourcelen, lzo_compress_buf, &compress_size, lzo_mem);
+	mutex_unlock(&deflate_mutex);
+
+	if (ret != LZO_E_OK)
+		return -1;
+
+	if (compress_size > *dstlen)
+		return -1;
+
+	memcpy(cpage_out, lzo_compress_buf, compress_size);
+	*dstlen = compress_size;
+
+	return 0;
+}
+
+static int jffs2_lzo_decompress(unsigned char *data_in, unsigned char *cpage_out,
+				 uint32_t srclen, uint32_t destlen, void *model)
+{
+	unsigned long dl = destlen;
+	int ret;
+
+	ret = lzo1x_decompress_safe(data_in, srclen, cpage_out, &dl, NULL);
+
+	if (ret != LZO_E_OK || dl != destlen)
+		return -1;
+
+	return 0;
+}
+
+static struct jffs2_compressor jffs2_lzo_comp = {
+	.priority = JFFS2_LZO_PRIORITY,
+	.name = "lzo",
+	.compr = JFFS2_COMPR_LZO,
+	.compress = &jffs2_lzo_compress,
+	.decompress = &jffs2_lzo_decompress,
+	.disabled = 0,
+};
+
+int __init jffs2_lzo_init(void)
+{
+	int ret;
+
+	ret = alloc_workspace();
+	if (ret < 0)
+		return ret;
+
+	ret = jffs2_register_compressor(&jffs2_lzo_comp);
+	if (ret)
+		free_workspace();
+
+	return ret;
+}
+
+void jffs2_lzo_exit(void)
+{
+	jffs2_unregister_compressor(&jffs2_lzo_comp);
+	free_workspace();
+}
diff --git a/include/linux/jffs2.h b/include/linux/jffs2.h
index 840631f..6b563ca 100644
--- a/include/linux/jffs2.h
+++ b/include/linux/jffs2.h
@@ -46,6 +46,7 @@
 #define JFFS2_COMPR_COPY	0x04
 #define JFFS2_COMPR_DYNRUBIN	0x05
 #define JFFS2_COMPR_ZLIB	0x06
+#define JFFS2_COMPR_LZO		0x07
 /* Compatibility flags. */
 #define JFFS2_COMPAT_MASK 0xc000      /* What do to if an unknown nodetype is found */
 #define JFFS2_NODE_ACCURATE 0x2000

Re: [PATCH 2/5] jffs2: Add LZO compression support to jffs2

[David Woodhouse]

On Fri, 2007-05-04 at 12:30 +0100, Richard Purdie wrote:
> +config JFFS2_LZO
> +       bool "JFFS2 LZO compression support" if JFFS2_COMPRESSION_OPTIONS
> +       select LZO 

Please make it _depend_ on LZO instead. If you do that, I can merge the
JFFS2 bits in anticipation of the core LZO code going upstream.

-- 
dwmw2

Re: [PATCH 2/5] jffs2: Add LZO compression support to jffs2

[Haavard Skinnemoen]

On Fri, 04 May 2007 12:35:32 +0100
David Woodhouse <dwmw2@infradead.org> wrote:

> On Fri, 2007-05-04 at 12:30 +0100, Richard Purdie wrote:
> > +config JFFS2_LZO
> > +       bool "JFFS2 LZO compression support" if JFFS2_COMPRESSION_OPTIONS
> > +       select LZO   
> 
> Please make it _depend_ on LZO instead. If you do that, I can merge the
> JFFS2 bits in anticipation of the core LZO code going upstream.

Hmm...everything else in lib/Kconfig is supposed to be _selected_, not
depended upon. And the only three items in there which can actually be
selected by the user all begin their helptexts with

"This option is provided for the case where no in-kernel-tree modules
require FOO functions".

Are you sure it's a good idea to break with existing practice? I'm
concerned that nobody will ever find the JFFS2_LZO option if it means
they will have to go all the way down to the "Library functions" menu
and select something to make the option visible...

Haavard

Re: [PATCH 2/5] jffs2: Add LZO compression support to jffs2

[David Woodhouse]

On Fri, 2007-05-04 at 14:41 +0200, Haavard Skinnemoen wrote:
> 
> Hmm...everything else in lib/Kconfig is supposed to be _selected_, not
> depended upon. And the only three items in there which can actually be
> selected by the user all begin their helptexts with
> 
> "This option is provided for the case where no in-kernel-tree modules
> require FOO functions".
> 
> Are you sure it's a good idea to break with existing practice? I'm
> concerned that nobody will ever find the JFFS2_LZO option if it means
> they will have to go all the way down to the "Library functions" menu
> and select something to make the option visible...

That would seem be a tools problem.

Nevertheless, if I merge it with 'select', won't it break while the core
LZO patches aren't yet upstream?

Of course, one could argue that that's also a tools problem. All this
proliferation of 'select' is doing is moving problems around rather than
solving them.

-- 
dwmw2

[PATCH 2/5] jffs2: Add LZO compression support to jffs2

[Richard Purdie]

Add LZO1X compression/decompression support to jffs2.

LZO's interface doesn't entirely match that required by jffs2 so a
buffer and memcpy is unavoidable.

Signed-off-by: Richard Purdie <rpurdie@openedhand.com>
---
 fs/Kconfig            |   10 ++++
 fs/jffs2/Makefile     |    1 +
 fs/jffs2/compr.c      |    6 +++
 fs/jffs2/compr.h      |    7 +++-
 fs/jffs2/compr_lzo.c  |  110 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/jffs2.h |    1 +
 6 files changed, 134 insertions(+), 1 deletions(-)

diff --git a/fs/Kconfig b/fs/Kconfig
index a42f767..6e7095f 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -1310,6 +1310,16 @@ config JFFS2_ZLIB
 
           Say 'Y' if unsure.
 
+config JFFS2_LZO
+	bool "JFFS2 LZO compression support" if JFFS2_COMPRESSION_OPTIONS
+	depends on JFFS2_FS && LZO
+	default y
+	help
+	  LZO based compression. Generally works faster than Zlib with 
+	  a slightly lower compression ratio.
+
+	  Say 'Y' if unsure.
+
 config JFFS2_RTIME
 	bool "JFFS2 RTIME compression support" if JFFS2_COMPRESSION_OPTIONS
 	depends on JFFS2_FS
diff --git a/fs/jffs2/Makefile b/fs/jffs2/Makefile
index c32b241..60e5d49 100644
--- a/fs/jffs2/Makefile
+++ b/fs/jffs2/Makefile
@@ -17,4 +17,5 @@ jffs2-$(CONFIG_JFFS2_FS_POSIX_ACL)	+= acl.o
 jffs2-$(CONFIG_JFFS2_RUBIN)	+= compr_rubin.o
 jffs2-$(CONFIG_JFFS2_RTIME)	+= compr_rtime.o
 jffs2-$(CONFIG_JFFS2_ZLIB)	+= compr_zlib.o
+jffs2-$(CONFIG_JFFS2_LZO)	+= compr_lzo.o
 jffs2-$(CONFIG_JFFS2_SUMMARY)   += summary.o
diff --git a/fs/jffs2/compr.c b/fs/jffs2/compr.c
index 485d065..6a23408 100644
--- a/fs/jffs2/compr.c
+++ b/fs/jffs2/compr.c
@@ -285,6 +285,9 @@ int __init jffs2_compressors_init(void)
         jffs2_rubinmips_init();
         jffs2_dynrubin_init();
 #endif
+#ifdef CONFIG_JFFS2_LZO
+        jffs2_lzo_init();
+#endif
 /* Setting default compression mode */
 #ifdef CONFIG_JFFS2_CMODE_NONE
         jffs2_compression_mode = JFFS2_COMPR_MODE_NONE;
@@ -303,6 +306,9 @@ int __init jffs2_compressors_init(void)
 int jffs2_compressors_exit(void)
 {
 /* Unregistering compressors */
+#ifdef CONFIG_JFFS2_LZO
+        jffs2_lzo_exit();
+#endif
 #ifdef CONFIG_JFFS2_RUBIN
         jffs2_dynrubin_exit();
         jffs2_rubinmips_exit();
diff --git a/fs/jffs2/compr.h b/fs/jffs2/compr.h
index 68cc701..8c6b2af 100644
--- a/fs/jffs2/compr.h
+++ b/fs/jffs2/compr.h
@@ -27,9 +27,10 @@
 #define JFFS2_RUBINMIPS_PRIORITY 10
 #define JFFS2_DYNRUBIN_PRIORITY  20
 #define JFFS2_LZARI_PRIORITY     30
-#define JFFS2_LZO_PRIORITY       40
 #define JFFS2_RTIME_PRIORITY     50
 #define JFFS2_ZLIB_PRIORITY      60
+#define JFFS2_LZO_PRIORITY       80
+
 
 #define JFFS2_RUBINMIPS_DISABLED /* RUBINs will be used only */
 #define JFFS2_DYNRUBIN_DISABLED  /*        for decompression */
@@ -90,5 +91,9 @@ void jffs2_rtime_exit(void);
 int jffs2_zlib_init(void);
 void jffs2_zlib_exit(void);
 #endif
+#ifdef CONFIG_JFFS2_LZO
+int jffs2_lzo_init(void);
+void jffs2_lzo_exit(void);
+#endif
 
 #endif /* __JFFS2_COMPR_H__ */
diff --git a/fs/jffs2/compr_lzo.c b/fs/jffs2/compr_lzo.c
new file mode 100644
index 0000000..6396682
--- /dev/null
+++ b/fs/jffs2/compr_lzo.c
@@ -0,0 +1,110 @@
+/*
+ * JFFS2 -- Journalling Flash File System, Version 2.
+ *
+ * LZO Compression Interface
+ *
+ * Copyright © 2007 Nokia Corporation. All rights reserved.
+ *
+ * Created by Richard Purdie <rpurdie@openedhand.com>
+ *
+ * For licensing information, see the file 'LICENCE' in this directory.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/init.h>
+#include <linux/lzo.h>
+#include "compr.h"
+
+static void *lzo_mem;
+static void *lzo_compress_buf;
+static DEFINE_MUTEX(deflate_mutex);
+
+static void free_workspace(void)
+{
+	vfree(lzo_mem);
+	vfree(lzo_compress_buf);
+}
+
+static int __init alloc_workspace(void)
+{
+	lzo_mem = vmalloc(LZO1X_MEM_COMPRESS);
+	lzo_compress_buf = vmalloc(lzo1x_worst_compress(PAGE_SIZE));
+
+	if (!lzo_mem || !lzo_compress_buf) {
+		printk(KERN_WARNING "Failed to allocate lzo deflate workspace\n");
+		free_workspace();
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int jffs2_lzo_compress(unsigned char *data_in, unsigned char *cpage_out,
+			      uint32_t *sourcelen, uint32_t *dstlen, void *model)
+{
+	unsigned long compress_size;
+	int ret;
+
+	mutex_lock(&deflate_mutex);
+	ret = lzo1x_1_compress(data_in, *sourcelen, lzo_compress_buf, &compress_size, lzo_mem);
+	mutex_unlock(&deflate_mutex);
+
+	if (ret != LZO_E_OK)
+		return -1;
+
+	if (compress_size > *dstlen)
+		return -1;
+
+	memcpy(cpage_out, lzo_compress_buf, compress_size);
+	*dstlen = compress_size;
+
+	return 0;
+}
+
+static int jffs2_lzo_decompress(unsigned char *data_in, unsigned char *cpage_out,
+				 uint32_t srclen, uint32_t destlen, void *model)
+{
+	unsigned long dl = destlen;
+	int ret;
+
+	ret = lzo1x_decompress_safe(data_in, srclen, cpage_out, &dl, NULL);
+
+	if (ret != LZO_E_OK || dl != destlen)
+		return -1;
+
+	return 0;
+}
+
+static struct jffs2_compressor jffs2_lzo_comp = {
+	.priority = JFFS2_LZO_PRIORITY,
+	.name = "lzo",
+	.compr = JFFS2_COMPR_LZO,
+	.compress = &jffs2_lzo_compress,
+	.decompress = &jffs2_lzo_decompress,
+	.disabled = 0,
+};
+
+int __init jffs2_lzo_init(void)
+{
+	int ret;
+
+	ret = alloc_workspace();
+	if (ret < 0)
+		return ret;
+
+	ret = jffs2_register_compressor(&jffs2_lzo_comp);
+	if (ret)
+		free_workspace();
+
+	return ret;
+}
+
+void jffs2_lzo_exit(void)
+{
+	jffs2_unregister_compressor(&jffs2_lzo_comp);
+	free_workspace();
+}
diff --git a/include/linux/jffs2.h b/include/linux/jffs2.h
index 840631f..6b563ca 100644
--- a/include/linux/jffs2.h
+++ b/include/linux/jffs2.h
@@ -46,6 +46,7 @@
 #define JFFS2_COMPR_COPY	0x04
 #define JFFS2_COMPR_DYNRUBIN	0x05
 #define JFFS2_COMPR_ZLIB	0x06
+#define JFFS2_COMPR_LZO		0x07
 /* Compatibility flags. */
 #define JFFS2_COMPAT_MASK 0xc000      /* What do to if an unknown nodetype is found */
 #define JFFS2_NODE_ACCURATE 0x2000