* [PATCH 1/2] Squashfs: add LZ4 compression support
2013-07-22 2:21 [PATCH 0/2] Squashfs: add LZ4 compression Phillip Lougher
@ 2013-07-22 2:21 ` Phillip Lougher
2013-07-22 4:35 ` Joe Perches
2013-07-22 2:21 ` [PATCH 2/2] Squashfs: Add LZ4 compression configuration option Phillip Lougher
` (2 subsequent siblings)
3 siblings, 1 reply; 21+ messages in thread
From: Phillip Lougher @ 2013-07-22 2:21 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel, linux-embedded; +Cc: Phillip Lougher
Add support for reading file systems compressed with the
LZ4 compression algorithm.
This patch adds the LZ4 decompressor wrapper code.
Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
---
fs/squashfs/lz4_wrapper.c | 163 +++++++++++++++++++++++++++++++++++++++++++++
fs/squashfs/squashfs_fs.h | 1 +
2 files changed, 164 insertions(+)
create mode 100644 fs/squashfs/lz4_wrapper.c
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
new file mode 100644
index 0000000..681ed94
--- /dev/null
+++ b/fs/squashfs/lz4_wrapper.c
@@ -0,0 +1,163 @@
+/*
+ * Squashfs - a compressed read only filesystem for Linux
+ *
+ * Copyright (c) 2013
+ * Phillip Lougher <phillip@squashfs.org.uk>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * lz4_wrapper.c
+ */
+
+#include <linux/buffer_head.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/lz4.h>
+
+#include "squashfs_fs.h"
+#include "squashfs_fs_sb.h"
+#include "squashfs.h"
+#include "decompressor.h"
+
+#define LZ4_LEGACY 1
+
+struct lz4_comp_opts {
+ __le32 version;
+ __le32 flags;
+};
+
+struct squashfs_lz4 {
+ void *input;
+ void *output;
+};
+
+
+
+static void *lz4_init(struct squashfs_sb_info *msblk, void *buff, int len)
+{
+ struct lz4_comp_opts *comp_opts = buff;
+ int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE);
+ struct squashfs_lz4 *stream;
+ int err = -ENOMEM;
+
+ /* LZ4 compressed filesystems always have compression options */
+ if(comp_opts == NULL || len < sizeof(*comp_opts)) {
+ err = -EIO;
+ goto failed;
+ }
+ if(le32_to_cpu(comp_opts->version) != LZ4_LEGACY) {
+ /* LZ4 format currently used by the kernel is the 'legacy'
+ * format */
+ ERROR("Unknown LZ4 version\n");
+ err = -EINVAL;
+ goto failed;
+ }
+
+ stream = kzalloc(sizeof(*stream), GFP_KERNEL);
+ if (stream == NULL)
+ goto failed;
+ stream->input = vmalloc(block_size);
+ if (stream->input == NULL)
+ goto failed2;
+ stream->output = vmalloc(block_size);
+ if (stream->output == NULL)
+ goto failed3;
+
+ return stream;
+
+failed3:
+ vfree(stream->input);
+failed2:
+ kfree(stream);
+failed:
+ ERROR("Failed to initialise LZ4 decompressor\n");
+ return ERR_PTR(err);
+}
+
+
+static void lz4_free(void *strm)
+{
+ struct squashfs_lz4 *stream = strm;
+
+ if (stream) {
+ vfree(stream->input);
+ vfree(stream->output);
+ }
+ kfree(stream);
+}
+
+
+static int lz4_uncompress(struct squashfs_sb_info *msblk, void **buffer,
+ struct buffer_head **bh, int b, int offset, int length, int srclength,
+ int pages)
+{
+ struct squashfs_lz4 *stream = msblk->stream;
+ void *buff = stream->input;
+ int avail, i, bytes = length, res;
+ size_t dest_len = srclength;
+
+ mutex_lock(&msblk->read_data_mutex);
+
+ for (i = 0; i < b; i++) {
+ wait_on_buffer(bh[i]);
+ if (!buffer_uptodate(bh[i]))
+ goto block_release;
+
+ avail = min(bytes, msblk->devblksize - offset);
+ memcpy(buff, bh[i]->b_data + offset, avail);
+ buff += avail;
+ bytes -= avail;
+ offset = 0;
+ put_bh(bh[i]);
+ }
+
+ res = lz4_decompress_unknownoutputsize(stream->input, length,
+ stream->output, &dest_len);
+ if (res)
+ goto failed;
+
+ bytes = dest_len;
+ for (i = 0, buff = stream->output; bytes && i < pages; i++) {
+ avail = min_t(int, bytes, PAGE_CACHE_SIZE);
+ memcpy(buffer[i], buff, avail);
+ buff += avail;
+ bytes -= avail;
+ }
+ if (bytes)
+ goto failed;
+
+ mutex_unlock(&msblk->read_data_mutex);
+ return dest_len;
+
+block_release:
+ for (; i < b; i++)
+ put_bh(bh[i]);
+
+failed:
+ mutex_unlock(&msblk->read_data_mutex);
+
+ ERROR("lz4 decompression failed, data probably corrupt\n");
+ return -EIO;
+}
+
+const struct squashfs_decompressor squashfs_lz4_comp_ops = {
+ .init = lz4_init,
+ .free = lz4_free,
+ .decompress = lz4_uncompress,
+ .id = LZ4_COMPRESSION,
+ .name = "lz4",
+ .supported = 1
+};
diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h
index 9e2349d..ee0be5c 100644
--- a/fs/squashfs/squashfs_fs.h
+++ b/fs/squashfs/squashfs_fs.h
@@ -237,6 +237,7 @@ struct meta_index {
#define LZMA_COMPRESSION 2
#define LZO_COMPRESSION 3
#define XZ_COMPRESSION 4
+#define LZ4_COMPRESSION 5
struct squashfs_super_block {
__le32 s_magic;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH 1/2] Squashfs: add LZ4 compression support
2013-07-22 2:21 ` [PATCH 1/2] Squashfs: add LZ4 compression support Phillip Lougher
@ 2013-07-22 4:35 ` Joe Perches
0 siblings, 0 replies; 21+ messages in thread
From: Joe Perches @ 2013-07-22 4:35 UTC (permalink / raw)
To: Phillip Lougher; +Cc: linux-kernel, linux-fsdevel, linux-embedded
On Mon, 2013-07-22 at 03:21 +0100, Phillip Lougher wrote:
> Add support for reading file systems compressed with the
> LZ4 compression algorithm.
Some whitespace trivia and a naming comment.
> diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
[]
> +static void *lz4_init(struct squashfs_sb_info *msblk, void *buff, int len)
[]
> + /* LZ4 compressed filesystems always have compression options */
> + if(comp_opts == NULL || len < sizeof(*comp_opts)) {
space after ifs please.
> + if(le32_to_cpu(comp_opts->version) != LZ4_LEGACY) {
> +
> +
> +static void lz4_free(void *strm)
Single blank line between functions
> +static int lz4_uncompress(struct squashfs_sb_info *msblk, void **buffer,
> + struct buffer_head **bh, int b, int offset, int length, int srclength,
> + int pages)
> +{
> + struct squashfs_lz4 *stream = msblk->stream;
> + void *buff = stream->input;
It's not particularly nice to have both buffer and buff
in the same function.
Maybe void *input though char *input would be better.
> + int avail, i, bytes = length, res;
> + size_t dest_len = srclength;
> +
> + mutex_lock(&msblk->read_data_mutex);
> +
> + for (i = 0; i < b; i++) {
> + wait_on_buffer(bh[i]);
> + if (!buffer_uptodate(bh[i]))
> + goto block_release;
> +
> + avail = min(bytes, msblk->devblksize - offset);
> + memcpy(buff, bh[i]->b_data + offset, avail);
> + buff += avail;
> + bytes -= avail;
> + offset = 0;
> + put_bh(bh[i]);
> + }
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH 2/2] Squashfs: Add LZ4 compression configuration option
2013-07-22 2:21 [PATCH 0/2] Squashfs: add LZ4 compression Phillip Lougher
2013-07-22 2:21 ` [PATCH 1/2] Squashfs: add LZ4 compression support Phillip Lougher
@ 2013-07-22 2:21 ` Phillip Lougher
2013-07-22 3:05 ` [PATCH 0/2] Squashfs: add LZ4 compression Gu Zheng
[not found] ` <20130723151009.GA5971@wolff.to>
3 siblings, 0 replies; 21+ messages in thread
From: Phillip Lougher @ 2013-07-22 2:21 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel, linux-embedded; +Cc: Phillip Lougher
Add the glue code, and also update the documentation.
Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
---
Documentation/filesystems/squashfs.txt | 8 ++++----
fs/squashfs/Kconfig | 15 +++++++++++++++
fs/squashfs/Makefile | 1 +
fs/squashfs/decompressor.c | 7 +++++++
fs/squashfs/decompressor.h | 4 ++++
5 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/Documentation/filesystems/squashfs.txt b/Documentation/filesystems/squashfs.txt
index 403c090..e5274f8 100644
--- a/Documentation/filesystems/squashfs.txt
+++ b/Documentation/filesystems/squashfs.txt
@@ -2,10 +2,10 @@ SQUASHFS 4.0 FILESYSTEM
=======================
Squashfs is a compressed read-only filesystem for Linux.
-It uses zlib/lzo/xz compression to compress files, inodes and directories.
-Inodes in the system are very small and all blocks are packed to minimise
-data overhead. Block sizes greater than 4K are supported up to a maximum
-of 1Mbytes (default block size 128K).
+It uses zlib, lz4, lzo, or xz compression to compress files, inodes and
+directories. Inodes in the system are very small and all blocks are packed to
+minimise data overhead. Block sizes greater than 4K are supported up to a
+maximum of 1Mbytes (default block size 128K).
Squashfs is intended for general read-only filesystem use, for archival
use (i.e. in cases where a .tar.gz file may be used), and in constrained
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
index c70111e..257f934 100644
--- a/fs/squashfs/Kconfig
+++ b/fs/squashfs/Kconfig
@@ -48,6 +48,21 @@ config SQUASHFS_ZLIB
If unsure, say Y.
+config SQUASHFS_LZ4
+ bool "Include support for LZ4 compressed file systems"
+ depends on SQUASHFS
+ select LZ4_DECOMPRESS
+ help
+ Saying Y here includes support for reading Squashfs file systems
+ compressed with LZ4 compression. LZ4 compression is mainly
+ aimed at embedded systems with slower CPUs where the overheads
+ of zlib are too high.
+
+ LZ4 is not the standard compression used in Squashfs and so most
+ file systems will be readable without selecting this option.
+
+ If unsure, say N.
+
config SQUASHFS_LZO
bool "Include support for LZO compressed file systems"
depends on SQUASHFS
diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile
index 110b047..4a80ca7 100644
--- a/fs/squashfs/Makefile
+++ b/fs/squashfs/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_SQUASHFS) += squashfs.o
squashfs-y += block.o cache.o dir.o export.o file.o fragment.o id.o inode.o
squashfs-y += namei.o super.o symlink.o decompressor.o
squashfs-$(CONFIG_SQUASHFS_XATTR) += xattr.o xattr_id.o
+squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o
squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o
squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o
squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o
diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c
index 3f6271d..ae60211 100644
--- a/fs/squashfs/decompressor.c
+++ b/fs/squashfs/decompressor.c
@@ -40,6 +40,12 @@ static const struct squashfs_decompressor squashfs_lzma_unsupported_comp_ops = {
NULL, NULL, NULL, LZMA_COMPRESSION, "lzma", 0
};
+#ifndef CONFIG_SQUASHFS_LZ4
+static const struct squashfs_decompressor squashfs_lz4_comp_ops = {
+ NULL, NULL, NULL, LZ4_COMPRESSION, "lz4", 0
+};
+#endif
+
#ifndef CONFIG_SQUASHFS_LZO
static const struct squashfs_decompressor squashfs_lzo_comp_ops = {
NULL, NULL, NULL, LZO_COMPRESSION, "lzo", 0
@@ -64,6 +70,7 @@ static const struct squashfs_decompressor squashfs_unknown_comp_ops = {
static const struct squashfs_decompressor *decompressor[] = {
&squashfs_zlib_comp_ops,
+ &squashfs_lz4_comp_ops,
&squashfs_lzo_comp_ops,
&squashfs_xz_comp_ops,
&squashfs_lzma_unsupported_comp_ops,
diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
index 330073e..a9fbdd2 100644
--- a/fs/squashfs/decompressor.h
+++ b/fs/squashfs/decompressor.h
@@ -52,6 +52,10 @@ static inline int squashfs_decompress(struct squashfs_sb_info *msblk,
extern const struct squashfs_decompressor squashfs_xz_comp_ops;
#endif
+#ifdef CONFIG_SQUASHFS_LZ4
+extern const struct squashfs_decompressor squashfs_lz4_comp_ops;
+#endif
+
#ifdef CONFIG_SQUASHFS_LZO
extern const struct squashfs_decompressor squashfs_lzo_comp_ops;
#endif
--
1.7.10.4
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2013-07-22 2:21 [PATCH 0/2] Squashfs: add LZ4 compression Phillip Lougher
2013-07-22 2:21 ` [PATCH 1/2] Squashfs: add LZ4 compression support Phillip Lougher
2013-07-22 2:21 ` [PATCH 2/2] Squashfs: Add LZ4 compression configuration option Phillip Lougher
@ 2013-07-22 3:05 ` Gu Zheng
2013-07-22 5:07 ` Phillip Lougher
[not found] ` <20130723151009.GA5971@wolff.to>
3 siblings, 1 reply; 21+ messages in thread
From: Gu Zheng @ 2013-07-22 3:05 UTC (permalink / raw)
To: Phillip Lougher; +Cc: linux-kernel, linux-fsdevel, linux-embedded
Hi Phillip,
Have some tests been carried out to confirm that Squashfs really
can get benefit from LZ4 compression, comparing with lzo?
Thanks,
Gu
On 07/22/2013 10:21 AM, Phillip Lougher wrote:
> Hi
>
> Now that LZ4 compression support is in 3.11-rc1, I have written the
> following two patches for Squashfs to use it.
>
> Phillip Lougher (2):
> Squashfs: add LZ4 compression support
> Squashfs: Add LZ4 compression configuration option
>
> Documentation/filesystems/squashfs.txt | 8 +-
> fs/squashfs/Kconfig | 15 +++
> fs/squashfs/Makefile | 1 +
> fs/squashfs/decompressor.c | 7 ++
> fs/squashfs/decompressor.h | 4 +
> fs/squashfs/lz4_wrapper.c | 163 ++++++++++++++++++++++++++++++++
> fs/squashfs/squashfs_fs.h | 1 +
> 7 files changed, 195 insertions(+), 4 deletions(-)
> create mode 100644 fs/squashfs/lz4_wrapper.c
>
> These patches are also available in the git tree here:
>
> browse: https://git.kernel.org/cgit/linux/kernel/git/pkl/squashfs-lz4.git
> git clone: git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-lz4.git
>
> LZ4 support has (obviously) also been added to the squashfs-tools
> (Mksquashfs and Unsquashfs). This is available from the Squashfs-tools
> git repository here:
>
> browse: https://git.kernel.org/cgit/fs/squashfs/squashfs-tools.git
> git clone: git://git.kernel.org/pub/scm/fs/squashfs/squashfs-tools.git
>
> When building the squashfs-tools edit the Makefile to enable LZ4
> support (by default it is disabled).
>
> LZ4 compression can be specified by using the -comp option, e.g.
> % mksquashfs xxx img.sqsh -comp lz4
>
> The use of LZ4 high compression can be specified using -Xhc, e.g.
>
> % mksquashfs xxx img.sqsh -comp lz4 -Xhc
>
> Phillip
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2013-07-22 3:05 ` [PATCH 0/2] Squashfs: add LZ4 compression Gu Zheng
@ 2013-07-22 5:07 ` Phillip Lougher
2013-07-22 6:04 ` Gu Zheng
0 siblings, 1 reply; 21+ messages in thread
From: Phillip Lougher @ 2013-07-22 5:07 UTC (permalink / raw)
To: Gu Zheng; +Cc: Phillip Lougher, linux-kernel, linux-fsdevel, linux-embedded
On 22 July 2013 04:05, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
> Hi Phillip,
> Have some tests been carried out to confirm that Squashfs really
> can get benefit from LZ4 compression, comparing with lzo?
This seems to be a loaded question, in that it seems to be trying to
reopen the "why add lz4 when we already have lzo" debate all over
again. As LZ4 has been merged to mainline, this appears to be a
question that has already been answered.
As far as Squashfs is concerned, I believe it is important to give
people the choice of using LZ4 to compress Squashfs filesystems now
its been mainlined. As far as expected benefits are concerned,
Squashfs' use in embedded systems is very similar to compressing
kernels and initramfs data, in that it tends to be used to compress
root filesystems. As such the benefits of using LZ4 in Squashfs
should be broadly similar to using LZ4 to compress kernels and
initramfs data. Ultimately it is up to people to experiment and
choose whatever compression is best for their systems.
People are welcome to try the patches out and report their findings.
Phillip
>
> Thanks,
> Gu
>
> On 07/22/2013 10:21 AM, Phillip Lougher wrote:
>
>> Hi
>>
>> Now that LZ4 compression support is in 3.11-rc1, I have written the
>> following two patches for Squashfs to use it.
>>
>> Phillip Lougher (2):
>> Squashfs: add LZ4 compression support
>> Squashfs: Add LZ4 compression configuration option
>>
>> Documentation/filesystems/squashfs.txt | 8 +-
>> fs/squashfs/Kconfig | 15 +++
>> fs/squashfs/Makefile | 1 +
>> fs/squashfs/decompressor.c | 7 ++
>> fs/squashfs/decompressor.h | 4 +
>> fs/squashfs/lz4_wrapper.c | 163 ++++++++++++++++++++++++++++++++
>> fs/squashfs/squashfs_fs.h | 1 +
>> 7 files changed, 195 insertions(+), 4 deletions(-)
>> create mode 100644 fs/squashfs/lz4_wrapper.c
>>
>> These patches are also available in the git tree here:
>>
>> browse: https://git.kernel.org/cgit/linux/kernel/git/pkl/squashfs-lz4.git
>> git clone: git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-lz4.git
>>
>> LZ4 support has (obviously) also been added to the squashfs-tools
>> (Mksquashfs and Unsquashfs). This is available from the Squashfs-tools
>> git repository here:
>>
>> browse: https://git.kernel.org/cgit/fs/squashfs/squashfs-tools.git
>> git clone: git://git.kernel.org/pub/scm/fs/squashfs/squashfs-tools.git
>>
>> When building the squashfs-tools edit the Makefile to enable LZ4
>> support (by default it is disabled).
>>
>> LZ4 compression can be specified by using the -comp option, e.g.
>> % mksquashfs xxx img.sqsh -comp lz4
>>
>> The use of LZ4 high compression can be specified using -Xhc, e.g.
>>
>> % mksquashfs xxx img.sqsh -comp lz4 -Xhc
>>
>> Phillip
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at http://vger.kernel.org/majordomo-info.html
>> Please read the FAQ at http://www.tux.org/lkml/
>>
>
>
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2013-07-22 5:07 ` Phillip Lougher
@ 2013-07-22 6:04 ` Gu Zheng
2013-07-22 13:38 ` Phillip Lougher
` (3 more replies)
0 siblings, 4 replies; 21+ messages in thread
From: Gu Zheng @ 2013-07-22 6:04 UTC (permalink / raw)
To: Phillip Lougher
Cc: Phillip Lougher, linux-kernel, linux-fsdevel, linux-embedded
On 07/22/2013 01:07 PM, Phillip Lougher wrote:
> On 22 July 2013 04:05, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>> Hi Phillip,
>> Have some tests been carried out to confirm that Squashfs really
>> can get benefit from LZ4 compression, comparing with lzo?
>
> This seems to be a loaded question, in that it seems to be trying to
> reopen the "why add lz4 when we already have lzo" debate all over
> again. As LZ4 has been merged to mainline, this appears to be a
> question that has already been answered.
No, they are different. LZ4 can be merged to mainline, because we can
see the benefit(faster compressing speed under the enabled unaligned
memory access) it brings to us comparing with lzo.
But it's hard to say that it also really can bring benefit to Squashfs.
>
> As far as Squashfs is concerned, I believe it is important to give
> people the choice of using LZ4 to compress Squashfs filesystems now
> its been mainlined. As far as expected benefits are concerned,
> Squashfs' use in embedded systems is very similar to compressing
> kernels and initramfs data, in that it tends to be used to compress
> root filesystems. As such the benefits of using LZ4 in Squashfs
> should be broadly similar to using LZ4 to compress kernels and
> initramfs data. Ultimately it is up to people to experiment and
> choose whatever compression is best for their systems.
Yes, but the premise is that using LZ4 in Squashfs is better than lzo
in at least one actual situation.
>
> People are welcome to try the patches out and report their findings.
Let's look and see other guys' feedbacks, they are more persuasive.:)
Regards,
Gu
>
> Phillip
>
>>
>> Thanks,
>> Gu
>>
>> On 07/22/2013 10:21 AM, Phillip Lougher wrote:
>>
>>> Hi
>>>
>>> Now that LZ4 compression support is in 3.11-rc1, I have written the
>>> following two patches for Squashfs to use it.
>>>
>>> Phillip Lougher (2):
>>> Squashfs: add LZ4 compression support
>>> Squashfs: Add LZ4 compression configuration option
>>>
>>> Documentation/filesystems/squashfs.txt | 8 +-
>>> fs/squashfs/Kconfig | 15 +++
>>> fs/squashfs/Makefile | 1 +
>>> fs/squashfs/decompressor.c | 7 ++
>>> fs/squashfs/decompressor.h | 4 +
>>> fs/squashfs/lz4_wrapper.c | 163 ++++++++++++++++++++++++++++++++
>>> fs/squashfs/squashfs_fs.h | 1 +
>>> 7 files changed, 195 insertions(+), 4 deletions(-)
>>> create mode 100644 fs/squashfs/lz4_wrapper.c
>>>
>>> These patches are also available in the git tree here:
>>>
>>> browse: https://git.kernel.org/cgit/linux/kernel/git/pkl/squashfs-lz4.git
>>> git clone: git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-lz4.git
>>>
>>> LZ4 support has (obviously) also been added to the squashfs-tools
>>> (Mksquashfs and Unsquashfs). This is available from the Squashfs-tools
>>> git repository here:
>>>
>>> browse: https://git.kernel.org/cgit/fs/squashfs/squashfs-tools.git
>>> git clone: git://git.kernel.org/pub/scm/fs/squashfs/squashfs-tools.git
>>>
>>> When building the squashfs-tools edit the Makefile to enable LZ4
>>> support (by default it is disabled).
>>>
>>> LZ4 compression can be specified by using the -comp option, e.g.
>>> % mksquashfs xxx img.sqsh -comp lz4
>>>
>>> The use of LZ4 high compression can be specified using -Xhc, e.g.
>>>
>>> % mksquashfs xxx img.sqsh -comp lz4 -Xhc
>>>
>>> Phillip
>>> --
>>> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>>> the body of a message to majordomo@vger.kernel.org
>>> More majordomo info at http://vger.kernel.org/majordomo-info.html
>>> Please read the FAQ at http://www.tux.org/lkml/
>>>
>>
>>
>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2013-07-22 6:04 ` Gu Zheng
@ 2013-07-22 13:38 ` Phillip Lougher
2013-07-26 22:00 ` Rob Landley
` (2 subsequent siblings)
3 siblings, 0 replies; 21+ messages in thread
From: Phillip Lougher @ 2013-07-22 13:38 UTC (permalink / raw)
To: Gu Zheng; +Cc: Phillip Lougher, linux-kernel, linux-fsdevel, linux-embedded
On 22 July 2013 07:04, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
> On 07/22/2013 01:07 PM, Phillip Lougher wrote:
>
>> On 22 July 2013 04:05, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>>> Hi Phillip,
>>> Have some tests been carried out to confirm that Squashfs really
>>> can get benefit from LZ4 compression, comparing with lzo?
>>
>> This seems to be a loaded question, in that it seems to be trying to
>> reopen the "why add lz4 when we already have lzo" debate all over
>> again. As LZ4 has been merged to mainline, this appears to be a
>> question that has already been answered.
>
> No, they are different. LZ4 can be merged to mainline, because we can
> see the benefit(faster compressing speed under the enabled unaligned
> memory access) it brings to us comparing with lzo.
The users of LZ4 that has been merged is for decompression of kernel
and initramfs, not compression. So far I don't see any users of the
LZ4 compression code in the kernel. So you're saying it was merged
for fast compression despite there being no users of it?
> But it's hard to say that it also really can bring benefit to Squashfs.
Prejudiced statement without any evidence, if that's your contention prove it.
>
>>
>> As far as Squashfs is concerned, I believe it is important to give
>> people the choice of using LZ4 to compress Squashfs filesystems now
>> its been mainlined. As far as expected benefits are concerned,
>> Squashfs' use in embedded systems is very similar to compressing
>> kernels and initramfs data, in that it tends to be used to compress
>> root filesystems. As such the benefits of using LZ4 in Squashfs
>> should be broadly similar to using LZ4 to compress kernels and
>> initramfs data. Ultimately it is up to people to experiment and
>> choose whatever compression is best for their systems.
>
> Yes, but the premise is that using LZ4 in Squashfs is better than lzo
> in at least one actual situation.
>
No. All it takes is for LZ4 to be better/faster at decompressing the
kernel or initramfs to make it advantageous to use LZ4 for Squashfs,
thus avoiding the overhead of another decompressor in the kernel..
>>
>> People are welcome to try the patches out and report their findings.
>
> Let's look and see other guys' feedbacks, they are more persuasive.:)
Suit yourself, maybe when you start using evidence rather than
unsubstantiated assertions, you'll be more persuasive too. So far all
I see is wind.
Phillip
>
> Regards,
> Gu
>
>>
>> Phillip
>>
>>>
>>> Thanks,
>>> Gu
>>>
>>> On 07/22/2013 10:21 AM, Phillip Lougher wrote:
>>>
>>>> Hi
>>>>
>>>> Now that LZ4 compression support is in 3.11-rc1, I have written the
>>>> following two patches for Squashfs to use it.
>>>>
>>>> Phillip Lougher (2):
>>>> Squashfs: add LZ4 compression support
>>>> Squashfs: Add LZ4 compression configuration option
>>>>
>>>> Documentation/filesystems/squashfs.txt | 8 +-
>>>> fs/squashfs/Kconfig | 15 +++
>>>> fs/squashfs/Makefile | 1 +
>>>> fs/squashfs/decompressor.c | 7 ++
>>>> fs/squashfs/decompressor.h | 4 +
>>>> fs/squashfs/lz4_wrapper.c | 163 ++++++++++++++++++++++++++++++++
>>>> fs/squashfs/squashfs_fs.h | 1 +
>>>> 7 files changed, 195 insertions(+), 4 deletions(-)
>>>> create mode 100644 fs/squashfs/lz4_wrapper.c
>>>>
>>>> These patches are also available in the git tree here:
>>>>
>>>> browse: https://git.kernel.org/cgit/linux/kernel/git/pkl/squashfs-lz4.git
>>>> git clone: git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-lz4.git
>>>>
>>>> LZ4 support has (obviously) also been added to the squashfs-tools
>>>> (Mksquashfs and Unsquashfs). This is available from the Squashfs-tools
>>>> git repository here:
>>>>
>>>> browse: https://git.kernel.org/cgit/fs/squashfs/squashfs-tools.git
>>>> git clone: git://git.kernel.org/pub/scm/fs/squashfs/squashfs-tools.git
>>>>
>>>> When building the squashfs-tools edit the Makefile to enable LZ4
>>>> support (by default it is disabled).
>>>>
>>>> LZ4 compression can be specified by using the -comp option, e.g.
>>>> % mksquashfs xxx img.sqsh -comp lz4
>>>>
>>>> The use of LZ4 high compression can be specified using -Xhc, e.g.
>>>>
>>>> % mksquashfs xxx img.sqsh -comp lz4 -Xhc
>>>>
>>>> Phillip
>>>> --
>>>> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>>>> the body of a message to majordomo@vger.kernel.org
>>>> More majordomo info at http://vger.kernel.org/majordomo-info.html
>>>> Please read the FAQ at http://www.tux.org/lkml/
>>>>
>>>
>>>
>>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2013-07-22 6:04 ` Gu Zheng
2013-07-22 13:38 ` Phillip Lougher
@ 2013-07-26 22:00 ` Rob Landley
2014-09-05 10:01 ` Guan, Xin
2014-09-06 20:29 ` Guan, Xin
3 siblings, 0 replies; 21+ messages in thread
From: Rob Landley @ 2013-07-26 22:00 UTC (permalink / raw)
To: Gu Zheng
Cc: Phillip Lougher, Phillip Lougher, linux-kernel, linux-fsdevel,
linux-embedded
On 07/22/2013 01:04:59 AM, Gu Zheng wrote:
> On 07/22/2013 01:07 PM, Phillip Lougher wrote:
>
> > On 22 July 2013 04:05, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
> >> Hi Phillip,
> >> Have some tests been carried out to confirm that Squashfs
> really
> >> can get benefit from LZ4 compression, comparing with lzo?
> >
> > This seems to be a loaded question, in that it seems to be trying to
> > reopen the "why add lz4 when we already have lzo" debate all over
> > again. As LZ4 has been merged to mainline, this appears to be a
> > question that has already been answered.
>
> No, they are different. LZ4 can be merged to mainline, because we can
> see the benefit(faster compressing speed under the enabled unaligned
> memory access) it brings to us comparing with lzo.
> But it's hard to say that it also really can bring benefit to
> Squashfs.
A compression format was added to the kernel. Philip hooked up the code
that was already in the kernel to a filesystem that was already in the
kernel.
You consider this action controversial...
Rob
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2013-07-22 6:04 ` Gu Zheng
2013-07-22 13:38 ` Phillip Lougher
2013-07-26 22:00 ` Rob Landley
@ 2014-09-05 10:01 ` Guan, Xin
2014-09-06 20:29 ` Guan, Xin
3 siblings, 0 replies; 21+ messages in thread
From: Guan, Xin @ 2014-09-05 10:01 UTC (permalink / raw)
To: linux-fsdevel
Gu Zheng <guz.fnst <at> cn.fujitsu.com> writes:
> But it's hard to say that it also really can bring benefit to Squashfs.
I did observe faster I/O in my daily use. The start up time of
applications on my squashfs+aufs root filesystem generally decreased by
10% to 20% compared to lzo. Not very much, but the absence of lag
with lz4 greatly improved user experience.
> Yes, but the premise is that using LZ4 in Squashfs is better than lzo
> in at least one actual situation.
I update my squashfs image (about 8GB compressed) ~ once every month.
Lz4 yields slightly worse compression ratio (+20%) than lzo but its
compression speed is about fifty (no typo) times faster.
In order to get rid of the endless waiting on "mksquashfs -comp lzo",
I had to port Phillip's patch (in particular the "lz4_wrapper.c") to
Linux 3.14.x so I can use lz4 compressed squashfs images everyday.
> Let's look and see other guys' feedbacks, they are more persuasive.:)
IMHO, a 50x boost of compression speed is persuasive.
Lz4 also helped me to get rid of occasional lags on file I/O,
greatly improved user experience with my Linux desktop.
Adding lz4 support to squashfs isn't complicated. Everyone can
review the patch in a couple of minutes.
So I vote on its inclusion in mainline.
Guan
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2013-07-22 6:04 ` Gu Zheng
` (2 preceding siblings ...)
2014-09-05 10:01 ` Guan, Xin
@ 2014-09-06 20:29 ` Guan, Xin
3 siblings, 0 replies; 21+ messages in thread
From: Guan, Xin @ 2014-09-06 20:29 UTC (permalink / raw)
To: linux-fsdevel
Gu Zheng <guz.fnst <at> cn.fujitsu.com> writes:
> On 07/22/2013 01:07 PM, Phillip Lougher wrote:
>
> But it's hard to say that it also really can bring benefit to Squashfs.
In addition to my previous message:
Tried HC mode of lz4 -- Compression ratio of my rootfs (the whole system) is
3~4% better than lzo; compression speed is 5x of lzo's.
Decompression, as already mentioned, was markedly faster than lzo.
Lz4 is a clear win over lzo from every aspect.
Guan
^ permalink raw reply [flat|nested] 21+ messages in thread
[parent not found: <20130723151009.GA5971@wolff.to>]
* Re: [PATCH 0/2] Squashfs: add LZ4 compression
[not found] ` <20130723151009.GA5971@wolff.to>
@ 2013-07-23 16:17 ` Phillip Lougher
2013-07-23 16:29 ` Bruno Wolff III
0 siblings, 1 reply; 21+ messages in thread
From: Phillip Lougher @ 2013-07-23 16:17 UTC (permalink / raw)
To: Bruno Wolff III; +Cc: Phillip Lougher, linux-kernel, linux-fsdevel
On 23 July 2013 16:10, Bruno Wolff III <bruno@wolff.to> wrote:
> On Mon, Jul 22, 2013 at 03:21:01 +0100,
> Phillip Lougher <phillip@squashfs.org.uk> wrote:
>>
>> Hi
>>
>> Now that LZ4 compression support is in 3.11-rc1, I have written the
>> following two patches for Squashfs to use it.
>
>
> If this gets accepted are you going to move the LZ4 changes to
> squashfs-tools into the stable branch? I'd like to make LZ4 work in
> mksquashfs in at least rawhide once LZ4 is usable for squashfs file systems
> in the kernel. So I was wondering whether to expect this to appear in stable
> shortly or to expect to start doing builds using the master branch.
Hmm, good point.... I was intending to merge the LZ4 work onto stable
yesterday or today, but, I've received more resistance to the idea of
LZ4 in Squashfs than I expected.... I'm not sure that merging LZ4
onto stable now gives the right message, because if it does not go
into mainline, it will effectively be experimental or abandoned.
In otherwords I don't think it's wise yet to merge LZ4 onto stable,
not until at least there's some positive feedback on the mailing list.
Thoughts? Maybe some positive feedback? :-)
A V2 of the patches will be coming ASAP hopefully with some performance stats.
Thanks
Phillip
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2013-07-23 16:17 ` Phillip Lougher
@ 2013-07-23 16:29 ` Bruno Wolff III
0 siblings, 0 replies; 21+ messages in thread
From: Bruno Wolff III @ 2013-07-23 16:29 UTC (permalink / raw)
To: Phillip Lougher; +Cc: Phillip Lougher, linux-kernel, linux-fsdevel
On Tue, Jul 23, 2013 at 17:17:30 +0100,
Phillip Lougher <phillip.lougher@gmail.com> wrote:
>
>In otherwords I don't think it's wise yet to merge LZ4 onto stable,
>not until at least there's some positive feedback on the mailing list.
>
>Thoughts? Maybe some positive feedback? :-)
I think it makes sense to tie inclusion in the kernel with inclusion
into the stable branch. If this gets into the 3.11 kernel, I'm going
to want to get it into Fedora releases that use that kernel. This
will initially be Fedora 20 (which will branch from rawhide in a couple
of weeks) and will eventually be in Fedora 19 and probably Fedora 18.
So I'd like to see it in stable, since I would feel less comfortable
using the master branch.
If LZ4 support isn't in the kernel I think that there isn't a lot of
need for support in Fedora since I don't think many people use it
for archiving. It mostly gets used for live images. I'll make sure
that LZ4 can be used for them, but I suspect most people will still
prefer xz for them. But probably at least some people would prefer
faster decompression rather than minimal size. As flash drives get
bigger (for the same cost) more people will probably prefer speed over
size.
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2014-11-27 8:00 Phillip Lougher
@ 2014-11-27 13:37 ` Bruno Wolff III
[not found] ` <CAJZww4gzTnbyB4jbFQc0N4G6GJ31pVwv-PNZtDMbUxXMoH-orQ@mail.gmail.com>
2014-12-12 7:44 ` Dave Vasilevsky
` (2 subsequent siblings)
3 siblings, 1 reply; 21+ messages in thread
From: Bruno Wolff III @ 2014-11-27 13:37 UTC (permalink / raw)
To: Phillip Lougher
Cc: linux-kernel, linux-fsdevel, texstar, martin, guanx.bac, dave,
szycha, blyons, tokiclover, afm404, hugochevrain
On Thu, Nov 27, 2014 at 08:00:47 +0000,
Phillip Lougher <phillip@squashfs.org.uk> wrote:
>
>My intention is to submit them in the next kernel merge window.
>If you want LZ4 support in Squashfs now is a good time to publically
>support the inclusion of these patches.
Fedora has been supporting LZ4 functionallity in mksquashfs and unsquashfs
for about 6 months (in F19+), but the Fedora kernel team won't add support
in the kernel until LZ4 support is accepted for the upstream kernel.
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2014-11-27 8:00 Phillip Lougher
2014-11-27 13:37 ` Bruno Wolff III
@ 2014-12-12 7:44 ` Dave Vasilevsky
2014-12-12 8:31 ` Anthony F McInerney
2014-12-12 10:27 ` Guan Xin
[not found] ` <CAGw_SrN6tZyXRx6pxkpnhWc2U5845HRbfzhOK5v9qkjWejFpmA@mail.gmail.com>
3 siblings, 1 reply; 21+ messages in thread
From: Dave Vasilevsky @ 2014-12-12 7:44 UTC (permalink / raw)
To: Phillip Lougher
Cc: linux-kernel, linux-fsdevel, texstar, martin, Guan Xin, bruno,
szycha, blyons, tokiclover, afm404, hugochevrain
I'd like to support the inclusion of these patches.
There's enough demand for LZ4-compressed squashfs that I received an
unsolicited patch adding support to my FUSE implementation,
squashfuse: https://github.com/vasi/squashfuse/pull/2
At least a few people are currently using squashfuse to mount their
LZ4-compressed squashfs images. Ideally these users would be able to
use the in-kernel squashfs for improved performance.
Cheers,
Dave Vasilevsky
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2014-12-12 7:44 ` Dave Vasilevsky
@ 2014-12-12 8:31 ` Anthony F McInerney
0 siblings, 0 replies; 21+ messages in thread
From: Anthony F McInerney @ 2014-12-12 8:31 UTC (permalink / raw)
To: Dave Vasilevsky
Cc: Phillip Lougher, linux-kernel, linux-fsdevel, texstar, martin,
Guan Xin, bruno, szycha, blyons, tokiclover, hugochevrain
Thanks very much for keeping me in the loop.
I'd like to put a big +1 on this patch inclusion.
Looking forward to in-kernel support for LZ4 squashfs images.
Thanks
Anthony
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2014-11-27 8:00 Phillip Lougher
2014-11-27 13:37 ` Bruno Wolff III
2014-12-12 7:44 ` Dave Vasilevsky
@ 2014-12-12 10:27 ` Guan Xin
[not found] ` <CAGw_SrN6tZyXRx6pxkpnhWc2U5845HRbfzhOK5v9qkjWejFpmA@mail.gmail.com>
3 siblings, 0 replies; 21+ messages in thread
From: Guan Xin @ 2014-12-12 10:27 UTC (permalink / raw)
To: Phillip Lougher
Cc: linux-kernel, linux-fsdevel, texstar, martin, Bruno Wolff III,
dave, szycha, blyons, tokiclover, afm404, hugochevrain
Hi Philip,
Thank you for your work on squashfs and your persistent effort on mainlining
squashfs-lz4 support!
I've been using squashfs with lz4 compression (together with aufs by Okajima)
for my desktop, laptops, home router and bluetooth music player for about
half a year. There was not a single issue. All I got were reduced latency
compared to lzo compression, and drastically increased compression speed.
The later is important to me because my application requires re-mksquashfs of
the whole system in order to make software updates permanent.
lz4 support (especially with -Xhc) is not only helpful, but also a
valuable addition
to squashfs.
Regards,
Guan
^ permalink raw reply [flat|nested] 21+ messages in thread[parent not found: <CAGw_SrN6tZyXRx6pxkpnhWc2U5845HRbfzhOK5v9qkjWejFpmA@mail.gmail.com>]
* Re: [PATCH 0/2] Squashfs: add LZ4 compression
[not found] ` <CAGw_SrN6tZyXRx6pxkpnhWc2U5845HRbfzhOK5v9qkjWejFpmA@mail.gmail.com>
@ 2014-12-12 15:56 ` Bruno Wolff III
2014-12-12 22:45 ` Phillip Lougher
0 siblings, 1 reply; 21+ messages in thread
From: Bruno Wolff III @ 2014-12-12 15:56 UTC (permalink / raw)
To: toki clover
Cc: Phillip Lougher, linux-kernel, linux-fsdevel, texstar, martin,
Guan Xin, dave, szycha, blyons, afm404, hugochevrain
On Fri, Dec 12, 2014 at 13:23:19 +0100,
toki clover <tokiclover@gmail.com> wrote:
>
>Now, I did not see any Linux FS devs activity/response to this... What
>a waste of time because if those patch don't make it for this merge
>window, rebasing/reposting will be, again, necessary.
The patches got pulled into linux-next. For XZ, the patches sat in
linux-next for a full release cycle. I don't know whether or not
if that is the plan for the lz4 patches.
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 0/2] Squashfs: add LZ4 compression
2014-12-12 15:56 ` Bruno Wolff III
@ 2014-12-12 22:45 ` Phillip Lougher
0 siblings, 0 replies; 21+ messages in thread
From: Phillip Lougher @ 2014-12-12 22:45 UTC (permalink / raw)
To: Bruno Wolff III, toki clover
Cc: Phillip Lougher, linux-kernel, linux-fsdevel, texstar, martin,
Guan Xin, dave, szycha, blyons, afm404, hugochevrain
On 12/12/14 15:56, Bruno Wolff III wrote:
> On Fri, Dec 12, 2014 at 13:23:19 +0100,
> toki clover <tokiclover@gmail.com> wrote:
>>
>> Now, I did not see any Linux FS devs activity/response to this... What
>> a waste of time because if those patch don't make it for this merge
>> window, rebasing/reposting will be, again, necessary.
>
> The patches got pulled into linux-next. For XZ, the patches sat in linux-next for a full release cycle. I don't know whether or not if that is the plan for the lz4 patches.
> .
>
The plan is to send out the merge request this weekend.
I've been too busy this week with my main job to do the merge request,
sorry (I'll probably be working past midnight again tonight, the
weeks leading up to Christmas are always really busy).
Phillip
^ permalink raw reply [flat|nested] 21+ messages in thread