[PATCH 1/5] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[PATCH 1/5] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[Albin Tonnerre]

These includes were added by 079effb6933f34b9b1b67b08bd4fd7fb672d16ef to
fix the build when using kmemtrace. However this is not necessary when
used to create a compressed kernel, and actually creates issues (brings
a lot of things unavailable in the decompression environment), so don't
include it if STATIC is defined.

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 lib/decompress_bunzip2.c |    2 +-
 lib/decompress_inflate.c |    2 +-
 lib/decompress_unlzma.c  |    2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c
index 708e2a8..14b0c7d 100644
--- a/lib/decompress_bunzip2.c
+++ b/lib/decompress_bunzip2.c
@@ -47,10 +47,10 @@
 
 #ifndef STATIC
 #include <linux/decompress/bunzip2.h>
+#include <linux/slab.h>
 #endif /* !STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #ifndef INT_MAX
 #define INT_MAX 0x7fffffff
diff --git a/lib/decompress_inflate.c b/lib/decompress_inflate.c
index e36b296..fc30d50 100644
--- a/lib/decompress_inflate.c
+++ b/lib/decompress_inflate.c
@@ -19,11 +19,11 @@
 #include "zlib_inflate/inflate.h"
 
 #include "zlib_inflate/infutil.h"
+#include <linux/slab.h>
 
 #endif /* STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #define INBUF_LEN (16*1024)
 
diff --git a/lib/decompress_unlzma.c b/lib/decompress_unlzma.c
index 32123a1..f078c88 100644
--- a/lib/decompress_unlzma.c
+++ b/lib/decompress_unlzma.c
@@ -31,10 +31,10 @@
 
 #ifndef STATIC
 #include <linux/decompress/unlzma.h>
+#include <linux/slab.h>
 #endif /* STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #define	MIN(a, b) (((a) < (b)) ? (a) : (b))
 
-- 
1.6.0.4

[PATCH 2/5] include/linux/unaligned/{l,b}e_byteshift.h: Fix usage for compressed kernels

[Albin Tonnerre]

When unaligned accesses are required for uncompressing a kernel (such as
for LZO decompression on ARM in a patch that follows), including
<linux/kernel.h> causes issues as it brings in a lot of things that are
not available in the decompression environment.
However, those files apparently use nothing from <linux/kernel.h>, all
they need is the declaration of types such as u32 or u64, so
<linux/types.h> should be enough

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 include/linux/unaligned/be_byteshift.h |    2 +-
 include/linux/unaligned/le_byteshift.h |    2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/linux/unaligned/be_byteshift.h b/include/linux/unaligned/be_byteshift.h
index 46dd12c..9356b24 100644
--- a/include/linux/unaligned/be_byteshift.h
+++ b/include/linux/unaligned/be_byteshift.h
@@ -1,7 +1,7 @@
 #ifndef _LINUX_UNALIGNED_BE_BYTESHIFT_H
 #define _LINUX_UNALIGNED_BE_BYTESHIFT_H
 
-#include <linux/kernel.h>
+#include <linux/types.h>
 
 static inline u16 __get_unaligned_be16(const u8 *p)
 {
diff --git a/include/linux/unaligned/le_byteshift.h b/include/linux/unaligned/le_byteshift.h
index 59777e9..be376fb 100644
--- a/include/linux/unaligned/le_byteshift.h
+++ b/include/linux/unaligned/le_byteshift.h
@@ -1,7 +1,7 @@
 #ifndef _LINUX_UNALIGNED_LE_BYTESHIFT_H
 #define _LINUX_UNALIGNED_LE_BYTESHIFT_H
 
-#include <linux/kernel.h>
+#include <linux/types.h>
 
 static inline u16 __get_unaligned_le16(const u8 *p)
 {
-- 
1.6.0.4

[PATCH 3/5] Add support for LZO-compressed kernels

[Albin Tonnerre]

This is the first part of the lzo patch
The lzo compressor is worse than gzip at compression, but faster at
extraction. Here are some figures for an ARM board I'm working on:

Uncompressed size: 3.24Mo
gzip  1.61Mo 0.72s
lzo   1.75Mo 0.48s

So for a compression ratio that is still relatively close to gzip, it's
much faster to extract, at least in that case.

This version applies to kernel 2.6.31-rc3

This part contains:
 - Makefile routine to support lzo compression
 - Fixes to the existing lzo compressor so that it can be used in
   compressed kernels
 - wrapper around the existing lzo1x_decompress, as it only extracts one
   block at a time, while we need to extract a whole file here
 - config dialog for kernel compression

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 include/linux/decompress/unlzo.h |   10 +++
 init/Kconfig                     |   18 ++++-
 lib/decompress_unlzo.c           |  138 ++++++++++++++++++++++++++++++++++++++
 lib/lzo/lzo1x_decompress.c       |    9 ++-
 scripts/Makefile.lib             |    5 ++
 5 files changed, 173 insertions(+), 7 deletions(-)
 create mode 100644 include/linux/decompress/unlzo.h
 create mode 100644 lib/decompress_unlzo.c

diff --git a/include/linux/decompress/unlzo.h b/include/linux/decompress/unlzo.h
new file mode 100644
index 0000000..d1925ea
--- /dev/null
+++ b/include/linux/decompress/unlzo.h
@@ -0,0 +1,10 @@
+#ifndef DECOMPRESS_UNLZO_H
+#define DECOMPRESS_UNLZO_H
+
+int unlzo(unsigned char *inbuf, int len,
+          int(*fill)(void*, unsigned int),
+          int(*flush)(void*, unsigned int),
+          unsigned char *output,
+          int *pos,
+          void(*error)(char *x));
+#endif
diff --git a/init/Kconfig b/init/Kconfig
index 1ce05a4..66281fb 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -115,10 +115,13 @@ config HAVE_KERNEL_BZIP2
 config HAVE_KERNEL_LZMA
 	bool
 
+config HAVE_KERNEL_LZO
+	bool
+
 choice
 	prompt "Kernel compression mode"
 	default KERNEL_GZIP
-	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA
+	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA || HAVE_KERNEL_LZO
 	help
 	  The linux kernel is a kind of self-extracting executable.
 	  Several compression algorithms are available, which differ
@@ -141,9 +144,8 @@ config KERNEL_GZIP
 	bool "Gzip"
 	depends on HAVE_KERNEL_GZIP
 	help
-	  The old and tried gzip compression. Its compression ratio is
-	  the poorest among the 3 choices; however its speed (both
-	  compression and decompression) is the fastest.
+	  The old and tried gzip compression. It provides a good balance
+	  between compression ration and decompression speed.
 
 config KERNEL_BZIP2
 	bool "Bzip2"
@@ -164,6 +166,14 @@ config KERNEL_LZMA
 	  two. Compression is slowest.	The kernel size is about 33%
 	  smaller with LZMA in comparison to gzip.
 
+config KERNEL_LZO
+	bool "LZO"
+	depends on HAVE_KERNEL_LZO
+	help
+	  Its compression ratio is the poorest among the 4. The kernel
+	  size is about about 10% bigger than gzip; however its speed
+	  (both compression and decompression) is the fastest.
+
 endchoice
 
 config SWAP
diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c
new file mode 100644
index 0000000..d908b35
--- /dev/null
+++ b/lib/decompress_unlzo.c
@@ -0,0 +1,138 @@
+/*
+ * LZO decompressor for the Linux kernel. Code borrowed from the lzo
+ * implementation by Markus Franz Xaver Johannes Oberhumer.
+ *
+ * Linux kernel adaptation:
+ * Copyright (C) 2009
+ * Albin Tonnerre, Free Electrons <albin.tonnerre@free-electrons.com>
+ *
+ * Original code:
+ * Copyright (C) 1996-2005 Markus Franz Xaver Johannes Oberhumer
+ * All Rights Reserved.
+ *
+ * lzop and the LZO library are free software; you can redistribute them
+ * and/or modify them under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.
+ * If not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Markus F.X.J. Oberhumer
+ * <markus@oberhumer.com>
+ * http://www.oberhumer.com/opensource/lzop/
+ */
+
+#ifdef STATIC
+#include "lzo/lzo1x_decompress.c"
+#endif
+
+#include <linux/lzo.h>
+#include <linux/decompress/mm.h>
+#include <linux/decompress/unlzo.h>
+
+#include <linux/compiler.h>
+#include <linux/types.h>
+#include <asm/unaligned.h>
+
+static const unsigned char lzop_magic[] =
+    { 0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a };
+
+#define BLOCK_SIZE        (256*1024l)
+#define HEADER_HAS_FILTER      0x00000800L
+
+STATIC inline int INIT parse_header(u8 *input, u8 *skip)
+{
+	int l;
+	u8 *parse = input;
+	u8 level = 0;
+	u16 version;
+
+	/* read magic: 9 first bits */
+	for (l = 0; l < 9; l++) {
+		if (*parse++ != lzop_magic[l])
+			return 0;
+	}
+	/* get version (2bytes), skip library version (2),
+	 * 'need to be extracted' version (2) and
+	 * method (1) */
+	version = get_unaligned_be16(parse);
+	parse += 7;
+	if (version >= 0x0940)
+		level = *parse++;
+	if (get_unaligned_be32(parse) & HEADER_HAS_FILTER)
+		parse += 8; /* flags + filter info */
+	else
+		parse += 4; /* flags */
+
+	/* skip mode and mtime_low */
+	parse += 8;
+	if (version >= 0x0940)
+		parse += 4;	/* skip mtime_high */
+
+	l = *parse++;
+	/* don't care about the file name, and skip checksum */
+	parse += l + 4;
+
+	*skip = parse - input;
+	return 1;
+}
+
+STATIC inline int INIT lzo_decompress(u8 *input, int in_len,
+				      int (*fill) (void *, unsigned int),
+				      int (*flush) (void *, unsigned int),
+				      u8 *output, int *posp,
+				      void (*error_fn) (char *x))
+{
+	u8 skip = 0, r = 0;
+	u32 src_len, dst_len;
+	size_t tmp;
+	u8 *in_buf = input;
+	u8 *out_buf = output;
+	int obytes_processed = 0;
+
+	if (!parse_header(input, &skip))
+		error("invalid header");
+
+	in_buf += skip;
+	for (;;) {
+		/* read uncompressed block size */
+		dst_len = get_unaligned_be32(in_buf);
+		in_buf += 4;
+
+		/* exit if last block */
+		if (dst_len == 0)
+			break;
+
+		if (dst_len > BLOCK_SIZE)
+			error("dest len longer than block size");
+
+		/* read compressed block size, and skip block checksum info */
+		src_len = get_unaligned_be32(in_buf);
+		in_buf += 8;
+
+		if (src_len <= 0 || src_len > dst_len)
+			error("file corrupted");
+
+		/* decompress */
+		tmp = dst_len;
+		r = lzo1x_decompress_safe((u8 *) in_buf, src_len, out_buf, &tmp);
+
+		if (r != LZO_E_OK || dst_len != tmp)
+			error("Compressed data violation");
+
+		obytes_processed += dst_len;
+		in_buf += src_len;
+		out_buf += dst_len;
+	}
+	return obytes_processed;
+}
+
+#define decompress lzo_decompress
diff --git a/lib/lzo/lzo1x_decompress.c b/lib/lzo/lzo1x_decompress.c
index 5dc6b29..f2fd098 100644
--- a/lib/lzo/lzo1x_decompress.c
+++ b/lib/lzo/lzo1x_decompress.c
@@ -11,11 +11,13 @@
  *  Richard Purdie <rpurdie@openedhand.com>
  */
 
+#ifndef STATIC
 #include <linux/module.h>
 #include <linux/kernel.h>
-#include <linux/lzo.h>
-#include <asm/byteorder.h>
+#endif
+
 #include <asm/unaligned.h>
+#include <linux/lzo.h>
 #include "lzodefs.h"
 
 #define HAVE_IP(x, ip_end, ip) ((size_t)(ip_end - ip) < (x))
@@ -244,9 +246,10 @@ lookbehind_overrun:
 	*out_len = op - out;
 	return LZO_E_LOOKBEHIND_OVERRUN;
 }
-
+#ifndef STATIC
 EXPORT_SYMBOL_GPL(lzo1x_decompress_safe);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("LZO1X Decompressor");
 
+#endif
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 7a77787..7b721d1 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -230,3 +230,8 @@ quiet_cmd_lzma = LZMA    $@
 cmd_lzma = (cat $(filter-out FORCE,$^) | \
 	lzma -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
 	(rm -f $@ ; false)
+
+quiet_cmd_lzo = LZO    $@
+cmd_lzo = (cat $(filter-out FORCE,$^) | \
+	lzop -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
+	(rm -f $@ ; false)
-- 
1.6.0.4

[PATCH 4/5] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

This is the second part of patch. This part includes:
 - changes to ach/arch/boot/Makefile to make it easier to add new
   compression types
 - new piggy.lzo.S necessary for lzo compression
 - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
   gzip, depending on the config
 - Kconfig support

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 arch/arm/Kconfig                      |    2 +
 arch/arm/boot/compressed/Makefile     |   16 +++--
 arch/arm/boot/compressed/misc.c       |  110 ++++++++------------------------
 arch/arm/boot/compressed/piggy.S      |    6 --
 arch/arm/boot/compressed/piggy.gzip.S |    6 ++
 arch/arm/boot/compressed/piggy.lzo.S  |    6 ++
 6 files changed, 52 insertions(+), 94 deletions(-)
 delete mode 100644 arch/arm/boot/compressed/piggy.S
 create mode 100644 arch/arm/boot/compressed/piggy.gzip.S
 create mode 100644 arch/arm/boot/compressed/piggy.lzo.S

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index aef63c8..ea71c0c 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -18,6 +18,8 @@ config ARM
 	select HAVE_KRETPROBES if (HAVE_KPROBES)
 	select HAVE_FUNCTION_TRACER if (!XIP_KERNEL)
 	select HAVE_GENERIC_DMA_COHERENT
+	select HAVE_KERNEL_GZIP
+	select HAVE_KERNEL_LZO
 	help
 	  The ARM series is a line of low-power-consumption RISC chip designs
 	  licensed by ARM Ltd and targeted at embedded applications and
diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile
index ce39dc5..4ea8f25 100644
--- a/arch/arm/boot/compressed/Makefile
+++ b/arch/arm/boot/compressed/Makefile
@@ -63,8 +63,12 @@ endif
 
 SEDFLAGS	= s/TEXT_START/$(ZTEXTADDR)/;s/BSS_START/$(ZBSSADDR)/
 
-targets       := vmlinux vmlinux.lds piggy.gz piggy.o font.o font.c \
-		 head.o misc.o $(OBJS)
+suffix_$(CONFIG_KERNEL_GZIP) = gzip
+suffix_$(CONFIG_KERNEL_LZO)  = lzo
+
+targets       := vmlinux vmlinux.lds \
+		 piggy.$(suffix_y) piggy.$(suffix_y).o \
+		 font.o font.c head.o misc.o $(OBJS)
 
 ifeq ($(CONFIG_FUNCTION_TRACER),y)
 ORIG_CFLAGS := $(KBUILD_CFLAGS)
@@ -94,15 +98,15 @@ LDFLAGS_vmlinux += -p --no-undefined -X \
 # would otherwise mess up our GOT table
 CFLAGS_misc.o := -Dstatic=
 
-$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.o \
+$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.$(suffix_y).o \
 	 	$(addprefix $(obj)/, $(OBJS)) FORCE
 	$(call if_changed,ld)
 	@:
 
-$(obj)/piggy.gz: $(obj)/../Image FORCE
-	$(call if_changed,gzip)
+$(obj)/piggy.$(suffix_y): $(obj)/../Image FORCE
+	$(call if_changed,$(suffix_y))
 
-$(obj)/piggy.o:  $(obj)/piggy.gz FORCE
+$(obj)/piggy.$(suffix_y).o:  $(obj)/piggy.$(suffix_y) FORCE
 
 CFLAGS_font.o := -Dstatic=
 
diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c
index 9e6e512..c4ec564 100644
--- a/arch/arm/boot/compressed/misc.c
+++ b/arch/arm/boot/compressed/misc.c
@@ -18,11 +18,15 @@
 
 unsigned int __machine_arch_type;
 
+#define _LINUX_STRING_H_
+
 #include <linux/compiler.h>	/* for inline */
 #include <linux/types.h>	/* for size_t */
 #include <linux/stddef.h>	/* for NULL */
 #include <asm/string.h>
 
+#include <asm/unaligned.h>
+
 #ifdef STANDALONE_DEBUG
 #define putstr printf
 #else
@@ -189,34 +193,8 @@ static inline __ptr_t memcpy(__ptr_t __dest, __const __ptr_t __src,
 /*
  * gzip delarations
  */
-#define OF(args)  args
 #define STATIC static
 
-typedef unsigned char  uch;
-typedef unsigned short ush;
-typedef unsigned long  ulg;
-
-#define WSIZE 0x8000		/* Window size must be at least 32k, */
-				/* and a power of two */
-
-static uch *inbuf;		/* input buffer */
-static uch window[WSIZE];	/* Sliding window buffer */
-
-static unsigned insize;		/* valid bytes in inbuf */
-static unsigned inptr;		/* index of next byte to be processed in inbuf */
-static unsigned outcnt;		/* bytes in output buffer */
-
-/* gzip flag byte */
-#define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
-#define CONTINUATION 0x02 /* bit 1 set: continuation of multi-part gzip file */
-#define EXTRA_FIELD  0x04 /* bit 2 set: extra field present */
-#define ORIG_NAME    0x08 /* bit 3 set: original file name present */
-#define COMMENT      0x10 /* bit 4 set: file comment present */
-#define ENCRYPTED    0x20 /* bit 5 set: file is encrypted */
-#define RESERVED     0xC0 /* bit 6,7:   reserved */
-
-#define get_byte()  (inptr < insize ? inbuf[inptr++] : fill_inbuf())
-
 /* Diagnostic functions */
 #ifdef DEBUG
 #  define Assert(cond,msg) {if(!(cond)) error(msg);}
@@ -234,24 +212,20 @@ static unsigned outcnt;		/* bytes in output buffer */
 #  define Tracecv(c,x)
 #endif
 
-static int  fill_inbuf(void);
-static void flush_window(void);
 static void error(char *m);
 
 extern char input_data[];
 extern char input_data_end[];
 
-static uch *output_data;
-static ulg output_ptr;
-static ulg bytes_out;
+static unsigned char *output_data;
+static unsigned long output_ptr;
 
 static void error(char *m);
 
 static void putstr(const char *);
 
-extern int end;
-static ulg free_mem_ptr;
-static ulg free_mem_end_ptr;
+static unsigned long free_mem_ptr;
+static unsigned long free_mem_end_ptr;
 
 #ifdef STANDALONE_DEBUG
 #define NO_INFLATE_MALLOC
@@ -259,46 +233,13 @@ static ulg free_mem_end_ptr;
 
 #define ARCH_HAS_DECOMP_WDOG
 
-#include "../../../../lib/inflate.c"
-
-/* ===========================================================================
- * Fill the input buffer. This is called only when the buffer is empty
- * and at least one byte is really needed.
- */
-int fill_inbuf(void)
-{
-	if (insize != 0)
-		error("ran out of input data");
-
-	inbuf = input_data;
-	insize = &input_data_end[0] - &input_data[0];
-
-	inptr = 1;
-	return inbuf[0];
-}
+#ifdef CONFIG_KERNEL_GZIP
+#include "../../../../lib/decompress_inflate.c"
+#endif
 
-/* ===========================================================================
- * Write the output window window[0..outcnt-1] and update crc and bytes_out.
- * (Used for the decompressed data only.)
- */
-void flush_window(void)
-{
-	ulg c = crc;
-	unsigned n;
-	uch *in, *out, ch;
-
-	in = window;
-	out = &output_data[output_ptr];
-	for (n = 0; n < outcnt; n++) {
-		ch = *out++ = *in++;
-		c = crc_32_tab[((int)c ^ ch) & 0xff] ^ (c >> 8);
-	}
-	crc = c;
-	bytes_out += (ulg)outcnt;
-	output_ptr += (ulg)outcnt;
-	outcnt = 0;
-	putstr(".");
-}
+#ifdef CONFIG_KERNEL_LZO
+#include "../../../../lib/decompress_unlzo.c"
+#endif
 
 #ifndef arch_error
 #define arch_error(x)
@@ -317,20 +258,26 @@ static void error(char *x)
 
 #ifndef STANDALONE_DEBUG
 
-ulg
-decompress_kernel(ulg output_start, ulg free_mem_ptr_p, ulg free_mem_ptr_end_p,
-		  int arch_id)
+unsigned long
+decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p,
+                  unsigned long free_mem_ptr_end_p,
+                  int arch_id)
 {
-	output_data		= (uch *)output_start;	/* Points to kernel start */
+	unsigned char *tmp;
+
+	output_data		= (unsigned char *)output_start;
 	free_mem_ptr		= free_mem_ptr_p;
 	free_mem_end_ptr	= free_mem_ptr_end_p;
 	__machine_arch_type	= arch_id;
 
 	arch_decomp_setup();
 
-	makecrc();
+	tmp = (unsigned char *) (((unsigned long)input_data_end) - 4);
+	output_ptr = get_unaligned_le32(tmp);
+
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+	           NULL, NULL, output_data, NULL, error);
 	putstr(" done, booting the kernel.\n");
 	return output_ptr;
 }
@@ -342,11 +289,10 @@ int main()
 {
 	output_data = output_buffer;
 
-	makecrc();
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+	           NULL, NULL, output_data, NULL, error);
 	putstr("done.\n");
 	return 0;
 }
 #endif
-	
diff --git a/arch/arm/boot/compressed/piggy.S b/arch/arm/boot/compressed/piggy.S
deleted file mode 100644
index 54c9518..0000000
--- a/arch/arm/boot/compressed/piggy.S
+++ /dev/null
@@ -1,6 +0,0 @@
-	.section .piggydata,#alloc
-	.globl	input_data
-input_data:
-	.incbin	"arch/arm/boot/compressed/piggy.gz"
-	.globl	input_data_end
-input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.gzip.S b/arch/arm/boot/compressed/piggy.gzip.S
new file mode 100644
index 0000000..a68adf9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.gzip.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.gzip"
+	.globl	input_data_end
+input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.lzo.S b/arch/arm/boot/compressed/piggy.lzo.S
new file mode 100644
index 0000000..a425ad9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.lzo.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.lzo"
+	.globl	input_data_end
+input_data_end:
-- 
1.6.0.4

[PATCH 5/5] Add support for LZO-compressed kernels on x86

[Albin Tonnerre]

This is the third and last part of the patch, which contains the
necessary changes to the x86 Kconfig and boot/compressed to allow the
use of this new compression method

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 arch/x86/Kconfig                  |    1 +
 arch/x86/boot/compressed/Makefile |    5 ++++-
 arch/x86/boot/compressed/misc.c   |    4 ++++
 3 files changed, 9 insertions(+), 1 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 738bdc6..49974d3 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -48,6 +48,7 @@ config X86
 	select HAVE_KERNEL_GZIP
 	select HAVE_KERNEL_BZIP2
 	select HAVE_KERNEL_LZMA
+	select HAVE_KERNEL_LZO
 	select HAVE_ARCH_KMEMCHECK
 
 config OUTPUT_FORMAT
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index e2ff504..95bfe0e 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -4,7 +4,7 @@
 # create a compressed vmlinux image from the original vmlinux
 #
 
-targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma head_$(BITS).o misc.o piggy.o
+targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.lzo head_$(BITS).o misc.o piggy.o
 
 KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2
 KBUILD_CFLAGS += -fno-strict-aliasing -fPIC
@@ -48,10 +48,13 @@ $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE
 	$(call if_changed,bzip2)
 $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE
 	$(call if_changed,lzma)
+$(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE
+	$(call if_changed,lzo)
 
 suffix-$(CONFIG_KERNEL_GZIP)	:= gz
 suffix-$(CONFIG_KERNEL_BZIP2)	:= bz2
 suffix-$(CONFIG_KERNEL_LZMA)	:= lzma
+suffix-$(CONFIG_KERNEL_LZO) 	:= lzo
 
 quiet_cmd_mkpiggy = MKPIGGY $@
       cmd_mkpiggy = $(obj)/mkpiggy $< > $@ || ( rm -f $@ ; false )
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 842b2a3..3b22fe8 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -162,6 +162,10 @@ static int lines, cols;
 #include "../../../../lib/decompress_unlzma.c"
 #endif
 
+#ifdef CONFIG_KERNEL_LZO
+#include "../../../../lib/decompress_unlzo.c"
+#endif
+
 static void scroll(void)
 {
 	int i;
-- 
1.6.0.4

[PATCH] Add LZO compression support for initramfs and old-style initrd

[Albin Tonnerre]

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 lib/Kconfig      |    4 ++++
 lib/Makefile     |    1 +
 lib/decompress.c |    5 +++++
 usr/Kconfig      |   25 ++++++++++++++++++++-----
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/lib/Kconfig b/lib/Kconfig
index bb1326d..8639349 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -117,6 +117,10 @@ config DECOMPRESS_BZIP2
 config DECOMPRESS_LZMA
 	tristate
 
+config DECOMPRESS_LZO
+	select LZO_DECOMPRESS
+	tristate
+
 #
 # Generic allocator support is selected if needed
 #
diff --git a/lib/Makefile b/lib/Makefile
index b6d1857..cd3d37b 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -69,6 +69,7 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
 lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
 lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
 lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o
+lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
 
 obj-$(CONFIG_TEXTSEARCH) += textsearch.o
 obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
diff --git a/lib/decompress.c b/lib/decompress.c
index d2842f5..a760681 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -9,6 +9,7 @@
 #include <linux/decompress/bunzip2.h>
 #include <linux/decompress/unlzma.h>
 #include <linux/decompress/inflate.h>
+#include <linux/decompress/unlzo.h>
 
 #include <linux/types.h>
 #include <linux/string.h>
@@ -22,6 +23,9 @@
 #ifndef CONFIG_DECOMPRESS_LZMA
 # define unlzma NULL
 #endif
+#ifndef CONFIG_DECOMPRESS_LZO
+# define unlzo NULL
+#endif
 
 static const struct compress_format {
 	unsigned char magic[2];
@@ -32,6 +36,7 @@ static const struct compress_format {
 	{ {037, 0236}, "gzip", gunzip },
 	{ {0x42, 0x5a}, "bzip2", bunzip2 },
 	{ {0x5d, 0x00}, "lzma", unlzma },
+	{ {0x89, 0x4c}, "lzo", unlzo },
 	{ {0, 0}, NULL, NULL }
 };
 
diff --git a/usr/Kconfig b/usr/Kconfig
index 1c3039f..04a826e 100644
--- a/usr/Kconfig
+++ b/usr/Kconfig
@@ -72,6 +72,15 @@ config RD_LZMA
 	  Support loading of a LZMA encoded initial ramdisk or cpio buffer
 	  If unsure, say N.
 
+config RD_LZO
+	bool "Support initial ramdisks compressed using LZO" if EMBEDDED
+	default !EMBEDDED
+	depends on BLK_DEV_INITRD
+	select DECOMPRESS_LZO
+	help
+	  Support loading of a LZO encoded initial ramdisk or cpio buffer
+	  If unsure, say N.
+
 choice
 	prompt "Built-in initramfs compression mode" if INITRAMFS_SOURCE!=""
 	help
@@ -108,16 +117,15 @@ config INITRAMFS_COMPRESSION_GZIP
 	bool "Gzip"
 	depends on RD_GZIP
 	help
-	  The old and tried gzip compression. Its compression ratio is
-	  the poorest among the 3 choices; however its speed (both
-	  compression and decompression) is the fastest.
+	  The old and tried gzip compression. It provides a good balance
+	  between compression ration and decompression speed.
 
 config INITRAMFS_COMPRESSION_BZIP2
 	bool "Bzip2"
 	depends on RD_BZIP2
 	help
 	  Its compression ratio and speed is intermediate.
-	  Decompression speed is slowest among the three.  The initramfs
+	  Decompression speed is slowest among the four.  The initramfs
 	  size is about 10% smaller with bzip2, in comparison to gzip.
 	  Bzip2 uses a large amount of memory. For modern kernels you
 	  will need at least 8MB RAM or more for booting.
@@ -128,7 +136,14 @@ config INITRAMFS_COMPRESSION_LZMA
 	help
 	  The most recent compression algorithm.
 	  Its ratio is best, decompression speed is between the other
-	  two. Compression is slowest.	The initramfs size is about 33%
+	  three. Compression is slowest. The initramfs size is about 33%
 	  smaller with LZMA in comparison to gzip.
 
+config INITRAMFS_COMPRESSION_LZO
+	bool "LZO"
+	depends on RD_LZO
+	help
+	  Its compression ratio is the poorest among the four. The kernel
+	  size is about about 10% bigger than gzip; however its speed
+
 endchoice
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 5/5] Add support for LZO-compressed kernels on x86

[H. Peter Anvin]

On 07/22/2009 07:01 AM, Albin Tonnerre wrote:
> This is the third and last part of the patch, which contains the
> necessary changes to the x86 Kconfig and boot/compressed to allow the
> use of this new compression method
> 
> Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>

Acked-by: H. Peter Anvin <hpa@zytor.com>

Since the patchset otherwise isn't really x86-related it probably makes
more sense to pass this through the kbuild tree, or perhaps via akpm,
rather than -tip?

	-hpa

Re: [PATCH 5/5] Add support for LZO-compressed kernels on x86

[Sam Ravnborg]

On Wed, Jul 29, 2009 at 01:00:36PM -0700, H. Peter Anvin wrote:
> On 07/22/2009 07:01 AM, Albin Tonnerre wrote:
> > This is the third and last part of the patch, which contains the
> > necessary changes to the x86 Kconfig and boot/compressed to allow the
> > use of this new compression method
> > 
> > Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
> 
> Acked-by: H. Peter Anvin <hpa@zytor.com>
> 
> Since the patchset otherwise isn't really x86-related it probably makes
> more sense to pass this through the kbuild tree, or perhaps via akpm,
> rather than -tip?

I can take it via kbuild if I get a fewsh patch-set with proper
ack's added.
I've long lost the original patch-set.

	Sam

Re: [PATCH 5/5] Add support for LZO-compressed kernels on x86

[Albin Tonnerre]

[-- Attachment #1: Type: text/plain, Size: 1195 bytes --]

On Wed, Jul 29, 2009 at 11:02:58PM +0200, Sam Ravnborg wrote :
> On Wed, Jul 29, 2009 at 01:00:36PM -0700, H. Peter Anvin wrote:
> > On 07/22/2009 07:01 AM, Albin Tonnerre wrote:
> > > This is the third and last part of the patch, which contains the
> > > necessary changes to the x86 Kconfig and boot/compressed to allow the
> > > use of this new compression method
> > > 
> > > Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
> > 
> > Acked-by: H. Peter Anvin <hpa@zytor.com>
> > 
> > Since the patchset otherwise isn't really x86-related it probably makes
> > more sense to pass this through the kbuild tree, or perhaps via akpm,
> > rather than -tip?
> 
> I can take it via kbuild if I get a fewsh patch-set with proper
> ack's added.
> I've long lost the original patch-set.

Only the x86-specific part of the patch has been acked so far, and it relies on
3 other patches which have not been acked. If there's anything I can do to get
feedback on the remaining patches, please let me know.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 835 bytes --]

Re: [PATCH 5/5] Add support for LZO-compressed kernels on x86

[Sam Ravnborg]

On Fri, Jul 31, 2009 at 09:51:19AM +0200, Albin Tonnerre wrote:
> On Wed, Jul 29, 2009 at 11:02:58PM +0200, Sam Ravnborg wrote :
> > On Wed, Jul 29, 2009 at 01:00:36PM -0700, H. Peter Anvin wrote:
> > > On 07/22/2009 07:01 AM, Albin Tonnerre wrote:
> > > > This is the third and last part of the patch, which contains the
> > > > necessary changes to the x86 Kconfig and boot/compressed to allow the
> > > > use of this new compression method
> > > > 
> > > > Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
> > > 
> > > Acked-by: H. Peter Anvin <hpa@zytor.com>
> > > 
> > > Since the patchset otherwise isn't really x86-related it probably makes
> > > more sense to pass this through the kbuild tree, or perhaps via akpm,
> > > rather than -tip?
> > 
> > I can take it via kbuild if I get a fewsh patch-set with proper
> > ack's added.
> > I've long lost the original patch-set.
> 
> Only the x86-specific part of the patch has been acked so far, and it relies on
> 3 other patches which have not been acked. If there's anything I can do to get
> feedback on the remaining patches, please let me know.

Please resend the full serie then I may take a short look at it.

	Sam

[PATCH 1/6] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[Albin Tonnerre]

These includes were added by 079effb6933f34b9b1b67b08bd4fd7fb672d16ef to
fix the build when using kmemtrace. However this is not necessary when
used to create a compressed kernel, and actually creates issues (brings
a lot of things unavailable in the decompression environment), so don't
include it if STATIC is defined.

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 lib/decompress_bunzip2.c |    2 +-
 lib/decompress_inflate.c |    2 +-
 lib/decompress_unlzma.c  |    2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c
index 708e2a8..14b0c7d 100644
--- a/lib/decompress_bunzip2.c
+++ b/lib/decompress_bunzip2.c
@@ -47,10 +47,10 @@
 
 #ifndef STATIC
 #include <linux/decompress/bunzip2.h>
+#include <linux/slab.h>
 #endif /* !STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #ifndef INT_MAX
 #define INT_MAX 0x7fffffff
diff --git a/lib/decompress_inflate.c b/lib/decompress_inflate.c
index e36b296..fc30d50 100644
--- a/lib/decompress_inflate.c
+++ b/lib/decompress_inflate.c
@@ -19,11 +19,11 @@
 #include "zlib_inflate/inflate.h"
 
 #include "zlib_inflate/infutil.h"
+#include <linux/slab.h>
 
 #endif /* STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #define INBUF_LEN (16*1024)
 
diff --git a/lib/decompress_unlzma.c b/lib/decompress_unlzma.c
index 32123a1..f078c88 100644
--- a/lib/decompress_unlzma.c
+++ b/lib/decompress_unlzma.c
@@ -31,10 +31,10 @@
 
 #ifndef STATIC
 #include <linux/decompress/unlzma.h>
+#include <linux/slab.h>
 #endif /* STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #define	MIN(a, b) (((a) < (b)) ? (a) : (b))
 
-- 
1.6.3.3

[PATCH 2/6] include/linux/unaligned/{l,b}e_byteshift.h: Fix usage for compressed kernels

[Albin Tonnerre]

When unaligned accesses are required for uncompressing a kernel (such as
for LZO decompression on ARM in a patch that follows), including
<linux/kernel.h> causes issues as it brings in a lot of things that are
not available in the decompression environment.
However, those files apparently use nothing from <linux/kernel.h>, all
they need is the declaration of types such as u32 or u64, so
<linux/types.h> should be enough

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 include/linux/unaligned/be_byteshift.h |    2 +-
 include/linux/unaligned/le_byteshift.h |    2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/linux/unaligned/be_byteshift.h b/include/linux/unaligned/be_byteshift.h
index 46dd12c..9356b24 100644
--- a/include/linux/unaligned/be_byteshift.h
+++ b/include/linux/unaligned/be_byteshift.h
@@ -1,7 +1,7 @@
 #ifndef _LINUX_UNALIGNED_BE_BYTESHIFT_H
 #define _LINUX_UNALIGNED_BE_BYTESHIFT_H
 
-#include <linux/kernel.h>
+#include <linux/types.h>
 
 static inline u16 __get_unaligned_be16(const u8 *p)
 {
diff --git a/include/linux/unaligned/le_byteshift.h b/include/linux/unaligned/le_byteshift.h
index 59777e9..be376fb 100644
--- a/include/linux/unaligned/le_byteshift.h
+++ b/include/linux/unaligned/le_byteshift.h
@@ -1,7 +1,7 @@
 #ifndef _LINUX_UNALIGNED_LE_BYTESHIFT_H
 #define _LINUX_UNALIGNED_LE_BYTESHIFT_H
 
-#include <linux/kernel.h>
+#include <linux/types.h>
 
 static inline u16 __get_unaligned_le16(const u8 *p)
 {
-- 
1.6.3.3

[PATCH 3/6] Add support for LZO-compressed kernels

[Albin Tonnerre]

This is the first part of the lzo patch
The lzo compressor is worse than gzip at compression, but faster at
extraction. Here are some figures for an ARM board I'm working on:

Uncompressed size: 3.24Mo
gzip  1.61Mo 0.72s
lzo   1.75Mo 0.48s

So for a compression ratio that is still relatively close to gzip, it's
much faster to extract, at least in that case.

This version applies to kernel 2.6.31-rc3

This part contains:
 - Makefile routine to support lzo compression
 - Fixes to the existing lzo compressor so that it can be used in
   compressed kernels
 - wrapper around the existing lzo1x_decompress, as it only extracts one
   block at a time, while we need to extract a whole file here
 - config dialog for kernel compression

Changelog since v1:

lib/decompress_unlzo.c
 - Rename lzo_decompress to unlzo to match the prototype in decompress/unlzo.h
 - Use LZO_BLOCK_SIZE instead of BLOCK_SIZE to avoid confusion
 - Add support for using the posp, fill and flush arguments
 - Reorder includes so that linux/types.h is included before
   linux/lzo.h. This prevents issue when not used as bootstrap code, as
   lzo.h needs things like size_t
 - When we are not using unlzo as part of kernel bootstrap code, we need
   linux/slab.h for memory allocation functions.
 - ... and we also need a call to set_error_fn so that the provided error
   function is used correctly

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 include/linux/decompress/unlzo.h |   10 ++
 init/Kconfig                     |   18 +++-
 lib/decompress_unlzo.c           |  206 ++++++++++++++++++++++++++++++++++++++
 lib/lzo/lzo1x_decompress.c       |    9 +-
 scripts/Makefile.lib             |    5 +
 5 files changed, 241 insertions(+), 7 deletions(-)
 create mode 100644 include/linux/decompress/unlzo.h
 create mode 100644 lib/decompress_unlzo.c

diff --git a/include/linux/decompress/unlzo.h b/include/linux/decompress/unlzo.h
new file mode 100644
index 0000000..d1925ea
--- /dev/null
+++ b/include/linux/decompress/unlzo.h
@@ -0,0 +1,10 @@
+#ifndef DECOMPRESS_UNLZO_H
+#define DECOMPRESS_UNLZO_H
+
+int unlzo(unsigned char *inbuf, int len,
+          int(*fill)(void*, unsigned int),
+          int(*flush)(void*, unsigned int),
+          unsigned char *output,
+          int *pos,
+          void(*error)(char *x));
+#endif
diff --git a/init/Kconfig b/init/Kconfig
index cb2c092..ada6182 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -115,10 +115,13 @@ config HAVE_KERNEL_BZIP2
 config HAVE_KERNEL_LZMA
 	bool
 
+config HAVE_KERNEL_LZO
+	bool
+
 choice
 	prompt "Kernel compression mode"
 	default KERNEL_GZIP
-	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA
+	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA || HAVE_KERNEL_LZO
 	help
 	  The linux kernel is a kind of self-extracting executable.
 	  Several compression algorithms are available, which differ
@@ -141,9 +144,8 @@ config KERNEL_GZIP
 	bool "Gzip"
 	depends on HAVE_KERNEL_GZIP
 	help
-	  The old and tried gzip compression. Its compression ratio is
-	  the poorest among the 3 choices; however its speed (both
-	  compression and decompression) is the fastest.
+	  The old and tried gzip compression. It provides a good balance
+	  between compression ration and decompression speed.
 
 config KERNEL_BZIP2
 	bool "Bzip2"
@@ -164,6 +166,14 @@ config KERNEL_LZMA
 	  two. Compression is slowest.	The kernel size is about 33%
 	  smaller with LZMA in comparison to gzip.
 
+config KERNEL_LZO
+	bool "LZO"
+	depends on HAVE_KERNEL_LZO
+	help
+	  Its compression ratio is the poorest among the 4. The kernel
+	  size is about about 10% bigger than gzip; however its speed
+	  (both compression and decompression) is the fastest.
+
 endchoice
 
 config SWAP
diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c
new file mode 100644
index 0000000..a18417a
--- /dev/null
+++ b/lib/decompress_unlzo.c
@@ -0,0 +1,206 @@
+/*
+ * LZO decompressor for the Linux kernel. Code borrowed from the lzo
+ * implementation by Markus Franz Xaver Johannes Oberhumer.
+ *
+ * Linux kernel adaptation:
+ * Copyright (C) 2009
+ * Albin Tonnerre, Free Electrons <albin.tonnerre@free-electrons.com>
+ *
+ * Original code:
+ * Copyright (C) 1996-2005 Markus Franz Xaver Johannes Oberhumer
+ * All Rights Reserved.
+ *
+ * lzop and the LZO library are free software; you can redistribute them
+ * and/or modify them under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.
+ * If not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Markus F.X.J. Oberhumer
+ * <markus@oberhumer.com>
+ * http://www.oberhumer.com/opensource/lzop/
+ */
+
+#ifdef STATIC
+#include "lzo/lzo1x_decompress.c"
+#else
+#include <linux/slab.h>
+#endif
+
+#include <linux/types.h>
+#include <linux/lzo.h>
+#include <linux/decompress/mm.h>
+#include <linux/decompress/unlzo.h>
+
+#include <linux/compiler.h>
+#include <asm/unaligned.h>
+
+static const unsigned char lzop_magic[] =
+    { 0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a };
+
+#define LZO_BLOCK_SIZE        (256*1024l)
+#define HEADER_HAS_FILTER      0x00000800L
+
+STATIC inline int INIT parse_header(u8 *input, u8 *skip)
+{
+	int l;
+	u8 *parse = input;
+	u8 level = 0;
+	u16 version;
+
+	/* read magic: 9 first bits */
+	for (l = 0; l < 9; l++) {
+		if (*parse++ != lzop_magic[l])
+			return 0;
+	}
+	/* get version (2bytes), skip library version (2),
+	 * 'need to be extracted' version (2) and
+	 * method (1) */
+	version = get_unaligned_be16(parse);
+	parse += 7;
+	if (version >= 0x0940)
+		level = *parse++;
+	if (get_unaligned_be32(parse) & HEADER_HAS_FILTER)
+		parse += 8; /* flags + filter info */
+	else
+		parse += 4; /* flags */
+
+	/* skip mode and mtime_low */
+	parse += 8;
+	if (version >= 0x0940)
+		parse += 4;	/* skip mtime_high */
+
+	l = *parse++;
+	/* don't care about the file name, and skip checksum */
+	parse += l + 4;
+
+	*skip = parse - input;
+	return 1;
+}
+
+STATIC inline int INIT unlzo(u8 *input, int in_len,
+                             int (*fill) (void *, unsigned int),
+                             int (*flush) (void *, unsigned int),
+                             u8 *output, int *posp,
+                             void (*error_fn) (char *x))
+{
+	u8 skip = 0, r = 0;
+	u32 src_len, dst_len;
+	size_t tmp;
+	u8 *in_buf, *in_buf_save, *out_buf;
+	int obytes_processed = 0;
+
+	set_error_fn(error_fn);
+
+	if (output)
+		out_buf = output;
+	else if (!flush) {
+		error("NULL output pointer and no flush function provided");
+		goto exit;
+	}
+	else if (!(out_buf = malloc(LZO_BLOCK_SIZE))) {
+		error("Could not allocate output buffer");
+		goto exit;
+	}
+
+	if (input && fill) {
+		error("Both input pointer and fill function provided, don't know what to do");
+		goto exit_1;
+	}
+	else if (input)
+		in_buf = input;
+	else if (!fill || !posp) {
+		error("NULL input pointer and missing position pointer or fill function");
+		goto exit_1;
+	}
+	else if (!(in_buf = malloc(lzo1x_worst_compress(LZO_BLOCK_SIZE)))) {
+		error("Could not allocate input buffer");
+		goto exit_1;
+	}
+	in_buf_save = in_buf;
+
+	if (posp)
+		*posp = 0;
+
+	if (fill)
+		fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE));
+
+	if (!parse_header(input, &skip)) {
+		error("invalid header");
+		goto exit_2;
+	}
+	in_buf += skip;
+
+	if (posp)
+		*posp = skip;
+
+	for (;;) {
+		/* read uncompressed block size */
+		dst_len = get_unaligned_be32(in_buf);
+		in_buf += 4;
+
+		/* exit if last block */
+		if (dst_len == 0) {
+			if (posp)
+				*posp += 4;
+			break;
+		}
+
+		if (dst_len > LZO_BLOCK_SIZE) {
+			error("dest len longer than block size");
+			goto exit_2;
+		}
+
+		/* read compressed block size, and skip block checksum info */
+		src_len = get_unaligned_be32(in_buf);
+		in_buf += 8;
+
+		if (src_len <= 0 || src_len > dst_len) {
+			error("file corrupted");
+			goto exit_2;
+		}
+
+		/* decompress */
+		tmp = dst_len;
+		r = lzo1x_decompress_safe((u8 *) in_buf, src_len, out_buf, &tmp);
+
+		if (r != LZO_E_OK || dst_len != tmp) {
+			error("Compressed data violation");
+			goto exit_2;
+		}
+
+		obytes_processed += dst_len;
+		if (flush)
+			flush(out_buf, dst_len);
+		if (output)
+			out_buf += dst_len;
+		if (posp)
+			*posp += src_len + 12;
+		if (fill) {
+			in_buf = in_buf_save;
+			fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE));
+		}
+		else
+			in_buf += src_len;
+	}
+
+exit_2:
+	if (!input)
+		free(in_buf);
+exit_1:
+	if (!output)
+		free(out_buf);
+exit:
+	return obytes_processed;
+}
+
+#define decompress unlzo
diff --git a/lib/lzo/lzo1x_decompress.c b/lib/lzo/lzo1x_decompress.c
index 5dc6b29..f2fd098 100644
--- a/lib/lzo/lzo1x_decompress.c
+++ b/lib/lzo/lzo1x_decompress.c
@@ -11,11 +11,13 @@
  *  Richard Purdie <rpurdie@openedhand.com>
  */
 
+#ifndef STATIC
 #include <linux/module.h>
 #include <linux/kernel.h>
-#include <linux/lzo.h>
-#include <asm/byteorder.h>
+#endif
+
 #include <asm/unaligned.h>
+#include <linux/lzo.h>
 #include "lzodefs.h"
 
 #define HAVE_IP(x, ip_end, ip) ((size_t)(ip_end - ip) < (x))
@@ -244,9 +246,10 @@ lookbehind_overrun:
 	*out_len = op - out;
 	return LZO_E_LOOKBEHIND_OVERRUN;
 }
-
+#ifndef STATIC
 EXPORT_SYMBOL_GPL(lzo1x_decompress_safe);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("LZO1X Decompressor");
 
+#endif
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 7a77787..7b721d1 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -230,3 +230,8 @@ quiet_cmd_lzma = LZMA    $@
 cmd_lzma = (cat $(filter-out FORCE,$^) | \
 	lzma -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
 	(rm -f $@ ; false)
+
+quiet_cmd_lzo = LZO    $@
+cmd_lzo = (cat $(filter-out FORCE,$^) | \
+	lzop -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
+	(rm -f $@ ; false)
-- 
1.6.3.3

[PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

This is the second part of patch. This part includes:
 - changes to ach/arch/boot/Makefile to make it easier to add new
   compression types
 - new piggy.lzo.S necessary for lzo compression
 - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
   gzip, depending on the config
 - Kconfig support

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 arch/arm/Kconfig                      |    2 +
 arch/arm/boot/compressed/Makefile     |   16 +++--
 arch/arm/boot/compressed/misc.c       |  110 ++++++++------------------------
 arch/arm/boot/compressed/piggy.S      |    6 --
 arch/arm/boot/compressed/piggy.gzip.S |    6 ++
 arch/arm/boot/compressed/piggy.lzo.S  |    6 ++
 6 files changed, 52 insertions(+), 94 deletions(-)
 delete mode 100644 arch/arm/boot/compressed/piggy.S
 create mode 100644 arch/arm/boot/compressed/piggy.gzip.S
 create mode 100644 arch/arm/boot/compressed/piggy.lzo.S

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index aef63c8..ea71c0c 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -18,6 +18,8 @@ config ARM
 	select HAVE_KRETPROBES if (HAVE_KPROBES)
 	select HAVE_FUNCTION_TRACER if (!XIP_KERNEL)
 	select HAVE_GENERIC_DMA_COHERENT
+	select HAVE_KERNEL_GZIP
+	select HAVE_KERNEL_LZO
 	help
 	  The ARM series is a line of low-power-consumption RISC chip designs
 	  licensed by ARM Ltd and targeted at embedded applications and
diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile
index ce39dc5..4ea8f25 100644
--- a/arch/arm/boot/compressed/Makefile
+++ b/arch/arm/boot/compressed/Makefile
@@ -63,8 +63,12 @@ endif
 
 SEDFLAGS	= s/TEXT_START/$(ZTEXTADDR)/;s/BSS_START/$(ZBSSADDR)/
 
-targets       := vmlinux vmlinux.lds piggy.gz piggy.o font.o font.c \
-		 head.o misc.o $(OBJS)
+suffix_$(CONFIG_KERNEL_GZIP) = gzip
+suffix_$(CONFIG_KERNEL_LZO)  = lzo
+
+targets       := vmlinux vmlinux.lds \
+		 piggy.$(suffix_y) piggy.$(suffix_y).o \
+		 font.o font.c head.o misc.o $(OBJS)
 
 ifeq ($(CONFIG_FUNCTION_TRACER),y)
 ORIG_CFLAGS := $(KBUILD_CFLAGS)
@@ -94,15 +98,15 @@ LDFLAGS_vmlinux += -p --no-undefined -X \
 # would otherwise mess up our GOT table
 CFLAGS_misc.o := -Dstatic=
 
-$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.o \
+$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.$(suffix_y).o \
 	 	$(addprefix $(obj)/, $(OBJS)) FORCE
 	$(call if_changed,ld)
 	@:
 
-$(obj)/piggy.gz: $(obj)/../Image FORCE
-	$(call if_changed,gzip)
+$(obj)/piggy.$(suffix_y): $(obj)/../Image FORCE
+	$(call if_changed,$(suffix_y))
 
-$(obj)/piggy.o:  $(obj)/piggy.gz FORCE
+$(obj)/piggy.$(suffix_y).o:  $(obj)/piggy.$(suffix_y) FORCE
 
 CFLAGS_font.o := -Dstatic=
 
diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c
index 9e6e512..c4ec564 100644
--- a/arch/arm/boot/compressed/misc.c
+++ b/arch/arm/boot/compressed/misc.c
@@ -18,11 +18,15 @@
 
 unsigned int __machine_arch_type;
 
+#define _LINUX_STRING_H_
+
 #include <linux/compiler.h>	/* for inline */
 #include <linux/types.h>	/* for size_t */
 #include <linux/stddef.h>	/* for NULL */
 #include <asm/string.h>
 
+#include <asm/unaligned.h>
+
 #ifdef STANDALONE_DEBUG
 #define putstr printf
 #else
@@ -189,34 +193,8 @@ static inline __ptr_t memcpy(__ptr_t __dest, __const __ptr_t __src,
 /*
  * gzip delarations
  */
-#define OF(args)  args
 #define STATIC static
 
-typedef unsigned char  uch;
-typedef unsigned short ush;
-typedef unsigned long  ulg;
-
-#define WSIZE 0x8000		/* Window size must be at least 32k, */
-				/* and a power of two */
-
-static uch *inbuf;		/* input buffer */
-static uch window[WSIZE];	/* Sliding window buffer */
-
-static unsigned insize;		/* valid bytes in inbuf */
-static unsigned inptr;		/* index of next byte to be processed in inbuf */
-static unsigned outcnt;		/* bytes in output buffer */
-
-/* gzip flag byte */
-#define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
-#define CONTINUATION 0x02 /* bit 1 set: continuation of multi-part gzip file */
-#define EXTRA_FIELD  0x04 /* bit 2 set: extra field present */
-#define ORIG_NAME    0x08 /* bit 3 set: original file name present */
-#define COMMENT      0x10 /* bit 4 set: file comment present */
-#define ENCRYPTED    0x20 /* bit 5 set: file is encrypted */
-#define RESERVED     0xC0 /* bit 6,7:   reserved */
-
-#define get_byte()  (inptr < insize ? inbuf[inptr++] : fill_inbuf())
-
 /* Diagnostic functions */
 #ifdef DEBUG
 #  define Assert(cond,msg) {if(!(cond)) error(msg);}
@@ -234,24 +212,20 @@ static unsigned outcnt;		/* bytes in output buffer */
 #  define Tracecv(c,x)
 #endif
 
-static int  fill_inbuf(void);
-static void flush_window(void);
 static void error(char *m);
 
 extern char input_data[];
 extern char input_data_end[];
 
-static uch *output_data;
-static ulg output_ptr;
-static ulg bytes_out;
+static unsigned char *output_data;
+static unsigned long output_ptr;
 
 static void error(char *m);
 
 static void putstr(const char *);
 
-extern int end;
-static ulg free_mem_ptr;
-static ulg free_mem_end_ptr;
+static unsigned long free_mem_ptr;
+static unsigned long free_mem_end_ptr;
 
 #ifdef STANDALONE_DEBUG
 #define NO_INFLATE_MALLOC
@@ -259,46 +233,13 @@ static ulg free_mem_end_ptr;
 
 #define ARCH_HAS_DECOMP_WDOG
 
-#include "../../../../lib/inflate.c"
-
-/* ===========================================================================
- * Fill the input buffer. This is called only when the buffer is empty
- * and at least one byte is really needed.
- */
-int fill_inbuf(void)
-{
-	if (insize != 0)
-		error("ran out of input data");
-
-	inbuf = input_data;
-	insize = &input_data_end[0] - &input_data[0];
-
-	inptr = 1;
-	return inbuf[0];
-}
+#ifdef CONFIG_KERNEL_GZIP
+#include "../../../../lib/decompress_inflate.c"
+#endif
 
-/* ===========================================================================
- * Write the output window window[0..outcnt-1] and update crc and bytes_out.
- * (Used for the decompressed data only.)
- */
-void flush_window(void)
-{
-	ulg c = crc;
-	unsigned n;
-	uch *in, *out, ch;
-
-	in = window;
-	out = &output_data[output_ptr];
-	for (n = 0; n < outcnt; n++) {
-		ch = *out++ = *in++;
-		c = crc_32_tab[((int)c ^ ch) & 0xff] ^ (c >> 8);
-	}
-	crc = c;
-	bytes_out += (ulg)outcnt;
-	output_ptr += (ulg)outcnt;
-	outcnt = 0;
-	putstr(".");
-}
+#ifdef CONFIG_KERNEL_LZO
+#include "../../../../lib/decompress_unlzo.c"
+#endif
 
 #ifndef arch_error
 #define arch_error(x)
@@ -317,20 +258,26 @@ static void error(char *x)
 
 #ifndef STANDALONE_DEBUG
 
-ulg
-decompress_kernel(ulg output_start, ulg free_mem_ptr_p, ulg free_mem_ptr_end_p,
-		  int arch_id)
+unsigned long
+decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p,
+                  unsigned long free_mem_ptr_end_p,
+                  int arch_id)
 {
-	output_data		= (uch *)output_start;	/* Points to kernel start */
+	unsigned char *tmp;
+
+	output_data		= (unsigned char *)output_start;
 	free_mem_ptr		= free_mem_ptr_p;
 	free_mem_end_ptr	= free_mem_ptr_end_p;
 	__machine_arch_type	= arch_id;
 
 	arch_decomp_setup();
 
-	makecrc();
+	tmp = (unsigned char *) (((unsigned long)input_data_end) - 4);
+	output_ptr = get_unaligned_le32(tmp);
+
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+	           NULL, NULL, output_data, NULL, error);
 	putstr(" done, booting the kernel.\n");
 	return output_ptr;
 }
@@ -342,11 +289,10 @@ int main()
 {
 	output_data = output_buffer;
 
-	makecrc();
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+	           NULL, NULL, output_data, NULL, error);
 	putstr("done.\n");
 	return 0;
 }
 #endif
-	
diff --git a/arch/arm/boot/compressed/piggy.S b/arch/arm/boot/compressed/piggy.S
deleted file mode 100644
index 54c9518..0000000
--- a/arch/arm/boot/compressed/piggy.S
+++ /dev/null
@@ -1,6 +0,0 @@
-	.section .piggydata,#alloc
-	.globl	input_data
-input_data:
-	.incbin	"arch/arm/boot/compressed/piggy.gz"
-	.globl	input_data_end
-input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.gzip.S b/arch/arm/boot/compressed/piggy.gzip.S
new file mode 100644
index 0000000..a68adf9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.gzip.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.gzip"
+	.globl	input_data_end
+input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.lzo.S b/arch/arm/boot/compressed/piggy.lzo.S
new file mode 100644
index 0000000..a425ad9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.lzo.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.lzo"
+	.globl	input_data_end
+input_data_end:
-- 
1.6.3.3

[PATCH 5/6] Add support for LZO-compressed kernels on x86

[Albin Tonnerre]

This is the third and last part of the patch, which contains the
necessary changes to the x86 Kconfig and boot/compressed to allow the
use of this new compression method

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 arch/x86/Kconfig                  |    1 +
 arch/x86/boot/compressed/Makefile |    5 ++++-
 arch/x86/boot/compressed/misc.c   |    4 ++++
 3 files changed, 9 insertions(+), 1 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 738bdc6..49974d3 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -48,6 +48,7 @@ config X86
 	select HAVE_KERNEL_GZIP
 	select HAVE_KERNEL_BZIP2
 	select HAVE_KERNEL_LZMA
+	select HAVE_KERNEL_LZO
 	select HAVE_ARCH_KMEMCHECK
 
 config OUTPUT_FORMAT
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index e2ff504..95bfe0e 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -4,7 +4,7 @@
 # create a compressed vmlinux image from the original vmlinux
 #
 
-targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma head_$(BITS).o misc.o piggy.o
+targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.lzo head_$(BITS).o misc.o piggy.o
 
 KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2
 KBUILD_CFLAGS += -fno-strict-aliasing -fPIC
@@ -48,10 +48,13 @@ $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE
 	$(call if_changed,bzip2)
 $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE
 	$(call if_changed,lzma)
+$(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE
+	$(call if_changed,lzo)
 
 suffix-$(CONFIG_KERNEL_GZIP)	:= gz
 suffix-$(CONFIG_KERNEL_BZIP2)	:= bz2
 suffix-$(CONFIG_KERNEL_LZMA)	:= lzma
+suffix-$(CONFIG_KERNEL_LZO) 	:= lzo
 
 quiet_cmd_mkpiggy = MKPIGGY $@
       cmd_mkpiggy = $(obj)/mkpiggy $< > $@ || ( rm -f $@ ; false )
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 842b2a3..3b22fe8 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -162,6 +162,10 @@ static int lines, cols;
 #include "../../../../lib/decompress_unlzma.c"
 #endif
 
+#ifdef CONFIG_KERNEL_LZO
+#include "../../../../lib/decompress_unlzo.c"
+#endif
+
 static void scroll(void)
 {
 	int i;
-- 
1.6.3.3

[PATCH 6/6] Add LZO compression support for initramfs and old-style initrd

[Albin Tonnerre]

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 lib/Kconfig      |    4 ++++
 lib/Makefile     |    1 +
 lib/decompress.c |    5 +++++
 usr/Kconfig      |   25 ++++++++++++++++++++-----
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/lib/Kconfig b/lib/Kconfig
index bb1326d..8639349 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -117,6 +117,10 @@ config DECOMPRESS_BZIP2
 config DECOMPRESS_LZMA
 	tristate
 
+config DECOMPRESS_LZO
+	select LZO_DECOMPRESS
+	tristate
+
 #
 # Generic allocator support is selected if needed
 #
diff --git a/lib/Makefile b/lib/Makefile
index 2e78277..cfa4041 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -69,6 +69,7 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
 lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
 lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
 lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o
+lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
 
 obj-$(CONFIG_TEXTSEARCH) += textsearch.o
 obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
diff --git a/lib/decompress.c b/lib/decompress.c
index d2842f5..a760681 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -9,6 +9,7 @@
 #include <linux/decompress/bunzip2.h>
 #include <linux/decompress/unlzma.h>
 #include <linux/decompress/inflate.h>
+#include <linux/decompress/unlzo.h>
 
 #include <linux/types.h>
 #include <linux/string.h>
@@ -22,6 +23,9 @@
 #ifndef CONFIG_DECOMPRESS_LZMA
 # define unlzma NULL
 #endif
+#ifndef CONFIG_DECOMPRESS_LZO
+# define unlzo NULL
+#endif
 
 static const struct compress_format {
 	unsigned char magic[2];
@@ -32,6 +36,7 @@ static const struct compress_format {
 	{ {037, 0236}, "gzip", gunzip },
 	{ {0x42, 0x5a}, "bzip2", bunzip2 },
 	{ {0x5d, 0x00}, "lzma", unlzma },
+	{ {0x89, 0x4c}, "lzo", unlzo },
 	{ {0, 0}, NULL, NULL }
 };
 
diff --git a/usr/Kconfig b/usr/Kconfig
index 1c3039f..04a826e 100644
--- a/usr/Kconfig
+++ b/usr/Kconfig
@@ -72,6 +72,15 @@ config RD_LZMA
 	  Support loading of a LZMA encoded initial ramdisk or cpio buffer
 	  If unsure, say N.
 
+config RD_LZO
+	bool "Support initial ramdisks compressed using LZO" if EMBEDDED
+	default !EMBEDDED
+	depends on BLK_DEV_INITRD
+	select DECOMPRESS_LZO
+	help
+	  Support loading of a LZO encoded initial ramdisk or cpio buffer
+	  If unsure, say N.
+
 choice
 	prompt "Built-in initramfs compression mode" if INITRAMFS_SOURCE!=""
 	help
@@ -108,16 +117,15 @@ config INITRAMFS_COMPRESSION_GZIP
 	bool "Gzip"
 	depends on RD_GZIP
 	help
-	  The old and tried gzip compression. Its compression ratio is
-	  the poorest among the 3 choices; however its speed (both
-	  compression and decompression) is the fastest.
+	  The old and tried gzip compression. It provides a good balance
+	  between compression ration and decompression speed.
 
 config INITRAMFS_COMPRESSION_BZIP2
 	bool "Bzip2"
 	depends on RD_BZIP2
 	help
 	  Its compression ratio and speed is intermediate.
-	  Decompression speed is slowest among the three.  The initramfs
+	  Decompression speed is slowest among the four.  The initramfs
 	  size is about 10% smaller with bzip2, in comparison to gzip.
 	  Bzip2 uses a large amount of memory. For modern kernels you
 	  will need at least 8MB RAM or more for booting.
@@ -128,7 +136,14 @@ config INITRAMFS_COMPRESSION_LZMA
 	help
 	  The most recent compression algorithm.
 	  Its ratio is best, decompression speed is between the other
-	  two. Compression is slowest.	The initramfs size is about 33%
+	  three. Compression is slowest. The initramfs size is about 33%
 	  smaller with LZMA in comparison to gzip.
 
+config INITRAMFS_COMPRESSION_LZO
+	bool "LZO"
+	depends on RD_LZO
+	help
+	  Its compression ratio is the poorest among the four. The kernel
+	  size is about about 10% bigger than gzip; however its speed
+
 endchoice
-- 
1.6.3.3

Re: [PATCH 6/6] Add LZO compression support for initramfs and old-style initrd

[H. Peter Anvin]

On 08/03/2009 07:58 AM, Albin Tonnerre wrote:
>  
> +config INITRAMFS_COMPRESSION_LZO
> +	bool "LZO"
> +	depends on RD_LZO
> +	help
> +	  Its compression ratio is the poorest among the four. The kernel
> +	  size is about about 10% bigger than gzip; however its speed
> +
>  endchoice

Truncated help text...

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH 6/6] Add LZO compression support for initramfs and old-style initrd

[Albin Tonnerre]

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---

Updated with a missing part of the description for INITRAMFS_COMPRESSION_LZO

 lib/Kconfig      |    4 ++++
 lib/Makefile     |    1 +
 lib/decompress.c |    5 +++++
 usr/Kconfig      |   26 +++++++++++++++++++++-----
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/lib/Kconfig b/lib/Kconfig
index bb1326d..8639349 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -117,6 +117,10 @@ config DECOMPRESS_BZIP2
 config DECOMPRESS_LZMA
 	tristate
 
+config DECOMPRESS_LZO
+	select LZO_DECOMPRESS
+	tristate
+
 #
 # Generic allocator support is selected if needed
 #
diff --git a/lib/Makefile b/lib/Makefile
index 2e78277..cfa4041 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -69,6 +69,7 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
 lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
 lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
 lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o
+lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
 
 obj-$(CONFIG_TEXTSEARCH) += textsearch.o
 obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
diff --git a/lib/decompress.c b/lib/decompress.c
index d2842f5..a760681 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -9,6 +9,7 @@
 #include <linux/decompress/bunzip2.h>
 #include <linux/decompress/unlzma.h>
 #include <linux/decompress/inflate.h>
+#include <linux/decompress/unlzo.h>
 
 #include <linux/types.h>
 #include <linux/string.h>
@@ -22,6 +23,9 @@
 #ifndef CONFIG_DECOMPRESS_LZMA
 # define unlzma NULL
 #endif
+#ifndef CONFIG_DECOMPRESS_LZO
+# define unlzo NULL
+#endif
 
 static const struct compress_format {
 	unsigned char magic[2];
@@ -32,6 +36,7 @@ static const struct compress_format {
 	{ {037, 0236}, "gzip", gunzip },
 	{ {0x42, 0x5a}, "bzip2", bunzip2 },
 	{ {0x5d, 0x00}, "lzma", unlzma },
+	{ {0x89, 0x4c}, "lzo", unlzo },
 	{ {0, 0}, NULL, NULL }
 };
 
diff --git a/usr/Kconfig b/usr/Kconfig
index 1c3039f..bf30c32 100644
--- a/usr/Kconfig
+++ b/usr/Kconfig
@@ -72,6 +72,15 @@ config RD_LZMA
 	  Support loading of a LZMA encoded initial ramdisk or cpio buffer
 	  If unsure, say N.
 
+config RD_LZO
+	bool "Support initial ramdisks compressed using LZO" if EMBEDDED
+	default !EMBEDDED
+	depends on BLK_DEV_INITRD
+	select DECOMPRESS_LZO
+	help
+	  Support loading of a LZO encoded initial ramdisk or cpio buffer
+	  If unsure, say N.
+
 choice
 	prompt "Built-in initramfs compression mode" if INITRAMFS_SOURCE!=""
 	help
@@ -108,16 +117,15 @@ config INITRAMFS_COMPRESSION_GZIP
 	bool "Gzip"
 	depends on RD_GZIP
 	help
-	  The old and tried gzip compression. Its compression ratio is
-	  the poorest among the 3 choices; however its speed (both
-	  compression and decompression) is the fastest.
+	  The old and tried gzip compression. It provides a good balance
+	  between compression ration and decompression speed.
 
 config INITRAMFS_COMPRESSION_BZIP2
 	bool "Bzip2"
 	depends on RD_BZIP2
 	help
 	  Its compression ratio and speed is intermediate.
-	  Decompression speed is slowest among the three.  The initramfs
+	  Decompression speed is slowest among the four.  The initramfs
 	  size is about 10% smaller with bzip2, in comparison to gzip.
 	  Bzip2 uses a large amount of memory. For modern kernels you
 	  will need at least 8MB RAM or more for booting.
@@ -128,7 +136,15 @@ config INITRAMFS_COMPRESSION_LZMA
 	help
 	  The most recent compression algorithm.
 	  Its ratio is best, decompression speed is between the other
-	  two. Compression is slowest.	The initramfs size is about 33%
+	  three. Compression is slowest. The initramfs size is about 33%
 	  smaller with LZMA in comparison to gzip.
 
+config INITRAMFS_COMPRESSION_LZO
+	bool "LZO"
+	depends on RD_LZO
+	help
+	  Its compression ratio is the poorest among the four. The kernel
+	  size is about about 10% bigger than gzip; however its speed
+	  (both compression and decompression) is the fastest.
+
 endchoice
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 5/6] Add support for LZO-compressed kernels on x86

[H. Peter Anvin]

On 08/03/2009 07:58 AM, Albin Tonnerre wrote:
> This is the third and last part of the patch, which contains the
> necessary changes to the x86 Kconfig and boot/compressed to allow the
> use of this new compression method
> 
> Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
> ---
>  arch/x86/Kconfig                  |    1 +
>  arch/x86/boot/compressed/Makefile |    5 ++++-
>  arch/x86/boot/compressed/misc.c   |    4 ++++
>  3 files changed, 9 insertions(+), 1 deletions(-)
> 

Acked-by: H. Peter Anvin <hpa@zytor.com>

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Russell King - ARM Linux]

On Mon, Aug 03, 2009 at 04:58:19PM +0200, Albin Tonnerre wrote:
> This is the second part of patch. This part includes:
>  - changes to ach/arch/boot/Makefile to make it easier to add new
>    compression types
>  - new piggy.lzo.S necessary for lzo compression
>  - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
>    gzip, depending on the config
>  - Kconfig support

FYI, with these patches applied and selecting GZIP method, I get
linker errors.  I've been unable to track down what's going on, but
it appears to be a libgcc issue.

In spite of the decompressor being built as an EABI object, gcc seems
to be issuing calls to __umodsi3, which isn't in the EABI libgcc
(they're called something different - don't ask.)

So I think these patches need further testing and evaluation on ARM
before they can be merged.  Moreover, I'd like to see some comparisons
between the _current_ gzip method, the new gzip method and the lzo
method on ARM.

I was going to get those numbers, but the platform I was going to run
the tests on has died on me today.

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Thu, Aug 06, 2009 at 11:40:55PM +0100, Russell King - ARM Linux wrote :
> On Mon, Aug 03, 2009 at 04:58:19PM +0200, Albin Tonnerre wrote:
> > This is the second part of patch. This part includes:
> >  - changes to ach/arch/boot/Makefile to make it easier to add new
> >    compression types
> >  - new piggy.lzo.S necessary for lzo compression
> >  - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
> >    gzip, depending on the config
> >  - Kconfig support

> FYI, with these patches applied and selecting GZIP method, I get
> linker errors.  I've been unable to track down what's going on, but
> it appears to be a libgcc issue.

> In spite of the decompressor being built as an EABI object, gcc seems
> to be issuing calls to __umodsi3, which isn't in the EABI libgcc
> (they're called something different - don't ask.)

I also happen to have issues related to the linker not finding __aeabi_uidivmod,
but only when compiling with CONFIG_CC_OPTIMIZE_FOR_SIZE. Is it similar for you,
or does it also fail when compiling with -O2 ?

> So I think these patches need further testing and evaluation on ARM
> before they can be merged.  Moreover, I'd like to see some comparisons
> between the _current_ gzip method, the new gzip method and the lzo
> method on ARM.

I don't have my ARM platforms at hand, but I'll so these comparisons in a couple
days when I access to them again.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Russell King - ARM Linux]

On Fri, Aug 07, 2009 at 11:24:52AM +0200, Albin Tonnerre wrote:
> I also happen to have issues related to the linker not finding __aeabi_uidivmod,
> but only when compiling with CONFIG_CC_OPTIMIZE_FOR_SIZE. Is it similar for you,
> or does it also fail when compiling with -O2 ?

I tend not to build with -O2 because it seems to tickle compiler bugs
(which is the reverse of -Os on x86.)  I'll give it a go over the
next few days.

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Alain Knaff]

On 08/07/09 11:24, Albin Tonnerre wrote:
> On Thu, Aug 06, 2009 at 11:40:55PM +0100, Russell King - ARM Linux wrote :
>> On Mon, Aug 03, 2009 at 04:58:19PM +0200, Albin Tonnerre wrote:
>>> This is the second part of patch. This part includes:
>>>  - changes to ach/arch/boot/Makefile to make it easier to add new
>>>    compression types
>>>  - new piggy.lzo.S necessary for lzo compression
>>>  - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
>>>    gzip, depending on the config
>>>  - Kconfig support
> 
>> FYI, with these patches applied and selecting GZIP method, I get
>> linker errors.  I've been unable to track down what's going on, but
>> it appears to be a libgcc issue.
> 
>> In spite of the decompressor being built as an EABI object, gcc seems
>> to be issuing calls to __umodsi3, which isn't in the EABI libgcc
>> (they're called something different - don't ask.)
> 
> I also happen to have issues related to the linker not finding __aeabi_uidivmod,
> but only when compiling with CONFIG_CC_OPTIMIZE_FOR_SIZE. Is it similar for you,
> or does it also fail when compiling with -O2 ?
> 
>> So I think these patches need further testing and evaluation on ARM
>> before they can be merged.  Moreover, I'd like to see some comparisons
>> between the _current_ gzip method, the new gzip method and the lzo
>> method on ARM.
> 
> I don't have my ARM platforms at hand, but I'll so these comparisons in a couple
> days when I access to them again.
> 
> Regards,

Could it be that the patches that remove division (zutil.h and inflate.c)
have somehow not been applied?

Regards,

Alain

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Fri, Aug 07, 2009 at 11:36:56AM +0200, Alain Knaff wrote :
> On 08/07/09 11:24, Albin Tonnerre wrote:
> > On Thu, Aug 06, 2009 at 11:40:55PM +0100, Russell King - ARM Linux wrote :
> >> On Mon, Aug 03, 2009 at 04:58:19PM +0200, Albin Tonnerre wrote:
> >>> This is the second part of patch. This part includes:
> >>>  - changes to ach/arch/boot/Makefile to make it easier to add new
> >>>    compression types
> >>>  - new piggy.lzo.S necessary for lzo compression
> >>>  - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
> >>>    gzip, depending on the config
> >>>  - Kconfig support

> >> FYI, with these patches applied and selecting GZIP method, I get
> >> linker errors.  I've been unable to track down what's going on, but
> >> it appears to be a libgcc issue.

> >> In spite of the decompressor being built as an EABI object, gcc seems
> >> to be issuing calls to __umodsi3, which isn't in the EABI libgcc
> >> (they're called something different - don't ask.)

> > I also happen to have issues related to the linker not finding __aeabi_uidivmod,
> > but only when compiling with CONFIG_CC_OPTIMIZE_FOR_SIZE. Is it similar for you,
> > or does it also fail when compiling with -O2 ?

> >> So I think these patches need further testing and evaluation on ARM
> >> before they can be merged.  Moreover, I'd like to see some comparisons
> >> between the _current_ gzip method, the new gzip method and the lzo
> >> method on ARM.

> > I don't have my ARM platforms at hand, but I'll so these comparisons in a couple
> > days when I access to them again.

> > Regards,

> Could it be that the patches that remove division (zutil.h and inflate.c)
> have somehow not been applied?

Indeed, they've not been applied. However, I'd rather try to understand why
exactly this is an issue when compiling with -Os and not -O2 instead of working
around it by removing the divisions.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Alain Knaff]

On 08/07/09 12:21, Albin Tonnerre wrote:

> Indeed, they've not been applied. However, I'd rather try to understand why
> exactly this is an issue when compiling with -Os and not -O2 instead of working
> around it by removing the divisions.
> 
> Regards,

Well, I for myself couldn't get it to compile _at_all_ for ARM as long as
there were any division operations in the code. My theory was that the ARM
processor doesn't have native division, and that this is being supplied by
some kind of run-time library, which is not linked with when building the
pre-boot environment. But if it works with some compilation flags but not
others, something more complex must be going on...

... or could it be that with some flags, the compiler generates "division
emulation" code, but not with others (such as when optimizing for size?)

Regards,

Alain

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Matthieu CASTET]

Albin Tonnerre a écrit :
> On Fri, Aug 07, 2009 at 11:36:56AM +0200, Alain Knaff wrote :
>> On 08/07/09 11:24, Albin Tonnerre wrote:
> 
>>> Regards,
> 
>> Could it be that the patches that remove division (zutil.h and inflate.c)
>> have somehow not been applied?
> 
> Indeed, they've not been applied. However, I'd rather try to understand why
> exactly this is an issue when compiling with -Os and not -O2 instead of working
> around it by removing the divisions.
> 
Look at the generated code.

Arm doesn't have division instruction.
May be at -Os gcc emit a call to the software division, but at -O2 it
manage to optimise the division (transform it in shift, inline some
builtin, ...).


Matthieu

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Fri, Aug 07, 2009 at 01:50:03PM +0200, Matthieu CASTET wrote :
> Albin Tonnerre a écrit :
> > On Fri, Aug 07, 2009 at 11:36:56AM +0200, Alain Knaff wrote :
> >> On 08/07/09 11:24, Albin Tonnerre wrote:

> >>> Regards,

> >> Could it be that the patches that remove division (zutil.h and inflate.c)
> >> have somehow not been applied?

> > Indeed, they've not been applied. However, I'd rather try to understand why
> > exactly this is an issue when compiling with -Os and not -O2 instead of working
> > around it by removing the divisions.

> Look at the generated code.

> Arm doesn't have division instruction.
> May be at -Os gcc emit a call to the software division, but at -O2 it
> manage to optimise the division (transform it in shift, inline some
> builtin, ...).

Yes, I figured that out. What I don't get, though, is that it fails while the
software division symbol (__aeabi_uidivmod here) does seem to be provided by
libgcc.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Matthieu CASTET]

Albin Tonnerre a écrit :
> On Fri, Aug 07, 2009 at 01:50:03PM +0200, Matthieu CASTET wrote :
>> Albin Tonnerre a écrit :
>>> On Fri, Aug 07, 2009 at 11:36:56AM +0200, Alain Knaff wrote :
>>>> On 08/07/09 11:24, Albin Tonnerre wrote:
> 
>>>>> Regards,
> 
>>>> Could it be that the patches that remove division (zutil.h and inflate.c)
>>>> have somehow not been applied?
> 
>>> Indeed, they've not been applied. However, I'd rather try to understand why
>>> exactly this is an issue when compiling with -Os and not -O2 instead of working
>>> around it by removing the divisions.
> 
>> Look at the generated code.
> 
>> Arm doesn't have division instruction.
>> May be at -Os gcc emit a call to the software division, but at -O2 it
>> manage to optimise the division (transform it in shift, inline some
>> builtin, ...).
> 
> Yes, I figured that out. What I don't get, though, is that it fails while the
> software division symbol (__aeabi_uidivmod here) does seem to be provided by
> libgcc.
> 
AFAIK we don't link the kernel with libgcc.
That's why the kernel provide __aeabi_* in arch/arm/lib

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Fri, Aug 07, 2009 at 03:27:00PM +0200, Matthieu CASTET wrote :
> Albin Tonnerre a écrit :
> > On Fri, Aug 07, 2009 at 01:50:03PM +0200, Matthieu CASTET wrote :
> >> Albin Tonnerre a écrit :
> >>> On Fri, Aug 07, 2009 at 11:36:56AM +0200, Alain Knaff wrote :
> >>>> On 08/07/09 11:24, Albin Tonnerre wrote:

> >>>>> Regards,

> >>>> Could it be that the patches that remove division (zutil.h and inflate.c)
> >>>> have somehow not been applied?

> >>> Indeed, they've not been applied. However, I'd rather try to understand why
> >>> exactly this is an issue when compiling with -Os and not -O2 instead of working
> >>> around it by removing the divisions.

> >> Look at the generated code.

> >> Arm doesn't have division instruction.
> >> May be at -Os gcc emit a call to the software division, but at -O2 it
> >> manage to optimise the division (transform it in shift, inline some
> >> builtin, ...).

> > Yes, I figured that out. What I don't get, though, is that it fails while the
> > software division symbol (__aeabi_uidivmod here) does seem to be provided by
> > libgcc.

> AFAIK we don't link the kernel with libgcc.
> That's why the kernel provide __aeabi_* in arch/arm/lib

That's true for the actual kernel image, but not for the bootstrap code we use
when compiling compressed kernels. arch/arm/boot/compressed/Makefile uses
libgcc, unless I'm overlooking something here:

  arm-unknown-linux-uclibcgnueabi-ld -EL    --defsym zreladdr=0x20008000
  --defsym initrd_phys=0x20410000 --defsym params_phys=0x20000100 -p
  --no-undefined -X
  /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a
  -T arch/arm/boot/compressed/vmlinux.lds arch/arm/boot/compressed/head.o
  arch/arm/boot/compressed/piggy.gzip.o arch/arm/boot/compressed/misc.o -o
  arch/arm/boot/compressed/vmlinux


Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Russell King - ARM Linux]

On Fri, Aug 07, 2009 at 03:55:24PM +0200, Albin Tonnerre wrote:
> That's true for the actual kernel image, but not for the bootstrap code we use
> when compiling compressed kernels. arch/arm/boot/compressed/Makefile uses
> libgcc, unless I'm overlooking something here:
> 
>   arm-unknown-linux-uclibcgnueabi-ld -EL    --defsym zreladdr=0x20008000
>   --defsym initrd_phys=0x20410000 --defsym params_phys=0x20000100 -p
>   --no-undefined -X
>   /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a
>   -T arch/arm/boot/compressed/vmlinux.lds arch/arm/boot/compressed/head.o
>   arch/arm/boot/compressed/piggy.gzip.o arch/arm/boot/compressed/misc.o -o
>   arch/arm/boot/compressed/vmlinux

It's because libgcc appears in the wrong place in the command line, and
due to the way kbuild works, we can't get it into the right place easily.

Linkers are sensitive to the order of archives on the command line - its
pointless having an archive as the first file argument because none of
the contained objects will ever be pulled in.

Sam - any ideas how to solve this?

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[H. Peter Anvin]

On 08/07/2009 01:00 PM, Russell King - ARM Linux wrote:
> On Fri, Aug 07, 2009 at 03:55:24PM +0200, Albin Tonnerre wrote:
>> That's true for the actual kernel image, but not for the bootstrap code we use
>> when compiling compressed kernels. arch/arm/boot/compressed/Makefile uses
>> libgcc, unless I'm overlooking something here:
>>
>>   arm-unknown-linux-uclibcgnueabi-ld -EL    --defsym zreladdr=0x20008000
>>   --defsym initrd_phys=0x20410000 --defsym params_phys=0x20000100 -p
>>   --no-undefined -X
>>   /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a
>>   -T arch/arm/boot/compressed/vmlinux.lds arch/arm/boot/compressed/head.o
>>   arch/arm/boot/compressed/piggy.gzip.o arch/arm/boot/compressed/misc.o -o
>>   arch/arm/boot/compressed/vmlinux
> 
> It's because libgcc appears in the wrong place in the command line, and
> due to the way kbuild works, we can't get it into the right place easily.
> 
> Linkers are sensitive to the order of archives on the command line - its
> pointless having an archive as the first file argument because none of
> the contained objects will ever be pulled in.
> 
> Sam - any ideas how to solve this?

Can we use the group feature of ld for this?

	-hpa

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Sam Ravnborg]

On Fri, Aug 07, 2009 at 09:00:01PM +0100, Russell King - ARM Linux wrote:
> On Fri, Aug 07, 2009 at 03:55:24PM +0200, Albin Tonnerre wrote:
> > That's true for the actual kernel image, but not for the bootstrap code we use
> > when compiling compressed kernels. arch/arm/boot/compressed/Makefile uses
> > libgcc, unless I'm overlooking something here:
> > 
> >   arm-unknown-linux-uclibcgnueabi-ld -EL    --defsym zreladdr=0x20008000
> >   --defsym initrd_phys=0x20410000 --defsym params_phys=0x20000100 -p
> >   --no-undefined -X
> >   /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a
> >   -T arch/arm/boot/compressed/vmlinux.lds arch/arm/boot/compressed/head.o
> >   arch/arm/boot/compressed/piggy.gzip.o arch/arm/boot/compressed/misc.o -o
> >   arch/arm/boot/compressed/vmlinux
> 
> It's because libgcc appears in the wrong place in the command line, and
> due to the way kbuild works, we can't get it into the right place easily.
> 
> Linkers are sensitive to the order of archives on the command line - its
> pointless having an archive as the first file argument because none of
> the contained objects will ever be pulled in.
> 
> Sam - any ideas how to solve this?

We could add libgcc as a prerequisite.
Untested patch below.


This is a ittle hackish - otherwise we have to change the
definition of ld or use a private version.

I added explanations for some of the linker symbols when
I was there. I cannot the ld options.

If you decide to use this it has my:

Signed-off-by: Sam Ravnborg <sam@ravnborg.org>


	Sam


diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile
index ce39dc5..4f980f2 100644
--- a/arch/arm/boot/compressed/Makefile
+++ b/arch/arm/boot/compressed/Makefile
@@ -87,15 +87,24 @@ endif
 ifneq ($(PARAMS_PHYS),)
 LDFLAGS_vmlinux += --defsym params_phys=$(PARAMS_PHYS)
 endif
-LDFLAGS_vmlinux += -p --no-undefined -X \
-       $(shell $(CC) $(KBUILD_CFLAGS) --print-libgcc-file-name) -T
+
+# ?
+LDFLAGS_vmlinux += -p
+# Report unresolved symbol references
+LDFLAGS_vmlinux += --no-undefined
+# Delete all temporary local symbols
+LDFLAGS_vmlinux += -X
+# Next argument is a linker script
+LDFLAGS_vmlinux += -T
+
+libgcc = $(shell $(CC) $(KBUILD_CFLAGS) --print-libgcc-file-name)
 
 # Don't allow any static data in misc.o, which
 # would otherwise mess up our GOT table
 CFLAGS_misc.o := -Dstatic=
 
 $(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.o \
-               $(addprefix $(obj)/, $(OBJS)) FORCE
+               $(addprefix $(obj)/, $(OBJS)) $(libgcc) FORCE
        $(call if_changed,ld)
        @:
 


	Sam

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

Hi Sam,

On Fri, Aug 07, 2009 at 11:08:16PM +0200, Sam Ravnborg wrote :
> We could add libgcc as a prerequisite.
> Untested patch below.

When compiling with this patch applied, I get the following error:

  Kernel: arch/arm/boot/Image is ready
    LD      arch/arm/boot/compressed/vmlinux
/home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a(_dvmd_lnx.o):
In function `__div0':
/home/albin/devel/free-electrons/toolchain/targets/src/gcc-4.3.2/libgcc/../gcc/config/arm/lib1funcs.asm:1079:
undefined reference to `raise'
make[2]: *** [arch/arm/boot/compressed/vmlinux] Error 1
make[1]: *** [arch/arm/boot/compressed/vmlinux] Error 2

I've got no idea where this symbol is defined, though. Has anyone an idea on
this ?

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Sam Ravnborg]

On Tue, Aug 11, 2009 at 11:44:18AM +0200, Albin Tonnerre wrote:
> Hi Sam,
> 
> On Fri, Aug 07, 2009 at 11:08:16PM +0200, Sam Ravnborg wrote :
> > We could add libgcc as a prerequisite.
> > Untested patch below.
> 
> When compiling with this patch applied, I get the following error:
> 
>   Kernel: arch/arm/boot/Image is ready
>     LD      arch/arm/boot/compressed/vmlinux
> /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a(_dvmd_lnx.o):
> In function `__div0':
> /home/albin/devel/free-electrons/toolchain/targets/src/gcc-4.3.2/libgcc/../gcc/config/arm/lib1funcs.asm:1079:
> undefined reference to `raise'
> make[2]: *** [arch/arm/boot/compressed/vmlinux] Error 1
> make[1]: *** [arch/arm/boot/compressed/vmlinux] Error 2
> 
> I've got no idea where this symbol is defined, though. Has anyone an idea on
> this ?

To make sure thet hack did what it was intended could you
try to build with V=1 and post output of the ld command.

	Sam

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Tue, Aug 11, 2009 at 03:39:43PM +0200, Sam Ravnborg wrote :
> On Tue, Aug 11, 2009 at 11:44:18AM +0200, Albin Tonnerre wrote:
> > Hi Sam,
> > 
> > On Fri, Aug 07, 2009 at 11:08:16PM +0200, Sam Ravnborg wrote :
> > > We could add libgcc as a prerequisite.
> > > Untested patch below.
> > 
> > When compiling with this patch applied, I get the following error:
> > 
> >   Kernel: arch/arm/boot/Image is ready
> >     LD      arch/arm/boot/compressed/vmlinux
> > /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a(_dvmd_lnx.o):
> > In function `__div0':
> > /home/albin/devel/free-electrons/toolchain/targets/src/gcc-4.3.2/libgcc/../gcc/config/arm/lib1funcs.asm:1079:
> > undefined reference to `raise'
> > make[2]: *** [arch/arm/boot/compressed/vmlinux] Error 1
> > make[1]: *** [arch/arm/boot/compressed/vmlinux] Error 2
> > 
> > I've got no idea where this symbol is defined, though. Has anyone an idea on
> > this ?
> 
> To make sure thet hack did what it was intended could you
> try to build with V=1 and post output of the ld command.
> 

Here it is:

make -f scripts/Makefile.build obj=arch/arm/boot/compressed
arch/arm/boot/compressed/vmlinux
  arm-unknown-linux-uclibcgnueabi-ld -EL    --defsym zreladdr=0x20008000
  --defsym initrd_phys=0x20410000 --defsym params_phys=0x20000100 -p
  --no-undefined -X -T arch/arm/boot/compressed/vmlinux.lds
  arch/arm/boot/compressed/head.o arch/arm/boot/compressed/piggy.gzip.o arch/arm/boot/compressed/misc.o
  /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a -o arch/arm/boot/compressed/vmlinux
  /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a(_dvmd_lnx.o):
  In function `__div0':
  /home/albin/devel/free-electrons/toolchain/targets/src/gcc-4.3.2/libgcc/../gcc/config/arm/lib1funcs.asm:1079:
  undefined reference to `raise'

So it seems like the hack indeed did what is was supposed to do.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Matthieu CASTET]

Albin Tonnerre a écrit :
> Hi Sam,
> 
> On Fri, Aug 07, 2009 at 11:08:16PM +0200, Sam Ravnborg wrote :
>> We could add libgcc as a prerequisite.
>> Untested patch below.
> 
> When compiling with this patch applied, I get the following error:
> 
>   Kernel: arch/arm/boot/Image is ready
>     LD      arch/arm/boot/compressed/vmlinux
> /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a(_dvmd_lnx.o):
> In function `__div0':
> /home/albin/devel/free-electrons/toolchain/targets/src/gcc-4.3.2/libgcc/../gcc/config/arm/lib1funcs.asm:1079:
> undefined reference to `raise'
> make[2]: *** [arch/arm/boot/compressed/vmlinux] Error 1
> make[1]: *** [arch/arm/boot/compressed/vmlinux] Error 2
> 
> I've got no idea where this symbol is defined, though. Has anyone an idea on
> this ?
It is defined in libc ;)
You need either to provide a fake one, or use don't use libgcc.

Matthieu

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[H. Peter Anvin]

On 08/11/2009 02:44 AM, Albin Tonnerre wrote:
> Hi Sam,
> 
> On Fri, Aug 07, 2009 at 11:08:16PM +0200, Sam Ravnborg wrote :
>> We could add libgcc as a prerequisite.
>> Untested patch below.
> 
> When compiling with this patch applied, I get the following error:
> 
>   Kernel: arch/arm/boot/Image is ready
>     LD      arch/arm/boot/compressed/vmlinux
> /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a(_dvmd_lnx.o):
> In function `__div0':
> /home/albin/devel/free-electrons/toolchain/targets/src/gcc-4.3.2/libgcc/../gcc/config/arm/lib1funcs.asm:1079:
> undefined reference to `raise'
> make[2]: *** [arch/arm/boot/compressed/vmlinux] Error 1
> make[1]: *** [arch/arm/boot/compressed/vmlinux] Error 2
> 
> I've got no idea where this symbol is defined, though. Has anyone an idea on
> this ?
> 

raise() gets called by libgcc to handle division by zero.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Tue, Aug 11, 2009 at 09:03:14AM -0700, H. Peter Anvin wrote :
> On 08/11/2009 02:44 AM, Albin Tonnerre wrote:
> > Hi Sam,
> > 
> > On Fri, Aug 07, 2009 at 11:08:16PM +0200, Sam Ravnborg wrote :
> >> We could add libgcc as a prerequisite.
> >> Untested patch below.
> > 
> > When compiling with this patch applied, I get the following error:
> > 
> >   Kernel: arch/arm/boot/Image is ready
> >     LD      arch/arm/boot/compressed/vmlinux
> > /home/albin/x-tools/arm-unknown-linux-uclibcgnueabi/lib/gcc/arm-unknown-linux-uclibcgnueabi/4.3.2/libgcc.a(_dvmd_lnx.o):
> > In function `__div0':
> > /home/albin/devel/free-electrons/toolchain/targets/src/gcc-4.3.2/libgcc/../gcc/config/arm/lib1funcs.asm:1079:
> > undefined reference to `raise'
> > make[2]: *** [arch/arm/boot/compressed/vmlinux] Error 1
> > make[1]: *** [arch/arm/boot/compressed/vmlinux] Error 2
> > 
> > I've got no idea where this symbol is defined, though. Has anyone an idea on
> > this ?
> > 
> 
> raise() gets called by libgcc to handle division by zero.

So I guess the only options left are either define a dummy raise() function, or
get rid of the divisions like Alain Knaff did in his patch ?

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[H. Peter Anvin]

On 08/11/2009 09:27 AM, Albin Tonnerre wrote:
> 
> So I guess the only options left are either define a dummy raise() function, or
> get rid of the divisions like Alain Knaff did in his patch ?
> 

Define a dummy raise, get rid of the divisions, or provide your own
division function (there is probably one in the kernel already...)

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Tue, Aug 11, 2009 at 09:31:25AM -0700, H. Peter Anvin wrote :
> On 08/11/2009 09:27 AM, Albin Tonnerre wrote:
> > 
> > So I guess the only options left are either define a dummy raise() function, or
> > get rid of the divisions like Alain Knaff did in his patch ?
> > 
> 
> Define a dummy raise, get rid of the divisions, or provide your own
> division function (there is probably one in the kernel already...)

Yes, there's such a function in arch/arm/lib/lib1funcs.S, which in turns
requires the __div0 symbol, defined in arch/arm/kernel/traps.c, and this one
cannot be used as it brings plenty of symbols that aren't available in the
pre-boot environment. So anyway, we have to define our own symbols there. Either
we use libgcc and define raise(), or lib1funcs and define __div0().

Russell, what's your take on this ?

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[H. Peter Anvin]

On 08/13/2009 02:30 AM, Albin Tonnerre wrote:
> On Tue, Aug 11, 2009 at 09:31:25AM -0700, H. Peter Anvin wrote :
>> On 08/11/2009 09:27 AM, Albin Tonnerre wrote:
>>> So I guess the only options left are either define a dummy raise() function, or
>>> get rid of the divisions like Alain Knaff did in his patch ?
>>>
>> Define a dummy raise, get rid of the divisions, or provide your own
>> division function (there is probably one in the kernel already...)
> 
> Yes, there's such a function in arch/arm/lib/lib1funcs.S, which in turns
> requires the __div0 symbol, defined in arch/arm/kernel/traps.c, and this one
> cannot be used as it brings plenty of symbols that aren't available in the
> pre-boot environment. So anyway, we have to define our own symbols there. Either
> we use libgcc and define raise(), or lib1funcs and define __div0().
> 

I would go with the variant in the kernel, as being a known quantity.

Your __div0 function can just loop forever.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH 4/6] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Thu, Aug 06, 2009 at 11:40:55PM +0100, Russell King - ARM Linux wrote :
> On Mon, Aug 03, 2009 at 04:58:19PM +0200, Albin Tonnerre wrote:
> > This is the second part of patch. This part includes:
> >  - changes to ach/arch/boot/Makefile to make it easier to add new
> >    compression types
> >  - new piggy.lzo.S necessary for lzo compression
> >  - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
> >    gzip, depending on the config
> >  - Kconfig support
> 
> FYI, with these patches applied and selecting GZIP method, I get
> linker errors.  I've been unable to track down what's going on, but
> it appears to be a libgcc issue.
> 
> In spite of the decompressor being built as an EABI object, gcc seems
> to be issuing calls to __umodsi3, which isn't in the EABI libgcc
> (they're called something different - don't ask.)

Looks like this one is on its way to getting solved.

> So I think these patches need further testing and evaluation on ARM
> before they can be merged.  Moreover, I'd like to see some comparisons
> between the _current_ gzip method, the new gzip method and the lzo
> method on ARM.

The figures I posted with my patch were from an ARM (AT91SAM9263) 180MHz CPU.
For the record (average on 25 boots with each compression method):

Uncompressed size: 3.24Mo
gzip  1.61Mo 0.72s
lzo   1.75Mo 0.48s

Some tests with the current gzip code give about 1.64s decompression time.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

[PATCH 4/6 v2] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

This is the second part of patch. This part includes:
 - changes to ach/arch/boot/Makefile to make it easier to add new
   compression types
 - new piggy.lzo.S necessary for lzo compression
 - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
   gzip, depending on the config
 - Kconfig support

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
Changes:
 Compiling with -Os failed due to missing __aeabi_uidivmod.
 Link using arch/arm/lib/lib1funcs.o which provides this symbol, and
 define a dummy __div0 function in arch/arm/boot/compressed/misc.c, as
 this symbol is required by lib1funcs.

 arch/arm/Kconfig                      |    2 +
 arch/arm/boot/compressed/Makefile     |   31 ++++++---
 arch/arm/boot/compressed/misc.c       |  116 ++++++++++-----------------------
 arch/arm/boot/compressed/piggy.S      |    6 --
 arch/arm/boot/compressed/piggy.gzip.S |    6 ++
 arch/arm/boot/compressed/piggy.lzo.S  |    6 ++
 6 files changed, 70 insertions(+), 97 deletions(-)
 delete mode 100644 arch/arm/boot/compressed/piggy.S
 create mode 100644 arch/arm/boot/compressed/piggy.gzip.S
 create mode 100644 arch/arm/boot/compressed/piggy.lzo.S

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index aef63c8..ea71c0c 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -18,6 +18,8 @@ config ARM
 	select HAVE_KRETPROBES if (HAVE_KPROBES)
 	select HAVE_FUNCTION_TRACER if (!XIP_KERNEL)
 	select HAVE_GENERIC_DMA_COHERENT
+	select HAVE_KERNEL_GZIP
+	select HAVE_KERNEL_LZO
 	help
 	  The ARM series is a line of low-power-consumption RISC chip designs
 	  licensed by ARM Ltd and targeted at embedded applications and
diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile
index ce39dc5..5b4629b 100644
--- a/arch/arm/boot/compressed/Makefile
+++ b/arch/arm/boot/compressed/Makefile
@@ -63,8 +63,12 @@ endif
 
 SEDFLAGS	= s/TEXT_START/$(ZTEXTADDR)/;s/BSS_START/$(ZBSSADDR)/
 
-targets       := vmlinux vmlinux.lds piggy.gz piggy.o font.o font.c \
-		 head.o misc.o $(OBJS)
+suffix_$(CONFIG_KERNEL_GZIP) = gzip
+suffix_$(CONFIG_KERNEL_LZO)  = lzo
+
+targets       := vmlinux vmlinux.lds \
+		 piggy.$(suffix_y) piggy.$(suffix_y).o \
+		 font.o font.c head.o misc.o $(OBJS)
 
 ifeq ($(CONFIG_FUNCTION_TRACER),y)
 ORIG_CFLAGS := $(KBUILD_CFLAGS)
@@ -87,22 +91,31 @@ endif
 ifneq ($(PARAMS_PHYS),)
 LDFLAGS_vmlinux += --defsym params_phys=$(PARAMS_PHYS)
 endif
-LDFLAGS_vmlinux += -p --no-undefined -X \
-	$(shell $(CC) $(KBUILD_CFLAGS) --print-libgcc-file-name) -T
+# ?
+LDFLAGS_vmlinux += -p
+# Report unresolved symbol references
+LDFLAGS_vmlinux += --no-undefined
+# Delete all temporary local symbols
+LDFLAGS_vmlinux += -X
+# Next argument is a linker script
+LDFLAGS_vmlinux += -T
+
+# For __aeabi_uidivmod
+lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o
 
 # Don't allow any static data in misc.o, which
 # would otherwise mess up our GOT table
 CFLAGS_misc.o := -Dstatic=
 
-$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.o \
-	 	$(addprefix $(obj)/, $(OBJS)) FORCE
+$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.$(suffix_y).o \
+	 	$(addprefix $(obj)/, $(OBJS)) $(lib1funcs) FORCE
 	$(call if_changed,ld)
 	@:
 
-$(obj)/piggy.gz: $(obj)/../Image FORCE
-	$(call if_changed,gzip)
+$(obj)/piggy.$(suffix_y): $(obj)/../Image FORCE
+	$(call if_changed,$(suffix_y))
 
-$(obj)/piggy.o:  $(obj)/piggy.gz FORCE
+$(obj)/piggy.$(suffix_y).o:  $(obj)/piggy.$(suffix_y) FORCE
 
 CFLAGS_font.o := -Dstatic=
 
diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c
index 17153b5..57077c8 100644
--- a/arch/arm/boot/compressed/misc.c
+++ b/arch/arm/boot/compressed/misc.c
@@ -18,10 +18,15 @@
 
 unsigned int __machine_arch_type;
 
+#define _LINUX_STRING_H_
+
 #include <linux/compiler.h>	/* for inline */
 #include <linux/types.h>	/* for size_t */
 #include <linux/stddef.h>	/* for NULL */
 #include <asm/string.h>
+#include <linux/linkage.h>
+
+#include <asm/unaligned.h>
 
 #ifdef STANDALONE_DEBUG
 #define putstr printf
@@ -188,34 +193,8 @@ static inline __ptr_t memcpy(__ptr_t __dest, __const __ptr_t __src,
 /*
  * gzip delarations
  */
-#define OF(args)  args
 #define STATIC static
 
-typedef unsigned char  uch;
-typedef unsigned short ush;
-typedef unsigned long  ulg;
-
-#define WSIZE 0x8000		/* Window size must be at least 32k, */
-				/* and a power of two */
-
-static uch *inbuf;		/* input buffer */
-static uch window[WSIZE];	/* Sliding window buffer */
-
-static unsigned insize;		/* valid bytes in inbuf */
-static unsigned inptr;		/* index of next byte to be processed in inbuf */
-static unsigned outcnt;		/* bytes in output buffer */
-
-/* gzip flag byte */
-#define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
-#define CONTINUATION 0x02 /* bit 1 set: continuation of multi-part gzip file */
-#define EXTRA_FIELD  0x04 /* bit 2 set: extra field present */
-#define ORIG_NAME    0x08 /* bit 3 set: original file name present */
-#define COMMENT      0x10 /* bit 4 set: file comment present */
-#define ENCRYPTED    0x20 /* bit 5 set: file is encrypted */
-#define RESERVED     0xC0 /* bit 6,7:   reserved */
-
-#define get_byte()  (inptr < insize ? inbuf[inptr++] : fill_inbuf())
-
 /* Diagnostic functions */
 #ifdef DEBUG
 #  define Assert(cond,msg) {if(!(cond)) error(msg);}
@@ -233,24 +212,20 @@ static unsigned outcnt;		/* bytes in output buffer */
 #  define Tracecv(c,x)
 #endif
 
-static int  fill_inbuf(void);
-static void flush_window(void);
 static void error(char *m);
 
 extern char input_data[];
 extern char input_data_end[];
 
-static uch *output_data;
-static ulg output_ptr;
-static ulg bytes_out;
+static unsigned char *output_data;
+static unsigned long output_ptr;
 
 static void error(char *m);
 
 static void putstr(const char *);
 
-extern int end;
-static ulg free_mem_ptr;
-static ulg free_mem_end_ptr;
+static unsigned long free_mem_ptr;
+static unsigned long free_mem_end_ptr;
 
 #ifdef STANDALONE_DEBUG
 #define NO_INFLATE_MALLOC
@@ -258,46 +233,13 @@ static ulg free_mem_end_ptr;
 
 #define ARCH_HAS_DECOMP_WDOG
 
-#include "../../../../lib/inflate.c"
-
-/* ===========================================================================
- * Fill the input buffer. This is called only when the buffer is empty
- * and at least one byte is really needed.
- */
-int fill_inbuf(void)
-{
-	if (insize != 0)
-		error("ran out of input data");
-
-	inbuf = input_data;
-	insize = &input_data_end[0] - &input_data[0];
-
-	inptr = 1;
-	return inbuf[0];
-}
+#ifdef CONFIG_KERNEL_GZIP
+#include "../../../../lib/decompress_inflate.c"
+#endif
 
-/* ===========================================================================
- * Write the output window window[0..outcnt-1] and update crc and bytes_out.
- * (Used for the decompressed data only.)
- */
-void flush_window(void)
-{
-	ulg c = crc;
-	unsigned n;
-	uch *in, *out, ch;
-
-	in = window;
-	out = &output_data[output_ptr];
-	for (n = 0; n < outcnt; n++) {
-		ch = *out++ = *in++;
-		c = crc_32_tab[((int)c ^ ch) & 0xff] ^ (c >> 8);
-	}
-	crc = c;
-	bytes_out += (ulg)outcnt;
-	output_ptr += (ulg)outcnt;
-	outcnt = 0;
-	putstr(".");
-}
+#ifdef CONFIG_KERNEL_LZO
+#include "../../../../lib/decompress_unlzo.c"
+#endif
 
 #ifndef arch_error
 #define arch_error(x)
@@ -314,22 +256,33 @@ static void error(char *x)
 	while(1);	/* Halt */
 }
 
+asmlinkage void __div0(void)
+{
+	error("Attempting division by 0!");
+}
+
 #ifndef STANDALONE_DEBUG
 
-ulg
-decompress_kernel(ulg output_start, ulg free_mem_ptr_p, ulg free_mem_ptr_end_p,
-		  int arch_id)
+unsigned long
+decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p,
+                  unsigned long free_mem_ptr_end_p,
+                  int arch_id)
 {
-	output_data		= (uch *)output_start;	/* Points to kernel start */
+	unsigned char *tmp;
+
+	output_data		= (unsigned char *)output_start;
 	free_mem_ptr		= free_mem_ptr_p;
 	free_mem_end_ptr	= free_mem_ptr_end_p;
 	__machine_arch_type	= arch_id;
 
 	arch_decomp_setup();
 
-	makecrc();
+	tmp = (unsigned char *) (((unsigned long)input_data_end) - 4);
+	output_ptr = get_unaligned_le32(tmp);
+
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+	           NULL, NULL, output_data, NULL, error);
 	putstr(" done, booting the kernel.\n");
 	return output_ptr;
 }
@@ -341,11 +294,10 @@ int main()
 {
 	output_data = output_buffer;
 
-	makecrc();
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+	           NULL, NULL, output_data, NULL, error);
 	putstr("done.\n");
 	return 0;
 }
 #endif
-	
diff --git a/arch/arm/boot/compressed/piggy.S b/arch/arm/boot/compressed/piggy.S
deleted file mode 100644
index 54c9518..0000000
--- a/arch/arm/boot/compressed/piggy.S
+++ /dev/null
@@ -1,6 +0,0 @@
-	.section .piggydata,#alloc
-	.globl	input_data
-input_data:
-	.incbin	"arch/arm/boot/compressed/piggy.gz"
-	.globl	input_data_end
-input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.gzip.S b/arch/arm/boot/compressed/piggy.gzip.S
new file mode 100644
index 0000000..a68adf9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.gzip.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.gzip"
+	.globl	input_data_end
+input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.lzo.S b/arch/arm/boot/compressed/piggy.lzo.S
new file mode 100644
index 0000000..a425ad9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.lzo.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.lzo"
+	.globl	input_data_end
+input_data_end:
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 4/6 v2] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Fri, Aug 14, 2009 at 12:08:28PM +0200, Albin Tonnerre wrote :
> This is the second part of patch. This part includes:
>  - changes to ach/arch/boot/Makefile to make it easier to add new
>    compression types
>  - new piggy.lzo.S necessary for lzo compression
>  - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
>    gzip, depending on the config
>  - Kconfig support
> 
> Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
> ---
> Changes:
>  Compiling with -Os failed due to missing __aeabi_uidivmod.
>  Link using arch/arm/lib/lib1funcs.o which provides this symbol, and
>  define a dummy __div0 function in arch/arm/boot/compressed/misc.c, as
>  this symbol is required by lib1funcs.

Hi Russell,
Does that solve the concerns/issues you had with this patch?
If so, it might be worth getting the updaed version in Andrew's branch.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 3/6] Add support for LZO-compressed kernels

[Andrew Morton]

On Mon,  3 Aug 2009 16:58:18 +0200
Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:

> This is the first part of the lzo patch
> The lzo compressor is worse than gzip at compression, but faster at
> extraction. Here are some figures for an ARM board I'm working on:
> 
> Uncompressed size: 3.24Mo
> gzip  1.61Mo 0.72s
> lzo   1.75Mo 0.48s
> 
> So for a compression ratio that is still relatively close to gzip, it's
> much faster to extract, at least in that case.

Is 3.2Mb a typical kernel size for small systems?  It sounds large.


0.24 seconds booting speedup sounds pretty thin.  Adding a new
decompression format will introduce more configuration/build/deployment
complexities.  How do we justify this?

Did anyone look into just speeding up the gzip decompressor?

> +#ifdef STATIC

What is this STATIC thing for?

Re: [PATCH 3/6] Add support for LZO-compressed kernels

[H. Peter Anvin]

On 08/04/2009 04:00 PM, Andrew Morton wrote:
> 
> 0.24 seconds booting speedup sounds pretty thin.  Adding a new
> decompression format will introduce more configuration/build/deployment
> complexities.  How do we justify this?
> 

Keep in mind this may be out of a 3-5 second boot budget.

	-hpa

Re: [PATCH 3/6] Add support for LZO-compressed kernels

[Albin Tonnerre]

On Tue, Aug 04, 2009 at 04:00:43PM -0700, Andrew Morton wrote :
> On Mon,  3 Aug 2009 16:58:18 +0200
> Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:

> > This is the first part of the lzo patch
> > The lzo compressor is worse than gzip at compression, but faster at
> > extraction. Here are some figures for an ARM board I'm working on:

> > Uncompressed size: 3.24Mo
> > gzip  1.61Mo 0.72s
> > lzo   1.75Mo 0.48s

> > So for a compression ratio that is still relatively close to gzip, it's
> > much faster to extract, at least in that case.

> Is 3.2Mb a typical kernel size for small systems?  It sounds large.

This one actually embeds an initramfs which accounts for about half of the size.

> > +#ifdef STATIC

> What is this STATIC thing for?

That's what is currently used to test whether you're compiling the pre-boot
environment. eg. include/linux/decompress/mm.h uses this to determine whether it
should provide a malloc() implementation or simply #define malloc(a) kmalloc(a, GFP_KERNEL),
and a lot of similar things

-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 3/6] Add support for LZO-compressed kernels

[Andrew Morton]

On Mon,  3 Aug 2009 16:58:18 +0200
Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:

> This is the first part of the lzo patch

Please pass this patch (and all others!) through scritps/checkpatch.pl.

checkpatch reports a number of trivial errors which you wouldn't have included
had you known about them.

Re: [PATCH 2/6] include/linux/unaligned/{l,b}e_byteshift.h: Fix usage for compressed kernels

[Andrew Morton]

On Mon,  3 Aug 2009 16:58:17 +0200
Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:

> When unaligned accesses are required for uncompressing a kernel (such as
> for LZO decompression on ARM in a patch that follows), including
> <linux/kernel.h> causes issues as it brings in a lot of things that are
> not available in the decompression environment.
> However, those files apparently use nothing from <linux/kernel.h>, all
> they need is the declaration of types such as u32 or u64, so
> <linux/types.h> should be enough

Again, please provide a full description of thes "issues" which a patch
addresses so that the patch's importance can be understood by others,
thanks.

Re: [PATCH 2/6] include/linux/unaligned/{l,b}e_byteshift.h: Fix usage for compressed kernels

[Albin Tonnerre]

On Tue, Aug 04, 2009 at 03:55:50PM -0700, Andrew Morton wrote :
> On Mon,  3 Aug 2009 16:58:17 +0200
> Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:

> > When unaligned accesses are required for uncompressing a kernel (such as
> > for LZO decompression on ARM in a patch that follows), including
> > <linux/kernel.h> causes issues as it brings in a lot of things that are
> > not available in the decompression environment.
> > However, those files apparently use nothing from <linux/kernel.h>, all
> > they need is the declaration of types such as u32 or u64, so
> > <linux/types.h> should be enough

> Again, please provide a full description of thes "issues" which a patch
> addresses so that the patch's importance can be understood by others,
> thanks.

linux/kernel.h brings at least:
extern int console_printk[];
extern const char hex_asc[];
which causes errors at link-time as they are not available when
compiling the pre-boot environement. There are also a few others:

arch/arm/boot/compressed/misc.o: In function `valid_user_regs':
/home/albin/devel/free-electrons/gits/linux-2.6/arch/arm/include/asm/ptrace.h:158: undefined reference to `elf_hwcap'
arch/arm/boot/compressed/misc.o: In function `console_silent':
/home/albin/devel/free-electrons/gits/linux-2.6/include/linux/kernel.h:292: undefined reference to `console_printk'
arch/arm/boot/compressed/misc.o: In function `console_verbose':
/home/albin/devel/free-electrons/gits/linux-2.6/include/linux/kernel.h:297: undefined reference to `console_printk'
arch/arm/boot/compressed/misc.o: In function `pack_hex_byte':
/home/albin/devel/free-electrons/gits/linux-2.6/include/linux/kernel.h:360: undefined reference to `hex_asc'
arch/arm/boot/compressed/misc.o: In function `hweight_long':
/home/albin/devel/free-electrons/gits/linux-2.6/include/linux/bitops.h:45: undefined reference to `hweight32'
arch/arm/boot/compressed/misc.o: In function `__cmpxchg_local_generic':
/home/albin/devel/free-electrons/gits/linux-2.6/include/asm-generic/cmpxchg-local.h:21: undefined reference to `wrong_size_cmpxchg'
/home/albin/devel/free-electrons/gits/linux-2.6/include/asm-generic/cmpxchg-local.h:42: undefined reference to `wrong_size_cmpxchg'
arch/arm/boot/compressed/misc.o: In function `__xchg':
/home/albin/devel/free-electrons/gits/linux-2.6/arch/arm/include/asm/system.h:309: undefined reference to `__bad_xchg'
make[2]: *** [arch/arm/boot/compressed/vmlinux] Error 1
make[1]: *** [arch/arm/boot/compressed/vmlinux] Error 2

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH 2/6] include/linux/unaligned/{l,b}e_byteshift.h: Fix usage for compressed kernels

[Russell King - ARM Linux]

On Wed, Aug 05, 2009 at 11:29:56AM +0200, Albin Tonnerre wrote:
> linux/kernel.h brings at least:
> extern int console_printk[];
> extern const char hex_asc[];
> which causes errors at link-time as they are not available when
> compiling the pre-boot environement. There are also a few others:

To be clear, for Andrew's benefit, the reason this happens is because
we disable the 'static' when building on ARM - this is because we build
the decompressor with -fPIC, so we can relocate _just_ the data segment
without writing at all to the text segment.

This is so that we can flash the compressed image into read only memory
and have the kernel boot directly from that memory, without having to
have an expensive and unnecessary copy.

Having static data and functions causes complications which make this
impossible to otherwise achieve.  I don't remember the details, I'm not
an expert on the ARM shared library stuff.

Re: [PATCH 1/6] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[Andrew Morton]

On Mon,  3 Aug 2009 16:58:16 +0200
Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:

> These includes were added by 079effb6933f34b9b1b67b08bd4fd7fb672d16ef to
> fix the build when using kmemtrace. However this is not necessary when
> used to create a compressed kernel, and actually creates issues (brings
> a lot of things unavailable in the decompression environment), so don't
> include it if STATIC is defined.
> 

The description "actually creates issues (brings a lot of things
unavailable in the decompression environment)" is inadequate.  Please
describe te problem this patch fixes more completely so that others
(ie: me) can decide whether this patch is needed in 2.6.32, 2.6.31. 
2.6.30, ...


This patch conflicts heavily with

http://userweb.kernel.org/~akpm/mmotm/broken-out/bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch


What should we do about that?

Re: [PATCH 1/6] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[Phillip Lougher]

Andrew Morton wrote:
> On Mon,  3 Aug 2009 16:58:16 +0200
> Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:
> 
>> These includes were added by 079effb6933f34b9b1b67b08bd4fd7fb672d16ef to
>> fix the build when using kmemtrace. However this is not necessary when
>> used to create a compressed kernel, and actually creates issues (brings
>> a lot of things unavailable in the decompression environment), so don't
>> include it if STATIC is defined.
>>
> 
> The description "actually creates issues (brings a lot of things
> unavailable in the decompression environment)" is inadequate.  Please
> describe te problem this patch fixes more completely so that others
> (ie: me) can decide whether this patch is needed in 2.6.32, 2.6.31. 
> 2.6.30, ...
> 
> 
> This patch conflicts heavily with
> 
> http://userweb.kernel.org/~akpm/mmotm/broken-out/bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
> 
> 
> What should we do about that?


What do you normally do in this situation?  I'm happy to send a revised
bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
that would apply cleanly on-top of Alvin's patch, but, this will obviously
create dependencies on his patch being applied.

Phillip

Re: [PATCH 1/6] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[H. Peter Anvin]

On 08/04/2009 05:47 PM, Phillip Lougher wrote:
> Andrew Morton wrote:
>> On Mon,  3 Aug 2009 16:58:16 +0200
>> Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:
>>
>>> These includes were added by 079effb6933f34b9b1b67b08bd4fd7fb672d16ef to
>>> fix the build when using kmemtrace. However this is not necessary when
>>> used to create a compressed kernel, and actually creates issues (brings
>>> a lot of things unavailable in the decompression environment), so don't
>>> include it if STATIC is defined.
>>>
>>
>> The description "actually creates issues (brings a lot of things
>> unavailable in the decompression environment)" is inadequate.  Please
>> describe te problem this patch fixes more completely so that others
>> (ie: me) can decide whether this patch is needed in 2.6.32, 2.6.31.
>> 2.6.30, ...
>>
>> This patch conflicts heavily with
>>
>> http://userweb.kernel.org/~akpm/mmotm/broken-out/bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
>>
>> What should we do about that?
> 
> 
> What do you normally do in this situation?  I'm happy to send a revised
> bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
> 
> that would apply cleanly on-top of Alvin's patch, but, this will obviously
> create dependencies on his patch being applied.
> 

The general principle is that if A alone creates a more functional
environment than B alone, then B should be applied on top of A, and vice
versa.  This is especially so if A is a stable candidate.

It *sounds* like your patch is B here, but I am not sure from the
description.

	-hpa

Re: [PATCH 1/6] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[Phillip Lougher]

H. Peter Anvin wrote:
> On 08/04/2009 05:47 PM, Phillip Lougher wrote:
>> Andrew Morton wrote:
>>> On Mon,  3 Aug 2009 16:58:16 +0200
>>> Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:
>>>
>>>> These includes were added by 079effb6933f34b9b1b67b08bd4fd7fb672d16ef to
>>>> fix the build when using kmemtrace. However this is not necessary when
>>>> used to create a compressed kernel, and actually creates issues (brings
>>>> a lot of things unavailable in the decompression environment), so don't
>>>> include it if STATIC is defined.
>>>>
>>> The description "actually creates issues (brings a lot of things
>>> unavailable in the decompression environment)" is inadequate.  Please
>>> describe te problem this patch fixes more completely so that others
>>> (ie: me) can decide whether this patch is needed in 2.6.32, 2.6.31.
>>> 2.6.30, ...
>>>
>>> This patch conflicts heavily with
>>>
>>> http://userweb.kernel.org/~akpm/mmotm/broken-out/bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
>>>
>>> What should we do about that?
>>
>> What do you normally do in this situation?  I'm happy to send a revised
>> bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
>>
>> that would apply cleanly on-top of Alvin's patch, but, this will obviously
>> create dependencies on his patch being applied.
>>
> 
> The general principle is that if A alone creates a more functional
> environment than B alone, then B should be applied on top of A, and vice
> versa.  This is especially so if A is a stable candidate.
> 
> It *sounds* like your patch is B here, but I am not sure from the
> description.
>

Gosh, who wants to get into the my patch is better than yours
argument.  I certainly don't...

My patch series cleans up the code and fixes a number of
rough edges (which I expect to hit when I try to make Squashfs
use the new decompression code).  Albin's looks to be adding a
new set of LZO functionality.  Regarding the conflicting
patches in question, my patch removes a hack, Albin's moves
#include <slab.h> into code covered by #ifndef STATIC, so it
doesn't pull in loads of unnecessary definitions when the
file is being built in the pre-boot environment.

I personally can't decide which is A or B.

Phillip

Re: [PATCH 1/6] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[Andrew Morton]

On Wed, 05 Aug 2009 01:47:57 +0100 Phillip Lougher <phillip@lougher.demon.co.uk> wrote:

> Andrew Morton wrote:
> > On Mon,  3 Aug 2009 16:58:16 +0200
> > Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:
> > 
> >> These includes were added by 079effb6933f34b9b1b67b08bd4fd7fb672d16ef to
> >> fix the build when using kmemtrace. However this is not necessary when
> >> used to create a compressed kernel, and actually creates issues (brings
> >> a lot of things unavailable in the decompression environment), so don't
> >> include it if STATIC is defined.
> >>
> > 
> > The description "actually creates issues (brings a lot of things
> > unavailable in the decompression environment)" is inadequate.  Please
> > describe te problem this patch fixes more completely so that others
> > (ie: me) can decide whether this patch is needed in 2.6.32, 2.6.31. 
> > 2.6.30, ...
> > 
> > 
> > This patch conflicts heavily with
> > 
> > http://userweb.kernel.org/~akpm/mmotm/broken-out/bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
> > 
> > 
> > What should we do about that?
> 
> 
> What do you normally do in this situation?

I normally fix the rejects ;)

But I'd like to confirm that the two patches don't fix the same thing
via different means.  Lacking a full description of Albin's "issues",
that's hard to determine.  They do appear to be unrelated.

>  I'm happy to send a revised
> bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
> that would apply cleanly on-top of Alvin's patch, but, this will obviously
> create dependencies on his patch being applied.

Reworked
lib-decompress_-only-include-linux-slabh-if-static-is-not-defined.patch:

 lib/decompress_bunzip2.c |    2 +-
 lib/decompress_inflate.c |    2 +-
 lib/decompress_unlzma.c  |    2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff -puN lib/decompress_bunzip2.c~lib-decompress_-only-include-linux-slabh-if-static-is-not-defined lib/decompress_bunzip2.c
--- a/lib/decompress_bunzip2.c~lib-decompress_-only-include-linux-slabh-if-static-is-not-defined
+++ a/lib/decompress_bunzip2.c
@@ -49,10 +49,10 @@
 #define PREBOOT
 #else
 #include <linux/decompress/bunzip2.h>
+#include <linux/slab.h>
 #endif /* STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #ifndef INT_MAX
 #define INT_MAX 0x7fffffff
diff -puN lib/decompress_inflate.c~lib-decompress_-only-include-linux-slabh-if-static-is-not-defined lib/decompress_inflate.c
--- a/lib/decompress_inflate.c~lib-decompress_-only-include-linux-slabh-if-static-is-not-defined
+++ a/lib/decompress_inflate.c
@@ -19,11 +19,11 @@
 #include "zlib_inflate/inflate.h"
 
 #include "zlib_inflate/infutil.h"
+#include <linux/slab.h>
 
 #endif /* STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #define GZIP_IOBUF_SIZE (16*1024)
 
diff -puN lib/decompress_unlzma.c~lib-decompress_-only-include-linux-slabh-if-static-is-not-defined lib/decompress_unlzma.c
--- a/lib/decompress_unlzma.c~lib-decompress_-only-include-linux-slabh-if-static-is-not-defined
+++ a/lib/decompress_unlzma.c
@@ -33,10 +33,10 @@
 #define PREBOOT
 #else
 #include <linux/decompress/unlzma.h>
+#include <linux/slab.h>
 #endif /* STATIC */
 
 #include <linux/decompress/mm.h>
-#include <linux/slab.h>
 
 #define	MIN(a, b) (((a) < (b)) ? (a) : (b))
 
_

Re: [PATCH 1/6] lib/decompress_*: only include <linux/slab.h> if STATIC is not defined

[Phillip Lougher]

Andrew Morton wrote:
> On Wed, 05 Aug 2009 01:47:57 +0100 Phillip Lougher <phillip@lougher.demon.co.uk> wrote:
> 
>>
>> What do you normally do in this situation?
> 
> I normally fix the rejects ;)
> 
> But I'd like to confirm that the two patches don't fix the same thing
> via different means.  Lacking a full description of Albin's "issues",
> that's hard to determine.  They do appear to be unrelated.
> 

No they're not fixing the same thing.

Albin's patch is moving #include <slab.h> inside the #ifndef STATIC ...
#endif code segment.  This ensures that <slab.h>
isn't included when the file is being built in the stripped down
pre-boot environment.  I imagine Albin's issues is that slab.h
pulls in a lot of definitions unnecessary in the pre-boot
environment and which rely on things which are missing
in the stripped down pre-boot environment.

My changes to the #ifndef STATIC logic defines PREBOOT if
STATIC is defined.  My patch uses the PREBOOT definition
later to define the decompress wrapper function, which is only
needed in the preboot environment.

i.e.

#ifdef STATIC
#define PREBOOT
#else
#include <linux/decompress/unlzma.h>
#endif

...
Lots of code
...

#ifdef PREBOOT
static int INIT decompress.....
#endif


Obvious question, why doesn't my patch use STATIC here rather than
PREBOOT?  The header file <linux/decompress/unlzma.h> defines STATIC, .i.e
the #ifndef STATIC case defines STATIC via an include file, which makes
decisions on STATIC later in the file impossible.

>>  I'm happy to send a revised
>> bzip2-lzma-remove-nasty-uncompressed-size-hack-in-pre-boot-environment.patch
>> that would apply cleanly on-top of Alvin's patch, but, this will obviously
>> create dependencies on his patch being applied.
> 
> Reworked
> lib-decompress_-only-include-linux-slabh-if-static-is-not-defined.patch:

The patch looks OK.

Thanks

Phillip

Re: [PATCH 4/5] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

[-- Attachment #1: Type: text/plain, Size: 766 bytes --]

On Wed, Jul 22, 2009 at 04:01:18PM +0200, Albin Tonnerre wrote :
> This is the second part of patch. This part includes:
>  - changes to ach/arch/boot/Makefile to make it easier to add new
>    compression types
>  - new piggy.lzo.S necessary for lzo compression
>  - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
>    gzip, depending on the config
>  - Kconfig support
> 

I failed to mention this in the first place, but for proper crediting it should
be noted that the ARM part of the patch is based on Alain Knaff's work on the
LZMA/Bzip compression patch for ARM.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 835 bytes --]

Re: [PATCH 3/5] Add support for LZO-compressed kernels

[H. Peter Anvin]

kevin.granade@gmail.com wrote:
>  >
>  > So for a compression ratio that is still relatively close to gzip, it's
>  > much faster to extract, at least in that case.
> 
> Is that "time to run the extraction algorithm", or "time to read in 
> image from media and extract"? I think the time to read from the media 
> would tend to dominate the decompression time.
> Either way, could you provide the other time for each algorithm in order 
> to give a sense of how this might scale to other CPU speeds/media read 
> speeds?
> 

If you have very slow media, you probably want to use LZMA.  If you have 
a very slow CPU and comparatively fast media, LZO might be a good 
option... I have heard people asking for *uncompressed* kernels for this 
reason, but LZO runs at a significant fraction of memcpy() speed.

	-hpa

Re: [PATCH 3/5] Add support for LZO-compressed kernels

[Albin Tonnerre]

[-- Attachment #1: Type: text/plain, Size: 1639 bytes --]

On Wed, Jul 22, 2009 at 03:28:09PM +0000, kevin.granade@gmail.com wrote :
> On Jul 22, 2009 9:01am, Albin Tonnerre <albin.tonnerre@free-electrons.com>
> wrote:
> > This is the first part of the lzo patch
> >
> > The lzo compressor is worse than gzip at compression, but faster at
> >
> > extraction. Here are some figures for an ARM board I'm working on:
> >
> >
> >
> > Uncompressed size: 3.24Mo
> >
> > gzip 1.61Mo 0.72s
> >
> > lzo 1.75Mo 0.48s
> >
> >
> >
> > So for a compression ratio that is still relatively close to gzip, it's
> >
> > much faster to extract, at least in that case.
> 
> Is that "time to run the extraction algorithm", or "time to read in image from
> media and extract"? I think the time to read from the media would tend to
> dominate the decompression time.
> Either way, could you provide the other time for each algorithm in order to
> give a sense of how this might scale to other CPU speeds/media read speeds?
> 

That's the time to run the extraction algorithm. As H. Peter Anvin pointed out,
you can have a fast media and a somewhat slow CPU, for which lzo makes sense.
As for other data, I don't have all the figures handy, but:

 - LZMA: compressed size 1.19Mo, decompression time: several *seconds* (that's
   on a 180MHz ARM9 board, using a patch to implement LZMA compression similar
   to this one)
 - Bzip2 eats a lot of RAM, and head.S only gives 64Ko of malloc() space on
   ARM, so I didn't try.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 835 bytes --]

[PATCH 3/5 v2] Add support for LZO-compressed kernels

[Albin Tonnerre]

This is the first part of the lzo patch
The lzo compressor is worse than gzip at compression, but faster at
extraction. Here are some figures for an ARM board I'm working on:

Uncompressed size: 3.24Mo
gzip  1.61Mo 0.72s
lzo   1.75Mo 0.48s

So for a compression ratio that is still relatively close to gzip, it's
much faster to extract, at least in that case.

This version applies to kernel 2.6.31-rc3

This part contains:
 - Makefile routine to support lzo compression
 - Fixes to the existing lzo compressor so that it can be used in
   compressed kernels
 - wrapper around the existing lzo1x_decompress, as it only extracts one
   block at a time, while we need to extract a whole file here
 - config dialog for kernel compression

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
Changelog since v1:

lib/decompress_unlzo.c
 - Rename lzo_decompress to unlzo to match the prototype in decompress/unlzo.h
 - Use LZO_BLOCK_SIZE instead of BLOCK_SIZE to avoid confusion
 - Add support for using the posp, fill and flush arguments
 - Reorder includes so that linux/types.h is included before
   linux/lzo.h. This prevents issue when not used as bootstrap code, as
   lzo.h needs things like size_t
 - When we are not using unlzo as part of kernel bootstrap code, we need
   linux/slab.h for memory allocation functions.
 - ... and we also need a call to set_error_fn so that the provided error
   function is used correctly

 include/linux/decompress/unlzo.h |   10 ++
 init/Kconfig                     |   18 +++-
 lib/decompress_unlzo.c           |  206 ++++++++++++++++++++++++++++++++++++++
 lib/lzo/lzo1x_decompress.c       |    9 +-
 scripts/Makefile.lib             |    5 +
 5 files changed, 241 insertions(+), 7 deletions(-)
 create mode 100644 include/linux/decompress/unlzo.h
 create mode 100644 lib/decompress_unlzo.c

diff --git a/include/linux/decompress/unlzo.h b/include/linux/decompress/unlzo.h
new file mode 100644
index 0000000..d1925ea
--- /dev/null
+++ b/include/linux/decompress/unlzo.h
@@ -0,0 +1,10 @@
+#ifndef DECOMPRESS_UNLZO_H
+#define DECOMPRESS_UNLZO_H
+
+int unlzo(unsigned char *inbuf, int len,
+          int(*fill)(void*, unsigned int),
+          int(*flush)(void*, unsigned int),
+          unsigned char *output,
+          int *pos,
+          void(*error)(char *x));
+#endif
diff --git a/init/Kconfig b/init/Kconfig
index cb2c092..ada6182 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -115,10 +115,13 @@ config HAVE_KERNEL_BZIP2
 config HAVE_KERNEL_LZMA
 	bool
 
+config HAVE_KERNEL_LZO
+	bool
+
 choice
 	prompt "Kernel compression mode"
 	default KERNEL_GZIP
-	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA
+	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA || HAVE_KERNEL_LZO
 	help
 	  The linux kernel is a kind of self-extracting executable.
 	  Several compression algorithms are available, which differ
@@ -141,9 +144,8 @@ config KERNEL_GZIP
 	bool "Gzip"
 	depends on HAVE_KERNEL_GZIP
 	help
-	  The old and tried gzip compression. Its compression ratio is
-	  the poorest among the 3 choices; however its speed (both
-	  compression and decompression) is the fastest.
+	  The old and tried gzip compression. It provides a good balance
+	  between compression ration and decompression speed.
 
 config KERNEL_BZIP2
 	bool "Bzip2"
@@ -164,6 +166,14 @@ config KERNEL_LZMA
 	  two. Compression is slowest.	The kernel size is about 33%
 	  smaller with LZMA in comparison to gzip.
 
+config KERNEL_LZO
+	bool "LZO"
+	depends on HAVE_KERNEL_LZO
+	help
+	  Its compression ratio is the poorest among the 4. The kernel
+	  size is about about 10% bigger than gzip; however its speed
+	  (both compression and decompression) is the fastest.
+
 endchoice
 
 config SWAP
diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c
new file mode 100644
index 0000000..a18417a
--- /dev/null
+++ b/lib/decompress_unlzo.c
@@ -0,0 +1,206 @@
+/*
+ * LZO decompressor for the Linux kernel. Code borrowed from the lzo
+ * implementation by Markus Franz Xaver Johannes Oberhumer.
+ *
+ * Linux kernel adaptation:
+ * Copyright (C) 2009
+ * Albin Tonnerre, Free Electrons <albin.tonnerre@free-electrons.com>
+ *
+ * Original code:
+ * Copyright (C) 1996-2005 Markus Franz Xaver Johannes Oberhumer
+ * All Rights Reserved.
+ *
+ * lzop and the LZO library are free software; you can redistribute them
+ * and/or modify them under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.
+ * If not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Markus F.X.J. Oberhumer
+ * <markus@oberhumer.com>
+ * http://www.oberhumer.com/opensource/lzop/
+ */
+
+#ifdef STATIC
+#include "lzo/lzo1x_decompress.c"
+#else
+#include <linux/slab.h>
+#endif
+
+#include <linux/types.h>
+#include <linux/lzo.h>
+#include <linux/decompress/mm.h>
+#include <linux/decompress/unlzo.h>
+
+#include <linux/compiler.h>
+#include <asm/unaligned.h>
+
+static const unsigned char lzop_magic[] =
+    { 0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a };
+
+#define LZO_BLOCK_SIZE        (256*1024l)
+#define HEADER_HAS_FILTER      0x00000800L
+
+STATIC inline int INIT parse_header(u8 *input, u8 *skip)
+{
+	int l;
+	u8 *parse = input;
+	u8 level = 0;
+	u16 version;
+
+	/* read magic: 9 first bits */
+	for (l = 0; l < 9; l++) {
+		if (*parse++ != lzop_magic[l])
+			return 0;
+	}
+	/* get version (2bytes), skip library version (2),
+	 * 'need to be extracted' version (2) and
+	 * method (1) */
+	version = get_unaligned_be16(parse);
+	parse += 7;
+	if (version >= 0x0940)
+		level = *parse++;
+	if (get_unaligned_be32(parse) & HEADER_HAS_FILTER)
+		parse += 8; /* flags + filter info */
+	else
+		parse += 4; /* flags */
+
+	/* skip mode and mtime_low */
+	parse += 8;
+	if (version >= 0x0940)
+		parse += 4;	/* skip mtime_high */
+
+	l = *parse++;
+	/* don't care about the file name, and skip checksum */
+	parse += l + 4;
+
+	*skip = parse - input;
+	return 1;
+}
+
+STATIC inline int INIT unlzo(u8 *input, int in_len,
+                             int (*fill) (void *, unsigned int),
+                             int (*flush) (void *, unsigned int),
+                             u8 *output, int *posp,
+                             void (*error_fn) (char *x))
+{
+	u8 skip = 0, r = 0;
+	u32 src_len, dst_len;
+	size_t tmp;
+	u8 *in_buf, *in_buf_save, *out_buf;
+	int obytes_processed = 0;
+
+	set_error_fn(error_fn);
+
+	if (output)
+		out_buf = output;
+	else if (!flush) {
+		error("NULL output pointer and no flush function provided");
+		goto exit;
+	}
+	else if (!(out_buf = malloc(LZO_BLOCK_SIZE))) {
+		error("Could not allocate output buffer");
+		goto exit;
+	}
+
+	if (input && fill) {
+		error("Both input pointer and fill function provided, don't know what to do");
+		goto exit_1;
+	}
+	else if (input)
+		in_buf = input;
+	else if (!fill || !posp) {
+		error("NULL input pointer and missing position pointer or fill function");
+		goto exit_1;
+	}
+	else if (!(in_buf = malloc(lzo1x_worst_compress(LZO_BLOCK_SIZE)))) {
+		error("Could not allocate input buffer");
+		goto exit_1;
+	}
+	in_buf_save = in_buf;
+
+	if (posp)
+		*posp = 0;
+
+	if (fill)
+		fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE));
+
+	if (!parse_header(input, &skip)) {
+		error("invalid header");
+		goto exit_2;
+	}
+	in_buf += skip;
+
+	if (posp)
+		*posp = skip;
+
+	for (;;) {
+		/* read uncompressed block size */
+		dst_len = get_unaligned_be32(in_buf);
+		in_buf += 4;
+
+		/* exit if last block */
+		if (dst_len == 0) {
+			if (posp)
+				*posp += 4;
+			break;
+		}
+
+		if (dst_len > LZO_BLOCK_SIZE) {
+			error("dest len longer than block size");
+			goto exit_2;
+		}
+
+		/* read compressed block size, and skip block checksum info */
+		src_len = get_unaligned_be32(in_buf);
+		in_buf += 8;
+
+		if (src_len <= 0 || src_len > dst_len) {
+			error("file corrupted");
+			goto exit_2;
+		}
+
+		/* decompress */
+		tmp = dst_len;
+		r = lzo1x_decompress_safe((u8 *) in_buf, src_len, out_buf, &tmp);
+
+		if (r != LZO_E_OK || dst_len != tmp) {
+			error("Compressed data violation");
+			goto exit_2;
+		}
+
+		obytes_processed += dst_len;
+		if (flush)
+			flush(out_buf, dst_len);
+		if (output)
+			out_buf += dst_len;
+		if (posp)
+			*posp += src_len + 12;
+		if (fill) {
+			in_buf = in_buf_save;
+			fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE));
+		}
+		else
+			in_buf += src_len;
+	}
+
+exit_2:
+	if (!input)
+		free(in_buf);
+exit_1:
+	if (!output)
+		free(out_buf);
+exit:
+	return obytes_processed;
+}
+
+#define decompress unlzo
diff --git a/lib/lzo/lzo1x_decompress.c b/lib/lzo/lzo1x_decompress.c
index 5dc6b29..f2fd098 100644
--- a/lib/lzo/lzo1x_decompress.c
+++ b/lib/lzo/lzo1x_decompress.c
@@ -11,11 +11,13 @@
  *  Richard Purdie <rpurdie@openedhand.com>
  */
 
+#ifndef STATIC
 #include <linux/module.h>
 #include <linux/kernel.h>
-#include <linux/lzo.h>
-#include <asm/byteorder.h>
+#endif
+
 #include <asm/unaligned.h>
+#include <linux/lzo.h>
 #include "lzodefs.h"
 
 #define HAVE_IP(x, ip_end, ip) ((size_t)(ip_end - ip) < (x))
@@ -244,9 +246,10 @@ lookbehind_overrun:
 	*out_len = op - out;
 	return LZO_E_LOOKBEHIND_OVERRUN;
 }
-
+#ifndef STATIC
 EXPORT_SYMBOL_GPL(lzo1x_decompress_safe);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("LZO1X Decompressor");
 
+#endif
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 7a77787..7b721d1 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -230,3 +230,8 @@ quiet_cmd_lzma = LZMA    $@
 cmd_lzma = (cat $(filter-out FORCE,$^) | \
 	lzma -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
 	(rm -f $@ ; false)
+
+quiet_cmd_lzo = LZO    $@
+cmd_lzo = (cat $(filter-out FORCE,$^) | \
+	lzop -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
+	(rm -f $@ ; false)
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com