[PATCH v3 0/4] Add support for LZO-compressed kernel images

[PATCH v3 0/4] Add support for LZO-compressed kernel images

[Albin Tonnerre]

This patch series adds generic support for creating and extracting
LZO-compressed kernel images, as well as support for using such images on the
x86 and ARM architectures, and support for creating and using LZO-compressed
initrd and initramfs images.

Changes in v2:
	- On ARM, compiling with -Os failed due to missing __aeabi_uidivmod.
	  Link using arch/arm/lib/lib1funcs.o which provides this symbol, and
	  define a dummy __div0 function in arch/arm/boot/compressed/misc.c, as
	  this symbol is required by lib1funcs.
	- lib/decompress_unlzo.c: include linux/decompress/unlzo.h only when
	  we're not compiling the pre-boot environment to prevent compile
	  issues
	- Formatting fixes

Changes in v3:
	- Add a couple missing semicolons to fix compile issues
	- Fix typos in {init,usr}/Kconfig

Andrew: since it used to be in -mm, can you pick this up again?

Russell: since I believe your initial concerns regarding the ARM part of the
patch got fixed, do I have your approval?

[PATCH v3 1/4] Add support for LZO-compressed kernels

[Albin Tonnerre]

This is the first part of the lzo patch
The lzo compressor is worse than gzip at compression, but faster at
extraction. Here are some figures for an ARM board I'm working on:

Uncompressed size: 3.24Mo
gzip  1.61Mo 0.72s
lzo   1.75Mo 0.48s

So for a compression ratio that is still relatively close to gzip, it's
much faster to extract, at least in that case.

This version applies to kernel 2.6.31-rc3

This part contains:
 - Makefile routine to support lzo compression
 - Fixes to the existing lzo compressor so that it can be used in
   compressed kernels
 - wrapper around the existing lzo1x_decompress, as it only extracts one
   block at a time, while we need to extract a whole file here
 - config dialog for kernel compression

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 include/linux/decompress/unlzo.h |   10 ++
 init/Kconfig                     |   18 +++-
 lib/decompress_unlzo.c           |  208 ++++++++++++++++++++++++++++++++++++++
 lib/lzo/lzo1x_decompress.c       |    9 +-
 scripts/Makefile.lib             |    5 +
 5 files changed, 243 insertions(+), 7 deletions(-)
 create mode 100644 include/linux/decompress/unlzo.h
 create mode 100644 lib/decompress_unlzo.c

diff --git a/include/linux/decompress/unlzo.h b/include/linux/decompress/unlzo.h
new file mode 100644
index 0000000..9872297
--- /dev/null
+++ b/include/linux/decompress/unlzo.h
@@ -0,0 +1,10 @@
+#ifndef DECOMPRESS_UNLZO_H
+#define DECOMPRESS_UNLZO_H
+
+int unlzo(unsigned char *inbuf, int len,
+	int(*fill)(void*, unsigned int),
+	int(*flush)(void*, unsigned int),
+	unsigned char *output,
+	int *pos,
+	void(*error)(char *x));
+#endif
diff --git a/init/Kconfig b/init/Kconfig
index f515864..eb65318 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -115,10 +115,13 @@ config HAVE_KERNEL_BZIP2
 config HAVE_KERNEL_LZMA
 	bool
 
+config HAVE_KERNEL_LZO
+	bool
+
 choice
 	prompt "Kernel compression mode"
 	default KERNEL_GZIP
-	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA
+	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA || HAVE_KERNEL_LZO
 	help
 	  The linux kernel is a kind of self-extracting executable.
 	  Several compression algorithms are available, which differ
@@ -141,9 +144,8 @@ config KERNEL_GZIP
 	bool "Gzip"
 	depends on HAVE_KERNEL_GZIP
 	help
-	  The old and tried gzip compression. Its compression ratio is
-	  the poorest among the 3 choices; however its speed (both
-	  compression and decompression) is the fastest.
+	  The old and tried gzip compression. It provides a good balance
+	  between compression ratio and decompression speed.
 
 config KERNEL_BZIP2
 	bool "Bzip2"
@@ -164,6 +166,14 @@ config KERNEL_LZMA
 	  two. Compression is slowest.	The kernel size is about 33%
 	  smaller with LZMA in comparison to gzip.
 
+config KERNEL_LZO
+	bool "LZO"
+	depends on HAVE_KERNEL_LZO
+	help
+	  Its compression ratio is the poorest among the 4. The kernel
+	  size is about about 10% bigger than gzip; however its speed
+	  (both compression and decompression) is the fastest.
+
 endchoice
 
 config SWAP
diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c
new file mode 100644
index 0000000..2bb736f
--- /dev/null
+++ b/lib/decompress_unlzo.c
@@ -0,0 +1,208 @@
+/*
+ * LZO decompressor for the Linux kernel. Code borrowed from the lzo
+ * implementation by Markus Franz Xaver Johannes Oberhumer.
+ *
+ * Linux kernel adaptation:
+ * Copyright (C) 2009
+ * Albin Tonnerre, Free Electrons <albin.tonnerre@free-electrons.com>
+ *
+ * Original code:
+ * Copyright (C) 1996-2005 Markus Franz Xaver Johannes Oberhumer
+ * All Rights Reserved.
+ *
+ * lzop and the LZO library are free software; you can redistribute them
+ * and/or modify them under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.
+ * If not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Markus F.X.J. Oberhumer
+ * <markus@oberhumer.com>
+ * http://www.oberhumer.com/opensource/lzop/
+ */
+
+#ifdef STATIC
+#include "lzo/lzo1x_decompress.c"
+#else
+#include <linux/slab.h>
+#include <linux/decompress/unlzo.h>
+#endif
+
+#include <linux/types.h>
+#include <linux/lzo.h>
+#include <linux/decompress/mm.h>
+
+#include <linux/compiler.h>
+#include <asm/unaligned.h>
+
+static const unsigned char lzop_magic[] =
+	{ 0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a };
+
+#define LZO_BLOCK_SIZE        (256*1024l)
+#define HEADER_HAS_FILTER      0x00000800L
+
+STATIC inline int INIT parse_header(u8 *input, u8 *skip)
+{
+	int l;
+	u8 *parse = input;
+	u8 level = 0;
+	u16 version;
+
+	/* read magic: 9 first bits */
+	for (l = 0; l < 9; l++) {
+		if (*parse++ != lzop_magic[l])
+			return 0;
+	}
+	/* get version (2bytes), skip library version (2),
+	 * 'need to be extracted' version (2) and
+	 * method (1) */
+	version = get_unaligned_be16(parse);
+	parse += 7;
+	if (version >= 0x0940)
+		level = *parse++;
+	if (get_unaligned_be32(parse) & HEADER_HAS_FILTER)
+		parse += 8; /* flags + filter info */
+	else
+		parse += 4; /* flags */
+
+	/* skip mode and mtime_low */
+	parse += 8;
+	if (version >= 0x0940)
+		parse += 4;	/* skip mtime_high */
+
+	l = *parse++;
+	/* don't care about the file name, and skip checksum */
+	parse += l + 4;
+
+	*skip = parse - input;
+	return 1;
+}
+
+STATIC inline int INIT unlzo(u8 *input, int in_len,
+				int (*fill) (void *, unsigned int),
+				int (*flush) (void *, unsigned int),
+				u8 *output, int *posp,
+				void (*error_fn) (char *x))
+{
+	u8 skip = 0, r = 0;
+	u32 src_len, dst_len;
+	size_t tmp;
+	u8 *in_buf, *in_buf_save, *out_buf;
+	int obytes_processed = 0;
+
+	set_error_fn(error_fn);
+
+	if (output)
+		out_buf = output;
+	else if (!flush) {
+		error("NULL output pointer and no flush function provided");
+		goto exit;
+	} else {
+		out_buf = malloc(LZO_BLOCK_SIZE);
+		if (!out_buf) {
+			error("Could not allocate output buffer");
+			goto exit;
+		}
+	}
+
+	if (input && fill) {
+		error("Both input pointer and fill function provided, don't know what to do");
+		goto exit_1;
+	} else if (input)
+		in_buf = input;
+	else if (!fill || !posp) {
+		error("NULL input pointer and missing position pointer or fill function");
+		goto exit_1;
+	} else {
+		in_buf = malloc(lzo1x_worst_compress(LZO_BLOCK_SIZE));
+		if (!in_buf) {
+			error("Could not allocate input buffer");
+			goto exit_1;
+		}
+	}
+	in_buf_save = in_buf;
+
+	if (posp)
+		*posp = 0;
+
+	if (fill)
+		fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE));
+
+	if (!parse_header(input, &skip)) {
+		error("invalid header");
+		goto exit_2;
+	}
+	in_buf += skip;
+
+	if (posp)
+		*posp = skip;
+
+	for (;;) {
+		/* read uncompressed block size */
+		dst_len = get_unaligned_be32(in_buf);
+		in_buf += 4;
+
+		/* exit if last block */
+		if (dst_len == 0) {
+			if (posp)
+				*posp += 4;
+			break;
+		}
+
+		if (dst_len > LZO_BLOCK_SIZE) {
+			error("dest len longer than block size");
+			goto exit_2;
+		}
+
+		/* read compressed block size, and skip block checksum info */
+		src_len = get_unaligned_be32(in_buf);
+		in_buf += 8;
+
+		if (src_len <= 0 || src_len > dst_len) {
+			error("file corrupted");
+			goto exit_2;
+		}
+
+		/* decompress */
+		tmp = dst_len;
+		r = lzo1x_decompress_safe((u8 *) in_buf, src_len, out_buf, &tmp);
+
+		if (r != LZO_E_OK || dst_len != tmp) {
+			error("Compressed data violation");
+			goto exit_2;
+		}
+
+		obytes_processed += dst_len;
+		if (flush)
+			flush(out_buf, dst_len);
+		if (output)
+			out_buf += dst_len;
+		if (posp)
+			*posp += src_len + 12;
+		if (fill) {
+			in_buf = in_buf_save;
+			fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE));
+		} else
+			in_buf += src_len;
+	}
+
+exit_2:
+	if (!input)
+		free(in_buf);
+exit_1:
+	if (!output)
+		free(out_buf);
+exit:
+	return obytes_processed;
+}
+
+#define decompress unlzo
diff --git a/lib/lzo/lzo1x_decompress.c b/lib/lzo/lzo1x_decompress.c
index 5dc6b29..f2fd098 100644
--- a/lib/lzo/lzo1x_decompress.c
+++ b/lib/lzo/lzo1x_decompress.c
@@ -11,11 +11,13 @@
  *  Richard Purdie <rpurdie@openedhand.com>
  */
 
+#ifndef STATIC
 #include <linux/module.h>
 #include <linux/kernel.h>
-#include <linux/lzo.h>
-#include <asm/byteorder.h>
+#endif
+
 #include <asm/unaligned.h>
+#include <linux/lzo.h>
 #include "lzodefs.h"
 
 #define HAVE_IP(x, ip_end, ip) ((size_t)(ip_end - ip) < (x))
@@ -244,9 +246,10 @@ lookbehind_overrun:
 	*out_len = op - out;
 	return LZO_E_LOOKBEHIND_OVERRUN;
 }
-
+#ifndef STATIC
 EXPORT_SYMBOL_GPL(lzo1x_decompress_safe);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("LZO1X Decompressor");
 
+#endif
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index ffdafb2..39c3483 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -230,3 +230,8 @@ quiet_cmd_lzma = LZMA    $@
 cmd_lzma = (cat $(filter-out FORCE,$^) | \
 	lzma -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
 	(rm -f $@ ; false)
+
+quiet_cmd_lzo = LZO    $@
+cmd_lzo = (cat $(filter-out FORCE,$^) | \
+	lzop -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
+	(rm -f $@ ; false)
-- 
1.6.3.3

[PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

This is the second part of patch. This part includes:
 - changes to ach/arch/boot/Makefile to make it easier to add new
   compression types
 - new piggy.lzo.S necessary for lzo compression
 - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
   gzip, depending on the config
 - Kconfig support

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 arch/arm/Kconfig                      |    2 +
 arch/arm/boot/compressed/Makefile     |   31 ++++++---
 arch/arm/boot/compressed/misc.c       |  116 ++++++++++-----------------------
 arch/arm/boot/compressed/piggy.S      |    6 --
 arch/arm/boot/compressed/piggy.gzip.S |    6 ++
 arch/arm/boot/compressed/piggy.lzo.S  |    6 ++
 6 files changed, 70 insertions(+), 97 deletions(-)
 delete mode 100644 arch/arm/boot/compressed/piggy.S
 create mode 100644 arch/arm/boot/compressed/piggy.gzip.S
 create mode 100644 arch/arm/boot/compressed/piggy.lzo.S

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 1c4119c..350921d 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -18,6 +18,8 @@ config ARM
 	select HAVE_KRETPROBES if (HAVE_KPROBES)
 	select HAVE_FUNCTION_TRACER if (!XIP_KERNEL)
 	select HAVE_GENERIC_DMA_COHERENT
+	select HAVE_KERNEL_GZIP
+	select HAVE_KERNEL_LZO
 	help
 	  The ARM series is a line of low-power-consumption RISC chip designs
 	  licensed by ARM Ltd and targeted at embedded applications and
diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile
index ce39dc5..5b4629b 100644
--- a/arch/arm/boot/compressed/Makefile
+++ b/arch/arm/boot/compressed/Makefile
@@ -63,8 +63,12 @@ endif
 
 SEDFLAGS	= s/TEXT_START/$(ZTEXTADDR)/;s/BSS_START/$(ZBSSADDR)/
 
-targets       := vmlinux vmlinux.lds piggy.gz piggy.o font.o font.c \
-		 head.o misc.o $(OBJS)
+suffix_$(CONFIG_KERNEL_GZIP) = gzip
+suffix_$(CONFIG_KERNEL_LZO)  = lzo
+
+targets       := vmlinux vmlinux.lds \
+		 piggy.$(suffix_y) piggy.$(suffix_y).o \
+		 font.o font.c head.o misc.o $(OBJS)
 
 ifeq ($(CONFIG_FUNCTION_TRACER),y)
 ORIG_CFLAGS := $(KBUILD_CFLAGS)
@@ -87,22 +91,31 @@ endif
 ifneq ($(PARAMS_PHYS),)
 LDFLAGS_vmlinux += --defsym params_phys=$(PARAMS_PHYS)
 endif
-LDFLAGS_vmlinux += -p --no-undefined -X \
-	$(shell $(CC) $(KBUILD_CFLAGS) --print-libgcc-file-name) -T
+# ?
+LDFLAGS_vmlinux += -p
+# Report unresolved symbol references
+LDFLAGS_vmlinux += --no-undefined
+# Delete all temporary local symbols
+LDFLAGS_vmlinux += -X
+# Next argument is a linker script
+LDFLAGS_vmlinux += -T
+
+# For __aeabi_uidivmod
+lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o
 
 # Don't allow any static data in misc.o, which
 # would otherwise mess up our GOT table
 CFLAGS_misc.o := -Dstatic=
 
-$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.o \
-	 	$(addprefix $(obj)/, $(OBJS)) FORCE
+$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.$(suffix_y).o \
+	 	$(addprefix $(obj)/, $(OBJS)) $(lib1funcs) FORCE
 	$(call if_changed,ld)
 	@:
 
-$(obj)/piggy.gz: $(obj)/../Image FORCE
-	$(call if_changed,gzip)
+$(obj)/piggy.$(suffix_y): $(obj)/../Image FORCE
+	$(call if_changed,$(suffix_y))
 
-$(obj)/piggy.o:  $(obj)/piggy.gz FORCE
+$(obj)/piggy.$(suffix_y).o:  $(obj)/piggy.$(suffix_y) FORCE
 
 CFLAGS_font.o := -Dstatic=
 
diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c
index 17153b5..7e0fe4d 100644
--- a/arch/arm/boot/compressed/misc.c
+++ b/arch/arm/boot/compressed/misc.c
@@ -18,10 +18,15 @@
 
 unsigned int __machine_arch_type;
 
+#define _LINUX_STRING_H_
+
 #include <linux/compiler.h>	/* for inline */
 #include <linux/types.h>	/* for size_t */
 #include <linux/stddef.h>	/* for NULL */
 #include <asm/string.h>
+#include <linux/linkage.h>
+
+#include <asm/unaligned.h>
 
 #ifdef STANDALONE_DEBUG
 #define putstr printf
@@ -188,34 +193,8 @@ static inline __ptr_t memcpy(__ptr_t __dest, __const __ptr_t __src,
 /*
  * gzip delarations
  */
-#define OF(args)  args
 #define STATIC static
 
-typedef unsigned char  uch;
-typedef unsigned short ush;
-typedef unsigned long  ulg;
-
-#define WSIZE 0x8000		/* Window size must be at least 32k, */
-				/* and a power of two */
-
-static uch *inbuf;		/* input buffer */
-static uch window[WSIZE];	/* Sliding window buffer */
-
-static unsigned insize;		/* valid bytes in inbuf */
-static unsigned inptr;		/* index of next byte to be processed in inbuf */
-static unsigned outcnt;		/* bytes in output buffer */
-
-/* gzip flag byte */
-#define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
-#define CONTINUATION 0x02 /* bit 1 set: continuation of multi-part gzip file */
-#define EXTRA_FIELD  0x04 /* bit 2 set: extra field present */
-#define ORIG_NAME    0x08 /* bit 3 set: original file name present */
-#define COMMENT      0x10 /* bit 4 set: file comment present */
-#define ENCRYPTED    0x20 /* bit 5 set: file is encrypted */
-#define RESERVED     0xC0 /* bit 6,7:   reserved */
-
-#define get_byte()  (inptr < insize ? inbuf[inptr++] : fill_inbuf())
-
 /* Diagnostic functions */
 #ifdef DEBUG
 #  define Assert(cond,msg) {if(!(cond)) error(msg);}
@@ -233,24 +212,20 @@ static unsigned outcnt;		/* bytes in output buffer */
 #  define Tracecv(c,x)
 #endif
 
-static int  fill_inbuf(void);
-static void flush_window(void);
 static void error(char *m);
 
 extern char input_data[];
 extern char input_data_end[];
 
-static uch *output_data;
-static ulg output_ptr;
-static ulg bytes_out;
+static unsigned char *output_data;
+static unsigned long output_ptr;
 
 static void error(char *m);
 
 static void putstr(const char *);
 
-extern int end;
-static ulg free_mem_ptr;
-static ulg free_mem_end_ptr;
+static unsigned long free_mem_ptr;
+static unsigned long free_mem_end_ptr;
 
 #ifdef STANDALONE_DEBUG
 #define NO_INFLATE_MALLOC
@@ -258,46 +233,13 @@ static ulg free_mem_end_ptr;
 
 #define ARCH_HAS_DECOMP_WDOG
 
-#include "../../../../lib/inflate.c"
-
-/* ===========================================================================
- * Fill the input buffer. This is called only when the buffer is empty
- * and at least one byte is really needed.
- */
-int fill_inbuf(void)
-{
-	if (insize != 0)
-		error("ran out of input data");
-
-	inbuf = input_data;
-	insize = &input_data_end[0] - &input_data[0];
-
-	inptr = 1;
-	return inbuf[0];
-}
+#ifdef CONFIG_KERNEL_GZIP
+#include "../../../../lib/decompress_inflate.c"
+#endif
 
-/* ===========================================================================
- * Write the output window window[0..outcnt-1] and update crc and bytes_out.
- * (Used for the decompressed data only.)
- */
-void flush_window(void)
-{
-	ulg c = crc;
-	unsigned n;
-	uch *in, *out, ch;
-
-	in = window;
-	out = &output_data[output_ptr];
-	for (n = 0; n < outcnt; n++) {
-		ch = *out++ = *in++;
-		c = crc_32_tab[((int)c ^ ch) & 0xff] ^ (c >> 8);
-	}
-	crc = c;
-	bytes_out += (ulg)outcnt;
-	output_ptr += (ulg)outcnt;
-	outcnt = 0;
-	putstr(".");
-}
+#ifdef CONFIG_KERNEL_LZO
+#include "../../../../lib/decompress_unlzo.c"
+#endif
 
 #ifndef arch_error
 #define arch_error(x)
@@ -314,22 +256,33 @@ static void error(char *x)
 	while(1);	/* Halt */
 }
 
+asmlinkage void __div0(void)
+{
+	error("Attempting division by 0!");
+}
+
 #ifndef STANDALONE_DEBUG
 
-ulg
-decompress_kernel(ulg output_start, ulg free_mem_ptr_p, ulg free_mem_ptr_end_p,
-		  int arch_id)
+unsigned long
+decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p,
+		unsigned long free_mem_ptr_end_p,
+		int arch_id)
 {
-	output_data		= (uch *)output_start;	/* Points to kernel start */
+	unsigned char *tmp;
+
+	output_data		= (unsigned char *)output_start;
 	free_mem_ptr		= free_mem_ptr_p;
 	free_mem_end_ptr	= free_mem_ptr_end_p;
 	__machine_arch_type	= arch_id;
 
 	arch_decomp_setup();
 
-	makecrc();
+	tmp = (unsigned char *) (((unsigned long)input_data_end) - 4);
+	output_ptr = get_unaligned_le32(tmp);
+
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+			NULL, NULL, output_data, NULL, error);
 	putstr(" done, booting the kernel.\n");
 	return output_ptr;
 }
@@ -341,11 +294,10 @@ int main()
 {
 	output_data = output_buffer;
 
-	makecrc();
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+			NULL, NULL, output_data, NULL, error);
 	putstr("done.\n");
 	return 0;
 }
 #endif
-	
diff --git a/arch/arm/boot/compressed/piggy.S b/arch/arm/boot/compressed/piggy.S
deleted file mode 100644
index 54c9518..0000000
--- a/arch/arm/boot/compressed/piggy.S
+++ /dev/null
@@ -1,6 +0,0 @@
-	.section .piggydata,#alloc
-	.globl	input_data
-input_data:
-	.incbin	"arch/arm/boot/compressed/piggy.gz"
-	.globl	input_data_end
-input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.gzip.S b/arch/arm/boot/compressed/piggy.gzip.S
new file mode 100644
index 0000000..a68adf9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.gzip.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.gzip"
+	.globl	input_data_end
+input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.lzo.S b/arch/arm/boot/compressed/piggy.lzo.S
new file mode 100644
index 0000000..a425ad9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.lzo.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.lzo"
+	.globl	input_data_end
+input_data_end:
-- 
1.6.3.3

[PATCH v3 3/4] Add support for LZO-compressed kernels on x86

[Albin Tonnerre]

This is the third part of the patch, which contains the necessary
changes to the x86 Kconfig and boot/compressed to allow the use of this
new compression method

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
Acked-by: H. Peter Anvin <hpa@zytor.com>
---
 arch/x86/Kconfig                  |    1 +
 arch/x86/boot/compressed/Makefile |    5 ++++-
 arch/x86/boot/compressed/misc.c   |    4 ++++
 3 files changed, 9 insertions(+), 1 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 72ace95..89f7771 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -49,6 +49,7 @@ config X86
 	select HAVE_KERNEL_GZIP
 	select HAVE_KERNEL_BZIP2
 	select HAVE_KERNEL_LZMA
+	select HAVE_KERNEL_LZO
 	select HAVE_ARCH_KMEMCHECK
 
 config OUTPUT_FORMAT
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index f8ed065..0e835d0 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -4,7 +4,7 @@
 # create a compressed vmlinux image from the original vmlinux
 #
 
-targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma head_$(BITS).o misc.o piggy.o
+targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.lzo head_$(BITS).o misc.o piggy.o
 
 KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2
 KBUILD_CFLAGS += -fno-strict-aliasing -fPIC
@@ -48,10 +48,13 @@ $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE
 	$(call if_changed,bzip2)
 $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE
 	$(call if_changed,lzma)
+$(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE
+	$(call if_changed,lzo)
 
 suffix-$(CONFIG_KERNEL_GZIP)	:= gz
 suffix-$(CONFIG_KERNEL_BZIP2)	:= bz2
 suffix-$(CONFIG_KERNEL_LZMA)	:= lzma
+suffix-$(CONFIG_KERNEL_LZO) 	:= lzo
 
 quiet_cmd_mkpiggy = MKPIGGY $@
       cmd_mkpiggy = $(obj)/mkpiggy $< > $@ || ( rm -f $@ ; false )
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 842b2a3..3b22fe8 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -162,6 +162,10 @@ static int lines, cols;
 #include "../../../../lib/decompress_unlzma.c"
 #endif
 
+#ifdef CONFIG_KERNEL_LZO
+#include "../../../../lib/decompress_unlzo.c"
+#endif
+
 static void scroll(void)
 {
 	int i;
-- 
1.6.3.3

[PATCH v3 4/4] Add LZO compression support for initramfs and old-style initrd

[Albin Tonnerre]

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
 lib/Kconfig      |    4 ++++
 lib/Makefile     |    1 +
 lib/decompress.c |    5 +++++
 usr/Kconfig      |   26 +++++++++++++++++++++-----
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/lib/Kconfig b/lib/Kconfig
index bb1326d..8639349 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -117,6 +117,10 @@ config DECOMPRESS_BZIP2
 config DECOMPRESS_LZMA
 	tristate
 
+config DECOMPRESS_LZO
+	select LZO_DECOMPRESS
+	tristate
+
 #
 # Generic allocator support is selected if needed
 #
diff --git a/lib/Makefile b/lib/Makefile
index 2e78277..cfa4041 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -69,6 +69,7 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
 lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
 lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
 lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o
+lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
 
 obj-$(CONFIG_TEXTSEARCH) += textsearch.o
 obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
diff --git a/lib/decompress.c b/lib/decompress.c
index d2842f5..a760681 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -9,6 +9,7 @@
 #include <linux/decompress/bunzip2.h>
 #include <linux/decompress/unlzma.h>
 #include <linux/decompress/inflate.h>
+#include <linux/decompress/unlzo.h>
 
 #include <linux/types.h>
 #include <linux/string.h>
@@ -22,6 +23,9 @@
 #ifndef CONFIG_DECOMPRESS_LZMA
 # define unlzma NULL
 #endif
+#ifndef CONFIG_DECOMPRESS_LZO
+# define unlzo NULL
+#endif
 
 static const struct compress_format {
 	unsigned char magic[2];
@@ -32,6 +36,7 @@ static const struct compress_format {
 	{ {037, 0236}, "gzip", gunzip },
 	{ {0x42, 0x5a}, "bzip2", bunzip2 },
 	{ {0x5d, 0x00}, "lzma", unlzma },
+	{ {0x89, 0x4c}, "lzo", unlzo },
 	{ {0, 0}, NULL, NULL }
 };
 
diff --git a/usr/Kconfig b/usr/Kconfig
index 1c3039f..e2721f5 100644
--- a/usr/Kconfig
+++ b/usr/Kconfig
@@ -72,6 +72,15 @@ config RD_LZMA
 	  Support loading of a LZMA encoded initial ramdisk or cpio buffer
 	  If unsure, say N.
 
+config RD_LZO
+	bool "Support initial ramdisks compressed using LZO" if EMBEDDED
+	default !EMBEDDED
+	depends on BLK_DEV_INITRD
+	select DECOMPRESS_LZO
+	help
+	  Support loading of a LZO encoded initial ramdisk or cpio buffer
+	  If unsure, say N.
+
 choice
 	prompt "Built-in initramfs compression mode" if INITRAMFS_SOURCE!=""
 	help
@@ -108,16 +117,15 @@ config INITRAMFS_COMPRESSION_GZIP
 	bool "Gzip"
 	depends on RD_GZIP
 	help
-	  The old and tried gzip compression. Its compression ratio is
-	  the poorest among the 3 choices; however its speed (both
-	  compression and decompression) is the fastest.
+	  The old and tried gzip compression. It provides a good balance
+	  between compression ratio and decompression speed.
 
 config INITRAMFS_COMPRESSION_BZIP2
 	bool "Bzip2"
 	depends on RD_BZIP2
 	help
 	  Its compression ratio and speed is intermediate.
-	  Decompression speed is slowest among the three.  The initramfs
+	  Decompression speed is slowest among the four.  The initramfs
 	  size is about 10% smaller with bzip2, in comparison to gzip.
 	  Bzip2 uses a large amount of memory. For modern kernels you
 	  will need at least 8MB RAM or more for booting.
@@ -128,7 +136,15 @@ config INITRAMFS_COMPRESSION_LZMA
 	help
 	  The most recent compression algorithm.
 	  Its ratio is best, decompression speed is between the other
-	  two. Compression is slowest.	The initramfs size is about 33%
+	  three. Compression is slowest. The initramfs size is about 33%
 	  smaller with LZMA in comparison to gzip.
 
+config INITRAMFS_COMPRESSION_LZO
+	bool "LZO"
+	depends on RD_LZO
+	help
+	  Its compression ratio is the poorest among the four. The kernel
+	  size is about about 10% bigger than gzip; however its speed
+	  (both compression and decompression) is the fastest.
+
 endchoice
-- 
1.6.3.3

Re: [PATCH v3 1/4] Add support for LZO-compressed kernels

[Wu Zhangjin]

Hi Albin,

Just tested it with the Compressed kernel support of MIPS in Ralf's
linux-queue git repository and with an extra patch for this LZO support
of MIPS on a YeeLoong2F netbook:

http://www.linux-mips.org/archives/linux-mips/2009-10/msg00143.html

It works well.

Thanks & Regards,
	Wu Zhangjin

On Wed, 2009-11-11 at 13:27 +0100, Albin Tonnerre wrote:
> This is the first part of the lzo patch
> The lzo compressor is worse than gzip at compression, but faster at
> extraction. Here are some figures for an ARM board I'm working on:
> 
> Uncompressed size: 3.24Mo
> gzip  1.61Mo 0.72s
> lzo   1.75Mo 0.48s
> 
> So for a compression ratio that is still relatively close to gzip, it's
> much faster to extract, at least in that case.
> 
> This version applies to kernel 2.6.31-rc3
> 
> This part contains:
>  - Makefile routine to support lzo compression
>  - Fixes to the existing lzo compressor so that it can be used in
>    compressed kernels
>  - wrapper around the existing lzo1x_decompress, as it only extracts one
>    block at a time, while we need to extract a whole file here
>  - config dialog for kernel compression
> 
> Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
> ---
>  include/linux/decompress/unlzo.h |   10 ++
>  init/Kconfig                     |   18 +++-
>  lib/decompress_unlzo.c           |  208 ++++++++++++++++++++++++++++++++++++++
>  lib/lzo/lzo1x_decompress.c       |    9 +-
>  scripts/Makefile.lib             |    5 +
>  5 files changed, 243 insertions(+), 7 deletions(-)
>  create mode 100644 include/linux/decompress/unlzo.h
>  create mode 100644 lib/decompress_unlzo.c
> 
> diff --git a/include/linux/decompress/unlzo.h b/include/linux/decompress/unlzo.h
> new file mode 100644
> index 0000000..9872297
> --- /dev/null
> +++ b/include/linux/decompress/unlzo.h
> @@ -0,0 +1,10 @@
> +#ifndef DECOMPRESS_UNLZO_H
> +#define DECOMPRESS_UNLZO_H
> +
> +int unlzo(unsigned char *inbuf, int len,
> +	int(*fill)(void*, unsigned int),
> +	int(*flush)(void*, unsigned int),
> +	unsigned char *output,
> +	int *pos,
> +	void(*error)(char *x));
> +#endif
> diff --git a/init/Kconfig b/init/Kconfig
> index f515864..eb65318 100644
> --- a/init/Kconfig
> +++ b/init/Kconfig
> @@ -115,10 +115,13 @@ config HAVE_KERNEL_BZIP2
>  config HAVE_KERNEL_LZMA
>  	bool
>  
> +config HAVE_KERNEL_LZO
> +	bool
> +
>  choice
>  	prompt "Kernel compression mode"
>  	default KERNEL_GZIP
> -	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA
> +	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA || HAVE_KERNEL_LZO
>  	help
>  	  The linux kernel is a kind of self-extracting executable.
>  	  Several compression algorithms are available, which differ
> @@ -141,9 +144,8 @@ config KERNEL_GZIP
>  	bool "Gzip"
>  	depends on HAVE_KERNEL_GZIP
>  	help
> -	  The old and tried gzip compression. Its compression ratio is
> -	  the poorest among the 3 choices; however its speed (both
> -	  compression and decompression) is the fastest.
> +	  The old and tried gzip compression. It provides a good balance
> +	  between compression ratio and decompression speed.
>  
>  config KERNEL_BZIP2
>  	bool "Bzip2"
> @@ -164,6 +166,14 @@ config KERNEL_LZMA
>  	  two. Compression is slowest.	The kernel size is about 33%
>  	  smaller with LZMA in comparison to gzip.
>  
> +config KERNEL_LZO
> +	bool "LZO"
> +	depends on HAVE_KERNEL_LZO
> +	help
> +	  Its compression ratio is the poorest among the 4. The kernel
> +	  size is about about 10% bigger than gzip; however its speed
> +	  (both compression and decompression) is the fastest.
> +
>  endchoice
>  
>  config SWAP
> diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c
> new file mode 100644
> index 0000000..2bb736f
> --- /dev/null
> +++ b/lib/decompress_unlzo.c
> @@ -0,0 +1,208 @@
> +/*
> + * LZO decompressor for the Linux kernel. Code borrowed from the lzo
> + * implementation by Markus Franz Xaver Johannes Oberhumer.
> + *
> + * Linux kernel adaptation:
> + * Copyright (C) 2009
> + * Albin Tonnerre, Free Electrons <albin.tonnerre@free-electrons.com>
> + *
> + * Original code:
> + * Copyright (C) 1996-2005 Markus Franz Xaver Johannes Oberhumer
> + * All Rights Reserved.
> + *
> + * lzop and the LZO library are free software; you can redistribute them
> + * and/or modify them under the terms of the GNU General Public License as
> + * published by the Free Software Foundation; either version 2 of
> + * the License, or (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; see the file COPYING.
> + * If not, write to the Free Software Foundation, Inc.,
> + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
> + *
> + * Markus F.X.J. Oberhumer
> + * <markus@oberhumer.com>
> + * http://www.oberhumer.com/opensource/lzop/
> + */
> +
> +#ifdef STATIC
> +#include "lzo/lzo1x_decompress.c"
> +#else
> +#include <linux/slab.h>
> +#include <linux/decompress/unlzo.h>
> +#endif
> +
> +#include <linux/types.h>
> +#include <linux/lzo.h>
> +#include <linux/decompress/mm.h>
> +
> +#include <linux/compiler.h>
> +#include <asm/unaligned.h>
> +
> +static const unsigned char lzop_magic[] =
> +	{ 0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a };
> +
> +#define LZO_BLOCK_SIZE        (256*1024l)
> +#define HEADER_HAS_FILTER      0x00000800L
> +
> +STATIC inline int INIT parse_header(u8 *input, u8 *skip)
> +{
> +	int l;
> +	u8 *parse = input;
> +	u8 level = 0;
> +	u16 version;
> +
> +	/* read magic: 9 first bits */
> +	for (l = 0; l < 9; l++) {
> +		if (*parse++ != lzop_magic[l])
> +			return 0;
> +	}
> +	/* get version (2bytes), skip library version (2),
> +	 * 'need to be extracted' version (2) and
> +	 * method (1) */
> +	version = get_unaligned_be16(parse);
> +	parse += 7;
> +	if (version >= 0x0940)
> +		level = *parse++;
> +	if (get_unaligned_be32(parse) & HEADER_HAS_FILTER)
> +		parse += 8; /* flags + filter info */
> +	else
> +		parse += 4; /* flags */
> +
> +	/* skip mode and mtime_low */
> +	parse += 8;
> +	if (version >= 0x0940)
> +		parse += 4;	/* skip mtime_high */
> +
> +	l = *parse++;
> +	/* don't care about the file name, and skip checksum */
> +	parse += l + 4;
> +
> +	*skip = parse - input;
> +	return 1;
> +}
> +
> +STATIC inline int INIT unlzo(u8 *input, int in_len,
> +				int (*fill) (void *, unsigned int),
> +				int (*flush) (void *, unsigned int),
> +				u8 *output, int *posp,
> +				void (*error_fn) (char *x))
> +{
> +	u8 skip = 0, r = 0;
> +	u32 src_len, dst_len;
> +	size_t tmp;
> +	u8 *in_buf, *in_buf_save, *out_buf;
> +	int obytes_processed = 0;
> +
> +	set_error_fn(error_fn);
> +
> +	if (output)
> +		out_buf = output;
> +	else if (!flush) {
> +		error("NULL output pointer and no flush function provided");
> +		goto exit;
> +	} else {
> +		out_buf = malloc(LZO_BLOCK_SIZE);
> +		if (!out_buf) {
> +			error("Could not allocate output buffer");
> +			goto exit;
> +		}
> +	}
> +
> +	if (input && fill) {
> +		error("Both input pointer and fill function provided, don't know what to do");
> +		goto exit_1;
> +	} else if (input)
> +		in_buf = input;
> +	else if (!fill || !posp) {
> +		error("NULL input pointer and missing position pointer or fill function");
> +		goto exit_1;
> +	} else {
> +		in_buf = malloc(lzo1x_worst_compress(LZO_BLOCK_SIZE));
> +		if (!in_buf) {
> +			error("Could not allocate input buffer");
> +			goto exit_1;
> +		}
> +	}
> +	in_buf_save = in_buf;
> +
> +	if (posp)
> +		*posp = 0;
> +
> +	if (fill)
> +		fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE));
> +
> +	if (!parse_header(input, &skip)) {
> +		error("invalid header");
> +		goto exit_2;
> +	}
> +	in_buf += skip;
> +
> +	if (posp)
> +		*posp = skip;
> +
> +	for (;;) {
> +		/* read uncompressed block size */
> +		dst_len = get_unaligned_be32(in_buf);
> +		in_buf += 4;
> +
> +		/* exit if last block */
> +		if (dst_len == 0) {
> +			if (posp)
> +				*posp += 4;
> +			break;
> +		}
> +
> +		if (dst_len > LZO_BLOCK_SIZE) {
> +			error("dest len longer than block size");
> +			goto exit_2;
> +		}
> +
> +		/* read compressed block size, and skip block checksum info */
> +		src_len = get_unaligned_be32(in_buf);
> +		in_buf += 8;
> +
> +		if (src_len <= 0 || src_len > dst_len) {
> +			error("file corrupted");
> +			goto exit_2;
> +		}
> +
> +		/* decompress */
> +		tmp = dst_len;
> +		r = lzo1x_decompress_safe((u8 *) in_buf, src_len, out_buf, &tmp);
> +
> +		if (r != LZO_E_OK || dst_len != tmp) {
> +			error("Compressed data violation");
> +			goto exit_2;
> +		}
> +
> +		obytes_processed += dst_len;
> +		if (flush)
> +			flush(out_buf, dst_len);
> +		if (output)
> +			out_buf += dst_len;
> +		if (posp)
> +			*posp += src_len + 12;
> +		if (fill) {
> +			in_buf = in_buf_save;
> +			fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE));
> +		} else
> +			in_buf += src_len;
> +	}
> +
> +exit_2:
> +	if (!input)
> +		free(in_buf);
> +exit_1:
> +	if (!output)
> +		free(out_buf);
> +exit:
> +	return obytes_processed;
> +}
> +
> +#define decompress unlzo
> diff --git a/lib/lzo/lzo1x_decompress.c b/lib/lzo/lzo1x_decompress.c
> index 5dc6b29..f2fd098 100644
> --- a/lib/lzo/lzo1x_decompress.c
> +++ b/lib/lzo/lzo1x_decompress.c
> @@ -11,11 +11,13 @@
>   *  Richard Purdie <rpurdie@openedhand.com>
>   */
>  
> +#ifndef STATIC
>  #include <linux/module.h>
>  #include <linux/kernel.h>
> -#include <linux/lzo.h>
> -#include <asm/byteorder.h>
> +#endif
> +
>  #include <asm/unaligned.h>
> +#include <linux/lzo.h>
>  #include "lzodefs.h"
>  
>  #define HAVE_IP(x, ip_end, ip) ((size_t)(ip_end - ip) < (x))
> @@ -244,9 +246,10 @@ lookbehind_overrun:
>  	*out_len = op - out;
>  	return LZO_E_LOOKBEHIND_OVERRUN;
>  }
> -
> +#ifndef STATIC
>  EXPORT_SYMBOL_GPL(lzo1x_decompress_safe);
>  
>  MODULE_LICENSE("GPL");
>  MODULE_DESCRIPTION("LZO1X Decompressor");
>  
> +#endif
> diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
> index ffdafb2..39c3483 100644
> --- a/scripts/Makefile.lib
> +++ b/scripts/Makefile.lib
> @@ -230,3 +230,8 @@ quiet_cmd_lzma = LZMA    $@
>  cmd_lzma = (cat $(filter-out FORCE,$^) | \
>  	lzma -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
>  	(rm -f $@ ; false)
> +
> +quiet_cmd_lzo = LZO    $@
> +cmd_lzo = (cat $(filter-out FORCE,$^) | \
> +	lzop -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \
> +	(rm -f $@ ; false)

Re: [PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Martin Michlmayr]

* Albin Tonnerre <albin.tonnerre@free-electrons.com> [2009-11-11 13:27]:
> This is the second part of patch. This part includes:
>  - changes to ach/arch/boot/Makefile to make it easier to add new
>    compression types
> +
> +# For __aeabi_uidivmod
> +lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o

I build my kernel in a different directory to where the sources are
stored and I get:

make[7]: *** No rule to make target `/home/tbm/kernel/linux-2.6-2.6.32~rc6/debian/build/source_armel_none/arch/arm/lib/lib1funcs.o', needed by `arch/arm/boot/compressed/vmlinux'.  Stop.

The following patch makes it compile for me:

--- a/arm/boot/compressed/Makefile	2009-11-12 12:26:47.000000000 +0000
+++ b/arm/boot/compressed/Makefile	2009-11-12 12:26:58.000000000 +0000
@@ -101,7 +101,7 @@
 LDFLAGS_vmlinux += -T
 
 # For __aeabi_uidivmod
-lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o
+lib1funcs = arch/$(SRCARCH)/lib/lib1funcs.o
 
 # Don't allow any static data in misc.o, which
 # would otherwise mess up our GOT table

-- 
Martin Michlmayr
http://www.cyrius.com/

Re: [PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Martin Michlmayr]

* Albin Tonnerre <albin.tonnerre@free-electrons.com> [2009-11-11 13:27]:
> This is the second part of patch. This part includes:
>  - changes to ach/arch/boot/Makefile to make it easier to add new
>    compression types
>  - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
>    gzip, depending on the config

I tested your arm patch together with your unsubmitted patch to allow
lzma and it works great!  Thanks a lot for working on this.  Feel free
to add my:

Tested-by: Martin Michlmayr <tbm@cyrius.com>

I tested lzma compressed kernels on the following ARM machines:
 - Linksys NSLU2 (ixp4xx)
 - SheevaPlug (kirkwood)
 - QNAP TS-219 (kirkwood)
 - Thecus N2100 (iop32x)

-- 
Martin Michlmayr
http://www.cyrius.com/

Re: [PATCH v3 0/4] Add support for LZO-compressed kernel images

[Russell King]

On Wed, Nov 11, 2009 at 01:27:54PM +0100, Albin Tonnerre wrote:
> Russell: since I believe your initial concerns regarding the ARM part of the
> patch got fixed, do I have your approval?

I was going to run some tests on it to compare this with the existing
stuff, but I haven't managed to get around to it (due to the platforms
I was going to run the tests on breaking - twice.)

I hope I'll be able to run tests tomorrow/this weekend.

-- 
Russell King
 Linux kernel    2.6 ARM Linux   - http://www.arm.linux.org.uk/
 maintainer of:

Re: [PATCH v3 1/4] Add support for LZO-compressed kernels

[Andrew Morton]

On Wed, 11 Nov 2009 13:27:55 +0100
Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:

> +STATIC inline int INIT unlzo(u8 *input, int in_len,

I assume the macro gunk is to support testing of the code in a
userspace test haress?

Where does INIT get defined?

Re: [PATCH v3 1/4] Add support for LZO-compressed kernels

[Albin Tonnerre]

On Thu, 12 Nov 2009 14:33 -0800, Andrew Morton wrote :
> On Wed, 11 Nov 2009 13:27:55 +0100
> Albin Tonnerre <albin.tonnerre@free-electrons.com> wrote:
> 
> > +STATIC inline int INIT unlzo(u8 *input, int in_len,
> 
> I assume the macro gunk is to support testing of the code in a
> userspace test haress?

> Where does INIT get defined?

STATIC and INIT are both defined in include/linux/decompress/mm.h. They're
needed to support using the decompression code both in the kernel bootstrap code
if the kernel image is compressed, and as part of the kernel code if it used to
extract an initrd/initramfs image.

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Sebastian Andrzej Siewior]

* Martin Michlmayr | 2009-11-12 12:59:38 [+0000]:

>* Albin Tonnerre <albin.tonnerre@free-electrons.com> [2009-11-11 13:27]:
>> This is the second part of patch. This part includes:
>>  - changes to ach/arch/boot/Makefile to make it easier to add new
>>    compression types
>> +
>> +# For __aeabi_uidivmod
>> +lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o
>
>I build my kernel in a different directory to where the sources are
>stored and I get:
>
>make[7]: *** No rule to make target `/home/tbm/kernel/linux-2.6-2.6.32~rc6/debian/build/source_armel_none/arch/arm/lib/lib1funcs.o', needed by `arch/arm/boot/compressed/vmlinux'.  Stop.
>
>The following patch makes it compile for me:
>
>--- a/arm/boot/compressed/Makefile	2009-11-12 12:26:47.000000000 +0000
>+++ b/arm/boot/compressed/Makefile	2009-11-12 12:26:58.000000000 +0000
>@@ -101,7 +101,7 @@
> LDFLAGS_vmlinux += -T
> 
> # For __aeabi_uidivmod
>-lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o
>+lib1funcs = arch/$(SRCARCH)/lib/lib1funcs.o

+lib1funcs = $(obj)/arch/$(SRCARCH)/lib/lib1funcs.o

This is untested but it should refer to obj because you break out of
tree builds (O=) if you don't :)

> 
> # Don't allow any static data in misc.o, which
> # would otherwise mess up our GOT table
>
>-- 
>Martin Michlmayr
>http://www.cyrius.com/

Sebastian

Re: [PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Martin Michlmayr]

* Sebastian Andrzej Siewior <sebastian@breakpoint.cc> [2009-11-13 15:29]:
> > # For __aeabi_uidivmod
> >-lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o
> >+lib1funcs = arch/$(SRCARCH)/lib/lib1funcs.o
> 
> +lib1funcs = $(obj)/arch/$(SRCARCH)/lib/lib1funcs.o
> 
> This is untested but it should refer to obj because you break out of
> tree builds (O=) if you don't :)

With this I get:
 make[4]: *** No rule to make target `arch/arm/boot/compressed/arch/arm/lib/lib1funcs.o', needed by `arch/arm/boot/compressed/vmlinux'.  Stop.

-- 
Martin Michlmayr
http://www.cyrius.com/

Re: [PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

On Fri, 13 Nov 2009 14:50 +0000, Martin Michlmayr wrote :
> * Sebastian Andrzej Siewior <sebastian@breakpoint.cc> [2009-11-13 15:29]:
> > > # For __aeabi_uidivmod
> > >-lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o
> > >+lib1funcs = arch/$(SRCARCH)/lib/lib1funcs.o
> > 
> > +lib1funcs = $(obj)/arch/$(SRCARCH)/lib/lib1funcs.o
> > 
> > This is untested but it should refer to obj because you break out of
> > tree builds (O=) if you don't :)
> 
> With this I get:
>  make[4]: *** No rule to make target `arch/arm/boot/compressed/arch/arm/lib/lib1funcs.o', needed by `arch/arm/boot/compressed/vmlinux'.  Stop.

Sam: what's the correct way to solve this issue?

Regards,
-- 
Albin Tonnerre, Free Electrons
Kernel, drivers and embedded Linux development,
consulting, training and support.
http://free-electrons.com

Re: [PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Russell King]

On Fri, Nov 13, 2009 at 03:54:00PM +0100, Albin Tonnerre wrote:
> On Fri, 13 Nov 2009 14:50 +0000, Martin Michlmayr wrote :
> > * Sebastian Andrzej Siewior <sebastian@breakpoint.cc> [2009-11-13 15:29]:
> > > > # For __aeabi_uidivmod
> > > >-lib1funcs = $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.o
> > > >+lib1funcs = arch/$(SRCARCH)/lib/lib1funcs.o
> > > 
> > > +lib1funcs = $(obj)/arch/$(SRCARCH)/lib/lib1funcs.o
> > > 
> > > This is untested but it should refer to obj because you break out of
> > > tree builds (O=) if you don't :)
> > 
> > With this I get:
> >  make[4]: *** No rule to make target `arch/arm/boot/compressed/arch/arm/lib/lib1funcs.o', needed by `arch/arm/boot/compressed/vmlinux'.  Stop.
> 
> Sam: what's the correct way to solve this issue?

We already have something similar - with the font support.  Just do this:

lib1funcs = $(obj)/lib1funcs.o

$(obj)/lib1funcs.S: $(srctree)/arch/arm/lib/lib1funcs.S FORCE
        $(call cmd,shipped)

-- 
Russell King
 Linux kernel    2.6 ARM Linux   - http://www.arm.linux.org.uk/
 maintainer of:

Re: [PATCH v3 1/4] Add support for LZO-compressed kernels

[Russell King]

On Wed, Nov 11, 2009 at 01:27:55PM +0100, Albin Tonnerre wrote:
> This is the first part of the lzo patch
> The lzo compressor is worse than gzip at compression, but faster at
> extraction. Here are some figures for an ARM board I'm working on:
> 
> Uncompressed size: 3.24Mo
> gzip  1.61Mo 0.72s
> lzo   1.75Mo 0.48s

>From my testing on a Cortex A9 model:
- lzo decompressor is 65% of the time gzip takes to decompress a kernel
- lzo kernel is 9% larger than a gzip kernel

which I'm happy to say confirms your figures when comparing the two.

However, when comparing your new gzip code to the old gzip code:
- new is 99% of the size of the old code
- new takes 42% of the time to decompress than the old code

What this means is that for a proper comparison, the results get even better:
- lzo is 7.5% larger than the old gzip'd kernel image
- lzo takes 28% of the time that the old gzip code took

So the expense seems definitely worth the effort.  The only reason I
can think of ever using gzip would be if you needed the additional
compression (eg, because you have limited flash to store the image.)

I would argue that the default for ARM should therefore be LZO.

-- 
Russell King
 Linux kernel    2.6 ARM Linux   - http://www.arm.linux.org.uk/
 maintainer of:

Re: [PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Martin Michlmayr]

* Russell King <rmk+lkml@arm.linux.org.uk> [2009-11-13 15:21]:
> > >  make[4]: *** No rule to make target `arch/arm/boot/compressed/arch/arm/lib/lib1funcs.o', needed by `arch/arm/boot/compressed/vmlinux'.  Stop.
> > 
> > Sam: what's the correct way to solve this issue?
> 
> We already have something similar - with the font support.  Just do this:

That works.
-- 
Martin Michlmayr
http://www.cyrius.com/

[PATCH v3 2/4] Add support for LZO-compressed kernels for ARM

[Albin Tonnerre]

This is the second part of patch. This part includes:
 - changes to ach/arch/boot/Makefile to make it easier to add new
   compression types
 - new piggy.lzo.S necessary for lzo compression
 - changes in arch/arm/boot/compressed/misc.c to allow the use of lzo or
   gzip, depending on the config
 - Kconfig support

Signed-off-by: Albin Tonnerre <albin.tonnerre@free-electrons.com>
---
Updated patch to fix the out-of-tree build issue, thanks to Martin Michlmayr
and Russell King

 arch/arm/Kconfig                      |    2 +
 arch/arm/boot/compressed/Makefile     |   34 +++++++---
 arch/arm/boot/compressed/misc.c       |  116 ++++++++++-----------------------
 arch/arm/boot/compressed/piggy.S      |    6 --
 arch/arm/boot/compressed/piggy.gzip.S |    6 ++
 arch/arm/boot/compressed/piggy.lzo.S  |    6 ++
 6 files changed, 73 insertions(+), 97 deletions(-)
 delete mode 100644 arch/arm/boot/compressed/piggy.S
 create mode 100644 arch/arm/boot/compressed/piggy.gzip.S
 create mode 100644 arch/arm/boot/compressed/piggy.lzo.S

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 1c4119c..350921d 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -18,6 +18,8 @@ config ARM
 	select HAVE_KRETPROBES if (HAVE_KPROBES)
 	select HAVE_FUNCTION_TRACER if (!XIP_KERNEL)
 	select HAVE_GENERIC_DMA_COHERENT
+	select HAVE_KERNEL_GZIP
+	select HAVE_KERNEL_LZO
 	help
 	  The ARM series is a line of low-power-consumption RISC chip designs
 	  licensed by ARM Ltd and targeted at embedded applications and
diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile
index ce39dc5..2d4d88b 100644
--- a/arch/arm/boot/compressed/Makefile
+++ b/arch/arm/boot/compressed/Makefile
@@ -63,8 +63,12 @@ endif
 
 SEDFLAGS	= s/TEXT_START/$(ZTEXTADDR)/;s/BSS_START/$(ZBSSADDR)/
 
-targets       := vmlinux vmlinux.lds piggy.gz piggy.o font.o font.c \
-		 head.o misc.o $(OBJS)
+suffix_$(CONFIG_KERNEL_GZIP) = gzip
+suffix_$(CONFIG_KERNEL_LZO)  = lzo
+
+targets       := vmlinux vmlinux.lds \
+		 piggy.$(suffix_y) piggy.$(suffix_y).o \
+		 font.o font.c head.o misc.o $(OBJS)
 
 ifeq ($(CONFIG_FUNCTION_TRACER),y)
 ORIG_CFLAGS := $(KBUILD_CFLAGS)
@@ -87,22 +91,34 @@ endif
 ifneq ($(PARAMS_PHYS),)
 LDFLAGS_vmlinux += --defsym params_phys=$(PARAMS_PHYS)
 endif
-LDFLAGS_vmlinux += -p --no-undefined -X \
-	$(shell $(CC) $(KBUILD_CFLAGS) --print-libgcc-file-name) -T
+# ?
+LDFLAGS_vmlinux += -p
+# Report unresolved symbol references
+LDFLAGS_vmlinux += --no-undefined
+# Delete all temporary local symbols
+LDFLAGS_vmlinux += -X
+# Next argument is a linker script
+LDFLAGS_vmlinux += -T
+
+# For __aeabi_uidivmod
+lib1funcs = $(obj)/lib1funcs.o
+
+$(obj)/lib1funcs.S: $(srctree)/arch/$(SRCARCH)/lib/lib1funcs.S FORCE
+	$(call cmd,shipped)
 
 # Don't allow any static data in misc.o, which
 # would otherwise mess up our GOT table
 CFLAGS_misc.o := -Dstatic=
 
-$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.o \
-	 	$(addprefix $(obj)/, $(OBJS)) FORCE
+$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/$(HEAD) $(obj)/piggy.$(suffix_y).o \
+	 	$(addprefix $(obj)/, $(OBJS)) $(lib1funcs) FORCE
 	$(call if_changed,ld)
 	@:
 
-$(obj)/piggy.gz: $(obj)/../Image FORCE
-	$(call if_changed,gzip)
+$(obj)/piggy.$(suffix_y): $(obj)/../Image FORCE
+	$(call if_changed,$(suffix_y))
 
-$(obj)/piggy.o:  $(obj)/piggy.gz FORCE
+$(obj)/piggy.$(suffix_y).o:  $(obj)/piggy.$(suffix_y) FORCE
 
 CFLAGS_font.o := -Dstatic=
 
diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c
index 17153b5..7e0fe4d 100644
--- a/arch/arm/boot/compressed/misc.c
+++ b/arch/arm/boot/compressed/misc.c
@@ -18,10 +18,15 @@
 
 unsigned int __machine_arch_type;
 
+#define _LINUX_STRING_H_
+
 #include <linux/compiler.h>	/* for inline */
 #include <linux/types.h>	/* for size_t */
 #include <linux/stddef.h>	/* for NULL */
 #include <asm/string.h>
+#include <linux/linkage.h>
+
+#include <asm/unaligned.h>
 
 #ifdef STANDALONE_DEBUG
 #define putstr printf
@@ -188,34 +193,8 @@ static inline __ptr_t memcpy(__ptr_t __dest, __const __ptr_t __src,
 /*
  * gzip delarations
  */
-#define OF(args)  args
 #define STATIC static
 
-typedef unsigned char  uch;
-typedef unsigned short ush;
-typedef unsigned long  ulg;
-
-#define WSIZE 0x8000		/* Window size must be at least 32k, */
-				/* and a power of two */
-
-static uch *inbuf;		/* input buffer */
-static uch window[WSIZE];	/* Sliding window buffer */
-
-static unsigned insize;		/* valid bytes in inbuf */
-static unsigned inptr;		/* index of next byte to be processed in inbuf */
-static unsigned outcnt;		/* bytes in output buffer */
-
-/* gzip flag byte */
-#define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
-#define CONTINUATION 0x02 /* bit 1 set: continuation of multi-part gzip file */
-#define EXTRA_FIELD  0x04 /* bit 2 set: extra field present */
-#define ORIG_NAME    0x08 /* bit 3 set: original file name present */
-#define COMMENT      0x10 /* bit 4 set: file comment present */
-#define ENCRYPTED    0x20 /* bit 5 set: file is encrypted */
-#define RESERVED     0xC0 /* bit 6,7:   reserved */
-
-#define get_byte()  (inptr < insize ? inbuf[inptr++] : fill_inbuf())
-
 /* Diagnostic functions */
 #ifdef DEBUG
 #  define Assert(cond,msg) {if(!(cond)) error(msg);}
@@ -233,24 +212,20 @@ static unsigned outcnt;		/* bytes in output buffer */
 #  define Tracecv(c,x)
 #endif
 
-static int  fill_inbuf(void);
-static void flush_window(void);
 static void error(char *m);
 
 extern char input_data[];
 extern char input_data_end[];
 
-static uch *output_data;
-static ulg output_ptr;
-static ulg bytes_out;
+static unsigned char *output_data;
+static unsigned long output_ptr;
 
 static void error(char *m);
 
 static void putstr(const char *);
 
-extern int end;
-static ulg free_mem_ptr;
-static ulg free_mem_end_ptr;
+static unsigned long free_mem_ptr;
+static unsigned long free_mem_end_ptr;
 
 #ifdef STANDALONE_DEBUG
 #define NO_INFLATE_MALLOC
@@ -258,46 +233,13 @@ static ulg free_mem_end_ptr;
 
 #define ARCH_HAS_DECOMP_WDOG
 
-#include "../../../../lib/inflate.c"
-
-/* ===========================================================================
- * Fill the input buffer. This is called only when the buffer is empty
- * and at least one byte is really needed.
- */
-int fill_inbuf(void)
-{
-	if (insize != 0)
-		error("ran out of input data");
-
-	inbuf = input_data;
-	insize = &input_data_end[0] - &input_data[0];
-
-	inptr = 1;
-	return inbuf[0];
-}
+#ifdef CONFIG_KERNEL_GZIP
+#include "../../../../lib/decompress_inflate.c"
+#endif
 
-/* ===========================================================================
- * Write the output window window[0..outcnt-1] and update crc and bytes_out.
- * (Used for the decompressed data only.)
- */
-void flush_window(void)
-{
-	ulg c = crc;
-	unsigned n;
-	uch *in, *out, ch;
-
-	in = window;
-	out = &output_data[output_ptr];
-	for (n = 0; n < outcnt; n++) {
-		ch = *out++ = *in++;
-		c = crc_32_tab[((int)c ^ ch) & 0xff] ^ (c >> 8);
-	}
-	crc = c;
-	bytes_out += (ulg)outcnt;
-	output_ptr += (ulg)outcnt;
-	outcnt = 0;
-	putstr(".");
-}
+#ifdef CONFIG_KERNEL_LZO
+#include "../../../../lib/decompress_unlzo.c"
+#endif
 
 #ifndef arch_error
 #define arch_error(x)
@@ -314,22 +256,33 @@ static void error(char *x)
 	while(1);	/* Halt */
 }
 
+asmlinkage void __div0(void)
+{
+	error("Attempting division by 0!");
+}
+
 #ifndef STANDALONE_DEBUG
 
-ulg
-decompress_kernel(ulg output_start, ulg free_mem_ptr_p, ulg free_mem_ptr_end_p,
-		  int arch_id)
+unsigned long
+decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p,
+		unsigned long free_mem_ptr_end_p,
+		int arch_id)
 {
-	output_data		= (uch *)output_start;	/* Points to kernel start */
+	unsigned char *tmp;
+
+	output_data		= (unsigned char *)output_start;
 	free_mem_ptr		= free_mem_ptr_p;
 	free_mem_end_ptr	= free_mem_ptr_end_p;
 	__machine_arch_type	= arch_id;
 
 	arch_decomp_setup();
 
-	makecrc();
+	tmp = (unsigned char *) (((unsigned long)input_data_end) - 4);
+	output_ptr = get_unaligned_le32(tmp);
+
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+			NULL, NULL, output_data, NULL, error);
 	putstr(" done, booting the kernel.\n");
 	return output_ptr;
 }
@@ -341,11 +294,10 @@ int main()
 {
 	output_data = output_buffer;
 
-	makecrc();
 	putstr("Uncompressing Linux...");
-	gunzip();
+	decompress(input_data, input_data_end - input_data,
+			NULL, NULL, output_data, NULL, error);
 	putstr("done.\n");
 	return 0;
 }
 #endif
-	
diff --git a/arch/arm/boot/compressed/piggy.S b/arch/arm/boot/compressed/piggy.S
deleted file mode 100644
index 54c9518..0000000
--- a/arch/arm/boot/compressed/piggy.S
+++ /dev/null
@@ -1,6 +0,0 @@
-	.section .piggydata,#alloc
-	.globl	input_data
-input_data:
-	.incbin	"arch/arm/boot/compressed/piggy.gz"
-	.globl	input_data_end
-input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.gzip.S b/arch/arm/boot/compressed/piggy.gzip.S
new file mode 100644
index 0000000..a68adf9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.gzip.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.gzip"
+	.globl	input_data_end
+input_data_end:
diff --git a/arch/arm/boot/compressed/piggy.lzo.S b/arch/arm/boot/compressed/piggy.lzo.S
new file mode 100644
index 0000000..a425ad9
--- /dev/null
+++ b/arch/arm/boot/compressed/piggy.lzo.S
@@ -0,0 +1,6 @@
+	.section .piggydata,#alloc
+	.globl	input_data
+input_data:
+	.incbin	"arch/arm/boot/compressed/piggy.lzo"
+	.globl	input_data_end
+input_data_end:
-- 
1.6.3.3