[PATCHv3 0/3] x86: boot protocol updates.

[PATCHv3 0/3] x86: boot protocol updates.

[Ian Campbell]

Updates since last time:
  - Rebased to latest x86.git#mm (no changes required).

Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>

[PATCHv3 1/3] x86: use ELF format in compressed images.

[Ian Campbell]

This allows other boot loaders such as the Xen domain builder the
opportunity to extract the ELF file.

Signed-off-by: Ian Campbell <ijc@hellion.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: virtualization@lists.linux-foundation.org
---
 Documentation/i386/boot.txt       |   18 ++++++++++++
 arch/x86/boot/Makefile            |   14 +++++++++
 arch/x86/boot/compressed/Makefile |    2 +-
 arch/x86/boot/compressed/misc.c   |   56 +++++++++++++++++++++++++++++++++++++
 arch/x86/boot/header.S            |    6 ++++
 5 files changed, 95 insertions(+), 1 deletions(-)

diff --git a/Documentation/i386/boot.txt b/Documentation/i386/boot.txt
index fc49b79..b5f5ba1 100644
--- a/Documentation/i386/boot.txt
+++ b/Documentation/i386/boot.txt
@@ -170,6 +170,8 @@ Offset	Proto	Name		Meaning
 0238/4	2.06+	cmdline_size	Maximum size of the kernel command line
 023C/4	2.07+	hardware_subarch Hardware subarchitecture
 0240/8	2.07+	hardware_subarch_data Subarchitecture-specific data
+0248/4	2.08+	compressed_payload_offset
+024C/4	2.08+	compressed_payload_length
 
 (1) For backwards compatibility, if the setup_sects field contains 0, the
     real value is 4.
@@ -512,6 +514,22 @@ Protocol:	2.07+
 
   A pointer to data that is specific to hardware subarch
 
+Field name:	compressed_payload_offset
+Type:		read
+Offset/size:	0x248/4
+Protocol:	2.08+
+
+  If non-zero then this field contains the offset from the end of the
+  real-mode code to the compressed payload. The compression format
+  should be determined using the standard magic number, currently only
+  gzip is used.
+  
+Field name:	compressed_payload_length
+Type:		read
+Offset/size:	0x24c/4
+Protocol:	2.08+
+
+  The length of the compressed payload.
 
 **** THE KERNEL COMMAND LINE
 
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index f88458e..9695aff 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -94,6 +94,20 @@ $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE
 
 SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
 
+sed-offsets := -e 's/^00*/0/' \
+        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/\#define \2 0x\1/p'
+
+quiet_cmd_offsets = OFFSETS $@
+      cmd_offsets = $(NM) $< | sed -n $(sed-offsets) > $@
+
+$(obj)/offsets.h: $(obj)/compressed/vmlinux FORCE
+	$(call if_changed,offsets)
+
+targets += offsets.h
+
+AFLAGS_header.o += -I$(obj)
+$(obj)/header.o: $(obj)/offsets.h
+
 LDFLAGS_setup.elf	:= -T
 $(obj)/setup.elf: $(src)/setup.ld $(SETUP_OBJS) FORCE
 	$(call if_changed,ld)
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index d2b9f3b..92fdd35 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -22,7 +22,7 @@ $(obj)/vmlinux: $(src)/vmlinux_$(BITS).lds $(obj)/head_$(BITS).o $(obj)/misc.o $
 	$(call if_changed,ld)
 	@:
 
-OBJCOPYFLAGS_vmlinux.bin := -O binary -R .note -R .comment -S
+OBJCOPYFLAGS_vmlinux.bin :=  -R .comment -S
 $(obj)/vmlinux.bin: vmlinux FORCE
 	$(call if_changed,objcopy)
 
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 8182e32..69aec2f 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -15,6 +15,10 @@
  * we just keep it from happening
  */
 #undef CONFIG_PARAVIRT
+#ifdef CONFIG_X86_32
+#define _ASM_DESC_H_ 1
+#endif
+
 #ifdef CONFIG_X86_64
 #define _LINUX_STRING_H_ 1
 #define __LINUX_BITMAP_H 1
@@ -22,6 +26,7 @@
 
 #include <linux/linkage.h>
 #include <linux/screen_info.h>
+#include <linux/elf.h>
 #include <asm/io.h>
 #include <asm/page.h>
 #include <asm/boot.h>
@@ -365,6 +370,56 @@ static void error(char *x)
 		asm("hlt");
 }
 
+static void parse_elf(void *output)
+{
+#ifdef CONFIG_X86_64
+	Elf64_Ehdr ehdr;
+	Elf64_Phdr *phdrs, *phdr;
+#else
+	Elf32_Ehdr ehdr;
+	Elf32_Phdr *phdrs, *phdr;
+#endif
+	void *dest;
+	int i;
+
+	memcpy(&ehdr, output, sizeof(ehdr));
+	if(ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
+	   ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
+	   ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
+	   ehdr.e_ident[EI_MAG3] != ELFMAG3)
+	{
+		error("Kernel is not a valid ELF file");
+		return;
+	}
+
+	putstr("Parsing ELF... ");
+
+	phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum);
+	if (!phdrs)
+		error("Failed to allocate space for phdrs");
+
+	memcpy(phdrs, output + ehdr.e_phoff, sizeof(*phdrs) * ehdr.e_phnum);
+
+	for (i=0; i<ehdr.e_phnum; i++) {
+		phdr = &phdrs[i];
+
+		switch (phdr->p_type) {
+		case PT_LOAD:
+#ifdef CONFIG_RELOCATABLE
+			dest = output;
+			dest += (phdr->p_paddr - LOAD_PHYSICAL_ADDR);
+#else
+			dest = (void*)(phdr->p_paddr);
+#endif
+			memcpy(dest,
+			       output + phdr->p_offset,
+			       phdr->p_filesz);
+			break;
+		default: /* Ignore other PT_* */ break;
+		}
+	}
+}
+
 asmlinkage void decompress_kernel(void *rmode, memptr heap,
 				  uch *input_data, unsigned long input_len,
 				  uch *output)
@@ -408,6 +463,7 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
 	makecrc();
 	putstr("\nDecompressing Linux... ");
 	gunzip();
+	parse_elf(output);
 	putstr("done.\nBooting the kernel.\n");
 	return;
 }
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index 64ad901..8471658 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -22,6 +22,7 @@
 #include <asm/page.h>
 #include <asm/setup.h>
 #include "boot.h"
+#include "offsets.h"
 
 SETUPSECTS	= 4			/* default nr of setup-sectors */
 BOOTSEG		= 0x07C0		/* original address of boot-sector */
@@ -223,6 +224,11 @@ hardware_subarch:	.long 0			# subarchitecture, added with 2.07
 
 hardware_subarch_data:	.quad 0
 
+compressed_payload_offset:
+			.long input_data
+compressed_payload_length:
+			.long input_data_end-input_data
+
 # End of setup header #####################################################
 
 	.section ".inittext", "ax"
-- 
1.5.4

[PATCHv3 2/3] x86: add a crc32 checksum to the kernel image.

[Ian Campbell]

Signed-off-by: Ian Campbell <ijc@hellion.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
---
 Documentation/i386/boot.txt |    8 ++++
 arch/x86/boot/tools/build.c |   88 ++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 95 insertions(+), 1 deletions(-)

diff --git a/Documentation/i386/boot.txt b/Documentation/i386/boot.txt
index b5f5ba1..05c24df 100644
--- a/Documentation/i386/boot.txt
+++ b/Documentation/i386/boot.txt
@@ -531,6 +531,14 @@ Protocol:	2.08+
 
   The length of the compressed payload.
 
+**** THE IMAGE CHECKSUM
+
+From boot protocol version 2.08 onwards the CRC-32 is calculated over
+the entire file using the characteristic polynomial 0x04C11DB7 and an
+initial remainder of 0xffffffff.  The checksum is appended to the
+file; therefore the CRC of the file up to the limit specified in the
+syssize field of the header is always 0.
+
 **** THE KERNEL COMMAND LINE
 
 The kernel command line has become an important way for the boot
diff --git a/arch/x86/boot/tools/build.c b/arch/x86/boot/tools/build.c
index b424874..44dc192 100644
--- a/arch/x86/boot/tools/build.c
+++ b/arch/x86/boot/tools/build.c
@@ -50,6 +50,75 @@ typedef unsigned long  u32;
 u8 buf[SETUP_SECT_MAX*512];
 int is_big_kernel;
 
+/*----------------------------------------------------------------------*/
+
+static const u32 crctab32[] = {
+	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419,
+	0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4,
+	0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
+	0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+	0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856,
+	0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
+	0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
+	0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+	0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3,
+	0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a,
+	0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
+	0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190,
+	0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
+	0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e,
+	0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+	0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed,
+	0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+	0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
+	0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+	0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
+	0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5,
+	0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010,
+	0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17,
+	0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6,
+	0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
+	0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+	0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344,
+	0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+	0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a,
+	0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+	0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1,
+	0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c,
+	0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
+	0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe,
+	0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31,
+	0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c,
+	0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+	0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b,
+	0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+	0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
+	0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+	0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278,
+	0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7,
+	0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66,
+	0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
+	0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8,
+	0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
+	0x2d02ef8d
+};
+
+static u32 partial_crc32_one(u8 c, u32 crc)
+{
+	return crctab32[(crc ^ c) & 0xff] ^ (crc >> 8);
+}
+
+static u32 partial_crc32(const u8 *s, int len, u32 crc)
+{
+	while (len--)
+		crc = partial_crc32_one(*s++, crc);
+	return crc;
+}
+
 static void die(const char * str, ...)
 {
 	va_list args;
@@ -74,6 +143,7 @@ int main(int argc, char ** argv)
 	FILE *file;
 	int fd;
 	void *kernel;
+	u32 crc = 0xffffffffUL;
 
 	if (argc > 2 && !strcmp(argv[1], "-b"))
 	  {
@@ -144,7 +214,8 @@ int main(int argc, char ** argv)
 	kernel = mmap(NULL, sz, PROT_READ, MAP_SHARED, fd, 0);
 	if (kernel == MAP_FAILED)
 		die("Unable to mmap '%s': %m", argv[2]);
-	sys_size = (sz + 15) / 16;
+	/* Number of 16-byte paragraphs, including space for a 4-byte CRC */
+	sys_size = (sz + 15 + 4) / 16;
 	if (!is_big_kernel && sys_size > DEF_SYSSIZE)
 		die("System is too big. Try using bzImage or modules.");
 
@@ -155,12 +226,27 @@ int main(int argc, char ** argv)
 	buf[0x1f6] = sys_size >> 16;
 	buf[0x1f7] = sys_size >> 24;
 
+	crc = partial_crc32(buf, i, crc);
 	if (fwrite(buf, 1, i, stdout) != i)
 		die("Writing setup failed");
 
 	/* Copy the kernel code */
+	crc = partial_crc32(kernel, sz, crc);
 	if (fwrite(kernel, 1, sz, stdout) != sz)
 		die("Writing kernel failed");
+
+	/* Add padding leaving 4 bytes for the checksum */
+	while (sz++ < (sys_size*16) - 4) {
+		crc = partial_crc32_one('\0', crc);
+		if (fwrite("\0", 1, 1, stdout) != 1)
+			die("Writing padding failed");
+	}
+
+	/* Write the CRC */
+	fprintf(stderr, "CRC %lx\n", crc);
+	if (fwrite(&crc, 1, 4, stdout) != 4)
+		die("Writing CRC failed");
+
 	close(fd);
 
 	/* Everything is OK */
-- 
1.5.4

[PATCHv3 3/3] x86: bump image header to version 2.08.

[Ian Campbell]

Signed-off-by: Ian Campbell <ijc@hellion.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
---
 arch/x86/boot/header.S |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index 8471658..40c91bb 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -120,7 +120,7 @@ _start:
 	# Part 2 of the header, from the old setup.S
 
 		.ascii	"HdrS"		# header signature
-		.word	0x0207		# header version number (>= 0x0105)
+		.word	0x0208		# header version number (>= 0x0105)
 					# or else old loadlin-1.5 will fail)
 		.globl realmode_swtch
 realmode_swtch:	.word	0, 0		# default_switch, SETUPSEG
-- 
1.5.4

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Mark McLoughlin]

On Wed, 2008-02-13 at 20:54 +0000, Ian Campbell wrote:
> This allows other boot loaders such as the Xen domain builder the
> opportunity to extract the ELF file.

	Right, Xen currently can't boot bzImage (it needs the ELF image) so you
still can't use the same kernel image on Xen as bare-metal.

> +Field name:	compressed_payload_offset
> +Type:		read
> +Offset/size:	0x248/4
> +Protocol:	2.08+
> +
> +  If non-zero then this field contains the offset from the end of the
> +  real-mode code to the compressed payload. The compression format
> +  should be determined using the standard magic number, currently only
> +  gzip is used.

	Should probably mention that the payload format is expected to be ELF.
 
> diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
> index f88458e..9695aff 100644
> --- a/arch/x86/boot/Makefile
> +++ b/arch/x86/boot/Makefile
> @@ -94,6 +94,20 @@ $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE
>  
>  SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
>  
> +sed-offsets := -e 's/^00*/0/' \
> +        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/\#define \2 0x\1/p'
> +
> +quiet_cmd_offsets = OFFSETS $@
> +      cmd_offsets = $(NM) $< | sed -n $(sed-offsets) > $@
> +
> +$(obj)/offsets.h: $(obj)/compressed/vmlinux FORCE
> +	$(call if_changed,offsets)
> +
> +targets += offsets.h
> +
> +AFLAGS_header.o += -I$(obj)
> +$(obj)/header.o: $(obj)/offsets.h

	How about this?

+sed-offsets := -e 's/^00*/0/' \
+        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/-D\2=0x\1 /p'
+
+$(obj)/header.o: AFLAGS_header.o += $(shell $(NM) $(obj)/compressed/vmlinux | sed -n $(sed-offsets))
+$(obj)/header.o: $(obj)/compressed/vmlinux FORCE

Cheers,
Mark.

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Ian Campbell]

On Thu, 2008-02-14 at 11:34 +0000, Mark McLoughlin wrote:
> On Wed, 2008-02-13 at 20:54 +0000, Ian Campbell wrote:
> > This allows other boot loaders such as the Xen domain builder the
> > opportunity to extract the ELF file.
> 
> 	Right, Xen currently can't boot bzImage (it needs the ELF image) so you
> still can't use the same kernel image on Xen as bare-metal.

I have a xen domain builder patch as well. I was waiting for the Linux
side to gain some traction before putting it forward (I'd attach it now
but it's at home on a laptop which is sleeping).

> > +Field name:	compressed_payload_offset
> > +Type:		read
> > +Offset/size:	0x248/4
> > +Protocol:	2.08+
> > +
> > +  If non-zero then this field contains the offset from the end of the
> > +  real-mode code to the compressed payload. The compression format
> > +  should be determined using the standard magic number, currently only
> > +  gzip is used.
> 
> 	Should probably mention that the payload format is expected to be ELF.

Agreed. Probably the same deal as the compression format, i.e. use the
magic number but only ELF is possible today (even less likely to change
than the compression format I guess...).

> 	How about this?
> 
> +sed-offsets := -e 's/^00*/0/' \
> +        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/-D\2=0x\1 /p'
> +
> +$(obj)/header.o: AFLAGS_header.o += $(shell $(NM) $(obj)/compressed/vmlinux | sed -n $(sed-offsets))
> +$(obj)/header.o: $(obj)/compressed/vmlinux FORCE

That's probably a neater way of doing it. Although the ".../header.o:
AFLAGS_header.o" is redundant, either 
	header.o: AFLAGS += foo
or
	AFLAGS_header.o += foo
with the second being preferred in Linux Makefiles I think.

I'll try and get an updated patch out before I head for my flight
tomorrow.

Ian.

-- 
Ian Campbell
Current Noise: Reverend Bizarre - The Festival

While money can't buy happiness, it certainly lets you choose your own
form of misery.

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Mark McLoughlin]

On Thu, 2008-02-14 at 17:01 +0000, Ian Campbell wrote:
> On Thu, 2008-02-14 at 11:34 +0000, Mark McLoughlin wrote:
> > On Wed, 2008-02-13 at 20:54 +0000, Ian Campbell wrote:
> > > This allows other boot loaders such as the Xen domain builder the
> > > opportunity to extract the ELF file.
> > 
> > 	Right, Xen currently can't boot bzImage (it needs the ELF image) so you
> > still can't use the same kernel image on Xen as bare-metal.
> 
> I have a xen domain builder patch as well. I was waiting for the Linux
> side to gain some traction before putting it forward (I'd attach it now
> but it's at home on a laptop which is sleeping).

Yep, just want to highlight to people that your patches (or an
alternative) are needed before the same pv_ops kernel truly can be used
on bare-metal and Xen.

> > 	How about this?
> > 
> > +sed-offsets := -e 's/^00*/0/' \
> > +        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/-D\2=0x\1 /p'
> > +
> > +$(obj)/header.o: AFLAGS_header.o += $(shell $(NM) $(obj)/compressed/vmlinux | sed -n $(sed-offsets))
> > +$(obj)/header.o: $(obj)/compressed/vmlinux FORCE
> 
> That's probably a neater way of doing it. Although the ".../header.o:
> AFLAGS_header.o" is redundant, either 
> 	header.o: AFLAGS += foo

With this, AFLAGS would apply to building when building the
prerequisites of header.o too, which you don't want

The make manual says:

  "when you define a target-specific variable that variable value is
   also in effect for all prerequisites of this target, and all their
   prerequisites"

> or
> 	AFLAGS_header.o += foo
> with the second being preferred in Linux Makefiles I think.

And with this, it would try and read vmlinux before it is built.

But, hmm, given the fact that the variable is defined when building
prerequisites, you'd think that even in my version AFLAGS would be
evaluated before building vmlinux.

Cheers,
Mark.

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Ian Campbell]

On Thu, 2008-02-14 at 17:37 +0000, Mark McLoughlin wrote:
> On Thu, 2008-02-14 at 17:01 +0000, Ian Campbell wrote:
> With this, AFLAGS would apply to building when building the
> prerequisites of header.o too, which you don't want
> 
> The make manual says:
> 
>   "when you define a target-specific variable that variable value is
>    also in effect for all prerequisites of this target, and all their
>    prerequisites"

Ah yes, quite right.

I think given that subtleties of these kinds of variable definitions
it's probably clearer to stick with the header file way of doing things.
it's nice and straight forward.

Ian.
-- 
Ian Campbell

Davis' Law of Traffic Density:
	The density of rush-hour traffic is directly proportional to
	1.5 times the amount of extra time you allow to arrive on time.

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Ian Campbell]

On Thu, 2008-02-14 at 17:01 +0000, Ian Campbell wrote:
> 
> I have a xen domain builder patch as well. I was waiting for the Linux
> side to gain some traction before putting it forward (I'd attach it
> now but it's at home on a laptop which is sleeping).

Here it is:

# HG changeset patch
# User ijc@hellion.org.uk
# Date 1203011758 0
# Node ID 3079b4b3835e3aba52bb6548bbbced70471a9f32
# Parent  42369d21641d6297dc369441c3bfd355880d28c0
Support loading Linux bzImage v2.08 and up.

Signed-off-by : Ian Campbell <ijc@hellion.org.uk>

diff -r 42369d21641d -r 3079b4b3835e tools/libxc/Makefile
--- a/tools/libxc/Makefile	Thu Jan 31 16:23:35 2008 +0000
+++ b/tools/libxc/Makefile	Thu Feb 14 17:55:58 2008 +0000
@@ -40,6 +40,7 @@ GUEST_SRCS-y += libelf-dominfo.c libelf-
 # new domain builder
 GUEST_SRCS-y += xc_dom_core.c xc_dom_boot.c
 GUEST_SRCS-y += xc_dom_elfloader.c
+GUEST_SRCS-y += xc_dom_bzimageloader.c
 GUEST_SRCS-y += xc_dom_binloader.c
 GUEST_SRCS-y += xc_dom_compat_linux.c
 
diff -r 42369d21641d -r 3079b4b3835e tools/libxc/xc_dom_bzimageloader.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/libxc/xc_dom_bzimageloader.c	Thu Feb 14 17:55:58 2008 +0000
@@ -0,0 +1,159 @@
+/*
+ * Xen domain builder -- bzImage bits
+ *
+ * Parse and load bzImage kernel images.
+ *
+ * This relies on version 2.08 of the boot protocol, which contains an
+ * ELF file embedded in the bzImage.  The loader extracts this ELF
+ * image and passes it off to the standard ELF loader.
+ *
+ * This code is licenced under the GPL.
+ * written 2006 by Gerd Hoffmann <kraxel@suse.de>.
+ * written 2007 by Jeremy Fitzhardinge <jeremy@xensource.com>
+ * written 2008 by Ian Campbell <ijc@hellion.org.uk>
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom.h"
+
+struct setup_header {
+	uint8_t		_pad0[0x1f1];		/* skip uninteresting stuff */
+	uint8_t		setup_sects;
+	uint16_t	root_flags;
+	uint32_t	syssize;
+	uint16_t	ram_size;
+	uint16_t	vid_mode;
+	uint16_t	root_dev;
+	uint16_t	boot_flag;
+	uint16_t	jump;
+	uint32_t	header;
+#define HDR_MAGIC		"HdrS"
+#define HDR_MAGIC_SZ	4
+	uint16_t	version;
+#define VERSION(h,l)	(((h)<<8) | (l))
+	uint32_t	realmode_swtch;
+	uint16_t	start_sys;
+	uint16_t	kernel_version;
+	uint8_t		type_of_loader;
+	uint8_t		loadflags;
+	uint16_t	setup_move_size;
+	uint32_t	code32_start;
+	uint32_t	ramdisk_image;
+	uint32_t	ramdisk_size;
+	uint32_t	bootsect_kludge;
+	uint16_t	heap_end_ptr;
+	uint16_t	_pad1;
+	uint32_t	cmd_line_ptr;
+	uint32_t	initrd_addr_max;
+	uint32_t	kernel_alignment;
+	uint8_t		relocatable_kernel;
+	uint8_t		_pad2[3];
+	uint32_t	cmdline_size;
+	uint32_t	hardware_subarch;
+	uint64_t	hardware_subarch_data;
+	uint32_t	compressed_payload_offset;
+	uint32_t	compressed_payload_length;
+} __attribute__((packed));
+
+extern struct xc_dom_loader elf_loader;
+
+static unsigned int compressed_offset(struct setup_header *hdr)
+{
+    unsigned int off;
+
+    off = (hdr->setup_sects + 1) * 512;
+    off += hdr->compressed_payload_offset;
+    return off;
+}
+
+static int check_bzimage_kernel(struct xc_dom_image *dom, int verbose)
+{
+    struct setup_header *hdr;
+
+    if ( dom->kernel_blob == NULL )
+    {
+        if ( verbose )
+            xc_dom_panic(XC_INTERNAL_ERROR, "%s: no kernel image loaded\n",
+                         __FUNCTION__);
+        return -EINVAL;
+    }
+    if ( dom->kernel_size < sizeof(struct setup_header) )
+    {
+        if ( verbose )
+            xc_dom_panic(XC_INTERNAL_ERROR, "%s: kernel image too small\n",
+                         __FUNCTION__);
+        return -EINVAL;
+    }
+
+    hdr = dom->kernel_blob;
+
+    if ( memcmp(&hdr->header, HDR_MAGIC, HDR_MAGIC_SZ) != 0 )
+    {
+        if ( verbose )
+            xc_dom_panic(XC_INVALID_KERNEL, "%s: kernel is not a bzImage\n",
+                         __FUNCTION__);
+        return -EINVAL;
+    }
+
+    if ( hdr->version < VERSION(2,8) )
+    {
+        if ( verbose )
+            xc_dom_panic(XC_INVALID_KERNEL, "%s: boot protocol too old (%04x)\n",
+                         __FUNCTION__, hdr->version);
+        return -EINVAL;
+    }
+
+    dom->kernel_blob = dom->kernel_blob + compressed_offset(hdr);
+    dom->kernel_size = hdr->compressed_payload_length;
+
+    if ( xc_dom_try_gunzip(dom, &dom->kernel_blob, &dom->kernel_size) == -1 )
+    {
+        if ( verbose )
+            xc_dom_panic(XC_INVALID_KERNEL, "%s: unable to decompress kernel\n",
+                         __FUNCTION__);
+        return -EINVAL;
+    }
+
+    return elf_loader.probe(dom);
+}
+
+static int xc_dom_probe_bzimage_kernel(struct xc_dom_image *dom)
+{
+    return check_bzimage_kernel(dom, 0);
+}
+
+static int xc_dom_parse_bzimage_kernel(struct xc_dom_image *dom)
+{
+    return elf_loader.parser(dom);
+}
+
+static int xc_dom_load_bzimage_kernel(struct xc_dom_image *dom)
+{
+    return elf_loader.loader(dom);
+}
+
+static struct xc_dom_loader bzimage_loader = {
+    .name = "Linux bzImage",
+    .probe = xc_dom_probe_bzimage_kernel,
+    .parser = xc_dom_parse_bzimage_kernel,
+    .loader = xc_dom_load_bzimage_kernel,
+};
+
+static void __init register_loader(void)
+{
+    xc_dom_register_loader(&bzimage_loader);
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff -r 42369d21641d -r 3079b4b3835e tools/libxc/xc_dom_elfloader.c
--- a/tools/libxc/xc_dom_elfloader.c	Thu Jan 31 16:23:35 2008 +0000
+++ b/tools/libxc/xc_dom_elfloader.c	Thu Feb 14 17:55:58 2008 +0000
@@ -281,7 +281,7 @@ static int xc_dom_load_elf_kernel(struct
 
 /* ------------------------------------------------------------------------ */
 
-static struct xc_dom_loader elf_loader = {
+struct xc_dom_loader elf_loader = {
     .name = "ELF-generic",
     .probe = xc_dom_probe_elf_kernel,
     .parser = xc_dom_parse_elf_kernel,

-- 
Ian Campbell

The English have no respect for their language, and will not teach
their children to speak it.
		-- G. B. Shaw

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Jeremy Fitzhardinge]

Ian Campbell wrote:
> On Thu, 2008-02-14 at 17:01 +0000, Ian Campbell wrote:
>   
>> I have a xen domain builder patch as well. I was waiting for the Linux
>> side to gain some traction before putting it forward (I'd attach it
>> now but it's at home on a laptop which is sleeping).
>>     
>
> Here it is:
>
> # HG changeset patch
> # User ijc@hellion.org.uk
> # Date 1203011758 0
> # Node ID 3079b4b3835e3aba52bb6548bbbced70471a9f32
> # Parent  42369d21641d6297dc369441c3bfd355880d28c0
> Support loading Linux bzImage v2.08 and up.
>   

Do you also have a patch to update the boot protocol?

    J

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[H. Peter Anvin]

Jeremy Fitzhardinge wrote:
> 
> Do you also have a patch to update the boot protocol?
> 

Looking for anything different than the root of this thread?

	-hpa

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Jeremy Fitzhardinge]

H. Peter Anvin wrote:
> Jeremy Fitzhardinge wrote:
>>
>> Do you also have a patch to update the boot protocol?
>>
>
> Looking for anything different than the root of this thread?

Yes, the patch for the Xen domain builder to boot a bzImage using the 
Linux boot protocol rather than the Xen one.  Ian's patch will extract 
the ELF file from the bzImage, but still boot it by finding the Xen 
entrypoint in the notes, with %esi pointing to the Xen start_info rather 
than the boot_params (unless I'm missing something).

    J

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Ian Campbell]

On Thu, 2008-02-14 at 17:01 +0000, Ian Campbell wrote:
> 
> > > +Field name:        compressed_payload_offset
> > > +Type:              read
> > > +Offset/size:       0x248/4
> > > +Protocol:  2.08+
> > > +
> > > +  If non-zero then this field contains the offset from the end of
> the
> > > +  real-mode code to the compressed payload. The compression
> format
> > > +  should be determined using the standard magic number, currently
> only
> > > +  gzip is used.
> > 
> >       Should probably mention that the payload format is expected to
> be ELF.
> 
> Agreed. Probably the same deal as the compression format, i.e. use the
> magic number but only ELF is possible today (even less likely to
> change than the compression format I guess...).

Updated with a note about ELF format payload.

I've also changed the fields to just payload_{offset,length} and
adjusted the description to allow for the possibility of non-compressed
ELF payloads. I don't have a use for it myself but I can see how it
might be useful (embedded systems?) so it seems reasonable not to rule
it out. ELF-in-gzip and plain ELF can both be identified by magic
numbers.

Ian.
--- 
>From 544c003d4067d895556180fc11a951e211202d0d Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@hellion.org.uk>
Date: Thu, 14 Feb 2008 18:29:01 +0000
Subject: [PATCH] x86: use ELF format in compressed images.

This allows other boot loaders such as the Xen domain builder the
opportunity to extract the ELF file.

Signed-off-by: Ian Campbell <ijc@hellion.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: virtualization@lists.linux-foundation.org
---
 Documentation/i386/boot.txt       |   20 +++++++++++++
 arch/x86/boot/Makefile            |   14 +++++++++
 arch/x86/boot/compressed/Makefile |    2 +-
 arch/x86/boot/compressed/misc.c   |   56 +++++++++++++++++++++++++++++++++++++
 arch/x86/boot/header.S            |    4 ++
 5 files changed, 95 insertions(+), 1 deletions(-)

diff --git a/Documentation/i386/boot.txt b/Documentation/i386/boot.txt
index fc49b79..f2e54e5 100644
--- a/Documentation/i386/boot.txt
+++ b/Documentation/i386/boot.txt
@@ -170,6 +170,8 @@ Offset	Proto	Name		Meaning
 0238/4	2.06+	cmdline_size	Maximum size of the kernel command line
 023C/4	2.07+	hardware_subarch Hardware subarchitecture
 0240/8	2.07+	hardware_subarch_data Subarchitecture-specific data
+0248/4	2.08+	payload_offset	Offset of kernel payload
+024C/4	2.08+	payload_length	Length of kernel payload
 
 (1) For backwards compatibility, if the setup_sects field contains 0, the
     real value is 4.
@@ -512,6 +514,24 @@ Protocol:	2.07+
 
   A pointer to data that is specific to hardware subarch
 
+Field name:	payload_offset
+Type:		read
+Offset/size:	0x248/4
+Protocol:	2.08+
+
+  If non-zero then this field contains the offset from the end of the
+  real-mode code to the payload.
+
+  The payload may be compressed. The format of both the compressed and
+  uncompressed data should be determined using the standard magic
+  numbers. Currently only gzip compressed ELF is used.
+  
+Field name:	payload_length
+Type:		read
+Offset/size:	0x24c/4
+Protocol:	2.08+
+
+  The length of the payload.
 
 **** THE KERNEL COMMAND LINE
 
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index f88458e..9695aff 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -94,6 +94,20 @@ $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE
 
 SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
 
+sed-offsets := -e 's/^00*/0/' \
+        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/\#define \2 0x\1/p'
+
+quiet_cmd_offsets = OFFSETS $@
+      cmd_offsets = $(NM) $< | sed -n $(sed-offsets) > $@
+
+$(obj)/offsets.h: $(obj)/compressed/vmlinux FORCE
+	$(call if_changed,offsets)
+
+targets += offsets.h
+
+AFLAGS_header.o += -I$(obj)
+$(obj)/header.o: $(obj)/offsets.h
+
 LDFLAGS_setup.elf	:= -T
 $(obj)/setup.elf: $(src)/setup.ld $(SETUP_OBJS) FORCE
 	$(call if_changed,ld)
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index d2b9f3b..92fdd35 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -22,7 +22,7 @@ $(obj)/vmlinux: $(src)/vmlinux_$(BITS).lds $(obj)/head_$(BITS).o $(obj)/misc.o $
 	$(call if_changed,ld)
 	@:
 
-OBJCOPYFLAGS_vmlinux.bin := -O binary -R .note -R .comment -S
+OBJCOPYFLAGS_vmlinux.bin :=  -R .comment -S
 $(obj)/vmlinux.bin: vmlinux FORCE
 	$(call if_changed,objcopy)
 
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 8182e32..69aec2f 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -15,6 +15,10 @@
  * we just keep it from happening
  */
 #undef CONFIG_PARAVIRT
+#ifdef CONFIG_X86_32
+#define _ASM_DESC_H_ 1
+#endif
+
 #ifdef CONFIG_X86_64
 #define _LINUX_STRING_H_ 1
 #define __LINUX_BITMAP_H 1
@@ -22,6 +26,7 @@
 
 #include <linux/linkage.h>
 #include <linux/screen_info.h>
+#include <linux/elf.h>
 #include <asm/io.h>
 #include <asm/page.h>
 #include <asm/boot.h>
@@ -365,6 +370,56 @@ static void error(char *x)
 		asm("hlt");
 }
 
+static void parse_elf(void *output)
+{
+#ifdef CONFIG_X86_64
+	Elf64_Ehdr ehdr;
+	Elf64_Phdr *phdrs, *phdr;
+#else
+	Elf32_Ehdr ehdr;
+	Elf32_Phdr *phdrs, *phdr;
+#endif
+	void *dest;
+	int i;
+
+	memcpy(&ehdr, output, sizeof(ehdr));
+	if(ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
+	   ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
+	   ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
+	   ehdr.e_ident[EI_MAG3] != ELFMAG3)
+	{
+		error("Kernel is not a valid ELF file");
+		return;
+	}
+
+	putstr("Parsing ELF... ");
+
+	phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum);
+	if (!phdrs)
+		error("Failed to allocate space for phdrs");
+
+	memcpy(phdrs, output + ehdr.e_phoff, sizeof(*phdrs) * ehdr.e_phnum);
+
+	for (i=0; i<ehdr.e_phnum; i++) {
+		phdr = &phdrs[i];
+
+		switch (phdr->p_type) {
+		case PT_LOAD:
+#ifdef CONFIG_RELOCATABLE
+			dest = output;
+			dest += (phdr->p_paddr - LOAD_PHYSICAL_ADDR);
+#else
+			dest = (void*)(phdr->p_paddr);
+#endif
+			memcpy(dest,
+			       output + phdr->p_offset,
+			       phdr->p_filesz);
+			break;
+		default: /* Ignore other PT_* */ break;
+		}
+	}
+}
+
 asmlinkage void decompress_kernel(void *rmode, memptr heap,
 				  uch *input_data, unsigned long input_len,
 				  uch *output)
@@ -408,6 +463,7 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
 	makecrc();
 	putstr("\nDecompressing Linux... ");
 	gunzip();
+	parse_elf(output);
 	putstr("done.\nBooting the kernel.\n");
 	return;
 }
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index 64ad901..4085616 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -22,6 +22,7 @@
 #include <asm/page.h>
 #include <asm/setup.h>
 #include "boot.h"
+#include "offsets.h"
 
 SETUPSECTS	= 4			/* default nr of setup-sectors */
 BOOTSEG		= 0x07C0		/* original address of boot-sector */
@@ -223,6 +224,9 @@ hardware_subarch:	.long 0			# subarchitecture, added with 2.07
 
 hardware_subarch_data:	.quad 0
 
+payload_offset:		.long input_data
+payload_length:		.long input_data_end-input_data
+
 # End of setup header #####################################################
 
 	.section ".inittext", "ax"
-- 
1.5.4



-- 
Ian Campbell

The moss on the tree does not fear the talons of the hawk.

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[H. Peter Anvin]

Ian Campbell wrote:
> 
> Agreed. Probably the same deal as the compression format, i.e. use the
> magic number but only ELF is possible today (even less likely to change
> than the compression format I guess...).
> 

Quite.

	-hpa

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Yinghai Lu]

On Wed, Feb 13, 2008 at 1:54 PM, Ian Campbell <ijc@hellion.org.uk> wrote:
> This allows other boot loaders such as the Xen domain builder the
>  opportunity to extract the ELF file.
>
>  Signed-off-by: Ian Campbell <ijc@hellion.org.uk>
>  Cc: Thomas Gleixner <tglx@linutronix.de>
>  Cc: Ingo Molnar <mingo@redhat.com>
>  Cc: H. Peter Anvin <hpa@zytor.com>
>  Cc: Jeremy Fitzhardinge <jeremy@goop.org>
>  Cc: virtualization@lists.linux-foundation.org
>  ---
>   Documentation/i386/boot.txt       |   18 ++++++++++++
>   arch/x86/boot/Makefile            |   14 +++++++++
>   arch/x86/boot/compressed/Makefile |    2 +-
>   arch/x86/boot/compressed/misc.c   |   56 +++++++++++++++++++++++++++++++++++++
>   arch/x86/boot/header.S            |    6 ++++
>   5 files changed, 95 insertions(+), 1 deletions(-)
>
>  diff --git a/Documentation/i386/boot.txt b/Documentation/i386/boot.txt
>  index fc49b79..b5f5ba1 100644
>  --- a/Documentation/i386/boot.txt
>  +++ b/Documentation/i386/boot.txt
>  @@ -170,6 +170,8 @@ Offset      Proto   Name            Meaning
>   0238/4 2.06+   cmdline_size    Maximum size of the kernel command line
>   023C/4 2.07+   hardware_subarch Hardware subarchitecture
>   0240/8 2.07+   hardware_subarch_data Subarchitecture-specific data
>  +0248/4 2.08+   compressed_payload_offset
>  +024C/4 2.08+   compressed_payload_length
>
>   (1) For backwards compatibility, if the setup_sects field contains 0, the
>      real value is 4.
>  @@ -512,6 +514,22 @@ Protocol:  2.07+
>
>    A pointer to data that is specific to hardware subarch
>
>  +Field name:    compressed_payload_offset
>  +Type:          read
>  +Offset/size:   0x248/4
>  +Protocol:      2.08+
>  +
>  +  If non-zero then this field contains the offset from the end of the
>  +  real-mode code to the compressed payload. The compression format
>  +  should be determined using the standard magic number, currently only
>  +  gzip is used.
>  +
>  +Field name:    compressed_payload_length
>  +Type:          read
>  +Offset/size:   0x24c/4
>  +Protocol:      2.08+
>  +
>  +  The length of the compressed payload.
>
>   **** THE KERNEL COMMAND LINE
>
>  diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
>  index f88458e..9695aff 100644
>  --- a/arch/x86/boot/Makefile
>  +++ b/arch/x86/boot/Makefile
>  @@ -94,6 +94,20 @@ $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE
>
>   SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
>
>  +sed-offsets := -e 's/^00*/0/' \
>  +        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/\#define \2 0x\1/p'
>  +
>  +quiet_cmd_offsets = OFFSETS $@
>  +      cmd_offsets = $(NM) $< | sed -n $(sed-offsets) > $@
>  +
>  +$(obj)/offsets.h: $(obj)/compressed/vmlinux FORCE
>  +       $(call if_changed,offsets)
>  +
>  +targets += offsets.h
>  +
>  +AFLAGS_header.o += -I$(obj)
>  +$(obj)/header.o: $(obj)/offsets.h
>  +
>   LDFLAGS_setup.elf      := -T
>   $(obj)/setup.elf: $(src)/setup.ld $(SETUP_OBJS) FORCE
>         $(call if_changed,ld)
>  diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
>  index d2b9f3b..92fdd35 100644
>  --- a/arch/x86/boot/compressed/Makefile
>  +++ b/arch/x86/boot/compressed/Makefile
>  @@ -22,7 +22,7 @@ $(obj)/vmlinux: $(src)/vmlinux_$(BITS).lds $(obj)/head_$(BITS).o $(obj)/misc.o $
>         $(call if_changed,ld)
>         @:
>
>  -OBJCOPYFLAGS_vmlinux.bin := -O binary -R .note -R .comment -S
>  +OBJCOPYFLAGS_vmlinux.bin :=  -R .comment -S
>   $(obj)/vmlinux.bin: vmlinux FORCE
>         $(call if_changed,objcopy)
>
>  diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
>  index 8182e32..69aec2f 100644
>  --- a/arch/x86/boot/compressed/misc.c
>  +++ b/arch/x86/boot/compressed/misc.c
>  @@ -15,6 +15,10 @@
>   * we just keep it from happening
>   */
>   #undef CONFIG_PARAVIRT
>  +#ifdef CONFIG_X86_32
>  +#define _ASM_DESC_H_ 1
>  +#endif
>  +
>   #ifdef CONFIG_X86_64
>   #define _LINUX_STRING_H_ 1
>   #define __LINUX_BITMAP_H 1
>  @@ -22,6 +26,7 @@
>
>   #include <linux/linkage.h>
>   #include <linux/screen_info.h>
>  +#include <linux/elf.h>
>   #include <asm/io.h>
>   #include <asm/page.h>
>   #include <asm/boot.h>
>  @@ -365,6 +370,56 @@ static void error(char *x)
>                 asm("hlt");
>   }
>
>  +static void parse_elf(void *output)
>  +{
>  +#ifdef CONFIG_X86_64
>  +       Elf64_Ehdr ehdr;
>  +       Elf64_Phdr *phdrs, *phdr;
>  +#else
>  +       Elf32_Ehdr ehdr;
>  +       Elf32_Phdr *phdrs, *phdr;
>  +#endif
>  +       void *dest;
>  +       int i;
>  +
>  +       memcpy(&ehdr, output, sizeof(ehdr));
>  +       if(ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
>  +          ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
>  +          ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
>  +          ehdr.e_ident[EI_MAG3] != ELFMAG3)
>  +       {
>  +               error("Kernel is not a valid ELF file");
>  +               return;
>  +       }
>  +
>  +       putstr("Parsing ELF... ");
>  +
>  +       phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum);
>  +       if (!phdrs)
>  +               error("Failed to allocate space for phdrs");
>  +
>  +       memcpy(phdrs, output + ehdr.e_phoff, sizeof(*phdrs) * ehdr.e_phnum);
>  +
>  +       for (i=0; i<ehdr.e_phnum; i++) {
>  +               phdr = &phdrs[i];
>  +
>  +               switch (phdr->p_type) {
>  +               case PT_LOAD:
>  +#ifdef CONFIG_RELOCATABLE
>  +                       dest = output;
>  +                       dest += (phdr->p_paddr - LOAD_PHYSICAL_ADDR);
>  +#else
>  +                       dest = (void*)(phdr->p_paddr);
>  +#endif
>  +                       memcpy(dest,
>  +                              output + phdr->p_offset,
>  +                              phdr->p_filesz);
>  +                       break;

so will cost every bzImage extra memory copy? that could be 18M or
even more big.

wonder if can have one some other kind elf format, and offset load
address to avoid that mem copy.

YH

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Ian Campbell]

[-- Attachment #1: Type: text/plain, Size: 1061 bytes --]


On Sun, 2008-04-06 at 00:03 -0700, Yinghai Lu wrote:
> so will cost every bzImage extra memory copy? that could be 18M or
> even more big.

Is it a huge deal for something that happens exactly once on boot? How
does the time take compare with the time to do the decompression?

18M seems like an awfully large kernel image to me. My distro (Debian)
kernel is more like 3M uncompressed. If someone was interested in fast
booting presumably they would either build a kernel with exactly what
they need.

> wonder if can have one some other kind elf format, and offset load
> address to avoid that mem copy.

I guess you could reduce the load address by sizeof(headers) and add a
dest == (output + phdr->p_offset) test before the copy.

Another possibility, which is much more complex, is that the ELF parsing
could be integrated into the decompressor such that the data is
decompressed into the correct location directly.

Ian.

-- 
Ian Campbell

People who have no faults are terrible; there is no way of taking
advantage of them.

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[H. Peter Anvin]

Yinghai Lu wrote:
> so will cost every bzImage extra memory copy? that could be 18M or
> even more big.

I wouldn't worry about that.  You will typically have several copies of 
the images during the execution of the boot loader.

	-hpa

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Yinghai Lu]

On Sun, Apr 6, 2008 at 9:38 AM, H. Peter Anvin <hpa@zytor.com> wrote:
> Yinghai Lu wrote:
>
> > so will cost every bzImage extra memory copy? that could be 18M or
> > even more big.
> >
>
>  I wouldn't worry about that.  You will typically have several copies of the
> images during the execution of the boot loader.

i put all drivers needed in kernel.
1. bootloader copy bzImage (6M) to memory
2. arch/x86/boot/compressed/head_32.S, will copy bzImage to end of
buffer to do uncompress on possiton.
3. parse_elf will copy the vmlinux (the uncompressed, that is some big, 18M)

I suggest that could have special elf header, and will only have one
PT_LOAD, and avoid the copy, and just offset start address of
uncompressed kernel for jump later.

YH

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[H. Peter Anvin]

Yinghai Lu wrote:
> On Sun, Apr 6, 2008 at 9:38 AM, H. Peter Anvin <hpa@zytor.com> wrote:
>> Yinghai Lu wrote:
>>
>>> so will cost every bzImage extra memory copy? that could be 18M or
>>> even more big.
>>>
>>  I wouldn't worry about that.  You will typically have several copies of the
>> images during the execution of the boot loader.
> 
> i put all drivers needed in kernel.
> 1. bootloader copy bzImage (6M) to memory
> 2. arch/x86/boot/compressed/head_32.S, will copy bzImage to end of
> buffer to do uncompress on possiton.
> 3. parse_elf will copy the vmlinux (the uncompressed, that is some big, 18M)
> 
> I suggest that could have special elf header, and will only have one
> PT_LOAD, and avoid the copy, and just offset start address of
> uncompressed kernel for jump later.

Once again, I think you will have a hard time measuring the time difference.

The start address of the uncompressed kernel is defined at compile time. 
  Typically it is set based on alignment constraints in the hardware; 
for x86-32 is is normally 1 MB but for reasonably large hardware 16 MB 
is better (in fact, I have proposed making 16 MB the default.)  Recovery 
kernels are frequently compiled with completely different addresses.

With your proposal, this would no longer be possible to be a 
configurable option, which would be a major loss -- a major loss of 
functionality for a small fraction of time at bootup time (which is 
almost certainly dwarfed by all the other copying and initialization 
that happens at boot time.)

	-hpa

Re: [PATCHv3 1/3] x86: use ELF format in compressed images.

[Yinghai Lu]

On Sun, Apr 6, 2008 at 10:25 AM, H. Peter Anvin <hpa@zytor.com> wrote:
>
> Yinghai Lu wrote:
>
> > On Sun, Apr 6, 2008 at 9:38 AM, H. Peter Anvin <hpa@zytor.com> wrote:
> >
> > > Yinghai Lu wrote:
> > >
> > >
> > > > so will cost every bzImage extra memory copy? that could be 18M or
> > > > even more big.
> > > >
> > > >
> > >  I wouldn't worry about that.  You will typically have several copies of
> the
> > > images during the execution of the boot loader.
> > >
> >
> > i put all drivers needed in kernel.
> > 1. bootloader copy bzImage (6M) to memory
> > 2. arch/x86/boot/compressed/head_32.S, will copy bzImage to end of
> > buffer to do uncompress on possiton.
> > 3. parse_elf will copy the vmlinux (the uncompressed, that is some big,
> 18M)
> >
> > I suggest that could have special elf header, and will only have one
> > PT_LOAD, and avoid the copy, and just offset start address of
> > uncompressed kernel for jump later.
> >
>
>  Once again, I think you will have a hard time measuring the time
> difference.

ok.

forget about my proposal..

YH

Re: [PATCHv3 0/3] x86: boot protocol updates.

[Thomas Gleixner]

On Wed, 13 Feb 2008, Ian Campbell wrote:

> Updates since last time:
>   - Rebased to latest x86.git#mm (no changes required).
> 

Applied. Thanks,

	 tglx

Re: [PATCHv3 0/3] x86: boot protocol updates.

[Ian Campbell]

On Sun, 2008-02-17 at 15:04 +0100, Thomas Gleixner wrote:
> On Wed, 13 Feb 2008, Ian Campbell wrote:
> 
> > Updates since last time:
> >   - Rebased to latest x86.git#mm (no changes required).
> > 
> 
> Applied. Thanks,

Thank you.

Could you take this version of "1/3 x86: use ELF format in compressed
images" instead? It contains a slightly improved version of the doc
section making it clear that ELF is used for the uncompressed data. It
also renames the field to just payload_{offset,length} so as to not rule
out the possibility of uncompressed payloads.

>From 7d2b37d7b0342e748320e2104adbdd8ba0b3666d Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@hellion.org.uk>
Date: Thu, 14 Feb 2008 18:29:01 +0000
Subject: [PATCH] x86: use ELF format in compressed images.

This allows other boot loaders such as the Xen domain builder the
opportunity to extract the ELF file.

Signed-off-by: Ian Campbell <ijc@hellion.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: virtualization@lists.linux-foundation.org
---
 Documentation/i386/boot.txt       |   20 +++++++++++++
 arch/x86/boot/Makefile            |   14 +++++++++
 arch/x86/boot/compressed/Makefile |    2 +-
 arch/x86/boot/compressed/misc.c   |   56 +++++++++++++++++++++++++++++++++++++
 arch/x86/boot/header.S            |    4 ++
 5 files changed, 95 insertions(+), 1 deletions(-)

diff --git a/Documentation/i386/boot.txt b/Documentation/i386/boot.txt
index fc49b79..f2e54e5 100644
--- a/Documentation/i386/boot.txt
+++ b/Documentation/i386/boot.txt
@@ -170,6 +170,8 @@ Offset	Proto	Name		Meaning
 0238/4	2.06+	cmdline_size	Maximum size of the kernel command line
 023C/4	2.07+	hardware_subarch Hardware subarchitecture
 0240/8	2.07+	hardware_subarch_data Subarchitecture-specific data
+0248/4	2.08+	payload_offset	Offset of kernel payload
+024C/4	2.08+	payload_length	Length of kernel payload
 
 (1) For backwards compatibility, if the setup_sects field contains 0, the
     real value is 4.
@@ -512,6 +514,24 @@ Protocol:	2.07+
 
   A pointer to data that is specific to hardware subarch
 
+Field name:	payload_offset
+Type:		read
+Offset/size:	0x248/4
+Protocol:	2.08+
+
+  If non-zero then this field contains the offset from the end of the
+  real-mode code to the payload.
+
+  The payload may be compressed. The format of both the compressed and
+  uncompressed data should be determined using the standard magic
+  numbers. Currently only gzip compressed ELF is used.
+  
+Field name:	payload_length
+Type:		read
+Offset/size:	0x24c/4
+Protocol:	2.08+
+
+  The length of the payload.
 
 **** THE KERNEL COMMAND LINE
 
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index f88458e..9695aff 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -94,6 +94,20 @@ $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE
 
 SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
 
+sed-offsets := -e 's/^00*/0/' \
+        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/\#define \2 0x\1/p'
+
+quiet_cmd_offsets = OFFSETS $@
+      cmd_offsets = $(NM) $< | sed -n $(sed-offsets) > $@
+
+$(obj)/offsets.h: $(obj)/compressed/vmlinux FORCE
+	$(call if_changed,offsets)
+
+targets += offsets.h
+
+AFLAGS_header.o += -I$(obj)
+$(obj)/header.o: $(obj)/offsets.h
+
 LDFLAGS_setup.elf	:= -T
 $(obj)/setup.elf: $(src)/setup.ld $(SETUP_OBJS) FORCE
 	$(call if_changed,ld)
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index d2b9f3b..92fdd35 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -22,7 +22,7 @@ $(obj)/vmlinux: $(src)/vmlinux_$(BITS).lds $(obj)/head_$(BITS).o $(obj)/misc.o $
 	$(call if_changed,ld)
 	@:
 
-OBJCOPYFLAGS_vmlinux.bin := -O binary -R .note -R .comment -S
+OBJCOPYFLAGS_vmlinux.bin :=  -R .comment -S
 $(obj)/vmlinux.bin: vmlinux FORCE
 	$(call if_changed,objcopy)
 
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 8182e32..69aec2f 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -15,6 +15,10 @@
  * we just keep it from happening
  */
 #undef CONFIG_PARAVIRT
+#ifdef CONFIG_X86_32
+#define _ASM_DESC_H_ 1
+#endif
+
 #ifdef CONFIG_X86_64
 #define _LINUX_STRING_H_ 1
 #define __LINUX_BITMAP_H 1
@@ -22,6 +26,7 @@
 
 #include <linux/linkage.h>
 #include <linux/screen_info.h>
+#include <linux/elf.h>
 #include <asm/io.h>
 #include <asm/page.h>
 #include <asm/boot.h>
@@ -365,6 +370,56 @@ static void error(char *x)
 		asm("hlt");
 }
 
+static void parse_elf(void *output)
+{
+#ifdef CONFIG_X86_64
+	Elf64_Ehdr ehdr;
+	Elf64_Phdr *phdrs, *phdr;
+#else
+	Elf32_Ehdr ehdr;
+	Elf32_Phdr *phdrs, *phdr;
+#endif
+	void *dest;
+	int i;
+
+	memcpy(&ehdr, output, sizeof(ehdr));
+	if(ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
+	   ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
+	   ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
+	   ehdr.e_ident[EI_MAG3] != ELFMAG3)
+	{
+		error("Kernel is not a valid ELF file");
+		return;
+	}
+
+	putstr("Parsing ELF... ");
+
+	phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum);
+	if (!phdrs)
+		error("Failed to allocate space for phdrs");
+
+	memcpy(phdrs, output + ehdr.e_phoff, sizeof(*phdrs) * ehdr.e_phnum);
+
+	for (i=0; i<ehdr.e_phnum; i++) {
+		phdr = &phdrs[i];
+
+		switch (phdr->p_type) {
+		case PT_LOAD:
+#ifdef CONFIG_RELOCATABLE
+			dest = output;
+			dest += (phdr->p_paddr - LOAD_PHYSICAL_ADDR);
+#else
+			dest = (void*)(phdr->p_paddr);
+#endif
+			memcpy(dest,
+			       output + phdr->p_offset,
+			       phdr->p_filesz);
+			break;
+		default: /* Ignore other PT_* */ break;
+		}
+	}
+}
+
 asmlinkage void decompress_kernel(void *rmode, memptr heap,
 				  uch *input_data, unsigned long input_len,
 				  uch *output)
@@ -408,6 +463,7 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
 	makecrc();
 	putstr("\nDecompressing Linux... ");
 	gunzip();
+	parse_elf(output);
 	putstr("done.\nBooting the kernel.\n");
 	return;
 }
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index 64ad901..4085616 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -22,6 +22,7 @@
 #include <asm/page.h>
 #include <asm/setup.h>
 #include "boot.h"
+#include "offsets.h"
 
 SETUPSECTS	= 4			/* default nr of setup-sectors */
 BOOTSEG		= 0x07C0		/* original address of boot-sector */
@@ -223,6 +224,9 @@ hardware_subarch:	.long 0			# subarchitecture, added with 2.07
 
 hardware_subarch_data:	.quad 0
 
+payload_offset:		.long input_data
+payload_length:		.long input_data_end-input_data
+
 # End of setup header #####################################################
 
 	.section ".inittext", "ax"
-- 
1.5.4


-- 
Ian Campbell

Modern art is what happens when painters stop looking at girls and persuade
themselves that they have a better idea.
		-- John Ciardi