Re: how to increase commandline size (fwd)

Re: how to increase commandline size (fwd)

[Jeff Chua]

On Sun, 8 Oct 2006, Marco Gerards wrote:

>> It seems to stop at 256 characters counting the input length
> 
> That's weird.  Could you please have a look at loader/i386/pc/linux.c
> to see if it gets the entire line or just a part.  Perhaps this
> limitation is in the protocol used to pass the command line to Linux,
> I don't this protocol.

Marco,

I managed to trace down the limitation to GRUB_LINUX_CL_END_OFFSET.
This limits to 255 characters. I've expanded to 1536 and it seems ok.
I've modified the code to rely on COMMAND_LINE_SIZE defined in the kernel.

ps. Marco had requested that I post this to the mailing list so that you 
all can help verify this is ok.


Here's the patch ...

Thanks for your help,

Jeff.


--- grub2/include/grub/i386/linux.h.org	2006-10-10 01:06:33 +0800
+++ grub2/include/grub/i386/linux.h	2006-10-10 01:07:54 +0800
@@ -17,6 +17,8 @@
   *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */

+#include <asm/param.h>
+
  #ifndef GRUB_LINUX_MACHINE_HEADER
  #define GRUB_LINUX_MACHINE_HEADER	1

@@ -41,8 +43,9 @@
  #define GRUB_LINUX_VID_MODE_ASK		0xFFFD

  #define GRUB_LINUX_CL_OFFSET		0x9000
-#define GRUB_LINUX_CL_END_OFFSET	0x90FF
-#define GRUB_LINUX_SETUP_MOVE_SIZE	0x9100
+#define GRUB_LINUX_CL_END_OFFSET	\
+		(GRUB_LINUX_CL_OFFSET + COMMAND_LINE_SIZE)
+#define GRUB_LINUX_SETUP_MOVE_SIZE	(GRUB_LINUX_CL_END_OFFSET + 1)
  #define GRUB_LINUX_CL_MAGIC		0xA33F

  #define GRUB_LINUX_EFI_SIGNATURE	\
--- grub2/kern/i386/pc/startup.S.org	2006-10-10 01:05:30 +0800
+++ grub2/kern/i386/pc/startup.S	2006-10-10 01:05:45 +0800
@@ -51,6 +51,7 @@
  #include <grub/cpu/linux.h>
  #include <grub/machine/kernel.h>
  #include <grub/machine/multiboot.h>
+#include <asm/param.h>

  #define ABS(x)	((x) - EXT_C(start) + GRUB_BOOT_MACHINE_KERNEL_ADDR + 0x200)

@@ -760,7 +761,7 @@
  	/* copy the real mode code */
  	movl	EXT_C(grub_linux_tmp_addr), %esi
  	movl	%ebx, %edi
-	movl	$GRUB_LINUX_SETUP_MOVE_SIZE, %ecx
+	movl	$(GRUB_LINUX_CL_OFFSET + COMMAND_LINE_SIZE + 1), %ecx
  	cld
  	rep
  	movsb
--- grub2/Makefile.org	2006-10-10 01:05:59 +0800
+++ grub2/Makefile	2006-10-10 01:06:06 +0800
@@ -61,7 +61,7 @@
  	 -DGRUB_LIBDIR=\"$(pkglibdir)\"
  TARGET_CC = gcc
  TARGET_CFLAGS = -Wall -W -Wshadow -Wpointer-arith -Wmissing-prototypes                  -Wundef -Wstrict-prototypes -g -Os -falign-jumps=1 -falign-loops=1 -falign-functions=1
-TARGET_CPPFLAGS =  -I. -Iinclude -I$(srcdir)/include \
+TARGET_CPPFLAGS =  -I/usr/include -I. -Iinclude -I$(srcdir)/include \
  	-Wall -W
  TARGET_LDFLAGS =
  OBJCOPY = objcopy

Re: how to increase commandline size (patch + changelog)

[Jeff Chua]

On Tue, 10 Oct 2006, Jeff Chua wrote:

Here's with ChangeLog included.


> On Sun, 8 Oct 2006, Marco Gerards wrote:
>
>>> It seems to stop at 256 characters counting the input length
>> 
>> That's weird.  Could you please have a look at loader/i386/pc/linux.c
>> to see if it gets the entire line or just a part.  Perhaps this
>> limitation is in the protocol used to pass the command line to Linux,
>> I don't this protocol.
>
> Marco,
>
> I managed to trace down the limitation to GRUB_LINUX_CL_END_OFFSET.
> This limits to 255 characters. I've expanded to 1536 and it seems ok.
> I've modified the code to rely on COMMAND_LINE_SIZE defined in the kernel.
>
> ps. Marco had requested that I post this to the mailing list so that you all 
> can help verify this is ok.
>
>
> Here's the patch ...
>
> Thanks for your help,
>
> Jeff.
>
--- grub2/include/grub/i386/linux.h.org	2006-10-10 01:06:33 +0800
+++ grub2/include/grub/i386/linux.h	2006-10-10 01:07:54 +0800
@@ -17,6 +17,8 @@
   *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */

+#include <asm/param.h>
+
  #ifndef GRUB_LINUX_MACHINE_HEADER
  #define GRUB_LINUX_MACHINE_HEADER	1

@@ -41,8 +43,9 @@
  #define GRUB_LINUX_VID_MODE_ASK		0xFFFD

  #define GRUB_LINUX_CL_OFFSET		0x9000
-#define GRUB_LINUX_CL_END_OFFSET	0x90FF
-#define GRUB_LINUX_SETUP_MOVE_SIZE	0x9100
+#define GRUB_LINUX_CL_END_OFFSET	\
+		(GRUB_LINUX_CL_OFFSET + COMMAND_LINE_SIZE)
+#define GRUB_LINUX_SETUP_MOVE_SIZE	(GRUB_LINUX_CL_END_OFFSET + 1)
  #define GRUB_LINUX_CL_MAGIC		0xA33F

  #define GRUB_LINUX_EFI_SIGNATURE	\
--- grub2/kern/i386/pc/startup.S.org	2006-10-10 01:05:30 +0800
+++ grub2/kern/i386/pc/startup.S	2006-10-10 01:05:45 +0800
@@ -51,6 +51,7 @@
  #include <grub/cpu/linux.h>
  #include <grub/machine/kernel.h>
  #include <grub/machine/multiboot.h>
+#include <asm/param.h>

  #define ABS(x)	((x) - EXT_C(start) + GRUB_BOOT_MACHINE_KERNEL_ADDR + 0x200)

@@ -760,7 +761,7 @@
  	/* copy the real mode code */
  	movl	EXT_C(grub_linux_tmp_addr), %esi
  	movl	%ebx, %edi
-	movl	$GRUB_LINUX_SETUP_MOVE_SIZE, %ecx
+	movl	$(GRUB_LINUX_CL_OFFSET + COMMAND_LINE_SIZE + 1), %ecx
  	cld
  	rep
  	movsb
--- grub2/Makefile.org	2006-10-10 01:05:59 +0800
+++ grub2/Makefile	2006-10-10 01:06:06 +0800
@@ -61,7 +61,7 @@
  	 -DGRUB_LIBDIR=\"$(pkglibdir)\"
  TARGET_CC = gcc
  TARGET_CFLAGS = -Wall -W -Wshadow -Wpointer-arith -Wmissing-prototypes                  -Wundef -Wstrict-prototypes -g -Os -falign-jumps=1 -falign-loops=1 -falign-functions=1
-TARGET_CPPFLAGS =  -I. -Iinclude -I$(srcdir)/include \
+TARGET_CPPFLAGS =  -I/usr/include -I. -Iinclude -I$(srcdir)/include \
  	-Wall -W
  TARGET_LDFLAGS =
  OBJCOPY = objcopy
--- grub2/ChangeLog.org	2006-10-10 08:46:03 +0800
+++ grub2/ChangeLog	2006-10-10 08:59:49 +0800
@@ -1,3 +1,14 @@
+2006-10-10  Jeff Chua <jchua@fedex.com>
+
+	* grub2/include/grub/i386/linux.h
+	(GRUB_LINUX_CL_END_OFFSET): determined from COMMAND_LINE_SIZE
+	(GRUB_LINUX_SETUP_MOVE_SIZE): Likewise.
+
+	* grub2/kern/i386/pc/startup.S (grub_linux_boot_bzimage):
+	Calculate command line offset based on COMMAND_LINE_SIZE
+
+	* grub2/Makefile: /usr/include added to locate <asm/param.h>
+
  2006-10-05  Tristan Gingold  <tristan.gingold@bull.net>

  	* kern/misc.c (grub_strtoull): Guess the base only if not

Re: how to increase commandline size (patch + changelog)

[Hollis Blanchard]

On Tue, 2006-10-10 at 09:03 +0800, Jeff Chua wrote:
> 
> +#include <asm/param.h>

I don't think this is a good idea because it requires we build on a
Linux host.

-Hollis

Re: how to increase commandline size (patch + changelog)

[Jeff Chua]

On Mon, 9 Oct 2006, Hollis Blanchard wrote:

>> +#include <asm/param.h>
>
> I don't think this is a good idea because it requires we build on a
> Linux host.

Shall I use '#ifdef __linux__' to include <asm/param.h> only on linux, and
'#define COMMAND_LINE_SIZE 255' for non-linux system?

Sorry, I only thought of using grub2 on linux.

Thanks,
Jeff.

Re: how to increase commandline size (patch + changelog)

[Hollis Blanchard]

On Tue, 2006-10-10 at 11:08 +0800, Jeff Chua wrote:
> On Mon, 9 Oct 2006, Hollis Blanchard wrote:
> 
> >> +#include <asm/param.h>
> >
> > I don't think this is a good idea because it requires we build on a
> > Linux host.
> 
> Shall I use '#ifdef __linux__' to include <asm/param.h> only on linux, and
> '#define COMMAND_LINE_SIZE 255' for non-linux system?

Well, does this number change often? If not, let's just copy it.

If so, then we're screwed anyways because asm/param.h would only match
the installed kernel's limit, so may not work for other kernel versions.
In that case, let's just copy it and shrink it in a "lowest common
denominator" approach.

-Hollis

Re: how to increase commandline size (patch + changelog)

[Jeff Chua]

On Mon, 9 Oct 2006, Hollis Blanchard wrote:

>> Shall I use '#ifdef __linux__' to include <asm/param.h> only on linux, and
>> '#define COMMAND_LINE_SIZE 255' for non-linux system?
>
5B> Well, does this number change often? If not, let's just copy it.
>
> If so, then we're screwed anyways because asm/param.h would only match
> the installed kernel's limit, so may not work for other kernel versions.

You're right. asm/param.h determines the limit. We've read this from here 
for linux.

The idea is that there are users out there who wants to pass quite a few 
parameters during boot, and loadlin currently handles this. I'm passing 
network address, netmask, broadcast, hostname, modules_to_load and boot 
the system up using ramdisk. This allow every system to be unique and yet 
able to boot from the same CDROM/image.

So, another approach is to use "configure --CommandLineSize=1024" to 
specify the size, but I thought reading from asm/param.h would be easier.

Setting the current limit to 255 essentially means grub sets it's own 
limit, and even the kernel supports 1024, grub doesn't.


Jeff.

Re: how to increase commandline size (patch + changelog)

[Marco Gerards]

Jeff Chua <jeff84@silk.corp.fedex.com> writes:

> On Mon, 9 Oct 2006, Hollis Blanchard wrote:
>
>>> Shall I use '#ifdef __linux__' to include <asm/param.h> only on linux, and
>>> '#define COMMAND_LINE_SIZE 255' for non-linux system?
>>
> 5B> Well, does this number change often? If not, let's just copy it.
>>
>> If so, then we're screwed anyways because asm/param.h would only match
>> the installed kernel's limit, so may not work for other kernel versions.
>
> You're right. asm/param.h determines the limit. We've read this from
> here for linux.
>
> The idea is that there are users out there who wants to pass quite a
> few parameters during boot, and loadlin currently handles this. I'm
> passing network address, netmask, broadcast, hostname, modules_to_load
> and boot the system up using ramdisk. This allow every system to be
> unique and yet able to boot from the same CDROM/image.
>
> So, another approach is to use "configure --CommandLineSize=1024" to
> specify the size, but I thought reading from asm/param.h would be
> easier.

I don't like limits that are introduced at compile time.  It means
that either things break when a kernel with other limits are used.  Or
that users will get binaries with limits (for example, when using a
GRUB 2 package).

--
Marco

Re: how to increase commandline size (patch + changelog)

[Marco Gerards]

Hollis Blanchard <hollis@penguinppc.org> writes:

> On Tue, 2006-10-10 at 11:08 +0800, Jeff Chua wrote:
>> On Mon, 9 Oct 2006, Hollis Blanchard wrote:
>> 
>> >> +#include <asm/param.h>
>> >
>> > I don't think this is a good idea because it requires we build on a
>> > Linux host.
>> 
>> Shall I use '#ifdef __linux__' to include <asm/param.h> only on linux, and
>> '#define COMMAND_LINE_SIZE 255' for non-linux system?
>
> Well, does this number change often? If not, let's just copy it.
>
> If so, then we're screwed anyways because asm/param.h would only match
> the installed kernel's limit, so may not work for other kernel versions.
> In that case, let's just copy it and shrink it in a "lowest common
> denominator" approach.

Isn't it possible to use the kernel binary to determine this?  For
example, is the maximum commandline size stored somewhere?  Perhaps,
we can determine it by looking at some version field?  According to
the documentation it is has a maximum length of 255 bytes for every
Linux version:

"The kernel command line is a null-terminated string up to 255
characters long, plus the final null."

I don't want the loader to rely on being compiled on GNU/Linux.  GRUB
2, when compiled on *BSD, should be capable of loading Linux just as
GRUB 2 compiled on GNU/Linux would.

--
Marco

Re: how to increase commandline size (patch + changelog)

[Jeff Chua]

On Tue, 10 Oct 2006, Marco Gerards wrote:

> Isn't it possible to use the kernel binary to determine this?  For
> example, is the maximum commandline size stored somewhere?  Perhaps,
> we can determine it by looking at some version field?  According to
> the documentation it is has a maximum length of 255 bytes for every
> Linux version:

But, user can overwrite this to expand the commandline. It's documented, 
and tested. See http://www.x86-64.org/lists/discuss/msg06193.html

> I don't want the loader to rely on being compiled on GNU/Linux.  GRUB
> 2, when compiled on *BSD, should be capable of loading Linux just as
> GRUB 2 compiled on GNU/Linux would.

Agree. So, standard 255 is ok, but then it'll be nice to allow user to
compile "grub2" to expand the commandline. I don't know whether any 
distributions expand more than 255 characters.

Thanks,
Jeff.

Re: how to increase commandline size (patch + changelog)

[Markus Laire]

On 10/10/06, Jeff Chua <jeff84@silk.corp.fedex.com> wrote:
> Agree. So, standard 255 is ok, but then it'll be nice to allow user to
> compile "grub2" to expand the commandline. I don't know whether any
> distributions expand more than 255 characters.

At least Knoppix 4.0.2 uses a patch to extend the boot commandline to
512 characters.

-- 
Markus Laire

Re: how to increase commandline size (patch + changelog)

[Marco Gerards]

Jeff Chua <jeff84@silk.corp.fedex.com> writes:

> On Tue, 10 Oct 2006, Marco Gerards wrote:
>
>> Isn't it possible to use the kernel binary to determine this?  For
>> example, is the maximum commandline size stored somewhere?  Perhaps,
>> we can determine it by looking at some version field?  According to
>> the documentation it is has a maximum length of 255 bytes for every
>> Linux version:
>
> But, user can overwrite this to expand the commandline. It's
> documented, and tested. See
> http://www.x86-64.org/lists/discuss/msg06193.html

Well, a protocol change which is not official and an email is not
really documentation.

>> I don't want the loader to rely on being compiled on GNU/Linux.  GRUB
>> 2, when compiled on *BSD, should be capable of loading Linux just as
>> GRUB 2 compiled on GNU/Linux would.
>
> Agree. So, standard 255 is ok, but then it'll be nice to allow user to
> compile "grub2" to expand the commandline. I don't know whether any
> distributions expand more than 255 characters.

Compile time arguments are not really an option to me.  In that case I
prefer something run-time.  Or something that can be detected at
run-time.  It seems all the official versions do not make this change,
so changing this is actually non-standard.

You can not rely on the header files when compiling GRUB.  Who says
the headers you use is for the kernel you load?  The same is true for
compile time arguments.

So there are two things we can do:

- Detect this somehow by looking at the binary.  This should be
  reliable.  I don't think this is possible.

- Add some run-time argument to the loader to override the command
  line length.  But I do not like this either because it allows
  breaking the Linux boot protocol.

--
Marco

Re: how to increase commandline size (patch + changelog)

[Hollis Blanchard]

On Tue, 2006-10-10 at 16:38 +0200, Marco Gerards wrote:
> 
> - Add some run-time argument to the loader to override the command
>   line length.  But I do not like this either because it allows
>   breaking the Linux boot protocol. 

What exactly happens if we give the kernel a longer cmdline than it
supports? Does it just truncate, or fail to boot?

Why not allow users to pass as much as they want, and just warn them if
it's >255?

-Hollis

Re: how to increase commandline size (patch + changelog)

[Marco Gerards]

Hollis Blanchard <hollis@penguinppc.org> writes:

> On Tue, 2006-10-10 at 16:38 +0200, Marco Gerards wrote:
>> 
>> - Add some run-time argument to the loader to override the command
>>   line length.  But I do not like this either because it allows
>>   breaking the Linux boot protocol. 
>
> What exactly happens if we give the kernel a longer cmdline than it
> supports? Does it just truncate, or fail to boot?

It truncates.

> Why not allow users to pass as much as they want, and just warn them if
> it's >255?

I am not sure what the impact is.  Linux has a memory map and I assume
the memory after the commandline is used for other things, or might be
overwritten.

--
Marco

Re: how to increase commandline size (patch + changelog)

[Jeff Chua]

On Tue, 10 Oct 2006, Marco Gerards wrote:

> Hollis Blanchard <hollis@penguinppc.org> writes:
>> What exactly happens if we give the kernel a longer cmdline than it
>> supports? Does it just truncate, or fail to boot?
>
> It truncates.
>
>> Why not allow users to pass as much as they want, and just warn them if
>> it's >255?
>
> I am not sure what the impact is.  Linux has a memory map and I assume
> the memory after the commandline is used for other things, or might be
> overwritten.

Is it possible to "calculate" the needed offset for the exact cmdline 
length user pass in to grub and set this in grub2/kern/i386/pc/startup.S, 
then we don't need to worry about what the user pass in.


Jeff.

Re: how to increase commandline size (patch + changelog)

[Yoshinori K. Okuji]

On Tuesday 10 October 2006 19:17, Jeff Chua wrote:
> On Tue, 10 Oct 2006, Marco Gerards wrote:
> > Hollis Blanchard <hollis@penguinppc.org> writes:
> >> What exactly happens if we give the kernel a longer cmdline than it
> >> supports? Does it just truncate, or fail to boot?
> >
> > It truncates.
> >
> >> Why not allow users to pass as much as they want, and just warn them if
> >> it's >255?
> >
> > I am not sure what the impact is.  Linux has a memory map and I assume
> > the memory after the commandline is used for other things, or might be
> > overwritten.
>
> Is it possible to "calculate" the needed offset for the exact cmdline
> length user pass in to grub and set this in grub2/kern/i386/pc/startup.S,
> then we don't need to worry about what the user pass in.

I do not like it very much. Linux specifies how it is loaded precisely, so we 
should follow it carefully. Even if loadlin passes anything and it is useful 
for you, it is a bug feature, and I don't appreciate that way.

BTW this is a kind of FAQ. I have answered the same question several times 
until now. All I have been saying is that you should improve Linux before 
changing GRUB. I thought someone took an action for this, but the 
specification hasn't changed AFAIK.

Okuji