[PATCH v2] char drivers: Ram oops/panic logger

[PATCH v2] char drivers: Ram oops/panic logger

[Marco Stornelli]

Ramoops, like mtdoops, can log oops/panic information but in RAM. It can
be used with persistent RAM for systems without flash support. In
addition, for this systems, with this driver, it's no more needed
add to the kernel the mtd subsystem with advantage in footprint.

Signed-off-by: Marco Stornelli <marco.stornelli@gmail.com>
---

diff -uprN linux-2.6.33-orig/drivers/char/Kconfig linux-2.6.33/drivers/char/Kconfig
--- linux-2.6.33-orig/drivers/char/Kconfig	2010-02-24 19:52:17.000000000 +0100
+++ linux-2.6.33/drivers/char/Kconfig	2010-02-28 10:47:29.000000000 +0100
@@ -1105,5 +1105,12 @@ config DEVPORT
 
 source "drivers/s390/char/Kconfig"
 
+config RAMOOPS
+	tristate "Log panic/oops to a RAM buffer"
+	default n
+	help
+	  This enables panic and oops messages to be logged to a circular
+	  buffer in RAM where it can be read back at some later point.
+
 endmenu
 
diff -uprN linux-2.6.33-orig/drivers/char/Makefile linux-2.6.33/drivers/char/Makefile
--- linux-2.6.33-orig/drivers/char/Makefile	2010-02-24 19:52:17.000000000 +0100
+++ linux-2.6.33/drivers/char/Makefile	2010-02-28 10:49:17.000000000 +0100
@@ -107,6 +107,7 @@ obj-$(CONFIG_HANGCHECK_TIMER)	+= hangche
 obj-$(CONFIG_TCG_TPM)		+= tpm/
 
 obj-$(CONFIG_PS3_FLASH)		+= ps3flash.o
+obj-$(CONFIG_RAMOOPS)		+= ramoops.o
 
 obj-$(CONFIG_JS_RTC)		+= js-rtc.o
 js-rtc-y = rtc.o
diff -uprN linux-2.6.33-orig/drivers/char/ramoops.c linux-2.6.33/drivers/char/ramoops.c
--- linux-2.6.33-orig/drivers/char/ramoops.c	1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.33/drivers/char/ramoops.c	2010-03-07 10:41:10.000000000 +0100
@@ -0,0 +1,162 @@
+/*
+ * RAM Oops/Panic logger
+ *
+ * Copyright (C) 2009 Marco Stornelli <marco.stornelli@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/kmsg_dump.h>
+#include <linux/time.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+
+#define RAMOOPS_KERNMSG_HDR "===="
+#define RAMOOPS_HEADER_SIZE   (5 + sizeof(struct timeval))
+
+#define RECORD_SIZE 4096
+
+static ulong mem_address;
+module_param(mem_address, ulong, 0600);
+MODULE_PARM_DESC(mem_address,
+		"start of reserved RAM used to store oops/panic logs");
+
+static ulong mem_size;
+module_param(mem_size, ulong, 0600);
+MODULE_PARM_DESC(mem_size,
+		"size of reserved RAM used to store oops/panic logs");
+
+static int dump_oops = 1;
+module_param(dump_oops, int, 0600);
+MODULE_PARM_DESC(dump_oops,
+		"set to 1 to dump oopses, 0 to only dump panics (default 1)");
+
+static struct ramoops_context {
+	struct kmsg_dumper dump;
+	void *virt_addr;
+	phys_addr_t phys_addr;
+	unsigned long size;
+	int count;
+	int max_count;
+} oops_cxt;
+
+static void ramoops_do_dump(struct kmsg_dumper *dumper,
+		enum kmsg_dump_reason reason, const char *s1, unsigned long l1,
+		const char *s2, unsigned long l2)
+{
+	struct ramoops_context *cxt = container_of(dumper,
+			struct ramoops_context, dump);
+	unsigned long s1_start, s2_start;
+	unsigned long l1_cpy, l2_cpy;
+	int res;
+	char *buf;
+	struct timeval timestamp;
+
+	/* Only dump oopses if dump_oops is set */
+	if (reason == KMSG_DUMP_OOPS && !dump_oops)
+		return;
+
+	buf = (char *)(cxt->virt_addr + (cxt->count * RECORD_SIZE));
+	memset(buf, '\0', RECORD_SIZE);
+	res = sprintf(buf, "%s", RAMOOPS_KERNMSG_HDR);
+	buf += res;
+	do_gettimeofday(&timestamp);
+	res = sprintf(buf, "%lu.%lu\n", (long)timestamp.tv_sec, (long)timestamp.tv_usec);
+	buf += res;
+
+	l2_cpy = min(l2, (unsigned long)(RECORD_SIZE - RAMOOPS_HEADER_SIZE));
+	l1_cpy = min(l1, (unsigned long)(RECORD_SIZE - RAMOOPS_HEADER_SIZE) - l2_cpy);
+
+	s2_start = l2 - l2_cpy;
+	s1_start = l1 - l1_cpy;
+
+	memcpy(buf, s1 + s1_start, l1_cpy);
+	memcpy(buf + l1_cpy, s2 + s2_start, l2_cpy);
+
+	cxt->count = (cxt->count + 1) % cxt->max_count;
+}
+
+static int __init ramoops_init(void)
+{
+	struct ramoops_context *cxt = &oops_cxt;
+	int err = -EINVAL;
+
+	if (!mem_size) {
+		printk(KERN_ERR "Invalid size specification");
+		goto fail3;
+	}
+
+	rounddown_pow_of_two(mem_size);
+
+	if (mem_size < RECORD_SIZE) {
+		printk(KERN_ERR "size too small");
+		goto fail3;
+	}
+
+	cxt->max_count = mem_size / RECORD_SIZE;
+	cxt->count = 0;
+	cxt->size = mem_size;
+	cxt->phys_addr = mem_address;
+
+	if (!request_mem_region(cxt->phys_addr, cxt->size, "ramoops")) {
+		printk(KERN_ERR "ramoops: request mem region failed");
+		err = -EINVAL;
+		goto fail3;
+	}
+
+	cxt->virt_addr = ioremap(cxt->phys_addr,  cxt->size);
+	if (!cxt->virt_addr) {
+		printk(KERN_ERR "ramoops: ioremap failed");
+		goto fail2;
+	}
+
+	cxt->dump.dump = ramoops_do_dump;
+	err = kmsg_dump_register(&cxt->dump);
+	if (err) {
+		printk(KERN_ERR "ramoops: registering kmsg dumper failed");
+		goto fail1;
+	}
+
+	return 0;
+
+fail1:
+	iounmap(cxt->virt_addr);
+fail2:
+	release_mem_region(cxt->phys_addr, cxt->size);
+fail3:
+	return err;
+}
+
+static void __exit ramoops_exit(void)
+{
+	struct ramoops_context *cxt = &oops_cxt;
+
+	if (kmsg_dump_unregister(&cxt->dump) < 0)
+		printk(KERN_WARNING "ramoops: could not unregister kmsg_dumper\n");
+
+	iounmap(cxt->virt_addr);
+	release_mem_region(cxt->phys_addr, cxt->size);
+}
+
+
+module_init(ramoops_init);
+module_exit(ramoops_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Marco Stornelli <marco.stornelli@gmail.com>");
+MODULE_DESCRIPTION("RAM Oops/Panic logger/driver");

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Yuasa Yoichi]

Hi,

2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
> Ramoops, like mtdoops, can log oops/panic information but in RAM.

What is different from mtdoops + mtd-ram?

Yoichi

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Marco Stornelli]

2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
> Hi,
>
> 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
>> Ramoops, like mtdoops, can log oops/panic information but in RAM.
>
> What is different from mtdoops + mtd-ram?
>
> Yoichi
>

It can be used in a very easy way with persistent RAM for systems
without flash support. For this systems, with this driver, it's no
more needed add to the kernel the mtd subsystem with advantage in
footprint as I said in the description. In addition, you can save
flash space and store this information only in RAM. I think it's very
useful for embedded systems.

Marco

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Yuasa Yoichi]

2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
> 2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
>> Hi,
>>
>> 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
>>> Ramoops, like mtdoops, can log oops/panic information but in RAM.
>>
>> What is different from mtdoops + mtd-ram?
>>
>> Yoichi
>>
>
> It can be used in a very easy way with persistent RAM for systems
> without flash support. For this systems, with this driver, it's no
> more needed add to the kernel the mtd subsystem with advantage in
> footprint as I said in the description.

right.
But,

> In addition, you can save
> flash space and store this information only in RAM. I think it's very
> useful for embedded systems.

CONFIG_MTD_RAM uses only RAM.
I think there's no big difference about this point.

Yoichi

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Marco Stornelli]

2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
> 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
>> 2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
>>> Hi,
>>>
>>> 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
>>>> Ramoops, like mtdoops, can log oops/panic information but in RAM.
>>>
>>> What is different from mtdoops + mtd-ram?
>>>
>>> Yoichi
>>>
>>
>> It can be used in a very easy way with persistent RAM for systems
>> without flash support. For this systems, with this driver, it's no
>> more needed add to the kernel the mtd subsystem with advantage in
>> footprint as I said in the description.
>
> right.
> But,
>
>> In addition, you can save
>> flash space and store this information only in RAM. I think it's very
>> useful for embedded systems.
>
> CONFIG_MTD_RAM uses only RAM.
> I think there's no big difference about this point.
>

I meant with the "classic" use of mtdoops, therefore with a flash
partition without use MTD_RAM. Using MTD_RAM, it's more or less the
same thing, with the exception of "where" you want deploy the log. For
example: if in your system you have got a nvram you can use it without
problem, you need to specify the address of the nvram to the module.
Very simple. I  think it's a small driver but very useful, feedback
from other embedded guys are welcome.

Marco

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Andrew Morton]

On Wed, 10 Mar 2010 13:15:25 +0100
Marco Stornelli <marco.stornelli@gmail.com> wrote:

> 2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
> > 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
> >> 2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
> >>> Hi,
> >>>
> >>> 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
> >>>> Ramoops, like mtdoops, can log oops/panic information but in RAM.
> >>>
> >>> What is different from mtdoops + mtd-ram?
> >>>
> >>> Yoichi
> >>>
> >>
> >> It can be used in a very easy way with persistent RAM for systems
> >> without flash support. For this systems, with this driver, it's no
> >> more needed add to the kernel the mtd subsystem with advantage in
> >> footprint as I said in the description.
> >
> > right.
> > But,
> >
> >> In addition, you can save
> >> flash space and store this information only in RAM. I think it's very
> >> useful for embedded systems.
> >
> > CONFIG_MTD_RAM uses only RAM.
> > I think there's no big difference about this point.
> >
> 
> I meant with the "classic" use of mtdoops, therefore with a flash
> partition without use MTD_RAM. Using MTD_RAM, it's more or less the
> same thing, with the exception of "where" you want deploy the log. For
> example: if in your system you have got a nvram you can use it without
> problem, you need to specify the address of the nvram to the module.
> Very simple. I  think it's a small driver but very useful, feedback
> from other embedded guys are welcome.

Seems sensible to me.  If you have a machine whose memory is persistent
across reboots then you reserve an arbitrary 4k hunk of memory for
collecting oops traces, yes?

What tools are used for displaying that memory on the next boot?  How
do those tools distinguish between "valid oops trace" and "garbage
because it was just powered on"?  A magic signature?

Should the kernel provide the 4k of memory rather than (or in addition
to) requiring that the system administrator reserve it and tell the
kernel about it?  That'd be a matter of creating a linker section which
isn't cleared out by the startup code.

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Jamie Lokier]

Andrew Morton wrote:
> > I meant with the "classic" use of mtdoops, therefore with a flash
> > partition without use MTD_RAM. Using MTD_RAM, it's more or less the
> > same thing, with the exception of "where" you want deploy the log. For
> > example: if in your system you have got a nvram you can use it without
> > problem, you need to specify the address of the nvram to the module.
> > Very simple. I  think it's a small driver but very useful, feedback
> > from other embedded guys are welcome.
> 
> Seems sensible to me.  If you have a machine whose memory is persistent
> across reboots then you reserve an arbitrary 4k hunk of memory for
> collecting oops traces, yes?

Me too, I think it's a great idea which sounds simpler to use than MTD-RAM.

> What tools are used for displaying that memory on the next boot?  How
> do those tools distinguish between "valid oops trace" and "garbage
> because it was just powered on"?  A magic signature?
> 
> Should the kernel provide the 4k of memory rather than (or in addition
> to) requiring that the system administrator reserve it and tell the
> kernel about it?  That'd be a matter of creating a linker section which
> isn't cleared out by the startup code.

It's good if there's an option to make the location not vary between
kernels, and be known to the bootloader.

Then you can debug kernels which always crash during boot, by either
booting into another kernel which works and looking at the oops, or by
a bootloader command to dump it.

That'd be fine if the kernel link scripts choose the address, as long
as it's consistent between different compiles and similar
configurations.  That'd be a bit simpler than the admin having to know
the memory map well enough to choose an address.

-- Jamie

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Marco Stornelli]

Il 13/03/2010 00:31, Jamie Lokier ha scritto:
> That'd be fine if the kernel link scripts choose the address, as long
> as it's consistent between different compiles and similar
> configurations.  That'd be a bit simpler than the admin having to know
> the memory map well enough to choose an address.
> 
> -- Jamie
> 

I agree, but the bootloader should be aware of it. I mean, usually
bootloaders at boot, reset the RAM, so you have to tell to the
bootloader that you are using a piece of RAM as persistent RAM, for
example U-Boot has got a specific option CONFIG_PRAM. I don't know if
all the process can be completely transparent to the admin in all
situations.

Marco

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Jamie Lokier]

Marco Stornelli wrote:
> Il 13/03/2010 00:31, Jamie Lokier ha scritto:
> > That'd be fine if the kernel link scripts choose the address, as long
> > as it's consistent between different compiles and similar
> > configurations.  That'd be a bit simpler than the admin having to know
> > the memory map well enough to choose an address.
> > 
> > -- Jamie
> > 
> 
> I agree, but the bootloader should be aware of it. I mean, usually
> bootloaders at boot, reset the RAM, so you have to tell to the
> bootloader that you are using a piece of RAM as persistent RAM, for
> example U-Boot has got a specific option CONFIG_PRAM. I don't know if
> all the process can be completely transparent to the admin in all
> situations.

Sometimes you can't change the bootloader (they don't always come with
source code).  Or you could, but you don't want to risk it (there
isn't always a way to recover if you break it).

Obviously then the feature is only useful when the bootloader doesn't
clear all the RAM :-)

On slow boards in consumer devices, they sometimes avoid clearing the
RAM because that adds measurable boot time.

-- Jamie

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Marco Stornelli]

2010/3/15 Jamie Lokier <jamie@shareable.org>:
> Marco Stornelli wrote:
>> Il 13/03/2010 00:31, Jamie Lokier ha scritto:
>> I agree, but the bootloader should be aware of it. I mean, usually
>> bootloaders at boot, reset the RAM, so you have to tell to the
>> bootloader that you are using a piece of RAM as persistent RAM, for
>> example U-Boot has got a specific option CONFIG_PRAM. I don't know if
>> all the process can be completely transparent to the admin in all
>> situations.
>
> Sometimes you can't change the bootloader (they don't always come with
> source code).  Or you could, but you don't want to risk it (there
> isn't always a way to recover if you break it).
>
> Obviously then the feature is only useful when the bootloader doesn't
> clear all the RAM :-)
>
> On slow boards in consumer devices, they sometimes avoid clearing the
> RAM because that adds measurable boot time.
>

In the embedded world, usually, you can change/write the fw and you
know well the memory map, so no problem to know the address to use. In
other cases, it can be possible to use a "transparent" approach, but
in my opinion the general approach used by the driver is enough.

Marco

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Marco Stornelli]

Il 12/03/2010 23:48, Andrew Morton ha scritto:
> On Wed, 10 Mar 2010 13:15:25 +0100
> Marco Stornelli <marco.stornelli@gmail.com> wrote:
> 
>> 2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
>>> 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
>>>> 2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
>> I meant with the "classic" use of mtdoops, therefore with a flash
>> partition without use MTD_RAM. Using MTD_RAM, it's more or less the
>> same thing, with the exception of "where" you want deploy the log. For
>> example: if in your system you have got a nvram you can use it without
>> problem, you need to specify the address of the nvram to the module.
>> Very simple. I  think it's a small driver but very useful, feedback
>> from other embedded guys are welcome.
> 
> Seems sensible to me.  If you have a machine whose memory is persistent
> across reboots then you reserve an arbitrary 4k hunk of memory for
> collecting oops traces, yes?

Yes.

> 
> What tools are used for displaying that memory on the next boot?  How
> do those tools distinguish between "valid oops trace" and "garbage
> because it was just powered on"?  A magic signature?

For my test I used the program devmem2 to dump the log. In general, you
can read the memory via /dev/mem. There's an header plus a timestamp of
the log. The memory is initialized with blank spaces and the size of the
record is fixed at 4k, so if a program/script doesn't find the header at
next 4k, it means there's garbage and it can stop the read operation.

> 
> Should the kernel provide the 4k of memory rather than (or in addition
> to) requiring that the system administrator reserve it and tell the
> kernel about it?  That'd be a matter of creating a linker section which
> isn't cleared out by the startup code.
> 
> 

Yes, it can be an option. My first idea was to write a "general" driver,
with an address in input that it can be related to the reserved RAM as
an NVRAM in the system, however it can be a good idea, why not.

Marco

Re: [PATCH v2] char drivers: Ram oops/panic logger

[Geert Uytterhoeven]

On Fri, Mar 12, 2010 at 23:48, Andrew Morton <akpm@linux-foundation.org> wrote:
> On Wed, 10 Mar 2010 13:15:25 +0100
> Marco Stornelli <marco.stornelli@gmail.com> wrote:
>
>> 2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
>> > 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
>> >> 2010/3/10 Yuasa Yoichi <yuasa@linux-mips.org>:
>> >>> Hi,
>> >>>
>> >>> 2010/3/10 Marco Stornelli <marco.stornelli@gmail.com>:
>> >>>> Ramoops, like mtdoops, can log oops/panic information but in RAM.
>> >>>
>> >>> What is different from mtdoops + mtd-ram?
>> >>>
>> >>> Yoichi
>> >>>
>> >>
>> >> It can be used in a very easy way with persistent RAM for systems
>> >> without flash support. For this systems, with this driver, it's no
>> >> more needed add to the kernel the mtd subsystem with advantage in
>> >> footprint as I said in the description.
>> >
>> > right.
>> > But,
>> >
>> >> In addition, you can save
>> >> flash space and store this information only in RAM. I think it's very
>> >> useful for embedded systems.
>> >
>> > CONFIG_MTD_RAM uses only RAM.
>> > I think there's no big difference about this point.
>> >
>>
>> I meant with the "classic" use of mtdoops, therefore with a flash
>> partition without use MTD_RAM. Using MTD_RAM, it's more or less the
>> same thing, with the exception of "where" you want deploy the log. For
>> example: if in your system you have got a nvram you can use it without
>> problem, you need to specify the address of the nvram to the module.
>> Very simple. I  think it's a small driver but very useful, feedback
>> from other embedded guys are welcome.
>
> Seems sensible to me.  If you have a machine whose memory is persistent
> across reboots then you reserve an arbitrary 4k hunk of memory for
> collecting oops traces, yes?
>
> What tools are used for displaying that memory on the next boot?  How
> do those tools distinguish between "valid oops trace" and "garbage
> because it was just powered on"?  A magic signature?

On Amiga, `debug=mem' enables something like that, cfr. git grep dmesg
arch/m68k.

Gr{oetje,eeting}s,

						Geert

--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- geert@linux-m68k.org

In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
							    -- Linus Torvalds