[BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Alexey Korolev]

Hi all,

We faced JFFS2 kernel panic issues on Linux 2.6.24.
Bug is easy reproducible if to perform random file system operations (esepcially trunkate). We have made a trivial tool which reproduces the problem on JFFS2 partiton. Usually less than 10 min is needed to catch a panic. 

The problem is related to introduction  of write_begin and write_end functions instead of original prepare_write & commit_write. The kernel panic has disappeared when we rolled back write_begin and write_end changes in JFFS2. We tried to fix it - but it seems problem is bit tough for us. I would be much appreciate if someone would fix the problem or suggest where the problem is. We made several attempts to identify the source but still have not idea about what exactly is the source of the bug. Each time panic messages are different. 

Here is the log of one of kernel panic messages:

4/103: write f0 [915918,86035] 28
4/104: unlink d3/d6/df/f16 0
4/106: getdents d3 0
4/107: write d3/d6/fd [1352490,60815] 28
10/209: truncate d5/f9 1534232 28
10/210Unable to handle kernel NULL pointer dereference at virtual
address 000000
00
: chown d5/df/d12/d1e 156 28
10/211: rename d5/df/d10/d15/f18 tpgd = c310c000
o d5/d20/f3c 28
10/212: mkdir d5/df/d10/d15/d28/d3d 28
10/213:[00000000] *pgd=a3107031 creat d5/df/d10/d15/f3e x:0 28 0
10/214: fsync d5/df/d10/d15/d, *pte=000000001b/f22 0
10/215: rename d5/df/d12/d1e to d5/df/d12/d3f 28
10/2, *ppte=0000000016: readlink - no filename
10/217: write d5/df/f1c [484960,1048
83] 28
10/218: readlink - no filename
87/134: mkdir d3/d8/d32 Internal error: Oops: 817 [#7]
Modules linked in:
CPU: 0    Tainted: G      D  (2.6.24.2-pxa27x #18)
PC is at shrink_dcache_parent+0x6c/0x108
LR is at __init_begin+0x3fff8000/0x34
pc : [<c008bdbc>]    lr : [<00000000>]    psr: 00000013
sp : c310becc  ip : c3b00324  fp : c310bef4
r10: c3b00358  r9 : 00000000  r8 : c021403c
r7 : c0214010  r6 : c310a000  r5 : c383781c  r4 : 0000000b
r3 : 00000000  r2 : 00000000  r1 : c3b00348  r0 : c383781c
Flags: nzcv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment user
Control: 0000397f  Table: a310c000  DAC: 00000015
Process fsstress (pid: 825, stack limit = 0xc310a268)
Stack: (0xc310becc to 0xc310c000)
bec0:                            c383781c 00000000 c3838098 c310bf28 c001e004
bee0: c310a000 0000f510 c310bf08 c310bef8 c0082d80 c008bd5c c383781c c310bf24
bf00: c310bf0c c00830f4 c0082d3c c383781c 00000000 c3fba000 c310bf94 c310bf28
bf20: c0084ff0 c00830b0 c3837e68 c3c0d120 00031837 00000002 c3fba000 00000010
bf40: 00000000 00000000 00000006 c001e004 c310bf6c c310bf60 c007c030 c007bd38
bf60: c310bf88 c310bf70 c0078f58 c007c008 00000000 c30847a0 0000000a bed68de4
bf80: 0000f484 00000028 c310bfa4 c310bf98 c0085080 c0084f50 00000000 c310bfa8
bfa0: c001de60 c0085074 0000000a bed68de4 400006b0 bed689dc bed689d4 bed689dc
bfc0: 0000000a bed68de4 0000f484 00000000 000081cc 00000000 0000f510 bed689c4
bfe0: bed689c8 bed685a4 0000aaa0 0001ffc4 20000010 400006b0 00000000 00000000
Backtrace:
[<c008bd50>] (shrink_dcache_parent+0x0/0x108) from [<c0082d80>]
(dentry_unhash+0
x50/0x9c)
[<c0082d30>] (dentry_unhash+0x0/0x9c) from [<c00830f4>]
(vfs_rmdir+0x50/0xbc)
 r4:c383781c
[<c00830a4>] (vfs_rmdir+0x0/0xbc) from [<c0084ff0>] (do_rmdir+0xac/0xf0)
 r6:c3fba000 r5:00000000 r4:c383781c
[<c0084f44>] (do_rmdir+0x0/0xf0) from [<c0085080>] (sys_rmdir+0x18/0x1c)
 r7:00000028 r6:0000f484 r5:bed68de4 r4:0000000a
[<c0085068>] (sys_rmdir+0x0/0x1c) from [<c001de60>]
(ret_fast_syscall+0x0/0x2c)
Code: e2433001 e5873004 e5912004 e59c3024 (e5823000)
28


Here is a code of trivial tool which reproduces the issue very fast:

/*Code is based on fsstress GPL program*/
#define _LARGEFILE64_SOURCE
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/time.h>
#include <sys/ioctl.h>
#include <sys/wait.h>
#include <sys/types.h>
#include <fcntl.h>
#include <malloc.h>
#include <dirent.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <error.h>

#define MAX_FILE_NAME_LEN 256
#define FILE_MASK "file"
#define MNT_POINT "/mnt/mtd8/"
#define BUF_SIZE (128*1024)

char rnd_buf[BUF_SIZE];
int files_created;
char mnt_pnt[256];

enum ops
{
  OPS_CREAT,
  OPS_WRITE,
  OPS_TRUNC,
  OPS_TOTAL
};

void creat_f()
{
  int fd;
  char fname[MAX_FILE_NAME_LEN];
  sprintf(fname,"%s%s.%d",mnt_pnt,FILE_MASK,files_created);
  printf("creat fname=%s\n",fname);
  fd=creat(fname,0666);
  if(!fd)
	return;
  close(fd);
  files_created++;
}

void write_f(int file_num)
{
  printf("write file_num:%d\n",file_num);
  struct stat64 stb;
  int fd;
  __int64_t lr = ((__int64_t)random() << 32) + random();
  off64_t writeofft=0;
  long writelen=0;
  char fname[MAX_FILE_NAME_LEN];
  sprintf(fname,"%s%s.%d",mnt_pnt,FILE_MASK,file_num);
  
  fd=open(fname,O_WRONLY);
  if(!fd)
	return;
  if(fstat64(fd,&stb)<0)
  {
	close(fd);
	return;
  }
  printf("stb.st_size=%lld\n",stb.st_size);
  if(stb.st_size!=0)
	writeofft=(off64_t)(lr%(stb.st_size + BUF_SIZE*4));
  writelen=random()%BUF_SIZE;
  if(writeofft!=lseek64(fd,writeofft,SEEK_SET))
  {
	printf("lseek64 failed with errno %d
%s\n",errno,strerror(errno));
	return;
  }
  memset(rnd_buf,(file_num%127)&0xFF,BUF_SIZE);
  printf("fname=%s writeofft=%lld
writelen=%ld\n",fname,writeofft,writelen);
  write(fd,rnd_buf,writelen);
  printf("after write\n");
  close(fd);
}

void trunc_f(int file_num)
{
  printf("trunc file_num:%d\n",file_num);
  int fd;
  struct stat64 stb;
  stb.st_size=0;
  __int64_t lr = ((__int64_t)random() << 32) + random();
  off64_t truncofft=0;
  char fname[MAX_FILE_NAME_LEN];
  sprintf(fname,"%s%s.%d",mnt_pnt,FILE_MASK,file_num);
  fd=open(fname,O_RDONLY);
  if(!fd)
	return;
  if(fstat64(fd,&stb)<0)
  {
	close(fd);
	return;
  }
  close(fd);
  
  truncofft=(off64_t)(lr%(stb.st_size + BUF_SIZE*4));	
  printf("fname=%s truncofft=%lld\n",fname,truncofft);
  truncate64(fname,truncofft);
}

int main(int argc,char** argv)
{
  int op_num;
  int file_num;
  files_created=0;

  if(argc != 2 || (argc==2 && (strcmp(argv[1],"--help")==0 ||
strcmp(argv[1],"-h")==0)))
  {
	printf("Using: rndops dir\n");
	return 0;
  }
  else
  {
	strcpy(mnt_pnt,argv[1]);
  }

  if(mnt_pnt[strlen(mnt_pnt)-1]!='/')
	strcat(mnt_pnt,"/");

  creat_f();
  
  while(1)
  {
	op_num=random()%OPS_TOTAL;
	file_num=random()%files_created;
	switch(op_num)
	{
	  case OPS_CREAT:
	  creat_f();
	  break;
	  case OPS_WRITE:
		write_f(file_num);
		break;
	  case OPS_TRUNC:
		trunc_f(file_num);
		break;
	}
  }
	
  return 0;
}

Thanks,
Alexey

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Jean Pihet]

[-- Attachment #1: Type: text/plain, Size: 7612 bytes --]

Alexey,

Here is the patch (attached) I applied to get a stable JFFS2 filesystem.
The changes have been cherry picked from 
http://sourceforge.net/projects/mtd-mods.

I stressed it with intensive read/write, flash fill-up and data removal. A few 
GB's of data have been transferred without any problem. I didn't try the tool 
you mentionned though.
Tested on OMAP3 platform.

Now I am testing with PREEMPT_RT config.

Regards,
Jean.

On Thursday 10 April 2008 18:53:55 Alexey Korolev wrote:
> Hi all,
>
> We faced JFFS2 kernel panic issues on Linux 2.6.24.
> Bug is easy reproducible if to perform random file system operations
> (esepcially trunkate). We have made a trivial tool which reproduces the
> problem on JFFS2 partiton. Usually less than 10 min is needed to catch a
> panic.
>
> The problem is related to introduction  of write_begin and write_end
> functions instead of original prepare_write & commit_write. The kernel
> panic has disappeared when we rolled back write_begin and write_end changes
> in JFFS2. We tried to fix it - but it seems problem is bit tough for us. I
> would be much appreciate if someone would fix the problem or suggest where
> the problem is. We made several attempts to identify the source but still
> have not idea about what exactly is the source of the bug. Each time panic
> messages are different.
>
> Here is the log of one of kernel panic messages:
>
> 4/103: write f0 [915918,86035] 28
> 4/104: unlink d3/d6/df/f16 0
> 4/106: getdents d3 0
> 4/107: write d3/d6/fd [1352490,60815] 28
> 10/209: truncate d5/f9 1534232 28
> 10/210Unable to handle kernel NULL pointer dereference at virtual
> address 000000
> 00
>
> : chown d5/df/d12/d1e 156 28
>
> 10/211: rename d5/df/d10/d15/f18 tpgd = c310c000
> o d5/d20/f3c 28
> 10/212: mkdir d5/df/d10/d15/d28/d3d 28
> 10/213:[00000000] *pgd=a3107031 creat d5/df/d10/d15/f3e x:0 28 0
> 10/214: fsync d5/df/d10/d15/d, *pte=000000001b/f22 0
> 10/215: rename d5/df/d12/d1e to d5/df/d12/d3f 28
> 10/2, *ppte=0000000016: readlink - no filename
> 10/217: write d5/df/f1c [484960,1048
> 83] 28
> 10/218: readlink - no filename
> 87/134: mkdir d3/d8/d32 Internal error: Oops: 817 [#7]
> Modules linked in:
> CPU: 0    Tainted: G      D  (2.6.24.2-pxa27x #18)
> PC is at shrink_dcache_parent+0x6c/0x108
> LR is at __init_begin+0x3fff8000/0x34
> pc : [<c008bdbc>]    lr : [<00000000>]    psr: 00000013
> sp : c310becc  ip : c3b00324  fp : c310bef4
> r10: c3b00358  r9 : 00000000  r8 : c021403c
> r7 : c0214010  r6 : c310a000  r5 : c383781c  r4 : 0000000b
> r3 : 00000000  r2 : 00000000  r1 : c3b00348  r0 : c383781c
> Flags: nzcv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment user
> Control: 0000397f  Table: a310c000  DAC: 00000015
> Process fsstress (pid: 825, stack limit = 0xc310a268)
> Stack: (0xc310becc to 0xc310c000)
> bec0:                            c383781c 00000000 c3838098 c310bf28
> c001e004 bee0: c310a000 0000f510 c310bf08 c310bef8 c0082d80 c008bd5c
> c383781c c310bf24 bf00: c310bf0c c00830f4 c0082d3c c383781c 00000000
> c3fba000 c310bf94 c310bf28 bf20: c0084ff0 c00830b0 c3837e68 c3c0d120
> 00031837 00000002 c3fba000 00000010 bf40: 00000000 00000000 00000006
> c001e004 c310bf6c c310bf60 c007c030 c007bd38 bf60: c310bf88 c310bf70
> c0078f58 c007c008 00000000 c30847a0 0000000a bed68de4 bf80: 0000f484
> 00000028 c310bfa4 c310bf98 c0085080 c0084f50 00000000 c310bfa8 bfa0:
> c001de60 c0085074 0000000a bed68de4 400006b0 bed689dc bed689d4 bed689dc
> bfc0: 0000000a bed68de4 0000f484 00000000 000081cc 00000000 0000f510
> bed689c4 bfe0: bed689c8 bed685a4 0000aaa0 0001ffc4 20000010 400006b0
> 00000000 00000000 Backtrace:
> [<c008bd50>] (shrink_dcache_parent+0x0/0x108) from [<c0082d80>]
> (dentry_unhash+0
> x50/0x9c)
> [<c0082d30>] (dentry_unhash+0x0/0x9c) from [<c00830f4>]
> (vfs_rmdir+0x50/0xbc)
>  r4:c383781c
> [<c00830a4>] (vfs_rmdir+0x0/0xbc) from [<c0084ff0>] (do_rmdir+0xac/0xf0)
>  r6:c3fba000 r5:00000000 r4:c383781c
> [<c0084f44>] (do_rmdir+0x0/0xf0) from [<c0085080>] (sys_rmdir+0x18/0x1c)
>  r7:00000028 r6:0000f484 r5:bed68de4 r4:0000000a
> [<c0085068>] (sys_rmdir+0x0/0x1c) from [<c001de60>]
> (ret_fast_syscall+0x0/0x2c)
> Code: e2433001 e5873004 e5912004 e59c3024 (e5823000)
> 28
>
>
> Here is a code of trivial tool which reproduces the issue very fast:
>
> /*Code is based on fsstress GPL program*/
> #define _LARGEFILE64_SOURCE
> #include <sys/stat.h>
> #include <sys/statvfs.h>
> #include <sys/time.h>
> #include <sys/ioctl.h>
> #include <sys/wait.h>
> #include <sys/types.h>
> #include <fcntl.h>
> #include <malloc.h>
> #include <dirent.h>
> #include <errno.h>
> #include <string.h>
> #include <stdlib.h>
> #include <stdio.h>
> #include <unistd.h>
> #include <error.h>
>
> #define MAX_FILE_NAME_LEN 256
> #define FILE_MASK "file"
> #define MNT_POINT "/mnt/mtd8/"
> #define BUF_SIZE (128*1024)
>
> char rnd_buf[BUF_SIZE];
> int files_created;
> char mnt_pnt[256];
>
> enum ops
> {
>   OPS_CREAT,
>   OPS_WRITE,
>   OPS_TRUNC,
>   OPS_TOTAL
> };
>
> void creat_f()
> {
>   int fd;
>   char fname[MAX_FILE_NAME_LEN];
>   sprintf(fname,"%s%s.%d",mnt_pnt,FILE_MASK,files_created);
>   printf("creat fname=%s\n",fname);
>   fd=creat(fname,0666);
>   if(!fd)
> 	return;
>   close(fd);
>   files_created++;
> }
>
> void write_f(int file_num)
> {
>   printf("write file_num:%d\n",file_num);
>   struct stat64 stb;
>   int fd;
>   __int64_t lr = ((__int64_t)random() << 32) + random();
>   off64_t writeofft=0;
>   long writelen=0;
>   char fname[MAX_FILE_NAME_LEN];
>   sprintf(fname,"%s%s.%d",mnt_pnt,FILE_MASK,file_num);
>
>   fd=open(fname,O_WRONLY);
>   if(!fd)
> 	return;
>   if(fstat64(fd,&stb)<0)
>   {
> 	close(fd);
> 	return;
>   }
>   printf("stb.st_size=%lld\n",stb.st_size);
>   if(stb.st_size!=0)
> 	writeofft=(off64_t)(lr%(stb.st_size + BUF_SIZE*4));
>   writelen=random()%BUF_SIZE;
>   if(writeofft!=lseek64(fd,writeofft,SEEK_SET))
>   {
> 	printf("lseek64 failed with errno %d
> %s\n",errno,strerror(errno));
> 	return;
>   }
>   memset(rnd_buf,(file_num%127)&0xFF,BUF_SIZE);
>   printf("fname=%s writeofft=%lld
> writelen=%ld\n",fname,writeofft,writelen);
>   write(fd,rnd_buf,writelen);
>   printf("after write\n");
>   close(fd);
> }
>
> void trunc_f(int file_num)
> {
>   printf("trunc file_num:%d\n",file_num);
>   int fd;
>   struct stat64 stb;
>   stb.st_size=0;
>   __int64_t lr = ((__int64_t)random() << 32) + random();
>   off64_t truncofft=0;
>   char fname[MAX_FILE_NAME_LEN];
>   sprintf(fname,"%s%s.%d",mnt_pnt,FILE_MASK,file_num);
>   fd=open(fname,O_RDONLY);
>   if(!fd)
> 	return;
>   if(fstat64(fd,&stb)<0)
>   {
> 	close(fd);
> 	return;
>   }
>   close(fd);
>
>   truncofft=(off64_t)(lr%(stb.st_size + BUF_SIZE*4));
>   printf("fname=%s truncofft=%lld\n",fname,truncofft);
>   truncate64(fname,truncofft);
> }
>
> int main(int argc,char** argv)
> {
>   int op_num;
>   int file_num;
>   files_created=0;
>
>   if(argc != 2 || (argc==2 && (strcmp(argv[1],"--help")==0 ||
> strcmp(argv[1],"-h")==0)))
>   {
> 	printf("Using: rndops dir\n");
> 	return 0;
>   }
>   else
>   {
> 	strcpy(mnt_pnt,argv[1]);
>   }
>
>   if(mnt_pnt[strlen(mnt_pnt)-1]!='/')
> 	strcat(mnt_pnt,"/");
>
>   creat_f();
>
>   while(1)
>   {
> 	op_num=random()%OPS_TOTAL;
> 	file_num=random()%files_created;
> 	switch(op_num)
> 	{
> 	  case OPS_CREAT:
> 	  creat_f();
> 	  break;
> 	  case OPS_WRITE:
> 		write_f(file_num);
> 		break;
> 	  case OPS_TRUNC:
> 		trunc_f(file_num);
> 		break;
> 	}
>   }
>
>   return 0;
> }
>
> Thanks,
> Alexey
>
> ______________________________________________________
> Linux MTD discussion mailing list
> http://lists.infradead.org/mailman/listinfo/linux-mtd/



[-- Attachment #2: 0001-mtd-jffs2-Fix-non-contiguous-write-BUG-warning-abo.patch --]
[-- Type: text/x-diff, Size: 1973 bytes --]

From 04574ba1c6233a09578988d565a4cc3f2e27f723 Mon Sep 17 00:00:00 2001
From: Jean Pihet <jpihet@mvista.com>
Date: Wed, 9 Apr 2008 11:04:32 +0200
Subject: [PATCH] mtd jffs2: Fix non contiguous write BUG, warning about 'len too short'

Merged 2 fixes from mtd-jffs2-stable-linux-2.6.23 at http://sourceforge.net/projects/mtd-mods:
- non contiguous write BUG
- false warning about 'len too short'

Cf. http://sourceforge.net/project/showfiles.php?group_id=207166&package_id=261645
 for archive download

Tested with intensive (2+ GB data) simultaneous read/write, flash fill-up, data removal cycles

Signed-off-by: Jean Pihet <jpihet@mvista.com>
---
 fs/jffs2/readinode.c |    2 +-
 fs/jffs2/wbuf.c      |    4 ++++
 2 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/fs/jffs2/readinode.c b/fs/jffs2/readinode.c
index 2eae5d2..20efdda 100644
--- a/fs/jffs2/readinode.c
+++ b/fs/jffs2/readinode.c
@@ -63,7 +63,7 @@ static int check_node_data(struct jffs2_sb_info *c, struct jffs2_tmp_dnode_info
 	 * adding and jffs2_flash_read_end() interface. */
 	if (c->mtd->point) {
 		err = c->mtd->point(c->mtd, ofs, len, &retlen, &buffer);
-		if (!err && retlen < tn->csize) {
+		if (!err && retlen < len) {
 			JFFS2_WARNING("MTD point returned len too short: %zu instead of %u.\n", retlen, tn->csize);
 			c->mtd->unpoint(c->mtd, buffer, ofs, retlen);
 		} else if (err)
diff --git a/fs/jffs2/wbuf.c b/fs/jffs2/wbuf.c
index d1d4f27..e023511 100644
--- a/fs/jffs2/wbuf.c
+++ b/fs/jffs2/wbuf.c
@@ -888,6 +888,10 @@ int jffs2_flash_writev(struct jffs2_sb_info *c, const struct kvec *invecs,
 		donelen += wbuf_retlen;
 	}
 
+	/* adjust write buffer offset, else we get a non contiguous write bug */
+	if (!(c->wbuf_ofs % c->sector_size) && !c->wbuf_len)
+		c->wbuf_ofs = 0xffffffff;
+
 	/*
 	 * If there's a remainder in the wbuf and it's a non-GC write,
 	 * remember that the wbuf affects this ino
-- 
1.5.4.1.159.gc4a6c

RE: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Joakim Tjernlund]

> -----Original Message-----
> From: linux-mtd-bounces@lists.infradead.org [mailto:linux-mtd-bounces@lists.infradead.org] On Behalf
> Of Jean Pihet
> Sent: den 10 april 2008 19:18
> To: linux-mtd@lists.infradead.org
> Cc: nickpiggin@yahoo.com.au; joern@logfs.org; dwmw2@infradead.org; Alexey Korolev; akpm@linux-
> foundation.org
> Subject: Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic
> 
> Alexey,
> 
> Here is the patch (attached) I applied to get a stable JFFS2 filesystem.
> The changes have been cherry picked from
> http://sourceforge.net/projects/mtd-mods.
> 
> I stressed it with intensive read/write, flash fill-up and data removal. A few
> GB's of data have been transferred without any problem. I didn't try the tool
> you mentionned though.
> Tested on OMAP3 platform.
> 
> Now I am testing with PREEMPT_RT config.

Is this in Linus tree too?

    Jocke

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Jean Pihet]

Jocke,

> Is this in Linus tree too?
No, it isn't.
2.6.24 JFFS2 as such is pretty unstable.

Jean

RE: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Joakim Tjernlund]

> -----Original Message-----
> From: Jean Pihet [mailto:jpihet@mvista.com]
> Sent: den 10 april 2008 20:52
> To: Joakim Tjernlund
> Cc: linux-mtd@lists.infradead.org; nickpiggin@yahoo.com.au; joern@logfs.org; dwmw2@infradead.org;
> 'Alexey Korolev'; akpm@linux-foundation.org
> Subject: Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic
> 
> Jocke,
> 
> > Is this in Linus tree too?
> No, it isn't.
> 2.6.24 JFFS2 as such is pretty unstable.

ehh, that's no good. I browsed http://sourceforge.net/projects/mtd-mods and there
seems to be lots of patches there that fix different problems. Are
any of these in the soon to be released 2.6.25? which ones?

    Jocke

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Alexey Korolev]

Hi Jean,
> Alexey,
> 
> Here is the patch (attached) I applied to get a stable JFFS2 filesystem.
> The changes have been cherry picked from 
> http://sourceforge.net/projects/mtd-mods.
> 
> I stressed it with intensive read/write, flash fill-up and data removal. A few 
> GB's of data have been transferred without any problem. I didn't try the tool 
> you mentionned though.
> Tested on OMAP3 platform.
> 
> Now I am testing with PREEMPT_RT config.

Oh very interesting link. I'm glad to hear that people use it - even
before announce. Actually I know this project very well - I'm the author and maintainer of this ;-). The project will be announced soon.

You can find the workaround  for the kernel panic problem which disables write_begin and write_end in JFFS2 already exists in the newest release. But fix for the problem does not ready yet. We found one bug in implementation but post validation is not completed yet.  

The patch you have sent is intended to fix another kernel panic issue. It doesn't depend on write_begin/write_end functionalities. Sometime ago we posted the fix to get included, but it was rejected. 

Thanks,
Alexey

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Artem Bityutskiy]

On Fri, 2008-04-11 at 19:00 +0100, Alexey Korolev wrote:
> Hi Jean,
> > Alexey,
> > 
> > Here is the patch (attached) I applied to get a stable JFFS2 filesystem.
> > The changes have been cherry picked from 
> > http://sourceforge.net/projects/mtd-mods.
> > 
> > I stressed it with intensive read/write, flash fill-up and data removal. A few 
> > GB's of data have been transferred without any problem. I didn't try the tool 
> > you mentionned though.
> > Tested on OMAP3 platform.
> > 
> > Now I am testing with PREEMPT_RT config.
> 
> Oh very interesting link. I'm glad to hear that people use it - even
> before announce. Actually I know this project very well - I'm the author and maintainer of this ;-). The project will be announced soon.

So why did you start this project? How about just making all your work
to go into mainline instead?

-- 
Best regards,
Artem Bityutskiy (Битюцкий Артём)

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Jörn Engel]

On Fri, 11 April 2008 21:05:33 +0300, Artem Bityutskiy wrote:
> 
> So why did you start this project? How about just making all your work
> to go into mainline instead?

At least one of the patches should not go to mainline.  It is one of
those strange "If I change the code here, all symptoms disappear"
patches that doesn't cure the cause.  Someone should spend some time to
find the root couse, but noone has.

Keeping such patches somewhere easily accessable - but not in mainline -
might be a good idea.  Would be even better if the root cause was found
and fixed, of course.

Jörn

-- 
Fantasy is more important than knowledge. Knowledge is limited,
while fantasy embraces the whole world.
-- Albert Einstein

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[David Woodhouse]

On Sun, 2008-04-13 at 12:50 +0200, Jörn Engel wrote:
> On Fri, 11 April 2008 21:05:33 +0300, Artem Bityutskiy wrote:
> > 
> > So why did you start this project? How about just making all your work
> > to go into mainline instead?
> 
> At least one of the patches should not go to mainline.  It is one of
> those strange "If I change the code here, all symptoms disappear"
> patches that doesn't cure the cause.  Someone should spend some time to
> find the root couse, but noone has.

Indeed. I'm very reluctant to merge band-aids when the root cause isn't
properly understood. I'm normally a bit better at taking over from such
'hints' and tracking problems down for myself, but over the last few
months I haven't had time for that (or indeed for _anything_ much).
That situation should be improved now -- I have no plans to leave the
country between now and OLS.

> Keeping such patches somewhere easily accessable - but not in mainline -
> might be a good idea.  Would be even better if the root cause was found
> and fixed, of course.

That's the reason I give out accounts on git.infradead.org. I believe
Alexey has such an account.

-- 
dwmw2

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Alexander Belyakov]

On Sun, Apr 13, 2008 at 2:50 PM, Jörn Engel <joern@logfs.org> wrote:
> On Fri, 11 April 2008 21:05:33 +0300, Artem Bityutskiy wrote:
>  >
>  > So why did you start this project? How about just making all your work
>  > to go into mainline instead?
>
>  At least one of the patches should not go to mainline.  It is one of
>  those strange "If I change the code here, all symptoms disappear"
>  patches that doesn't cure the cause.  Someone should spend some time to
>  find the root couse, but noone has.

Joern, I do know the patch you are talking about ;) It was about jffs2
gc path bug.

There are two options: to constantly hit the bug because noone is
interested (have no time) in fixing it "properly" or to use cure
(tested and stressed) until someone would make better fix in the
mainline.

First time (more than half of year ago) I've submitted the patch (with
detailed bug analysis) it was simply ignored. The second time Joern
claimed it as cure not a fix. btw Joern was the only person who
replied - thanks.

So bug does exist, no fix in mainline, and no fruitful discussion in
mailing list. IMO answer to Artem's question becomes quite obvious.

-a

RE: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Joakim Tjernlund]

> -----Original Message-----
> From: linux-mtd-bounces@lists.infradead.org [mailto:linux-mtd-bounces@lists.infradead.org] On Behalf
> Of Alexey Korolev
> Sent: den 11 april 2008 20:01
> To: Jean Pihet
> Cc: nickpiggin@yahoo.com.au; joern@logfs.org; linux-mtd@lists.infradead.org; dwmw2@infradead.org;
> akpm@linux-foundation.org
> Subject: Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic
> 
> Hi Jean,
> > Alexey,
> >
> > Here is the patch (attached) I applied to get a stable JFFS2 filesystem.
> > The changes have been cherry picked from
> > http://sourceforge.net/projects/mtd-mods.
> >
> > I stressed it with intensive read/write, flash fill-up and data removal. A few
> > GB's of data have been transferred without any problem. I didn't try the tool
> > you mentionned though.
> > Tested on OMAP3 platform.
> >
> > Now I am testing with PREEMPT_RT config.
> 
> Oh very interesting link. I'm glad to hear that people use it - even
> before announce. Actually I know this project very well - I'm the author and maintainer of this ;-).
> The project will be announced soon.
> 
> You can find the workaround  for the kernel panic problem which disables write_begin and write_end in
> JFFS2 already exists in the newest release. But fix for the problem does not ready yet. We found one
> bug in implementation but post validation is not completed yet.
> 
> The patch you have sent is intended to fix another kernel panic issue. It doesn't depend on
> write_begin/write_end functionalities. Sometime ago we posted the fix to get included, but it was
> rejected.

Alexey,
are there other patches in your repo that isn't in Linus git tree? If so, which
ones and are they for NOR, NAND or both?

   Jocke

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[David Woodhouse]

On Thu, 2008-04-10 at 17:53 +0100, Alexey Korolev wrote:
> The problem is related to introduction  of write_begin and write_end
> functions instead of original prepare_write & commit_write. The kernel
> panic has disappeared when we rolled back write_begin and write_end
> changes in JFFS2. We tried to fix it - but it seems problem is bit
> tough for us.

Hm, that's very strange. Are you 100% sure it's related to the
write_begin/write_end changes? How long did you run the test for without
a failure, after reverting those changes? And what was the maximum
length of time you could run it without failure beforehand?

Can you show some of the different panic messages you saw? Was there any
kind of pattern to them at all? I suspect memory corruption of some kind
-- can you enable slab debugging?

The output you show includes strings such as 'readlink - no filename'.
Can you explain those? They don't seem to come from your test program.

-- 
dwmw2

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Alexey Korolev]

Hi David, 

> > The problem is related to introduction  of write_begin and write_end
> > functions instead of original prepare_write & commit_write. The kernel
> > panic has disappeared when we rolled back write_begin and write_end
> > changes in JFFS2. We tried to fix it - but it seems problem is bit
> > tough for us.
> 
> Hm, that's very strange. Are you 100% sure it's related to the
> write_begin/write_end changes? How long did you run the test for without
> a failure, after reverting those changes? And what was the maximum
> length of time you could run it without failure beforehand?
> 
Yes. I am 100% sure that the problem is related to
write_begin/write_end. We have quite extensive test suite.  Bug did
not appear if we roll back the changes. 
The failure reproduction depens on different option. In case of disbled
compression and volume size of 32MB we reproduce the issue within 10min
stabily. 
In case of write_begin/write_end disabled we do not catch the problem
after 2 hours of testing. 
> Can you show some of the different panic messages you saw? Was there any
> kind of pattern to them at all? I suspect memory corruption of some kind
> -- can you enable slab debugging?
> 
Sure. 
==========================================
fname=/mnt/mtd9/file.591 writeofft=0 writelen=24991
after write
write file_num:602
stb.st_size=0
fname=/mnt/mtd9/file.602 writeofft=0 writelen=97627
Unable to handle kernel NULL pointer dereference at virtual address 00000000
pgd = c3f00000
[00000000] *pgd=a3e83031, *pte=00000000, *ppte=00000000
Internal error: Oops: 817 [#1]
Modules linked in:
CPU: 0    Not tainted  (2.6.24.2-pxa27x #2)
PC is at cache_alloc_refill+0x18c/0x5b8
LR is at 0xc0cdd000
pc : [<c0075d3c>]    lr : [<c0cdd000>]    psr: 80000093
sp : c3d91c08  ip : 00200200  fp : c3d91c40
r10: ffffffff  r9 : c3c915f0  r8 : c3c904e0
r7 : c3c915e8  r6 : c3c3a000  r5 : c3c915e0  r4 : c0cdd01c
r3 : 00000000  r2 : 00000000  r1 : 0000003c  r0 : 00000000
Flags: Nzcv  IRQs off  FIQs on  Mode SVC_32  ISA ARM  Segment user
Control: 0000397f  Table: a3f00000  DAC: 00000015
Process rndops (pid: 775, stack limit = 0xc3d90268)
Stack: (0xc3d91c08 to 0xc3d92000)
1c00:                   000000d0 000000d0 00000000 000000d0 20000013 00012000
1c20: c3c904e0 000000d0 c3858a4c c3ef9e00 c3fb4c48 c3d91c60 c3d91c44 c0075b78
1c40: c0075bbc c3fb4c48 00012000 00001000 c3fb4c48 c3d91c70 c3d91c64 c00c15cc
1c60: c0075b34 c3d91c90 c3d91c74 c00c0eb8 c00c15bc c3fb4c48 00000000 00000006
1c80: c3c354f8 c3d91cbc c3d91c94 c00c0f28 c00c0ea8 c3fb4c48 00000000 00000006
1ca0: c3c354f8 c3858a4c c3ef9e00 c3fb4c48 c3d91d10 c3d91cc0 c00c546c c00c0f04
1cc0: 00000031 00000003 00001000 c041e000 00000000 00000000 00000031 00001000
1ce0: 00003540 c3ee4000 00000000 c3c354f8 c3858a78 c02433c0 00000000 00000000
1d00: 00000000 c3d91d60 c3d91d14 c00c0068 c00c5254 00012000 00001000 c3d91d34
1d20: 00001000 00012000 00000000 c3ef9e00 00001000 00000000 00001000 00012000
1d40: 00000000 c3d90000 00001000 c3d91dac 00000000 c3d91dec c3d91d64 c0059420
1d60: c00bff58 00001000 00001000 c02433c0 c3858a78 c3858b0c c3d91ea8 c3eddea0
1d80: c3858b0c c01a5218 c3858a78 00012000 00001000 00000000 c01a5218 c00962f4
1da0: c00ca770 c3d91db0 c00379d0 c3d91f30 00000001 00012000 00005d5b c3858a78
1dc0: c02433c0 00000000 c3858a78 00000000 00000000 c3858a78 00000001 00017d5b
1de0: c3d91e60 c3d91df4 c0059ce8 c0059298 00000000 00000000 c3d91ef0 00017d5b
1e00: 00000000 c3d91ef0 c3d91f30 c3d91ea8 c3eddea0 c3858b0c 00000000 00000001
1e20: c3eddea0 c3c262c0 00000001 c3d91f0c 00000001 00017d5b c3858ae0 c3858a78
1e40: c3d91ea8 00000000 00000000 00000001 c3d91f30 c3d91e9c c3d91e64 c0059db8
1e60: c00598c8 00000001 00000000 c3eddea0 c3858b0c 00000000 c3d91ea8 c3eddea0
1e80: c3d91f30 c3d91f80 c3d90000 fffffdee c3d91f5c c3d91ea4 c0078d84 c0059d48
1ea0: 00000000 00000000 c3d91ec8 c3d91eb8 00000000 00000001 ffffffff c3eddea0
1ec0: 00000000 00000000 00000000 00000000 c3c41100 00000000 00000000 c3d91f00
1ee0: c3c41100 c0046d9c c3d91ee8 c3d91ee8 00000000 00000000 c3d91f04 c0102d88
1f00: c002e498 00000000 c3808180 00017d5b c3d91f1c c01031d8 c0102d30 c3c270a0
1f20: c0106058 00000034 c3d3a80c 00000054 00011c08 00017d5b 00017d5b c3eddea0
1f40: 00011c08 c3d91f80 c001e004 40138ff0 c3d91f7c c3d91f60 c00795f0 c0078cc4
1f60: 00000000 00000000 c3eddea0 00000004 c3d91fa4 c3d91f80 c0079b58 c0079548
1f80: 00000000 00000000 00000000 00000000 7d7a5202 00000002 00000000 c3d91fa8
1fa0: c001de60 c0079b18 00000000 7d7a5202 00000003 00011c08 00017d5b 00017d5b
1fc0: 00000000 7d7a5202 00000002 000097d8 00008d48 00000000 40138ff0 bee21de0
1fe0: 00000000 bee21c3c 00005264 400deb00 60000010 00000003 e468e5b3 101e974f
Backtrace:
[<c0075bb0>] (cache_alloc_refill+0x0/0x5b8) from [<c0075b78>] (kmem_cache_alloc+
0x50/0x88)
[<c0075b28>] (kmem_cache_alloc+0x0/0x88) from [<c00c15cc>] (jffs2_alloc_node_fra
g+0x1c/0x24)
 r7:c3fb4c48 r6:00001000 r5:00012000 r4:c3fb4c48
[<c00c15b0>] (jffs2_alloc_node_frag+0x0/0x24) from [<c00c0eb8>] (new_fragment+0x
1c/0x5c)
[<c00c0e9c>] (new_fragment+0x0/0x5c) from [<c00c0f28>] (jffs2_add_full_dnode_to_
inode+0x30/0x3f0)
 r7:c3c354f8 r6:00000006 r5:00000000 r4:c3fb4c48
[<c00c0ef8>] (jffs2_add_full_dnode_to_inode+0x0/0x3f0) from [<c00c546c>] (jffs2_
write_inode_range+0x224/0x344)
[<c00c5248>] (jffs2_write_inode_range+0x0/0x344) from [<c00c0068>] (jffs2_write_
end+0x11c/0x268)
[<c00bff4c>] (jffs2_write_end+0x0/0x268) from [<c0059420>] (generic_file_buffere
d_write+0x198/0x634)
[<c005928c>] (generic_file_buffered_write+0x4/0x634) from [<c0059ce8>] (__generi
c_file_aio_write_nolock+0x42c/0x47c)
[<c00598bc>] (__generic_file_aio_write_nolock+0x0/0x47c) from [<c0059db8>] (gene
ric_file_aio_write+0x80/0xfc)
[<c0059d3c>] (generic_file_aio_write+0x4/0xfc) from [<c0078d84>] (do_sync_write+
0xcc/0x11c)
[<c0078cb8>] (do_sync_write+0x0/0x11c) from [<c00795f0>] (vfs_write+0xb4/0xf4)
[<c007953c>] (vfs_write+0x0/0xf4) from [<c0079b58>] (sys_write+0x4c/0x80)
 r7:00000004 r6:c3eddea0 r5:00000000 r4:00000000
[<c0079b0c>] (sys_write+0x0/0x80) from [<c001de60>] (ret_fast_syscall+0x0/0x2c)
 r6:00000002 r5:7d7a5202 r4:00000000
Code: e25aa001 2affffea e89e000c e59fc3fc (e5832000)
---[ end trace 0b8c0fce588b6607 ]---
Segmentation fault
===========================================================================================

fname=/mnt/mtd9/file.126 writeofft=432278 writelen=114785
after write
write file_num:903
stb.st_size=395371Unable to handle kernel NULL pointer dereference at virtual ad
dress 00000004

fname=/mnt/mtd9/file.903 writeofft=496167 writelen=68119
pgd = c3ef4000
[00000004] *pgd=a3ef8031, *pte=00000000, *ppte=00000000
Internal error: Oops: 17 [#1]
Modules linked in:
CPU: 0    Not tainted  (2.6.24.2-pxa27x #2)
PC is at jffs2_read_dnode+0x3c/0x2f8
LR is at jffs2_alloc_raw_inode+0x1c/0x24
pc : [<c00c1858>]    lr : [<c00c16e4>]    psr: a0000013
sp : c3d4fc4c  ip : c3c3a600  fp : c3d4fc88
r10: c2375000  r9 : c3d90e00  r8 : 00000dd9
r7 : c2375000  r6 : c08e2a58  r5 : c09873f8  r4 : c3d644f8
r3 : 00000000  r2 : 00000001  r1 : fffffffc  r0 : c3d90e00
Flags: NzCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment user
Control: 0000397f  Table: a3ef4000  DAC: 00000015
Process rndops (pid: 775, stack limit = 0xc3d4e268)
Stack: (0xc3d4fc4c to 0xc3d50000)
fc40:                            c3c40080 c0281ea0 c09029f4 c0902934 00000000
fc60: 00001000 00079000 c08e2a58 c2375000 00000dd9 c0902934 0007a000 c3d4fcc0
fc80: c3d4fc8c c00c1c24 c00c1828 00079000 00001000 c3d90e00 c0281ea0 00000000
fca0: c0902960 c0902934 00000dd9 00000227 00000227 c3d4fcd8 c3d4fcc4 c00bfe68
fcc0: c00c1b20 00001000 c0281ea0 c3d4fd60 c3d4fcdc c00c0508 c00bfe0c 000008f7
fce0: 00004a80 00000227 00000000 c0281ea0 c3d4fd10 c3d4fd00 c0057f20 c0046d44
fd00: 00000000 c3d4fd60 c3d4fd34 c3d4fd18 c0089df0 c00e2ccc c3d4fda0 c09014f4
fd20: c3d4fd58 c3d4fd30 c008a804 c00e73e4 c0901500 00000107 c3d4fdac 00079227
fd40: 00000000 c3d4e000 00000dd9 c3d4fdac 00000227 c3d4fdec c3d4fd64 c00593ac
fd60: c00c0230 00000dd9 00000000 c3d4fdc0 c3d4fdbc c09029f4 c3d4fea8 c3e5a4a0
fd80: c09029f4 c01a5218 c0902960 00000000 00000dd9 00000000 c01a5218 00000000
fda0: c3d4fdcc c3d4fdb0 c00379d0 c3d4ff30 00000001 00000000 00010a17 c0902960
fdc0: c0281ea0 00000000 c0902960 00079227 00000000 c0902960 00000001 00010a17
fde0: c3d4fe60 c3d4fdf4 c0059ce8 c0059298 00079227 00000000 c3d4fef0 00010a17
fe00: 00000000 c3d4fef0 c3d4ff30 c3d4fea8 c3e5a4a0 c09029f4 00000000 00000001
fe20: c00380e4 00000016 c0212554 00000000 00000001 00010a17 c09029c8 c0902960
fe40: c3d4fea8 00079227 00000000 00000001 c3d4ff30 c3d4fe9c c3d4fe64 c0059db8
fe60: c00598c8 c0220ab4 c3c400b0 c3e5a4a0 c09029f4 00000000 c3d4fea8 c3e5a4a0
fe80: c3d4ff30 c3d4ff80 c3d4e000 fffffdee c3d4ff5c c3d4fea4 c0078d84 c0059d48
fea0: 00079227 00000000 c3d4fec8 c3d4feb8 00000000 00000001 ffffffff c3e5a4a0
fec0: 00000000 00000000 00000000 00000000 c3c40080 00000000 00000000 c3d4ff00
fee0: c3c40080 c0046d9c c3d4fee8 c3d4fee8 00079227 00000000 c3d4ff04 c0102d88
ff00: c002e498 00000000 c3808180 00010a17 c3d4ff1c c01031d8 c0102d30 c3c55540
ff20: c0106058 00000039 c3d3a80c 00000063 00011c08 00010a17 00010a17 c3e5a4a0
ff40: 00011c08 c3d4ff80 c001e004 40138ff0 c3d4ff7c c3d4ff60 c00795f0 c0078cc4
ff60: 00079227 00000000 c3e5a4a0 00000004 c3d4ffa4 c3d4ff80 c0079b58 c0079548
ff80: 00079227 00000000 00000000 00000000 000e086b 00000000 00000000 c3d4ffa8
ffa0: c001de60 c0079b18 00000000 000e086b 00000003 00011c08 00010a17 00010a17
ffc0: 00000000 000e086b 00000000 000097d8 00008d48 00000000 40138ff0 bec44de0
ffe0: 00000000 bec44c3c 00005264 400deb00 60000010 00000003 00000000 00000000
Backtrace:
[<c00c181c>] (jffs2_read_dnode+0x0/0x2f8) from [<c00c1c24>] (jffs2_read_inode_ra
nge+0x110/0x15c)
[<c00c1b14>] (jffs2_read_inode_range+0x0/0x15c) from [<c00bfe68>] (jffs2_do_read
page_nolock+0x68/0xfc)
[<c00bfe00>] (jffs2_do_readpage_nolock+0x0/0xfc) from [<c00c0508>] (jffs2_write_
begin+0x2e4/0x338)
 r4:c0281ea0
[<c00c0224>] (jffs2_write_begin+0x0/0x338) from [<c00593ac>] (generic_file_buffe
red_write+0x124/0x634)
[<c005928c>] (generic_file_buffered_write+0x4/0x634) from [<c0059ce8>] (__generi
c_file_aio_write_nolock+0x42c/0x47c)
[<c00598bc>] (__generic_file_aio_write_nolock+0x0/0x47c) from [<c0059db8>] (gene
ric_file_aio_write+0x80/0xfc)
[<c0059d3c>] (generic_file_aio_write+0x4/0xfc) from [<c0078d84>] (do_sync_write+
0xcc/0x11c)
[<c0078cb8>] (do_sync_write+0x0/0x11c) from [<c00795f0>] (vfs_write+0xb4/0xf4)
[<c007953c>] (vfs_write+0x0/0xf4) from [<c0079b58>] (sys_write+0x4c/0x80)
 r7:00000004 r6:c3e5a4a0 r5:00000000 r4:00079227
[<c0079b0c>] (sys_write+0x0/0x80) from [<c001de60>] (ret_fast_syscall+0x0/0x2c)
 r6:00000000 r5:000e086b r4:00000000
Code: 0a0000a9 e5953000 e3e01003 e1a00009 (e593c004)
---[ end trace 8f0e605e2cdf29a9 ]---
==============================================================================

fname=/mnt/mtd9/file.92 writeofft=175629 writelen=93922
after write
trunc file_num:275
fname=/mnt/mtd9/file.275 truncofUnable to handle kernel NULL pointer dereference
 at virtual address 00000004
ft=301959
write file_num:719
stb.st_size=20361
fname=/mnt/mtdpgd = c3efc000
9/file.719 writeofft=110950 writelen=115736
[00000004] *pgd=a3ef6031, *pte=00000000, *ppte=00000000
Internal error: Oops: 17 [#1]
Modules linked in:
CPU: 0    Not tainted  (2.6.24.2-pxa27x #2)
PC is at jffs2_read_dnode+0x3c/0x2f8
LR is at jffs2_alloc_raw_inode+0x1c/0x24
pc : [<c00c1858>]    lr : [<c00c16e4>]    psr: a0000013
sp : c3d91c4c  ip : c3c3a600  fp : c3d91c88
r10: c0447000  r9 : c3ef9e00  r8 : 00000e9a
r7 : c0447000  r6 : c0ec1518  r5 : c0cc2a68  r4 : c3c8c4f8
r3 : 00000000  r2 : 00000001  r1 : fffffffc  r0 : c3ef9e00
Flags: NzCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment user
Control: 0000397f  Table: a3efc000  DAC: 00000015
Process rndops (pid: 776, stack limit = 0xc3d90268)
Stack: (0xc3d91c4c to 0xc3d92000)
1c40:                            0000001b 00000014 00000000 c3867e34 c3d91c68
1c60: 00001000 0001b000 c0ec1518 c0447000 00000e9a c3867e34 0001c000 c3d91cc0
1c80: c3d91c8c c00c1c24 c00c1828 0001b000 00001000 c3ef9e00 c02438e0 00000000
1ca0: c3867e60 c3867e34 00000e9a 00000166 00000166 c3d91cd8 c3d91cc4 c00bfe68
1cc0: c00c1b20 00001000 c02438e0 c3d91d60 c3d91cdc c00c0508 c00bfe0c 00000cef
1ce0: 0001fc40 00000166 00000000 c02438e0 c3d91d10 c3d91d00 c0057f20 c0046d44
1d00: 00000000 c3d91d60 c3d91d34 c3d91d18 c0089df0 c00e2ccc c3d91da0 c3866e48
1d20: c3d91d58 c3d91d30 c008a804 c00e73e4 c3866e54 00000107 c3d91dac 0001b166
1d40: 00000000 c3d90000 00000e9a c3d91dac 00000166 c3d91dec c3d91d64 c00593ac
1d60: c00c0230 00000e9a 00000000 c3d91dc0 c3d91dbc c3867ef4 c3d91ea8 c3ec38c0
1d80: c3867ef4 c01a5218 c3867e60 00000000 00000e9a 00000000 c01a5218 c00962f4
1da0: c00ca770 c3d91db0 c00379d0 c3d91f30 00000001 00000000 0001c418 c3867e60
1dc0: c02438e0 00000000 c3867e60 0001b166 00000000 c3867e60 00000001 0001c418
1de0: c3d91e60 c3d91df4 c0059ce8 c0059298 0001b166 00000000 c3d91ef0 0001c418
1e00: 00000000 c3d91ef0 c3d91f30 c3d91ea8 c3ec38c0 c3867ef4 00000000 00000001
1e20: c3ec38c0 c3c262e0 00000001 c3d91f0c 00000001 0001c418 c3867ec8 c3867e60
1e40: c3d91ea8 0001b166 00000000 00000001 c3d91f30 c3d91e9c c3d91e64 c0059db8
1e60: c00598c8 00000001 00000000 c3ec38c0 c3867ef4 00000000 c3d91ea8 c3ec38c0
1e80: c3d91f30 c3d91f80 c3d90000 fffffdee c3d91f5c c3d91ea4 c0078d84 c0059d48
1ea0: 0001b166 00000000 c3d91ec8 c3d91eb8 00000000 00000001 ffffffff c3ec38c0
1ec0: 00000000 00000000 00000000 00000000 c3d67180 00000000 00000000 c3d91f00
1ee0: c3d67180 c0046d9c c3d91ee8 c3d91ee8 0001b166 00000000 c3d91f04 c0102d88
1f00: c002e498 00000000 c3808180 0001c418 c3d91f1c c01031d8 c0102d30 c3c27120
1f20: c0106058 0000003a c3d3a80c 00000054 00011c08 0001c418 0001c418 c3ec38c0
1f40: 00011c08 c3d91f80 c001e004 40138ff0 c3d91f7c c3d91f60 c00795f0 c0078cc4
1f60: 0001b166 00000000 c3ec38c0 00000004 c3d91fa4 c3d91f80 c0079b58 c0079548
1f80: 0001b166 00000000 00000000 00000000 00084f89 00000000 00000000 c3d91fa8
1fa0: c001de60 c0079b18 00000000 00084f89 00000003 00011c08 0001c418 0001c418
1fc0: 00000000 00084f89 00000000 000097d8 00008d48 00000000 40138ff0 be86bde0
1fe0: 00000000 be86bc3c 00005264 400deb00 60000010 00000003 e468e5b3 101e974f
Backtrace:
[<c00c181c>] (jffs2_read_dnode+0x0/0x2f8) from [<c00c1c24>] (jffs2_read_inode_ra
nge+0x110/0x15c)
[<c00c1b14>] (jffs2_read_inode_range+0x0/0x15c) from [<c00bfe68>] (jffs2_do_read
page_nolock+0x68/0xfc)
[<c00bfe00>] (jffs2_do_readpage_nolock+0x0/0xfc) from [<c00c0508>] (jffs2_write_
begin+0x2e4/0x338)
 r4:c02438e0
[<c00c0224>] (jffs2_write_begin+0x0/0x338) from [<c00593ac>] (generic_file_buffe
red_write+0x124/0x634)
[<c005928c>] (generic_file_buffered_write+0x4/0x634) from [<c0059ce8>] (__generi
c_file_aio_write_nolock+0x42c/0x47c)
[<c00598bc>] (__generic_file_aio_write_nolock+0x0/0x47c) from [<c0059db8>] (gene
ric_file_aio_write+0x80/0xfc)
[<c0059d3c>] (generic_file_aio_write+0x4/0xfc) from [<c0078d84>] (do_sync_write+
0xcc/0x11c)
[<c0078cb8>] (do_sync_write+0x0/0x11c) from [<c00795f0>] (vfs_write+0xb4/0xf4)
[<c007953c>] (vfs_write+0x0/0xf4) from [<c0079b58>] (sys_write+0x4c/0x80)
 r7:00000004 r6:c3ec38c0 r5:00000000 r4:0001b166
[<c0079b0c>] (sys_write+0x0/0x80) from [<c001de60>] (ret_fast_syscall+0x0/0x2c)
 r6:00000000 r5:00084f89 r4:00000000
Code: 0a0000a9 e5953000 e3e01003 e1a00009 (e593c004)



> The output you show includes strings such as 'readlink - no filename'.
> Can you explain those? They don't seem to come from your test program.
> 
Oh you are quite onservant. I have several panic logs from different
applications. By mistake I sent bug message which was cathed by another application(it is required more time to cacth the
problem in case of using it ). 

Thanks,
Alexey

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Alexey Korolev]

Hi 

 
> > > The problem is related to introduction  of write_begin and write_end
> > > functions instead of original prepare_write & commit_write. The kernel
> > > panic has disappeared when we rolled back write_begin and write_end
> > > changes in JFFS2. We tried to fix it - but it seems problem is bit
> > > tough for us.
> > 

Seems the problem is overcome. See the patch in message "[PATCH] JFFS2
Fix of panics caused by wrong condition for hole fragcreation in
write_begin". Validation is done. The problem does not take palce if to apply the patch. 

Thanks,
Alexey

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Anders Grafström]

Alexey Korolev wrote:
>>>> The problem is related to introduction  of write_begin and write_end
>>>> functions instead of original prepare_write & commit_write. The kernel
>>>> panic has disappeared when we rolled back write_begin and write_end
>>>> changes in JFFS2. We tried to fix it - but it seems problem is bit
>>>> tough for us.
> 
> Seems the problem is overcome. See the patch in message "[PATCH] JFFS2
> Fix of panics caused by wrong condition for hole fragcreation in
> write_begin". Validation is done. The problem does not take palce if to apply the patch. 

I've been getting a bunch of these panics while testing the erase suspend issue.
I'm using current mtd-2.6.git.

[    4.251504] Unable to handle kernel NULL pointer dereference at virtual address 00000000
[    4.261504] pgd = c3cd8000
[    4.271504] [00000000] *pgd=03d9b031, *pte=000010cf, *ppte=0000100e
[    4.291504] Internal error: Oops: 3cd981f [#1]
[    4.291504] Modules linked in:
[    4.291504] CPU: 0    Not tainted  (2.6.25-1 #5)
[    4.291504] PC is at jffs2_write_end+0x54/0x264
[    4.291504] LR is at generic_file_buffered_write+0x19c/0x668
[    4.291504] pc : [<c00daa84>]    lr : [<c005baa8>]    psr: 60000013
[    4.291504] sp : c3c99d18  ip : c3c99d68  fp : c3c99d64
[    4.291504] r10: c3c98000  r9 : c3c99db8  r8 : 00000000
[    4.291504] r7 : c02934e0  r6 : c38208e8  r5 : 0007c000  r4 : 00001000
[    4.291504] r3 : 00001000  r2 : 0007c000  r1 : 00000000  r0 : c35d2440
[    4.291504] Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment user
[    4.291504] Control: 03cd917f  Table: 03cd917f  DAC: 00000015
[    4.291504] Process swap (pid: 358, stack limit = 0xc3c98260)
[    4.291504] Stack: (0xc3c99d18 to 0xc3c9a000)
[    4.291504] 9d00:                                                       0007b000 00001000
[    4.291504] 9d20: c3c99d38 00001000 0007c000 00000000 c3503e00 c1fa7000 00000000 00001000
[    4.291504] 9d40: 0007c000 00000000 00001000 00000000 c3c99db8 c3c98000 c3c99df8 c3c99d68
[    4.291504] 9d60: c005baa8 c00daa3c 00001000 00001000 c02934e0 c3c99dfc c3820980 c3c99ebc
[    4.291504] 9d80: c35d2440 c3820980 c01ba6e0 c38208e8 0001c000 00001000 00000000 c01ba6e0
[    4.291504] 9da0: c3c99dd8 c35d2440 c3c99dd4 c3c99db8 c003aed0 c003a96c c3c99f40 00000001
[    4.291504] 9dc0: 0001c000 00004000 c3c99dfc c02934e0 00000000 c38208e8 00060000 00000000
[    4.291504] 9de0: 00000000 00000001 00020000 c3c99e74 c3c99e00 c005c3b4 c005b91c 00060000
[    4.291504] 9e00: 00000000 c3c99f04 00020000 00000000 c3c99f04 c3c99f40 c3c99ebc 0005ffff
[    4.291504] 9e20: 00000000 c35d2440 c3820980 00000000 00000001 c3c99e74 c3c99e44 c005b5a4
[    4.291504] 9e40: c005b140 00000000 00020000 c3820954 c3c99ebc c38208e8 00060000 00000000
[    4.291504] 9e60: 00000001 c3c99f40 c3c99eb0 c3c99e78 c005c4a0 c005bf80 00020000 00000000
[    4.291504] 9e80: c35d2440 c3820980 c3c99ebc c35d2440 c3c99f80 00000004 c0020c64 c3c98000
[    4.291504] 9ea0: 40035138 c3c99f60 c3c99eb8 c0078194 c005c434 00060000 00000000 c3c99ec8
[    4.291504] 9ec0: c0113da0 00000000 00000001 ffffffff c35d2440 00000000 00000000 00000000
[    4.291504] 9ee0: 00000000 c3c39080 00000000 00000000 00000000 c3c39080 c0049ec0 c3c99efc
[    4.291504] 9f00: c3c99efc 00060000 00000000 00000031 c3c99f60 c023dd5c ce92ad14 00020000
[    4.291504] 9f20: c3c71090 c3c99f54 c3c99f34 c3c99f5c c3c99f3c c004d9b0 c002b5ac c35d2440
[    4.291504] 9f40: 40842c08 00020000 c35d2440 40842c08 c3c99f80 c3c99f7c c3c99f64 c00789e8
[    4.291504] 9f60: c00780e0 c35d2440 00060000 00000000 c3c99fa4 c3c99f80 c0078f28 c0078940
[    4.291504] 9f80: 00060000 00000000 00000000 00020000 00060000 00020000 00000000 c3c99fa8
[    4.291504] 9fa0: c0020ac0 c0078ee8 00020000 00060000 00000004 40842c08 00020000 00000000
[    4.291504] 9fc0: 00020000 00060000 00020000 00000004 005e1064 407e2c08 40035138 becb50b4
[    4.291504] 9fe0: 00000000 becb5080 000036c4 400e0ec0 60000010 00000004 3ac9b6c3 0dd9cf30
[    4.291504] Backtrace:
[    4.291504] [<c00daa30>] (jffs2_write_end+0x0/0x264) from [<c005baa8>] (generic_file_buffered_write+0x19c/0x668)
[    4.291504] [<c005b910>] (generic_file_buffered_write+0x4/0x668) from [<c005c3b4>] (__generic_file_aio_write_nolock+0x440/0x4b0)
[    4.291504] [<c005bf74>] (__generic_file_aio_write_nolock+0x0/0x4b0) from [<c005c4a0>] (generic_file_aio_write+0x7c/0xf0)
[    4.291504] [<c005c428>] (generic_file_aio_write+0x4/0xf0) from [<c0078194>] (do_sync_write+0xc0/0x114)
[    4.291504] [<c00780d4>] (do_sync_write+0x0/0x114) from [<c00789e8>] (vfs_write+0xb4/0xdc)
[    4.291504]  r6:c3c99f80 r5:40842c08 r4:c35d2440
[    4.291504] [<c0078934>] (vfs_write+0x0/0xdc) from [<c0078f28>] (sys_write+0x4c/0x7c)
[    4.291504]  r6:00000000 r5:00060000 r4:c35d2440
[    4.291504] [<c0078edc>] (sys_write+0x0/0x7c) from [<c0020ac0>] (ret_fast_syscall+0x0/0x2c)
[    4.291504]  r6:00020000 r5:00060000 r4:00020000
[    4.291504] Code: e1a08a28 e2111001 e0883003 e50b3040 (05811000)
[    4.291504] Kernel panic - not syncing: Fatal exception

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[David Woodhouse]

On Thu, 2008-04-24 at 23:10 +0200, Anders Grafström wrote:
> [    4.251504] Unable to handle kernel NULL pointer dereference at virtual address 00000000
> [    4.261504] pgd = c3cd8000
> [    4.271504] [00000000] *pgd=03d9b031, *pte=000010cf, *ppte=0000100e
> [    4.291504] Internal error: Oops: 3cd981f [#1]
> [    4.291504] Modules linked in:
> [    4.291504] CPU: 0    Not tainted  (2.6.25-1 #5)
> [    4.291504] PC is at jffs2_write_end+0x54/0x264

list *jffs2_write_end+0x54

And can you reproduce with CONFIG_JFFS2_FS_DEBUG=1? 

-- 
dwmw2

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Alexey Korolev]

Anders,

Are you reproducing it after failed erase? 
What tool do you use? The last one in excessive erase thread?
If so we could try to reproduce it as well.

Thanks,
Alexey

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Anders Grafström]

David Woodhouse wrote:
> list *jffs2_write_end+0x54
> 
> And can you reproduce with CONFIG_JFFS2_FS_DEBUG=1? 

I think I need an mtdoops partition to catch anything useful from CONFIG_JFFS2_FS_DEBUG.
Too much output for the console. It slows it down so much that it doesn't trigger the panic.

It is triggered by an application that writes a file (about 6MB large) to the jffs2 file system.
It replaces an old file and the use percentage on the file system is high so it triggers
a lot of erases while it's writing. The panic happens before I see any failed erases.

Filesystem           1k-blocks      Used Available Use% Mounted on
/dev/mtdblock2           15616     13780      1836  88% /flash

The panic occurs at fs/jffs2/file.c:251 which is
BUG_ON(!PageUptodate(pg));


Data CRC 33c102e9 != calculated CRC 0ef77e7b for node at 005d42e4
kernel BUG at /local/git/kernel-mtd/kernel/fs/jffs2/file.c:251!
Unable to handle kernel NULL pointer dereference at virtual address 00000000
pgd = c3d0c000
[00000000] *pgd=010fa031, *pte=000010cf, *ppte=0000100e
Internal error: Oops: 3d0d81f [#1]
Modules linked in:
CPU: 0    Not tainted  (2.6.25-1 #11)
PC is at __bug+0x20/0x2c
LR is at 0xc0240544
pc : [<c002465c>]    lr : [<c0240544>]    psr: 60000013
sp : c3d33d04  ip : c0240544  fp : c3d33d10
r10: c3d32000  r9 : 00001000  r8 : c02c0e60
r7 : 00000000  r6 : 00000000  r5 : c38208e8  r4 : 00000000
r3 : 00000000  r2 : 00000001  r1 : 00000001  r0 : 00000043
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment user
Control: 03d0d17f  Table: 03d0d17f  DAC: 00000015
Process swap (pid: 205, stack limit = 0xc3d32260)
Stack: (0xc3d33d04 to 0xc3d34000)
3d00:          c3d33d64 c3d33d14 c00dd7ec c0024648 00001000 00000203 c3d33d38
3d20: 0043d000 00000000 0043d000 00000000 c351fe00 00000000 00000000 00001000
3d40: 0043d000 00000000 00001000 00000000 c3d33db8 c3d32000 c3d33df8 c3d33d68
3d60: c005d854 c00dd77c 00001000 00001000 c02c0e60 c3d33dfc c3820980 c3d33ebc
3d80: c3dca720 c3820980 c01c1f6c c38208e8 0001d000 00001000 00000000 c01c1f6c
3da0: c3d33dd8 c3dca720 c3d33dd4 c3d33db8 c003c824 c003c3b0 c3d33f40 00000001
3dc0: 0001d000 00003000 c3d33dfc c02c0e60 00000000 c38208e8 00420000 00000000
3de0: 00420000 00000000 00020000 c3d33e74 c3d33e00 c005e160 c005d6c8 00420000
3e00: 00000000 c3d33f04 00020000 00000000 c3d33f04 c3d33f40 c3d33ebc 0041ffff
3e20: 00000000 c3dca720 c3820980 00000000 00000001 c3d33e74 c3d33e44 c005d350
3e40: c005ceec 00000000 00020000 c3820954 c3d33ebc c3d33f40 c38208e8 00420000
3e60: 00000000 00000001 c3d33eb0 c3d33e78 c005e250 c005dd2c 00020000 00000000
3e80: c3dca720 c3820980 c3d33ebc c3dca720 c3d33f80 00000004 c0020c64 c3d32000
3ea0: 40035138 c3d33f60 c3d33eb8 c007a35c c005e1e0 00420000 00000000 c3d33ec8
3ec0: c011acb0 00000000 00000001 ffffffff c3dca720 00000000 00000000 00000000
3ee0: 00000000 c35e2380 00000000 00000000 00000000 c35e2380 c004b944 c3d33efc
3f00: c3d33efc 00420000 00000000 00000031 c3d33f60 c3d33f20 c01155c4 00020000
3f20: c35e3bc0 c0118230 c352a40c 00000031 4811f800 2474fa30 c35e3bc0 c3dca720
3f40: 40c09c08 00020000 c3dca720 40c09c08 c3d33f80 c3d33f7c c3d33f64 c007abb0
3f60: c007a2a8 c3dca720 00420000 00000000 c3d33fa4 c3d33f80 c007b0f0 c007ab08
3f80: 00420000 00000000 00000000 00020000 00420000 00020000 00000000 c3d33fa8
3fa0: c0020ac0 c007b0b0 00020000 00420000 00000005 40c09c08 00020000 00000000
3fc0: 00020000 00420000 00020000 00000005 005e84e0 407e9c08 40035138 be8900b4
3fe0: 00000000 be890080 000036c4 400e0ec0 60000010 00000005 5fd6b7be d06feaea
Backtrace:
[<c002463c>] (__bug+0x0/0x2c) from [<c00dd7ec>] (jffs2_write_end+0x7c/0x2b8)
[<c00dd770>] (jffs2_write_end+0x0/0x2b8) from [<c005d854>] (generic_file_buffered_write+0x19c/0x668)
[<c005d6bc>] (generic_file_buffered_write+0x4/0x668) from [<c005e160>] (__generic_file_aio_write_nolock+0x440/0x4b0)
[<c005dd20>] (__generic_file_aio_write_nolock+0x0/0x4b0) from [<c005e250>] (generic_file_aio_write+0x80/0xfc)
[<c005e1d4>] (generic_file_aio_write+0x4/0xfc) from [<c007a35c>] (do_sync_write+0xc0/0x114)
[<c007a29c>] (do_sync_write+0x0/0x114) from [<c007abb0>] (vfs_write+0xb4/0xdc)
  r6:c3d33f80 r5:40c09c08 r4:c3dca720
[<c007aafc>] (vfs_write+0x0/0xdc) from [<c007b0f0>] (sys_write+0x4c/0x7c)
  r6:00000000 r5:00420000 r4:c3dca720
[<c007b0a4>] (sys_write+0x0/0x7c) from [<c0020ac0>] (ret_fast_syscall+0x0/0x2c)
  r6:00020000 r5:00420000 r4:00020000
Code: e1a01000 e59f000c eb0051ba e3a03000 (e5833000)
Kernel panic - not syncing: Fatal exception

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[David Woodhouse]

On Fri, 2008-04-25 at 18:09 +0200, Anders Grafström wrote:
> 
> 
> I think I need an mtdoops partition to catch anything useful from
> CONFIG_JFFS2_FS_DEBUG.
> Too much output for the console. It slows it down so much that it
> doesn't trigger the panic.

Just the lines in the kernel's ring buffer leading up to the BUG should
be sufficient, if you can get them -- just the output of 'dmesg' ought
to be fairly useful. Why is it a panic anyway -- have you set
panic-on-oops?

-- 
dwmw2

Re: [BUG] JFFS2 usage of write_begin and write_end functions causes kernel panic

[Anders Grafström]

Anders Grafström wrote:
> The panic occurs at fs/jffs2/file.c:251 which is
> BUG_ON(!PageUptodate(pg));
> 
> Data CRC 33c102e9 != calculated CRC 0ef77e7b for node at 005d42e4

I think this issue might be a sideeffect of (or related to) the erase suspend problems
I'm having. Sometimes there are data CRC errors.
The error is propagated upwards from jffs2_read_dnode to jffs2_do_readpage_nolock
which calls ClearPageUptodate and returns 0 then jffs2_write_end traps on !PageUptodate.

I hope this illustrates the scenario:

jffs2_write_begin {
     jffs2_do_readpage_nolock {
         jffs2_read_inode_range {
             jffs2_read_dnode -> Data CRC 33c102e9 != calculated CRC 0ef77e7b for node at 005d42e4
         }
         ClearPageUptodate(pg);
         return 0
     }
}

jffs2_write_end {
     BUG_ON(!PageUptodate(pg));
}

Should the error value perhaps be forwarded upwards?
Would that inadvertently prevent a broken node from being replaced by a new one?

diff --git a/fs/jffs2/file.c b/fs/jffs2/file.c
index 5e92034..a146bf4 100644
--- a/fs/jffs2/file.c
+++ b/fs/jffs2/file.c
@@ -99,7 +99,7 @@ static int jffs2_do_readpage_nolock (struct inode *inode, struct page *pg)
  	kunmap(pg);

  	D2(printk(KERN_DEBUG "readpage finished\n"));
-	return 0;
+	return ret;
  }

  int jffs2_do_readpage_unlock(struct inode *inode, struct page *pg)