Reiser4 unlink performance

Reiser4 unlink performance

[Alec H. Peterson]

Hi all,

I have a scenario where large numbers of unlinks spread across 256  
directories with 20,000 files in each directory results in these  
kernel messages:

Feb 22 09:00:19 ecbuild-10 kernel: <4>reiser4[umount(13646)]:  
commit_current_atom (fs/reiser4/txnmgr.c:1092)[nikita-3176]:
Feb 22 09:00:19 ecbuild-10 kernel: WARNING: Flushing like mad: 16384

When this happens the filesystem blocks for a rather long period of  
time.

Interestingly, when I reduce the number of directories to 120 and  
increase the number of files in each directory to 40,000 the  
performance improves remarkably, although I do still get some of  
those kernel messages, albeit fewer of them.

I am curious if there are tunable parameters (either compile-time or  
mount-time) that I can modify to help alleviate this problem, or if  
there are any other suggestions.

Thanks!

Alec

Re: Reiser4 unlink performance

[Alec H. Peterson]

Hi Alexander,

On Feb 26, 2006, at 13:42, Alexander Zarochentsev wrote:

>
> Which kernel version and reiser4 patches were used?

2.6.15.4r4 kernel,
reiser4-for-2.6.15-1.patch

> Is the system SMP?

Yes.

>
> No, "Flushing like mad" usually indicates a bug.

Good to know.

Thanks!

Alec

Re: Reiser4 unlink performance

[Hans Reiser]

Alec H. Peterson wrote:

> Hi Alexander,
>
> On Feb 26, 2006, at 13:42, Alexander Zarochentsev wrote:
>
>>
>> Which kernel version and reiser4 patches were used?
>
>
> 2.6.15.4r4 kernel,
> reiser4-for-2.6.15-1.patch
>
>> Is the system SMP?
>
>
> Yes.
>
>>
>> No, "Flushing like mad" usually indicates a bug.
>
>
Please describe it in more detail.

> Good to know.
>
> Thanks!
>
> Alec
>
>
>
>

Re: Reiser4 unlink performance

[Alec H. Peterson]

Hi Hans,

On Feb 28, 2006, at 12:11, Hans Reiser wrote:

> Please describe it in more detail.

Well, the error I'm getting is this:

Feb 21 13:10:22 ecbuild-10 kernel: WARNING: Flushing like mad: 16384

The C program I'm running to do the test is this:

main(int argc, char **argv)
{
   char buf[1024];
   char dir[1024];
   int i,j,k;
   int ret;
   int fd;
   int *sequence;
   struct timeval start, unlinktime, end;

   if(argc != 2)
   {
     fprintf(stderr, "Usage: %s <base directory>\n", argv[0]);
     exit(1);
   }

   strncpy(dir, argv[1], 1024);

   if(chdir(dir))
   {
     perror("chdir");
     exit(1);
   }

   for(i = 0; i < 256; i++)
   {
     snprintf(buf, 1024, "%d", i);
     ret = mkdir(buf, S_IRWXU);
     if(ret && (errno != EEXIST))
     {
       perror("mkdir");
       exit(1);
     }
   }


   {
     sequence = sequence20000;
     gettimeofday(&start, NULL);
     fprintf(stderr, "Starting %d run %d.%d.\n",
             256 * 20000, start.tv_sec, start.tv_usec);

     for(i = 0; i < 256; i++)
     {
       for(j = 0; j < 20000; j++)
       {
         snprintf(buf, 1024, "%d/%d", i, j);
         fd = open(buf, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
         if(fd == -1)
         {
           perror("open");
           exit(1);
         }
         close(fd);
       }
       fprintf(stderr, "Done %d\n", i);
     }

     gettimeofday(&unlinktime, NULL);
     fprintf(stderr, "Starting %d unlink %d.%d, elapsed time was %d.\n",
             256 * 20000, unlinktime.tv_sec, unlinktime.tv_usec,
             unlinktime.tv_sec - start.tv_sec);

     for(i = 0; i < 20000; i++)
     {
       for(j = 0; j < 256; j++)
       {
         snprintf(buf, 1024, "%d/%d", j, sequence[i]);
         unlink(buf);
         if(fd == -1)
         {
           perror("unlink");
           exit(1);
         }
       }
       fprintf(stderr, "Done %d\n", i);
     }

     gettimeofday(&end, NULL);
     fprintf(stderr, "Done with %d unlink %d.%d, elapsed time was %d. 
\n",
             20000 * 256, end.tv_sec, end.tv_usec,
             end.tv_sec - unlinktime.tv_sec);
   }

   exit(0);
}

sequence20000 is an array which contains a sequence from 1 to 20000  
randomly ordered.

Alec

Re: Reiser4 unlink performance

[Hans Reiser]

I meant for Zam to describe it in more detail, sorry about that.  :-/

Hans

Alec H. Peterson wrote:

> Hi Hans,
>
> On Feb 28, 2006, at 12:11, Hans Reiser wrote:
>
>> Please describe it in more detail.
>
>
> Well, the error I'm getting is this:
>
> Feb 21 13:10:22 ecbuild-10 kernel: WARNING: Flushing like mad: 16384
>
> The C program I'm running to do the test is this:
>
> main(int argc, char **argv)
> {
>   char buf[1024];
>   char dir[1024];
>   int i,j,k;
>   int ret;
>   int fd;
>   int *sequence;
>   struct timeval start, unlinktime, end;
>
>   if(argc != 2)
>   {
>     fprintf(stderr, "Usage: %s <base directory>\n", argv[0]);
>     exit(1);
>   }
>
>   strncpy(dir, argv[1], 1024);
>
>   if(chdir(dir))
>   {
>     perror("chdir");
>     exit(1);
>   }
>
>   for(i = 0; i < 256; i++)
>   {
>     snprintf(buf, 1024, "%d", i);
>     ret = mkdir(buf, S_IRWXU);
>     if(ret && (errno != EEXIST))
>     {
>       perror("mkdir");
>       exit(1);
>     }
>   }
>
>
>   {
>     sequence = sequence20000;
>     gettimeofday(&start, NULL);
>     fprintf(stderr, "Starting %d run %d.%d.\n",
>             256 * 20000, start.tv_sec, start.tv_usec);
>
>     for(i = 0; i < 256; i++)
>     {
>       for(j = 0; j < 20000; j++)
>       {
>         snprintf(buf, 1024, "%d/%d", i, j);
>         fd = open(buf, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
>         if(fd == -1)
>         {
>           perror("open");
>           exit(1);
>         }
>         close(fd);
>       }
>       fprintf(stderr, "Done %d\n", i);
>     }
>
>     gettimeofday(&unlinktime, NULL);
>     fprintf(stderr, "Starting %d unlink %d.%d, elapsed time was %d.\n",
>             256 * 20000, unlinktime.tv_sec, unlinktime.tv_usec,
>             unlinktime.tv_sec - start.tv_sec);
>
>     for(i = 0; i < 20000; i++)
>     {
>       for(j = 0; j < 256; j++)
>       {
>         snprintf(buf, 1024, "%d/%d", j, sequence[i]);
>         unlink(buf);
>         if(fd == -1)
>         {
>           perror("unlink");
>           exit(1);
>         }
>       }
>       fprintf(stderr, "Done %d\n", i);
>     }
>
>     gettimeofday(&end, NULL);
>     fprintf(stderr, "Done with %d unlink %d.%d, elapsed time was %d. \n",
>             20000 * 256, end.tv_sec, end.tv_usec,
>             end.tv_sec - unlinktime.tv_sec);
>   }
>
>   exit(0);
> }
>
> sequence20000 is an array which contains a sequence from 1 to 20000 
> randomly ordered.
>
> Alec
>
>
>

Re: Reiser4 unlink performance

[Alexander Zarochentsev]

On Tuesday 28 February 2006 22:11, Hans Reiser wrote:
> Alec H. Peterson wrote:
> > Hi Alexander,
> >
> > On Feb 26, 2006, at 13:42, Alexander Zarochentsev wrote:
> >> Which kernel version and reiser4 patches were used?
> >
> > 2.6.15.4r4 kernel,
> > reiser4-for-2.6.15-1.patch
> >
> >> Is the system SMP?
> >
> > Yes.
> >
> >> No, "Flushing like mad" usually indicates a bug.
>
> Please describe it in more detail.
>

The flush algorithm designed to make progress in any iteration.  
"flushing like mad" messages usually indicates that the flush can't 
make progress.

> > Good to know.
> >
> > Thanks!
> >
> > Alec
>
> !DSPAM:4404a06b108681480917376!

-- 
Alex.

Re: Reiser4 unlink performance

[Hans Reiser]

Alexander Zarochentsev wrote:

>On Tuesday 28 February 2006 22:11, Hans Reiser wrote:
>  
>
>>Alec H. Peterson wrote:
>>    
>>
>>>Hi Alexander,
>>>
>>>On Feb 26, 2006, at 13:42, Alexander Zarochentsev wrote:
>>>      
>>>
>>>>Which kernel version and reiser4 patches were used?
>>>>        
>>>>
>>>2.6.15.4r4 kernel,
>>>reiser4-for-2.6.15-1.patch
>>>
>>>      
>>>
>>>>Is the system SMP?
>>>>        
>>>>
>>>Yes.
>>>
>>>      
>>>
>>>>No, "Flushing like mad" usually indicates a bug.
>>>>        
>>>>
>>Please describe it in more detail.
>>
>>    
>>
>
>The flush algorithm designed to make progress in any iteration.  
>"flushing like mad" messages usually indicates that the flush can't 
>make progress.
>  
>
Yet more detail please.

>  
>
>>>Good to know.
>>>
>>>Thanks!
>>>
>>>Alec
>>>      
>>>
>>!DSPAM:4404a06b108681480917376!
>>    
>>
>
>  
>

Re: Reiser4 unlink performance

[Alexander Zarochentsev]

On Tuesday 28 February 2006 22:56, Hans Reiser wrote:
> Alexander Zarochentsev wrote:
> >On Tuesday 28 February 2006 22:11, Hans Reiser wrote:
> >>Alec H. Peterson wrote:
> >>>Hi Alexander,
> >>>
> >>>On Feb 26, 2006, at 13:42, Alexander Zarochentsev wrote:
> >>>>Which kernel version and reiser4 patches were used?
> >>>
> >>>2.6.15.4r4 kernel,
> >>>reiser4-for-2.6.15-1.patch
> >>>
> >>>>Is the system SMP?
> >>>
> >>>Yes.
> >>>
> >>>>No, "Flushing like mad" usually indicates a bug.
> >>
> >>Please describe it in more detail.
> >
> >The flush algorithm designed to make progress in any iteration.
> >"flushing like mad" messages usually indicates that the flush can't
> >make progress.
> Yet more detail please.

the algorithm does not work as designed, it is a design bug or an 
implementation bug ;)

> >>>Good to know.
> >>>
> >>>Thanks!
> >>>
> >>>Alec

-- 
Alex.

Re: Reiser4 unlink performance

[Hans Reiser]

Alexander Zarochentsev wrote:

>On Tuesday 28 February 2006 22:56, Hans Reiser wrote:
>  
>
>>Alexander Zarochentsev wrote:
>>    
>>
>>>On Tuesday 28 February 2006 22:11, Hans Reiser wrote:
>>>      
>>>
>>>>Alec H. Peterson wrote:
>>>>        
>>>>
>>>>>Hi Alexander,
>>>>>
>>>>>On Feb 26, 2006, at 13:42, Alexander Zarochentsev wrote:
>>>>>          
>>>>>
>>>>>>Which kernel version and reiser4 patches were used?
>>>>>>            
>>>>>>
>>>>>2.6.15.4r4 kernel,
>>>>>reiser4-for-2.6.15-1.patch
>>>>>
>>>>>          
>>>>>
>>>>>>Is the system SMP?
>>>>>>            
>>>>>>
>>>>>Yes.
>>>>>
>>>>>          
>>>>>
>>>>>>No, "Flushing like mad" usually indicates a bug.
>>>>>>            
>>>>>>
>>>>Please describe it in more detail.
>>>>        
>>>>
>>>The flush algorithm designed to make progress in any iteration.
>>>"flushing like mad" messages usually indicates that the flush can't
>>>make progress.
>>>      
>>>
>>Yet more detail please.
>>    
>>
>
>the algorithm does not work as designed, it is a design bug or an 
>implementation bug ;)
>
>  
>
I seem to remember vs saying this message could be ignored safely at
times in the past, which is why I am asking you for more details.

Hans

Re: Reiser4 unlink performance

[Alexander Zarochentsev]

On Tuesday 28 February 2006 23:42, Hans Reiser wrote:
> Alexander Zarochentsev wrote:
> >On Tuesday 28 February 2006 22:56, Hans Reiser wrote:
> >>Alexander Zarochentsev wrote:
> >>>On Tuesday 28 February 2006 22:11, Hans Reiser wrote:
> >>>>Alec H. Peterson wrote:
> >>>>>Hi Alexander,
> >>>>>
> >>>>>On Feb 26, 2006, at 13:42, Alexander Zarochentsev wrote:
> >>>>>>Which kernel version and reiser4 patches were used?
> >>>>>
> >>>>>2.6.15.4r4 kernel,
> >>>>>reiser4-for-2.6.15-1.patch
> >>>>>
> >>>>>>Is the system SMP?
> >>>>>
> >>>>>Yes.
> >>>>>
> >>>>>>No, "Flushing like mad" usually indicates a bug.
> >>>>
> >>>>Please describe it in more detail.
> >>>
> >>>The flush algorithm designed to make progress in any iteration.
> >>>"flushing like mad" messages usually indicates that the flush
> >>> can't make progress.
> >>
> >>Yet more detail please.
> >
> >the algorithm does not work as designed, it is a design bug or an
> >implementation bug ;)
>
> I seem to remember vs saying this message could be ignored safely at
> times in the past, which is why I am asking you for more details.

We have improved handling of truncate/flush races, especially if 
formatted nodes are deleted.  Now I think that any truncate/flush race 
can be resolved w/o having those too many useless iterations of 
jnode_flush.

> Hans

-- 
Alex.

Re: Reiser4 unlink performance

[Alexander Zarochentsev]

On Tuesday 28 February 2006 22:19, Alec H. Peterson wrote:
> Hi Hans,
>
> On Feb 28, 2006, at 12:11, Hans Reiser wrote:
> > Please describe it in more detail.
>
> Well, the error I'm getting is this:
>
> Feb 21 13:10:22 ecbuild-10 kernel: WARNING: Flushing like mad: 16384
>
> The C program I'm running to do the test is this:

All files created by the program below have zero lentgh, correct?
Is it the same program as which triggered 'flushing like mad' ?
 
>
> main(int argc, char **argv)
> {
>    char buf[1024];
>    char dir[1024];
>    int i,j,k;
>    int ret;
>    int fd;
>    int *sequence;
>    struct timeval start, unlinktime, end;
>
>    if(argc != 2)
>    {
>      fprintf(stderr, "Usage: %s <base directory>\n", argv[0]);
>      exit(1);
>    }
>
>    strncpy(dir, argv[1], 1024);
>
>    if(chdir(dir))
>    {
>      perror("chdir");
>      exit(1);
>    }
>
>    for(i = 0; i < 256; i++)
>    {
>      snprintf(buf, 1024, "%d", i);
>      ret = mkdir(buf, S_IRWXU);
>      if(ret && (errno != EEXIST))
>      {
>        perror("mkdir");
>        exit(1);
>      }
>    }
>
>
>    {
>      sequence = sequence20000;
>      gettimeofday(&start, NULL);
>      fprintf(stderr, "Starting %d run %d.%d.\n",
>              256 * 20000, start.tv_sec, start.tv_usec);
>
>      for(i = 0; i < 256; i++)
>      {
>        for(j = 0; j < 20000; j++)
>        {
>          snprintf(buf, 1024, "%d/%d", i, j);
>          fd = open(buf, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
>          if(fd == -1)
>          {
>            perror("open");
>            exit(1);
>          }
>          close(fd);
>        }
>        fprintf(stderr, "Done %d\n", i);
>      }
>
>      gettimeofday(&unlinktime, NULL);
>      fprintf(stderr, "Starting %d unlink %d.%d, elapsed time was
> %d.\n", 256 * 20000, unlinktime.tv_sec, unlinktime.tv_usec,
> unlinktime.tv_sec - start.tv_sec);
>
>      for(i = 0; i < 20000; i++)
>      {
>        for(j = 0; j < 256; j++)
>        {
>          snprintf(buf, 1024, "%d/%d", j, sequence[i]);
>          unlink(buf);
>          if(fd == -1)
>          {
>            perror("unlink");
>            exit(1);
>          }
>        }
>        fprintf(stderr, "Done %d\n", i);
>      }
>
>      gettimeofday(&end, NULL);
>      fprintf(stderr, "Done with %d unlink %d.%d, elapsed time was %d.
> \n",
>              20000 * 256, end.tv_sec, end.tv_usec,
>              end.tv_sec - unlinktime.tv_sec);
>    }
>
>    exit(0);
> }
>
> sequence20000 is an array which contains a sequence from 1 to 20000
> randomly ordered.
>
> Alec
>

-- 
Alex.

Re: Reiser4 unlink performance

[Alec H. Peterson]

Hi Hans,

On Mar 1, 2006, at 8:27, Alexander Zarochentsev wrote:

>
> All files created by the program below have zero lentgh, correct?
> Is it the same program as which triggered 'flushing like mad' ?

Correct.

I would be happy to modify it and re-run it however you like.  You  
are correct though that that isn't a real use case for us (0 length  
files).

Alec