Re: Random file I/O regressions in 2.6

[Ram Pai <linuxram@us.ibm.com>]

[-- Attachment #1: Type: text/plain, Size: 2111 bytes --]

On Tue, 2004-05-04 at 08:03, Ram Pai wrote:
> On Mon, 2004-05-03 at 23:29, Andrew Morton wrote:
>  
> > 
> > Putting a semaphore around do_generic_file_read() or maintaining the state
> > as below fixes it up.
> > 
> > I wonder if we should bother fixing this?  I guess as long as the app is
> > using pread() it is a legitimate thing to be doing, so I guess we should...
> > 
> > 
> > 
> Yes this patch makes sense. I have setup sysbench on my lab machine. Let
> me see how much improvement the patch provides.
I ran the following command:

/root/sysbench-0.2.5/sysbench/sysbench --num-threads=256 --test=fileio
--file-total-size=2800M --file-test-mode=rndrw run 



Without the patch:
------------------

Operations performed:  5959 Read, 4041 Write, 10752 Other = 20752 Total
Read 93Mb  Written 63Mb  Total Transferred 156Mb
   7.549Mb/sec  Transferred
  483.89 Requests/sec executed

Test execution Statistics summary:
Time spent for test:  20.6661s

no of times window reset because of hits: 0
no of times window reset because of misses: 7
no of times window was shrunk because of hits: 6716
no of times the page request was non-contiguous: 5880
no of times the page request was contiguous : 19639


With the patch:
--------------

Operations performed:  5960 Read, 4040 Write, 10880 Other = 20880 Total
Read 93Mb  Written 63Mb  Total Transferred 156Mb
   7.985Mb/sec  Transferred
   511.85 Requests/sec executed

Test execution Statistics summary:
Time spent for test:  19.5370s


no of times window got reset because of hits: 0
no of times window got reset because of misses: 0
no of times window was shrunk because of hits: 5844
no of times the page request was non-contiguous: 5830
no of times the page request was contiguous : 20232


I have enclosed the patch that collects the hit/miss related counts.

        In general I am not seeing any major difference with or without
        andrew's ra-copy patch; except for readahead window getting
        closed because of misses when run without the patch.

Would be nice if Alexey tries the patch on his machine and sees any
major difference.

RP





[-- Attachment #2: ra_instrumentation.patch --]
[-- Type: text/x-patch, Size: 4017 bytes --]

diff -urNp linux-2.6.6-rc3/include/linux/sysctl.h linux-2.6.6-rc3.new/include/linux/sysctl.h
--- linux-2.6.6-rc3/include/linux/sysctl.h	2004-04-27 18:35:49.000000000 -0700
+++ linux-2.6.6-rc3.new/include/linux/sysctl.h	2004-05-04 18:26:37.911973080 -0700
@@ -643,6 +643,11 @@ enum
 	FS_XFS=17,	/* struct: control xfs parameters */
 	FS_AIO_NR=18,	/* current system-wide number of aio requests */
 	FS_AIO_MAX_NR=19,	/* system-wide maximum number of aio requests */
+	FS_READ_MISS_RESET=20,
+	FS_READ_HIT_RESET=21,
+	FS_CONTIGUOUS_CNT=22,
+	FS_NON_CONTIGUOUS_CNT=22,
+	FS_HIT_COUNT=23,
 };
 
 /* /proc/sys/fs/quota/ */
diff -urNp linux-2.6.6-rc3/kernel/sysctl.c linux-2.6.6-rc3.new/kernel/sysctl.c
--- linux-2.6.6-rc3/kernel/sysctl.c	2004-04-27 18:35:08.000000000 -0700
+++ linux-2.6.6-rc3.new/kernel/sysctl.c	2004-05-04 18:38:58.774344880 -0700
@@ -64,6 +64,13 @@ extern int sysctl_lower_zone_protection;
 extern int min_free_kbytes;
 extern int printk_ratelimit_jiffies;
 extern int printk_ratelimit_burst;
+extern int printk_ratelimit_burst;
+extern int printk_ratelimit_burst;
+extern atomic_t hit_reset;
+extern atomic_t miss_reset;
+extern atomic_t hit_count;
+extern atomic_t contiguous_cnt;
+extern atomic_t non_contiguous_cnt;
 
 /* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */
 static int maxolduid = 65535;
@@ -897,6 +904,46 @@ static ctl_table fs_table[] = {
 		.mode		= 0644,
 		.proc_handler	= &proc_dointvec,
 	},
+	{
+		.ctl_name	= FS_READ_MISS_RESET,
+		.procname	= "read-miss-reset",
+		.data		= &miss_reset,
+		.maxlen		= sizeof(miss_reset),
+		.mode		= 0444,
+		.proc_handler	= &proc_dointvec,
+	},
+	{
+		.ctl_name	= FS_READ_HIT_RESET,
+		.procname	= "read-hit-reset",
+		.data		= &hit_reset,
+		.maxlen		= sizeof(hit_reset),
+		.mode		= 0444,
+		.proc_handler	= &proc_dointvec,
+	},
+	{
+		.ctl_name	= FS_CONTIGUOUS_CNT,
+		.procname	= "read-contiguous-cnt",
+		.data		= &contiguous_cnt,
+		.maxlen		= sizeof(contiguous_cnt),
+		.mode		= 0444,
+		.proc_handler	= &proc_dointvec,
+	},
+	{
+		.ctl_name	= FS_NON_CONTIGUOUS_CNT,
+		.procname	= "read-non-contiguous-cnt",
+		.data		= &non_contiguous_cnt,
+		.maxlen		= sizeof(non_contiguous_cnt),
+		.mode		= 0444,
+		.proc_handler	= &proc_dointvec,
+	},
+	{
+		.ctl_name	= FS_HIT_COUNT,
+		.procname	= "read-hit-count",
+		.data		= &hit_count,
+		.maxlen		= sizeof(hit_count),
+		.mode		= 0444,
+		.proc_handler	= &proc_dointvec,
+	},
 	{ .ctl_name = 0 }
 };
 
diff -urNp linux-2.6.6-rc3/mm/readahead.c linux-2.6.6-rc3.new/mm/readahead.c
--- linux-2.6.6-rc3/mm/readahead.c	2004-04-27 18:35:06.000000000 -0700
+++ linux-2.6.6-rc3.new/mm/readahead.c	2004-05-04 18:37:20.681257296 -0700
@@ -316,6 +316,12 @@ int do_page_cache_readahead(struct addre
 	return 0;
 }
 
+atomic_t hit_reset= ATOMIC_INIT(0);
+atomic_t miss_reset= ATOMIC_INIT(0);
+atomic_t hit_count= ATOMIC_INIT(0);
+atomic_t contiguous_cnt = ATOMIC_INIT(0);
+atomic_t non_contiguous_cnt= ATOMIC_INIT(0);
+
 /*
  * Check how effective readahead is being.  If the amount of started IO is
  * less than expected then the file is partly or fully in pagecache and
@@ -331,11 +337,13 @@ check_ra_success(struct file_ra_state *r
 	if (actual == 0) {
 		if (orig_next_size > 1) {
 			ra->next_size = orig_next_size - 1;
+			atomic_inc(&hit_count);
 			if (ra->ahead_size)
 				ra->ahead_size = ra->next_size;
 		} else {
 			ra->next_size = -1UL;
 			ra->size = 0;
+			atomic_inc(&hit_reset);
 		}
 	}
 }
@@ -406,17 +414,20 @@ page_cache_readahead(struct address_spac
 		 * page beyond the end.  Expand the next readahead size.
 		 */
 		ra->next_size += 2;
+		atomic_inc(&contiguous_cnt);
 	} else {
 		/*
 		 * A miss - lseek, pagefault, pread, etc.  Shrink the readahead
 		 * window.
 		 */
 		ra->next_size -= 2;
+		atomic_inc(&non_contiguous_cnt);
 	}
 
 	if ((long)ra->next_size > (long)max)
 		ra->next_size = max;
 	if ((long)ra->next_size <= 0L) {
+		atomic_inc(&miss_reset);
 		ra->next_size = -1UL;
 		ra->size = 0;
 		goto out;		/* Readahead is off */