[PATCH 00/14] Adaptive readahead update

[PATCH 00/14] Adaptive readahead update

[Fengguang Wu]

Andrew,

Here is one more adaptive readahead update.

It fixes:
                - state based readahead
                - thrashing recovery readahead
and cleanups:
                - call scheme
                - events/accounting
		- rescue_pages()

Still in the todo list are:
                - convert to Martin's statistic infrastructure
                - a replacement of the stock readahead
                  to be simple, reliable and fast

Regards,
Fengguang Wu
---

Recommended patch ordering:

--- broken-out/series	2007-03-08 11:45:54.000000000 +0800
+++ patches/series	2007-03-16 12:08:17.000000000 +0800
@@ -993,6 +993,9 @@ readahead-min-max-sizes.patch
 readahead-state-based-method-aging-accounting.patch
 readahead-state-based-method-routines.patch
 readahead-state-based-method.patch
+readahead-state-based-method-check-node-id.patch
+readahead-state-based-method-decouple-readahead_ratio-from-growth_limit.patch
+readahead-state-based-method-cancel-lookahead-gracefully.patch
 readahead-context-based-method.patch
 readahead-initial-method-guiding-sizes.patch
 readahead-initial-method-thrashing-guard-size.patch
@@ -1000,10 +1003,21 @@ readahead-initial-method-user-recommende
 readahead-initial-method.patch
 readahead-backward-prefetching-method.patch
 readahead-thrashing-recovery-method.patch
+readahead-thrashing-recovery-method-check-unbalanced-aging.patch
+readahead-thrashing-recovery-method-refill-holes.patch
 readahead-call-scheme.patch
+readahead-call-scheme-cleanup.patch
+readahead-call-scheme-catch-thrashing-on-lookahead-time.patch
 readahead-laptop-mode.patch
 readahead-loop-case.patch
 readahead-nfsd-case.patch
+readahead-remove-parameter-ra_max-from-thrashing_recovery_readahead.patch
+readahead-remove-parameter-ra_max-from-adjust_rala.patch
+readahead-state-based-method-protect-against-tiny-size.patch
+readahead-rename-state_based_readahead-to-clock_based_readahead.patch
+readahead-account-io-block-times-for-stock-readahead.patch
+readahead-rescue_pages-updates.patch
+readahead-remove-noaction-shrink-events.patch
 readahead-remove-size-limit-on-read_ahead_kb.patch
 readahead-remove-size-limit-of-max_sectors_kb-on-read_ahead_kb.patch
 readahead-partial-sendfile-fix.patch

[PATCH 01/14] readahead: state based method: check node id

[Fengguang Wu]

[-- Attachment #1: readahead-state-based-method-check-node-id.patch --]
[-- Type: text/plain, Size: 3060 bytes --]

The file_ra_state.age is node specific,
comparing ages between two nodes is a bug.

So add code to
- take down the node id in file_ra_state.flags;
- ensure that we are comparing the ages from the same node.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 include/linux/fs.h |    2 +-
 mm/readahead.c     |   30 +++++++++++++++++++++++++-----
 2 files changed, 26 insertions(+), 6 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/include/linux/fs.h
+++ linux-2.6.21-rc3-mm2/include/linux/fs.h
@@ -738,7 +738,7 @@ struct file_ra_state {
 	unsigned long mmap_hit;		/* Cache hit stat for mmap accesses */
 	unsigned long mmap_miss;	/* Cache miss stat for mmap accesses */
 
-	unsigned long flags;	/* RA_FLAG_xxx | ra_class_old | ra_class_new */
+	unsigned long flags;	/* RA_FLAG_xxx | node_id | class_old | class_new */
 	unsigned long prev_page;	/* Cache last read() position */
 	unsigned long ra_pages;		/* Maximum readahead window */
 };
--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -52,11 +52,13 @@ EXPORT_SYMBOL_GPL(readahead_ratio);
 int readahead_hit_rate = 1;
 #endif /* CONFIG_ADAPTIVE_READAHEAD */
 
+#define RA_CLASS_SHIFT 4
+#define RA_CLASS_MASK  ((1 << RA_CLASS_SHIFT) - 1)
+#define RA_NODE_SHIFT  (2 * RA_CLASS_SHIFT)
+#define RA_NODE_MASK   ((MAX_NUMNODES-1) << RA_NODE_SHIFT)
 /*
  * Detailed classification of read-ahead behaviors.
  */
-#define RA_CLASS_SHIFT 4
-#define RA_CLASS_MASK  ((1 << RA_CLASS_SHIFT) - 1)
 enum ra_class {
 	RA_CLASS_ALL,
 	RA_CLASS_INITIAL,
@@ -802,6 +804,11 @@ static inline enum ra_class ra_class_old
 	return (ra->flags >> RA_CLASS_SHIFT) & RA_CLASS_MASK;
 }
 
+static inline int ra_node_id(struct file_ra_state *ra)
+{
+	return (ra->flags >> RA_NODE_SHIFT) & RA_NODE_MASK;
+}
+
 static unsigned long ra_readahead_size(struct file_ra_state *ra)
 {
 	return ra->readahead_index - ra->ra_index;
@@ -864,6 +871,18 @@ static void ra_set_size(struct file_ra_s
 }
 
 /*
+ * Save the current node id and age.
+ */
+static void ra_save_node_age(struct file_ra_state *ra)
+{
+	int nid = numa_node_id();
+
+	ra->flags &= ~RA_NODE_MASK;
+	ra->flags |= nid << RA_NODE_SHIFT;
+	ra->age = nr_scanned_pages_node(nid);
+}
+
+/*
  * Submit IO for the read-ahead request in file_ra_state.
  */
 static unsigned long ra_submit(struct file_ra_state *ra,
@@ -901,7 +920,7 @@ static unsigned long ra_submit(struct fi
 	}
 
 	/* Take down the current read-ahead aging value. */
-	ra->age = nr_scanned_pages_node(numa_node_id());
+	ra_save_node_age(ra);
 
 	ra_size = ra_readahead_size(ra);
 	la_size = ra_lookahead_size(ra);
@@ -1025,9 +1044,10 @@ static unsigned long compute_thrashing_t
 	unsigned long stream_shift;
 	unsigned long ra_size;
 	uint64_t ll;
+	int nid = ra_node_id(ra);
 
-	global_size = nr_free_inactive_pages_node(numa_node_id());
-	global_shift = nr_scanned_pages_node(numa_node_id()) - ra->age;
+	global_size = nr_free_inactive_pages_node(nid);
+	global_shift = nr_scanned_pages_node(nid) - ra->age;
 	global_shift |= 1UL;
 	stream_shift = ra_invoke_interval(ra);
 

--

[PATCH 02/14] readahead: state based method: decouple readahead_ratio from growth_limit

[Fengguang Wu]

[-- Attachment #1: readahead-state-based-method-decouple-readahead_ratio-from-growth_limit.patch --]
[-- Type: text/plain, Size: 1129 bytes --]

Remove readahead_ratio from the growth_limit computing in state based method.

It simplifies the size ramp up rules to:

        ahead_window_size = request_size +
                            current_window_size * 2 +
                            readahead_max / 16;

The first two sizes are apparent. The last one has two effects:

1) Ensure that we get acceptable readahead size in early start up phase.
   It is an analog to the old 'first x4 then x2' trick.

2) Ensure that we reach readahead_max within 8 steps.
   It is good for the laptop mode. A user may set a huge readahead_max, and
   expect the kernel to do large I/Os right away.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1099,7 +1099,7 @@ state_based_readahead(struct address_spa
 
 	growth_limit = req_size;
 	growth_limit += ra_max / 16;
-	growth_limit += (2 + readahead_ratio / 64) * ra_old;
+	growth_limit += 2 * ra_old;
 	if (growth_limit > ra_max)
 	    growth_limit = ra_max;
 

--

[PATCH 03/14] readahead: state based method: cancel lookahead gracefully

[Fengguang Wu]

[-- Attachment #1: readahead-state-based-method-cancel-lookahead-gracefully.patch --]
[-- Type: text/plain, Size: 1080 bytes --]

When canceling lookahead, update ra->lookahead_index to the next invoke index.
So that we see a consistent state in the next time.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |    8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1094,7 +1094,7 @@ state_based_readahead(struct address_spa
 
 	if (page && remain_space <= la_size) {
 		rescue_pages(page, la_size);
-		return 0;
+		goto cancel_lookahead;
 	}
 
 	growth_limit = req_size;
@@ -1104,7 +1104,7 @@ state_based_readahead(struct address_spa
 	    growth_limit = ra_max;
 
 	if (!adjust_rala(growth_limit, &ra_size, &la_size))
-		return 0;
+		goto cancel_lookahead;
 
 	limit_rala(growth_limit, la_old, &ra_size, &la_size);
 
@@ -1113,6 +1113,10 @@ state_based_readahead(struct address_spa
 	ra_set_size(ra, ra_size, la_size);
 
 	return ra_submit(ra, mapping, filp);
+
+cancel_lookahead:
+	ra->lookahead_index = ra->readahead_index;
+	return 0;
 }
 
 #endif /* CONFIG_ADAPTIVE_READAHEAD */

--

[PATCH 04/14] readahead: thrashing recovery method: check unbalanced aging

[Fengguang Wu]

[-- Attachment #1: readahead-thrashing-recovery-method-check-unbalanced-aging.patch --]
[-- Type: text/plain, Size: 1177 bytes --]

Always check for unbalanced aging in thrashing_recovery_readahead(), and
account RA_EVENT_READAHEAD_MUTILATE/RA_EVENT_READAHEAD_THRASHING events.

Unbalanced zone/node aging can come from abnormal system loads, or
misconfigured NUMA policies.

Thrashings are rare events, hence the cost of probe_page() is not a concern.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |   11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1546,14 +1546,11 @@ thrashing_recovery_readahead(struct addr
 				pgoff_t offset, unsigned long ra_max)
 {
 	unsigned long ra_size;
+	int unbalanced_aging = probe_page(mapping, offset - 1);
 
-#ifdef CONFIG_DEBUG_READAHEAD
-	if (probe_page(mapping, offset - 1))
-		ra_account(ra, RA_EVENT_READAHEAD_MUTILATE,
-						ra->readahead_index - offset);
-	ra_account(ra, RA_EVENT_READAHEAD_THRASHING,
-						ra->readahead_index - offset);
-#endif
+	ra_account(ra, unbalanced_aging ? RA_EVENT_READAHEAD_MUTILATE :
+					  RA_EVENT_READAHEAD_THRASHING,
+					  ra->readahead_index - offset);
 
 	if (offset < ra->ra_index) {
 		/*

--

[PATCH 05/14] readahead: thrashing recovery method: refill holes

[Fengguang Wu]

[-- Attachment #1: readahead-thrashing-recovery-method-refill-holes.patch --]
[-- Type: text/plain, Size: 2107 bytes --]

When thrashing happened in the following abnormal cases:
	1) the old chunk is lost;
	2) random pages are lost due to unbalanced aging.
There will be hole(s) in the otherwise continuous readahead pages.

We recover from it by simply refilling all possible holes and
turning off lookahead.

These kind of abnormal situations are not expected to repeatable for the same
stream. They happen in a random fashion and do not tell much about the system
load. So the best thing we can do is to do nothing more than refilling the
holes.  If thrashing persists, it will quickly fail into the 3) case where we
get the exact thrashing threshold:
	3) the new chunk is thrashed.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |   19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1552,20 +1552,23 @@ thrashing_recovery_readahead(struct addr
 					  RA_EVENT_READAHEAD_THRASHING,
 					  ra->readahead_index - offset);
 
-	if (offset < ra->ra_index) {
+	if (offset < ra->ra_index || unbalanced_aging) {
 		/*
-		 * Thrashed when we are in [la_index, ra_index), i.e.
-		 * the old chunk is lost soon after the new one is allocated.
-		 * Ensure that we recover all needed pages in the old chunk.
-		 * And futher keep the lookahead_index untouched.
+		 * 1) The old chunk is lost.
+		 * 2) Some random pages are lost due to unbalanced zone/node aging.
+		 * Refill the hole(s).
+		 * Further thrashings will bring us back to case (3) below.
 		 */
-		ra_size = ra->lookahead_index - offset;
+		ra_size = ra->readahead_index - offset;
 	} else {
-		/* After thrashing, we know the exact thrashing-threshold. */
+		/*
+		 * 3) The new chunk is lost.
+		 * It tells us about the thrashing-threshold.
+		 */
 		ra_size = offset - ra->la_index;
 		update_ra_thrash_bytes(mapping->backing_dev_info, ra_size);
 
-		/* And be cooperative: the system may be hunting for memory. */
+		/* Be cooperative: the system may be hunting for memory. */
 		ra_size = MIN_RA_PAGES + ra_size / 2;
 	}
 

--

[PATCH 06/14] readahead: call scheme: cleanup

[Fengguang Wu]

[-- Attachment #1: readahead-call-scheme-cleanup.patch --]
[-- Type: text/plain, Size: 1597 bytes --]

Merge two similar page_cache_readahead_adaptive() calls into one.

No behavior change.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/filemap.c |   18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/filemap.c
+++ linux-2.6.21-rc3-mm2/mm/filemap.c
@@ -934,17 +934,13 @@ void do_generic_mapping_read(struct addr
 find_page:
 		page = find_get_page(mapping, index);
 		if (prefer_adaptive_readahead()) {
-			if (unlikely(page == NULL)) {
-				ra.prev_page = prev_index;
-				page_cache_readahead_adaptive(mapping,
-						&ra, filp, NULL,
-						index, last_index - index);
-				page = find_get_page(mapping, index);
-			} else if (PageReadahead(page)) {
+			if (!page || PageReadahead(page)) {
 				ra.prev_page = prev_index;
 				page_cache_readahead_adaptive(mapping,
 						&ra, filp, page,
 						index, last_index - index);
+				if (!page)
+					page = find_get_page(mapping, index);
 			}
 		}
 		if (unlikely(page == NULL)) {
@@ -1394,13 +1390,11 @@ struct page *filemap_fault(struct vm_are
 retry_find:
 	page = find_lock_page(mapping, fdata->pgoff);
 	if (prefer_adaptive_readahead() && VM_SequentialReadHint(vma)) {
-		if (!page) {
-			page_cache_readahead_adaptive(mapping, ra, file, NULL,
-							fdata->pgoff, 1);
-			page = find_lock_page(mapping, fdata->pgoff);
-		} else if (PageReadahead(page)) {
+		if (!page || PageReadahead(page)) {
 			page_cache_readahead_adaptive(mapping, ra, file, page,
 							fdata->pgoff, 1);
+			if (!page)
+				page = find_lock_page(mapping, fdata->pgoff);
 		}
 	}
 	if (!page) {

--

[PATCH 07/14] readahead: call scheme: catch thrashing on lookahead time

[Fengguang Wu]

[-- Attachment #1: readahead-call-scheme-catch-thrashing-on-lookahead-time.patch --]
[-- Type: text/plain, Size: 1368 bytes --]

Move the call to thrashing_recovery_readahead() before state_based_readahead().

That catches the rare case where thrashing happened on the time we are to read
the page at ra->lookahead_index. Obviously this case should be handled by
thrashing_recovery_readahead() instead of state_based_readahead().

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |   14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1638,6 +1638,13 @@ page_cache_readahead_adaptive(struct add
 		return initial_readahead(mapping, filp, ra, req_size);
 
 	/*
+	 * Recover from possible thrashing.
+	 */
+	if (!page && offset - ra->prev_page <= 1 && ra_has_index(ra, offset))
+		return thrashing_recovery_readahead(mapping, filp, ra,
+								offset, ra_max);
+
+	/*
 	 * State based sequential read-ahead.
 	 */
 	if (offset == ra->prev_page + 1 &&
@@ -1647,13 +1654,6 @@ page_cache_readahead_adaptive(struct add
 						offset, req_size, ra_max);
 
 	/*
-	 * Recover from possible thrashing.
-	 */
-	if (!page && offset - ra->prev_page <= 1 && ra_has_index(ra, offset))
-		return thrashing_recovery_readahead(mapping, filp, ra,
-								offset, ra_max);
-
-	/*
 	 * Backward read-ahead.
 	 */
 	if (!page && try_backward_prefetching(ra, offset, req_size, ra_max))

--

[PATCH 08/14] readahead: remove parameter ra_max from thrashing_recovery_readahead()

[Fengguang Wu]

[-- Attachment #1: readahead-remove-parameter-ra_max-from-thrashing_recovery_readahead.patch --]
[-- Type: text/plain, Size: 1162 bytes --]

Remove the unused @ra_max from thrashing_recovery_readahead().

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |    8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1567,9 +1567,8 @@ try_backward_prefetching(struct file_ra_
  * Readahead thrashing recovery.
  */
 static unsigned long
-thrashing_recovery_readahead(struct address_space *mapping,
-				struct file *filp, struct file_ra_state *ra,
-				pgoff_t offset, unsigned long ra_max)
+thrashing_recovery_readahead(struct address_space *mapping, struct file *filp,
+			     struct file_ra_state *ra, pgoff_t offset)
 {
 	unsigned long ra_size;
 	int unbalanced_aging = probe_page(mapping, offset - 1);
@@ -1682,8 +1681,7 @@ page_cache_readahead_adaptive(struct add
 	 * Recover from possible thrashing.
 	 */
 	if (!page && offset - ra->prev_page <= 1 && ra_has_index(ra, offset))
-		return thrashing_recovery_readahead(mapping, filp, ra,
-								offset, ra_max);
+		return thrashing_recovery_readahead(mapping, filp, ra, offset);
 
 	/*
 	 * State based sequential read-ahead.

--

[PATCH 09/14] readahead: remove parameter ra_max from adjust_rala*()

[Fengguang Wu]

[-- Attachment #1: readahead-remove-parameter-ra_max-from-adjust_rala.patch --]
[-- Type: text/plain, Size: 2120 bytes --]

Remove the unused parameter @ra_max from adjust_rala() and
adjust_rala_aggressive().

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |   17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1000,8 +1000,7 @@ static unsigned long ra_submit(struct fi
  *	- @ra_size stores the estimated thrashing-threshold.
  *	- @la_size stores the look-ahead size of previous request.
  */
-static int adjust_rala(unsigned long ra_max,
-			unsigned long *ra_size, unsigned long *la_size)
+static int adjust_rala(unsigned long *ra_size, unsigned long *la_size)
 {
 	/*
 	 * Cancel asynchrous read-ahead,
@@ -1143,15 +1142,15 @@ state_based_readahead(struct address_spa
 		goto cancel_lookahead;
 	}
 
+	if (!adjust_rala(&ra_size, &la_size))
+		goto cancel_lookahead;
+
 	growth_limit = req_size;
 	growth_limit += ra_max / 16;
 	growth_limit += 2 * ra_old;
 	if (growth_limit > ra_max)
 	    growth_limit = ra_max;
 
-	if (!adjust_rala(growth_limit, &ra_size, &la_size))
-		goto cancel_lookahead;
-
 	limit_rala(growth_limit, la_old, &ra_size, &la_size);
 
 	/* ra_size in its _steady_ state reflects thrashing threshold */
@@ -1345,8 +1344,8 @@ out_unlock:
  * which is safe: the tailing look-ahead part is 'unsafe'. However it will be
  * safeguarded by rescue_pages() when the previous chunks are lost.
  */
-static void adjust_rala_aggressive(unsigned long ra_max,
-				unsigned long *ra_size, unsigned long *la_size)
+static void adjust_rala_aggressive(unsigned long *ra_size,
+				   unsigned long *la_size)
 {
 	pgoff_t offset = *ra_size;
 
@@ -1463,12 +1462,12 @@ has_history_pages:
 
 	if (ra_size >= offset) {
 		ra_size = offset;
-		adjust_rala_aggressive(ra_max, &ra_size, &la_size);
+		adjust_rala_aggressive(&ra_size, &la_size);
 		ra_set_class(ra, RA_CLASS_CONTEXT_AGGRESSIVE);
 	} else {
 		if (ra_size < ra_min)
 		    ra_size = ra_min;
-		if (!adjust_rala(ra_max, &ra_size, &la_size))
+		if (!adjust_rala(&ra_size, &la_size))
 			return -1;
 		ra_set_class(ra, RA_CLASS_CONTEXT);
 	}

--

[PATCH 10/14] readahead: state based method: protect against tiny size

[Fengguang Wu]

[-- Attachment #1: readahead-state-based-method-protect-against-tiny-size.patch --]
[-- Type: text/plain, Size: 1159 bytes --]

Move the tiny I/O size protection code from limit_rala() to
state_based_readahead().

limit_rala() is also called by context based readahead, which has its own
minimal readahead size.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |   14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1031,13 +1031,6 @@ static void limit_rala(unsigned long ra_
 	unsigned long stream_shift;
 
 	/*
-	 * Protect against too small I/O sizes,
-	 * by mapping [0, 4*min] to [min, 4*min].
-	 */
-	if (*ra_size < 4 * MIN_RA_PAGES)
-		*ra_size = MIN_RA_PAGES + *ra_size * 3 / 4;
-
-	/*
 	 * Apply basic upper limits.
 	 */
 	if (*ra_size > ra_max)
@@ -1145,6 +1138,13 @@ state_based_readahead(struct address_spa
 	if (!adjust_rala(&ra_size, &la_size))
 		goto cancel_lookahead;
 
+	/*
+	 * Protect against too small I/O sizes,
+	 * by mapping [0, 4*min] to [min, 4*min].
+	 */
+	if (ra_size < 4 * MIN_RA_PAGES)
+		ra_size = MIN_RA_PAGES + ra_size * 3 / 4;
+
 	growth_limit = req_size;
 	growth_limit += ra_max / 16;
 	growth_limit += 2 * ra_old;

--

[PATCH 11/14] readahead: rename state_based_readahead() to clock_based_readahead()

[Fengguang Wu]

[-- Attachment #1: readahead-rename-state_based_readahead-to-clock_based_readahead.patch --]
[-- Type: text/plain, Size: 2662 bytes --]

Rename state_based_readahead() to clock_based_readahead().
It better reflects the property of that readahead method.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |   18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -66,7 +66,7 @@ int readahead_hit_rate = 1;
 enum ra_class {
 	RA_CLASS_ALL,
 	RA_CLASS_INITIAL,
-	RA_CLASS_STATE,
+	RA_CLASS_CLOCK,
 	RA_CLASS_CONTEXT,
 	RA_CLASS_CONTEXT_AGGRESSIVE,
 	RA_CLASS_BACKWARD,
@@ -101,7 +101,7 @@ enum ra_event {
 
 #ifdef CONFIG_DEBUG_READAHEAD
 static u32 readahead_debug_level = 1;
-static u32 disable_stateful_method;
+static u32 disable_clock_readahead;
 static const char * const ra_class_name[];
 static void ra_account(struct file_ra_state *ra, enum ra_event e, int pages);
 #  define debug_inc(var)		do { var++; } while (0)
@@ -1115,7 +1115,7 @@ static unsigned long compute_thrashing_t
  * Main function for file_ra_state based read-ahead.
  */
 static unsigned long
-state_based_readahead(struct address_space *mapping, struct file *filp,
+clock_based_readahead(struct address_space *mapping, struct file *filp,
 			struct file_ra_state *ra,
 			struct page *page, pgoff_t offset,
 			unsigned long req_size, unsigned long ra_max)
@@ -1157,7 +1157,7 @@ state_based_readahead(struct address_spa
 	if (page && ra_old + ra_old / 8 >= ra_size)
 		update_ra_thrash_bytes(mapping->backing_dev_info, ra_size);
 
-	ra_set_class(ra, RA_CLASS_STATE);
+	ra_set_class(ra, RA_CLASS_CLOCK);
 	ra_set_index(ra, offset, ra->readahead_index);
 	ra_set_size(ra, ra_size, la_size);
 
@@ -1687,8 +1687,8 @@ page_cache_readahead_adaptive(struct add
 	 */
 	if (offset == ra->prev_page + 1 &&
 	    offset == ra->lookahead_index &&
-					!debug_option(disable_stateful_method))
-		return state_based_readahead(mapping, filp, ra, page,
+					!debug_option(disable_clock_readahead))
+		return clock_based_readahead(mapping, filp, ra, page,
 						offset, req_size, ra_max);
 
 	/*
@@ -1788,7 +1788,7 @@ void readahead_cache_hit(struct file_ra_
 static const char * const ra_class_name[] = {
 	"total",
 	"initial",
-	"state",
+	"clock",
 	"context",
 	"contexta",
 	"backward",
@@ -1970,8 +1970,8 @@ static int __init readahead_init(void)
 	debugfs_create_file("events", 0644, root, NULL, &ra_events_fops);
 
 	debugfs_create_u32("debug_level", 0644, root, &readahead_debug_level);
-	debugfs_create_bool("disable_stateful_method", 0644, root,
-			    &disable_stateful_method);
+	debugfs_create_bool("disable_clock_readahead", 0644, root,
+			    &disable_clock_readahead);
 
 	return 0;
 }

--

[PATCH 12/14] readahead: account I/O block times for stock readahead

[Fengguang Wu]

[-- Attachment #1: readahead-account-io-block-times-for-stock-readahead.patch --]
[-- Type: text/plain, Size: 704 bytes --]

Account I/O block times for the stock readahead, too.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>

---
 mm/readahead.c |    6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -1756,15 +1756,15 @@ EXPORT_SYMBOL_GPL(page_cache_readahead_a
  */
 void readahead_cache_hit(struct file_ra_state *ra, struct page *page)
 {
-	if (!prefer_adaptive_readahead())
-		return;
-
 	if (PageActive(page) || PageReferenced(page))
 		return;
 
 	if (!PageUptodate(page))
 		ra_account(ra, RA_EVENT_IO_BLOCK, 1);
 
+	if (!prefer_adaptive_readahead())
+		return;
+
 	if (!ra_has_index(ra, page->index))
 		return;
 

--

[PATCH 13/14] readahead: rescue_pages() updates

[Fengguang Wu]

[-- Attachment #1: readahead-rescue_pages-updates.patch --]
[-- Type: text/plain, Size: 3876 bytes --]

- Replace @page with @mapping and @index,
  which is a more usable interface.

- Add a new parameter @ra to rescue_pages(),
  which enables detailed accounting for individual readahead methods.

- Scan all pages in the range, instead of bailing out on first uncached page.

- ClearPageReadahead() on each page.
  It's harmful to have any lookahead marks when thrashing is pending.

- Add a call to rescue_pages() in thrashing_recovery_readahead().

Stress tested in qemu.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>
---
 mm/readahead.c |   67 +++++++++++++++++++++++------------------------
 1 file changed, 34 insertions(+), 33 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -731,60 +731,60 @@ unsigned long max_sane_readahead(unsigne
 /*
  * Move pages in danger (of thrashing) to the head of inactive_list.
  * Not expected to happen frequently.
- *
- * @page will be skipped: it's grabbed and won't die away.
- * The following @nr_pages-1 pages will be protected.
  */
-static unsigned long rescue_pages(struct page *page, unsigned long nr_pages)
+static unsigned long rescue_pages(struct address_space *mapping,
+				  struct file_ra_state *ra,
+				  pgoff_t index, unsigned long nr_pages)
 {
-	int pgrescue = 0;
-	pgoff_t index = page_index(page);
-	struct address_space *mapping = page_mapping(page);
-	struct page *grabbed_page = NULL;
+	struct page *grabbed_page;
+	struct page *page;
 	struct zone *zone;
+	int pgrescue = 0;
 
-	dprintk("rescue_pages(ino=%lu, index=%lu nr=%lu)\n",
+	dprintk("rescue_pages(ino=%lu, index=%lu, nr=%lu)\n",
 			mapping->host->i_ino, index, nr_pages);
 
-	for(;;) {
+	for(; nr_pages;) {
+		grabbed_page = page = find_get_page(mapping, index);
+		if (!page) {
+			index++;
+			nr_pages--;
+			continue;
+		}
+
 		zone = page_zone(page);
 		spin_lock_irq(&zone->lru_lock);
 
-		if (!PageLRU(page))
-			goto out_unlock;
+		if (!PageLRU(page)) {
+			index++;
+			nr_pages--;
+			goto next_unlock;
+		}
 
-		while (page_mapping(page) == mapping &&
-				page_index(page) == index) {
+		do {
 			struct page *the_page = page;
 			page = list_entry((page)->lru.prev, struct page, lru);
+			index++;
+			nr_pages--;
+			ClearPageReadahead(the_page);
 			if (!PageActive(the_page) &&
 					!PageLocked(the_page) &&
 					page_count(the_page) == 1) {
 				list_move(&the_page->lru, &zone->inactive_list);
 				pgrescue++;
 			}
-			index++;
-			if (!--nr_pages)
-				goto out_unlock;
-		}
+		} while (nr_pages &&
+				page_mapping(page) == mapping &&
+				page_index(page) == index);
 
+next_unlock:
 		spin_unlock_irq(&zone->lru_lock);
+		page_cache_release(grabbed_page);
 		cond_resched();
-
-		if (grabbed_page)
-			page_cache_release(grabbed_page);
-		grabbed_page = page = find_get_page(mapping, index);
-		if (!page)
-			goto out;
 	}
 
-out_unlock:
-	spin_unlock_irq(&zone->lru_lock);
-out:
-	if (grabbed_page)
-		page_cache_release(grabbed_page);
-	ra_account(NULL, RA_EVENT_READAHEAD_RESCUE, pgrescue);
-	return nr_pages;
+	ra_account(ra, RA_EVENT_READAHEAD_RESCUE, pgrescue);
+	return pgrescue;
 }
 
 /*
@@ -1131,7 +1131,7 @@ clock_based_readahead(struct address_spa
 	ra_size = ra_size * readahead_ratio / 100;
 
 	if (page && remain_space <= la_size) {
-		rescue_pages(page, la_size);
+		rescue_pages(mapping, ra, offset, la_size);
 		goto cancel_lookahead;
 	}
 
@@ -1456,7 +1456,7 @@ has_history_pages:
 	la_size = start - offset;
 	if (page && ra_size < la_size) {
 		if (ra_size < offset)
-			rescue_pages(page, la_size);
+			rescue_pages(mapping, ra, offset, la_size);
 		return -1;
 	}
 
@@ -1584,6 +1584,7 @@ thrashing_recovery_readahead(struct addr
 		 * Further thrashings will bring us back to case (3) below.
 		 */
 		ra_size = ra->readahead_index - offset;
+		rescue_pages(mapping, ra, offset, ra_size);
 	} else {
 		/*
 		 * 3) The new chunk is lost.

--

[PATCH 14/14] readahead: remove noaction/shrink events

[Fengguang Wu]

[-- Attachment #1: readahead-remove-noaction-shrink-events.patch --]
[-- Type: text/plain, Size: 2851 bytes --]

Tear down the following ra_account(NULL, ...) calls:

	ra_account(RA_EVENT_LOOKAHEAD_NOACTION)
	ra_account(RA_EVENT_READAHEAD_SHRINK)

The two events are not as useful and do not have clear meanings.

Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn>

---
 mm/readahead.c |   20 +++-----------------
 1 file changed, 3 insertions(+), 17 deletions(-)

--- linux-2.6.21-rc3-mm2.orig/mm/readahead.c
+++ linux-2.6.21-rc3-mm2/mm/readahead.c
@@ -87,10 +87,8 @@ enum ra_event {
 	RA_EVENT_READAHEAD_HIT,		/* read-ahead page hit */
 	RA_EVENT_LOOKAHEAD,		/* look-ahead issued */
 	RA_EVENT_LOOKAHEAD_HIT,		/* look-ahead mark hit */
-	RA_EVENT_LOOKAHEAD_NOACTION,	/* look-ahead mark ignored */
 	RA_EVENT_READAHEAD_MMAP,	/* read-ahead for mmap access */
 	RA_EVENT_READAHEAD_EOF,		/* read-ahead reaches EOF */
-	RA_EVENT_READAHEAD_SHRINK,	/* ra_size falls under previous la_size */
 	RA_EVENT_READAHEAD_THRASHING,	/* read-ahead thrashing happened */
 	RA_EVENT_READAHEAD_MUTILATE,	/* read-ahead mutilated by imbalanced aging */
 	RA_EVENT_READAHEAD_RESCUE,	/* read-ahead rescued */
@@ -452,8 +450,6 @@ int force_page_cache_readahead(struct ad
 		nr_to_read -= this_chunk;
 	}
 
-	ra_account(NULL, RA_EVENT_READAHEAD, ret);
-
 	return ret;
 }
 
@@ -496,8 +492,6 @@ int do_page_cache_readahead(struct addre
 
 	ret = __do_page_cache_readahead(mapping, filp, offset, nr_to_read, 0);
 
-	ra_account(NULL, RA_EVENT_READAHEAD, ret);
-
 	return ret;
 }
 
@@ -520,7 +514,6 @@ blockable_page_cache_readahead(struct ad
 
 	actual = __do_page_cache_readahead(mapping, filp, offset, nr_to_read, 0);
 
-	ra_account(NULL, RA_EVENT_READAHEAD, actual);
 	dprintk("blockable-readahead(ino=%lu, ra=%lu+%lu) = %d\n",
 			mapping->host->i_ino, offset, nr_to_read, actual);
 
@@ -1007,7 +1000,6 @@ static int adjust_rala(unsigned long *ra
 	 * if there is a major upsurge of load, or fall of this stream's speed.
 	 */
 	if (*ra_size <= *la_size * 2) {
-		ra_account(NULL, RA_EVENT_READAHEAD_SHRINK, *ra_size);
 		return 0;
 	}
 
@@ -1710,8 +1702,6 @@ page_cache_readahead_adaptive(struct add
 
 	/* No action on look-ahead time? */
 	if (page) {
-		ra_account(ra, RA_EVENT_LOOKAHEAD_NOACTION,
-						ra->readahead_index - offset);
 		return 0;
 	}
 
@@ -1807,10 +1797,8 @@ static const char * const ra_event_name[
 	"readahead_hit",
 	"lookahead",
 	"lookahead_hit",
-	"lookahead_ignore",
 	"readahead_mmap",
 	"readahead_eof",
-	"readahead_shrink",
 	"readahead_thrash",
 	"readahead_mutilt",
 	"readahead_rescue"
@@ -1825,13 +1813,11 @@ static void ra_account(struct file_ra_st
 	if (!readahead_debug_level)
 		return;
 
-	if (e == RA_EVENT_READAHEAD_HIT && pages < 0) {
-		c = ra_class_old(ra);
+	if (pages < 0) {
 		pages = -pages;
-	} else if (ra)
+		c = ra_class_old(ra);
+	} else
 		c = ra_class_new(ra);
-	else
-		c = RA_CLASS_NONE;
 
 	if (!c)
 		c = RA_CLASS_NONE;

--