Re: Kernel falls apart under light memory pressure (i.e. linking vmlinux)

[Andrew Lutomirski <luto@mit.edu>]

[-- Attachment #1: Type: text/plain, Size: 4915 bytes --]

On Fri, May 20, 2011 at 6:11 AM, Andrea Arcangeli <aarcange@redhat.com> wrote:
> I figure it's not easily reproducible but you can easily rule out THP
> issues by reproducing at least once after booting with
> transparent_hugepage=never or by building the kernel with
> CONFIG_TRANSPARENT_HUGEPAGE=n.

Reproduced with CONFIG_TRANSPARENT_HUGEPAGE=n with and without
compaction and migration.

I applied the attached patch (which includes Minchan's !pgdat_balanced
and need_resched changes).  I see:

[  121.468339] firefox shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea00019217a8) w/ prev = 100000000002000D
[  121.469236] firefox shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea00016596b8) w/ prev = 100000000002000D
[  121.470207] firefox: shrink_page_list (nr_scanned=94
nr_reclaimed=19 nr_to_reclaim=32 gfp_mask=201DA) found inactive page
ffffea00019217a8 with flags=100000000002004D
[  121.472451] firefox: shrink_page_list (nr_scanned=94
nr_reclaimed=19 nr_to_reclaim=32 gfp_mask=201DA) found inactive page
ffffea00016596b8 with flags=100000000002004D
[  121.482782] dd shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea00013a8938) w/ prev = 100000000002000D
[  121.489820] dd shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea00017a4e88) w/ prev = 1000000000000801
[  121.490626] dd shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea000005edb0) w/ prev = 1000000000000801
[  121.491499] dd: shrink_page_list (nr_scanned=62 nr_reclaimed=0
nr_to_reclaim=32 gfp_mask=200D2) found inactive page ffffea00017a4e88
with flags=1000000000000841
[  121.494337] dd: shrink_page_list (nr_scanned=62 nr_reclaimed=0
nr_to_reclaim=32 gfp_mask=200D2) found inactive page ffffea000005edb0
with flags=1000000000000841
[  121.499219] dd shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea000129c788) w/ prev = 1000000000080009
[  121.500363] dd shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea000129c830) w/ prev = 1000000000080009
[  121.502270] kswapd0 shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0001146470) w/ prev = 100000000008001D
[  121.661545] kworker/1:1 shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0000058168) w/ prev = 1000000000000801
[  121.662791] kworker/1:1 shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea000166f288) w/ prev = 1000000000000801
[  121.665727] kworker/1:1 shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0001681c40) w/ prev = 1000000000000801
[  121.666857] kworker/1:1 shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0001693130) w/ prev = 1000000000000801
[  121.667988] kworker/1:1 shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0000c790d8) w/ prev = 1000000000000801
[  121.669105] kworker/1:1 shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea000113fe48) w/ prev = 1000000000000801
[  121.670238] kworker/1:1: shrink_page_list (nr_scanned=102
nr_reclaimed=20 nr_to_reclaim=32 gfp_mask=11212) found inactive page
ffffea0000058168 with flags=1000000000000841
[  121.674061] kworker/1:1: shrink_page_list (nr_scanned=102
nr_reclaimed=20 nr_to_reclaim=32 gfp_mask=11212) found inactive page
ffffea000166f288 with flags=1000000000000841
[  121.678054] kworker/1:1: shrink_page_list (nr_scanned=102
nr_reclaimed=20 nr_to_reclaim=32 gfp_mask=11212) found inactive page
ffffea0001681c40 with flags=1000000000000841
[  121.682069] kworker/1:1: shrink_page_list (nr_scanned=102
nr_reclaimed=20 nr_to_reclaim=32 gfp_mask=11212) found inactive page
ffffea0001693130 with flags=1000000000000841
[  121.686074] kworker/1:1: shrink_page_list (nr_scanned=102
nr_reclaimed=20 nr_to_reclaim=32 gfp_mask=11212) found inactive page
ffffea0000c790d8 with flags=1000000000000841
[  121.690045] kworker/1:1: shrink_page_list (nr_scanned=102
nr_reclaimed=20 nr_to_reclaim=32 gfp_mask=11212) found inactive page
ffffea000113fe48 with flags=1000000000000841
[  121.866205] test_mempressur shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea000165d5b8) w/ prev = 100000000002000D
[  121.868204] test_mempressur shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0001661288) w/ prev = 100000000002000D
[  121.870203] test_mempressur shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0001661250) w/ prev = 100000000002000D
[  121.872195] test_mempressur shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea000100cee8) w/ prev = 100000000002000D
[  121.873486] test_mempressur shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0000eafab8) w/ prev = 100000000002000D
[  121.874718] test_mempressur shrink_page_list+0x4f3/0x5ca:
SetPageActive(ffffea0000eafaf0) w/ prev = 100000000002000D

This is interesting: it looks like shrink_page_list is making its way
through the list more than once.  It could be reentering itself
somehow or it could have something screwed up with the linked list.

I'll keep slowly debugging, but maybe this is enough for someone
familiar with this code to beat me to it.

Minchan, I think this means that your fixes are just hiding and not
fixing the underlying problem.

[-- Attachment #2: vm_tests.patch --]
[-- Type: application/octet-stream, Size: 3074 bytes --]

diff --git a/mm/swap.c b/mm/swap.c
index c02f936..74118d2 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -40,6 +40,25 @@ int page_cluster;
 static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
 
+static noinline void SetPageActiveCheck(struct page *page)
+{
+	unsigned long x;
+	while(true) {
+		unsigned long flags = page->flags;
+		x = cmpxchg(&(page)->flags, flags,
+			    flags | (1 << PG_active));
+		if (x == flags) break;
+	}
+	if (~x & (1 << PG_lru)) {
+                char name[sizeof(current->comm)];
+		printk(KERN_ERR "%s %pS: SetPageActive(%p) w/ prev = %lX\n",
+		       get_task_comm(name, current),
+		       __builtin_return_address(0), page, x);
+	}
+}
+
+#define SetPageActive SetPageActiveCheck
+
 /*
  * This path almost never happens for VM activity - pages are normally
  * freed via pagevecs.  But it gets used by networking.
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 3f44b81..dc417ab 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -53,6 +53,25 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/vmscan.h>
 
+static noinline void SetPageActiveCheck(struct page *page)
+{
+	unsigned long x;
+	while(true) {
+		unsigned long flags = page->flags;
+		x = cmpxchg(&(page)->flags, flags,
+			    flags | (1 << PG_active));
+		if (x == flags) break;
+	}
+	if (~x & (1 << PG_lru)) {
+                char name[sizeof(current->comm)];
+		printk(KERN_ERR "%s %pS: SetPageActive(%p) w/ prev = %lX\n",
+		       get_task_comm(name, current),
+		       __builtin_return_address(0), page, x);
+	}
+}
+
+#define SetPageActive SetPageActiveCheck
+
 /*
  * reclaim_mode determines how the inactive list is shrunk
  * RECLAIM_MODE_SINGLE: Reclaim only order-0 pages
@@ -729,7 +748,17 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		if (!trylock_page(page))
 			goto keep;
 
-		VM_BUG_ON(PageActive(page));
+		if (PageActive(page)) {
+			char name[sizeof(current->comm)];
+			printk(KERN_ERR "%s: shrink_page_list (nr_scanned=%lu nr_reclaimed=%lu nr_to_reclaim=%lu gfp_mask=%X) found inactive page %p with flags=%lX\n",
+			       get_task_comm(name, current),
+			       sc->nr_scanned, sc->nr_reclaimed,
+			       sc->nr_to_reclaim, sc->gfp_mask, page,
+			       page->flags);
+			//VM_BUG_ON(PageActive(page));
+			msleep(1);
+			continue;
+		}
 		VM_BUG_ON(page_zone(page) != zone);
 
 		sc->nr_scanned++;
@@ -2247,6 +2276,10 @@ static bool sleeping_prematurely(pg_data_t *pgdat, int order, long remaining,
 	unsigned long balanced = 0;
 	bool all_zones_ok = true;
 
+	/* If kswapd has been running too long, just sleep */
+	if (need_resched())
+		return false;
+
 	/* If a direct reclaimer woke kswapd within HZ/10, it's premature */
 	if (remaining)
 		return true;
@@ -2282,7 +2315,7 @@ static bool sleeping_prematurely(pg_data_t *pgdat, int order, long remaining,
 	 * must be balanced
 	 */
 	if (order)
-		return pgdat_balanced(pgdat, balanced, classzone_idx);
+		return !pgdat_balanced(pgdat, balanced, classzone_idx);
 	else
 		return !all_zones_ok;
 }