Re: [GIT PULL] (xen) stable/for-jens-3.10 xenwatch: page allocation failure: order:7, mode:0x10c0d0

["Roger Pau Monné" <roger.pau@citrix.com>]

[-- Attachment #1: Type: text/plain, Size: 6555 bytes --]

On 25/04/13 14:39, Roger Pau Monné wrote:
> On 25/04/13 14:32, Sander Eikelenboom wrote:
>>
>> Thursday, April 25, 2013, 10:43:33 AM, you wrote:
>>
>>> On 25/04/13 10:35, Roger Pau Monné wrote:
>>>> On 24/04/13 20:16, Sander Eikelenboom wrote:
>>>>> Friday, April 19, 2013, 4:44:01 PM, you wrote:
>>>>>
>>>>>> Hey Jens,
>>>>>
>>>>>> Please in your spare time (if there is such a thing at a conference)
>>>>>> pull this branch:
>>>>>
>>>>>>  git://git.kernel.org/pub/scm/linux/kernel/git/konrad/xen.git stable/for-jens-3.10
>>>>>
>>>>>> for your v3.10 branch. Sorry for being so late with this.
>>>>>
>>>>> <big snip></big snip>
>>>>>
>>>>>> Anyhow, please pull and if possible include the nice overview I typed up in the
>>>>>> merge commit.
>>>>>
>>>>>>  Documentation/ABI/stable/sysfs-bus-xen-backend |  18 +
>>>>>>  drivers/block/xen-blkback/blkback.c            | 843 ++++++++++++++++---------
>>>>>>  drivers/block/xen-blkback/common.h             | 145 ++++-
>>>>>>  drivers/block/xen-blkback/xenbus.c             |  38 ++
>>>>>>  drivers/block/xen-blkfront.c                   | 490 +++++++++++---
>>>>>>  include/xen/interface/io/blkif.h               |  53 ++
>>>>>>  6 files changed, 1188 insertions(+), 399 deletions(-)
>>>>>
>>>>>> Roger Pau Monne (7):
>>>>>>       xen-blkback: print stats about persistent grants
>>>>>>       xen-blkback: use balloon pages for all mappings
>>>>>>       xen-blkback: implement LRU mechanism for persistent grants
>>>>>>       xen-blkback: move pending handles list from blkbk to pending_req
>>>>>>       xen-blkback: make the queue of free requests per backend
>>>>>>       xen-blkback: expand map/unmap functions
>>>>>>       xen-block: implement indirect descriptors
>>>>>
>>>>>
>>>>> Hi Konrad / Roger,
>>>>>
>>>>> I tried this pull on top of latest Linus latest linux-3.9 tree, but although it seems to boot and work fine at first, i seem to get trouble after running for about a day.
>>>>> Without this pull it runs fine for several days.
>>>>>
>>>>> Trying to start a new guest I ended up with the splat below. In the output of xl-dmesg i seem to see more of these than before:
>>>>> (XEN) [2013-04-24 14:37:40] grant_table.c:1250:d1 Expanding dom (1) grant table from (9) to (10) frames
>>>>
>>>> Hello Sander,
>>>>
>>>> Thanks for the report, it is expected to see more messages regarding
>>>> grant table expansion with this patch, since we are using up to 1056
>>>> persistent grants for each backend. Could you try lowering down the
>>>> maximum number of persistent grants to see if that prevents running out
>>>> of memory:
>>>>
>>>> # echo 384 > /sys/module/xen_blkback/parameters/max_persistent_grants
>>
>>> And the number of free pages keep in blkback cache:
>>
>> # echo 256 >> /sys/module/xen_blkback/parameters/max_buffer_pages
>>
>> With both set .. it still bails out after sometime when trying to start a new guest.
> 
> OK, will work on a patch to split memory allocation instead of doing it
> all in a big chunk.

Could you try the following patch?

---
>From db0163f74dafb085457d843c6ab7935d5dc7bd65 Mon Sep 17 00:00:00 2001
From: Roger Pau Monne <roger.pau@citrix.com>
Date: Thu, 25 Apr 2013 17:48:32 +0200
Subject: [PATCH] xen-blkback: allocate list of pending reqs in small chunks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Allocate pending requests in smaller chunks instead of allocating them
all at the same time.

Also remove the global array of pending_reqs, it is no longer
necessary.

Signed-off-by: Roger Pau Monné <roger.pau@citrix.com>
---
 drivers/block/xen-blkback/common.h |    2 -
 drivers/block/xen-blkback/xenbus.c |   41 ++++++++++++++++++++++-------------
 2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h
index 1ac53da..2bbda8637 100644
--- a/drivers/block/xen-blkback/common.h
+++ b/drivers/block/xen-blkback/common.h
@@ -297,8 +297,6 @@ struct xen_blkif {
 	int			free_pages_num;
 	struct list_head	free_pages;
 
-	/* Allocation of pending_reqs */
-	struct pending_req	*pending_reqs;
 	/* List of all 'pending_req' available */
 	struct list_head	pending_free;
 	/* And its spinlock. */
diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c
index afab208..ceefbe4 100644
--- a/drivers/block/xen-blkback/xenbus.c
+++ b/drivers/block/xen-blkback/xenbus.c
@@ -105,6 +105,7 @@ static void xen_update_blkif_status(struct xen_blkif *blkif)
 static struct xen_blkif *xen_blkif_alloc(domid_t domid)
 {
 	struct xen_blkif *blkif;
+	struct pending_req *req, *n;
 	int i;
 
 	BUILD_BUG_ON(MAX_INDIRECT_PAGES > BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST);
@@ -127,22 +128,29 @@ static struct xen_blkif *xen_blkif_alloc(domid_t domid)
 	blkif->free_pages_num = 0;
 	atomic_set(&blkif->persistent_gnt_in_use, 0);
 
-	blkif->pending_reqs = kcalloc(XEN_BLKIF_REQS,
-	                              sizeof(blkif->pending_reqs[0]),
-	                              GFP_KERNEL);
-	if (!blkif->pending_reqs) {
-		kmem_cache_free(xen_blkif_cachep, blkif);
-		return ERR_PTR(-ENOMEM);
-	}
 	INIT_LIST_HEAD(&blkif->pending_free);
+
+	for (i = 0; i < XEN_BLKIF_REQS; i++) {
+		req = kzalloc(sizeof(*req), GFP_KERNEL);
+		if (!req)
+			goto fail;
+		list_add_tail(&req->free_list,
+		              &blkif->pending_free);
+	}
 	spin_lock_init(&blkif->pending_free_lock);
 	init_waitqueue_head(&blkif->pending_free_wq);
 
-	for (i = 0; i < XEN_BLKIF_REQS; i++)
-		list_add_tail(&blkif->pending_reqs[i].free_list,
-			      &blkif->pending_free);
-
 	return blkif;
+
+fail:
+	list_for_each_entry_safe(req, n, &blkif->pending_free, free_list) {
+		list_del(&req->free_list);
+		kfree(req);
+	}
+
+	kmem_cache_free(xen_blkif_cachep, blkif);
+
+	return ERR_PTR(-ENOMEM);
 }
 
 static int xen_blkif_map(struct xen_blkif *blkif, unsigned long shared_page,
@@ -221,18 +229,21 @@ static void xen_blkif_disconnect(struct xen_blkif *blkif)
 
 static void xen_blkif_free(struct xen_blkif *blkif)
 {
-	struct pending_req *req;
+	struct pending_req *req, *n;
 	int i = 0;
 
 	if (!atomic_dec_and_test(&blkif->refcnt))
 		BUG();
 
 	/* Check that there is no request in use */
-	list_for_each_entry(req, &blkif->pending_free, free_list)
+	list_for_each_entry_safe(req, n, &blkif->pending_free, free_list) {
+		list_del(&req->free_list);
+		kfree(req);
 		i++;
-	BUG_ON(i != XEN_BLKIF_REQS);
+	}
+
+	WARN_ON(i != XEN_BLKIF_REQS);
 
-	kfree(blkif->pending_reqs);
 	kmem_cache_free(xen_blkif_cachep, blkif);
 }
 
-- 
1.7.7.5 (Apple Git-26)




[-- Attachment #2: 0001-xen-blkback-allocate-list-of-pending-reqs-in-small-c.patch --]
[-- Type: text/plain, Size: 3528 bytes --]

>From db0163f74dafb085457d843c6ab7935d5dc7bd65 Mon Sep 17 00:00:00 2001
From: Roger Pau Monne <roger.pau@citrix.com>
Date: Thu, 25 Apr 2013 17:48:32 +0200
Subject: [PATCH] xen-blkback: allocate list of pending reqs in small chunks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Allocate pending requests in smaller chunks instead of allocating them
all at the same time.

Also remove the global array of pending_reqs, it is no longer
necessary.

Signed-off-by: Roger Pau Monné <roger.pau@citrix.com>
---
 drivers/block/xen-blkback/common.h |    2 -
 drivers/block/xen-blkback/xenbus.c |   41 ++++++++++++++++++++++-------------
 2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h
index 1ac53da..2bbda8637 100644
--- a/drivers/block/xen-blkback/common.h
+++ b/drivers/block/xen-blkback/common.h
@@ -297,8 +297,6 @@ struct xen_blkif {
 	int			free_pages_num;
 	struct list_head	free_pages;
 
-	/* Allocation of pending_reqs */
-	struct pending_req	*pending_reqs;
 	/* List of all 'pending_req' available */
 	struct list_head	pending_free;
 	/* And its spinlock. */
diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c
index afab208..ceefbe4 100644
--- a/drivers/block/xen-blkback/xenbus.c
+++ b/drivers/block/xen-blkback/xenbus.c
@@ -105,6 +105,7 @@ static void xen_update_blkif_status(struct xen_blkif *blkif)
 static struct xen_blkif *xen_blkif_alloc(domid_t domid)
 {
 	struct xen_blkif *blkif;
+	struct pending_req *req, *n;
 	int i;
 
 	BUILD_BUG_ON(MAX_INDIRECT_PAGES > BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST);
@@ -127,22 +128,29 @@ static struct xen_blkif *xen_blkif_alloc(domid_t domid)
 	blkif->free_pages_num = 0;
 	atomic_set(&blkif->persistent_gnt_in_use, 0);
 
-	blkif->pending_reqs = kcalloc(XEN_BLKIF_REQS,
-	                              sizeof(blkif->pending_reqs[0]),
-	                              GFP_KERNEL);
-	if (!blkif->pending_reqs) {
-		kmem_cache_free(xen_blkif_cachep, blkif);
-		return ERR_PTR(-ENOMEM);
-	}
 	INIT_LIST_HEAD(&blkif->pending_free);
+
+	for (i = 0; i < XEN_BLKIF_REQS; i++) {
+		req = kzalloc(sizeof(*req), GFP_KERNEL);
+		if (!req)
+			goto fail;
+		list_add_tail(&req->free_list,
+		              &blkif->pending_free);
+	}
 	spin_lock_init(&blkif->pending_free_lock);
 	init_waitqueue_head(&blkif->pending_free_wq);
 
-	for (i = 0; i < XEN_BLKIF_REQS; i++)
-		list_add_tail(&blkif->pending_reqs[i].free_list,
-			      &blkif->pending_free);
-
 	return blkif;
+
+fail:
+	list_for_each_entry_safe(req, n, &blkif->pending_free, free_list) {
+		list_del(&req->free_list);
+		kfree(req);
+	}
+
+	kmem_cache_free(xen_blkif_cachep, blkif);
+
+	return ERR_PTR(-ENOMEM);
 }
 
 static int xen_blkif_map(struct xen_blkif *blkif, unsigned long shared_page,
@@ -221,18 +229,21 @@ static void xen_blkif_disconnect(struct xen_blkif *blkif)
 
 static void xen_blkif_free(struct xen_blkif *blkif)
 {
-	struct pending_req *req;
+	struct pending_req *req, *n;
 	int i = 0;
 
 	if (!atomic_dec_and_test(&blkif->refcnt))
 		BUG();
 
 	/* Check that there is no request in use */
-	list_for_each_entry(req, &blkif->pending_free, free_list)
+	list_for_each_entry_safe(req, n, &blkif->pending_free, free_list) {
+		list_del(&req->free_list);
+		kfree(req);
 		i++;
-	BUG_ON(i != XEN_BLKIF_REQS);
+	}
+
+	WARN_ON(i != XEN_BLKIF_REQS);
 
-	kfree(blkif->pending_reqs);
 	kmem_cache_free(xen_blkif_cachep, blkif);
 }
 
-- 
1.7.7.5 (Apple Git-26)


[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel