[Cluster-devel] [GFS2 PATCH 0/2] GFS2: Bulk submit ordered writes

[Cluster-devel] [GFS2 PATCH 0/2] GFS2: Bulk submit ordered writes

[Bob Peterson]

This patch set is designed to submit the ordered writes list in bulk.
This is a performance enhancement from the way we currently do it,
which is submit->wait->submit->wait. Submitting them in bulk takes
better advantage of the io elevator and vfs below us.

The first patch merely moves function gfs2_ordered_wait. The second
patch implements the actual change.

Bob Peterson (2):
  GFS2: Move function gfs2_ordered_wait before ordered_write
  GFS2: Submit all ordered writes in bulk, wait after that

 fs/gfs2/log.c | 48 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 31 insertions(+), 17 deletions(-)

-- 
2.17.0

[Cluster-devel] [GFS2 PATCH 1/2] GFS2: Move function gfs2_ordered_wait before ordered_write

[Bob Peterson]

Although it may not be obvious from the patch contents, this
patch simply moves function gfs2_ordered_wait before its sister
function, gfs2_ordered_write. There are no changes to the code.
This was done to make a follow-on patch more clear and easy to
read.

Signed-off-by: Bob Peterson <rpeterso@redhat.com>
---
 fs/gfs2/log.c | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
index 0248835625f1..bedd1c056ca2 100644
--- a/fs/gfs2/log.c
+++ b/fs/gfs2/log.c
@@ -530,44 +530,44 @@ static int ip_cmp(void *priv, struct list_head *a, struct list_head *b)
 	return 0;
 }
 
-static void gfs2_ordered_write(struct gfs2_sbd *sdp)
+static void gfs2_ordered_wait(struct gfs2_sbd *sdp)
 {
 	struct gfs2_inode *ip;
-	LIST_HEAD(written);
 
 	spin_lock(&sdp->sd_ordered_lock);
-	list_sort(NULL, &sdp->sd_log_le_ordered, &ip_cmp);
 	while (!list_empty(&sdp->sd_log_le_ordered)) {
 		ip = list_entry(sdp->sd_log_le_ordered.next, struct gfs2_inode, i_ordered);
-		if (ip->i_inode.i_mapping->nrpages == 0) {
-			test_and_clear_bit(GIF_ORDERED, &ip->i_flags);
-			list_del(&ip->i_ordered);
+		list_del(&ip->i_ordered);
+		WARN_ON(!test_and_clear_bit(GIF_ORDERED, &ip->i_flags));
+		if (ip->i_inode.i_mapping->nrpages == 0)
 			continue;
-		}
-		list_move(&ip->i_ordered, &written);
 		spin_unlock(&sdp->sd_ordered_lock);
-		filemap_fdatawrite(ip->i_inode.i_mapping);
+		filemap_fdatawait(ip->i_inode.i_mapping);
 		spin_lock(&sdp->sd_ordered_lock);
 	}
-	list_splice(&written, &sdp->sd_log_le_ordered);
 	spin_unlock(&sdp->sd_ordered_lock);
 }
 
-static void gfs2_ordered_wait(struct gfs2_sbd *sdp)
+static void gfs2_ordered_write(struct gfs2_sbd *sdp)
 {
 	struct gfs2_inode *ip;
+	LIST_HEAD(written);
 
 	spin_lock(&sdp->sd_ordered_lock);
+	list_sort(NULL, &sdp->sd_log_le_ordered, &ip_cmp);
 	while (!list_empty(&sdp->sd_log_le_ordered)) {
 		ip = list_entry(sdp->sd_log_le_ordered.next, struct gfs2_inode, i_ordered);
-		list_del(&ip->i_ordered);
-		WARN_ON(!test_and_clear_bit(GIF_ORDERED, &ip->i_flags));
-		if (ip->i_inode.i_mapping->nrpages == 0)
+		if (ip->i_inode.i_mapping->nrpages == 0) {
+			test_and_clear_bit(GIF_ORDERED, &ip->i_flags);
+			list_del(&ip->i_ordered);
 			continue;
+		}
+		list_move(&ip->i_ordered, &written);
 		spin_unlock(&sdp->sd_ordered_lock);
-		filemap_fdatawait(ip->i_inode.i_mapping);
+		filemap_fdatawrite(ip->i_inode.i_mapping);
 		spin_lock(&sdp->sd_ordered_lock);
 	}
+	list_splice(&written, &sdp->sd_log_le_ordered);
 	spin_unlock(&sdp->sd_ordered_lock);
 }
 
-- 
2.17.0

[Cluster-devel] [GFS2 PATCH 2/2] GFS2: Submit all ordered writes in bulk, wait after that

[Bob Peterson]

Before this patch, the ordered_write function would submit all
the ordered writes with filemap_fdatawrite, which waited for each
one to complete before moving on to the next. This patch allows it
to submit them all, then wait for them after they're submitted.

Signed-off-by: Bob Peterson <rpeterso@redhat.com>
---
 fs/gfs2/log.c | 34 ++++++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
index bedd1c056ca2..b05d0fbc3d05 100644
--- a/fs/gfs2/log.c
+++ b/fs/gfs2/log.c
@@ -22,6 +22,7 @@
 #include <linux/blkdev.h>
 #include <linux/writeback.h>
 #include <linux/list_sort.h>
+#include <linux/fs.h>
 
 #include "gfs2.h"
 #include "incore.h"
@@ -530,15 +531,20 @@ static int ip_cmp(void *priv, struct list_head *a, struct list_head *b)
 	return 0;
 }
 
-static void gfs2_ordered_wait(struct gfs2_sbd *sdp)
+static void gfs2_ordered_wait(struct gfs2_sbd *sdp, struct list_head *src,
+			      struct list_head *dst)
 {
 	struct gfs2_inode *ip;
 
 	spin_lock(&sdp->sd_ordered_lock);
-	while (!list_empty(&sdp->sd_log_le_ordered)) {
-		ip = list_entry(sdp->sd_log_le_ordered.next, struct gfs2_inode, i_ordered);
-		list_del(&ip->i_ordered);
-		WARN_ON(!test_and_clear_bit(GIF_ORDERED, &ip->i_flags));
+	while (!list_empty(src)) {
+		ip = list_entry(src->next, struct gfs2_inode, i_ordered);
+		if (dst)
+			list_move_tail(&ip->i_ordered, dst);
+		else {
+			list_del(&ip->i_ordered);
+			WARN_ON(!test_and_clear_bit(GIF_ORDERED, &ip->i_flags));
+		}
 		if (ip->i_inode.i_mapping->nrpages == 0)
 			continue;
 		spin_unlock(&sdp->sd_ordered_lock);
@@ -551,24 +557,32 @@ static void gfs2_ordered_wait(struct gfs2_sbd *sdp)
 static void gfs2_ordered_write(struct gfs2_sbd *sdp)
 {
 	struct gfs2_inode *ip;
+	struct address_space *mapping;
 	LIST_HEAD(written);
+	struct writeback_control wbc = {
+		.sync_mode = WB_SYNC_NONE,
+		.nr_to_write = LONG_MAX,
+		.range_start = 0,
+		.range_end = LLONG_MAX,
+	};
 
 	spin_lock(&sdp->sd_ordered_lock);
 	list_sort(NULL, &sdp->sd_log_le_ordered, &ip_cmp);
 	while (!list_empty(&sdp->sd_log_le_ordered)) {
 		ip = list_entry(sdp->sd_log_le_ordered.next, struct gfs2_inode, i_ordered);
-		if (ip->i_inode.i_mapping->nrpages == 0) {
+		mapping = ip->i_inode.i_mapping;
+		if (mapping->nrpages == 0) {
 			test_and_clear_bit(GIF_ORDERED, &ip->i_flags);
 			list_del(&ip->i_ordered);
 			continue;
 		}
-		list_move(&ip->i_ordered, &written);
+		list_move_tail(&ip->i_ordered, &written);
 		spin_unlock(&sdp->sd_ordered_lock);
-		filemap_fdatawrite(ip->i_inode.i_mapping);
+		mapping->a_ops->writepages(mapping, &wbc);
 		spin_lock(&sdp->sd_ordered_lock);
 	}
-	list_splice(&written, &sdp->sd_log_le_ordered);
 	spin_unlock(&sdp->sd_ordered_lock);
+	gfs2_ordered_wait(sdp, &written, &sdp->sd_log_le_ordered);
 }
 
 void gfs2_ordered_del_inode(struct gfs2_inode *ip)
@@ -741,7 +755,7 @@ static void log_write_header(struct gfs2_sbd *sdp, u32 flags)
 	tail = current_tail(sdp);
 
 	if (test_bit(SDF_NOBARRIERS, &sdp->sd_flags)) {
-		gfs2_ordered_wait(sdp);
+		gfs2_ordered_wait(sdp, &sdp->sd_log_le_ordered, NULL);
 		log_flush_wait(sdp);
 		op_flags = REQ_SYNC | REQ_META | REQ_PRIO;
 	}
-- 
2.17.0

[Cluster-devel] [GFS2 PATCH 2/2] GFS2: Submit all ordered writes in bulk, wait after that

[Andreas Gruenbacher]

Hi Bob,

On 14 May 2018 at 16:18, Bob Peterson <rpeterso@redhat.com> wrote:
> Before this patch, the ordered_write function would submit all
> the ordered writes with filemap_fdatawrite, which waited for each
> one to complete before moving on to the next. This patch allows it
> to submit them all, then wait for them after they're submitted.

this looks reasonable.  I think it's not very helpful to reuse
gfs2_ordered_wait here though: the folowing incremental patch should
work just as well or better.

Andreas

---
 fs/gfs2/log.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
index b05d0fbc3d05..2a9b1c008325 100644
--- a/fs/gfs2/log.c
+++ b/fs/gfs2/log.c
@@ -577,12 +577,15 @@ static void gfs2_ordered_write(struct gfs2_sbd *sdp)
 			continue;
 		}
 		list_move_tail(&ip->i_ordered, &written);
-		spin_unlock(&sdp->sd_ordered_lock);
-		mapping->a_ops->writepages(mapping, &wbc);
-		spin_lock(&sdp->sd_ordered_lock);
 	}
 	spin_unlock(&sdp->sd_ordered_lock);
-	gfs2_ordered_wait(sdp, &written, &sdp->sd_log_le_ordered);
+	list_for_each_entry(ip, &written, i_ordered)
+		mapping->a_ops->writepages(ip->i_inode.i_mapping, &wbc);
+	list_for_each_entry(ip, &written, i_ordered)
+		filemap_fdatawait(ip->i_inode.i_mapping);
+	spin_lock(&sdp->sd_ordered_lock);
+	list_splice_tail(&written, &sdp->sd_log_le_ordered);
+	spin_unlock(&sdp->sd_ordered_lock);
 }
 
 void gfs2_ordered_del_inode(struct gfs2_inode *ip)
-- 
2.17.0