* [PATCH 1/7] writeback: move dirty inodes from super_block to backing_dev_info
2009-03-12 14:33 [PATCH 0/7] Per-bdi writeback flusher threads Jens Axboe
@ 2009-03-12 14:33 ` Jens Axboe
2009-03-24 16:17 ` Jan Kara
2009-03-12 14:33 ` [PATCH 2/7] writeback: switch to per-bdi threads for flushing data Jens Axboe
` (5 subsequent siblings)
6 siblings, 1 reply; 31+ messages in thread
From: Jens Axboe @ 2009-03-12 14:33 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel; +Cc: chris.mason, david, npiggin, Jens Axboe
This is a first step at introducing per-bdi flusher threads. We should
have no change in behaviour, although sb_has_dirty_inodes() is now
ridiculously expensive, as there's no easy way to answer that question.
Not a huge problem, since it'll be deleted in subsequent patches.
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
fs/fs-writeback.c | 186 +++++++++++++++++++++++++++----------------
fs/super.c | 3 -
include/linux/backing-dev.h | 9 ++
include/linux/fs.h | 5 +-
mm/backing-dev.c | 31 +++++++-
mm/page-writeback.c | 1 -
6 files changed, 156 insertions(+), 79 deletions(-)
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index e5eaa62..c107cff 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -25,6 +25,7 @@
#include <linux/buffer_head.h>
#include "internal.h"
+#define inode_to_bdi(inode) (inode)->i_mapping->backing_dev_info
/**
* writeback_acquire - attempt to get exclusive writeback access to a device
@@ -158,12 +159,13 @@ void __mark_inode_dirty(struct inode *inode, int flags)
goto out;
/*
- * If the inode was already on s_dirty/s_io/s_more_io, don't
- * reposition it (that would break s_dirty time-ordering).
+ * If the inode was already on b_dirty/b_io/b_more_io, don't
+ * reposition it (that would break b_dirty time-ordering).
*/
if (!was_dirty) {
inode->dirtied_when = jiffies;
- list_move(&inode->i_list, &sb->s_dirty);
+ list_move(&inode->i_list,
+ &inode_to_bdi(inode)->b_dirty);
}
}
out:
@@ -184,31 +186,30 @@ static int write_inode(struct inode *inode, int sync)
* furthest end of its superblock's dirty-inode list.
*
* Before stamping the inode's ->dirtied_when, we check to see whether it is
- * already the most-recently-dirtied inode on the s_dirty list. If that is
+ * already the most-recently-dirtied inode on the b_dirty list. If that is
* the case then the inode must have been redirtied while it was being written
* out and we don't reset its dirtied_when.
*/
static void redirty_tail(struct inode *inode)
{
- struct super_block *sb = inode->i_sb;
+ struct backing_dev_info *bdi = inode_to_bdi(inode);
- if (!list_empty(&sb->s_dirty)) {
- struct inode *tail_inode;
+ if (!list_empty(&bdi->b_dirty)) {
+ struct inode *tail;
- tail_inode = list_entry(sb->s_dirty.next, struct inode, i_list);
- if (!time_after_eq(inode->dirtied_when,
- tail_inode->dirtied_when))
+ tail = list_entry(bdi->b_dirty.next, struct inode, i_list);
+ if (!time_after_eq(inode->dirtied_when, tail->dirtied_when))
inode->dirtied_when = jiffies;
}
- list_move(&inode->i_list, &sb->s_dirty);
+ list_move(&inode->i_list, &bdi->b_dirty);
}
/*
- * requeue inode for re-scanning after sb->s_io list is exhausted.
+ * requeue inode for re-scanning after bdi->b_io list is exhausted.
*/
static void requeue_io(struct inode *inode)
{
- list_move(&inode->i_list, &inode->i_sb->s_more_io);
+ list_move(&inode->i_list, &inode_to_bdi(inode)->b_more_io);
}
static void inode_sync_complete(struct inode *inode)
@@ -240,18 +241,50 @@ static void move_expired_inodes(struct list_head *delaying_queue,
/*
* Queue all expired dirty inodes for io, eldest first.
*/
-static void queue_io(struct super_block *sb,
- unsigned long *older_than_this)
+static void queue_io(struct backing_dev_info *bdi,
+ unsigned long *older_than_this)
{
- list_splice_init(&sb->s_more_io, sb->s_io.prev);
- move_expired_inodes(&sb->s_dirty, &sb->s_io, older_than_this);
+ list_splice_init(&bdi->b_more_io, bdi->b_io.prev);
+ move_expired_inodes(&bdi->b_dirty, &bdi->b_io, older_than_this);
+}
+
+static int sb_on_inode_list(struct super_block *sb, struct list_head *list)
+{
+ struct inode *inode;
+ int ret = 0;
+
+ spin_lock(&inode_lock);
+ list_for_each_entry(inode, list, i_list) {
+ if (inode->i_sb == sb) {
+ ret = 1;
+ break;
+ }
+ }
+ spin_unlock(&inode_lock);
+ return ret;
}
int sb_has_dirty_inodes(struct super_block *sb)
{
- return !list_empty(&sb->s_dirty) ||
- !list_empty(&sb->s_io) ||
- !list_empty(&sb->s_more_io);
+ struct backing_dev_info *bdi;
+ int ret = 0;
+
+ /*
+ * This is REALLY expensive right now, but it'll go away
+ * when the bdi writeback is introduced
+ */
+ rcu_read_lock();
+ list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
+ if (sb_on_inode_list(sb, &bdi->b_dirty) ||
+ sb_on_inode_list(sb, &bdi->b_io) ||
+ sb_on_inode_list(sb, &bdi->b_more_io)) {
+ ret = 1;
+ break;
+ }
+ }
+ rcu_read_unlock();
+
+ return ret;
}
EXPORT_SYMBOL(sb_has_dirty_inodes);
@@ -305,11 +338,11 @@ __sync_single_inode(struct inode *inode, struct writeback_control *wbc)
/*
* We didn't write back all the pages. nfs_writepages()
* sometimes bales out without doing anything. Redirty
- * the inode; Move it from s_io onto s_more_io/s_dirty.
+ * the inode; Move it from b_io onto b_more_io/b_dirty.
*/
/*
* akpm: if the caller was the kupdate function we put
- * this inode at the head of s_dirty so it gets first
+ * this inode at the head of b_dirty so it gets first
* consideration. Otherwise, move it to the tail, for
* the reasons described there. I'm not really sure
* how much sense this makes. Presumably I had a good
@@ -319,7 +352,7 @@ __sync_single_inode(struct inode *inode, struct writeback_control *wbc)
if (wbc->for_kupdate) {
/*
* For the kupdate function we move the inode
- * to s_more_io so it will get more writeout as
+ * to b_more_io so it will get more writeout as
* soon as the queue becomes uncongested.
*/
inode->i_state |= I_DIRTY_PAGES;
@@ -385,10 +418,10 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
if ((wbc->sync_mode != WB_SYNC_ALL) && (inode->i_state & I_SYNC)) {
/*
* We're skipping this inode because it's locked, and we're not
- * doing writeback-for-data-integrity. Move it to s_more_io so
- * that writeback can proceed with the other inodes on s_io.
+ * doing writeback-for-data-integrity. Move it to b_more_io so
+ * that writeback can proceed with the other inodes on b_io.
* We'll have another go at writing back this inode when we
- * completed a full scan of s_io.
+ * completed a full scan of b_io.
*/
requeue_io(inode);
return 0;
@@ -411,51 +444,24 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
return __sync_single_inode(inode, wbc);
}
-/*
- * Write out a superblock's list of dirty inodes. A wait will be performed
- * upon no inodes, all inodes or the final one, depending upon sync_mode.
- *
- * If older_than_this is non-NULL, then only write out inodes which
- * had their first dirtying at a time earlier than *older_than_this.
- *
- * If we're a pdlfush thread, then implement pdflush collision avoidance
- * against the entire list.
- *
- * If `bdi' is non-zero then we're being asked to writeback a specific queue.
- * This function assumes that the blockdev superblock's inodes are backed by
- * a variety of queues, so all inodes are searched. For other superblocks,
- * assume that all inodes are backed by the same queue.
- *
- * FIXME: this linear search could get expensive with many fileystems. But
- * how to fix? We need to go from an address_space to all inodes which share
- * a queue with that address_space. (Easy: have a global "dirty superblocks"
- * list).
- *
- * The inodes to be written are parked on sb->s_io. They are moved back onto
- * sb->s_dirty as they are selected for writing. This way, none can be missed
- * on the writer throttling path, and we get decent balancing between many
- * throttled threads: we don't want them all piling up on inode_sync_wait.
- */
-void generic_sync_sb_inodes(struct super_block *sb,
- struct writeback_control *wbc)
+static void generic_sync_bdi_inodes(struct backing_dev_info *bdi,
+ struct writeback_control *wbc,
+ int is_blkdev)
{
const unsigned long start = jiffies; /* livelock avoidance */
- int sync = wbc->sync_mode == WB_SYNC_ALL;
spin_lock(&inode_lock);
- if (!wbc->for_kupdate || list_empty(&sb->s_io))
- queue_io(sb, wbc->older_than_this);
+ if (!wbc->for_kupdate || list_empty(&bdi->b_io))
+ queue_io(bdi, wbc->older_than_this);
- while (!list_empty(&sb->s_io)) {
- struct inode *inode = list_entry(sb->s_io.prev,
+ while (!list_empty(&bdi->b_io)) {
+ struct inode *inode = list_entry(bdi->b_io.prev,
struct inode, i_list);
- struct address_space *mapping = inode->i_mapping;
- struct backing_dev_info *bdi = mapping->backing_dev_info;
long pages_skipped;
if (!bdi_cap_writeback_dirty(bdi)) {
redirty_tail(inode);
- if (sb_is_blkdev_sb(sb)) {
+ if (is_blkdev) {
/*
* Dirty memory-backed blockdev: the ramdisk
* driver does this. Skip just this inode
@@ -472,14 +478,14 @@ void generic_sync_sb_inodes(struct super_block *sb,
if (wbc->nonblocking && bdi_write_congested(bdi)) {
wbc->encountered_congestion = 1;
- if (!sb_is_blkdev_sb(sb))
+ if (!is_blkdev)
break; /* Skip a congested fs */
requeue_io(inode);
continue; /* Skip a congested blockdev */
}
if (wbc->bdi && bdi != wbc->bdi) {
- if (!sb_is_blkdev_sb(sb))
+ if (!is_blkdev)
break; /* fs has the wrong queue */
requeue_io(inode);
continue; /* blockdev has wrong queue */
@@ -514,13 +520,55 @@ void generic_sync_sb_inodes(struct super_block *sb,
wbc->more_io = 1;
break;
}
- if (!list_empty(&sb->s_more_io))
+ if (!list_empty(&bdi->b_more_io))
wbc->more_io = 1;
}
- if (sync) {
+ spin_unlock(&inode_lock);
+ /* Leave any unwritten inodes on b_io */
+}
+
+/*
+ * Write out a superblock's list of dirty inodes. A wait will be performed
+ * upon no inodes, all inodes or the final one, depending upon sync_mode.
+ *
+ * If older_than_this is non-NULL, then only write out inodes which
+ * had their first dirtying at a time earlier than *older_than_this.
+ *
+ * If we're a pdlfush thread, then implement pdflush collision avoidance
+ * against the entire list.
+ *
+ * If `bdi' is non-zero then we're being asked to writeback a specific queue.
+ * This function assumes that the blockdev superblock's inodes are backed by
+ * a variety of queues, so all inodes are searched. For other superblocks,
+ * assume that all inodes are backed by the same queue.
+ *
+ * FIXME: this linear search could get expensive with many fileystems. But
+ * how to fix? We need to go from an address_space to all inodes which share
+ * a queue with that address_space. (Easy: have a global "dirty superblocks"
+ * list).
+ *
+ * The inodes to be written are parked on bdi->b_io. They are moved back onto
+ * bdi->b_dirty as they are selected for writing. This way, none can be missed
+ * on the writer throttling path, and we get decent balancing between many
+ * throttled threads: we don't want them all piling up on inode_sync_wait.
+ */
+void generic_sync_sb_inodes(struct super_block *sb,
+ struct writeback_control *wbc)
+{
+ const int is_blkdev = sb_is_blkdev_sb(sb);
+ struct backing_dev_info *bdi;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
+ generic_sync_bdi_inodes(bdi, wbc, is_blkdev);
+ rcu_read_unlock();
+
+ if (wbc->sync_mode == WB_SYNC_ALL) {
struct inode *inode, *old_inode = NULL;
+ spin_lock(&inode_lock);
+
/*
* Data integrity sync. Must wait for all pages under writeback,
* because there may have been pages dirtied before our sync
@@ -557,10 +605,8 @@ void generic_sync_sb_inodes(struct super_block *sb,
}
spin_unlock(&inode_lock);
iput(old_inode);
- } else
- spin_unlock(&inode_lock);
+ }
- return; /* Leave any unwritten inodes on s_io */
}
EXPORT_SYMBOL_GPL(generic_sync_sb_inodes);
@@ -575,8 +621,8 @@ static void sync_sb_inodes(struct super_block *sb,
*
* Note:
* We don't need to grab a reference to superblock here. If it has non-empty
- * ->s_dirty it's hadn't been killed yet and kill_super() won't proceed
- * past sync_inodes_sb() until the ->s_dirty/s_io/s_more_io lists are all
+ * ->b_dirty it's hadn't been killed yet and kill_super() won't proceed
+ * past sync_inodes_sb() until the ->b_dirty/b_io/b_more_io lists are all
* empty. Since __sync_single_inode() regains inode_lock before it finally moves
* inode from superblock lists we are OK.
*
diff --git a/fs/super.c b/fs/super.c
index 8349ed6..e3c5b6f 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -64,9 +64,6 @@ static struct super_block *alloc_super(struct file_system_type *type)
s = NULL;
goto out;
}
- INIT_LIST_HEAD(&s->s_dirty);
- INIT_LIST_HEAD(&s->s_io);
- INIT_LIST_HEAD(&s->s_more_io);
INIT_LIST_HEAD(&s->s_files);
INIT_LIST_HEAD(&s->s_instances);
INIT_HLIST_HEAD(&s->s_anon);
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index bee52ab..bb58c95 100644
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -40,6 +40,8 @@ enum bdi_stat_item {
#define BDI_STAT_BATCH (8*(1+ilog2(nr_cpu_ids)))
struct backing_dev_info {
+ struct list_head bdi_list;
+
unsigned long ra_pages; /* max readahead in PAGE_CACHE_SIZE units */
unsigned long state; /* Always use atomic bitops on this */
unsigned int capabilities; /* Device capabilities */
@@ -58,6 +60,10 @@ struct backing_dev_info {
struct device *dev;
+ struct list_head b_dirty; /* dirty inodes */
+ struct list_head b_io; /* parked for writeback */
+ struct list_head b_more_io; /* parked for more writeback */
+
#ifdef CONFIG_DEBUG_FS
struct dentry *debug_dir;
struct dentry *debug_stats;
@@ -72,6 +78,9 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev);
void bdi_unregister(struct backing_dev_info *bdi);
+extern spinlock_t bdi_lock;
+extern struct list_head bdi_list;
+
static inline void __add_bdi_stat(struct backing_dev_info *bdi,
enum bdi_stat_item item, s64 amount)
{
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 92734c0..3c90eb4 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -648,7 +648,7 @@ static inline int mapping_writably_mapped(struct address_space *mapping)
struct inode {
struct hlist_node i_hash;
- struct list_head i_list;
+ struct list_head i_list; /* backing dev IO list */
struct list_head i_sb_list;
struct list_head i_dentry;
unsigned long i_ino;
@@ -1155,9 +1155,6 @@ struct super_block {
struct xattr_handler **s_xattr;
struct list_head s_inodes; /* all inodes */
- struct list_head s_dirty; /* dirty inodes */
- struct list_head s_io; /* parked for writeback */
- struct list_head s_more_io; /* parked for more writeback */
struct hlist_head s_anon; /* anonymous dentries for (nfs) exporting */
struct list_head s_files;
/* s_dentry_lru and s_nr_dentry_unused are protected by dcache_lock */
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index 8e85874..cf1528b 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -7,8 +7,9 @@
#include <linux/writeback.h>
#include <linux/device.h>
-
static struct class *bdi_class;
+DEFINE_SPINLOCK(bdi_lock);
+LIST_HEAD(bdi_list);
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
@@ -187,6 +188,10 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
goto exit;
}
+ spin_lock(&bdi_lock);
+ list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
+ spin_unlock(&bdi_lock);
+
bdi->dev = dev;
bdi_debug_register(bdi, dev_name(dev));
@@ -201,9 +206,23 @@ int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
}
EXPORT_SYMBOL(bdi_register_dev);
+static void bdi_remove_from_list(struct backing_dev_info *bdi)
+{
+ spin_lock(&bdi_lock);
+ list_del_rcu(&bdi->bdi_list);
+ spin_unlock(&bdi_lock);
+
+ /*
+ * In case the bdi is freed right after unregister, we need to
+ * make sure any RCU sections have exited
+ */
+ synchronize_rcu();
+}
+
void bdi_unregister(struct backing_dev_info *bdi)
{
if (bdi->dev) {
+ bdi_remove_from_list(bdi);
bdi_debug_unregister(bdi);
device_unregister(bdi->dev);
bdi->dev = NULL;
@@ -221,6 +240,10 @@ int bdi_init(struct backing_dev_info *bdi)
bdi->min_ratio = 0;
bdi->max_ratio = 100;
bdi->max_prop_frac = PROP_FRAC_BASE;
+ INIT_LIST_HEAD(&bdi->bdi_list);
+ INIT_LIST_HEAD(&bdi->b_io);
+ INIT_LIST_HEAD(&bdi->b_dirty);
+ INIT_LIST_HEAD(&bdi->b_more_io);
for (i = 0; i < NR_BDI_STAT_ITEMS; i++) {
err = percpu_counter_init(&bdi->bdi_stat[i], 0);
@@ -235,6 +258,8 @@ int bdi_init(struct backing_dev_info *bdi)
err:
while (i--)
percpu_counter_destroy(&bdi->bdi_stat[i]);
+
+ bdi_remove_from_list(bdi);
}
return err;
@@ -245,6 +270,10 @@ void bdi_destroy(struct backing_dev_info *bdi)
{
int i;
+ WARN_ON(!list_empty(&bdi->b_dirty));
+ WARN_ON(!list_empty(&bdi->b_io));
+ WARN_ON(!list_empty(&bdi->b_more_io));
+
bdi_unregister(bdi);
for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 74dc57c..3ec11d8 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -319,7 +319,6 @@ static void task_dirty_limit(struct task_struct *tsk, long *pdirty)
/*
*
*/
-static DEFINE_SPINLOCK(bdi_lock);
static unsigned int bdi_min_ratio;
int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
--
1.6.2
^ permalink raw reply related [flat|nested] 31+ messages in thread* Re: [PATCH 1/7] writeback: move dirty inodes from super_block to backing_dev_info
2009-03-12 14:33 ` [PATCH 1/7] writeback: move dirty inodes from super_block to backing_dev_info Jens Axboe
@ 2009-03-24 16:17 ` Jan Kara
2009-03-24 18:45 ` Jens Axboe
0 siblings, 1 reply; 31+ messages in thread
From: Jan Kara @ 2009-03-24 16:17 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
> This is a first step at introducing per-bdi flusher threads. We should
> have no change in behaviour, although sb_has_dirty_inodes() is now
> ridiculously expensive, as there's no easy way to answer that question.
> Not a huge problem, since it'll be deleted in subsequent patches.
Could you maybe expand the changelog a bit? If I read the patch right
the only thing it does is that it moves from per-sb inode lists to
per-bdi inode lists, right? Also sync_sb_inodes() now writes all the
inodes in the system, not just the ones for that superblock, doesn't it?
Honza
>
> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
> ---
> fs/fs-writeback.c | 186 +++++++++++++++++++++++++++----------------
> fs/super.c | 3 -
> include/linux/backing-dev.h | 9 ++
> include/linux/fs.h | 5 +-
> mm/backing-dev.c | 31 +++++++-
> mm/page-writeback.c | 1 -
> 6 files changed, 156 insertions(+), 79 deletions(-)
>
> diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
> index e5eaa62..c107cff 100644
> --- a/fs/fs-writeback.c
> +++ b/fs/fs-writeback.c
> @@ -25,6 +25,7 @@
> #include <linux/buffer_head.h>
> #include "internal.h"
>
> +#define inode_to_bdi(inode) (inode)->i_mapping->backing_dev_info
>
> /**
> * writeback_acquire - attempt to get exclusive writeback access to a device
> @@ -158,12 +159,13 @@ void __mark_inode_dirty(struct inode *inode, int flags)
> goto out;
>
> /*
> - * If the inode was already on s_dirty/s_io/s_more_io, don't
> - * reposition it (that would break s_dirty time-ordering).
> + * If the inode was already on b_dirty/b_io/b_more_io, don't
> + * reposition it (that would break b_dirty time-ordering).
> */
> if (!was_dirty) {
> inode->dirtied_when = jiffies;
> - list_move(&inode->i_list, &sb->s_dirty);
> + list_move(&inode->i_list,
> + &inode_to_bdi(inode)->b_dirty);
> }
> }
> out:
> @@ -184,31 +186,30 @@ static int write_inode(struct inode *inode, int sync)
> * furthest end of its superblock's dirty-inode list.
> *
> * Before stamping the inode's ->dirtied_when, we check to see whether it is
> - * already the most-recently-dirtied inode on the s_dirty list. If that is
> + * already the most-recently-dirtied inode on the b_dirty list. If that is
> * the case then the inode must have been redirtied while it was being written
> * out and we don't reset its dirtied_when.
> */
> static void redirty_tail(struct inode *inode)
> {
> - struct super_block *sb = inode->i_sb;
> + struct backing_dev_info *bdi = inode_to_bdi(inode);
>
> - if (!list_empty(&sb->s_dirty)) {
> - struct inode *tail_inode;
> + if (!list_empty(&bdi->b_dirty)) {
> + struct inode *tail;
>
> - tail_inode = list_entry(sb->s_dirty.next, struct inode, i_list);
> - if (!time_after_eq(inode->dirtied_when,
> - tail_inode->dirtied_when))
> + tail = list_entry(bdi->b_dirty.next, struct inode, i_list);
> + if (!time_after_eq(inode->dirtied_when, tail->dirtied_when))
> inode->dirtied_when = jiffies;
> }
> - list_move(&inode->i_list, &sb->s_dirty);
> + list_move(&inode->i_list, &bdi->b_dirty);
> }
>
> /*
> - * requeue inode for re-scanning after sb->s_io list is exhausted.
> + * requeue inode for re-scanning after bdi->b_io list is exhausted.
> */
> static void requeue_io(struct inode *inode)
> {
> - list_move(&inode->i_list, &inode->i_sb->s_more_io);
> + list_move(&inode->i_list, &inode_to_bdi(inode)->b_more_io);
> }
>
> static void inode_sync_complete(struct inode *inode)
> @@ -240,18 +241,50 @@ static void move_expired_inodes(struct list_head *delaying_queue,
> /*
> * Queue all expired dirty inodes for io, eldest first.
> */
> -static void queue_io(struct super_block *sb,
> - unsigned long *older_than_this)
> +static void queue_io(struct backing_dev_info *bdi,
> + unsigned long *older_than_this)
> {
> - list_splice_init(&sb->s_more_io, sb->s_io.prev);
> - move_expired_inodes(&sb->s_dirty, &sb->s_io, older_than_this);
> + list_splice_init(&bdi->b_more_io, bdi->b_io.prev);
> + move_expired_inodes(&bdi->b_dirty, &bdi->b_io, older_than_this);
> +}
> +
> +static int sb_on_inode_list(struct super_block *sb, struct list_head *list)
> +{
> + struct inode *inode;
> + int ret = 0;
> +
> + spin_lock(&inode_lock);
> + list_for_each_entry(inode, list, i_list) {
> + if (inode->i_sb == sb) {
> + ret = 1;
> + break;
> + }
> + }
> + spin_unlock(&inode_lock);
> + return ret;
> }
>
> int sb_has_dirty_inodes(struct super_block *sb)
> {
> - return !list_empty(&sb->s_dirty) ||
> - !list_empty(&sb->s_io) ||
> - !list_empty(&sb->s_more_io);
> + struct backing_dev_info *bdi;
> + int ret = 0;
> +
> + /*
> + * This is REALLY expensive right now, but it'll go away
> + * when the bdi writeback is introduced
> + */
> + rcu_read_lock();
> + list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
> + if (sb_on_inode_list(sb, &bdi->b_dirty) ||
> + sb_on_inode_list(sb, &bdi->b_io) ||
> + sb_on_inode_list(sb, &bdi->b_more_io)) {
> + ret = 1;
> + break;
> + }
> + }
> + rcu_read_unlock();
> +
> + return ret;
> }
> EXPORT_SYMBOL(sb_has_dirty_inodes);
>
> @@ -305,11 +338,11 @@ __sync_single_inode(struct inode *inode, struct writeback_control *wbc)
> /*
> * We didn't write back all the pages. nfs_writepages()
> * sometimes bales out without doing anything. Redirty
> - * the inode; Move it from s_io onto s_more_io/s_dirty.
> + * the inode; Move it from b_io onto b_more_io/b_dirty.
> */
> /*
> * akpm: if the caller was the kupdate function we put
> - * this inode at the head of s_dirty so it gets first
> + * this inode at the head of b_dirty so it gets first
> * consideration. Otherwise, move it to the tail, for
> * the reasons described there. I'm not really sure
> * how much sense this makes. Presumably I had a good
> @@ -319,7 +352,7 @@ __sync_single_inode(struct inode *inode, struct writeback_control *wbc)
> if (wbc->for_kupdate) {
> /*
> * For the kupdate function we move the inode
> - * to s_more_io so it will get more writeout as
> + * to b_more_io so it will get more writeout as
> * soon as the queue becomes uncongested.
> */
> inode->i_state |= I_DIRTY_PAGES;
> @@ -385,10 +418,10 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
> if ((wbc->sync_mode != WB_SYNC_ALL) && (inode->i_state & I_SYNC)) {
> /*
> * We're skipping this inode because it's locked, and we're not
> - * doing writeback-for-data-integrity. Move it to s_more_io so
> - * that writeback can proceed with the other inodes on s_io.
> + * doing writeback-for-data-integrity. Move it to b_more_io so
> + * that writeback can proceed with the other inodes on b_io.
> * We'll have another go at writing back this inode when we
> - * completed a full scan of s_io.
> + * completed a full scan of b_io.
> */
> requeue_io(inode);
> return 0;
> @@ -411,51 +444,24 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
> return __sync_single_inode(inode, wbc);
> }
>
> -/*
> - * Write out a superblock's list of dirty inodes. A wait will be performed
> - * upon no inodes, all inodes or the final one, depending upon sync_mode.
> - *
> - * If older_than_this is non-NULL, then only write out inodes which
> - * had their first dirtying at a time earlier than *older_than_this.
> - *
> - * If we're a pdlfush thread, then implement pdflush collision avoidance
> - * against the entire list.
> - *
> - * If `bdi' is non-zero then we're being asked to writeback a specific queue.
> - * This function assumes that the blockdev superblock's inodes are backed by
> - * a variety of queues, so all inodes are searched. For other superblocks,
> - * assume that all inodes are backed by the same queue.
> - *
> - * FIXME: this linear search could get expensive with many fileystems. But
> - * how to fix? We need to go from an address_space to all inodes which share
> - * a queue with that address_space. (Easy: have a global "dirty superblocks"
> - * list).
> - *
> - * The inodes to be written are parked on sb->s_io. They are moved back onto
> - * sb->s_dirty as they are selected for writing. This way, none can be missed
> - * on the writer throttling path, and we get decent balancing between many
> - * throttled threads: we don't want them all piling up on inode_sync_wait.
> - */
> -void generic_sync_sb_inodes(struct super_block *sb,
> - struct writeback_control *wbc)
> +static void generic_sync_bdi_inodes(struct backing_dev_info *bdi,
> + struct writeback_control *wbc,
> + int is_blkdev)
> {
> const unsigned long start = jiffies; /* livelock avoidance */
> - int sync = wbc->sync_mode == WB_SYNC_ALL;
>
> spin_lock(&inode_lock);
> - if (!wbc->for_kupdate || list_empty(&sb->s_io))
> - queue_io(sb, wbc->older_than_this);
> + if (!wbc->for_kupdate || list_empty(&bdi->b_io))
> + queue_io(bdi, wbc->older_than_this);
>
> - while (!list_empty(&sb->s_io)) {
> - struct inode *inode = list_entry(sb->s_io.prev,
> + while (!list_empty(&bdi->b_io)) {
> + struct inode *inode = list_entry(bdi->b_io.prev,
> struct inode, i_list);
> - struct address_space *mapping = inode->i_mapping;
> - struct backing_dev_info *bdi = mapping->backing_dev_info;
> long pages_skipped;
>
> if (!bdi_cap_writeback_dirty(bdi)) {
> redirty_tail(inode);
> - if (sb_is_blkdev_sb(sb)) {
> + if (is_blkdev) {
> /*
> * Dirty memory-backed blockdev: the ramdisk
> * driver does this. Skip just this inode
> @@ -472,14 +478,14 @@ void generic_sync_sb_inodes(struct super_block *sb,
>
> if (wbc->nonblocking && bdi_write_congested(bdi)) {
> wbc->encountered_congestion = 1;
> - if (!sb_is_blkdev_sb(sb))
> + if (!is_blkdev)
> break; /* Skip a congested fs */
> requeue_io(inode);
> continue; /* Skip a congested blockdev */
> }
>
> if (wbc->bdi && bdi != wbc->bdi) {
> - if (!sb_is_blkdev_sb(sb))
> + if (!is_blkdev)
> break; /* fs has the wrong queue */
> requeue_io(inode);
> continue; /* blockdev has wrong queue */
> @@ -514,13 +520,55 @@ void generic_sync_sb_inodes(struct super_block *sb,
> wbc->more_io = 1;
> break;
> }
> - if (!list_empty(&sb->s_more_io))
> + if (!list_empty(&bdi->b_more_io))
> wbc->more_io = 1;
> }
>
> - if (sync) {
> + spin_unlock(&inode_lock);
> + /* Leave any unwritten inodes on b_io */
> +}
> +
> +/*
> + * Write out a superblock's list of dirty inodes. A wait will be performed
> + * upon no inodes, all inodes or the final one, depending upon sync_mode.
> + *
> + * If older_than_this is non-NULL, then only write out inodes which
> + * had their first dirtying at a time earlier than *older_than_this.
> + *
> + * If we're a pdlfush thread, then implement pdflush collision avoidance
> + * against the entire list.
> + *
> + * If `bdi' is non-zero then we're being asked to writeback a specific queue.
> + * This function assumes that the blockdev superblock's inodes are backed by
> + * a variety of queues, so all inodes are searched. For other superblocks,
> + * assume that all inodes are backed by the same queue.
> + *
> + * FIXME: this linear search could get expensive with many fileystems. But
> + * how to fix? We need to go from an address_space to all inodes which share
> + * a queue with that address_space. (Easy: have a global "dirty superblocks"
> + * list).
> + *
> + * The inodes to be written are parked on bdi->b_io. They are moved back onto
> + * bdi->b_dirty as they are selected for writing. This way, none can be missed
> + * on the writer throttling path, and we get decent balancing between many
> + * throttled threads: we don't want them all piling up on inode_sync_wait.
> + */
> +void generic_sync_sb_inodes(struct super_block *sb,
> + struct writeback_control *wbc)
> +{
> + const int is_blkdev = sb_is_blkdev_sb(sb);
> + struct backing_dev_info *bdi;
> +
> + rcu_read_lock();
> + list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
> + generic_sync_bdi_inodes(bdi, wbc, is_blkdev);
> + rcu_read_unlock();
> +
> + if (wbc->sync_mode == WB_SYNC_ALL) {
> struct inode *inode, *old_inode = NULL;
>
> + spin_lock(&inode_lock);
> +
> /*
> * Data integrity sync. Must wait for all pages under writeback,
> * because there may have been pages dirtied before our sync
> @@ -557,10 +605,8 @@ void generic_sync_sb_inodes(struct super_block *sb,
> }
> spin_unlock(&inode_lock);
> iput(old_inode);
> - } else
> - spin_unlock(&inode_lock);
> + }
>
> - return; /* Leave any unwritten inodes on s_io */
> }
> EXPORT_SYMBOL_GPL(generic_sync_sb_inodes);
>
> @@ -575,8 +621,8 @@ static void sync_sb_inodes(struct super_block *sb,
> *
> * Note:
> * We don't need to grab a reference to superblock here. If it has non-empty
> - * ->s_dirty it's hadn't been killed yet and kill_super() won't proceed
> - * past sync_inodes_sb() until the ->s_dirty/s_io/s_more_io lists are all
> + * ->b_dirty it's hadn't been killed yet and kill_super() won't proceed
> + * past sync_inodes_sb() until the ->b_dirty/b_io/b_more_io lists are all
> * empty. Since __sync_single_inode() regains inode_lock before it finally moves
> * inode from superblock lists we are OK.
> *
> diff --git a/fs/super.c b/fs/super.c
> index 8349ed6..e3c5b6f 100644
> --- a/fs/super.c
> +++ b/fs/super.c
> @@ -64,9 +64,6 @@ static struct super_block *alloc_super(struct file_system_type *type)
> s = NULL;
> goto out;
> }
> - INIT_LIST_HEAD(&s->s_dirty);
> - INIT_LIST_HEAD(&s->s_io);
> - INIT_LIST_HEAD(&s->s_more_io);
> INIT_LIST_HEAD(&s->s_files);
> INIT_LIST_HEAD(&s->s_instances);
> INIT_HLIST_HEAD(&s->s_anon);
> diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
> index bee52ab..bb58c95 100644
> --- a/include/linux/backing-dev.h
> +++ b/include/linux/backing-dev.h
> @@ -40,6 +40,8 @@ enum bdi_stat_item {
> #define BDI_STAT_BATCH (8*(1+ilog2(nr_cpu_ids)))
>
> struct backing_dev_info {
> + struct list_head bdi_list;
> +
> unsigned long ra_pages; /* max readahead in PAGE_CACHE_SIZE units */
> unsigned long state; /* Always use atomic bitops on this */
> unsigned int capabilities; /* Device capabilities */
> @@ -58,6 +60,10 @@ struct backing_dev_info {
>
> struct device *dev;
>
> + struct list_head b_dirty; /* dirty inodes */
> + struct list_head b_io; /* parked for writeback */
> + struct list_head b_more_io; /* parked for more writeback */
> +
> #ifdef CONFIG_DEBUG_FS
> struct dentry *debug_dir;
> struct dentry *debug_stats;
> @@ -72,6 +78,9 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
> int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev);
> void bdi_unregister(struct backing_dev_info *bdi);
>
> +extern spinlock_t bdi_lock;
> +extern struct list_head bdi_list;
> +
> static inline void __add_bdi_stat(struct backing_dev_info *bdi,
> enum bdi_stat_item item, s64 amount)
> {
> diff --git a/include/linux/fs.h b/include/linux/fs.h
> index 92734c0..3c90eb4 100644
> --- a/include/linux/fs.h
> +++ b/include/linux/fs.h
> @@ -648,7 +648,7 @@ static inline int mapping_writably_mapped(struct address_space *mapping)
>
> struct inode {
> struct hlist_node i_hash;
> - struct list_head i_list;
> + struct list_head i_list; /* backing dev IO list */
> struct list_head i_sb_list;
> struct list_head i_dentry;
> unsigned long i_ino;
> @@ -1155,9 +1155,6 @@ struct super_block {
> struct xattr_handler **s_xattr;
>
> struct list_head s_inodes; /* all inodes */
> - struct list_head s_dirty; /* dirty inodes */
> - struct list_head s_io; /* parked for writeback */
> - struct list_head s_more_io; /* parked for more writeback */
> struct hlist_head s_anon; /* anonymous dentries for (nfs) exporting */
> struct list_head s_files;
> /* s_dentry_lru and s_nr_dentry_unused are protected by dcache_lock */
> diff --git a/mm/backing-dev.c b/mm/backing-dev.c
> index 8e85874..cf1528b 100644
> --- a/mm/backing-dev.c
> +++ b/mm/backing-dev.c
> @@ -7,8 +7,9 @@
> #include <linux/writeback.h>
> #include <linux/device.h>
>
> -
> static struct class *bdi_class;
> +DEFINE_SPINLOCK(bdi_lock);
> +LIST_HEAD(bdi_list);
>
> #ifdef CONFIG_DEBUG_FS
> #include <linux/debugfs.h>
> @@ -187,6 +188,10 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
> goto exit;
> }
>
> + spin_lock(&bdi_lock);
> + list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
> + spin_unlock(&bdi_lock);
> +
> bdi->dev = dev;
> bdi_debug_register(bdi, dev_name(dev));
>
> @@ -201,9 +206,23 @@ int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
> }
> EXPORT_SYMBOL(bdi_register_dev);
>
> +static void bdi_remove_from_list(struct backing_dev_info *bdi)
> +{
> + spin_lock(&bdi_lock);
> + list_del_rcu(&bdi->bdi_list);
> + spin_unlock(&bdi_lock);
> +
> + /*
> + * In case the bdi is freed right after unregister, we need to
> + * make sure any RCU sections have exited
> + */
> + synchronize_rcu();
> +}
> +
> void bdi_unregister(struct backing_dev_info *bdi)
> {
> if (bdi->dev) {
> + bdi_remove_from_list(bdi);
> bdi_debug_unregister(bdi);
> device_unregister(bdi->dev);
> bdi->dev = NULL;
> @@ -221,6 +240,10 @@ int bdi_init(struct backing_dev_info *bdi)
> bdi->min_ratio = 0;
> bdi->max_ratio = 100;
> bdi->max_prop_frac = PROP_FRAC_BASE;
> + INIT_LIST_HEAD(&bdi->bdi_list);
> + INIT_LIST_HEAD(&bdi->b_io);
> + INIT_LIST_HEAD(&bdi->b_dirty);
> + INIT_LIST_HEAD(&bdi->b_more_io);
>
> for (i = 0; i < NR_BDI_STAT_ITEMS; i++) {
> err = percpu_counter_init(&bdi->bdi_stat[i], 0);
> @@ -235,6 +258,8 @@ int bdi_init(struct backing_dev_info *bdi)
> err:
> while (i--)
> percpu_counter_destroy(&bdi->bdi_stat[i]);
> +
> + bdi_remove_from_list(bdi);
> }
>
> return err;
> @@ -245,6 +270,10 @@ void bdi_destroy(struct backing_dev_info *bdi)
> {
> int i;
>
> + WARN_ON(!list_empty(&bdi->b_dirty));
> + WARN_ON(!list_empty(&bdi->b_io));
> + WARN_ON(!list_empty(&bdi->b_more_io));
> +
> bdi_unregister(bdi);
>
> for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
> diff --git a/mm/page-writeback.c b/mm/page-writeback.c
> index 74dc57c..3ec11d8 100644
> --- a/mm/page-writeback.c
> +++ b/mm/page-writeback.c
> @@ -319,7 +319,6 @@ static void task_dirty_limit(struct task_struct *tsk, long *pdirty)
> /*
> *
> */
> -static DEFINE_SPINLOCK(bdi_lock);
> static unsigned int bdi_min_ratio;
>
> int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
> --
> 1.6.2
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
--
Jan Kara <jack@suse.cz>
SuSE CR Labs
^ permalink raw reply [flat|nested] 31+ messages in thread* Re: [PATCH 1/7] writeback: move dirty inodes from super_block to backing_dev_info
2009-03-24 16:17 ` Jan Kara
@ 2009-03-24 18:45 ` Jens Axboe
0 siblings, 0 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-24 18:45 UTC (permalink / raw)
To: Jan Kara; +Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
On Tue, Mar 24 2009, Jan Kara wrote:
> > This is a first step at introducing per-bdi flusher threads. We should
> > have no change in behaviour, although sb_has_dirty_inodes() is now
> > ridiculously expensive, as there's no easy way to answer that question.
> > Not a huge problem, since it'll be deleted in subsequent patches.
> Could you maybe expand the changelog a bit? If I read the patch right
> the only thing it does is that it moves from per-sb inode lists to
> per-bdi inode lists, right? Also sync_sb_inodes() now writes all the
> inodes in the system, not just the ones for that superblock, doesn't it?
That is correct, it just moves the dirty lists to the bdi instead of
keeping them in the superblock. It does appear that this intermediate
step doesn't honor the sb passed in, later in the series it works
though. I'll get that fixed up, as the changelog mentions we should not
have much change in behaviour at this point :-)
--
Jens Axboe
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-12 14:33 [PATCH 0/7] Per-bdi writeback flusher threads Jens Axboe
2009-03-12 14:33 ` [PATCH 1/7] writeback: move dirty inodes from super_block to backing_dev_info Jens Axboe
@ 2009-03-12 14:33 ` Jens Axboe
2009-03-13 5:33 ` Andrew Morton
2009-03-16 10:22 ` Christoph Hellwig
2009-03-12 14:33 ` [PATCH 3/7] writeback: get rid of pdflush_operation() in emergency sync and remount Jens Axboe
` (4 subsequent siblings)
6 siblings, 2 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-12 14:33 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel; +Cc: chris.mason, david, npiggin, Jens Axboe
This gets rid of pdflush for bdi writeout and kupdated style cleaning.
This is an experiment to see if we get better writeout behaviour with
per-bdi flushing. Some initial tests look pretty encouraging. A sample
ffsb workload that does random writes to files is about 8% faster here
on a simple SATA drive during the benchmark phase. File layout also seems
a LOT more smooth in vmstat:
r b swpd free buff cache si so bi bo in cs us sy id wa
0 1 0 608848 2652 375372 0 0 0 71024 604 24 1 10 48 42
0 1 0 549644 2712 433736 0 0 0 60692 505 27 1 8 48 44
1 0 0 476928 2784 505192 0 0 4 29540 553 24 0 9 53 37
0 1 0 457972 2808 524008 0 0 0 54876 331 16 0 4 38 58
0 1 0 366128 2928 614284 0 0 4 92168 710 58 0 13 53 34
0 1 0 295092 3000 684140 0 0 0 62924 572 23 0 9 53 37
0 1 0 236592 3064 741704 0 0 4 58256 523 17 0 8 48 44
0 1 0 165608 3132 811464 0 0 0 57460 560 21 0 8 54 38
0 1 0 102952 3200 873164 0 0 4 74748 540 29 1 10 48 41
0 1 0 48604 3252 926472 0 0 0 53248 469 29 0 7 47 45
where vanilla tends to fluctuate a lot in the creation phase:
r b swpd free buff cache si so bi bo in cs us sy id wa
1 1 0 678716 5792 303380 0 0 0 74064 565 50 1 11 52 36
1 0 0 662488 5864 319396 0 0 4 352 302 329 0 2 47 51
0 1 0 599312 5924 381468 0 0 0 78164 516 55 0 9 51 40
0 1 0 519952 6008 459516 0 0 4 78156 622 56 1 11 52 37
1 1 0 436640 6092 541632 0 0 0 82244 622 54 0 11 48 41
0 1 0 436640 6092 541660 0 0 0 8 152 39 0 0 51 49
0 1 0 332224 6200 644252 0 0 4 102800 728 46 1 13 49 36
1 0 0 274492 6260 701056 0 0 4 12328 459 49 0 7 50 43
0 1 0 211220 6324 763356 0 0 0 106940 515 37 1 10 51 39
1 0 0 160412 6376 813468 0 0 0 8224 415 43 0 6 49 45
1 1 0 85980 6452 886556 0 0 4 113516 575 39 1 11 54 34
0 2 0 85968 6452 886620 0 0 0 1640 158 211 0 0 46 54
So apart from seemingly behaving better for buffered writeout, this also
allows us to potentially have more than one bdi thread flushing out data.
This may be useful for NUMA type setups.
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
fs/buffer.c | 2 +-
fs/fs-writeback.c | 318 ++++++++++++++++++++++++++-----------------
fs/ntfs/super.c | 32 +----
fs/sync.c | 2 +-
include/linux/backing-dev.h | 19 +++
include/linux/fs.h | 3 +-
include/linux/writeback.h | 2 +-
mm/backing-dev.c | 17 ++-
mm/page-writeback.c | 139 +------------------
mm/vmscan.c | 2 +-
10 files changed, 244 insertions(+), 292 deletions(-)
diff --git a/fs/buffer.c b/fs/buffer.c
index 9f69741..89a26bc 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -426,7 +426,7 @@ static void free_more_memory(void)
struct zone *zone;
int nid;
- wakeup_pdflush(1024);
+ wakeup_flusher_threads(1024);
yield();
for_each_online_node(nid) {
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index c107cff..209c6fa 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -19,6 +19,8 @@
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/mm.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
@@ -61,10 +63,184 @@ int writeback_in_progress(struct backing_dev_info *bdi)
*/
static void writeback_release(struct backing_dev_info *bdi)
{
- BUG_ON(!writeback_in_progress(bdi));
+ WARN_ON_ONCE(!writeback_in_progress(bdi));
+ bdi->wb_arg.nr_pages = 0;
+ bdi->wb_arg.sb = NULL;
clear_bit(BDI_pdflush, &bdi->state);
}
+void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
+ long nr_pages)
+{
+ /*
+ * Should not happen, complain?
+ */
+ if (unlikely(!bdi->task))
+ return;
+
+ if (writeback_acquire(bdi)) {
+ bdi->wb_arg.nr_pages = nr_pages;
+ bdi->wb_arg.sb = sb;
+ /*
+ * make above store seen before the task is woken
+ */
+ smp_mb();
+ wake_up(&bdi->wait);
+ }
+}
+
+/*
+ * The maximum number of pages to writeout in a single bdi flush/kupdate
+ * operation. We do this so we don't hold I_SYNC against an inode for
+ * enormous amounts of time, which would block a userspace task which has
+ * been forced to throttle against that inode. Also, the code reevaluates
+ * the dirty each time it has written this many pages.
+ */
+#define MAX_WRITEBACK_PAGES 1024
+
+/*
+ * Periodic writeback of "old" data.
+ *
+ * Define "old": the first time one of an inode's pages is dirtied, we mark the
+ * dirtying-time in the inode's address_space. So this periodic writeback code
+ * just walks the superblock inode list, writing back any inodes which are
+ * older than a specific point in time.
+ *
+ * Try to run once per dirty_writeback_interval. But if a writeback event
+ * takes longer than a dirty_writeback_interval interval, then leave a
+ * one-second gap.
+ *
+ * older_than_this takes precedence over nr_to_write. So we'll only write back
+ * all dirty pages if they are all attached to "old" mappings.
+ */
+static void bdi_kupdated(struct backing_dev_info *bdi)
+{
+ long nr_to_write;
+ struct writeback_control wbc = {
+ .bdi = bdi,
+ .sync_mode = WB_SYNC_NONE,
+ .nr_to_write = 0,
+ .for_kupdate = 1,
+ .range_cyclic = 1,
+ };
+
+ sync_supers();
+
+ nr_to_write = global_page_state(NR_FILE_DIRTY) +
+ global_page_state(NR_UNSTABLE_NFS) +
+ (inodes_stat.nr_inodes - inodes_stat.nr_unused);
+
+ while (nr_to_write > 0) {
+ wbc.more_io = 0;
+ wbc.encountered_congestion = 0;
+ wbc.nr_to_write = MAX_WRITEBACK_PAGES;
+ generic_sync_bdi_inodes(NULL, &wbc);
+ if (wbc.nr_to_write > 0)
+ break; /* All the old data is written */
+ nr_to_write -= MAX_WRITEBACK_PAGES;
+ }
+}
+
+static void bdi_pdflush(struct backing_dev_info *bdi)
+{
+ struct writeback_control wbc = {
+ .bdi = bdi,
+ .sync_mode = WB_SYNC_NONE,
+ .older_than_this = NULL,
+ .range_cyclic = 1,
+ };
+ long nr_pages = bdi->wb_arg.nr_pages;
+
+ for (;;) {
+ unsigned long background_thresh, dirty_thresh;
+ get_dirty_limits(&background_thresh, &dirty_thresh, NULL, NULL);
+ if ((global_page_state(NR_FILE_DIRTY) +
+ global_page_state(NR_UNSTABLE_NFS) < background_thresh) &&
+ nr_pages <= 0)
+ break;
+
+ wbc.more_io = 0;
+ wbc.encountered_congestion = 0;
+ wbc.nr_to_write = MAX_WRITEBACK_PAGES;
+ wbc.pages_skipped = 0;
+ generic_sync_bdi_inodes(bdi->wb_arg.sb, &wbc);
+ nr_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
+ /*
+ * If we ran out of stuff to write, bail unless more_io got set
+ */
+ if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) {
+ if (wbc.more_io)
+ continue;
+ break;
+ }
+ }
+}
+
+/*
+ * Handle writeback of dirty data for the device backed by this bdi. Also
+ * wakes up periodically and does kupdated style flushing.
+ */
+int bdi_writeback_task(void *ptr)
+{
+ struct backing_dev_info *bdi = ptr;
+ struct task_struct *tsk = current;
+
+ tsk->flags |= PF_FLUSHER | PF_SWAPWRITE;
+ set_freezable();
+
+ /*
+ * Our parent may run at a different priority, just set us to normal
+ */
+ set_user_nice(tsk, 0);
+
+ while (!kthread_should_stop()) {
+ DECLARE_WAITQUEUE(wait, tsk);
+
+ add_wait_queue(&bdi->wait, &wait);
+ set_task_state(tsk, TASK_INTERRUPTIBLE);
+ schedule_timeout(dirty_writeback_interval);
+ try_to_freeze();
+
+ /*
+ * We get here in two cases:
+ *
+ * schedule_timeout() returned because the dirty writeback
+ * interval has elapsed. If that happens, we will be able
+ * to acquire the writeback lock and will proceed to do
+ * kupdated style writeout.
+ *
+ * Someone called bdi_start_writeback(), which will acquire
+ * the writeback lock. This means our writeback_acquire()
+ * below will fail and we call into bdi_pdflush() for
+ * pdflush style writeout.
+ *
+ */
+ if (writeback_acquire(bdi))
+ bdi_kupdated(bdi);
+ else
+ bdi_pdflush(bdi);
+
+ writeback_release(bdi);
+ set_task_state(tsk, TASK_RUNNING);
+ finish_wait(&bdi->wait, &wait);
+ }
+
+ return 0;
+}
+
+void bdi_writeback_all(struct super_block *sb, long nr_pages)
+{
+ struct backing_dev_info *bdi;
+
+ rcu_read_lock();
+
+ list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
+ if (bdi_has_dirty_io(bdi))
+ bdi_start_writeback(bdi, sb, nr_pages);
+
+ rcu_read_unlock();
+}
+
/**
* __mark_inode_dirty - internal function
* @inode: inode to mark
@@ -248,46 +424,6 @@ static void queue_io(struct backing_dev_info *bdi,
move_expired_inodes(&bdi->b_dirty, &bdi->b_io, older_than_this);
}
-static int sb_on_inode_list(struct super_block *sb, struct list_head *list)
-{
- struct inode *inode;
- int ret = 0;
-
- spin_lock(&inode_lock);
- list_for_each_entry(inode, list, i_list) {
- if (inode->i_sb == sb) {
- ret = 1;
- break;
- }
- }
- spin_unlock(&inode_lock);
- return ret;
-}
-
-int sb_has_dirty_inodes(struct super_block *sb)
-{
- struct backing_dev_info *bdi;
- int ret = 0;
-
- /*
- * This is REALLY expensive right now, but it'll go away
- * when the bdi writeback is introduced
- */
- rcu_read_lock();
- list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
- if (sb_on_inode_list(sb, &bdi->b_dirty) ||
- sb_on_inode_list(sb, &bdi->b_io) ||
- sb_on_inode_list(sb, &bdi->b_more_io)) {
- ret = 1;
- break;
- }
- }
- rcu_read_unlock();
-
- return ret;
-}
-EXPORT_SYMBOL(sb_has_dirty_inodes);
-
/*
* Write a single inode's dirty pages and inode data out to disk.
* If `wait' is set, wait on the writeout.
@@ -444,10 +580,11 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
return __sync_single_inode(inode, wbc);
}
-static void generic_sync_bdi_inodes(struct backing_dev_info *bdi,
- struct writeback_control *wbc,
- int is_blkdev)
+void generic_sync_bdi_inodes(struct super_block *sb,
+ struct writeback_control *wbc)
{
+ const int is_blkdev_sb = sb_is_blkdev_sb(sb);
+ struct backing_dev_info *bdi = wbc->bdi;
const unsigned long start = jiffies; /* livelock avoidance */
spin_lock(&inode_lock);
@@ -459,9 +596,17 @@ static void generic_sync_bdi_inodes(struct backing_dev_info *bdi,
struct inode, i_list);
long pages_skipped;
+ /*
+ * super block given and doesn't match, skip this inode
+ */
+ if (sb && sb != inode->i_sb) {
+ redirty_tail(inode);
+ continue;
+ }
+
if (!bdi_cap_writeback_dirty(bdi)) {
redirty_tail(inode);
- if (is_blkdev) {
+ if (is_blkdev_sb) {
/*
* Dirty memory-backed blockdev: the ramdisk
* driver does this. Skip just this inode
@@ -478,33 +623,20 @@ static void generic_sync_bdi_inodes(struct backing_dev_info *bdi,
if (wbc->nonblocking && bdi_write_congested(bdi)) {
wbc->encountered_congestion = 1;
- if (!is_blkdev)
+ if (!is_blkdev_sb)
break; /* Skip a congested fs */
requeue_io(inode);
continue; /* Skip a congested blockdev */
}
- if (wbc->bdi && bdi != wbc->bdi) {
- if (!is_blkdev)
- break; /* fs has the wrong queue */
- requeue_io(inode);
- continue; /* blockdev has wrong queue */
- }
-
/* Was this inode dirtied after sync_sb_inodes was called? */
if (time_after(inode->dirtied_when, start))
break;
- /* Is another pdflush already flushing this queue? */
- if (current_is_pdflush() && !writeback_acquire(bdi))
- break;
-
BUG_ON(inode->i_state & I_FREEING);
__iget(inode);
pages_skipped = wbc->pages_skipped;
__writeback_single_inode(inode, wbc);
- if (current_is_pdflush())
- writeback_release(bdi);
if (wbc->pages_skipped != pages_skipped) {
/*
* writeback is not making progress due to locked
@@ -543,11 +675,6 @@ static void generic_sync_bdi_inodes(struct backing_dev_info *bdi,
* a variety of queues, so all inodes are searched. For other superblocks,
* assume that all inodes are backed by the same queue.
*
- * FIXME: this linear search could get expensive with many fileystems. But
- * how to fix? We need to go from an address_space to all inodes which share
- * a queue with that address_space. (Easy: have a global "dirty superblocks"
- * list).
- *
* The inodes to be written are parked on bdi->b_io. They are moved back onto
* bdi->b_dirty as they are selected for writing. This way, none can be missed
* on the writer throttling path, and we get decent balancing between many
@@ -556,13 +683,10 @@ static void generic_sync_bdi_inodes(struct backing_dev_info *bdi,
void generic_sync_sb_inodes(struct super_block *sb,
struct writeback_control *wbc)
{
- const int is_blkdev = sb_is_blkdev_sb(sb);
- struct backing_dev_info *bdi;
-
- rcu_read_lock();
- list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
- generic_sync_bdi_inodes(bdi, wbc, is_blkdev);
- rcu_read_unlock();
+ if (wbc->bdi)
+ bdi_start_writeback(wbc->bdi, sb, 0);
+ else
+ bdi_writeback_all(sb, 0);
if (wbc->sync_mode == WB_SYNC_ALL) {
struct inode *inode, *old_inode = NULL;
@@ -617,58 +741,6 @@ static void sync_sb_inodes(struct super_block *sb,
}
/*
- * Start writeback of dirty pagecache data against all unlocked inodes.
- *
- * Note:
- * We don't need to grab a reference to superblock here. If it has non-empty
- * ->b_dirty it's hadn't been killed yet and kill_super() won't proceed
- * past sync_inodes_sb() until the ->b_dirty/b_io/b_more_io lists are all
- * empty. Since __sync_single_inode() regains inode_lock before it finally moves
- * inode from superblock lists we are OK.
- *
- * If `older_than_this' is non-zero then only flush inodes which have a
- * flushtime older than *older_than_this.
- *
- * If `bdi' is non-zero then we will scan the first inode against each
- * superblock until we find the matching ones. One group will be the dirty
- * inodes against a filesystem. Then when we hit the dummy blockdev superblock,
- * sync_sb_inodes will seekout the blockdev which matches `bdi'. Maybe not
- * super-efficient but we're about to do a ton of I/O...
- */
-void
-writeback_inodes(struct writeback_control *wbc)
-{
- struct super_block *sb;
-
- might_sleep();
- spin_lock(&sb_lock);
-restart:
- list_for_each_entry_reverse(sb, &super_blocks, s_list) {
- if (sb_has_dirty_inodes(sb)) {
- /* we're making our own get_super here */
- sb->s_count++;
- spin_unlock(&sb_lock);
- /*
- * If we can't get the readlock, there's no sense in
- * waiting around, most of the time the FS is going to
- * be unmounted by the time it is released.
- */
- if (down_read_trylock(&sb->s_umount)) {
- if (sb->s_root)
- sync_sb_inodes(sb, wbc);
- up_read(&sb->s_umount);
- }
- spin_lock(&sb_lock);
- if (__put_super_and_need_restart(sb))
- goto restart;
- }
- if (wbc->nr_to_write <= 0)
- break;
- }
- spin_unlock(&sb_lock);
-}
-
-/*
* writeback and wait upon the filesystem's dirty inodes. The caller will
* do this in two passes - one to write, and one to wait.
*
diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c
index 4a46743..ffa4853 100644
--- a/fs/ntfs/super.c
+++ b/fs/ntfs/super.c
@@ -2373,39 +2373,13 @@ static void ntfs_put_super(struct super_block *sb)
vol->mftmirr_ino = NULL;
}
/*
- * If any dirty inodes are left, throw away all mft data page cache
- * pages to allow a clean umount. This should never happen any more
- * due to mft.c::ntfs_mft_writepage() cleaning all the dirty pages as
- * the underlying mft records are written out and cleaned. If it does,
+ * We should have no dirty inodes left, due to
+ * mft.c::ntfs_mft_writepage() cleaning all the dirty pages as
+ * the underlying mft records are written out and cleaned.
* happen anyway, we want to know...
*/
ntfs_commit_inode(vol->mft_ino);
write_inode_now(vol->mft_ino, 1);
- if (sb_has_dirty_inodes(sb)) {
- const char *s1, *s2;
-
- mutex_lock(&vol->mft_ino->i_mutex);
- truncate_inode_pages(vol->mft_ino->i_mapping, 0);
- mutex_unlock(&vol->mft_ino->i_mutex);
- write_inode_now(vol->mft_ino, 1);
- if (sb_has_dirty_inodes(sb)) {
- static const char *_s1 = "inodes";
- static const char *_s2 = "";
- s1 = _s1;
- s2 = _s2;
- } else {
- static const char *_s1 = "mft pages";
- static const char *_s2 = "They have been thrown "
- "away. ";
- s1 = _s1;
- s2 = _s2;
- }
- ntfs_error(sb, "Dirty %s found at umount time. %sYou should "
- "run chkdsk. Please email "
- "linux-ntfs-dev@lists.sourceforge.net and say "
- "that you saw this message. Thank you.", s1,
- s2);
- }
#endif /* NTFS_RW */
iput(vol->mft_ino);
diff --git a/fs/sync.c b/fs/sync.c
index a16d53e..f4ddcef 100644
--- a/fs/sync.c
+++ b/fs/sync.c
@@ -23,7 +23,7 @@
*/
static void do_sync(unsigned long wait)
{
- wakeup_pdflush(0);
+ wakeup_flusher_threads(0);
sync_inodes(0); /* All mappings, inodes and their blockdevs */
DQUOT_SYNC(NULL);
sync_supers(); /* Write the superblocks */
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index bb58c95..3c94fbd 100644
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -39,6 +39,11 @@ enum bdi_stat_item {
#define BDI_STAT_BATCH (8*(1+ilog2(nr_cpu_ids)))
+struct bdi_writeback_arg {
+ unsigned long nr_pages;
+ struct super_block *sb;
+};
+
struct backing_dev_info {
struct list_head bdi_list;
@@ -60,6 +65,9 @@ struct backing_dev_info {
struct device *dev;
+ struct task_struct *task; /* writeback task */
+ wait_queue_head_t wait;
+ struct bdi_writeback_arg wb_arg; /* protected by BDI_pdflush */
struct list_head b_dirty; /* dirty inodes */
struct list_head b_io; /* parked for writeback */
struct list_head b_more_io; /* parked for more writeback */
@@ -77,10 +85,21 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
const char *fmt, ...);
int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev);
void bdi_unregister(struct backing_dev_info *bdi);
+void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
+ long nr_pages);
+int bdi_writeback_task(void *);
+void bdi_writeback_all(struct super_block *sb, long nr_pages);
extern spinlock_t bdi_lock;
extern struct list_head bdi_list;
+static inline int bdi_has_dirty_io(struct backing_dev_info *bdi)
+{
+ return !list_empty(&bdi->b_dirty) ||
+ !list_empty(&bdi->b_io) ||
+ !list_empty(&bdi->b_more_io);
+}
+
static inline void __add_bdi_stat(struct backing_dev_info *bdi,
enum bdi_stat_item item, s64 amount)
{
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 3c90eb4..7d44bda 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1832,6 +1832,8 @@ extern int invalidate_inode_pages2_range(struct address_space *mapping,
pgoff_t start, pgoff_t end);
extern void generic_sync_sb_inodes(struct super_block *sb,
struct writeback_control *wbc);
+extern void generic_sync_bdi_inodes(struct super_block *sb,
+ struct writeback_control *);
extern int write_inode_now(struct inode *, int);
extern int filemap_fdatawrite(struct address_space *);
extern int filemap_flush(struct address_space *);
@@ -1950,7 +1952,6 @@ extern int bdev_read_only(struct block_device *);
extern int set_blocksize(struct block_device *, int);
extern int sb_set_blocksize(struct super_block *, int);
extern int sb_min_blocksize(struct super_block *, int);
-extern int sb_has_dirty_inodes(struct super_block *);
extern int generic_file_mmap(struct file *, struct vm_area_struct *);
extern int generic_file_readonly_mmap(struct file *, struct vm_area_struct *);
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 7300ecd..3697e91 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -99,7 +99,7 @@ static inline void inode_sync_wait(struct inode *inode)
/*
* mm/page-writeback.c
*/
-int wakeup_pdflush(long nr_pages);
+int wakeup_flusher_threads(long nr_pages);
void laptop_io_completion(void);
void laptop_sync_completion(void);
void throttle_vm_writeout(gfp_t gfp_mask);
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index cf1528b..bf9c0e0 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -1,8 +1,10 @@
#include <linux/wait.h>
#include <linux/backing-dev.h>
+#include <linux/kthread.h>
#include <linux/fs.h>
#include <linux/sched.h>
+#include <linux/mm.h>
#include <linux/module.h>
#include <linux/writeback.h>
#include <linux/device.h>
@@ -188,6 +190,13 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
goto exit;
}
+ bdi->task = kthread_run(bdi_writeback_task, bdi, "bdi-%s",
+ dev_name(dev));
+ if (!bdi->task) {
+ ret = -ENOMEM;
+ goto exit;
+ }
+
spin_lock(&bdi_lock);
list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
spin_unlock(&bdi_lock);
@@ -223,6 +232,10 @@ void bdi_unregister(struct backing_dev_info *bdi)
{
if (bdi->dev) {
bdi_remove_from_list(bdi);
+ if (bdi->task) {
+ kthread_stop(bdi->task);
+ bdi->task = NULL;
+ }
bdi_debug_unregister(bdi);
device_unregister(bdi->dev);
bdi->dev = NULL;
@@ -232,14 +245,14 @@ EXPORT_SYMBOL(bdi_unregister);
int bdi_init(struct backing_dev_info *bdi)
{
- int i;
- int err;
+ int i, err;
bdi->dev = NULL;
bdi->min_ratio = 0;
bdi->max_ratio = 100;
bdi->max_prop_frac = PROP_FRAC_BASE;
+ init_waitqueue_head(&bdi->wait);
INIT_LIST_HEAD(&bdi->bdi_list);
INIT_LIST_HEAD(&bdi->b_io);
INIT_LIST_HEAD(&bdi->b_dirty);
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 3ec11d8..9d37be6 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -36,15 +36,6 @@
#include <linux/pagevec.h>
/*
- * The maximum number of pages to writeout in a single bdflush/kupdate
- * operation. We do this so we don't hold I_SYNC against an inode for
- * enormous amounts of time, which would block a userspace task which has
- * been forced to throttle against that inode. Also, the code reevaluates
- * the dirty each time it has written this many pages.
- */
-#define MAX_WRITEBACK_PAGES 1024
-
-/*
* After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
* will look to see if it needs to force writeback or throttling.
*/
@@ -117,8 +108,6 @@ EXPORT_SYMBOL(laptop_mode);
/* End of sysctl-exported parameters */
-static void background_writeout(unsigned long _min_pages);
-
/*
* Scale the writeback cache size proportional to the relative writeout speeds.
*
@@ -541,7 +530,7 @@ static void balance_dirty_pages(struct address_space *mapping)
* been flushed to permanent storage.
*/
if (bdi_nr_reclaimable) {
- writeback_inodes(&wbc);
+ generic_sync_bdi_inodes(NULL, &wbc);
pages_written += write_chunk - wbc.nr_to_write;
get_dirty_limits(&background_thresh, &dirty_thresh,
&bdi_thresh, bdi);
@@ -592,7 +581,7 @@ static void balance_dirty_pages(struct address_space *mapping)
(!laptop_mode && (global_page_state(NR_FILE_DIRTY)
+ global_page_state(NR_UNSTABLE_NFS)
> background_thresh)))
- pdflush_operation(background_writeout, 0);
+ bdi_start_writeback(bdi, NULL, 0);
}
void set_page_dirty_balance(struct page *page, int page_mkwrite)
@@ -677,151 +666,36 @@ void throttle_vm_writeout(gfp_t gfp_mask)
}
/*
- * writeback at least _min_pages, and keep writing until the amount of dirty
- * memory is less than the background threshold, or until we're all clean.
- */
-static void background_writeout(unsigned long _min_pages)
-{
- long min_pages = _min_pages;
- struct writeback_control wbc = {
- .bdi = NULL,
- .sync_mode = WB_SYNC_NONE,
- .older_than_this = NULL,
- .nr_to_write = 0,
- .nonblocking = 1,
- .range_cyclic = 1,
- };
-
- for ( ; ; ) {
- unsigned long background_thresh;
- unsigned long dirty_thresh;
-
- get_dirty_limits(&background_thresh, &dirty_thresh, NULL, NULL);
- if (global_page_state(NR_FILE_DIRTY) +
- global_page_state(NR_UNSTABLE_NFS) < background_thresh
- && min_pages <= 0)
- break;
- wbc.more_io = 0;
- wbc.encountered_congestion = 0;
- wbc.nr_to_write = MAX_WRITEBACK_PAGES;
- wbc.pages_skipped = 0;
- writeback_inodes(&wbc);
- min_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
- if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) {
- /* Wrote less than expected */
- if (wbc.encountered_congestion || wbc.more_io)
- congestion_wait(WRITE, HZ/10);
- else
- break;
- }
- }
-}
-
-/*
* Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back
* the whole world. Returns 0 if a pdflush thread was dispatched. Returns
* -1 if all pdflush threads were busy.
*/
-int wakeup_pdflush(long nr_pages)
+int wakeup_flusher_threads(long nr_pages)
{
if (nr_pages == 0)
nr_pages = global_page_state(NR_FILE_DIRTY) +
global_page_state(NR_UNSTABLE_NFS);
- return pdflush_operation(background_writeout, nr_pages);
+ bdi_writeback_all(NULL, nr_pages);
+ return 0;
}
-static void wb_timer_fn(unsigned long unused);
static void laptop_timer_fn(unsigned long unused);
-static DEFINE_TIMER(wb_timer, wb_timer_fn, 0, 0);
static DEFINE_TIMER(laptop_mode_wb_timer, laptop_timer_fn, 0, 0);
/*
- * Periodic writeback of "old" data.
- *
- * Define "old": the first time one of an inode's pages is dirtied, we mark the
- * dirtying-time in the inode's address_space. So this periodic writeback code
- * just walks the superblock inode list, writing back any inodes which are
- * older than a specific point in time.
- *
- * Try to run once per dirty_writeback_interval. But if a writeback event
- * takes longer than a dirty_writeback_interval interval, then leave a
- * one-second gap.
- *
- * older_than_this takes precedence over nr_to_write. So we'll only write back
- * all dirty pages if they are all attached to "old" mappings.
- */
-static void wb_kupdate(unsigned long arg)
-{
- unsigned long oldest_jif;
- unsigned long start_jif;
- unsigned long next_jif;
- long nr_to_write;
- struct writeback_control wbc = {
- .bdi = NULL,
- .sync_mode = WB_SYNC_NONE,
- .older_than_this = &oldest_jif,
- .nr_to_write = 0,
- .nonblocking = 1,
- .for_kupdate = 1,
- .range_cyclic = 1,
- };
-
- sync_supers();
-
- oldest_jif = jiffies - dirty_expire_interval;
- start_jif = jiffies;
- next_jif = start_jif + dirty_writeback_interval;
- nr_to_write = global_page_state(NR_FILE_DIRTY) +
- global_page_state(NR_UNSTABLE_NFS) +
- (inodes_stat.nr_inodes - inodes_stat.nr_unused);
- while (nr_to_write > 0) {
- wbc.more_io = 0;
- wbc.encountered_congestion = 0;
- wbc.nr_to_write = MAX_WRITEBACK_PAGES;
- writeback_inodes(&wbc);
- if (wbc.nr_to_write > 0) {
- if (wbc.encountered_congestion || wbc.more_io)
- congestion_wait(WRITE, HZ/10);
- else
- break; /* All the old data is written */
- }
- nr_to_write -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
- }
- if (time_before(next_jif, jiffies + HZ))
- next_jif = jiffies + HZ;
- if (dirty_writeback_interval)
- mod_timer(&wb_timer, next_jif);
-}
-
-/*
* sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
*/
int dirty_writeback_centisecs_handler(ctl_table *table, int write,
struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
{
proc_dointvec_userhz_jiffies(table, write, file, buffer, length, ppos);
- if (dirty_writeback_interval)
- mod_timer(&wb_timer, jiffies + dirty_writeback_interval);
- else
- del_timer(&wb_timer);
return 0;
}
-static void wb_timer_fn(unsigned long unused)
-{
- if (pdflush_operation(wb_kupdate, 0) < 0)
- mod_timer(&wb_timer, jiffies + HZ); /* delay 1 second */
-}
-
-static void laptop_flush(unsigned long unused)
-{
- sys_sync();
-}
-
static void laptop_timer_fn(unsigned long unused)
{
- pdflush_operation(laptop_flush, 0);
+ wakeup_flusher_threads(0);
}
/*
@@ -904,7 +778,6 @@ void __init page_writeback_init(void)
{
int shift;
- mod_timer(&wb_timer, jiffies + dirty_writeback_interval);
writeback_set_ratelimit();
register_cpu_notifier(&ratelimit_nb);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 6177e3b..b5e2802 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1649,7 +1649,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
*/
if (total_scanned > sc->swap_cluster_max +
sc->swap_cluster_max / 2) {
- wakeup_pdflush(laptop_mode ? 0 : total_scanned);
+ wakeup_flusher_threads(laptop_mode ? 0 : total_scanned);
sc->may_writepage = 1;
}
--
1.6.2
^ permalink raw reply related [flat|nested] 31+ messages in thread* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-12 14:33 ` [PATCH 2/7] writeback: switch to per-bdi threads for flushing data Jens Axboe
@ 2009-03-13 5:33 ` Andrew Morton
2009-03-13 10:54 ` Jens Axboe
2009-03-16 10:22 ` Christoph Hellwig
1 sibling, 1 reply; 31+ messages in thread
From: Andrew Morton @ 2009-03-13 5:33 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
On Thu, 12 Mar 2009 15:33:43 +0100 Jens Axboe <jens.axboe@oracle.com> wrote:
> This gets rid of pdflush for bdi writeout and kupdated style cleaning.
> This is an experiment to see if we get better writeout behaviour with
> per-bdi flushing. Some initial tests look pretty encouraging. A sample
> ffsb workload that does random writes to files is about 8% faster here
> on a simple SATA drive during the benchmark phase. File layout also seems
> a LOT more smooth in vmstat:
>
> r b swpd free buff cache si so bi bo in cs us sy id wa
> 0 1 0 608848 2652 375372 0 0 0 71024 604 24 1 10 48 42
> 0 1 0 549644 2712 433736 0 0 0 60692 505 27 1 8 48 44
> 1 0 0 476928 2784 505192 0 0 4 29540 553 24 0 9 53 37
> 0 1 0 457972 2808 524008 0 0 0 54876 331 16 0 4 38 58
> 0 1 0 366128 2928 614284 0 0 4 92168 710 58 0 13 53 34
> 0 1 0 295092 3000 684140 0 0 0 62924 572 23 0 9 53 37
> 0 1 0 236592 3064 741704 0 0 4 58256 523 17 0 8 48 44
> 0 1 0 165608 3132 811464 0 0 0 57460 560 21 0 8 54 38
> 0 1 0 102952 3200 873164 0 0 4 74748 540 29 1 10 48 41
> 0 1 0 48604 3252 926472 0 0 0 53248 469 29 0 7 47 45
>
> where vanilla tends to fluctuate a lot in the creation phase:
>
> r b swpd free buff cache si so bi bo in cs us sy id wa
> 1 1 0 678716 5792 303380 0 0 0 74064 565 50 1 11 52 36
> 1 0 0 662488 5864 319396 0 0 4 352 302 329 0 2 47 51
> 0 1 0 599312 5924 381468 0 0 0 78164 516 55 0 9 51 40
> 0 1 0 519952 6008 459516 0 0 4 78156 622 56 1 11 52 37
> 1 1 0 436640 6092 541632 0 0 0 82244 622 54 0 11 48 41
> 0 1 0 436640 6092 541660 0 0 0 8 152 39 0 0 51 49
> 0 1 0 332224 6200 644252 0 0 4 102800 728 46 1 13 49 36
> 1 0 0 274492 6260 701056 0 0 4 12328 459 49 0 7 50 43
> 0 1 0 211220 6324 763356 0 0 0 106940 515 37 1 10 51 39
> 1 0 0 160412 6376 813468 0 0 0 8224 415 43 0 6 49 45
> 1 1 0 85980 6452 886556 0 0 4 113516 575 39 1 11 54 34
> 0 2 0 85968 6452 886620 0 0 0 1640 158 211 0 0 46 54
Confused. The two should be equivalent for
one-filesystem-per-physical-disk. What made it change?
> So apart from seemingly behaving better for buffered writeout, this also
> allows us to potentially have more than one bdi thread flushing out data.
> This may be useful for NUMA type setups.
Bear in mind that the XFS guys found that one thread per fs had
insufficient CPU power to keep up with fast devices.
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-13 5:33 ` Andrew Morton
@ 2009-03-13 10:54 ` Jens Axboe
2009-03-15 22:52 ` Dave Chinner
0 siblings, 1 reply; 31+ messages in thread
From: Jens Axboe @ 2009-03-13 10:54 UTC (permalink / raw)
To: Andrew Morton; +Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
On Thu, Mar 12 2009, Andrew Morton wrote:
> On Thu, 12 Mar 2009 15:33:43 +0100 Jens Axboe <jens.axboe@oracle.com> wrote:
>
> > This gets rid of pdflush for bdi writeout and kupdated style cleaning.
> > This is an experiment to see if we get better writeout behaviour with
> > per-bdi flushing. Some initial tests look pretty encouraging. A sample
> > ffsb workload that does random writes to files is about 8% faster here
> > on a simple SATA drive during the benchmark phase. File layout also seems
> > a LOT more smooth in vmstat:
> >
> > r b swpd free buff cache si so bi bo in cs us sy id wa
> > 0 1 0 608848 2652 375372 0 0 0 71024 604 24 1 10 48 42
> > 0 1 0 549644 2712 433736 0 0 0 60692 505 27 1 8 48 44
> > 1 0 0 476928 2784 505192 0 0 4 29540 553 24 0 9 53 37
> > 0 1 0 457972 2808 524008 0 0 0 54876 331 16 0 4 38 58
> > 0 1 0 366128 2928 614284 0 0 4 92168 710 58 0 13 53 34
> > 0 1 0 295092 3000 684140 0 0 0 62924 572 23 0 9 53 37
> > 0 1 0 236592 3064 741704 0 0 4 58256 523 17 0 8 48 44
> > 0 1 0 165608 3132 811464 0 0 0 57460 560 21 0 8 54 38
> > 0 1 0 102952 3200 873164 0 0 4 74748 540 29 1 10 48 41
> > 0 1 0 48604 3252 926472 0 0 0 53248 469 29 0 7 47 45
> >
> > where vanilla tends to fluctuate a lot in the creation phase:
> >
> > r b swpd free buff cache si so bi bo in cs us sy id wa
> > 1 1 0 678716 5792 303380 0 0 0 74064 565 50 1 11 52 36
> > 1 0 0 662488 5864 319396 0 0 4 352 302 329 0 2 47 51
> > 0 1 0 599312 5924 381468 0 0 0 78164 516 55 0 9 51 40
> > 0 1 0 519952 6008 459516 0 0 4 78156 622 56 1 11 52 37
> > 1 1 0 436640 6092 541632 0 0 0 82244 622 54 0 11 48 41
> > 0 1 0 436640 6092 541660 0 0 0 8 152 39 0 0 51 49
> > 0 1 0 332224 6200 644252 0 0 4 102800 728 46 1 13 49 36
> > 1 0 0 274492 6260 701056 0 0 4 12328 459 49 0 7 50 43
> > 0 1 0 211220 6324 763356 0 0 0 106940 515 37 1 10 51 39
> > 1 0 0 160412 6376 813468 0 0 0 8224 415 43 0 6 49 45
> > 1 1 0 85980 6452 886556 0 0 4 113516 575 39 1 11 54 34
> > 0 2 0 85968 6452 886620 0 0 0 1640 158 211 0 0 46 54
>
> Confused. The two should be equivalent for
> one-filesystem-per-physical-disk. What made it change?
There's at least one difference, and that is the lack of congestion
checking. With one thread per bdi, we don't need to do nonblocking IO to
the device. Whether that is the factor that makes the difference, I
don't know yet. I'll investigate for sure :-)
> > So apart from seemingly behaving better for buffered writeout, this also
> > allows us to potentially have more than one bdi thread flushing out data.
> > This may be useful for NUMA type setups.
>
> Bear in mind that the XFS guys found that one thread per fs had
> insufficient CPU power to keep up with fast devices.
Yes, I definitely want to experiment with > 1 thread per device in the
near future.
--
Jens Axboe
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-13 10:54 ` Jens Axboe
@ 2009-03-15 22:52 ` Dave Chinner
2009-03-16 7:33 ` Jens Axboe
0 siblings, 1 reply; 31+ messages in thread
From: Dave Chinner @ 2009-03-15 22:52 UTC (permalink / raw)
To: Jens Axboe
Cc: Andrew Morton, linux-kernel, linux-fsdevel, chris.mason, npiggin
On Fri, Mar 13, 2009 at 11:54:46AM +0100, Jens Axboe wrote:
> On Thu, Mar 12 2009, Andrew Morton wrote:
> > On Thu, 12 Mar 2009 15:33:43 +0100 Jens Axboe <jens.axboe@oracle.com> wrote:
> > Bear in mind that the XFS guys found that one thread per fs had
> > insufficient CPU power to keep up with fast devices.
>
> Yes, I definitely want to experiment with > 1 thread per device in the
> near future.
The question here is how to do this efficiently. Even if XFS is
operating on a single device, it is not optimal just to throw
multiple threads at the bdi. Ideally we want a thread per region
(allocation group) of the filesystem as each allocation group has
it's own inode cache (radix tree) to traverse. These traversals can
be done completely in parallel and won't contend either at the
traversal level or in the IO hardware....
i.e. what I'd like to see is the ability so any new flushing
mechanism to be able to offload responsibility of tracking,
traversing and flushing of dirty inodes to the filesystem.
Filesystems that don't do such things could use a generic
bdi-based implementation.
FWIW, we also want to avoid the current pattern of flushing
data, then the inode, then data, then the inode, ....
By offloading into the filesystem, this writeback ordering can
be done as efficiently as possible for each given filesystem.
XFs already has all the hooks to be able to do this
effectively....
I know that Christoph was doing some work towards this end;
perhaps he can throw his 2c worth in here...
Cheers,
Dave.
--
Dave Chinner
david@fromorbit.com
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-15 22:52 ` Dave Chinner
@ 2009-03-16 7:33 ` Jens Axboe
2009-03-16 10:17 ` Christoph Hellwig
2009-03-16 23:38 ` Dave Chinner
0 siblings, 2 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-16 7:33 UTC (permalink / raw)
To: Dave Chinner
Cc: Andrew Morton, linux-kernel, linux-fsdevel, chris.mason, npiggin
On Mon, Mar 16 2009, Dave Chinner wrote:
> On Fri, Mar 13, 2009 at 11:54:46AM +0100, Jens Axboe wrote:
> > On Thu, Mar 12 2009, Andrew Morton wrote:
> > > On Thu, 12 Mar 2009 15:33:43 +0100 Jens Axboe <jens.axboe@oracle.com> wrote:
> > > Bear in mind that the XFS guys found that one thread per fs had
> > > insufficient CPU power to keep up with fast devices.
> >
> > Yes, I definitely want to experiment with > 1 thread per device in the
> > near future.
>
> The question here is how to do this efficiently. Even if XFS is
> operating on a single device, it is not optimal just to throw
> multiple threads at the bdi. Ideally we want a thread per region
> (allocation group) of the filesystem as each allocation group has
> it's own inode cache (radix tree) to traverse. These traversals can
> be done completely in parallel and won't contend either at the
> traversal level or in the IO hardware....
>
> i.e. what I'd like to see is the ability so any new flushing
> mechanism to be able to offload responsibility of tracking,
> traversing and flushing of dirty inodes to the filesystem.
> Filesystems that don't do such things could use a generic
> bdi-based implementation.
>
> FWIW, we also want to avoid the current pattern of flushing
> data, then the inode, then data, then the inode, ....
> By offloading into the filesystem, this writeback ordering can
> be done as efficiently as possible for each given filesystem.
> XFs already has all the hooks to be able to do this
> effectively....
>
> I know that Christoph was doing some work towards this end;
> perhaps he can throw his 2c worth in here...
This is very useful feedback, thanks Dave. So on the filesystem vs bdi
side, XFS could register a bdi per allocation group. Then set the proper
inode->mapping->backing_dev_info from sb->s_op->alloc_inode and
__mark_inode_dirty() should get the placement right. For private
traverse and flush, provide some address_space op to override
generic_sync_bdi_inodes().
It sounds like I should move the bdi flushing bits separate from the bdi
itself. Embed one in the bdi, but allow outside registration of others.
Will fit better with the need for more than one flusher per backing
device.
--
Jens Axboe
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-16 7:33 ` Jens Axboe
@ 2009-03-16 10:17 ` Christoph Hellwig
2009-03-16 10:21 ` Jens Axboe
2009-03-16 23:38 ` Dave Chinner
1 sibling, 1 reply; 31+ messages in thread
From: Christoph Hellwig @ 2009-03-16 10:17 UTC (permalink / raw)
To: Jens Axboe
Cc: Dave Chinner, Andrew Morton, linux-kernel, linux-fsdevel,
chris.mason, npiggin
On Mon, Mar 16, 2009 at 08:33:21AM +0100, Jens Axboe wrote:
> This is very useful feedback, thanks Dave. So on the filesystem vs bdi
> side, XFS could register a bdi per allocation group. Then set the proper
> inode->mapping->backing_dev_info from sb->s_op->alloc_inode and
> __mark_inode_dirty() should get the placement right. For private
> traverse and flush, provide some address_space op to override
> generic_sync_bdi_inodes().
>
> It sounds like I should move the bdi flushing bits separate from the bdi
> itself. Embed one in the bdi, but allow outside registration of others.
> Will fit better with the need for more than one flusher per backing
> device.
Yes, having a separate flushing container is a good idea. Either way
I'm not sure how much use all this is for XFS. We're working on using
our own flushing code as we want to iterate the inodes using the
knowledge about their placement on disk. Due to that we don't use
the dirty inode list anymore (already in mainline since 2.6.29-rc) but
the per-ag radix-tree. The finer granularity pdflush management might
come in handy, but we'll hand off to some XFS-specific code at a pretty
high level in there.
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-16 10:17 ` Christoph Hellwig
@ 2009-03-16 10:21 ` Jens Axboe
0 siblings, 0 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-16 10:21 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Dave Chinner, Andrew Morton, linux-kernel, linux-fsdevel,
chris.mason, npiggin
On Mon, Mar 16 2009, Christoph Hellwig wrote:
> On Mon, Mar 16, 2009 at 08:33:21AM +0100, Jens Axboe wrote:
> > This is very useful feedback, thanks Dave. So on the filesystem vs bdi
> > side, XFS could register a bdi per allocation group. Then set the proper
> > inode->mapping->backing_dev_info from sb->s_op->alloc_inode and
> > __mark_inode_dirty() should get the placement right. For private
> > traverse and flush, provide some address_space op to override
> > generic_sync_bdi_inodes().
> >
> > It sounds like I should move the bdi flushing bits separate from the bdi
> > itself. Embed one in the bdi, but allow outside registration of others.
> > Will fit better with the need for more than one flusher per backing
> > device.
>
> Yes, having a separate flushing container is a good idea. Either way
> I'm not sure how much use all this is for XFS. We're working on using
> our own flushing code as we want to iterate the inodes using the
> knowledge about their placement on disk. Due to that we don't use
> the dirty inode list anymore (already in mainline since 2.6.29-rc) but
> the per-ag radix-tree. The finer granularity pdflush management might
> come in handy, but we'll hand off to some XFS-specific code at a pretty
> high level in there.
Nobody says that you have to use the inode lists for iteration. Or
perhaps it would be possible to plug __mark_inode_dirty() and allow for
alternate placement of the inode upfront.
I'll definitely proceed with the flushing container approach. And I'd
very much like to work with XFS to ensure that it caters to that end as
well, makes it a lot more applicable imho.
--
Jens Axboe
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-16 7:33 ` Jens Axboe
2009-03-16 10:17 ` Christoph Hellwig
@ 2009-03-16 23:38 ` Dave Chinner
2009-03-17 9:37 ` Jens Axboe
2009-03-17 13:21 ` Chris Mason
1 sibling, 2 replies; 31+ messages in thread
From: Dave Chinner @ 2009-03-16 23:38 UTC (permalink / raw)
To: Jens Axboe
Cc: Andrew Morton, linux-kernel, linux-fsdevel, chris.mason, npiggin
On Mon, Mar 16, 2009 at 08:33:21AM +0100, Jens Axboe wrote:
> On Mon, Mar 16 2009, Dave Chinner wrote:
> > On Fri, Mar 13, 2009 at 11:54:46AM +0100, Jens Axboe wrote:
> > > On Thu, Mar 12 2009, Andrew Morton wrote:
> > > > On Thu, 12 Mar 2009 15:33:43 +0100 Jens Axboe <jens.axboe@oracle.com> wrote:
> > > > Bear in mind that the XFS guys found that one thread per fs had
> > > > insufficient CPU power to keep up with fast devices.
> > >
> > > Yes, I definitely want to experiment with > 1 thread per device in the
> > > near future.
> >
> > The question here is how to do this efficiently. Even if XFS is
> > operating on a single device, it is not optimal just to throw
> > multiple threads at the bdi. Ideally we want a thread per region
> > (allocation group) of the filesystem as each allocation group has
> > it's own inode cache (radix tree) to traverse. These traversals can
> > be done completely in parallel and won't contend either at the
> > traversal level or in the IO hardware....
> >
> > i.e. what I'd like to see is the ability so any new flushing
> > mechanism to be able to offload responsibility of tracking,
> > traversing and flushing of dirty inodes to the filesystem.
> > Filesystems that don't do such things could use a generic
> > bdi-based implementation.
> >
> > FWIW, we also want to avoid the current pattern of flushing
> > data, then the inode, then data, then the inode, ....
> > By offloading into the filesystem, this writeback ordering can
> > be done as efficiently as possible for each given filesystem.
> > XFs already has all the hooks to be able to do this
> > effectively....
> >
> > I know that Christoph was doing some work towards this end;
> > perhaps he can throw his 2c worth in here...
>
> This is very useful feedback, thanks Dave. So on the filesystem vs bdi
> side, XFS could register a bdi per allocation group.
How do multiple bdis on a single block device interact?
> Then set the proper
> inode->mapping->backing_dev_info from sb->s_op->alloc_inode and
> __mark_inode_dirty() should get the placement right. For private
> traverse and flush, provide some address_space op to override
> generic_sync_bdi_inodes().
Yes, that seems like it would support the sort of internal XFS
structure I've been thinking of.
> It sounds like I should move the bdi flushing bits separate from the bdi
> itself. Embed one in the bdi, but allow outside registration of others.
> Will fit better with the need for more than one flusher per backing
> device.
*nod*
Cheers,
Dave.
--
Dave Chinner
david@fromorbit.com
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-16 23:38 ` Dave Chinner
@ 2009-03-17 9:37 ` Jens Axboe
2009-03-17 13:21 ` Chris Mason
1 sibling, 0 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-17 9:37 UTC (permalink / raw)
To: Dave Chinner
Cc: Andrew Morton, linux-kernel, linux-fsdevel, chris.mason, npiggin
On Tue, Mar 17 2009, Dave Chinner wrote:
> On Mon, Mar 16, 2009 at 08:33:21AM +0100, Jens Axboe wrote:
> > On Mon, Mar 16 2009, Dave Chinner wrote:
> > > On Fri, Mar 13, 2009 at 11:54:46AM +0100, Jens Axboe wrote:
> > > > On Thu, Mar 12 2009, Andrew Morton wrote:
> > > > > On Thu, 12 Mar 2009 15:33:43 +0100 Jens Axboe <jens.axboe@oracle.com> wrote:
> > > > > Bear in mind that the XFS guys found that one thread per fs had
> > > > > insufficient CPU power to keep up with fast devices.
> > > >
> > > > Yes, I definitely want to experiment with > 1 thread per device in the
> > > > near future.
> > >
> > > The question here is how to do this efficiently. Even if XFS is
> > > operating on a single device, it is not optimal just to throw
> > > multiple threads at the bdi. Ideally we want a thread per region
> > > (allocation group) of the filesystem as each allocation group has
> > > it's own inode cache (radix tree) to traverse. These traversals can
> > > be done completely in parallel and won't contend either at the
> > > traversal level or in the IO hardware....
> > >
> > > i.e. what I'd like to see is the ability so any new flushing
> > > mechanism to be able to offload responsibility of tracking,
> > > traversing and flushing of dirty inodes to the filesystem.
> > > Filesystems that don't do such things could use a generic
> > > bdi-based implementation.
> > >
> > > FWIW, we also want to avoid the current pattern of flushing
> > > data, then the inode, then data, then the inode, ....
> > > By offloading into the filesystem, this writeback ordering can
> > > be done as efficiently as possible for each given filesystem.
> > > XFs already has all the hooks to be able to do this
> > > effectively....
> > >
> > > I know that Christoph was doing some work towards this end;
> > > perhaps he can throw his 2c worth in here...
> >
> > This is very useful feedback, thanks Dave. So on the filesystem vs bdi
> > side, XFS could register a bdi per allocation group.
>
> How do multiple bdis on a single block device interact?
I think this should be revised. So the structure I have now has a list
of flusher containers hanging off the bdi. backing device registration
will fork the single flusher, then the intention is that users like XFS
could add more flusher threads to the bdi. Basically what I wrote
further down :-)
> > Then set the proper
> > inode->mapping->backing_dev_info from sb->s_op->alloc_inode and
> > __mark_inode_dirty() should get the placement right. For private
> > traverse and flush, provide some address_space op to override
> > generic_sync_bdi_inodes().
>
> Yes, that seems like it would support the sort of internal XFS
> structure I've been thinking of.
Goodie!
> > It sounds like I should move the bdi flushing bits separate from the bdi
> > itself. Embed one in the bdi, but allow outside registration of others.
> > Will fit better with the need for more than one flusher per backing
> > device.
>
> *nod*
>
> Cheers,
>
> Dave.
> --
> Dave Chinner
> david@fromorbit.com
--
Jens Axboe
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-16 23:38 ` Dave Chinner
2009-03-17 9:37 ` Jens Axboe
@ 2009-03-17 13:21 ` Chris Mason
1 sibling, 0 replies; 31+ messages in thread
From: Chris Mason @ 2009-03-17 13:21 UTC (permalink / raw)
To: Dave Chinner
Cc: Jens Axboe, Andrew Morton, linux-kernel, linux-fsdevel, npiggin
On Tue, 2009-03-17 at 10:38 +1100, Dave Chinner wrote:
> On Mon, Mar 16, 2009 at 08:33:21AM +0100, Jens Axboe wrote:
> > On Mon, Mar 16 2009, Dave Chinner wrote:
> > > On Fri, Mar 13, 2009 at 11:54:46AM +0100, Jens Axboe wrote:
> > > > On Thu, Mar 12 2009, Andrew Morton wrote:
> > > > > On Thu, 12 Mar 2009 15:33:43 +0100 Jens Axboe <jens.axboe@oracle.com> wrote:
> > > > > Bear in mind that the XFS guys found that one thread per fs had
> > > > > insufficient CPU power to keep up with fast devices.
> > > >
> > > > Yes, I definitely want to experiment with > 1 thread per device in the
> > > > near future.
> > >
> > > The question here is how to do this efficiently. Even if XFS is
> > > operating on a single device, it is not optimal just to throw
> > > multiple threads at the bdi. Ideally we want a thread per region
> > > (allocation group) of the filesystem as each allocation group has
> > > it's own inode cache (radix tree) to traverse. These traversals can
> > > be done completely in parallel and won't contend either at the
> > > traversal level or in the IO hardware....
> > >
> > > i.e. what I'd like to see is the ability so any new flushing
> > > mechanism to be able to offload responsibility of tracking,
> > > traversing and flushing of dirty inodes to the filesystem.
> > > Filesystems that don't do such things could use a generic
> > > bdi-based implementation.
> > >
> > > FWIW, we also want to avoid the current pattern of flushing
> > > data, then the inode, then data, then the inode, ....
> > > By offloading into the filesystem, this writeback ordering can
> > > be done as efficiently as possible for each given filesystem.
> > > XFs already has all the hooks to be able to do this
> > > effectively....
> > >
> > > I know that Christoph was doing some work towards this end;
> > > perhaps he can throw his 2c worth in here...
> >
> > This is very useful feedback, thanks Dave. So on the filesystem vs bdi
> > side, XFS could register a bdi per allocation group.
>
> How do multiple bdis on a single block device interact?
The main difference is that dirty page tracking for balance_dirty_pages
and friends is done per-bdi. So, you'll end up with uneven memory
pressure on ags that don't have much dirty data, but hopefully that's a
good thing.
-chris
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-12 14:33 ` [PATCH 2/7] writeback: switch to per-bdi threads for flushing data Jens Axboe
@ 2009-03-16 10:22 ` Christoph Hellwig
2009-03-16 10:22 ` Christoph Hellwig
1 sibling, 0 replies; 31+ messages in thread
From: Christoph Hellwig @ 2009-03-16 10:22 UTC (permalink / raw)
To: Jens Axboe
Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin,
linux-mm
On Thu, Mar 12, 2009 at 03:33:43PM +0100, Jens Axboe wrote:
> +static void bdi_kupdated(struct backing_dev_info *bdi)
> +{
> + long nr_to_write;
> + struct writeback_control wbc = {
> + .bdi = bdi,
> + .sync_mode = WB_SYNC_NONE,
> + .nr_to_write = 0,
> + .for_kupdate = 1,
> + .range_cyclic = 1,
> + };
> +
> + sync_supers();
Not directly related to your patch, but can someone explain WTF
sync_supers is doing here or in the old kupdated? We're writing back
dirty pages from the VM, and for some reason we try to also write back
superblocks. This doesn't really make any sense.
^ permalink raw reply [flat|nested] 31+ messages in thread* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
@ 2009-03-16 10:22 ` Christoph Hellwig
0 siblings, 0 replies; 31+ messages in thread
From: Christoph Hellwig @ 2009-03-16 10:22 UTC (permalink / raw)
To: Jens Axboe
Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin,
linux-mm
On Thu, Mar 12, 2009 at 03:33:43PM +0100, Jens Axboe wrote:
> +static void bdi_kupdated(struct backing_dev_info *bdi)
> +{
> + long nr_to_write;
> + struct writeback_control wbc = {
> + .bdi = bdi,
> + .sync_mode = WB_SYNC_NONE,
> + .nr_to_write = 0,
> + .for_kupdate = 1,
> + .range_cyclic = 1,
> + };
> +
> + sync_supers();
Not directly related to your patch, but can someone explain WTF
sync_supers is doing here or in the old kupdated? We're writing back
dirty pages from the VM, and for some reason we try to also write back
superblocks. This doesn't really make any sense.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 31+ messages in thread* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-16 10:22 ` Christoph Hellwig
@ 2009-03-16 13:30 ` Chris Mason
-1 siblings, 0 replies; 31+ messages in thread
From: Chris Mason @ 2009-03-16 13:30 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Jens Axboe, linux-kernel, linux-fsdevel, david, npiggin, linux-mm
On Mon, 2009-03-16 at 06:22 -0400, Christoph Hellwig wrote:
> On Thu, Mar 12, 2009 at 03:33:43PM +0100, Jens Axboe wrote:
> > +static void bdi_kupdated(struct backing_dev_info *bdi)
> > +{
> > + long nr_to_write;
> > + struct writeback_control wbc = {
> > + .bdi = bdi,
> > + .sync_mode = WB_SYNC_NONE,
> > + .nr_to_write = 0,
> > + .for_kupdate = 1,
> > + .range_cyclic = 1,
> > + };
> > +
> > + sync_supers();
>
> Not directly related to your patch, but can someone explain WTF
> sync_supers is doing here or in the old kupdated? We're writing back
> dirty pages from the VM, and for some reason we try to also write back
> superblocks. This doesn't really make any sense.
Some of our poor filesystem cousins don't write the super until kupdate
kicks them (see ext2_write_super). kupdate has always been the periodic
FS thread of last resort.
-chris
^ permalink raw reply [flat|nested] 31+ messages in thread* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
@ 2009-03-16 13:30 ` Chris Mason
0 siblings, 0 replies; 31+ messages in thread
From: Chris Mason @ 2009-03-16 13:30 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Jens Axboe, linux-kernel, linux-fsdevel, david, npiggin, linux-mm
On Mon, 2009-03-16 at 06:22 -0400, Christoph Hellwig wrote:
> On Thu, Mar 12, 2009 at 03:33:43PM +0100, Jens Axboe wrote:
> > +static void bdi_kupdated(struct backing_dev_info *bdi)
> > +{
> > + long nr_to_write;
> > + struct writeback_control wbc = {
> > + .bdi = bdi,
> > + .sync_mode = WB_SYNC_NONE,
> > + .nr_to_write = 0,
> > + .for_kupdate = 1,
> > + .range_cyclic = 1,
> > + };
> > +
> > + sync_supers();
>
> Not directly related to your patch, but can someone explain WTF
> sync_supers is doing here or in the old kupdated? We're writing back
> dirty pages from the VM, and for some reason we try to also write back
> superblocks. This doesn't really make any sense.
Some of our poor filesystem cousins don't write the super until kupdate
kicks them (see ext2_write_super). kupdate has always been the periodic
FS thread of last resort.
-chris
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 31+ messages in thread* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
2009-03-16 13:30 ` Chris Mason
@ 2009-03-16 13:39 ` Christoph Hellwig
-1 siblings, 0 replies; 31+ messages in thread
From: Christoph Hellwig @ 2009-03-16 13:39 UTC (permalink / raw)
To: Chris Mason
Cc: Christoph Hellwig, Jens Axboe, linux-kernel, linux-fsdevel, david,
npiggin, linux-mm
On Mon, Mar 16, 2009 at 09:30:14AM -0400, Chris Mason wrote:
> Some of our poor filesystem cousins don't write the super until kupdate
> kicks them (see ext2_write_super). kupdate has always been the periodic
> FS thread of last resort.
Yikes, looks like this is indeed the only peridocial sb update for many
simpler filesystems. We should really have a separate thread for that
instead of hacking it into VM writeback. Especially with the per-bdi
one where the current setup doesn't make any sense.
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 2/7] writeback: switch to per-bdi threads for flushing data
@ 2009-03-16 13:39 ` Christoph Hellwig
0 siblings, 0 replies; 31+ messages in thread
From: Christoph Hellwig @ 2009-03-16 13:39 UTC (permalink / raw)
To: Chris Mason
Cc: Christoph Hellwig, Jens Axboe, linux-kernel, linux-fsdevel, david,
npiggin, linux-mm
On Mon, Mar 16, 2009 at 09:30:14AM -0400, Chris Mason wrote:
> Some of our poor filesystem cousins don't write the super until kupdate
> kicks them (see ext2_write_super). kupdate has always been the periodic
> FS thread of last resort.
Yikes, looks like this is indeed the only peridocial sb update for many
simpler filesystems. We should really have a separate thread for that
instead of hacking it into VM writeback. Especially with the per-bdi
one where the current setup doesn't make any sense.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH 3/7] writeback: get rid of pdflush_operation() in emergency sync and remount
2009-03-12 14:33 [PATCH 0/7] Per-bdi writeback flusher threads Jens Axboe
2009-03-12 14:33 ` [PATCH 1/7] writeback: move dirty inodes from super_block to backing_dev_info Jens Axboe
2009-03-12 14:33 ` [PATCH 2/7] writeback: switch to per-bdi threads for flushing data Jens Axboe
@ 2009-03-12 14:33 ` Jens Axboe
2009-03-16 10:13 ` Christoph Hellwig
2009-03-12 14:33 ` [PATCH 4/7] writeback: get rid of task/current_is_pdflush() Jens Axboe
` (3 subsequent siblings)
6 siblings, 1 reply; 31+ messages in thread
From: Jens Axboe @ 2009-03-12 14:33 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel; +Cc: chris.mason, david, npiggin, Jens Axboe
Opencode a cheasy approach with kevent. The idea here is that we'll
add some generic delayed work infrastructure, which probably wont be
based on pdflush (or maybe it will, in which case we can just add it
back).
Kill off mm/pdflush.c
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
fs/fs-writeback.c | 5 +
fs/super.c | 11 ++-
fs/sync.c | 14 +++-
include/linux/writeback.h | 1 -
mm/Makefile | 2 +-
mm/pdflush.c | 251 ---------------------------------------------
6 files changed, 28 insertions(+), 256 deletions(-)
delete mode 100644 mm/pdflush.c
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 209c6fa..37b042f 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -29,6 +29,11 @@
#define inode_to_bdi(inode) (inode)->i_mapping->backing_dev_info
+/*
+ * We don't actually have pdflush, but this one is exported though /proc...
+ */
+int nr_pdflush_threads = 0;
+
/**
* writeback_acquire - attempt to get exclusive writeback access to a device
* @bdi: the device's backing_dev_info structure
diff --git a/fs/super.c b/fs/super.c
index e3c5b6f..3c715fd 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -668,7 +668,7 @@ int do_remount_sb(struct super_block *sb, int flags, void *data, int force)
return 0;
}
-static void do_emergency_remount(unsigned long foo)
+static void do_emergency_remount(struct work_struct *work)
{
struct super_block *sb;
@@ -691,12 +691,19 @@ static void do_emergency_remount(unsigned long foo)
spin_lock(&sb_lock);
}
spin_unlock(&sb_lock);
+ kfree(work);
printk("Emergency Remount complete\n");
}
void emergency_remount(void)
{
- pdflush_operation(do_emergency_remount, 0);
+ struct work_struct *work;
+
+ work = kmalloc(sizeof(*work), GFP_ATOMIC);
+ if (work) {
+ INIT_WORK(work, do_emergency_remount);
+ schedule_work(work);
+ }
}
/*
diff --git a/fs/sync.c b/fs/sync.c
index f4ddcef..4552be6 100644
--- a/fs/sync.c
+++ b/fs/sync.c
@@ -42,9 +42,21 @@ SYSCALL_DEFINE0(sync)
return 0;
}
+static void do_sync_work(struct work_struct *work)
+{
+ do_sync(0);
+ kfree(work);
+}
+
void emergency_sync(void)
{
- pdflush_operation(do_sync, 0);
+ struct work_struct *work;
+
+ work = kmalloc(sizeof(*work), GFP_ATOMIC);
+ if (work) {
+ INIT_WORK(work, do_sync_work);
+ schedule_work(work);
+ }
}
/*
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 3697e91..724327b 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -151,7 +151,6 @@ balance_dirty_pages_ratelimited(struct address_space *mapping)
typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc,
void *data);
-int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0);
int generic_writepages(struct address_space *mapping,
struct writeback_control *wbc);
int write_cache_pages(struct address_space *mapping,
diff --git a/mm/Makefile b/mm/Makefile
index 72255be..6443e19 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -8,7 +8,7 @@ mmu-$(CONFIG_MMU) := fremap.o highmem.o madvise.o memory.o mincore.o \
vmalloc.o
obj-y := bootmem.o filemap.o mempool.o oom_kill.o fadvise.o \
- maccess.o page_alloc.o page-writeback.o pdflush.o \
+ maccess.o page_alloc.o page-writeback.o \
readahead.o swap.o truncate.o vmscan.o shmem.o \
prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \
page_isolation.o mm_init.o $(mmu-y)
diff --git a/mm/pdflush.c b/mm/pdflush.c
deleted file mode 100644
index 15de509..0000000
--- a/mm/pdflush.c
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * mm/pdflush.c - worker threads for writing back filesystem data
- *
- * Copyright (C) 2002, Linus Torvalds.
- *
- * 09Apr2002 Andrew Morton
- * Initial version
- * 29Feb2004 kaos@sgi.com
- * Move worker thread creation to kthread to avoid chewing
- * up stack space with nested calls to kernel_thread.
- */
-
-#include <linux/sched.h>
-#include <linux/list.h>
-#include <linux/signal.h>
-#include <linux/spinlock.h>
-#include <linux/gfp.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/fs.h> /* Needed by writeback.h */
-#include <linux/writeback.h> /* Prototypes pdflush_operation() */
-#include <linux/kthread.h>
-#include <linux/cpuset.h>
-#include <linux/freezer.h>
-
-
-/*
- * Minimum and maximum number of pdflush instances
- */
-#define MIN_PDFLUSH_THREADS 2
-#define MAX_PDFLUSH_THREADS 8
-
-static void start_one_pdflush_thread(void);
-
-
-/*
- * The pdflush threads are worker threads for writing back dirty data.
- * Ideally, we'd like one thread per active disk spindle. But the disk
- * topology is very hard to divine at this level. Instead, we take
- * care in various places to prevent more than one pdflush thread from
- * performing writeback against a single filesystem. pdflush threads
- * have the PF_FLUSHER flag set in current->flags to aid in this.
- */
-
-/*
- * All the pdflush threads. Protected by pdflush_lock
- */
-static LIST_HEAD(pdflush_list);
-static DEFINE_SPINLOCK(pdflush_lock);
-
-/*
- * The count of currently-running pdflush threads. Protected
- * by pdflush_lock.
- *
- * Readable by sysctl, but not writable. Published to userspace at
- * /proc/sys/vm/nr_pdflush_threads.
- */
-int nr_pdflush_threads = 0;
-
-/*
- * The time at which the pdflush thread pool last went empty
- */
-static unsigned long last_empty_jifs;
-
-/*
- * The pdflush thread.
- *
- * Thread pool management algorithm:
- *
- * - The minimum and maximum number of pdflush instances are bound
- * by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS.
- *
- * - If there have been no idle pdflush instances for 1 second, create
- * a new one.
- *
- * - If the least-recently-went-to-sleep pdflush thread has been asleep
- * for more than one second, terminate a thread.
- */
-
-/*
- * A structure for passing work to a pdflush thread. Also for passing
- * state information between pdflush threads. Protected by pdflush_lock.
- */
-struct pdflush_work {
- struct task_struct *who; /* The thread */
- void (*fn)(unsigned long); /* A callback function */
- unsigned long arg0; /* An argument to the callback */
- struct list_head list; /* On pdflush_list, when idle */
- unsigned long when_i_went_to_sleep;
-};
-
-static int __pdflush(struct pdflush_work *my_work)
-{
- current->flags |= PF_FLUSHER | PF_SWAPWRITE;
- set_freezable();
- my_work->fn = NULL;
- my_work->who = current;
- INIT_LIST_HEAD(&my_work->list);
-
- spin_lock_irq(&pdflush_lock);
- nr_pdflush_threads++;
- for ( ; ; ) {
- struct pdflush_work *pdf;
-
- set_current_state(TASK_INTERRUPTIBLE);
- list_move(&my_work->list, &pdflush_list);
- my_work->when_i_went_to_sleep = jiffies;
- spin_unlock_irq(&pdflush_lock);
- schedule();
- try_to_freeze();
- spin_lock_irq(&pdflush_lock);
- if (!list_empty(&my_work->list)) {
- /*
- * Someone woke us up, but without removing our control
- * structure from the global list. swsusp will do this
- * in try_to_freeze()->refrigerator(). Handle it.
- */
- my_work->fn = NULL;
- continue;
- }
- if (my_work->fn == NULL) {
- printk("pdflush: bogus wakeup\n");
- continue;
- }
- spin_unlock_irq(&pdflush_lock);
-
- (*my_work->fn)(my_work->arg0);
-
- /*
- * Thread creation: For how long have there been zero
- * available threads?
- */
- if (time_after(jiffies, last_empty_jifs + 1 * HZ)) {
- /* unlocked list_empty() test is OK here */
- if (list_empty(&pdflush_list)) {
- /* unlocked test is OK here */
- if (nr_pdflush_threads < MAX_PDFLUSH_THREADS)
- start_one_pdflush_thread();
- }
- }
-
- spin_lock_irq(&pdflush_lock);
- my_work->fn = NULL;
-
- /*
- * Thread destruction: For how long has the sleepiest
- * thread slept?
- */
- if (list_empty(&pdflush_list))
- continue;
- if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS)
- continue;
- pdf = list_entry(pdflush_list.prev, struct pdflush_work, list);
- if (time_after(jiffies, pdf->when_i_went_to_sleep + 1 * HZ)) {
- /* Limit exit rate */
- pdf->when_i_went_to_sleep = jiffies;
- break; /* exeunt */
- }
- }
- nr_pdflush_threads--;
- spin_unlock_irq(&pdflush_lock);
- return 0;
-}
-
-/*
- * Of course, my_work wants to be just a local in __pdflush(). It is
- * separated out in this manner to hopefully prevent the compiler from
- * performing unfortunate optimisations against the auto variables. Because
- * these are visible to other tasks and CPUs. (No problem has actually
- * been observed. This is just paranoia).
- */
-static int pdflush(void *dummy)
-{
- struct pdflush_work my_work;
- cpumask_var_t cpus_allowed;
-
- /*
- * Since the caller doesn't even check kthread_run() worked, let's not
- * freak out too much if this fails.
- */
- if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
- printk(KERN_WARNING "pdflush failed to allocate cpumask\n");
- return 0;
- }
-
- /*
- * pdflush can spend a lot of time doing encryption via dm-crypt. We
- * don't want to do that at keventd's priority.
- */
- set_user_nice(current, 0);
-
- /*
- * Some configs put our parent kthread in a limited cpuset,
- * which kthread() overrides, forcing cpus_allowed == CPU_MASK_ALL.
- * Our needs are more modest - cut back to our cpusets cpus_allowed.
- * This is needed as pdflush's are dynamically created and destroyed.
- * The boottime pdflush's are easily placed w/o these 2 lines.
- */
- cpuset_cpus_allowed(current, cpus_allowed);
- set_cpus_allowed_ptr(current, cpus_allowed);
- free_cpumask_var(cpus_allowed);
-
- return __pdflush(&my_work);
-}
-
-/*
- * Attempt to wake up a pdflush thread, and get it to do some work for you.
- * Returns zero if it indeed managed to find a worker thread, and passed your
- * payload to it.
- */
-int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
-{
- unsigned long flags;
- int ret = 0;
-
- BUG_ON(fn == NULL); /* Hard to diagnose if it's deferred */
-
- spin_lock_irqsave(&pdflush_lock, flags);
- if (list_empty(&pdflush_list)) {
- ret = -1;
- } else {
- struct pdflush_work *pdf;
-
- pdf = list_entry(pdflush_list.next, struct pdflush_work, list);
- list_del_init(&pdf->list);
- if (list_empty(&pdflush_list))
- last_empty_jifs = jiffies;
- pdf->fn = fn;
- pdf->arg0 = arg0;
- wake_up_process(pdf->who);
- }
- spin_unlock_irqrestore(&pdflush_lock, flags);
-
- return ret;
-}
-
-static void start_one_pdflush_thread(void)
-{
- kthread_run(pdflush, NULL, "pdflush");
-}
-
-static int __init pdflush_init(void)
-{
- int i;
-
- for (i = 0; i < MIN_PDFLUSH_THREADS; i++)
- start_one_pdflush_thread();
- return 0;
-}
-
-module_init(pdflush_init);
--
1.6.2
^ permalink raw reply related [flat|nested] 31+ messages in thread* Re: [PATCH 3/7] writeback: get rid of pdflush_operation() in emergency sync and remount
2009-03-12 14:33 ` [PATCH 3/7] writeback: get rid of pdflush_operation() in emergency sync and remount Jens Axboe
@ 2009-03-16 10:13 ` Christoph Hellwig
0 siblings, 0 replies; 31+ messages in thread
From: Christoph Hellwig @ 2009-03-16 10:13 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
On Thu, Mar 12, 2009 at 03:33:44PM +0100, Jens Axboe wrote:
> Opencode a cheasy approach with kevent. The idea here is that we'll
> add some generic delayed work infrastructure, which probably wont be
> based on pdflush (or maybe it will, in which case we can just add it
> back).
Good idea. Care to submit this early outside of the series?
>
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH 4/7] writeback: get rid of task/current_is_pdflush()
2009-03-12 14:33 [PATCH 0/7] Per-bdi writeback flusher threads Jens Axboe
` (2 preceding siblings ...)
2009-03-12 14:33 ` [PATCH 3/7] writeback: get rid of pdflush_operation() in emergency sync and remount Jens Axboe
@ 2009-03-12 14:33 ` Jens Axboe
2009-03-16 10:14 ` Christoph Hellwig
2009-03-12 14:33 ` [PATCH 5/7] writeback: move the default backing_dev_info out of readahead Jens Axboe
` (2 subsequent siblings)
6 siblings, 1 reply; 31+ messages in thread
From: Jens Axboe @ 2009-03-12 14:33 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel; +Cc: chris.mason, david, npiggin, Jens Axboe
There's just a single user in the kernel and Chris says it's safe to kill.
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
fs/btrfs/disk-io.c | 2 +-
include/linux/writeback.h | 11 -----------
2 files changed, 1 insertions(+), 12 deletions(-)
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 3e18175..6ec80c0 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -2385,7 +2385,7 @@ void btrfs_btree_balance_dirty(struct btrfs_root *root, unsigned long nr)
unsigned long thresh = 32 * 1024 * 1024;
tree = &BTRFS_I(root->fs_info->btree_inode)->io_tree;
- if (current_is_pdflush() || current->flags & PF_MEMALLOC)
+ if (current->flags & PF_MEMALLOC)
return;
num_dirty = count_range_bits(tree, &start, (u64)-1,
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 724327b..f96fea3 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -14,17 +14,6 @@ extern struct list_head inode_in_use;
extern struct list_head inode_unused;
/*
- * Yes, writeback.h requires sched.h
- * No, sched.h is not included from here.
- */
-static inline int task_is_pdflush(struct task_struct *task)
-{
- return task->flags & PF_FLUSHER;
-}
-
-#define current_is_pdflush() task_is_pdflush(current)
-
-/*
* fs/fs-writeback.c
*/
enum writeback_sync_modes {
--
1.6.2
^ permalink raw reply related [flat|nested] 31+ messages in thread* Re: [PATCH 4/7] writeback: get rid of task/current_is_pdflush()
2009-03-12 14:33 ` [PATCH 4/7] writeback: get rid of task/current_is_pdflush() Jens Axboe
@ 2009-03-16 10:14 ` Christoph Hellwig
2009-03-16 10:22 ` Jens Axboe
2009-03-16 13:26 ` Chris Mason
0 siblings, 2 replies; 31+ messages in thread
From: Christoph Hellwig @ 2009-03-16 10:14 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
On Thu, Mar 12, 2009 at 03:33:45PM +0100, Jens Axboe wrote:
> There's just a single user in the kernel and Chris says it's safe to kill.
I'll defer about the use to Chris, but if this can go it should now,
not deep inside the series.
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 4/7] writeback: get rid of task/current_is_pdflush()
2009-03-16 10:14 ` Christoph Hellwig
@ 2009-03-16 10:22 ` Jens Axboe
2009-03-16 13:26 ` Chris Mason
1 sibling, 0 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-16 10:22 UTC (permalink / raw)
To: Christoph Hellwig
Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
On Mon, Mar 16 2009, Christoph Hellwig wrote:
> On Thu, Mar 12, 2009 at 03:33:45PM +0100, Jens Axboe wrote:
> > There's just a single user in the kernel and Chris says it's safe to kill.
>
> I'll defer about the use to Chris, but if this can go it should now,
> not deep inside the series.
Oh agree, the series was more of an RFC, it wasn't a submission. It's
not even well tested yet. I'll snip the emergency sync/remount stuff and
this out of the series and put it at the front, so it can go in seperate
and earlier.
--
Jens Axboe
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 4/7] writeback: get rid of task/current_is_pdflush()
2009-03-16 10:14 ` Christoph Hellwig
2009-03-16 10:22 ` Jens Axboe
@ 2009-03-16 13:26 ` Chris Mason
1 sibling, 0 replies; 31+ messages in thread
From: Chris Mason @ 2009-03-16 13:26 UTC (permalink / raw)
To: Christoph Hellwig; +Cc: Jens Axboe, linux-kernel, linux-fsdevel, david, npiggin
On Mon, 2009-03-16 at 06:14 -0400, Christoph Hellwig wrote:
> On Thu, Mar 12, 2009 at 03:33:45PM +0100, Jens Axboe wrote:
> > There's just a single user in the kernel and Chris says it's safe to kill.
>
> I'll defer about the use to Chris, but if this can go it should now,
> not deep inside the series.
>
I'll queue it up for 2.6.30-rc1
-chris
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH 5/7] writeback: move the default backing_dev_info out of readahead
2009-03-12 14:33 [PATCH 0/7] Per-bdi writeback flusher threads Jens Axboe
` (3 preceding siblings ...)
2009-03-12 14:33 ` [PATCH 4/7] writeback: get rid of task/current_is_pdflush() Jens Axboe
@ 2009-03-12 14:33 ` Jens Axboe
2009-03-16 10:19 ` Christoph Hellwig
2009-03-12 14:33 ` [PATCH 6/7] writeback: add lazy bdi->task creation Jens Axboe
2009-03-12 14:33 ` [PATCH 7/7] writeback: add some debug inode list counters to bdi stats Jens Axboe
6 siblings, 1 reply; 31+ messages in thread
From: Jens Axboe @ 2009-03-12 14:33 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel; +Cc: chris.mason, david, npiggin, Jens Axboe
It belongs in mm/backing-dev.c
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
mm/backing-dev.c | 27 ++++++++++++++++++++++++++-
mm/readahead.c | 25 -------------------------
2 files changed, 26 insertions(+), 26 deletions(-)
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index bf9c0e0..0096b96 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -5,10 +5,24 @@
#include <linux/fs.h>
#include <linux/sched.h>
#include <linux/mm.h>
+#include <linux/pagemap.h>
#include <linux/module.h>
#include <linux/writeback.h>
#include <linux/device.h>
+void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
+{
+}
+EXPORT_SYMBOL(default_unplug_io_fn);
+
+struct backing_dev_info default_backing_dev_info = {
+ .ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE,
+ .state = 0,
+ .capabilities = BDI_CAP_MAP_COPY,
+ .unplug_io_fn = default_unplug_io_fn,
+};
+EXPORT_SYMBOL_GPL(default_backing_dev_info);
+
static struct class *bdi_class;
DEFINE_SPINLOCK(bdi_lock);
LIST_HEAD(bdi_list);
@@ -169,9 +183,20 @@ static __init int bdi_class_init(void)
bdi_debug_init();
return 0;
}
-
postcore_initcall(bdi_class_init);
+static int __init default_bdi_init(void)
+{
+ int err;
+
+ err = bdi_init(&default_backing_dev_info);
+ if (!err)
+ bdi_register(&default_backing_dev_info, NULL, "default");
+
+ return err;
+}
+subsys_initcall(default_bdi_init);
+
int bdi_register(struct backing_dev_info *bdi, struct device *parent,
const char *fmt, ...)
{
diff --git a/mm/readahead.c b/mm/readahead.c
index bec83c1..9ce303d 100644
--- a/mm/readahead.c
+++ b/mm/readahead.c
@@ -17,19 +17,6 @@
#include <linux/pagevec.h>
#include <linux/pagemap.h>
-void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
-{
-}
-EXPORT_SYMBOL(default_unplug_io_fn);
-
-struct backing_dev_info default_backing_dev_info = {
- .ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE,
- .state = 0,
- .capabilities = BDI_CAP_MAP_COPY,
- .unplug_io_fn = default_unplug_io_fn,
-};
-EXPORT_SYMBOL_GPL(default_backing_dev_info);
-
/*
* Initialise a struct file's readahead state. Assumes that the caller has
* memset *ra to zero.
@@ -233,18 +220,6 @@ unsigned long max_sane_readahead(unsigned long nr)
+ node_page_state(numa_node_id(), NR_FREE_PAGES)) / 2);
}
-static int __init readahead_init(void)
-{
- int err;
-
- err = bdi_init(&default_backing_dev_info);
- if (!err)
- bdi_register(&default_backing_dev_info, NULL, "default");
-
- return err;
-}
-subsys_initcall(readahead_init);
-
/*
* Submit IO for the read-ahead request in file_ra_state.
*/
--
1.6.2
^ permalink raw reply related [flat|nested] 31+ messages in thread* Re: [PATCH 5/7] writeback: move the default backing_dev_info out of readahead
2009-03-12 14:33 ` [PATCH 5/7] writeback: move the default backing_dev_info out of readahead Jens Axboe
@ 2009-03-16 10:19 ` Christoph Hellwig
2009-03-16 10:23 ` Jens Axboe
0 siblings, 1 reply; 31+ messages in thread
From: Christoph Hellwig @ 2009-03-16 10:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
On Thu, Mar 12, 2009 at 03:33:46PM +0100, Jens Axboe wrote:
> It belongs in mm/backing-dev.c
Good cleanup, should be queued up ASAP.
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 5/7] writeback: move the default backing_dev_info out of readahead
2009-03-16 10:19 ` Christoph Hellwig
@ 2009-03-16 10:23 ` Jens Axboe
0 siblings, 0 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-16 10:23 UTC (permalink / raw)
To: Christoph Hellwig
Cc: linux-kernel, linux-fsdevel, chris.mason, david, npiggin
On Mon, Mar 16 2009, Christoph Hellwig wrote:
> On Thu, Mar 12, 2009 at 03:33:46PM +0100, Jens Axboe wrote:
> > It belongs in mm/backing-dev.c
>
> Good cleanup, should be queued up ASAP.
Yep, will place that at the front as well.
--
Jens Axboe
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH 6/7] writeback: add lazy bdi->task creation
2009-03-12 14:33 [PATCH 0/7] Per-bdi writeback flusher threads Jens Axboe
` (4 preceding siblings ...)
2009-03-12 14:33 ` [PATCH 5/7] writeback: move the default backing_dev_info out of readahead Jens Axboe
@ 2009-03-12 14:33 ` Jens Axboe
2009-03-12 14:33 ` [PATCH 7/7] writeback: add some debug inode list counters to bdi stats Jens Axboe
6 siblings, 0 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-12 14:33 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel; +Cc: chris.mason, david, npiggin, Jens Axboe
Instead of creating the bdi flusher threads when the bdi is registered,
defer that to the point where we have dirty IO pending and someone
attempts to start the flushing.
A bdi is put on the normal bdi_list when it is registered. When someone
attempts to schedule writeback on this bdi, we move it to a pending list
and wake up the default bdi forker thread to take care of setting up a
task and putting the bdi back on the normal bdi_list. If task creation
should fail, the forker thread will writeout some data on behalf of the
pending bdi. This should ensure progress always.
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
fs/fs-writeback.c | 42 +++++-----
include/linux/backing-dev.h | 8 ++-
mm/backing-dev.c | 196 +++++++++++++++++++++++++++++++++++++++----
3 files changed, 206 insertions(+), 40 deletions(-)
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 37b042f..c25c261 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -74,14 +74,17 @@ static void writeback_release(struct backing_dev_info *bdi)
clear_bit(BDI_pdflush, &bdi->state);
}
-void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
+int bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
long nr_pages)
{
/*
- * Should not happen, complain?
+ * This only happens the first time someone kicks this bdi, so put
+ * it out-of-line.
*/
- if (unlikely(!bdi->task))
- return;
+ if (unlikely(!bdi->task)) {
+ bdi_add_flusher_task(bdi);
+ return 1;
+ }
if (writeback_acquire(bdi)) {
bdi->wb_arg.nr_pages = nr_pages;
@@ -92,6 +95,8 @@ void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
smp_mb();
wake_up(&bdi->wait);
}
+
+ return 0;
}
/*
@@ -185,24 +190,13 @@ static void bdi_pdflush(struct backing_dev_info *bdi)
* Handle writeback of dirty data for the device backed by this bdi. Also
* wakes up periodically and does kupdated style flushing.
*/
-int bdi_writeback_task(void *ptr)
+int bdi_writeback_task(struct backing_dev_info *bdi)
{
- struct backing_dev_info *bdi = ptr;
- struct task_struct *tsk = current;
-
- tsk->flags |= PF_FLUSHER | PF_SWAPWRITE;
- set_freezable();
-
- /*
- * Our parent may run at a different priority, just set us to normal
- */
- set_user_nice(tsk, 0);
-
while (!kthread_should_stop()) {
- DECLARE_WAITQUEUE(wait, tsk);
+ DECLARE_WAITQUEUE(wait, current);
add_wait_queue(&bdi->wait, &wait);
- set_task_state(tsk, TASK_INTERRUPTIBLE);
+ set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(dirty_writeback_interval);
try_to_freeze();
@@ -226,7 +220,7 @@ int bdi_writeback_task(void *ptr)
bdi_pdflush(bdi);
writeback_release(bdi);
- set_task_state(tsk, TASK_RUNNING);
+ set_current_state(TASK_RUNNING);
finish_wait(&bdi->wait, &wait);
}
@@ -239,9 +233,13 @@ void bdi_writeback_all(struct super_block *sb, long nr_pages)
rcu_read_lock();
- list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
- if (bdi_has_dirty_io(bdi))
- bdi_start_writeback(bdi, sb, nr_pages);
+restart:
+ list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
+ if (!bdi_has_dirty_io(bdi))
+ continue;
+ if (bdi_start_writeback(bdi, sb, nr_pages))
+ goto restart;
+ }
rcu_read_unlock();
}
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index 3c94fbd..b9e2085 100644
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -24,6 +24,7 @@ struct dentry;
*/
enum bdi_state {
BDI_pdflush, /* A pdflush thread is working this device */
+ BDI_pending, /* On its way to being activated */
BDI_write_congested, /* The write queue is getting full */
BDI_read_congested, /* The read queue is getting full */
BDI_unused, /* Available bits start here */
@@ -46,6 +47,7 @@ struct bdi_writeback_arg {
struct backing_dev_info {
struct list_head bdi_list;
+ struct rcu_head rcu_head;
unsigned long ra_pages; /* max readahead in PAGE_CACHE_SIZE units */
unsigned long state; /* Always use atomic bitops on this */
@@ -85,10 +87,11 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
const char *fmt, ...);
int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev);
void bdi_unregister(struct backing_dev_info *bdi);
-void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
+int bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
long nr_pages);
-int bdi_writeback_task(void *);
+int bdi_writeback_task(struct backing_dev_info *bdi);
void bdi_writeback_all(struct super_block *sb, long nr_pages);
+void bdi_add_flusher_task(struct backing_dev_info *bdi);
extern spinlock_t bdi_lock;
extern struct list_head bdi_list;
@@ -215,6 +218,7 @@ int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);
#define BDI_CAP_EXEC_MAP 0x00000040
#define BDI_CAP_NO_ACCT_WB 0x00000080
#define BDI_CAP_SWAP_BACKED 0x00000100
+#define BDI_CAP_FLUSH_FORKER 0x00000200
#define BDI_CAP_VMFLAGS \
(BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP | BDI_CAP_EXEC_MAP)
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index 0096b96..500d1fc 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -2,6 +2,7 @@
#include <linux/wait.h>
#include <linux/backing-dev.h>
#include <linux/kthread.h>
+#include <linux/freezer.h>
#include <linux/fs.h>
#include <linux/sched.h>
#include <linux/mm.h>
@@ -18,7 +19,7 @@ EXPORT_SYMBOL(default_unplug_io_fn);
struct backing_dev_info default_backing_dev_info = {
.ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE,
.state = 0,
- .capabilities = BDI_CAP_MAP_COPY,
+ .capabilities = BDI_CAP_MAP_COPY | BDI_CAP_FLUSH_FORKER,
.unplug_io_fn = default_unplug_io_fn,
};
EXPORT_SYMBOL_GPL(default_backing_dev_info);
@@ -26,6 +27,7 @@ EXPORT_SYMBOL_GPL(default_backing_dev_info);
static struct class *bdi_class;
DEFINE_SPINLOCK(bdi_lock);
LIST_HEAD(bdi_list);
+LIST_HEAD(bdi_pending_list);
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
@@ -197,6 +199,147 @@ static int __init default_bdi_init(void)
}
subsys_initcall(default_bdi_init);
+static int bdi_start_fn(void *ptr)
+{
+ struct backing_dev_info *bdi = ptr;
+ struct task_struct *tsk = current;
+
+ /*
+ * Add us to the active bdi_list
+ */
+ spin_lock_bh(&bdi_lock);
+ list_add_rcu(&bdi->bdi_list, &bdi_list);
+ spin_unlock_bh(&bdi_lock);
+
+ tsk->flags |= PF_FLUSHER | PF_SWAPWRITE;
+ set_freezable();
+
+ /*
+ * Our parent may run at a different priority, just set us to normal
+ */
+ set_user_nice(tsk, 0);
+
+ /*
+ * Clear pending bit and wakeup anybody waiting to tear us down
+ */
+ clear_bit(BDI_pending, &bdi->state);
+ wake_up_bit(&bdi->state, BDI_pending);
+
+ return bdi_writeback_task(bdi);
+}
+
+static int bdi_forker_task(void *ptr)
+{
+ struct backing_dev_info *bdi = ptr;
+ struct task_struct *tsk = current;
+
+ for (;;) {
+ DECLARE_WAITQUEUE(wait, tsk);
+
+ /*
+ * Should never trigger on the default bdi
+ */
+ WARN_ON(bdi_has_dirty_io(bdi));
+
+ add_wait_queue(&bdi->wait, &wait);
+ set_task_state(tsk, TASK_INTERRUPTIBLE);
+ smp_mb();
+ if (list_empty(&bdi_pending_list))
+ schedule();
+ else {
+ struct backing_dev_info *bdi = NULL;
+
+ spin_lock_bh(&bdi_lock);
+ if (!list_empty(&bdi_pending_list)) {
+ bdi = list_entry(bdi_pending_list.next,
+ struct backing_dev_info,
+ bdi_list);
+ list_del_init(&bdi->bdi_list);
+ }
+ spin_unlock_bh(&bdi_lock);
+
+ /*
+ * If no bdi or bdi already got setup, continue
+ */
+ if (!bdi || bdi->task)
+ continue;
+
+ bdi->task = kthread_run(bdi_start_fn, bdi, "bdi-%s",
+ dev_name(bdi->dev));
+ /*
+ * If task creation fails, then readd the bdi to
+ * the pending list and force writeout of the bdi
+ * from this forker thread. That will free some memory
+ * and we can try again.
+ */
+ if (!bdi->task) {
+ struct writeback_control wbc = {
+ .bdi = bdi,
+ .sync_mode = WB_SYNC_NONE,
+ .older_than_this = NULL,
+ .range_cyclic = 1,
+ };
+
+ /*
+ * Add this 'bdi' to the back, so we get
+ * a chance to flush other bdi's to free
+ * memory.
+ */
+ spin_lock_bh(&bdi_lock);
+ list_add_tail(&bdi->bdi_list,
+ &bdi_pending_list);
+ spin_unlock_bh(&bdi_lock);
+
+ wbc.nr_to_write = 1024;
+ generic_sync_bdi_inodes(NULL, &wbc);
+ }
+ }
+
+ set_task_state(tsk, TASK_RUNNING);
+ finish_wait(&bdi->wait, &wait);
+ }
+
+ return 0;
+}
+
+/*
+ * Grace period has now ended, init bdi->bdi_list and add us to the
+ * list of bdi's that are pending for task creation. Wake up
+ * bdi_forker_task() to finish the job and add us back to the
+ * active bdi_list.
+ */
+static void bdi_add_to_pending(struct rcu_head *head)
+{
+ struct backing_dev_info *bdi;
+
+ bdi = container_of(head, struct backing_dev_info, rcu_head);
+ INIT_LIST_HEAD(&bdi->bdi_list);
+
+ spin_lock(&bdi_lock);
+ list_add_tail(&bdi->bdi_list, &bdi_pending_list);
+ spin_unlock(&bdi_lock);
+
+ wake_up(&default_backing_dev_info.wait);
+}
+
+void bdi_add_flusher_task(struct backing_dev_info *bdi)
+{
+ if (test_and_set_bit(BDI_pending, &bdi->state))
+ return;
+
+ spin_lock_bh(&bdi_lock);
+ list_del_rcu(&bdi->bdi_list);
+ spin_unlock_bh(&bdi_lock);
+
+ /*
+ * We need to wait for the current grace period to end,
+ * in case others were browsing the bdi_list as well.
+ * So defer the adding and wakeup to after the RCU
+ * grace period has ended.
+ */
+ call_rcu(&bdi->rcu_head, bdi_add_to_pending);
+}
+
int bdi_register(struct backing_dev_info *bdi, struct device *parent,
const char *fmt, ...)
{
@@ -215,17 +358,24 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
goto exit;
}
- bdi->task = kthread_run(bdi_writeback_task, bdi, "bdi-%s",
- dev_name(dev));
- if (!bdi->task) {
- ret = -ENOMEM;
- goto exit;
+ /*
+ * Just start the forker thread for our default backing_dev_info,
+ * and add other bdi's to the list. They will get a thread created
+ * on-demand when they need it.
+ */
+ if (bdi->capabilities & BDI_CAP_FLUSH_FORKER) {
+ bdi->task = kthread_run(bdi_forker_task, bdi, "bdi-%s",
+ dev_name(dev));
+ if (!bdi->task) {
+ ret = -ENOMEM;
+ goto exit;
+ }
+ } else {
+ spin_lock_bh(&bdi_lock);
+ list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
+ spin_unlock_bh(&bdi_lock);
}
- spin_lock(&bdi_lock);
- list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
- spin_unlock(&bdi_lock);
-
bdi->dev = dev;
bdi_debug_register(bdi, dev_name(dev));
@@ -240,11 +390,22 @@ int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
}
EXPORT_SYMBOL(bdi_register_dev);
+static int sched_wait(void *word)
+{
+ schedule();
+ return 0;
+}
+
static void bdi_remove_from_list(struct backing_dev_info *bdi)
{
- spin_lock(&bdi_lock);
+ /*
+ * If setup is pending, wait for that to complete first
+ */
+ wait_on_bit(&bdi->state, BDI_pending, sched_wait, TASK_UNINTERRUPTIBLE);
+
+ spin_lock_bh(&bdi_lock);
list_del_rcu(&bdi->bdi_list);
- spin_unlock(&bdi_lock);
+ spin_unlock_bh(&bdi_lock);
/*
* In case the bdi is freed right after unregister, we need to
@@ -256,10 +417,12 @@ static void bdi_remove_from_list(struct backing_dev_info *bdi)
void bdi_unregister(struct backing_dev_info *bdi)
{
if (bdi->dev) {
- bdi_remove_from_list(bdi);
- if (bdi->task) {
- kthread_stop(bdi->task);
- bdi->task = NULL;
+ if (!(bdi->capabilities & BDI_CAP_FLUSH_FORKER)) {
+ bdi_remove_from_list(bdi);
+ if (bdi->task) {
+ kthread_stop(bdi->task);
+ bdi->task = NULL;
+ }
}
bdi_debug_unregister(bdi);
device_unregister(bdi->dev);
@@ -272,6 +435,7 @@ int bdi_init(struct backing_dev_info *bdi)
{
int i, err;
+ INIT_RCU_HEAD(&bdi->rcu_head);
bdi->dev = NULL;
bdi->min_ratio = 0;
--
1.6.2
^ permalink raw reply related [flat|nested] 31+ messages in thread* [PATCH 7/7] writeback: add some debug inode list counters to bdi stats
2009-03-12 14:33 [PATCH 0/7] Per-bdi writeback flusher threads Jens Axboe
` (5 preceding siblings ...)
2009-03-12 14:33 ` [PATCH 6/7] writeback: add lazy bdi->task creation Jens Axboe
@ 2009-03-12 14:33 ` Jens Axboe
6 siblings, 0 replies; 31+ messages in thread
From: Jens Axboe @ 2009-03-12 14:33 UTC (permalink / raw)
To: linux-kernel, linux-fsdevel; +Cc: chris.mason, david, npiggin, Jens Axboe
Not meant for inclusion, just to monitor what is going on while testing
this stuff.
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
mm/backing-dev.c | 19 +++++++++++++++++--
1 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index 500d1fc..76a7809 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -46,6 +46,18 @@ static int bdi_debug_stats_show(struct seq_file *m, void *v)
unsigned long background_thresh;
unsigned long dirty_thresh;
unsigned long bdi_thresh;
+ unsigned long nr_dirty, nr_io, nr_more_io;
+ struct inode *inode;
+
+ nr_dirty = nr_io = nr_more_io = 0;
+ spin_lock(&inode_lock);
+ list_for_each_entry(inode, &bdi->b_dirty, i_list)
+ nr_dirty++;
+ list_for_each_entry(inode, &bdi->b_io, i_list)
+ nr_io++;
+ list_for_each_entry(inode, &bdi->b_more_io, i_list)
+ nr_more_io++;
+ spin_unlock(&inode_lock);
get_dirty_limits(&background_thresh, &dirty_thresh, &bdi_thresh, bdi);
@@ -55,12 +67,15 @@ static int bdi_debug_stats_show(struct seq_file *m, void *v)
"BdiReclaimable: %8lu kB\n"
"BdiDirtyThresh: %8lu kB\n"
"DirtyThresh: %8lu kB\n"
- "BackgroundThresh: %8lu kB\n",
+ "BackgroundThresh: %8lu kB\n"
+ "b_dirty: %8lu\n"
+ "b_io: %8lu\n"
+ "b_more_io: %8lu\n",
(unsigned long) K(bdi_stat(bdi, BDI_WRITEBACK)),
(unsigned long) K(bdi_stat(bdi, BDI_RECLAIMABLE)),
K(bdi_thresh),
K(dirty_thresh),
- K(background_thresh));
+ K(background_thresh), nr_dirty, nr_io, nr_more_io);
#undef K
return 0;
--
1.6.2
^ permalink raw reply related [flat|nested] 31+ messages in thread