From: Dave Chinner <david@fromorbit.com>
To: linux-fsdevel@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, akpm@linux-foundation.org,
davej@redhat.com, viro@zeniv.linux.org.uk, jack@suse.cz,
glommer@parallels.com
Subject: [PATCH 09/11] fs: Use RCU lookups for inode cache
Date: Wed, 31 Jul 2013 14:15:48 +1000 [thread overview]
Message-ID: <1375244150-27296-10-git-send-email-david@fromorbit.com> (raw)
In-Reply-To: <1375244150-27296-1-git-send-email-david@fromorbit.com>
From: Dave Chinner <dchinner@redhat.com>
Convert inode cache lookups to be protected by RCU locking rather
than the global inode_hash_lock. This will improve scalability of
inode lookup intensive workloads.
Tested w/ ext4 and btrfs on concurrent fsmark/lookup/unlink
workloads only. It removes the inode hash lock from the inode lookup
paths, but does not solve the problem of the inode hash lock being a
bottleneck on the inode cache insert/remove paths.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
---
fs/inode.c | 76 +++++++++++++++++++++++++++++++++++---------------------------
1 file changed, 43 insertions(+), 33 deletions(-)
diff --git a/fs/inode.c b/fs/inode.c
index 7eea591..810386e 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -465,7 +465,7 @@ void __insert_inode_hash(struct inode *inode, unsigned long hashval)
spin_lock(&inode_hash_lock);
spin_lock(&inode->i_lock);
- hlist_add_head(&inode->i_hash, b);
+ hlist_add_head_rcu(&inode->i_hash, b);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_hash_lock);
}
@@ -481,7 +481,7 @@ void __remove_inode_hash(struct inode *inode)
{
spin_lock(&inode_hash_lock);
spin_lock(&inode->i_lock);
- hlist_del_init(&inode->i_hash);
+ hlist_del_init_rcu(&inode->i_hash);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_hash_lock);
}
@@ -776,13 +776,18 @@ static void __wait_on_freeing_inode(struct inode *inode);
static struct inode *find_inode(struct super_block *sb,
struct hlist_head *head,
int (*test)(struct inode *, void *),
- void *data)
+ void *data, bool locked)
{
struct inode *inode = NULL;
repeat:
- hlist_for_each_entry(inode, head, i_hash) {
+ rcu_read_lock();
+ hlist_for_each_entry_rcu(inode, head, i_hash) {
spin_lock(&inode->i_lock);
+ if (inode_unhashed(inode)) {
+ spin_unlock(&inode->i_lock);
+ continue;
+ }
if (inode->i_sb != sb) {
spin_unlock(&inode->i_lock);
continue;
@@ -792,13 +797,20 @@ repeat:
continue;
}
if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
+ rcu_read_unlock();
+ if (locked)
+ spin_unlock(&inode_hash_lock);
__wait_on_freeing_inode(inode);
+ if (locked)
+ spin_lock(&inode_hash_lock);
goto repeat;
}
__iget(inode);
spin_unlock(&inode->i_lock);
+ rcu_read_unlock();
return inode;
}
+ rcu_read_unlock();
return NULL;
}
@@ -807,13 +819,19 @@ repeat:
* iget_locked for details.
*/
static struct inode *find_inode_fast(struct super_block *sb,
- struct hlist_head *head, unsigned long ino)
+ struct hlist_head *head,
+ unsigned long ino, bool locked)
{
struct inode *inode = NULL;
repeat:
- hlist_for_each_entry(inode, head, i_hash) {
+ rcu_read_lock();
+ hlist_for_each_entry_rcu(inode, head, i_hash) {
spin_lock(&inode->i_lock);
+ if (inode_unhashed(inode)) {
+ spin_unlock(&inode->i_lock);
+ continue;
+ }
if (inode->i_ino != ino) {
spin_unlock(&inode->i_lock);
continue;
@@ -823,13 +841,20 @@ repeat:
continue;
}
if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
+ rcu_read_unlock();
+ if (locked)
+ spin_unlock(&inode_hash_lock);
__wait_on_freeing_inode(inode);
+ if (locked)
+ spin_lock(&inode_hash_lock);
goto repeat;
}
__iget(inode);
spin_unlock(&inode->i_lock);
+ rcu_read_unlock();
return inode;
}
+ rcu_read_unlock();
return NULL;
}
@@ -984,9 +1009,7 @@ struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
struct hlist_head *head = inode_hashtable + hash(sb, hashval);
struct inode *inode;
- spin_lock(&inode_hash_lock);
- inode = find_inode(sb, head, test, data);
- spin_unlock(&inode_hash_lock);
+ inode = find_inode(sb, head, test, data, false);
if (inode) {
wait_on_inode(inode);
@@ -998,8 +1021,7 @@ struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
struct inode *old;
spin_lock(&inode_hash_lock);
- /* We released the lock, so.. */
- old = find_inode(sb, head, test, data);
+ old = find_inode(sb, head, test, data, true);
if (!old) {
if (set(inode, data))
goto set_failed;
@@ -1054,9 +1076,7 @@ struct inode *iget_locked(struct super_block *sb, unsigned long ino)
struct hlist_head *head = inode_hashtable + hash(sb, ino);
struct inode *inode;
- spin_lock(&inode_hash_lock);
- inode = find_inode_fast(sb, head, ino);
- spin_unlock(&inode_hash_lock);
+ inode = find_inode_fast(sb, head, ino, false);
if (inode) {
wait_on_inode(inode);
return inode;
@@ -1067,8 +1087,7 @@ struct inode *iget_locked(struct super_block *sb, unsigned long ino)
struct inode *old;
spin_lock(&inode_hash_lock);
- /* We released the lock, so.. */
- old = find_inode_fast(sb, head, ino);
+ old = find_inode_fast(sb, head, ino, true);
if (!old) {
inode->i_ino = ino;
spin_lock(&inode->i_lock);
@@ -1110,14 +1129,15 @@ static int test_inode_iunique(struct super_block *sb, unsigned long ino)
struct hlist_head *b = inode_hashtable + hash(sb, ino);
struct inode *inode;
- spin_lock(&inode_hash_lock);
- hlist_for_each_entry(inode, b, i_hash) {
- if (inode->i_ino == ino && inode->i_sb == sb) {
- spin_unlock(&inode_hash_lock);
+ rcu_read_lock();
+ hlist_for_each_entry_rcu(inode, b, i_hash) {
+ if (inode->i_ino == ino && inode->i_sb == sb &&
+ !inode_unhashed(inode)) {
+ rcu_read_unlock();
return 0;
}
}
- spin_unlock(&inode_hash_lock);
+ rcu_read_unlock();
return 1;
}
@@ -1198,13 +1218,8 @@ struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
int (*test)(struct inode *, void *), void *data)
{
struct hlist_head *head = inode_hashtable + hash(sb, hashval);
- struct inode *inode;
-
- spin_lock(&inode_hash_lock);
- inode = find_inode(sb, head, test, data);
- spin_unlock(&inode_hash_lock);
- return inode;
+ return find_inode(sb, head, test, data, false);
}
EXPORT_SYMBOL(ilookup5_nowait);
@@ -1249,10 +1264,7 @@ struct inode *ilookup(struct super_block *sb, unsigned long ino)
struct hlist_head *head = inode_hashtable + hash(sb, ino);
struct inode *inode;
- spin_lock(&inode_hash_lock);
- inode = find_inode_fast(sb, head, ino);
- spin_unlock(&inode_hash_lock);
-
+ inode = find_inode_fast(sb, head, ino, false);
if (inode)
wait_on_inode(inode);
return inode;
@@ -1696,10 +1708,8 @@ static void __wait_on_freeing_inode(struct inode *inode)
wq = bit_waitqueue(&inode->i_state, __I_NEW);
prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
spin_unlock(&inode->i_lock);
- spin_unlock(&inode_hash_lock);
schedule();
finish_wait(wq, &wait.wait);
- spin_lock(&inode_hash_lock);
}
static __initdata unsigned long ihash_entries;
--
1.8.3.2
next prev parent reply other threads:[~2013-07-31 4:16 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2013-07-31 4:15 [PATCH 00/11] Sync and VFS scalability improvements Dave Chinner
2013-07-31 4:15 ` [PATCH 01/11] writeback: plug writeback at a high level Dave Chinner
2013-07-31 14:40 ` Jan Kara
2013-08-01 5:48 ` Dave Chinner
2013-08-01 8:34 ` Jan Kara
2013-07-31 4:15 ` [PATCH 02/11] inode: add IOP_NOTHASHED to avoid inode hash lock in evict Dave Chinner
2013-07-31 14:44 ` Jan Kara
2013-08-01 8:12 ` Christoph Hellwig
2013-08-02 1:11 ` Dave Chinner
2013-08-02 14:32 ` Christoph Hellwig
2013-07-31 4:15 ` [PATCH 03/11] inode: convert inode_sb_list_lock to per-sb Dave Chinner
2013-07-31 14:48 ` Jan Kara
2013-07-31 4:15 ` [PATCH 04/11] sync: serialise per-superblock sync operations Dave Chinner
2013-07-31 15:12 ` Jan Kara
2013-07-31 4:15 ` [PATCH 05/11] inode: rename i_wb_list to i_io_list Dave Chinner
2013-07-31 14:51 ` Jan Kara
2013-07-31 4:15 ` [PATCH 06/11] bdi: add a new writeback list for sync Dave Chinner
2013-07-31 15:11 ` Jan Kara
2013-08-01 5:59 ` Dave Chinner
2013-07-31 4:15 ` [PATCH 07/11] writeback: periodically trim the writeback list Dave Chinner
2013-07-31 15:15 ` Jan Kara
2013-08-01 6:16 ` Dave Chinner
2013-08-01 9:03 ` Jan Kara
2013-07-31 4:15 ` [PATCH 08/11] inode: convert per-sb inode list to a list_lru Dave Chinner
2013-08-01 8:19 ` Christoph Hellwig
2013-08-02 1:06 ` Dave Chinner
2013-07-31 4:15 ` Dave Chinner [this message]
2013-07-31 4:15 ` [PATCH 10/11] list_lru: don't need node lock in list_lru_count_node Dave Chinner
2013-07-31 4:15 ` [PATCH 11/11] list_lru: don't lock during add/del if unnecessary Dave Chinner
2013-07-31 6:48 ` [PATCH 00/11] Sync and VFS scalability improvements Sedat Dilek
2013-08-01 6:19 ` Dave Chinner
2013-08-01 6:31 ` Sedat Dilek
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1375244150-27296-10-git-send-email-david@fromorbit.com \
--to=david@fromorbit.com \
--cc=akpm@linux-foundation.org \
--cc=davej@redhat.com \
--cc=glommer@parallels.com \
--cc=jack@suse.cz \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=viro@zeniv.linux.org.uk \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).