From: npiggin@kernel.dk
To: linux-fsdevel@vger.kernel.org, linux-kernel@vger.kernel.org,
npiggin@kernel.dk
Subject: [patch 06/14] fs: icache lock inode hash
Date: Fri, 22 Oct 2010 00:08:35 +1100 [thread overview]
Message-ID: <20101021131016.533198770@kernel.dk> (raw)
In-Reply-To: 20101021130829.442910807@kernel.dk
[-- Attachment #1: fs-inode_lock-scale-2.patch --]
[-- Type: text/plain, Size: 9660 bytes --]
Add a new lock, inode_hash_lock, to protect the inode hash table lists.
[note: inode_lock can't be lifted much further here, because hash lookups
tend to involve insertions etc onto other data structures]
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
---
fs/inode.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 75 insertions(+), 13 deletions(-)
Index: linux-2.6/fs/inode.c
===================================================================
--- linux-2.6.orig/fs/inode.c 2010-10-21 23:50:27.000000000 +1100
+++ linux-2.6/fs/inode.c 2010-10-21 23:50:44.000000000 +1100
@@ -39,11 +39,14 @@
* i_sb_list
* sb_inode_list_lock protects:
* s_inodes, i_sb_list
+ * inode_hash_lock protects:
+ * inode hash table, i_hash
*
* Ordering:
* inode_lock
* i_lock
* sb_inode_list_lock
+ * inode_hash_lock
*/
/*
* This is needed for the following functions:
@@ -104,6 +107,7 @@ static struct hlist_head *inode_hashtabl
*/
DEFINE_SPINLOCK(inode_lock);
DEFINE_SPINLOCK(sb_inode_list_lock);
+static DEFINE_SPINLOCK(inode_hash_lock);
/*
* iprune_sem provides exclusion between the kswapd or try_to_free_pages
@@ -390,7 +394,9 @@ static void dispose_list(struct list_hea
spin_lock(&inode_lock);
spin_lock(&inode->i_lock);
+ spin_lock(&inode_hash_lock);
hlist_del_init(&inode->i_hash);
+ spin_unlock(&inode_hash_lock);
spin_lock(&sb_inode_list_lock);
list_del_init(&inode->i_sb_list);
spin_unlock(&sb_inode_list_lock);
@@ -615,7 +621,12 @@ static struct inode *find_inode(struct s
continue;
if (!test(inode, data))
continue;
- spin_lock(&inode->i_lock);
+ if (!spin_trylock(&inode->i_lock)) {
+ spin_unlock(&inode_hash_lock);
+ cpu_relax();
+ spin_lock(&inode_hash_lock);
+ goto repeat;
+ }
if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
__wait_on_freeing_inode(inode);
goto repeat;
@@ -641,7 +652,12 @@ static struct inode *find_inode_fast(str
continue;
if (inode->i_sb != sb)
continue;
- spin_lock(&inode->i_lock);
+ if (!spin_trylock(&inode->i_lock)) {
+ spin_unlock(&inode_hash_lock);
+ cpu_relax();
+ spin_lock(&inode_hash_lock);
+ goto repeat;
+ }
if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
__wait_on_freeing_inode(inode);
goto repeat;
@@ -670,8 +686,11 @@ __inode_add_to_lists(struct super_block
spin_lock(&sb_inode_list_lock);
list_add(&inode->i_sb_list, &sb->s_inodes);
spin_unlock(&sb_inode_list_lock);
- if (head)
+ if (head) {
+ spin_lock(&inode_hash_lock);
hlist_add_head(&inode->i_hash, head);
+ spin_unlock(&inode_hash_lock);
+ }
}
/**
@@ -790,15 +809,18 @@ static struct inode *get_new_inode(struc
struct inode *old;
spin_lock(&inode_lock);
+ spin_lock(&inode_hash_lock);
/* We released the lock, so.. */
old = find_inode(sb, head, test, data);
if (!old) {
if (set(inode, data))
goto set_failed;
- spin_lock(&inode->i_lock);
+ BUG_ON(!spin_trylock(&inode->i_lock));
inode->i_state = I_NEW;
- __inode_add_to_lists(sb, head, inode);
+ hlist_add_head(&inode->i_hash, head);
+ spin_unlock(&inode_hash_lock);
+ __inode_add_to_lists(sb, NULL, inode);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
@@ -814,6 +836,7 @@ static struct inode *get_new_inode(struc
* allocated.
*/
inode_get_ilock(old);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&old->i_lock);
spin_unlock(&inode_lock);
destroy_inode(inode);
@@ -823,6 +846,7 @@ static struct inode *get_new_inode(struc
return inode;
set_failed:
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode_lock);
destroy_inode(inode);
return NULL;
@@ -842,13 +866,16 @@ static struct inode *get_new_inode_fast(
struct inode *old;
spin_lock(&inode_lock);
+ spin_lock(&inode_hash_lock);
/* We released the lock, so.. */
old = find_inode_fast(sb, head, ino);
if (!old) {
- spin_lock(&inode->i_lock);
+ BUG_ON(!spin_trylock(&inode->i_lock));
inode->i_ino = ino;
inode->i_state = I_NEW;
- __inode_add_to_lists(sb, head, inode);
+ hlist_add_head(&inode->i_hash, head);
+ spin_unlock(&inode_hash_lock);
+ __inode_add_to_lists(sb, NULL, inode);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
@@ -864,6 +891,7 @@ static struct inode *get_new_inode_fast(
* allocated.
*/
inode_get_ilock(old);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&old->i_lock);
spin_unlock(&inode_lock);
destroy_inode(inode);
@@ -900,15 +928,17 @@ ino_t iunique(struct super_block *sb, in
ino_t res;
spin_lock(&inode_lock);
+ spin_lock(&inode_hash_lock);
do {
if (counter <= max_reserved)
counter = max_reserved + 1;
res = counter++;
head = inode_hashtable + hash(sb, res);
inode = find_inode_fast(sb, head, res);
- if (inode)
- spin_unlock(&inode->i_lock);
} while (inode != NULL);
+ spin_unlock(&inode_hash_lock);
+ if (inode)
+ spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
return res;
@@ -963,15 +993,18 @@ static struct inode *ifind(struct super_
struct inode *inode;
spin_lock(&inode_lock);
+ spin_lock(&inode_hash_lock);
inode = find_inode(sb, head, test, data);
if (inode) {
inode_get_ilock(inode);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
if (likely(wait))
wait_on_inode(inode);
return inode;
}
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode_lock);
return NULL;
}
@@ -997,14 +1030,17 @@ static struct inode *ifind_fast(struct s
struct inode *inode;
spin_lock(&inode_lock);
+ spin_lock(&inode_hash_lock);
inode = find_inode_fast(sb, head, ino);
if (inode) {
inode_get_ilock(inode);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
wait_on_inode(inode);
return inode;
}
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode_lock);
return NULL;
}
@@ -1169,26 +1205,34 @@ int insert_inode_locked(struct inode *in
struct hlist_node *node;
struct inode *old = NULL;
spin_lock(&inode_lock);
+lock_again:
+ spin_lock(&inode_hash_lock);
hlist_for_each_entry(old, node, head, i_hash) {
if (old->i_ino != ino)
continue;
if (old->i_sb != sb)
continue;
- spin_lock(&old->i_lock);
+ if (!spin_trylock(&old->i_lock)) {
+ spin_unlock(&inode_hash_lock);
+ cpu_relax();
+ goto lock_again;
+ }
if (old->i_state & (I_FREEING|I_WILL_FREE)) {
spin_unlock(&old->i_lock);
continue;
}
goto found_old;
}
- spin_lock(&inode->i_lock);
+ BUG_ON(!spin_trylock(&inode->i_lock)); /* XXX: init locked */
hlist_add_head(&inode->i_hash, head);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
return 0;
found_old:
inode_get_ilock(old);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&old->i_lock);
spin_unlock(&inode_lock);
wait_on_inode(old);
@@ -1214,26 +1258,34 @@ int insert_inode_locked4(struct inode *i
struct inode *old = NULL;
spin_lock(&inode_lock);
+lock_again:
+ spin_lock(&inode_hash_lock);
hlist_for_each_entry(old, node, head, i_hash) {
if (old->i_sb != sb)
continue;
if (!test(old, data))
continue;
- spin_lock(&old->i_lock);
+ if (!spin_trylock(&old->i_lock)) {
+ spin_unlock(&inode_hash_lock);
+ cpu_relax();
+ goto lock_again;
+ }
if (old->i_state & (I_FREEING|I_WILL_FREE)) {
spin_unlock(&old->i_lock);
continue;
}
goto found_old;
}
- spin_lock(&inode->i_lock);
+ BUG_ON(!spin_trylock(&inode->i_lock)); /* XXX: init locked */
hlist_add_head(&inode->i_hash, head);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
return 0;
found_old:
inode_get_ilock(old);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&old->i_lock);
spin_unlock(&inode_lock);
wait_on_inode(old);
@@ -1259,7 +1311,9 @@ void __insert_inode_hash(struct inode *i
struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
spin_lock(&inode_lock);
spin_lock(&inode->i_lock);
+ spin_lock(&inode_hash_lock);
hlist_add_head(&inode->i_hash, head);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
}
@@ -1275,7 +1329,9 @@ void remove_inode_hash(struct inode *ino
{
spin_lock(&inode_lock);
spin_lock(&inode->i_lock);
+ spin_lock(&inode_hash_lock);
hlist_del_init(&inode->i_hash);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
}
@@ -1338,7 +1394,9 @@ static void iput_final(struct inode *ino
WARN_ON(inode->i_state & I_NEW);
inode->i_state &= ~I_WILL_FREE;
inodes_stat.nr_unused--;
+ spin_lock(&inode_hash_lock);
hlist_del_init(&inode->i_hash);
+ spin_unlock(&inode_hash_lock);
}
list_del_init(&inode->i_list);
spin_lock(&sb_inode_list_lock);
@@ -1352,7 +1410,9 @@ static void iput_final(struct inode *ino
evict(inode);
spin_lock(&inode_lock);
spin_lock(&inode->i_lock);
+ spin_lock(&inode_hash_lock);
hlist_del_init(&inode->i_hash);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
wake_up_inode(inode);
@@ -1573,11 +1633,13 @@ static void __wait_on_freeing_inode(stru
DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
wq = bit_waitqueue(&inode->i_state, __I_NEW);
prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
+ spin_unlock(&inode_hash_lock);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_lock);
schedule();
finish_wait(wq, &wait.wait);
spin_lock(&inode_lock);
+ spin_lock(&inode_hash_lock);
}
static __initdata unsigned long ihash_entries;
next prev parent reply other threads:[~2010-10-21 13:25 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-10-21 13:08 [patch 00/14] reworked minimal inode_lock breaking series npiggin
2010-10-21 13:08 ` [patch 01/14] fs: icache begin inode_lock lock breaking npiggin
2010-10-21 13:08 ` [patch 02/14] fs: icache lock i_count npiggin
2010-10-21 13:08 ` [patch 03/14] fs: icache lock inodes icache state npiggin
2010-10-21 13:08 ` [patch 04/14] fs: icache unmount code cleanup npiggin
2010-10-21 13:08 ` [patch 05/14] fs: icache lock s_inodes list npiggin
2010-10-21 13:08 ` npiggin [this message]
2010-10-21 13:08 ` [patch 07/14] fs: icache lock lru/writeback lists npiggin
2010-10-21 13:08 ` [patch 08/14] fs: icache make nr_inodes and nr_unused atomic npiggin
2010-10-21 13:08 ` [patch 09/14] fs: inode atomic last_ino, iunique lock npiggin
2010-10-21 13:08 ` [patch 10/14] fs: icache remove inode_lock npiggin
2010-10-21 13:08 ` [patch 11/14] fs: icache factor hash lock into functions npiggin
2010-10-21 13:08 ` [patch 12/14] fs: icache lazy inode lru npiggin
2010-10-21 13:08 ` [patch 13/14] fs: icache split IO and LRU lists npiggin
2010-10-21 15:28 ` Christoph Lameter
2010-10-22 0:00 ` Nick Piggin
2010-10-22 1:05 ` Nick Piggin
2010-10-21 13:08 ` [patch 14/14] fs: icache split writeback and lru locks npiggin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20101021131016.533198770@kernel.dk \
--to=npiggin@kernel.dk \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.