From: npiggin@suse.de
To: linux-fsdevel@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [patch 19/27] fs: dcache per-inode inode alias locking
Date: Sat, 25 Apr 2009 11:20:39 +1000 [thread overview]
Message-ID: <20090425012211.798273449@suse.de> (raw)
In-Reply-To: 20090425012020.457460929@suse.de
[-- Attachment #1: dcache-split-inode_lock.patch --]
[-- Type: text/plain, Size: 15691 bytes --]
dcache_inode_lock can be replaced with per-inode locking. Use existing
inode->i_lock for this. This is slightly non-trivial because we sometimes
need to find the inode from the dentry, which requires d_inode to be
stabilised (either with refcount or d_lock).
---
fs/affs/amigaffs.c | 4 -
fs/dcache.c | 118 +++++++++++++++++++++++++-------------------
fs/exportfs/expfs.c | 12 ++--
fs/nfs/getroot.c | 4 -
fs/notify/inotify/inotify.c | 4 -
fs/ocfs2/dcache.c | 4 -
fs/sysfs/dir.c | 6 +-
include/linux/dcache.h | 1
8 files changed, 87 insertions(+), 66 deletions(-)
Index: linux-2.6/fs/dcache.c
===================================================================
--- linux-2.6.orig/fs/dcache.c
+++ linux-2.6/fs/dcache.c
@@ -36,7 +36,7 @@
/*
* Usage:
- * dcache_inode_lock protects:
+ * dcache->d_inode->i_lock protects:
* - the inode alias lists, d_inode
* dcache_hash_bucket->lock protects:
* - the dcache hash table
@@ -50,7 +50,7 @@
* - d_subdirs and children's d_child
*
* Ordering:
- * dcache_inode_lock
+ * dcache->d_inode->i_lock
* dentry->d_lock
* dcache_lru_lock
* dcache_hash_bucket->lock
@@ -58,12 +58,9 @@
int sysctl_vfs_cache_pressure __read_mostly = 100;
EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
-__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_inode_lock);
__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
-EXPORT_SYMBOL(dcache_inode_lock);
-
static struct kmem_cache *dentry_cache __read_mostly;
#define DNAME_INLINE_LEN (sizeof(struct dentry)-offsetof(struct dentry,d_iname))
@@ -138,14 +135,13 @@ static void d_free(struct dentry *dentry
*/
static void dentry_iput(struct dentry * dentry)
__releases(dentry->d_lock)
- __releases(dcache_inode_lock)
{
struct inode *inode = dentry->d_inode;
if (inode) {
dentry->d_inode = NULL;
list_del_init(&dentry->d_alias);
spin_unlock(&dentry->d_lock);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
if (!inode->i_nlink)
fsnotify_inoderemove(inode);
if (dentry->d_op && dentry->d_op->d_iput)
@@ -154,7 +150,6 @@ static void dentry_iput(struct dentry *
iput(inode);
} else {
spin_unlock(&dentry->d_lock);
- spin_unlock(&dcache_inode_lock);
}
}
@@ -225,7 +220,6 @@ static void dentry_lru_del_init(struct d
*/
static struct dentry *d_kill(struct dentry *dentry)
__releases(dentry->d_lock)
- __releases(dcache_inode_lock)
{
struct dentry *parent;
@@ -341,6 +335,7 @@ EXPORT_SYMBOL(dget_parent);
void dput(struct dentry *dentry)
{
struct dentry *parent;
+ struct inode *inode;
if (!dentry)
return;
@@ -376,17 +371,24 @@ repeat:
spin_unlock(&dentry->d_lock);
return;
-kill_it:
- spin_unlock(&dentry->d_lock);
- spin_lock(&dcache_inode_lock);
-relock:
+relock1:
spin_lock(&dentry->d_lock);
+kill_it:
+ inode = dentry->d_inode;
+ if (inode) {
+ if (!spin_trylock(&inode->i_lock)) {
+relock2:
+ spin_unlock(&dentry->d_lock);
+ goto relock1;
+ }
+ }
parent = dentry->d_parent;
if (parent) {
BUG_ON(parent == dentry);
if (!spin_trylock(&parent->d_lock)) {
- spin_unlock(&dentry->d_lock);
- goto relock;
+ if (inode)
+ spin_unlock(&inode->i_lock);
+ goto relock2;
}
}
dentry->d_count--;
@@ -394,7 +396,8 @@ relock:
/* This case should be fine */
spin_unlock(&dentry->d_lock);
spin_unlock(&parent->d_lock);
- spin_unlock(&dcache_inode_lock);
+ if (inode)
+ spin_unlock(&inode->i_lock);
return;
}
/* if dentry was on the d_lru list delete it from there */
@@ -510,9 +513,9 @@ struct dentry * d_find_alias(struct inod
struct dentry *de = NULL;
if (!list_empty(&inode->i_dentry)) {
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
de = __d_find_alias(inode, 0);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
}
return de;
}
@@ -525,20 +528,20 @@ void d_prune_aliases(struct inode *inode
{
struct dentry *dentry;
restart:
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
spin_lock(&dentry->d_lock);
if (!dentry->d_count) {
__dget_locked_dlock(dentry);
__d_drop(dentry);
spin_unlock(&dentry->d_lock);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
dput(dentry);
goto restart;
}
spin_unlock(&dentry->d_lock);
}
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
}
/*
@@ -560,8 +563,10 @@ static void prune_one_dentry(struct dent
*/
while (dentry) {
struct dentry *parent = NULL;
+ struct inode *inode = dentry->d_inode;
- spin_lock(&dcache_inode_lock);
+ if (inode)
+ spin_lock(&inode->i_lock);
again:
spin_lock(&dentry->d_lock);
if (dentry->d_parent && dentry != dentry->d_parent) {
@@ -576,7 +581,8 @@ again:
if (parent)
spin_unlock(&parent->d_lock);
spin_unlock(&dentry->d_lock);
- spin_unlock(&dcache_inode_lock);
+ if (inode)
+ spin_unlock(&inode->i_lock);
return;
}
@@ -645,10 +651,11 @@ restart:
}
spin_unlock(&dcache_lru_lock);
- spin_lock(&dcache_inode_lock);
again:
spin_lock(&dcache_lru_lock); /* lru_lock also protects tmp list */
while (!list_empty(&tmp)) {
+ struct inode *inode;
+
dentry = list_entry(tmp.prev, struct dentry, d_lru);
if (!spin_trylock(&dentry->d_lock)) {
@@ -666,11 +673,18 @@ again1:
spin_unlock(&dentry->d_lock);
continue;
}
+ inode = dentry->d_inode;
+ if (inode && !spin_trylock(&inode->i_lock)) {
+again2:
+ spin_unlock(&dentry->d_lock);
+ goto again1;
+ }
if (dentry->d_parent) {
BUG_ON(dentry == dentry->d_parent);
if (!spin_trylock(&dentry->d_parent->d_lock)) {
- spin_unlock(&dentry->d_lock);
- goto again1;
+ if (inode)
+ spin_unlock(&inode->i_lock);
+ goto again2;
}
}
__dentry_lru_del_init(dentry);
@@ -678,10 +692,8 @@ again1:
prune_one_dentry(dentry);
/* dentry->d_lock dropped */
- spin_lock(&dcache_inode_lock);
spin_lock(&dcache_lru_lock);
}
- spin_unlock(&dcache_inode_lock);
if (count == NULL && !list_empty(&sb->s_dentry_lru))
goto restart;
@@ -1244,9 +1256,11 @@ static void __d_instantiate(struct dentr
void d_instantiate(struct dentry *entry, struct inode * inode)
{
BUG_ON(!list_empty(&entry->d_alias));
- spin_lock(&dcache_inode_lock);
+ if (inode)
+ spin_lock(&inode->i_lock);
__d_instantiate(entry, inode);
- spin_unlock(&dcache_inode_lock);
+ if (inode)
+ spin_unlock(&inode->i_lock);
security_d_instantiate(entry, inode);
}
@@ -1304,9 +1318,11 @@ struct dentry *d_instantiate_unique(stru
BUG_ON(!list_empty(&entry->d_alias));
- spin_lock(&dcache_inode_lock);
+ if (inode)
+ spin_lock(&inode->i_lock);
result = __d_instantiate_unique(entry, inode);
- spin_unlock(&dcache_inode_lock);
+ if (inode)
+ spin_unlock(&inode->i_lock);
if (!result) {
security_d_instantiate(entry, inode);
@@ -1386,10 +1402,10 @@ struct dentry *d_obtain_alias(struct ino
}
tmp->d_parent = tmp; /* make sure dput doesn't croak */
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
res = __d_find_alias(inode, 0);
if (res) {
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
dput(tmp);
goto out_iput;
}
@@ -1403,7 +1419,7 @@ struct dentry *d_obtain_alias(struct ino
list_add(&tmp->d_alias, &inode->i_dentry);
hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon);
spin_unlock(&tmp->d_lock);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
return tmp;
@@ -1434,19 +1450,19 @@ struct dentry *d_splice_alias(struct ino
struct dentry *new = NULL;
if (inode && S_ISDIR(inode->i_mode)) {
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
new = __d_find_alias(inode, 1);
if (new) {
BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
security_d_instantiate(new, inode);
d_rehash(dentry);
d_move(new, dentry);
iput(inode);
} else {
- /* already taken dcache_inode_lock, d_add() by hand */
+ /* already taken inode->i_lock, d_add() by hand */
__d_instantiate(dentry, inode);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
security_d_instantiate(dentry, inode);
d_rehash(dentry);
}
@@ -1518,10 +1534,10 @@ struct dentry *d_add_ci(struct dentry *d
* Negative dentry: instantiate it unless the inode is a directory and
* already has a dentry.
*/
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
if (!S_ISDIR(inode->i_mode) || list_empty(&inode->i_dentry)) {
__d_instantiate(found, inode);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
security_d_instantiate(found, inode);
return found;
}
@@ -1532,7 +1548,7 @@ struct dentry *d_add_ci(struct dentry *d
*/
new = list_entry(inode->i_dentry.next, struct dentry, d_alias);
dget_locked(new);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
security_d_instantiate(found, inode);
d_move(new, found);
iput(inode);
@@ -1742,15 +1758,17 @@ out:
void d_delete(struct dentry * dentry)
{
+ struct inode *inode;
int isdir = 0;
/*
* Are we the only user?
*/
again:
spin_lock(&dentry->d_lock);
- isdir = S_ISDIR(dentry->d_inode->i_mode);
+ inode = dentry->d_inode;
+ isdir = S_ISDIR(inode->i_mode);
if (dentry->d_count == 1) {
- if (!spin_trylock(&dcache_inode_lock)) {
+ if (inode && !spin_trylock(&inode->i_lock)) {
spin_unlock(&dentry->d_lock);
goto again;
}
@@ -1983,6 +2001,7 @@ static struct dentry *__d_unalias(struct
{
struct mutex *m1 = NULL, *m2 = NULL;
struct dentry *ret;
+ struct inode *inode;
/* If alias and dentry share a parent, then no extra locks required */
if (alias->d_parent == dentry->d_parent)
@@ -1998,14 +2017,15 @@ static struct dentry *__d_unalias(struct
if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
goto out_err;
m1 = &dentry->d_sb->s_vfs_rename_mutex;
- if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex))
+ inode = alias->d_parent->d_inode;
+ if (!mutex_trylock(&inode->i_mutex))
goto out_err;
- m2 = &alias->d_parent->d_inode->i_mutex;
+ m2 = &inode->i_mutex;
out_unalias:
d_move_locked(alias, dentry);
ret = alias;
out_err:
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
if (m2)
mutex_unlock(m2);
if (m1)
@@ -2076,7 +2096,7 @@ struct dentry *d_materialise_unique(stru
goto out_nolock;
}
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
if (S_ISDIR(inode->i_mode)) {
struct dentry *alias;
@@ -2112,7 +2132,7 @@ struct dentry *d_materialise_unique(stru
found:
_d_rehash(actual);
spin_unlock(&actual->d_lock);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
out_nolock:
if (actual == dentry) {
security_d_instantiate(dentry, inode);
Index: linux-2.6/fs/sysfs/dir.c
===================================================================
--- linux-2.6.orig/fs/sysfs/dir.c
+++ linux-2.6/fs/sysfs/dir.c
@@ -547,7 +547,7 @@ static void sysfs_drop_dentry(struct sys
* dput to immediately free the dentry if it is not in use.
*/
repeat:
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
spin_lock(&dentry->d_lock);
if (d_unhashed(dentry)) {
@@ -557,11 +557,11 @@ repeat:
dget_locked_dlock(dentry);
__d_drop(dentry);
spin_unlock(&dentry->d_lock);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
dput(dentry);
goto repeat;
}
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
/* adjust nlink and update timestamp */
mutex_lock(&inode->i_mutex);
Index: linux-2.6/include/linux/dcache.h
===================================================================
--- linux-2.6.orig/include/linux/dcache.h
+++ linux-2.6/include/linux/dcache.h
@@ -184,7 +184,6 @@ d_iput: no no yes
#define DCACHE_COOKIE 0x0040 /* For use by dcookie subsystem */
-extern spinlock_t dcache_inode_lock;
extern seqlock_t rename_lock;
/**
Index: linux-2.6/fs/notify/inotify/inotify.c
===================================================================
--- linux-2.6.orig/fs/notify/inotify/inotify.c
+++ linux-2.6/fs/notify/inotify/inotify.c
@@ -184,7 +184,7 @@ static void set_dentry_child_flags(struc
{
struct dentry *alias;
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
list_for_each_entry(alias, &inode->i_dentry, d_alias) {
struct dentry *child;
@@ -202,7 +202,7 @@ static void set_dentry_child_flags(struc
}
spin_unlock(&alias->d_lock);
}
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
}
/*
Index: linux-2.6/fs/exportfs/expfs.c
===================================================================
--- linux-2.6.orig/fs/exportfs/expfs.c
+++ linux-2.6/fs/exportfs/expfs.c
@@ -43,24 +43,26 @@ find_acceptable_alias(struct dentry *res
void *context)
{
struct dentry *dentry, *toput = NULL;
+ struct inode *inode;
if (acceptable(context, result))
return result;
- spin_lock(&dcache_inode_lock);
- list_for_each_entry(dentry, &result->d_inode->i_dentry, d_alias) {
+ inode = result->d_inode;
+ spin_lock(&inode->i_lock);
+ list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
dget_locked(dentry);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
if (toput)
dput(toput);
if (dentry != result && acceptable(context, dentry)) {
dput(result);
return dentry;
}
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
toput = dentry;
}
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
if (toput)
dput(toput);
Index: linux-2.6/fs/affs/amigaffs.c
===================================================================
--- linux-2.6.orig/fs/affs/amigaffs.c
+++ linux-2.6/fs/affs/amigaffs.c
@@ -128,7 +128,7 @@ affs_fix_dcache(struct dentry *dentry, u
void *data = dentry->d_fsdata;
struct list_head *head, *next;
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
head = &inode->i_dentry;
next = head->next;
while (next != head) {
@@ -139,7 +139,7 @@ affs_fix_dcache(struct dentry *dentry, u
}
next = next->next;
}
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
}
Index: linux-2.6/fs/nfs/getroot.c
===================================================================
--- linux-2.6.orig/fs/nfs/getroot.c
+++ linux-2.6/fs/nfs/getroot.c
@@ -65,11 +65,11 @@ static int nfs_superblock_set_dummy_root
* This again causes shrink_dcache_for_umount_subtree() to
* Oops, since the test for IS_ROOT() will fail.
*/
- spin_lock(&dcache_inode_lock);
+ spin_lock(&sb->s_root->d_inode->i_lock);
spin_lock(&sb->s_root->d_lock);
list_del_init(&sb->s_root->d_alias);
spin_unlock(&sb->s_root->d_lock);
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&sb->s_root->d_inode->i_lock);
}
return 0;
}
Index: linux-2.6/fs/ocfs2/dcache.c
===================================================================
--- linux-2.6.orig/fs/ocfs2/dcache.c
+++ linux-2.6/fs/ocfs2/dcache.c
@@ -140,7 +140,7 @@ struct dentry *ocfs2_find_local_alias(st
struct list_head *p;
struct dentry *dentry = NULL;
- spin_lock(&dcache_inode_lock);
+ spin_lock(&inode->i_lock);
list_for_each(p, &inode->i_dentry) {
dentry = list_entry(p, struct dentry, d_alias);
@@ -158,7 +158,7 @@ struct dentry *ocfs2_find_local_alias(st
dentry = NULL;
}
- spin_unlock(&dcache_inode_lock);
+ spin_unlock(&inode->i_lock);
return dentry;
}
next prev parent reply other threads:[~2009-04-25 1:34 UTC|newest]
Thread overview: 50+ messages / expand[flat|nested] mbox.gz Atom feed top
2009-04-25 1:20 [patch 00/27] [rfc] vfs scalability patchset npiggin
2009-04-25 1:20 ` [patch 01/27] fs: cleanup files_lock npiggin
2009-04-25 3:20 ` Al Viro
2009-04-25 5:35 ` Eric W. Biederman
2009-04-26 6:12 ` Nick Piggin
2009-04-25 9:42 ` Alan Cox
2009-04-26 6:15 ` Nick Piggin
2009-04-25 1:20 ` [patch 02/27] fs: scale files_lock npiggin
2009-04-25 3:32 ` Al Viro
2009-04-25 1:20 ` [patch 03/27] fs: mnt_want_write speedup npiggin
2009-04-25 1:20 ` [patch 04/27] fs: introduce mnt_clone_write npiggin
2009-04-25 3:35 ` Al Viro
2009-04-25 1:20 ` [patch 05/27] fs: brlock vfsmount_lock npiggin
2009-04-25 3:50 ` Al Viro
2009-04-26 6:36 ` Nick Piggin
2009-04-25 1:20 ` [patch 06/27] fs: dcache fix LRU ordering npiggin
2009-04-25 1:20 ` [patch 07/27] fs: dcache scale hash npiggin
2009-04-25 1:20 ` [patch 08/27] fs: dcache scale lru npiggin
2009-04-25 1:20 ` [patch 09/27] fs: dcache scale nr_dentry npiggin
2009-04-25 1:20 ` [patch 10/27] fs: dcache scale dentry refcount npiggin
2009-04-25 1:20 ` [patch 11/27] fs: dcache scale d_unhashed npiggin
2009-04-25 1:20 ` [patch 12/27] fs: dcache scale subdirs npiggin
2009-04-25 1:20 ` [patch 13/27] fs: scale inode alias list npiggin
2009-04-25 1:20 ` [patch 14/27] fs: use RCU / seqlock logic for reverse and multi-step operaitons npiggin
2009-04-25 1:20 ` [patch 15/27] fs: dcache remove dcache_lock npiggin
2009-04-25 1:20 ` [patch 16/27] fs: dcache reduce dput locking npiggin
2009-04-25 1:20 ` [patch 17/27] fs: dcache per-bucket dcache hash locking npiggin
2009-04-25 1:20 ` [patch 18/27] fs: dcache reduce dcache_inode_lock npiggin
2009-04-25 1:20 ` npiggin [this message]
2009-04-25 1:20 ` [patch 20/27] fs: icache lock s_inodes list npiggin
2009-04-25 1:20 ` [patch 21/27] fs: icache lock inode hash npiggin
2009-04-25 1:20 ` [patch 22/27] fs: icache lock i_state npiggin
2009-04-25 1:20 ` [patch 23/27] fs: icache lock i_count npiggin
2009-04-25 1:20 ` [patch 24/27] fs: icache atomic inodes_stat npiggin
2009-04-25 1:20 ` [patch 25/27] fs: icache lock lru/writeback lists npiggin
2009-04-25 1:20 ` [patch 26/27] fs: icache protect inode state npiggin
2009-04-25 1:20 ` [patch 27/27] fs: icache remove inode_lock npiggin
2009-04-25 4:18 ` [patch 00/27] [rfc] vfs scalability patchset Al Viro
2009-04-25 5:02 ` Nick Piggin
2009-04-25 8:01 ` Christoph Hellwig
2009-04-25 8:06 ` Al Viro
2009-04-28 9:09 ` Christoph Hellwig
2009-04-28 9:48 ` Nick Piggin
2009-04-28 10:58 ` Peter Zijlstra
2009-04-28 11:32 ` Eric W. Biederman
2009-04-30 6:14 ` Nick Piggin
2009-04-25 19:08 ` Eric W. Biederman
2009-04-25 19:31 ` Al Viro
2009-04-25 20:29 ` Eric W. Biederman
2009-04-25 22:05 ` Theodore Tso
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20090425012211.798273449@suse.de \
--to=npiggin@suse.de \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.