From: npiggin@suse.de
To: linux-fsdevel@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [patch 17/27] fs: dcache per-bucket dcache hash locking
Date: Sat, 25 Apr 2009 11:20:37 +1000 [thread overview]
Message-ID: <20090425012211.509344338@suse.de> (raw)
In-Reply-To: 20090425012020.457460929@suse.de
[-- Attachment #1: dcache-chain-hashlock.patch --]
[-- Type: text/plain, Size: 11164 bytes --]
We can turn the dcache hash locking from a global dcache_hash_lock into
per-bucket locking.
XXX: should probably use a bit lock in the first bit of the hash pointers
to avoid any space bloating (non-atomic unlock means no extra atomics either)
---
fs/dcache.c | 197 ++++++++++++++++++++++++++++---------------------
include/linux/dcache.h | 20 ----
2 files changed, 115 insertions(+), 102 deletions(-)
Index: linux-2.6/fs/dcache.c
===================================================================
--- linux-2.6.orig/fs/dcache.c
+++ linux-2.6/fs/dcache.c
@@ -38,7 +38,7 @@
* Usage:
* dcache_inode_lock protects:
* - the inode alias lists, d_inode
- * dcache_hash_lock protects:
+ * dcache_hash_bucket->lock protects:
* - the dcache hash table
* dcache_lru_lock protects:
* - the dcache lru lists and counters
@@ -53,18 +53,16 @@
* dcache_inode_lock
* dentry->d_lock
* dcache_lru_lock
- * dcache_hash_lock
+ * dcache_hash_bucket->lock
*/
int sysctl_vfs_cache_pressure __read_mostly = 100;
EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_inode_lock);
-__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);
__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
EXPORT_SYMBOL(dcache_inode_lock);
-EXPORT_SYMBOL(dcache_hash_lock);
static struct kmem_cache *dentry_cache __read_mostly;
@@ -83,7 +81,12 @@ static struct kmem_cache *dentry_cache _
static unsigned int d_hash_mask __read_mostly;
static unsigned int d_hash_shift __read_mostly;
-static struct hlist_head *dentry_hashtable __read_mostly;
+
+struct dcache_hash_bucket {
+ spinlock_t lock;
+ struct hlist_head head;
+};
+static struct dcache_hash_bucket *dentry_hashtable __read_mostly;
/* Statistics gathering. */
struct dentry_stat_t dentry_stat = {
@@ -91,6 +94,14 @@ struct dentry_stat_t dentry_stat = {
.age_limit = 45,
};
+static inline struct dcache_hash_bucket *d_hash(struct dentry *parent,
+ unsigned long hash)
+{
+ hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
+ hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
+ return dentry_hashtable + (hash & D_HASHMASK);
+}
+
static void __d_free(struct dentry *dentry)
{
WARN_ON(!list_empty(&dentry->d_alias));
@@ -231,6 +242,73 @@ static struct dentry *d_kill(struct dent
return parent;
}
+void __d_drop(struct dentry *dentry)
+{
+ if (!(dentry->d_flags & DCACHE_UNHASHED)) {
+ struct dcache_hash_bucket *b;
+ b = d_hash(dentry->d_parent, dentry->d_name.hash);
+ dentry->d_flags |= DCACHE_UNHASHED;
+ spin_lock(&b->lock);
+ hlist_del_rcu(&dentry->d_hash);
+ spin_unlock(&b->lock);
+ }
+}
+
+void d_drop(struct dentry *dentry)
+{
+ spin_lock(&dentry->d_lock);
+ __d_drop(dentry);
+ spin_unlock(&dentry->d_lock);
+}
+
+/* This should be called _only_ with a lock pinning the dentry */
+static inline struct dentry * __dget_locked_dlock(struct dentry *dentry)
+{
+ dentry->d_count++;
+ dentry_lru_del_init(dentry);
+ return dentry;
+}
+
+static inline struct dentry * __dget_locked(struct dentry *dentry)
+{
+ spin_lock(&dentry->d_lock);
+ __dget_locked_dlock(dentry);
+ spin_lock(&dentry->d_lock);
+ return dentry;
+}
+
+struct dentry * dget_locked_dlock(struct dentry *dentry)
+{
+ return __dget_locked_dlock(dentry);
+}
+
+struct dentry * dget_locked(struct dentry *dentry)
+{
+ return __dget_locked(dentry);
+}
+
+struct dentry *dget_parent(struct dentry *dentry)
+{
+ struct dentry *ret;
+
+repeat:
+ spin_lock(&dentry->d_lock);
+ ret = dentry->d_parent;
+ if (!ret)
+ goto out;
+ if (!spin_trylock(&ret->d_lock)) {
+ spin_unlock(&dentry->d_lock);
+ goto repeat;
+ }
+ BUG_ON(!ret->d_count);
+ ret->d_count++;
+ spin_unlock(&ret->d_lock);
+out:
+ spin_unlock(&dentry->d_lock);
+ return ret;
+}
+EXPORT_SYMBOL(dget_parent);
+
/*
* This is dput
*
@@ -380,54 +458,6 @@ int d_invalidate(struct dentry * dentry)
return 0;
}
-/* This should be called _only_ with a lock pinning the dentry */
-static inline struct dentry * __dget_locked_dlock(struct dentry *dentry)
-{
- dentry->d_count++;
- dentry_lru_del_init(dentry);
- return dentry;
-}
-
-static inline struct dentry * __dget_locked(struct dentry *dentry)
-{
- spin_lock(&dentry->d_lock);
- __dget_locked_dlock(dentry);
- spin_lock(&dentry->d_lock);
- return dentry;
-}
-
-struct dentry * dget_locked_dlock(struct dentry *dentry)
-{
- return __dget_locked_dlock(dentry);
-}
-
-struct dentry * dget_locked(struct dentry *dentry)
-{
- return __dget_locked(dentry);
-}
-
-struct dentry *dget_parent(struct dentry *dentry)
-{
- struct dentry *ret;
-
-repeat:
- spin_lock(&dentry->d_lock);
- ret = dentry->d_parent;
- if (!ret)
- goto out;
- if (!spin_trylock(&ret->d_lock)) {
- spin_unlock(&dentry->d_lock);
- goto repeat;
- }
- BUG_ON(!ret->d_count);
- ret->d_count++;
- spin_unlock(&ret->d_lock);
-out:
- spin_unlock(&dentry->d_lock);
- return ret;
-}
-EXPORT_SYMBOL(dget_parent);
-
/**
* d_find_alias - grab a hashed alias of inode
* @inode: inode in question
@@ -1316,14 +1346,6 @@ struct dentry * d_alloc_root(struct inod
return res;
}
-static inline struct hlist_head *d_hash(struct dentry *parent,
- unsigned long hash)
-{
- hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
- hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
- return dentry_hashtable + (hash & D_HASHMASK);
-}
-
/**
* d_obtain_alias - find or allocate a dentry for a given inode
* @inode: inode to allocate the dentry for
@@ -1570,7 +1592,8 @@ struct dentry * __d_lookup(struct dentry
unsigned int len = name->len;
unsigned int hash = name->hash;
const unsigned char *str = name->name;
- struct hlist_head *head = d_hash(parent,hash);
+ struct dcache_hash_bucket *b = d_hash(parent, hash);
+ struct hlist_head *head = &b->head;
struct dentry *found = NULL;
struct hlist_node *node;
struct dentry *dentry;
@@ -1664,6 +1687,7 @@ out:
int d_validate(struct dentry *dentry, struct dentry *dparent)
{
+ struct dcache_hash_bucket *b;
struct hlist_head *base;
struct hlist_node *lhp;
@@ -1675,20 +1699,21 @@ int d_validate(struct dentry *dentry, st
goto out;
spin_lock(&dentry->d_lock);
- spin_lock(&dcache_hash_lock);
- base = d_hash(dparent, dentry->d_name.hash);
- hlist_for_each(lhp,base) {
+ b = d_hash(dparent, dentry->d_name.hash);
+ base = &b->head;
+ spin_lock(&b->lock);
+ hlist_for_each(lhp, base) {
/* hlist_for_each_entry_rcu() not required for d_hash list
- * as it is parsed under dcache_hash_lock
+ * as it is parsed under dcache_hash_bucket->lock
*/
if (dentry == hlist_entry(lhp, struct dentry, d_hash)) {
- spin_unlock(&dcache_hash_lock);
+ spin_unlock(&b->lock);
__dget_locked_dlock(dentry);
spin_unlock(&dentry->d_lock);
return 1;
}
}
- spin_unlock(&dcache_hash_lock);
+ spin_unlock(&b->lock);
spin_unlock(&dentry->d_lock);
out:
return 0;
@@ -1739,11 +1764,12 @@ void d_delete(struct dentry * dentry)
fsnotify_nameremove(dentry, isdir);
}
-static void __d_rehash(struct dentry * entry, struct hlist_head *list)
+static void __d_rehash(struct dentry * entry, struct dcache_hash_bucket *b)
{
-
entry->d_flags &= ~DCACHE_UNHASHED;
- hlist_add_head_rcu(&entry->d_hash, list);
+ spin_lock(&b->lock);
+ hlist_add_head_rcu(&entry->d_hash, &b->head);
+ spin_unlock(&b->lock);
}
static void _d_rehash(struct dentry * entry)
@@ -1761,9 +1787,7 @@ static void _d_rehash(struct dentry * en
void d_rehash(struct dentry * entry)
{
spin_lock(&entry->d_lock);
- spin_lock(&dcache_hash_lock);
_d_rehash(entry);
- spin_unlock(&dcache_hash_lock);
spin_unlock(&entry->d_lock);
}
@@ -1841,6 +1865,7 @@ static void switch_names(struct dentry *
*/
static void d_move_locked(struct dentry * dentry, struct dentry * target)
{
+ struct dcache_hash_bucket *b;
if (!dentry->d_inode)
printk(KERN_WARNING "VFS: moving negative dcache entry\n");
@@ -1869,11 +1894,13 @@ static void d_move_locked(struct dentry
}
/* Move the dentry to the target hash queue, if on different bucket */
- spin_lock(&dcache_hash_lock);
- if (!d_unhashed(dentry))
+ if (!d_unhashed(dentry)) {
+ b = d_hash(dentry->d_parent, dentry->d_name.hash);
+ spin_lock(&b->lock);
hlist_del_rcu(&dentry->d_hash);
+ spin_unlock(&b->lock);
+ }
__d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
- spin_unlock(&dcache_hash_lock);
/* Unhash the target: dput() will then get rid of it */
__d_drop(target);
@@ -2080,9 +2107,7 @@ struct dentry *d_materialise_unique(stru
found_lock:
spin_lock(&actual->d_lock);
found:
- spin_lock(&dcache_hash_lock);
_d_rehash(actual);
- spin_unlock(&dcache_hash_lock);
spin_unlock(&actual->d_lock);
spin_unlock(&dcache_inode_lock);
out_nolock:
@@ -2534,7 +2559,7 @@ static void __init dcache_init_early(voi
dentry_hashtable =
alloc_large_system_hash("Dentry cache",
- sizeof(struct hlist_head),
+ sizeof(struct dcache_hash_bucket),
dhash_entries,
13,
HASH_EARLY,
@@ -2542,8 +2567,10 @@ static void __init dcache_init_early(voi
&d_hash_mask,
0);
- for (loop = 0; loop < (1 << d_hash_shift); loop++)
- INIT_HLIST_HEAD(&dentry_hashtable[loop]);
+ for (loop = 0; loop < (1 << d_hash_shift); loop++) {
+ spin_lock_init(&dentry_hashtable[loop].lock);
+ INIT_HLIST_HEAD(&dentry_hashtable[loop].head);
+ }
}
static void __init dcache_init(void)
@@ -2566,7 +2593,7 @@ static void __init dcache_init(void)
dentry_hashtable =
alloc_large_system_hash("Dentry cache",
- sizeof(struct hlist_head),
+ sizeof(struct dcache_hash_bucket),
dhash_entries,
13,
0,
@@ -2574,8 +2601,10 @@ static void __init dcache_init(void)
&d_hash_mask,
0);
- for (loop = 0; loop < (1 << d_hash_shift); loop++)
- INIT_HLIST_HEAD(&dentry_hashtable[loop]);
+ for (loop = 0; loop < (1 << d_hash_shift); loop++) {
+ spin_lock_init(&dentry_hashtable[loop].lock);
+ INIT_HLIST_HEAD(&dentry_hashtable[loop].head);
+ }
}
/* SLAB cache for __getname() consumers */
Index: linux-2.6/include/linux/dcache.h
===================================================================
--- linux-2.6.orig/include/linux/dcache.h
+++ linux-2.6/include/linux/dcache.h
@@ -185,7 +185,6 @@ d_iput: no no yes
#define DCACHE_COOKIE 0x0040 /* For use by dcookie subsystem */
extern spinlock_t dcache_inode_lock;
-extern spinlock_t dcache_hash_lock;
extern seqlock_t rename_lock;
/**
@@ -203,23 +202,8 @@ extern seqlock_t rename_lock;
*
* __d_drop requires dentry->d_lock.
*/
-
-static inline void __d_drop(struct dentry *dentry)
-{
- if (!(dentry->d_flags & DCACHE_UNHASHED)) {
- dentry->d_flags |= DCACHE_UNHASHED;
- spin_lock(&dcache_hash_lock);
- hlist_del_rcu(&dentry->d_hash);
- spin_unlock(&dcache_hash_lock);
- }
-}
-
-static inline void d_drop(struct dentry *dentry)
-{
- spin_lock(&dentry->d_lock);
- __d_drop(dentry);
- spin_unlock(&dentry->d_lock);
-}
+void d_drop(struct dentry *dentry);
+void __d_drop(struct dentry *dentry);
static inline int dname_external(struct dentry *dentry)
{
next prev parent reply other threads:[~2009-04-25 1:32 UTC|newest]
Thread overview: 50+ messages / expand[flat|nested] mbox.gz Atom feed top
2009-04-25 1:20 [patch 00/27] [rfc] vfs scalability patchset npiggin
2009-04-25 1:20 ` [patch 01/27] fs: cleanup files_lock npiggin
2009-04-25 3:20 ` Al Viro
2009-04-25 5:35 ` Eric W. Biederman
2009-04-26 6:12 ` Nick Piggin
2009-04-25 9:42 ` Alan Cox
2009-04-26 6:15 ` Nick Piggin
2009-04-25 1:20 ` [patch 02/27] fs: scale files_lock npiggin
2009-04-25 3:32 ` Al Viro
2009-04-25 1:20 ` [patch 03/27] fs: mnt_want_write speedup npiggin
2009-04-25 1:20 ` [patch 04/27] fs: introduce mnt_clone_write npiggin
2009-04-25 3:35 ` Al Viro
2009-04-25 1:20 ` [patch 05/27] fs: brlock vfsmount_lock npiggin
2009-04-25 3:50 ` Al Viro
2009-04-26 6:36 ` Nick Piggin
2009-04-25 1:20 ` [patch 06/27] fs: dcache fix LRU ordering npiggin
2009-04-25 1:20 ` [patch 07/27] fs: dcache scale hash npiggin
2009-04-25 1:20 ` [patch 08/27] fs: dcache scale lru npiggin
2009-04-25 1:20 ` [patch 09/27] fs: dcache scale nr_dentry npiggin
2009-04-25 1:20 ` [patch 10/27] fs: dcache scale dentry refcount npiggin
2009-04-25 1:20 ` [patch 11/27] fs: dcache scale d_unhashed npiggin
2009-04-25 1:20 ` [patch 12/27] fs: dcache scale subdirs npiggin
2009-04-25 1:20 ` [patch 13/27] fs: scale inode alias list npiggin
2009-04-25 1:20 ` [patch 14/27] fs: use RCU / seqlock logic for reverse and multi-step operaitons npiggin
2009-04-25 1:20 ` [patch 15/27] fs: dcache remove dcache_lock npiggin
2009-04-25 1:20 ` [patch 16/27] fs: dcache reduce dput locking npiggin
2009-04-25 1:20 ` npiggin [this message]
2009-04-25 1:20 ` [patch 18/27] fs: dcache reduce dcache_inode_lock npiggin
2009-04-25 1:20 ` [patch 19/27] fs: dcache per-inode inode alias locking npiggin
2009-04-25 1:20 ` [patch 20/27] fs: icache lock s_inodes list npiggin
2009-04-25 1:20 ` [patch 21/27] fs: icache lock inode hash npiggin
2009-04-25 1:20 ` [patch 22/27] fs: icache lock i_state npiggin
2009-04-25 1:20 ` [patch 23/27] fs: icache lock i_count npiggin
2009-04-25 1:20 ` [patch 24/27] fs: icache atomic inodes_stat npiggin
2009-04-25 1:20 ` [patch 25/27] fs: icache lock lru/writeback lists npiggin
2009-04-25 1:20 ` [patch 26/27] fs: icache protect inode state npiggin
2009-04-25 1:20 ` [patch 27/27] fs: icache remove inode_lock npiggin
2009-04-25 4:18 ` [patch 00/27] [rfc] vfs scalability patchset Al Viro
2009-04-25 5:02 ` Nick Piggin
2009-04-25 8:01 ` Christoph Hellwig
2009-04-25 8:06 ` Al Viro
2009-04-28 9:09 ` Christoph Hellwig
2009-04-28 9:48 ` Nick Piggin
2009-04-28 10:58 ` Peter Zijlstra
2009-04-28 11:32 ` Eric W. Biederman
2009-04-30 6:14 ` Nick Piggin
2009-04-25 19:08 ` Eric W. Biederman
2009-04-25 19:31 ` Al Viro
2009-04-25 20:29 ` Eric W. Biederman
2009-04-25 22:05 ` Theodore Tso
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20090425012211.509344338@suse.de \
--to=npiggin@suse.de \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.