[PATCH 05/11] quota: make per-sb hash array

[Dmitry Monakhov <dmonakhov@openvz.org>]

Currently quota_hash[] is global, which is bad for scalability.
Also is is the last user of global dq_list_lock.
It is reasonable to introduce didicated hash for each super_block
which use quota.

per-sb hash will be allocated only when necessery (on first quota_on())
Protected by per-sb dq_list_lock.

Signed-off-by: Dmitry Monakhov <dmonakhov@gmail.com>
---
 fs/quota/dquot.c      |  111 ++++++++++++++++++++++++++++++++----------------
 include/linux/quota.h |   10 ++++-
 2 files changed, 83 insertions(+), 38 deletions(-)

diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
index f2092d1..e93b323 100644
--- a/fs/quota/dquot.c
+++ b/fs/quota/dquot.c
@@ -220,7 +220,7 @@ static void put_quota_format(struct quota_format_type *fmt)
 /*
  * Dquot List Management:
  * The quota code uses three lists for dquot management: the inuse_list,
- * free_dquots, and dquot_hash[] array. A single dquot structure may be
+ * free_dquots, and dq_hash[] array. A single dquot structure may be
  * on all three lists, depending on its current state.
  *
  * All dquots are placed to the end of inuse_list when first created, and this
@@ -233,13 +233,10 @@ static void put_quota_format(struct quota_format_type *fmt)
  * dquot is invalidated it's completely released from memory.
  *
  * Dquots with a specific identity (device, type and id) are placed on
- * one of the dquot_hash[] hash chains. The provides an efficient search
+ * one of the dq_hash[] hash chains. The provides an efficient search
  * mechanism to locate a specific dquot.
  */
 
-static unsigned int dq_hash_bits, dq_hash_mask;
-static struct hlist_head *dquot_hash;
-
 struct dqstats dqstats;
 EXPORT_SYMBOL(dqstats);
 
@@ -251,8 +248,9 @@ hashfn(const struct super_block *sb, unsigned int id, int type)
 {
 	unsigned long tmp;
 
-	tmp = (((unsigned long)sb>>L1_CACHE_SHIFT) ^ id) * (MAXQUOTAS - type);
-	return (tmp + (tmp >> dq_hash_bits)) & dq_hash_mask;
+	tmp = id * (MAXQUOTAS - type);
+	return (tmp + (tmp >> sb->s_dquot.dq_hash.bits)) &
+		sb->s_dquot.dq_hash.mask;
 }
 
 /*
@@ -261,7 +259,8 @@ hashfn(const struct super_block *sb, unsigned int id, int type)
 static inline void insert_dquot_hash(struct dquot *dquot)
 {
 	struct hlist_head *head;
-	head = dquot_hash + hashfn(dquot->dq_sb, dquot->dq_id, dquot->dq_type);
+	head = dq_opt(dquot)->dq_hash.head +
+		hashfn(dquot->dq_sb, dquot->dq_id, dquot->dq_type);
 	hlist_add_head(&dquot->dq_hash, head);
 }
 
@@ -270,13 +269,17 @@ static inline void remove_dquot_hash(struct dquot *dquot)
 	hlist_del_init(&dquot->dq_hash);
 }
 
-static struct dquot *find_dquot(unsigned int hashent, struct super_block *sb,
+static struct dquot *find_dquot(struct super_block *sb,
 				unsigned int id, int type)
 {
 	struct hlist_node *node;
 	struct dquot *dquot;
+	unsigned int hashent = hashfn(sb, id, type);
+
+	if (!sb_dqopt(sb)->dq_hash.head)
+		return NULL;
 
-	hlist_for_each (node, dquot_hash+hashent) {
+	hlist_for_each(node, sb_dqopt(sb)->dq_hash.head + hashent) {
 		dquot = hlist_entry(node, struct dquot, dq_hash);
 		if (dquot->dq_sb == sb && dquot->dq_id == id &&
 		    dquot->dq_type == type)
@@ -846,7 +849,6 @@ static struct dquot *get_empty_dquot(struct super_block *sb, int type)
  */
 struct dquot *dqget(struct super_block *sb, unsigned int id, int type)
 {
-	unsigned int hashent = hashfn(sb, id, type);
 	struct dquot *dquot = NULL, *empty = NULL;
 	struct quota_info *dqopt = sb_dqopt(sb);
 
@@ -864,7 +866,7 @@ we_slept:
 	}
 	spin_unlock(&sb_dqopt(sb)->dq_state_lock);
 
-	dquot = find_dquot(hashent, sb, id, type);
+	dquot = find_dquot(sb, id, type);
 	if (!dquot) {
 		if (!empty) {
 			spin_unlock(&dqopt->dq_list_lock);
@@ -1925,6 +1927,47 @@ int dquot_file_open(struct inode *inode, struct file *file)
 }
 EXPORT_SYMBOL(dquot_file_open);
 
+int dquot_hash_alloc(struct super_block *sb, int order)
+{
+	unsigned long nr_hash, i;
+	struct hlist_head *hash_array;
+	struct dquot_hash *dq_hash = &sb_dqopt(sb)->dq_hash;
+
+	hash_array = (struct hlist_head *)__get_free_pages(GFP_KERNEL, order);
+	if (!hash_array)
+		return -ENOMEM;
+
+	/* Find power-of-two hlist_heads which can fit into allocation */
+	nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
+	dq_hash->bits = 0;
+	do {
+		dq_hash->bits++;
+	} while (nr_hash >> dq_hash->bits);
+	dq_hash->bits--;
+
+	nr_hash = 1UL << dq_hash->bits;
+	dq_hash->mask = nr_hash - 1;
+	for (i = 0; i < nr_hash; i++)
+		INIT_HLIST_HEAD(hash_array + i);
+	dq_hash->order = order;
+	dq_hash->head = hash_array;
+	return 0;
+}
+
+void dquot_hash_destroy(struct super_block *sb)
+{
+
+	struct dquot_hash * dq_hash = &sb_dqopt(sb)->dq_hash;
+	unsigned long i, nr_hash = 1UL << dq_hash->bits;
+	unsigned long addr = (unsigned long )dq_hash->head;
+
+	for (i = 0; i < nr_hash; i++)
+		BUG_ON(!hlist_empty(dq_hash->head + i));
+
+	dq_hash->head = NULL;
+	free_pages(addr, dq_hash->order);
+}
+
 /*
  * Turn quota off on a device. type == -1 ==> quotaoff for all types (umount)
  */
@@ -2006,6 +2049,9 @@ int dquot_disable(struct super_block *sb, int type, unsigned int flags)
 		dqopt->info[cnt].dqi_bgrace = 0;
 		dqopt->ops[cnt] = NULL;
 	}
+	if (!sb_any_quota_loaded(sb))
+		/* All quotas was unloaded, hash no longer needed */
+		dquot_hash_destroy(sb);
 	mutex_unlock(&dqopt->dqonoff_mutex);
 
 	/* Skip syncing and setting flags if quota files are hidden */
@@ -2081,7 +2127,7 @@ static int vfs_load_quota_inode(struct inode *inode, int type, int format_id,
 	struct quota_format_type *fmt = find_quota_format(format_id);
 	struct super_block *sb = inode->i_sb;
 	struct quota_info *dqopt = sb_dqopt(sb);
-	int error;
+	int error, hash_alloc;
 	int oldflags = -1;
 
 	if (!fmt)
@@ -2115,6 +2161,16 @@ static int vfs_load_quota_inode(struct inode *inode, int type, int format_id,
 		invalidate_bdev(sb->s_bdev);
 	}
 	mutex_lock(&dqopt->dqonoff_mutex);
+
+	hash_alloc = !sb_any_quota_loaded(sb);
+	if (hash_alloc) {
+		error = dquot_hash_alloc(sb, 0);
+		if (error) {
+			hash_alloc = 0;
+			goto out_lock;
+		}
+	}
+
 	if (sb_has_quota_loaded(sb, type)) {
 		error = -EBUSY;
 		goto out_lock;
@@ -2168,6 +2224,8 @@ out_file_init:
 	dqopt->files[type] = NULL;
 	iput(inode);
 out_lock:
+	if (hash_alloc)
+		dquot_hash_destroy(sb);
 	if (oldflags != -1) {
 		mutex_lock_nested(&inode->i_mutex, I_MUTEX_QUOTA);
 		/* Set the flags back (in the case of accidental quotaon()
@@ -2224,8 +2282,10 @@ int dquot_quota_on(struct super_block *sb, int type, int format_id,
 		   struct path *path)
 {
 	int error = security_quota_on(path->dentry);
+
 	if (error)
 		return error;
+
 	/* Quota file not on the same filesystem? */
 	if (path->mnt->mnt_sb != sb)
 		error = -EXDEV;
@@ -2643,8 +2703,7 @@ static ctl_table sys_table[] = {
 static int __init dquot_init(void)
 {
 	int i, ret;
-	unsigned long nr_hash, order;
-
+	
 	printk(KERN_NOTICE "VFS: Disk quotas %s\n", __DQUOT_VERSION__);
 
 	register_sysctl_table(sys_table);
@@ -2655,33 +2714,11 @@ static int __init dquot_init(void)
 				SLAB_MEM_SPREAD|SLAB_PANIC),
 			NULL);
 
-	order = 0;
-	dquot_hash = (struct hlist_head *)__get_free_pages(GFP_ATOMIC, order);
-	if (!dquot_hash)
-		panic("Cannot create dquot hash table");
-
 	for (i = 0; i < _DQST_DQSTAT_LAST; i++) {
 		ret = percpu_counter_init(&dqstats.counter[i], 0);
 		if (ret)
 			panic("Cannot create dquot stat counters");
 	}
-
-	/* Find power-of-two hlist_heads which can fit into allocation */
-	nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
-	dq_hash_bits = 0;
-	do {
-		dq_hash_bits++;
-	} while (nr_hash >> dq_hash_bits);
-	dq_hash_bits--;
-
-	nr_hash = 1UL << dq_hash_bits;
-	dq_hash_mask = nr_hash - 1;
-	for (i = 0; i < nr_hash; i++)
-		INIT_HLIST_HEAD(dquot_hash + i);
-
-	printk("Dquot-cache hash table entries: %ld (order %ld, %ld bytes)\n",
-			nr_hash, order, (PAGE_SIZE << order));
-
 	register_shrinker(&dqcache_shrinker);
 
 	return 0;
diff --git a/include/linux/quota.h b/include/linux/quota.h
index 134c18d..eb083fc 100644
--- a/include/linux/quota.h
+++ b/include/linux/quota.h
@@ -393,15 +393,23 @@ static inline void quota_send_warning(short type, unsigned int id, dev_t dev,
 }
 #endif /* CONFIG_QUOTA_NETLINK_INTERFACE */
 
+struct dquot_hash {
+	struct hlist_head *head;
+	unsigned int  order;
+	unsigned int  bits;
+	unsigned int  mask;
+};
+
 struct quota_info {
 	unsigned int flags;			/* Flags for diskquotas on this device */
 	struct mutex dqio_mutex;		/* lock device while I/O in progress */
 	struct mutex dqonoff_mutex;		/* Serialize quotaon & quotaoff */
 	struct rw_semaphore dqptr_sem;		/* serialize ops using quota_info struct, pointers from inode to dquots */
 	spinlock_t dq_state_lock;		/* serialize quota state changes*/
-	spinlock_t dq_list_lock;		/* protect lists */
+	spinlock_t dq_list_lock;		/* protect lists and hash */
 	struct list_head dq_inuse_list;		/* list of inused dquotas */
 	struct list_head dq_free_list;		/* list of free dquotas */
+	struct dquot_hash dq_hash;		/* dquot lookup hash  */
 
 	struct inode *files[MAXQUOTAS];		/* inodes of quotafiles */
 	struct mem_dqinfo info[MAXQUOTAS];	/* Information for each quota type */
-- 
1.6.6.1