[PATCH 07/19] quota: make per-sb hash array

[Dmitry Monakhov <dmonakhov@openvz.org>]

From: Dmitry Monakhov <dmonakhov@gmail.com>

Currently quota_hash[] is global, which is bad for scalability.
Also is is the last user of global dq_list_lock.
It is reasonable to introduce dedicated hash for each super_block
which use quota.

per-sb hash will be allocated only when necessary (on first quota_on())
Protected by per-sb dq_list_lock.

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 fs/quota/dquot.c      |   97 ++++++++++++++++++++++++++++++-------------------
 include/linux/quota.h |   11 +++++-
 2 files changed, 69 insertions(+), 39 deletions(-)

diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
index 324f124..822d7ad 100644
--- a/fs/quota/dquot.c
+++ b/fs/quota/dquot.c
@@ -220,7 +220,7 @@ static void put_quota_format(struct quota_format_type *fmt)
 /*
  * Dquot List Management:
  * The quota code uses three lists for dquot management: the inuse_list,
- * free_dquots, and dquot_hash[] array. A single dquot structure may be
+ * free_dquots, and dq_hash[] array. A single dquot structure may be
  * on all three lists, depending on its current state.
  *
  * All dquots are placed to the end of inuse_list when first created, and this
@@ -233,13 +233,10 @@ static void put_quota_format(struct quota_format_type *fmt)
  * dquot is invalidated it's completely released from memory.
  *
  * Dquots with a specific identity (device, type and id) are placed on
- * one of the dquot_hash[] hash chains. The provides an efficient search
+ * one of the dq_hash[] hash chains. The provides an efficient search
  * mechanism to locate a specific dquot.
  */
 
-static unsigned int dq_hash_bits, dq_hash_mask;
-static struct hlist_head *dquot_hash;
-
 struct dqstats dqstats;
 EXPORT_SYMBOL(dqstats);
 
@@ -251,8 +248,9 @@ hashfn(const struct super_block *sb, unsigned int id, int type)
 {
 	unsigned long tmp;
 
-	tmp = (((unsigned long)sb>>L1_CACHE_SHIFT) ^ id) * (MAXQUOTAS - type);
-	return (tmp + (tmp >> dq_hash_bits)) & dq_hash_mask;
+	tmp = id * (MAXQUOTAS - type);
+	return (tmp + (tmp >> dqopts(sb)->dq_hash.bits)) &
+		dqopts(sb)->dq_hash.mask;
 }
 
 /*
@@ -261,7 +259,8 @@ hashfn(const struct super_block *sb, unsigned int id, int type)
 static inline void insert_dquot_hash(struct dquot *dquot)
 {
 	struct hlist_head *head;
-	head = dquot_hash + hashfn(dquot->dq_sb, dquot->dq_id, dquot->dq_type);
+	head = sb_dqopts(dquot)->dq_hash.head +
+		hashfn(dquot->dq_sb, dquot->dq_id, dquot->dq_type);
 	hlist_add_head(&dquot->dq_hash, head);
 }
 
@@ -270,13 +269,17 @@ static inline void remove_dquot_hash(struct dquot *dquot)
 	hlist_del_init(&dquot->dq_hash);
 }
 
-static struct dquot *find_dquot(unsigned int hashent, struct super_block *sb,
+static struct dquot *find_dquot(struct super_block *sb,
 				unsigned int id, int type)
 {
 	struct hlist_node *node;
 	struct dquot *dquot;
+	unsigned int hashent = hashfn(sb, id, type);
 
-	hlist_for_each (node, dquot_hash+hashent) {
+	if (!dqopts(sb)->dq_hash.head)
+		return NULL;
+
+	hlist_for_each(node, dqopts(sb)->dq_hash.head + hashent) {
 		dquot = hlist_entry(node, struct dquot, dq_hash);
 		if (dquot->dq_sb == sb && dquot->dq_id == id &&
 		    dquot->dq_type == type)
@@ -854,7 +857,6 @@ static struct dquot *get_empty_dquot(struct super_block *sb, int type)
  */
 struct dquot *dqget(struct super_block *sb, unsigned int id, int type)
 {
-	unsigned int hashent = hashfn(sb, id, type);
 	struct dquot *dquot = NULL, *empty = NULL;
 	struct quota_info *dqopt = dqopts(sb);
 
@@ -872,7 +874,7 @@ we_slept:
 	}
 	spin_unlock(&dqopt->dq_state_lock);
 
-	dquot = find_dquot(hashent, sb, id, type);
+	dquot = find_dquot(sb, id, type);
 	if (!dquot) {
 		if (!empty) {
 			spin_unlock(&dqopt->dq_list_lock);
@@ -1927,6 +1929,42 @@ const struct dquot_operations dquot_operations = {
 };
 EXPORT_SYMBOL(dquot_operations);
 
+int alloc_quota_hash(struct quota_info *dqopt, int order)
+{
+	unsigned long nr_hash, i;
+	struct hlist_head *hash_array;
+	struct dquot_hash *dq_hash = &dqopt->dq_hash;
+
+	hash_array = (struct hlist_head *)__get_free_pages(GFP_KERNEL, order);
+	if (!hash_array)
+		return -ENOMEM;
+
+	/* Find power-of-two hlist_heads which can fit into allocation */
+	nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
+	dq_hash->bits = ilog2(nr_hash);
+	nr_hash = 1UL << dq_hash->bits;
+	dq_hash->mask = nr_hash - 1;
+	for (i = 0; i < nr_hash; i++)
+		INIT_HLIST_HEAD(hash_array + i);
+	dq_hash->order = order;
+	dq_hash->head = hash_array;
+	return 0;
+}
+
+void free_quota_hash(struct quota_info *dqopt)
+{
+
+	struct dquot_hash *dq_hash = &dqopt->dq_hash;
+	unsigned long i, nr_hash = 1UL << dq_hash->bits;
+	unsigned long addr = (unsigned long )dq_hash->head;
+
+	for (i = 0; i < nr_hash; i++)
+		WARN_ON(!hlist_empty(dq_hash->head + i));
+
+	dq_hash->head = NULL;
+	free_pages(addr, dq_hash->order);
+}
+
 /*
  * Generic helper for ->open on filesystems supporting disk quotas.
  */
@@ -1956,14 +1994,21 @@ static int alloc_quota_info(struct quota_ctl_info *dqctl) {
 	spin_lock_init(&dqopt->dq_list_lock);
 	INIT_LIST_HEAD(&dqopt->dq_inuse_list);
 	INIT_LIST_HEAD(&dqopt->dq_free_list);
-
 	dqctl->dq_opt = dqopt;
-	return 0;
+	err = alloc_quota_hash(dqopt, 0);
+
+	if (err && dqopt) {
+		free_quota_hash(dqopt);
+		kfree(dqopt);
+		dqctl->dq_opt = NULL;
+	}
+	return err;
 }
 
 static void free_quota_info(struct quota_ctl_info *dqctl)
 {
 	if (dqctl->dq_opt) {
+		free_quota_hash(dqctl->dq_opt);
 		kfree(dqctl->dq_opt);
 		dqctl->dq_opt = NULL;
 	}
@@ -2709,8 +2754,6 @@ static ctl_table sys_table[] = {
 static int __init dquot_init(void)
 {
 	int i, ret;
-	unsigned long nr_hash, order;
-
 	printk(KERN_NOTICE "VFS: Disk quotas %s\n", __DQUOT_VERSION__);
 
 	register_sysctl_table(sys_table);
@@ -2721,33 +2764,11 @@ static int __init dquot_init(void)
 				SLAB_MEM_SPREAD|SLAB_PANIC),
 			NULL);
 
-	order = 0;
-	dquot_hash = (struct hlist_head *)__get_free_pages(GFP_ATOMIC, order);
-	if (!dquot_hash)
-		panic("Cannot create dquot hash table");
-
 	for (i = 0; i < _DQST_DQSTAT_LAST; i++) {
 		ret = percpu_counter_init(&dqstats.counter[i], 0);
 		if (ret)
 			panic("Cannot create dquot stat counters");
 	}
-
-	/* Find power-of-two hlist_heads which can fit into allocation */
-	nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
-	dq_hash_bits = 0;
-	do {
-		dq_hash_bits++;
-	} while (nr_hash >> dq_hash_bits);
-	dq_hash_bits--;
-
-	nr_hash = 1UL << dq_hash_bits;
-	dq_hash_mask = nr_hash - 1;
-	for (i = 0; i < nr_hash; i++)
-		INIT_HLIST_HEAD(dquot_hash + i);
-
-	printk("Dquot-cache hash table entries: %ld (order %ld, %ld bytes)\n",
-			nr_hash, order, (PAGE_SIZE << order));
-
 	register_shrinker(&dqcache_shrinker);
 
 	return 0;
diff --git a/include/linux/quota.h b/include/linux/quota.h
index bb63abf..eaa9f91 100644
--- a/include/linux/quota.h
+++ b/include/linux/quota.h
@@ -401,13 +401,22 @@ struct quota_ctl_info {
 	const struct dquot_operations *dq_op;
 	struct quota_info *dq_opt;
 };
+
+struct dquot_hash {
+	struct hlist_head *head;
+	unsigned int  order;
+	unsigned int  bits;
+	unsigned int  mask;
+};
+
 struct quota_info {
 	struct mutex dqio_mutex;		/* lock device while I/O in progress */
 	struct mem_dqinfo info[MAXQUOTAS];	/* Information for each quota type */
 	spinlock_t dq_state_lock;	/* serialize quota state changes*/
-	spinlock_t dq_list_lock;		/* protect lists */
+	spinlock_t dq_list_lock;		/* protect lists and hash*/
 	struct list_head dq_inuse_list;		/* list of inused dquotas */
 	struct list_head dq_free_list;		/* list of free dquotas */
+	struct dquot_hash dq_hash;		/* dquot lookup hash  */
 
 	struct inode *files[MAXQUOTAS];	/* inodes of quotafiles */
 	const struct quota_format_ops *fmt_ops[MAXQUOTAS];	/* Operations for each type */
-- 
1.6.5.2