[PATCH v4 08/11] futex: Resize futex hash table based on number of threads.

Sebastian Andrzej Siewior
2024-12-03T16:43:42Z

Automatically size hash bucket based on the number of threads. The logic
tries to allocate between 16 and futex_hashsize (the default for the
system wide hash bucket) and uses 4 * number-of-threads.

If the upper limit is reached, the HB will be made invariant.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/fork.c | 4 ----
kernel/futex/core.c | 39 +++++++++++++++++++++++++++++++++------
2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/kernel/fork.c b/kernel/fork.c
index 6267d600af991..35ec9958707c5 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2134,10 +2134,6 @@ static bool need_futex_hash_allocate_default(u64 clone_flags)
{
if ((clone_flags & (CLONE_THREAD | CLONE_VM)) != (CLONE_THREAD | CLONE_VM))
return false;
- if (!thread_group_empty(current))
- return false;
- if (current->mm->futex_hash_bucket)
- return false;
return true;
}

diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 1abea8f9abd22..19515aa5a6430 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -65,6 +65,8 @@ struct futex_hash_bucket_private {
struct futex_hash_bucket queues[];
};

+static unsigned int futex_default_max_buckets;
+
/*
* Fault injections for futexes.
*/
@@ -1262,7 +1264,7 @@ bool futex_check_hb_valid(struct futex_hash_bucket *hb)
return hb_p_now == hb_p;
}

-static int futex_hash_allocate(unsigned int hash_slots)
+static int futex_hash_allocate(unsigned int hash_slots, bool slots_invariant)
{
struct futex_hash_bucket_private *hb_p, *hb_p_old = NULL;
struct mm_struct *mm;
@@ -1274,8 +1276,8 @@ static int futex_hash_allocate(unsigned int hash_slots)
hash_slots = 16;
if (hash_slots < 2)
hash_slots = 2;
- if (hash_slots > 131072)
- hash_slots = 131072;
+ if (hash_slots > futex_default_max_buckets)
+ hash_slots = futex_default_max_buckets;
if (!is_power_of_2(hash_slots))
hash_slots = rounddown_pow_of_two(hash_slots);

@@ -1293,7 +1295,7 @@ static int futex_hash_allocate(unsigned int hash_slots)

rcuref_init(&hb_p->users, 1);
hb_p->hash_mask = hash_slots - 1;
- hb_p->slots_invariant = false;
+ hb_p->slots_invariant = slots_invariant;

for (i = 0; i < hash_slots; i++)
futex_hash_bucket_init(&hb_p->queues[i], i + 1);
@@ -1321,7 +1323,31 @@ static int futex_hash_allocate(unsigned int hash_slots)

int futex_hash_allocate_default(void)
{
- return futex_hash_allocate(0);
+ unsigned int threads;
+ unsigned int buckets;
+ unsigned int current_buckets = 0;
+ struct futex_hash_bucket_private *hb_p;
+
+ if (!current->mm)
+ return 0;
+
+ scoped_guard(rcu) {
+ threads = get_nr_threads(current);
+ hb_p = rcu_dereference(current->mm->futex_hash_bucket);
+ if (hb_p) {
+ if (hb_p->slots_invariant)
+ return 0;
+ current_buckets = hb_p->hash_mask + 1;
+ }
+ }
+
+ buckets = roundup_pow_of_two(4 * threads);
+ buckets = max(buckets, 16);
+ buckets = min(buckets, futex_default_max_buckets);
+ if (current_buckets > buckets)
+ return 0;
+
+ return futex_hash_allocate(buckets, buckets == futex_default_max_buckets);
}

static int futex_hash_get_slots(void)
@@ -1372,7 +1398,7 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3,

switch (arg2) {
case PR_FUTEX_HASH_SET_SLOTS:
- ret = futex_hash_allocate(arg3);
+ ret = futex_hash_allocate(arg3, false);
break;

case PR_FUTEX_HASH_GET_SLOTS:
@@ -1404,6 +1430,7 @@ static int __init futex_init(void)
#else
futex_hashsize = roundup_pow_of_two(256 * num_possible_cpus());
#endif
+ futex_default_max_buckets = futex_hashsize;

futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),
futex_hashsize, 0, 0,
--
2.45.2