[PATCH 2/2] sched/fair: Honor asymmetric SMT priority in idle selection

[Andrea Righi <arighi@nvidia.com>]

SD_ASYM_PACKING orders CPUs that share an SMT core, but idle CPU
selection does not consult that order. A task can therefore wake on an
arbitrary sibling and remain there until load balancing corrects the
placement. On SMT implementations where changing the active sibling
repartitions core resources, that initial choice can cause a large and
persistent performance loss.

When idle selection finds an available CPU in an SMT core, choose the
highest-priority available sibling. On SMT2 this only changes selection
on fully idle cores, because a partially idle core has only one
available CPU. On wider SMT cores it also fills available siblings in
priority order while the core is partially busy.

Apply the preference to idle-core and idle-CPU scans, asymmetric
capacity scans, and the target, previous, and recently-used CPU fast
paths.

Keep physical-core capacity selection independent from SMT sibling
ordering. SD_ASYM_CPUCAPACITY can first select among cores with
different maximum capacities, SD_ASYM_PACKING then selects the preferred
available sibling inside the chosen core, whose siblings continue to
share equal capacity.

Signed-off-by: Andrea Righi <arighi@nvidia.com>
---
 kernel/sched/fair.c     | 81 ++++++++++++++++++++++++++++++++++++-----
 kernel/sched/sched.h    |  6 +++
 kernel/sched/topology.c | 36 ++++++++++++++++++
 3 files changed, 114 insertions(+), 9 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 8dff37059faf7..3c49aa63742cb 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -8587,6 +8587,65 @@ static inline bool test_idle_cores(int cpu)
 	return false;
 }
 
+/*
+ * Return true when @cpu has a higher asymmetric-packing priority than @other in their SMT
+ * scheduling domain.
+ */
+static bool sched_smt_asym_prefer(int cpu, int other)
+{
+	struct sched_domain *sd;
+
+	for_each_domain(cpu, sd) {
+		/*
+		 * Only honor priorities declared at shared-capacity SMT levels.
+		 * SD_ASYM_PACKING at higher levels may describe core ordering.
+		 */
+		if (!(sd->flags & SD_SHARE_CPUCAPACITY))
+			break;
+
+		if ((sd->flags & SD_ASYM_PACKING) && cpumask_test_cpu(other, sched_domain_span(sd)))
+			return sched_asym_prefer(cpu, other);
+	}
+
+	return false;
+}
+
+/*
+ * Return the highest-priority available CPU in @cpu's SMT core that is also in @cpus.
+ */
+static int __select_idle_smt_cpu(struct task_struct *p, int cpu, const struct cpumask *cpus)
+{
+	int best = cpu;
+	int sibling;
+
+	for_each_cpu_and(sibling, cpu_smt_mask(cpu), cpus) {
+		if (sibling == best || !choose_idle_cpu(sibling, p))
+			continue;
+
+		if (sched_smt_asym_prefer(sibling, best))
+			best = sibling;
+	}
+
+	return best;
+}
+
+static inline int
+select_idle_smt_cpu(struct task_struct *p, int cpu, const struct cpumask *cpus)
+{
+	if (!sched_smt_asym_active())
+		return cpu;
+
+	return __select_idle_smt_cpu(p, cpu, cpus);
+}
+
+/*
+ * Redirect an available SMT CPU to a higher-priority available sibling allowed by task affinity.
+ */
+static inline int select_idle_smt_priority(struct task_struct *p, int cpu)
+{
+	return select_idle_smt_cpu(p, cpu, p->cpus_ptr);
+}
+
 /*
  * Scans the local SMT mask to see if the entire core is idle, and records this
  * information in sd_balance_shared->has_idle_cores.
@@ -8645,7 +8704,7 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu
 	}
 
 	if (idle)
-		return core;
+		return select_idle_smt_cpu(p, core, cpus);
 
 	cpumask_andnot(cpus, cpus, cpu_smt_mask(core));
 	return -1;
@@ -8668,7 +8727,7 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t
 		if (!cpumask_test_cpu(cpu, sched_domain_span(sd)))
 			continue;
 		if (choose_idle_cpu(cpu, p))
-			return cpu;
+			return select_idle_smt_priority(p, cpu);
 	}
 
 	return -1;
@@ -8720,7 +8779,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 						return -1;
 					idle_cpu = __select_idle_cpu(cpu, p);
 					if ((unsigned int)idle_cpu < nr_cpumask_bits)
-						return idle_cpu;
+						return select_idle_smt_priority(p, idle_cpu);
 				}
 			}
 			cpumask_andnot(cpus, cpus, sched_group_span(sg));
@@ -8745,7 +8804,8 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 	if (has_idle_core)
 		set_idle_cores(target, false);
 
-	return idle_cpu;
+	return (unsigned int)idle_cpu < nr_cpumask_bits ?
+		select_idle_smt_priority(p, idle_cpu) : idle_cpu;
 }
 
 /*
@@ -8858,7 +8918,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
 		 * immediately.
 		 */
 		if (fits > 0 && preferred_core)
-			return cpu;
+			return select_idle_smt_cpu(p, cpu, cpus);
 		/*
 		 * Only the min performance hint (i.e. uclamp_min) doesn't fit.
 		 * Look for the CPU with best capacity.
@@ -8915,6 +8975,9 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
 	if (has_idle_core && best_fits > ASYM_IDLE_COMPLETE_MISFIT)
 		set_idle_cores(target, false);
 
+	if (best_cpu >= 0)
+		best_cpu = select_idle_smt_cpu(p, best_cpu, cpus);
+
 	return best_cpu;
 }
 
@@ -8971,7 +9034,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 
 	if (choose_idle_cpu(target, p) &&
 	    asym_fits_cpu(task_util, util_min, util_max, target))
-		return target;
+		return select_idle_smt_priority(p, target);
 
 	/*
 	 * If the previous CPU is cache affine and idle, don't be stupid:
@@ -8982,9 +9045,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 
 		if (!static_branch_unlikely(&sched_cluster_active) ||
 		    cpus_share_resources(prev, target))
-			return prev;
+			return select_idle_smt_priority(p, prev);
 
-		prev_aff = prev;
+		prev_aff = select_idle_smt_priority(p, prev);
 	}
 
 	/*
@@ -9015,7 +9078,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 
 		if (!static_branch_unlikely(&sched_cluster_active) ||
 		    cpus_share_resources(recent_used_cpu, target))
-			return recent_used_cpu;
+			return select_idle_smt_priority(p, recent_used_cpu);
 
 	} else {
 		recent_used_cpu = -1;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index e656c7059bf86..73731e9439b97 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2240,6 +2240,7 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
 
 extern struct static_key_false sched_asym_cpucapacity;
+extern struct static_key_false sched_smt_asym_packing;
 extern struct static_key_false sched_cluster_active;
 
 static __always_inline bool sched_asym_cpucap_active(void)
@@ -2247,6 +2248,11 @@ static __always_inline bool sched_asym_cpucap_active(void)
 	return static_branch_unlikely(&sched_asym_cpucapacity);
 }
 
+static __always_inline bool sched_smt_asym_active(void)
+{
+	return static_branch_unlikely(&sched_smt_asym_packing);
+}
+
 struct sched_group_capacity {
 	atomic_t		ref;
 	/*
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 0248227d983a7..06c40eb5932af 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -683,8 +683,24 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
 
 DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);
+DEFINE_STATIC_KEY_FALSE(sched_smt_asym_packing);
 DEFINE_STATIC_KEY_FALSE(sched_cluster_active);
 
+static bool has_asym_smt_domain(int cpu)
+{
+	struct sched_domain *sd;
+
+	for_each_domain(cpu, sd) {
+		if (!(sd->flags & SD_SHARE_CPUCAPACITY))
+			break;
+
+		if (sd->flags & SD_ASYM_PACKING)
+			return true;
+	}
+
+	return false;
+}
+
 static void update_top_cache_domain(int cpu)
 {
 	struct sched_domain_shared *sds = NULL;
@@ -3084,6 +3100,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 	struct rq *rq = NULL;
 	int i, ret = -ENOMEM;
 	bool has_asym = false;
+	bool has_asym_smt = false;
 	bool has_cluster = false;
 
 	if (WARN_ON(cpumask_empty(cpu_map)))
@@ -3202,6 +3219,9 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 
 		cpu_attach_domain(sd, d.rd, i);
 
+		if (has_asym_smt_domain(i))
+			has_asym_smt = true;
+
 		if (lowest_flag_domain(i, SD_CLUSTER))
 			has_cluster = true;
 	}
@@ -3210,6 +3230,9 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 	if (has_asym)
 		static_branch_inc_cpuslocked(&sched_asym_cpucapacity);
 
+	if (has_asym_smt)
+		static_branch_inc_cpuslocked(&sched_smt_asym_packing);
+
 	if (has_cluster)
 		static_branch_inc_cpuslocked(&sched_cluster_active);
 
@@ -3310,11 +3333,24 @@ int __init sched_init_domains(const struct cpumask *cpu_map)
 static void detach_destroy_domains(const struct cpumask *cpu_map)
 {
 	unsigned int cpu = cpumask_any(cpu_map);
+	bool has_asym_smt = false;
 	int i;
 
+	rcu_read_lock();
+	for_each_cpu(i, cpu_map) {
+		if (has_asym_smt_domain(i)) {
+			has_asym_smt = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
 	if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu)))
 		static_branch_dec_cpuslocked(&sched_asym_cpucapacity);
 
+	if (has_asym_smt)
+		static_branch_dec_cpuslocked(&sched_smt_asym_packing);
+
 	if (static_branch_unlikely(&sched_cluster_active))
 		static_branch_dec_cpuslocked(&sched_cluster_active);
 
-- 
2.55.0