[PATCH] cpu/hotplug: dont offline the last non-isolated CPU

[PATCH] cpu/hotplug: dont offline the last non-isolated CPU

[yang.yang29]

From: Ran Xiaokai <ran.xiaokai@zte.com.cn>

If system has some isolate cpus with "isolcpus=" parameter,
and user try to offline all the non-isolated CPUs, kernel will
first report a warning and then panic when taking the last
non-isolated CPU offline.

Kernel should not panic when user managing CPUs with this
meaningless behavior, so we add a check for this, which
prevent user taking the last non-isolated CPU offline.

[   19.415123] WARNING: CPU: 4 PID: 80 at kernel/sched/topology.c:2366 build_sched_domains+0x120c/0x1408
[   19.430451] Call trace:
[   19.430792]  build_sched_domains+0x120c/0x1408
[   19.431334]  partition_sched_domains_locked+0x234/0x880
[   19.431878]  rebuild_sched_domains_locked+0x37c/0x798
[   19.432436]  rebuild_sched_domains+0x30/0x58
[   19.432902]  cpuset_hotplug_workfn+0x2a8/0x930
[   19.433383]  process_scheduled_works+0x178/0x3e0
[   19.433878]  worker_thread+0x174/0x2f0
[   19.435204] ---[ end trace 0000000000000000 ]---
[   19.438650] Unable to handle kernel paging request at virtual address fffe80027ab37080
[   19.456414]  partition_sched_domains_locked+0x318/0x880
[   19.456899]  rebuild_sched_domains_locked+0x37c/0x798
[   19.457361]  rebuild_sched_domains+0x30/0x58
[   19.457761]  cpuset_hotplug_workfn+0x2a8/0x930
[   19.458175]  process_scheduled_works+0x178/0x3e0
[   19.458599]  worker_thread+0x174/0x2f0
[   19.458948]  kthread+0x10c/0x128
[   19.459268]  ret_from_fork+0x10/0x20
[   19.459728] Code: 1a850042 b9441883 f862dae0 8b000021 (f945003b)

Signed-off-by: Ran Xiaokai <ran.xiaokai@zte.com.cn>
Cc: Xuexin Jiang <jiang.xuexin@zte.com.cn>
Cc: Yang Yang <yang.yang29@zte.com.cn>
---
 kernel/cpu.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/kernel/cpu.c b/kernel/cpu.c
index 6de7c6bb74ee..09ecc19d2999 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -1502,6 +1502,7 @@ static long __cpu_down_maps_locked(void *arg)
 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
 {
 	struct cpu_down_work work = { .cpu = cpu, .target = target, };
+	struct cpumask tmp_mask;

 	/*
 	 * If the platform does not support hotplug, report it explicitly to
@@ -1512,11 +1513,16 @@ static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
 	if (cpu_hotplug_disabled)
 		return -EBUSY;

+	/*
+	 * Ensure the last non-isolated CPU is not offlined.
+	 */
+	cpumask_and(&tmp_mask, cpu_online_mask, housekeeping_cpumask(HK_TYPE_DOMAIN));
+
 	/*
 	 * Ensure that the control task does not run on the to be offlined
 	 * CPU to prevent a deadlock against cfs_b->period_timer.
 	 */
-	cpu = cpumask_any_but(cpu_online_mask, cpu);
+	cpu = cpumask_any_but(&tmp_mask, cpu);
 	if (cpu >= nr_cpu_ids)
 		return -EBUSY;
 	return work_on_cpu(cpu, __cpu_down_maps_locked, &work);
-- 
2.15.2

Re: [PATCH] cpu/hotplug: dont offline the last non-isolated CPU

[Thomas Gleixner]

On Sat, Sep 16 2023 at 10:37, yang wrote:
> @@ -1502,6 +1502,7 @@ static long __cpu_down_maps_locked(void *arg)
>  static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
>  {
>  	struct cpu_down_work work = { .cpu = cpu, .target = target, };
> +	struct cpumask tmp_mask;

Allocating a cpumask on stack is not really a good idea as it takes up
to 1K stack space.

>  	/*
>  	 * If the platform does not support hotplug, report it explicitly to
> @@ -1512,11 +1513,16 @@ static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
>  	if (cpu_hotplug_disabled)
>  		return -EBUSY;
>
> +	/*
> +	 * Ensure the last non-isolated CPU is not offlined.
> +	 */
> +	cpumask_and(&tmp_mask, cpu_online_mask, housekeeping_cpumask(HK_TYPE_DOMAIN));

You can spare that excercise. See below.

>  	/*
>  	 * Ensure that the control task does not run on the to be offlined
>  	 * CPU to prevent a deadlock against cfs_b->period_timer.
>  	 */
> -	cpu = cpumask_any_but(cpu_online_mask, cpu);
> +	cpu = cpumask_any_but(&tmp_mask, cpu);

Just open code it this way:

        for_each_cpu_and(cpu, cpu_online_mask, housekeeping_cpumask(HK_TYPE_DOMAIN)) {
        	if (cpu != work.cpu)
  			return work_on_cpu(cpu, __cpu_down_maps_locked, &work);
	}
        return -EBUSY;

Hmm?

Thanks,

        tglx

[PATCH V2] cpu/hotplug: dont offline the last non-isolated CPU

[yang.yang29]

From: Ran Xiaokai <ran.xiaokai@zte.com.cn>

If system has some isolate cpus with "isolcpus=" parameter,
and user try to offline all the non-isolated CPUs, kernel will
first report a warning and then panic when taking the last
non-isolated CPU offline.

When offline the last non-isolate cpu,
cpuset_hotplug_workfn()
  rebuild_sched_domains_locked()
    ndoms = generate_sched_domains(&doms, &attr);
      cpumask_and(doms[0], top_cpuset.effective_cpus,
                  housekeeping_cpumask(HK_FLAG_DOMAIN));

As isolated cpus are excluded, "doms" passes to partition_sched_domains_locked()
will be an empty cpumask.

WARNING: CPU: 4 PID: 80 at kernel/sched/topology.c:2366 build_sched_domains+0x120c/0x1408
Call trace:
 build_sched_domains+0x120c/0x1408
 partition_sched_domains_locked+0x234/0x880
 rebuild_sched_domains_locked+0x37c/0x798
 rebuild_sched_domains+0x30/0x58
 cpuset_hotplug_workfn+0x2a8/0x930
 process_scheduled_works+0x178/0x3e0
 worker_thread+0x174/0x2f0
---[ end trace 0000000000000000 ]---
Unable to handle kernel paging request at virtual address fffe80027ab37080
 partition_sched_domains_locked+0x318/0x880
 rebuild_sched_domains_locked+0x37c/0x798
 rebuild_sched_domains+0x30/0x58
 cpuset_hotplug_workfn+0x2a8/0x930
 process_scheduled_works+0x178/0x3e0
 worker_thread+0x174/0x2f0
 kthread+0x10c/0x128
 ret_from_fork+0x10/0x20
Code: 1a850042 b9441883 f862dae0 8b000021 (f945003b)

Kernel should not panic when user managing CPUs with this
meaningless behavior, so we add a check for this, which
prevent user taking the last non-isolated CPU offline.

Signed-off-by: Ran Xiaokai <ran.xiaokai@zte.com.cn>
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Yang Yang <yang.yang29@zte.com.cn>
Cc: Xuexin Jiang <jiang.xuexin@zte.com.cn>
---
 kernel/cpu.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/kernel/cpu.c b/kernel/cpu.c
index 6de7c6bb74ee..94430ea8b4a5 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -1515,11 +1515,14 @@ static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
 	/*
 	 * Ensure that the control task does not run on the to be offlined
 	 * CPU to prevent a deadlock against cfs_b->period_timer.
+	 * Also keep at least one housekeeping cpu onlined to avoid generating
+	 * an empty sched_domain span.
 	 */
-	cpu = cpumask_any_but(cpu_online_mask, cpu);
-	if (cpu >= nr_cpu_ids)
-		return -EBUSY;
-	return work_on_cpu(cpu, __cpu_down_maps_locked, &work);
+	for_each_cpu_and(cpu, cpu_online_mask, housekeeping_cpumask(HK_TYPE_DOMAIN)) {
+		if (cpu != work.cpu)
+			return work_on_cpu(cpu, __cpu_down_maps_locked, &work);
+	}
+	return -EBUSY;
 }

 static int cpu_down(unsigned int cpu, enum cpuhp_state target)
-- 
2.15.2