[PATCH v4 0/2] Drivers: hv: vmbus: use cpu_hotplug_enable/disable for CPU offlining prevention

[PATCH v4 0/2] Drivers: hv: vmbus: use cpu_hotplug_enable/disable for CPU offlining prevention

[Vitaly Kuznetsov]

Changes since v3:
- add WARN_ON when decreasing cpu_hotplug_disabled [Peter Zijlstra]

Changes since v2:
- Rebase on top of current Greg's char-misc-next tree [K. Y. Srinivasan]

Changes since v1:
- Make cpu_hotplug_disabled a counter [Radim Krčmář]

Export cpu_hotplug_enable/cpu_hotplug_disable functions from cpu.c and use
them instead of altering smp_ops.cpu_disable in Hyper-V vmbus module.

Vitaly Kuznetsov (2):
  cpu-hotplug: export cpu_hotplug_enable/cpu_hotplug_disable
  Drivers: hv: vmbus: use cpu_hotplug_enable/disable

 Documentation/power/suspend-and-cpuhotplug.txt |  6 ++--
 drivers/hv/vmbus_drv.c                         | 38 +++-----------------------
 kernel/cpu.c                                   | 11 ++++----
 3 files changed, 13 insertions(+), 42 deletions(-)

-- 
2.4.2

[PATCH v4 1/2] cpu-hotplug: export cpu_hotplug_enable/cpu_hotplug_disable

[Vitaly Kuznetsov]

Loaded Hyper-V module will use these functions to disable CPU hotplug
under certain circumstances. Convert cpu_hotplug_disabled to a counter
(protected by cpu_add_remove_lock) to support e.g. disable -> disable ->
enable call sequences.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 Documentation/power/suspend-and-cpuhotplug.txt |  6 +++---
 kernel/cpu.c                                   | 11 ++++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/Documentation/power/suspend-and-cpuhotplug.txt b/Documentation/power/suspend-and-cpuhotplug.txt
index 2850df3..2fc9095 100644
--- a/Documentation/power/suspend-and-cpuhotplug.txt
+++ b/Documentation/power/suspend-and-cpuhotplug.txt
@@ -72,7 +72,7 @@ More details follow:
                                         |
                                         v
                            Disable regular cpu hotplug
-                        by setting cpu_hotplug_disabled=1
+                        by increasing cpu_hotplug_disabled
                                         |
                                         v
                             Release cpu_add_remove_lock
@@ -89,7 +89,7 @@ Resuming back is likewise, with the counterparts being (in the order of
 execution during resume):
 * enable_nonboot_cpus() which involves:
    |  Acquire cpu_add_remove_lock
-   |  Reset cpu_hotplug_disabled to 0, thereby enabling regular cpu hotplug
+   |  Decrease cpu_hotplug_disabled, thereby enabling regular cpu hotplug
    |  Call _cpu_up() [for all those cpus in the frozen_cpus mask, in a loop]
    |  Release cpu_add_remove_lock
    v
@@ -120,7 +120,7 @@ after the entire cycle is complete (i.e., suspend + resume).
                            Acquire cpu_add_remove_lock
                                         |
                                         v
-                          If cpu_hotplug_disabled is 1
+                          If cpu_hotplug_disabled > 0
                                 return gracefully
                                         |
                                         |
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 94bbe46..b6e42d2 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -190,17 +190,18 @@ void cpu_hotplug_done(void)
 void cpu_hotplug_disable(void)
 {
 	cpu_maps_update_begin();
-	cpu_hotplug_disabled = 1;
+	cpu_hotplug_disabled++;
 	cpu_maps_update_done();
 }
+EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 
 void cpu_hotplug_enable(void)
 {
 	cpu_maps_update_begin();
-	cpu_hotplug_disabled = 0;
+	WARN_ON(--cpu_hotplug_disabled < 0);
 	cpu_maps_update_done();
 }
-
+EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 #endif	/* CONFIG_HOTPLUG_CPU */
 
 /* Need to know about CPUs going up/down? */
@@ -600,7 +601,7 @@ int disable_nonboot_cpus(void)
 	if (!error) {
 		BUG_ON(num_online_cpus() > 1);
 		/* Make sure the CPUs won't be enabled by someone else */
-		cpu_hotplug_disabled = 1;
+		cpu_hotplug_disabled++;
 	} else {
 		pr_err("Non-boot CPUs are not disabled\n");
 	}
@@ -622,7 +623,7 @@ void __ref enable_nonboot_cpus(void)
 
 	/* Allow everyone to use the CPU hotplug again */
 	cpu_maps_update_begin();
-	cpu_hotplug_disabled = 0;
+	WARN_ON(--cpu_hotplug_disabled < 0);
 	if (cpumask_empty(frozen_cpus))
 		goto out;
 
-- 
2.4.2

Re: [PATCH v4 1/2] cpu-hotplug: export cpu_hotplug_enable/cpu_hotplug_disable

[Vitaly Kuznetsov]

Vitaly Kuznetsov <vkuznets@redhat.com> writes:

> Loaded Hyper-V module will use these functions to disable CPU hotplug
> under certain circumstances. Convert cpu_hotplug_disabled to a counter
> (protected by cpu_add_remove_lock) to support e.g. disable -> disable ->
> enable call sequences.
>
> Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
> ---
>  Documentation/power/suspend-and-cpuhotplug.txt |  6 +++---
>  kernel/cpu.c                                   | 11 ++++++-----
>  2 files changed, 9 insertions(+), 8 deletions(-)
>
> diff --git a/Documentation/power/suspend-and-cpuhotplug.txt b/Documentation/power/suspend-and-cpuhotplug.txt
> index 2850df3..2fc9095 100644
> --- a/Documentation/power/suspend-and-cpuhotplug.txt
> +++ b/Documentation/power/suspend-and-cpuhotplug.txt
> @@ -72,7 +72,7 @@ More details follow:
>                                          |
>                                          v
>                             Disable regular cpu hotplug
> -                        by setting cpu_hotplug_disabled=1
> +                        by increasing cpu_hotplug_disabled
>                                          |
>                                          v
>                              Release cpu_add_remove_lock
> @@ -89,7 +89,7 @@ Resuming back is likewise, with the counterparts being (in the order of
>  execution during resume):
>  * enable_nonboot_cpus() which involves:
>     |  Acquire cpu_add_remove_lock
> -   |  Reset cpu_hotplug_disabled to 0, thereby enabling regular cpu hotplug
> +   |  Decrease cpu_hotplug_disabled, thereby enabling regular cpu hotplug
>     |  Call _cpu_up() [for all those cpus in the frozen_cpus mask, in a loop]
>     |  Release cpu_add_remove_lock
>     v
> @@ -120,7 +120,7 @@ after the entire cycle is complete (i.e., suspend + resume).
>                             Acquire cpu_add_remove_lock
>                                          |
>                                          v
> -                          If cpu_hotplug_disabled is 1
> +                          If cpu_hotplug_disabled > 0
>                                  return gracefully
>                                          |
>                                          |
> diff --git a/kernel/cpu.c b/kernel/cpu.c
> index 94bbe46..b6e42d2 100644
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -190,17 +190,18 @@ void cpu_hotplug_done(void)
>  void cpu_hotplug_disable(void)
>  {
>  	cpu_maps_update_begin();
> -	cpu_hotplug_disabled = 1;
> +	cpu_hotplug_disabled++;
>  	cpu_maps_update_done();
>  }
> +EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
>
>  void cpu_hotplug_enable(void)
>  {
>  	cpu_maps_update_begin();
> -	cpu_hotplug_disabled = 0;
> +	WARN_ON(--cpu_hotplug_disabled < 0);
>  	cpu_maps_update_done();
>  }
> -
> +EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
>  #endif	/* CONFIG_HOTPLUG_CPU */
>
>  /* Need to know about CPUs going up/down? */
> @@ -600,7 +601,7 @@ int disable_nonboot_cpus(void)
>  	if (!error) {
>  		BUG_ON(num_online_cpus() > 1);
>  		/* Make sure the CPUs won't be enabled by someone else */
> -		cpu_hotplug_disabled = 1;
> +		cpu_hotplug_disabled++;

Actually, there is an issue here: all disable_nonboot_cpus() users do
enable_nonboot_cpus() in case of failure (yesterday I sent a patch to
fix the last one: https://lkml.org/lkml/2015/6/24/379) but we do
cpu_hotplug_disabled++ only in case of success. I think we need to do
cpu_hotplug_disabled++ here unconditionally.

I'll fix and resend.


>  	} else {
>  		pr_err("Non-boot CPUs are not disabled\n");
>  	}
> @@ -622,7 +623,7 @@ void __ref enable_nonboot_cpus(void)
>
>  	/* Allow everyone to use the CPU hotplug again */
>  	cpu_maps_update_begin();
> -	cpu_hotplug_disabled = 0;
> +	WARN_ON(--cpu_hotplug_disabled < 0);

(here we don't really know if we had a success or a failure in
previously called disable_nonboot_cpus()).

>  	if (cpumask_empty(frozen_cpus))
>  		goto out;

-- 
  Vitaly

[PATCH v4 2/2] Drivers: hv: vmbus: use cpu_hotplug_enable/disable

[Vitaly Kuznetsov]

Commit e513229b4c38 ("Drivers: hv: vmbus: prevent cpu offlining on newer
hypervisors") was altering smp_ops.cpu_disable to prevent CPU offlining.
We can bo better by using cpu_hotplug_enable/disable functions instead of
such hard-coding.

Reported-by: Radim Krčmář <rkrcmar@redhat.com>
Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
Signed-off-by: K. Y. Srinivasan <kys@microsoft.com>
---
 drivers/hv/vmbus_drv.c | 38 ++++----------------------------------
 1 file changed, 4 insertions(+), 34 deletions(-)

diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index cf20400..6de65fb 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -763,38 +763,6 @@ static void vmbus_isr(void)
 	}
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-static int hyperv_cpu_disable(void)
-{
-	return -ENOSYS;
-}
-
-static void hv_cpu_hotplug_quirk(bool vmbus_loaded)
-{
-	static void *previous_cpu_disable;
-
-	/*
-	 * Offlining a CPU when running on newer hypervisors (WS2012R2, Win8,
-	 * ...) is not supported at this moment as channel interrupts are
-	 * distributed across all of them.
-	 */
-
-	if ((vmbus_proto_version == VERSION_WS2008) ||
-	    (vmbus_proto_version == VERSION_WIN7))
-		return;
-
-	if (vmbus_loaded) {
-		previous_cpu_disable = smp_ops.cpu_disable;
-		smp_ops.cpu_disable = hyperv_cpu_disable;
-		pr_notice("CPU offlining is not supported by hypervisor\n");
-	} else if (previous_cpu_disable)
-		smp_ops.cpu_disable = previous_cpu_disable;
-}
-#else
-static void hv_cpu_hotplug_quirk(bool vmbus_loaded)
-{
-}
-#endif
 
 /*
  * vmbus_bus_init -Main vmbus driver initialization routine.
@@ -836,7 +804,8 @@ static int vmbus_bus_init(int irq)
 	if (ret)
 		goto err_alloc;
 
-	hv_cpu_hotplug_quirk(true);
+	if (vmbus_proto_version > VERSION_WIN7)
+		cpu_hotplug_disable();
 
 	/*
 	 * Only register if the crash MSRs are available
@@ -1121,7 +1090,8 @@ static void __exit vmbus_exit(void)
 		smp_call_function_single(cpu, hv_synic_cleanup, NULL, 1);
 	}
 	acpi_bus_unregister_driver(&vmbus_acpi_driver);
-	hv_cpu_hotplug_quirk(false);
+	if (vmbus_proto_version > VERSION_WIN7)
+		cpu_hotplug_enable();
 }
 
 
-- 
2.4.2