[PATCH v3] x86: correct socket_cpumask allocation

[Chao Peng <chao.p.peng@linux.intel.com>]

For booting cpu, the socket number is not needed to be 0 so
it needs to be computed by cpu number.

For secondary cpu, phys_proc_id is not valid in CPU_PREPARE
notifier(cpu_smpboot_alloc), so cpu_to_socket(cpu) can't be used.
Instead, pre-allocate secondary_cpu_mask in cpu_smpboot_alloc()
and later consume it in smp_store_cpu_info().

This patch also change socket_cpumask type from 'cpumask_var_t *'
to 'cpumask_t **' so that smaller NR_CPUS works.

Reported-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Tested-by: Dario Faggioli <dario.faggioli@citrix.com>
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
Changes in v3:
* use type safe xzalloc(cpumask_t)
Changes in v2:
* Fix case that booting cpu is on the socket other than socket0.
* cpumask_var_t => cpumask_t * to make smaller NR_CPUS builds.
---
 xen/arch/x86/smpboot.c    | 25 ++++++++++++++++++-------
 xen/include/asm-x86/smp.h |  2 +-
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index c73aa1b..0f03364 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -61,7 +61,8 @@ cpumask_t cpu_online_map __read_mostly;
 EXPORT_SYMBOL(cpu_online_map);
 
 unsigned int __read_mostly nr_sockets;
-cpumask_var_t *__read_mostly socket_cpumask;
+cpumask_t **__read_mostly socket_cpumask;
+static cpumask_t *secondary_socket_cpumask;
 
 struct cpuinfo_x86 cpu_data[NR_CPUS];
 
@@ -84,11 +85,21 @@ void *stack_base[NR_CPUS];
 static void smp_store_cpu_info(int id)
 {
     struct cpuinfo_x86 *c = cpu_data + id;
+    unsigned int socket;
 
     *c = boot_cpu_data;
     if ( id != 0 )
+    {
         identify_cpu(c);
 
+        socket = cpu_to_socket(id);
+        if ( !socket_cpumask[socket] )
+        {
+            socket_cpumask[socket] = secondary_socket_cpumask;
+            secondary_socket_cpumask = NULL;
+        }
+    }
+
     /*
      * Certain Athlons might work (for various values of 'work') in SMP
      * but they are not certified as MP capable.
@@ -658,7 +669,7 @@ static void cpu_smpboot_free(unsigned int cpu)
 
     if ( cpumask_empty(socket_cpumask[socket]) )
     {
-        free_cpumask_var(socket_cpumask[socket]);
+        xfree(socket_cpumask[socket]);
         socket_cpumask[socket] = NULL;
     }
 
@@ -705,7 +716,6 @@ static int cpu_smpboot_alloc(unsigned int cpu)
     nodeid_t node = cpu_to_node(cpu);
     struct desc_struct *gdt;
     unsigned long stub_page;
-    unsigned int socket = cpu_to_socket(cpu);
 
     if ( node != NUMA_NO_NODE )
         memflags = MEMF_node(node);
@@ -748,8 +758,8 @@ static int cpu_smpboot_alloc(unsigned int cpu)
         goto oom;
     per_cpu(stubs.addr, cpu) = stub_page + STUB_BUF_CPU_OFFS(cpu);
 
-    if ( !socket_cpumask[socket] &&
-         !zalloc_cpumask_var(socket_cpumask + socket) )
+    if ( secondary_socket_cpumask == NULL &&
+         (secondary_socket_cpumask = xzalloc(cpumask_t)) == NULL )
         goto oom;
 
     if ( zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) &&
@@ -804,8 +814,9 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 
     set_nr_sockets();
 
-    socket_cpumask = xzalloc_array(cpumask_var_t, nr_sockets);
-    if ( !socket_cpumask || !zalloc_cpumask_var(socket_cpumask) )
+    socket_cpumask = xzalloc_array(cpumask_t *, nr_sockets);
+    if ( socket_cpumask == NULL ||
+         (socket_cpumask[cpu_to_socket(0)] = xzalloc(cpumask_t)) == NULL )
         panic("No memory for socket CPU siblings map");
 
     if ( !zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, 0)) ||
diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h
index e594062..ea07888 100644
--- a/xen/include/asm-x86/smp.h
+++ b/xen/include/asm-x86/smp.h
@@ -67,7 +67,7 @@ extern unsigned int nr_sockets;
 void set_nr_sockets(void);
 
 /* Representing HT and core siblings in each socket. */
-extern cpumask_var_t *socket_cpumask;
+extern cpumask_t **socket_cpumask;
 
 #endif /* !__ASSEMBLY__ */
 
-- 
1.9.1