[PATCH 0/3] Octeon SMP initialization and HOTPLUG_CPU fixes.

[PATCH 0/3] Octeon SMP initialization and HOTPLUG_CPU fixes.

[David Daney]

These three basically improve arch/mips/cavium-octeon/smp.c.  The
individual patch logs have the details.

David Daney (3):
  MIPS: Octeon: Clean up SMP CPU numbering.
  MIPS: Octeon: Simplify hotcpu_notifier registration.
  MIPS: Octeon: HOTPLUG_CPU fixes.

 arch/mips/cavium-octeon/octeon_boot.h |   16 ++--
 arch/mips/cavium-octeon/smp.c         |  170 ++++++++++++++++----------------
 arch/mips/include/asm/octeon/octeon.h |    2 +
 3 files changed, 96 insertions(+), 92 deletions(-)

[PATCH 1/3] MIPS: Octeon: Clean up SMP CPU numbering.

[David Daney]

Also number offline CPUs that could potentially be brought on-line
later.

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
---
 arch/mips/cavium-octeon/smp.c |   37 ++++++++++++++++++++++++++++++-------
 1 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c
index 6d99b9d..8ff2c7b 100644
--- a/arch/mips/cavium-octeon/smp.c
+++ b/arch/mips/cavium-octeon/smp.c
@@ -3,7 +3,7 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
- * Copyright (C) 2004-2008 Cavium Networks
+ * Copyright (C) 2004-2008, 2009, 2010 Cavium Networks
  */
 #include <linux/cpu.h>
 #include <linux/init.h>
@@ -102,24 +102,47 @@ static void octeon_smp_setup(void)
 	const int coreid = cvmx_get_core_num();
 	int cpus;
 	int id;
-
 	int core_mask = octeon_get_boot_coremask();
+#ifdef CONFIG_HOTPLUG_CPU
+	unsigned int num_cores = cvmx_octeon_num_cores();
+#endif
+
+	/* The present CPUs are initially just the boot cpu (CPU 0). */
+	for (id = 0; id < NR_CPUS; id++) {
+		set_cpu_possible(id, id == 0);
+		set_cpu_present(id, id == 0);
+	}
 
-	cpus_clear(cpu_possible_map);
 	__cpu_number_map[coreid] = 0;
 	__cpu_logical_map[0] = coreid;
-	cpu_set(0, cpu_possible_map);
 
+	/* The present CPUs get the lowest CPU numbers. */
 	cpus = 1;
-	for (id = 0; id < 16; id++) {
+	for (id = 0; id < NR_CPUS; id++) {
 		if ((id != coreid) && (core_mask & (1 << id))) {
-			cpu_set(cpus, cpu_possible_map);
+			set_cpu_possible(cpus, true);
+			set_cpu_present(cpus, true);
 			__cpu_number_map[id] = cpus;
 			__cpu_logical_map[cpus] = id;
 			cpus++;
 		}
 	}
-	cpu_present_map = cpu_possible_map;
+
+#ifdef CONFIG_HOTPLUG_CPU
+	/*
+	 * The possible CPUs are all those present on the chip.  We
+	 * will assign CPU numbers for possible cores as well.  Cores
+	 * are always consecutively numberd from 0.
+	 */
+	for (id = 0; id < num_cores && id < NR_CPUS; id++) {
+		if (!(core_mask & (1 << id))) {
+			set_cpu_possible(cpus, true);
+			__cpu_number_map[id] = cpus;
+			__cpu_logical_map[cpus] = id;
+			cpus++;
+		}
+	}
+#endif
 
 	octeon_smp_hotplug_setup();
 }
-- 
1.7.1.1

Re: [PATCH 1/3] MIPS: Octeon: Clean up SMP CPU numbering.

[Ralf Baechle]

Queued for 2.6.36.  Thanks,

  Ralf

[PATCH 2/3] MIPS: Octeon: Simplify hotcpu_notifier registration.

[David Daney]

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
---
 arch/mips/cavium-octeon/smp.c |    8 +-------
 1 files changed, 1 insertions(+), 7 deletions(-)

diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c
index 8ff2c7b..ceb7649 100644
--- a/arch/mips/cavium-octeon/smp.c
+++ b/arch/mips/cavium-octeon/smp.c
@@ -428,17 +428,11 @@ static int __cpuinit octeon_cpu_callback(struct notifier_block *nfb,
 	return NOTIFY_OK;
 }
 
-static struct notifier_block __cpuinitdata octeon_cpu_notifier = {
-	.notifier_call = octeon_cpu_callback,
-};
-
 static int __cpuinit register_cavium_notifier(void)
 {
-	register_hotcpu_notifier(&octeon_cpu_notifier);
-
+	hotcpu_notifier(octeon_cpu_callback, 0);
 	return 0;
 }
-
 late_initcall(register_cavium_notifier);
 
 #endif  /* CONFIG_HOTPLUG_CPU */
-- 
1.7.1.1

Re: [PATCH 2/3] MIPS: Octeon: Simplify hotcpu_notifier registration.

[Ralf Baechle]

Queued for 2.6.36.  Thanks,

  Ralf

[PATCH 3/3] MIPS: Octeon: HOTPLUG_CPU fixes.

[David Daney]

* Rename camel-case InitTLBStart_addr to octeon_bootloader_entry_addr.

* Convert calls to cvmx_read64_uint32(), to simple pointer
  dereferences.

* Set proper ebase.

* Don't confuse coreid and cpu numbers.

* Try to maintain consistent bootloader coremask.

* Update the signature and boot_init_vector of supported bootloaders.

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
---
 arch/mips/cavium-octeon/octeon_boot.h |   16 +++--
 arch/mips/cavium-octeon/smp.c         |  125 ++++++++++++++-------------------
 arch/mips/include/asm/octeon/octeon.h |    2 +
 3 files changed, 65 insertions(+), 78 deletions(-)

diff --git a/arch/mips/cavium-octeon/octeon_boot.h b/arch/mips/cavium-octeon/octeon_boot.h
index 0f7f84a..428864b 100644
--- a/arch/mips/cavium-octeon/octeon_boot.h
+++ b/arch/mips/cavium-octeon/octeon_boot.h
@@ -23,14 +23,16 @@
 #include <linux/types.h>
 
 struct boot_init_vector {
-	uint32_t stack_addr;
-	uint32_t code_addr;
+	/* First stage address - in ram instead of flash */
+	uint64_t code_addr;
+	/* Setup code for application, NOT application entry point */
 	uint32_t app_start_func_addr;
+	/* k0 is used for global data - needs to be passed to other cores */
 	uint32_t k0_val;
-	uint32_t flags;
-	uint32_t boot_info_addr;
+	/* Address of boot info block structure */
+	uint64_t boot_info_addr;
+	uint32_t flags;         /* flags */
 	uint32_t pad;
-	uint32_t pad2;
 };
 
 /* similar to bootloader's linux_app_boot_info but without global data */
@@ -40,7 +42,7 @@ struct linux_app_boot_info {
 	uint32_t avail_coremask;
 	uint32_t pci_console_active;
 	uint32_t icache_prefetch_disable;
-	uint32_t InitTLBStart_addr;
+	uint64_t InitTLBStart_addr;
 	uint32_t start_app_addr;
 	uint32_t cur_exception_base;
 	uint32_t no_mark_private_data;
@@ -58,7 +60,7 @@ struct linux_app_boot_info {
 
 #define LINUX_APP_BOOT_BLOCK_NAME "linux-app-boot"
 
-#define LABI_SIGNATURE 0xAABBCCDD
+#define LABI_SIGNATURE 0xAABBCC01
 
 /*  from uboot-headers/octeon_mem_map.h */
 #define EXCEPTION_BASE_INCR     (4 * 1024)
diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c
index ceb7649..183e641 100644
--- a/arch/mips/cavium-octeon/smp.c
+++ b/arch/mips/cavium-octeon/smp.c
@@ -27,7 +27,8 @@ volatile unsigned long octeon_processor_sp;
 volatile unsigned long octeon_processor_gp;
 
 #ifdef CONFIG_HOTPLUG_CPU
-static unsigned int InitTLBStart_addr;
+uint64_t octeon_bootloader_entry_addr;
+EXPORT_SYMBOL(octeon_bootloader_entry_addr);
 #endif
 
 static irqreturn_t mailbox_interrupt(int irq, void *dev_id)
@@ -80,20 +81,13 @@ static inline void octeon_send_ipi_mask(const struct cpumask *mask,
 static void octeon_smp_hotplug_setup(void)
 {
 #ifdef CONFIG_HOTPLUG_CPU
-	uint32_t labi_signature;
-
-	labi_signature =
-		cvmx_read64_uint32(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-					LABI_ADDR_IN_BOOTLOADER +
-					offsetof(struct linux_app_boot_info,
-						    labi_signature)));
-	if (labi_signature != LABI_SIGNATURE)
-		pr_err("The bootloader version on this board is incorrect\n");
-	InitTLBStart_addr =
-		cvmx_read64_uint32(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-				   LABI_ADDR_IN_BOOTLOADER +
-					   offsetof(struct linux_app_boot_info,
-						    InitTLBStart_addr)));
+	struct linux_app_boot_info *labi;
+
+	labi = (struct linux_app_boot_info *)PHYS_TO_XKSEG_CACHED(LABI_ADDR_IN_BOOTLOADER);
+	if (labi->labi_signature != LABI_SIGNATURE)
+		panic("The bootloader version on this board is incorrect.");
+
+	octeon_bootloader_entry_addr = labi->InitTLBStart_addr;
 #endif
 }
 
@@ -181,18 +175,21 @@ static void octeon_init_secondary(void)
 {
 	const int coreid = cvmx_get_core_num();
 	union cvmx_ciu_intx_sum0 interrupt_enable;
+	unsigned int sr;
 
 #ifdef CONFIG_HOTPLUG_CPU
-	unsigned int cur_exception_base;
-
-	cur_exception_base = cvmx_read64_uint32(
-		CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-			     LABI_ADDR_IN_BOOTLOADER +
-			     offsetof(struct linux_app_boot_info,
-				      cur_exception_base)));
-	/* cur_exception_base is incremented in bootloader after setting */
-	write_c0_ebase((unsigned int)(cur_exception_base - EXCEPTION_BASE_INCR));
+	struct linux_app_boot_info *labi;
+
+	labi = (struct linux_app_boot_info *)PHYS_TO_XKSEG_CACHED(LABI_ADDR_IN_BOOTLOADER);
+
+	if (labi->labi_signature != LABI_SIGNATURE)
+		panic("The bootloader version on this board is incorrect.");
 #endif
+
+	sr = set_c0_status(ST0_BEV);
+	write_c0_ebase((u32)ebase);
+	write_c0_status(sr);
+
 	octeon_check_cpu_bist();
 	octeon_init_cvmcount();
 	/*
@@ -299,8 +296,8 @@ static int octeon_cpu_disable(void)
 static void octeon_cpu_die(unsigned int cpu)
 {
 	int coreid = cpu_logical_map(cpu);
-	uint32_t avail_coremask;
-	struct cvmx_bootmem_named_block_desc *block_desc;
+	uint32_t mask, new_mask;
+	const struct cvmx_bootmem_named_block_desc *block_desc;
 
 	while (per_cpu(cpu_state, cpu) != CPU_DEAD)
 		cpu_relax();
@@ -309,52 +306,40 @@ static void octeon_cpu_die(unsigned int cpu)
 	 * This is a bit complicated strategics of getting/settig available
 	 * cores mask, copied from bootloader
 	 */
+
+	mask = 1 << coreid;
 	/* LINUX_APP_BOOT_BLOCK is initialized in bootoct binary */
 	block_desc = cvmx_bootmem_find_named_block(LINUX_APP_BOOT_BLOCK_NAME);
 
 	if (!block_desc) {
-		avail_coremask =
-			cvmx_read64_uint32(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-						   LABI_ADDR_IN_BOOTLOADER +
-						   offsetof
-						   (struct linux_app_boot_info,
-						    avail_coremask)));
-	} else {		       /* alternative, already initialized */
-	       avail_coremask =
-		   cvmx_read64_uint32(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-						   block_desc->base_addr +
-						  AVAIL_COREMASK_OFFSET_IN_LINUX_APP_BOOT_BLOCK));
-	}
+		struct linux_app_boot_info *labi;
 
-	avail_coremask |= 1 << coreid;
+		labi = (struct linux_app_boot_info *)PHYS_TO_XKSEG_CACHED(LABI_ADDR_IN_BOOTLOADER);
 
-	/* Setting avail_coremask for bootoct binary */
-	if (!block_desc) {
-		cvmx_write64_uint32(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-						LABI_ADDR_IN_BOOTLOADER +
-						offsetof(struct linux_app_boot_info,
-							 avail_coremask)),
-				   avail_coremask);
-	} else {
-		cvmx_write64_uint32(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-						block_desc->base_addr +
-						AVAIL_COREMASK_OFFSET_IN_LINUX_APP_BOOT_BLOCK),
-				   avail_coremask);
+		labi->avail_coremask |= mask;
+		new_mask = labi->avail_coremask;
+	} else {		       /* alternative, already initialized */
+		uint32_t *p = (uint32_t *)PHYS_TO_XKSEG_CACHED(block_desc->base_addr +
+							       AVAIL_COREMASK_OFFSET_IN_LINUX_APP_BOOT_BLOCK);
+		*p |= mask;
+		new_mask = *p;
 	}
 
-	pr_info("Reset core %d. Available Coremask = %x\n", coreid,
-		avail_coremask);
+	pr_info("Reset core %d. Available Coremask = 0x%x \n", coreid, new_mask);
+	mb();
 	cvmx_write_csr(CVMX_CIU_PP_RST, 1 << coreid);
 	cvmx_write_csr(CVMX_CIU_PP_RST, 0);
 }
 
 void play_dead(void)
 {
-	int coreid = cvmx_get_core_num();
+	int cpu = cpu_number_map(cvmx_get_core_num());
 
 	idle_task_exit();
 	octeon_processor_boot = 0xff;
-	per_cpu(cpu_state, coreid) = CPU_DEAD;
+	per_cpu(cpu_state, cpu) = CPU_DEAD;
+
+	mb();
 
 	while (1)	/* core will be reset here */
 		;
@@ -367,29 +352,27 @@ static void start_after_reset(void)
 	kernel_entry(0, 0, 0);  /* set a2 = 0 for secondary core */
 }
 
-int octeon_update_boot_vector(unsigned int cpu)
+static int octeon_update_boot_vector(unsigned int cpu)
 {
 
 	int coreid = cpu_logical_map(cpu);
-	unsigned int avail_coremask;
-	struct cvmx_bootmem_named_block_desc *block_desc;
+	uint32_t avail_coremask;
+	const struct cvmx_bootmem_named_block_desc *block_desc;
 	struct boot_init_vector *boot_vect =
-		(struct boot_init_vector *) cvmx_phys_to_ptr(0x0 +
-						  BOOTLOADER_BOOT_VECTOR);
+		(struct boot_init_vector *)PHYS_TO_XKSEG_CACHED(BOOTLOADER_BOOT_VECTOR);
 
 	block_desc = cvmx_bootmem_find_named_block(LINUX_APP_BOOT_BLOCK_NAME);
 
 	if (!block_desc) {
-		avail_coremask =
-			cvmx_read64_uint32(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-					   LABI_ADDR_IN_BOOTLOADER +
-						offsetof(struct linux_app_boot_info,
-						avail_coremask)));
+		struct linux_app_boot_info *labi;
+
+		labi = (struct linux_app_boot_info *)PHYS_TO_XKSEG_CACHED(LABI_ADDR_IN_BOOTLOADER);
+
+        	avail_coremask = labi->avail_coremask;
+		labi->avail_coremask &= ~(1 << coreid);
 	} else {		       /* alternative, already initialized */
-	       avail_coremask =
-		   cvmx_read64_uint32(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
-						   block_desc->base_addr +
-						   AVAIL_COREMASK_OFFSET_IN_LINUX_APP_BOOT_BLOCK));
+		avail_coremask = *(uint32_t *)PHYS_TO_XKSEG_CACHED(
+			block_desc->base_addr + AVAIL_COREMASK_OFFSET_IN_LINUX_APP_BOOT_BLOCK);
 	}
 
 	if (!(avail_coremask & (1 << coreid))) {
@@ -400,9 +383,9 @@ int octeon_update_boot_vector(unsigned int cpu)
 
 	boot_vect[coreid].app_start_func_addr =
 		(uint32_t) (unsigned long) start_after_reset;
-	boot_vect[coreid].code_addr = InitTLBStart_addr;
+	boot_vect[coreid].code_addr = octeon_bootloader_entry_addr;
 
-	CVMX_SYNC;
+	mb();
 
 	cvmx_write_csr(CVMX_CIU_NMI, (1 << coreid) & avail_coremask);
 
diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h
index 8a3eb3b..917a6c4 100644
--- a/arch/mips/include/asm/octeon/octeon.h
+++ b/arch/mips/include/asm/octeon/octeon.h
@@ -254,4 +254,6 @@ static inline uint32_t octeon_npi_read32(uint64_t address)
 
 extern struct cvmx_bootinfo *octeon_bootinfo;
 
+extern uint64_t octeon_bootloader_entry_addr;
+
 #endif /* __ASM_OCTEON_OCTEON_H */
-- 
1.7.1.1

Re: [PATCH 3/3] MIPS: Octeon: HOTPLUG_CPU fixes.

[Ralf Baechle]

Queued for 2.6.36.  Thanks,

  Ralf