[PATCH v6 0/9] MIPS: Support I6500 multi-cluster configuration

[PATCH v6 0/9] MIPS: Support I6500 multi-cluster configuration

[Aleksandar Rikalo]

Taken from Paul Burton MIPS repo with minor changes from Chao-ying Fu.
Tested with 64r6el_defconfig on Boston board in 2 cluster/2 VPU and
1 cluster/4 VPU configurations.

v6:
 - Re-base onto the master branch, with no functionality impact.
 - Correct the issue reported by the kernel test robot.

v5:
 - Drop FDC related changes (patches 12, 13, and 14).
 - Apply changes suggested by Thomas Gleixner (patches 3 and 4).
 - Add #include <linux/cpumask.h> to patch 1, suggested by Thomas Bogendoerfer.
 - Add Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> for the patch 08/11.
 - Add Tested-by: Serge Semin <fancer.lancer@gmail.com> for the entire series.
 - Correct some commit messages.

v4:
 - Re-base onto the master branch, with no functionality impact.
 - Refactor MIPS FDC driver in the context of multicluster support.

v3:
 - Add Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com> for the patch 02/12.
 - Add the changes requested by Marc Zyngier for the 3/12 patch.
 - Remove the patch 11/12 (a consequence of a discussion between Jiaxun Yang
   and Marc Zyngier.
 - Re-base onto the master branch, with no functionality impact.

v2:
 - Apply correct Signed-off-by to avoid confusion.

Chao-ying Fu (1):
  irqchip/mips-gic: Setup defaults in each cluster

Paul Burton (8):
  irqchip/mips-gic: Introduce for_each_online_cpu_gic()
  irqchip/mips-gic: Support multi-cluster in for_each_online_cpu_gic()
  irqchip/mips-gic: Multi-cluster support
  clocksource: mips-gic-timer: Always use cluster 0 counter as
    clocksource
  clocksource: mips-gic-timer: Enable counter when CPUs start
  MIPS: pm-cps: Use per-CPU variables as per-CPU, not per-core
  MIPS: CPS: Introduce struct cluster_boot_config
  MIPS: CPS: Boot CPUs in secondary clusters

 arch/mips/include/asm/mips-cm.h      |  18 ++
 arch/mips/include/asm/smp-cps.h      |   7 +-
 arch/mips/kernel/asm-offsets.c       |   3 +
 arch/mips/kernel/cps-vec.S           |  19 +-
 arch/mips/kernel/mips-cm.c           |   4 +-
 arch/mips/kernel/pm-cps.c            |  35 ++--
 arch/mips/kernel/smp-cps.c           | 285 ++++++++++++++++++++++-----
 drivers/clocksource/mips-gic-timer.c |  45 ++++-
 drivers/irqchip/Kconfig              |   1 +
 drivers/irqchip/irq-mips-gic.c       | 257 ++++++++++++++++++++----
 10 files changed, 560 insertions(+), 114 deletions(-)

-- 
2.25.1

[PATCH v6 1/9] irqchip/mips-gic: Introduce for_each_online_cpu_gic()

[Aleksandar Rikalo]

From: Paul Burton <paulburton@kernel.org>

Parts of code in the MIPS GIC driver operate on the GIC local register
block for each online CPU, accessing each via the GIC's other/redirect
register block.

Abstract the process of iterating over online CPUs & configuring the
other/redirect region to access their registers through a new
for_each_online_cpu_gic() macro.

Signed-off-by: Paul Burton <paulburton@kernel.org>
Signed-off-by: Chao-ying Fu <cfu@wavecomp.com>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 drivers/irqchip/irq-mips-gic.c | 59 +++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 18 deletions(-)

diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index 76253e864f23..6c7a7d2f0438 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -66,6 +66,44 @@ static struct gic_all_vpes_chip_data {
 	bool	mask;
 } gic_all_vpes_chip_data[GIC_NUM_LOCAL_INTRS];
 
+static int __gic_with_next_online_cpu(int prev)
+{
+	unsigned int cpu;
+
+	/* Discover the next online CPU */
+	cpu = cpumask_next(prev, cpu_online_mask);
+
+	/* If there isn't one, we're done */
+	if (cpu >= nr_cpu_ids)
+		return cpu;
+
+	/*
+	 * Move the access lock to the next CPU's GIC local register block.
+	 *
+	 * Set GIC_VL_OTHER. Since the caller holds gic_lock nothing can
+	 * clobber the written value.
+	 */
+	write_gic_vl_other(mips_cm_vp_id(cpu));
+
+	return cpu;
+}
+
+/**
+ * for_each_online_cpu_gic() - Iterate over online CPUs, access local registers
+ * @cpu: An integer variable to hold the current CPU number
+ * @gic_lock: A pointer to raw spin lock used as a guard
+ *
+ * Iterate over online CPUs & configure the other/redirect register region to
+ * access each CPUs GIC local register block, which can be accessed from the
+ * loop body using read_gic_vo_*() or write_gic_vo_*() accessor functions or
+ * their derivatives.
+ */
+#define for_each_online_cpu_gic(cpu, gic_lock)		\
+	guard(raw_spinlock_irqsave)(gic_lock);		\
+	for ((cpu) = __gic_with_next_online_cpu(-1);	\
+	     (cpu) < nr_cpu_ids;			\
+	     (cpu) = __gic_with_next_online_cpu(cpu))
+
 static void gic_clear_pcpu_masks(unsigned int intr)
 {
 	unsigned int i;
@@ -350,37 +388,27 @@ static struct irq_chip gic_local_irq_controller = {
 static void gic_mask_local_irq_all_vpes(struct irq_data *d)
 {
 	struct gic_all_vpes_chip_data *cd;
-	unsigned long flags;
 	int intr, cpu;
 
 	intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 	cd = irq_data_get_irq_chip_data(d);
 	cd->mask = false;
 
-	raw_spin_lock_irqsave(&gic_lock, flags);
-	for_each_online_cpu(cpu) {
-		write_gic_vl_other(mips_cm_vp_id(cpu));
+	for_each_online_cpu_gic(cpu, &gic_lock)
 		write_gic_vo_rmask(BIT(intr));
-	}
-	raw_spin_unlock_irqrestore(&gic_lock, flags);
 }
 
 static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
 {
 	struct gic_all_vpes_chip_data *cd;
-	unsigned long flags;
 	int intr, cpu;
 
 	intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 	cd = irq_data_get_irq_chip_data(d);
 	cd->mask = true;
 
-	raw_spin_lock_irqsave(&gic_lock, flags);
-	for_each_online_cpu(cpu) {
-		write_gic_vl_other(mips_cm_vp_id(cpu));
+	for_each_online_cpu_gic(cpu, &gic_lock)
 		write_gic_vo_smask(BIT(intr));
-	}
-	raw_spin_unlock_irqrestore(&gic_lock, flags);
 }
 
 static void gic_all_vpes_irq_cpu_online(void)
@@ -469,7 +497,6 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
 			      irq_hw_number_t hwirq)
 {
 	struct gic_all_vpes_chip_data *cd;
-	unsigned long flags;
 	unsigned int intr;
 	int err, cpu;
 	u32 map;
@@ -533,12 +560,8 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
 	if (!gic_local_irq_is_routable(intr))
 		return -EPERM;
 
-	raw_spin_lock_irqsave(&gic_lock, flags);
-	for_each_online_cpu(cpu) {
-		write_gic_vl_other(mips_cm_vp_id(cpu));
+	for_each_online_cpu_gic(cpu, &gic_lock)
 		write_gic_vo_map(mips_gic_vx_map_reg(intr), map);
-	}
-	raw_spin_unlock_irqrestore(&gic_lock, flags);
 
 	return 0;
 }
-- 
2.25.1

[PATCH v6 2/9] irqchip/mips-gic: Support multi-cluster in for_each_online_cpu_gic()

[Aleksandar Rikalo]

From: Paul Burton <paulburton@kernel.org>

Use CM's GCR_CL_REDIRECT register to access registers in remote clusters,
so users of gic_with_each_online_cpu() gain support for multi-cluster with
no further changes.

Signed-off-by: Paul Burton <paulburton@kernel.org>
Signed-off-by: Chao-ying Fu <cfu@wavecomp.com>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 drivers/irqchip/irq-mips-gic.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index 6c7a7d2f0438..29bdfdce2123 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -88,6 +88,12 @@ static int __gic_with_next_online_cpu(int prev)
 	return cpu;
 }
 
+static inline void gic_unlock_cluster(void)
+{
+	if (mips_cps_multicluster_cpus())
+		mips_cm_unlock_other();
+}
+
 /**
  * for_each_online_cpu_gic() - Iterate over online CPUs, access local registers
  * @cpu: An integer variable to hold the current CPU number
@@ -102,6 +108,7 @@ static int __gic_with_next_online_cpu(int prev)
 	guard(raw_spinlock_irqsave)(gic_lock);		\
 	for ((cpu) = __gic_with_next_online_cpu(-1);	\
 	     (cpu) < nr_cpu_ids;			\
+	     gic_unlock_cluster(),			\
 	     (cpu) = __gic_with_next_online_cpu(cpu))
 
 static void gic_clear_pcpu_masks(unsigned int intr)
-- 
2.25.1

[PATCH v6 3/9] irqchip/mips-gic: Setup defaults in each cluster

[Aleksandar Rikalo]

From: Chao-ying Fu <cfu@wavecomp.com>

In multi-cluster MIPS I6500 systems, there is a GIC per cluster.
The default shared interrupt setup configured in gic_of_init() will only
apply to the GIC in the cluster containing the boot CPU, leaving the GICs
of other clusters unconfigured.

Similarly configure other clusters.

Signed-off-by: Chao-ying Fu <cfu@wavecomp.com>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 drivers/irqchip/irq-mips-gic.c | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index 29bdfdce2123..d93a076620c7 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -764,7 +764,7 @@ static int gic_cpu_startup(unsigned int cpu)
 static int __init gic_of_init(struct device_node *node,
 			      struct device_node *parent)
 {
-	unsigned int cpu_vec, i, gicconfig;
+	unsigned int cpu_vec, i, gicconfig, cl, nclusters;
 	unsigned long reserved;
 	phys_addr_t gic_base;
 	struct resource res;
@@ -845,11 +845,29 @@ static int __init gic_of_init(struct device_node *node,
 
 	board_bind_eic_interrupt = &gic_bind_eic_interrupt;
 
-	/* Setup defaults */
-	for (i = 0; i < gic_shared_intrs; i++) {
-		change_gic_pol(i, GIC_POL_ACTIVE_HIGH);
-		change_gic_trig(i, GIC_TRIG_LEVEL);
-		write_gic_rmask(i);
+	/*
+	 * Initialise each cluster's GIC shared registers to sane default
+	 * values.
+	 * Otherwise, the IPI set up will be erased if we move code
+	 * to gic_cpu_startup for each cpu.
+	 */
+	nclusters = mips_cps_numclusters();
+	for (cl = 0; cl < nclusters; cl++) {
+		if (cl == cpu_cluster(&current_cpu_data)) {
+			for (i = 0; i < gic_shared_intrs; i++) {
+				change_gic_pol(i, GIC_POL_ACTIVE_HIGH);
+				change_gic_trig(i, GIC_TRIG_LEVEL);
+				write_gic_rmask(i);
+			}
+		} else {
+			mips_cm_lock_other(cl, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+			for (i = 0; i < gic_shared_intrs; i++) {
+				change_gic_redir_pol(i, GIC_POL_ACTIVE_HIGH);
+				change_gic_redir_trig(i, GIC_TRIG_LEVEL);
+				write_gic_redir_rmask(i);
+			}
+			mips_cm_unlock_other();
+		}
 	}
 
 	return cpuhp_setup_state(CPUHP_AP_IRQ_MIPS_GIC_STARTING,
-- 
2.25.1

[PATCH v6 4/9] irqchip/mips-gic: Multi-cluster support

[Aleksandar Rikalo]

From: Paul Burton <paulburton@kernel.org>

The MIPS I6500 CPU & CM (Coherence Manager) 3.5 introduce the concept of
multiple clusters to the system. In these systems, each cluster contains
its own GIC, so the GIC isn't truly global any longer. Access to
registers in the GICs of remote clusters is possible using a redirect
register block much like the redirect register blocks provided by the
CM & CPC, and configured through the same GCR_REDIRECT register that
mips_cm_lock_other() abstraction builds upon.

It is expected that external interrupts are connected identically on all
clusters. That is, if there is a device providing an interrupt connected
to GIC interrupt pin 0 then it should be connected to pin 0 of every GIC
in the system. For the most part, the GIC can be treated as though it is
still truly global, so long as interrupts in the cluster are configured
properly.

This patch introduces support for such multi-cluster systems in the
MIPS GIC irqchip driver. A newly introduced gic_irq_lock_cluster()
function allows:

  1) Configure access to a GIC in a remote cluster via the redirect
     register block, using mips_cm_lock_other().

Or:

  2) Detect that the interrupt in question is affine to the local
     cluster and plain old GIC register access to the GIC in the
     local cluster should be used.

It is possible to access the local cluster's GIC registers via the
redirect block, but keeping the special case for them is both good for
performance (because we avoid the locking & indirection overhead of
using the redirect block) and necessary to maintain compatibility with
systems using CM revisions prior to 3.5 which don't support the redirect
block.

The gic_irq_lock_cluster() function relies upon an IRQs effective
affinity in order to discover which cluster the IRQ is affine to. In
order to track this & allow it to be updated at an appropriate point
during gic_set_affinity() we select the generic support for effective
affinity using CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK.

gic_set_affinity() is the one function which gains much complexity. It
now deconfigures routing to any VP(E), ie. CPU, on the old cluster when
moving affinity to a new cluster.

gic_shared_irq_domain_map() moves its update of the IRQs effective
affinity to before its use of gic_irq_lock_cluster(), to ensure that
operation is on the cluster the IRQ is affine to.

The remaining changes are straightforward use of the
gic_irq_lock_cluster() function to select between local cluster & remote
cluster code-paths when configuring interrupts.

Signed-off-by: Paul Burton <paulburton@kernel.org>
Signed-off-by: Chao-ying Fu <cfu@wavecomp.com>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 drivers/irqchip/Kconfig        |   1 +
 drivers/irqchip/irq-mips-gic.c | 161 +++++++++++++++++++++++++++++----
 2 files changed, 143 insertions(+), 19 deletions(-)

diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index d078bdc48c38..455e512d5f45 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -345,6 +345,7 @@ config KEYSTONE_IRQ
 
 config MIPS_GIC
 	bool
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
 	select GENERIC_IRQ_IPI if SMP
 	select IRQ_DOMAIN_HIERARCHY
 	select MIPS_CM
diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index d93a076620c7..f42f69bbd6fb 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -111,6 +111,41 @@ static inline void gic_unlock_cluster(void)
 	     gic_unlock_cluster(),			\
 	     (cpu) = __gic_with_next_online_cpu(cpu))
 
+/**
+ * gic_irq_lock_cluster() - Lock redirect block access to IRQ's cluster
+ * @d: struct irq_data corresponding to the interrupt we're interested in
+ *
+ * Locks redirect register block access to the global register block of the GIC
+ * within the remote cluster that the IRQ corresponding to @d is affine to,
+ * returning true when this redirect block setup & locking has been performed.
+ *
+ * If @d is affine to the local cluster then no locking is performed and this
+ * function will return false, indicating to the caller that it should access
+ * the local clusters registers without the overhead of indirection through the
+ * redirect block.
+ *
+ * In summary, if this function returns true then the caller should access GIC
+ * registers using redirect register block accessors & then call
+ * mips_cm_unlock_other() when done. If this function returns false then the
+ * caller should trivially access GIC registers in the local cluster.
+ *
+ * Returns true if locking performed, else false.
+ */
+static bool gic_irq_lock_cluster(struct irq_data *d)
+{
+	unsigned int cpu, cl;
+
+	cpu = cpumask_first(irq_data_get_effective_affinity_mask(d));
+	BUG_ON(cpu >= NR_CPUS);
+
+	cl = cpu_cluster(&cpu_data[cpu]);
+	if (cl == cpu_cluster(&current_cpu_data))
+		return false;
+
+	mips_cm_lock_other(cl, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+	return true;
+}
+
 static void gic_clear_pcpu_masks(unsigned int intr)
 {
 	unsigned int i;
@@ -157,7 +192,12 @@ static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
 {
 	irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
 
-	write_gic_wedge(GIC_WEDGE_RW | hwirq);
+	if (gic_irq_lock_cluster(d)) {
+		write_gic_redir_wedge(GIC_WEDGE_RW | hwirq);
+		mips_cm_unlock_other();
+	} else {
+		write_gic_wedge(GIC_WEDGE_RW | hwirq);
+	}
 }
 
 int gic_get_c0_compare_int(void)
@@ -225,7 +265,13 @@ static void gic_mask_irq(struct irq_data *d)
 {
 	unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq);
 
-	write_gic_rmask(intr);
+	if (gic_irq_lock_cluster(d)) {
+		write_gic_redir_rmask(intr);
+		mips_cm_unlock_other();
+	} else {
+		write_gic_rmask(intr);
+	}
+
 	gic_clear_pcpu_masks(intr);
 }
 
@@ -234,7 +280,12 @@ static void gic_unmask_irq(struct irq_data *d)
 	unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq);
 	unsigned int cpu;
 
-	write_gic_smask(intr);
+	if (gic_irq_lock_cluster(d)) {
+		write_gic_redir_smask(intr);
+		mips_cm_unlock_other();
+	} else {
+		write_gic_smask(intr);
+	}
 
 	gic_clear_pcpu_masks(intr);
 	cpu = cpumask_first(irq_data_get_effective_affinity_mask(d));
@@ -245,7 +296,12 @@ static void gic_ack_irq(struct irq_data *d)
 {
 	unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 
-	write_gic_wedge(irq);
+	if (gic_irq_lock_cluster(d)) {
+		write_gic_redir_wedge(irq);
+		mips_cm_unlock_other();
+	} else {
+		write_gic_wedge(irq);
+	}
 }
 
 static int gic_set_type(struct irq_data *d, unsigned int type)
@@ -285,9 +341,16 @@ static int gic_set_type(struct irq_data *d, unsigned int type)
 		break;
 	}
 
-	change_gic_pol(irq, pol);
-	change_gic_trig(irq, trig);
-	change_gic_dual(irq, dual);
+	if (gic_irq_lock_cluster(d)) {
+		change_gic_redir_pol(irq, pol);
+		change_gic_redir_trig(irq, trig);
+		change_gic_redir_dual(irq, dual);
+		mips_cm_unlock_other();
+	} else {
+		change_gic_pol(irq, pol);
+		change_gic_trig(irq, trig);
+		change_gic_dual(irq, dual);
+	}
 
 	if (trig == GIC_TRIG_EDGE)
 		irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
@@ -305,25 +368,72 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
 			    bool force)
 {
 	unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
+	unsigned int cpu, cl, old_cpu, old_cl;
 	unsigned long flags;
-	unsigned int cpu;
 
+	/*
+	 * The GIC specifies that we can only route an interrupt to one VP(E),
+	 * ie. CPU in Linux parlance, at a time. Therefore we always route to
+	 * the first online CPU in the mask.
+	 */
 	cpu = cpumask_first_and(cpumask, cpu_online_mask);
 	if (cpu >= NR_CPUS)
 		return -EINVAL;
 
-	/* Assumption : cpumask refers to a single CPU */
-	raw_spin_lock_irqsave(&gic_lock, flags);
+	old_cpu = cpumask_first(irq_data_get_effective_affinity_mask(d));
+	old_cl = cpu_cluster(&cpu_data[old_cpu]);
+	cl = cpu_cluster(&cpu_data[cpu]);
 
-	/* Re-route this IRQ */
-	write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu)));
+	raw_spin_lock_irqsave(&gic_lock, flags);
 
-	/* Update the pcpu_masks */
-	gic_clear_pcpu_masks(irq);
-	if (read_gic_mask(irq))
-		set_bit(irq, per_cpu_ptr(pcpu_masks, cpu));
+	/*
+	 * If we're moving affinity between clusters, stop routing the
+	 * interrupt to any VP(E) in the old cluster.
+	 */
+	if (cl != old_cl) {
+		if (gic_irq_lock_cluster(d)) {
+			write_gic_redir_map_vp(irq, 0);
+			mips_cm_unlock_other();
+		} else {
+			write_gic_map_vp(irq, 0);
+		}
+	}
 
+	/*
+	 * Update effective affinity - after this gic_irq_lock_cluster() will
+	 * begin operating on the new cluster.
+	 */
 	irq_data_update_effective_affinity(d, cpumask_of(cpu));
+
+	/*
+	 * If we're moving affinity between clusters, configure the interrupt
+	 * trigger type in the new cluster.
+	 */
+	if (cl != old_cl)
+		gic_set_type(d, irqd_get_trigger_type(d));
+
+	/* Route the interrupt to its new VP(E) */
+	if (gic_irq_lock_cluster(d)) {
+		write_gic_redir_map_pin(irq,
+					GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+		write_gic_redir_map_vp(irq, BIT(mips_cm_vp_id(cpu)));
+
+		/* Update the pcpu_masks */
+		gic_clear_pcpu_masks(irq);
+		if (read_gic_redir_mask(irq))
+			set_bit(irq, per_cpu_ptr(pcpu_masks, cpu));
+
+		mips_cm_unlock_other();
+	} else {
+		write_gic_map_pin(irq, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+		write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu)));
+
+		/* Update the pcpu_masks */
+		gic_clear_pcpu_masks(irq);
+		if (read_gic_mask(irq))
+			set_bit(irq, per_cpu_ptr(pcpu_masks, cpu));
+	}
+
 	raw_spin_unlock_irqrestore(&gic_lock, flags);
 
 	return IRQ_SET_MASK_OK;
@@ -471,11 +581,21 @@ static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
 	unsigned long flags;
 
 	data = irq_get_irq_data(virq);
+	irq_data_update_effective_affinity(data, cpumask_of(cpu));
 
 	raw_spin_lock_irqsave(&gic_lock, flags);
-	write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
-	write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu)));
-	irq_data_update_effective_affinity(data, cpumask_of(cpu));
+
+	/* Route the interrupt to its VP(E) */
+	if (gic_irq_lock_cluster(data)) {
+		write_gic_redir_map_pin(intr,
+					GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+		write_gic_redir_map_vp(intr, BIT(mips_cm_vp_id(cpu)));
+		mips_cm_unlock_other();
+	} else {
+		write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+		write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu)));
+	}
+
 	raw_spin_unlock_irqrestore(&gic_lock, flags);
 
 	return 0;
@@ -651,6 +771,9 @@ static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
 		if (ret)
 			goto error;
 
+		/* Set affinity to cpu.  */
+		irq_data_update_effective_affinity(irq_get_irq_data(virq + i),
+						   cpumask_of(cpu));
 		ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
 		if (ret)
 			goto error;
-- 
2.25.1

[PATCH v6 5/9] clocksource: mips-gic-timer: Always use cluster 0 counter as clocksource

[Aleksandar Rikalo]

From: Paul Burton <paulburton@kernel.org>

In a multi-cluster MIPS system, there are multiple GICs - one in each
cluster - each of which has its independent counter. The counters in
each GIC are not synchronized in any way, so they can drift relative
to one another through the lifetime of the system. This is problematic
for a clock source which ought to be global.

Avoid problems by always accessing cluster 0's counter, using
cross-cluster register access. This adds overhead so it is applied only
on multi-cluster systems.

Signed-off-by: Paul Burton <paulburton@kernel.org>
Signed-off-by: Chao-ying Fu <cfu@wavecomp.com>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 drivers/clocksource/mips-gic-timer.c | 39 +++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index 110347707ff9..7907b740497a 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -166,6 +166,37 @@ static u64 gic_hpt_read(struct clocksource *cs)
 	return gic_read_count();
 }
 
+static u64 gic_hpt_read_multicluster(struct clocksource *cs)
+{
+	unsigned int hi, hi2, lo;
+	u64 count;
+
+	mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+
+	if (mips_cm_is64) {
+		count = read_gic_redir_counter();
+		goto out;
+	}
+
+	hi = read_gic_redir_counter_32h();
+	while (true) {
+		lo = read_gic_redir_counter_32l();
+
+		/* If hi didn't change then lo didn't wrap & we're done */
+		hi2 = read_gic_redir_counter_32h();
+		if (hi2 == hi)
+			break;
+
+		/* Otherwise, repeat with the latest hi value */
+		hi = hi2;
+	}
+
+	count = (((u64)hi) << 32) + lo;
+out:
+	mips_cm_unlock_other();
+	return count;
+}
+
 static struct clocksource gic_clocksource = {
 	.name			= "GIC",
 	.read			= gic_hpt_read,
@@ -203,6 +234,11 @@ static int __init __gic_clocksource_init(void)
 		gic_clocksource.rating = 200;
 	gic_clocksource.rating += clamp(gic_frequency / 10000000, 0, 99);
 
+	if (mips_cps_multicluster_cpus()) {
+		gic_clocksource.read = &gic_hpt_read_multicluster;
+		gic_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
+	}
+
 	ret = clocksource_register_hz(&gic_clocksource, gic_frequency);
 	if (ret < 0)
 		pr_warn("Unable to register clocksource\n");
@@ -261,7 +297,8 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 	 * stable CPU frequency or on the platforms with CM3 and CPU frequency
 	 * change performed by the CPC core clocks divider.
 	 */
-	if (mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) {
+	if ((mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) &&
+	     !mips_cps_multicluster_cpus()) {
 		sched_clock_register(mips_cm_is64 ?
 				     gic_read_count_64 : gic_read_count_2x32,
 				     gic_count_width, gic_frequency);
-- 
2.25.1

[PATCH v6 6/9] clocksource: mips-gic-timer: Enable counter when CPUs start

[Aleksandar Rikalo]

From: Paul Burton <paulburton@kernel.org>

In multi-cluster MIPS I6500 systems there is a GIC in each cluster,
each with its own counter. When a cluster powers up the counter will
be stopped, with the COUNTSTOP bit set in the GIC_CONFIG register.

In single cluster systems, it has been fine to clear COUNTSTOP once
in gic_clocksource_of_init() to start the counter. In multi-cluster
systems, this will only have started the counter in the boot cluster,
and any CPUs in other clusters will find their counter stopped which
will break the GIC clock_event_device.

Resolve this by having CPUs clear the COUNTSTOP bit when they come
online, using the existing gic_starting_cpu() CPU hotplug callback. This
will allow CPUs in secondary clusters to ensure that the cluster's GIC
counter is running as expected.

Signed-off-by: Paul Burton <paulburton@kernel.org>
Signed-off-by: Chao-ying Fu <cfu@wavecomp.com>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 drivers/clocksource/mips-gic-timer.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index 7907b740497a..abb685a080a5 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -115,6 +115,9 @@ static void gic_update_frequency(void *data)
 
 static int gic_starting_cpu(unsigned int cpu)
 {
+	/* Ensure the GIC counter is running */
+	clear_gic_config(GIC_CONFIG_COUNTSTOP);
+
 	gic_clockevent_cpu_init(cpu, this_cpu_ptr(&gic_clockevent_device));
 	return 0;
 }
@@ -288,9 +291,6 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 			pr_warn("Unable to register clock notifier\n");
 	}
 
-	/* And finally start the counter */
-	clear_gic_config(GIC_CONFIG_COUNTSTOP);
-
 	/*
 	 * It's safe to use the MIPS GIC timer as a sched clock source only if
 	 * its ticks are stable, which is true on either the platforms with
-- 
2.25.1

[PATCH v6 7/9] MIPS: pm-cps: Use per-CPU variables as per-CPU, not per-core

[Aleksandar Rikalo]

From: Paul Burton <paulburton@kernel.org>

The pm-cps code has up until now used per-CPU variables indexed by core,
rather than CPU number, in order to share data amongst sibling CPUs (ie.
VPs/threads in a core). This works fine for single cluster systems, but
with multi-cluster systems a core number is no longer unique in the
system, leading to sharing between CPUs that are not actually siblings.

Avoid this issue by using per-CPU variables as they are more generally
used - ie. access them using CPU numbers rather than core numbers.
Sharing between siblings is then accomplished by:
 - Assigning the same pointer to entries for each sibling CPU for the
   nc_asm_enter & ready_count variables, which allow this by virtue of
   being per-CPU pointers.

 - Indexing by the first CPU set in a CPUs cpu_sibling_map in the case
   of pm_barrier, for which we can't use the previous approach because
   the per-CPU variable is not a pointer.

Signed-off-by: Paul Burton <paulburton@kernel.org>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 arch/mips/kernel/pm-cps.c | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/arch/mips/kernel/pm-cps.c b/arch/mips/kernel/pm-cps.c
index d09ca77e624d..9369a8dc385e 100644
--- a/arch/mips/kernel/pm-cps.c
+++ b/arch/mips/kernel/pm-cps.c
@@ -57,10 +57,7 @@ static DEFINE_PER_CPU_ALIGNED(u32*, ready_count);
 /* Indicates online CPUs coupled with the current CPU */
 static DEFINE_PER_CPU_ALIGNED(cpumask_t, online_coupled);
 
-/*
- * Used to synchronize entry to deep idle states. Actually per-core rather
- * than per-CPU.
- */
+/* Used to synchronize entry to deep idle states */
 static DEFINE_PER_CPU_ALIGNED(atomic_t, pm_barrier);
 
 /* Saved CPU state across the CPS_PM_POWER_GATED state */
@@ -112,9 +109,10 @@ int cps_pm_enter_state(enum cps_pm_state state)
 	cps_nc_entry_fn entry;
 	struct core_boot_config *core_cfg;
 	struct vpe_boot_config *vpe_cfg;
+	atomic_t *barrier;
 
 	/* Check that there is an entry function for this state */
-	entry = per_cpu(nc_asm_enter, core)[state];
+	entry = per_cpu(nc_asm_enter, cpu)[state];
 	if (!entry)
 		return -EINVAL;
 
@@ -150,7 +148,7 @@ int cps_pm_enter_state(enum cps_pm_state state)
 	smp_mb__after_atomic();
 
 	/* Create a non-coherent mapping of the core ready_count */
-	core_ready_count = per_cpu(ready_count, core);
+	core_ready_count = per_cpu(ready_count, cpu);
 	nc_addr = kmap_noncoherent(virt_to_page(core_ready_count),
 				   (unsigned long)core_ready_count);
 	nc_addr += ((unsigned long)core_ready_count & ~PAGE_MASK);
@@ -158,7 +156,8 @@ int cps_pm_enter_state(enum cps_pm_state state)
 
 	/* Ensure ready_count is zero-initialised before the assembly runs */
 	WRITE_ONCE(*nc_core_ready_count, 0);
-	coupled_barrier(&per_cpu(pm_barrier, core), online);
+	barrier = &per_cpu(pm_barrier, cpumask_first(&cpu_sibling_map[cpu]));
+	coupled_barrier(barrier, online);
 
 	/* Run the generated entry code */
 	left = entry(online, nc_core_ready_count);
@@ -629,12 +628,14 @@ static void *cps_gen_entry_code(unsigned cpu, enum cps_pm_state state)
 
 static int cps_pm_online_cpu(unsigned int cpu)
 {
-	enum cps_pm_state state;
-	unsigned core = cpu_core(&cpu_data[cpu]);
+	unsigned int sibling, core;
 	void *entry_fn, *core_rc;
+	enum cps_pm_state state;
+
+	core = cpu_core(&cpu_data[cpu]);
 
 	for (state = CPS_PM_NC_WAIT; state < CPS_PM_STATE_COUNT; state++) {
-		if (per_cpu(nc_asm_enter, core)[state])
+		if (per_cpu(nc_asm_enter, cpu)[state])
 			continue;
 		if (!test_bit(state, state_support))
 			continue;
@@ -646,16 +647,19 @@ static int cps_pm_online_cpu(unsigned int cpu)
 			clear_bit(state, state_support);
 		}
 
-		per_cpu(nc_asm_enter, core)[state] = entry_fn;
+		for_each_cpu(sibling, &cpu_sibling_map[cpu])
+			per_cpu(nc_asm_enter, sibling)[state] = entry_fn;
 	}
 
-	if (!per_cpu(ready_count, core)) {
+	if (!per_cpu(ready_count, cpu)) {
 		core_rc = kmalloc(sizeof(u32), GFP_KERNEL);
 		if (!core_rc) {
 			pr_err("Failed allocate core %u ready_count\n", core);
 			return -ENOMEM;
 		}
-		per_cpu(ready_count, core) = core_rc;
+
+		for_each_cpu(sibling, &cpu_sibling_map[cpu])
+			per_cpu(ready_count, sibling) = core_rc;
 	}
 
 	return 0;
-- 
2.25.1

[PATCH v6 8/9] MIPS: CPS: Introduce struct cluster_boot_config

[Aleksandar Rikalo]

From: Paul Burton <paulburton@kernel.org>

In preparation for supporting multi-cluster systems, introduce a struct
cluster_boot_config as an extra layer in the boot configuration
maintained by the MIPS Coherent Processing System (CPS) SMP
implementation. For now only one struct cluster_boot_config will be
allocated & we'll simply defererence its core_config field to find the
struct core_boot_config array which can be used to boot as usual.

Signed-off-by: Paul Burton <paulburton@kernel.org>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 arch/mips/include/asm/smp-cps.h |  6 ++-
 arch/mips/kernel/asm-offsets.c  |  3 ++
 arch/mips/kernel/cps-vec.S      | 19 ++++++--
 arch/mips/kernel/pm-cps.c       |  5 +-
 arch/mips/kernel/smp-cps.c      | 82 +++++++++++++++++++++------------
 5 files changed, 81 insertions(+), 34 deletions(-)

diff --git a/arch/mips/include/asm/smp-cps.h b/arch/mips/include/asm/smp-cps.h
index ab94e50f62b8..a629e948a6fd 100644
--- a/arch/mips/include/asm/smp-cps.h
+++ b/arch/mips/include/asm/smp-cps.h
@@ -22,7 +22,11 @@ struct core_boot_config {
 	struct vpe_boot_config *vpe_config;
 };
 
-extern struct core_boot_config *mips_cps_core_bootcfg;
+struct cluster_boot_config {
+	struct core_boot_config *core_config;
+};
+
+extern struct cluster_boot_config *mips_cps_cluster_bootcfg;
 
 extern void mips_cps_core_boot(int cca, void __iomem *gcr_base);
 extern void mips_cps_core_init(void);
diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c
index cb1045ebab06..b29944160b28 100644
--- a/arch/mips/kernel/asm-offsets.c
+++ b/arch/mips/kernel/asm-offsets.c
@@ -404,6 +404,9 @@ void output_cps_defines(void)
 {
 	COMMENT(" MIPS CPS offsets. ");
 
+	OFFSET(CLUSTERBOOTCFG_CORECONFIG, cluster_boot_config, core_config);
+	DEFINE(CLUSTERBOOTCFG_SIZE, sizeof(struct cluster_boot_config));
+
 	OFFSET(COREBOOTCFG_VPEMASK, core_boot_config, vpe_mask);
 	OFFSET(COREBOOTCFG_VPECONFIG, core_boot_config, vpe_config);
 	DEFINE(COREBOOTCFG_SIZE, sizeof(struct core_boot_config));
diff --git a/arch/mips/kernel/cps-vec.S b/arch/mips/kernel/cps-vec.S
index f876309130ad..2ae7034a3d5c 100644
--- a/arch/mips/kernel/cps-vec.S
+++ b/arch/mips/kernel/cps-vec.S
@@ -19,6 +19,10 @@
 #define GCR_CPC_BASE_OFS	0x0088
 #define GCR_CL_COHERENCE_OFS	0x2008
 #define GCR_CL_ID_OFS		0x2028
+#define  CM3_GCR_Cx_ID_CLUSTER_SHF	8
+#define  CM3_GCR_Cx_ID_CLUSTER_MSK	(0xff << 8)
+#define  CM3_GCR_Cx_ID_CORENUM_SHF	0
+#define  CM3_GCR_Cx_ID_CORENUM_MSK	(0xff << 0)
 
 #define CPC_CL_VC_STOP_OFS	0x2020
 #define CPC_CL_VC_RUN_OFS	0x2028
@@ -271,12 +275,21 @@ LEAF(mips_cps_core_init)
  */
 LEAF(mips_cps_get_bootcfg)
 	/* Calculate a pointer to this cores struct core_boot_config */
+	PTR_LA	v0, mips_cps_cluster_bootcfg
+	PTR_L	v0, 0(v0)
 	lw	t0, GCR_CL_ID_OFS(s1)
+#ifdef CONFIG_CPU_MIPSR6
+	ext	t1, t0, CM3_GCR_Cx_ID_CLUSTER_SHF, 8
+	li	t2, CLUSTERBOOTCFG_SIZE
+	mul	t1, t1, t2
+	PTR_ADDU \
+		v0, v0, t1
+#endif
+	PTR_L	v0, CLUSTERBOOTCFG_CORECONFIG(v0)
+	andi	t0, t0, CM3_GCR_Cx_ID_CORENUM_MSK
 	li	t1, COREBOOTCFG_SIZE
 	mul	t0, t0, t1
-	PTR_LA	t1, mips_cps_core_bootcfg
-	PTR_L	t1, 0(t1)
-	PTR_ADDU v0, t0, t1
+	PTR_ADDU v0, v0, t0
 
 	/* Calculate this VPEs ID. If the core doesn't support MT use 0 */
 	li	t9, 0
diff --git a/arch/mips/kernel/pm-cps.c b/arch/mips/kernel/pm-cps.c
index 9369a8dc385e..3de0e05e0511 100644
--- a/arch/mips/kernel/pm-cps.c
+++ b/arch/mips/kernel/pm-cps.c
@@ -101,12 +101,14 @@ static void coupled_barrier(atomic_t *a, unsigned online)
 int cps_pm_enter_state(enum cps_pm_state state)
 {
 	unsigned cpu = smp_processor_id();
+	unsigned int cluster = cpu_cluster(&current_cpu_data);
 	unsigned core = cpu_core(&current_cpu_data);
 	unsigned online, left;
 	cpumask_t *coupled_mask = this_cpu_ptr(&online_coupled);
 	u32 *core_ready_count, *nc_core_ready_count;
 	void *nc_addr;
 	cps_nc_entry_fn entry;
+	struct cluster_boot_config *cluster_cfg;
 	struct core_boot_config *core_cfg;
 	struct vpe_boot_config *vpe_cfg;
 	atomic_t *barrier;
@@ -136,7 +138,8 @@ int cps_pm_enter_state(enum cps_pm_state state)
 		if (!mips_cps_smp_in_use())
 			return -EINVAL;
 
-		core_cfg = &mips_cps_core_bootcfg[core];
+		cluster_cfg = &mips_cps_cluster_bootcfg[cluster];
+		core_cfg = &cluster_cfg->core_config[core];
 		vpe_cfg = &core_cfg->vpe_config[cpu_vpe_id(&current_cpu_data)];
 		vpe_cfg->pc = (unsigned long)mips_cps_pm_restore;
 		vpe_cfg->gp = (unsigned long)current_thread_info();
diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c
index 395622c37325..f71e2bb58318 100644
--- a/arch/mips/kernel/smp-cps.c
+++ b/arch/mips/kernel/smp-cps.c
@@ -40,7 +40,7 @@ static DECLARE_BITMAP(core_power, NR_CPUS);
 static uint32_t core_entry_reg;
 static phys_addr_t cps_vec_pa;
 
-struct core_boot_config *mips_cps_core_bootcfg;
+struct cluster_boot_config *mips_cps_cluster_bootcfg;
 
 static unsigned __init core_vpe_count(unsigned int cluster, unsigned core)
 {
@@ -212,8 +212,10 @@ static void __init cps_smp_setup(void)
 
 static void __init cps_prepare_cpus(unsigned int max_cpus)
 {
-	unsigned ncores, core_vpes, c, cca;
+	unsigned int nclusters, ncores, core_vpes, c, cl, cca;
 	bool cca_unsuitable, cores_limited;
+	struct cluster_boot_config *cluster_bootcfg;
+	struct core_boot_config *core_bootcfg;
 
 	mips_mt_set_cpuoptions();
 
@@ -255,40 +257,54 @@ static void __init cps_prepare_cpus(unsigned int max_cpus)
 
 	setup_cps_vecs();
 
-	/* Allocate core boot configuration structs */
-	ncores = mips_cps_numcores(0);
-	mips_cps_core_bootcfg = kcalloc(ncores, sizeof(*mips_cps_core_bootcfg),
-					GFP_KERNEL);
-	if (!mips_cps_core_bootcfg) {
-		pr_err("Failed to allocate boot config for %u cores\n", ncores);
-		goto err_out;
-	}
+	/* Allocate cluster boot configuration structs */
+	nclusters = mips_cps_numclusters();
+	mips_cps_cluster_bootcfg = kcalloc(nclusters,
+					   sizeof(*mips_cps_cluster_bootcfg),
+					   GFP_KERNEL);
 
-	/* Allocate VPE boot configuration structs */
-	for (c = 0; c < ncores; c++) {
-		core_vpes = core_vpe_count(0, c);
-		mips_cps_core_bootcfg[c].vpe_config = kcalloc(core_vpes,
-				sizeof(*mips_cps_core_bootcfg[c].vpe_config),
-				GFP_KERNEL);
-		if (!mips_cps_core_bootcfg[c].vpe_config) {
-			pr_err("Failed to allocate %u VPE boot configs\n",
-			       core_vpes);
+	for (cl = 0; cl < nclusters; cl++) {
+		/* Allocate core boot configuration structs */
+		ncores = mips_cps_numcores(cl);
+		core_bootcfg = kcalloc(ncores, sizeof(*core_bootcfg),
+					GFP_KERNEL);
+		if (!core_bootcfg)
 			goto err_out;
+		mips_cps_cluster_bootcfg[cl].core_config = core_bootcfg;
+
+		/* Allocate VPE boot configuration structs */
+		for (c = 0; c < ncores; c++) {
+			core_vpes = core_vpe_count(cl, c);
+			core_bootcfg[c].vpe_config = kcalloc(core_vpes,
+					sizeof(*core_bootcfg[c].vpe_config),
+					GFP_KERNEL);
+			if (!core_bootcfg[c].vpe_config)
+				goto err_out;
 		}
 	}
 
 	/* Mark this CPU as booted */
-	atomic_set(&mips_cps_core_bootcfg[cpu_core(&current_cpu_data)].vpe_mask,
-		   1 << cpu_vpe_id(&current_cpu_data));
+	cl = cpu_cluster(&current_cpu_data);
+	c = cpu_core(&current_cpu_data);
+	cluster_bootcfg = &mips_cps_cluster_bootcfg[cl];
+	core_bootcfg = &cluster_bootcfg->core_config[c];
+	atomic_set(&core_bootcfg->vpe_mask, 1 << cpu_vpe_id(&current_cpu_data));
 
 	return;
 err_out:
 	/* Clean up allocations */
-	if (mips_cps_core_bootcfg) {
-		for (c = 0; c < ncores; c++)
-			kfree(mips_cps_core_bootcfg[c].vpe_config);
-		kfree(mips_cps_core_bootcfg);
-		mips_cps_core_bootcfg = NULL;
+	if (mips_cps_cluster_bootcfg) {
+		for (cl = 0; cl < nclusters; cl++) {
+			cluster_bootcfg = &mips_cps_cluster_bootcfg[cl];
+			ncores = mips_cps_numcores(cl);
+			for (c = 0; c < ncores; c++) {
+				core_bootcfg = &cluster_bootcfg->core_config[c];
+				kfree(core_bootcfg->vpe_config);
+			}
+			kfree(mips_cps_cluster_bootcfg[c].core_config);
+		}
+		kfree(mips_cps_cluster_bootcfg);
+		mips_cps_cluster_bootcfg = NULL;
 	}
 
 	/* Effectively disable SMP by declaring CPUs not present */
@@ -376,17 +392,23 @@ static void boot_core(unsigned int core, unsigned int vpe_id)
 
 static void remote_vpe_boot(void *dummy)
 {
+	unsigned int cluster = cpu_cluster(&current_cpu_data);
 	unsigned core = cpu_core(&current_cpu_data);
-	struct core_boot_config *core_cfg = &mips_cps_core_bootcfg[core];
+	struct cluster_boot_config *cluster_cfg =
+		&mips_cps_cluster_bootcfg[cluster];
+	struct core_boot_config *core_cfg = &cluster_cfg->core_config[core];
 
 	mips_cps_boot_vpes(core_cfg, cpu_vpe_id(&current_cpu_data));
 }
 
 static int cps_boot_secondary(int cpu, struct task_struct *idle)
 {
+	unsigned int cluster = cpu_cluster(&cpu_data[cpu]);
 	unsigned core = cpu_core(&cpu_data[cpu]);
 	unsigned vpe_id = cpu_vpe_id(&cpu_data[cpu]);
-	struct core_boot_config *core_cfg = &mips_cps_core_bootcfg[core];
+	struct cluster_boot_config *cluster_cfg =
+		&mips_cps_cluster_bootcfg[cluster];
+	struct core_boot_config *core_cfg = &cluster_cfg->core_config[core];
 	struct vpe_boot_config *vpe_cfg = &core_cfg->vpe_config[vpe_id];
 	unsigned int remote;
 	int err;
@@ -544,12 +566,14 @@ static void cps_kexec_nonboot_cpu(void)
 static int cps_cpu_disable(void)
 {
 	unsigned cpu = smp_processor_id();
+	struct cluster_boot_config *cluster_cfg;
 	struct core_boot_config *core_cfg;
 
 	if (!cps_pm_support_state(CPS_PM_POWER_GATED))
 		return -EINVAL;
 
-	core_cfg = &mips_cps_core_bootcfg[cpu_core(&current_cpu_data)];
+	cluster_cfg = &mips_cps_cluster_bootcfg[cpu_cluster(&current_cpu_data)];
+	core_cfg = &cluster_cfg->core_config[cpu_core(&current_cpu_data)];
 	atomic_sub(1 << cpu_vpe_id(&current_cpu_data), &core_cfg->vpe_mask);
 	smp_mb__after_atomic();
 	set_cpu_online(cpu, false);
-- 
2.25.1

[PATCH v6 9/9] MIPS: CPS: Boot CPUs in secondary clusters

[Aleksandar Rikalo]

From: Paul Burton <paulburton@kernel.org>

Probe for & boot CPUs (cores & VPs) in secondary clusters (ie. not the
cluster that began booting Linux) when they are present in systems with
CM 3.5 or higher.

Signed-off-by: Paul Burton <paulburton@kernel.org>
Signed-off-by: Chao-ying Fu <cfu@wavecomp.com>
Signed-off-by: Dragan Mladjenovic <dragan.mladjenovic@syrmia.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
Tested-by: Serge Semin <fancer.lancer@gmail.com>
---
 arch/mips/include/asm/mips-cm.h |  18 +++
 arch/mips/include/asm/smp-cps.h |   1 +
 arch/mips/kernel/mips-cm.c      |   4 +-
 arch/mips/kernel/smp-cps.c      | 205 ++++++++++++++++++++++++++++----
 4 files changed, 207 insertions(+), 21 deletions(-)

diff --git a/arch/mips/include/asm/mips-cm.h b/arch/mips/include/asm/mips-cm.h
index 1e782275850a..4d47163647dd 100644
--- a/arch/mips/include/asm/mips-cm.h
+++ b/arch/mips/include/asm/mips-cm.h
@@ -255,6 +255,12 @@ GCR_ACCESSOR_RW(32, 0x130, l2_config)
 GCR_ACCESSOR_RO(32, 0x150, sys_config2)
 #define CM_GCR_SYS_CONFIG2_MAXVPW		GENMASK(3, 0)
 
+/* GCR_L2-RAM_CONFIG - Configuration & status of L2 cache RAMs */
+GCR_ACCESSOR_RW(64, 0x240, l2_ram_config)
+#define CM_GCR_L2_RAM_CONFIG_PRESENT		BIT(31)
+#define CM_GCR_L2_RAM_CONFIG_HCI_DONE		BIT(30)
+#define CM_GCR_L2_RAM_CONFIG_HCI_SUPPORTED	BIT(29)
+
 /* GCR_L2_PFT_CONTROL - Controls hardware L2 prefetching */
 GCR_ACCESSOR_RW(32, 0x300, l2_pft_control)
 #define CM_GCR_L2_PFT_CONTROL_PAGEMASK		GENMASK(31, 12)
@@ -266,6 +272,18 @@ GCR_ACCESSOR_RW(32, 0x308, l2_pft_control_b)
 #define CM_GCR_L2_PFT_CONTROL_B_CEN		BIT(8)
 #define CM_GCR_L2_PFT_CONTROL_B_PORTID		GENMASK(7, 0)
 
+/* GCR_L2_TAG_ADDR - Access addresses in L2 cache tags */
+GCR_ACCESSOR_RW(64, 0x600, l2_tag_addr)
+
+/* GCR_L2_TAG_STATE - Access L2 cache tag state */
+GCR_ACCESSOR_RW(64, 0x608, l2_tag_state)
+
+/* GCR_L2_DATA - Access data in L2 cache lines */
+GCR_ACCESSOR_RW(64, 0x610, l2_data)
+
+/* GCR_L2_ECC - Access ECC information from L2 cache lines */
+GCR_ACCESSOR_RW(64, 0x618, l2_ecc)
+
 /* GCR_L2SM_COP - L2 cache op state machine control */
 GCR_ACCESSOR_RW(32, 0x620, l2sm_cop)
 #define CM_GCR_L2SM_COP_PRESENT			BIT(31)
diff --git a/arch/mips/include/asm/smp-cps.h b/arch/mips/include/asm/smp-cps.h
index a629e948a6fd..10d3ebd890cb 100644
--- a/arch/mips/include/asm/smp-cps.h
+++ b/arch/mips/include/asm/smp-cps.h
@@ -23,6 +23,7 @@ struct core_boot_config {
 };
 
 struct cluster_boot_config {
+	unsigned long *core_power;
 	struct core_boot_config *core_config;
 };
 
diff --git a/arch/mips/kernel/mips-cm.c b/arch/mips/kernel/mips-cm.c
index 3eb2cfb893e1..9854bc2b6895 100644
--- a/arch/mips/kernel/mips-cm.c
+++ b/arch/mips/kernel/mips-cm.c
@@ -308,7 +308,9 @@ void mips_cm_lock_other(unsigned int cluster, unsigned int core,
 		      FIELD_PREP(CM3_GCR_Cx_OTHER_VP, vp);
 
 		if (cm_rev >= CM_REV_CM3_5) {
-			val |= CM_GCR_Cx_OTHER_CLUSTER_EN;
+			if (cluster != cpu_cluster(&current_cpu_data))
+				val |= CM_GCR_Cx_OTHER_CLUSTER_EN;
+			val |= CM_GCR_Cx_OTHER_GIC_EN;
 			val |= FIELD_PREP(CM_GCR_Cx_OTHER_CLUSTER, cluster);
 			val |= FIELD_PREP(CM_GCR_Cx_OTHER_BLOCK, block);
 		} else {
diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c
index f71e2bb58318..4f344c890a23 100644
--- a/arch/mips/kernel/smp-cps.c
+++ b/arch/mips/kernel/smp-cps.c
@@ -36,12 +36,56 @@ enum label_id {
 
 UASM_L_LA(_not_nmi)
 
-static DECLARE_BITMAP(core_power, NR_CPUS);
 static uint32_t core_entry_reg;
 static phys_addr_t cps_vec_pa;
 
 struct cluster_boot_config *mips_cps_cluster_bootcfg;
 
+static void power_up_other_cluster(unsigned int cluster)
+{
+	u32 stat, seq_state;
+	unsigned int timeout;
+
+	mips_cm_lock_other(cluster, CM_GCR_Cx_OTHER_CORE_CM, 0,
+			   CM_GCR_Cx_OTHER_BLOCK_LOCAL);
+	stat = read_cpc_co_stat_conf();
+	mips_cm_unlock_other();
+
+	seq_state = stat & CPC_Cx_STAT_CONF_SEQSTATE;
+	seq_state >>= __ffs(CPC_Cx_STAT_CONF_SEQSTATE);
+	if (seq_state == CPC_Cx_STAT_CONF_SEQSTATE_U5)
+		return;
+
+	/* Set endianness & power up the CM */
+	mips_cm_lock_other(cluster, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+	write_cpc_redir_sys_config(IS_ENABLED(CONFIG_CPU_BIG_ENDIAN));
+	write_cpc_redir_pwrup_ctl(1);
+	mips_cm_unlock_other();
+
+	/* Wait for the CM to start up */
+	timeout = 1000;
+	mips_cm_lock_other(cluster, CM_GCR_Cx_OTHER_CORE_CM, 0,
+			   CM_GCR_Cx_OTHER_BLOCK_LOCAL);
+	while (1) {
+		stat = read_cpc_co_stat_conf();
+		seq_state = stat & CPC_Cx_STAT_CONF_SEQSTATE;
+		seq_state >>= __ffs(CPC_Cx_STAT_CONF_SEQSTATE);
+		if (seq_state == CPC_Cx_STAT_CONF_SEQSTATE_U5)
+			break;
+
+		if (timeout) {
+			mdelay(1);
+			timeout--;
+		} else {
+			pr_warn("Waiting for cluster %u CM to power up... STAT_CONF=0x%x\n",
+				cluster, stat);
+			mdelay(1000);
+		}
+	}
+
+	mips_cm_unlock_other();
+}
+
 static unsigned __init core_vpe_count(unsigned int cluster, unsigned core)
 {
 	return min(smp_max_threads, mips_cps_numvps(cluster, core));
@@ -152,6 +196,9 @@ static void __init cps_smp_setup(void)
 			pr_cont(",");
 		pr_cont("{");
 
+		if (mips_cm_revision() >= CM_REV_CM3_5)
+			power_up_other_cluster(cl);
+
 		ncores = mips_cps_numcores(cl);
 		for (c = 0; c < ncores; c++) {
 			core_vpes = core_vpe_count(cl, c);
@@ -179,8 +226,8 @@ static void __init cps_smp_setup(void)
 
 	/* Indicate present CPUs (CPU being synonymous with VPE) */
 	for (v = 0; v < min_t(unsigned, nvpes, NR_CPUS); v++) {
-		set_cpu_possible(v, cpu_cluster(&cpu_data[v]) == 0);
-		set_cpu_present(v, cpu_cluster(&cpu_data[v]) == 0);
+		set_cpu_possible(v, true);
+		set_cpu_present(v, true);
 		__cpu_number_map[v] = v;
 		__cpu_logical_map[v] = v;
 	}
@@ -188,9 +235,6 @@ static void __init cps_smp_setup(void)
 	/* Set a coherent default CCA (CWB) */
 	change_c0_config(CONF_CM_CMASK, 0x5);
 
-	/* Core 0 is powered up (we're running on it) */
-	bitmap_set(core_power, 0, 1);
-
 	/* Initialise core 0 */
 	mips_cps_core_init();
 
@@ -272,6 +316,10 @@ static void __init cps_prepare_cpus(unsigned int max_cpus)
 			goto err_out;
 		mips_cps_cluster_bootcfg[cl].core_config = core_bootcfg;
 
+		mips_cps_cluster_bootcfg[cl].core_power =
+			kcalloc(BITS_TO_LONGS(ncores), sizeof(unsigned long),
+				GFP_KERNEL);
+
 		/* Allocate VPE boot configuration structs */
 		for (c = 0; c < ncores; c++) {
 			core_vpes = core_vpe_count(cl, c);
@@ -283,11 +331,12 @@ static void __init cps_prepare_cpus(unsigned int max_cpus)
 		}
 	}
 
-	/* Mark this CPU as booted */
+	/* Mark this CPU as powered up & booted */
 	cl = cpu_cluster(&current_cpu_data);
 	c = cpu_core(&current_cpu_data);
 	cluster_bootcfg = &mips_cps_cluster_bootcfg[cl];
 	core_bootcfg = &cluster_bootcfg->core_config[c];
+	bitmap_set(cluster_bootcfg->core_power, cpu_core(&current_cpu_data), 1);
 	atomic_set(&core_bootcfg->vpe_mask, 1 << cpu_vpe_id(&current_cpu_data));
 
 	return;
@@ -315,13 +364,118 @@ static void __init cps_prepare_cpus(unsigned int max_cpus)
 	}
 }
 
-static void boot_core(unsigned int core, unsigned int vpe_id)
+static void init_cluster_l2(void)
 {
-	u32 stat, seq_state;
-	unsigned timeout;
+	u32 l2_cfg, l2sm_cop, result;
+
+	while (1) {
+		l2_cfg = read_gcr_redir_l2_ram_config();
+
+		/* If HCI is not supported, use the state machine below */
+		if (!(l2_cfg & CM_GCR_L2_RAM_CONFIG_PRESENT))
+			break;
+		if (!(l2_cfg & CM_GCR_L2_RAM_CONFIG_HCI_SUPPORTED))
+			break;
+
+		/* If the HCI_DONE bit is set, we're finished */
+		if (l2_cfg & CM_GCR_L2_RAM_CONFIG_HCI_DONE)
+			return;
+	}
+
+	l2sm_cop = read_gcr_redir_l2sm_cop();
+	if (WARN(!(l2sm_cop & CM_GCR_L2SM_COP_PRESENT),
+		 "L2 init not supported on this system yet"))
+		return;
+
+	/* Clear L2 tag registers */
+	write_gcr_redir_l2_tag_state(0);
+	write_gcr_redir_l2_ecc(0);
+
+	/* Ensure the L2 tag writes complete before the state machine starts */
+	mb();
+
+	/* Wait for the L2 state machine to be idle */
+	do {
+		l2sm_cop = read_gcr_redir_l2sm_cop();
+	} while (l2sm_cop & CM_GCR_L2SM_COP_RUNNING);
+
+	/* Start a store tag operation */
+	l2sm_cop = CM_GCR_L2SM_COP_TYPE_IDX_STORETAG;
+	l2sm_cop <<= __ffs(CM_GCR_L2SM_COP_TYPE);
+	l2sm_cop |= CM_GCR_L2SM_COP_CMD_START;
+	write_gcr_redir_l2sm_cop(l2sm_cop);
+
+	/* Ensure the state machine starts before we poll for completion */
+	mb();
+
+	/* Wait for the operation to be complete */
+	do {
+		l2sm_cop = read_gcr_redir_l2sm_cop();
+		result = l2sm_cop & CM_GCR_L2SM_COP_RESULT;
+		result >>= __ffs(CM_GCR_L2SM_COP_RESULT);
+	} while (!result);
+
+	WARN(result != CM_GCR_L2SM_COP_RESULT_DONE_OK,
+	     "L2 state machine failed cache init with error %u\n", result);
+}
+
+static void boot_core(unsigned int cluster, unsigned int core,
+		      unsigned int vpe_id)
+{
+	struct cluster_boot_config *cluster_cfg;
+	u32 access, stat, seq_state;
+	unsigned int timeout, ncores;
+
+	cluster_cfg = &mips_cps_cluster_bootcfg[cluster];
+	ncores = mips_cps_numcores(cluster);
+
+	if ((cluster != cpu_cluster(&current_cpu_data)) &&
+	    bitmap_empty(cluster_cfg->core_power, ncores)) {
+		power_up_other_cluster(cluster);
+
+		mips_cm_lock_other(cluster, core, 0,
+				   CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+
+		/* Ensure cluster GCRs are where we expect */
+		write_gcr_redir_base(read_gcr_base());
+		write_gcr_redir_cpc_base(read_gcr_cpc_base());
+		write_gcr_redir_gic_base(read_gcr_gic_base());
+
+		init_cluster_l2();
+
+		/* Mirror L2 configuration */
+		write_gcr_redir_l2_only_sync_base(read_gcr_l2_only_sync_base());
+		write_gcr_redir_l2_pft_control(read_gcr_l2_pft_control());
+		write_gcr_redir_l2_pft_control_b(read_gcr_l2_pft_control_b());
+
+		/* Mirror ECC/parity setup */
+		write_gcr_redir_err_control(read_gcr_err_control());
+
+		/* Set BEV base */
+		write_gcr_redir_bev_base(core_entry_reg);
+
+		mips_cm_unlock_other();
+	}
+
+	if (cluster != cpu_cluster(&current_cpu_data)) {
+		mips_cm_lock_other(cluster, core, 0,
+				   CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+
+		/* Ensure the core can access the GCRs */
+		access = read_gcr_redir_access();
+		access |= BIT(core);
+		write_gcr_redir_access(access);
+
+		mips_cm_unlock_other();
+	} else {
+		/* Ensure the core can access the GCRs */
+		access = read_gcr_access();
+		access |= BIT(core);
+		write_gcr_access(access);
+	}
 
 	/* Select the appropriate core */
-	mips_cm_lock_other(0, core, 0, CM_GCR_Cx_OTHER_BLOCK_LOCAL);
+	mips_cm_lock_other(cluster, core, 0, CM_GCR_Cx_OTHER_BLOCK_LOCAL);
 
 	/* Set its reset vector */
 	write_gcr_co_reset_base(core_entry_reg);
@@ -387,7 +541,17 @@ static void boot_core(unsigned int core, unsigned int vpe_id)
 	mips_cm_unlock_other();
 
 	/* The core is now powered up */
-	bitmap_set(core_power, core, 1);
+	bitmap_set(cluster_cfg->core_power, core, 1);
+
+	/*
+	 * Restore CM_PWRUP=0 so that the CM can power down if all the cores in
+	 * the cluster do (eg. if they're all removed via hotplug.
+	 */
+	if (mips_cm_revision() >= CM_REV_CM3_5) {
+		mips_cm_lock_other(cluster, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+		write_cpc_redir_pwrup_ctl(0);
+		mips_cm_unlock_other();
+	}
 }
 
 static void remote_vpe_boot(void *dummy)
@@ -413,10 +577,6 @@ static int cps_boot_secondary(int cpu, struct task_struct *idle)
 	unsigned int remote;
 	int err;
 
-	/* We don't yet support booting CPUs in other clusters */
-	if (cpu_cluster(&cpu_data[cpu]) != cpu_cluster(&raw_current_cpu_data))
-		return -ENOSYS;
-
 	vpe_cfg->pc = (unsigned long)&smp_bootstrap;
 	vpe_cfg->sp = __KSTK_TOS(idle);
 	vpe_cfg->gp = (unsigned long)task_thread_info(idle);
@@ -425,14 +585,15 @@ static int cps_boot_secondary(int cpu, struct task_struct *idle)
 
 	preempt_disable();
 
-	if (!test_bit(core, core_power)) {
+	if (!test_bit(core, cluster_cfg->core_power)) {
 		/* Boot a VPE on a powered down core */
-		boot_core(core, vpe_id);
+		boot_core(cluster, core, vpe_id);
 		goto out;
 	}
 
 	if (cpu_has_vp) {
-		mips_cm_lock_other(0, core, vpe_id, CM_GCR_Cx_OTHER_BLOCK_LOCAL);
+		mips_cm_lock_other(cluster, core, vpe_id,
+				   CM_GCR_Cx_OTHER_BLOCK_LOCAL);
 		write_gcr_co_reset_base(core_entry_reg);
 		mips_cm_unlock_other();
 	}
@@ -639,11 +800,15 @@ static void cps_cpu_die(unsigned int cpu) { }
 
 static void cps_cleanup_dead_cpu(unsigned cpu)
 {
+	unsigned int cluster = cpu_cluster(&cpu_data[cpu]);
 	unsigned core = cpu_core(&cpu_data[cpu]);
 	unsigned int vpe_id = cpu_vpe_id(&cpu_data[cpu]);
 	ktime_t fail_time;
 	unsigned stat;
 	int err;
+	struct cluster_boot_config *cluster_cfg;
+
+	cluster_cfg = &mips_cps_cluster_bootcfg[cluster];
 
 	/*
 	 * Now wait for the CPU to actually offline. Without doing this that
@@ -695,7 +860,7 @@ static void cps_cleanup_dead_cpu(unsigned cpu)
 		} while (1);
 
 		/* Indicate the core is powered off */
-		bitmap_clear(core_power, core, 1);
+		bitmap_clear(cluster_cfg->core_power, core, 1);
 	} else if (cpu_has_mipsmt) {
 		/*
 		 * Have a CPU with access to the offlined CPUs registers wait
-- 
2.25.1

Re: [PATCH v6 0/9] MIPS: Support I6500 multi-cluster configuration

[Aleksandar Rikalo]

Would anyone be able to take a look at this?
Thank you.

--
Aleksandar

Re: [PATCH v6 0/9] MIPS: Support I6500 multi-cluster configuration

[Gregory CLEMENT]

Hello Aleksandar,

> Would anyone be able to take a look at this?
> Thank you.

I have tested your series on EyeQ6H, with the following topology:

VP topology {4,4,4,4},{4,4,4,4} total 32

And the core of the second cluster failed to boot:

CPU16: failed to start
No online CPU in core 0 to start CPU17
CPU17: failed to start
No online CPU in core 0 to start CPU18
CPU18: failed to start
No online CPU in core 0 to start CPU19
CPU19: failed to start
CPU20: failed to start
No online CPU in core 1 to start CPU21
CPU21: failed to start
No online CPU in core 1 to start CPU22
CPU22: failed to start
No online CPU in core 1 to start CPU23
CPU23: failed to start
CPU24: failed to start
No online CPU in core 2 to start CPU25
CPU25: failed to start
No online CPU in core 2 to start CPU26
CPU26: failed to start
No online CPU in core 2 to start CPU27
CPU27: failed to start
CPU28: failed to start
No online CPU in core 3 to start CPU29
CPU29: failed to start
No online CPU in core 3 to start CPU30
CPU30: failed to start
No online CPU in core 3 to start CPU31
CPU31: failed to start

I'm really interested in making it work and getting it merged. So if you
have any advice on how to debug this issue, please let me know.

For completeness, let me add the full boot log:

[    0.000000] Linux version 6.12.0-rc2-00012-g6a27d531a1b3 (gclement@BLaptop) (mips-img-linux-gnu-gcc (Codescape GNU Tools 2021.09-01 for MIPS IMG Linux) 11.2.0, GNU ld (Codescape GNU Tools 2021.09-01 for MIPS IMG Linux) 2.31.1) #311 SMP Tue Oct  8 15:52:31 CEST 2024
[    0.000000] CPU0 revision is: 0001b031 (MIPS I6500)
[    0.000000] FPU revision is: 20f30320
[    0.000000] MSA revision is: 00000320
[    0.000000] MIPS: machine is Mobile EyeQ6H MP6 Evaluation board
[    0.000000] earlycon: pl11 at MMIO32 0x00000000d3331000 (options '921600n8')
[    0.000000] printk: legacy bootconsole [pl11] enabled
[    0.000000] Initrd not found or empty - disabling initrd
[    0.000000] VP topology {4,4,4,4},{4,4,4,4} total 32
[    0.000000] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.000000] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.000000] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.000000] Zone ranges:
[    0.000000]   Normal   [mem 0x0000000100000000-0x00000001ffffffff]
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x0000000100000000-0x00000001ffffffff]
[    0.000000] Initmem setup node 0 [mem 0x0000000100000000-0x00000001ffffffff]
[    0.000000] percpu: Embedded 14 pages/cpu s177952 r8192 d43232 u229376
[    0.000000] Kernel command line: earlycon
[    0.000000] printk: log_buf_len individual max cpu contribution: 4096 bytes
[    0.000000] printk: log_buf_len total cpu_extra contributions: 126976 bytes
[    0.000000] printk: log_buf_len min size: 131072 bytes
[    0.000000] printk: log_buf_len: 262144 bytes
[    0.000000] printk: early log buf free: 129008(98%)
[    0.000000] Dentry cache hash table entries: 524288 (order: 8, 4194304 bytes, linear)
[    0.000000] Inode-cache hash table entries: 262144 (order: 7, 2097152 bytes, linear)
[    0.000000] ebase(0x0000000100008000) should better be in KSeg0
[    0.000000] Cache parity protection enabled
[    0.000000] Built 1 zonelists, mobility grouping on.  Total pages: 262144
[    0.000000] mem auto-init: stack:off, heap alloc:off, heap free:off
[    0.000000] MAAR configuration:
[    0.000000]   [0]: 0x0000000100000000-0x00000001ffffffff speculate
[    0.000000]   [1]: disabled
[    0.000000]   [2]: disabled
[    0.000000] SLUB: HWalign=128, Order=0-3, MinObjects=0, CPUs=32, Nodes=1
[    0.000000] ftrace: allocating 31777 entries in 32 pages
[    0.000000] ftrace: allocated 32 pages with 1 groups
[    0.000000] trace event string verifier disabled
[    0.000000] rcu: Hierarchical RCU implementation.
[    0.000000]  Rude variant of Tasks RCU enabled.
[    0.000000]  Tracing variant of Tasks RCU enabled.
[    0.000000] rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.
[    0.000000] RCU Tasks Rude: Setting shift to 5 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=32.
[    0.000000] RCU Tasks Trace: Setting shift to 5 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=32.
[    0.000000] NR_IRQS: 256
[    0.000000] rcu: srcu_init: Setting srcu_struct sizes based on contention.
[    0.000000] clocksource: GIC: mask: 0xffffffffffffffff max_cycles: 0x39a85c9bff6, max_idle_ns: 881590591483 ns
[    0.000000] clocksource: MIPS: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 1911260446 ns
[    0.000001] sched_clock: 32 bits at 1000MHz, resolution 1ns, wraps every 2147483647ns
[    0.001025] Console: colour dummy device 80x25
[    0.001564] printk: legacy console [tty0] enabled
[    0.002130] printk: legacy bootconsole [pl11] disabled
[    0.002789] Calibrating delay loop... 1988.60 BogoMIPS (lpj=3977216)
[    0.018803] pid_max: default: 32768 minimum: 301
[    0.018976] Mount-cache hash table entries: 8192 (order: 2, 65536 bytes, linear)
[    0.019013] Mountpoint-cache hash table entries: 8192 (order: 2, 65536 bytes, linear)
[    0.035650] MMID allocator initialised with 65536 entries
[    0.035798] rcu: Hierarchical SRCU implementation.
[    0.035810] rcu:     Max phase no-delay instances is 1000.
[    0.035992] Timer migration: 2 hierarchy levels; 8 children per group; 2 crossnode level
[    0.037059] smp: Bringing up secondary CPUs ...
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU1 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.058779] Counter synchronization [CPU#0 -> CPU#1]:
[    0.058779] Measured 12 cycles counter warp between CPUs
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU2 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.082779] Counter synchronization [CPU#0 -> CPU#2]:
[    0.082779] Measured 5 cycles counter warp between CPUs
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU3 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.110779] Counter synchronization [CPU#0 -> CPU#3]:
[    0.110779] Measured 10 cycles counter warp between CPUs
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU4 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.130779] Counter synchronization [CPU#0 -> CPU#4]: passed
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU5 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.158779] Counter synchronization [CPU#0 -> CPU#5]: passed
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU6 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.186779] Counter synchronization [CPU#0 -> CPU#6]:
[    0.186779] Measured 7 cycles counter warp between CPUs
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU7 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.214779] Counter synchronization [CPU#0 -> CPU#7]:
[    0.214779] Measured 1 cycles counter warp between CPUs
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU8 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.242779] Counter synchronization [CPU#0 -> CPU#8]: passed
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU9 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.266779] Counter synchronization [CPU#0 -> CPU#9]: passed
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU10 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.294779] Counter synchronization [CPU#0 -> CPU#10]: passed
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU11 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.322779] Counter synchronization [CPU#0 -> CPU#11]: passed
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU12 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.350779] Counter synchronization [CPU#0 -> CPU#12]: passed
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU13 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.370779] Counter synchronization [CPU#0 -> CPU#13]:
[    0.370779] Measured 1 cycles counter warp between CPUs
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU14 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.394779] Counter synchronization [CPU#0 -> CPU#14]: passed
[    0.006779] Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
[    0.006779] Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
[    0.006779] MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
[    0.006779] CPU15 revision is: 0001b031 (MIPS I6500)
[    0.006779] FPU revision is: 20f30320
[    0.006779] MSA revision is: 00000320
[    0.414779] Counter synchronization [CPU#0 -> CPU#15]:
[    0.414779] Measured 9 cycles counter warp between CPUs
[    1.442790] CPU16: failed to start
[    1.446881] No online CPU in core 0 to start CPU17
[    2.466789] CPU17: failed to start
[    2.471011] No online CPU in core 0 to start CPU18
[    3.490787] CPU18: failed to start
[    3.491263] No online CPU in core 0 to start CPU19
[    4.514787] CPU19: failed to start
[    5.538789] CPU20: failed to start
[    5.542939] No online CPU in core 1 to start CPU21
[    6.562787] CPU21: failed to start
[    6.567002] No online CPU in core 1 to start CPU22
[    7.586787] CPU22: failed to start
[    7.587290] No online CPU in core 1 to start CPU23
[    8.610788] CPU23: failed to start
[    9.634788] CPU24: failed to start
[    9.635298] No online CPU in core 2 to start CPU25
[   10.658787] CPU25: failed to start
[   10.662945] No online CPU in core 2 to start CPU26
[   11.682788] CPU26: failed to start
[   11.683291] No online CPU in core 2 to start CPU27
[   12.706788] CPU27: failed to start
[   13.730790] CPU28: failed to start
[   13.734869] No online CPU in core 3 to start CPU29
[   14.754788] CPU29: failed to start
[   14.758958] No online CPU in core 3 to start CPU30
[   15.778788] CPU30: failed to start
[   15.782954] No online CPU in core 3 to start CPU31
[   16.802788] CPU31: failed to start
[   16.802862] smp: Brought up 1 node, 16 CPUs
[   16.803221] Memory: 4126256K/4194304K available (11685K kernel code, 1724K rwdata, 2728K rodata, 5536K init, 474K bss, 53120K reserved, 0K cma-reserved)
[   16.803222] devtmpfs: initialized
[   16.804034] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
[   16.804034] futex hash table entries: 8192 (order: 6, 1048576 bytes, linear)
[   16.804034] pinctrl core: initialized pinctrl subsystem
[   16.807011] NET: Registered PF_NETLINK/PF_ROUTE protocol family
[   16.807553] Serial: AMBA PL011 UART driver
[   16.807966] d3331000.serial: ttyAMA0 at MMIO 0xd3331000 (irq = 74, base_baud = 0) is a PL011 rev3
[   16.808039] printk: legacy console [ttyAMA0] enabled
[   16.935278] SCSI subsystem initialized
[   16.936329] vgaarb: loaded
[   16.936329] clocksource: Switched to clocksource GIC
[   16.941569] NET: Registered PF_INET protocol family
[   16.942513] IP idents hash table entries: 65536 (order: 5, 524288 bytes, linear)
[   16.946756] tcp_listen_portaddr_hash hash table entries: 2048 (order: 1, 32768 bytes, linear)
[   16.947838] Table-perturb hash table entries: 65536 (order: 4, 262144 bytes, linear)
[   16.948761] TCP established hash table entries: 32768 (order: 4, 262144 bytes, linear)
[   16.949897] TCP bind hash table entries: 32768 (order: 6, 1048576 bytes, linear)
[   16.951403] TCP: Hash tables configured (established 32768 bind 32768)
[   16.952424] UDP hash table entries: 2048 (order: 2, 65536 bytes, linear)
[   16.953276] UDP-Lite hash table entries: 2048 (order: 2, 65536 bytes, linear)
[   16.954387] NET: Registered PF_UNIX/PF_LOCAL protocol family
[   16.956053] RPC: Registered named UNIX socket transport module.
[   16.956783] RPC: Registered udp transport module.
[   16.957338] RPC: Registered tcp transport module.
[   16.957893] RPC: Registered tcp-with-tls transport module.
[   16.958536] RPC: Registered tcp NFSv4.1 backchannel transport module.
[   16.959328] PCI: CLS 0 bytes, default 64
[   16.961807] workingset: timestamp_bits=46 max_order=18 bucket_order=0
[   16.963666] NFS: Registering the id_resolver key type
[   16.964329] Key type id_resolver registered
[   16.964843] Key type id_legacy registered
[   16.965371] nfs4filelayout_init: NFSv4 File Layout Driver Registering...
[   16.966173] nfs4flexfilelayout_init: NFSv4 Flexfile Layout Driver Registering...
[   16.967127] fuse: init (API version 7.41)
[   16.968175] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 254)
[   16.969091] io scheduler mq-deadline registered
[   16.969646] io scheduler kyber registered
[   16.970208] io scheduler bfq registered
[   17.042855] CAN device driver interface
[   17.043539] sdhci: Secure Digital Host Controller Interface driver
[   17.044259] sdhci: Copyright(c) Pierre Ossman
[   17.045205] NET: Registered PF_INET6 protocol family
[   17.047131] Segment Routing with IPv6
[   17.047637] In-situ OAM (IOAM) with IPv6
[   17.048151] sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver
[   17.049323] NET: Registered PF_PACKET protocol family
[   17.049938] NET: Registered PF_KEY protocol family
[   17.050500] can: controller area network core
[   17.051113] NET: Registered PF_CAN protocol family
[   17.051687] can: raw protocol
[   17.052049] can: broadcast manager protocol
[   17.052545] can: netlink gateway - max_hops=1
[   17.053202] Key type dns_resolver registered
[   17.056915] sched_clock: Marking stable (17052019968, 2779751)->(661362667, 16393437052)
[   17.057977] registered taskstats version 1
[   17.063277] Key type .fscrypt registered
[   17.063776] Key type fscrypt-provisioning registered
[   17.618713] clk: Disabling unused clocks
[   17.639990] Freeing unused kernel image (initmem) memory: 5536K
[   17.640719] This architecture does not have kernel memory protection.
[   17.641473] Run /init as init process

Gregory

Re: [PATCH v6 0/9] MIPS: Support I6500 multi-cluster configuration

[Gregory CLEMENT]

[-- Attachment #1: Type: text/plain, Size: 14181 bytes --]

Hello Aleksandar,

> Taken from Paul Burton MIPS repo with minor changes from Chao-ying Fu.
> Tested with 64r6el_defconfig on Boston board in 2 cluster/2 VPU and
> 1 cluster/4 VPU configurations.

With all three patches applied, I was able to get my second CPU cluster
working.

If you re-send the series with these three patches included, I'd be
happy to add my Tested-by tag.

Here are some relevant extracts from the boot log:

Linux version 6.12.0-rc2-00015-g44e960e85e4b (gclement@BLaptop)
(mips-img-linux-gnu-gcc (Codescape GNU Tools 2021.09-01 for MIPS IMG
Linux) 11.2.0, GNU ld (Codescape GNU Tools 2021.09-01 for MIPS IMG
Linux) 2.31.1) #368 SMP Fri Oct 18 14:53:35 CEST 2024
[...]
VP topology {4,4,4,4},{4,4,4,4} total 32
[...]
HCI (Hardware Cache Init for the L2 cache) in GCR_L2_RAM_CONFIG from the CM3 is broken
MMID allocator initialised with 65536 entries
rcu: Hierarchical SRCU implementation.
rcu:    Max phase no-delay instances is 1000.
Timer migration: 2 hierarchy levels; 8 children per group; 2 crossnode level
smp: Bringing up secondary CPUs ...
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU1 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#1]:
Measured 24 cycles counter warp between CPUs
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU2 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#2]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU3 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#3]:
Measured 2 cycles counter warp between CPUs
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU4 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#4]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU5 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#5]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU6 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#6]:
Measured 2 cycles counter warp between CPUs
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU7 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#7]:
Measured 16 cycles counter warp between CPUs
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU8 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#8]:
Measured 4 cycles counter warp between CPUs
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU9 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#9]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU10 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#10]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU11 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#11]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU12 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#12]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU13 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#13]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU14 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#14]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU15 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#15]:
Measured 2 cycles counter warp between CPUs
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU16 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#16]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU17 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#17]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU18 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#18]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU19 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#19]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU20 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#20]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU21 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#21]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU22 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#22]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU23 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#23]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU24 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#24]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU25 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#25]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU26 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#26]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU27 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#27]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU28 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#28]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU29 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#29]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU30 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#30]: passed
Primary instruction cache 64kB, VIPT, 4-way, linesize 64 bytes.
Primary data cache 32kB, 4-way, PIPT, no aliases, linesize 64 bytes
MIPS secondary cache 1024kB, 16-way, linesize 64 bytes.
CPU31 revision is: 0001b031 (MIPS I6500)
FPU revision is: 20f30320
MSA revision is: 00000320
Counter synchronization [CPU#0 -> CPU#31]: passed
smp: Brought up 1 node, 32 CPUs

Regards,

Gregory


>
> v6:
>  - Re-base onto the master branch, with no functionality impact.
>  - Correct the issue reported by the kernel test robot.
>
> v5:
>  - Drop FDC related changes (patches 12, 13, and 14).
>  - Apply changes suggested by Thomas Gleixner (patches 3 and 4).
>  - Add #include <linux/cpumask.h> to patch 1, suggested by Thomas Bogendoerfer.
>  - Add Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> for the patch 08/11.
>  - Add Tested-by: Serge Semin <fancer.lancer@gmail.com> for the entire series.
>  - Correct some commit messages.
>
> v4:
>  - Re-base onto the master branch, with no functionality impact.
>  - Refactor MIPS FDC driver in the context of multicluster support.
>
> v3:
>  - Add Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com> for the patch 02/12.
>  - Add the changes requested by Marc Zyngier for the 3/12 patch.
>  - Remove the patch 11/12 (a consequence of a discussion between Jiaxun Yang
>    and Marc Zyngier.
>  - Re-base onto the master branch, with no functionality impact.
>
> v2:
>  - Apply correct Signed-off-by to avoid confusion.
>
> Chao-ying Fu (1):
>   irqchip/mips-gic: Setup defaults in each cluster
>
> Paul Burton (8):
>   irqchip/mips-gic: Introduce for_each_online_cpu_gic()
>   irqchip/mips-gic: Support multi-cluster in for_each_online_cpu_gic()
>   irqchip/mips-gic: Multi-cluster support
>   clocksource: mips-gic-timer: Always use cluster 0 counter as
>     clocksource
>   clocksource: mips-gic-timer: Enable counter when CPUs start
>   MIPS: pm-cps: Use per-CPU variables as per-CPU, not per-core
>   MIPS: CPS: Introduce struct cluster_boot_config
>   MIPS: CPS: Boot CPUs in secondary clusters
>
>  arch/mips/include/asm/mips-cm.h      |  18 ++
>  arch/mips/include/asm/smp-cps.h      |   7 +-
>  arch/mips/kernel/asm-offsets.c       |   3 +
>  arch/mips/kernel/cps-vec.S           |  19 +-
>  arch/mips/kernel/mips-cm.c           |   4 +-
>  arch/mips/kernel/pm-cps.c            |  35 ++--
>  arch/mips/kernel/smp-cps.c           | 285 ++++++++++++++++++++++-----
>  drivers/clocksource/mips-gic-timer.c |  45 ++++-
>  drivers/irqchip/Kconfig              |   1 +
>  drivers/irqchip/irq-mips-gic.c       | 257 ++++++++++++++++++++----
>  10 files changed, 560 insertions(+), 114 deletions(-)
>
> -- 
> 2.25.1


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-dt-bindings-mips-cpu-Add-property-for-broken-HCI-inf.patch --]
[-- Type: text/x-diff, Size: 1284 bytes --]

From 5fcfc8fc29c3c16be3bed4d8502e0b9fabbdac5c Mon Sep 17 00:00:00 2001
From: Gregory CLEMENT <gregory.clement@bootlin.com>
Date: Thu, 17 Oct 2024 16:47:25 +0200
Subject: [PATCH 1/3] dt-bindings: mips: cpu: Add property for broken HCI
 information

Some CM3.5 reports show that Hardware Cache Initialization is
complete, but in reality it's not the case. They also incorrectly
indicate that Hardware Cache Initialization is supported. This
optional property allows warning about this broken feature that cannot
be detected at runtime.

Signed-off-by: Gregory CLEMENT <gregory.clement@bootlin.com>
---
 Documentation/devicetree/bindings/mips/cpus.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Documentation/devicetree/bindings/mips/cpus.yaml b/Documentation/devicetree/bindings/mips/cpus.yaml
index a85137add6689..57e93c07ab1be 100644
--- a/Documentation/devicetree/bindings/mips/cpus.yaml
+++ b/Documentation/devicetree/bindings/mips/cpus.yaml
@@ -47,6 +47,12 @@ properties:
   clocks:
     maxItems: 1
 
+  cm3-l2-config-hci-broken:
+    type: boolean
+    description:
+      If present, indicates that the HCI (Hardware Cache Initialization)
+      information for the L2 cache in multi-cluster configuration is broken.
+
   device_type: true
 
 allOf:
-- 
2.45.2


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #3: 0002-MIPS-CPS-Support-broken-HCI-for-multicluster.patch --]
[-- Type: text/x-diff, Size: 2456 bytes --]

From ea0744030f1ec47116f0a28f2d5c1bf6f9e3597e Mon Sep 17 00:00:00 2001
From: Gregory CLEMENT <gregory.clement@bootlin.com>
Date: Thu, 17 Oct 2024 16:57:09 +0200
Subject: [PATCH 2/3] MIPS: CPS: Support broken HCI for multicluster

Some CM3.5 devices incorrectly report that hardware cache
initialization has completed, and also claim to support hardware cache
initialization when they don't actually do so. This commit fixes this
issue by retrieving the correct information from the device tree and
allowing the system to bypass the hardware cache initialization
step. Instead, it relies on manual operation. As a result, multi-user
support is now possible for these CPUs.

Signed-off-by: Gregory CLEMENT <gregory.clement@bootlin.com>
---
 arch/mips/kernel/smp-cps.c | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c
index 34e977ea8a69c..75b26865a17f7 100644
--- a/arch/mips/kernel/smp-cps.c
+++ b/arch/mips/kernel/smp-cps.c
@@ -39,6 +39,7 @@ UASM_L_LA(_not_nmi)
 static uint64_t core_entry_reg;
 static phys_addr_t cps_vec_pa;
 
+static bool l2_hci_broken;
 struct cluster_boot_config *mips_cps_cluster_bootcfg;
 
 static void power_up_other_cluster(unsigned int cluster)
@@ -262,6 +263,22 @@ static void __init cps_smp_setup(void)
 #endif /* CONFIG_MIPS_MT_FPAFF */
 }
 
+static void __init check_hci_quirk(void)
+{
+	struct device_node *np;
+
+	np = of_cpu_device_node_get(0);
+	if (!np) {
+		pr_debug("%s: No cpu node in the device tree\n", __func__);
+		return;
+	}
+
+	if (of_property_read_bool(np, "cm3-l2-config-hci-broken")) {
+		pr_info("HCI (Hardware Cache Init for the L2 cache) in GCR_L2_RAM_CONFIG from the CM3 is broken");
+		l2_hci_broken = true;
+	}
+}
+
 static void __init cps_prepare_cpus(unsigned int max_cpus)
 {
 	unsigned int nclusters, ncores, core_vpes, c, cl, cca;
@@ -315,6 +332,9 @@ static void __init cps_prepare_cpus(unsigned int max_cpus)
 					   sizeof(*mips_cps_cluster_bootcfg),
 					   GFP_KERNEL);
 
+	if (nclusters > 1)
+		check_hci_quirk();
+
 	for (cl = 0; cl < nclusters; cl++) {
 		/* Allocate core boot configuration structs */
 		ncores = mips_cps_numcores(cl);
@@ -376,7 +396,7 @@ static void init_cluster_l2(void)
 {
 	u32 l2_cfg, l2sm_cop, result;
 
-	while (1) {
+	while (!l2_hci_broken) {
 		l2_cfg = read_gcr_redir_l2_ram_config();
 
 		/* If HCI is not supported, use the state machine below */
-- 
2.45.2


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #4: 0003-MIPS-mobileye-dts-eyeq6h-Enable-cluster-support.patch --]
[-- Type: text/x-diff, Size: 974 bytes --]

From 44e960e85e4b24dae6a9c28e898338366ea16c72 Mon Sep 17 00:00:00 2001
From: Gregory CLEMENT <gregory.clement@bootlin.com>
Date: Fri, 18 Oct 2024 14:42:29 +0200
Subject: [PATCH 3/3] MIPS: mobileye: dts: eyeq6h: Enable cluster support

The CM3.5 device used in EyeQ6H SoCs incorrectly reports the status
for Hardware Cache Initialization (HCI). This commit adds a property
to acknowledge this issue, which enables the use of the second CPU
cluster.

Signed-off-by: Gregory CLEMENT <gregory.clement@bootlin.com>
---
 arch/mips/boot/dts/mobileye/eyeq6h.dtsi | 1 +
 1 file changed, 1 insertion(+)

diff --git a/arch/mips/boot/dts/mobileye/eyeq6h.dtsi b/arch/mips/boot/dts/mobileye/eyeq6h.dtsi
index 1db3c3cda2e39..4ea85dfd4eed9 100644
--- a/arch/mips/boot/dts/mobileye/eyeq6h.dtsi
+++ b/arch/mips/boot/dts/mobileye/eyeq6h.dtsi
@@ -18,6 +18,7 @@ cpu@0 {
 			compatible = "img,i6500";
 			reg = <0>;
 			clocks = <&occ_cpu>;
+			cm3-l2-config-hci-broken;
 		};
 	};
 
-- 
2.45.2