[PATCH v2 0/2] Fix timer initialization and Add support hotplug.

[PATCH v2 0/2] Fix timer initialization and Add support hotplug.

[Atish Patra]

The patch (1/2)fixes issues around timer initialization. This fix is
required for CPU hotplug to work. That's why they are clubbed into one
series. I can separate them if required.

Changes from v1 to v2:
1. Removed compiler warnings about unused variables.

Atish Patra (2):
  RISCV: Register clocksource and events correctly
  RISCV: Support cpu hotplug.

 arch/riscv/Kconfig                | 11 ++++++-
 arch/riscv/include/asm/csr.h      |  1 +
 arch/riscv/include/asm/smp.h      |  9 ++++--
 arch/riscv/kernel/head.S          | 12 +++++++
 arch/riscv/kernel/process.c       |  7 +++++
 arch/riscv/kernel/setup.c         | 17 ++++++++++
 arch/riscv/kernel/smpboot.c       | 66 +++++++++++++++++++++++++++++++++++++--
 arch/riscv/kernel/time.c          |  9 +-----
 arch/riscv/kernel/traps.c         |  6 ++--
 drivers/clocksource/riscv_timer.c | 44 +++++++++++++++++---------
 include/linux/cpuhotplug.h        |  1 +
 11 files changed, 150 insertions(+), 33 deletions(-)

-- 
2.7.4

[PATCH v2 1/2] RISCV: Register clocksource and events correctly

[Atish Patra]

Currently, timer_probe() is called for every cpu and clocksource
is registered multiple times for each cpu which is wrong.

Probe timer only once during init and register the clock source at
that time. Move the clock event registration cpu online notification
callback. Take this opportunity to remove redundant functions as well.

Signed-off-by: Atish Patra <atish.patra@wdc.com>
---
 arch/riscv/include/asm/smp.h      |  2 +-
 arch/riscv/kernel/time.c          |  9 +-------
 drivers/clocksource/riscv_timer.c | 44 ++++++++++++++++++++++++++-------------
 include/linux/cpuhotplug.h        |  1 +
 4 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index 85e4220..01b8df8 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -25,7 +25,7 @@
 #ifdef CONFIG_SMP
 
 /* SMP initialization hook for setup_arch */
-void __init init_clockevent(void);
+void init_clockevent(void);
 
 /* SMP initialization hook for setup_arch */
 void __init setup_smp(void);
diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c
index 67709cb..bcd3e76 100644
--- a/arch/riscv/kernel/time.c
+++ b/arch/riscv/kernel/time.c
@@ -39,13 +39,6 @@ void riscv_timer_interrupt(void)
 #endif
 }
 
-void __init init_clockevent(void)
-{
-	timer_probe();
-	csr_set(sie, SIE_STIE);
-}
-
-
 static long __init timebase_frequency(void)
 {
 	struct device_node *cpu;
@@ -65,5 +58,5 @@ void __init time_init(void)
 {
 	riscv_timebase = timebase_frequency();
 	lpj_fine = riscv_timebase / HZ;
-	init_clockevent();
+	timer_probe();
 }
diff --git a/drivers/clocksource/riscv_timer.c b/drivers/clocksource/riscv_timer.c
index 59a734c..8b45af2 100644
--- a/drivers/clocksource/riscv_timer.c
+++ b/drivers/clocksource/riscv_timer.c
@@ -17,6 +17,7 @@
 #include <linux/delay.h>
 #include <linux/timer_riscv.h>
 #include <linux/sched_clock.h>
+#include <linux/cpu.h>
 #include <asm/sbi.h>
 
 #define MINDELTA 100
@@ -71,16 +72,6 @@ DEFINE_PER_CPU(struct clocksource, riscv_clocksource) = {
 	.read = rdtime,
 };
 
-void timer_riscv_init(int cpu_id,
-		      unsigned long riscv_timebase,
-		      int (*next)(unsigned long, struct clock_event_device*))
-{
-	struct clock_event_device *ce = per_cpu_ptr(&riscv_clock_event, cpu_id);
-
-	ce->cpumask = cpumask_of(cpu_id);
-	clockevents_config_and_register(ce, riscv_timebase, MINDELTA, MAXDELTA);
-}
-
 static int hart_of_timer(struct device_node *dev)
 {
 	u32 hart;
@@ -100,21 +91,44 @@ static u64 notrace timer_riscv_sched_read(void)
 	return get_cycles64();
 }
 
+static int timer_riscv_starting_cpu(unsigned int cpu)
+{
+	struct clock_event_device *ce = per_cpu_ptr(&riscv_clock_event, cpu);
+
+	ce->cpumask = cpumask_of(cpu);
+	clockevents_config_and_register(ce, riscv_timebase, MINDELTA, MAXDELTA);
+	/* Enable timer interrupt for this cpu */
+	csr_set(sie, SIE_STIE);
+
+	return 0;
+}
+
+static int timer_riscv_dying_cpu(unsigned int cpu)
+{
+	/* Disable timer interrupt for this cpu */
+	csr_clear(sie, SIE_STIE);
+
+	return 0;
+}
+
 static int __init timer_riscv_init_dt(struct device_node *n)
 {
+	int err = 0;
 	int cpu_id = hart_of_timer(n);
-	struct clock_event_device *ce = per_cpu_ptr(&riscv_clock_event, cpu_id);
 	struct clocksource *cs = per_cpu_ptr(&riscv_clocksource, cpu_id);
 
 	if (cpu_id == smp_processor_id()) {
 		clocksource_register_hz(cs, riscv_timebase);
 		sched_clock_register(timer_riscv_sched_read, 64, riscv_timebase);
 
-		ce->cpumask = cpumask_of(cpu_id);
-		clockevents_config_and_register(ce, riscv_timebase, MINDELTA, MAXDELTA);
+		err = cpuhp_setup_state(CPUHP_AP_RISCV_TIMER_STARTING,
+			 "clockevents/riscv/timer:starting",
+			 timer_riscv_starting_cpu, timer_riscv_dying_cpu);
+		if (err)
+			pr_err("RISCV timer register failed [%d] for cpu = [%d]\n",
+			       err, cpu_id);
 	}
-
-	return 0;
+	return err;
 }
 
 TIMER_OF_DECLARE(riscv_timer, "riscv", timer_riscv_init_dt);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 1a32e55..c68f924 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -126,6 +126,7 @@ enum cpuhp_state {
 	CPUHP_AP_MARCO_TIMER_STARTING,
 	CPUHP_AP_MIPS_GIC_TIMER_STARTING,
 	CPUHP_AP_ARC_TIMER_STARTING,
+	CPUHP_AP_RISCV_TIMER_STARTING,
 	CPUHP_AP_KVM_STARTING,
 	CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING,
 	CPUHP_AP_KVM_ARM_VGIC_STARTING,
-- 
2.7.4

[PATCH v2 2/2] RISCV: Support cpu hotplug.

[Atish Patra]

This patch enable support for cpu hotplug in RISC-V.

In absensece of generic cpu stop functions, WFI is used
to put the cpu in low power state during offline. An IPI
is sent to bring it out of WFI during online operation.

Tested both on QEMU and HighFive Unleashed board with
4 cpus. Test result follows.

$ echo 0 > /sys/devices/system/cpu/cpu2/online
[   31.828562] CPU2: shutdown
$ cat /proc/cpuinfo
hart    : 1
isa     : rv64imafdc
mmu     : sv39
uarch   : sifive,rocket0

hart    : 3
isa     : rv64imafdc
mmu     : sv39
uarch   : sifive,rocket0

hart    : 4
isa     : rv64imafdc
mmu     : sv39
uarch   : sifive,rocket0

$ echo 0 > /sys/devices/system/cpu/cpu4/online
[   52.968495] CPU4: shutdown
$ cat /proc/cpuinfo
hart    : 1
isa     : rv64imafdc
mmu     : sv39
uarch   : sifive,rocket0

hart    : 3
isa     : rv64imafdc
mmu     : sv39
uarch   : sifive,rocket0

$ echo 1 > /sys/devices/system/cpu/cpu4/online
[   64.298250] CPU4: online
$ cat /proc/cpuinfo
hart    : 1
isa     : rv64imafdc
mmu     : sv39
uarch   : sifive,rocket0

hart    : 3
isa     : rv64imafdc
mmu     : sv39
uarch   : sifive,rocket0

hart    : 4
isa     : rv64imafdc
mmu     : sv39
uarch   : sifive,rocket0

Signed-off-by: Atish Patra <atish.patra@wdc.com>
---
 arch/riscv/Kconfig           | 11 +++++++-
 arch/riscv/include/asm/csr.h |  1 +
 arch/riscv/include/asm/smp.h |  9 ++++--
 arch/riscv/kernel/head.S     | 12 ++++++++
 arch/riscv/kernel/process.c  |  7 +++++
 arch/riscv/kernel/setup.c    | 17 ++++++++++++
 arch/riscv/kernel/smpboot.c  | 66 ++++++++++++++++++++++++++++++++++++++++++--
 arch/riscv/kernel/traps.c    |  6 ++--
 8 files changed, 119 insertions(+), 10 deletions(-)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 0c57fac..d1f7cd2 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -14,7 +14,6 @@ config RISCV
 	select CLONE_BACKWARDS
 	select COMMON_CLK
 	select GENERIC_CLOCKEVENTS
-	select GENERIC_CPU_DEVICES
 	select GENERIC_IRQ_SHOW
 	select GENERIC_PCI_IOMAP
 	select GENERIC_STRNCPY_FROM_USER
@@ -187,6 +186,16 @@ config NR_CPUS
 	depends on SMP
 	default "8"
 
+config HOTPLUG_CPU
+	bool "Support for hot-pluggable CPUs"
+	select GENERIC_IRQ_MIGRATION
+	help
+
+	  Say Y here to experiment with turning CPUs off and on.  CPUs
+	  can be controlled through /sys/devices/system/cpu.
+
+	  Say N if you want to disable CPU hotplug.
+
 config CPU_SUPPORTS_32BIT_KERNEL
 	bool
 config CPU_SUPPORTS_64BIT_KERNEL
diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
index 421fa35..1baf8e0 100644
--- a/arch/riscv/include/asm/csr.h
+++ b/arch/riscv/include/asm/csr.h
@@ -54,6 +54,7 @@
 /* Interrupt Enable and Interrupt Pending flags */
 #define SIE_SSIE _AC(0x00000002, UL) /* Software Interrupt Enable */
 #define SIE_STIE _AC(0x00000020, UL) /* Timer Interrupt Enable */
+#define SIE_SEIE _AC(0x000000200, UL) /* External Interrupt Enable */
 
 #define EXC_INST_MISALIGNED     0
 #define EXC_INST_ACCESS         1
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index 01b8df8..e78b7f1 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -25,9 +25,6 @@
 #ifdef CONFIG_SMP
 
 /* SMP initialization hook for setup_arch */
-void init_clockevent(void);
-
-/* SMP initialization hook for setup_arch */
 void __init setup_smp(void);
 
 /* Hook for the generic smp_call_function_many() routine. */
@@ -47,6 +44,12 @@ void arch_send_call_function_single_ipi(int cpu);
 /* Interprocessor interrupt handler */
 irqreturn_t handle_ipi(void);
 
+#ifdef CONFIG_HOTPLUG_CPU
+extern int __cpu_disable(void);
+extern void __cpu_die(unsigned int cpu);
+extern void cpu_play_dead(void);
+extern void boot_sec_cpu(void);
+#endif
 #endif /* CONFIG_SMP */
 
 #endif /* _ASM_RISCV_SMP_H */
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index 226eeb1..63d478d 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -149,6 +149,18 @@ relocate:
 	j .Lsecondary_park
 END(_start)
 
+.section .text
+.global boot_sec_cpu
+
+boot_sec_cpu:
+	/* clear all pending flags */
+	csrw sip, zero
+	/* Mask all interrupts */
+	csrw sie, zero
+	fence
+
+	tail smp_callin
+
 __PAGE_ALIGNED_BSS
 	/* Empty zero page */
 	.balign PAGE_SIZE
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index d74d4ad..c5e2234 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -42,6 +42,13 @@ void arch_cpu_idle(void)
 	local_irq_enable();
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+void arch_cpu_idle_dead(void)
+{
+	cpu_play_dead();
+}
+#endif
+
 void show_regs(struct pt_regs *regs)
 {
 	show_regs_print_info(KERN_DEFAULT);
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index a1d5853..4ef8a8b 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -81,6 +81,7 @@ EXPORT_SYMBOL(empty_zero_page);
 
 /* The lucky hart to first increment this variable will boot the other cores */
 atomic_t hart_lottery;
+static DEFINE_PER_CPU(struct cpu, cpu_devices);
 
 #ifdef CONFIG_BLK_DEV_INITRD
 static void __init setup_initrd(void)
@@ -248,6 +249,22 @@ void __init setup_arch(char **cmdline_p)
 	riscv_fill_hwcap();
 }
 
+static int __init topology_init(void)
+{
+	int i;
+
+	for_each_possible_cpu(i) {
+		struct cpu *cpu = &per_cpu(cpu_devices, i);
+#ifdef CONFIG_HOTPLUG_CPU
+		cpu->hotpluggable = 1;
+#endif
+		register_cpu(cpu, i);
+	}
+
+	return 0;
+}
+subsys_initcall(topology_init);
+
 static int __init riscv_device_init(void)
 {
 	return of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
index f741458..d7c66a1 100644
--- a/arch/riscv/kernel/smpboot.c
+++ b/arch/riscv/kernel/smpboot.c
@@ -30,6 +30,7 @@
 #include <linux/irq.h>
 #include <linux/of.h>
 #include <linux/sched/task_stack.h>
+#include <linux/sched/hotplug.h>
 #include <asm/irq.h>
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
@@ -82,9 +83,11 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 	__cpu_up_stack_pointer[cpu] = task_stack_page(tidle) + THREAD_SIZE;
 	__cpu_up_task_pointer[cpu] = tidle;
 
+	arch_send_call_function_single_ipi(cpu);
 	while (!cpu_online(cpu))
 		cpu_relax();
 
+	pr_notice("CPU%u: online\n", cpu);
 	return 0;
 }
 
@@ -92,10 +95,68 @@ void __init smp_cpus_done(unsigned int max_cpus)
 {
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * __cpu_disable runs on the processor to be shutdown.
+ */
+int __cpu_disable(void)
+{
+	unsigned int cpu = smp_processor_id();
+
+	set_cpu_online(cpu, false);
+	irq_migrate_all_off_this_cpu();
+
+	return 0;
+}
+/*
+ * called on the thread which is asking for a CPU to be shutdown -
+ * waits until shutdown has completed, or it is timed out.
+ */
+void __cpu_die(unsigned int cpu)
+{
+	if (!cpu_wait_death(cpu, 5)) {
+		pr_err("CPU %u: didn't die\n", cpu);
+		return;
+	}
+	pr_notice("CPU%u: shutdown\n", cpu);
+}
+/*
+ * Called from the idle thread for the CPU which has been shutdown.
+ */
+void cpu_play_dead(void)
+{
+	int sipval, sieval, scauseval;
+
+	idle_task_exit();
+
+	(void)cpu_report_death();
+
+	/* Do not disable software interrupt to restart cpu after WFI */
+	csr_clear(sie, SIE_STIE | SIE_SEIE);
+	//local_irq_disable();
+	/* clear all pending flags */
+	csr_write(sip, 0);
+	/* clear any previous scause data */
+	csr_write(scause, 0);
+
+	do {
+		wait_for_interrupt();
+		sipval = csr_read(sip);
+		sieval = csr_read(sie);
+		scauseval = csr_read(scause);
+	/* only break if wfi returns for an enabled interrupt */
+	} while ((sipval & sieval) == 0 &&
+		 scauseval != INTERRUPT_CAUSE_SOFTWARE);
+
+	boot_sec_cpu();
+}
+
+
+#endif
 /*
  * C entry point for a secondary processor.
  */
-asmlinkage void __init smp_callin(void)
+asmlinkage void smp_callin(void)
 {
 	struct mm_struct *mm = &init_mm;
 
@@ -104,9 +165,8 @@ asmlinkage void __init smp_callin(void)
 	current->active_mm = mm;
 
 	trap_init();
-	init_clockevent();
 	notify_cpu_starting(smp_processor_id());
-	set_cpu_online(smp_processor_id(), 1);
+	set_cpu_online(smp_processor_id(), true);
 	local_flush_tlb_all();
 	local_irq_enable();
 	preempt_disable();
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 93132cb..b99adf1 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -166,7 +166,7 @@ int is_valid_bugaddr(unsigned long pc)
 }
 #endif /* CONFIG_GENERIC_BUG */
 
-void __init trap_init(void)
+void trap_init(void)
 {
 	/*
 	 * Set sup0 scratch register to 0, indicating to exception vector
@@ -175,6 +175,6 @@ void __init trap_init(void)
 	csr_write(sscratch, 0);
 	/* Set the exception vector address */
 	csr_write(stvec, &handle_exception);
-	/* Enable all interrupts */
-	csr_write(sie, -1);
+	/* Enable all interrupts but timer interrupt*/
+	csr_set(sie, SIE_SSIE | SIE_SEIE);
 }
-- 
2.7.4

[PATCH v2 0/2] Fix timer initialization and Add support hotplug.

[Palmer Dabbelt]

On Wed, 09 May 2018 15:02:20 PDT (-0700), atish.patra at wdc.com wrote:
> The patch (1/2)fixes issues around timer initialization. This fix is
> required for CPU hotplug to work. That's why they are clubbed into one
> series. I can separate them if required.
>
> Changes from v1 to v2:
> 1. Removed compiler warnings about unused variables.
>
> Atish Patra (2):
>   RISCV: Register clocksource and events correctly
>   RISCV: Support cpu hotplug.
>
>  arch/riscv/Kconfig                | 11 ++++++-
>  arch/riscv/include/asm/csr.h      |  1 +
>  arch/riscv/include/asm/smp.h      |  9 ++++--
>  arch/riscv/kernel/head.S          | 12 +++++++
>  arch/riscv/kernel/process.c       |  7 +++++
>  arch/riscv/kernel/setup.c         | 17 ++++++++++
>  arch/riscv/kernel/smpboot.c       | 66 +++++++++++++++++++++++++++++++++++++--
>  arch/riscv/kernel/time.c          |  9 +-----
>  arch/riscv/kernel/traps.c         |  6 ++--
>  drivers/clocksource/riscv_timer.c | 44 +++++++++++++++++---------
>  include/linux/cpuhotplug.h        |  1 +
>  11 files changed, 150 insertions(+), 33 deletions(-)

Sorry I'm a bit slow here: I was meaning to fix this up.  I think this patch 
set was made against something like riscv-all, which has a bunch of additional 
cruft in it.  Do you mind updating it to be against Linus' master?

[PATCH v2 0/2] Fix timer initialization and Add support hotplug.

[Atish Patra]

On 5/18/18 5:37 PM, Palmer Dabbelt wrote:
> On Wed, 09 May 2018 15:02:20 PDT (-0700), atish.patra at wdc.com wrote:
>> The patch (1/2)fixes issues around timer initialization. This fix is
>> required for CPU hotplug to work. That's why they are clubbed into one
>> series. I can separate them if required.
>>
>> Changes from v1 to v2:
>> 1. Removed compiler warnings about unused variables.
>>
>> Atish Patra (2):
>>    RISCV: Register clocksource and events correctly
>>    RISCV: Support cpu hotplug.
>>
>>   arch/riscv/Kconfig                | 11 ++++++-
>>   arch/riscv/include/asm/csr.h      |  1 +
>>   arch/riscv/include/asm/smp.h      |  9 ++++--
>>   arch/riscv/kernel/head.S          | 12 +++++++
>>   arch/riscv/kernel/process.c       |  7 +++++
>>   arch/riscv/kernel/setup.c         | 17 ++++++++++
>>   arch/riscv/kernel/smpboot.c       | 66 +++++++++++++++++++++++++++++++++++++--
>>   arch/riscv/kernel/time.c          |  9 +-----
>>   arch/riscv/kernel/traps.c         |  6 ++--
>>   drivers/clocksource/riscv_timer.c | 44 +++++++++++++++++---------
>>   include/linux/cpuhotplug.h        |  1 +
>>   11 files changed, 150 insertions(+), 33 deletions(-)
> 
> Sorry I'm a bit slow here: I was meaning to fix this up.  I think this patch
> set was made against something like riscv-all, which has a bunch of additional
> cruft in it.  

Yeah. It is based on riscv-all because my changes also depend on initial 
timer patches.

Do you mind updating it to be against Linus' master?

I can do it. But my patches won't work without the initial timer 
patches. I am not sure how to proceed in that case.

Do you want only hotplug patch (2/2) or I can squash the entire 
wip-timer patches into couple of sane ones and post them to mailing list ?

Regards,
Atish

> 

[PATCH v2 0/2] Fix timer initialization and Add support hotplug.

[Atish Patra]

On 5/21/18 1:43 PM, Atish Patra wrote:
> On 5/18/18 5:37 PM, Palmer Dabbelt wrote:
>> On Wed, 09 May 2018 15:02:20 PDT (-0700), atish.patra at wdc.com wrote:
>>> The patch (1/2)fixes issues around timer initialization. This fix is
>>> required for CPU hotplug to work. That's why they are clubbed into one
>>> series. I can separate them if required.
>>>
>>> Changes from v1 to v2:
>>> 1. Removed compiler warnings about unused variables.
>>>
>>> Atish Patra (2):
>>>     RISCV: Register clocksource and events correctly
>>>     RISCV: Support cpu hotplug.
>>>
>>>    arch/riscv/Kconfig                | 11 ++++++-
>>>    arch/riscv/include/asm/csr.h      |  1 +
>>>    arch/riscv/include/asm/smp.h      |  9 ++++--
>>>    arch/riscv/kernel/head.S          | 12 +++++++
>>>    arch/riscv/kernel/process.c       |  7 +++++
>>>    arch/riscv/kernel/setup.c         | 17 ++++++++++
>>>    arch/riscv/kernel/smpboot.c       | 66 +++++++++++++++++++++++++++++++++++++--
>>>    arch/riscv/kernel/time.c          |  9 +-----
>>>    arch/riscv/kernel/traps.c         |  6 ++--
>>>    drivers/clocksource/riscv_timer.c | 44 +++++++++++++++++---------
>>>    include/linux/cpuhotplug.h        |  1 +
>>>    11 files changed, 150 insertions(+), 33 deletions(-)
>>
>> Sorry I'm a bit slow here: I was meaning to fix this up.  I think this patch
>> set was made against something like riscv-all, which has a bunch of additional
>> cruft in it.
> 
> Yeah. It is based on riscv-all because my changes also depend on initial
> timer patches.
> 
> Do you mind updating it to be against Linus' master?
> 
> I can do it. But my patches won't work without the initial timer
> patches. I am not sure how to proceed in that case.
> 
> Do you want only hotplug patch (2/2) or I can squash the entire
> wip-timer patches into couple of sane ones and post them to mailing list ?
> 

As per our discussion, these patches has to be part of wip-timer until
interrupt issues are straightened and ready for upstream. All the timer 
related patches will go on top of it.

> Regards,
> Atish
> 
>>
> 
> 
I was reviewing the base timer patches. I have a question about the patch.
clocksource: New RISC-V SBI timer driver (ea8ec64).

The riscv_timer.h has a well defined per hart timer explanation in the 
comment and following function declaration. However, I can't find any 
reference to following functions.

+void timer_riscv_init(int cpu_id,
+		      unsigned long riscv_timebase,
+		      int (*next_event)(unsigned long, struct clock_event_device *));
+
+void clocksource_riscv_init(unsigned long long (*rdtime)(struct 
clocksource *));

Are these redundant code or I missed something ?

If you agree with me, I can send a v3 removing these (will move the 
comment to riscv_timer.c).


Regards,
Atish

[PATCH v2 0/2] Fix timer initialization and Add support hotplug.

[Palmer Dabbelt]

On Mon, 28 May 2018 18:15:03 PDT (-0700), atish.patra at wdc.com wrote:
> On 5/21/18 1:43 PM, Atish Patra wrote:
>> On 5/18/18 5:37 PM, Palmer Dabbelt wrote:
>>> On Wed, 09 May 2018 15:02:20 PDT (-0700), atish.patra at wdc.com wrote:
>>>> The patch (1/2)fixes issues around timer initialization. This fix is
>>>> required for CPU hotplug to work. That's why they are clubbed into one
>>>> series. I can separate them if required.
>>>>
>>>> Changes from v1 to v2:
>>>> 1. Removed compiler warnings about unused variables.
>>>>
>>>> Atish Patra (2):
>>>>     RISCV: Register clocksource and events correctly
>>>>     RISCV: Support cpu hotplug.
>>>>
>>>>    arch/riscv/Kconfig                | 11 ++++++-
>>>>    arch/riscv/include/asm/csr.h      |  1 +
>>>>    arch/riscv/include/asm/smp.h      |  9 ++++--
>>>>    arch/riscv/kernel/head.S          | 12 +++++++
>>>>    arch/riscv/kernel/process.c       |  7 +++++
>>>>    arch/riscv/kernel/setup.c         | 17 ++++++++++
>>>>    arch/riscv/kernel/smpboot.c       | 66 +++++++++++++++++++++++++++++++++++++--
>>>>    arch/riscv/kernel/time.c          |  9 +-----
>>>>    arch/riscv/kernel/traps.c         |  6 ++--
>>>>    drivers/clocksource/riscv_timer.c | 44 +++++++++++++++++---------
>>>>    include/linux/cpuhotplug.h        |  1 +
>>>>    11 files changed, 150 insertions(+), 33 deletions(-)
>>>
>>> Sorry I'm a bit slow here: I was meaning to fix this up.  I think this patch
>>> set was made against something like riscv-all, which has a bunch of additional
>>> cruft in it.
>>
>> Yeah. It is based on riscv-all because my changes also depend on initial
>> timer patches.
>>
>> Do you mind updating it to be against Linus' master?
>>
>> I can do it. But my patches won't work without the initial timer
>> patches. I am not sure how to proceed in that case.
>>
>> Do you want only hotplug patch (2/2) or I can squash the entire
>> wip-timer patches into couple of sane ones and post them to mailing list ?
>>
>
> As per our discussion, these patches has to be part of wip-timer until
> interrupt issues are straightened and ready for upstream. All the timer
> related patches will go on top of it.
>
>> Regards,
>> Atish
>>
>>>
>>
>>
> I was reviewing the base timer patches. I have a question about the patch.
> clocksource: New RISC-V SBI timer driver (ea8ec64).
>
> The riscv_timer.h has a well defined per hart timer explanation in the
> comment and following function declaration. However, I can't find any
> reference to following functions.
>
> +void timer_riscv_init(int cpu_id,
> +		      unsigned long riscv_timebase,
> +		      int (*next_event)(unsigned long, struct clock_event_device *));
> +
> +void clocksource_riscv_init(unsigned long long (*rdtime)(struct
> clocksource *));
>
> Are these redundant code or I missed something ?
>
> If you agree with me, I can send a v3 removing these (will move the
> comment to riscv_timer.c).

I see these in riscv_timer.c on my copy of wip-timer

    e56e8bd48eca (HEAD -> wip-timer, kernel.org-palmer/wip-timer) RISCV: Disable timer interrupt in handler to fix nohz.
    2b030aa93236 RISCV: Support cpu hotplug.
    1b35cd828f29 RISCV: Register clocksource and events correctly
    30540340a51e riscv_timer: add support for sched_clock
    d6ad991df6fa clocksource: RISC-V: Match up with the current DTS
    c48b6c686f01 Revert "clocksource: riscv_timer: split boot and secondary cpu timer init"
    baf107699d0f RISC-V: Support per-hart timebase-frequency
    1e955d9ecfbe clocksource: riscv_timer: split boot and secondary cpu timer init
    ea8ec6497122 clocksource: New RISC-V SBI timer driver
    60cc43fc8884 (tag: v4.17-rc1, wip-cpu_hotplog) Linux 4.17-rc1

did I screw something up?

[PATCH v2 0/2] Fix timer initialization and Add support hotplug.

[Atish Patra]

On 6/4/18 3:20 PM, Palmer Dabbelt wrote:
> On Mon, 28 May 2018 18:15:03 PDT (-0700), atish.patra at wdc.com wrote:
>> On 5/21/18 1:43 PM, Atish Patra wrote:
>>> On 5/18/18 5:37 PM, Palmer Dabbelt wrote:
>>>> On Wed, 09 May 2018 15:02:20 PDT (-0700), atish.patra at wdc.com wrote:
>>>>> The patch (1/2)fixes issues around timer initialization. This fix is
>>>>> required for CPU hotplug to work. That's why they are clubbed into one
>>>>> series. I can separate them if required.
>>>>>
>>>>> Changes from v1 to v2:
>>>>> 1. Removed compiler warnings about unused variables.
>>>>>
>>>>> Atish Patra (2):
>>>>>      RISCV: Register clocksource and events correctly
>>>>>      RISCV: Support cpu hotplug.
>>>>>
>>>>>     arch/riscv/Kconfig                | 11 ++++++-
>>>>>     arch/riscv/include/asm/csr.h      |  1 +
>>>>>     arch/riscv/include/asm/smp.h      |  9 ++++--
>>>>>     arch/riscv/kernel/head.S          | 12 +++++++
>>>>>     arch/riscv/kernel/process.c       |  7 +++++
>>>>>     arch/riscv/kernel/setup.c         | 17 ++++++++++
>>>>>     arch/riscv/kernel/smpboot.c       | 66 +++++++++++++++++++++++++++++++++++++--
>>>>>     arch/riscv/kernel/time.c          |  9 +-----
>>>>>     arch/riscv/kernel/traps.c         |  6 ++--
>>>>>     drivers/clocksource/riscv_timer.c | 44 +++++++++++++++++---------
>>>>>     include/linux/cpuhotplug.h        |  1 +
>>>>>     11 files changed, 150 insertions(+), 33 deletions(-)
>>>>
>>>> Sorry I'm a bit slow here: I was meaning to fix this up.  I think this patch
>>>> set was made against something like riscv-all, which has a bunch of additional
>>>> cruft in it.
>>>
>>> Yeah. It is based on riscv-all because my changes also depend on initial
>>> timer patches.
>>>
>>> Do you mind updating it to be against Linus' master?
>>>
>>> I can do it. But my patches won't work without the initial timer
>>> patches. I am not sure how to proceed in that case.
>>>
>>> Do you want only hotplug patch (2/2) or I can squash the entire
>>> wip-timer patches into couple of sane ones and post them to mailing list ?
>>>
>>
>> As per our discussion, these patches has to be part of wip-timer until
>> interrupt issues are straightened and ready for upstream. All the timer
>> related patches will go on top of it.
>>
>>> Regards,
>>> Atish
>>>
>>>>
>>>
>>>
>> I was reviewing the base timer patches. I have a question about the patch.
>> clocksource: New RISC-V SBI timer driver (ea8ec64).
>>
>> The riscv_timer.h has a well defined per hart timer explanation in the
>> comment and following function declaration. However, I can't find any
>> reference to following functions.
>>
>> +void timer_riscv_init(int cpu_id,
>> +		      unsigned long riscv_timebase,
>> +		      int (*next_event)(unsigned long, struct clock_event_device *));
>> +
>> +void clocksource_riscv_init(unsigned long long (*rdtime)(struct
>> clocksource *));
>>
>> Are these redundant code or I missed something ?
>>
>> If you agree with me, I can send a v3 removing these (will move the
>> comment to riscv_timer.c).
> 
> I see these in riscv_timer.c on my copy of wip-timer
> 
>      e56e8bd48eca (HEAD -> wip-timer, kernel.org-palmer/wip-timer) RISCV: Disable timer interrupt in handler to fix nohz.
>      2b030aa93236 RISCV: Support cpu hotplug.
>      1b35cd828f29 RISCV: Register clocksource and events correctly
>      30540340a51e riscv_timer: add support for sched_clock
>      d6ad991df6fa clocksource: RISC-V: Match up with the current DTS
>      c48b6c686f01 Revert "clocksource: riscv_timer: split boot and secondary cpu timer init"
>      baf107699d0f RISC-V: Support per-hart timebase-frequency
>      1e955d9ecfbe clocksource: riscv_timer: split boot and secondary cpu timer init
>      ea8ec6497122 clocksource: New RISC-V SBI timer driver
>      60cc43fc8884 (tag: v4.17-rc1, wip-cpu_hotplog) Linux 4.17-rc1
> 
> did I screw something up?

Nope. Sorry for not being clear enough.

+void timer_riscv_init(int cpu_id,
+		      unsigned long riscv_timebase,
+		      int (*next_event)(unsigned long, struct clock_event_device *));
+
+void clocksource_riscv_init(unsigned long long (*rdtime)(struct 
clocksource *));

I was asking about utility of above two functions added as a part of 
commit ea8ec6497122. I couldn't find any usage of these two.

Shall we remove them and move the comment in that file to riscv_timer.c ?

Regards,
Atish

[PATCH v2 0/2] Fix timer initialization and Add support hotplug.

[Palmer Dabbelt]

On Mon, 04 Jun 2018 15:34:59 PDT (-0700), atish.patra at wdc.com wrote:
> On 6/4/18 3:20 PM, Palmer Dabbelt wrote:
>> On Mon, 28 May 2018 18:15:03 PDT (-0700), atish.patra at wdc.com wrote:
>>> On 5/21/18 1:43 PM, Atish Patra wrote:
>>>> On 5/18/18 5:37 PM, Palmer Dabbelt wrote:
>>>>> On Wed, 09 May 2018 15:02:20 PDT (-0700), atish.patra at wdc.com wrote:
>>>>>> The patch (1/2)fixes issues around timer initialization. This fix is
>>>>>> required for CPU hotplug to work. That's why they are clubbed into one
>>>>>> series. I can separate them if required.
>>>>>>
>>>>>> Changes from v1 to v2:
>>>>>> 1. Removed compiler warnings about unused variables.
>>>>>>
>>>>>> Atish Patra (2):
>>>>>>      RISCV: Register clocksource and events correctly
>>>>>>      RISCV: Support cpu hotplug.
>>>>>>
>>>>>>     arch/riscv/Kconfig                | 11 ++++++-
>>>>>>     arch/riscv/include/asm/csr.h      |  1 +
>>>>>>     arch/riscv/include/asm/smp.h      |  9 ++++--
>>>>>>     arch/riscv/kernel/head.S          | 12 +++++++
>>>>>>     arch/riscv/kernel/process.c       |  7 +++++
>>>>>>     arch/riscv/kernel/setup.c         | 17 ++++++++++
>>>>>>     arch/riscv/kernel/smpboot.c       | 66 +++++++++++++++++++++++++++++++++++++--
>>>>>>     arch/riscv/kernel/time.c          |  9 +-----
>>>>>>     arch/riscv/kernel/traps.c         |  6 ++--
>>>>>>     drivers/clocksource/riscv_timer.c | 44 +++++++++++++++++---------
>>>>>>     include/linux/cpuhotplug.h        |  1 +
>>>>>>     11 files changed, 150 insertions(+), 33 deletions(-)
>>>>>
>>>>> Sorry I'm a bit slow here: I was meaning to fix this up.  I think this patch
>>>>> set was made against something like riscv-all, which has a bunch of additional
>>>>> cruft in it.
>>>>
>>>> Yeah. It is based on riscv-all because my changes also depend on initial
>>>> timer patches.
>>>>
>>>> Do you mind updating it to be against Linus' master?
>>>>
>>>> I can do it. But my patches won't work without the initial timer
>>>> patches. I am not sure how to proceed in that case.
>>>>
>>>> Do you want only hotplug patch (2/2) or I can squash the entire
>>>> wip-timer patches into couple of sane ones and post them to mailing list ?
>>>>
>>>
>>> As per our discussion, these patches has to be part of wip-timer until
>>> interrupt issues are straightened and ready for upstream. All the timer
>>> related patches will go on top of it.
>>>
>>>> Regards,
>>>> Atish
>>>>
>>>>>
>>>>
>>>>
>>> I was reviewing the base timer patches. I have a question about the patch.
>>> clocksource: New RISC-V SBI timer driver (ea8ec64).
>>>
>>> The riscv_timer.h has a well defined per hart timer explanation in the
>>> comment and following function declaration. However, I can't find any
>>> reference to following functions.
>>>
>>> +void timer_riscv_init(int cpu_id,
>>> +		      unsigned long riscv_timebase,
>>> +		      int (*next_event)(unsigned long, struct clock_event_device *));
>>> +
>>> +void clocksource_riscv_init(unsigned long long (*rdtime)(struct
>>> clocksource *));
>>>
>>> Are these redundant code or I missed something ?
>>>
>>> If you agree with me, I can send a v3 removing these (will move the
>>> comment to riscv_timer.c).
>>
>> I see these in riscv_timer.c on my copy of wip-timer
>>
>>      e56e8bd48eca (HEAD -> wip-timer, kernel.org-palmer/wip-timer) RISCV: Disable timer interrupt in handler to fix nohz.
>>      2b030aa93236 RISCV: Support cpu hotplug.
>>      1b35cd828f29 RISCV: Register clocksource and events correctly
>>      30540340a51e riscv_timer: add support for sched_clock
>>      d6ad991df6fa clocksource: RISC-V: Match up with the current DTS
>>      c48b6c686f01 Revert "clocksource: riscv_timer: split boot and secondary cpu timer init"
>>      baf107699d0f RISC-V: Support per-hart timebase-frequency
>>      1e955d9ecfbe clocksource: riscv_timer: split boot and secondary cpu timer init
>>      ea8ec6497122 clocksource: New RISC-V SBI timer driver
>>      60cc43fc8884 (tag: v4.17-rc1, wip-cpu_hotplog) Linux 4.17-rc1
>>
>> did I screw something up?
>
> Nope. Sorry for not being clear enough.
>
> +void timer_riscv_init(int cpu_id,
> +		      unsigned long riscv_timebase,
> +		      int (*next_event)(unsigned long, struct clock_event_device *));
> +
> +void clocksource_riscv_init(unsigned long long (*rdtime)(struct
> clocksource *));
>
> I was asking about utility of above two functions added as a part of
> commit ea8ec6497122. I couldn't find any usage of these two.
>
> Shall we remove them and move the comment in that file to riscv_timer.c ?

Ah, I think this will all need to be rebased at some point to clean up the 
commit history, as it's become a bit of a mess.  If they're dead in the head of 
wip-timer then it's sufficient for now.