[PATCHv2 03/16] irqchip: irq-armada-370-xp: suspend/resume support

[PATCHv2 03/16] irqchip: irq-armada-370-xp: suspend/resume support

[Thomas Petazzoni]

This commit adds suspend/resume support to the irqchip driver used on
Armada XP platforms (amongst others). It does so by adding a set of
suspend/resume syscore_ops, that will respectively save and restore
the necessary registers to ensure interrupts continue to work after
resume.

It is worth mentioning that the affinity is lost during a
suspend/resume cycle, because when a secondary CPU is brought
off-line, all interrupts that are assigned to this CPU in terms of
affinity gets re-assigned to a still running CPU. Therefore, right
before entering suspend, all interrupts are assigned to the boot CPU.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: linux-kernel@vger.kernel.org
---
 drivers/irqchip/irq-armada-370-xp.c | 52 +++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c
index 3e238cd..4ec137b 100644
--- a/drivers/irqchip/irq-armada-370-xp.c
+++ b/drivers/irqchip/irq-armada-370-xp.c
@@ -26,6 +26,7 @@
 #include <linux/of_pci.h>
 #include <linux/irqdomain.h>
 #include <linux/slab.h>
+#include <linux/syscore_ops.h>
 #include <linux/msi.h>
 #include <asm/mach/arch.h>
 #include <asm/exception.h>
@@ -66,6 +67,7 @@
 static void __iomem *per_cpu_int_base;
 static void __iomem *main_int_base;
 static struct irq_domain *armada_370_xp_mpic_domain;
+static u32 doorbell_mask_reg;
 #ifdef CONFIG_PCI_MSI
 static struct irq_domain *armada_370_xp_msi_domain;
 static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR);
@@ -474,6 +476,54 @@ armada_370_xp_handle_irq(struct pt_regs *regs)
 	} while (1);
 }
 
+static int armada_370_xp_mpic_suspend(void)
+{
+	doorbell_mask_reg = readl(per_cpu_int_base +
+				  ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
+	return 0;
+}
+
+static void armada_370_xp_mpic_resume(void)
+{
+	int nirqs;
+	irq_hw_number_t irq;
+
+	/* Re-enable interrupts */
+	nirqs = (readl(main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff;
+	for (irq = 0; irq < nirqs; irq++) {
+		struct irq_data *data;
+		int virq;
+
+		virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq);
+		if (virq == 0)
+			continue;
+
+		if (irq != ARMADA_370_XP_TIMER0_PER_CPU_IRQ)
+			writel(irq, per_cpu_int_base +
+			       ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
+		else
+			writel(irq, main_int_base +
+			       ARMADA_370_XP_INT_SET_ENABLE_OFFS);
+
+		data = irq_get_irq_data(virq);
+		if (!irqd_irq_disabled(data))
+			armada_370_xp_irq_unmask(data);
+	}
+
+	/* Reconfigure doorbells for IPIs and MSIs */
+	writel(doorbell_mask_reg,
+	       per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
+	if (doorbell_mask_reg & IPI_DOORBELL_MASK)
+		writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
+	if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK)
+		writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
+}
+
+struct syscore_ops armada_370_xp_mpic_syscore_ops = {
+	.suspend	= armada_370_xp_mpic_suspend,
+	.resume		= armada_370_xp_mpic_resume,
+};
+
 static int __init armada_370_xp_mpic_of_init(struct device_node *node,
 					     struct device_node *parent)
 {
@@ -530,6 +580,8 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node,
 					armada_370_xp_mpic_handle_cascade_irq);
 	}
 
+	register_syscore_ops(&armada_370_xp_mpic_syscore_ops);
+
 	return 0;
 }
 
-- 
2.1.0

[PATCHv2 04/16] clocksource: time-armada-370-xp: add suspend/resume support

[Thomas Petazzoni]

This commit adds a set of suspend/resume syscore_ops to respectively
save and restore a number of timer registers, in order to make sure
the clockevent and clocksource devices continue to work properly
across a suspend/resume cycle.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Cc: Daniel Lezcano <daniel.lezcano@linaro.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
---
 drivers/clocksource/time-armada-370-xp.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/drivers/clocksource/time-armada-370-xp.c b/drivers/clocksource/time-armada-370-xp.c
index 0451e62..ff37d3a 100644
--- a/drivers/clocksource/time-armada-370-xp.c
+++ b/drivers/clocksource/time-armada-370-xp.c
@@ -43,6 +43,7 @@
 #include <linux/module.h>
 #include <linux/sched_clock.h>
 #include <linux/percpu.h>
+#include <linux/syscore_ops.h>
 
 /*
  * Timer block registers.
@@ -223,6 +224,28 @@ static struct notifier_block armada_370_xp_timer_cpu_nb = {
 	.notifier_call = armada_370_xp_timer_cpu_notify,
 };
 
+static u32 timer0_ctrl_reg, timer0_local_ctrl_reg;
+
+static int armada_370_xp_timer_suspend(void)
+{
+	timer0_ctrl_reg = readl(timer_base + TIMER_CTRL_OFF);
+	timer0_local_ctrl_reg = readl(local_base + TIMER_CTRL_OFF);
+	return 0;
+}
+
+static void armada_370_xp_timer_resume(void)
+{
+	writel(0xffffffff, timer_base + TIMER0_VAL_OFF);
+	writel(0xffffffff, timer_base + TIMER0_RELOAD_OFF);
+	writel(timer0_ctrl_reg, timer_base + TIMER_CTRL_OFF);
+	writel(timer0_local_ctrl_reg, local_base + TIMER_CTRL_OFF);
+}
+
+struct syscore_ops armada_370_xp_timer_syscore_ops = {
+	.suspend	= armada_370_xp_timer_suspend,
+	.resume		= armada_370_xp_timer_resume,
+};
+
 static void __init armada_370_xp_timer_common_init(struct device_node *np)
 {
 	u32 clr = 0, set = 0;
@@ -285,6 +308,8 @@ static void __init armada_370_xp_timer_common_init(struct device_node *np)
 	/* Immediately configure the timer on the boot CPU */
 	if (!res)
 		armada_370_xp_timer_setup(this_cpu_ptr(armada_370_xp_evt));
+
+	register_syscore_ops(&armada_370_xp_timer_syscore_ops);
 }
 
 static void __init armada_xp_timer_init(struct device_node *np)
-- 
2.1.0

Re: [PATCHv2 04/16] clocksource: time-armada-370-xp: add suspend/resume support

[Daniel Lezcano]

On 11/14/2014 04:21 PM, Thomas Petazzoni wrote:
> This commit adds a set of suspend/resume syscore_ops to respectively
> save and restore a number of timer registers, in order to make sure
> the clockevent and clocksource devices continue to work properly
> across a suspend/resume cycle.
>
> Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
> Cc: Daniel Lezcano <daniel.lezcano@linaro.org>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Cc: linux-kernel@vger.kernel.org

Acked-by: Daniel Lezcano <daniel.lezcano@linaro.org>


> ---
>   drivers/clocksource/time-armada-370-xp.c | 25 +++++++++++++++++++++++++
>   1 file changed, 25 insertions(+)
>
> diff --git a/drivers/clocksource/time-armada-370-xp.c b/drivers/clocksource/time-armada-370-xp.c
> index 0451e62..ff37d3a 100644
> --- a/drivers/clocksource/time-armada-370-xp.c
> +++ b/drivers/clocksource/time-armada-370-xp.c
> @@ -43,6 +43,7 @@
>   #include <linux/module.h>
>   #include <linux/sched_clock.h>
>   #include <linux/percpu.h>
> +#include <linux/syscore_ops.h>
>
>   /*
>    * Timer block registers.
> @@ -223,6 +224,28 @@ static struct notifier_block armada_370_xp_timer_cpu_nb = {
>   	.notifier_call = armada_370_xp_timer_cpu_notify,
>   };
>
> +static u32 timer0_ctrl_reg, timer0_local_ctrl_reg;
> +
> +static int armada_370_xp_timer_suspend(void)
> +{
> +	timer0_ctrl_reg = readl(timer_base + TIMER_CTRL_OFF);
> +	timer0_local_ctrl_reg = readl(local_base + TIMER_CTRL_OFF);
> +	return 0;
> +}
> +
> +static void armada_370_xp_timer_resume(void)
> +{
> +	writel(0xffffffff, timer_base + TIMER0_VAL_OFF);
> +	writel(0xffffffff, timer_base + TIMER0_RELOAD_OFF);
> +	writel(timer0_ctrl_reg, timer_base + TIMER_CTRL_OFF);
> +	writel(timer0_local_ctrl_reg, local_base + TIMER_CTRL_OFF);
> +}
> +
> +struct syscore_ops armada_370_xp_timer_syscore_ops = {
> +	.suspend	= armada_370_xp_timer_suspend,
> +	.resume		= armada_370_xp_timer_resume,
> +};
> +
>   static void __init armada_370_xp_timer_common_init(struct device_node *np)
>   {
>   	u32 clr = 0, set = 0;
> @@ -285,6 +308,8 @@ static void __init armada_370_xp_timer_common_init(struct device_node *np)
>   	/* Immediately configure the timer on the boot CPU */
>   	if (!res)
>   		armada_370_xp_timer_setup(this_cpu_ptr(armada_370_xp_evt));
> +
> +	register_syscore_ops(&armada_370_xp_timer_syscore_ops);
>   }
>
>   static void __init armada_xp_timer_init(struct device_node *np)
>


-- 
  <http://www.linaro.org/> Linaro.org │ Open source software for ARM SoCs

Follow Linaro:  <http://www.facebook.com/pages/Linaro> Facebook |
<http://twitter.com/#!/linaroorg> Twitter |
<http://www.linaro.org/linaro-blog/> Blog

[PATCHv2 08/16] clk: mvebu: add suspend/resume for gatable clocks

[Thomas Petazzoni]

This commit adds suspend/resume support for the gatable clock driver
used on Marvell EBU platforms. When getting out of suspend, the
Marvell EBU platforms go through the bootloader, which re-enables all
gatable clocks. However, upon resume, the clock framework will not
disable again all gatable clocks that are not used.

Therefore, if the clock driver does not save/restore the state of the
gatable clocks, all gatable clocks that are not claimed by any device
driver will remain enabled after a resume. This is why this driver
saves and restores the state of those clocks.

Since clocks aren't real devices, we don't have the normal ->suspend()
and ->resume() of the device model, and have to use the ->suspend()
and ->resume() hooks of the syscore_ops mechanism. This mechanism has
the unfortunate idea of not providing a way of passing private data,
which requires us to change the driver to make the assumption that
there is only once instance of the gatable clock control structure.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Cc: Mike Turquette <mturquette@linaro.org>
Cc: linux-kernel@vger.kernel.org
Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
---
 drivers/clk/mvebu/common.c | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/drivers/clk/mvebu/common.c b/drivers/clk/mvebu/common.c
index b7fcb46..8799fb8 100644
--- a/drivers/clk/mvebu/common.c
+++ b/drivers/clk/mvebu/common.c
@@ -19,6 +19,7 @@
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/syscore_ops.h>
 
 #include "common.h"
 
@@ -177,14 +178,18 @@ struct clk_gating_ctrl {
 	spinlock_t *lock;
 	struct clk **gates;
 	int num_gates;
+	struct syscore_ops syscore_ops;
+	void __iomem *base;
+	u32 saved_reg;
 };
 
 #define to_clk_gate(_hw) container_of(_hw, struct clk_gate, hw)
 
+static struct clk_gating_ctrl *ctrl;
+
 static struct clk *clk_gating_get_src(
 	struct of_phandle_args *clkspec, void *data)
 {
-	struct clk_gating_ctrl *ctrl = (struct clk_gating_ctrl *)data;
 	int n;
 
 	if (clkspec->args_count < 1)
@@ -199,15 +204,30 @@ static struct clk *clk_gating_get_src(
 	return ERR_PTR(-ENODEV);
 }
 
+static int mvebu_clk_gating_suspend(void)
+{
+	ctrl->saved_reg = readl(ctrl->base);
+	return 0;
+}
+
+static void mvebu_clk_gating_resume(void)
+{
+	writel(ctrl->saved_reg, ctrl->base);
+}
+
 void __init mvebu_clk_gating_setup(struct device_node *np,
 				   const struct clk_gating_soc_desc *desc)
 {
-	struct clk_gating_ctrl *ctrl;
 	struct clk *clk;
 	void __iomem *base;
 	const char *default_parent = NULL;
 	int n;
 
+	if (ctrl) {
+		pr_err("mvebu-clk-gating: cannot instantiate more than one gatable clock device\n");
+		return;
+	}
+
 	base = of_iomap(np, 0);
 	if (WARN_ON(!base))
 		return;
@@ -225,6 +245,10 @@ void __init mvebu_clk_gating_setup(struct device_node *np,
 	/* lock must already be initialized */
 	ctrl->lock = &ctrl_gating_lock;
 
+	ctrl->base = base;
+	ctrl->syscore_ops.suspend = mvebu_clk_gating_suspend;
+	ctrl->syscore_ops.resume = mvebu_clk_gating_resume;
+
 	/* Count, allocate, and register clock gates */
 	for (n = 0; desc[n].name;)
 		n++;
@@ -246,6 +270,8 @@ void __init mvebu_clk_gating_setup(struct device_node *np,
 
 	of_clk_add_provider(np, clk_gating_get_src, ctrl);
 
+	register_syscore_ops(&ctrl->syscore_ops);
+
 	return;
 gates_out:
 	kfree(ctrl);
-- 
2.1.0

Re: [PATCHv2 08/16] clk: mvebu: add suspend/resume for gatable clocks

[Mike Turquette]

Quoting Thomas Petazzoni (2014-11-14 07:21:28)
> This commit adds suspend/resume support for the gatable clock driver
> used on Marvell EBU platforms. When getting out of suspend, the
> Marvell EBU platforms go through the bootloader, which re-enables all
> gatable clocks. However, upon resume, the clock framework will not
> disable again all gatable clocks that are not used.
> 
> Therefore, if the clock driver does not save/restore the state of the
> gatable clocks, all gatable clocks that are not claimed by any device
> driver will remain enabled after a resume. This is why this driver
> saves and restores the state of those clocks.

It might be a good idea to call clk_disable_unused() from the clk core
after resuming from suspend.

> 
> Since clocks aren't real devices, we don't have the normal ->suspend()
> and ->resume() of the device model, and have to use the ->suspend()
> and ->resume() hooks of the syscore_ops mechanism. This mechanism has
> the unfortunate idea of not providing a way of passing private data,
> which requires us to change the driver to make the assumption that
> there is only once instance of the gatable clock control structure.
> 
> Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
> Cc: Mike Turquette <mturquette@linaro.org>
> Cc: linux-kernel@vger.kernel.org
> Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
> ---
>  drivers/clk/mvebu/common.c | 30 ++++++++++++++++++++++++++++--
>  1 file changed, 28 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/clk/mvebu/common.c b/drivers/clk/mvebu/common.c
> index b7fcb46..8799fb8 100644
> --- a/drivers/clk/mvebu/common.c
> +++ b/drivers/clk/mvebu/common.c
> @@ -19,6 +19,7 @@
>  #include <linux/io.h>
>  #include <linux/of.h>
>  #include <linux/of_address.h>
> +#include <linux/syscore_ops.h>
>  
>  #include "common.h"
>  
> @@ -177,14 +178,18 @@ struct clk_gating_ctrl {
>         spinlock_t *lock;
>         struct clk **gates;
>         int num_gates;
> +       struct syscore_ops syscore_ops;

You are registering suspend/resume ops per clock. Have you considered
registering a single set of ops for your clock controller driver? See
drivers/clk/samsung/clk-exynos5420.c for an example.

Combined with a table of clocks registered by your driver, centralized
suspend/resume methods might be a cleaner solution.

Regards,
Mike

> +       void __iomem *base;
> +       u32 saved_reg;
>  };
>  
>  #define to_clk_gate(_hw) container_of(_hw, struct clk_gate, hw)
>  
> +static struct clk_gating_ctrl *ctrl;
> +
>  static struct clk *clk_gating_get_src(
>         struct of_phandle_args *clkspec, void *data)
>  {
> -       struct clk_gating_ctrl *ctrl = (struct clk_gating_ctrl *)data;
>         int n;
>  
>         if (clkspec->args_count < 1)
> @@ -199,15 +204,30 @@ static struct clk *clk_gating_get_src(
>         return ERR_PTR(-ENODEV);
>  }
>  
> +static int mvebu_clk_gating_suspend(void)
> +{
> +       ctrl->saved_reg = readl(ctrl->base);
> +       return 0;
> +}
> +
> +static void mvebu_clk_gating_resume(void)
> +{
> +       writel(ctrl->saved_reg, ctrl->base);
> +}
> +
>  void __init mvebu_clk_gating_setup(struct device_node *np,
>                                    const struct clk_gating_soc_desc *desc)
>  {
> -       struct clk_gating_ctrl *ctrl;
>         struct clk *clk;
>         void __iomem *base;
>         const char *default_parent = NULL;
>         int n;
>  
> +       if (ctrl) {
> +               pr_err("mvebu-clk-gating: cannot instantiate more than one gatable clock device\n");
> +               return;
> +       }
> +
>         base = of_iomap(np, 0);
>         if (WARN_ON(!base))
>                 return;
> @@ -225,6 +245,10 @@ void __init mvebu_clk_gating_setup(struct device_node *np,
>         /* lock must already be initialized */
>         ctrl->lock = &ctrl_gating_lock;
>  
> +       ctrl->base = base;
> +       ctrl->syscore_ops.suspend = mvebu_clk_gating_suspend;
> +       ctrl->syscore_ops.resume = mvebu_clk_gating_resume;
> +
>         /* Count, allocate, and register clock gates */
>         for (n = 0; desc[n].name;)
>                 n++;
> @@ -246,6 +270,8 @@ void __init mvebu_clk_gating_setup(struct device_node *np,
>  
>         of_clk_add_provider(np, clk_gating_get_src, ctrl);
>  
> +       register_syscore_ops(&ctrl->syscore_ops);
> +
>         return;
>  gates_out:
>         kfree(ctrl);
> -- 
> 2.1.0
> 

Re: [PATCHv2 08/16] clk: mvebu: add suspend/resume for gatable clocks

[Thomas Petazzoni]

Dear Mike Turquette,

On Mon, 17 Nov 2014 14:46:04 -0800, Mike Turquette wrote:
> Quoting Thomas Petazzoni (2014-11-14 07:21:28)
> > This commit adds suspend/resume support for the gatable clock driver
> > used on Marvell EBU platforms. When getting out of suspend, the
> > Marvell EBU platforms go through the bootloader, which re-enables all
> > gatable clocks. However, upon resume, the clock framework will not
> > disable again all gatable clocks that are not used.
> > 
> > Therefore, if the clock driver does not save/restore the state of the
> > gatable clocks, all gatable clocks that are not claimed by any device
> > driver will remain enabled after a resume. This is why this driver
> > saves and restores the state of those clocks.
> 
> It might be a good idea to call clk_disable_unused() from the clk core
> after resuming from suspend.

Yes, this might be an interesting clk core improvement.

> > @@ -177,14 +178,18 @@ struct clk_gating_ctrl {
> >         spinlock_t *lock;
> >         struct clk **gates;
> >         int num_gates;
> > +       struct syscore_ops syscore_ops;
> 
> You are registering suspend/resume ops per clock. Have you considered
> registering a single set of ops for your clock controller driver? See
> drivers/clk/samsung/clk-exynos5420.c for an example.
> 
> Combined with a table of clocks registered by your driver, centralized
> suspend/resume methods might be a cleaner solution.

Ok, I've changed. To be honest, I don't think it makes much change: if
we had two instances of a gatable clock controller, then we would have
two calls to mvebu_clk_gating_setup(), which would register twice the
same syscore_ops. But we were anyway already assuming that we have
already one instance of a gatable clock controller, since the
syscore_ops operation implementation already used a global pointer to
the gatable clock controller.

Will be part of the upcoming v3.

Thanks for the review!

Thomas
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux, Kernel and Android engineering
http://free-electrons.com