[PATCH] irqchip/loongarch-avec:Add multi-nodes topology support

[PATCH] irqchip/loongarch-avec:Add multi-nodes topology support

[Tianyang Zhang]

This patch enables the advanced interrupt controller function under
multiple-node of 3C600. The topology of the advanced interrupt controller
is consistent with NUMA node. We check the enable status of the node where
each CPU is located once when it goes online, which may cause some
additional operations, but it can ensure that the advanced interrupt
controller can still be used in situations where some CPUs cannot start

Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
---
 drivers/irqchip/irq-loongarch-avec.c | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/drivers/irqchip/irq-loongarch-avec.c b/drivers/irqchip/irq-loongarch-avec.c
index 0f6e465dd309..9e30198fa7e4 100644
--- a/drivers/irqchip/irq-loongarch-avec.c
+++ b/drivers/irqchip/irq-loongarch-avec.c
@@ -56,6 +56,18 @@ struct avecintc_data {
 	unsigned int		moving;
 };
 
+static inline void avecintc_enable(void)
+{
+	u64 value;
+
+	if (!loongarch_avec.domain)
+		return;
+
+	value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
+	value |= IOCSR_MISC_FUNC_AVEC_EN;
+	iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
+}
+
 static inline void avecintc_ack_irq(struct irq_data *d)
 {
 }
@@ -127,6 +139,8 @@ static int avecintc_cpu_online(unsigned int cpu)
 
 	guard(raw_spinlock)(&loongarch_avec.lock);
 
+	avecintc_enable();
+
 	irq_matrix_online(loongarch_avec.vector_matrix);
 
 	pending_list_init(cpu);
@@ -339,7 +353,6 @@ static int __init irq_matrix_init(void)
 static int __init avecintc_init(struct irq_domain *parent)
 {
 	int ret, parent_irq;
-	unsigned long value;
 
 	raw_spin_lock_init(&loongarch_avec.lock);
 
@@ -378,14 +391,13 @@ static int __init avecintc_init(struct irq_domain *parent)
 				  "irqchip/loongarch/avecintc:starting",
 				  avecintc_cpu_online, avecintc_cpu_offline);
 #endif
-	value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
-	value |= IOCSR_MISC_FUNC_AVEC_EN;
-	iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
+	avecintc_enable();
 
 	return ret;
 
 out_remove_domain:
 	irq_domain_remove(loongarch_avec.domain);
+	loongarch_avec.domain = NULL;
 out_free_handle:
 	irq_domain_free_fwnode(loongarch_avec.fwnode);
 out:
-- 
2.20.1

Re: [PATCH] irqchip/loongarch-avec:Add multi-nodes topology support

[Ming Wang]

On 2024/12/7 11:31, Tianyang Zhang wrote:
> This patch enables the advanced interrupt controller function under
> multiple-node of 3C600. The topology of the advanced interrupt controller
I think '3c600' is a typo. Shouldn't it be '3c6000'?
> is consistent with NUMA node. We check the enable status of the node where
> each CPU is located once when it goes online, which may cause some
> additional operations, but it can ensure that the advanced interrupt
> controller can still be used in situations where some CPUs cannot start
> 
> Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
> ---
>   drivers/irqchip/irq-loongarch-avec.c | 20 ++++++++++++++++----
>   1 file changed, 16 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/irqchip/irq-loongarch-avec.c b/drivers/irqchip/irq-loongarch-avec.c
> index 0f6e465dd309..9e30198fa7e4 100644
> --- a/drivers/irqchip/irq-loongarch-avec.c
> +++ b/drivers/irqchip/irq-loongarch-avec.c
> @@ -56,6 +56,18 @@ struct avecintc_data {
>   	unsigned int		moving;
>   };
>   
> +static inline void avecintc_enable(void)
> +{
> +	u64 value;
> +
> +	if (!loongarch_avec.domain)
> +		return;
> +
> +	value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
> +	value |= IOCSR_MISC_FUNC_AVEC_EN;
> +	iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
> +}
> +
>   static inline void avecintc_ack_irq(struct irq_data *d)
>   {
>   }
> @@ -127,6 +139,8 @@ static int avecintc_cpu_online(unsigned int cpu)
>   
>   	guard(raw_spinlock)(&loongarch_avec.lock);
>   
> +	avecintc_enable();
> +
>   	irq_matrix_online(loongarch_avec.vector_matrix);
>   
>   	pending_list_init(cpu);
> @@ -339,7 +353,6 @@ static int __init irq_matrix_init(void)
>   static int __init avecintc_init(struct irq_domain *parent)
>   {
>   	int ret, parent_irq;
> -	unsigned long value;
>   
>   	raw_spin_lock_init(&loongarch_avec.lock);
>   
> @@ -378,14 +391,13 @@ static int __init avecintc_init(struct irq_domain *parent)
>   				  "irqchip/loongarch/avecintc:starting",
>   				  avecintc_cpu_online, avecintc_cpu_offline);
>   #endif
> -	value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
> -	value |= IOCSR_MISC_FUNC_AVEC_EN;
> -	iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
> +	avecintc_enable();
>   
>   	return ret;
>   
>   out_remove_domain:
>   	irq_domain_remove(loongarch_avec.domain);
> +	loongarch_avec.domain = NULL;
>   out_free_handle:
>   	irq_domain_free_fwnode(loongarch_avec.fwnode);
>   out:

Re: [PATCH] irqchip/loongarch-avec:Add multi-nodes topology support

[Huacai Chen]

Hi, Tianyang,

On Sat, Dec 7, 2024 at 11:31 AM Tianyang Zhang
<zhangtianyang@loongson.cn> wrote:
>
> This patch enables the advanced interrupt controller function under
> multiple-node of 3C600. The topology of the advanced interrupt controller
> is consistent with NUMA node. We check the enable status of the node where
> each CPU is located once when it goes online, which may cause some
> additional operations, but it can ensure that the advanced interrupt
> controller can still be used in situations where some CPUs cannot start
The whole can be reworded:

Currently we only enable one chip's advanced interrupt controller
(AVEC) in hardware, so multi-chip machines with Loongson-3C6000 don't
work. This patch enables it for every chip (In theory every chip needs
to be enabled only once, but for simplicity we enable it on every core
in the CPU online hooks).

>
> Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
> ---
>  drivers/irqchip/irq-loongarch-avec.c | 20 ++++++++++++++++----
>  1 file changed, 16 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/irqchip/irq-loongarch-avec.c b/drivers/irqchip/irq-loongarch-avec.c
> index 0f6e465dd309..9e30198fa7e4 100644
> --- a/drivers/irqchip/irq-loongarch-avec.c
> +++ b/drivers/irqchip/irq-loongarch-avec.c
> @@ -56,6 +56,18 @@ struct avecintc_data {
>         unsigned int            moving;
>  };
>
> +static inline void avecintc_enable(void)
> +{
> +       u64 value;
> +
> +       if (!loongarch_avec.domain)
> +               return;
Is there any reason to check this? If domain is NULL, avecintc_init()
fails, there is no chance to execute this function.

Huacai

> +
> +       value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
> +       value |= IOCSR_MISC_FUNC_AVEC_EN;
> +       iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
> +}
> +
>  static inline void avecintc_ack_irq(struct irq_data *d)
>  {
>  }
> @@ -127,6 +139,8 @@ static int avecintc_cpu_online(unsigned int cpu)
>
>         guard(raw_spinlock)(&loongarch_avec.lock);
>
> +       avecintc_enable();
> +
>         irq_matrix_online(loongarch_avec.vector_matrix);
>
>         pending_list_init(cpu);
> @@ -339,7 +353,6 @@ static int __init irq_matrix_init(void)
>  static int __init avecintc_init(struct irq_domain *parent)
>  {
>         int ret, parent_irq;
> -       unsigned long value;
>
>         raw_spin_lock_init(&loongarch_avec.lock);
>
> @@ -378,14 +391,13 @@ static int __init avecintc_init(struct irq_domain *parent)
>                                   "irqchip/loongarch/avecintc:starting",
>                                   avecintc_cpu_online, avecintc_cpu_offline);
>  #endif
> -       value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
> -       value |= IOCSR_MISC_FUNC_AVEC_EN;
> -       iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
> +       avecintc_enable();
>
>         return ret;
>
>  out_remove_domain:
>         irq_domain_remove(loongarch_avec.domain);
> +       loongarch_avec.domain = NULL;
>  out_free_handle:
>         irq_domain_free_fwnode(loongarch_avec.fwnode);
>  out:
> --
> 2.20.1
>

Re: [PATCH] irqchip/loongarch-avec:Add multi-nodes topology support

[Thomas Gleixner]

On Sat, Dec 07 2024 at 17:11, Huacai Chen wrote:
> On Sat, Dec 7, 2024 at 11:31 AM Tianyang Zhang
> <zhangtianyang@loongson.cn> wrote:
>>
>> This patch enables the advanced interrupt controller function under
>> multiple-node of 3C600. The topology of the advanced interrupt controller
>> is consistent with NUMA node. We check the enable status of the node where
>> each CPU is located once when it goes online, which may cause some
>> additional operations, but it can ensure that the advanced interrupt
>> controller can still be used in situations where some CPUs cannot start
> The whole can be reworded:
>
> Currently we only enable one chip's advanced interrupt controller
> (AVEC) in hardware, so multi-chip machines with Loongson-3C6000 don't
> work. This patch enables it for every chip (In theory every chip needs
> to be enabled only once, but for simplicity we enable it on every core
> in the CPU online hooks).

It want's to be reworded without 'we' and 'this patch':

  avecintc_init() enables the Advanced Interrupt Controller (AVEC) of
  the boot CPU node, but nothing enables the AVEC on secondary nodes.

  Move the enablement to the CPU hotplug callback so that secondary
  nodes get the AVEC enabled too. In theory enabling it once per node
  would be sufficient, but redundant enabling does no hard, so keep the
  code simple and do it unconditionally.

Thanks,

        tglx

Re: [PATCH] irqchip/loongarch-avec:Add multi-nodes topology support

[Tianyang Zhang]

Hi, Ming


在 2024/12/7 下午2:03, Ming Wang 写道:
>
>
> On 2024/12/7 11:31, Tianyang Zhang wrote:
>> This patch enables the advanced interrupt controller function under
>> multiple-node of 3C600. The topology of the advanced interrupt 
>> controller
> I think '3c600' is a typo. Shouldn't it be '3c6000'?
ok, you are right , thanks
>> is consistent with NUMA node. We check the enable status of the node 
>> where
>> each CPU is located once when it goes online, which may cause some
>> additional operations, but it can ensure that the advanced interrupt
>> controller can still be used in situations where some CPUs cannot start
>>
>> Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
>> ---
>>   drivers/irqchip/irq-loongarch-avec.c | 20 ++++++++++++++++----
>>   1 file changed, 16 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/irqchip/irq-loongarch-avec.c 
>> b/drivers/irqchip/irq-loongarch-avec.c
>> index 0f6e465dd309..9e30198fa7e4 100644
>> --- a/drivers/irqchip/irq-loongarch-avec.c
>> +++ b/drivers/irqchip/irq-loongarch-avec.c
>> @@ -56,6 +56,18 @@ struct avecintc_data {
>>       unsigned int        moving;
>>   };
>>   +static inline void avecintc_enable(void)
>> +{
>> +    u64 value;
>> +
>> +    if (!loongarch_avec.domain)
>> +        return;
>> +
>> +    value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
>> +    value |= IOCSR_MISC_FUNC_AVEC_EN;
>> +    iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
>> +}
>> +
>>   static inline void avecintc_ack_irq(struct irq_data *d)
>>   {
>>   }
>> @@ -127,6 +139,8 @@ static int avecintc_cpu_online(unsigned int cpu)
>>         guard(raw_spinlock)(&loongarch_avec.lock);
>>   +    avecintc_enable();
>> +
>>       irq_matrix_online(loongarch_avec.vector_matrix);
>>         pending_list_init(cpu);
>> @@ -339,7 +353,6 @@ static int __init irq_matrix_init(void)
>>   static int __init avecintc_init(struct irq_domain *parent)
>>   {
>>       int ret, parent_irq;
>> -    unsigned long value;
>>         raw_spin_lock_init(&loongarch_avec.lock);
>>   @@ -378,14 +391,13 @@ static int __init avecintc_init(struct 
>> irq_domain *parent)
>>                     "irqchip/loongarch/avecintc:starting",
>>                     avecintc_cpu_online, avecintc_cpu_offline);
>>   #endif
>> -    value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
>> -    value |= IOCSR_MISC_FUNC_AVEC_EN;
>> -    iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
>> +    avecintc_enable();
>>         return ret;
>>     out_remove_domain:
>>       irq_domain_remove(loongarch_avec.domain);
>> +    loongarch_avec.domain = NULL;
>>   out_free_handle:
>>       irq_domain_free_fwnode(loongarch_avec.fwnode);
>>   out:

Re: [PATCH] irqchip/loongarch-avec:Add multi-nodes topology support

[Tianyang Zhang]

Hi, Huacai

在 2024/12/7 下午5:11, Huacai Chen 写道:
> Hi, Tianyang,
>
> On Sat, Dec 7, 2024 at 11:31 AM Tianyang Zhang
> <zhangtianyang@loongson.cn> wrote:
>> This patch enables the advanced interrupt controller function under
>> multiple-node of 3C600. The topology of the advanced interrupt controller
>> is consistent with NUMA node. We check the enable status of the node where
>> each CPU is located once when it goes online, which may cause some
>> additional operations, but it can ensure that the advanced interrupt
>> controller can still be used in situations where some CPUs cannot start
> The whole can be reworded:
>
> Currently we only enable one chip's advanced interrupt controller
> (AVEC) in hardware, so multi-chip machines with Loongson-3C6000 don't
> work. This patch enables it for every chip (In theory every chip needs
> to be enabled only once, but for simplicity we enable it on every core
> in the CPU online hooks).
Ok, I got it, thanks
>
>> Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
>> ---
>>   drivers/irqchip/irq-loongarch-avec.c | 20 ++++++++++++++++----
>>   1 file changed, 16 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/irqchip/irq-loongarch-avec.c b/drivers/irqchip/irq-loongarch-avec.c
>> index 0f6e465dd309..9e30198fa7e4 100644
>> --- a/drivers/irqchip/irq-loongarch-avec.c
>> +++ b/drivers/irqchip/irq-loongarch-avec.c
>> @@ -56,6 +56,18 @@ struct avecintc_data {
>>          unsigned int            moving;
>>   };
>>
>> +static inline void avecintc_enable(void)
>> +{
>> +       u64 value;
>> +
>> +       if (!loongarch_avec.domain)
>> +               return;
> Is there any reason to check this? If domain is NULL, avecintc_init()
> fails, there is no chance to execute this function.
>
> Huacai
Yes, you are right , it's redundancy, thanks
>
>> +
>> +       value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
>> +       value |= IOCSR_MISC_FUNC_AVEC_EN;
>> +       iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
>> +}
>> +
>>   static inline void avecintc_ack_irq(struct irq_data *d)
>>   {
>>   }
>> @@ -127,6 +139,8 @@ static int avecintc_cpu_online(unsigned int cpu)
>>
>>          guard(raw_spinlock)(&loongarch_avec.lock);
>>
>> +       avecintc_enable();
>> +
>>          irq_matrix_online(loongarch_avec.vector_matrix);
>>
>>          pending_list_init(cpu);
>> @@ -339,7 +353,6 @@ static int __init irq_matrix_init(void)
>>   static int __init avecintc_init(struct irq_domain *parent)
>>   {
>>          int ret, parent_irq;
>> -       unsigned long value;
>>
>>          raw_spin_lock_init(&loongarch_avec.lock);
>>
>> @@ -378,14 +391,13 @@ static int __init avecintc_init(struct irq_domain *parent)
>>                                    "irqchip/loongarch/avecintc:starting",
>>                                    avecintc_cpu_online, avecintc_cpu_offline);
>>   #endif
>> -       value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
>> -       value |= IOCSR_MISC_FUNC_AVEC_EN;
>> -       iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
>> +       avecintc_enable();
>>
>>          return ret;
>>
>>   out_remove_domain:
>>          irq_domain_remove(loongarch_avec.domain);
>> +       loongarch_avec.domain = NULL;
>>   out_free_handle:
>>          irq_domain_free_fwnode(loongarch_avec.fwnode);
>>   out:
>> --
>> 2.20.1
>>

Re: [PATCH] irqchip/loongarch-avec:Add multi-nodes topology support

[Tianyang Zhang]

Hi, Thomas

在 2024/12/9 上午4:22, Thomas Gleixner 写道:
> On Sat, Dec 07 2024 at 17:11, Huacai Chen wrote:
>> On Sat, Dec 7, 2024 at 11:31 AM Tianyang Zhang
>> <zhangtianyang@loongson.cn> wrote:
>>> This patch enables the advanced interrupt controller function under
>>> multiple-node of 3C600. The topology of the advanced interrupt controller
>>> is consistent with NUMA node. We check the enable status of the node where
>>> each CPU is located once when it goes online, which may cause some
>>> additional operations, but it can ensure that the advanced interrupt
>>> controller can still be used in situations where some CPUs cannot start
>> The whole can be reworded:
>>
>> Currently we only enable one chip's advanced interrupt controller
>> (AVEC) in hardware, so multi-chip machines with Loongson-3C6000 don't
>> work. This patch enables it for every chip (In theory every chip needs
>> to be enabled only once, but for simplicity we enable it on every core
>> in the CPU online hooks).
> It want's to be reworded without 'we' and 'this patch':
>
>    avecintc_init() enables the Advanced Interrupt Controller (AVEC) of
>    the boot CPU node, but nothing enables the AVEC on secondary nodes.
>
>    Move the enablement to the CPU hotplug callback so that secondary
>    nodes get the AVEC enabled too. In theory enabling it once per node
>    would be sufficient, but redundant enabling does no hard, so keep the
>    code simple and do it unconditionally.
>
> Thanks,
>
>          tglx
Ok, I got it , thanks