[PATCH v1 00/14] riscv: introduce basic UART support and interrupts for hypervisor mode

[PATCH v1 00/14] riscv: introduce basic UART support and interrupts for hypervisor mode

[Oleksii Kurochko]

The patch series introduces basic UART support (in interrupt mode) and support of
interrupts for hypervisor mode.

To implement this the following has been added:
- APLIC and IMISC initialization.
- Introduce of intc_hw_operations abstraction.
- Introduce some APLIC and IMSIC operations.
- Introduce init_IRQ(), platform_get_irq() and setup_irq() functions.
- Update do_trap() handler to handle IRQ_S_EXT.
- Introduce some other functions such as: get_s_time(), smp_clear_cpu_maps(),
  ioremap().
- Enable UART. 

Oleksii Kurochko (14):
  xen/riscv: implement get_s_time()
  xen/riscv: introduce smp_clear_cpu_maps()
  xen/riscv: introduce ioremap()
  xen/riscv: introduce init_IRQ()
  xen/riscv: introduce platform_get_irq()
  xen/riscv: riscv_of_processor_hartid() implementation
  xen/riscv: Introduce intc_hw_operations abstraction
  xen/riscv: imsic_init() implementation
  xen/riscv: aplic_init() implementation
  xen/riscv: implementation of aplic and imsic operations
  xen/riscv: add external interrupt handling for hypervisor mode
  xen/riscv: implement setup_irq()
  xen/riscv: initialize interrupt controller
  xen/riscv: add basic UART support

 xen/arch/riscv/Kconfig             |  13 +
 xen/arch/riscv/Makefile            |   3 +
 xen/arch/riscv/aplic.c             | 302 +++++++++++++++++++++++
 xen/arch/riscv/imsic.c             | 374 +++++++++++++++++++++++++++++
 xen/arch/riscv/include/asm/aplic.h |  89 +++++++
 xen/arch/riscv/include/asm/imsic.h |  88 +++++++
 xen/arch/riscv/include/asm/intc.h  |  31 +++
 xen/arch/riscv/include/asm/irq.h   |  37 ++-
 xen/arch/riscv/include/asm/smp.h   |   5 +
 xen/arch/riscv/include/asm/time.h  |   6 +
 xen/arch/riscv/intc.c              |  58 +++++
 xen/arch/riscv/irq.c               | 223 +++++++++++++++++
 xen/arch/riscv/pt.c                |  19 ++
 xen/arch/riscv/setup.c             |  23 ++
 xen/arch/riscv/smpboot.c           |  83 +++++++
 xen/arch/riscv/stubs.c             |  16 --
 xen/arch/riscv/time.c              |   7 +
 xen/arch/riscv/traps.c             |  18 ++
 xen/drivers/char/Kconfig           |   1 -
 19 files changed, 1378 insertions(+), 18 deletions(-)
 create mode 100644 xen/arch/riscv/imsic.c
 create mode 100644 xen/arch/riscv/include/asm/aplic.h
 create mode 100644 xen/arch/riscv/include/asm/imsic.h
 create mode 100644 xen/arch/riscv/irq.c
 create mode 100644 xen/arch/riscv/smpboot.c

-- 
2.49.0

[PATCH v1 01/14] xen/riscv: implement get_s_time()

[Oleksii Kurochko]

Also tick_to_ns() is implemeted as it is used in get_s_time().

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/include/asm/time.h | 6 ++++++
 xen/arch/riscv/stubs.c            | 5 -----
 xen/arch/riscv/time.c             | 7 +++++++
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/xen/arch/riscv/include/asm/time.h b/xen/arch/riscv/include/asm/time.h
index e8d9ffec57..0f6aa99ab1 100644
--- a/xen/arch/riscv/include/asm/time.h
+++ b/xen/arch/riscv/include/asm/time.h
@@ -3,6 +3,7 @@
 #define ASM__RISCV__TIME_H
 
 #include <xen/bug.h>
+#include <xen/lib.h>
 #include <xen/types.h>
 #include <asm/csr.h>
 
@@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void)
     return csr_read(CSR_TIME);
 }
 
+static inline s_time_t ticks_to_ns(uint64_t ticks)
+{
+    return muldiv64(ticks, SECONDS(1), 1000 * cpu_khz);
+}
+
 void preinit_xen_time(void);
 
 #endif /* ASM__RISCV__TIME_H */
diff --git a/xen/arch/riscv/stubs.c b/xen/arch/riscv/stubs.c
index a1d64534cd..83416d3350 100644
--- a/xen/arch/riscv/stubs.c
+++ b/xen/arch/riscv/stubs.c
@@ -27,11 +27,6 @@ nodemask_t __read_mostly node_online_map = { { [0] = 1UL } };
 
 /* time.c */
 
-s_time_t get_s_time(void)
-{
-    BUG_ON("unimplemented");
-}
-
 int reprogram_timer(s_time_t timeout)
 {
     BUG_ON("unimplemented");
diff --git a/xen/arch/riscv/time.c b/xen/arch/riscv/time.c
index 905bb13eb4..81e06781f8 100644
--- a/xen/arch/riscv/time.c
+++ b/xen/arch/riscv/time.c
@@ -4,10 +4,17 @@
 #include <xen/init.h>
 #include <xen/lib.h>
 #include <xen/sections.h>
+#include <xen/types.h>
 
 unsigned long __ro_after_init cpu_khz; /* CPU clock frequency in kHz. */
 uint64_t __ro_after_init boot_clock_cycles;
 
+s_time_t get_s_time(void)
+{
+    uint64_t ticks = get_cycles() - boot_clock_cycles;
+    return ticks_to_ns(ticks);
+}
+
 /* Set up the timer on the boot CPU (early init function) */
 static void __init preinit_dt_xen_time(void)
 {
-- 
2.49.0

Re: [PATCH v1 01/14] xen/riscv: implement get_s_time()

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> @@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void)
>      return csr_read(CSR_TIME);
>  }
>  
> +static inline s_time_t ticks_to_ns(uint64_t ticks)
> +{
> +    return muldiv64(ticks, SECONDS(1), 1000 * cpu_khz);
> +}

Why the extra multiplication by 1000? I.e. why not
"muldiv64(ticks, MILLISECONDS(1), cpu_khz)", getting away with just one
multiplication and a reduced risk of encountering intermediate overflow
(affecting only hypothetical above 4THz CPUs then)?

> --- a/xen/arch/riscv/time.c
> +++ b/xen/arch/riscv/time.c
> @@ -4,10 +4,17 @@
>  #include <xen/init.h>
>  #include <xen/lib.h>
>  #include <xen/sections.h>
> +#include <xen/types.h>
>  
>  unsigned long __ro_after_init cpu_khz; /* CPU clock frequency in kHz. */
>  uint64_t __ro_after_init boot_clock_cycles;
>  
> +s_time_t get_s_time(void)
> +{
> +    uint64_t ticks = get_cycles() - boot_clock_cycles;
> +    return ticks_to_ns(ticks);

Nit: Blank line between declaration(s) and statement(s) please, as well as
ahead of the main "return" of a function.

Happy to make both adjustments upon committing, so long as you agree; then:
Reviewed-by: Jan Beulich <jbeulich@suse.com>

Jan

Re: [PATCH v1 01/14] xen/riscv: implement get_s_time()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 1447 bytes --]


On 4/10/25 2:52 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> @@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void)
>>       return csr_read(CSR_TIME);
>>   }
>>   
>> +static inline s_time_t ticks_to_ns(uint64_t ticks)
>> +{
>> +    return muldiv64(ticks, SECONDS(1), 1000 * cpu_khz);
>> +}
> Why the extra multiplication by 1000? I.e. why not
> "muldiv64(ticks, MILLISECONDS(1), cpu_khz)", getting away with just one
> multiplication and a reduced risk of encountering intermediate overflow
> (affecting only hypothetical above 4THz CPUs then)?

Multiplication by 1000 was needed to convert khz to hz, but yes, your option
would be better.

>
>> --- a/xen/arch/riscv/time.c
>> +++ b/xen/arch/riscv/time.c
>> @@ -4,10 +4,17 @@
>>   #include <xen/init.h>
>>   #include <xen/lib.h>
>>   #include <xen/sections.h>
>> +#include <xen/types.h>
>>   
>>   unsigned long __ro_after_init cpu_khz; /* CPU clock frequency in kHz. */
>>   uint64_t __ro_after_init boot_clock_cycles;
>>   
>> +s_time_t get_s_time(void)
>> +{
>> +    uint64_t ticks = get_cycles() - boot_clock_cycles;
>> +    return ticks_to_ns(ticks);
> Nit: Blank line between declaration(s) and statement(s) please, as well as
> ahead of the main "return" of a function.
>
> Happy to make both adjustments upon committing, so long as you agree; then:
> Reviewed-by: Jan Beulich<jbeulich@suse.com>

I'll be happy with that.

Thank you very much.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 2301 bytes --]

[PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()

[Oleksii Kurochko]

Initialize cpu_{possible, online, present}_map by using smp_clear_cpu_maps().

Drop DEFINE_PER_CPU(unsigned int, cpu_id) from stubs.c as this variable isn't
expected to be used in RISC-V at all.

Move declaration of cpu_{possible,online,present}_map from stubs.c to smpboot.c
as now smpboot.c is now introduced.
Other defintions keep in stubs.c as they are not initialized and not needed, at
the moment.

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/Makefile          |  1 +
 xen/arch/riscv/include/asm/smp.h |  2 ++
 xen/arch/riscv/setup.c           |  2 ++
 xen/arch/riscv/smpboot.c         | 15 +++++++++++++++
 xen/arch/riscv/stubs.c           |  6 ------
 5 files changed, 20 insertions(+), 6 deletions(-)
 create mode 100644 xen/arch/riscv/smpboot.c

diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile
index 0c6c4a38a3..f551bf32a2 100644
--- a/xen/arch/riscv/Makefile
+++ b/xen/arch/riscv/Makefile
@@ -10,6 +10,7 @@ obj-y += sbi.o
 obj-y += setup.o
 obj-y += shutdown.o
 obj-y += smp.o
+obj-y += smpboot.o
 obj-y += stubs.o
 obj-y += time.o
 obj-y += traps.o
diff --git a/xen/arch/riscv/include/asm/smp.h b/xen/arch/riscv/include/asm/smp.h
index 5e170b57b3..188c033718 100644
--- a/xen/arch/riscv/include/asm/smp.h
+++ b/xen/arch/riscv/include/asm/smp.h
@@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,
 
 void setup_tp(unsigned int cpuid);
 
+void smp_clear_cpu_maps(void);
+
 #endif
 
 /*
diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c
index 4e416f6e44..7f68f3f5b7 100644
--- a/xen/arch/riscv/setup.c
+++ b/xen/arch/riscv/setup.c
@@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     remove_identity_mapping();
 
+    smp_clear_cpu_maps();
+
     set_processor_id(0);
 
     set_cpuid_to_hartid(0, bootcpu_id);
diff --git a/xen/arch/riscv/smpboot.c b/xen/arch/riscv/smpboot.c
new file mode 100644
index 0000000000..0f4dcc28e1
--- /dev/null
+++ b/xen/arch/riscv/smpboot.c
@@ -0,0 +1,15 @@
+#include <xen/cpumask.h>
+#include <xen/init.h>
+
+cpumask_t cpu_online_map;
+cpumask_t cpu_present_map;
+cpumask_t cpu_possible_map;
+
+void __init smp_clear_cpu_maps(void)
+{
+    cpumask_clear(&cpu_possible_map);
+    cpumask_clear(&cpu_online_map);
+    cpumask_set_cpu(0, &cpu_possible_map);
+    cpumask_set_cpu(0, &cpu_online_map);
+    cpumask_copy(&cpu_present_map, &cpu_possible_map);
+}
diff --git a/xen/arch/riscv/stubs.c b/xen/arch/riscv/stubs.c
index 83416d3350..fdcf91054e 100644
--- a/xen/arch/riscv/stubs.c
+++ b/xen/arch/riscv/stubs.c
@@ -11,12 +11,6 @@
 
 /* smpboot.c */
 
-cpumask_t cpu_online_map;
-cpumask_t cpu_present_map;
-cpumask_t cpu_possible_map;
-
-/* ID of the PCPU we're running on */
-DEFINE_PER_CPU(unsigned int, cpu_id);
 /* XXX these seem awfully x86ish... */
 /* representing HT siblings of each logical CPU */
 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_mask);
-- 
2.49.0

Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> Initialize cpu_{possible, online, present}_map by using smp_clear_cpu_maps().
> 
> Drop DEFINE_PER_CPU(unsigned int, cpu_id) from stubs.c as this variable isn't
> expected to be used in RISC-V at all.
> 
> Move declaration of cpu_{possible,online,present}_map from stubs.c to smpboot.c
> as now smpboot.c is now introduced.
> Other defintions keep in stubs.c as they are not initialized and not needed, at
> the moment.
> 
> Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
> ---
>  xen/arch/riscv/Makefile          |  1 +
>  xen/arch/riscv/include/asm/smp.h |  2 ++
>  xen/arch/riscv/setup.c           |  2 ++
>  xen/arch/riscv/smpboot.c         | 15 +++++++++++++++
>  xen/arch/riscv/stubs.c           |  6 ------
>  5 files changed, 20 insertions(+), 6 deletions(-)
>  create mode 100644 xen/arch/riscv/smpboot.c
> 
> diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile
> index 0c6c4a38a3..f551bf32a2 100644
> --- a/xen/arch/riscv/Makefile
> +++ b/xen/arch/riscv/Makefile
> @@ -10,6 +10,7 @@ obj-y += sbi.o
>  obj-y += setup.o
>  obj-y += shutdown.o
>  obj-y += smp.o
> +obj-y += smpboot.o
>  obj-y += stubs.o
>  obj-y += time.o
>  obj-y += traps.o
> diff --git a/xen/arch/riscv/include/asm/smp.h b/xen/arch/riscv/include/asm/smp.h
> index 5e170b57b3..188c033718 100644
> --- a/xen/arch/riscv/include/asm/smp.h
> +++ b/xen/arch/riscv/include/asm/smp.h
> @@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,
>  
>  void setup_tp(unsigned int cpuid);
>  
> +void smp_clear_cpu_maps(void);
> +
>  #endif
>  
>  /*
> diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c
> index 4e416f6e44..7f68f3f5b7 100644
> --- a/xen/arch/riscv/setup.c
> +++ b/xen/arch/riscv/setup.c
> @@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
>  
>      remove_identity_mapping();
>  
> +    smp_clear_cpu_maps();
> +
>      set_processor_id(0);
>  
>      set_cpuid_to_hartid(0, bootcpu_id);
> diff --git a/xen/arch/riscv/smpboot.c b/xen/arch/riscv/smpboot.c
> new file mode 100644
> index 0000000000..0f4dcc28e1
> --- /dev/null
> +++ b/xen/arch/riscv/smpboot.c
> @@ -0,0 +1,15 @@
> +#include <xen/cpumask.h>
> +#include <xen/init.h>
> +
> +cpumask_t cpu_online_map;
> +cpumask_t cpu_present_map;
> +cpumask_t cpu_possible_map;

__read_mostly for all of them, perhaps (if CPU hotplug isn't expected to
be supported) even __ro_after_init for the latter two?

As to cpu_possible_map - do you predict that you'll actually use it? Arm
does (and instead has only a fake cpu_present_map), but on x86 we get away
without.

> +void __init smp_clear_cpu_maps(void)
> +{
> +    cpumask_clear(&cpu_possible_map);
> +    cpumask_clear(&cpu_online_map);

What's the point of these? All three maps start out fully zeroed.

> +    cpumask_set_cpu(0, &cpu_possible_map);
> +    cpumask_set_cpu(0, &cpu_online_map);

These are contradicting the name of the function. The somewhat equivalent
function we have on x86 is smp_prepare_boot_cpu().

> +    cpumask_copy(&cpu_present_map, &cpu_possible_map);

Another cpumask_set_cpu() is probably cheaper here then.

Jan

Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 3995 bytes --]


On 4/10/25 3:10 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> Initialize cpu_{possible, online, present}_map by using smp_clear_cpu_maps().
>>
>> Drop DEFINE_PER_CPU(unsigned int, cpu_id) from stubs.c as this variable isn't
>> expected to be used in RISC-V at all.
>>
>> Move declaration of cpu_{possible,online,present}_map from stubs.c to smpboot.c
>> as now smpboot.c is now introduced.
>> Other defintions keep in stubs.c as they are not initialized and not needed, at
>> the moment.
>>
>> Signed-off-by: Oleksii Kurochko<oleksii.kurochko@gmail.com>
>> ---
>>   xen/arch/riscv/Makefile          |  1 +
>>   xen/arch/riscv/include/asm/smp.h |  2 ++
>>   xen/arch/riscv/setup.c           |  2 ++
>>   xen/arch/riscv/smpboot.c         | 15 +++++++++++++++
>>   xen/arch/riscv/stubs.c           |  6 ------
>>   5 files changed, 20 insertions(+), 6 deletions(-)
>>   create mode 100644 xen/arch/riscv/smpboot.c
>>
>> diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile
>> index 0c6c4a38a3..f551bf32a2 100644
>> --- a/xen/arch/riscv/Makefile
>> +++ b/xen/arch/riscv/Makefile
>> @@ -10,6 +10,7 @@ obj-y += sbi.o
>>   obj-y += setup.o
>>   obj-y += shutdown.o
>>   obj-y += smp.o
>> +obj-y += smpboot.o
>>   obj-y += stubs.o
>>   obj-y += time.o
>>   obj-y += traps.o
>> diff --git a/xen/arch/riscv/include/asm/smp.h b/xen/arch/riscv/include/asm/smp.h
>> index 5e170b57b3..188c033718 100644
>> --- a/xen/arch/riscv/include/asm/smp.h
>> +++ b/xen/arch/riscv/include/asm/smp.h
>> @@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,
>>   
>>   void setup_tp(unsigned int cpuid);
>>   
>> +void smp_clear_cpu_maps(void);
>> +
>>   #endif
>>   
>>   /*
>> diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c
>> index 4e416f6e44..7f68f3f5b7 100644
>> --- a/xen/arch/riscv/setup.c
>> +++ b/xen/arch/riscv/setup.c
>> @@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
>>   
>>       remove_identity_mapping();
>>   
>> +    smp_clear_cpu_maps();
>> +
>>       set_processor_id(0);
>>   
>>       set_cpuid_to_hartid(0, bootcpu_id);
>> diff --git a/xen/arch/riscv/smpboot.c b/xen/arch/riscv/smpboot.c
>> new file mode 100644
>> index 0000000000..0f4dcc28e1
>> --- /dev/null
>> +++ b/xen/arch/riscv/smpboot.c
>> @@ -0,0 +1,15 @@
>> +#include <xen/cpumask.h>
>> +#include <xen/init.h>
>> +
>> +cpumask_t cpu_online_map;
>> +cpumask_t cpu_present_map;
>> +cpumask_t cpu_possible_map;
> __read_mostly for all of them, perhaps (if CPU hotplug isn't expected to
> be supported) even __ro_after_init for the latter two?

We have been living without CPU hotplug support for a long time in the downstream
branch, but I can't say whether it is expected to be supported in the future or not.
To ensure we can add such an option later without changing the attributes of
cpu_online_map variable, I prefer to use|__read_mostly| here and __ro_after_init for
cpu_possible_map.

>
> As to cpu_possible_map - do you predict that you'll actually use it? Arm
> does (and instead has only a fake cpu_present_map), but on x86 we get away
> without.

I checked how it is used now in downstream latest branch and it isn't really used
only during initialization smp_clear_cpu_maps() and smp_prepare_cpus() so we can
skip it for RISC-V too.

>
>> +void __init smp_clear_cpu_maps(void)
>> +{
>> +    cpumask_clear(&cpu_possible_map);
>> +    cpumask_clear(&cpu_online_map);
> What's the point of these? All three maps start out fully zeroed.

It could be really dropped. I saw your patch for Arm, I'll align the current
patch with that changes.


>
>> +    cpumask_set_cpu(0, &cpu_possible_map);
>> +    cpumask_set_cpu(0, &cpu_online_map);
> These are contradicting the name of the function. The somewhat equivalent
> function we have on x86 is smp_prepare_boot_cpu().
>
>> +    cpumask_copy(&cpu_present_map, &cpu_possible_map);
> Another cpumask_set_cpu() is probably cheaper here then.

What do you mean by cheaper here?

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 5420 bytes --]

Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()

[Jan Beulich]

On 14.04.2025 17:05, Oleksii Kurochko wrote:
> On 4/10/25 3:10 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> +void __init smp_clear_cpu_maps(void)
>>> +{
>>> +    cpumask_clear(&cpu_possible_map);
>>> +    cpumask_clear(&cpu_online_map);
>> What's the point of these? All three maps start out fully zeroed.
> 
> It could be really dropped. I saw your patch for Arm, I'll align the current
> patch with that changes.
> 
>>> +    cpumask_set_cpu(0, &cpu_possible_map);
>>> +    cpumask_set_cpu(0, &cpu_online_map);
>> These are contradicting the name of the function. The somewhat equivalent
>> function we have on x86 is smp_prepare_boot_cpu().
>>
>>> +    cpumask_copy(&cpu_present_map, &cpu_possible_map);
>> Another cpumask_set_cpu() is probably cheaper here then.
> 
> What do you mean by cheaper here?

Less code to execute to achieve the same effect.

Jan

[PATCH v1 03/14] xen/riscv: introduce ioremap()

[Oleksii Kurochko]

Based on RISC-V unpriviliged spec ( Version 20240411 ):
```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension
and that ones is going to be supported by Xen have memory coherency
between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it
should support CMO extensions and Svpbmt. In this scenario, updates to
ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to
update ioremap() is not overlooked.

[1] https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/Kconfig | 12 ++++++++++++
 xen/arch/riscv/pt.c    | 19 +++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/xen/arch/riscv/Kconfig b/xen/arch/riscv/Kconfig
index d882e0a059..27086cca9c 100644
--- a/xen/arch/riscv/Kconfig
+++ b/xen/arch/riscv/Kconfig
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
 	string
 	default "arch/riscv/configs/tiny64_defconfig"
 
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.
+
 menu "Architecture Features"
 
 source "arch/Kconfig"
diff --git a/xen/arch/riscv/pt.c b/xen/arch/riscv/pt.c
index 857619d48d..e2f49e2f97 100644
--- a/xen/arch/riscv/pt.c
+++ b/xen/arch/riscv/pt.c
@@ -7,6 +7,7 @@
 #include <xen/pfn.h>
 #include <xen/pmap.h>
 #include <xen/spinlock.h>
+#include <xen/vmap.h>
 
 #include <asm/fixmap.h>
 #include <asm/flushtlb.h>
@@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map)
                               FIXMAP_ADDR(map) + PAGE_SIZE) != 0 )
         BUG();
 }
+
+void *ioremap(paddr_t pa, size_t len)
+{
+    mfn_t mfn = _mfn(PFN_DOWN(pa));
+    unsigned int offs = pa & (PAGE_SIZE - 1);
+    unsigned int nr = PFN_UP(offs + len);
+
+#ifdef CONFIG_HAS_SVPBMT
+    #error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()"
+#endif
+
+    void *ptr = __vmap(&mfn, nr, 1, 1, PAGE_HYPERVISOR, VMAP_DEFAULT);
+
+    if ( !ptr )
+        return NULL;
+
+    return ptr + offs;
+}
-- 
2.49.0

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> Based on RISC-V unpriviliged spec ( Version 20240411 ):
> ```
> For implementations that conform to the RISC-V Unix Platform Specification,
> I/O devices and DMA operations are required to access memory coherently and
> via strongly ordered I/O channels. Therefore, accesses to regular main memory
> regions that are concurrently accessed by external devices can also use the
> standard synchronization mechanisms. Implementations that do not conform
> to the Unix Platform Specification and/or in which devices do not access
> memory coherently will need to use mechanisms
> (which are currently platform-specific or device-specific) to enforce
> coherency.
> 
> I/O regions in the address space should be considered non-cacheable
> regions in the PMAs for those regions. Such regions can be considered coherent
> by the PMA if they are not cached by any agent.
> ```
> and [1]:
> ```
> The current riscv linux implementation requires SOC system to support
> memory coherence between all I/O devices and CPUs. But some SOC systems
> cannot maintain the coherence and they need support cache clean/invalid
> operations to synchronize data.
> 
> Current implementation is no problem with SiFive FU540, because FU540
> keeps all IO devices and DMA master devices coherence with CPU. But to a
> traditional SOC vendor, it may already have a stable non-coherency SOC
> system, the need is simply to replace the CPU with RV CPU and rebuild
> the whole system with IO-coherency is very expensive.
> ```
> 
> and the fact that all known ( to me ) CPUs that support the H-extension
> and that ones is going to be supported by Xen have memory coherency
> between all I/O devices and CPUs, so it is currently safe to use the
> PAGE_HYPERVISOR attribute.
> However, in cases where a platform does not support memory coherency, it
> should support CMO extensions and Svpbmt. In this scenario, updates to
> ioremap will be necessary.
> For now, a compilation error will be generated to ensure that the need to
> update ioremap() is not overlooked.
> 
> [1] https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/

But MMIO access correctness isn't just a matter of coherency. There may not
be any caching involved in most cases, or else you may observe significantly
delayed or even dropped (folded with later ones) writes, and reads may be
serviced from the cache instead of going to actual MMIO. Therefore ...

> --- a/xen/arch/riscv/Kconfig
> +++ b/xen/arch/riscv/Kconfig
> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>  	string
>  	default "arch/riscv/configs/tiny64_defconfig"
>  
> +config HAS_SVPBMT
> +	bool
> +	help
> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
> +	  page-based memory types).
> +
> +	  The memory type for a page contains a combination of attributes
> +	  that indicate the cacheability, idempotency, and ordering
> +	  properties for access to that page.
> +
> +	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need
making use of.

> @@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map)
>                                FIXMAP_ADDR(map) + PAGE_SIZE) != 0 )
>          BUG();
>  }
> +
> +void *ioremap(paddr_t pa, size_t len)
> +{
> +    mfn_t mfn = _mfn(PFN_DOWN(pa));
> +    unsigned int offs = pa & (PAGE_SIZE - 1);
> +    unsigned int nr = PFN_UP(offs + len);
> +
> +#ifdef CONFIG_HAS_SVPBMT
> +    #error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()"
> +#endif

While, as per above, I don't think this can stay, just in case: As indicated
earlier, pre-processor directives want to have the # in the first column.

Jan

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 4993 bytes --]


On 4/10/25 5:13 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>> ```
>> For implementations that conform to the RISC-V Unix Platform Specification,
>> I/O devices and DMA operations are required to access memory coherently and
>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>> regions that are concurrently accessed by external devices can also use the
>> standard synchronization mechanisms. Implementations that do not conform
>> to the Unix Platform Specification and/or in which devices do not access
>> memory coherently will need to use mechanisms
>> (which are currently platform-specific or device-specific) to enforce
>> coherency.
>>
>> I/O regions in the address space should be considered non-cacheable
>> regions in the PMAs for those regions. Such regions can be considered coherent
>> by the PMA if they are not cached by any agent.
>> ```
>> and [1]:
>> ```
>> The current riscv linux implementation requires SOC system to support
>> memory coherence between all I/O devices and CPUs. But some SOC systems
>> cannot maintain the coherence and they need support cache clean/invalid
>> operations to synchronize data.
>>
>> Current implementation is no problem with SiFive FU540, because FU540
>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>> traditional SOC vendor, it may already have a stable non-coherency SOC
>> system, the need is simply to replace the CPU with RV CPU and rebuild
>> the whole system with IO-coherency is very expensive.
>> ```
>>
>> and the fact that all known ( to me ) CPUs that support the H-extension
>> and that ones is going to be supported by Xen have memory coherency
>> between all I/O devices and CPUs, so it is currently safe to use the
>> PAGE_HYPERVISOR attribute.
>> However, in cases where a platform does not support memory coherency, it
>> should support CMO extensions and Svpbmt. In this scenario, updates to
>> ioremap will be necessary.
>> For now, a compilation error will be generated to ensure that the need to
>> update ioremap() is not overlooked.
>>
>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
> But MMIO access correctness isn't just a matter of coherency. There may not
> be any caching involved in most cases, or else you may observe significantly
> delayed or even dropped (folded with later ones) writes, and reads may be
> serviced from the cache instead of going to actual MMIO. Therefore ...
>
>> --- a/xen/arch/riscv/Kconfig
>> +++ b/xen/arch/riscv/Kconfig
>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>   	string
>>   	default "arch/riscv/configs/tiny64_defconfig"
>>   
>> +config HAS_SVPBMT
>> +	bool
>> +	help
>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>> +	  page-based memory types).
>> +
>> +	  The memory type for a page contains a combination of attributes
>> +	  that indicate the cacheability, idempotency, and ordering
>> +	  properties for access to that page.
>> +
>> +	  The Svpbmt extension is only available on 64-bit cpus.
> ... I kind of expect this extension (or anything else that there might be) will need
> making use of.

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
is used to control which memory regions are cacheable, non-cacheable, readable, writable,
etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
cores, or it can be fixed at design time, as in SiFive cores.

In the case of QEMU, I assume it is QEMU's responsibility to properly emulate accesses
to device memory regions. Since QEMU does not appear to provide registers for configuring
PMA, it seems that PMA is not emulated. Additionally, QEMU does not emulate caches.

Based on that, I expect that it is the responsibility of the firmware or the hardware
itself to provide the correct PMA configuration.

I want to note that even Svpbmt is available PMA settings could be used or be ovewritten
by Svpbmt's attribute value.

I will update the commit message  for more clearness.

>
>> @@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map)
>>                                 FIXMAP_ADDR(map) + PAGE_SIZE) != 0 )
>>           BUG();
>>   }
>> +
>> +void *ioremap(paddr_t pa, size_t len)
>> +{
>> +    mfn_t mfn = _mfn(PFN_DOWN(pa));
>> +    unsigned int offs = pa & (PAGE_SIZE - 1);
>> +    unsigned int nr = PFN_UP(offs + len);
>> +
>> +#ifdef CONFIG_HAS_SVPBMT
>> +    #error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()"
>> +#endif
> While, as per above, I don't think this can stay, just in case: As indicated
> earlier, pre-processor directives want to have the # in the first column.

I think it can be safely dropped now, and|PAGE_HYPERVISOR_IOREMAP| could be introduced
and simply PTE's PMBT bits can be ignored in|pt_update_entry()| if Svpbmt isn't implemented.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 6121 bytes --]

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Jan Beulich]

On 15.04.2025 12:29, Oleksii Kurochko wrote:
> 
> On 4/10/25 5:13 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>>> ```
>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>> I/O devices and DMA operations are required to access memory coherently and
>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>> regions that are concurrently accessed by external devices can also use the
>>> standard synchronization mechanisms. Implementations that do not conform
>>> to the Unix Platform Specification and/or in which devices do not access
>>> memory coherently will need to use mechanisms
>>> (which are currently platform-specific or device-specific) to enforce
>>> coherency.
>>>
>>> I/O regions in the address space should be considered non-cacheable
>>> regions in the PMAs for those regions. Such regions can be considered coherent
>>> by the PMA if they are not cached by any agent.
>>> ```
>>> and [1]:
>>> ```
>>> The current riscv linux implementation requires SOC system to support
>>> memory coherence between all I/O devices and CPUs. But some SOC systems
>>> cannot maintain the coherence and they need support cache clean/invalid
>>> operations to synchronize data.
>>>
>>> Current implementation is no problem with SiFive FU540, because FU540
>>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>>> traditional SOC vendor, it may already have a stable non-coherency SOC
>>> system, the need is simply to replace the CPU with RV CPU and rebuild
>>> the whole system with IO-coherency is very expensive.
>>> ```
>>>
>>> and the fact that all known ( to me ) CPUs that support the H-extension
>>> and that ones is going to be supported by Xen have memory coherency
>>> between all I/O devices and CPUs, so it is currently safe to use the
>>> PAGE_HYPERVISOR attribute.
>>> However, in cases where a platform does not support memory coherency, it
>>> should support CMO extensions and Svpbmt. In this scenario, updates to
>>> ioremap will be necessary.
>>> For now, a compilation error will be generated to ensure that the need to
>>> update ioremap() is not overlooked.
>>>
>>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
>> But MMIO access correctness isn't just a matter of coherency. There may not
>> be any caching involved in most cases, or else you may observe significantly
>> delayed or even dropped (folded with later ones) writes, and reads may be
>> serviced from the cache instead of going to actual MMIO. Therefore ...
>>
>>> --- a/xen/arch/riscv/Kconfig
>>> +++ b/xen/arch/riscv/Kconfig
>>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>>   	string
>>>   	default "arch/riscv/configs/tiny64_defconfig"
>>>   
>>> +config HAS_SVPBMT
>>> +	bool
>>> +	help
>>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>>> +	  page-based memory types).
>>> +
>>> +	  The memory type for a page contains a combination of attributes
>>> +	  that indicate the cacheability, idempotency, and ordering
>>> +	  properties for access to that page.
>>> +
>>> +	  The Svpbmt extension is only available on 64-bit cpus.
>> ... I kind of expect this extension (or anything else that there might be) will need
>> making use of.
> 
> In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
> is used to control which memory regions are cacheable, non-cacheable, readable, writable,
> etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
> cores, or it can be fixed at design time, as in SiFive cores.

How would things work if there was a need to map a RAM page uncacheable (via
ioremap() or otherwise)?

Jan

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 4151 bytes --]


On 4/15/25 1:02 PM, Jan Beulich wrote:
> On 15.04.2025 12:29, Oleksii Kurochko wrote:
>> On 4/10/25 5:13 PM, Jan Beulich wrote:
>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>>>> ```
>>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>>> I/O devices and DMA operations are required to access memory coherently and
>>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>>> regions that are concurrently accessed by external devices can also use the
>>>> standard synchronization mechanisms. Implementations that do not conform
>>>> to the Unix Platform Specification and/or in which devices do not access
>>>> memory coherently will need to use mechanisms
>>>> (which are currently platform-specific or device-specific) to enforce
>>>> coherency.
>>>>
>>>> I/O regions in the address space should be considered non-cacheable
>>>> regions in the PMAs for those regions. Such regions can be considered coherent
>>>> by the PMA if they are not cached by any agent.
>>>> ```
>>>> and [1]:
>>>> ```
>>>> The current riscv linux implementation requires SOC system to support
>>>> memory coherence between all I/O devices and CPUs. But some SOC systems
>>>> cannot maintain the coherence and they need support cache clean/invalid
>>>> operations to synchronize data.
>>>>
>>>> Current implementation is no problem with SiFive FU540, because FU540
>>>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>>>> traditional SOC vendor, it may already have a stable non-coherency SOC
>>>> system, the need is simply to replace the CPU with RV CPU and rebuild
>>>> the whole system with IO-coherency is very expensive.
>>>> ```
>>>>
>>>> and the fact that all known ( to me ) CPUs that support the H-extension
>>>> and that ones is going to be supported by Xen have memory coherency
>>>> between all I/O devices and CPUs, so it is currently safe to use the
>>>> PAGE_HYPERVISOR attribute.
>>>> However, in cases where a platform does not support memory coherency, it
>>>> should support CMO extensions and Svpbmt. In this scenario, updates to
>>>> ioremap will be necessary.
>>>> For now, a compilation error will be generated to ensure that the need to
>>>> update ioremap() is not overlooked.
>>>>
>>>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
>>> But MMIO access correctness isn't just a matter of coherency. There may not
>>> be any caching involved in most cases, or else you may observe significantly
>>> delayed or even dropped (folded with later ones) writes, and reads may be
>>> serviced from the cache instead of going to actual MMIO. Therefore ...
>>>
>>>> --- a/xen/arch/riscv/Kconfig
>>>> +++ b/xen/arch/riscv/Kconfig
>>>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>>>    	string
>>>>    	default "arch/riscv/configs/tiny64_defconfig"
>>>>    
>>>> +config HAS_SVPBMT
>>>> +	bool
>>>> +	help
>>>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>>>> +	  page-based memory types).
>>>> +
>>>> +	  The memory type for a page contains a combination of attributes
>>>> +	  that indicate the cacheability, idempotency, and ordering
>>>> +	  properties for access to that page.
>>>> +
>>>> +	  The Svpbmt extension is only available on 64-bit cpus.
>>> ... I kind of expect this extension (or anything else that there might be) will need
>>> making use of.
>> In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
>> is used to control which memory regions are cacheable, non-cacheable, readable, writable,
>> etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
>> cores, or it can be fixed at design time, as in SiFive cores.
> How would things work if there was a need to map a RAM page uncacheable (via
> ioremap() or otherwise)?

My understanding is that Svpbmt is only needed when someone wants to change the memory
attribute of a page set by PMA.

The question is if non-cacheable RAM page is really needed if we have a coherency?

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 4982 bytes --]

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Jan Beulich]

On 17.04.2025 16:20, Oleksii Kurochko wrote:
> On 4/15/25 1:02 PM, Jan Beulich wrote:
>> On 15.04.2025 12:29, Oleksii Kurochko wrote:
>>> On 4/10/25 5:13 PM, Jan Beulich wrote:
>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>>>>> ```
>>>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>>>> I/O devices and DMA operations are required to access memory coherently and
>>>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>>>> regions that are concurrently accessed by external devices can also use the
>>>>> standard synchronization mechanisms. Implementations that do not conform
>>>>> to the Unix Platform Specification and/or in which devices do not access
>>>>> memory coherently will need to use mechanisms
>>>>> (which are currently platform-specific or device-specific) to enforce
>>>>> coherency.
>>>>>
>>>>> I/O regions in the address space should be considered non-cacheable
>>>>> regions in the PMAs for those regions. Such regions can be considered coherent
>>>>> by the PMA if they are not cached by any agent.
>>>>> ```
>>>>> and [1]:
>>>>> ```
>>>>> The current riscv linux implementation requires SOC system to support
>>>>> memory coherence between all I/O devices and CPUs. But some SOC systems
>>>>> cannot maintain the coherence and they need support cache clean/invalid
>>>>> operations to synchronize data.
>>>>>
>>>>> Current implementation is no problem with SiFive FU540, because FU540
>>>>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>>>>> traditional SOC vendor, it may already have a stable non-coherency SOC
>>>>> system, the need is simply to replace the CPU with RV CPU and rebuild
>>>>> the whole system with IO-coherency is very expensive.
>>>>> ```
>>>>>
>>>>> and the fact that all known ( to me ) CPUs that support the H-extension
>>>>> and that ones is going to be supported by Xen have memory coherency
>>>>> between all I/O devices and CPUs, so it is currently safe to use the
>>>>> PAGE_HYPERVISOR attribute.
>>>>> However, in cases where a platform does not support memory coherency, it
>>>>> should support CMO extensions and Svpbmt. In this scenario, updates to
>>>>> ioremap will be necessary.
>>>>> For now, a compilation error will be generated to ensure that the need to
>>>>> update ioremap() is not overlooked.
>>>>>
>>>>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
>>>> But MMIO access correctness isn't just a matter of coherency. There may not
>>>> be any caching involved in most cases, or else you may observe significantly
>>>> delayed or even dropped (folded with later ones) writes, and reads may be
>>>> serviced from the cache instead of going to actual MMIO. Therefore ...
>>>>
>>>>> --- a/xen/arch/riscv/Kconfig
>>>>> +++ b/xen/arch/riscv/Kconfig
>>>>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>>>>    	string
>>>>>    	default "arch/riscv/configs/tiny64_defconfig"
>>>>>    
>>>>> +config HAS_SVPBMT
>>>>> +	bool
>>>>> +	help
>>>>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>>>>> +	  page-based memory types).
>>>>> +
>>>>> +	  The memory type for a page contains a combination of attributes
>>>>> +	  that indicate the cacheability, idempotency, and ordering
>>>>> +	  properties for access to that page.
>>>>> +
>>>>> +	  The Svpbmt extension is only available on 64-bit cpus.
>>>> ... I kind of expect this extension (or anything else that there might be) will need
>>>> making use of.
>>> In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
>>> is used to control which memory regions are cacheable, non-cacheable, readable, writable,
>>> etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
>>> cores, or it can be fixed at design time, as in SiFive cores.
>> How would things work if there was a need to map a RAM page uncacheable (via
>> ioremap() or otherwise)?
> 
> My understanding is that Svpbmt is only needed when someone wants to change the memory
> attribute of a page set by PMA.
> 
> The question is if non-cacheable RAM page is really needed if we have a coherency?

Aiui coherency here is among CPUs. Properties of devices in the system are
largely unknown? (Beyond this there may also be special situations in which
one really cares about data going directly to RAM.)

Jan

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 5641 bytes --]


On 4/17/25 4:24 PM, Jan Beulich wrote:
> On 17.04.2025 16:20, Oleksii Kurochko wrote:
>> On 4/15/25 1:02 PM, Jan Beulich wrote:
>>> On 15.04.2025 12:29, Oleksii Kurochko wrote:
>>>> On 4/10/25 5:13 PM, Jan Beulich wrote:
>>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>>>>>> ```
>>>>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>>>>> I/O devices and DMA operations are required to access memory coherently and
>>>>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>>>>> regions that are concurrently accessed by external devices can also use the
>>>>>> standard synchronization mechanisms. Implementations that do not conform
>>>>>> to the Unix Platform Specification and/or in which devices do not access
>>>>>> memory coherently will need to use mechanisms
>>>>>> (which are currently platform-specific or device-specific) to enforce
>>>>>> coherency.
>>>>>>
>>>>>> I/O regions in the address space should be considered non-cacheable
>>>>>> regions in the PMAs for those regions. Such regions can be considered coherent
>>>>>> by the PMA if they are not cached by any agent.
>>>>>> ```
>>>>>> and [1]:
>>>>>> ```
>>>>>> The current riscv linux implementation requires SOC system to support
>>>>>> memory coherence between all I/O devices and CPUs. But some SOC systems
>>>>>> cannot maintain the coherence and they need support cache clean/invalid
>>>>>> operations to synchronize data.
>>>>>>
>>>>>> Current implementation is no problem with SiFive FU540, because FU540
>>>>>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>>>>>> traditional SOC vendor, it may already have a stable non-coherency SOC
>>>>>> system, the need is simply to replace the CPU with RV CPU and rebuild
>>>>>> the whole system with IO-coherency is very expensive.
>>>>>> ```
>>>>>>
>>>>>> and the fact that all known ( to me ) CPUs that support the H-extension
>>>>>> and that ones is going to be supported by Xen have memory coherency
>>>>>> between all I/O devices and CPUs, so it is currently safe to use the
>>>>>> PAGE_HYPERVISOR attribute.
>>>>>> However, in cases where a platform does not support memory coherency, it
>>>>>> should support CMO extensions and Svpbmt. In this scenario, updates to
>>>>>> ioremap will be necessary.
>>>>>> For now, a compilation error will be generated to ensure that the need to
>>>>>> update ioremap() is not overlooked.
>>>>>>
>>>>>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
>>>>> But MMIO access correctness isn't just a matter of coherency. There may not
>>>>> be any caching involved in most cases, or else you may observe significantly
>>>>> delayed or even dropped (folded with later ones) writes, and reads may be
>>>>> serviced from the cache instead of going to actual MMIO. Therefore ...
>>>>>
>>>>>> --- a/xen/arch/riscv/Kconfig
>>>>>> +++ b/xen/arch/riscv/Kconfig
>>>>>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>>>>>     	string
>>>>>>     	default "arch/riscv/configs/tiny64_defconfig"
>>>>>>     
>>>>>> +config HAS_SVPBMT
>>>>>> +	bool
>>>>>> +	help
>>>>>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>>>>>> +	  page-based memory types).
>>>>>> +
>>>>>> +	  The memory type for a page contains a combination of attributes
>>>>>> +	  that indicate the cacheability, idempotency, and ordering
>>>>>> +	  properties for access to that page.
>>>>>> +
>>>>>> +	  The Svpbmt extension is only available on 64-bit cpus.
>>>>> ... I kind of expect this extension (or anything else that there might be) will need
>>>>> making use of.
>>>> In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
>>>> is used to control which memory regions are cacheable, non-cacheable, readable, writable,
>>>> etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
>>>> cores, or it can be fixed at design time, as in SiFive cores.
>>> How would things work if there was a need to map a RAM page uncacheable (via
>>> ioremap() or otherwise)?
>> My understanding is that Svpbmt is only needed when someone wants to change the memory
>> attribute of a page set by PMA.
>>
>> The question is if non-cacheable RAM page is really needed if we have a coherency?
> Aiui coherency here is among CPUs.

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.
```
Based on this from the spec, coherency here is not only among CPUs.


> Properties of devices in the system are
> largely unknown?

Yes, but still not sure what kind of property requires ioremap() which won't work
without Svpmbt. Could you please tell me an example?

> (Beyond this there may also be special situations in which
> one really cares about data going directly to RAM.)

If there are such special cases, I assume that the firmware or hardware (in the case
of fixed PMA) will provide a non-cacheable region. In that case, the user should be
aware of this region and use it for those specific scenarios.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 7042 bytes --]

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Jan Beulich]

On 17.04.2025 16:37, Oleksii Kurochko wrote:
> 
> On 4/17/25 4:24 PM, Jan Beulich wrote:
>> On 17.04.2025 16:20, Oleksii Kurochko wrote:
>>> On 4/15/25 1:02 PM, Jan Beulich wrote:
>>>> On 15.04.2025 12:29, Oleksii Kurochko wrote:
>>>>> On 4/10/25 5:13 PM, Jan Beulich wrote:
>>>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>>>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>>>>>>> ```
>>>>>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>>>>>> I/O devices and DMA operations are required to access memory coherently and
>>>>>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>>>>>> regions that are concurrently accessed by external devices can also use the
>>>>>>> standard synchronization mechanisms. Implementations that do not conform
>>>>>>> to the Unix Platform Specification and/or in which devices do not access
>>>>>>> memory coherently will need to use mechanisms
>>>>>>> (which are currently platform-specific or device-specific) to enforce
>>>>>>> coherency.
>>>>>>>
>>>>>>> I/O regions in the address space should be considered non-cacheable
>>>>>>> regions in the PMAs for those regions. Such regions can be considered coherent
>>>>>>> by the PMA if they are not cached by any agent.
>>>>>>> ```
>>>>>>> and [1]:
>>>>>>> ```
>>>>>>> The current riscv linux implementation requires SOC system to support
>>>>>>> memory coherence between all I/O devices and CPUs. But some SOC systems
>>>>>>> cannot maintain the coherence and they need support cache clean/invalid
>>>>>>> operations to synchronize data.
>>>>>>>
>>>>>>> Current implementation is no problem with SiFive FU540, because FU540
>>>>>>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>>>>>>> traditional SOC vendor, it may already have a stable non-coherency SOC
>>>>>>> system, the need is simply to replace the CPU with RV CPU and rebuild
>>>>>>> the whole system with IO-coherency is very expensive.
>>>>>>> ```
>>>>>>>
>>>>>>> and the fact that all known ( to me ) CPUs that support the H-extension
>>>>>>> and that ones is going to be supported by Xen have memory coherency
>>>>>>> between all I/O devices and CPUs, so it is currently safe to use the
>>>>>>> PAGE_HYPERVISOR attribute.
>>>>>>> However, in cases where a platform does not support memory coherency, it
>>>>>>> should support CMO extensions and Svpbmt. In this scenario, updates to
>>>>>>> ioremap will be necessary.
>>>>>>> For now, a compilation error will be generated to ensure that the need to
>>>>>>> update ioremap() is not overlooked.
>>>>>>>
>>>>>>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
>>>>>> But MMIO access correctness isn't just a matter of coherency. There may not
>>>>>> be any caching involved in most cases, or else you may observe significantly
>>>>>> delayed or even dropped (folded with later ones) writes, and reads may be
>>>>>> serviced from the cache instead of going to actual MMIO. Therefore ...
>>>>>>
>>>>>>> --- a/xen/arch/riscv/Kconfig
>>>>>>> +++ b/xen/arch/riscv/Kconfig
>>>>>>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>>>>>>     	string
>>>>>>>     	default "arch/riscv/configs/tiny64_defconfig"
>>>>>>>     
>>>>>>> +config HAS_SVPBMT
>>>>>>> +	bool
>>>>>>> +	help
>>>>>>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>>>>>>> +	  page-based memory types).
>>>>>>> +
>>>>>>> +	  The memory type for a page contains a combination of attributes
>>>>>>> +	  that indicate the cacheability, idempotency, and ordering
>>>>>>> +	  properties for access to that page.
>>>>>>> +
>>>>>>> +	  The Svpbmt extension is only available on 64-bit cpus.
>>>>>> ... I kind of expect this extension (or anything else that there might be) will need
>>>>>> making use of.
>>>>> In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
>>>>> is used to control which memory regions are cacheable, non-cacheable, readable, writable,
>>>>> etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
>>>>> cores, or it can be fixed at design time, as in SiFive cores.
>>>> How would things work if there was a need to map a RAM page uncacheable (via
>>>> ioremap() or otherwise)?
>>> My understanding is that Svpbmt is only needed when someone wants to change the memory
>>> attribute of a page set by PMA.
>>>
>>> The question is if non-cacheable RAM page is really needed if we have a coherency?
>> Aiui coherency here is among CPUs.
> 
> ```
> For implementations that conform to the RISC-V Unix Platform Specification,
> I/O devices and DMA operations are required to access memory coherently and
> via strongly ordered I/O channels. Therefore, accesses to regular main memory
> regions that are concurrently accessed by external devices can also use the
> standard synchronization mechanisms. Implementations that do not conform
> to the Unix Platform Specification and/or in which devices do not access
> memory coherently will need to use mechanisms
> (which are currently platform-specific or device-specific) to enforce
> coherency.
> ```
> Based on this from the spec, coherency here is not only among CPUs.
> 
> 
>> Properties of devices in the system are
>> largely unknown?
> 
> Yes, but still not sure what kind of property requires ioremap() which won't work
> without Svpmbt. Could you please tell me an example?

Well, above you said they all need to access memory coherently. That's the
"property" I was referring to.

>> (Beyond this there may also be special situations in which
>> one really cares about data going directly to RAM.)
> 
> If there are such special cases, I assume that the firmware or hardware (in the case
> of fixed PMA) will provide a non-cacheable region.

How could they? Firmware may be unaware of specific properties of specific
devices a user adds to a system.

Jan

> In that case, the user should be
> aware of this region and use it for those specific scenarios.
> 
> ~ Oleksii
> 

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 6969 bytes --]


On 4/17/25 4:49 PM, Jan Beulich wrote:
> On 17.04.2025 16:37, Oleksii Kurochko wrote:
>> On 4/17/25 4:24 PM, Jan Beulich wrote:
>>> On 17.04.2025 16:20, Oleksii Kurochko wrote:
>>>> On 4/15/25 1:02 PM, Jan Beulich wrote:
>>>>> On 15.04.2025 12:29, Oleksii Kurochko wrote:
>>>>>> On 4/10/25 5:13 PM, Jan Beulich wrote:
>>>>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>>>>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>>>>>>>> ```
>>>>>>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>>>>>>> I/O devices and DMA operations are required to access memory coherently and
>>>>>>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>>>>>>> regions that are concurrently accessed by external devices can also use the
>>>>>>>> standard synchronization mechanisms. Implementations that do not conform
>>>>>>>> to the Unix Platform Specification and/or in which devices do not access
>>>>>>>> memory coherently will need to use mechanisms
>>>>>>>> (which are currently platform-specific or device-specific) to enforce
>>>>>>>> coherency.
>>>>>>>>
>>>>>>>> I/O regions in the address space should be considered non-cacheable
>>>>>>>> regions in the PMAs for those regions. Such regions can be considered coherent
>>>>>>>> by the PMA if they are not cached by any agent.
>>>>>>>> ```
>>>>>>>> and [1]:
>>>>>>>> ```
>>>>>>>> The current riscv linux implementation requires SOC system to support
>>>>>>>> memory coherence between all I/O devices and CPUs. But some SOC systems
>>>>>>>> cannot maintain the coherence and they need support cache clean/invalid
>>>>>>>> operations to synchronize data.
>>>>>>>>
>>>>>>>> Current implementation is no problem with SiFive FU540, because FU540
>>>>>>>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>>>>>>>> traditional SOC vendor, it may already have a stable non-coherency SOC
>>>>>>>> system, the need is simply to replace the CPU with RV CPU and rebuild
>>>>>>>> the whole system with IO-coherency is very expensive.
>>>>>>>> ```
>>>>>>>>
>>>>>>>> and the fact that all known ( to me ) CPUs that support the H-extension
>>>>>>>> and that ones is going to be supported by Xen have memory coherency
>>>>>>>> between all I/O devices and CPUs, so it is currently safe to use the
>>>>>>>> PAGE_HYPERVISOR attribute.
>>>>>>>> However, in cases where a platform does not support memory coherency, it
>>>>>>>> should support CMO extensions and Svpbmt. In this scenario, updates to
>>>>>>>> ioremap will be necessary.
>>>>>>>> For now, a compilation error will be generated to ensure that the need to
>>>>>>>> update ioremap() is not overlooked.
>>>>>>>>
>>>>>>>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
>>>>>>> But MMIO access correctness isn't just a matter of coherency. There may not
>>>>>>> be any caching involved in most cases, or else you may observe significantly
>>>>>>> delayed or even dropped (folded with later ones) writes, and reads may be
>>>>>>> serviced from the cache instead of going to actual MMIO. Therefore ...
>>>>>>>
>>>>>>>> --- a/xen/arch/riscv/Kconfig
>>>>>>>> +++ b/xen/arch/riscv/Kconfig
>>>>>>>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>>>>>>>      	string
>>>>>>>>      	default "arch/riscv/configs/tiny64_defconfig"
>>>>>>>>      
>>>>>>>> +config HAS_SVPBMT
>>>>>>>> +	bool
>>>>>>>> +	help
>>>>>>>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>>>>>>>> +	  page-based memory types).
>>>>>>>> +
>>>>>>>> +	  The memory type for a page contains a combination of attributes
>>>>>>>> +	  that indicate the cacheability, idempotency, and ordering
>>>>>>>> +	  properties for access to that page.
>>>>>>>> +
>>>>>>>> +	  The Svpbmt extension is only available on 64-bit cpus.
>>>>>>> ... I kind of expect this extension (or anything else that there might be) will need
>>>>>>> making use of.
>>>>>> In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
>>>>>> is used to control which memory regions are cacheable, non-cacheable, readable, writable,
>>>>>> etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
>>>>>> cores, or it can be fixed at design time, as in SiFive cores.
>>>>> How would things work if there was a need to map a RAM page uncacheable (via
>>>>> ioremap() or otherwise)?
>>>> My understanding is that Svpbmt is only needed when someone wants to change the memory
>>>> attribute of a page set by PMA.
>>>>
>>>> The question is if non-cacheable RAM page is really needed if we have a coherency?
>>> Aiui coherency here is among CPUs.
>> ```
>> For implementations that conform to the RISC-V Unix Platform Specification,
>> I/O devices and DMA operations are required to access memory coherently and
>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>> regions that are concurrently accessed by external devices can also use the
>> standard synchronization mechanisms. Implementations that do not conform
>> to the Unix Platform Specification and/or in which devices do not access
>> memory coherently will need to use mechanisms
>> (which are currently platform-specific or device-specific) to enforce
>> coherency.
>> ```
>> Based on this from the spec, coherency here is not only among CPUs.
>>
>>
>>> Properties of devices in the system are
>>> largely unknown?
>> Yes, but still not sure what kind of property requires ioremap() which won't work
>> without Svpmbt. Could you please tell me an example?
> Well, above you said they all need to access memory coherently. That's the
> "property" I was referring to.

Do you mean that device could have a property which tell that it would like to have non-cachable
region used for that? I haven't seen such property in device tree files.

Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the
one option?

I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency
for everything).

Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice
and the support of Svpmbt is required.

>
>>> (Beyond this there may also be special situations in which
>>> one really cares about data going directly to RAM.)
>> If there are such special cases, I assume that the firmware or hardware (in the case
>> of fixed PMA) will provide a non-cacheable region.
> How could they? Firmware may be unaware of specific properties of specific
> devices a user adds to a system.

(this is not real case, just thoughts) Firmware could by default provide part of RAM as
non-cacheable region and then hypervisor/kernel use this region for allocation. But I agree
that it isn't the best thing to manage that.


~ Oleksii

>> In that case, the user should be
>> aware of this region and use it for those specific scenarios.
>>
>> ~ Oleksii
>>

[-- Attachment #2: Type: text/html, Size: 8997 bytes --]

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Jan Beulich]

On 22.04.2025 10:40, Oleksii Kurochko wrote:
> 
> On 4/17/25 4:49 PM, Jan Beulich wrote:
>> On 17.04.2025 16:37, Oleksii Kurochko wrote:
>>> On 4/17/25 4:24 PM, Jan Beulich wrote:
>>>> On 17.04.2025 16:20, Oleksii Kurochko wrote:
>>>>> On 4/15/25 1:02 PM, Jan Beulich wrote:
>>>>>> On 15.04.2025 12:29, Oleksii Kurochko wrote:
>>>>>>> On 4/10/25 5:13 PM, Jan Beulich wrote:
>>>>>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>>>>>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>>>>>>>>> ```
>>>>>>>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>>>>>>>> I/O devices and DMA operations are required to access memory coherently and
>>>>>>>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>>>>>>>> regions that are concurrently accessed by external devices can also use the
>>>>>>>>> standard synchronization mechanisms. Implementations that do not conform
>>>>>>>>> to the Unix Platform Specification and/or in which devices do not access
>>>>>>>>> memory coherently will need to use mechanisms
>>>>>>>>> (which are currently platform-specific or device-specific) to enforce
>>>>>>>>> coherency.
>>>>>>>>>
>>>>>>>>> I/O regions in the address space should be considered non-cacheable
>>>>>>>>> regions in the PMAs for those regions. Such regions can be considered coherent
>>>>>>>>> by the PMA if they are not cached by any agent.
>>>>>>>>> ```
>>>>>>>>> and [1]:
>>>>>>>>> ```
>>>>>>>>> The current riscv linux implementation requires SOC system to support
>>>>>>>>> memory coherence between all I/O devices and CPUs. But some SOC systems
>>>>>>>>> cannot maintain the coherence and they need support cache clean/invalid
>>>>>>>>> operations to synchronize data.
>>>>>>>>>
>>>>>>>>> Current implementation is no problem with SiFive FU540, because FU540
>>>>>>>>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>>>>>>>>> traditional SOC vendor, it may already have a stable non-coherency SOC
>>>>>>>>> system, the need is simply to replace the CPU with RV CPU and rebuild
>>>>>>>>> the whole system with IO-coherency is very expensive.
>>>>>>>>> ```
>>>>>>>>>
>>>>>>>>> and the fact that all known ( to me ) CPUs that support the H-extension
>>>>>>>>> and that ones is going to be supported by Xen have memory coherency
>>>>>>>>> between all I/O devices and CPUs, so it is currently safe to use the
>>>>>>>>> PAGE_HYPERVISOR attribute.
>>>>>>>>> However, in cases where a platform does not support memory coherency, it
>>>>>>>>> should support CMO extensions and Svpbmt. In this scenario, updates to
>>>>>>>>> ioremap will be necessary.
>>>>>>>>> For now, a compilation error will be generated to ensure that the need to
>>>>>>>>> update ioremap() is not overlooked.
>>>>>>>>>
>>>>>>>>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
>>>>>>>> But MMIO access correctness isn't just a matter of coherency. There may not
>>>>>>>> be any caching involved in most cases, or else you may observe significantly
>>>>>>>> delayed or even dropped (folded with later ones) writes, and reads may be
>>>>>>>> serviced from the cache instead of going to actual MMIO. Therefore ...
>>>>>>>>
>>>>>>>>> --- a/xen/arch/riscv/Kconfig
>>>>>>>>> +++ b/xen/arch/riscv/Kconfig
>>>>>>>>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>>>>>>>>      	string
>>>>>>>>>      	default "arch/riscv/configs/tiny64_defconfig"
>>>>>>>>>      
>>>>>>>>> +config HAS_SVPBMT
>>>>>>>>> +	bool
>>>>>>>>> +	help
>>>>>>>>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>>>>>>>>> +	  page-based memory types).
>>>>>>>>> +
>>>>>>>>> +	  The memory type for a page contains a combination of attributes
>>>>>>>>> +	  that indicate the cacheability, idempotency, and ordering
>>>>>>>>> +	  properties for access to that page.
>>>>>>>>> +
>>>>>>>>> +	  The Svpbmt extension is only available on 64-bit cpus.
>>>>>>>> ... I kind of expect this extension (or anything else that there might be) will need
>>>>>>>> making use of.
>>>>>>> In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
>>>>>>> is used to control which memory regions are cacheable, non-cacheable, readable, writable,
>>>>>>> etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
>>>>>>> cores, or it can be fixed at design time, as in SiFive cores.
>>>>>> How would things work if there was a need to map a RAM page uncacheable (via
>>>>>> ioremap() or otherwise)?
>>>>> My understanding is that Svpbmt is only needed when someone wants to change the memory
>>>>> attribute of a page set by PMA.
>>>>>
>>>>> The question is if non-cacheable RAM page is really needed if we have a coherency?
>>>> Aiui coherency here is among CPUs.
>>> ```
>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>> I/O devices and DMA operations are required to access memory coherently and
>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>> regions that are concurrently accessed by external devices can also use the
>>> standard synchronization mechanisms. Implementations that do not conform
>>> to the Unix Platform Specification and/or in which devices do not access
>>> memory coherently will need to use mechanisms
>>> (which are currently platform-specific or device-specific) to enforce
>>> coherency.
>>> ```
>>> Based on this from the spec, coherency here is not only among CPUs.
>>>
>>>
>>>> Properties of devices in the system are
>>>> largely unknown?
>>> Yes, but still not sure what kind of property requires ioremap() which won't work
>>> without Svpmbt. Could you please tell me an example?
>> Well, above you said they all need to access memory coherently. That's the
>> "property" I was referring to.
> 
> Do you mean that device could have a property which tell that it would like to have non-cachable
> region used for that? I haven't seen such property in device tree files.
> 
> Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the
> one option?

On x86 we have the case that IOMMUs may access memory non-coherently. This is
particular means that IOMMU page table updates (which necessarily live in RAM)
need to be done quite carefully. As it's all our code, we deal with the
situation by issuing cache flushes, avoiding the need for UC mappings.

Graphics engines may have similar constraints, aiui. With the driver code not
being part of Xen, we wouldn't be able to use a similar "simplification" there.
UC mappings would be pretty much unavoidable.

> I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency
> for everything).

Not sure what exactly you're asking here (if anything). An answer would very
likely depend on the specific kind of barrier you're thinking about. The
question would be what, if any, effect a barrier would have on the cache(s).

> Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice
> and the support of Svpmbt is required.

Quite possible.

Jan

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 9663 bytes --]


On 4/22/25 11:14 AM, Jan Beulich wrote:
> On 22.04.2025 10:40, Oleksii Kurochko wrote:
>> On 4/17/25 4:49 PM, Jan Beulich wrote:
>>> On 17.04.2025 16:37, Oleksii Kurochko wrote:
>>>> On 4/17/25 4:24 PM, Jan Beulich wrote:
>>>>> On 17.04.2025 16:20, Oleksii Kurochko wrote:
>>>>>> On 4/15/25 1:02 PM, Jan Beulich wrote:
>>>>>>> On 15.04.2025 12:29, Oleksii Kurochko wrote:
>>>>>>>> On 4/10/25 5:13 PM, Jan Beulich wrote:
>>>>>>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>>>>>>> Based on RISC-V unpriviliged spec ( Version 20240411 ):
>>>>>>>>>> ```
>>>>>>>>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>>>>>>>>> I/O devices and DMA operations are required to access memory coherently and
>>>>>>>>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>>>>>>>>> regions that are concurrently accessed by external devices can also use the
>>>>>>>>>> standard synchronization mechanisms. Implementations that do not conform
>>>>>>>>>> to the Unix Platform Specification and/or in which devices do not access
>>>>>>>>>> memory coherently will need to use mechanisms
>>>>>>>>>> (which are currently platform-specific or device-specific) to enforce
>>>>>>>>>> coherency.
>>>>>>>>>>
>>>>>>>>>> I/O regions in the address space should be considered non-cacheable
>>>>>>>>>> regions in the PMAs for those regions. Such regions can be considered coherent
>>>>>>>>>> by the PMA if they are not cached by any agent.
>>>>>>>>>> ```
>>>>>>>>>> and [1]:
>>>>>>>>>> ```
>>>>>>>>>> The current riscv linux implementation requires SOC system to support
>>>>>>>>>> memory coherence between all I/O devices and CPUs. But some SOC systems
>>>>>>>>>> cannot maintain the coherence and they need support cache clean/invalid
>>>>>>>>>> operations to synchronize data.
>>>>>>>>>>
>>>>>>>>>> Current implementation is no problem with SiFive FU540, because FU540
>>>>>>>>>> keeps all IO devices and DMA master devices coherence with CPU. But to a
>>>>>>>>>> traditional SOC vendor, it may already have a stable non-coherency SOC
>>>>>>>>>> system, the need is simply to replace the CPU with RV CPU and rebuild
>>>>>>>>>> the whole system with IO-coherency is very expensive.
>>>>>>>>>> ```
>>>>>>>>>>
>>>>>>>>>> and the fact that all known ( to me ) CPUs that support the H-extension
>>>>>>>>>> and that ones is going to be supported by Xen have memory coherency
>>>>>>>>>> between all I/O devices and CPUs, so it is currently safe to use the
>>>>>>>>>> PAGE_HYPERVISOR attribute.
>>>>>>>>>> However, in cases where a platform does not support memory coherency, it
>>>>>>>>>> should support CMO extensions and Svpbmt. In this scenario, updates to
>>>>>>>>>> ioremap will be necessary.
>>>>>>>>>> For now, a compilation error will be generated to ensure that the need to
>>>>>>>>>> update ioremap() is not overlooked.
>>>>>>>>>>
>>>>>>>>>> [1]https://patchwork.kernel.org/project/linux-riscv/patch/1555947870-23014-1-git-send-email-guoren@kernel.org/
>>>>>>>>> But MMIO access correctness isn't just a matter of coherency. There may not
>>>>>>>>> be any caching involved in most cases, or else you may observe significantly
>>>>>>>>> delayed or even dropped (folded with later ones) writes, and reads may be
>>>>>>>>> serviced from the cache instead of going to actual MMIO. Therefore ...
>>>>>>>>>
>>>>>>>>>> --- a/xen/arch/riscv/Kconfig
>>>>>>>>>> +++ b/xen/arch/riscv/Kconfig
>>>>>>>>>> @@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
>>>>>>>>>>       	string
>>>>>>>>>>       	default "arch/riscv/configs/tiny64_defconfig"
>>>>>>>>>>       
>>>>>>>>>> +config HAS_SVPBMT
>>>>>>>>>> +	bool
>>>>>>>>>> +	help
>>>>>>>>>> +	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
>>>>>>>>>> +	  page-based memory types).
>>>>>>>>>> +
>>>>>>>>>> +	  The memory type for a page contains a combination of attributes
>>>>>>>>>> +	  that indicate the cacheability, idempotency, and ordering
>>>>>>>>>> +	  properties for access to that page.
>>>>>>>>>> +
>>>>>>>>>> +	  The Svpbmt extension is only available on 64-bit cpus.
>>>>>>>>> ... I kind of expect this extension (or anything else that there might be) will need
>>>>>>>>> making use of.
>>>>>>>> In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes)
>>>>>>>> is used to control which memory regions are cacheable, non-cacheable, readable, writable,
>>>>>>>> etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes
>>>>>>>> cores, or it can be fixed at design time, as in SiFive cores.
>>>>>>> How would things work if there was a need to map a RAM page uncacheable (via
>>>>>>> ioremap() or otherwise)?
>>>>>> My understanding is that Svpbmt is only needed when someone wants to change the memory
>>>>>> attribute of a page set by PMA.
>>>>>>
>>>>>> The question is if non-cacheable RAM page is really needed if we have a coherency?
>>>>> Aiui coherency here is among CPUs.
>>>> ```
>>>> For implementations that conform to the RISC-V Unix Platform Specification,
>>>> I/O devices and DMA operations are required to access memory coherently and
>>>> via strongly ordered I/O channels. Therefore, accesses to regular main memory
>>>> regions that are concurrently accessed by external devices can also use the
>>>> standard synchronization mechanisms. Implementations that do not conform
>>>> to the Unix Platform Specification and/or in which devices do not access
>>>> memory coherently will need to use mechanisms
>>>> (which are currently platform-specific or device-specific) to enforce
>>>> coherency.
>>>> ```
>>>> Based on this from the spec, coherency here is not only among CPUs.
>>>>
>>>>
>>>>> Properties of devices in the system are
>>>>> largely unknown?
>>>> Yes, but still not sure what kind of property requires ioremap() which won't work
>>>> without Svpmbt. Could you please tell me an example?
>>> Well, above you said they all need to access memory coherently. That's the
>>> "property" I was referring to.
>> Do you mean that device could have a property which tell that it would like to have non-cachable
>> region used for that? I haven't seen such property in device tree files.
>>
>> Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the
>> one option?
> On x86 we have the case that IOMMUs may access memory non-coherently. This is
> particular means that IOMMU page table updates (which necessarily live in RAM)
> need to be done quite carefully. As it's all our code, we deal with the
> situation by issuing cache flushes, avoiding the need for UC mappings.
>
> Graphics engines may have similar constraints, aiui. With the driver code not
> being part of Xen, we wouldn't be able to use a similar "simplification" there.
> UC mappings would be pretty much unavoidable.

For this case, it would be better to have Svpmbt.

I would like to noted that Svpmbt isn't supported by RV32 architectures. For such cases, it will be still
needed to play with PMA.
I found today a patch in Linux kernel which does something similar to what I wrote in one of my previous
replies:
   [0]https://lore.kernel.org/all/20241102000843.1301099-1-samuel.holland@sifive.com/
In the cover letter [0] it is mentioned the following:
   On some RISC-V platforms, including StarFive JH7100 and ESWIN EIC7700,
   RAM is mapped to multiple physical address ranges, with each alias
   having a different set of statically-determined Physical Memory
   Attributes (PMAs). Software selects the PMAs for a page by choosing a
   PFN from the corresponding physical address range. On these platforms,
   this is the only way to allocate noncached memory for use with
   noncoherent DMA.

So firmware should configure PMAs so some part of RAM is noncached and then kernel get this info based
on the binding:
   https://patchew.org/linux/20241102000843.1301099-1-samuel.holland@sifive.com/20241102000843.1301099-2-samuel.holland@sifive.com/

Considering that this feature isn't available even in Linux kernel, we can start with assumption that all
our SoCs will support Svpmbt.

We don't really care about StarFive JH7100 as it doesn't support H extension, but we potentially should care
about ESWIN EIC7700, which support H extension and doesn't support Svpmbt extension according to a datasheet
publicly available:
   Each EIC7700X core is configured to support the RV64I base ISA, as well as the Multiply (M), Atomic(A),
   Single-Precision Floating Point (F), Double-Precision Floating Point (D), Compressed (C), CSR
   Instructions (Zicsr), Instruction-Fetch Fence (Zifencei), Address Calculation (Zba), Basic Bit
   Manipulation (Zbb), and Count Overflow and Mode-Based Filtering (Sscofpmf) RISC‑V extensions. This
   is captured by the RISC‑V extension string: RV64GC_Zba_Zbb_Sscofpmf.

>
>> I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency
>> for everything).
> Not sure what exactly you're asking here (if anything). An answer would very
> likely depend on the specific kind of barrier you're thinking about. The
> question would be what, if any, effect a barrier would have on the cache(s).

I confused barrier with cache flushes (when I wrote that I thought about the case of DMA that we don't really
should have requirement of non-cachable memory for DMA as it is enough to have memory fence between use of DMA
memory and MMIO that triggers the dma), basically I meant what you wrote above about x86's IOMMUs.

~ Oleksii

>
>> Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice
>> and the support of Svpmbt is required.
> Quite possible.
>
> Jan

[-- Attachment #2: Type: text/html, Size: 12133 bytes --]

[PATCH v1 04/14] xen/riscv: introduce init_IRQ()

[Oleksii Kurochko]

Implement init_IRQ() to initalize various IRQs.

Currently, this function initializes the irq_desc[] array,
which stores IRQ descriptors containing various information
about each IRQ, such as the type of hardware handling, whether
the IRQ is disabled, etc.
The initialization is basic at this point and includes setting
IRQ_TYPE_INVALID as the IRQ type, assigning the IRQ number ( which
is just a consequent index of irq_desc[] array ) to
desc->irq, and setting desc->action to NULL.

Additionally, the function init_irq_data() is introduced to
initialize the IRQ descriptors for all IRQs in the system.

Also, define IRQ_TYPE_* which are the same as the existing device
tree definitions for convenience.

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/Makefile          |  1 +
 xen/arch/riscv/include/asm/irq.h | 24 +++++++++++++++++
 xen/arch/riscv/irq.c             | 44 ++++++++++++++++++++++++++++++++
 xen/arch/riscv/setup.c           |  3 +++
 xen/arch/riscv/stubs.c           |  5 ----
 5 files changed, 72 insertions(+), 5 deletions(-)
 create mode 100644 xen/arch/riscv/irq.c

diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile
index f551bf32a2..457e8e88a4 100644
--- a/xen/arch/riscv/Makefile
+++ b/xen/arch/riscv/Makefile
@@ -3,6 +3,7 @@ obj-y += cpufeature.o
 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
 obj-y += entry.o
 obj-y += intc.o
+obj-y += irq.o
 obj-y += mm.o
 obj-y += pt.o
 obj-$(CONFIG_RISCV_64) += riscv64/
diff --git a/xen/arch/riscv/include/asm/irq.h b/xen/arch/riscv/include/asm/irq.h
index 2a48da2651..8f936b7d01 100644
--- a/xen/arch/riscv/include/asm/irq.h
+++ b/xen/arch/riscv/include/asm/irq.h
@@ -3,6 +3,28 @@
 #define ASM__RISCV__IRQ_H
 
 #include <xen/bug.h>
+#include <xen/device_tree.h>
+
+#define NR_IRQS 1024
+
+/*
+ * TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into
+ * #ifdef CONFIG_HAS_DEVICE_TREE?
+ */
+/*
+ * These defines correspond to the Xen internal representation of the
+ * IRQ types. We choose to make them the same as the existing device
+ * tree definitions for convenience.
+ */
+#define IRQ_TYPE_NONE           DT_IRQ_TYPE_NONE
+#define IRQ_TYPE_EDGE_RISING    DT_IRQ_TYPE_EDGE_RISING
+#define IRQ_TYPE_EDGE_FALLING   DT_IRQ_TYPE_EDGE_FALLING
+#define IRQ_TYPE_EDGE_BOTH      DT_IRQ_TYPE_EDGE_BOTH
+#define IRQ_TYPE_LEVEL_HIGH     DT_IRQ_TYPE_LEVEL_HIGH
+#define IRQ_TYPE_LEVEL_LOW      DT_IRQ_TYPE_LEVEL_LOW
+#define IRQ_TYPE_LEVEL_MASK     DT_IRQ_TYPE_LEVEL_MASK
+#define IRQ_TYPE_SENSE_MASK     DT_IRQ_TYPE_SENSE_MASK
+#define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
 /* TODO */
 #define nr_irqs 0U
@@ -25,6 +47,8 @@ static inline void arch_move_irqs(struct vcpu *v)
     BUG_ON("unimplemented");
 }
 
+void init_IRQ(void);
+
 #endif /* ASM__RISCV__IRQ_H */
 
 /*
diff --git a/xen/arch/riscv/irq.c b/xen/arch/riscv/irq.c
new file mode 100644
index 0000000000..99b8f2095e
--- /dev/null
+++ b/xen/arch/riscv/irq.c
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/*
+ * RISC-V Trap handlers
+ *
+ * Copyright (c) 2024 Vates
+ */
+
+#include <xen/bug.h>
+#include <xen/init.h>
+#include <xen/irq.h>
+
+static irq_desc_t irq_desc[NR_IRQS];
+
+int arch_init_one_irq_desc(struct irq_desc *desc)
+{
+    desc->arch.type = IRQ_TYPE_INVALID;
+    return 0;
+}
+
+static int __init init_irq_data(void)
+{
+    int irq;
+
+    for ( irq = 0; irq < NR_IRQS; irq++ )
+    {
+        struct irq_desc *desc = irq_to_desc(irq);
+        int rc = init_one_irq_desc(desc);
+
+        if ( rc )
+            return rc;
+
+        desc->irq = irq;
+        desc->action  = NULL;
+    }
+
+    return 0;
+}
+
+void __init init_IRQ(void)
+{
+    if ( init_irq_data() < 0 )
+        panic("initialization of IRQ data failed\n");
+}
diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c
index 7f68f3f5b7..a3189697da 100644
--- a/xen/arch/riscv/setup.c
+++ b/xen/arch/riscv/setup.c
@@ -6,6 +6,7 @@
 #include <xen/compile.h>
 #include <xen/device_tree.h>
 #include <xen/init.h>
+#include <xen/irq.h>
 #include <xen/mm.h>
 #include <xen/shutdown.h>
 #include <xen/vmap.h>
@@ -127,6 +128,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
         panic("Booting using ACPI isn't supported\n");
     }
 
+    init_IRQ();
+
     riscv_fill_hwcap();
 
     preinit_xen_time();
diff --git a/xen/arch/riscv/stubs.c b/xen/arch/riscv/stubs.c
index fdcf91054e..e396b67cd3 100644
--- a/xen/arch/riscv/stubs.c
+++ b/xen/arch/riscv/stubs.c
@@ -107,11 +107,6 @@ void irq_ack_none(struct irq_desc *desc)
     BUG_ON("unimplemented");
 }
 
-int arch_init_one_irq_desc(struct irq_desc *desc)
-{
-    BUG_ON("unimplemented");
-}
-
 void smp_send_state_dump(unsigned int cpu)
 {
     BUG_ON("unimplemented");
-- 
2.49.0

Re: [PATCH v1 04/14] xen/riscv: introduce init_IRQ()

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> --- a/xen/arch/riscv/include/asm/irq.h
> +++ b/xen/arch/riscv/include/asm/irq.h
> @@ -3,6 +3,28 @@
>  #define ASM__RISCV__IRQ_H
>  
>  #include <xen/bug.h>
> +#include <xen/device_tree.h>
> +
> +#define NR_IRQS 1024
> +
> +/*
> + * TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into
> + * #ifdef CONFIG_HAS_DEVICE_TREE?
> + */

Wouldn't that be more like asm-generic/dt-irq.h (or irq-dt.h)? The field where
these values are stored is an arch-specific one, after all.

> --- /dev/null
> +++ b/xen/arch/riscv/irq.c
> @@ -0,0 +1,44 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +
> +/*
> + * RISC-V Trap handlers
> + *
> + * Copyright (c) 2024 Vates
> + */
> +
> +#include <xen/bug.h>
> +#include <xen/init.h>
> +#include <xen/irq.h>
> +
> +static irq_desc_t irq_desc[NR_IRQS];
> +
> +int arch_init_one_irq_desc(struct irq_desc *desc)
> +{
> +    desc->arch.type = IRQ_TYPE_INVALID;
> +    return 0;
> +}
> +
> +static int __init init_irq_data(void)
> +{
> +    int irq;
> +
> +    for ( irq = 0; irq < NR_IRQS; irq++ )

For this the variable would better be of unsigned type. I realize though that ...

> +    {
> +        struct irq_desc *desc = irq_to_desc(irq);
> +        int rc = init_one_irq_desc(desc);
> +
> +        if ( rc )
> +            return rc;
> +
> +        desc->irq = irq;

... it's a plain int field that we're storing into here. I don't think I can spot
any place where a negative value would be stored into there.

Jan

Re: [PATCH v1 04/14] xen/riscv: introduce init_IRQ()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 633 bytes --]


On 4/10/25 5:25 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> --- a/xen/arch/riscv/include/asm/irq.h
>> +++ b/xen/arch/riscv/include/asm/irq.h
>> @@ -3,6 +3,28 @@
>>   #define ASM__RISCV__IRQ_H
>>   
>>   #include <xen/bug.h>
>> +#include <xen/device_tree.h>
>> +
>> +#define NR_IRQS 1024
>> +
>> +/*
>> + * TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into
>> + * #ifdef CONFIG_HAS_DEVICE_TREE?
>> + */
> Wouldn't that be more like asm-generic/dt-irq.h (or irq-dt.h)? The field where
> these values are stored is an arch-specific one, after all.

It would be much better. Thanks!

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 1125 bytes --]

[PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

[Oleksii Kurochko]

platform_get_irq() recieves information about device's irq ( type
and irq number ) from device tree node and using this information
update irq descriptor in irq_desc[] array.

Introduce dt_irq_xlate and initialize with aplic_irq_xlate() as
it is used by dt_device_get_irq() which is called by
platform_get_irq().

Co-developed-by: Romain Caritey <Romain.Caritey@microchip.com>
Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/aplic.c           | 19 +++++++++++++++
 xen/arch/riscv/include/asm/irq.h |  3 +++
 xen/arch/riscv/irq.c             | 41 ++++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+)

diff --git a/xen/arch/riscv/aplic.c b/xen/arch/riscv/aplic.c
index caba8f8993..6dc040af6f 100644
--- a/xen/arch/riscv/aplic.c
+++ b/xen/arch/riscv/aplic.c
@@ -11,6 +11,7 @@
 
 #include <xen/errno.h>
 #include <xen/init.h>
+#include <xen/irq.h>
 #include <xen/sections.h>
 #include <xen/types.h>
 
@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
     .hw_version = INTC_APLIC,
 };
 
+static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
+                           unsigned int *out_hwirq,
+                           unsigned int *out_type)
+{
+    if ( intsize < 2 )
+        return -EINVAL;
+
+    /* Mapping 1:1 */
+    *out_hwirq = intspec[0];
+
+    if ( out_type )
+        *out_type = intspec[1] & IRQ_TYPE_SENSE_MASK;
+
+    return 0;
+}
+
 static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 {
     if ( aplic_info.node )
@@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     aplic_info.node = node;
 
+    dt_irq_xlate = aplic_irq_xlate;
+
     return 0;
 }
 
diff --git a/xen/arch/riscv/include/asm/irq.h b/xen/arch/riscv/include/asm/irq.h
index 8f936b7d01..ff1c95e0be 100644
--- a/xen/arch/riscv/include/asm/irq.h
+++ b/xen/arch/riscv/include/asm/irq.h
@@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v)
     BUG_ON("unimplemented");
 }
 
+struct dt_device_node;
+int platform_get_irq(const struct dt_device_node *device, int index);
+
 void init_IRQ(void);
 
 #endif /* ASM__RISCV__IRQ_H */
diff --git a/xen/arch/riscv/irq.c b/xen/arch/riscv/irq.c
index 99b8f2095e..c332e000c4 100644
--- a/xen/arch/riscv/irq.c
+++ b/xen/arch/riscv/irq.c
@@ -7,11 +7,52 @@
  */
 
 #include <xen/bug.h>
+#include <xen/device_tree.h>
+#include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/irq.h>
 
 static irq_desc_t irq_desc[NR_IRQS];
 
+static bool irq_validate_new_type(unsigned int curr, unsigned int new)
+{
+    return (curr == IRQ_TYPE_INVALID || curr == new );
+}
+
+static int irq_set_type(unsigned int irq, unsigned int type)
+{
+    unsigned long flags;
+    struct irq_desc *desc = irq_to_desc(irq);
+    int ret = -EBUSY;
+
+    spin_lock_irqsave(&desc->lock, flags);
+
+    if ( !irq_validate_new_type(desc->arch.type, type) )
+        goto err;
+
+    desc->arch.type = type;
+
+    ret = 0;
+
+err:
+    spin_unlock_irqrestore(&desc->lock, flags);
+
+    return ret;
+}
+
+int platform_get_irq(const struct dt_device_node *device, int index)
+{
+    struct dt_irq dt_irq;
+
+    if ( dt_device_get_irq(device, index, &dt_irq) )
+        return -1;
+
+    if ( irq_set_type(dt_irq.irq, dt_irq.type) )
+        return -1;
+
+    return dt_irq.irq;
+}
+
 int arch_init_one_irq_desc(struct irq_desc *desc)
 {
     desc->arch.type = IRQ_TYPE_INVALID;
-- 
2.49.0

Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> @@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
>      .hw_version = INTC_APLIC,
>  };
>  
> +static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,

As you start adding functions calling indirectly, please consider adding cf_check
right away, even if right now this has no effect on RISC-V. That'll save you from
going through the entire RISC-V subtree later on to find them all.

> +                           unsigned int *out_hwirq,
> +                           unsigned int *out_type)
> +{
> +    if ( intsize < 2 )
> +        return -EINVAL;
> +
> +    /* Mapping 1:1 */
> +    *out_hwirq = intspec[0];
> +
> +    if ( out_type )
> +        *out_type = intspec[1] & IRQ_TYPE_SENSE_MASK;
> +
> +    return 0;
> +}
> +
>  static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
>  {
>      if ( aplic_info.node )
> @@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
>  
>      aplic_info.node = node;
>  
> +    dt_irq_xlate = aplic_irq_xlate;
> +
>      return 0;
>  }
>  
> --- a/xen/arch/riscv/include/asm/irq.h
> +++ b/xen/arch/riscv/include/asm/irq.h
> @@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v)
>      BUG_ON("unimplemented");
>  }
>  
> +struct dt_device_node;
> +int platform_get_irq(const struct dt_device_node *device, int index);

And I assume callers of this will appear later in the series.

> --- a/xen/arch/riscv/irq.c
> +++ b/xen/arch/riscv/irq.c
> @@ -7,11 +7,52 @@
>   */
>  
>  #include <xen/bug.h>
> +#include <xen/device_tree.h>
> +#include <xen/errno.h>
>  #include <xen/init.h>
>  #include <xen/irq.h>
>  
>  static irq_desc_t irq_desc[NR_IRQS];
>  
> +static bool irq_validate_new_type(unsigned int curr, unsigned int new)
> +{
> +    return (curr == IRQ_TYPE_INVALID || curr == new );
> +}
> +
> +static int irq_set_type(unsigned int irq, unsigned int type)
> +{
> +    unsigned long flags;
> +    struct irq_desc *desc = irq_to_desc(irq);
> +    int ret = -EBUSY;
> +
> +    spin_lock_irqsave(&desc->lock, flags);
> +
> +    if ( !irq_validate_new_type(desc->arch.type, type) )
> +        goto err;
> +
> +    desc->arch.type = type;
> +
> +    ret = 0;
> +
> +err:

Labels indented by at least one blank please.

> +    spin_unlock_irqrestore(&desc->lock, flags);
> +
> +    return ret;
> +}
> +
> +int platform_get_irq(const struct dt_device_node *device, int index)
> +{
> +    struct dt_irq dt_irq;
> +
> +    if ( dt_device_get_irq(device, index, &dt_irq) )
> +        return -1;
> +
> +    if ( irq_set_type(dt_irq.irq, dt_irq.type) )
> +        return -1;

Can you please return proper -E... values, perhaps ones coming back from the
functions called?

Jan

Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 3322 bytes --]


On 4/10/25 5:35 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> @@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
>>       .hw_version = INTC_APLIC,
>>   };
>>   
>> +static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
> As you start adding functions calling indirectly, please consider adding cf_check
> right away, even if right now this has no effect on RISC-V. That'll save you from
> going through the entire RISC-V subtree later on to find them all.

Sure. I thought that it is a feature for x86 as I haven't seen such attribute for
Arm and RISC-V in GCC manuals.

>
>> +                           unsigned int *out_hwirq,
>> +                           unsigned int *out_type)
>> +{
>> +    if ( intsize < 2 )
>> +        return -EINVAL;
>> +
>> +    /* Mapping 1:1 */
>> +    *out_hwirq = intspec[0];
>> +
>> +    if ( out_type )
>> +        *out_type = intspec[1] & IRQ_TYPE_SENSE_MASK;
>> +
>> +    return 0;
>> +}
>> +
>>   static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
>>   {
>>       if ( aplic_info.node )
>> @@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
>>   
>>       aplic_info.node = node;
>>   
>> +    dt_irq_xlate = aplic_irq_xlate;
>> +
>>       return 0;
>>   }
>>   
>> --- a/xen/arch/riscv/include/asm/irq.h
>> +++ b/xen/arch/riscv/include/asm/irq.h
>> @@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v)
>>       BUG_ON("unimplemented");
>>   }
>>   
>> +struct dt_device_node;
>> +int platform_get_irq(const struct dt_device_node *device, int index);
> And I assume callers of this will appear later in the series.

Yes, it will be called ns16550_uart_dt_init() when CONFIG_NS16550 will be enabled for RISC-V.

>
>> --- a/xen/arch/riscv/irq.c
>> +++ b/xen/arch/riscv/irq.c
>> @@ -7,11 +7,52 @@
>>    */
>>   
>>   #include <xen/bug.h>
>> +#include <xen/device_tree.h>
>> +#include <xen/errno.h>
>>   #include <xen/init.h>
>>   #include <xen/irq.h>
>>   
>>   static irq_desc_t irq_desc[NR_IRQS];
>>   
>> +static bool irq_validate_new_type(unsigned int curr, unsigned int new)
>> +{
>> +    return (curr == IRQ_TYPE_INVALID || curr == new );
>> +}
>> +
>> +static int irq_set_type(unsigned int irq, unsigned int type)
>> +{
>> +    unsigned long flags;
>> +    struct irq_desc *desc = irq_to_desc(irq);
>> +    int ret = -EBUSY;
>> +
>> +    spin_lock_irqsave(&desc->lock, flags);
>> +
>> +    if ( !irq_validate_new_type(desc->arch.type, type) )
>> +        goto err;
>> +
>> +    desc->arch.type = type;
>> +
>> +    ret = 0;
>> +
>> +err:
> Labels indented by at least one blank please.
>
>> +    spin_unlock_irqrestore(&desc->lock, flags);
>> +
>> +    return ret;
>> +}
>> +
>> +int platform_get_irq(const struct dt_device_node *device, int index)
>> +{
>> +    struct dt_irq dt_irq;
>> +
>> +    if ( dt_device_get_irq(device, index, &dt_irq) )
>> +        return -1;
>> +
>> +    if ( irq_set_type(dt_irq.irq, dt_irq.type) )
>> +        return -1;
> Can you please return proper -E... values, perhaps ones coming back from the
> functions called?

Sure, I will use -EINVAL. (or ,perhaps, it  will be better to introduce ret and
return what dt_device_get_irq()/irq_set_type() returns in the case of failure.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 4420 bytes --]

Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

[Jan Beulich]

On 15.04.2025 13:11, Oleksii Kurochko wrote:
> On 4/10/25 5:35 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> @@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
>>>       .hw_version = INTC_APLIC,
>>>   };
>>>   
>>> +static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
>> As you start adding functions calling indirectly, please consider adding cf_check
>> right away, even if right now this has no effect on RISC-V. That'll save you from
>> going through the entire RISC-V subtree later on to find them all.
> 
> Sure. I thought that it is a feature for x86 as I haven't seen such attribute for
> Arm and RISC-V in GCC manuals.

And that looks to be correct. I was under the (admittedly vague) impression
Arm64 had something equivalent in hardware, which then merely needs enabling
in the compiler. Not sure about RISC-V, but seeing the endless flow of
patches enabling new extensions in binutils, it would perhaps even be
surprising if nothing along these lines was already in the works somewhere.

Jan

Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 1364 bytes --]


On 4/15/25 1:23 PM, Jan Beulich wrote:
> On 15.04.2025 13:11, Oleksii Kurochko wrote:
>> On 4/10/25 5:35 PM, Jan Beulich wrote:
>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>> @@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
>>>>        .hw_version = INTC_APLIC,
>>>>    };
>>>>    
>>>> +static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
>>> As you start adding functions calling indirectly, please consider adding cf_check
>>> right away, even if right now this has no effect on RISC-V. That'll save you from
>>> going through the entire RISC-V subtree later on to find them all.
>> Sure. I thought that it is a feature for x86 as I haven't seen such attribute for
>> Arm and RISC-V in GCC manuals.
> And that looks to be correct. I was under the (admittedly vague) impression
> Arm64 had something equivalent in hardware, which then merely needs enabling
> in the compiler. Not sure about RISC-V, but seeing the endless flow of
> patches enabling new extensions in binutils, it would perhaps even be
> surprising if nothing along these lines was already in the works somewhere.

You are right, something is already in the work:
-https://github.com/riscv/riscv-cfi
-https://lore.kernel.org/lkml/20230213045351.3945824-1-debug@rivosinc.com/ (interesting that
   they are enabling it for U-mode)


~ Oleksii

[-- Attachment #2: Type: text/html, Size: 2367 bytes --]

[PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

[Oleksii Kurochko]

Implements riscv_of_processor_hartid() to get the hart ID of the given
device tree node and do some checks if CPU is available and given device
tree node has proper riscv,isa property.

As a helper function of_get_cpu_hwid() is introduced to deal specifically
with reg propery of a CPU device node.

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/include/asm/smp.h |  3 ++
 xen/arch/riscv/smpboot.c         | 68 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)

diff --git a/xen/arch/riscv/include/asm/smp.h b/xen/arch/riscv/include/asm/smp.h
index 188c033718..9b68f1e27a 100644
--- a/xen/arch/riscv/include/asm/smp.h
+++ b/xen/arch/riscv/include/asm/smp.h
@@ -26,6 +26,9 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,
 
 void setup_tp(unsigned int cpuid);
 
+struct dt_device_node;
+int riscv_of_processor_hartid(struct dt_device_node *node, unsigned long *hart);
+
 void smp_clear_cpu_maps(void);
 
 #endif
diff --git a/xen/arch/riscv/smpboot.c b/xen/arch/riscv/smpboot.c
index 0f4dcc28e1..3193639f00 100644
--- a/xen/arch/riscv/smpboot.c
+++ b/xen/arch/riscv/smpboot.c
@@ -1,5 +1,8 @@
 #include <xen/cpumask.h>
+#include <xen/device_tree.h>
+#include <xen/errno.h>
 #include <xen/init.h>
+#include <xen/types.h>
 
 cpumask_t cpu_online_map;
 cpumask_t cpu_present_map;
@@ -13,3 +16,68 @@ void __init smp_clear_cpu_maps(void)
     cpumask_set_cpu(0, &cpu_online_map);
     cpumask_copy(&cpu_present_map, &cpu_possible_map);
 }
+
+/**
+ * of_get_cpu_hwid - Get the hardware ID from a CPU device node
+ *
+ * @cpun: CPU number(logical index) for which device node is required
+ * @thread: The local thread number to get the hardware ID for.
+ *
+ * Return: The hardware ID for the CPU node or ~0ULL if not found.
+ */
+static uint64_t of_get_cpu_hwid(struct dt_device_node *cpun, unsigned int thread)
+{
+    const __be32 *cell;
+    int ac;
+    uint32_t len;
+
+    ac = dt_n_addr_cells(cpun);
+    cell = dt_get_property(cpun, "reg", &len);
+    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
+        return ~0ULL;
+
+    cell += ac * thread;
+    return dt_read_number(cell, ac);
+}
+
+/*
+ * Returns the hart ID of the given device tree node, or -ENODEV if the node
+ * isn't an enabled and valid RISC-V hart node.
+ */
+int riscv_of_processor_hartid(struct dt_device_node *node, unsigned long *hart)
+{
+    const char *isa;
+
+    if ( !dt_device_is_compatible(node, "riscv") )
+    {
+        printk("Found incompatible CPU\n");
+        return -ENODEV;
+    }
+
+    *hart = (unsigned long) of_get_cpu_hwid(node, 0);
+    if ( *hart == ~0UL )
+    {
+        printk("Found CPU without hart ID\n");
+        return -ENODEV;
+    }
+
+    if ( !dt_device_is_available(node))
+    {
+        printk("CPU with hartid=%lu is not available\n", *hart);
+        return -ENODEV;
+    }
+
+    if ( dt_property_read_string(node, "riscv,isa", &isa) )
+    {
+        printk("CPU with hartid=%lu has no \"riscv,isa\" property\n", *hart);
+        return -ENODEV;
+    }
+
+    if ( isa[0] != 'r' || isa[1] != 'v' )
+    {
+        printk("CPU with hartid=%lu has an invalid ISA of \"%s\"\n", *hart, isa);
+        return -ENODEV;
+    }
+
+    return 0;
+}
-- 
2.49.0

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> @@ -13,3 +16,68 @@ void __init smp_clear_cpu_maps(void)
>      cpumask_set_cpu(0, &cpu_online_map);
>      cpumask_copy(&cpu_present_map, &cpu_possible_map);
>  }
> +
> +/**
> + * of_get_cpu_hwid - Get the hardware ID from a CPU device node
> + *
> + * @cpun: CPU number(logical index) for which device node is required
> + * @thread: The local thread number to get the hardware ID for.
> + *
> + * Return: The hardware ID for the CPU node or ~0ULL if not found.
> + */
> +static uint64_t of_get_cpu_hwid(struct dt_device_node *cpun, unsigned int thread)

What does the "of" prefix stand for here? Looking at the function body I'm
really at a loss. (I was first guessing something like OpenFirmware, but
there's nothing here that would support that.)

As you're only fetching data - can cpun be pointer-to-const?

> +{
> +    const __be32 *cell;
> +    int ac;
> +    uint32_t len;
> +
> +    ac = dt_n_addr_cells(cpun);
> +    cell = dt_get_property(cpun, "reg", &len);
> +    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
> +        return ~0ULL;

I'm sorry for my lack of DT knowledge, but what's "thread" representing here?
You only pass in 0 below, so it's unclear whether it's what one might expect
(the thread number on a multi-threaded core).

> +    cell += ac * thread;
> +    return dt_read_number(cell, ac);

Nit (you know what)

> +/*
> + * Returns the hart ID of the given device tree node, or -ENODEV if the node
> + * isn't an enabled and valid RISC-V hart node.
> + */
> +int riscv_of_processor_hartid(struct dt_device_node *node, unsigned long *hart)

Similar question as above: What's "of" and what significance does the "riscv"
prefix have in RISC-V code?

Const-ness question again for "node".

> +{
> +    const char *isa;
> +
> +    if ( !dt_device_is_compatible(node, "riscv") )
> +    {
> +        printk("Found incompatible CPU\n");
> +        return -ENODEV;
> +    }
> +
> +    *hart = (unsigned long) of_get_cpu_hwid(node, 0);
> +    if ( *hart == ~0UL )

While for RV64 this won't matter, the difference in types (uint64_t returned,
unsigned long used) is still puzzling me. What's the deal?

Jan

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 3980 bytes --]


On 4/10/25 5:53 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> @@ -13,3 +16,68 @@ void __init smp_clear_cpu_maps(void)
>>       cpumask_set_cpu(0, &cpu_online_map);
>>       cpumask_copy(&cpu_present_map, &cpu_possible_map);
>>   }
>> +
>> +/**
>> + * of_get_cpu_hwid - Get the hardware ID from a CPU device node
>> + *
>> + * @cpun: CPU number(logical index) for which device node is required
>> + * @thread: The local thread number to get the hardware ID for.
>> + *
>> + * Return: The hardware ID for the CPU node or ~0ULL if not found.
>> + */
>> +static uint64_t of_get_cpu_hwid(struct dt_device_node *cpun, unsigned int thread)
> What does the "of" prefix stand for here? Looking at the function body I'm
> really at a loss. (I was first guessing something like OpenFirmware, but
> there's nothing here that would support that.)

I copy this function from Linux kernel. But you are right, "of" means OpenFirmware or
Open Firmware Device Tree and is used for the functions which work with device
tree.
I'll rename to dt_get_cpu_hwid() to follow the naming of device tree's functions
name in Xen.

>
> As you're only fetching data - can cpun be pointer-to-const?

Sure, it can  be. I'll update that.

>
>> +{
>> +    const __be32 *cell;
>> +    int ac;
>> +    uint32_t len;
>> +
>> +    ac = dt_n_addr_cells(cpun);
>> +    cell = dt_get_property(cpun, "reg", &len);
>> +    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
>> +        return ~0ULL;
> I'm sorry for my lack of DT knowledge, but what's "thread" representing here?
> You only pass in 0 below, so it's unclear whether it's what one might expect
> (the thread number on a multi-threaded core).

Based on the DT specification alone, the|`reg`| value could refer to either a CPU or a thread ID:
```
The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
the CPU/threads represented by the CPU node. If a CPU supports more than one thread
(i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
per thread.
```

My understanding is that the term/thread/ was used in the Linux kernel to cover both
cases.
When SMT isn't supported, the CPU can be considered to have a single thread.
For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).

Interestingly, the Linux kernel always uses|thread = 0|.

We could potentially drop this ambiguity and introduce an|ASSERT()| to check that
the|`reg`| property contains only one entry, representing the HART (CPU) ID:
```
   Software can determine the number of threads by dividing the size of reg by the parent
   node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
   is required.
```

Does that approach make sense, or should we stick with the current implementation?

>
>> +    cell += ac * thread;
>> +    return dt_read_number(cell, ac);
> Nit (you know what)
>
>> +/*
>> + * Returns the hart ID of the given device tree node, or -ENODEV if the node
>> + * isn't an enabled and valid RISC-V hart node.
>> + */
>> +int riscv_of_processor_hartid(struct dt_device_node *node, unsigned long *hart)
> Similar question as above: What's "of" and what significance does the "riscv"
> prefix have in RISC-V code?

I will drop usage of 'of' in Xen and change it to 'dt'.

>
> Const-ness question again for "node".
>
>> +{
>> +    const char *isa;
>> +
>> +    if ( !dt_device_is_compatible(node, "riscv") )
>> +    {
>> +        printk("Found incompatible CPU\n");
>> +        return -ENODEV;
>> +    }
>> +
>> +    *hart = (unsigned long) of_get_cpu_hwid(node, 0);
>> +    if ( *hart == ~0UL )
> While for RV64 this won't matter, the difference in types (uint64_t returned,
> unsigned long used) is still puzzling me. What's the deal?

No specific reason, just overlooked that moment. I think we could use just
drop this cast.
The reason for uint64_t as a return type is that dt_read_number() returns
u64.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 6222 bytes --]

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

[Jan Beulich]

On 15.04.2025 15:39, Oleksii Kurochko wrote:
> On 4/10/25 5:53 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> +{
>>> +    const __be32 *cell;
>>> +    int ac;
>>> +    uint32_t len;
>>> +
>>> +    ac = dt_n_addr_cells(cpun);
>>> +    cell = dt_get_property(cpun, "reg", &len);
>>> +    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
>>> +        return ~0ULL;
>> I'm sorry for my lack of DT knowledge, but what's "thread" representing here?
>> You only pass in 0 below, so it's unclear whether it's what one might expect
>> (the thread number on a multi-threaded core).
> 
> Based on the DT specification alone, the|`reg`| value could refer to either a CPU or a thread ID:
> ```
> The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
> the CPU/threads represented by the CPU node. If a CPU supports more than one thread
> (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
> per thread.
> ```
> 
> My understanding is that the term/thread/ was used in the Linux kernel to cover both
> cases.
> When SMT isn't supported, the CPU can be considered to have a single thread.
> For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).
> 
> Interestingly, the Linux kernel always uses|thread = 0|.
> 
> We could potentially drop this ambiguity and introduce an|ASSERT()| to check that
> the|`reg`| property contains only one entry, representing the HART (CPU) ID:
> ```
>    Software can determine the number of threads by dividing the size of reg by the parent
>    node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
>    is required.
> ```
> 
> Does that approach make sense, or should we stick with the current implementation?

If extra enabling is required to make multi-thread CPUs work, then panic()ing
(not so much ASSERT()ing) may make sense, for the time being. Better would be
if we could use all threads in a system right away.

Jan

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 2709 bytes --]


On 4/15/25 3:45 PM, Jan Beulich wrote:
> On 15.04.2025 15:39, Oleksii Kurochko wrote:
>> On 4/10/25 5:53 PM, Jan Beulich wrote:
>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>> +{
>>>> +    const __be32 *cell;
>>>> +    int ac;
>>>> +    uint32_t len;
>>>> +
>>>> +    ac = dt_n_addr_cells(cpun);
>>>> +    cell = dt_get_property(cpun, "reg", &len);
>>>> +    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
>>>> +        return ~0ULL;
>>> I'm sorry for my lack of DT knowledge, but what's "thread" representing here?
>>> You only pass in 0 below, so it's unclear whether it's what one might expect
>>> (the thread number on a multi-threaded core).
>> Based on the DT specification alone, the|`reg`| value could refer to either a CPU or a thread ID:
>> ```
>> The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
>> the CPU/threads represented by the CPU node. If a CPU supports more than one thread
>> (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
>> per thread.
>> ```
>>
>> My understanding is that the term/thread/ was used in the Linux kernel to cover both
>> cases.
>> When SMT isn't supported, the CPU can be considered to have a single thread.
>> For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).
>>
>> Interestingly, the Linux kernel always uses|thread = 0|.
>>
>> We could potentially drop this ambiguity and introduce an|ASSERT()| to check that
>> the|`reg`| property contains only one entry, representing the HART (CPU) ID:
>> ```
>>     Software can determine the number of threads by dividing the size of reg by the parent
>>     node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
>>     is required.
>> ```
>>
>> Does that approach make sense, or should we stick with the current implementation?
> If extra enabling is required to make multi-thread CPUs work, then panic()ing
> (not so much ASSERT()ing) may make sense, for the time being. Better would be
> if we could use all threads in a system right away.

Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id
by passing `thread` argument (`thread` -> the local thread number to get the hardware ID for).
So by calling:
  dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0
  dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
  ...

In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).

If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().

I will add a check in the caller.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 3439 bytes --]

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

[Jan Beulich]

On 25.04.2025 19:07, Oleksii Kurochko wrote:
> 
> On 4/15/25 3:45 PM, Jan Beulich wrote:
>> On 15.04.2025 15:39, Oleksii Kurochko wrote:
>>> On 4/10/25 5:53 PM, Jan Beulich wrote:
>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>> +{
>>>>> +    const __be32 *cell;
>>>>> +    int ac;
>>>>> +    uint32_t len;
>>>>> +
>>>>> +    ac = dt_n_addr_cells(cpun);
>>>>> +    cell = dt_get_property(cpun, "reg", &len);
>>>>> +    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
>>>>> +        return ~0ULL;
>>>> I'm sorry for my lack of DT knowledge, but what's "thread" representing here?
>>>> You only pass in 0 below, so it's unclear whether it's what one might expect
>>>> (the thread number on a multi-threaded core).
>>> Based on the DT specification alone, the|`reg`| value could refer to either a CPU or a thread ID:
>>> ```
>>> The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
>>> the CPU/threads represented by the CPU node. If a CPU supports more than one thread
>>> (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
>>> per thread.
>>> ```
>>>
>>> My understanding is that the term/thread/ was used in the Linux kernel to cover both
>>> cases.
>>> When SMT isn't supported, the CPU can be considered to have a single thread.
>>> For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).

Note the terminology ("CPU") you used here.

>>> Interestingly, the Linux kernel always uses|thread = 0|.
>>>
>>> We could potentially drop this ambiguity and introduce an|ASSERT()| to check that
>>> the|`reg`| property contains only one entry, representing the HART (CPU) ID:
>>> ```
>>>     Software can determine the number of threads by dividing the size of reg by the parent
>>>     node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
>>>     is required.
>>> ```
>>>
>>> Does that approach make sense, or should we stick with the current implementation?
>> If extra enabling is required to make multi-thread CPUs work, then panic()ing
>> (not so much ASSERT()ing) may make sense, for the time being. Better would be
>> if we could use all threads in a system right away.
> 
> Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id
> by passing `thread` argument (`thread` -> the local thread number to get the hardware ID for).
> So by calling:
>   dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0
>   dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
>   ...
> 
> In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).
> 
> If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().

I assume you meant SMT in both places you wrote SMP? But my main point here is:
If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you
need to be quite careful with what you call "CPU". Things need to be entirely
unambiguous, taking into account what internally in (common code) Xen we call a
"CPU". You certainly may call "CPU" what is a collection of threads / harts,
but you then need to clarify this in a prominent comment somewhere, and you
need to be entirely consistent throughout the RISC-V sub-tree.

Jan

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 6053 bytes --]


On 4/28/25 8:31 AM, Jan Beulich wrote:
> On 25.04.2025 19:07, Oleksii Kurochko wrote:
>> On 4/15/25 3:45 PM, Jan Beulich wrote:
>>> On 15.04.2025 15:39, Oleksii Kurochko wrote:
>>>> On 4/10/25 5:53 PM, Jan Beulich wrote:
>>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>>> +{
>>>>>> +    const __be32 *cell;
>>>>>> +    int ac;
>>>>>> +    uint32_t len;
>>>>>> +
>>>>>> +    ac = dt_n_addr_cells(cpun);
>>>>>> +    cell = dt_get_property(cpun, "reg", &len);
>>>>>> +    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
>>>>>> +        return ~0ULL;
>>>>> I'm sorry for my lack of DT knowledge, but what's "thread" representing here?
>>>>> You only pass in 0 below, so it's unclear whether it's what one might expect
>>>>> (the thread number on a multi-threaded core).
>>>> Based on the DT specification alone, the|`reg`| value could refer to either a CPU or a thread ID:
>>>> ```
>>>> The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
>>>> the CPU/threads represented by the CPU node. If a CPU supports more than one thread
>>>> (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
>>>> per thread.
>>>> ```
>>>>
>>>> My understanding is that the term/thread/ was used in the Linux kernel to cover both
>>>> cases.
>>>> When SMT isn't supported, the CPU can be considered to have a single thread.
>>>> For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).
> Note the terminology ("CPU") you used here.
>
>>>> Interestingly, the Linux kernel always uses|thread = 0|.
>>>>
>>>> We could potentially drop this ambiguity and introduce an|ASSERT()| to check that
>>>> the|`reg`| property contains only one entry, representing the HART (CPU) ID:
>>>> ```
>>>>      Software can determine the number of threads by dividing the size of reg by the parent
>>>>      node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
>>>>      is required.
>>>> ```
>>>>
>>>> Does that approach make sense, or should we stick with the current implementation?
>>> If extra enabling is required to make multi-thread CPUs work, then panic()ing
>>> (not so much ASSERT()ing) may make sense, for the time being. Better would be
>>> if we could use all threads in a system right away.
>> Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id
>> by passing `thread` argument (`thread` -> the local thread number to get the hardware ID for).
>> So by calling:
>>    dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0
>>    dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
>>    ...
>>
>> In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).
>>
>> If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().
> I assume you meant SMT in both places you wrote SMP?

Yes, it should be SMT.

>   But my main point here is:
> If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you
> need to be quite careful with what you call "CPU". Things need to be entirely
> unambiguous, taking into account what internally in (common code) Xen we call a
> "CPU". You certainly may call "CPU" what is a collection of threads / harts,
> but you then need to clarify this in a prominent comment somewhere, and you
> need to be entirely consistent throughout the RISC-V sub-tree.

╭────────────────────╮
│        CPU          │  ← 1 physical processor (chip)
│ ┌───────┬─────────┐ │
│ │ Core 0│ Core 1  │ │  ← 2 cores (for example)
│ │ ┌──┬──┐ ┌──┬──┐ │ │
│ │Thr0 Thr1 Thr0 Thr1│ ← 2 threads on each core (SMT)
│ └───────┴─────────┘ │
╰────────────────────╯
I want to double check what Xen call a "CPU". I thought that Xen uses word
CPU to describe a core, right?

What you wrote above "thread <N> of core <M> (using x86 terminology)" is also correlated
with RISC-V terminology:
   A component is termed a core if it contains an independent instruction fetch unit.
   A RISC-V-compatible core might support multiple RISC-V-compatible hardware threads,
   or harts, through multithreading

I checked RISC-V's DTS binding and it seems it is a little bit contradictory to DTS spec,
where it is mentioned that reg property is used to describe how many threads a cpu  has
when SMP is used, but in RISC-V's dts binding they are describing a hardware execution
context:
   This document uses some terminology common to the RISC-V community
   that is not widely used, the definitions of which are listed here:

   hart: A hardware execution context, which contains all the state
   mandated by the RISC-V ISA: a PC and some registers.  This
   terminology is designed to disambiguate software's view of execution
   contexts from any particular microarchitectural implementation
   strategy.  For example, an Intel laptop containing one socket with
   two cores, each of which has two hyperthreads, could be described as
   having four harts.

So in RISC-V's DTS binding they are describing only hardware threads what makes things more
confusing in terms what kind terminology from Xen point of view should be used.

And based on what is written in RISC-V's dts binding:
  For example, an Intel laptop containing one socket with
  two cores, each of which has two hyperthreads, could be described as
  having four harts.
It would be more logical to drop 'thread' argument of riscv_of_get_cpu_hwid(const struct dt_device_node *cpun).
And then the question is what to do with the name of variable cpun? As it could be still confusing. Or, at least,
I can add the comment that CPUn in terms of RISC-V means hart (hardware thread). And then will it be needed to
add such comment for each usage of word "CPU"?

~ Oleksii



[-- Attachment #2: Type: text/html, Size: 7470 bytes --]

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

[Jan Beulich]

On 28.04.2025 12:43, Oleksii Kurochko wrote:
> 
> On 4/28/25 8:31 AM, Jan Beulich wrote:
>> On 25.04.2025 19:07, Oleksii Kurochko wrote:
>>> On 4/15/25 3:45 PM, Jan Beulich wrote:
>>>> On 15.04.2025 15:39, Oleksii Kurochko wrote:
>>>>> On 4/10/25 5:53 PM, Jan Beulich wrote:
>>>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>>>> +{
>>>>>>> +    const __be32 *cell;
>>>>>>> +    int ac;
>>>>>>> +    uint32_t len;
>>>>>>> +
>>>>>>> +    ac = dt_n_addr_cells(cpun);
>>>>>>> +    cell = dt_get_property(cpun, "reg", &len);
>>>>>>> +    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
>>>>>>> +        return ~0ULL;
>>>>>> I'm sorry for my lack of DT knowledge, but what's "thread" representing here?
>>>>>> You only pass in 0 below, so it's unclear whether it's what one might expect
>>>>>> (the thread number on a multi-threaded core).
>>>>> Based on the DT specification alone, the|`reg`| value could refer to either a CPU or a thread ID:
>>>>> ```
>>>>> The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
>>>>> the CPU/threads represented by the CPU node. If a CPU supports more than one thread
>>>>> (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
>>>>> per thread.
>>>>> ```
>>>>>
>>>>> My understanding is that the term/thread/ was used in the Linux kernel to cover both
>>>>> cases.
>>>>> When SMT isn't supported, the CPU can be considered to have a single thread.
>>>>> For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).
>> Note the terminology ("CPU") you used here.
>>
>>>>> Interestingly, the Linux kernel always uses|thread = 0|.
>>>>>
>>>>> We could potentially drop this ambiguity and introduce an|ASSERT()| to check that
>>>>> the|`reg`| property contains only one entry, representing the HART (CPU) ID:
>>>>> ```
>>>>>      Software can determine the number of threads by dividing the size of reg by the parent
>>>>>      node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
>>>>>      is required.
>>>>> ```
>>>>>
>>>>> Does that approach make sense, or should we stick with the current implementation?
>>>> If extra enabling is required to make multi-thread CPUs work, then panic()ing
>>>> (not so much ASSERT()ing) may make sense, for the time being. Better would be
>>>> if we could use all threads in a system right away.
>>> Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id
>>> by passing `thread` argument (`thread` -> the local thread number to get the hardware ID for).
>>> So by calling:
>>>    dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0
>>>    dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
>>>    ...
>>>
>>> In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).
>>>
>>> If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().
>> I assume you meant SMT in both places you wrote SMP?
> 
> Yes, it should be SMT.
> 
>>   But my main point here is:
>> If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you
>> need to be quite careful with what you call "CPU". Things need to be entirely
>> unambiguous, taking into account what internally in (common code) Xen we call a
>> "CPU". You certainly may call "CPU" what is a collection of threads / harts,
>> but you then need to clarify this in a prominent comment somewhere, and you
>> need to be entirely consistent throughout the RISC-V sub-tree.
> 
> ╭────────────────────╮
> │        CPU          │  ← 1 physical processor (chip)
> │ ┌───────┬─────────┐ │
> │ │ Core 0│ Core 1  │ │  ← 2 cores (for example)
> │ │ ┌──┬──┐ ┌──┬──┐ │ │
> │ │Thr0 Thr1 Thr0 Thr1│ ← 2 threads on each core (SMT)
> │ └───────┴─────────┘ │
> ╰────────────────────╯
> I want to double check what Xen call a "CPU". I thought that Xen uses word
> CPU to describe a core, right?

No, see e.g. cpumask.h - it's a hart (as per below) that we internally describe
as CPU (leaving aside potentially ambiguous comments here and there, which is
what I'd like to prevent from the start for RISC-V).

> What you wrote above "thread <N> of core <M> (using x86 terminology)" is also correlated
> with RISC-V terminology:
>    A component is termed a core if it contains an independent instruction fetch unit.
>    A RISC-V-compatible core might support multiple RISC-V-compatible hardware threads,
>    or harts, through multithreading
> 
> I checked RISC-V's DTS binding and it seems it is a little bit contradictory to DTS spec,
> where it is mentioned that reg property is used to describe how many threads a cpu  has
> when SMP is used, but in RISC-V's dts binding they are describing a hardware execution
> context:
>    This document uses some terminology common to the RISC-V community
>    that is not widely used, the definitions of which are listed here:
> 
>    hart: A hardware execution context, which contains all the state
>    mandated by the RISC-V ISA: a PC and some registers.  This
>    terminology is designed to disambiguate software's view of execution
>    contexts from any particular microarchitectural implementation
>    strategy.  For example, an Intel laptop containing one socket with
>    two cores, each of which has two hyperthreads, could be described as
>    having four harts.
> 
> So in RISC-V's DTS binding they are describing only hardware threads what makes things more
> confusing in terms what kind terminology from Xen point of view should be used.
> 
> And based on what is written in RISC-V's dts binding:
>   For example, an Intel laptop containing one socket with
>   two cores, each of which has two hyperthreads, could be described as
>   having four harts.
> It would be more logical to drop 'thread' argument of riscv_of_get_cpu_hwid(const struct dt_device_node *cpun).
> And then the question is what to do with the name of variable cpun? As it could be still confusing. Or, at least,
> I can add the comment that CPUn in terms of RISC-V means hart (hardware thread).

If it's the normal thing you call a CPU, I don't think much commentary is
necessary. Then please just make sure you don't call anything else "CPU",
especially in identifiers.

> And then will it be needed to
> add such comment for each usage of word "CPU"?

No, that would go too far in any event. Hence why I said "clarify this in
a prominent comment somewhere".

Jan

[PATCH v1 07/14] xen/riscv: Introduce intc_hw_operations abstraction

[Oleksii Kurochko]

Introduce the `intc_hw_operations` structure to encapsulate interrupt
controller-specific data and operations. This structure includes:
- A pointer to interrupt controller information (`intc_info`)
- Callbacks to initialize the controller and set IRQ type/priority
- A reference to an interupt controller descriptor (`host_irq_type`)

Also introduce generic helper functions:
- `intc_init()`: Initializes the interrupt controller
- `register_intc_ops()`: Registers the `intc_hw_operations` implementation
- `intc_route_irq_to_xen()`: Configures IRQ routing to Xen, setting handler,
  type, and priority

Most of these functions act as thin wrappers around the corresponding
callbacks in `intc_hw_operations`.

This abstraction lays the groundwork for supporting multiple interrupt
controller types (e.g., PLIC, APLIC) in extensible way.

This patch is based on the changes from [1].

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7cfb4bd4748ca268142497ac5c327d2766fb342d

Co-developed-by: Romain Caritey <Romain.Caritey@microchip.com>
Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/include/asm/intc.h | 23 ++++++++++++++
 xen/arch/riscv/intc.c             | 51 +++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)

diff --git a/xen/arch/riscv/include/asm/intc.h b/xen/arch/riscv/include/asm/intc.h
index 52ba196d87..0d498b10f4 100644
--- a/xen/arch/riscv/include/asm/intc.h
+++ b/xen/arch/riscv/include/asm/intc.h
@@ -17,6 +17,29 @@ struct intc_info {
     const struct dt_device_node *node;
 };
 
+struct intc_hw_operations {
+    /* Hold intc hw information */
+    const struct intc_info *info;
+    /* Initialize the intc and the boot CPU */
+    int (*init)(void);
+
+    /* hw_irq_controller to enable/disable/eoi host irq */
+    hw_irq_controller *host_irq_type;
+
+    /* Set IRQ type */
+    void (*set_irq_type)(struct irq_desc *desc, unsigned int type);
+    /* Set IRQ priority */
+    void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
+
+};
+
 void intc_preinit(void);
 
+void intc_init(void);
+
+void register_intc_ops(const struct intc_hw_operations *ops);
+
+struct irq_desc;
+void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority);
+
 #endif /* ASM__RISCV__INTERRUPT_CONTOLLER_H */
diff --git a/xen/arch/riscv/intc.c b/xen/arch/riscv/intc.c
index 4061a3c457..8274897d8c 100644
--- a/xen/arch/riscv/intc.c
+++ b/xen/arch/riscv/intc.c
@@ -1,9 +1,21 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 
 #include <xen/acpi.h>
+#include <xen/bug.h>
 #include <xen/device_tree.h>
 #include <xen/init.h>
+#include <xen/irq.h>
 #include <xen/lib.h>
+#include <xen/spinlock.h>
+
+#include <asm/intc.h>
+
+static const struct intc_hw_operations *intc_hw_ops;
+
+void register_intc_ops(const struct intc_hw_operations *ops)
+{
+    intc_hw_ops = ops;
+}
 
 void __init intc_preinit(void)
 {
@@ -12,3 +24,42 @@ void __init intc_preinit(void)
     else
         panic("ACPI interrupt controller preinit() isn't implemented\n");
 }
+
+void __init intc_init(void)
+{
+    ASSERT(intc_hw_ops);
+
+    if ( intc_hw_ops->init() )
+        panic("Failed to initialize the interrupt controller drivers\n");
+}
+
+/* desc->irq needs to be disabled before calling this function */
+static void intc_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
+    ASSERT(spin_is_locked(&desc->lock));
+    ASSERT(type != IRQ_TYPE_INVALID);
+    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_type);
+
+    intc_hw_ops->set_irq_type(desc, type);
+}
+
+static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
+{
+    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_priority);
+
+    intc_hw_ops->set_irq_priority(desc, priority);
+}
+
+void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority)
+{
+    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
+    ASSERT(spin_is_locked(&desc->lock));
+    /* Can't route interrupts that don't exist */
+    ASSERT(intc_hw_ops && desc->irq < intc_hw_ops->info->nr_irqs);
+
+    desc->handler = intc_hw_ops->host_irq_type;
+
+    intc_set_irq_type(desc, desc->arch.type);
+    intc_set_irq_priority(desc, priority);
+}
-- 
2.49.0

Re: [PATCH v1 07/14] xen/riscv: Introduce intc_hw_operations abstraction

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> --- a/xen/arch/riscv/include/asm/intc.h
> +++ b/xen/arch/riscv/include/asm/intc.h
> @@ -17,6 +17,29 @@ struct intc_info {
>      const struct dt_device_node *node;
>  };
>  
> +struct intc_hw_operations {
> +    /* Hold intc hw information */
> +    const struct intc_info *info;
> +    /* Initialize the intc and the boot CPU */
> +    int (*init)(void);
> +
> +    /* hw_irq_controller to enable/disable/eoi host irq */
> +    hw_irq_controller *host_irq_type;

Pointer-to-const perhaps?

> +    /* Set IRQ type */
> +    void (*set_irq_type)(struct irq_desc *desc, unsigned int type);
> +    /* Set IRQ priority */
> +    void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
> +
> +};
> +
>  void intc_preinit(void);
>  
> +void intc_init(void);
> +
> +void register_intc_ops(const struct intc_hw_operations *ops);
> +
> +struct irq_desc;

If it's needed here at all, it needs to move up, as some of the hook pointers
already use the type.

> --- a/xen/arch/riscv/intc.c
> +++ b/xen/arch/riscv/intc.c
> @@ -1,9 +1,21 @@
>  /* SPDX-License-Identifier: GPL-2.0-only */
>  
>  #include <xen/acpi.h>
> +#include <xen/bug.h>
>  #include <xen/device_tree.h>
>  #include <xen/init.h>
> +#include <xen/irq.h>
>  #include <xen/lib.h>
> +#include <xen/spinlock.h>
> +
> +#include <asm/intc.h>
> +
> +static const struct intc_hw_operations *intc_hw_ops;

__ro_after_init perhaps?

> +
> +void register_intc_ops(const struct intc_hw_operations *ops)

__init perhaps?

> +{
> +    intc_hw_ops = ops;
> +}
>  
>  void __init intc_preinit(void)
>  {
> @@ -12,3 +24,42 @@ void __init intc_preinit(void)
>      else
>          panic("ACPI interrupt controller preinit() isn't implemented\n");
>  }
> +
> +void __init intc_init(void)
> +{
> +    ASSERT(intc_hw_ops);
> +
> +    if ( intc_hw_ops->init() )
> +        panic("Failed to initialize the interrupt controller drivers\n");
> +}
> +
> +/* desc->irq needs to be disabled before calling this function */
> +static void intc_set_irq_type(struct irq_desc *desc, unsigned int type)
> +{
> +    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
> +    ASSERT(spin_is_locked(&desc->lock));
> +    ASSERT(type != IRQ_TYPE_INVALID);
> +    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_type);
> +
> +    intc_hw_ops->set_irq_type(desc, type);
> +}
> +
> +static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
> +{
> +    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_priority);
> +
> +    intc_hw_ops->set_irq_priority(desc, priority);
> +}
> +
> +void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority)
> +{
> +    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
> +    ASSERT(spin_is_locked(&desc->lock));
> +    /* Can't route interrupts that don't exist */
> +    ASSERT(intc_hw_ops && desc->irq < intc_hw_ops->info->nr_irqs);
> +
> +    desc->handler = intc_hw_ops->host_irq_type;
> +
> +    intc_set_irq_type(desc, desc->arch.type);
> +    intc_set_irq_priority(desc, priority);

If these are going to remain the sole callers of the two functions, I'd question
the need for the separate functions. Some of the assertions done there would then
actually be redundant.

If not, is there a reason intc_set_irq_priority() doesn't have a lock-held check?

Jan

Re: [PATCH v1 07/14] xen/riscv: Introduce intc_hw_operations abstraction

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 4167 bytes --]


On 4/10/25 6:02 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> --- a/xen/arch/riscv/include/asm/intc.h
>> +++ b/xen/arch/riscv/include/asm/intc.h
>> @@ -17,6 +17,29 @@ struct intc_info {
>>       const struct dt_device_node *node;
>>   };
>>   
>> +struct intc_hw_operations {
>> +    /* Hold intc hw information */
>> +    const struct intc_info *info;
>> +    /* Initialize the intc and the boot CPU */
>> +    int (*init)(void);
>> +
>> +    /* hw_irq_controller to enable/disable/eoi host irq */
>> +    hw_irq_controller *host_irq_type;
> Pointer-to-const perhaps?

It could be pointer-to-const. I'll updat

>
>> +    /* Set IRQ type */
>> +    void (*set_irq_type)(struct irq_desc *desc, unsigned int type);
>> +    /* Set IRQ priority */
>> +    void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
>> +
>> +};
>> +
>>   void intc_preinit(void);
>>   
>> +void intc_init(void);
>> +
>> +void register_intc_ops(const struct intc_hw_operations *ops);
>> +
>> +struct irq_desc;
> If it's needed here at all, it needs to move up, as some of the hook pointers
> already use the type.

It could be dropped, but then we would need to include|<xen/irq.h>|, and I figured
the fewer includes we have, the better.

>
>> --- a/xen/arch/riscv/intc.c
>> +++ b/xen/arch/riscv/intc.c
>> @@ -1,9 +1,21 @@
>>   /* SPDX-License-Identifier: GPL-2.0-only */
>>   
>>   #include <xen/acpi.h>
>> +#include <xen/bug.h>
>>   #include <xen/device_tree.h>
>>   #include <xen/init.h>
>> +#include <xen/irq.h>
>>   #include <xen/lib.h>
>> +#include <xen/spinlock.h>
>> +
>> +#include <asm/intc.h>
>> +
>> +static const struct intc_hw_operations *intc_hw_ops;
> __ro_after_init perhaps?
>
>> +
>> +void register_intc_ops(const struct intc_hw_operations *ops)
> __init perhaps?

For both, yes, it should be __ro_after_init and __init.

>
>> +{
>> +    intc_hw_ops = ops;
>> +}
>>   
>>   void __init intc_preinit(void)
>>   {
>> @@ -12,3 +24,42 @@ void __init intc_preinit(void)
>>       else
>>           panic("ACPI interrupt controller preinit() isn't implemented\n");
>>   }
>> +
>> +void __init intc_init(void)
>> +{
>> +    ASSERT(intc_hw_ops);
>> +
>> +    if ( intc_hw_ops->init() )
>> +        panic("Failed to initialize the interrupt controller drivers\n");
>> +}
>> +
>> +/* desc->irq needs to be disabled before calling this function */
>> +static void intc_set_irq_type(struct irq_desc *desc, unsigned int type)
>> +{
>> +    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
>> +    ASSERT(spin_is_locked(&desc->lock));
>> +    ASSERT(type != IRQ_TYPE_INVALID);
>> +    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_type);
>> +
>> +    intc_hw_ops->set_irq_type(desc, type);
>> +}
>> +
>> +static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
>> +{
>> +    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_priority);
>> +
>> +    intc_hw_ops->set_irq_priority(desc, priority);
>> +}
>> +
>> +void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority)
>> +{
>> +    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
>> +    ASSERT(spin_is_locked(&desc->lock));
>> +    /* Can't route interrupts that don't exist */
>> +    ASSERT(intc_hw_ops && desc->irq < intc_hw_ops->info->nr_irqs);
>> +
>> +    desc->handler = intc_hw_ops->host_irq_type;
>> +
>> +    intc_set_irq_type(desc, desc->arch.type);
>> +    intc_set_irq_priority(desc, priority);
> If these are going to remain the sole callers of the two functions, I'd question
> the need for the separate functions. Some of the assertions done there would then
> actually be redundant.

Likely they will be reused for similar function but for guest to route an interrupt
to a guest. And, perhaps, only intc_set_irq_type() will be re-used outside
intc_route_irq_to_{xen,guest}...

>
> If not, is there a reason intc_set_irq_priority() doesn't have a lock-held check?

Do you mean this one: ASSERT(spin_is_locked(&desc->lock));

...
And it is the reason why intc_set_irq_priority() doesn't have this check as it is checked in the
caller.

But I think just to be on a safe side, it could be better to add this check to
intc_set_irq_priority().

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 5864 bytes --]

[PATCH v1 08/14] xen/riscv: imsic_init() implementation

[Oleksii Kurochko]

imsic_init() is introduced to parse device tree node, which has the following
bindings [2], and based on the parsed information update IMSIC configuration
which is stored in imsic_cfg.

The following helpers are introduces for imsic_init() usage:
  - imsic_parse_node() parses IMSIC node from DTS
  - imsic_get_parent_hartid() returns the hart ( CPU ) ID of the given device
    tree node.

This patch is based on the code from [1].

Since Microchip originally developed imsic.{c,h}, an internal discussion with
them led to the decision to use the MIT license.

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/0b1a94f2bc3bb1a81cd26bb75f0bf578f84cb4d4
[2] https://elixir.bootlin.com/linux/v6.12/source/Documentation/devicetree/bindings/interrupt-controller/riscv,imsics.yaml

Co-developed-by: Romain Caritey <Romain.Caritey@microchip.com>
Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/Makefile            |   1 +
 xen/arch/riscv/imsic.c             | 286 +++++++++++++++++++++++++++++
 xen/arch/riscv/include/asm/imsic.h |  66 +++++++
 3 files changed, 353 insertions(+)
 create mode 100644 xen/arch/riscv/imsic.c
 create mode 100644 xen/arch/riscv/include/asm/imsic.h

diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile
index 457e8e88a4..baa499a72d 100644
--- a/xen/arch/riscv/Makefile
+++ b/xen/arch/riscv/Makefile
@@ -2,6 +2,7 @@ obj-y += aplic.o
 obj-y += cpufeature.o
 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
 obj-y += entry.o
+obj-y += imsic.o
 obj-y += intc.o
 obj-y += irq.o
 obj-y += mm.o
diff --git a/xen/arch/riscv/imsic.c b/xen/arch/riscv/imsic.c
new file mode 100644
index 0000000000..99def9af2d
--- /dev/null
+++ b/xen/arch/riscv/imsic.c
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/imsic.c
+ *
+ * RISC-V Incoming MSI Controller support
+ *
+ * (c) 2023 Microchip Technology Inc.
+ * (c) 2024 Vates
+ */
+
+#include <xen/const.h>
+#include <xen/device_tree.h>
+#include <xen/errno.h>
+#include <xen/init.h>
+#include <xen/macros.h>
+#include <xen/xmalloc.h>
+
+#include <asm/imsic.h>
+
+static struct imsic_config imsic_cfg;
+
+const struct imsic_config *imsic_get_config(void)
+{
+    return &imsic_cfg;
+}
+
+static int __init imsic_get_parent_hartid(struct dt_device_node *node,
+                                          unsigned int index,
+                                          unsigned long *hartid)
+{
+    int res;
+    unsigned long hart;
+    struct dt_phandle_args args;
+
+    /* Try the new-style interrupts-extended first */
+    res = dt_parse_phandle_with_args(node, "interrupts-extended",
+                                     "#interrupt-cells", index, &args);
+    if ( !res )
+    {
+        res = riscv_of_processor_hartid(args.np->parent, &hart);
+        if ( res < 0 )
+            return -EINVAL;
+
+        *hartid = hart;
+    }
+    return res;
+}
+
+
+static int imsic_parse_node(struct dt_device_node *node,
+                     unsigned int *nr_parent_irqs)
+{
+    int rc;
+    unsigned int tmp;
+    paddr_t base_addr;
+
+    /* Find number of parent interrupts */
+    *nr_parent_irqs = dt_number_of_irq(node);
+    if ( !*nr_parent_irqs )
+    {
+        printk(XENLOG_ERR "%s: no parent irqs available\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find number of guest index bits in MSI address */
+    rc = dt_property_read_u32(node, "riscv,guest-index-bits",
+                              &imsic_cfg.guest_index_bits);
+    if ( !rc )
+        imsic_cfg.guest_index_bits = 0;
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT;
+    if ( tmp < imsic_cfg.guest_index_bits )
+    {
+        printk(XENLOG_ERR "%s: guest index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find number of HART index bits */
+    rc = dt_property_read_u32(node, "riscv,hart-index-bits",
+                              &imsic_cfg.hart_index_bits);
+    if ( !rc )
+    {
+        /* Assume default value */
+        imsic_cfg.hart_index_bits = fls(*nr_parent_irqs);
+        if ( BIT(imsic_cfg.hart_index_bits, UL) < *nr_parent_irqs )
+            imsic_cfg.hart_index_bits++;
+    }
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
+          imsic_cfg.guest_index_bits;
+    if ( tmp < imsic_cfg.hart_index_bits )
+    {
+        printk(XENLOG_ERR "%s: HART index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find number of group index bits */
+    rc = dt_property_read_u32(node, "riscv,group-index-bits",
+                              &imsic_cfg.group_index_bits);
+    if ( !rc )
+        imsic_cfg.group_index_bits = 0;
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
+          imsic_cfg.guest_index_bits - imsic_cfg.hart_index_bits;
+    if ( tmp < imsic_cfg.group_index_bits )
+    {
+        printk(XENLOG_ERR "%s: group index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find first bit position of group index */
+    tmp = IMSIC_MMIO_PAGE_SHIFT * 2;
+    rc = dt_property_read_u32(node, "riscv,group-index-shift",
+                              &imsic_cfg.group_index_shift);
+    if ( !rc )
+        imsic_cfg.group_index_shift = tmp;
+    if ( imsic_cfg.group_index_shift < tmp )
+    {
+        printk(XENLOG_ERR "%s: group index shift too small\n", node->name);
+        return -ENOENT;
+    }
+    tmp = imsic_cfg.group_index_bits + imsic_cfg.group_index_shift - 1;
+    if ( tmp >= BITS_PER_LONG )
+    {
+        printk(XENLOG_ERR "%s: group index shift too big\n", node->name);
+        return -EINVAL;
+    }
+
+    /* Find number of interrupt identities */
+    rc = dt_property_read_u32(node, "riscv,num-ids", &imsic_cfg.nr_ids);
+    if ( !rc )
+    {
+        printk(XENLOG_ERR "%s: number of interrupt identities not found\n",
+               node->name);
+        return -ENOENT;
+    }
+
+    if ( (imsic_cfg.nr_ids < IMSIC_MIN_ID) ||
+         (imsic_cfg.nr_ids >= IMSIC_MAX_ID) ||
+         ((imsic_cfg.nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) )
+    {
+        printk(XENLOG_ERR "%s: invalid number of interrupt identities\n",
+               node->name);
+        return -EINVAL;
+    }
+
+    /* Compute base address */
+    imsic_cfg.nr_mmios = 0;
+    rc = dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL);
+    if (rc)
+    {
+        printk(XENLOG_ERR "%s: first MMIO resource not found\n", node->name);
+        return -EINVAL;
+    }
+
+    imsic_cfg.base_addr = base_addr;
+    imsic_cfg.base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
+                   imsic_cfg.hart_index_bits +
+                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
+    imsic_cfg.base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
+                   imsic_cfg.group_index_shift);
+
+    /* Find number of MMIO register sets */
+    imsic_cfg.nr_mmios++;
+    while ( !dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL) )
+        imsic_cfg.nr_mmios++;
+
+    return 0;
+}
+
+int __init imsic_init(struct dt_device_node *node)
+{
+    int rc;
+    unsigned long reloff, hartid;
+    uint32_t nr_parent_irqs, index, nr_handlers = 0;
+    paddr_t base_addr;
+
+    /* Parse IMSIC node */
+    rc = imsic_parse_node(node, &nr_parent_irqs);
+    if ( rc )
+        return rc;
+
+    /* Allocate MMIO resource array */
+    imsic_cfg.mmios = xzalloc_array(struct imsic_mmios, imsic_cfg.nr_mmios);
+    if ( !imsic_cfg.mmios )
+        return -ENOMEM;
+
+    /* check MMIO register sets */
+    for ( int i = 0; i < imsic_cfg.nr_mmios; i++ )
+    {
+        rc = dt_device_get_address(node, i, &imsic_cfg.mmios[i].base_addr,
+                                   &imsic_cfg.mmios[i].size);
+        if ( rc )
+        {
+            printk(XENLOG_ERR "%s:  unable to parse MMIO regset %d\n",
+                   node->name, i);
+            goto imsic_init_err;
+        }
+
+        base_addr = imsic_cfg.mmios[i].base_addr;
+        base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
+                   imsic_cfg.hart_index_bits +
+                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
+        base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
+                   imsic_cfg.group_index_shift);
+        if ( base_addr != imsic_cfg.base_addr )
+        {
+            rc = -EINVAL;
+            printk(XENLOG_ERR "%s: address mismatch for regset %d\n",
+                   node->name, i);
+            goto imsic_init_err;
+        }
+    }
+
+    /* Configure handlers for target CPUs */
+    for ( int i = 0; i < nr_parent_irqs; i++ )
+    {
+        rc = imsic_get_parent_hartid(node, i, &hartid);
+        if ( rc )
+        {
+            printk(XENLOG_WARNING "%s: hart ID for parent irq%d not found\n",
+                   node->name, i);
+            continue;
+        }
+
+        if ( hartid > NR_CPUS )
+        {
+            printk(XENLOG_WARNING "%s: unsupported hart ID=%lu for parent irq%d\n",
+                   node->name, hartid, i);
+            continue;
+        }
+
+        /* Find MMIO location of MSI page */
+        index = imsic_cfg.nr_mmios;
+        reloff = i * BIT(imsic_cfg.guest_index_bits, UL) * IMSIC_MMIO_PAGE_SZ;
+        for ( int j = 0; imsic_cfg.nr_mmios; j++ )
+        {
+            if ( reloff < imsic_cfg.mmios[j].size )
+            {
+                index = j;
+                break;
+            }
+
+            /*
+             * MMIO region size may not be aligned to
+             * BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ
+             * if holes are present.
+             */
+            reloff -= ROUNDUP(imsic_cfg.mmios[j].size,
+                BIT(imsic_cfg.guest_index_bits, UL) * IMSIC_MMIO_PAGE_SZ);
+        }
+
+        if ( index >= imsic_cfg.nr_mmios )
+        {
+            printk(XENLOG_WARNING "%s: MMIO not found for parent irq%d\n",
+                   node->name, i);
+            continue;
+        }
+
+        if ( !IS_ALIGNED(imsic_cfg.msi[hartid].base_addr + reloff, PAGE_SIZE) )
+        {
+            printk(XENLOG_WARNING "%s: MMIO address 0x%lx is not aligned on a page\n",
+                   node->name, imsic_cfg.msi[hartid].base_addr + reloff);
+            imsic_cfg.msi[hartid].offset = 0;
+            imsic_cfg.msi[hartid].base_addr = 0;
+            continue;
+        }
+
+        imsic_cfg.mmios[index].harts[hartid] = true;
+        imsic_cfg.msi[hartid].base_addr = imsic_cfg.mmios[index].base_addr;
+        imsic_cfg.msi[hartid].offset = reloff;
+        nr_handlers++;
+    }
+
+    if ( !nr_handlers )
+    {
+        printk(XENLOG_ERR "%s: No CPU handlers found\n", node->name);
+        rc = -ENODEV;
+        goto imsic_init_err;
+    }
+
+    return 0;
+
+imsic_init_err:
+    xfree(imsic_cfg.mmios);
+
+    return rc;
+}
diff --git a/xen/arch/riscv/include/asm/imsic.h b/xen/arch/riscv/include/asm/imsic.h
new file mode 100644
index 0000000000..126e651863
--- /dev/null
+++ b/xen/arch/riscv/include/asm/imsic.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/imsic.h
+ *
+ * RISC-V Incoming MSI Controller support
+ *
+ * (c) 2023 Microchip Technology Inc.
+ */
+
+#ifndef ASM__RISCV__IMSIC_H
+#define ASM__RISCV__IMSIC_H
+
+#include <xen/types.h>
+
+#define IMSIC_MMIO_PAGE_SHIFT   12
+#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
+
+#define IMSIC_MIN_ID            63
+#define IMSIC_MAX_ID            2048
+
+struct imsic_msi {
+    paddr_t base_addr;
+    unsigned long offset;
+};
+
+struct imsic_mmios {
+    paddr_t base_addr;
+    unsigned long size;
+    bool harts[NR_CPUS];
+};
+
+struct imsic_config {
+    /* base address */
+    paddr_t base_addr;
+
+    /* Bits representing Guest index, HART index, and Group index */
+    unsigned int guest_index_bits;
+    unsigned int hart_index_bits;
+    unsigned int group_index_bits;
+    unsigned int group_index_shift;
+
+    /* imsic phandle */
+    unsigned int phandle;
+
+    /* number of parent irq */
+    unsigned int nr_parent_irqs;
+
+    /* number off interrupt identities */
+    unsigned int nr_ids;
+
+    /* mmios */
+    unsigned int nr_mmios;
+    struct imsic_mmios *mmios;
+
+    /* MSI */
+    struct imsic_msi msi[NR_CPUS];
+};
+
+struct dt_device_node;
+int imsic_init(struct dt_device_node *n);
+
+struct imsic_config;
+const struct imsic_config *imsic_get_config(void);
+
+#endif /* ASM__RISCV__IMSIC_H */
-- 
2.49.0

Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> --- /dev/null
> +++ b/xen/arch/riscv/imsic.c
> @@ -0,0 +1,286 @@
> +/* SPDX-License-Identifier: MIT */
> +
> +/*
> + * xen/arch/riscv/imsic.c
> + *
> + * RISC-V Incoming MSI Controller support
> + *
> + * (c) 2023 Microchip Technology Inc.
> + * (c) 2024 Vates

No 2025 here (if already the years matter)?

> + */
> +
> +#include <xen/const.h>
> +#include <xen/device_tree.h>
> +#include <xen/errno.h>
> +#include <xen/init.h>
> +#include <xen/macros.h>
> +#include <xen/xmalloc.h>
> +
> +#include <asm/imsic.h>
> +
> +static struct imsic_config imsic_cfg;
> +
> +const struct imsic_config *imsic_get_config(void)

Does this need to return a pointer to non-const?

> +{
> +    return &imsic_cfg;
> +}
> +
> +static int __init imsic_get_parent_hartid(struct dt_device_node *node,
> +                                          unsigned int index,
> +                                          unsigned long *hartid)
> +{
> +    int res;
> +    unsigned long hart;
> +    struct dt_phandle_args args;
> +
> +    /* Try the new-style interrupts-extended first */

The comment says "first", but then ...

> +    res = dt_parse_phandle_with_args(node, "interrupts-extended",
> +                                     "#interrupt-cells", index, &args);
> +    if ( !res )
> +    {
> +        res = riscv_of_processor_hartid(args.np->parent, &hart);
> +        if ( res < 0 )
> +            return -EINVAL;
> +
> +        *hartid = hart;
> +    }
> +    return res;
> +}

... nothing else is being tried.

Also, nit: Blank line please ahead of the main "return" of a function.

Further - any particular reason to discard riscv_of_processor_hartid()'s
error code on the error path?

> +
> +

Nit: No double blank lines please (and I wish I wouldn't need to repeat
this any further).

> +static int imsic_parse_node(struct dt_device_node *node,
> +                     unsigned int *nr_parent_irqs)
> +{
> +    int rc;
> +    unsigned int tmp;
> +    paddr_t base_addr;
> +
> +    /* Find number of parent interrupts */
> +    *nr_parent_irqs = dt_number_of_irq(node);
> +    if ( !*nr_parent_irqs )
> +    {
> +        printk(XENLOG_ERR "%s: no parent irqs available\n", node->name);
> +        return -ENOENT;
> +    }
> +
> +    /* Find number of guest index bits in MSI address */
> +    rc = dt_property_read_u32(node, "riscv,guest-index-bits",
> +                              &imsic_cfg.guest_index_bits);
> +    if ( !rc )

It is confusing to store a bool return value in a local "int" variable,
just to then use it as boolean. Is the local var needed at all here?

> +        imsic_cfg.guest_index_bits = 0;
> +    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT;
> +    if ( tmp < imsic_cfg.guest_index_bits )
> +    {
> +        printk(XENLOG_ERR "%s: guest index bits too big\n", node->name);
> +        return -ENOENT;
> +    }
> +
> +    /* Find number of HART index bits */
> +    rc = dt_property_read_u32(node, "riscv,hart-index-bits",
> +                              &imsic_cfg.hart_index_bits);
> +    if ( !rc )
> +    {
> +        /* Assume default value */
> +        imsic_cfg.hart_index_bits = fls(*nr_parent_irqs);
> +        if ( BIT(imsic_cfg.hart_index_bits, UL) < *nr_parent_irqs )
> +            imsic_cfg.hart_index_bits++;
> +    }
> +    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
> +          imsic_cfg.guest_index_bits;

    tmp -= imsic_cfg.guest_index_bits;

? (And then similarly further down.)

> +    if ( tmp < imsic_cfg.hart_index_bits )
> +    {
> +        printk(XENLOG_ERR "%s: HART index bits too big\n", node->name);
> +        return -ENOENT;
> +    }
> +
> +    /* Find number of group index bits */
> +    rc = dt_property_read_u32(node, "riscv,group-index-bits",
> +                              &imsic_cfg.group_index_bits);
> +    if ( !rc )
> +        imsic_cfg.group_index_bits = 0;
> +    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
> +          imsic_cfg.guest_index_bits - imsic_cfg.hart_index_bits;
> +    if ( tmp < imsic_cfg.group_index_bits )
> +    {
> +        printk(XENLOG_ERR "%s: group index bits too big\n", node->name);
> +        return -ENOENT;
> +    }
> +
> +    /* Find first bit position of group index */
> +    tmp = IMSIC_MMIO_PAGE_SHIFT * 2;
> +    rc = dt_property_read_u32(node, "riscv,group-index-shift",
> +                              &imsic_cfg.group_index_shift);
> +    if ( !rc )
> +        imsic_cfg.group_index_shift = tmp;
> +    if ( imsic_cfg.group_index_shift < tmp )
> +    {
> +        printk(XENLOG_ERR "%s: group index shift too small\n", node->name);
> +        return -ENOENT;
> +    }
> +    tmp = imsic_cfg.group_index_bits + imsic_cfg.group_index_shift - 1;
> +    if ( tmp >= BITS_PER_LONG )
> +    {
> +        printk(XENLOG_ERR "%s: group index shift too big\n", node->name);
> +        return -EINVAL;
> +    }
> +
> +    /* Find number of interrupt identities */
> +    rc = dt_property_read_u32(node, "riscv,num-ids", &imsic_cfg.nr_ids);
> +    if ( !rc )
> +    {
> +        printk(XENLOG_ERR "%s: number of interrupt identities not found\n",
> +               node->name);
> +        return -ENOENT;
> +    }
> +
> +    if ( (imsic_cfg.nr_ids < IMSIC_MIN_ID) ||
> +         (imsic_cfg.nr_ids >= IMSIC_MAX_ID) ||

Something named "max" normally wants to decribe the highest valid value,
not the first out-of-range one.

> +         ((imsic_cfg.nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) )
> +    {
> +        printk(XENLOG_ERR "%s: invalid number of interrupt identities\n",
> +               node->name);
> +        return -EINVAL;
> +    }
> +
> +    /* Compute base address */
> +    imsic_cfg.nr_mmios = 0;
> +    rc = dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL);
> +    if (rc)

Nit: Style.

> +    {
> +        printk(XENLOG_ERR "%s: first MMIO resource not found\n", node->name);
> +        return -EINVAL;

Discarding "rc" again?

> +    }
> +
> +    imsic_cfg.base_addr = base_addr;
> +    imsic_cfg.base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
> +                   imsic_cfg.hart_index_bits +
> +                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
> +    imsic_cfg.base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
> +                   imsic_cfg.group_index_shift);

Besides indentation being bogus here, why is it that you need to mask bits
off of the value read from DT? Wouldn't the expectation be that you get back
the true base address?

> +    /* Find number of MMIO register sets */
> +    imsic_cfg.nr_mmios++;
> +    while ( !dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL) )
> +        imsic_cfg.nr_mmios++;

And the base addresses of these aren't of interest? Oh, I see they're
fetched again further down.

Also - use do-while here?

> +    return 0;
> +}
> +
> +int __init imsic_init(struct dt_device_node *node)
> +{
> +    int rc;
> +    unsigned long reloff, hartid;
> +    uint32_t nr_parent_irqs, index, nr_handlers = 0;
> +    paddr_t base_addr;
> +
> +    /* Parse IMSIC node */
> +    rc = imsic_parse_node(node, &nr_parent_irqs);
> +    if ( rc )
> +        return rc;
> +
> +    /* Allocate MMIO resource array */
> +    imsic_cfg.mmios = xzalloc_array(struct imsic_mmios, imsic_cfg.nr_mmios);
> +    if ( !imsic_cfg.mmios )
> +        return -ENOMEM;
> +
> +    /* check MMIO register sets */
> +    for ( int i = 0; i < imsic_cfg.nr_mmios; i++ )

No plain int please for anything that can only be non-negative.

> +    {
> +        rc = dt_device_get_address(node, i, &imsic_cfg.mmios[i].base_addr,
> +                                   &imsic_cfg.mmios[i].size);
> +        if ( rc )
> +        {
> +            printk(XENLOG_ERR "%s:  unable to parse MMIO regset %d\n",
> +                   node->name, i);
> +            goto imsic_init_err;
> +        }
> +
> +        base_addr = imsic_cfg.mmios[i].base_addr;
> +        base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
> +                   imsic_cfg.hart_index_bits +
> +                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
> +        base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
> +                   imsic_cfg.group_index_shift);

Indentation again.

> +        if ( base_addr != imsic_cfg.base_addr )
> +        {
> +            rc = -EINVAL;
> +            printk(XENLOG_ERR "%s: address mismatch for regset %d\n",
> +                   node->name, i);
> +            goto imsic_init_err;
> +        }

Oh, all of the addresses need to (sufficiently) match.

> +    }
> +
> +    /* Configure handlers for target CPUs */
> +    for ( int i = 0; i < nr_parent_irqs; i++ )
> +    {
> +        rc = imsic_get_parent_hartid(node, i, &hartid);
> +        if ( rc )
> +        {
> +            printk(XENLOG_WARNING "%s: hart ID for parent irq%d not found\n",
> +                   node->name, i);
> +            continue;
> +        }
> +
> +        if ( hartid > NR_CPUS )
> +        {
> +            printk(XENLOG_WARNING "%s: unsupported hart ID=%lu for parent irq%d\n",
> +                   node->name, hartid, i);
> +            continue;
> +        }
> +
> +        /* Find MMIO location of MSI page */
> +        index = imsic_cfg.nr_mmios;
> +        reloff = i * BIT(imsic_cfg.guest_index_bits, UL) * IMSIC_MMIO_PAGE_SZ;
> +        for ( int j = 0; imsic_cfg.nr_mmios; j++ )
> +        {
> +            if ( reloff < imsic_cfg.mmios[j].size )
> +            {
> +                index = j;
> +                break;
> +            }
> +
> +            /*
> +             * MMIO region size may not be aligned to
> +             * BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ
> +             * if holes are present.
> +             */
> +            reloff -= ROUNDUP(imsic_cfg.mmios[j].size,
> +                BIT(imsic_cfg.guest_index_bits, UL) * IMSIC_MMIO_PAGE_SZ);
> +        }
> +
> +        if ( index >= imsic_cfg.nr_mmios )
> +        {
> +            printk(XENLOG_WARNING "%s: MMIO not found for parent irq%d\n",
> +                   node->name, i);
> +            continue;
> +        }
> +
> +        if ( !IS_ALIGNED(imsic_cfg.msi[hartid].base_addr + reloff, PAGE_SIZE) )
> +        {
> +            printk(XENLOG_WARNING "%s: MMIO address 0x%lx is not aligned on a page\n",
> +                   node->name, imsic_cfg.msi[hartid].base_addr + reloff);
> +            imsic_cfg.msi[hartid].offset = 0;
> +            imsic_cfg.msi[hartid].base_addr = 0;
> +            continue;
> +        }
> +
> +        imsic_cfg.mmios[index].harts[hartid] = true;
> +        imsic_cfg.msi[hartid].base_addr = imsic_cfg.mmios[index].base_addr;
> +        imsic_cfg.msi[hartid].offset = reloff;
> +        nr_handlers++;
> +    }
> +
> +    if ( !nr_handlers )
> +    {
> +        printk(XENLOG_ERR "%s: No CPU handlers found\n", node->name);
> +        rc = -ENODEV;
> +        goto imsic_init_err;
> +    }
> +
> +    return 0;
> +
> +imsic_init_err:

Labels indented by at least one blank please.

> +    xfree(imsic_cfg.mmios);

Better use XFREE() in cases like this one?

> --- /dev/null
> +++ b/xen/arch/riscv/include/asm/imsic.h
> @@ -0,0 +1,66 @@
> +/* SPDX-License-Identifier: MIT */
> +
> +/*
> + * xen/arch/riscv/imsic.h
> + *
> + * RISC-V Incoming MSI Controller support
> + *
> + * (c) 2023 Microchip Technology Inc.
> + */
> +
> +#ifndef ASM__RISCV__IMSIC_H
> +#define ASM__RISCV__IMSIC_H
> +
> +#include <xen/types.h>
> +
> +#define IMSIC_MMIO_PAGE_SHIFT   12
> +#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
> +
> +#define IMSIC_MIN_ID            63
> +#define IMSIC_MAX_ID            2048
> +
> +struct imsic_msi {
> +    paddr_t base_addr;
> +    unsigned long offset;
> +};
> +
> +struct imsic_mmios {
> +    paddr_t base_addr;
> +    unsigned long size;
> +    bool harts[NR_CPUS];

An array of bool - won't a bitmap do here? Even then I wouldn't be overly
happy to see it dimensioned by NR_CPUS.

> +};
> +
> +struct imsic_config {
> +    /* base address */
> +    paddr_t base_addr;
> +
> +    /* Bits representing Guest index, HART index, and Group index */
> +    unsigned int guest_index_bits;
> +    unsigned int hart_index_bits;
> +    unsigned int group_index_bits;
> +    unsigned int group_index_shift;
> +
> +    /* imsic phandle */
> +    unsigned int phandle;
> +
> +    /* number of parent irq */
> +    unsigned int nr_parent_irqs;
> +
> +    /* number off interrupt identities */
> +    unsigned int nr_ids;
> +
> +    /* mmios */
> +    unsigned int nr_mmios;
> +    struct imsic_mmios *mmios;
> +
> +    /* MSI */
> +    struct imsic_msi msi[NR_CPUS];

You surely can avoid wasting perhaps a lot of memory by allocating this
based on the number of CPUs in use?

> +};
> +
> +struct dt_device_node;
> +int imsic_init(struct dt_device_node *n);

Misra demands that parameter names match between declaration and definition.

> +struct imsic_config;

I don't think you need this, as it's ...

> +const struct imsic_config *imsic_get_config(void);

... not used as parameter type (where its scope would otherwise be wrongly
limited).

Jan

Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 11816 bytes --]


On 4/14/25 11:32 AM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> --- /dev/null
>> +++ b/xen/arch/riscv/imsic.c
>> @@ -0,0 +1,286 @@
>> +/* SPDX-License-Identifier: MIT */
>> +
>> +/*
>> + * xen/arch/riscv/imsic.c
>> + *
>> + * RISC-V Incoming MSI Controller support
>> + *
>> + * (c) 2023 Microchip Technology Inc.
>> + * (c) 2024 Vates
> No 2025 here (if already the years matter)?

I think it doesn't really matter and it could be just dropped.

>
>> + */
>> +
>> +#include <xen/const.h>
>> +#include <xen/device_tree.h>
>> +#include <xen/errno.h>
>> +#include <xen/init.h>
>> +#include <xen/macros.h>
>> +#include <xen/xmalloc.h>
>> +
>> +#include <asm/imsic.h>
>> +
>> +static struct imsic_config imsic_cfg;
>> +
>> +const struct imsic_config *imsic_get_config(void)
> Does this need to return a pointer to non-const?

No, I think it could be const struct imsic_config * const as a return type.
It isn't expected that the caller will change a pointer.

>
>> +{
>> +    return &imsic_cfg;
>> +}
>> +
>> +static int __init imsic_get_parent_hartid(struct dt_device_node *node,
>> +                                          unsigned int index,
>> +                                          unsigned long *hartid)
>> +{
>> +    int res;
>> +    unsigned long hart;
>> +    struct dt_phandle_args args;
>> +
>> +    /* Try the new-style interrupts-extended first */
> The comment says "first", but then ...
>
>> +    res = dt_parse_phandle_with_args(node, "interrupts-extended",
>> +                                     "#interrupt-cells", index, &args);
>> +    if ( !res )
>> +    {
>> +        res = riscv_of_processor_hartid(args.np->parent, &hart);
>> +        if ( res < 0 )
>> +            return -EINVAL;
>> +
>> +        *hartid = hart;
>> +    }
>> +    return res;
>> +}
> ... nothing else is being tried.

A stale comment, we decided to support only interrupt-extended to be in line with
Linux device tree bindings.
I'll drop this comment.

>
> Also, nit: Blank line please ahead of the main "return" of a function.
>
> Further - any particular reason to discard riscv_of_processor_hartid()'s
> error code on the error path?

No particular reason, just overlooked that we could really return just `res`.


>> +static int imsic_parse_node(struct dt_device_node *node,
>> +                     unsigned int *nr_parent_irqs)

(for me: fix an indentation)

>> +{
>> +    int rc;
>> +    unsigned int tmp;
>> +    paddr_t base_addr;
>> +
>> +    /* Find number of parent interrupts */
>> +    *nr_parent_irqs = dt_number_of_irq(node);
>> +    if ( !*nr_parent_irqs )
>> +    {
>> +        printk(XENLOG_ERR "%s: no parent irqs available\n", node->name);
>> +        return -ENOENT;
>> +    }
>> +
>> +    /* Find number of guest index bits in MSI address */
>> +    rc = dt_property_read_u32(node, "riscv,guest-index-bits",
>> +                              &imsic_cfg.guest_index_bits);
>> +    if ( !rc )
> It is confusing to store a bool return value in a local "int" variable,
> just to then use it as boolean. Is the local var needed at all here?

'int` is used because of ...:
   rc = dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL);
... at the end of imsic_parse_node().

Agree, we can just drop `rc` variable and return err code explicitly.

>
>> +        imsic_cfg.guest_index_bits = 0;
>> +    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT;
>> +    if ( tmp < imsic_cfg.guest_index_bits )
>> +    {
>> +        printk(XENLOG_ERR "%s: guest index bits too big\n", node->name);
>> +        return -ENOENT;
>> +    }
>> +
>> +    /* Find number of HART index bits */
>> +    rc = dt_property_read_u32(node, "riscv,hart-index-bits",
>> +                              &imsic_cfg.hart_index_bits);
>> +    if ( !rc )
>> +    {
>> +        /* Assume default value */
>> +        imsic_cfg.hart_index_bits = fls(*nr_parent_irqs);
>> +        if ( BIT(imsic_cfg.hart_index_bits, UL) < *nr_parent_irqs )
>> +            imsic_cfg.hart_index_bits++;
>> +    }
>> +    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
>> +          imsic_cfg.guest_index_bits;
>      tmp -= imsic_cfg.guest_index_bits;
>
> ? (And then similarly further down.)

Yes, it would be simpler.

>> +    if ( tmp < imsic_cfg.hart_index_bits )
>> +    {
>> +        printk(XENLOG_ERR "%s: HART index bits too big\n", node->name);
>> +        return -ENOENT;
>> +    }
>> +
>> +    /* Find number of group index bits */
>> +    rc = dt_property_read_u32(node, "riscv,group-index-bits",
>> +                              &imsic_cfg.group_index_bits);
>> +    if ( !rc )
>> +        imsic_cfg.group_index_bits = 0;
>> +    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
>> +          imsic_cfg.guest_index_bits - imsic_cfg.hart_index_bits;
>> +    if ( tmp < imsic_cfg.group_index_bits )
>> +    {
>> +        printk(XENLOG_ERR "%s: group index bits too big\n", node->name);
>> +        return -ENOENT;
>> +    }
>> +
>> +    /* Find first bit position of group index */
>> +    tmp = IMSIC_MMIO_PAGE_SHIFT * 2;
>> +    rc = dt_property_read_u32(node, "riscv,group-index-shift",
>> +                              &imsic_cfg.group_index_shift);
>> +    if ( !rc )
>> +        imsic_cfg.group_index_shift = tmp;
>> +    if ( imsic_cfg.group_index_shift < tmp )
>> +    {
>> +        printk(XENLOG_ERR "%s: group index shift too small\n", node->name);
>> +        return -ENOENT;
>> +    }
>> +    tmp = imsic_cfg.group_index_bits + imsic_cfg.group_index_shift - 1;
>> +    if ( tmp >= BITS_PER_LONG )
>> +    {
>> +        printk(XENLOG_ERR "%s: group index shift too big\n", node->name);
>> +        return -EINVAL;
>> +    }
>> +
>> +    /* Find number of interrupt identities */
>> +    rc = dt_property_read_u32(node, "riscv,num-ids", &imsic_cfg.nr_ids);
>> +    if ( !rc )
>> +    {
>> +        printk(XENLOG_ERR "%s: number of interrupt identities not found\n",
>> +               node->name);
>> +        return -ENOENT;
>> +    }
>> +
>> +    if ( (imsic_cfg.nr_ids < IMSIC_MIN_ID) ||
>> +         (imsic_cfg.nr_ids >= IMSIC_MAX_ID) ||
> Something named "max" normally wants to decribe the highest valid value,
> not the first out-of-range one.

But it is the maximum of identities for IMISIC according to the AIA specs:
```
( MRIF stands for Memory-Resident Interrupt File )

All MRIFs are the size to accommodate 2047 valid interrupt identities,
the maximum allowed for an IMSIC interrupt file. If a system’s actual IMSICs
have interrupt files that implement only N interrupt identities, N < 2047,
then the contents of MRIFs for identities greater than N may be ignored by
softwareand
```
and prefix IMSIC (IMO)emphasizes this. Would it be better to rename to IMSIC_MAX_ALLOWED_ID? 
And then IMSIC_MIN_ALLOWED_ID?

>
>> +         ((imsic_cfg.nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) )
>> +    {
>> +        printk(XENLOG_ERR "%s: invalid number of interrupt identities\n",
>> +               node->name);
>> +        return -EINVAL;
>> +    }
>> +
>> +    /* Compute base address */
>> +    imsic_cfg.nr_mmios = 0;
>> +    rc = dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL);
>> +    if (rc)
> Nit: Style.
>
>> +    {
>> +        printk(XENLOG_ERR "%s: first MMIO resource not found\n", node->name);
>> +        return -EINVAL;
> Discarding "rc" again?

Agree, it could be just `rc`, but considering of that this functions always returns
-EINVAL in case of the error and that `rc` could be dropped at all, we just explicitly
return here -EINVAL.

>
>> +    }
>> +
>> +    imsic_cfg.base_addr = base_addr;
>> +    imsic_cfg.base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
>> +                   imsic_cfg.hart_index_bits +
>> +                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
>> +    imsic_cfg.base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
>> +                   imsic_cfg.group_index_shift);
> Besides indentation being bogus here, why is it that you need to mask bits
> off of the value read from DT? Wouldn't the expectation be that you get back
> the true base address?

The group index is used to differentiate between clusters/groups. For 
example, consider two clusters: - Cluster 1 with cpu0 and cpu1 - Cluster 
2 with cpu2 and cpu3 Then, the reg property in the IMSIC node will 
include two entries: reg = <0x0 0xd100000 0x0 0x20000>, <0x0 0x2d100000 
0x0 0x20000>; riscv,guest-index-bits = <3>; riscv,hart-index-bits = <2>; 
riscv,group-index-bits = <1>; riscv,group-index-shift = <29>; In this 
example: The group index is 1 bit wide (group-index-bits = <1>), It is 
located at bit 29 (group-index-shift = <29>) of the address.

so imsic_cfg.group_index_bits will be used to distinguish clusters, but 
they must have

the same base address and this is the reason why the mask is applied.



>
>> +    /* Find number of MMIO register sets */
>> +    imsic_cfg.nr_mmios++;
>> +    while ( !dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL) )
>> +        imsic_cfg.nr_mmios++;
> And the base addresses of these aren't of interest? Oh, I see they're
> fetched again further down.

It is fetched again because we are using imsic_cfg.nr_mmios to calculate the size of
imsic_cfg.mmios array:
   imsic_cfg.mmios = xzalloc_array(struct imsic_mmios, imsic_cfg.nr_mmios);

>
> Also - use do-while here?

It could be, I'll update that.

>> +        if ( base_addr != imsic_cfg.base_addr )
>> +        {
>> +            rc = -EINVAL;
>> +            printk(XENLOG_ERR "%s: address mismatch for regset %d\n",
>> +                   node->name, i);
>> +            goto imsic_init_err;
>> +        }
> Oh, all of the addresses need to (sufficiently) match.
>
>> +    xfree(imsic_cfg.mmios);
> Better use XFREE() in cases like this one?

I think, yes.

>
>> --- /dev/null
>> +++ b/xen/arch/riscv/include/asm/imsic.h
>> @@ -0,0 +1,66 @@
>> +/* SPDX-License-Identifier: MIT */
>> +
>> +/*
>> + * xen/arch/riscv/imsic.h
>> + *
>> + * RISC-V Incoming MSI Controller support
>> + *
>> + * (c) 2023 Microchip Technology Inc.
>> + */
>> +
>> +#ifndef ASM__RISCV__IMSIC_H
>> +#define ASM__RISCV__IMSIC_H
>> +
>> +#include <xen/types.h>
>> +
>> +#define IMSIC_MMIO_PAGE_SHIFT   12
>> +#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
>> +
>> +#define IMSIC_MIN_ID            63
>> +#define IMSIC_MAX_ID            2048
>> +
>> +struct imsic_msi {
>> +    paddr_t base_addr;
>> +    unsigned long offset;
>> +};
>> +
>> +struct imsic_mmios {
>> +    paddr_t base_addr;
>> +    unsigned long size;
>> +    bool harts[NR_CPUS];
> An array of bool - won't a bitmap do here? Even then I wouldn't be overly
> happy to see it dimensioned by NR_CPUS.

Bitmap will fit here well. But for DECLARE_BITMAP() is necessary the size
of bitmap so NR_CPUS should be used again.
Could you please remind me why it isn't good to use it?
Because NR_CPUS not always equal to an amount of physical cpus?
Should I use non-static version of bitmap declaration? (if we have such...)

>
>> +};
>> +
>> +struct imsic_config {
>> +    /* base address */
>> +    paddr_t base_addr;
>> +
>> +    /* Bits representing Guest index, HART index, and Group index */
>> +    unsigned int guest_index_bits;
>> +    unsigned int hart_index_bits;
>> +    unsigned int group_index_bits;
>> +    unsigned int group_index_shift;
>> +
>> +    /* imsic phandle */
>> +    unsigned int phandle;
>> +
>> +    /* number of parent irq */
>> +    unsigned int nr_parent_irqs;
>> +
>> +    /* number off interrupt identities */
>> +    unsigned int nr_ids;
>> +
>> +    /* mmios */
>> +    unsigned int nr_mmios;
>> +    struct imsic_mmios *mmios;
>> +
>> +    /* MSI */
>> +    struct imsic_msi msi[NR_CPUS];
> You surely can avoid wasting perhaps a lot of memory by allocating this
> based on the number of CPUs in use?

It make sense. I'll allocate then this dynamically.

Thanks!

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 16549 bytes --]

Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 5090 bytes --]


On 4/16/25 8:31 AM, Jan Beulich wrote:
>>>> +    }
>>>> +
>>>> +    imsic_cfg.base_addr = base_addr;
>>>> +    imsic_cfg.base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
>>>> +                   imsic_cfg.hart_index_bits +
>>>> +                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
>>>> +    imsic_cfg.base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
>>>> +                   imsic_cfg.group_index_shift);
>>> Besides indentation being bogus here, why is it that you need to mask bits
>>> off of the value read from DT? Wouldn't the expectation be that you get back
>>> the true base address?
>> The group index is used to differentiate between clusters/groups. For
>> example, consider two clusters: - Cluster 1 with cpu0 and cpu1 - Cluster
>> 2 with cpu2 and cpu3 Then, the reg property in the IMSIC node will
>> include two entries: reg = <0x0 0xd100000 0x0 0x20000>, <0x0 0x2d100000
>> 0x0 0x20000>; riscv,guest-index-bits = <3>; riscv,hart-index-bits = <2>;
>> riscv,group-index-bits = <1>; riscv,group-index-shift = <29>; In this
>> example: The group index is 1 bit wide (group-index-bits = <1>), It is
>> located at bit 29 (group-index-shift = <29>) of the address.
>>
>> so imsic_cfg.group_index_bits will be used to distinguish clusters, but
>> they must have
>>
>> the same base address and this is the reason why the mask is applied.
> Well, yes, but that doesn't answer my question. All of what you say makes
> sense for the loop elsewhere retrieving all the addresses. Here you
> retrieved only the first of them, and in your example no masking would be
> needed here either. To re-phrase my question: Can the address observed in
> the first entry be other than the lowest one across the set of all
> entries?

It doesn't mentioned explicitly in riscv,imsiscs binding that reg property
is sorted.
I can write:
   reg = <0x0 0x2d100000 0x0 0x20000>,<0x0 0xd100000 0x0 0x20000>;
And a dtc compiler will compile.

> If not, can there be bits set across all entries that need
> masking off from all of them?

I think that other bits (Hart Index and Guest Index) excepts base address
bits are 0 in dts, but it doesn't guarantee explicitly by IMSIC's dts bindinding.

>
>>>> --- /dev/null
>>>> +++ b/xen/arch/riscv/include/asm/imsic.h
>>>> @@ -0,0 +1,66 @@
>>>> +/* SPDX-License-Identifier: MIT */
>>>> +
>>>> +/*
>>>> + * xen/arch/riscv/imsic.h
>>>> + *
>>>> + * RISC-V Incoming MSI Controller support
>>>> + *
>>>> + * (c) 2023 Microchip Technology Inc.
>>>> + */
>>>> +
>>>> +#ifndef ASM__RISCV__IMSIC_H
>>>> +#define ASM__RISCV__IMSIC_H
>>>> +
>>>> +#include <xen/types.h>
>>>> +
>>>> +#define IMSIC_MMIO_PAGE_SHIFT   12
>>>> +#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
>>>> +
>>>> +#define IMSIC_MIN_ID            63
>>>> +#define IMSIC_MAX_ID            2048
>>>> +
>>>> +struct imsic_msi {
>>>> +    paddr_t base_addr;
>>>> +    unsigned long offset;
>>>> +};
>>>> +
>>>> +struct imsic_mmios {
>>>> +    paddr_t base_addr;
>>>> +    unsigned long size;
>>>> +    bool harts[NR_CPUS];
>>> An array of bool - won't a bitmap do here? Even then I wouldn't be overly
>>> happy to see it dimensioned by NR_CPUS.
>> Bitmap will fit here well. But for DECLARE_BITMAP() is necessary the size
>> of bitmap so NR_CPUS should be used again.
>> Could you please remind me why it isn't good to use it?
>> Because NR_CPUS not always equal to an amount of physical cpus?
> "Not equal" wouldn't be overly problematic. But NR_CPUS=4000 and the actual
> number of CPUs being 4 would be wasteful in general. More when its wider
> than a bit that's needed per CPU, but where would you draw the line if you
> permitted use of NR_CPUS here?

Hard to say. It seems it will be just better to use apporoach you suggested below.

Thanks.

~ Oleksii

>
>> Should I use non-static version of bitmap declaration? (if we have such...)
> That's simply "unsigned long *" then, or - at the tail of a dynamically
> allocated struct - possibly unsigned long[].
>
>>>> +};
>>>> +
>>>> +struct imsic_config {
>>>> +    /* base address */
>>>> +    paddr_t base_addr;
>>>> +
>>>> +    /* Bits representing Guest index, HART index, and Group index */
>>>> +    unsigned int guest_index_bits;
>>>> +    unsigned int hart_index_bits;
>>>> +    unsigned int group_index_bits;
>>>> +    unsigned int group_index_shift;
>>>> +
>>>> +    /* imsic phandle */
>>>> +    unsigned int phandle;
>>>> +
>>>> +    /* number of parent irq */
>>>> +    unsigned int nr_parent_irqs;
>>>> +
>>>> +    /* number off interrupt identities */
>>>> +    unsigned int nr_ids;
>>>> +
>>>> +    /* mmios */
>>>> +    unsigned int nr_mmios;
>>>> +    struct imsic_mmios *mmios;
>>>> +
>>>> +    /* MSI */
>>>> +    struct imsic_msi msi[NR_CPUS];
>>> You surely can avoid wasting perhaps a lot of memory by allocating this
>>> based on the number of CPUs in use?
>> It make sense. I'll allocate then this dynamically.
> Or, as per above, when put at the tail and the struct itself is
> dynamically allocated, use struct imsic_msi[]. We even have dedicated
> xmalloc() flavors for this kind of allocation.
>
> Jan

[-- Attachment #2: Type: text/html, Size: 6813 bytes --]

Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 3542 bytes --]


On 4/16/25 8:31 AM, Jan Beulich wrote:
>>>> --- /dev/null
>>>> +++ b/xen/arch/riscv/include/asm/imsic.h
>>>> @@ -0,0 +1,66 @@
>>>> +/* SPDX-License-Identifier: MIT */
>>>> +
>>>> +/*
>>>> + * xen/arch/riscv/imsic.h
>>>> + *
>>>> + * RISC-V Incoming MSI Controller support
>>>> + *
>>>> + * (c) 2023 Microchip Technology Inc.
>>>> + */
>>>> +
>>>> +#ifndef ASM__RISCV__IMSIC_H
>>>> +#define ASM__RISCV__IMSIC_H
>>>> +
>>>> +#include <xen/types.h>
>>>> +
>>>> +#define IMSIC_MMIO_PAGE_SHIFT   12
>>>> +#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
>>>> +
>>>> +#define IMSIC_MIN_ID            63
>>>> +#define IMSIC_MAX_ID            2048
>>>> +
>>>> +struct imsic_msi {
>>>> +    paddr_t base_addr;
>>>> +    unsigned long offset;
>>>> +};
>>>> +
>>>> +struct imsic_mmios {
>>>> +    paddr_t base_addr;
>>>> +    unsigned long size;
>>>> +    bool harts[NR_CPUS];
>>> An array of bool - won't a bitmap do here? Even then I wouldn't be overly
>>> happy to see it dimensioned by NR_CPUS.
>> Bitmap will fit here well. But for DECLARE_BITMAP() is necessary the size
>> of bitmap so NR_CPUS should be used again.
>> Could you please remind me why it isn't good to use it?
>> Because NR_CPUS not always equal to an amount of physical cpus?
> "Not equal" wouldn't be overly problematic. But NR_CPUS=4000 and the actual
> number of CPUs being 4 would be wasteful in general. More when its wider
> than a bit that's needed per CPU, but where would you draw the line if you
> permitted use of NR_CPUS here?
>
>> Should I use non-static version of bitmap declaration? (if we have such...)
> That's simply "unsigned long *" then, or - at the tail of a dynamically
> allocated struct - possibly unsigned long[].
>
>>>> +};
>>>> +
>>>> +struct imsic_config {
>>>> +    /* base address */
>>>> +    paddr_t base_addr;
>>>> +
>>>> +    /* Bits representing Guest index, HART index, and Group index */
>>>> +    unsigned int guest_index_bits;
>>>> +    unsigned int hart_index_bits;
>>>> +    unsigned int group_index_bits;
>>>> +    unsigned int group_index_shift;
>>>> +
>>>> +    /* imsic phandle */
>>>> +    unsigned int phandle;
>>>> +
>>>> +    /* number of parent irq */
>>>> +    unsigned int nr_parent_irqs;
>>>> +
>>>> +    /* number off interrupt identities */
>>>> +    unsigned int nr_ids;
>>>> +
>>>> +    /* mmios */
>>>> +    unsigned int nr_mmios;
>>>> +    struct imsic_mmios *mmios;
>>>> +
>>>> +    /* MSI */
>>>> +    struct imsic_msi msi[NR_CPUS];
>>> You surely can avoid wasting perhaps a lot of memory by allocating this
>>> based on the number of CPUs in use?
>> It make sense. I'll allocate then this dynamically.
> Or, as per above, when put at the tail and the struct itself is
> dynamically allocated, use struct imsic_msi[]. We even have dedicated
> xmalloc() flavors for this kind of allocation.

Do you mean xzalloc_flex_struct()?

I think, I can't use for both of the cases (allocation of mmios and msi).
For msi[] then it is needed to allocate imsic_config also dynamically, isn't it?
So something like:
  imsic_config = xzalloc_flex_struct(struct imsic_config, msi, NR_CPUS).
But now it is allocated statically.

For *mmios and harts[] (a member inside struct imsic_mmios):
   mmios = xzalloc_flex_struct(struct imsic_mmios, harts, NR_CPUS); // NR_CPUs just for example...
It will allocate only one mmios, but it is needed mmios[nr_mmios].
Maybe, something like _xmalloc((offsetof(struct imsic_mmios, harts[NR_CPUS])) * NR_CPUS, sizeof(struct imsic_mmios)) will work.

Am I missing something?

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 4472 bytes --]

[PATCH v1 09/14] xen/riscv: aplic_init() implementation

[Oleksii Kurochko]

aplic_init() function does the following few things:
 - checks that IMSIC in device tree node  ( by checking msi-parent property
   in APLIC node ) is present as current one implmenetaion of AIA is
   supported only MSI method.
 - initialize IMSIC based on IMSIC device tree node
 - Read value of APLIC's paddr start/end and size.
 - Map aplic.regs
 - Setup APLIC initial state interrupts (disable all interrupts, set
   interrupt type and default priority, confgifure APLIC domaincfg) by
   calling aplic_init_hw_interrutps().

aplic_init() is based on the code from [1] and [2].

Since Microchip originally developed aplic.c, an internal discussion with
them led to the decision to use the MIT license.

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7cfb4bd4748ca268142497ac5c327d2766fb342d
[2] https://gitlab.com/xen-project/people/olkur/xen/-/commit/392a531bfad39bf4656ce8128e004b241b8b3f3e

Co-developed-by: Romain Caritey <Romain.Caritey@microchip.com>
Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/aplic.c             | 97 ++++++++++++++++++++++++++++++
 xen/arch/riscv/include/asm/aplic.h | 77 ++++++++++++++++++++++++
 xen/arch/riscv/include/asm/intc.h  |  3 +
 xen/arch/riscv/include/asm/irq.h   |  1 -
 4 files changed, 177 insertions(+), 1 deletion(-)
 create mode 100644 xen/arch/riscv/include/asm/aplic.h

diff --git a/xen/arch/riscv/aplic.c b/xen/arch/riscv/aplic.c
index 6dc040af6f..d1aa835c3e 100644
--- a/xen/arch/riscv/aplic.c
+++ b/xen/arch/riscv/aplic.c
@@ -9,19 +9,112 @@
  * Copyright (c) 2024-2025 Vates
  */
 
+#include <xen/device_tree.h>
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/irq.h>
+#include <xen/mm.h>
 #include <xen/sections.h>
 #include <xen/types.h>
+#include <xen/vmap.h>
 
+#include <asm/aplic.h>
 #include <asm/device.h>
+#include <asm/imsic.h>
 #include <asm/intc.h>
+#include <asm/riscv_encoding.h>
+
+#define APLIC_DEFAULT_PRIORITY  1
+
+static struct aplic_priv aplic;
 
 static struct intc_info __ro_after_init aplic_info = {
     .hw_version = INTC_APLIC,
 };
 
+static void __init aplic_init_hw_interrupts(void)
+{
+    int i;
+
+    /* Disable all interrupts */
+    for ( i = 0; i <= aplic_info.nr_irqs; i += 32 )
+        aplic.regs->clrie[i] = -1U;
+
+    /* Set interrupt type and default priority for all interrupts */
+    for ( i = 1; i <= aplic_info.nr_irqs; i++ )
+    {
+        aplic.regs->sourcecfg[i - 1] = 0;
+        aplic.regs->target[i - 1] = APLIC_DEFAULT_PRIORITY;
+    }
+
+    /* Clear APLIC domaincfg */
+    aplic.regs->domaincfg = APLIC_DOMAINCFG_IE | APLIC_DOMAINCFG_DM;
+}
+
+static int __init aplic_init(void)
+{
+    int rc;
+    dt_phandle imsic_phandle;
+    uint32_t irq_range[2];
+    const __be32 *prop;
+    uint64_t size, paddr;
+    struct dt_device_node *imsic_node;
+    const struct dt_device_node *node = aplic_info.node;
+
+    /* check for associated imsic node */
+    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
+    if ( !rc )
+        panic("%s: IDC mode not supported\n", node->full_name);
+
+    imsic_node = dt_find_node_by_phandle(imsic_phandle);
+    if ( !imsic_node )
+        panic("%s: unable to find IMSIC node\n", node->full_name);
+
+    /* check imsic mode */
+    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
+                                    irq_range, ARRAY_SIZE(irq_range));
+    if ( rc && (rc != -EOVERFLOW) )
+        panic("%s: unable to find interrupt-extended in %s node\n",
+               node->full_name, imsic_node->full_name);
+
+    if ( irq_range[1] == IRQ_M_EXT )
+        /* machine mode imsic node, ignore this aplic node */
+        return 0;
+
+    rc = imsic_init(imsic_node);
+    if ( rc )
+        panic("%s: Failded to initialize IMSIC\n", node->full_name);
+
+    /* Find out number of interrupt sources */
+    rc = dt_property_read_u32(node, "riscv,num-sources", &aplic_info.nr_irqs);
+    if ( !rc )
+        panic("%s: failed to get number of interrupt sources\n",
+              node->full_name);
+
+    prop = dt_get_property(node, "reg", NULL);
+    dt_get_range(&prop, node, &paddr, &size);
+    if ( !paddr )
+        panic("%s: first MMIO resource not found\n", node->full_name);
+
+    aplic.paddr_start = paddr;
+    aplic.paddr_end = paddr + size;
+    aplic.size = size;
+
+    aplic.regs = ioremap(paddr, size);
+    if ( !aplic.regs )
+        panic("%s: unable to map\n", node->full_name);
+
+    /* Setup initial state APLIC interrupts */
+    aplic_init_hw_interrupts();
+
+    return 0;
+}
+
+static const struct intc_hw_operations __ro_after_init aplic_ops = {
+    .info                = &aplic_info,
+    .init                = aplic_init,
+};
+
 static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
                            unsigned int *out_hwirq,
                            unsigned int *out_type)
@@ -52,8 +145,12 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     aplic_info.node = node;
 
+    aplic.imsic_cfg = imsic_get_config();
+
     dt_irq_xlate = aplic_irq_xlate;
 
+    register_intc_ops(&aplic_ops);
+
     return 0;
 }
 
diff --git a/xen/arch/riscv/include/asm/aplic.h b/xen/arch/riscv/include/asm/aplic.h
new file mode 100644
index 0000000000..94b3d0b616
--- /dev/null
+++ b/xen/arch/riscv/include/asm/aplic.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/aplic.h
+ *
+ * RISC-V Advanced Platform-Level Interrupt Controller support
+ *
+ * Copyright (c) 2023 Microchip.
+ */
+
+#ifndef ASM__RISCV__APLIC_H
+#define ASM__RISCV__APLIC_H
+
+#include <xen/types.h>
+
+#include <asm/imsic.h>
+
+#define APLIC_DOMAINCFG_IE      BIT(8, UL)
+#define APLIC_DOMAINCFG_DM      BIT(2, UL)
+
+struct aplic_regs {
+    uint32_t domaincfg;
+    uint32_t sourcecfg[1023];
+    uint8_t _reserved1[0xBC0];
+
+    uint32_t mmsiaddrcfg;
+    uint32_t mmsiaddrcfgh;
+    uint32_t smsiaddrcfg;
+    uint32_t smsiaddrcfgh;
+    uint8_t _reserved2[0x30];
+
+    uint32_t setip[32];
+    uint8_t _reserved3[92];
+
+    uint32_t setipnum;
+    uint8_t _reserved4[0x20];
+
+    uint32_t in_clrip[32];
+    uint8_t _reserved5[92];
+
+    uint32_t clripnum;
+    uint8_t _reserved6[32];
+
+    uint32_t setie[32];
+    uint8_t _reserved7[92];
+
+    uint32_t setienum;
+    uint8_t _reserved8[32];
+
+    uint32_t clrie[32];
+    uint8_t _reserved9[92];
+
+    uint32_t clrienum;
+    uint8_t _reserved10[32];
+
+    uint32_t setipnum_le;
+    uint32_t setipnum_be;
+    uint8_t _reserved11[4088];
+
+    uint32_t genmsi;
+    uint32_t target[1023];
+};
+
+struct aplic_priv {
+    /* base physical address and size */
+    paddr_t paddr_start;
+    paddr_t paddr_end;
+    size_t  size;
+
+    /* registers */
+    volatile struct aplic_regs *regs;
+
+    /* imsic configuration */
+    const struct imsic_config *imsic_cfg;
+};
+
+#endif /* ASM__RISCV__APLIC_H */
diff --git a/xen/arch/riscv/include/asm/intc.h b/xen/arch/riscv/include/asm/intc.h
index 0d498b10f4..db53caa07b 100644
--- a/xen/arch/riscv/include/asm/intc.h
+++ b/xen/arch/riscv/include/asm/intc.h
@@ -15,6 +15,9 @@ enum intc_version {
 struct intc_info {
     enum intc_version hw_version;
     const struct dt_device_node *node;
+
+    /* number of irqs */
+    unsigned int nr_irqs;
 };
 
 struct intc_hw_operations {
diff --git a/xen/arch/riscv/include/asm/irq.h b/xen/arch/riscv/include/asm/irq.h
index ff1c95e0be..163a478d78 100644
--- a/xen/arch/riscv/include/asm/irq.h
+++ b/xen/arch/riscv/include/asm/irq.h
@@ -27,7 +27,6 @@
 #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
 /* TODO */
-#define nr_irqs 0U
 #define nr_static_irqs 0
 #define arch_hwdom_irqs(domid) 0U
 
-- 
2.49.0

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> --- a/xen/arch/riscv/aplic.c
> +++ b/xen/arch/riscv/aplic.c
> @@ -9,19 +9,112 @@
>   * Copyright (c) 2024-2025 Vates
>   */
>  
> +#include <xen/device_tree.h>
>  #include <xen/errno.h>
>  #include <xen/init.h>
>  #include <xen/irq.h>
> +#include <xen/mm.h>
>  #include <xen/sections.h>
>  #include <xen/types.h>
> +#include <xen/vmap.h>
>  
> +#include <asm/aplic.h>
>  #include <asm/device.h>
> +#include <asm/imsic.h>
>  #include <asm/intc.h>
> +#include <asm/riscv_encoding.h>
> +
> +#define APLIC_DEFAULT_PRIORITY  1
> +
> +static struct aplic_priv aplic;
>  
>  static struct intc_info __ro_after_init aplic_info = {
>      .hw_version = INTC_APLIC,
>  };
>  
> +static void __init aplic_init_hw_interrupts(void)
> +{
> +    int i;
> +
> +    /* Disable all interrupts */
> +    for ( i = 0; i <= aplic_info.nr_irqs; i += 32 )
> +        aplic.regs->clrie[i] = -1U;
> +
> +    /* Set interrupt type and default priority for all interrupts */
> +    for ( i = 1; i <= aplic_info.nr_irqs; i++ )
> +    {
> +        aplic.regs->sourcecfg[i - 1] = 0;
> +        aplic.regs->target[i - 1] = APLIC_DEFAULT_PRIORITY;

A field named "target" is written with a priority value?

> +    }
> +
> +    /* Clear APLIC domaincfg */
> +    aplic.regs->domaincfg = APLIC_DOMAINCFG_IE | APLIC_DOMAINCFG_DM;

The statement doesn't like like there was any "clearing" here.

> +}
> +
> +static int __init aplic_init(void)
> +{
> +    int rc;
> +    dt_phandle imsic_phandle;
> +    uint32_t irq_range[2];
> +    const __be32 *prop;
> +    uint64_t size, paddr;
> +    struct dt_device_node *imsic_node;
> +    const struct dt_device_node *node = aplic_info.node;
> +
> +    /* check for associated imsic node */

Nit: Comment style (also elsewhere).

> +    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
> +    if ( !rc )
> +        panic("%s: IDC mode not supported\n", node->full_name);
> +
> +    imsic_node = dt_find_node_by_phandle(imsic_phandle);
> +    if ( !imsic_node )
> +        panic("%s: unable to find IMSIC node\n", node->full_name);
> +
> +    /* check imsic mode */
> +    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
> +                                    irq_range, ARRAY_SIZE(irq_range));
> +    if ( rc && (rc != -EOVERFLOW) )
> +        panic("%s: unable to find interrupt-extended in %s node\n",
> +               node->full_name, imsic_node->full_name);

Why exactly is EOVERFLOW tolerable here?

> +    if ( irq_range[1] == IRQ_M_EXT )
> +        /* machine mode imsic node, ignore this aplic node */
> +        return 0;
> +
> +    rc = imsic_init(imsic_node);
> +    if ( rc )
> +        panic("%s: Failded to initialize IMSIC\n", node->full_name);
> +
> +    /* Find out number of interrupt sources */
> +    rc = dt_property_read_u32(node, "riscv,num-sources", &aplic_info.nr_irqs);
> +    if ( !rc )
> +        panic("%s: failed to get number of interrupt sources\n",
> +              node->full_name);
> +
> +    prop = dt_get_property(node, "reg", NULL);
> +    dt_get_range(&prop, node, &paddr, &size);
> +    if ( !paddr )
> +        panic("%s: first MMIO resource not found\n", node->full_name);
> +
> +    aplic.paddr_start = paddr;
> +    aplic.paddr_end = paddr + size;
> +    aplic.size = size;

Why do all three need recording? Isn't a (start,size) tuple sufficient
(and unambiguous)?

> +    aplic.regs = ioremap(paddr, size);
> +    if ( !aplic.regs )
> +        panic("%s: unable to map\n", node->full_name);
> +
> +    /* Setup initial state APLIC interrupts */
> +    aplic_init_hw_interrupts();
> +
> +    return 0;
> +}
> +
> +static const struct intc_hw_operations __ro_after_init aplic_ops = {

const or __ro_after_init?

> --- /dev/null
> +++ b/xen/arch/riscv/include/asm/aplic.h
> @@ -0,0 +1,77 @@
> +/* SPDX-License-Identifier: MIT */
> +
> +/*
> + * xen/arch/riscv/aplic.h
> + *
> + * RISC-V Advanced Platform-Level Interrupt Controller support
> + *
> + * Copyright (c) 2023 Microchip.
> + */
> +
> +#ifndef ASM__RISCV__APLIC_H
> +#define ASM__RISCV__APLIC_H
> +
> +#include <xen/types.h>
> +
> +#include <asm/imsic.h>
> +
> +#define APLIC_DOMAINCFG_IE      BIT(8, UL)
> +#define APLIC_DOMAINCFG_DM      BIT(2, UL)
> +
> +struct aplic_regs {
> +    uint32_t domaincfg;
> +    uint32_t sourcecfg[1023];
> +    uint8_t _reserved1[0xBC0];
> +
> +    uint32_t mmsiaddrcfg;
> +    uint32_t mmsiaddrcfgh;
> +    uint32_t smsiaddrcfg;
> +    uint32_t smsiaddrcfgh;
> +    uint8_t _reserved2[0x30];
> +
> +    uint32_t setip[32];
> +    uint8_t _reserved3[92];
> +
> +    uint32_t setipnum;
> +    uint8_t _reserved4[0x20];
> +
> +    uint32_t in_clrip[32];
> +    uint8_t _reserved5[92];
> +
> +    uint32_t clripnum;
> +    uint8_t _reserved6[32];
> +
> +    uint32_t setie[32];
> +    uint8_t _reserved7[92];
> +
> +    uint32_t setienum;
> +    uint8_t _reserved8[32];
> +
> +    uint32_t clrie[32];
> +    uint8_t _reserved9[92];
> +
> +    uint32_t clrienum;
> +    uint8_t _reserved10[32];
> +
> +    uint32_t setipnum_le;
> +    uint32_t setipnum_be;
> +    uint8_t _reserved11[4088];
> +
> +    uint32_t genmsi;
> +    uint32_t target[1023];
> +};
> +
> +struct aplic_priv {
> +    /* base physical address and size */
> +    paddr_t paddr_start;
> +    paddr_t paddr_end;
> +    size_t  size;
> +
> +    /* registers */
> +    volatile struct aplic_regs *regs;
> +
> +    /* imsic configuration */
> +    const struct imsic_config *imsic_cfg;
> +};
> +
> +#endif /* ASM__RISCV__APLIC_H */

Does all of this really need to live in a non-private header?

> --- a/xen/arch/riscv/include/asm/irq.h
> +++ b/xen/arch/riscv/include/asm/irq.h
> @@ -27,7 +27,6 @@
>  #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
>  
>  /* TODO */
> -#define nr_irqs 0U

How come this is simply no longer needed, i.e. without any replacement?
Hmm, looks like the only use in common code has gone away. Yet then this
still doesn't really look to belong here (especially if not mentioned in
the description).

Jan

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 9080 bytes --]


On 4/14/25 12:04 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> --- a/xen/arch/riscv/aplic.c
>> +++ b/xen/arch/riscv/aplic.c
>> @@ -9,19 +9,112 @@
>>    * Copyright (c) 2024-2025 Vates
>>    */
>>   
>> +#include <xen/device_tree.h>
>>   #include <xen/errno.h>
>>   #include <xen/init.h>
>>   #include <xen/irq.h>
>> +#include <xen/mm.h>
>>   #include <xen/sections.h>
>>   #include <xen/types.h>
>> +#include <xen/vmap.h>
>>   
>> +#include <asm/aplic.h>
>>   #include <asm/device.h>
>> +#include <asm/imsic.h>
>>   #include <asm/intc.h>
>> +#include <asm/riscv_encoding.h>
>> +
>> +#define APLIC_DEFAULT_PRIORITY  1
>> +
>> +static struct aplic_priv aplic;
>>   
>>   static struct intc_info __ro_after_init aplic_info = {
>>       .hw_version = INTC_APLIC,
>>   };
>>   
>> +static void __init aplic_init_hw_interrupts(void)
>> +{
>> +    int i;
>> +
>> +    /* Disable all interrupts */
>> +    for ( i = 0; i <= aplic_info.nr_irqs; i += 32 )
>> +        aplic.regs->clrie[i] = -1U;
>> +
>> +    /* Set interrupt type and default priority for all interrupts */
>> +    for ( i = 1; i <= aplic_info.nr_irqs; i++ )
>> +    {
>> +        aplic.regs->sourcecfg[i - 1] = 0;
>> +        aplic.regs->target[i - 1] = APLIC_DEFAULT_PRIORITY;
> A field named "target" is written with a priority value?

Low bits of target register contains Interrupt Priority bits which can't be zero according to
AIA spec:
```
4.5.16.1. Active source, direct delivery mode
For an active interrupt source , if the domain is configured in direct delivery mode (domaincfg.DM = 0),
then register target[ ] has this format:
   bits 31:18 Hart Index (WLRL)
   bits 7:0 IPRIO (WARL)

All other register bits are reserved and read as zeros.
Hart Index is a WLRL field that specifies the hart to which interrupts from this source will be delivered.
Field IPRIO (Interrupt Priority) specifies the priority number for the interrupt source. This field is
a WARL unsigned integer of IPRIOLEN bits, where IPRIOLEN is a constant parameter for the given APLIC, in
the range of 1 to 8. Only values 1 through are allowed for IPRIO, not zero. A write to a target register
sets IPRIO equal to bits :0 of the 32-bit value written, unless those bits are all zeros, in which case
the priority number is set to 1 instead. (If IPRIOLEN = 1, these rules cause IPRIO to be effectively
read-only with value 1.)
```

>
>> +    }
>> +
>> +    /* Clear APLIC domaincfg */
>> +    aplic.regs->domaincfg = APLIC_DOMAINCFG_IE | APLIC_DOMAINCFG_DM;
> The statement doesn't like like there was any "clearing" here.

But all other bits, except|APLIC_DOMAINCFG_{IE, DM}|, are set to zero.
I think we can remove this comment to avoid confusion.

>> +    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
>> +    if ( !rc )
>> +        panic("%s: IDC mode not supported\n", node->full_name);
>> +
>> +    imsic_node = dt_find_node_by_phandle(imsic_phandle);
>> +    if ( !imsic_node )
>> +        panic("%s: unable to find IMSIC node\n", node->full_name);
>> +
>> +    /* check imsic mode */
>> +    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
>> +                                    irq_range, ARRAY_SIZE(irq_range));
>> +    if ( rc && (rc != -EOVERFLOW) )
>> +        panic("%s: unable to find interrupt-extended in %s node\n",
>> +               node->full_name, imsic_node->full_name);
> Why exactly is EOVERFLOW tolerable here?

QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode.
For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only
interested in the S-mode IMSIC node.

The IMSIC node includes this information in the|"interrupts-extended"| property,
which has the following format:
   interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.

For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:

   if ( irq_range[1] == IRQ_M_EXT )

Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom
has more then one CPU as we passed irq_range[2] as an argument but the amount of values
in "interrupts-extended" property will be (2 * CPUS_NUM).

I can update the comment above dt_property_read_u32_array() for more clearness.

>> +    if ( irq_range[1] == IRQ_M_EXT )
>> +        /* machine mode imsic node, ignore this aplic node */
>> +        return 0;
>> +
>> +    rc = imsic_init(imsic_node);
>> +    if ( rc )
>> +        panic("%s: Failded to initialize IMSIC\n", node->full_name);
>> +
>> +    /* Find out number of interrupt sources */
>> +    rc = dt_property_read_u32(node, "riscv,num-sources", &aplic_info.nr_irqs);
>> +    if ( !rc )
>> +        panic("%s: failed to get number of interrupt sources\n",
>> +              node->full_name);
>> +
>> +    prop = dt_get_property(node, "reg", NULL);
>> +    dt_get_range(&prop, node, &paddr, &size);
>> +    if ( !paddr )
>> +        panic("%s: first MMIO resource not found\n", node->full_name);
>> +
>> +    aplic.paddr_start = paddr;
>> +    aplic.paddr_end = paddr + size;
>> +    aplic.size = size;
> Why do all three need recording? Isn't a (start,size) tuple sufficient
> (and unambiguous)?

(start,size) will be enough. I'll drop aplic.paddr_end.

>
>> +    aplic.regs = ioremap(paddr, size);
>> +    if ( !aplic.regs )
>> +        panic("%s: unable to map\n", node->full_name);
>> +
>> +    /* Setup initial state APLIC interrupts */
>> +    aplic_init_hw_interrupts();
>> +
>> +    return 0;
>> +}
>> +
>> +static const struct intc_hw_operations __ro_after_init aplic_ops = {
> const or __ro_after_init?

What’s wrong with using both?|const| ensures the variable can't be changed at compile time,
while|__ro_after_init| makes it read-only at runtime after initialization is complete.

Probably,|__initconst| would be a better fit:
   static const struct intc_hw_operations __initconst aplic_ops = {

Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.

>
>> --- /dev/null
>> +++ b/xen/arch/riscv/include/asm/aplic.h
>> @@ -0,0 +1,77 @@
>> +/* SPDX-License-Identifier: MIT */
>> +
>> +/*
>> + * xen/arch/riscv/aplic.h
>> + *
>> + * RISC-V Advanced Platform-Level Interrupt Controller support
>> + *
>> + * Copyright (c) 2023 Microchip.
>> + */
>> +
>> +#ifndef ASM__RISCV__APLIC_H
>> +#define ASM__RISCV__APLIC_H
>> +
>> +#include <xen/types.h>
>> +
>> +#include <asm/imsic.h>
>> +
>> +#define APLIC_DOMAINCFG_IE      BIT(8, UL)
>> +#define APLIC_DOMAINCFG_DM      BIT(2, UL)
>> +
>> +struct aplic_regs {
>> +    uint32_t domaincfg;
>> +    uint32_t sourcecfg[1023];
>> +    uint8_t _reserved1[0xBC0];
>> +
>> +    uint32_t mmsiaddrcfg;
>> +    uint32_t mmsiaddrcfgh;
>> +    uint32_t smsiaddrcfg;
>> +    uint32_t smsiaddrcfgh;
>> +    uint8_t _reserved2[0x30];
>> +
>> +    uint32_t setip[32];
>> +    uint8_t _reserved3[92];
>> +
>> +    uint32_t setipnum;
>> +    uint8_t _reserved4[0x20];
>> +
>> +    uint32_t in_clrip[32];
>> +    uint8_t _reserved5[92];
>> +
>> +    uint32_t clripnum;
>> +    uint8_t _reserved6[32];
>> +
>> +    uint32_t setie[32];
>> +    uint8_t _reserved7[92];
>> +
>> +    uint32_t setienum;
>> +    uint8_t _reserved8[32];
>> +
>> +    uint32_t clrie[32];
>> +    uint8_t _reserved9[92];
>> +
>> +    uint32_t clrienum;
>> +    uint8_t _reserved10[32];
>> +
>> +    uint32_t setipnum_le;
>> +    uint32_t setipnum_be;
>> +    uint8_t _reserved11[4088];
>> +
>> +    uint32_t genmsi;
>> +    uint32_t target[1023];
>> +};
>> +
>> +struct aplic_priv {
>> +    /* base physical address and size */
>> +    paddr_t paddr_start;
>> +    paddr_t paddr_end;
>> +    size_t  size;
>> +
>> +    /* registers */
>> +    volatile struct aplic_regs *regs;
>> +
>> +    /* imsic configuration */
>> +    const struct imsic_config *imsic_cfg;
>> +};
>> +
>> +#endif /* ASM__RISCV__APLIC_H */
> Does all of this really need to live in a non-private header?

struct aplic_priv is used in different files:
- in aplic.c to define `aplic` variable.
- in vaplic.c (which isn't intoduced yet) is used in several places:https://gitlab.com/xen-project/people/olkur/xen/-/blob/latest/xen/arch/riscv/vaplic.c#L41

struct aplic_regs is used only in aplic.c (at least, at the moment) so could be moved to
aplic.c, but I don't see too much sense.

>
>> --- a/xen/arch/riscv/include/asm/irq.h
>> +++ b/xen/arch/riscv/include/asm/irq.h
>> @@ -27,7 +27,6 @@
>>   #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
>>   
>>   /* TODO */
>> -#define nr_irqs 0U
> How come this is simply no longer needed, i.e. without any replacement?
> Hmm, looks like the only use in common code has gone away. Yet then this
> still doesn't really look to belong here (especially if not mentioned in
> the description).

I missed that it is used in xen/common/domain.c when CONFIG_HAS_PIRQ=y, but this
config isn't selected for RISC-V.
I think that I have to revert this change.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 12222 bytes --]

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

[Jan Beulich]

On 16.04.2025 12:15, Oleksii Kurochko wrote:
> On 4/14/25 12:04 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> +    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
>>> +    if ( !rc )
>>> +        panic("%s: IDC mode not supported\n", node->full_name);
>>> +
>>> +    imsic_node = dt_find_node_by_phandle(imsic_phandle);
>>> +    if ( !imsic_node )
>>> +        panic("%s: unable to find IMSIC node\n", node->full_name);
>>> +
>>> +    /* check imsic mode */
>>> +    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
>>> +                                    irq_range, ARRAY_SIZE(irq_range));
>>> +    if ( rc && (rc != -EOVERFLOW) )
>>> +        panic("%s: unable to find interrupt-extended in %s node\n",
>>> +               node->full_name, imsic_node->full_name);
>> Why exactly is EOVERFLOW tolerable here?
> 
> QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode.
> For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only
> interested in the S-mode IMSIC node.
> 
> The IMSIC node includes this information in the|"interrupts-extended"| property,
> which has the following format:
>    interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
> The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.
> 
> For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
> 
>    if ( irq_range[1] == IRQ_M_EXT )
> 
> Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom
> has more then one CPU as we passed irq_range[2] as an argument but the amount of values
> in "interrupts-extended" property will be (2 * CPUS_NUM).
> 
> I can update the comment above dt_property_read_u32_array() for more clearness.

Yet my question remains: Why would it be okay to ignore the remaining entries,
and hence accept -EOVERFLOW as kind-of-success?

>>> +    aplic.regs = ioremap(paddr, size);
>>> +    if ( !aplic.regs )
>>> +        panic("%s: unable to map\n", node->full_name);
>>> +
>>> +    /* Setup initial state APLIC interrupts */
>>> +    aplic_init_hw_interrupts();
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +static const struct intc_hw_operations __ro_after_init aplic_ops = {
>> const or __ro_after_init?
> 
> What’s wrong with using both?|const| ensures the variable can't be changed at compile time,
> while|__ro_after_init| makes it read-only at runtime after initialization is complete.

No, const makes it read-only at compile- and run-time. __ro_after_init,
putting the item into a special section, makes it writable at init-time.
Due to the const, the compiler wouldn't emit any writes. But we can
also avoid stray writes by having the item live in .rodata.

> Probably,|__initconst| would be a better fit:
>    static const struct intc_hw_operations __initconst aplic_ops = {
> 
> Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.

Well, if this variable isn't accessed post-init, sure. That seems pretty
unlikely though, considering it contains pointers to hook functions.

>>> --- /dev/null
>>> +++ b/xen/arch/riscv/include/asm/aplic.h
>>> @@ -0,0 +1,77 @@
>>> +/* SPDX-License-Identifier: MIT */
>>> +
>>> +/*
>>> + * xen/arch/riscv/aplic.h
>>> + *
>>> + * RISC-V Advanced Platform-Level Interrupt Controller support
>>> + *
>>> + * Copyright (c) 2023 Microchip.
>>> + */
>>> +
>>> +#ifndef ASM__RISCV__APLIC_H
>>> +#define ASM__RISCV__APLIC_H
>>> +
>>> +#include <xen/types.h>
>>> +
>>> +#include <asm/imsic.h>
>>> +
>>> +#define APLIC_DOMAINCFG_IE      BIT(8, UL)
>>> +#define APLIC_DOMAINCFG_DM      BIT(2, UL)
>>> +
>>> +struct aplic_regs {
>>> +    uint32_t domaincfg;
>>> +    uint32_t sourcecfg[1023];
>>> +    uint8_t _reserved1[0xBC0];
>>> +
>>> +    uint32_t mmsiaddrcfg;
>>> +    uint32_t mmsiaddrcfgh;
>>> +    uint32_t smsiaddrcfg;
>>> +    uint32_t smsiaddrcfgh;
>>> +    uint8_t _reserved2[0x30];
>>> +
>>> +    uint32_t setip[32];
>>> +    uint8_t _reserved3[92];
>>> +
>>> +    uint32_t setipnum;
>>> +    uint8_t _reserved4[0x20];
>>> +
>>> +    uint32_t in_clrip[32];
>>> +    uint8_t _reserved5[92];
>>> +
>>> +    uint32_t clripnum;
>>> +    uint8_t _reserved6[32];
>>> +
>>> +    uint32_t setie[32];
>>> +    uint8_t _reserved7[92];
>>> +
>>> +    uint32_t setienum;
>>> +    uint8_t _reserved8[32];
>>> +
>>> +    uint32_t clrie[32];
>>> +    uint8_t _reserved9[92];
>>> +
>>> +    uint32_t clrienum;
>>> +    uint8_t _reserved10[32];
>>> +
>>> +    uint32_t setipnum_le;
>>> +    uint32_t setipnum_be;
>>> +    uint8_t _reserved11[4088];
>>> +
>>> +    uint32_t genmsi;
>>> +    uint32_t target[1023];
>>> +};
>>> +
>>> +struct aplic_priv {
>>> +    /* base physical address and size */
>>> +    paddr_t paddr_start;
>>> +    paddr_t paddr_end;
>>> +    size_t  size;
>>> +
>>> +    /* registers */
>>> +    volatile struct aplic_regs *regs;
>>> +
>>> +    /* imsic configuration */
>>> +    const struct imsic_config *imsic_cfg;
>>> +};
>>> +
>>> +#endif /* ASM__RISCV__APLIC_H */
>> Does all of this really need to live in a non-private header?
> 
> struct aplic_priv is used in different files:
> - in aplic.c to define `aplic` variable.
> - in vaplic.c (which isn't intoduced yet) is used in several places:https://gitlab.com/xen-project/people/olkur/xen/-/blob/latest/xen/arch/riscv/vaplic.c#L41

Which would still call for a private header (xen/arch/riscv/aplic.h).

> struct aplic_regs is used only in aplic.c (at least, at the moment) so could be moved to
> aplic.c, but I don't see too much sense.

It is generally good practice to limit the scope of things as much as
possible. Just to avoid (or make more noticeable) mis-uses or layering
violations, for example.

>>> --- a/xen/arch/riscv/include/asm/irq.h
>>> +++ b/xen/arch/riscv/include/asm/irq.h
>>> @@ -27,7 +27,6 @@
>>>   #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
>>>   
>>>   /* TODO */
>>> -#define nr_irqs 0U
>> How come this is simply no longer needed, i.e. without any replacement?
>> Hmm, looks like the only use in common code has gone away. Yet then this
>> still doesn't really look to belong here (especially if not mentioned in
>> the description).
> 
> I missed that it is used in xen/common/domain.c when CONFIG_HAS_PIRQ=y, but this
> config isn't selected for RISC-V.
> I think that I have to revert this change.

I don't think you need to, as long as you don't mean to select HAS_PIRQ
for RISC-V. It's just that the change looks entirely unrelated _here_.

Jan

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 3814 bytes --]


On 4/16/25 12:30 PM, Jan Beulich wrote:
> On 16.04.2025 12:15, Oleksii Kurochko wrote:
>> On 4/14/25 12:04 PM, Jan Beulich wrote:
>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>> +    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
>>>> +    if ( !rc )
>>>> +        panic("%s: IDC mode not supported\n", node->full_name);
>>>> +
>>>> +    imsic_node = dt_find_node_by_phandle(imsic_phandle);
>>>> +    if ( !imsic_node )
>>>> +        panic("%s: unable to find IMSIC node\n", node->full_name);
>>>> +
>>>> +    /* check imsic mode */
>>>> +    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
>>>> +                                    irq_range, ARRAY_SIZE(irq_range));
>>>> +    if ( rc && (rc != -EOVERFLOW) )
>>>> +        panic("%s: unable to find interrupt-extended in %s node\n",
>>>> +               node->full_name, imsic_node->full_name);
>>> Why exactly is EOVERFLOW tolerable here?
>> QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode.
>> For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only
>> interested in the S-mode IMSIC node.
>>
>> The IMSIC node includes this information in the|"interrupts-extended"| property,
>> which has the following format:
>>     interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
>> The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.
>>
>> For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
>>
>>     if ( irq_range[1] == IRQ_M_EXT )
>>
>> Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom
>> has more then one CPU as we passed irq_range[2] as an argument but the amount of values
>> in "interrupts-extended" property will be (2 * CPUS_NUM).
>>
>> I can update the comment above dt_property_read_u32_array() for more clearness.
> Yet my question remains: Why would it be okay to ignore the remaining entries,
> and hence accept -EOVERFLOW as kind-of-success?

Because for other entries the IMSIC mode will be the same and the difference will be only in
interrupt controller's phandle which we don't care about in this function and cares only about
in imisic_init(), look at usage of imsic_get_parent_hartid().

>
>>>> +    aplic.regs = ioremap(paddr, size);
>>>> +    if ( !aplic.regs )
>>>> +        panic("%s: unable to map\n", node->full_name);
>>>> +
>>>> +    /* Setup initial state APLIC interrupts */
>>>> +    aplic_init_hw_interrupts();
>>>> +
>>>> +    return 0;
>>>> +}
>>>> +
>>>> +static const struct intc_hw_operations __ro_after_init aplic_ops = {
>>> const or __ro_after_init?
>> What’s wrong with using both?|const| ensures the variable can't be changed at compile time,
>> while|__ro_after_init| makes it read-only at runtime after initialization is complete.
> No, const makes it read-only at compile- and run-time.__ro_after_init,
> putting the item into a special section, makes it writable at init-time.
> Due to the const, the compiler wouldn't emit any writes. But we can
> also avoid stray writes by having the item live in .rodata.

Oh, right, `const` will add the variable to .rodata.
Then I think it is enough to have `const` as aplic_ops is going to be initialized once and
then only read will happen.

>
>> Probably,|__initconst| would be a better fit:
>>     static const struct intc_hw_operations __initconst aplic_ops = {
>>
>> Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.
> Well, if this variable isn't accessed post-init, sure. That seems pretty
> unlikely though, considering it contains pointers to hook functions.

Sure, .init section is going to be freed after init-time.

~ Oleksii


[-- Attachment #2: Type: text/html, Size: 5358 bytes --]

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

[Jan Beulich]

On 17.04.2025 17:21, Oleksii Kurochko wrote:
> 
> On 4/16/25 12:30 PM, Jan Beulich wrote:
>> On 16.04.2025 12:15, Oleksii Kurochko wrote:
>>> On 4/14/25 12:04 PM, Jan Beulich wrote:
>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>> +    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
>>>>> +    if ( !rc )
>>>>> +        panic("%s: IDC mode not supported\n", node->full_name);
>>>>> +
>>>>> +    imsic_node = dt_find_node_by_phandle(imsic_phandle);
>>>>> +    if ( !imsic_node )
>>>>> +        panic("%s: unable to find IMSIC node\n", node->full_name);
>>>>> +
>>>>> +    /* check imsic mode */
>>>>> +    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
>>>>> +                                    irq_range, ARRAY_SIZE(irq_range));
>>>>> +    if ( rc && (rc != -EOVERFLOW) )
>>>>> +        panic("%s: unable to find interrupt-extended in %s node\n",
>>>>> +               node->full_name, imsic_node->full_name);
>>>> Why exactly is EOVERFLOW tolerable here?
>>> QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode.
>>> For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only
>>> interested in the S-mode IMSIC node.
>>>
>>> The IMSIC node includes this information in the|"interrupts-extended"| property,
>>> which has the following format:
>>>     interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
>>> The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.
>>>
>>> For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
>>>
>>>     if ( irq_range[1] == IRQ_M_EXT )
>>>
>>> Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom
>>> has more then one CPU as we passed irq_range[2] as an argument but the amount of values
>>> in "interrupts-extended" property will be (2 * CPUS_NUM).
>>>
>>> I can update the comment above dt_property_read_u32_array() for more clearness.
>> Yet my question remains: Why would it be okay to ignore the remaining entries,
>> and hence accept -EOVERFLOW as kind-of-success?
> 
> Because for other entries the IMSIC mode will be the same and the difference will be only in
> interrupt controller's phandle

And we can blindly take this for granted? Would you mind extending the
comment that's there to include this aspect?

Jan

> which we don't care about in this function and cares only about
> in imisic_init(), look at usage of imsic_get_parent_hartid().

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 2881 bytes --]


On 4/17/25 5:30 PM, Jan Beulich wrote:
> On 17.04.2025 17:21, Oleksii Kurochko wrote:
>> On 4/16/25 12:30 PM, Jan Beulich wrote:
>>> On 16.04.2025 12:15, Oleksii Kurochko wrote:
>>>> On 4/14/25 12:04 PM, Jan Beulich wrote:
>>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>>> +    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
>>>>>> +    if ( !rc )
>>>>>> +        panic("%s: IDC mode not supported\n", node->full_name);
>>>>>> +
>>>>>> +    imsic_node = dt_find_node_by_phandle(imsic_phandle);
>>>>>> +    if ( !imsic_node )
>>>>>> +        panic("%s: unable to find IMSIC node\n", node->full_name);
>>>>>> +
>>>>>> +    /* check imsic mode */
>>>>>> +    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
>>>>>> +                                    irq_range, ARRAY_SIZE(irq_range));
>>>>>> +    if ( rc && (rc != -EOVERFLOW) )
>>>>>> +        panic("%s: unable to find interrupt-extended in %s node\n",
>>>>>> +               node->full_name, imsic_node->full_name);
>>>>> Why exactly is EOVERFLOW tolerable here?
>>>> QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode.
>>>> For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only
>>>> interested in the S-mode IMSIC node.
>>>>
>>>> The IMSIC node includes this information in the|"interrupts-extended"| property,
>>>> which has the following format:
>>>>      interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
>>>> The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.
>>>>
>>>> For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
>>>>
>>>>      if ( irq_range[1] == IRQ_M_EXT )
>>>>
>>>> Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom
>>>> has more then one CPU as we passed irq_range[2] as an argument but the amount of values
>>>> in "interrupts-extended" property will be (2 * CPUS_NUM).
>>>>
>>>> I can update the comment above dt_property_read_u32_array() for more clearness.
>>> Yet my question remains: Why would it be okay to ignore the remaining entries,
>>> and hence accept -EOVERFLOW as kind-of-success?
>> Because for other entries the IMSIC mode will be the same and the difference will be only in
>> interrupt controller's phandle
> And we can blindly take this for granted? Would you mind extending the
> comment that's there to include this aspect?

I tried to compile dtc with different modes in interrupt-extends and compilation doesn't failed, so
it's not really granted.
Just to be sure, I'll check all items of interrupts-extend not just the first one.

~ Oleksii

>> which we don't care about in this function and cares only about
>> in imisic_init(), look at usage of imsic_get_parent_hartid().

[-- Attachment #2: Type: text/html, Size: 4158 bytes --]

[PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Oleksii Kurochko]

Introduce interrupt controller descriptor for host APLIC to describe
the low-lovel hardare. It includes implementation of the following functions:
 - aplic_irq_startup()
 - aplic_irq_shutdown()
 - aplic_irq_enable()
 - aplic_irq_disable()
 - aplic_irq_ack()
 - aplic_host_irq_end()
 - aplic_set_irq_affinity()

As APLIC is used in MSI mode it requires to enable/disable interrupts not
only for APLIC but also for IMSIC. Thereby for the purpose of
aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().

For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is
introduced to get hart id.

Also, introduce additional interrupt controller h/w operations and
host_irq_type for APLIC:
 - aplic_host_irq_type
 - aplic_set_irq_priority()
 - aplic_set_irq_type()

Patch is based on the code from [1].

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5

Co-developed-by: Romain Caritey <Romain.Caritey@microchip.com>
Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/aplic.c             | 169 ++++++++++++++++++++++++++++-
 xen/arch/riscv/imsic.c             |  63 +++++++++++
 xen/arch/riscv/include/asm/aplic.h |  12 ++
 xen/arch/riscv/include/asm/imsic.h |  15 +++
 4 files changed, 258 insertions(+), 1 deletion(-)

diff --git a/xen/arch/riscv/aplic.c b/xen/arch/riscv/aplic.c
index d1aa835c3e..4b60cb9a77 100644
--- a/xen/arch/riscv/aplic.c
+++ b/xen/arch/riscv/aplic.c
@@ -15,6 +15,7 @@
 #include <xen/irq.h>
 #include <xen/mm.h>
 #include <xen/sections.h>
+#include <xen/spinlock.h>
 #include <xen/types.h>
 #include <xen/vmap.h>
 
@@ -110,9 +111,173 @@ static int __init aplic_init(void)
     return 0;
 }
 
-static const struct intc_hw_operations __ro_after_init aplic_ops = {
+static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    unsigned int irq = desc->irq - 1;
+
+    spin_lock(&aplic.lock);
+    switch(type) {
+        case IRQ_TYPE_EDGE_RISING:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_RISE;
+            break;
+        case IRQ_TYPE_EDGE_FALLING:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
+            break;
+        case IRQ_TYPE_LEVEL_HIGH:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
+            break;
+        case IRQ_TYPE_LEVEL_LOW:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
+            break;
+        default:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
+            break;
+    }
+    spin_unlock(&aplic.lock);
+}
+
+static void aplic_set_irq_priority(struct irq_desc *desc,
+                                   unsigned int priority)
+{
+    /* No priority, do nothing */
+}
+
+static void aplic_irq_enable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(spin_is_locked(&desc->lock));
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    clear_bit(_IRQ_DISABLED, &desc->status);
+
+    /* enable interrupt in IMSIC */
+    imsic_irq_enable(desc->irq);
+
+    /* enable interrupt in APLIC */
+    aplic.regs->setienum = desc->irq;
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static void aplic_irq_disable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(spin_is_locked(&desc->lock));
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    set_bit(_IRQ_DISABLED, &desc->status);
+
+    /* disable interrupt in APLIC */
+    aplic.regs->clrienum = desc->irq;
+
+    /* disable interrupt in IMSIC */
+    imsic_irq_disable(desc->irq);
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static unsigned int aplic_irq_startup(struct irq_desc *desc)
+{
+    aplic_irq_enable(desc);
+
+    return 0;
+}
+
+static void aplic_irq_shutdown(struct irq_desc *desc)
+{
+    aplic_irq_disable(desc);
+}
+
+static void aplic_irq_ack(struct irq_desc *desc)
+{
+    /* nothing to do */
+}
+
+static void aplic_host_irq_end(struct irq_desc *desc)
+{
+    /* nothing to do */
+}
+
+static unsigned int aplic_get_cpu_from_mask(const cpumask_t *cpumask)
+{
+    unsigned int cpu;
+    cpumask_t possible_mask;
+
+    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
+    cpu = cpumask_any(&possible_mask);
+
+    return cpu;
+}
+
+static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
+{
+    unsigned int cpu;
+    uint64_t group_index, base_ppn;
+    uint32_t hhxw, lhxw ,hhxs, value;
+    const struct imsic_config *imsic = aplic.imsic_cfg;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(!cpumask_empty(mask));
+
+    spin_lock(&aplic.lock);
+
+    cpu = cpuid_to_hartid(aplic_get_cpu_from_mask(mask));
+    hhxw = imsic->group_index_bits;
+    lhxw = imsic->hart_index_bits;
+    hhxs = imsic->group_index_shift - IMSIC_MMIO_PAGE_SHIFT * 2;
+    base_ppn = imsic->msi[cpu].base_addr >> IMSIC_MMIO_PAGE_SHIFT;
+
+    /* update hart and EEID in the target register */
+    group_index = (base_ppn >> (hhxs + 12)) & (BIT(hhxw, UL) - 1);
+    value = desc->irq;
+    value |= cpu << APLIC_TARGET_HART_IDX_SHIFT;
+    value |= group_index << (lhxw + APLIC_TARGET_HART_IDX_SHIFT) ;
+    aplic.regs->target[desc->irq - 1] = value;
+
+    spin_unlock(&aplic.lock);
+}
+
+static hw_irq_controller aplic_host_irq_type = {
+    .typename     = "aplic",
+    .startup      = aplic_irq_startup,
+    .shutdown     = aplic_irq_shutdown,
+    .enable       = aplic_irq_enable,
+    .disable      = aplic_irq_disable,
+    .ack          = aplic_irq_ack,
+    .end          = aplic_host_irq_end,
+    .set_affinity = aplic_set_irq_affinity,
+};
+
+static const struct intc_hw_operations aplic_ops = {
     .info                = &aplic_info,
     .init                = aplic_init,
+    .host_irq_type       = &aplic_host_irq_type,
+    .set_irq_priority    = aplic_set_irq_priority,
+    .set_irq_type        = aplic_set_irq_type,
 };
 
 static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
@@ -149,6 +314,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     dt_irq_xlate = aplic_irq_xlate;
 
+    spin_lock_init(&aplic.lock);
+
     register_intc_ops(&aplic_ops);
 
     return 0;
diff --git a/xen/arch/riscv/imsic.c b/xen/arch/riscv/imsic.c
index 99def9af2d..8198d008ef 100644
--- a/xen/arch/riscv/imsic.c
+++ b/xen/arch/riscv/imsic.c
@@ -14,12 +14,68 @@
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/macros.h>
+#include <xen/spinlock.h>
 #include <xen/xmalloc.h>
 
 #include <asm/imsic.h>
 
 static struct imsic_config imsic_cfg;
 
+#define imsic_csr_set(c, v)     \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_set(CSR_SIREG, v);      \
+} while (0)
+
+#define imsic_csr_clear(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_clear(CSR_SIREG, v);    \
+} while (0)
+
+static void imsic_local_eix_update(unsigned long base_id, unsigned long num_id,
+                                   bool pend, bool val)
+{
+    unsigned long i, isel, ireg;
+    unsigned long id = base_id, last_id = base_id + num_id;
+
+    while ( id < last_id )
+    {
+        isel = id / __riscv_xlen;
+        isel *= __riscv_xlen / IMSIC_EIPx_BITS;
+        isel += (pend) ? IMSIC_EIP0 : IMSIC_EIE0;
+
+        ireg = 0;
+        for ( i = id & (__riscv_xlen - 1);
+              (id < last_id) && (i < __riscv_xlen);
+              i++, id++ )
+            ireg |= (1 << i);
+
+        if ( val )
+            imsic_csr_set(isel, ireg);
+        else
+            imsic_csr_clear(isel, ireg);
+    }
+}
+
+void imsic_irq_enable(unsigned int hwirq)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&imsic_cfg.lock, flags);
+    imsic_local_eix_update(hwirq, 1, false, true);
+    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
+}
+
+void imsic_irq_disable(unsigned int hwirq)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&imsic_cfg.lock, flags);
+    imsic_local_eix_update(hwirq, 1, false, false);
+    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
+}
+
 const struct imsic_config *imsic_get_config(void)
 {
     return &imsic_cfg;
@@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node)
         goto imsic_init_err;
     }
 
+    spin_lock_init(&imsic_cfg.lock);
+
+    /* Enable local interrupt delivery */
+    imsic_ids_local_delivery(true);
+
+    imsic_cfg.is_used = true;
+
     return 0;
 
 imsic_init_err:
diff --git a/xen/arch/riscv/include/asm/aplic.h b/xen/arch/riscv/include/asm/aplic.h
index 94b3d0b616..ce858663a9 100644
--- a/xen/arch/riscv/include/asm/aplic.h
+++ b/xen/arch/riscv/include/asm/aplic.h
@@ -18,6 +18,15 @@
 #define APLIC_DOMAINCFG_IE      BIT(8, UL)
 #define APLIC_DOMAINCFG_DM      BIT(2, UL)
 
+#define APLIC_SOURCECFG_SM_INACTIVE     0x0
+#define APLIC_SOURCECFG_SM_DETACH       0x1
+#define APLIC_SOURCECFG_SM_EDGE_RISE    0x4
+#define APLIC_SOURCECFG_SM_EDGE_FALL    0x5
+#define APLIC_SOURCECFG_SM_LEVEL_HIGH   0x6
+#define APLIC_SOURCECFG_SM_LEVEL_LOW    0x7
+
+#define APLIC_TARGET_HART_IDX_SHIFT 18
+
 struct aplic_regs {
     uint32_t domaincfg;
     uint32_t sourcecfg[1023];
@@ -70,6 +79,9 @@ struct aplic_priv {
     /* registers */
     volatile struct aplic_regs *regs;
 
+    /* lock */
+    spinlock_t lock;
+
     /* imsic configuration */
     const struct imsic_config *imsic_cfg;
 };
diff --git a/xen/arch/riscv/include/asm/imsic.h b/xen/arch/riscv/include/asm/imsic.h
index 126e651863..d2c0178529 100644
--- a/xen/arch/riscv/include/asm/imsic.h
+++ b/xen/arch/riscv/include/asm/imsic.h
@@ -11,6 +11,7 @@
 #ifndef ASM__RISCV__IMSIC_H
 #define ASM__RISCV__IMSIC_H
 
+#include <xen/spinlock.h>
 #include <xen/types.h>
 
 #define IMSIC_MMIO_PAGE_SHIFT   12
@@ -19,6 +20,11 @@
 #define IMSIC_MIN_ID            63
 #define IMSIC_MAX_ID            2048
 
+#define IMSIC_EIP0              0x80
+#define IMSIC_EIPx_BITS         32
+
+#define IMSIC_EIE0              0xC0
+
 struct imsic_msi {
     paddr_t base_addr;
     unsigned long offset;
@@ -55,6 +61,12 @@ struct imsic_config {
 
     /* MSI */
     struct imsic_msi msi[NR_CPUS];
+
+    /* a check that IMSIC is used */
+    bool is_used;
+
+    /* lock */
+    spinlock_t lock;
 };
 
 struct dt_device_node;
@@ -63,4 +75,7 @@ int imsic_init(struct dt_device_node *n);
 struct imsic_config;
 const struct imsic_config *imsic_get_config(void);
 
+void imsic_irq_enable(unsigned int hwirq);
+void imsic_irq_disable(unsigned int hwirq);
+
 #endif /* ASM__RISCV__IMSIC_H */
-- 
2.49.0

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> Introduce interrupt controller descriptor for host APLIC to describe
> the low-lovel hardare. It includes implementation of the following functions:
>  - aplic_irq_startup()
>  - aplic_irq_shutdown()
>  - aplic_irq_enable()
>  - aplic_irq_disable()
>  - aplic_irq_ack()
>  - aplic_host_irq_end()
>  - aplic_set_irq_affinity()
> 
> As APLIC is used in MSI mode it requires to enable/disable interrupts not
> only for APLIC but also for IMSIC. Thereby for the purpose of
> aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
> 
> For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is
> introduced to get hart id.
> 
> Also, introduce additional interrupt controller h/w operations and
> host_irq_type for APLIC:
>  - aplic_host_irq_type
>  - aplic_set_irq_priority()
>  - aplic_set_irq_type()

Yet these two functions nor the hooks they're used to populate are entirely
unused here. Since they're also outside of the common IRQ handling machinery,
it's unclear how one would sanely ack such a change.

> --- a/xen/arch/riscv/aplic.c
> +++ b/xen/arch/riscv/aplic.c
> @@ -15,6 +15,7 @@
>  #include <xen/irq.h>
>  #include <xen/mm.h>
>  #include <xen/sections.h>
> +#include <xen/spinlock.h>
>  #include <xen/types.h>
>  #include <xen/vmap.h>
>  
> @@ -110,9 +111,173 @@ static int __init aplic_init(void)
>      return 0;
>  }
>  
> -static const struct intc_hw_operations __ro_after_init aplic_ops = {
> +static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
> +{
> +    unsigned int irq = desc->irq - 1;

Why this adjustment by 1 (and yet both items being named "irq")?

> +    spin_lock(&aplic.lock);
> +    switch(type) {
> +        case IRQ_TYPE_EDGE_RISING:

Nit (style): Missing blanks, brace on its own line, case labels indented
like their containing switch().

> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_RISE;
> +            break;
> +        case IRQ_TYPE_EDGE_FALLING:

Blank lines please between non-fall-through case blocks.

> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
> +            break;
> +        case IRQ_TYPE_LEVEL_HIGH:
> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
> +            break;
> +        case IRQ_TYPE_LEVEL_LOW:
> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
> +            break;
> +        default:
> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
> +            break;

Is the default: label legitimate to be reached?

> +    }
> +    spin_unlock(&aplic.lock);
> +}
> +
> +static void aplic_set_irq_priority(struct irq_desc *desc,
> +                                   unsigned int priority)
> +{
> +    /* No priority, do nothing */
> +}

Since the function dopes nothing, wouldn't it be better to omit it and have
the (future) caller check for a NULL pointer ahead of making the (indirect)
call? Same remark for other handlers (below) which also do nothing.

> +static void aplic_irq_enable(struct irq_desc *desc)
> +{
> +    unsigned long flags;
> +
> +    /*
> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
> +     *       If APLIC without MSI interrupts is required in the future,
> +     *       this function will need to be updated accordingly.
> +     */
> +    ASSERT(aplic.imsic_cfg->is_used);

Such an extra field, used only for assertions, is pretty odd. Can't you
use any of the other fields to achieve the same effect?

> +    ASSERT(spin_is_locked(&desc->lock));

If this lock (which is an IRQ-safe one) is necessarily held, ...

> +    spin_lock_irqsave(&aplic.lock, flags);

... you can use just spin_lock() here.

> +    clear_bit(_IRQ_DISABLED, &desc->status);

Why an atomic bitop when desc is locked? (And yes, I ought to raise the same
question on Arm code also doing so.)

I'm uncertain about this bit setting anyway - on x86 we would only fiddle
with it for IRQs not in use, not while enabling/disabling one.

In any event this can be done outside of the APLIC-locked region, I think.

> +    /* enable interrupt in IMSIC */

May I remind you of Xen comment style?

> +    imsic_irq_enable(desc->irq);
> +
> +    /* enable interrupt in APLIC */
> +    aplic.regs->setienum = desc->irq;

Are you sure you want to use plain assignments for MMIO accesses? I'd have
expected writel() to be used here. (And only later I realized that I didn't
spot the same already higher up from here.)

From the vague understanding I've gained so far: Isn't the APLIC closer to
the CPU and the IMSIC closer to the device? If so, wouldn't you want to
enable at the APLIC before enabling at the IMSIC? But of course that also
depends on what exactly happens in the window while one is already enabled
and the other is still disabled. (Later) From the code you add to imsic.c
it looks like it's the other way around, as the IMSIC is accessed through
CSRs.

> +    spin_unlock_irqrestore(&aplic.lock, flags);
> +}
> +
> +static void aplic_irq_disable(struct irq_desc *desc)
> +{
> +    unsigned long flags;
> +
> +    /*
> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
> +     *       If APLIC without MSI interrupts is required in the future,
> +     *       this function will need to be updated accordingly.
> +     */
> +    ASSERT(aplic.imsic_cfg->is_used);
> +
> +    ASSERT(spin_is_locked(&desc->lock));
> +
> +    spin_lock_irqsave(&aplic.lock, flags);
> +
> +    set_bit(_IRQ_DISABLED, &desc->status);
> +
> +    /* disable interrupt in APLIC */
> +    aplic.regs->clrienum = desc->irq;
> +
> +    /* disable interrupt in IMSIC */
> +    imsic_irq_disable(desc->irq);
> +
> +    spin_unlock_irqrestore(&aplic.lock, flags);
> +}
> +
> +static unsigned int aplic_irq_startup(struct irq_desc *desc)
> +{
> +    aplic_irq_enable(desc);
> +
> +    return 0;
> +}
> +
> +static void aplic_irq_shutdown(struct irq_desc *desc)
> +{
> +    aplic_irq_disable(desc);
> +}

You don't really need a separate hook function here, do you?

> +static void aplic_irq_ack(struct irq_desc *desc)
> +{
> +    /* nothing to do */
> +}
> +
> +static void aplic_host_irq_end(struct irq_desc *desc)

What's the "host" in the identifier about?

> +{
> +    /* nothing to do */
> +}
> +
> +static unsigned int aplic_get_cpu_from_mask(const cpumask_t *cpumask)
> +{
> +    unsigned int cpu;

No real need for this variable?

> +    cpumask_t possible_mask;
> +
> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
> +    cpu = cpumask_any(&possible_mask);

Why would you use cpu_possible_map here? That includes any offline CPUs.
I think you need to use cpu_online_map here.

> +    return cpu;
> +}
> +
> +static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
> +{
> +    unsigned int cpu;
> +    uint64_t group_index, base_ppn;
> +    uint32_t hhxw, lhxw ,hhxs, value;
> +    const struct imsic_config *imsic = aplic.imsic_cfg;
> +
> +    /*
> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
> +     *       If APLIC without MSI interrupts is required in the future,
> +     *       this function will need to be updated accordingly.
> +     */
> +    ASSERT(aplic.imsic_cfg->is_used);

Use the local variable you have made yourself?

> +    ASSERT(!cpumask_empty(mask));
> +
> +    spin_lock(&aplic.lock);

Aiui the lock can be acquired quite a bit later. It ought to be needed only
around the actual write to the hardware register.

> +    cpu = cpuid_to_hartid(aplic_get_cpu_from_mask(mask));
> +    hhxw = imsic->group_index_bits;
> +    lhxw = imsic->hart_index_bits;
> +    hhxs = imsic->group_index_shift - IMSIC_MMIO_PAGE_SHIFT * 2;
> +    base_ppn = imsic->msi[cpu].base_addr >> IMSIC_MMIO_PAGE_SHIFT;
> +
> +    /* update hart and EEID in the target register */
> +    group_index = (base_ppn >> (hhxs + 12)) & (BIT(hhxw, UL) - 1);

What's this magic 12 in here? Not IMSIC_MMIO_PAGE_SHIFT I suppose?

> +    value = desc->irq;
> +    value |= cpu << APLIC_TARGET_HART_IDX_SHIFT;
> +    value |= group_index << (lhxw + APLIC_TARGET_HART_IDX_SHIFT) ;
> +    aplic.regs->target[desc->irq - 1] = value;
> +
> +    spin_unlock(&aplic.lock);
> +}
> +
> +static hw_irq_controller aplic_host_irq_type = {

const?

> --- a/xen/arch/riscv/imsic.c
> +++ b/xen/arch/riscv/imsic.c
> @@ -14,12 +14,68 @@
>  #include <xen/errno.h>
>  #include <xen/init.h>
>  #include <xen/macros.h>
> +#include <xen/spinlock.h>
>  #include <xen/xmalloc.h>
>  
>  #include <asm/imsic.h>
>  
>  static struct imsic_config imsic_cfg;
>  
> +#define imsic_csr_set(c, v)     \
> +do {                            \
> +    csr_write(CSR_SISELECT, c); \
> +    csr_set(CSR_SIREG, v);      \
> +} while (0)
> +
> +#define imsic_csr_clear(c, v)   \
> +do {                            \
> +    csr_write(CSR_SISELECT, c); \
> +    csr_clear(CSR_SIREG, v);    \
> +} while (0)
> +
> +static void imsic_local_eix_update(unsigned long base_id, unsigned long num_id,
> +                                   bool pend, bool val)
> +{
> +    unsigned long i, isel, ireg;

These can be constrained to inside the outer loop below.

> +    unsigned long id = base_id, last_id = base_id + num_id;
> +
> +    while ( id < last_id )
> +    {
> +        isel = id / __riscv_xlen;
> +        isel *= __riscv_xlen / IMSIC_EIPx_BITS;
> +        isel += (pend) ? IMSIC_EIP0 : IMSIC_EIE0;

Nit: Why the parentheses?

> +        ireg = 0;
> +        for ( i = id & (__riscv_xlen - 1);
> +              (id < last_id) && (i < __riscv_xlen);
> +              i++, id++ )
> +            ireg |= (1 << i);

I wonder if this calculation really needs a loop. Afaict it's just a
consecutive set of bits you mean to set.

> +        if ( val )
> +            imsic_csr_set(isel, ireg);
> +        else
> +            imsic_csr_clear(isel, ireg);
> +    }
> +}
> +
> +void imsic_irq_enable(unsigned int hwirq)
> +{
> +    unsigned long flags;
> +
> +    spin_lock_irqsave(&imsic_cfg.lock, flags);
> +    imsic_local_eix_update(hwirq, 1, false, true);

No subtraction of 1 here? Also, why "hwirq" and not just "irq"?

> +    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
> +}
> +
> +void imsic_irq_disable(unsigned int hwirq)
> +{
> +    unsigned long flags;
> +
> +    spin_lock_irqsave(&imsic_cfg.lock, flags);
> +    imsic_local_eix_update(hwirq, 1, false, false);
> +    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
> +}
> +
>  const struct imsic_config *imsic_get_config(void)
>  {
>      return &imsic_cfg;
> @@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node)
>          goto imsic_init_err;
>      }
>  
> +    spin_lock_init(&imsic_cfg.lock);
> +
> +    /* Enable local interrupt delivery */
> +    imsic_ids_local_delivery(true);

What's this? I can't find the function/macro here, nor in patch 08, nor in
staging.

Jan

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 15479 bytes --]


On 4/15/25 2:46 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> Introduce interrupt controller descriptor for host APLIC to describe
>> the low-lovel hardare. It includes implementation of the following functions:
>>   - aplic_irq_startup()
>>   - aplic_irq_shutdown()
>>   - aplic_irq_enable()
>>   - aplic_irq_disable()
>>   - aplic_irq_ack()
>>   - aplic_host_irq_end()
>>   - aplic_set_irq_affinity()
>>
>> As APLIC is used in MSI mode it requires to enable/disable interrupts not
>> only for APLIC but also for IMSIC. Thereby for the purpose of
>> aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
>>
>> For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is
>> introduced to get hart id.
>>
>> Also, introduce additional interrupt controller h/w operations and
>> host_irq_type for APLIC:
>>   - aplic_host_irq_type
>>   - aplic_set_irq_priority()
>>   - aplic_set_irq_type()
> Yet these two functions nor the hooks they're used to populate are entirely
> unused here. Since they're also outside of the common IRQ handling machinery,
> it's unclear how one would sanely ack such a change.

They will be called by intc_route_irq_to_xen() from setup_irq() during firt time
the IRQ is setup.

>
>> --- a/xen/arch/riscv/aplic.c
>> +++ b/xen/arch/riscv/aplic.c
>> @@ -15,6 +15,7 @@
>>   #include <xen/irq.h>
>>   #include <xen/mm.h>
>>   #include <xen/sections.h>
>> +#include <xen/spinlock.h>
>>   #include <xen/types.h>
>>   #include <xen/vmap.h>
>>   
>> @@ -110,9 +111,173 @@ static int __init aplic_init(void)
>>       return 0;
>>   }
>>   
>> -static const struct intc_hw_operations __ro_after_init aplic_ops = {
>> +static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
>> +{
>> +    unsigned int irq = desc->irq - 1;
> Why this adjustment by 1 (and yet both items being named "irq")?

Interrupt 0 isn't possible based on the spec:
```
Each of an APLIC’s interrupt sources has a fixed unique identity number 
in the range 1 to N, where N is the total number of sources at the 
APLIC. The number zero is not a valid interrupt identity number at an 
APLIC. The maximum number of interrupt sources an APLIC may support is 
1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt 
2 ->sourcecfg[1] and so on. And that is the reason why we need -1.

>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
>> +            break;
>> +        case IRQ_TYPE_LEVEL_HIGH:
>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
>> +            break;
>> +        case IRQ_TYPE_LEVEL_LOW:
>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
>> +            break;
>> +        default:
>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
>> +            break;
> Is the default: label legitimate to be reached?

 From the spec:
```
0 Inactive Inactive in this domain (and not delegated) 1 Detached 
Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active, 
edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active, 
edge-sensitive; interrupt asserted on falling edge 6 Level1 Active, 
level-sensitive; interrupt asserted when high 7 Level0 Active, 
level-sensitive; interrupt asserted when low ``` It seems to me like 
APLIC_SOURCECFG_SM_INACTIVE just covers cases (0-3) and inactive IRQ 
pretty safe to as a default value.

>
>> +    }
>> +    spin_unlock(&aplic.lock);
>> +}
>> +
>> +static void aplic_set_irq_priority(struct irq_desc *desc,
>> +                                   unsigned int priority)
>> +{
>> +    /* No priority, do nothing */
>> +}
> Since the function dopes nothing, wouldn't it be better to omit it and have
> the (future) caller check for a NULL pointer ahead of making the (indirect)
> call? Same remark for other handlers (below) which also do nothing.

I thought about that too, but it could be some cases when the stub is introduced
with temporary BUG_ON("unimplemented") inside just to not miss to implement it
when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.

>
>> +static void aplic_irq_enable(struct irq_desc *desc)
>> +{
>> +    unsigned long flags;
>> +
>> +    /*
>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>> +     *       If APLIC without MSI interrupts is required in the future,
>> +     *       this function will need to be updated accordingly.
>> +     */
>> +    ASSERT(aplic.imsic_cfg->is_used);
> Such an extra field, used only for assertions, is pretty odd. Can't you
> use any of the other fields to achieve the same effect?

in aplic_init() there is:
     /* check for associated imsic node */
     rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
     if ( !rc )
         panic("%s: IDC mode not supported\n", node->full_name);

So we will have panic() anyway if MSI mode isn't supported. As an option we
can just drop the ASSERT.

Or introduce static variable in aplic.c `aplic_mode`, init it in aplic_init()
and use it in ASSERT().

>
>> +    ASSERT(spin_is_locked(&desc->lock));
> If this lock (which is an IRQ-safe one) is necessarily held, ...
>
>> +    spin_lock_irqsave(&aplic.lock, flags);
> ... you can use just spin_lock() here.
>
>> +    clear_bit(_IRQ_DISABLED, &desc->status);
> Why an atomic bitop when desc is locked? (And yes, I ought to raise the same
> question on Arm code also doing so.)

I haven't thought about that. Likely non-atomic bitop could be used here.

>
> I'm uncertain about this bit setting anyway - on x86 we would only fiddle
> with it for IRQs not in use, not while enabling/disabling one.
>
> In any event this can be done outside of the APLIC-locked region, I think.

Considering that we are doing that under desc->lock, agree we can move that outside
the APLIC-locked region.

>> +    imsic_irq_enable(desc->irq);
>> +
>> +    /* enable interrupt in APLIC */
>> +    aplic.regs->setienum = desc->irq;
> Are you sure you want to use plain assignments for MMIO accesses? I'd have
> expected writel() to be used here. (And only later I realized that I didn't
> spot the same already higher up from here.)

Good point. I have to update that with writel()...

>
>  From the vague understanding I've gained so far: Isn't the APLIC closer to
> the CPU and the IMSIC closer to the device? If so, wouldn't you want to
> enable at the APLIC before enabling at the IMSIC? But of course that also
> depends on what exactly happens in the window while one is already enabled
> and the other is still disabled. (Later) From the code you add to imsic.c
> it looks like it's the other way around, as the IMSIC is accessed through
> CSRs.

 From the AIA spec:
```
An Incoming MSI Controller (IMSIC) is an optional RISC-V hardware component
that is closely coupled with a hart, one IMSIC per hart. An IMSIC receives
and records incoming message-signaled interrupts (MSIs) for a hart, and
signals to the hart when there are pending and enabled interrupts to be
serviced.
```

Based on the figure 2 (Interrupt delivery by MSIs when harts have IMSICs for receiving them)
of AIA spechttps://github.com/riscv/riscv-aia/blob/main/src/intrsWithIMSICs.png
IMSIC is more close to CPU and APLIC is more close to the device. The external interrupt
controller is APLIC and it only sends a MSI message for a CPU.

The logical flow of an interrupt to a hart with an IMSIC would be:
1. A physical interrupt signal arrives at the APLIC.
2. The APLIC, if configured for MSI delivery mode (domaincfg.DM = 1) and if the specific
    interrupt source is active and enabled within its domain (controlled by sourcecfg[i]
    and the global Interrupt Enable bit IE in domaincfg), will generate an MSI.
3. This MSI is then sent to the target hart's IMSIC. The APLIC needs to know the MSI
    target address for each hart, which can be hardwired or configured through registers
    like mmsiaddrcfg and mmsiaddrcfgh.
4. The receiving hart's IMSIC records this MSI as a pending interrupt.
5. If the corresponding interrupt identity is enabled within the IMSIC's interrupt file,
    the IMSIC will signal the hart, typically by setting the MEIP or SEIP bit in the mip
    CSR (or sip CSR).

Generally, I think that the order in which enable interrupts doesn't really matter as
if you were to enable the IMSIC to receive a certain interrupt before the APLIC was
configured to send it (or had a pending interrupt from the device), the IMSIC would
simply be waiting for an MSI that wouldn't arrive.
Similarly, if the APLIC sends an MSI for an interrupt that is not enabled in the IMSIC,
the interrupt would remain pending in the IMSIC but wouldn't trigger an interrupt at
the hart.

IMO, the order which is used now in the code is pretty logical.

Does it make sense?

>
>> +    spin_unlock_irqrestore(&aplic.lock, flags);
>> +}
>> +
>> +static void aplic_irq_disable(struct irq_desc *desc)
>> +{
>> +    unsigned long flags;
>> +
>> +    /*
>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>> +     *       If APLIC without MSI interrupts is required in the future,
>> +     *       this function will need to be updated accordingly.
>> +     */
>> +    ASSERT(aplic.imsic_cfg->is_used);
>> +
>> +    ASSERT(spin_is_locked(&desc->lock));
>> +
>> +    spin_lock_irqsave(&aplic.lock, flags);
>> +
>> +    set_bit(_IRQ_DISABLED, &desc->status);
>> +
>> +    /* disable interrupt in APLIC */
>> +    aplic.regs->clrienum = desc->irq;
>> +
>> +    /* disable interrupt in IMSIC */
>> +    imsic_irq_disable(desc->irq);
>> +
>> +    spin_unlock_irqrestore(&aplic.lock, flags);
>> +}
>> +
>> +static unsigned int aplic_irq_startup(struct irq_desc *desc)
>> +{
>> +    aplic_irq_enable(desc);
>> +
>> +    return 0;
>> +}
>> +
>> +static void aplic_irq_shutdown(struct irq_desc *desc)
>> +{
>> +    aplic_irq_disable(desc);
>> +}
> You don't really need a separate hook function here, do you?

With such implementation it is really not needed to have a hook so
I will drop it.

>> +static void aplic_irq_ack(struct irq_desc *desc)
>> +{
>> +    /* nothing to do */
>> +}
>> +
>> +static void aplic_host_irq_end(struct irq_desc *desc)
> What's the "host" in the identifier about?

It was copied that from Arm and my understanding that it means
Xen-related IRQ as they also have:
```
/* XXX different for level vs edge */
static hw_irq_controller gicv2_host_irq_type = {
...
     .end          = gicv2_host_irq_end,
...
};

static hw_irq_controller gicv2_guest_irq_type = {
...
     .end          = gicv2_guest_irq_end,
...
};
```


>
>> +{
>> +    /* nothing to do */
>> +}
>> +
>> +static unsigned int aplic_get_cpu_from_mask(const cpumask_t *cpumask)
>> +{
>> +    unsigned int cpu;
> No real need for this variable?

Yes, it could be dropped.

>
>> +    cpumask_t possible_mask;
>> +
>> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
>> +    cpu = cpumask_any(&possible_mask);
> Why would you use cpu_possible_map here? That includes any offline CPUs.
> I think you need to use cpu_online_map here.

It makes sense, Ill switch to cpu_online_map.

>
>> +    return cpu;
>> +}
>> +
>> +static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
>> +{
>> +    unsigned int cpu;
>> +    uint64_t group_index, base_ppn;
>> +    uint32_t hhxw, lhxw ,hhxs, value;
>> +    const struct imsic_config *imsic = aplic.imsic_cfg;
>> +
>> +    /*
>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>> +     *       If APLIC without MSI interrupts is required in the future,
>> +     *       this function will need to be updated accordingly.
>> +     */
>> +    ASSERT(aplic.imsic_cfg->is_used);
> Use the local variable you have made yourself?

What do you mean by local here?

>
>> +    ASSERT(!cpumask_empty(mask));
>> +
>> +    spin_lock(&aplic.lock);
> Aiui the lock can be acquired quite a bit later. It ought to be needed only
> around the actual write to the hardware register.
>
>> +    cpu = cpuid_to_hartid(aplic_get_cpu_from_mask(mask));
>> +    hhxw = imsic->group_index_bits;
>> +    lhxw = imsic->hart_index_bits;
>> +    hhxs = imsic->group_index_shift - IMSIC_MMIO_PAGE_SHIFT * 2;
>> +    base_ppn = imsic->msi[cpu].base_addr >> IMSIC_MMIO_PAGE_SHIFT;
>> +
>> +    /* update hart and EEID in the target register */
>> +    group_index = (base_ppn >> (hhxs + 12)) & (BIT(hhxw, UL) - 1);
> What's this magic 12 in here? Not IMSIC_MMIO_PAGE_SHIFT I suppose?

In the AIA spec they are using 12 explicitly:https://github.com/riscv/riscv-aia/blob/main/src/AdvPLIC.adoc#AdvPLIC-MSIAddrs

>> +    unsigned long id = base_id, last_id = base_id + num_id;
>> +
>> +    while ( id < last_id )
>> +    {
>> +        isel = id / __riscv_xlen;
>> +        isel *= __riscv_xlen / IMSIC_EIPx_BITS;
>> +        isel += (pend) ? IMSIC_EIP0 : IMSIC_EIE0;
> Nit: Why the parentheses?
>
>> +        ireg = 0;
>> +        for ( i = id & (__riscv_xlen - 1);
>> +              (id < last_id) && (i < __riscv_xlen);
>> +              i++, id++ )
>> +            ireg |= (1 << i);
> I wonder if this calculation really needs a loop. Afaict it's just a
> consecutive set of bits you mean to set.

Good point. I will double-check.

>
>> +        if ( val )
>> +            imsic_csr_set(isel, ireg);
>> +        else
>> +            imsic_csr_clear(isel, ireg);
>> +    }
>> +}
>> +
>> +void imsic_irq_enable(unsigned int hwirq)
>> +{
>> +    unsigned long flags;
>> +
>> +    spin_lock_irqsave(&imsic_cfg.lock, flags);
>> +    imsic_local_eix_update(hwirq, 1, false, true);
> No subtraction of 1 here? Also, why "hwirq" and not just "irq"?

 From the spec:
```

When an interrupt file supports distinct interrupt identities, valid identity numbers are between 1
and inclusive. The identity numbers within this range are said to be implemented by the interrupt
file; numbers outside this range are not implemented. The number zero is never a valid interrupt
identity.
...

Bit positions in a valid eiek register that don’t correspond to a 
supported interrupt identity (such as bit 0 of eie0) are read-only zeros.


```

So in EIx registers interrupt i corresponds to bit i in comparison wiht APLIC's sourcecfg which starts from 0.

>
>> +    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
>> +}
>> +
>> +void imsic_irq_disable(unsigned int hwirq)
>> +{
>> +    unsigned long flags;
>> +
>> +    spin_lock_irqsave(&imsic_cfg.lock, flags);
>> +    imsic_local_eix_update(hwirq, 1, false, false);
>> +    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
>> +}
>> +
>>   const struct imsic_config *imsic_get_config(void)
>>   {
>>       return &imsic_cfg;
>> @@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node)
>>           goto imsic_init_err;
>>       }
>>   
>> +    spin_lock_init(&imsic_cfg.lock);
>> +
>> +    /* Enable local interrupt delivery */
>> +    imsic_ids_local_delivery(true);
> What's this? I can't find the function/macro here, nor in patch 08, nor in
> staging.

It is defined in imsic.c:
```
void imsic_ids_local_delivery(bool enable)
{
     if ( enable )
     {
         imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
         imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
     }
     else
     {
         imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
         imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
     }
}
```

Thanks for review.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 22061 bytes --]

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Jan Beulich]

On 16.04.2025 21:05, Oleksii Kurochko wrote:
> On 4/15/25 2:46 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> Introduce interrupt controller descriptor for host APLIC to describe
>>> the low-lovel hardare. It includes implementation of the following functions:
>>>   - aplic_irq_startup()
>>>   - aplic_irq_shutdown()
>>>   - aplic_irq_enable()
>>>   - aplic_irq_disable()
>>>   - aplic_irq_ack()
>>>   - aplic_host_irq_end()
>>>   - aplic_set_irq_affinity()
>>>
>>> As APLIC is used in MSI mode it requires to enable/disable interrupts not
>>> only for APLIC but also for IMSIC. Thereby for the purpose of
>>> aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
>>>
>>> For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is
>>> introduced to get hart id.
>>>
>>> Also, introduce additional interrupt controller h/w operations and
>>> host_irq_type for APLIC:
>>>   - aplic_host_irq_type
>>>   - aplic_set_irq_priority()
>>>   - aplic_set_irq_type()
>> Yet these two functions nor the hooks they're used to populate are entirely
>> unused here. Since they're also outside of the common IRQ handling machinery,
>> it's unclear how one would sanely ack such a change.
> 
> They will be called by intc_route_irq_to_xen() from setup_irq() during firt time
> the IRQ is setup.

Perhaps move their introduction to there then? We don't do any Misra checking
yet lon RISC-V, but imo it's still good practice to avoid introducing new
violations, even if only temporarily.

>>> --- a/xen/arch/riscv/aplic.c
>>> +++ b/xen/arch/riscv/aplic.c
>>> @@ -15,6 +15,7 @@
>>>   #include <xen/irq.h>
>>>   #include <xen/mm.h>
>>>   #include <xen/sections.h>
>>> +#include <xen/spinlock.h>
>>>   #include <xen/types.h>
>>>   #include <xen/vmap.h>
>>>   
>>> @@ -110,9 +111,173 @@ static int __init aplic_init(void)
>>>       return 0;
>>>   }
>>>   
>>> -static const struct intc_hw_operations __ro_after_init aplic_ops = {
>>> +static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
>>> +{
>>> +    unsigned int irq = desc->irq - 1;
>> Why this adjustment by 1 (and yet both items being named "irq")?
> 
> Interrupt 0 isn't possible based on the spec:
> ```
> Each of an APLIC’s interrupt sources has a fixed unique identity number 
> in the range 1 to N, where N is the total number of sources at the 
> APLIC. The number zero is not a valid interrupt identity number at an 
> APLIC. The maximum number of interrupt sources an APLIC may support is 
> 1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt 
> 2 ->sourcecfg[1] and so on. And that is the reason why we need -1.

Okay, fine. But what about the part of the question in parentheses?

>>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
>>> +            break;
>>> +        case IRQ_TYPE_LEVEL_HIGH:
>>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
>>> +            break;
>>> +        case IRQ_TYPE_LEVEL_LOW:
>>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
>>> +            break;
>>> +        default:
>>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
>>> +            break;
>> Is the default: label legitimate to be reached?
> 
>  From the spec:
> ```
> 0 Inactive Inactive in this domain (and not delegated) 1 Detached 
> Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active, 
> edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active, 
> edge-sensitive; interrupt asserted on falling edge 6 Level1 Active, 
> level-sensitive; interrupt asserted when high 7 Level0 Active, 
> level-sensitive; interrupt asserted when low ``` It seems to me like 
> APLIC_SOURCECFG_SM_INACTIVE just covers cases (0-3) and inactive IRQ 
> pretty safe to as a default value.

I fear this doesn't answer my question, which is to a large part related
to the Xen code, and only to some degree to the spec.

>>> +static void aplic_set_irq_priority(struct irq_desc *desc,
>>> +                                   unsigned int priority)
>>> +{
>>> +    /* No priority, do nothing */
>>> +}
>> Since the function dopes nothing, wouldn't it be better to omit it and have
>> the (future) caller check for a NULL pointer ahead of making the (indirect)
>> call? Same remark for other handlers (below) which also do nothing.
> 
> I thought about that too, but it could be some cases when the stub is introduced
> with temporary BUG_ON("unimplemented") inside just to not miss to implement it
> when it will be necessary.
> If we will have only the caller check then we could miss to implement such stubs.

I guess I don't understand the concern.

>>> +static void aplic_irq_enable(struct irq_desc *desc)
>>> +{
>>> +    unsigned long flags;
>>> +
>>> +    /*
>>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>>> +     *       If APLIC without MSI interrupts is required in the future,
>>> +     *       this function will need to be updated accordingly.
>>> +     */
>>> +    ASSERT(aplic.imsic_cfg->is_used);
>> Such an extra field, used only for assertions, is pretty odd. Can't you
>> use any of the other fields to achieve the same effect?
> 
> in aplic_init() there is:
>      /* check for associated imsic node */
>      rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
>      if ( !rc )
>          panic("%s: IDC mode not supported\n", node->full_name);
> 
> So we will have panic() anyway if MSI mode isn't supported. As an option we
> can just drop the ASSERT.

Since they serve primarily as a reminder where changes would need making,
I'd prefer if they could be kept.

> Or introduce static variable in aplic.c `aplic_mode`, init it in aplic_init()
> and use it in ASSERT().

This would then again be used solely for assertions, aiui? As said, I
think it would be preferable if some already existing indicator could be
used for this purpose.

>>> +    ASSERT(spin_is_locked(&desc->lock));
>> If this lock (which is an IRQ-safe one) is necessarily held, ...
>>
>>> +    spin_lock_irqsave(&aplic.lock, flags);
>> ... you can use just spin_lock() here.
>>
>>> +    clear_bit(_IRQ_DISABLED, &desc->status);
>> Why an atomic bitop when desc is locked? (And yes, I ought to raise the same
>> question on Arm code also doing so.)
> 
> I haven't thought about that. Likely non-atomic bitop could be used here.

And then - does it need to be a bitop? Aiui that's what Arm uses, while x86
doesn't. And I see no reason to use other than plain C operators here. If
Arm was switched, presumably all the redundant (and misnamed) _IRQ_*
constants could go away, with just the IRQ_* ones left.

>> I'm uncertain about this bit setting anyway - on x86 we would only fiddle
>> with it for IRQs not in use, not while enabling/disabling one.

What about this part?

>> In any event this can be done outside of the APLIC-locked region, I think.
> 
> Considering that we are doing that under desc->lock, agree we can move that outside
> the APLIC-locked region.
> 
>>> +    imsic_irq_enable(desc->irq);
>>> +
>>> +    /* enable interrupt in APLIC */
>>> +    aplic.regs->setienum = desc->irq;
>> Are you sure you want to use plain assignments for MMIO accesses? I'd have
>> expected writel() to be used here. (And only later I realized that I didn't
>> spot the same already higher up from here.)
> 
> Good point. I have to update that with writel()...
> 
>>
>>  From the vague understanding I've gained so far: Isn't the APLIC closer to
>> the CPU and the IMSIC closer to the device? If so, wouldn't you want to
>> enable at the APLIC before enabling at the IMSIC? But of course that also
>> depends on what exactly happens in the window while one is already enabled
>> and the other is still disabled. (Later) From the code you add to imsic.c
>> it looks like it's the other way around, as the IMSIC is accessed through
>> CSRs.
> 
>  From the AIA spec:
> ```
> An Incoming MSI Controller (IMSIC) is an optional RISC-V hardware component
> that is closely coupled with a hart, one IMSIC per hart. An IMSIC receives
> and records incoming message-signaled interrupts (MSIs) for a hart, and
> signals to the hart when there are pending and enabled interrupts to be
> serviced.
> ```
> 
> Based on the figure 2 (Interrupt delivery by MSIs when harts have IMSICs for receiving them)
> of AIA spechttps://github.com/riscv/riscv-aia/blob/main/src/intrsWithIMSICs.png
> IMSIC is more close to CPU and APLIC is more close to the device. The external interrupt
> controller is APLIC and it only sends a MSI message for a CPU.
> 
> The logical flow of an interrupt to a hart with an IMSIC would be:
> 1. A physical interrupt signal arrives at the APLIC.
> 2. The APLIC, if configured for MSI delivery mode (domaincfg.DM = 1) and if the specific
>     interrupt source is active and enabled within its domain (controlled by sourcecfg[i]
>     and the global Interrupt Enable bit IE in domaincfg), will generate an MSI.
> 3. This MSI is then sent to the target hart's IMSIC. The APLIC needs to know the MSI
>     target address for each hart, which can be hardwired or configured through registers
>     like mmsiaddrcfg and mmsiaddrcfgh.
> 4. The receiving hart's IMSIC records this MSI as a pending interrupt.
> 5. If the corresponding interrupt identity is enabled within the IMSIC's interrupt file,
>     the IMSIC will signal the hart, typically by setting the MEIP or SEIP bit in the mip
>     CSR (or sip CSR).
> 
> Generally, I think that the order in which enable interrupts doesn't really matter as
> if you were to enable the IMSIC to receive a certain interrupt before the APLIC was
> configured to send it (or had a pending interrupt from the device), the IMSIC would
> simply be waiting for an MSI that wouldn't arrive.
> Similarly, if the APLIC sends an MSI for an interrupt that is not enabled in the IMSIC,
> the interrupt would remain pending in the IMSIC but wouldn't trigger an interrupt at
> the hart.
> 
> IMO, the order which is used now in the code is pretty logical.
> 
> Does it make sense?

Except for the "doesn't really matter" - yes. In a reply to a later patch I
indicated I realized that IMSIC is what's closer to the CPU (and hence later
in the chain of interrupt delivery actions).

>>> +    spin_unlock_irqrestore(&aplic.lock, flags);
>>> +}
>>> +
>>> +static void aplic_irq_disable(struct irq_desc *desc)
>>> +{
>>> +    unsigned long flags;
>>> +
>>> +    /*
>>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>>> +     *       If APLIC without MSI interrupts is required in the future,
>>> +     *       this function will need to be updated accordingly.
>>> +     */
>>> +    ASSERT(aplic.imsic_cfg->is_used);
>>> +
>>> +    ASSERT(spin_is_locked(&desc->lock));
>>> +
>>> +    spin_lock_irqsave(&aplic.lock, flags);
>>> +
>>> +    set_bit(_IRQ_DISABLED, &desc->status);
>>> +
>>> +    /* disable interrupt in APLIC */
>>> +    aplic.regs->clrienum = desc->irq;
>>> +
>>> +    /* disable interrupt in IMSIC */
>>> +    imsic_irq_disable(desc->irq);
>>> +
>>> +    spin_unlock_irqrestore(&aplic.lock, flags);
>>> +}
>>> +
>>> +static unsigned int aplic_irq_startup(struct irq_desc *desc)
>>> +{
>>> +    aplic_irq_enable(desc);
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +static void aplic_irq_shutdown(struct irq_desc *desc)
>>> +{
>>> +    aplic_irq_disable(desc);
>>> +}
>> You don't really need a separate hook function here, do you?
> 
> With such implementation it is really not needed to have a hook so
> I will drop it.
> 
>>> +static void aplic_irq_ack(struct irq_desc *desc)
>>> +{
>>> +    /* nothing to do */
>>> +}
>>> +
>>> +static void aplic_host_irq_end(struct irq_desc *desc)
>> What's the "host" in the identifier about?
> 
> It was copied that from Arm and my understanding that it means
> Xen-related IRQ as they also have:
> ```
> /* XXX different for level vs edge */
> static hw_irq_controller gicv2_host_irq_type = {
> ...
>      .end          = gicv2_host_irq_end,
> ...
> };
> 
> static hw_irq_controller gicv2_guest_irq_type = {
> ...
>      .end          = gicv2_guest_irq_end,
> ...
> };
> ```

And you expect to end up with a similar distinction on RISC-V? There's
nothing like that on x86, just to mention it.

>>> +static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
>>> +{
>>> +    unsigned int cpu;
>>> +    uint64_t group_index, base_ppn;
>>> +    uint32_t hhxw, lhxw ,hhxs, value;
>>> +    const struct imsic_config *imsic = aplic.imsic_cfg;
>>> +
>>> +    /*
>>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>>> +     *       If APLIC without MSI interrupts is required in the future,
>>> +     *       this function will need to be updated accordingly.
>>> +     */
>>> +    ASSERT(aplic.imsic_cfg->is_used);
>> Use the local variable you have made yourself?
> 
> What do you mean by local here?

Just a few lines up you latch aplic.imsic_cfg into the local "imsic".

>>> +    ASSERT(!cpumask_empty(mask));
>>> +
>>> +    spin_lock(&aplic.lock);
>> Aiui the lock can be acquired quite a bit later. It ought to be needed only
>> around the actual write to the hardware register.
>>
>>> +    cpu = cpuid_to_hartid(aplic_get_cpu_from_mask(mask));
>>> +    hhxw = imsic->group_index_bits;
>>> +    lhxw = imsic->hart_index_bits;
>>> +    hhxs = imsic->group_index_shift - IMSIC_MMIO_PAGE_SHIFT * 2;
>>> +    base_ppn = imsic->msi[cpu].base_addr >> IMSIC_MMIO_PAGE_SHIFT;
>>> +
>>> +    /* update hart and EEID in the target register */
>>> +    group_index = (base_ppn >> (hhxs + 12)) & (BIT(hhxw, UL) - 1);
>> What's this magic 12 in here? Not IMSIC_MMIO_PAGE_SHIFT I suppose?
> 
> In the AIA spec they are using 12 explicitly:https://github.com/riscv/riscv-aia/blob/main/src/AdvPLIC.adoc#AdvPLIC-MSIAddrs

In the spec that's fine, but please make yourself a constant with a suitable
name then, to be used here. Just consider what would happen if we used literal
12 everywhere PAGE_SHIFT was meant.

>>> +void imsic_irq_enable(unsigned int hwirq)
>>> +{
>>> +    unsigned long flags;
>>> +
>>> +    spin_lock_irqsave(&imsic_cfg.lock, flags);
>>> +    imsic_local_eix_update(hwirq, 1, false, true);
>> No subtraction of 1 here? Also, why "hwirq" and not just "irq"?
> 
>  From the spec:
> ```
> 
> When an interrupt file supports distinct interrupt identities, valid identity numbers are between 1
> and inclusive. The identity numbers within this range are said to be implemented by the interrupt
> file; numbers outside this range are not implemented. The number zero is never a valid interrupt
> identity.
> ...
> 
> Bit positions in a valid eiek register that don’t correspond to a 
> supported interrupt identity (such as bit 0 of eie0) are read-only zeros.
> 
> 
> ```
> 
> So in EIx registers interrupt i corresponds to bit i in comparison wiht APLIC's sourcecfg which starts from 0.

Confusing, but what do you do.

>>> @@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node)
>>>           goto imsic_init_err;
>>>       }
>>>   
>>> +    spin_lock_init(&imsic_cfg.lock);
>>> +
>>> +    /* Enable local interrupt delivery */
>>> +    imsic_ids_local_delivery(true);
>> What's this? I can't find the function/macro here, nor in patch 08, nor in
>> staging.
> 
> It is defined in imsic.c:
> ```
> void imsic_ids_local_delivery(bool enable)
> {
>      if ( enable )
>      {
>          imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
>          imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
>      }
>      else
>      {
>          imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
>          imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
>      }
> }
> ```

No, it's not. As noted in the reply to a later patch, it's only introduced
there. Hence the build will break between the two patches.

Jan

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 12839 bytes --]


On 4/17/25 8:25 AM, Jan Beulich wrote:
> On 16.04.2025 21:05, Oleksii Kurochko wrote:
>> On 4/15/25 2:46 PM, Jan Beulich wrote:
>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>> Introduce interrupt controller descriptor for host APLIC to describe
>>>> the low-lovel hardare. It includes implementation of the following functions:
>>>>    - aplic_irq_startup()
>>>>    - aplic_irq_shutdown()
>>>>    - aplic_irq_enable()
>>>>    - aplic_irq_disable()
>>>>    - aplic_irq_ack()
>>>>    - aplic_host_irq_end()
>>>>    - aplic_set_irq_affinity()
>>>>
>>>> As APLIC is used in MSI mode it requires to enable/disable interrupts not
>>>> only for APLIC but also for IMSIC. Thereby for the purpose of
>>>> aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
>>>>
>>>> For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is
>>>> introduced to get hart id.
>>>>
>>>> Also, introduce additional interrupt controller h/w operations and
>>>> host_irq_type for APLIC:
>>>>    - aplic_host_irq_type
>>>>    - aplic_set_irq_priority()
>>>>    - aplic_set_irq_type()
>>> Yet these two functions nor the hooks they're used to populate are entirely
>>> unused here. Since they're also outside of the common IRQ handling machinery,
>>> it's unclear how one would sanely ack such a change.
>> They will be called by intc_route_irq_to_xen() from setup_irq() during firt time
>> the IRQ is setup.
> Perhaps move their introduction to there then? We don't do any Misra checking
> yet lon RISC-V, but imo it's still good practice to avoid introducing new
> violations, even if only temporarily.

Okay, I will move their introduction to there.

Btw, what is needed to add Misra checking for RISC-V? I started to think that, probably,
it will make sense to do that from the start.

>>>> --- a/xen/arch/riscv/aplic.c
>>>> +++ b/xen/arch/riscv/aplic.c
>>>> @@ -15,6 +15,7 @@
>>>>    #include <xen/irq.h>
>>>>    #include <xen/mm.h>
>>>>    #include <xen/sections.h>
>>>> +#include <xen/spinlock.h>
>>>>    #include <xen/types.h>
>>>>    #include <xen/vmap.h>
>>>>    
>>>> @@ -110,9 +111,173 @@ static int __init aplic_init(void)
>>>>        return 0;
>>>>    }
>>>>    
>>>> -static const struct intc_hw_operations __ro_after_init aplic_ops = {
>>>> +static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
>>>> +{
>>>> +    unsigned int irq = desc->irq - 1;
>>> Why this adjustment by 1 (and yet both items being named "irq")?
>> Interrupt 0 isn't possible based on the spec:
>> ```
>> Each of an APLIC’s interrupt sources has a fixed unique identity number
>> in the range 1 to N, where N is the total number of sources at the
>> APLIC. The number zero is not a valid interrupt identity number at an
>> APLIC. The maximum number of interrupt sources an APLIC may support is
>> 1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt
>> 2 ->sourcecfg[1] and so on. And that is the reason why we need -1.
> Okay, fine. But what about the part of the question in parentheses?

Sorry, missed to write that I'll change irq to irq_bit or something like that.

>>>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
>>>> +            break;
>>>> +        case IRQ_TYPE_LEVEL_HIGH:
>>>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
>>>> +            break;
>>>> +        case IRQ_TYPE_LEVEL_LOW:
>>>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
>>>> +            break;
>>>> +        default:
>>>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
>>>> +            break;
>>> Is the default: label legitimate to be reached?
>>   From the spec:
>> ```
>> 0 Inactive Inactive in this domain (and not delegated) 1 Detached
>> Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active,
>> edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active,
>> edge-sensitive; interrupt asserted on falling edge 6 Level1 Active,
>> level-sensitive; interrupt asserted when high 7 Level0 Active,
>> level-sensitive; interrupt asserted when low ``` It seems to me like
>> APLIC_SOURCECFG_SM_INACTIVE just covers cases (0-3) and inactive IRQ
>> pretty safe to as a default value.
> I fear this doesn't answer my question, which is to a large part related
> to the Xen code, and only to some degree to the spec.

 From Xen code point of view, I am not sure if it legitimate or not. I've not any
issue, at the moment, with such implementation, but to be on a safe side I'll
implement default case as panic("...").

>>>> +static void aplic_set_irq_priority(struct irq_desc *desc,
>>>> +                                   unsigned int priority)
>>>> +{
>>>> +    /* No priority, do nothing */
>>>> +}
>>> Since the function dopes nothing, wouldn't it be better to omit it and have
>>> the (future) caller check for a NULL pointer ahead of making the (indirect)
>>> call? Same remark for other handlers (below) which also do nothing.
>> I thought about that too, but it could be some cases when the stub is introduced
>> with temporary BUG_ON("unimplemented") inside just to not miss to implement it
>> when it will be necessary.
>> If we will have only the caller check then we could miss to implement such stubs.
> I guess I don't understand the concern.

for example, if we will have the following code:
   void some_caller(struct irq_desc *desc)
   {
       if ( desc->handler->set_affinity )
           desc->handler->set_affinity(desc, cpu_mask);
   }

Then we will skip the call of handler->set_affinity() (if it was just initialized with
.set_affinity = NULL) without any notification. And it is fine specifically in this
case as aplic_set_irq_priority() does nothing.

But what about the cases if it is a function which will have some implementation in the
future but doesn't have implementation for now. Then without notification that this
function is unimplemented we could skip something what really matters.

But I think that your initial comment was just about the function which basically
does nothing. Am i right?

>>>> +static void aplic_irq_enable(struct irq_desc *desc)
>>>> +{
>>>> +    unsigned long flags;
>>>> +
>>>> +    /*
>>>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>>>> +     *       If APLIC without MSI interrupts is required in the future,
>>>> +     *       this function will need to be updated accordingly.
>>>> +     */
>>>> +    ASSERT(aplic.imsic_cfg->is_used);
>>> Such an extra field, used only for assertions, is pretty odd. Can't you
>>> use any of the other fields to achieve the same effect?
>> in aplic_init() there is:
>>       /* check for associated imsic node */
>>       rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
>>       if ( !rc )
>>           panic("%s: IDC mode not supported\n", node->full_name);
>>
>> So we will have panic() anyway if MSI mode isn't supported. As an option we
>> can just drop the ASSERT.
> Since they serve primarily as a reminder where changes would need making,
> I'd prefer if they could be kept.
>
>> Or introduce static variable in aplic.c `aplic_mode`, init it in aplic_init()
>> and use it in ASSERT().
> This would then again be used solely for assertions, aiui? As said, I
> think it would be preferable if some already existing indicator could be
> used for this purpose.

I think that not solely, for example, if IMSIC isn't available then we should skip
the calls of imsic_irq_enable(), at least, and this variable could be used for that
purpose.

But I will double check if we have better indicator. At the moment, I don't think
we have better, probably, except checking of aplic.regs->domaincfg if bit APLIC_DOMAINCFG_DM
is set.

>>>> +    ASSERT(spin_is_locked(&desc->lock));
>>> If this lock (which is an IRQ-safe one) is necessarily held, ...
>>>
>>>> +    spin_lock_irqsave(&aplic.lock, flags);
>>> ... you can use just spin_lock() here.
>>>
>>>> +    clear_bit(_IRQ_DISABLED, &desc->status);
>>> Why an atomic bitop when desc is locked? (And yes, I ought to raise the same
>>> question on Arm code also doing so.)
>> I haven't thought about that. Likely non-atomic bitop could be used here.
> And then - does it need to be a bitop? Aiui that's what Arm uses, while x86
> doesn't. And I see no reason to use other than plain C operators here. If
> Arm was switched, presumably all the redundant (and misnamed) _IRQ_*
> constants could go away, with just the IRQ_* ones left.

The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473
but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two 
places one in __setup_irq() but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and in vgic_v2_fold_lr_state(). 
Maybe, I'm missing something.|
|RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but __setup_irq() is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock))
and then it will also safe to use non-bitop function.
Probably, it is a little bit safer to use always bitops for desc->status.
||

>>> I'm uncertain about this bit setting anyway - on x86 we would only fiddle
>>> with it for IRQs not in use, not while enabling/disabling one.
> What about this part?

As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned
to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt
and not knowing what to do with it. So it is needed for enablement of IRQs when the guest
requests it and not unconditionally at boot time.

>>>> +    spin_unlock_irqrestore(&aplic.lock, flags);
>>>> +}
>>>> +
>>>> +static void aplic_irq_disable(struct irq_desc *desc)
>>>> +{
>>>> +    unsigned long flags;
>>>> +
>>>> +    /*
>>>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>>>> +     *       If APLIC without MSI interrupts is required in the future,
>>>> +     *       this function will need to be updated accordingly.
>>>> +     */
>>>> +    ASSERT(aplic.imsic_cfg->is_used);
>>>> +
>>>> +    ASSERT(spin_is_locked(&desc->lock));
>>>> +
>>>> +    spin_lock_irqsave(&aplic.lock, flags);
>>>> +
>>>> +    set_bit(_IRQ_DISABLED, &desc->status);
>>>> +
>>>> +    /* disable interrupt in APLIC */
>>>> +    aplic.regs->clrienum = desc->irq;
>>>> +
>>>> +    /* disable interrupt in IMSIC */
>>>> +    imsic_irq_disable(desc->irq);
>>>> +
>>>> +    spin_unlock_irqrestore(&aplic.lock, flags);
>>>> +}
>>>> +
>>>> +static unsigned int aplic_irq_startup(struct irq_desc *desc)
>>>> +{
>>>> +    aplic_irq_enable(desc);
>>>> +
>>>> +    return 0;
>>>> +}
>>>> +
>>>> +static void aplic_irq_shutdown(struct irq_desc *desc)
>>>> +{
>>>> +    aplic_irq_disable(desc);
>>>> +}
>>> You don't really need a separate hook function here, do you?
>> With such implementation it is really not needed to have a hook so
>> I will drop it.
>>
>>>> +static void aplic_irq_ack(struct irq_desc *desc)
>>>> +{
>>>> +    /* nothing to do */
>>>> +}
>>>> +
>>>> +static void aplic_host_irq_end(struct irq_desc *desc)
>>> What's the "host" in the identifier about?
>> It was copied that from Arm and my understanding that it means
>> Xen-related IRQ as they also have:
>> ```
>> /* XXX different for level vs edge */
>> static hw_irq_controller gicv2_host_irq_type = {
>> ...
>>       .end          = gicv2_host_irq_end,
>> ...
>> };
>>
>> static hw_irq_controller gicv2_guest_irq_type = {
>> ...
>>       .end          = gicv2_guest_irq_end,
>> ...
>> };
>> ```
> And you expect to end up with a similar distinction on RISC-V? There's
> nothing like that on x86, just to mention it.

Yes, if one day support for guest interrupts without IMSIC support will be added for APLIC.
(at the moment, we are planning only to have APLIC+IMSIC support as this way is hypervisor-aware)

>>>> +static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
>>>> +{
>>>> +    unsigned int cpu;
>>>> +    uint64_t group_index, base_ppn;
>>>> +    uint32_t hhxw, lhxw ,hhxs, value;
>>>> +    const struct imsic_config *imsic = aplic.imsic_cfg;
>>>> +
>>>> +    /*
>>>> +     * TODO: Currently, APLIC is supported only with MSI interrupts.
>>>> +     *       If APLIC without MSI interrupts is required in the future,
>>>> +     *       this function will need to be updated accordingly.
>>>> +     */
>>>> +    ASSERT(aplic.imsic_cfg->is_used);
>>> Use the local variable you have made yourself?
>> What do you mean by local here?
> Just a few lines up you latch aplic.imsic_cfg into the local "imsic".

Oh, sure, if *->is_used will still present in the next patch series then I'll re-use here "imsic".

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 18159 bytes --]

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Jan Beulich]

On 28.04.2025 10:12, Oleksii Kurochko wrote:
> On 4/17/25 8:25 AM, Jan Beulich wrote:
>> On 16.04.2025 21:05, Oleksii Kurochko wrote:
>>> On 4/15/25 2:46 PM, Jan Beulich wrote:
>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>> Introduce interrupt controller descriptor for host APLIC to describe
>>>>> the low-lovel hardare. It includes implementation of the following functions:
>>>>>    - aplic_irq_startup()
>>>>>    - aplic_irq_shutdown()
>>>>>    - aplic_irq_enable()
>>>>>    - aplic_irq_disable()
>>>>>    - aplic_irq_ack()
>>>>>    - aplic_host_irq_end()
>>>>>    - aplic_set_irq_affinity()
>>>>>
>>>>> As APLIC is used in MSI mode it requires to enable/disable interrupts not
>>>>> only for APLIC but also for IMSIC. Thereby for the purpose of
>>>>> aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
>>>>>
>>>>> For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is
>>>>> introduced to get hart id.
>>>>>
>>>>> Also, introduce additional interrupt controller h/w operations and
>>>>> host_irq_type for APLIC:
>>>>>    - aplic_host_irq_type
>>>>>    - aplic_set_irq_priority()
>>>>>    - aplic_set_irq_type()
>>>> Yet these two functions nor the hooks they're used to populate are entirely
>>>> unused here. Since they're also outside of the common IRQ handling machinery,
>>>> it's unclear how one would sanely ack such a change.
>>> They will be called by intc_route_irq_to_xen() from setup_irq() during firt time
>>> the IRQ is setup.
>> Perhaps move their introduction to there then? We don't do any Misra checking
>> yet lon RISC-V, but imo it's still good practice to avoid introducing new
>> violations, even if only temporarily.
> 
> Okay, I will move their introduction to there.
> 
> Btw, what is needed to add Misra checking for RISC-V? I started to think that, probably,
> it will make sense to do that from the start.

Best I can do is point you at what is done for Arm and x86. You may want to
ask people more familiar with the CI aspects involved here.

>>>>> +static void aplic_set_irq_priority(struct irq_desc *desc,
>>>>> +                                   unsigned int priority)
>>>>> +{
>>>>> +    /* No priority, do nothing */
>>>>> +}
>>>> Since the function dopes nothing, wouldn't it be better to omit it and have
>>>> the (future) caller check for a NULL pointer ahead of making the (indirect)
>>>> call? Same remark for other handlers (below) which also do nothing.
>>> I thought about that too, but it could be some cases when the stub is introduced
>>> with temporary BUG_ON("unimplemented") inside just to not miss to implement it
>>> when it will be necessary.
>>> If we will have only the caller check then we could miss to implement such stubs.
>> I guess I don't understand the concern.
> 
> for example, if we will have the following code:
>    void some_caller(struct irq_desc *desc)
>    {
>        if ( desc->handler->set_affinity )
>            desc->handler->set_affinity(desc, cpu_mask);
>    }
> 
> Then we will skip the call of handler->set_affinity() (if it was just initialized with
> .set_affinity = NULL) without any notification. And it is fine specifically in this
> case as aplic_set_irq_priority() does nothing.
> 
> But what about the cases if it is a function which will have some implementation in the
> future but doesn't have implementation for now. Then without notification that this
> function is unimplemented we could skip something what really matters.
> 
> But I think that your initial comment was just about the function which basically
> does nothing. Am i right?

Since indirect calls are not only more expensive (often; not sure about
RISC-V) but also pose speculative concerns, having such just to do nothing
simply seems like moving in the wrong direction.

>>>>> +    ASSERT(spin_is_locked(&desc->lock));
>>>> If this lock (which is an IRQ-safe one) is necessarily held, ...
>>>>
>>>>> +    spin_lock_irqsave(&aplic.lock, flags);
>>>> ... you can use just spin_lock() here.
>>>>
>>>>> +    clear_bit(_IRQ_DISABLED, &desc->status);
>>>> Why an atomic bitop when desc is locked? (And yes, I ought to raise the same
>>>> question on Arm code also doing so.)
>>> I haven't thought about that. Likely non-atomic bitop could be used here.
>> And then - does it need to be a bitop? Aiui that's what Arm uses, while x86
>> doesn't. And I see no reason to use other than plain C operators here. If
>> Arm was switched, presumably all the redundant (and misnamed) _IRQ_*
>> constants could go away, with just the IRQ_* ones left.
> 
> The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473
> but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two 
> places one in __setup_irq() but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and in vgic_v2_fold_lr_state(). 
> Maybe, I'm missing something.|
> |RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but __setup_irq() is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock))
> and then it will also safe to use non-bitop function.
> Probably, it is a little bit safer to use always bitops for desc->status.
> ||

I question that. If any accesses outside of locked regions were needed (as the
description of that commit suggests), then the situation would be different.

Btw, you not wrapping lines and you adding strange | instances doesn't help
readability of your replies.

>>>> I'm uncertain about this bit setting anyway - on x86 we would only fiddle
>>>> with it for IRQs not in use, not while enabling/disabling one.
>> What about this part?
> 
> As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned
> to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt
> and not knowing what to do with it. So it is needed for enablement of IRQs when the guest
> requests it and not unconditionally at boot time.

I fear I don't understand this. The way we do things on x86 doesn't leave us
in such a situation.

Jan

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 3232 bytes --]


On 4/28/25 10:54 AM, Jan Beulich wrote:
>>>>>> +    ASSERT(spin_is_locked(&desc->lock));
>>>>> If this lock (which is an IRQ-safe one) is necessarily held, ...
>>>>>
>>>>>> +    spin_lock_irqsave(&aplic.lock, flags);
>>>>> ... you can use just spin_lock() here.
>>>>>
>>>>>> +    clear_bit(_IRQ_DISABLED, &desc->status);
>>>>> Why an atomic bitop when desc is locked? (And yes, I ought to raise the same
>>>>> question on Arm code also doing so.)
>>>> I haven't thought about that. Likely non-atomic bitop could be used here.
>>> And then - does it need to be a bitop? Aiui that's what Arm uses, while x86
>>> doesn't. And I see no reason to use other than plain C operators here. If
>>> Arm was switched, presumably all the redundant (and misnamed) _IRQ_*
>>> constants could go away, with just the IRQ_* ones left.
>> The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473
>> but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two
>> places one in __setup_irq() but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and in vgic_v2_fold_lr_state().
>> Maybe, I'm missing something.|
>> |RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but __setup_irq() is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock))
>> and then it will also safe to use non-bitop function.
>> Probably, it is a little bit safer to use always bitops for desc->status.
>> ||
> I question that. If any accesses outside of locked regions were needed (as the
> description of that commit suggests), then the situation would be different.

Okay, then at the moment there is no such cases and I'll use plain C operator instead of
clear/set_bit().

>
> Btw, you not wrapping lines and you adding strange | instances doesn't help
> readability of your replies.
>
>>>>> I'm uncertain about this bit setting anyway - on x86 we would only fiddle
>>>>> with it for IRQs not in use, not while enabling/disabling one.
>>> What about this part?
>> As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned
>> to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt
>> and not knowing what to do with it. So it is needed for enablement of IRQs when the guest
>> requests it and not unconditionally at boot time.
> I fear I don't understand this. The way we do things on x86 doesn't leave us
> in such a situation.

On Arm, the physical interrupts would be enabled when the interrupt is initially routed and in case guest
is booting with interrupt disabled, it could introduce a problem when guest enabled interrupts it will
already have a pending interrupt for which it isn't ready.

How is it handled the case when a device isn't quiescing at the boot time in x86?

But I just realized the way how interrupts are enabled in RISC-V for guest won't lead to such case. The interrupt
will be enabled only when guest's device driver will request that. So this setting/clearing of IRQ_DISABLED could
be dropped for RISC-V.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 5092 bytes --]

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> --- a/xen/arch/riscv/imsic.c
> +++ b/xen/arch/riscv/imsic.c
> @@ -14,12 +14,68 @@
>  #include <xen/errno.h>
>  #include <xen/init.h>
>  #include <xen/macros.h>
> +#include <xen/spinlock.h>
>  #include <xen/xmalloc.h>
>  
>  #include <asm/imsic.h>
>  
>  static struct imsic_config imsic_cfg;
>  
> +#define imsic_csr_set(c, v)     \
> +do {                            \
> +    csr_write(CSR_SISELECT, c); \
> +    csr_set(CSR_SIREG, v);      \
> +} while (0)
> +
> +#define imsic_csr_clear(c, v)   \
> +do {                            \
> +    csr_write(CSR_SISELECT, c); \
> +    csr_clear(CSR_SIREG, v);    \
> +} while (0)

Coming back to these (the later patch adds one more here): How expensive are
these CSR writes? IOW would it perhaps make sense to maintain a local cache
of the last written SISELECT value, to avoid writing the same one again if
the same windowed register needs accessing twice in a row?

Jan

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 1112 bytes --]


On 4/15/25 4:53 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> --- a/xen/arch/riscv/imsic.c
>> +++ b/xen/arch/riscv/imsic.c
>> @@ -14,12 +14,68 @@
>>   #include <xen/errno.h>
>>   #include <xen/init.h>
>>   #include <xen/macros.h>
>> +#include <xen/spinlock.h>
>>   #include <xen/xmalloc.h>
>>   
>>   #include <asm/imsic.h>
>>   
>>   static struct imsic_config imsic_cfg;
>>   
>> +#define imsic_csr_set(c, v)     \
>> +do {                            \
>> +    csr_write(CSR_SISELECT, c); \
>> +    csr_set(CSR_SIREG, v);      \
>> +} while (0)
>> +
>> +#define imsic_csr_clear(c, v)   \
>> +do {                            \
>> +    csr_write(CSR_SISELECT, c); \
>> +    csr_clear(CSR_SIREG, v);    \
>> +} while (0)
> Coming back to these (the later patch adds one more here): How expensive are
> these CSR writes? IOW would it perhaps make sense to maintain a local cache
> of the last written SISELECT value, to avoid writing the same one again if
> the same windowed register needs accessing twice in a row?

CSRs belong to the HART, so access to them is very fast.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 1633 bytes --]

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Jan Beulich]

On 18.04.2025 12:43, Oleksii Kurochko wrote:
> 
> On 4/15/25 4:53 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> --- a/xen/arch/riscv/imsic.c
>>> +++ b/xen/arch/riscv/imsic.c
>>> @@ -14,12 +14,68 @@
>>>   #include <xen/errno.h>
>>>   #include <xen/init.h>
>>>   #include <xen/macros.h>
>>> +#include <xen/spinlock.h>
>>>   #include <xen/xmalloc.h>
>>>   
>>>   #include <asm/imsic.h>
>>>   
>>>   static struct imsic_config imsic_cfg;
>>>   
>>> +#define imsic_csr_set(c, v)     \
>>> +do {                            \
>>> +    csr_write(CSR_SISELECT, c); \
>>> +    csr_set(CSR_SIREG, v);      \
>>> +} while (0)
>>> +
>>> +#define imsic_csr_clear(c, v)   \
>>> +do {                            \
>>> +    csr_write(CSR_SISELECT, c); \
>>> +    csr_clear(CSR_SIREG, v);    \
>>> +} while (0)
>> Coming back to these (the later patch adds one more here): How expensive are
>> these CSR writes? IOW would it perhaps make sense to maintain a local cache
>> of the last written SISELECT value, to avoid writing the same one again if
>> the same windowed register needs accessing twice in a row?
> 
> CSRs belong to the HART, so access to them is very fast.

Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs,
and access (writes in particular) to some of them is rather slow.

Jan

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 1479 bytes --]


On 4/22/25 9:02 AM, Jan Beulich wrote:
> On 18.04.2025 12:43, Oleksii Kurochko wrote:
>> On 4/15/25 4:53 PM, Jan Beulich wrote:
>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>> --- a/xen/arch/riscv/imsic.c
>>>> +++ b/xen/arch/riscv/imsic.c
>>>> @@ -14,12 +14,68 @@
>>>>    #include <xen/errno.h>
>>>>    #include <xen/init.h>
>>>>    #include <xen/macros.h>
>>>> +#include <xen/spinlock.h>
>>>>    #include <xen/xmalloc.h>
>>>>    
>>>>    #include <asm/imsic.h>
>>>>    
>>>>    static struct imsic_config imsic_cfg;
>>>>    
>>>> +#define imsic_csr_set(c, v)     \
>>>> +do {                            \
>>>> +    csr_write(CSR_SISELECT, c); \
>>>> +    csr_set(CSR_SIREG, v);      \
>>>> +} while (0)
>>>> +
>>>> +#define imsic_csr_clear(c, v)   \
>>>> +do {                            \
>>>> +    csr_write(CSR_SISELECT, c); \
>>>> +    csr_clear(CSR_SIREG, v);    \
>>>> +} while (0)
>>> Coming back to these (the later patch adds one more here): How expensive are
>>> these CSR writes? IOW would it perhaps make sense to maintain a local cache
>>> of the last written SISELECT value, to avoid writing the same one again if
>>> the same windowed register needs accessing twice in a row?
>> CSRs belong to the HART, so access to them is very fast.
> Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs,
> and access (writes in particular) to some of them is rather slow.

CSR read 1 cycle, CSR write 7 cycles on Microchip platform. ~ Oleksii

[-- Attachment #2: Type: text/html, Size: 2240 bytes --]

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

[Jan Beulich]

On 25.04.2025 21:31, Oleksii Kurochko wrote:
> 
> On 4/22/25 9:02 AM, Jan Beulich wrote:
>> On 18.04.2025 12:43, Oleksii Kurochko wrote:
>>> On 4/15/25 4:53 PM, Jan Beulich wrote:
>>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>>> --- a/xen/arch/riscv/imsic.c
>>>>> +++ b/xen/arch/riscv/imsic.c
>>>>> @@ -14,12 +14,68 @@
>>>>>    #include <xen/errno.h>
>>>>>    #include <xen/init.h>
>>>>>    #include <xen/macros.h>
>>>>> +#include <xen/spinlock.h>
>>>>>    #include <xen/xmalloc.h>
>>>>>    
>>>>>    #include <asm/imsic.h>
>>>>>    
>>>>>    static struct imsic_config imsic_cfg;
>>>>>    
>>>>> +#define imsic_csr_set(c, v)     \
>>>>> +do {                            \
>>>>> +    csr_write(CSR_SISELECT, c); \
>>>>> +    csr_set(CSR_SIREG, v);      \
>>>>> +} while (0)
>>>>> +
>>>>> +#define imsic_csr_clear(c, v)   \
>>>>> +do {                            \
>>>>> +    csr_write(CSR_SISELECT, c); \
>>>>> +    csr_clear(CSR_SIREG, v);    \
>>>>> +} while (0)
>>>> Coming back to these (the later patch adds one more here): How expensive are
>>>> these CSR writes? IOW would it perhaps make sense to maintain a local cache
>>>> of the last written SISELECT value, to avoid writing the same one again if
>>>> the same windowed register needs accessing twice in a row?
>>> CSRs belong to the HART, so access to them is very fast.
>> Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs,
>> and access (writes in particular) to some of them is rather slow.
> 
> CSR read 1 cycle, CSR write 7 cycles on Microchip platform. ~ Oleksii

And that's an in-order platform, i.e. cycle count being all that matters for
performance? No other (e.g. latency) effect on subsequent insns?

Further, how does this compare to the outlined alternative, especially if we
assumed that the respective cacheline would be hot and hence usually in L1
cache?

Jan

[PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode

[Oleksii Kurochko]

Implement functions necessarry to have working external interrupts in
hypervisor mode. The following changes are done:
 - Add a common function intc_handle_external_irq() to call APLIC specific
   function to handle an interrupt.
 - Update do_trap() function to handle IRQ_S_EXT case; add the check to catch
   case when cause of trap is an interrupt.
 - Add handle_interrrupt() member to intc_hw_operations structure.
 - Enable local interrupt delivery for IMSIC by implementation and calling of
   imsic_ids_local_delivery() in imsic_init(); additionally introduce helper
   imsic_csr_write() to update IMSIC_EITHRESHOLD and IMSIC_EITHRESHOLD.
 - Enable hypervisor external interrupts.
 - Implement aplic_handler_interrupt() and use it to init ->handle_interrupt
   member of intc_hw_operations for APLIC.
 - Add implementation of do_IRQ() to dispatch the interrupt.

The current patch is based on the code from [1].

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5

Co-developed-by: Romain Caritey <Romain.Caritey@microchip.com>
Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/aplic.c             | 19 +++++++++++++
 xen/arch/riscv/imsic.c             | 25 +++++++++++++++++
 xen/arch/riscv/include/asm/imsic.h |  7 +++++
 xen/arch/riscv/include/asm/intc.h  |  5 ++++
 xen/arch/riscv/include/asm/irq.h   |  3 +++
 xen/arch/riscv/intc.c              |  7 +++++
 xen/arch/riscv/irq.c               | 43 ++++++++++++++++++++++++++++++
 xen/arch/riscv/traps.c             | 18 +++++++++++++
 8 files changed, 127 insertions(+)

diff --git a/xen/arch/riscv/aplic.c b/xen/arch/riscv/aplic.c
index 4b60cb9a77..38b57ed1ac 100644
--- a/xen/arch/riscv/aplic.c
+++ b/xen/arch/riscv/aplic.c
@@ -261,6 +261,21 @@ static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
     spin_unlock(&aplic.lock);
 }
 
+static void aplic_handle_interrupt(unsigned long cause, struct cpu_user_regs *regs)
+{
+    /* disable to avoid more external interrupts */
+    csr_clear(CSR_SIE, 1UL << IRQ_S_EXT);
+
+    /* clear the pending bit */
+    csr_clear(CSR_SIP, 1UL << IRQ_S_EXT);
+
+    /* dispatch the interrupt */
+    do_IRQ(regs, csr_swap(CSR_STOPEI, 0) >> TOPI_IID_SHIFT);
+
+    /* enable external interrupts */
+    csr_set(CSR_SIE, 1UL << IRQ_S_EXT);
+}
+
 static hw_irq_controller aplic_host_irq_type = {
     .typename     = "aplic",
     .startup      = aplic_irq_startup,
@@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = {
     .host_irq_type       = &aplic_host_irq_type,
     .set_irq_priority    = aplic_set_irq_priority,
     .set_irq_type        = aplic_set_irq_type,
+    .handle_interrupt    = aplic_handle_interrupt,
 };
 
 static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
@@ -318,6 +334,9 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     register_intc_ops(&aplic_ops);
 
+    /* Enable supervisor external interrupt */
+    csr_set(CSR_SIE, 1UL << IRQ_S_EXT);
+
     return 0;
 }
 
diff --git a/xen/arch/riscv/imsic.c b/xen/arch/riscv/imsic.c
index 8198d008ef..e00f2d69df 100644
--- a/xen/arch/riscv/imsic.c
+++ b/xen/arch/riscv/imsic.c
@@ -19,8 +19,19 @@
 
 #include <asm/imsic.h>
 
+#define IMSIC_DISABLE_EIDELIVERY    0
+#define IMSIC_ENABLE_EIDELIVERY     1
+#define IMSIC_DISABLE_EITHRESHOLD   1
+#define IMSIC_ENABLE_EITHRESHOLD    0
+
 static struct imsic_config imsic_cfg;
 
+#define imsic_csr_write(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_write(CSR_SIREG, v);    \
+} while (0)
+
 #define imsic_csr_set(c, v)     \
 do {                            \
     csr_write(CSR_SISELECT, c); \
@@ -33,6 +44,20 @@ do {                            \
     csr_clear(CSR_SIREG, v);    \
 } while (0)
 
+void imsic_ids_local_delivery(bool enable)
+{
+    if ( enable )
+    {
+        imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
+        imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
+    }
+    else
+    {
+        imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
+        imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
+    }
+}
+
 static void imsic_local_eix_update(unsigned long base_id, unsigned long num_id,
                                    bool pend, bool val)
 {
diff --git a/xen/arch/riscv/include/asm/imsic.h b/xen/arch/riscv/include/asm/imsic.h
index d2c0178529..b2c674f271 100644
--- a/xen/arch/riscv/include/asm/imsic.h
+++ b/xen/arch/riscv/include/asm/imsic.h
@@ -12,6 +12,7 @@
 #define ASM__RISCV__IMSIC_H
 
 #include <xen/spinlock.h>
+#include <xen/stdbool.h>
 #include <xen/types.h>
 
 #define IMSIC_MMIO_PAGE_SHIFT   12
@@ -20,6 +21,10 @@
 #define IMSIC_MIN_ID            63
 #define IMSIC_MAX_ID            2048
 
+#define IMSIC_EIDELIVERY        0x70
+
+#define IMSIC_EITHRESHOLD       0x72
+
 #define IMSIC_EIP0              0x80
 #define IMSIC_EIPx_BITS         32
 
@@ -78,4 +83,6 @@ const struct imsic_config *imsic_get_config(void);
 void imsic_irq_enable(unsigned int hwirq);
 void imsic_irq_disable(unsigned int hwirq);
 
+void imsic_ids_local_delivery(bool enable);
+
 #endif /* ASM__RISCV__IMSIC_H */
diff --git a/xen/arch/riscv/include/asm/intc.h b/xen/arch/riscv/include/asm/intc.h
index db53caa07b..e4363af87d 100644
--- a/xen/arch/riscv/include/asm/intc.h
+++ b/xen/arch/riscv/include/asm/intc.h
@@ -34,6 +34,8 @@ struct intc_hw_operations {
     /* Set IRQ priority */
     void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
 
+    /* handle external interrupt */
+    void (*handle_interrupt)(unsigned long cause, struct cpu_user_regs *regs);
 };
 
 void intc_preinit(void);
@@ -45,4 +47,7 @@ void register_intc_ops(const struct intc_hw_operations *ops);
 struct irq_desc;
 void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority);
 
+struct cpu_user_regs;
+void intc_handle_external_irqs(unsigned long cause, struct cpu_user_regs *regs);
+
 #endif /* ASM__RISCV__INTERRUPT_CONTOLLER_H */
diff --git a/xen/arch/riscv/include/asm/irq.h b/xen/arch/riscv/include/asm/irq.h
index 163a478d78..9558d3fa61 100644
--- a/xen/arch/riscv/include/asm/irq.h
+++ b/xen/arch/riscv/include/asm/irq.h
@@ -51,6 +51,9 @@ int platform_get_irq(const struct dt_device_node *device, int index);
 
 void init_IRQ(void);
 
+struct cpu_user_regs;
+void do_IRQ(struct cpu_user_regs *regs, unsigned int irq);
+
 #endif /* ASM__RISCV__IRQ_H */
 
 /*
diff --git a/xen/arch/riscv/intc.c b/xen/arch/riscv/intc.c
index 8274897d8c..41a4310ead 100644
--- a/xen/arch/riscv/intc.c
+++ b/xen/arch/riscv/intc.c
@@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
     intc_hw_ops->set_irq_priority(desc, priority);
 }
 
+void intc_handle_external_irqs(unsigned long cause, struct cpu_user_regs *regs)
+{
+    ASSERT(intc_hw_ops && intc_hw_ops->handle_interrupt);
+
+    intc_hw_ops->handle_interrupt(cause, regs);
+}
+
 void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority)
 {
     ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
diff --git a/xen/arch/riscv/irq.c b/xen/arch/riscv/irq.c
index c332e000c4..3c0b95220a 100644
--- a/xen/arch/riscv/irq.c
+++ b/xen/arch/riscv/irq.c
@@ -11,6 +11,10 @@
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/irq.h>
+#include <xen/spinlock.h>
+
+#include <asm/hardirq.h>
+#include <asm/intc.h>
 
 static irq_desc_t irq_desc[NR_IRQS];
 
@@ -83,3 +87,42 @@ void __init init_IRQ(void)
     if ( init_irq_data() < 0 )
         panic("initialization of IRQ data failed\n");
 }
+
+/* Dispatch an interrupt */
+void do_IRQ(struct cpu_user_regs *regs, unsigned int irq)
+{
+    struct irq_desc *desc = irq_to_desc(irq);
+    struct irqaction *action;
+
+    irq_enter();
+
+    spin_lock(&desc->lock);
+    desc->handler->ack(desc);
+
+    if ( test_bit(_IRQ_DISABLED, &desc->status) )
+        goto out;
+
+    set_bit(_IRQ_INPROGRESS, &desc->status);
+
+    action = desc->action;
+
+    spin_unlock_irq(&desc->lock);
+
+#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION
+    action->handler(irq, action->dev_id);
+#else
+    do {
+        action->handler(irq, action->dev_id);
+        action = action->next;
+    } while ( action );
+#endif /* CONFIG_IRQ_HAS_MULTIPLE_ACTION */
+
+    spin_lock_irq(&desc->lock);
+
+    clear_bit(_IRQ_INPROGRESS, &desc->status);
+
+out:
+    desc->handler->end(desc);
+    spin_unlock(&desc->lock);
+    irq_exit();
+}
diff --git a/xen/arch/riscv/traps.c b/xen/arch/riscv/traps.c
index ea3638a54f..da5813e34a 100644
--- a/xen/arch/riscv/traps.c
+++ b/xen/arch/riscv/traps.c
@@ -11,6 +11,7 @@
 #include <xen/nospec.h>
 #include <xen/sched.h>
 
+#include <asm/intc.h>
 #include <asm/processor.h>
 #include <asm/riscv_encoding.h>
 #include <asm/traps.h>
@@ -128,6 +129,23 @@ void do_trap(struct cpu_user_regs *cpu_regs)
         }
         fallthrough;
     default:
+        if ( cause & CAUSE_IRQ_FLAG )
+        {
+            /* Handle interrupt */
+            unsigned long icause = cause & ~CAUSE_IRQ_FLAG;
+
+            switch ( icause )
+            {
+            case IRQ_S_EXT:
+                intc_handle_external_irqs(cause, cpu_regs);
+                break;
+            default:
+                break;
+            }
+
+            break;
+        }
+
         do_unexpected_trap(cpu_regs);
         break;
     }
-- 
2.49.0

Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> Implement functions necessarry to have working external interrupts in
> hypervisor mode. The following changes are done:
>  - Add a common function intc_handle_external_irq() to call APLIC specific
>    function to handle an interrupt.
>  - Update do_trap() function to handle IRQ_S_EXT case; add the check to catch
>    case when cause of trap is an interrupt.
>  - Add handle_interrrupt() member to intc_hw_operations structure.
>  - Enable local interrupt delivery for IMSIC by implementation and calling of
>    imsic_ids_local_delivery() in imsic_init();

Ah, here is where that call really belongs (see my question on the earlier
patch). Please make sure you series builds okay at every patch boundary.

> --- a/xen/arch/riscv/aplic.c
> +++ b/xen/arch/riscv/aplic.c
> @@ -261,6 +261,21 @@ static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
>      spin_unlock(&aplic.lock);
>  }
>  
> +static void aplic_handle_interrupt(unsigned long cause, struct cpu_user_regs *regs)
> +{
> +    /* disable to avoid more external interrupts */
> +    csr_clear(CSR_SIE, 1UL << IRQ_S_EXT);

Didn't I see you use BIT() elsewhere? Would be nice to be overall consistent
at least within related code.

> +    /* clear the pending bit */
> +    csr_clear(CSR_SIP, 1UL << IRQ_S_EXT);
> +
> +    /* dispatch the interrupt */
> +    do_IRQ(regs, csr_swap(CSR_STOPEI, 0) >> TOPI_IID_SHIFT);
> +
> +    /* enable external interrupts */
> +    csr_set(CSR_SIE, 1UL << IRQ_S_EXT);
> +}

Why does "cause" need passing into here? I realize the function is used ...

> @@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = {
>      .host_irq_type       = &aplic_host_irq_type,
>      .set_irq_priority    = aplic_set_irq_priority,
>      .set_irq_type        = aplic_set_irq_type,
> +    .handle_interrupt    = aplic_handle_interrupt,
>  };

... as a hook, yet it's still unclear whether (why) any other such hook
would need the cause to be passed in.

> @@ -33,6 +44,20 @@ do {                            \
>      csr_clear(CSR_SIREG, v);    \
>  } while (0)
>  
> +void imsic_ids_local_delivery(bool enable)

__init as long as the sole caller is such?

> --- a/xen/arch/riscv/include/asm/intc.h
> +++ b/xen/arch/riscv/include/asm/intc.h
> @@ -34,6 +34,8 @@ struct intc_hw_operations {
>      /* Set IRQ priority */
>      void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
>  
> +    /* handle external interrupt */
> +    void (*handle_interrupt)(unsigned long cause, struct cpu_user_regs *regs);
>  };
>  
>  void intc_preinit(void);
> @@ -45,4 +47,7 @@ void register_intc_ops(const struct intc_hw_operations *ops);
>  struct irq_desc;
>  void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority);
>  
> +struct cpu_user_regs;

This is too late - you've used it above already. It either can be dropped,
or needs to move up.

> --- a/xen/arch/riscv/intc.c
> +++ b/xen/arch/riscv/intc.c
> @@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
>      intc_hw_ops->set_irq_priority(desc, priority);
>  }
>  
> +void intc_handle_external_irqs(unsigned long cause, struct cpu_user_regs *regs)
> +{
> +    ASSERT(intc_hw_ops && intc_hw_ops->handle_interrupt);

I don't view such checks (on every interrupt) as very useful. If you checked
once early on - okay. But here you gain nothing at all ...

> +    intc_hw_ops->handle_interrupt(cause, regs);

... towards the use here, when considering a release build.


> @@ -83,3 +87,42 @@ void __init init_IRQ(void)
>      if ( init_irq_data() < 0 )
>          panic("initialization of IRQ data failed\n");
>  }
> +
> +/* Dispatch an interrupt */
> +void do_IRQ(struct cpu_user_regs *regs, unsigned int irq)
> +{
> +    struct irq_desc *desc = irq_to_desc(irq);
> +    struct irqaction *action;
> +
> +    irq_enter();
> +
> +    spin_lock(&desc->lock);
> +    desc->handler->ack(desc);
> +
> +    if ( test_bit(_IRQ_DISABLED, &desc->status) )
> +        goto out;
> +
> +    set_bit(_IRQ_INPROGRESS, &desc->status);

Same comment as on the earlier patch - atomic bitop when in a suitably
locked region?

> +    action = desc->action;
> +
> +    spin_unlock_irq(&desc->lock);
> +
> +#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION

Stolen from Arm? What's this about?

> +    action->handler(irq, action->dev_id);
> +#else
> +    do {
> +        action->handler(irq, action->dev_id);
> +        action = action->next;
> +    } while ( action );
> +#endif /* CONFIG_IRQ_HAS_MULTIPLE_ACTION */
> +
> +    spin_lock_irq(&desc->lock);
> +
> +    clear_bit(_IRQ_INPROGRESS, &desc->status);

See above.

> +out:

Nit (you know what).

> @@ -128,6 +129,23 @@ void do_trap(struct cpu_user_regs *cpu_regs)
>          }
>          fallthrough;
>      default:
> +        if ( cause & CAUSE_IRQ_FLAG )
> +        {
> +            /* Handle interrupt */
> +            unsigned long icause = cause & ~CAUSE_IRQ_FLAG;
> +
> +            switch ( icause )
> +            {
> +            case IRQ_S_EXT:
> +                intc_handle_external_irqs(cause, cpu_regs);
> +                break;
> +            default:

Nit: Blank line please between non-fall-through case blocks.

Jan

Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 3198 bytes --]


On 4/15/25 4:42 PM, Jan Beulich wrote:
>
>> +    /* clear the pending bit */
>> +    csr_clear(CSR_SIP, 1UL << IRQ_S_EXT);
>> +
>> +    /* dispatch the interrupt */
>> +    do_IRQ(regs, csr_swap(CSR_STOPEI, 0) >> TOPI_IID_SHIFT);
>> +
>> +    /* enable external interrupts */
>> +    csr_set(CSR_SIE, 1UL << IRQ_S_EXT);
>> +}
> Why does "cause" need passing into here? I realize the function is used ...
>
>> @@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = {
>>       .host_irq_type       = &aplic_host_irq_type,
>>       .set_irq_priority    = aplic_set_irq_priority,
>>       .set_irq_type        = aplic_set_irq_type,
>> +    .handle_interrupt    = aplic_handle_interrupt,
>>   };
> ... as a hook, yet it's still unclear whether (why) any other such hook
> would need the cause to be passed in.

I don't remember a particular reason, but it could have been dropped. If, for some reason,
the cause is needed in|handle_interrupt()|, it can be retrieved explicitly from a register.

>
>> @@ -33,6 +44,20 @@ do {                            \
>>       csr_clear(CSR_SIREG, v);    \
>>   } while (0)
>>   
>> +void imsic_ids_local_delivery(bool enable)
> __init as long as the sole caller is such?

Yes, it make sense. Also, I noticed some other functions that could be __init in imsic.c (but likely
you mentioned that in the previous patches).

>> --- a/xen/arch/riscv/intc.c
>> +++ b/xen/arch/riscv/intc.c
>> @@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
>>       intc_hw_ops->set_irq_priority(desc, priority);
>>   }
>>   
>> +void intc_handle_external_irqs(unsigned long cause, struct cpu_user_regs *regs)
>> +{
>> +    ASSERT(intc_hw_ops && intc_hw_ops->handle_interrupt);
> I don't view such checks (on every interrupt) as very useful. If you checked
> once early on - okay. But here you gain nothing at all ...
>
>> +    intc_hw_ops->handle_interrupt(cause, regs);
> ... towards the use here, when considering a release build.

I will try to find a better place then.

>
>
>> @@ -83,3 +87,42 @@ void __init init_IRQ(void)
>>       if ( init_irq_data() < 0 )
>>           panic("initialization of IRQ data failed\n");
>>   }
>> +
>> +/* Dispatch an interrupt */
>> +void do_IRQ(struct cpu_user_regs *regs, unsigned int irq)
>> +{
>> +    struct irq_desc *desc = irq_to_desc(irq);
>> +    struct irqaction *action;
>> +
>> +    irq_enter();
>> +
>> +    spin_lock(&desc->lock);
>> +    desc->handler->ack(desc);
>> +
>> +    if ( test_bit(_IRQ_DISABLED, &desc->status) )
>> +        goto out;
>> +
>> +    set_bit(_IRQ_INPROGRESS, &desc->status);
> Same comment as on the earlier patch - atomic bitop when in a suitably
> locked region?

Agree, it could be used non-atomic bitop.

>
>> +    action = desc->action;
>> +
>> +    spin_unlock_irq(&desc->lock);
>> +
>> +#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION
> Stolen from Arm? What's this about?

Yes, it is stolen from Arm. I thought that it is a generic one, but the config is defined
inside Arm's config.h.
Then it could be dropped now as I don't know, at the moment, the cases when it is neeeded
to exectute several handler for an irq for RISC-V.

Thanks.

~ Oleksii


[-- Attachment #2: Type: text/html, Size: 5183 bytes --]

Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 577 bytes --]


On 4/17/25 10:44 AM, Oleksii Kurochko wrote:
>>> +    action = desc->action;
>>> +
>>> +    spin_unlock_irq(&desc->lock);
>>> +
>>> +#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION
>> Stolen from Arm? What's this about?
> Yes, it is stolen from Arm. I thought that it is a generic one, but the config is defined
> inside Arm's config.h.
> Then it could be dropped now as I don't know, at the moment, the cases when it is neeeded
> to exectute several handler for an irq for RISC-V.

Probably, IOMMU may setup multiple handler for the same interrupt. I'll double check that.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 1259 bytes --]

[PATCH v1 12/14] xen/riscv: implement setup_irq()

[Oleksii Kurochko]

Introduce support for IRQ setup on RISC-V by implementing `setup_irq()` and
`__setup_irq()`, adapted and extended from an initial implementation by [1].

__setup_irq() does the following:
  - Sets up an IRQ action.
  - Validates that shared IRQs have non-NULL `dev_id` and are only used when
    existing handlers allow sharing.
  - Uses `smp_mb()` to enforce memory ordering after assigning `desc->action`
    to ensure visibility before enabling the IRQ.
  - Supports multi-action setups via `CONFIG_IRQ_HAS_MULTIPLE_ACTION`.

setup_irq() does the following:
  - Converts IRQ number to descriptor and acquires its lock.
  - Rejects registration if the IRQ is already assigned to a guest domain,
    printing an error.
  - Delegates the core setup to `__setup_irq()`.
  - On first-time setup, disables the IRQ, routes it to Xen using
    `intc_route_irq_to_xen()`, sets default CPU affinity (current CPU),
    calls the handler’s startup routine, and finally enables the IRQ.

irq_set_affinity() invokes `set_affinity` callback from the IRQ handler
if present.

Defined `IRQ_NO_PRIORITY` as default priority used when routing IRQs to Xen.

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5

Co-developed-by: Romain Caritey <Romain.Caritey@microchip.com>
Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/include/asm/irq.h |  6 ++
 xen/arch/riscv/irq.c             | 95 ++++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)

diff --git a/xen/arch/riscv/include/asm/irq.h b/xen/arch/riscv/include/asm/irq.h
index 9558d3fa61..bba3a97e3e 100644
--- a/xen/arch/riscv/include/asm/irq.h
+++ b/xen/arch/riscv/include/asm/irq.h
@@ -26,6 +26,8 @@
 #define IRQ_TYPE_SENSE_MASK     DT_IRQ_TYPE_SENSE_MASK
 #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
+#define IRQ_NO_PRIORITY 0
+
 /* TODO */
 #define nr_static_irqs 0
 #define arch_hwdom_irqs(domid) 0U
@@ -54,6 +56,10 @@ void init_IRQ(void);
 struct cpu_user_regs;
 void do_IRQ(struct cpu_user_regs *regs, unsigned int irq);
 
+struct irq_desc;
+struct cpumask_t;
+void irq_set_affinity(struct irq_desc *desc, const cpumask_t *cpu_mask);
+
 #endif /* ASM__RISCV__IRQ_H */
 
 /*
diff --git a/xen/arch/riscv/irq.c b/xen/arch/riscv/irq.c
index 3c0b95220a..1e937d4306 100644
--- a/xen/arch/riscv/irq.c
+++ b/xen/arch/riscv/irq.c
@@ -6,7 +6,9 @@
  * Copyright (c) 2024 Vates
  */
 
+#include <xen/bitops.h>
 #include <xen/bug.h>
+#include <xen/cpumask.h>
 #include <xen/device_tree.h>
 #include <xen/errno.h>
 #include <xen/init.h>
@@ -57,6 +59,99 @@ int platform_get_irq(const struct dt_device_node *device, int index)
     return dt_irq.irq;
 }
 
+static int __setup_irq(struct irq_desc *desc, unsigned int irqflags,
+                       struct irqaction *new)
+{
+    bool shared = irqflags & IRQF_SHARED;
+
+    ASSERT(new != NULL);
+
+    /* Sanity checks:
+     *  - if the IRQ is marked as shared
+     *  - dev_id is not NULL when IRQF_SHARED is set
+     */
+    if ( desc->action != NULL && (!test_bit(_IRQF_SHARED, &desc->status)
+         || !shared) )
+        return -EINVAL;
+    if ( shared && new->dev_id == NULL )
+        return -EINVAL;
+
+    if ( shared )
+        set_bit(_IRQF_SHARED, &desc->status);
+
+#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION
+    new->next = desc->action;
+    smp_mb();
+#endif
+
+    desc->action = new;
+    smp_mb();
+
+    return 0;
+}
+
+void irq_set_affinity(struct irq_desc *desc, const cpumask_t *cpu_mask)
+{
+    if ( desc != NULL )
+        desc->handler->set_affinity(desc, cpu_mask);
+}
+
+int setup_irq(unsigned int irq, unsigned int irqflags, struct irqaction *new)
+{
+    int rc;
+    unsigned long flags;
+    struct irq_desc *desc;
+    bool disabled;
+
+    desc = irq_to_desc(irq);
+
+    spin_lock_irqsave(&desc->lock, flags);
+
+    disabled = (desc->action == NULL);
+
+    if ( test_bit(_IRQ_GUEST, &desc->status) )
+    {
+        spin_unlock_irqrestore(&desc->lock, flags);
+        /*
+         * TODO: would be nice to have functionality to print which domain owns
+         *       an IRQ.
+         */
+        printk(XENLOG_ERR "ERROR: IRQ %u is already in use by a domain\n", irq);
+        return -EBUSY;
+    }
+
+    rc = __setup_irq(desc, irqflags, new);
+    if ( rc )
+        goto err;
+
+    /* First time the IRQ is setup */
+    if ( disabled )
+    {
+        /* disable irq by default */
+        set_bit(_IRQ_DISABLED, &desc->status);
+
+        /* route interrupt to xen */
+        intc_route_irq_to_xen(desc, IRQ_NO_PRIORITY);
+
+        /*
+         * we don't care for now which CPU will receive the
+         * interrupt
+         *
+         * TODO: Handle case where IRQ is setup on different CPU than
+         * the targeted CPU and the priority.
+         */
+        irq_set_affinity(desc, cpumask_of(smp_processor_id()));
+        desc->handler->startup(desc);
+        /* enable irq */
+        clear_bit(_IRQ_DISABLED, &desc->status);
+    }
+
+err:
+    spin_unlock_irqrestore(&desc->lock, flags);
+
+    return rc;
+}
+
 int arch_init_one_irq_desc(struct irq_desc *desc)
 {
     desc->arch.type = IRQ_TYPE_INVALID;
-- 
2.49.0

Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> --- a/xen/arch/riscv/include/asm/irq.h
> +++ b/xen/arch/riscv/include/asm/irq.h
> @@ -26,6 +26,8 @@
>  #define IRQ_TYPE_SENSE_MASK     DT_IRQ_TYPE_SENSE_MASK
>  #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
>  
> +#define IRQ_NO_PRIORITY 0
> +
>  /* TODO */
>  #define nr_static_irqs 0
>  #define arch_hwdom_irqs(domid) 0U
> @@ -54,6 +56,10 @@ void init_IRQ(void);
>  struct cpu_user_regs;

Seeing this and ...

>  void do_IRQ(struct cpu_user_regs *regs, unsigned int irq);
>  
> +struct irq_desc;
> +struct cpumask_t;

... now these - all of such forward decls may want to collectively live in a
central place higher up in the file.

> @@ -57,6 +59,99 @@ int platform_get_irq(const struct dt_device_node *device, int index)
>      return dt_irq.irq;
>  }
>  
> +static int __setup_irq(struct irq_desc *desc, unsigned int irqflags,
> +                       struct irqaction *new)

Irrespective of you possibly having found it like this elsewhere, may I
suggest that in new code we avoid leading double underscores? A single one
will do here.

> +{
> +    bool shared = irqflags & IRQF_SHARED;
> +
> +    ASSERT(new != NULL);
> +
> +    /* Sanity checks:

Nit: Comment style (and there are many more issues below).

> +     *  - if the IRQ is marked as shared
> +     *  - dev_id is not NULL when IRQF_SHARED is set
> +     */
> +    if ( desc->action != NULL && (!test_bit(_IRQF_SHARED, &desc->status)
> +         || !shared) )

Nit: Operator placement and indentation.

You're probably better off this way anyway:

    if ( desc->action != NULL &&
         (!test_bit(_IRQF_SHARED, &desc->status) || !shared) )

> +        return -EINVAL;
> +    if ( shared && new->dev_id == NULL )
> +        return -EINVAL;
> +
> +    if ( shared )
> +        set_bit(_IRQF_SHARED, &desc->status);

See comments on earlier patches.

> +#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION
> +    new->next = desc->action;
> +    smp_mb();
> +#endif
> +
> +    desc->action = new;
> +    smp_mb();

Aren't smp_wmb() sufficient on both places? If not, I think comments
want adding.

> +    return 0;
> +}
> +
> +void irq_set_affinity(struct irq_desc *desc, const cpumask_t *cpu_mask)
> +{
> +    if ( desc != NULL )

Can desc really be NULL here? Isn't desc->lock required to be held?

> +        desc->handler->set_affinity(desc, cpu_mask);
> +}
> +
> +int setup_irq(unsigned int irq, unsigned int irqflags, struct irqaction *new)
> +{
> +    int rc;
> +    unsigned long flags;
> +    struct irq_desc *desc;
> +    bool disabled;
> +
> +    desc = irq_to_desc(irq);

Make this the variable's initializer?

> +    spin_lock_irqsave(&desc->lock, flags);
> +
> +    disabled = (desc->action == NULL);
> +
> +    if ( test_bit(_IRQ_GUEST, &desc->status) )
> +    {
> +        spin_unlock_irqrestore(&desc->lock, flags);
> +        /*
> +         * TODO: would be nice to have functionality to print which domain owns
> +         *       an IRQ.
> +         */
> +        printk(XENLOG_ERR "ERROR: IRQ %u is already in use by a domain\n", irq);
> +        return -EBUSY;
> +    }
> +
> +    rc = __setup_irq(desc, irqflags, new);
> +    if ( rc )
> +        goto err;
> +
> +    /* First time the IRQ is setup */
> +    if ( disabled )
> +    {
> +        /* disable irq by default */
> +        set_bit(_IRQ_DISABLED, &desc->status);

Shouldn't this be set when we make it here?

> +        /* route interrupt to xen */
> +        intc_route_irq_to_xen(desc, IRQ_NO_PRIORITY);
> +
> +        /*
> +         * we don't care for now which CPU will receive the
> +         * interrupt
> +         *
> +         * TODO: Handle case where IRQ is setup on different CPU than
> +         * the targeted CPU and the priority.
> +         */
> +        irq_set_affinity(desc, cpumask_of(smp_processor_id()));
> +        desc->handler->startup(desc);
> +        /* enable irq */
> +        clear_bit(_IRQ_DISABLED, &desc->status);

Now it turns out this is really done twice: Once in aplic_irq_enable(),
and once here.

> +    }
> +
> +err:

Nit (yet once again).

Jan

Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 2573 bytes --]


On 4/15/25 5:55 PM, Jan Beulich wrote
>> +#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION
>> +    new->next = desc->action;
>> +    smp_mb();
>> +#endif
>> +
>> +    desc->action = new;
>> +    smp_mb();
> Aren't smp_wmb() sufficient on both places? If not, I think comments
> want adding.

smp_wmb() will be sufficient but I think the barrier could be dropped at all
as __setup_irq() is called only in setup_irq() and __setup_irq() call is wrapped
by spinlock_{un}lock_irqsave() where spinlock_unlock_*() will put barrier.

>
>> +    return 0;
>> +}
>> +
>> +void irq_set_affinity(struct irq_desc *desc, const cpumask_t *cpu_mask)
>> +{
>> +    if ( desc != NULL )
> Can desc really be NULL here?

It can't as irq_desc[] isn't dynamically allocated.

>   Isn't desc->lock required to be held?

It is and it is called in setup_irq() which calls spin_lock_irqsave().
Anyway, I think it could be dropped at all and use 'desc->handler->set_affinity(desc, cpu_mask);'
explicitly in setup_irq().

>> +    spin_lock_irqsave(&desc->lock, flags);
>> +
>> +    disabled = (desc->action == NULL);
>> +
>> +    if ( test_bit(_IRQ_GUEST, &desc->status) )
>> +    {
>> +        spin_unlock_irqrestore(&desc->lock, flags);
>> +        /*
>> +         * TODO: would be nice to have functionality to print which domain owns
>> +         *       an IRQ.
>> +         */
>> +        printk(XENLOG_ERR "ERROR: IRQ %u is already in use by a domain\n", irq);
>> +        return -EBUSY;
>> +    }
>> +
>> +    rc = __setup_irq(desc, irqflags, new);
>> +    if ( rc )
>> +        goto err;
>> +
>> +    /* First time the IRQ is setup */
>> +    if ( disabled )
>> +    {
>> +        /* disable irq by default */
>> +        set_bit(_IRQ_DISABLED, &desc->status);
> Shouldn't this be set when we make it here?

It should be. I'll drop the setting of _IRQ_DISABLED.

>
>> +        /* route interrupt to xen */
>> +        intc_route_irq_to_xen(desc, IRQ_NO_PRIORITY);
>> +
>> +        /*
>> +         * we don't care for now which CPU will receive the
>> +         * interrupt
>> +         *
>> +         * TODO: Handle case where IRQ is setup on different CPU than
>> +         * the targeted CPU and the priority.
>> +         */
>> +        irq_set_affinity(desc, cpumask_of(smp_processor_id()));
>> +        desc->handler->startup(desc);
>> +        /* enable irq */
>> +        clear_bit(_IRQ_DISABLED, &desc->status);
> Now it turns out this is really done twice: Once in aplic_irq_enable(),
> and once here.

Agree, this is a job of *_startup()->*_aplic_irq_enable(). I'll drop that too.

~ Oleksii


[-- Attachment #2: Type: text/html, Size: 4051 bytes --]

Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()

[Jan Beulich]

On 17.04.2025 12:10, Oleksii Kurochko wrote:
> On 4/15/25 5:55 PM, Jan Beulich wrote
>>> +        /*
>>> +         * we don't care for now which CPU will receive the
>>> +         * interrupt
>>> +         *
>>> +         * TODO: Handle case where IRQ is setup on different CPU than
>>> +         * the targeted CPU and the priority.
>>> +         */
>>> +        irq_set_affinity(desc, cpumask_of(smp_processor_id()));
>>> +        desc->handler->startup(desc);
>>> +        /* enable irq */
>>> +        clear_bit(_IRQ_DISABLED, &desc->status);
>> Now it turns out this is really done twice: Once in aplic_irq_enable(),
>> and once here.
> 
> Agree, this is a job of *_startup()->*_aplic_irq_enable(). I'll drop that too.

Wait - see my comment there. I think it belongs here, not there.

Jan

[PATCH v1 13/14] xen/riscv: initialize interrupt controller

[Oleksii Kurochko]

Call intc_init() to do basic initialization steps for APLIC and IMISC.

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/setup.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c
index a3189697da..9765bcbb08 100644
--- a/xen/arch/riscv/setup.c
+++ b/xen/arch/riscv/setup.c
@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     intc_preinit();
 
+    intc_init();
+
     printk("All set up\n");
 
     machine_halt();
-- 
2.49.0

Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> Call intc_init() to do basic initialization steps for APLIC and IMISC.
> 
> Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>

Acked-by: Jan Beulich <jbeulich@suse.com>
yet ...

> --- a/xen/arch/riscv/setup.c
> +++ b/xen/arch/riscv/setup.c
> @@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
>  
>      intc_preinit();
>  
> +    intc_init();
> +
>      printk("All set up\n");
>  
>      machine_halt();

... this being everything here I wonder if this can't be folded with the
patch where the function is introduced.

Jan

Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 745 bytes --]


On 4/15/25 5:59 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> Call intc_init() to do basic initialization steps for APLIC and IMISC.
>>
>> Signed-off-by: Oleksii Kurochko<oleksii.kurochko@gmail.com>
> Acked-by: Jan Beulich<jbeulich@suse.com>
> yet ...
>
>> --- a/xen/arch/riscv/setup.c
>> +++ b/xen/arch/riscv/setup.c
>> @@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
>>   
>>       intc_preinit();
>>   
>> +    intc_init();
>> +
>>       printk("All set up\n");
>>   
>>       machine_halt();
> ... this being everything here I wonder if this can't be folded with the
> patch where the function is introduced.

Sure, it can be folded. I will do that to reduce patch series.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 1529 bytes --]

Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 1166 bytes --]


On 4/17/25 12:11 PM, Oleksii Kurochko wrote:
>
>
> On 4/15/25 5:59 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> Call intc_init() to do basic initialization steps for APLIC and IMISC.
>>>
>>> Signed-off-by: Oleksii Kurochko<oleksii.kurochko@gmail.com>
>> Acked-by: Jan Beulich<jbeulich@suse.com>
>> yet ...
>>
>>> --- a/xen/arch/riscv/setup.c
>>> +++ b/xen/arch/riscv/setup.c
>>> @@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
>>>   
>>>       intc_preinit();
>>>   
>>> +    intc_init();
>>> +
>>>       printk("All set up\n");
>>>   
>>>       machine_halt();
>> ... this being everything here I wonder if this can't be folded with the
>> patch where the function is introduced.
> Sure, it can be folded. I will do that to reduce patch series.

I doubled checked and, at the moment, when intc_init() is introduced:
void __init intc_init(void)
{
     ASSERT(intc_hw_ops);

     if ( intc_hw_ops->init() )
         panic("Failed to initialize the interrupt controller drivers\n");
}

intc_hw_ops isn't registered as they are registered in the next two patches after
intriduction of intc_hw_ops.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 2426 bytes --]

Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller

[Jan Beulich]

On 30.04.2025 17:34, Oleksii Kurochko wrote:
> 
> On 4/17/25 12:11 PM, Oleksii Kurochko wrote:
>>
>>
>> On 4/15/25 5:59 PM, Jan Beulich wrote:
>>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>>> Call intc_init() to do basic initialization steps for APLIC and IMISC.
>>>>
>>>> Signed-off-by: Oleksii Kurochko<oleksii.kurochko@gmail.com>
>>> Acked-by: Jan Beulich<jbeulich@suse.com>
>>> yet ...
>>>
>>>> --- a/xen/arch/riscv/setup.c
>>>> +++ b/xen/arch/riscv/setup.c
>>>> @@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
>>>>   
>>>>       intc_preinit();
>>>>   
>>>> +    intc_init();
>>>> +
>>>>       printk("All set up\n");
>>>>   
>>>>       machine_halt();
>>> ... this being everything here I wonder if this can't be folded with the
>>> patch where the function is introduced.
>> Sure, it can be folded. I will do that to reduce patch series.
> 
> I doubled checked and, at the moment, when intc_init() is introduced:
> void __init intc_init(void)
> {
>      ASSERT(intc_hw_ops);
> 
>      if ( intc_hw_ops->init() )
>          panic("Failed to initialize the interrupt controller drivers\n");
> }
> 
> intc_hw_ops isn't registered as they are registered in the next two patches after
> intriduction of intc_hw_ops.

Which then feels wrong anyway; you're then merely leveraging that the function
has no caller, which (as said elsewhere) shouldn't be the case at the very least
for Misra's sake. So I expect some re-ordering to be necessary. Or you may want
to introduce the function empty and add the intc_hw_ops uses as intc_hw_ops is
introduced.

Jan

[PATCH v1 14/14] xen/riscv: add basic UART support

[Oleksii Kurochko]

Update Kconfig to select GENERIC_UART_INIT for basic UART init ( find a dt node
and call device specific device_init() ).

Drop `default n if RISCV` statement for config HAS_NS16550 as now ns16550 is
ready to be compiled and used by RISC-V.

Initialize a minimal amount of stuff to have UART and Xen console:
 - Initialize uart by calling uart_init().
 - Initialize Xen console by calling console_init_{pre,post}irq().
 - Initialize timer and its internal lists which are used by
   init_timer() which is called by ns16550_init_postirq(); otherwise
   "Unhandled exception: Store/AMO Page Fault" occurs.
 - Enable local interrupt to recieve an input from UART

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
 xen/arch/riscv/Kconfig   |  1 +
 xen/arch/riscv/setup.c   | 16 ++++++++++++++++
 xen/drivers/char/Kconfig |  1 -
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/xen/arch/riscv/Kconfig b/xen/arch/riscv/Kconfig
index 27086cca9c..f5ba7a5a78 100644
--- a/xen/arch/riscv/Kconfig
+++ b/xen/arch/riscv/Kconfig
@@ -2,6 +2,7 @@ config RISCV
 	def_bool y
 	select FUNCTION_ALIGNMENT_16B
 	select GENERIC_BUG_FRAME
+	select GENERIC_UART_INIT
 	select HAS_DEVICE_TREE
 	select HAS_PMAP
 	select HAS_UBSAN
diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c
index 9765bcbb08..b5fd660a4b 100644
--- a/xen/arch/riscv/setup.c
+++ b/xen/arch/riscv/setup.c
@@ -4,11 +4,16 @@
 #include <xen/bug.h>
 #include <xen/bootfdt.h>
 #include <xen/compile.h>
+#include <xen/console.h>
 #include <xen/device_tree.h>
 #include <xen/init.h>
 #include <xen/irq.h>
+#include <xen/keyhandler.h>
 #include <xen/mm.h>
+#include <xen/percpu.h>
+#include <xen/serial.h>
 #include <xen/shutdown.h>
+#include <xen/timer.h>
 #include <xen/vmap.h>
 #include <xen/xvmalloc.h>
 
@@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     remove_identity_mapping();
 
+    percpu_init_areas();
+
     smp_clear_cpu_maps();
 
     set_processor_id(0);
@@ -136,8 +143,17 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     intc_preinit();
 
+    uart_init();
+    console_init_preirq();
+
     intc_init();
 
+    timer_init();
+
+    local_irq_enable();
+
+    console_init_postirq();
+
     printk("All set up\n");
 
     machine_halt();
diff --git a/xen/drivers/char/Kconfig b/xen/drivers/char/Kconfig
index e6e12bb413..01fa31fb2b 100644
--- a/xen/drivers/char/Kconfig
+++ b/xen/drivers/char/Kconfig
@@ -3,7 +3,6 @@ config GENERIC_UART_INIT
 
 config HAS_NS16550
 	bool "NS16550 UART driver" if ARM
-	default n if RISCV
 	default y
 	help
 	  This selects the 16550-series UART support. For most systems, say Y.
-- 
2.49.0

Re: [PATCH v1 14/14] xen/riscv: add basic UART support

[Jan Beulich]

On 08.04.2025 17:57, Oleksii Kurochko wrote:
> --- a/xen/arch/riscv/setup.c
> +++ b/xen/arch/riscv/setup.c
> @@ -4,11 +4,16 @@
>  #include <xen/bug.h>
>  #include <xen/bootfdt.h>
>  #include <xen/compile.h>
> +#include <xen/console.h>
>  #include <xen/device_tree.h>
>  #include <xen/init.h>
>  #include <xen/irq.h>
> +#include <xen/keyhandler.h>

Why's this one needed?

>  #include <xen/mm.h>
> +#include <xen/percpu.h>
> +#include <xen/serial.h>
>  #include <xen/shutdown.h>
> +#include <xen/timer.h>
>  #include <xen/vmap.h>
>  #include <xen/xvmalloc.h>
>  
> @@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
>  
>      remove_identity_mapping();
>  
> +    percpu_init_areas();

I'll trust you that it's needed now, but the addition looks unrelated here,
and also isn't mentioned as intentional in the description.

> --- a/xen/drivers/char/Kconfig
> +++ b/xen/drivers/char/Kconfig
> @@ -3,7 +3,6 @@ config GENERIC_UART_INIT
>  
>  config HAS_NS16550
>  	bool "NS16550 UART driver" if ARM
> -	default n if RISCV
>  	default y

Just to double-check: Unlike Arm you don't want this to be user-(un)selectable
on RISC-V?

Jan

Re: [PATCH v1 14/14] xen/riscv: add basic UART support

[Oleksii Kurochko]

[-- Attachment #1: Type: text/plain, Size: 1784 bytes --]


On 4/15/25 6:03 PM, Jan Beulich wrote:
> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>> --- a/xen/arch/riscv/setup.c
>> +++ b/xen/arch/riscv/setup.c
>> @@ -4,11 +4,16 @@
>>   #include <xen/bug.h>
>>   #include <xen/bootfdt.h>
>>   #include <xen/compile.h>
>> +#include <xen/console.h>
>>   #include <xen/device_tree.h>
>>   #include <xen/init.h>
>>   #include <xen/irq.h>
>> +#include <xen/keyhandler.h>
> Why's this one needed?

It isn't needed anymore, just missed to drop.
I thought that it would be needed to test UART working fine by checking if Ctrl+AAA is working.

>
>>   #include <xen/mm.h>
>> +#include <xen/percpu.h>
>> +#include <xen/serial.h>
>>   #include <xen/shutdown.h>
>> +#include <xen/timer.h>
>>   #include <xen/vmap.h>
>>   #include <xen/xvmalloc.h>
>>   
>> @@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
>>   
>>       remove_identity_mapping();
>>   
>> +    percpu_init_areas();
> I'll trust you that it's needed now, but the addition looks unrelated here,
> and also isn't mentioned as intentional in the description.

I had  this patch when I used polling mode for UART, for this case percpu is used to receive
serial port info:
   struct serial_port *port = this_cpu(poll_port);

So percpu isn't really needed at the current development state. I'll drop this change or as an option
move to separate patch.

>> --- a/xen/drivers/char/Kconfig
>> +++ b/xen/drivers/char/Kconfig
>> @@ -3,7 +3,6 @@ config GENERIC_UART_INIT
>>   
>>   config HAS_NS16550
>>   	bool "NS16550 UART driver" if ARM
>> -	default n if RISCV
>>   	default y
> Just to double-check: Unlike Arm you don't want this to be user-(un)selectable
> on RISC-V?

Thanks for noticing that. I want to have this selectable by user. I will add RISC-V here.

~ Oleksii

[-- Attachment #2: Type: text/html, Size: 2923 bytes --]

Re: [PATCH v1 14/14] xen/riscv: add basic UART support

[Jan Beulich]

On 17.04.2025 12:31, Oleksii Kurochko wrote:
> On 4/15/25 6:03 PM, Jan Beulich wrote:
>> On 08.04.2025 17:57, Oleksii Kurochko wrote:
>>> --- a/xen/drivers/char/Kconfig
>>> +++ b/xen/drivers/char/Kconfig
>>> @@ -3,7 +3,6 @@ config GENERIC_UART_INIT
>>>   
>>>   config HAS_NS16550
>>>   	bool "NS16550 UART driver" if ARM
>>> -	default n if RISCV
>>>   	default y
>> Just to double-check: Unlike Arm you don't want this to be user-(un)selectable
>> on RISC-V?
> 
> Thanks for noticing that. I want to have this selectable by user. I will add RISC-V here.

At which point we may want to consider whether the condition on the prompt
wouldn't better become "!X86".

Jan