[RFC PATCH v3 13/37] kvx: Add build infrastructure

[RFC PATCH v3 13/37] kvx: Add build infrastructure

[ysionneau]

From: Yann Sionneau <ysionneau@kalrayinc.com>

Add Kbuild, Makefile, Kconfig and link script for kvx build infrastructure.

Co-developed-by: Clement Leger <clement@clement-leger.fr>
Signed-off-by: Clement Leger <clement@clement-leger.fr>
Co-developed-by: Guillaume Thouvenin <thouveng@gmail.com>
Signed-off-by: Guillaume Thouvenin <thouveng@gmail.com>
Co-developed-by: Jonathan Borne <jborne@kalrayinc.com>
Signed-off-by: Jonathan Borne <jborne@kalrayinc.com>
Co-developed-by: Jules Maselbas <jmaselbas@zdiv.net>
Signed-off-by: Jules Maselbas <jmaselbas@zdiv.net>
Co-developed-by: Julian Vetter <jvetter@kalrayinc.com>
Signed-off-by: Julian Vetter <jvetter@kalrayinc.com>
Co-developed-by: Marc Poulhiès <dkm@kataplop.net>
Signed-off-by: Marc Poulhiès <dkm@kataplop.net>
Co-developed-by: Marius Gligor <mgligor@kalrayinc.com>
Signed-off-by: Marius Gligor <mgligor@kalrayinc.com>
Co-developed-by: Samuel Jones <sjones@kalrayinc.com>
Signed-off-by: Samuel Jones <sjones@kalrayinc.com>
Co-developed-by: Vincent Chardon <vincent.chardon@elsys-design.com>
Signed-off-by: Vincent Chardon <vincent.chardon@elsys-design.com>
Signed-off-by: Yann Sionneau <ysionneau@kalrayinc.com>
---

Notes:
V1 -> V2:
- typos and formatting fixes, removed int from PGTABLE_LEVELS
- renamed default_defconfig to defconfig in arch/kvx/Makefile
- fix clean target raising an error from gcc (LIBGCC)

V2 -> V3:
- use generic entry framework
- add smem in kernel direct map
- remove DEBUG_EXCEPTION_STACK
- use global addresses for smem
- now boot from PE (KALRAY_BOOT_BY_PE)
- do not link with libgcc anymore
---
 arch/kvx/Kconfig                 | 226 +++++++++++++++++++++++++++++++
 arch/kvx/Kconfig.debug           |  60 ++++++++
 arch/kvx/Makefile                |  47 +++++++
 arch/kvx/include/asm/Kbuild      |  20 +++
 arch/kvx/include/uapi/asm/Kbuild |   1 +
 arch/kvx/kernel/Makefile         |  15 ++
 arch/kvx/kernel/kvx_ksyms.c      |  18 +++
 arch/kvx/kernel/vmlinux.lds.S    | 150 ++++++++++++++++++++
 arch/kvx/lib/Makefile            |   6 +
 arch/kvx/mm/Makefile             |   8 ++
 10 files changed, 551 insertions(+)
 create mode 100644 arch/kvx/Kconfig
 create mode 100644 arch/kvx/Kconfig.debug
 create mode 100644 arch/kvx/Makefile
 create mode 100644 arch/kvx/include/asm/Kbuild
 create mode 100644 arch/kvx/include/uapi/asm/Kbuild
 create mode 100644 arch/kvx/kernel/Makefile
 create mode 100644 arch/kvx/kernel/kvx_ksyms.c
 create mode 100644 arch/kvx/kernel/vmlinux.lds.S
 create mode 100644 arch/kvx/lib/Makefile
 create mode 100644 arch/kvx/mm/Makefile

diff --git a/arch/kvx/Kconfig b/arch/kvx/Kconfig
new file mode 100644
index 0000000000000..a98c1fbd8131d
--- /dev/null
+++ b/arch/kvx/Kconfig
@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# For a description of the syntax of this configuration file,
+# see Documentation/kbuild/kconfig-language.txt.
+#
+
+config 64BIT
+	def_bool y
+
+config GENERIC_CALIBRATE_DELAY
+	def_bool y
+
+config FIX_EARLYCON_MEM
+	def_bool y
+
+config MMU
+	def_bool y
+
+config KALLSYMS_BASE_RELATIVE
+	def_bool n
+
+config GENERIC_CSUM
+	def_bool y
+
+config RWSEM_GENERIC_SPINLOCK
+	def_bool y
+
+config GENERIC_HWEIGHT
+	def_bool y
+
+config ARCH_MMAP_RND_BITS_MAX
+	default 24
+
+config ARCH_MMAP_RND_BITS_MIN
+	default 18
+
+config STACKTRACE_SUPPORT
+	def_bool y
+
+config LOCKDEP_SUPPORT
+	def_bool y
+
+config GENERIC_BUG
+	def_bool y
+	depends on BUG
+
+config KVX_4K_PAGES
+	def_bool y
+
+config KVX
+	def_bool y
+	select ARCH_CLOCKSOURCE_DATA
+	select ARCH_DMA_ADDR_T_64BIT
+	select ARCH_HAS_DEVMEM_IS_ALLOWED
+	select ARCH_HAS_DMA_PREP_COHERENT
+	select ARCH_HAS_ELF_RANDOMIZE
+	select ARCH_HAS_PTE_SPECIAL
+	select ARCH_HAS_SETUP_DMA_OPS if IOMMU_SUPPORT
+	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
+	select ARCH_HAS_SYNC_DMA_FOR_CPU
+	select ARCH_HAS_TEARDOWN_DMA_OPS if IOMMU_SUPPORT
+	select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT
+	select ARCH_USE_QUEUED_SPINLOCKS
+	select ARCH_USE_QUEUED_RWLOCKS
+	select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
+	select ARCH_WANT_FRAME_POINTERS
+	select CLKSRC_OF
+	select COMMON_CLK
+	select DMA_DIRECT_REMAP
+	select GENERIC_ALLOCATOR
+	select GENERIC_CLOCKEVENTS
+	select GENERIC_CLOCKEVENTS
+	select GENERIC_CPU_DEVICES
+	select GENERIC_ENTRY
+	select GENERIC_IOMAP
+	select GENERIC_IOREMAP
+	select GENERIC_IRQ_CHIP
+	select GENERIC_IRQ_MULTI_HANDLER
+	select GENERIC_IRQ_PROBE
+	select GENERIC_IRQ_SHOW
+	select GENERIC_SCHED_CLOCK
+	select HAVE_ARCH_AUDITSYSCALL
+	select HAVE_ARCH_BITREVERSE
+	select HAVE_ARCH_MMAP_RND_BITS
+	select HAVE_ARCH_SECCOMP_FILTER
+	select HAVE_ASM_MODVERSIONS
+	select HAVE_DEBUG_KMEMLEAK
+	select HAVE_EFFICIENT_UNALIGNED_ACCESS
+	select HAVE_FUTEX_CMPXCHG if FUTEX
+	select HAVE_IOREMAP_PROT
+	select HAVE_MEMBLOCK_NODE_MAP
+	select HAVE_PCI
+	select HAVE_STACKPROTECTOR
+	select HAVE_SYSCALL_TRACEPOINTS
+	select IOMMU_DMA if IOMMU_SUPPORT
+	select KVX_IPI_CTRL if SMP
+	select KVX_APIC_GIC
+	select KVX_APIC_MAILBOX
+	select KVX_CORE_INTC
+	select KVX_ITGEN
+	select KVX_WATCHDOG
+	select MODULES_USE_ELF_RELA
+	select OF
+	select OF_EARLY_FLATTREE
+	select OF_RESERVED_MEM
+	select PCI_DOMAINS_GENERIC if PCI
+	select SPARSE_IRQ
+	select SYSCTL_EXCEPTION_TRACE
+	select THREAD_INFO_IN_TASK
+	select TIMER_OF
+	select TRACE_IRQFLAGS_SUPPORT
+	select WATCHDOG
+	select ZONE_DMA32
+
+config PGTABLE_LEVELS
+	default 3
+
+config HAVE_KPROBES
+	def_bool n
+
+menu "System setup"
+
+config POISON_INITMEM
+	bool "Enable to poison freed initmem"
+	default y
+	help
+	  In order to debug initmem, using poison allows to verify if some
+	  data/code is still using it. Enable this for debug purposes.
+
+config KVX_PHYS_OFFSET
+	hex "RAM address of memory base"
+	default 0x0
+
+config KVX_PAGE_OFFSET
+	hex "kernel virtual address of memory base"
+	default 0xFFFFFF8000000000
+
+config ARCH_SPARSEMEM_ENABLE
+	def_bool y
+
+config ARCH_SPARSEMEM_DEFAULT
+	def_bool ARCH_SPARSEMEM_ENABLE
+
+config ARCH_SELECT_MEMORY_MODEL
+	def_bool ARCH_SPARSEMEM_ENABLE
+
+config STACK_MAX_DEPTH_TO_PRINT
+	int "Maximum depth of stack to print"
+	range 1 128
+	default "24"
+
+config SECURE_DAME_HANDLING
+	bool "Secure DAME handling"
+	default y
+	help
+	  In order to securely handle Data Asynchronous Memory Errors, we need
+	  to do a barrier upon kernel entry when coming from userspace. This
+	  barrier guarantees us that any pending DAME will be serviced right
+	  away. We also need to do a barrier when returning from kernel to user.
+	  This way, if the kernel or the user triggered a DAME, it will be
+	  serviced by knowing we are coming from kernel or user and avoid
+	  pulling the wrong lever (panic for kernel or sigfault for user).
+	  This can be costly but ensures that user cannot interfere with kernel.
+	  /!\ Do not disable unless you want to open a giant breach between
+	  user and kernel /!\
+
+config CACHECTL_UNSAFE_PHYS_OPERATIONS
+	bool "Enable cachectl syscall unsafe physical operations"
+	default n
+	help
+	  Enable the cachectl syscall to allow writing back/invalidating memory
+	  ranges based on physical addresses. These operations require the
+	  CAP_SYS_ADMIN capability.
+
+config ENABLE_TCA
+	bool "Enable TCA coprocessor support"
+	default y
+	help
+	  This option enables TCA coprocessor support. It allows the user to
+	  use the coprocessor and save registers on context switch if used.
+	  Registers content will also be cleared when switching.
+
+config SMP
+	bool "Symmetric multi-processing support"
+	default n
+	select GENERIC_SMP_IDLE_THREAD
+	select GENERIC_IRQ_IPI
+	select IRQ_DOMAIN_HIERARCHY
+	select IRQ_DOMAIN
+	help
+	  This enables support for systems with more than one CPU. If you have
+	  a system with only one CPU, say N. If you have a system with more
+	  than one CPU, say Y.
+
+	  If you say N here, the kernel will run on uni- and multiprocessor
+	  machines, but will use only one CPU of a multiprocessor machine. If
+	  you say Y here, the kernel will run on many, but not all,
+	  uniprocessor machines. On a uniprocessor machine, the kernel
+	  will run faster if you say N here.
+
+config NR_CPUS
+	int "Maximum number of CPUs"
+	range 1 16
+	default "16"
+	depends on SMP
+	help
+	  Kalray support can handle a maximum of 16 CPUs.
+
+config KVX_PAGE_SHIFT
+	int
+	default 12
+
+config CMDLINE
+	string "Default kernel command string"
+	default ""
+	help
+	  On some architectures there is currently no way for the boot loader
+	  to pass arguments to the kernel. For these architectures, you should
+	  supply some command-line options at build time by entering them
+	  here.
+
+endmenu
+
+menu "Kernel Features"
+source "kernel/Kconfig.hz"
+endmenu
diff --git a/arch/kvx/Kconfig.debug b/arch/kvx/Kconfig.debug
new file mode 100644
index 0000000000000..0d526802e9221
--- /dev/null
+++ b/arch/kvx/Kconfig.debug
@@ -0,0 +1,60 @@
+menu "KVX debugging"
+
+config KVX_DEBUG_ASN
+	bool "Check ASN before writing TLB entry"
+	default n
+	help
+	  This option allows to check if the ASN of the current
+	  process matches the ASN found in MMC. If it is not the
+	  case an error will be printed.
+
+config KVX_DEBUG_TLB_WRITE
+	bool "Enable TLBs write checks"
+	default n
+	help
+	  Enabling this option will enable TLB access checks. This is
+	  particularly helpful when modifying the assembly code responsible
+	  for TLB refill. If set, mmc.e will be checked each time the TLB are
+	  written and a panic will be thrown on error.
+
+config KVX_DEBUG_TLB_ACCESS
+	bool "Enable TLBs accesses logging"
+	default n
+	help
+	  Enabling this option will enable TLB entry manipulation logging.
+	  Each time an entry is added to the TLBs, it is logged in an array
+	  readable via gdb scripts. This can be useful to understand strange
+	  crashes related to suspicious virtual/physical addresses.
+
+config KVX_DEBUG_TLB_ACCESS_BITS
+	int "Number of bits used as index of entries in log table"
+	default 12
+	depends on KVX_DEBUG_TLB_ACCESS
+	help
+	  Set the number of bits used as index of entries that will be logged
+	  in a ring buffer called kvx_tlb_access. One entry in the table
+	  contains registers TEL, TEH and MMC. It also logs the type of the
+	  operations (0:read, 1:write, 2:probe). Buffer is per CPU. For one
+	  entry 24 bytes are used. So by default it uses 96KB of memory per
+	  CPU to store 2^12 (4096) entries.
+
+config KVX_MMU_STATS
+	bool "Register MMU stats debugfs entries"
+	default n
+	depends on DEBUG_FS
+	help
+	  Enable debugfs attribute which will allow inspecting various metrics
+	  regarding MMU:
+	  - Number of nomapping traps handled
+	  - avg/min/max time for nomapping refill (user/kernel)
+
+config DEBUG_SFR_SET_MASK
+	bool "Enable SFR set_mask debugging"
+	default n
+	help
+	  Verify that values written using kvx_sfr_set_mask match the mask.
+	  This ensure that no extra bits of SFR will be overridden by some
+	  incorrectly truncated values. This can lead to huge problems by
+	  modifying important bits in system registers.
+
+endmenu
diff --git a/arch/kvx/Makefile b/arch/kvx/Makefile
new file mode 100644
index 0000000000000..305fa1d5d8692
--- /dev/null
+++ b/arch/kvx/Makefile
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2018-2024 Kalray Inc.
+
+ifeq ($(CROSS_COMPILE),)
+CROSS_COMPILE := kvx-mbr-
+endif
+
+KBUILD_DEFCONFIG := defconfig
+
+LDFLAGS_vmlinux := -X
+OBJCOPYFLAGS := -O binary -R .comment -R .note -R .bootloader -S
+
+DEFAULT_OPTS := -nostdlib -fno-builtin -march=kv3-1
+KBUILD_CFLAGS += $(DEFAULT_OPTS)
+KBUILD_AFLAGS += $(DEFAULT_OPTS)
+KBUILD_CFLAGS_MODULE += -mfarcall
+
+KBUILD_LDFLAGS += -m elf64kvx
+
+head-y	:= arch/kvx/kernel/head.o
+libs-y  += arch/kvx/lib/
+core-y += arch/kvx/kernel/ \
+          arch/kvx/mm/
+# Final targets
+all: vmlinux
+
+BOOT_TARGETS = bImage bImage.bin bImage.bz2 bImage.gz bImage.lzma bImage.lzo
+
+$(BOOT_TARGETS): vmlinux
+	$(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
+
+install:
+	$(Q)$(MAKE) $(build)=$(boot) BOOTIMAGE=$(KBUILD_IMAGE) install
+
+define archhelp
+  echo  '* bImage         - Alias to selected kernel format (bImage.gz by default)'
+  echo  '  bImage.bin     - Uncompressed Kernel-only image for barebox (arch/$(ARCH)/boot/bImage.bin)'
+  echo  '  bImage.bz2     - Kernel-only image for barebox (arch/$(ARCH)/boot/bImage.bz2)'
+  echo  '* bImage.gz      - Kernel-only image for barebox (arch/$(ARCH)/boot/bImage.gz)'
+  echo  '  bImage.lzma    - Kernel-only image for barebox (arch/$(ARCH)/boot/bImage.lzma)'
+  echo  '  bImage.lzo     - Kernel-only image for barebox (arch/$(ARCH)/boot/bImage.lzo)'
+  echo  '  install        - Install kernel using'
+  echo  '                     (your) ~/bin/$(INSTALLKERNEL) or'
+  echo  '                     (distribution) PATH: $(INSTALLKERNEL) or'
+  echo  '                     install to $$(INSTALL_PATH)'
+endef
diff --git a/arch/kvx/include/asm/Kbuild b/arch/kvx/include/asm/Kbuild
new file mode 100644
index 0000000000000..ea73552faa103
--- /dev/null
+++ b/arch/kvx/include/asm/Kbuild
@@ -0,0 +1,20 @@
+generic-y += asm-offsets.h
+generic-y += clkdev.h
+generic-y += auxvec.h
+generic-y += bpf_perf_event.h
+generic-y += cmpxchg-local.h
+generic-y += errno.h
+generic-y += extable.h
+generic-y += export.h
+generic-y += kvm_para.h
+generic-y += mcs_spinlock.h
+generic-y += mman.h
+generic-y += param.h
+generic-y += qrwlock.h
+generic-y += qspinlock.h
+generic-y += rwsem.h
+generic-y += sockios.h
+generic-y += stat.h
+generic-y += statfs.h
+generic-y += ucontext.h
+generic-y += user.h
diff --git a/arch/kvx/include/uapi/asm/Kbuild b/arch/kvx/include/uapi/asm/Kbuild
new file mode 100644
index 0000000000000..8b137891791fe
--- /dev/null
+++ b/arch/kvx/include/uapi/asm/Kbuild
@@ -0,0 +1 @@
+
diff --git a/arch/kvx/kernel/Makefile b/arch/kvx/kernel/Makefile
new file mode 100644
index 0000000000000..a57309bbdbefd
--- /dev/null
+++ b/arch/kvx/kernel/Makefile
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2019-2024 Kalray Inc.
+#
+
+obj-y	:= head.o setup.o process.o traps.o common.o time.o prom.o kvx_ksyms.o \
+	   irq.o cpuinfo.o entry.o ptrace.o syscall_table.o signal.o sys_kvx.o \
+	   stacktrace.o dame_handler.o vdso.o debug.o break_hook.o \
+	   reset.o io.o cpu.o
+
+obj-$(CONFIG_SMP) 			+= smp.o smpboot.o
+obj-$(CONFIG_MODULES)			+= module.o
+CFLAGS_module.o				+= -Wstrict-overflow -fstrict-overflow
+
+extra-y					+= vmlinux.lds
diff --git a/arch/kvx/kernel/kvx_ksyms.c b/arch/kvx/kernel/kvx_ksyms.c
new file mode 100644
index 0000000000000..6a24ade5be3d9
--- /dev/null
+++ b/arch/kvx/kernel/kvx_ksyms.c
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * derived from arch/nios2/kernel/nios2_ksyms.c
+ *
+ * Copyright (C) 2017-2024 Kalray Inc.
+ * Author(s): Clement Leger
+ *            Yann Sionneau
+ */
+
+#include <linux/kernel.h>
+#include <linux/export.h>
+
+#define DECLARE_EXPORT(name)	extern void name(void); EXPORT_SYMBOL(name)
+
+DECLARE_EXPORT(clear_page);
+DECLARE_EXPORT(copy_page);
+DECLARE_EXPORT(memset);
+DECLARE_EXPORT(asm_clear_user);
diff --git a/arch/kvx/kernel/vmlinux.lds.S b/arch/kvx/kernel/vmlinux.lds.S
new file mode 100644
index 0000000000000..6e064e8867b82
--- /dev/null
+++ b/arch/kvx/kernel/vmlinux.lds.S
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2017-2024 Kalray Inc.
+ * Author(s): Clement Leger
+ *            Guillaume Thouvenin
+ *            Marius Gligor
+ *            Marc Poulhiès
+ *            Yann Sionneau
+ */
+
+#include <asm/thread_info.h>
+#include <asm/asm-offsets.h>
+#include <asm/sys_arch.h>
+#include <asm/cache.h>
+#include <asm/page.h>
+#include <asm/fixmap.h>
+
+/* Entry is linked at smem global address */
+#define BOOT_ENTRY		(16 * 1024 * 1024)
+#define DTB_DEFAULT_SIZE	(64 * 1024)
+
+#define LOAD_OFFSET  (DDR_VIRT_OFFSET - DDR_PHYS_OFFSET)
+#include <asm-generic/vmlinux.lds.h>
+
+OUTPUT_FORMAT("elf64-kvx")
+ENTRY(kvx_start)
+
+#define HANDLER_SECTION(__sec, __name) \
+	__sec ## _ ## __name ## _start = .; \
+	KEEP(*(.##__sec ##.## __name)); \
+	. = __sec ## _ ##__name ## _start + EXCEPTION_STRIDE;
+
+/**
+ * Generate correct section positioning for exception handling
+ * Since we need it twice for early exception handler and normal
+ * exception handler, factorize it here.
+ */
+#define EXCEPTION_SECTIONS(__sec) \
+	__ ## __sec ## _start = ABSOLUTE(.); \
+	HANDLER_SECTION(__sec,debug) \
+	HANDLER_SECTION(__sec,trap) \
+	HANDLER_SECTION(__sec,interrupt) \
+	HANDLER_SECTION(__sec,syscall)
+
+jiffies = jiffies_64;
+KALRAY_BOOT_BY_PE = 0;
+SECTIONS
+{
+	. = BOOT_ENTRY;
+	.boot :
+	{
+		__kernel_smem_code_start = .;
+		KEEP(*(.boot.startup));
+		KEEP(*(.boot.*));
+		__kernel_smem_code_end = .;
+	}
+
+	. = DDR_VIRT_OFFSET;
+	_start = .;
+
+	_stext = .;
+	__init_begin = .;
+	__inittext_start = .;
+	.exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET)
+	{
+		EXIT_TEXT
+	}
+
+	.early_exception ALIGN(EXCEPTION_ALIGNMENT) :
+				AT(ADDR(.early_exception) - LOAD_OFFSET)
+	{
+		EXCEPTION_SECTIONS(early_exception)
+	}
+
+	HEAD_TEXT_SECTION
+	INIT_TEXT_SECTION(PAGE_SIZE)
+	. = ALIGN(PAGE_SIZE);
+	__inittext_end = .;
+	__initdata_start = .;
+	INIT_DATA_SECTION(16)
+
+	/* we have to discard exit text and such at runtime, not link time */
+	.exit.data : AT(ADDR(.exit.data) - LOAD_OFFSET)
+	{
+		EXIT_DATA
+	}
+
+	PERCPU_SECTION(L1_CACHE_BYTES)
+	. = ALIGN(PAGE_SIZE);
+	__initdata_end = .;
+	__init_end = .;
+
+	/* Everything below this point will be mapped RO EXEC up to _etext */
+	.text ALIGN(PAGE_SIZE) : AT(ADDR(.text) - LOAD_OFFSET)
+	{
+		_text = .;
+		EXCEPTION_SECTIONS(exception)
+		*(.exception.text)
+		. = ALIGN(PAGE_SIZE);
+		__exception_end = .;
+		TEXT_TEXT
+		SCHED_TEXT
+		LOCK_TEXT
+		KPROBES_TEXT
+		ENTRY_TEXT
+		IRQENTRY_TEXT
+		SOFTIRQENTRY_TEXT
+		*(.fixup)
+	}
+	. = ALIGN(PAGE_SIZE);
+	_etext = .;
+
+	/* Everything below this point will be mapped RO NOEXEC up to _sdata */
+	__rodata_start = .;
+	RO_DATA(PAGE_SIZE)
+	EXCEPTION_TABLE(8)
+	. = ALIGN(32);
+	.dtb : AT(ADDR(.dtb) - LOAD_OFFSET)
+	{
+		__dtb_start = .;
+		. += DTB_DEFAULT_SIZE;
+		__dtb_end = .;
+	}
+	. = ALIGN(PAGE_SIZE);
+	__rodata_end = .;
+
+	/* Everything below this point will be mapped RW NOEXEC up to _end */
+	_sdata = .;
+	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE)
+	_edata = .;
+
+	BSS_SECTION(32, 32, 32)
+	. = ALIGN(PAGE_SIZE);
+	_end = .;
+
+	/* This page will be mapped using a FIXMAP */
+	.gdb_page ALIGN(PAGE_SIZE) : AT(ADDR(.gdb_page) - LOAD_OFFSET)
+	{
+		_debug_start = ADDR(.gdb_page) - LOAD_OFFSET;
+		. += PAGE_SIZE;
+	}
+	_debug_start_lma = ASM_FIX_TO_VIRT(FIX_GDB_MEM_BASE_IDX);
+
+	/* Debugging sections */
+	STABS_DEBUG
+	DWARF_DEBUG
+
+	/* Sections to be discarded -- must be last */
+	DISCARDS
+}
diff --git a/arch/kvx/lib/Makefile b/arch/kvx/lib/Makefile
new file mode 100644
index 0000000000000..3e9904d1f6f9c
--- /dev/null
+++ b/arch/kvx/lib/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2017-2024 Kalray Inc.
+#
+
+lib-y := usercopy.o clear_page.o copy_page.o memcpy.o memset.o strlen.o delay.o div.o
diff --git a/arch/kvx/mm/Makefile b/arch/kvx/mm/Makefile
new file mode 100644
index 0000000000000..a96a3764aae4a
--- /dev/null
+++ b/arch/kvx/mm/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2017-2024 Kalray Inc.
+#
+
+obj-y := init.o mmu.o fault.o tlb.o extable.o dma-mapping.o cacheflush.o
+obj-$(CONFIG_KVX_MMU_STATS) += mmu_stats.o
+obj-$(CONFIG_STRICT_DEVMEM) += mmap.o
-- 
2.45.2

Re: [RFC PATCH v3 13/37] kvx: Add build infrastructure

[Arnd Bergmann]

On Mon, Jul 22, 2024, at 09:41, ysionneau@kalrayinc.com wrote:
> +
> +config GENERIC_CALIBRATE_DELAY
> +	def_bool y

You don't seem to define ARCH_HAS_READ_CURRENT_TIMER 
or calibrate_delay_is_known(), so it appears that you are
measuring the loops per jiffy. Since you are using a
cycle counter for your delay loop, you should be able
to just set the value directly instead.

> +	select GENERIC_IOMAP
> +	select GENERIC_IOREMAP

GENERIC_IOMAP is something you should normally not
set, it is a bit misnamed and not at all generic ;-)

> +config ARCH_SPARSEMEM_ENABLE
> +	def_bool y
> +
> +config ARCH_SPARSEMEM_DEFAULT
> +	def_bool ARCH_SPARSEMEM_ENABLE
> +
> +config ARCH_SELECT_MEMORY_MODEL
> +	def_bool ARCH_SPARSEMEM_ENABLE

There has been some discussion about removing traditional
sparsemem support in favor of just using SPARSEMEM_VMEMMAP,
which is simpler and more efficient on 64-bit architectures.

You should probably have a look at that.

> +config SMP
> +	bool "Symmetric multi-processing support"
> +	default n
> +	select GENERIC_SMP_IDLE_THREAD
> +	select GENERIC_IRQ_IPI
> +	select IRQ_DOMAIN_HIERARCHY

Do you have a use case for turning SMP off in practice?
Non-SMP kernels can be a little more efficient, but it's
getting rare these days to actually have systems with
just one CPU.

> +config KVX_PAGE_SHIFT
> +	int
> +	default 12

There is now a generic CONFIG_PAGE_SIZE and CONFIG_PAGE_SHIFT
setting, so you can remove the custom ones and just use
'select HAVE_PAGE_SIZE_4KB'

> --- /dev/null
> +++ b/arch/kvx/include/asm/Kbuild
> @@ -0,0 +1,20 @@
> +generic-y += asm-offsets.h
> +generic-y += clkdev.h
> +generic-y += auxvec.h
> +generic-y += bpf_perf_event.h
> +generic-y += cmpxchg-local.h
> +generic-y += errno.h
> +generic-y += extable.h
> +generic-y += export.h
...

You seem to have more entries here than necessary, e.g.
clkdev.h and auxvec.h are no longer in asm-generic, and
export.h is deprecated.

      Arnd

[RFC PATCH v3 30/37] kvx: Add multi-processor (SMP) support

[ysionneau]

From: Yann Sionneau <ysionneau@kalrayinc.com>

Coolidge v1 SoC has 5 clusters of 17 kvx cores:
 - 16 application cores aka PE
 - 1 privileged core, the Resource Manager, aka RM.

Linux can run in SMP config on the 16 cores of a Cluster.

Memory coherency between all cores is guaranteed by the L2 cache.

Co-developed-by: Clement Leger <clement@clement-leger.fr>
Signed-off-by: Clement Leger <clement@clement-leger.fr>
Co-developed-by: Julian Vetter <jvetter@kalrayinc.com>
Signed-off-by: Julian Vetter <jvetter@kalrayinc.com>
Co-developed-by: Julien Hascoet <jhascoet@kalrayinc.com>
Signed-off-by: Julien Hascoet <jhascoet@kalrayinc.com>
Co-developed-by: Louis Morhet <lmorhet@kalrayinc.com>
Signed-off-by: Louis Morhet <lmorhet@kalrayinc.com>
Co-developed-by: Luc Michel <luc@lmichel.fr>
Signed-off-by: Luc Michel <luc@lmichel.fr>
Co-developed-by: Marius Gligor <mgligor@kalrayinc.com>
Signed-off-by: Marius Gligor <mgligor@kalrayinc.com>
Signed-off-by: Yann Sionneau <ysionneau@kalrayinc.com>
---

Notes:
V1 -> V2:
- removed L2 cache driver
- removed ipi and pwr-ctrl driver (split into their own patch)

V2 -> V3:
- Refactored smp_init_cpus function to use
  `of_get_cpu_hwid` and `set_cpu_possible`
- boot secondary CPUs via "smp_ops" / DT enable-method
- put most IPI code in its own driver in drivers/irqchip
  which fills a smp_cross_call func pointer.
  see remarks in there:
  - https://lore.kernel.org/bpf/Y8qlOpYgDefMPqWH@zx2c4.com/T/#m5786c05e937e99a4c7e5353a4394f870240853d8
- don't hardcode probing of pwr-ctrl in smpboot.c
  instead let a driver in drivers/soc/kvx probe and register smp_ops
  see remarks in there:
  - https://lore.kernel.org/bpf/Y8qlOpYgDefMPqWH@zx2c4.com/T/#mf43bfb87d7a8f03ec98fb15e66f0bec19e85839c
  - https://lore.kernel.org/bpf/Y8qlOpYgDefMPqWH@zx2c4.com/T/#m1da9ac16c5ed93f895a82687b3e53ba9cdb26578
---
 arch/kvx/include/asm/smp.h |  63 ++++++++++++++++
 arch/kvx/kernel/smp.c      |  83 +++++++++++++++++++++
 arch/kvx/kernel/smpboot.c  | 146 +++++++++++++++++++++++++++++++++++++
 include/linux/cpuhotplug.h |   2 +
 4 files changed, 294 insertions(+)
 create mode 100644 arch/kvx/include/asm/smp.h
 create mode 100644 arch/kvx/kernel/smp.c
 create mode 100644 arch/kvx/kernel/smpboot.c

diff --git a/arch/kvx/include/asm/smp.h b/arch/kvx/include/asm/smp.h
new file mode 100644
index 0000000000000..7a8dab6b1300e
--- /dev/null
+++ b/arch/kvx/include/asm/smp.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2017-2024 Kalray Inc.
+ * Author(s): Clement Leger
+ *            Jonathan Borne
+ *            Yann Sionneau
+ */
+
+#ifndef _ASM_KVX_SMP_H
+#define _ASM_KVX_SMP_H
+
+#include <linux/cpumask.h>
+#include <linux/irqreturn.h>
+
+#include <asm/sfr.h>
+
+void smp_init_cpus(void);
+
+#ifdef CONFIG_SMP
+
+extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
+asmlinkage void __init start_kernel_secondary(void);
+
+/* Hook for the generic smp_call_function_many() routine. */
+void arch_send_call_function_ipi_mask(struct cpumask *mask);
+
+/* Hook for the generic smp_call_function_single() routine. */
+void arch_send_call_function_single_ipi(int cpu);
+
+void __init setup_processor(void);
+int __init setup_smp(void);
+
+#define raw_smp_processor_id() ((int) \
+	((kvx_sfr_get(PCR) & KVX_SFR_PCR_PID_MASK) \
+					>> KVX_SFR_PCR_PID_SHIFT))
+
+#define flush_cache_vmap(start, end)		do { } while (0)
+#define flush_cache_vunmap(start, end)		do { } while (0)
+extern void handle_IPI(unsigned long ops);
+
+struct smp_operations {
+	int  (*smp_boot_secondary)(unsigned int cpu);
+};
+
+struct of_cpu_method {
+	const char *method;
+	const struct smp_operations *ops;
+};
+
+#define CPU_METHOD_OF_DECLARE(name, _method, _ops)			\
+	static const struct of_cpu_method __cpu_method_of_table_##name	\
+		__used __section("__cpu_method_of_table")		\
+		= { .method = _method, .ops = _ops }
+
+extern void smp_set_ops(const struct smp_operations *ops);
+
+#else
+
+void smp_init_cpus(void) {}
+
+#endif /* CONFIG_SMP */
+
+#endif
diff --git a/arch/kvx/kernel/smp.c b/arch/kvx/kernel/smp.c
new file mode 100644
index 0000000000000..c2cb96797c90b
--- /dev/null
+++ b/arch/kvx/kernel/smp.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017-2024 Kalray Inc.
+ * Author(s): Clement Leger
+ *            Jonathan Borne
+ *            Yann Sionneau
+ */
+
+#include <linux/cpumask.h>
+#include <linux/irq_work.h>
+
+static void (*smp_cross_call)(const struct cpumask *, unsigned int);
+
+enum ipi_message_type {
+	IPI_RESCHEDULE,
+	IPI_CALL_FUNC,
+	IPI_IRQ_WORK,
+	IPI_MAX
+};
+
+void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
+{
+	smp_cross_call = fn;
+}
+
+static void send_ipi_message(const struct cpumask *mask,
+			     enum ipi_message_type operation)
+{
+	if (!smp_cross_call)
+		panic("ipi controller init failed\n");
+	smp_cross_call(mask, (unsigned int)operation);
+}
+
+void arch_send_call_function_ipi_mask(struct cpumask *mask)
+{
+	send_ipi_message(mask, IPI_CALL_FUNC);
+}
+
+void arch_send_call_function_single_ipi(int cpu)
+{
+	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
+}
+
+#ifdef CONFIG_IRQ_WORK
+void arch_irq_work_raise(void)
+{
+	send_ipi_message(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
+}
+#endif
+
+static void ipi_stop(void *unused)
+{
+	local_cpu_stop();
+}
+
+void smp_send_stop(void)
+{
+	struct cpumask targets;
+
+	cpumask_copy(&targets, cpu_online_mask);
+	cpumask_clear_cpu(smp_processor_id(), &targets);
+
+	smp_call_function_many(&targets, ipi_stop, NULL, 0);
+}
+
+void arch_smp_send_reschedule(int cpu)
+{
+	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
+}
+
+void handle_IPI(unsigned long ops)
+{
+	if (ops & (1 << IPI_RESCHEDULE))
+		scheduler_ipi();
+
+	if (ops & (1 << IPI_CALL_FUNC))
+		generic_smp_call_function_interrupt();
+
+	if (ops & (1 << IPI_IRQ_WORK))
+		irq_work_run();
+
+	WARN_ON_ONCE((ops >> IPI_MAX) != 0);
+}
diff --git a/arch/kvx/kernel/smpboot.c b/arch/kvx/kernel/smpboot.c
new file mode 100644
index 0000000000000..ab7f29708fed2
--- /dev/null
+++ b/arch/kvx/kernel/smpboot.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017-2024 Kalray Inc.
+ * Author(s): Clement Leger
+ *            Julian Vetter
+ *            Yann Sionneau
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/smp.h>
+#include <linux/cpu.h>
+#include <linux/sched.h>
+#include <linux/cpumask.h>
+#include <linux/sched/mm.h>
+#include <linux/mm_types.h>
+#include <linux/of_platform.h>
+#include <linux/sched/task_stack.h>
+
+#include <asm/tlbflush.h>
+#include <asm/ipi.h>
+
+void *__cpu_up_stack_pointer[NR_CPUS];
+void *__cpu_up_task_pointer[NR_CPUS];
+static struct smp_operations smp_ops __ro_after_init;
+extern struct of_cpu_method __cpu_method_of_table[];
+
+void __init smp_prepare_boot_cpu(void)
+{
+}
+
+void __init smp_set_ops(const struct smp_operations *ops)
+{
+	if (ops)
+		smp_ops = *ops;
+};
+
+int __cpu_up(unsigned int cpu, struct task_struct *tidle)
+{
+	int ret;
+
+	__cpu_up_stack_pointer[cpu] = task_stack_page(tidle) + THREAD_SIZE;
+	__cpu_up_task_pointer[cpu] = tidle;
+	/* We need to be sure writes are committed */
+	smp_mb();
+
+	if (!smp_ops.smp_boot_secondary) {
+		pr_err_once("No smp_ops registered: could not bring up secondary CPUs\n");
+		return -ENOSYS;
+	}
+
+	ret = smp_ops.smp_boot_secondary(cpu);
+	if (ret == 0) {
+		/* CPU was successfully started */
+		while (!cpu_online(cpu))
+			cpu_relax();
+	} else {
+		pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
+	}
+
+	return ret;
+}
+
+
+
+static int __init set_smp_ops_by_method(struct device_node *node)
+{
+	const char *method;
+	struct of_cpu_method *m = __cpu_method_of_table;
+
+	if (of_property_read_string(node, "enable-method", &method))
+		return 0;
+
+	for (; m->method; m++)
+		if (!strcmp(m->method, method)) {
+			smp_set_ops(m->ops);
+			return 1;
+		}
+
+	return 0;
+}
+
+void __init smp_cpus_done(unsigned int max_cpus)
+{
+}
+
+void __init smp_init_cpus(void)
+{
+	struct device_node *cpu, *cpus;
+	u32 cpu_id;
+	unsigned int nr_cpus = 0;
+	int found_method = 0;
+
+	cpus = of_find_node_by_path("/cpus");
+	for_each_of_cpu_node(cpu) {
+		if (!of_device_is_available(cpu))
+			continue;
+
+		cpu_id = of_get_cpu_hwid(cpu, 0);
+		if ((cpu_id < NR_CPUS) && (nr_cpus < nr_cpu_ids)) {
+			nr_cpus++;
+			set_cpu_possible(cpu_id, true);
+			if (!found_method)
+				found_method = set_smp_ops_by_method(cpu);
+		}
+	}
+
+	if (!found_method)
+		set_smp_ops_by_method(cpus);
+
+	pr_info("%d possible cpus\n", nr_cpus);
+}
+
+void __init smp_prepare_cpus(unsigned int max_cpus)
+{
+	if (num_present_cpus() <= 1)
+		init_cpu_present(cpu_possible_mask);
+}
+
+/*
+ * C entry point for a secondary processor.
+ */
+asmlinkage void __init start_kernel_secondary(void)
+{
+	struct mm_struct *mm = &init_mm;
+	unsigned int cpu = smp_processor_id();
+
+	setup_processor();
+	kvx_mmu_early_setup();
+
+	/* All kernel threads share the same mm context.  */
+	mmgrab(mm);
+	current->active_mm = mm;
+	cpumask_set_cpu(cpu, mm_cpumask(mm));
+
+	notify_cpu_starting(cpu);
+	set_cpu_online(cpu, true);
+	trace_hardirqs_off();
+
+	local_flush_tlb_all();
+
+	local_irq_enable();
+	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
+}
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 7a5785f405b62..aa35c19dbd99a 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -147,6 +147,7 @@ enum cpuhp_state {
 	CPUHP_AP_IRQ_LOONGARCH_STARTING,
 	CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING,
 	CPUHP_AP_IRQ_RISCV_IMSIC_STARTING,
+	CPUHP_AP_IRQ_KVX_STARTING,
 	CPUHP_AP_ARM_MVEBU_COHERENCY,
 	CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
 	CPUHP_AP_PERF_X86_STARTING,
@@ -176,6 +177,7 @@ enum cpuhp_state {
 	CPUHP_AP_CSKY_TIMER_STARTING,
 	CPUHP_AP_TI_GP_TIMER_STARTING,
 	CPUHP_AP_HYPERV_TIMER_STARTING,
+	CPUHP_AP_KVX_TIMER_STARTING,
 	/* Must be the last timer callback */
 	CPUHP_AP_DUMMY_TIMER_STARTING,
 	CPUHP_AP_ARM_XEN_STARTING,
-- 
2.45.2

Re: [RFC PATCH v3 30/37] kvx: Add multi-processor (SMP) support

[Thomas Gleixner]

On Mon, Jul 22 2024 at 11:41, ysionneau@kalrayinc.com wrote:
> +
> +int __cpu_up(unsigned int cpu, struct task_struct *tidle)
> +{
> +	int ret;
> +
> +	__cpu_up_stack_pointer[cpu] = task_stack_page(tidle) + THREAD_SIZE;
> +	__cpu_up_task_pointer[cpu] = tidle;
> +	/* We need to be sure writes are committed */
> +	smp_mb();
> +
> +	if (!smp_ops.smp_boot_secondary) {
> +		pr_err_once("No smp_ops registered: could not bring up secondary CPUs\n");
> +		return -ENOSYS;
> +	}
> +
> +	ret = smp_ops.smp_boot_secondary(cpu);
> +	if (ret == 0) {
> +		/* CPU was successfully started */
> +		while (!cpu_online(cpu))
> +			cpu_relax();

Please use the generic CPU hotplug synchronization mechanisms CONFIG_HOTPLUG_*_SYNC

Thanks,

        tglx

[RFC PATCH v3 34/37] kvx: Add power controller driver

[ysionneau]

From: Yann Sionneau <ysionneau@kalrayinc.com>

The Power Controller (pwr-ctrl) controls cores reset and wake-up
procedure.

Co-developed-by: Clement Leger <clement@clement-leger.fr>
Signed-off-by: Clement Leger <clement@clement-leger.fr>
Co-developed-by: Julian Vetter <jvetter@kalrayinc.com>
Signed-off-by: Julian Vetter <jvetter@kalrayinc.com>
Co-developed-by: Louis Morhet <lmorhet@kalrayinc.com>
Signed-off-by: Louis Morhet <lmorhet@kalrayinc.com>
Co-developed-by: Marius Gligor <mgligor@kalrayinc.com>
Signed-off-by: Marius Gligor <mgligor@kalrayinc.com>
Signed-off-by: Jules Maselbas <jmaselbas@zdiv.net>
Signed-off-by: Yann Sionneau <ysionneau@kalrayinc.com>
---

Notes:
V1 -> V2: new patch
V2 -> V3:
- Moved driver from arch/kvx/platform to drivers/soc/kvx/
  see discussions there:
  - https://lore.kernel.org/bpf/Y8qlOpYgDefMPqWH@zx2c4.com/T/#m722d8f7c7501615251e4f97705198f5485865ce2
- indent
- add missing static qualifier
- driver now registers a cpu_method/smp_op via CPU_METHOD_OF_DECLARE
  like arm and sh, it puts a struct into a __cpu_method_of_table ELF section.
  the smp_ops is used by smpboot.c if its name matches the DT 'cpus' node
  enable-method property.
---
 arch/kvx/include/asm/pwr_ctrl.h     | 57 ++++++++++++++++++++
 drivers/soc/Kconfig                 |  1 +
 drivers/soc/Makefile                |  1 +
 drivers/soc/kvx/Kconfig             | 10 ++++
 drivers/soc/kvx/Makefile            |  2 +
 drivers/soc/kvx/coolidge_pwr_ctrl.c | 84 +++++++++++++++++++++++++++++
 6 files changed, 155 insertions(+)
 create mode 100644 arch/kvx/include/asm/pwr_ctrl.h
 create mode 100644 drivers/soc/kvx/Kconfig
 create mode 100644 drivers/soc/kvx/Makefile
 create mode 100644 drivers/soc/kvx/coolidge_pwr_ctrl.c

diff --git a/arch/kvx/include/asm/pwr_ctrl.h b/arch/kvx/include/asm/pwr_ctrl.h
new file mode 100644
index 0000000000000..715eddd45a88c
--- /dev/null
+++ b/arch/kvx/include/asm/pwr_ctrl.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2017-2024 Kalray Inc.
+ * Author(s): Clement Leger
+ *            Marius Gligor
+ *            Julian Vetter
+ *            Yann Sionneau
+ */
+
+#ifndef _ASM_KVX_PWR_CTRL_H
+#define _ASM_KVX_PWR_CTRL_H
+
+#ifndef __ASSEMBLY__
+
+static int kvx_pwr_ctrl_probe(void);
+
+int kvx_pwr_ctrl_cpu_poweron(unsigned int cpu);
+
+#endif
+
+#define PWR_CTRL_ADDR                           0xA40000
+
+/* Power controller vector register definitions */
+#define KVX_PWR_CTRL_VEC_OFFSET                 0x1000
+#define KVX_PWR_CTRL_VEC_WUP_SET_OFFSET         0x10
+#define KVX_PWR_CTRL_VEC_WUP_CLEAR_OFFSET       0x20
+
+/* Power controller PE reset PC register definitions */
+#define KVX_PWR_CTRL_RESET_PC_OFFSET            0x2000
+
+/* Power controller global register definitions */
+#define KVX_PWR_CTRL_GLOBAL_OFFSET              0x4040
+
+#define KVX_PWR_CTRL_GLOBAL_SET_OFFSET          0x10
+#define KVX_PWR_CTRL_GLOBAL_CLEAR_OFFSET        0x20
+#define KVX_PWR_CTRL_GLOBAL_SET_PE_EN_SHIFT     0x1
+
+#define PWR_CTRL_WUP_SET_OFFSET  \
+		(KVX_PWR_CTRL_VEC_OFFSET + \
+		 KVX_PWR_CTRL_VEC_WUP_SET_OFFSET)
+
+#define PWR_CTRL_WUP_CLEAR_OFFSET  \
+		(KVX_PWR_CTRL_VEC_OFFSET + \
+		 KVX_PWR_CTRL_VEC_WUP_CLEAR_OFFSET)
+
+#define PWR_CTRL_GLOBAL_CONFIG_SET_OFFSET \
+		(KVX_PWR_CTRL_GLOBAL_OFFSET + \
+		 KVX_PWR_CTRL_GLOBAL_SET_OFFSET)
+
+#define PWR_CTRL_GLOBAL_CONFIG_CLEAR_OFFSET \
+		(KVX_PWR_CTRL_GLOBAL_OFFSET + \
+		 KVX_PWR_CTRL_GLOBAL_CLEAR_OFFSET)
+
+#define PWR_CTRL_GLOBAL_CONFIG_PE_EN \
+	(1 << KVX_PWR_CTRL_GLOBAL_SET_PE_EN_SHIFT)
+
+#endif /* _ASM_KVX_PWR_CTRL_H */
diff --git a/drivers/soc/Kconfig b/drivers/soc/Kconfig
index 5d924e946507b..f28078620da14 100644
--- a/drivers/soc/Kconfig
+++ b/drivers/soc/Kconfig
@@ -12,6 +12,7 @@ source "drivers/soc/fujitsu/Kconfig"
 source "drivers/soc/hisilicon/Kconfig"
 source "drivers/soc/imx/Kconfig"
 source "drivers/soc/ixp4xx/Kconfig"
+source "drivers/soc/kvx/Kconfig"
 source "drivers/soc/litex/Kconfig"
 source "drivers/soc/loongson/Kconfig"
 source "drivers/soc/mediatek/Kconfig"
diff --git a/drivers/soc/Makefile b/drivers/soc/Makefile
index fb2bd31387d07..240e148eaaff8 100644
--- a/drivers/soc/Makefile
+++ b/drivers/soc/Makefile
@@ -16,6 +16,7 @@ obj-$(CONFIG_ARCH_GEMINI)	+= gemini/
 obj-y				+= hisilicon/
 obj-y				+= imx/
 obj-y				+= ixp4xx/
+obj-$(CONFIG_KVX)		+= kvx/
 obj-$(CONFIG_SOC_XWAY)		+= lantiq/
 obj-$(CONFIG_LITEX_SOC_CONTROLLER) += litex/
 obj-y				+= loongson/
diff --git a/drivers/soc/kvx/Kconfig b/drivers/soc/kvx/Kconfig
new file mode 100644
index 0000000000000..96d05efe4bfb5
--- /dev/null
+++ b/drivers/soc/kvx/Kconfig
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config COOLIDGE_POWER_CONTROLLER
+	bool "Coolidge power controller"
+	default n
+	depends on KVX
+	help
+	  The Kalray Coolidge Power Controller is used to manage the power
+	  state of secondary CPU cores. Currently only powering up is
+	  supported.
diff --git a/drivers/soc/kvx/Makefile b/drivers/soc/kvx/Makefile
new file mode 100644
index 0000000000000..c7b0b3e99eabc
--- /dev/null
+++ b/drivers/soc/kvx/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_COOLIDGE_POWER_CONTROLLER)	+= coolidge_pwr_ctrl.o
diff --git a/drivers/soc/kvx/coolidge_pwr_ctrl.c b/drivers/soc/kvx/coolidge_pwr_ctrl.c
new file mode 100644
index 0000000000000..67af3e446d0e7
--- /dev/null
+++ b/drivers/soc/kvx/coolidge_pwr_ctrl.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017-2024 Kalray Inc.
+ * Author(s): Clement Leger
+ *            Yann Sionneau
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/slab.h>
+#include <linux/smp.h>
+#include <linux/types.h>
+
+#include <asm/pwr_ctrl.h>
+#include <asm/symbols.h>
+
+struct kvx_pwr_ctrl {
+	void __iomem *regs;
+};
+
+static struct kvx_pwr_ctrl kvx_pwr_controller;
+
+static bool pwr_ctrl_not_initialized = true;
+
+/**
+ * kvx_pwr_ctrl_cpu_poweron() - Wakeup a cpu
+ * @cpu: cpu to wakeup
+ */
+int __init kvx_pwr_ctrl_cpu_poweron(unsigned int cpu)
+{
+	int ret = 0;
+
+	if (pwr_ctrl_not_initialized) {
+		pr_err("KVX power controller not initialized!\n");
+		return -ENODEV;
+	}
+
+	/* Set PE boot address */
+	writeq((unsigned long long)kvx_start,
+			kvx_pwr_controller.regs + KVX_PWR_CTRL_RESET_PC_OFFSET);
+	/* Wake up processor ! */
+	writeq(1ULL << cpu,
+	       kvx_pwr_controller.regs + PWR_CTRL_WUP_SET_OFFSET);
+	/* Then clear wakeup to allow processor to sleep */
+	writeq(1ULL << cpu,
+	       kvx_pwr_controller.regs + PWR_CTRL_WUP_CLEAR_OFFSET);
+
+	return ret;
+}
+
+static const struct smp_operations coolidge_smp_ops __initconst = {
+	.smp_boot_secondary = kvx_pwr_ctrl_cpu_poweron,
+};
+
+static int __init kvx_pwr_ctrl_probe(void)
+{
+	struct device_node *ctrl;
+
+	ctrl = of_find_compatible_node(NULL, NULL, "kalray,coolidge-pwr-ctrl");
+	if (!ctrl) {
+		pr_err("Failed to get power controller node\n");
+		return -EINVAL;
+	}
+
+	kvx_pwr_controller.regs = of_iomap(ctrl, 0);
+	if (!kvx_pwr_controller.regs) {
+		pr_err("Failed ioremap\n");
+		return -EINVAL;
+	}
+
+	pwr_ctrl_not_initialized = false;
+	pr_info("KVX power controller probed\n");
+
+	return 0;
+}
+
+CPU_METHOD_OF_DECLARE(coolidge_pwr_ctrl, "kalray,coolidge-pwr-ctrl",
+		      &coolidge_smp_ops);
+
+early_initcall(kvx_pwr_ctrl_probe);
-- 
2.45.2

Re: [RFC PATCH v3 34/37] kvx: Add power controller driver

[Krzysztof Kozlowski]

On 22/07/2024 11:41, ysionneau@kalrayinc.com wrote:
> From: Yann Sionneau <ysionneau@kalrayinc.com>
> 
> The Power Controller (pwr-ctrl) controls cores reset and wake-up
> procedure.
> 
> Co-developed-by: Clement Leger <clement@clement-leger.fr>
> Signed-off-by: Clement Leger <clement@clement-leger.fr>
> Co-developed-by: Julian Vetter <jvetter@kalrayinc.com>
> Signed-off-by: Julian Vetter <jvetter@kalrayinc.com>
> Co-developed-by: Louis Morhet <lmorhet@kalrayinc.com>
> Signed-off-by: Louis Morhet <lmorhet@kalrayinc.com>
> Co-developed-by: Marius Gligor <mgligor@kalrayinc.com>
> Signed-off-by: Marius Gligor <mgligor@kalrayinc.com>
> Signed-off-by: Jules Maselbas <jmaselbas@zdiv.net>
> Signed-off-by: Yann Sionneau <ysionneau@kalrayinc.com>
> ---
> 
> Notes:
> V1 -> V2: new patch
> V2 -> V3:
> - Moved driver from arch/kvx/platform to drivers/soc/kvx/
>   see discussions there:
>   - https://lore.kernel.org/bpf/Y8qlOpYgDefMPqWH@zx2c4.com/T/#m722d8f7c7501615251e4f97705198f5485865ce2
> - indent
> - add missing static qualifier
> - driver now registers a cpu_method/smp_op via CPU_METHOD_OF_DECLARE
>   like arm and sh, it puts a struct into a __cpu_method_of_table ELF section.
>   the smp_ops is used by smpboot.c if its name matches the DT 'cpus' node
>   enable-method property.
> ---
>  arch/kvx/include/asm/pwr_ctrl.h     | 57 ++++++++++++++++++++
>  drivers/soc/Kconfig                 |  1 +
>  drivers/soc/Makefile                |  1 +
>  drivers/soc/kvx/Kconfig             | 10 ++++
>  drivers/soc/kvx/Makefile            |  2 +
>  drivers/soc/kvx/coolidge_pwr_ctrl.c | 84 +++++++++++++++++++++++++++++
>  6 files changed, 155 insertions(+)
>  create mode 100644 arch/kvx/include/asm/pwr_ctrl.h
>  create mode 100644 drivers/soc/kvx/Kconfig
>  create mode 100644 drivers/soc/kvx/Makefile
>  create mode 100644 drivers/soc/kvx/coolidge_pwr_ctrl.c
> 
> diff --git a/arch/kvx/include/asm/pwr_ctrl.h b/arch/kvx/include/asm/pwr_ctrl.h
> new file mode 100644
> index 0000000000000..715eddd45a88c
> --- /dev/null
> +++ b/arch/kvx/include/asm/pwr_ctrl.h
> @@ -0,0 +1,57 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * Copyright (C) 2017-2024 Kalray Inc.
> + * Author(s): Clement Leger
> + *            Marius Gligor
> + *            Julian Vetter
> + *            Yann Sionneau
> + */
> +
> +#ifndef _ASM_KVX_PWR_CTRL_H
> +#define _ASM_KVX_PWR_CTRL_H
> +
> +#ifndef __ASSEMBLY__
> +
> +static int kvx_pwr_ctrl_probe(void);
> +
> +int kvx_pwr_ctrl_cpu_poweron(unsigned int cpu);
> +
> +#endif
> +
> +#define PWR_CTRL_ADDR                           0xA40000
> +
> +/* Power controller vector register definitions */
> +#define KVX_PWR_CTRL_VEC_OFFSET                 0x1000
> +#define KVX_PWR_CTRL_VEC_WUP_SET_OFFSET         0x10
> +#define KVX_PWR_CTRL_VEC_WUP_CLEAR_OFFSET       0x20
> +
> +/* Power controller PE reset PC register definitions */
> +#define KVX_PWR_CTRL_RESET_PC_OFFSET            0x2000
> +
> +/* Power controller global register definitions */
> +#define KVX_PWR_CTRL_GLOBAL_OFFSET              0x4040
> +
> +#define KVX_PWR_CTRL_GLOBAL_SET_OFFSET          0x10
> +#define KVX_PWR_CTRL_GLOBAL_CLEAR_OFFSET        0x20
> +#define KVX_PWR_CTRL_GLOBAL_SET_PE_EN_SHIFT     0x1
> +
> +#define PWR_CTRL_WUP_SET_OFFSET  \
> +		(KVX_PWR_CTRL_VEC_OFFSET + \
> +		 KVX_PWR_CTRL_VEC_WUP_SET_OFFSET)
> +
> +#define PWR_CTRL_WUP_CLEAR_OFFSET  \
> +		(KVX_PWR_CTRL_VEC_OFFSET + \
> +		 KVX_PWR_CTRL_VEC_WUP_CLEAR_OFFSET)
> +
> +#define PWR_CTRL_GLOBAL_CONFIG_SET_OFFSET \
> +		(KVX_PWR_CTRL_GLOBAL_OFFSET + \
> +		 KVX_PWR_CTRL_GLOBAL_SET_OFFSET)
> +
> +#define PWR_CTRL_GLOBAL_CONFIG_CLEAR_OFFSET \
> +		(KVX_PWR_CTRL_GLOBAL_OFFSET + \
> +		 KVX_PWR_CTRL_GLOBAL_CLEAR_OFFSET)
> +
> +#define PWR_CTRL_GLOBAL_CONFIG_PE_EN \
> +	(1 << KVX_PWR_CTRL_GLOBAL_SET_PE_EN_SHIFT)
> +
> +#endif /* _ASM_KVX_PWR_CTRL_H */
> diff --git a/drivers/soc/Kconfig b/drivers/soc/Kconfig
> index 5d924e946507b..f28078620da14 100644
> --- a/drivers/soc/Kconfig
> +++ b/drivers/soc/Kconfig
> @@ -12,6 +12,7 @@ source "drivers/soc/fujitsu/Kconfig"
>  source "drivers/soc/hisilicon/Kconfig"
>  source "drivers/soc/imx/Kconfig"
>  source "drivers/soc/ixp4xx/Kconfig"
> +source "drivers/soc/kvx/Kconfig"
>  source "drivers/soc/litex/Kconfig"
>  source "drivers/soc/loongson/Kconfig"
>  source "drivers/soc/mediatek/Kconfig"
> diff --git a/drivers/soc/Makefile b/drivers/soc/Makefile
> index fb2bd31387d07..240e148eaaff8 100644
> --- a/drivers/soc/Makefile
> +++ b/drivers/soc/Makefile
> @@ -16,6 +16,7 @@ obj-$(CONFIG_ARCH_GEMINI)	+= gemini/
>  obj-y				+= hisilicon/
>  obj-y				+= imx/
>  obj-y				+= ixp4xx/
> +obj-$(CONFIG_KVX)		+= kvx/
>  obj-$(CONFIG_SOC_XWAY)		+= lantiq/
>  obj-$(CONFIG_LITEX_SOC_CONTROLLER) += litex/
>  obj-y				+= loongson/
> diff --git a/drivers/soc/kvx/Kconfig b/drivers/soc/kvx/Kconfig
> new file mode 100644
> index 0000000000000..96d05efe4bfb5
> --- /dev/null
> +++ b/drivers/soc/kvx/Kconfig
> @@ -0,0 +1,10 @@
> +# SPDX-License-Identifier: GPL-2.0
> +
> +config COOLIDGE_POWER_CONTROLLER
> +	bool "Coolidge power controller"
> +	default n
> +	depends on KVX
> +	help
> +	  The Kalray Coolidge Power Controller is used to manage the power
> +	  state of secondary CPU cores. Currently only powering up is
> +	  supported.
> diff --git a/drivers/soc/kvx/Makefile b/drivers/soc/kvx/Makefile
> new file mode 100644
> index 0000000000000..c7b0b3e99eabc
> --- /dev/null
> +++ b/drivers/soc/kvx/Makefile
> @@ -0,0 +1,2 @@
> +# SPDX-License-Identifier: GPL-2.0
> +obj-$(CONFIG_COOLIDGE_POWER_CONTROLLER)	+= coolidge_pwr_ctrl.o
> diff --git a/drivers/soc/kvx/coolidge_pwr_ctrl.c b/drivers/soc/kvx/coolidge_pwr_ctrl.c
> new file mode 100644
> index 0000000000000..67af3e446d0e7
> --- /dev/null
> +++ b/drivers/soc/kvx/coolidge_pwr_ctrl.c
> @@ -0,0 +1,84 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (C) 2017-2024 Kalray Inc.
> + * Author(s): Clement Leger
> + *            Yann Sionneau
> + */
> +
> +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
> +
> +#include <linux/io.h>
> +#include <linux/module.h>
> +#include <linux/of_address.h>
> +#include <linux/of_platform.h>
> +#include <linux/slab.h>
> +#include <linux/smp.h>
> +#include <linux/types.h>
> +
> +#include <asm/pwr_ctrl.h>
> +#include <asm/symbols.h>
> +
> +struct kvx_pwr_ctrl {
> +	void __iomem *regs;
> +};
> +
> +static struct kvx_pwr_ctrl kvx_pwr_controller;
> +
> +static bool pwr_ctrl_not_initialized = true;

Do not use inverted meanings.

> +
> +/**
> + * kvx_pwr_ctrl_cpu_poweron() - Wakeup a cpu
> + * @cpu: cpu to wakeup
> + */
> +int __init kvx_pwr_ctrl_cpu_poweron(unsigned int cpu)
> +{
> +	int ret = 0;
> +
> +	if (pwr_ctrl_not_initialized) {
> +		pr_err("KVX power controller not initialized!\n");
> +		return -ENODEV;
> +	}
> +
> +	/* Set PE boot address */
> +	writeq((unsigned long long)kvx_start,

Addresses use kernel_ulong_t

> +			kvx_pwr_controller.regs + KVX_PWR_CTRL_RESET_PC_OFFSET);
> +	/* Wake up processor ! */
> +	writeq(1ULL << cpu,

That's BIT

> +	       kvx_pwr_controller.regs + PWR_CTRL_WUP_SET_OFFSET);
> +	/* Then clear wakeup to allow processor to sleep */
> +	writeq(1ULL << cpu,

BIT

> +	       kvx_pwr_controller.regs + PWR_CTRL_WUP_CLEAR_OFFSET);
> +
> +	return ret;
> +}
> +
> +static const struct smp_operations coolidge_smp_ops __initconst = {
> +	.smp_boot_secondary = kvx_pwr_ctrl_cpu_poweron,
> +};
> +
> +static int __init kvx_pwr_ctrl_probe(void)

That's not a probe, please rename to avoid confusion. Or make it a
proper device driver.

> +{
> +	struct device_node *ctrl;
> +
> +	ctrl = of_find_compatible_node(NULL, NULL, "kalray,coolidge-pwr-ctrl");
> +	if (!ctrl) {
> +		pr_err("Failed to get power controller node\n");
> +		return -EINVAL;
> +	}
> +
> +	kvx_pwr_controller.regs = of_iomap(ctrl, 0);
> +	if (!kvx_pwr_controller.regs) {
> +		pr_err("Failed ioremap\n");
> +		return -EINVAL;
> +	}
> +
> +	pwr_ctrl_not_initialized = false;
> +	pr_info("KVX power controller probed\n");
> +
> +	return 0;
> +}
> +
> +CPU_METHOD_OF_DECLARE(coolidge_pwr_ctrl, "kalray,coolidge-pwr-ctrl",
> +		      &coolidge_smp_ops);
> +
> +early_initcall(kvx_pwr_ctrl_probe);

Best regards,
Krzysztof

Re: [RFC PATCH v3 34/37] kvx: Add power controller driver

[Yann Sionneau]

Hello Krzysztof,

On 22/07/2024 14:37, Krzysztof Kozlowski wrote:
> On 22/07/2024 11:41, ysionneau@kalrayinc.com wrote:
>> From: Yann Sionneau <ysionneau@kalrayinc.com>
>>
>> The Power Controller (pwr-ctrl) controls cores reset and wake-up
>> procedure.
>> [...]
>> +
>> +static bool pwr_ctrl_not_initialized = true;
> Do not use inverted meanings.
Ack, I will fix this.
>
>> +
>> +/**
>> + * kvx_pwr_ctrl_cpu_poweron() - Wakeup a cpu
>> + * @cpu: cpu to wakeup
>> + */
>> +int __init kvx_pwr_ctrl_cpu_poweron(unsigned int cpu)
>> +{
>> +	int ret = 0;
>> +
>> +	if (pwr_ctrl_not_initialized) {
>> +		pr_err("KVX power controller not initialized!\n");
>> +		return -ENODEV;
>> +	}
>> +
>> +	/* Set PE boot address */
>> +	writeq((unsigned long long)kvx_start,
> Addresses use kernel_ulong_t
Ack, I will fix this.
>
>> +			kvx_pwr_controller.regs + KVX_PWR_CTRL_RESET_PC_OFFSET);
>> +	/* Wake up processor ! */
>> +	writeq(1ULL << cpu,
> That's BIT
Ack, I will fix this and replace with BITULL(cpu).
>
>> +	       kvx_pwr_controller.regs + PWR_CTRL_WUP_SET_OFFSET);
>> +	/* Then clear wakeup to allow processor to sleep */
>> +	writeq(1ULL << cpu,
> BIT
Ack.
>
>> +	       kvx_pwr_controller.regs + PWR_CTRL_WUP_CLEAR_OFFSET);
>> +
>> +	return ret;
>> +}
>> +
>> +static const struct smp_operations coolidge_smp_ops __initconst = {
>> +	.smp_boot_secondary = kvx_pwr_ctrl_cpu_poweron,
>> +};
>> +
>> +static int __init kvx_pwr_ctrl_probe(void)
> That's not a probe, please rename to avoid confusion. Or make it a
> proper device driver.

Ok, I will probably rename it kvx_pwr_ctrl_init()

Thanks!

-- 

Yann

[RFC PATCH v3 35/37] kvx: Add IPI driver

[ysionneau]

From: Yann Sionneau <ysionneau@kalrayinc.com>

The Inter-Processor Interrupt Controller (IPI) provides a fast
synchronization mechanism to the software. It exposes eight independent
sets of registers that can be used to notify each processor in the cluster.

Co-developed-by: Clement Leger <clement@clement-leger.fr>
Signed-off-by: Clement Leger <clement@clement-leger.fr>
Co-developed-by: Guillaume Thouvenin <thouveng@gmail.com>
Signed-off-by: Guillaume Thouvenin <thouveng@gmail.com>
Co-developed-by: Julian Vetter <jvetter@kalrayinc.com>
Signed-off-by: Julian Vetter <jvetter@kalrayinc.com>
Co-developed-by: Luc Michel <luc@lmichel.fr>
Signed-off-by: Luc Michel <luc@lmichel.fr>
Signed-off-by: Jules Maselbas <jmaselbas@zdiv.net>
Signed-off-by: Yann Sionneau <ysionneau@kalrayinc.com>
---

Notes:
V1 -> V2: new patch
V2 -> V3:
- Restructured IPI code according to reviewer feedback
  - move from arch/kvx/platform to drivers/irqchip/
  - remove bogus comment
  - call set_smp_cross_call() to set smpboot.c's ipi function pointer
  - feedbacks: https://lore.kernel.org/bpf/Y8qlOpYgDefMPqWH@zx2c4.com/T/#mb02884ea498e627c2621973157330f2ea9977190
---
 arch/kvx/include/asm/ipi.h         |  16 ++++
 drivers/irqchip/Kconfig            |   4 +
 drivers/irqchip/Makefile           |   1 +
 drivers/irqchip/irq-kvx-ipi-ctrl.c | 143 +++++++++++++++++++++++++++++
 4 files changed, 164 insertions(+)
 create mode 100644 arch/kvx/include/asm/ipi.h
 create mode 100644 drivers/irqchip/irq-kvx-ipi-ctrl.c

diff --git a/arch/kvx/include/asm/ipi.h b/arch/kvx/include/asm/ipi.h
new file mode 100644
index 0000000000000..a23275d19d225
--- /dev/null
+++ b/arch/kvx/include/asm/ipi.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2017-2023 Kalray Inc.
+ * Author(s): Clement Leger
+ */
+
+#ifndef _ASM_KVX_IPI_H
+#define _ASM_KVX_IPI_H
+
+#include <linux/irqreturn.h>
+
+int kvx_ipi_ctrl_init(struct device_node *node, struct device_node *parent);
+
+void kvx_ipi_send(const struct cpumask *mask, unsigned int operation);
+
+#endif /* _ASM_KVX_IPI_H */
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index da1dbd79dab54..65db9990cf475 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -337,6 +337,10 @@ config KVX_CORE_INTC
 	depends on KVX
 	select IRQ_DOMAIN
 
+config KVX_IPI_CTRL
+	bool
+	depends on KVX
+
 config KVX_APIC_GIC
 	bool
 	depends on KVX
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 30b69db8789f7..f8fa246df74d2 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -69,6 +69,7 @@ obj-$(CONFIG_BRCMSTB_L2_IRQ)		+= irq-brcmstb-l2.o
 obj-$(CONFIG_KEYSTONE_IRQ)		+= irq-keystone.o
 obj-$(CONFIG_MIPS_GIC)			+= irq-mips-gic.o
 obj-$(CONFIG_KVX_CORE_INTC)		+= irq-kvx-core-intc.o
+obj-$(CONFIG_KVX_IPI_CTRL)		+= irq-kvx-ipi-ctrl.o
 obj-$(CONFIG_KVX_APIC_GIC)		+= irq-kvx-apic-gic.o
 obj-$(CONFIG_KVX_ITGEN)			+= irq-kvx-itgen.o
 obj-$(CONFIG_KVX_APIC_MAILBOX)		+= irq-kvx-apic-mailbox.o
diff --git a/drivers/irqchip/irq-kvx-ipi-ctrl.c b/drivers/irqchip/irq-kvx-ipi-ctrl.c
new file mode 100644
index 0000000000000..09d955a5c109a
--- /dev/null
+++ b/drivers/irqchip/irq-kvx-ipi-ctrl.c
@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017-2024 Kalray Inc.
+ *
+ * Author(s): Clement Leger
+ *            Jonathan Borne
+ *            Luc Michel
+ */
+
+#define pr_fmt(fmt)	"kvx_ipi_ctrl: " fmt
+
+#include <linux/smp.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/irqchip.h>
+#include <linux/of_irq.h>
+#include <linux/cpumask.h>
+#include <linux/interrupt.h>
+#include <linux/cpuhotplug.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+
+#include <asm/ipi.h>
+
+#define IPI_INTERRUPT_OFFSET	0x0
+#define IPI_MASK_OFFSET		0x20
+
+/*
+ * IPI controller can signal RM and PE0 -> 15
+ * In order to restrict that to the PE, write the corresponding mask
+ */
+#define KVX_IPI_CPU_MASK	(~0xFFFF)
+
+/* A collection of single bit ipi messages.  */
+static DEFINE_PER_CPU_ALIGNED(unsigned long, ipi_data);
+
+struct kvx_ipi_ctrl {
+	void __iomem *regs;
+	unsigned int ipi_irq;
+};
+
+static struct kvx_ipi_ctrl kvx_ipi_controller;
+
+void kvx_ipi_send(const struct cpumask *mask, unsigned int operation)
+{
+	const unsigned long *maskb = cpumask_bits(mask);
+	unsigned long flags;
+	int cpu;
+
+	/* Set operation that must be done by receiver */
+	for_each_cpu(cpu, mask)
+		set_bit(operation, &per_cpu(ipi_data, cpu));
+
+	/* Commit the write before sending IPI */
+	smp_wmb();
+
+	local_irq_save(flags);
+
+	WARN_ON(*maskb & KVX_IPI_CPU_MASK);
+	writel(*maskb, kvx_ipi_controller.regs + IPI_INTERRUPT_OFFSET);
+
+	local_irq_restore(flags);
+}
+
+static int kvx_ipi_starting_cpu(unsigned int cpu)
+{
+	enable_percpu_irq(kvx_ipi_controller.ipi_irq, IRQ_TYPE_NONE);
+
+	return 0;
+}
+
+static int kvx_ipi_dying_cpu(unsigned int cpu)
+{
+	disable_percpu_irq(kvx_ipi_controller.ipi_irq);
+
+	return 0;
+}
+
+static irqreturn_t ipi_irq_handler(int irq, void *dev_id)
+{
+	unsigned long *pending_ipis = &per_cpu(ipi_data, smp_processor_id());
+
+	while (true) {
+		unsigned long ops = xchg(pending_ipis, 0);
+
+		if (!ops)
+			return IRQ_HANDLED;
+
+		handle_IPI(ops);
+	}
+
+	return IRQ_HANDLED;
+}
+
+int __init kvx_ipi_ctrl_init(struct device_node *node,
+			     struct device_node *parent)
+{
+	int ret;
+	unsigned int ipi_irq;
+	void __iomem *ipi_base;
+
+	BUG_ON(!node);
+
+	ipi_base = of_iomap(node, 0);
+	BUG_ON(!ipi_base);
+
+	kvx_ipi_controller.regs = ipi_base;
+
+	/* Init mask for interrupts to PE0 -> PE15 */
+	writel(KVX_IPI_CPU_MASK, kvx_ipi_controller.regs + IPI_MASK_OFFSET);
+
+	ipi_irq = irq_of_parse_and_map(node, 0);
+	if (!ipi_irq) {
+		pr_err("Failed to parse irq: %d\n", ipi_irq);
+		return -EINVAL;
+	}
+
+	ret = request_percpu_irq(ipi_irq, ipi_irq_handler,
+						"kvx_ipi", &kvx_ipi_controller);
+	if (ret) {
+		pr_err("can't register interrupt %d (%d)\n",
+						ipi_irq, ret);
+		return ret;
+	}
+	kvx_ipi_controller.ipi_irq = ipi_irq;
+
+	ret = cpuhp_setup_state(CPUHP_AP_IRQ_KVX_STARTING,
+				"kvx/ipi:online",
+				kvx_ipi_starting_cpu,
+				kvx_ipi_dying_cpu);
+	if (ret < 0) {
+		pr_err("Failed to setup hotplug state");
+		return ret;
+	}
+
+	set_smp_cross_call(kvx_ipi_send);
+	pr_info("controller probed\n");
+
+	return 0;
+}
+IRQCHIP_DECLARE(kvx_ipi_ctrl, "kalray,coolidge-ipi-ctrl", kvx_ipi_ctrl_init);
-- 
2.45.2

Re: [RFC PATCH v3 35/37] kvx: Add IPI driver

[Krzysztof Kozlowski]

On 22/07/2024 11:41, ysionneau@kalrayinc.com wrote:
> From: Yann Sionneau <ysionneau@kalrayinc.com>
> 
> The Inter-Processor Interrupt Controller (IPI) provides a fast
> synchronization mechanism to the software. It exposes eight independent
> sets of registers that can be used to notify each processor in the cluster.
> 
> Co-developed-by: Clement Leger <clement@clement-leger.fr>
> Signed-off-by: Clement Leger <clement@clement-leger.fr>
> Co-developed-by: Guillaume Thouvenin <thouveng@gmail.com>
> Signed-off-by: Guillaume Thouvenin <thouveng@gmail.com>
> Co-developed-by: Julian Vetter <jvetter@kalrayinc.com>
> Signed-off-by: Julian Vetter <jvetter@kalrayinc.com>
> Co-developed-by: Luc Michel <luc@lmichel.fr>
> Signed-off-by: Luc Michel <luc@lmichel.fr>
> Signed-off-by: Jules Maselbas <jmaselbas@zdiv.net>
> Signed-off-by: Yann Sionneau <ysionneau@kalrayinc.com>

...

> +
> +int __init kvx_ipi_ctrl_init(struct device_node *node,
> +			     struct device_node *parent)
> +{
> +	int ret;
> +	unsigned int ipi_irq;
> +	void __iomem *ipi_base;
> +
> +	BUG_ON(!node);

Fix your code instead.

> +
> +	ipi_base = of_iomap(node, 0);
> +	BUG_ON(!ipi_base);

No, handle it by returning.

> +
> +	kvx_ipi_controller.regs = ipi_base;
> +
> +	/* Init mask for interrupts to PE0 -> PE15 */
> +	writel(KVX_IPI_CPU_MASK, kvx_ipi_controller.regs + IPI_MASK_OFFSET);
> +
> +	ipi_irq = irq_of_parse_and_map(node, 0);
> +	if (!ipi_irq) {
> +		pr_err("Failed to parse irq: %d\n", ipi_irq);
> +		return -EINVAL;
> +	}
> +
> +	ret = request_percpu_irq(ipi_irq, ipi_irq_handler,
> +						"kvx_ipi", &kvx_ipi_controller);
> +	if (ret) {
> +		pr_err("can't register interrupt %d (%d)\n",
> +						ipi_irq, ret);
> +		return ret;
> +	}
> +	kvx_ipi_controller.ipi_irq = ipi_irq;
> +
> +	ret = cpuhp_setup_state(CPUHP_AP_IRQ_KVX_STARTING,
> +				"kvx/ipi:online",
> +				kvx_ipi_starting_cpu,
> +				kvx_ipi_dying_cpu);
> +	if (ret < 0) {
> +		pr_err("Failed to setup hotplug state");
> +		return ret;
> +	}
> +
> +	set_smp_cross_call(kvx_ipi_send);
> +	pr_info("controller probed\n");

Drop this simple probe successes. This is not the way to trace system
bootup. Keep only reasonable amount, not every driver printing that its
initcall started.

Best regards,
Krzysztof

Re: [RFC PATCH v3 35/37] kvx: Add IPI driver

[Yann Sionneau]

Hello Krzysztof,

On 22/07/2024 14:39, Krzysztof Kozlowski wrote:
> On 22/07/2024 11:41, ysionneau@kalrayinc.com wrote:
>> From: Yann Sionneau <ysionneau@kalrayinc.com>
>>
>> [...]
>> +
>> +int __init kvx_ipi_ctrl_init(struct device_node *node,
>> +			     struct device_node *parent)
>> +{
>> +	int ret;
>> +	unsigned int ipi_irq;
>> +	void __iomem *ipi_base;
>> +
>> +	BUG_ON(!node);
> Fix your code instead.

I am not sure I understand your comment here, I don't have the control over what the kernel passes to my driver, do I?

On the other hand, "node" being the node that matches the compatible, maybe it can never be NULL, is that what you're saying?

After doing some archeology in our old code base it seems indeed this line is an artefact of this previous code snippet:

```

np = of_find_compatible_node(NULL, NULL, "kalray,coolidge-ipi-ctrl");
BUG_ON(!np);

```

Now that this is a real driver declared via IRQCHIP_DECLARE(), I guess that this check isn't needed anymore.

>
>> +
>> +	ipi_base = of_iomap(node, 0);
>> +	BUG_ON(!ipi_base);
> No, handle it by returning.
Ack
>
>> +
>> +	kvx_ipi_controller.regs = ipi_base;
>> +
>> +	/* Init mask for interrupts to PE0 -> PE15 */
>> +	writel(KVX_IPI_CPU_MASK, kvx_ipi_controller.regs + IPI_MASK_OFFSET);
>> +
>> +	ipi_irq = irq_of_parse_and_map(node, 0);
>> +	if (!ipi_irq) {
>> +		pr_err("Failed to parse irq: %d\n", ipi_irq);
>> +		return -EINVAL;
>> +	}
>> +
>> +	ret = request_percpu_irq(ipi_irq, ipi_irq_handler,
>> +						"kvx_ipi", &kvx_ipi_controller);
>> +	if (ret) {
>> +		pr_err("can't register interrupt %d (%d)\n",
>> +						ipi_irq, ret);
>> +		return ret;
>> +	}
>> +	kvx_ipi_controller.ipi_irq = ipi_irq;
>> +
>> +	ret = cpuhp_setup_state(CPUHP_AP_IRQ_KVX_STARTING,
>> +				"kvx/ipi:online",
>> +				kvx_ipi_starting_cpu,
>> +				kvx_ipi_dying_cpu);
>> +	if (ret < 0) {
>> +		pr_err("Failed to setup hotplug state");
>> +		return ret;
>> +	}
>> +
>> +	set_smp_cross_call(kvx_ipi_send);
>> +	pr_info("controller probed\n");
> Drop this simple probe successes. This is not the way to trace system
> bootup. Keep only reasonable amount, not every driver printing that its
> initcall started.
Ack.

Thanks for the review!

Regards,

-- 

Yann

Re: [RFC PATCH v3 35/37] kvx: Add IPI driver

[Krzysztof Kozlowski]

On 23/08/2024 16:46, Yann Sionneau wrote:
> Hello Krzysztof,
> 
> On 22/07/2024 14:39, Krzysztof Kozlowski wrote:
>> On 22/07/2024 11:41, ysionneau@kalrayinc.com wrote:
>>> From: Yann Sionneau <ysionneau@kalrayinc.com>
>>>
>>> [...]
>>> +
>>> +int __init kvx_ipi_ctrl_init(struct device_node *node,
>>> +			     struct device_node *parent)
>>> +{
>>> +	int ret;
>>> +	unsigned int ipi_irq;
>>> +	void __iomem *ipi_base;
>>> +
>>> +	BUG_ON(!node);
>> Fix your code instead.
> 
> I am not sure I understand your comment here, I don't have the control over what the kernel passes to my driver, do I?

In general you have. Investigate the path and check whether NULL is
allowed. If it is allowed, then this should be handled correctly and
gracefully. If it is not allowed, then BUG_ON() is not welcomed in general.

> 
> On the other hand, "node" being the node that matches the compatible, maybe it can never be NULL, is that what you're saying?

I don't remember the context anymore. You responded one month after my
review. But if this is about matching, then obviously this cannot happen
for DT platforms. If this can be matched via different methods then it
should not be BUG_ON...


Best regards,
Krzysztof

Re: [RFC PATCH v3 35/37] kvx: Add IPI driver

[Thomas Gleixner]

On Mon, Jul 22 2024 at 11:41, ysionneau@kalrayinc.com wrote:
> +/*
> + * IPI controller can signal RM and PE0 -> 15
> + * In order to restrict that to the PE, write the corresponding mask

This comment is undecodable

> + */
> +#define KVX_IPI_CPU_MASK	(~0xFFFF)
> +
> +/* A collection of single bit ipi messages.  */
> +static DEFINE_PER_CPU_ALIGNED(unsigned long, ipi_data);
> +
> +struct kvx_ipi_ctrl {
> +	void __iomem *regs;
> +	unsigned int ipi_irq;
> +};
> +
> +static struct kvx_ipi_ctrl kvx_ipi_controller;
> +
> +void kvx_ipi_send(const struct cpumask *mask, unsigned int operation)

Why is this global? It's only used in this file, no?

> +{
> +	const unsigned long *maskb = cpumask_bits(mask);
> +	unsigned long flags;
> +	int cpu;
> +
> +	/* Set operation that must be done by receiver */
> +	for_each_cpu(cpu, mask)
> +		set_bit(operation, &per_cpu(ipi_data, cpu));
> +
> +	/* Commit the write before sending IPI */
> +	smp_wmb();
> +
> +	local_irq_save(flags);
> +
> +	WARN_ON(*maskb & KVX_IPI_CPU_MASK);

> +#define KVX_IPI_CPU_MASK	(~0xFFFF)

This means the system is limited to 16 CPUs, right?

How should a bit >= NR_CPUs be set in a valid cpu mask?  Also above you
happily iterate the full cpumask. This does not make sense.

> +	writel(*maskb, kvx_ipi_controller.regs + IPI_INTERRUPT_OFFSET);
> +
> +	local_irq_restore(flags);
> +}
> +
> +static int kvx_ipi_starting_cpu(unsigned int cpu)
> +{
> +	enable_percpu_irq(kvx_ipi_controller.ipi_irq, IRQ_TYPE_NONE);
> +
> +	return 0;
> +}
> +
> +static int kvx_ipi_dying_cpu(unsigned int cpu)
> +{
> +	disable_percpu_irq(kvx_ipi_controller.ipi_irq);
> +
> +	return 0;
> +}
> +
> +static irqreturn_t ipi_irq_handler(int irq, void *dev_id)
> +{
> +	unsigned long *pending_ipis = &per_cpu(ipi_data, smp_processor_id());

  this_cpu_ptr() ?

> +	while (true) {
> +		unsigned long ops = xchg(pending_ipis, 0);
> +
> +		if (!ops)
> +			return IRQ_HANDLED;
> +
> +		handle_IPI(ops);
> +	}

        for (ops = xchg(pending_ipis, 0); ops; ops = xchg(pending_ipis, 0))
        	handle_IPI(ops);

Hmm?

> +	return IRQ_HANDLED;
> +}
> +
> +int __init kvx_ipi_ctrl_init(struct device_node *node,
> +			     struct device_node *parent)
> +{
> +	int ret;
> +	unsigned int ipi_irq;
> +	void __iomem *ipi_base;
> +
> +	BUG_ON(!node);
> +
> +	ipi_base = of_iomap(node, 0);

What's the point of this ipi_base indirection? Just use controller.regs
directly.

> +	BUG_ON(!ipi_base);
> +
> +	kvx_ipi_controller.regs = ipi_base;
> +
> +	/* Init mask for interrupts to PE0 -> PE15 */
> +	writel(KVX_IPI_CPU_MASK, kvx_ipi_controller.regs + IPI_MASK_OFFSET);
> +
> +	ipi_irq = irq_of_parse_and_map(node, 0);
> +	if (!ipi_irq) {
> +		pr_err("Failed to parse irq: %d\n", ipi_irq);
> +		return -EINVAL;
> +	}
> +
> +	ret = request_percpu_irq(ipi_irq, ipi_irq_handler,
> +						"kvx_ipi", &kvx_ipi_controller);
> +	if (ret) {
> +		pr_err("can't register interrupt %d (%d)\n",
> +						ipi_irq, ret);
> +		return ret;
> +	}
> +	kvx_ipi_controller.ipi_irq = ipi_irq;
> +
> +	ret = cpuhp_setup_state(CPUHP_AP_IRQ_KVX_STARTING,
> +				"kvx/ipi:online",
> +				kvx_ipi_starting_cpu,
> +				kvx_ipi_dying_cpu);
> +	if (ret < 0) {
> +		pr_err("Failed to setup hotplug state");

That leaves the half initialized IPI handler around.

> +		return ret;

Thanks,

        tglx