[PATCH v2 13/19] Execute crypto initcalls during module initialization

[Jay Wang <wanjay@amazon.com>]

After moving builtin cryptos into the standalone crypto module, to ensure
all such cryptos are properly initialized as they were builtin, the
initcalls of the cryptos should be executed at proper booting time.

To do so, this patch adds run_initcalls() function to execute the
initialization calls of the collected cryptos right after the module is
loaded. The function iterates through initcalls in the __fips_initcalls
section (which stores all function addresses of the collected crypto
initcalls) and executes them.

A key consideration is to ensure the crypto initcalls are executed in
proper order, for instance, some crypto initcalls are ought to execute
at `late_initcall` boot time level while some should be executed at
`module_init` boot time level. To do so, this patch enables coordination
between kernel and the crypto module initialization to preserve proper
execution order by hijacking the kernel initialization function
`do_initcall_level` with added synchronization helpers.

Signed-off-by: Jay Wang <wanjay@amazon.com>
---
 crypto/fips140/fips140-loader.c | 93 +++++++++++++++++++++++++++++++++
 crypto/fips140/fips140-module.c | 67 ++++++++++++++++++++++++
 crypto/fips140/fips140-module.h | 12 +++++
 include/linux/init.h            | 10 ++++
 init/main.c                     |  4 ++
 5 files changed, 186 insertions(+)

diff --git a/crypto/fips140/fips140-loader.c b/crypto/fips140/fips140-loader.c
index 490fb7276befb..ab9a3f9c5599e 100644
--- a/crypto/fips140/fips140-loader.c
+++ b/crypto/fips140/fips140-loader.c
@@ -100,3 +100,96 @@ static int fips_loader_init(void)
 	vfree(vmalloc_mem); // Free after successful module loading
 	return ret;
 }
+
+/* FIPS140 synchronization between kernel and module
+ *
+ * Synchronization levels map kernel initcall levels to FIPS module levels:
+ * - Level 0: subsys_initcall (kernel init level 4) - Basic subsystem initialization
+ * - Level 1: device_initcall (kernel init level 6) - Device driver initialization  
+ * - Level 2: late_initcall (kernel init level 7) - Late system initialization
+ *
+ * The kernel marks each level complete and waits for the FIPS module to
+ * complete the corresponding level before proceeding to ensure proper
+ * initialization ordering between kernel crypto and FIPS module.
+ */
+atomic_t fips140_kernel_level_complete = ATOMIC_INIT(0);
+atomic_t fips140_module_level_complete = ATOMIC_INIT(0);
+
+/* Wait queues for efficient synchronization */
+DECLARE_WAIT_QUEUE_HEAD(fips140_kernel_wq);
+DECLARE_WAIT_QUEUE_HEAD(fips140_module_wq);
+
+void fips140_mark_kernel_level_complete(int level)
+{
+	atomic_or(1 << level, &fips140_kernel_level_complete);
+	wake_up(&fips140_kernel_wq);
+}
+
+bool fips140_is_kernel_level_complete(int level)
+{
+	return atomic_read(&fips140_kernel_level_complete) & (1 << level);
+}
+
+bool fips140_is_module_level_complete(int level)
+{
+	return atomic_read(&fips140_module_level_complete) & (1 << level);
+}
+
+void fips140_mark_module_level_complete(int level)
+{
+	atomic_or(1 << level, &fips140_module_level_complete);
+	wake_up(&fips140_module_wq);
+}
+
+static int fips140_sync_thread(void *data)
+{
+	pr_info("FIPS 140: starting sync thread\n");
+	
+	/* Call FIPS loader explicitly */
+	int ret = fips_loader_init();
+	if (ret)
+		panic("FIPS 140: loader initialization failed: %d\n", ret);
+	
+	pr_info("FIPS 140: sync thread finished\n");
+	return 0;
+}
+
+void __init start_fips140_loader(void)
+{
+	struct task_struct *task;
+	
+	task = kthread_run(fips140_sync_thread, NULL, "fips140_sync");
+	if (IS_ERR(task)) {
+		panic("FIPS 140: failed to create sync thread\n");
+	}
+}
+
+void __init wait_until_fips140_level_sync(int level)
+{
+	/* Map kernel initcall levels to FIPS module levels */
+	int fips_level = -1;
+	if (level == 3) { /* Start FIPS loader thread at arch_initcall_sync level */
+		start_fips140_loader();
+		fips_level = 0;
+	} else if (level == 4) /* subsys_initcall */
+		fips_level = 1;
+	else if (level == 6) /* device_initcall */
+		fips_level = 2;
+	else if (level == 7) /* late_initcall */
+		fips_level = 3;
+
+	if (fips_level >= 0) {
+		/* Mark kernel level complete and wait for module level completion */
+		fips140_mark_kernel_level_complete(fips_level);
+		wait_event(fips140_module_wq, fips140_is_module_level_complete(fips_level));
+	}
+}
+
+EXPORT_SYMBOL(fips140_kernel_level_complete);
+EXPORT_SYMBOL(fips140_module_level_complete);
+EXPORT_SYMBOL(fips140_kernel_wq);
+EXPORT_SYMBOL(fips140_module_wq);
+EXPORT_SYMBOL(fips140_mark_kernel_level_complete);
+EXPORT_SYMBOL(fips140_is_kernel_level_complete);
+EXPORT_SYMBOL(fips140_is_module_level_complete);
+EXPORT_SYMBOL(fips140_mark_module_level_complete);
\ No newline at end of file
diff --git a/crypto/fips140/fips140-module.c b/crypto/fips140/fips140-module.c
index a942de8780efb..8767d98baacd9 100644
--- a/crypto/fips140/fips140-module.c
+++ b/crypto/fips140/fips140-module.c
@@ -16,9 +16,76 @@
 
 #define CRYPTO_INTERNAL "CRYPTO_INTERNAL"
 
+/*
+ * Run FIPS module initcalls level by level, synchronizing with the
+ * kernel's initcall progression.
+ *
+ * Initcall section mapping (see include/linux/module.h):
+ *   Level 0 (.fips_initcall0) <- subsys_initcall()
+ *                                Syncs with kernel subsys_initcall (initcall level 4)
+ *   Level 1 (.fips_initcall1) <- module_init()
+ *                                Syncs with kernel device_initcall (initcall level 6)
+ *   Level 2 (.fips_initcall2) <- late_initcall()
+ *                                Syncs with kernel late_initcall (initcall level 7)
+ */
+
+static int __init run_initcalls(void)
+{
+	typedef int (*initcall_t)(void);
+	
+	extern initcall_t __fips140_initcall0_start[], __fips140_initcall0_end[];
+	extern initcall_t __fips140_initcall1_start[], __fips140_initcall1_end[];
+	extern initcall_t __fips140_initcall2_start[], __fips140_initcall2_end[];
+
+	initcall_t *starts[] = {
+		__fips140_initcall0_start,
+		__fips140_initcall1_start,
+		__fips140_initcall2_start,
+	};
+	
+	initcall_t *ends[] = {
+		__fips140_initcall0_end,
+		__fips140_initcall1_end,
+		__fips140_initcall2_end,
+	};
+
+	pr_info("FIPS 140: run_initcalls starting\n");
+
+	for (int level = 0; level < ARRAY_SIZE(starts); level++) {
+		
+		/* Run FIPS initcalls for this level */
+		for (initcall_t *initcall = starts[level]; initcall < ends[level]; ++initcall) {
+			int ret;
+			initcall_t fn = *initcall;
+			
+			ret = fn();
+			if (!ret || ret == -ENODEV)
+				continue;
+
+			pr_err("FIPS 140: initcall %pS failed: %d\n", fn, ret);
+		}
+	
+		if (level < 2)
+			fips140_mark_module_level_complete(level + 1);
+		/* Wait for kernel to complete this level */
+		wait_event(fips140_kernel_wq, fips140_is_kernel_level_complete(level + 1));
+	}
+
+	pr_info("FIPS 140: run_initcalls finished\n");
+	return 0;
+}
+
 /* Initialize the FIPS 140 module */
 static int __init fips140_init(void)
 {
+	/* Signal that module is loaded and address placeholders are populated */
+	fips140_mark_module_level_complete(0);
+	wait_event(fips140_kernel_wq, fips140_is_kernel_level_complete(0));
+
+    pr_info("loading " FIPS140_MODULE_NAME "\n");
+
+	run_initcalls();
+	fips140_mark_module_level_complete(3);
     return 0;
 }
 
diff --git a/crypto/fips140/fips140-module.h b/crypto/fips140/fips140-module.h
index ed2b6e17969fc..e95dac8eeda9e 100644
--- a/crypto/fips140/fips140-module.h
+++ b/crypto/fips140/fips140-module.h
@@ -10,5 +10,17 @@
 #include <linux/crypto.h>
 #include <crypto/algapi.h>
 #include <linux/init.h>
+#include <linux/atomic.h>
+#include <linux/wait.h>
+
+/* FIPS140 synchronization between kernel and module */
+extern atomic_t fips140_kernel_level_complete;
+extern atomic_t fips140_module_level_complete;
+extern wait_queue_head_t fips140_kernel_wq;
+
+void fips140_mark_kernel_level_complete(int level);
+bool fips140_is_kernel_level_complete(int level);
+bool fips140_is_module_level_complete(int level);
+void fips140_mark_module_level_complete(int level);
 
 #endif /* _CRYPTO_FIPS140_MODULE_H */
diff --git a/include/linux/init.h b/include/linux/init.h
index 40331923b9f4a..eefbdfac1d41b 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -392,4 +392,14 @@ void __init parse_early_options(char *cmdline);
 #define __exit_p(x) NULL
 #endif
 
+#if defined(CONFIG_CRYPTO_FIPS140_EXTMOD) && !defined(__ASSEMBLY__)
+/* FIPS140 synchronization between kernel and module */
+void fips140_mark_kernel_level_complete(int level);
+bool fips140_is_kernel_level_complete(int level);
+bool fips140_is_module_level_complete(int level);
+void fips140_mark_module_level_complete(int level);
+void start_fips140_loader(void);
+void wait_until_fips140_level_sync(int level);
+#endif /* CONFIG_CRYPTO_FIPS140_EXTMOD && !__ASSEMBLY__ */
+
 #endif /* _LINUX_INIT_H */
diff --git a/init/main.c b/init/main.c
index 96f93bb06c490..8ef50419b42e4 100644
--- a/init/main.c
+++ b/init/main.c
@@ -1452,6 +1452,10 @@ static void __init do_initcall_level(int level, char *command_line)
 	do_trace_initcall_level(initcall_level_names[level]);
 	for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
 		do_one_initcall(initcall_from_entry(fn));
+
+#ifdef CONFIG_CRYPTO_FIPS140_EXTMOD
+	wait_until_fips140_level_sync(level);
+#endif
 }
 
 static void __init do_initcalls(void)
-- 
2.47.3