[PATCH 06/17] crypto: add pluggable interface for builtin crypto modules

[Jay Wang <wanjay@amazon.com>]

Traditionally, builtin cryptos interact with the main kernel through
exported functions and variables via EXPORT_SYMBOL. The exported
symbols are statically tied with the main kernel, so not suitable
after those builtin cryptos moved to a standalone kernel module
(e.g., main kernel will not even be compilable since users need the
exact function/variable address).

To address this, introduce a pluggable interface between builtin
crypto functions and variables by placing address placeholders.
During runtime once the crypto kernel module is loaded, the address
placeholder gets updated (by `do_crypto_api` and `do_crypto_var`) to
correct address.

In more details, there are two types of address holders: for function
addresses, "static call" mechanism is used; for variable addresses, a
var_type* pointer is used. To apply this pluggable interface, just
wrap the function/variable header declaration with the placeholder
declaration wrappers (implemented as `DECLARE_STATIC_CALL()`/
`DECLARE_CRYPTO_VAR()`), and place the placeholder definition wrappers
for each crypto function/variable in fips140-api.c (implemented as
`DEFINE_CRYPTO_API_STUB()`/`DEFINE_CRYPTO_VAR_STUB()`). Those wrappers
will be compiled differently for main kernel and the crypto kernel
module using different compilation flags. To make such pluggable
interface only affect cryptos chosen as builtin but not crypto chosen
built as modules already, associate wrappers with CONFIG_CRYPTO_{NAME}
parameter.

In addition to the pluggable interface, this patch also adds a way to
collect crypto init functions, such as `module_init()` into a
dedicated ELF section for later module entry to initialize each
crypto.

The idea of using "static call" as pluggable interface for function
address stems from Vegard Nossum <vegard.nossum@oracle.com>, while
we additionally make the following core differences: avoid duplicate
crypto code in main kernel, allow variable pluggable, make this
feature not affect cryptos already chosen to be module-built, and
make this adapt to any .config choices.

Signed-off-by: Jay Wang <wanjay@amazon.com>
---
 crypto/fips140/Makefile           |   5 +-
 crypto/fips140/fips140-api.c      |   7 ++
 include/asm-generic/vmlinux.lds.h |   1 +
 include/crypto/api.h              | 197 ++++++++++++++++++++++++++++++
 kernel/module/main.c              |  46 ++++++-
 5 files changed, 253 insertions(+), 3 deletions(-)
 create mode 100644 crypto/fips140/fips140-api.c
 create mode 100644 include/crypto/api.h

diff --git a/crypto/fips140/Makefile b/crypto/fips140/Makefile
index 3b4a74ccf41e..fb083022efbb 100644
--- a/crypto/fips140/Makefile
+++ b/crypto/fips140/Makefile
@@ -1,5 +1,8 @@
 
 crypto-objs-y += \
-	fips140-module.o 
+	fips140-module.o \
+	fips140-api.o
+
+obj-y += fips140-api.o
 
 clean-files:= .fips140.order .fips140.symvers
\ No newline at end of file
diff --git a/crypto/fips140/fips140-api.c b/crypto/fips140/fips140-api.c
new file mode 100644
index 000000000000..a11e898ff4bc
--- /dev/null
+++ b/crypto/fips140/fips140-api.c
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+/*
+ * Define static call keys for any functions which are part of the crypto
+ * API and used by the standalone FIPS module but which are not built into
+ * vmlinux.
+ */
\ No newline at end of file
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index eeb070f330bd..e25b44d29362 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -724,6 +724,7 @@
 	KERNEL_CTORS()							\
 	MCOUNT_REC()							\
 	*(.init.rodata .init.rodata.*)					\
+	BOUNDED_SECTION(__crypto_api_keys)				\
 	FTRACE_EVENTS()							\
 	TRACE_SYSCALLS()						\
 	KPROBE_BLACKLIST()						\
diff --git a/include/crypto/api.h b/include/crypto/api.h
new file mode 100644
index 000000000000..b58240ffd173
--- /dev/null
+++ b/include/crypto/api.h
@@ -0,0 +1,197 @@
+#ifndef _CRYPTO_API_H
+#define _CRYPTO_API_H
+
+#include <linux/static_call.h>
+
+#define CRYPTO_VAR_NAME(name) __crypto_##name##_ptr
+
+#define __CAT(a,b) a##b
+#define _CAT(a,b)  __CAT(a,b)
+
+#define __IF_1(...) __VA_ARGS__
+#define __IF_0(...)
+#define __IFNOT_1(...)
+#define __IFNOT_0(...) __VA_ARGS__
+
+/* Emit __VA_ARGS__ only if cfg is built into vmlinux (=y) */
+#define IF_BUILTIN(cfg, ...)     _CAT(__IF_,    IS_BUILTIN(cfg))(__VA_ARGS__)
+/* Emit __VA_ARGS__ only if cfg is NOT built in (i.e., =m or unset) */
+#define IF_NOT_BUILTIN(cfg, ...) _CAT(__IFNOT_, IS_BUILTIN(cfg))(__VA_ARGS__)
+
+#if !defined(CONFIG_CRYPTO_FIPS140_EXTMOD)
+
+/*
+ * These are the definitions that get used when no standalone FIPS module
+ * is used: we simply forward everything to normal functions and function
+ * calls.
+ */
+
+#define DECLARE_CRYPTO_API(cfg, name, ret_type, args_decl, args_call)       \
+	ret_type name args_decl;
+
+#define DECLARE_CRYPTO_VAR(cfg, name, var_type, ...) \
+	extern var_type name __VA_ARGS__;
+
+#define crypto_module_init(fn) module_init(fn)
+#define crypto_module_exit(fn) module_exit(fn)
+
+#define crypto_subsys_initcall(fn)	subsys_initcall(fn)
+#define crypto_late_initcall(fn)	late_initcall(fn)
+
+#define crypto_module_cpu_feature_match(x, __initfunc) \
+	module_cpu_feature_match(x, __initfunc)
+
+#else
+
+struct crypto_api_key {
+	struct static_call_key *key;
+	void *tramp;
+	void *func;
+};
+
+struct crypto_var_key {
+	void **ptr;
+	void *var;
+};
+
+#ifndef FIPS_MODULE
+
+/*
+ * These are the definitions that get used for vmlinux and in-tree
+ * kernel modules.
+ *
+ * In this case, all references to the kernel crypto API functions will
+ * be replaced by wrappers that perform a call using the kernel's static_call
+ * functionality.
+ */
+
+/* Consolidated version of different DECLARE_CRYPTO_API versions */
+
+/*
+ *  - If cfg is built-in (=y): declare nonfips_<name>, a static_call key,
+ *    and an inline wrapper <name>() that dispatches via static_call.
+ *  - Else (cfg =m or unset): only declare <name>() prototype.
+ */
+#define DECLARE_CRYPTO_API(cfg, name, ret_type, args_decl, args_call)       \
+	IF_BUILTIN(cfg,                                                      \
+		ret_type nonfips_##name args_decl;                           \
+		DECLARE_STATIC_CALL(crypto_##name##_key, nonfips_##name);    \
+		static inline ret_type name args_decl                        \
+		{                                                            \
+			return static_call(crypto_##name##_key) args_call;   \
+		}                                                            \
+	)                                                                   \
+	IF_NOT_BUILTIN(cfg,                                                 \
+		ret_type name args_decl;                                    \
+	)
+
+#define DECLARE_CRYPTO_VAR(cfg, name, var_type, ...) \
+	IF_BUILTIN(cfg, \
+		extern void *CRYPTO_VAR_NAME(name); \
+	) \
+	IF_NOT_BUILTIN(cfg, \
+		extern var_type name __VA_ARGS__; \
+	)
+
+#define DEFINE_CRYPTO_API_STUB(name) \
+	DEFINE_STATIC_CALL_NULL(crypto_##name##_key, name); \
+	EXPORT_STATIC_CALL(crypto_##name##_key)
+
+#define DEFINE_CRYPTO_VAR_STUB(name) \
+	void* CRYPTO_VAR_NAME(name) = NULL;\
+	EXPORT_SYMBOL(CRYPTO_VAR_NAME(name));
+
+#define crypto_module_init(fn) module_init(fn)
+#define crypto_module_exit(fn) module_exit(fn)
+
+#define crypto_subsys_initcall(fn)	subsys_initcall(fn)
+#define crypto_late_initcall(fn)	late_initcall(fn)
+
+#define crypto_module_cpu_feature_match(x, __initfunc) \
+	module_cpu_feature_match(x, __initfunc)
+	
+#else /* defined(FIPS_MODULE) */
+
+/* Consolidated version of different DECLARE_CRYPTO_API versions,
+   within FIPS module, API remains the same, only declare static 
+   call key */
+#define DECLARE_CRYPTO_API(cfg, name, ret_type, args_decl, args_call) \
+	IF_BUILTIN(cfg,                                                   \
+		ret_type name args_decl;                                      \
+		DECLARE_STATIC_CALL(crypto_##name##_key, name)                \
+	)                                                                 \
+	IF_NOT_BUILTIN(cfg,                                               \
+		ret_type name args_decl;                                      \
+	)
+
+#define DECLARE_CRYPTO_VAR(cfg, name, var_type, ...)               \
+	IF_BUILTIN(cfg,                                             \
+		extern var_type name __VA_ARGS__;                   \
+		extern void *CRYPTO_VAR_NAME(name);                  \
+	)                                                            \
+	IF_NOT_BUILTIN(cfg,                                          \
+		extern var_type name __VA_ARGS__;                   \
+	)
+
+#define DEFINE_CRYPTO_VAR_STUB(name) \
+	static struct crypto_var_key __crypto_##name##_var_key \
+		__used \
+		__section("__crypto_var_keys") \
+		__aligned(__alignof__(struct crypto_var_key)) = \
+	{ \
+		.ptr = &CRYPTO_VAR_NAME(name), \
+		.var = (void*)&name, \
+	};
+
+/*
+ * These are the definitions that get used for the main kernel.
+ *
+ * In this case, initialize crypto static call key with original name
+ */
+
+#define DEFINE_CRYPTO_API_STUB(name) \
+	static struct crypto_api_key __##name##_key \
+		__used \
+		__section("__crypto_api_keys") \
+		__aligned(__alignof__(struct crypto_api_key)) = \
+	{ \
+		.key = &STATIC_CALL_KEY(crypto_##name##_key), \
+		.tramp = STATIC_CALL_TRAMP_ADDR(crypto_##name##_key), \
+		.func = &name, \
+	};
+
+#define crypto_module_init(fn) \
+	static initcall_t __used __section(".fips_initcall1") \
+		__fips_##fn = fn;
+#define crypto_module_exit(fn) \
+		static unsigned long __used __section(".fips_exitcall") \
+		__fips_##fn = (unsigned long) &fn;
+#define crypto_subsys_initcall(fn) \
+	static initcall_t __used __section(".fips_initcall0") \
+		__fips_##fn = fn;
+#define crypto_subsys_exitcall(fn) \
+		static unsigned long __used __section(".fips_exitcall") \
+		__fips_##fn = (unsigned long) &fn;
+#define crypto_late_initcall(fn) \
+	static initcall_t __used __section(".fips_initcall2") \
+		__fips_##fn = fn;
+#define crypto_late_exitcall(fn) \
+		static unsigned long __used __section(".fips_exitcall") \
+		__fips_##fn = (unsigned long) &fn;
+
+#define crypto_module_cpu_feature_match(x, __initfunc) \
+static struct cpu_feature const __maybe_unused cpu_feature_match_ ## x[] = \
+	{ { .feature = cpu_feature(x) }, { } }; \
+MODULE_DEVICE_TABLE(cpu, cpu_feature_match_ ## x); \
+static int __init cpu_feature_match_ ## x ## _init(void) \
+{ \
+	if (!cpu_have_feature(cpu_feature(x))) \
+		return -ENODEV; \
+	return __initfunc(); \
+} \
+crypto_module_init(cpu_feature_match_ ## x ## _init)
+
+#endif /* defined(FIPS_MODULE) */
+#endif /* defined(CONFIG_CRYPTO_FIPS140_EXTMOD) */
+
+#endif /* !_CRYPTO_API_H */
diff --git a/kernel/module/main.c b/kernel/module/main.c
index 2914e7619766..dad84f0548ac 100644
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -7,6 +7,7 @@
 
 #define INCLUDE_VERMAGIC
 
+#include <crypto/api.h>
 #include <linux/export.h>
 #include <linux/extable.h>
 #include <linux/moduleloader.h>
@@ -2956,6 +2957,39 @@ static int post_relocation(struct module *mod, const struct load_info *info)
 	return module_finalize(info->hdr, info->sechdrs, mod);
 }
 
+#ifdef CONFIG_CRYPTO_FIPS140_EXTMOD
+static void do_crypto_api(struct load_info *info)
+{
+	struct crypto_api_key *crypto_api_keys;
+	unsigned int num_crypto_api_keys;
+
+	unsigned int i;
+
+	crypto_api_keys = section_objs(info, "__crypto_api_keys",
+		sizeof(*crypto_api_keys), &num_crypto_api_keys);
+
+	for (i = 0; i < num_crypto_api_keys; ++i) {
+		struct crypto_api_key *key = &crypto_api_keys[i];
+		__static_call_update(key->key, key->tramp, key->func);
+	}
+}
+
+static void do_crypto_var(struct load_info *info)
+{
+	struct crypto_var_key *crypto_var_keys;
+	unsigned int num_crypto_var_keys;
+	unsigned int i;
+
+	crypto_var_keys = section_objs(info, "__crypto_var_keys",
+		sizeof(*crypto_var_keys), &num_crypto_var_keys);
+
+	for (i = 0; i < num_crypto_var_keys; ++i) {
+		struct crypto_var_key *var_key = &crypto_var_keys[i];
+		*(var_key->ptr) = var_key->var;
+	}
+}
+#endif
+
 /* Call module constructors. */
 static void do_mod_ctors(struct module *mod)
 {
@@ -3010,7 +3044,7 @@ module_param(async_probe, bool, 0644);
  * Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
  * helper command 'lx-symbols'.
  */
-static noinline int do_init_module(struct module *mod, int flags)
+static noinline int do_init_module(struct load_info *info, struct module *mod, int flags)
 {
 	int ret = 0;
 	struct mod_initfree *freeinit;
@@ -3036,6 +3070,14 @@ static noinline int do_init_module(struct module *mod, int flags)
 	freeinit->init_data = mod->mem[MOD_INIT_DATA].base;
 	freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base;
 
+#ifdef CONFIG_CRYPTO_FIPS140_EXTMOD
+	if (flags & MODULE_INIT_CRYPTO_FROM_MEM)
+		{
+			do_crypto_api(info);
+			do_crypto_var(info);
+		}
+#endif
+
 	do_mod_ctors(mod);
 	/* Start the module */
 	if (mod->init != NULL)
@@ -3499,7 +3541,7 @@ static int _load_module(struct load_info *info, const char __user *uargs,
 	/* Done! */
 	trace_module_load(mod);
 
-	return do_init_module(mod, flags);
+	return do_init_module(info, mod, flags);
 
  sysfs_cleanup:
 	mod_sysfs_teardown(mod);
-- 
2.47.3