[LTP] [PATCH] build: Parallelize subdirectory recursion in generic_trunk_target

[Andrea Cervesato <andrea.cervesato@suse.de>]

From: Andrea Cervesato <andrea.cervesato@mailbox.org>

Currently, generic_trunk_target traverses $(SUBDIRS) using a serial shell
'for' loop. This serializes directory traversal and prevents 'make -jN'
from building sibling directories in parallel.

Replace the shell loop with target-based recursion:
1. Map each subdirectory to a phony target: <dir>-all, <dir>-install,
   and <dir>-check.
2. Make 'all', 'install', and 'check' depend on these targets so GNU Make
   can dispatch them concurrently across available job slots.
3. In each recipe, strip the suffix with $(patsubst %-<action>,%,$@) to
   extract the directory name and invoke $(MAKE) -C <dir>.
4. Add explicit ordering dependencies where subdirectories produce
   libraries required by sibling directories (tools before testcases,
   kernel/lib before syscalls and mem, rpc-tirpc before basic_tests).

This cuts full build time by over 20% on multi-core systems.

Signed-off-by: Andrea Cervesato <andrea.cervesato@suse.com>
---
The following results show how this optimization moves compile times
really far from what we have now. With a single threaded build, LTP
would take approximately 12 minutes to compile. With a multi-threaded
build on 16 CPUs, this drops down to ~46 seconds.

time make -j16

Executed in   45.96 secs    fish           external
   usr time  519.11 secs    0.24 millis  519.11 secs
   sys time  167.13 secs    1.08 millis  167.13 secs
---
 Makefile                            |  2 ++
 include/mk/generic_trunk_target.inc | 35 ++++++++++++++++++++++++++++-------
 testcases/kernel/Makefile           |  3 +++
 testcases/network/rpc/Makefile      |  3 +++
 4 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index 2efbe8c6c..0abb5c6b4 100644
--- a/Makefile
+++ b/Makefile
@@ -63,6 +63,8 @@ CHECK_TARGETS		:= $(addsuffix -check,testcases lib)
 all: $(addsuffix -all,$(COMMON_TARGETS)) Version
 
 $(MAKE_TARGETS): lib-all libs-all
+# tools contains apicmds/tst_kvercmp required by some testcases
+testcases-all: tools-all
 
 .PHONY: include-all include-install
 include-install: $(top_builddir)/include/config.h include/mk/config.mk include-all
diff --git a/include/mk/generic_trunk_target.inc b/include/mk/generic_trunk_target.inc
index f9db7896a..6975507e8 100644
--- a/include/mk/generic_trunk_target.inc
+++ b/include/mk/generic_trunk_target.inc
@@ -83,15 +83,36 @@ ifeq ($(strip $(SUBDIRS)),)
 	$(warning CURDIR is: $(CURDIR))
 	$(error SUBDIRS empty -- did you want generic_leaf_target instead?)
 else
-$(RECURSIVE_TARGETS): %: | $(SUBDIRS)
+# Recurse into subdirectories in parallel instead of using a serial shell loop.
+# Each subdirectory is mapped to a phony target (<dir>-all, <dir>-install, etc.).
+# GNU Make can then schedule sibling directories concurrently under -jN.
+all:     $(addsuffix -all,$(SUBDIRS))
+install: $(addsuffix -install,$(SUBDIRS))
+check:   $(addsuffix -check,$(SUBDIRS))
+
+.PHONY: $(addsuffix -all,$(SUBDIRS)) $(addsuffix -install,$(SUBDIRS)) $(addsuffix -check,$(SUBDIRS))
+
+# For each target (e.g. 'open-all'), strip the suffix to get the directory name,
+# build trunk-local targets first, and ensure the subdirectory exists.
+$(addsuffix -all,$(SUBDIRS)): trunk-all | $(SUBDIRS) $(MAKE_DEPS)
 ifdef VERBOSE
-	@set -e; for dir in $(SUBDIRS); do \
-	    $(MAKE) -C $$dir -f "$(abs_srcdir)/$$dir/Makefile" $@; \
-	done
+	$(MAKE) -C "$(patsubst %-all,%,$@)" -f "$(abs_srcdir)/$(patsubst %-all,%,$@)/Makefile" all
 else
-	@set -e; for dir in $(SUBDIRS); do \
-	    $(MAKE) --no-print-directory -C $$dir -f "$(abs_srcdir)/$$dir/Makefile" $@; \
-	done
+	@$(MAKE) --no-print-directory -C "$(patsubst %-all,%,$@)" -f "$(abs_srcdir)/$(patsubst %-all,%,$@)/Makefile" all
+endif
+
+$(addsuffix -install,$(SUBDIRS)): trunk-install | $(SUBDIRS) $(MAKE_DEPS)
+ifdef VERBOSE
+	$(MAKE) -C "$(patsubst %-install,%,$@)" -f "$(abs_srcdir)/$(patsubst %-install,%,$@)/Makefile" install
+else
+	@$(MAKE) --no-print-directory -C "$(patsubst %-install,%,$@)" -f "$(abs_srcdir)/$(patsubst %-install,%,$@)/Makefile" install
+endif
+
+$(addsuffix -check,$(SUBDIRS)): trunk-check | $(SUBDIRS) $(MAKE_DEPS)
+ifdef VERBOSE
+	$(MAKE) -C "$(patsubst %-check,%,$@)" -f "$(abs_srcdir)/$(patsubst %-check,%,$@)/Makefile" check
+else
+	@$(MAKE) --no-print-directory -C "$(patsubst %-check,%,$@)" -f "$(abs_srcdir)/$(patsubst %-check,%,$@)/Makefile" check
 endif
 endif
 
diff --git a/testcases/kernel/Makefile b/testcases/kernel/Makefile
index 29e26742d..3bf5e433e 100644
--- a/testcases/kernel/Makefile
+++ b/testcases/kernel/Makefile
@@ -54,3 +54,6 @@ FILTER_OUT_DIRS		+= containers controllers device-drivers fs io ipc mem \
 endif
 
 include $(top_srcdir)/include/mk/generic_trunk_target.mk
+
+# libkerntest.a in lib/ must be built before syscalls and mem consumers
+syscalls-all mem-all: lib-all
diff --git a/testcases/network/rpc/Makefile b/testcases/network/rpc/Makefile
index 6ecb1a665..8d12bdf1e 100644
--- a/testcases/network/rpc/Makefile
+++ b/testcases/network/rpc/Makefile
@@ -9,3 +9,6 @@ include $(top_srcdir)/include/mk/env_pre.mk
 INSTALL_TARGETS		:= *.sh
 
 include $(top_srcdir)/include/mk/generic_trunk_target.mk
+
+# librpc-tirpc.a in rpc-tirpc/ must be built before basic_tests
+basic_tests-all: rpc-tirpc-all

---
base-commit: 2ae4cfe69b9010018f2e5df6fa4bbba21810d4f2
change-id: 20261020-build_system_optimize-6cd0343f08e1

Best regards,
--  
Andrea Cervesato <andrea.cervesato@suse.com>


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp