[PATCH 1/3] scripts: kconfig: merge_config.sh: refactor from shell/sed/grep to awk

[PATCH 1/3] scripts: kconfig: merge_config.sh: refactor from shell/sed/grep to awk

[Mikko Rapeli]

From: Anders Roxell <anders.roxell@linaro.org>

merge_config.sh shell/sed/grep loop scales poorly and is slow.
With Yocto genericarm64 kernel and around 190 config fragments
the script takes more than 20 minutes to run on a fast build machine.
Re-implementation with awk does the same job in 10 seconds.
Using awk since it is likely available in the build environments
and using perl, python etc would introduce more complex runtime
dependencies. awk is good enough and lot better than shell/sed/grep.

Signed-off-by: Anders Roxell <anders.roxell@linaro.org>
Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
---
 scripts/kconfig/merge_config.sh | 161 ++++++++++++++++++++++++--------
 1 file changed, 123 insertions(+), 38 deletions(-)

diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
index 79c09b378be8..46397d7c6957 100755
--- a/scripts/kconfig/merge_config.sh
+++ b/scripts/kconfig/merge_config.sh
@@ -16,8 +16,8 @@
 set -e
 
 clean_up() {
-	rm -f $TMP_FILE
-	rm -f $MERGE_FILE
+	rm -f "${TMP_FILE}"
+	rm -f "${TMP_FILE}.new"
 }
 
 usage() {
@@ -121,7 +121,6 @@ SED_CONFIG_EXP1="s/^\(${CONFIG_PREFIX}[a-zA-Z0-9_]*\)=.*/\1/p"
 SED_CONFIG_EXP2="s/^# \(${CONFIG_PREFIX}[a-zA-Z0-9_]*\) is not set$/\1/p"
 
 TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX)
-MERGE_FILE=$(mktemp ./.merge_tmp.config.XXXXXXXXXX)
 
 echo "Using $INITFILE as base"
 
@@ -136,42 +135,128 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
 		echo "The merge file '$ORIG_MERGE_FILE' does not exist.  Exit." >&2
 		exit 1
 	fi
-	cat $ORIG_MERGE_FILE > $MERGE_FILE
-	CFG_LIST=$(sed -n -e "$SED_CONFIG_EXP1" -e "$SED_CONFIG_EXP2" $MERGE_FILE)
-
-	for CFG in $CFG_LIST ; do
-		grep -q -w $CFG $TMP_FILE || continue
-		PREV_VAL=$(grep -w $CFG $TMP_FILE)
-		NEW_VAL=$(grep -w $CFG $MERGE_FILE)
-		BUILTIN_FLAG=false
-		if [ "$BUILTIN" = "true" ] && [ "${NEW_VAL#CONFIG_*=}" = "m" ] && [ "${PREV_VAL#CONFIG_*=}" = "y" ]; then
-			${WARNOVERRIDE} Previous  value: $PREV_VAL
-			${WARNOVERRIDE} New value:       $NEW_VAL
-			${WARNOVERRIDE} -y passed, will not demote y to m
-			${WARNOVERRIDE}
-			BUILTIN_FLAG=true
-		elif [ "x$PREV_VAL" != "x$NEW_VAL" ] ; then
-			${WARNOVERRIDE} Value of $CFG is redefined by fragment $ORIG_MERGE_FILE:
-			${WARNOVERRIDE} Previous  value: $PREV_VAL
-			${WARNOVERRIDE} New value:       $NEW_VAL
-			${WARNOVERRIDE}
-			if [ "$STRICT" = "true" ]; then
-				STRICT_MODE_VIOLATED=true
-			fi
-		elif [ "$WARNREDUN" = "true" ]; then
-			${WARNOVERRIDE} Value of $CFG is redundant by fragment $ORIG_MERGE_FILE:
-		fi
-		if [ "$BUILTIN_FLAG" = "false" ]; then
-			sed -i "/$CFG[ =]/d" $TMP_FILE
-		else
-			sed -i "/$CFG[ =]/d" $MERGE_FILE
-		fi
-	done
-	# In case the previous file lacks a new line at the end
-	echo >> $TMP_FILE
-	cat $MERGE_FILE >> $TMP_FILE
-done
+	# Use awk for single-pass processing instead of per-symbol grep/sed
+	if ! awk -v prefix="$CONFIG_PREFIX" \
+		-v warnoverride="$WARNOVERRIDE" \
+		-v strict="$STRICT" \
+		-v builtin="$BUILTIN" \
+		-v warnredun="$WARNREDUN" '
+	BEGIN {
+		strict_violated = 0
+		cfg_regex = "^" prefix "[a-zA-Z0-9_]+"
+		notset_regex = "^# " prefix "[a-zA-Z0-9_]+ is not set$"
+	}
+
+	# Extract config name from a line, returns "" if not a config line
+	function get_cfg(line) {
+		if (match(line, cfg_regex)) {
+			return substr(line, RSTART, RLENGTH)
+		} else if (match(line, notset_regex)) {
+			# Extract CONFIG_FOO from "# CONFIG_FOO is not set"
+			sub(/^# /, "", line)
+			sub(/ is not set$/, "", line)
+			return line
+		}
+		return ""
+	}
+
+	# Normalize: strip trailing comments, convert "is not set" to "=n"
+	function normalize(line) {
+		if (line == "") return ""
+		sub(/[[:space:]]+#.*/, "", line)
+		if (line ~ / is not set$/) {
+			sub(/^# /, "", line)
+			sub(/ is not set$/, "=n", line)
+		}
+		return line
+	}
+
+	function warn_builtin(cfg, prev, new) {
+		if (warnoverride == "true") return
+		print cfg ": -y passed, will not demote y to m" > "/dev/stderr"
+		print "  Previous value: " prev > "/dev/stderr"
+		print "  New value:	 " new > "/dev/stderr"
+		print "" > "/dev/stderr"
+	}
+
+	function warn_redefined(cfg, prev, new) {
+		if (warnoverride == "true") return
+		print "Value of " cfg " is redefined by fragment " mergefile ":" > "/dev/stderr"
+		print "  Previous value: " prev > "/dev/stderr"
+		print "  New value:	 " new > "/dev/stderr"
+		print "" > "/dev/stderr"
+	}
+
+	function warn_redundant(cfg) {
+		if (warnredun != "true" || warnoverride == "true") return
+		print "Value of " cfg " is redundant by fragment " mergefile ":" > "/dev/stderr"
+	}
+
+	# First pass: read merge file, store all lines and index
+	FILENAME == ARGV[1] {
+	        mergefile = FILENAME
+		merge_lines[FNR] = $0
+		merge_total = FNR
+		cfg = get_cfg($0)
+		if (cfg != "") {
+			merge_cfg[cfg] = $0
+			merge_cfg_line[cfg] = FNR
+		}
+		next
+	}
 
+	# Second pass: process base file (TMP_FILE)
+		cfg = get_cfg($0)
+
+		# Not a config or not in merge file - keep it
+		if (cfg == "" || !(cfg in merge_cfg)) {
+			print $0
+			next
+		}
+
+		prev_val = normalize($0)
+		new_val = normalize(merge_cfg[cfg])
+
+		# BUILTIN: do not demote y to m
+		if (builtin == "true" && new_val ~ /=m$/ && prev_val ~ /=y$/) {
+			warn_builtin(cfg, prev_val, new_val)
+			print $0
+			skip_merge[merge_cfg_line[cfg]] = 1
+			next
+		}
+
+		# Values equal - redundant
+		if (prev_val == new_val) {
+			warn_redundant(cfg)
+			next
+		}
+
+		# Values differ - redefined
+		warn_redefined(cfg, prev_val, new_val)
+		if (strict == "true") {
+			strict_violated = 1
+		}
+	}
+
+	END {
+		# Newline in case base file lacks trailing newline
+		print ""
+		# Append merge file, skipping lines marked for builtin preservation
+		for (i = 1; i <= merge_total; i++) {
+			if (!(i in skip_merge)) {
+				print merge_lines[i]
+			}
+		}
+		if (strict_violated) {
+			exit 1
+		}
+	}' \
+	"$ORIG_MERGE_FILE" "$TMP_FILE" > "${TMP_FILE}.new"; then
+		# awk exited non-zero, strict mode was violated
+		STRICT_MODE_VIOLATED=true
+	fi
+	mv "${TMP_FILE}.new" "$TMP_FILE"
+done
 if [ "$STRICT_MODE_VIOLATED" = "true" ]; then
 	echo "The fragment redefined a value and strict mode had been passed."
 	exit 1
-- 
2.34.1

[PATCH 2/3] scripts: kconfig: merge_config.sh: use awk in checks too

[Mikko Rapeli]

Converting from shell/sed/grep loop to awk improves runtime
checks of Yocto genericarm64 kernel config from 20 seconds
to under 1 second. Also fix the tests to take into account:

 * kernel config lines with comments
   CONFIG_FOO=m # this is a comment
 * match "# CONFIG_FOO is not set" with "CONFIG_FOO=n"
 * match unset config value to "CONFIG_FOO=n"

Cc: Anders Roxell <anders.roxell@linaro.org>
Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
---
 scripts/kconfig/merge_config.sh | 99 +++++++++++++++++++++++++++++----
 1 file changed, 88 insertions(+), 11 deletions(-)

diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
index 46397d7c6957..bc6b5d0a69bc 100755
--- a/scripts/kconfig/merge_config.sh
+++ b/scripts/kconfig/merge_config.sh
@@ -283,16 +283,93 @@ fi
 # allnoconfig: Fills in any missing symbols with # CONFIG_* is not set
 make KCONFIG_ALLCONFIG=$TMP_FILE $OUTPUT_ARG $ALLTARGET
 
+# Check all specified config values took effect (might have missed-dependency issues)
+if ! awk -v prefix="$CONFIG_PREFIX" \
+	-v warnoverride="$WARNOVERRIDE" \
+	-v strict="$STRICT" \
+	-v warnredun="$WARNREDUN" '
+BEGIN {
+	strict_violated = 0
+	cfg_regex = "^" prefix "[a-zA-Z0-9_]+"
+	notset_regex = "^# " prefix "[a-zA-Z0-9_]+ is not set$"
+}
 
-# Check all specified config values took (might have missed-dependency issues)
-for CFG in $(sed -n -e "$SED_CONFIG_EXP1" -e "$SED_CONFIG_EXP2" $TMP_FILE); do
+# Extract config name from a line, returns "" if not a config line
+function get_cfg(line) {
+	if (match(line, cfg_regex)) {
+		return substr(line, RSTART, RLENGTH)
+	} else if (match(line, notset_regex)) {
+		# Extract CONFIG_FOO from "# CONFIG_FOO is not set"
+		sub(/^# /, "", line)
+		sub(/ is not set$/, "", line)
+		return line
+	}
+	return ""
+}
 
-	REQUESTED_VAL=$(grep -w -e "$CFG" $TMP_FILE)
-	ACTUAL_VAL=$(grep -w -e "$CFG" "$KCONFIG_CONFIG" || true)
-	if [ "x$REQUESTED_VAL" != "x$ACTUAL_VAL" ] ; then
-		echo "Value requested for $CFG not in final .config"
-		echo "Requested value:  $REQUESTED_VAL"
-		echo "Actual value:     $ACTUAL_VAL"
-		echo ""
-	fi
-done
+# Normalize: strip trailing comments, convert "is not set" to "=n"
+function normalize(line) {
+	if (line == "") return ""
+	sub(/[[:space:]]+#.*/, "", line)
+	if (line ~ / is not set$/) {
+		sub(/^# /, "", line)
+		sub(/ is not set$/, "=n", line)
+	}
+	return line
+}
+
+function warn_mismatch(cfg, merged, final) {
+	if (warnredun == "true") return
+	if (final == "" && !(merged ~ /=n$/)) {
+		print "WARNING: " cfg " not in final config but set in merged config to: " merged > "/dev/stderr"
+	} else if (final == "" && merged ~ /=n$/) {
+		# not set is the same as =n, pass
+	} else if (merged == "" && final != "") {
+		print "WARNING: " cfg " not in merged config but added in final config: " final > "/dev/stderr"
+	} else {
+		print "WARNING: " cfg " differs:" > "/dev/stderr"
+		print "Merged value: " merged > "/dev/stderr"
+		print "Final  value: " final > "/dev/stderr"
+	}
+}
+
+# First pass: read effective config file, store all lines
+FILENAME == ARGV[1] {
+	cfg = get_cfg($0)
+	if (cfg != "") {
+		config_cfg[cfg] = $0
+	}
+	next
+}
+
+# Second pass: process merged config and compare against effective config
+{
+	cfg = get_cfg($0)
+	if (cfg == "") next
+
+	merged_val = normalize($0)
+	final_val = normalize(config_cfg[cfg])
+
+	if (merged_val == final_val) next
+
+	warn_mismatch(cfg, merged_val, final_val)
+
+	if (strict == "true") {
+		strict_violated = 1
+	}
+}
+
+END {
+	if (strict_violated) {
+		exit 1
+	}
+}' \
+"$KCONFIG_CONFIG" "$TMP_FILE"; then
+	# awk exited non-zero, strict mode was violated
+	STRICT_MODE_VIOLATED=true
+fi
+
+if [ "$STRICT" == "true" ] && [ "$STRICT_MODE_VIOLATED" == "true" ]; then
+	echo "Requested and effective config differ"
+	exit 1
+fi
-- 
2.34.1

[PATCH 3/3] scripts: kconfig: merge_config.sh: warn on duplicate input files

[Mikko Rapeli]

External scripts like yocto kernel scc may provide
same input config fragment multiple times. This may
be a bug since processing same fragments multiple times
can be time consuming.

Cc: Anders Roxell <anders.roxell@linaro.org>
Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
---
 scripts/kconfig/merge_config.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
index bc6b5d0a69bc..0996bf118325 100755
--- a/scripts/kconfig/merge_config.sh
+++ b/scripts/kconfig/merge_config.sh
@@ -128,6 +128,8 @@ trap clean_up EXIT
 
 cat $INITFILE > $TMP_FILE
 
+PROCESSED_FILES=""
+
 # Merge files, printing warnings on overridden values
 for ORIG_MERGE_FILE in $MERGE_LIST ; do
 	echo "Merging $ORIG_MERGE_FILE"
@@ -135,6 +137,14 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
 		echo "The merge file '$ORIG_MERGE_FILE' does not exist.  Exit." >&2
 		exit 1
 	fi
+
+	# Check for duplicate input files
+	case " $PROCESSED_FILES " in
+		*" $ORIG_MERGE_FILE "*)
+			${WARNOVERRIDE} "WARNING: Input file provided multiple times: $ORIG_MERGE_FILE"
+			;;
+	esac
+
 	# Use awk for single-pass processing instead of per-symbol grep/sed
 	if ! awk -v prefix="$CONFIG_PREFIX" \
 		-v warnoverride="$WARNOVERRIDE" \
@@ -256,6 +266,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
 		STRICT_MODE_VIOLATED=true
 	fi
 	mv "${TMP_FILE}.new" "$TMP_FILE"
+	PROCESSED_FILES="$PROCESSED_FILES $ORIG_MERGE_FILE"
 done
 if [ "$STRICT_MODE_VIOLATED" = "true" ]; then
 	echo "The fragment redefined a value and strict mode had been passed."
-- 
2.34.1

Re: [PATCH 1/3] scripts: kconfig: merge_config.sh: refactor from shell/sed/grep to awk

[Nathan Chancellor]

Hi Mikko,

I don't really know awk well so I won't give as much review on that
part.

On Mon, Dec 29, 2025 at 01:44:45PM +0200, Mikko Rapeli wrote:
> From: Anders Roxell <anders.roxell@linaro.org>
> 
> merge_config.sh shell/sed/grep loop scales poorly and is slow.
> With Yocto genericarm64 kernel and around 190 config fragments
> the script takes more than 20 minutes to run on a fast build machine.
> Re-implementation with awk does the same job in 10 seconds.
> Using awk since it is likely available in the build environments
> and using perl, python etc would introduce more complex runtime
> dependencies. awk is good enough and lot better than shell/sed/grep.
> 
> Signed-off-by: Anders Roxell <anders.roxell@linaro.org>
> Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
> ---
>  scripts/kconfig/merge_config.sh | 161 ++++++++++++++++++++++++--------
>  1 file changed, 123 insertions(+), 38 deletions(-)
> 
> diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
> index 79c09b378be8..46397d7c6957 100755
> --- a/scripts/kconfig/merge_config.sh
> +++ b/scripts/kconfig/merge_config.sh
> @@ -16,8 +16,8 @@
>  set -e
>  
>  clean_up() {
> -	rm -f $TMP_FILE
> -	rm -f $MERGE_FILE
> +	rm -f "${TMP_FILE}"
> +	rm -f "${TMP_FILE}.new"
>  }
>  
>  usage() {
> @@ -121,7 +121,6 @@ SED_CONFIG_EXP1="s/^\(${CONFIG_PREFIX}[a-zA-Z0-9_]*\)=.*/\1/p"
>  SED_CONFIG_EXP2="s/^# \(${CONFIG_PREFIX}[a-zA-Z0-9_]*\) is not set$/\1/p"

The SED_CONFIG_EXP variables are unused now so they can be removed.

>  TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX)
> -MERGE_FILE=$(mktemp ./.merge_tmp.config.XXXXXXXXXX)
>  
>  echo "Using $INITFILE as base"
>  
> @@ -136,42 +135,128 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
>  		echo "The merge file '$ORIG_MERGE_FILE' does not exist.  Exit." >&2
>  		exit 1
>  	fi
> -	cat $ORIG_MERGE_FILE > $MERGE_FILE
> -	CFG_LIST=$(sed -n -e "$SED_CONFIG_EXP1" -e "$SED_CONFIG_EXP2" $MERGE_FILE)
> -
> -	for CFG in $CFG_LIST ; do
> -		grep -q -w $CFG $TMP_FILE || continue
> -		PREV_VAL=$(grep -w $CFG $TMP_FILE)
> -		NEW_VAL=$(grep -w $CFG $MERGE_FILE)
> -		BUILTIN_FLAG=false
> -		if [ "$BUILTIN" = "true" ] && [ "${NEW_VAL#CONFIG_*=}" = "m" ] && [ "${PREV_VAL#CONFIG_*=}" = "y" ]; then
> -			${WARNOVERRIDE} Previous  value: $PREV_VAL
> -			${WARNOVERRIDE} New value:       $NEW_VAL
> -			${WARNOVERRIDE} -y passed, will not demote y to m
> -			${WARNOVERRIDE}
> -			BUILTIN_FLAG=true
> -		elif [ "x$PREV_VAL" != "x$NEW_VAL" ] ; then
> -			${WARNOVERRIDE} Value of $CFG is redefined by fragment $ORIG_MERGE_FILE:
> -			${WARNOVERRIDE} Previous  value: $PREV_VAL
> -			${WARNOVERRIDE} New value:       $NEW_VAL
> -			${WARNOVERRIDE}
> -			if [ "$STRICT" = "true" ]; then
> -				STRICT_MODE_VIOLATED=true
> -			fi
> -		elif [ "$WARNREDUN" = "true" ]; then
> -			${WARNOVERRIDE} Value of $CFG is redundant by fragment $ORIG_MERGE_FILE:
> -		fi
> -		if [ "$BUILTIN_FLAG" = "false" ]; then
> -			sed -i "/$CFG[ =]/d" $TMP_FILE
> -		else
> -			sed -i "/$CFG[ =]/d" $MERGE_FILE
> -		fi
> -	done
> -	# In case the previous file lacks a new line at the end
> -	echo >> $TMP_FILE
> -	cat $MERGE_FILE >> $TMP_FILE
> -done
> +	# Use awk for single-pass processing instead of per-symbol grep/sed
> +	if ! awk -v prefix="$CONFIG_PREFIX" \

Please use the AWK variable from Kbuild here:

  if ${AWK} -v prefix="$CONFIG_PREFIX" \

You can do something like:

  if [ -z "${AWK}" ]; then
      AWK=awk
  fi

towards the top of the script to have the script continue to work
outside of Kbuild as well.

> +		-v warnoverride="$WARNOVERRIDE" \
> +		-v strict="$STRICT" \
> +		-v builtin="$BUILTIN" \
> +		-v warnredun="$WARNREDUN" '
> +	BEGIN {
> +		strict_violated = 0
> +		cfg_regex = "^" prefix "[a-zA-Z0-9_]+"
> +		notset_regex = "^# " prefix "[a-zA-Z0-9_]+ is not set$"
> +	}
> +
> +	# Extract config name from a line, returns "" if not a config line
> +	function get_cfg(line) {
> +		if (match(line, cfg_regex)) {
> +			return substr(line, RSTART, RLENGTH)
> +		} else if (match(line, notset_regex)) {
> +			# Extract CONFIG_FOO from "# CONFIG_FOO is not set"
> +			sub(/^# /, "", line)
> +			sub(/ is not set$/, "", line)
> +			return line
> +		}
> +		return ""
> +	}
> +
> +	# Normalize: strip trailing comments, convert "is not set" to "=n"
> +	function normalize(line) {
> +		if (line == "") return ""
> +		sub(/[[:space:]]+#.*/, "", line)
> +		if (line ~ / is not set$/) {
> +			sub(/^# /, "", line)
> +			sub(/ is not set$/, "=n", line)
> +		}
> +		return line
> +	}
> +
> +	function warn_builtin(cfg, prev, new) {
> +		if (warnoverride == "true") return
> +		print cfg ": -y passed, will not demote y to m" > "/dev/stderr"
> +		print "  Previous value: " prev > "/dev/stderr"
> +		print "  New value:	 " new > "/dev/stderr"
> +		print "" > "/dev/stderr"
> +	}
> +
> +	function warn_redefined(cfg, prev, new) {
> +		if (warnoverride == "true") return
> +		print "Value of " cfg " is redefined by fragment " mergefile ":" > "/dev/stderr"
> +		print "  Previous value: " prev > "/dev/stderr"
> +		print "  New value:	 " new > "/dev/stderr"
> +		print "" > "/dev/stderr"
> +	}
> +
> +	function warn_redundant(cfg) {
> +		if (warnredun != "true" || warnoverride == "true") return
> +		print "Value of " cfg " is redundant by fragment " mergefile ":" > "/dev/stderr"
> +	}
> +
> +	# First pass: read merge file, store all lines and index
> +	FILENAME == ARGV[1] {
> +	        mergefile = FILENAME
> +		merge_lines[FNR] = $0
> +		merge_total = FNR
> +		cfg = get_cfg($0)
> +		if (cfg != "") {
> +			merge_cfg[cfg] = $0
> +			merge_cfg_line[cfg] = FNR
> +		}
> +		next
> +	}
>  
> +	# Second pass: process base file (TMP_FILE)
> +		cfg = get_cfg($0)
> +
> +		# Not a config or not in merge file - keep it
> +		if (cfg == "" || !(cfg in merge_cfg)) {

  I get an error on this line when trying to use this script via the arm64
  virtconfig target:

  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux- clean virtconfig
  awk: cmd. line:70:              if (cfg == "" || !(cfg in merge_cfg)) {
  awk: cmd. line:70:              ^ syntax error
  make[2]: *** [arch/arm64/Makefile:222: virtconfig] Error 1
  ...

  $ awk --version | head -1
  GNU Awk 5.3.2, API 4.0, PMA Avon 8-g1, (GNU MPFR 4.2.2, GNU MP 6.3.0)

> +			print $0
> +			next
> +		}
> +
> +		prev_val = normalize($0)
> +		new_val = normalize(merge_cfg[cfg])
> +
> +		# BUILTIN: do not demote y to m
> +		if (builtin == "true" && new_val ~ /=m$/ && prev_val ~ /=y$/) {
> +			warn_builtin(cfg, prev_val, new_val)
> +			print $0
> +			skip_merge[merge_cfg_line[cfg]] = 1
> +			next
> +		}
> +
> +		# Values equal - redundant
> +		if (prev_val == new_val) {
> +			warn_redundant(cfg)
> +			next
> +		}
> +
> +		# Values differ - redefined
> +		warn_redefined(cfg, prev_val, new_val)
> +		if (strict == "true") {
> +			strict_violated = 1
> +		}
> +	}
> +
> +	END {
> +		# Newline in case base file lacks trailing newline
> +		print ""
> +		# Append merge file, skipping lines marked for builtin preservation
> +		for (i = 1; i <= merge_total; i++) {
> +			if (!(i in skip_merge)) {
> +				print merge_lines[i]
> +			}
> +		}
> +		if (strict_violated) {
> +			exit 1
> +		}
> +	}' \
> +	"$ORIG_MERGE_FILE" "$TMP_FILE" > "${TMP_FILE}.new"; then
> +		# awk exited non-zero, strict mode was violated
> +		STRICT_MODE_VIOLATED=true
> +	fi
> +	mv "${TMP_FILE}.new" "$TMP_FILE"
> +done
>  if [ "$STRICT_MODE_VIOLATED" = "true" ]; then
>  	echo "The fragment redefined a value and strict mode had been passed."
>  	exit 1
> -- 
> 2.34.1
> 

Re: [PATCH 1/3] scripts: kconfig: merge_config.sh: refactor from shell/sed/grep to awk

[Mikko Rapeli]

Hi,

On Tue, Dec 30, 2025 at 01:55:49PM -0700, Nathan Chancellor wrote:
> Hi Mikko,
> 
> I don't really know awk well so I won't give as much review on that
> part.
> 
> On Mon, Dec 29, 2025 at 01:44:45PM +0200, Mikko Rapeli wrote:
> > From: Anders Roxell <anders.roxell@linaro.org>
> > 
> > merge_config.sh shell/sed/grep loop scales poorly and is slow.
> > With Yocto genericarm64 kernel and around 190 config fragments
> > the script takes more than 20 minutes to run on a fast build machine.
> > Re-implementation with awk does the same job in 10 seconds.
> > Using awk since it is likely available in the build environments
> > and using perl, python etc would introduce more complex runtime
> > dependencies. awk is good enough and lot better than shell/sed/grep.
> > 
> > Signed-off-by: Anders Roxell <anders.roxell@linaro.org>
> > Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
> > ---
> >  scripts/kconfig/merge_config.sh | 161 ++++++++++++++++++++++++--------
> >  1 file changed, 123 insertions(+), 38 deletions(-)
> > 
> > diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
> > index 79c09b378be8..46397d7c6957 100755
> > --- a/scripts/kconfig/merge_config.sh
> > +++ b/scripts/kconfig/merge_config.sh
> > @@ -16,8 +16,8 @@
> >  set -e
> >  
> >  clean_up() {
> > -	rm -f $TMP_FILE
> > -	rm -f $MERGE_FILE
> > +	rm -f "${TMP_FILE}"
> > +	rm -f "${TMP_FILE}.new"
> >  }
> >  
> >  usage() {
> > @@ -121,7 +121,6 @@ SED_CONFIG_EXP1="s/^\(${CONFIG_PREFIX}[a-zA-Z0-9_]*\)=.*/\1/p"
> >  SED_CONFIG_EXP2="s/^# \(${CONFIG_PREFIX}[a-zA-Z0-9_]*\) is not set$/\1/p"
> 
> The SED_CONFIG_EXP variables are unused now so they can be removed.

Right, removing in v2.

> >  TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX)
> > -MERGE_FILE=$(mktemp ./.merge_tmp.config.XXXXXXXXXX)
> >  
> >  echo "Using $INITFILE as base"
> >  
> > @@ -136,42 +135,128 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
> >  		echo "The merge file '$ORIG_MERGE_FILE' does not exist.  Exit." >&2
> >  		exit 1
> >  	fi
> > -	cat $ORIG_MERGE_FILE > $MERGE_FILE
> > -	CFG_LIST=$(sed -n -e "$SED_CONFIG_EXP1" -e "$SED_CONFIG_EXP2" $MERGE_FILE)
> > -
> > -	for CFG in $CFG_LIST ; do
> > -		grep -q -w $CFG $TMP_FILE || continue
> > -		PREV_VAL=$(grep -w $CFG $TMP_FILE)
> > -		NEW_VAL=$(grep -w $CFG $MERGE_FILE)
> > -		BUILTIN_FLAG=false
> > -		if [ "$BUILTIN" = "true" ] && [ "${NEW_VAL#CONFIG_*=}" = "m" ] && [ "${PREV_VAL#CONFIG_*=}" = "y" ]; then
> > -			${WARNOVERRIDE} Previous  value: $PREV_VAL
> > -			${WARNOVERRIDE} New value:       $NEW_VAL
> > -			${WARNOVERRIDE} -y passed, will not demote y to m
> > -			${WARNOVERRIDE}
> > -			BUILTIN_FLAG=true
> > -		elif [ "x$PREV_VAL" != "x$NEW_VAL" ] ; then
> > -			${WARNOVERRIDE} Value of $CFG is redefined by fragment $ORIG_MERGE_FILE:
> > -			${WARNOVERRIDE} Previous  value: $PREV_VAL
> > -			${WARNOVERRIDE} New value:       $NEW_VAL
> > -			${WARNOVERRIDE}
> > -			if [ "$STRICT" = "true" ]; then
> > -				STRICT_MODE_VIOLATED=true
> > -			fi
> > -		elif [ "$WARNREDUN" = "true" ]; then
> > -			${WARNOVERRIDE} Value of $CFG is redundant by fragment $ORIG_MERGE_FILE:
> > -		fi
> > -		if [ "$BUILTIN_FLAG" = "false" ]; then
> > -			sed -i "/$CFG[ =]/d" $TMP_FILE
> > -		else
> > -			sed -i "/$CFG[ =]/d" $MERGE_FILE
> > -		fi
> > -	done
> > -	# In case the previous file lacks a new line at the end
> > -	echo >> $TMP_FILE
> > -	cat $MERGE_FILE >> $TMP_FILE
> > -done
> > +	# Use awk for single-pass processing instead of per-symbol grep/sed
> > +	if ! awk -v prefix="$CONFIG_PREFIX" \
> 
> Please use the AWK variable from Kbuild here:
> 
>   if ${AWK} -v prefix="$CONFIG_PREFIX" \
> 
> You can do something like:
> 
>   if [ -z "${AWK}" ]; then
>       AWK=awk
>   fi
> 
> towards the top of the script to have the script continue to work
> outside of Kbuild as well.

Ok will add this in v2.

> > +		-v warnoverride="$WARNOVERRIDE" \
> > +		-v strict="$STRICT" \
> > +		-v builtin="$BUILTIN" \
> > +		-v warnredun="$WARNREDUN" '
> > +	BEGIN {
> > +		strict_violated = 0
> > +		cfg_regex = "^" prefix "[a-zA-Z0-9_]+"
> > +		notset_regex = "^# " prefix "[a-zA-Z0-9_]+ is not set$"
> > +	}
> > +
> > +	# Extract config name from a line, returns "" if not a config line
> > +	function get_cfg(line) {
> > +		if (match(line, cfg_regex)) {
> > +			return substr(line, RSTART, RLENGTH)
> > +		} else if (match(line, notset_regex)) {
> > +			# Extract CONFIG_FOO from "# CONFIG_FOO is not set"
> > +			sub(/^# /, "", line)
> > +			sub(/ is not set$/, "", line)
> > +			return line
> > +		}
> > +		return ""
> > +	}
> > +
> > +	# Normalize: strip trailing comments, convert "is not set" to "=n"
> > +	function normalize(line) {
> > +		if (line == "") return ""
> > +		sub(/[[:space:]]+#.*/, "", line)
> > +		if (line ~ / is not set$/) {
> > +			sub(/^# /, "", line)
> > +			sub(/ is not set$/, "=n", line)
> > +		}
> > +		return line
> > +	}
> > +
> > +	function warn_builtin(cfg, prev, new) {
> > +		if (warnoverride == "true") return
> > +		print cfg ": -y passed, will not demote y to m" > "/dev/stderr"
> > +		print "  Previous value: " prev > "/dev/stderr"
> > +		print "  New value:	 " new > "/dev/stderr"
> > +		print "" > "/dev/stderr"
> > +	}
> > +
> > +	function warn_redefined(cfg, prev, new) {
> > +		if (warnoverride == "true") return
> > +		print "Value of " cfg " is redefined by fragment " mergefile ":" > "/dev/stderr"
> > +		print "  Previous value: " prev > "/dev/stderr"
> > +		print "  New value:	 " new > "/dev/stderr"
> > +		print "" > "/dev/stderr"
> > +	}
> > +
> > +	function warn_redundant(cfg) {
> > +		if (warnredun != "true" || warnoverride == "true") return
> > +		print "Value of " cfg " is redundant by fragment " mergefile ":" > "/dev/stderr"
> > +	}
> > +
> > +	# First pass: read merge file, store all lines and index
> > +	FILENAME == ARGV[1] {
> > +	        mergefile = FILENAME
> > +		merge_lines[FNR] = $0
> > +		merge_total = FNR
> > +		cfg = get_cfg($0)
> > +		if (cfg != "") {
> > +			merge_cfg[cfg] = $0
> > +			merge_cfg_line[cfg] = FNR
> > +		}
> > +		next
> > +	}
> >  
> > +	# Second pass: process base file (TMP_FILE)
> > +		cfg = get_cfg($0)
> > +
> > +		# Not a config or not in merge file - keep it
> > +		if (cfg == "" || !(cfg in merge_cfg)) {
> 
>   I get an error on this line when trying to use this script via the arm64
>   virtconfig target:
> 
>   $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux- clean virtconfig
>   awk: cmd. line:70:              if (cfg == "" || !(cfg in merge_cfg)) {
>   awk: cmd. line:70:              ^ syntax error
>   make[2]: *** [arch/arm64/Makefile:222: virtconfig] Error 1
>   ...
> 
>   $ awk --version | head -1
>   GNU Awk 5.3.2, API 4.0, PMA Avon 8-g1, (GNU MPFR 4.2.2, GNU MP 6.3.0)

Oops. Sorry, there is a curly brace missing:

@@ -216,6 +214,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
        }

        # Second pass: process base file (TMP_FILE)
+       {
                cfg = get_cfg($0)

                # Not a config or not in merge file - keep it

but this is fatal so I had been testing an old version of the patch
in my yocto/bitbake setup. I will double check this with v2.

Cheers,

-Mikko

[PATCH v2 1/3] scripts: kconfig: merge_config.sh: refactor from shell/sed/grep to awk

[Mikko Rapeli]

From: Anders Roxell <anders.roxell@linaro.org>

merge_config.sh shell/sed/grep loop scales poorly and is slow.
With Yocto genericarm64 kernel and around 190 config fragments
the script takes more than 20 minutes to run on a fast build machine.
Re-implementation with awk does the same job in 10 seconds.
Using awk since it is likely available in the build environments
and using perl, python etc would introduce more complex runtime
dependencies. awk is good enough and lot better than shell/sed/grep.

Signed-off-by: Anders Roxell <anders.roxell@linaro.org>
Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
---
 scripts/kconfig/merge_config.sh | 168 ++++++++++++++++++++++++--------
 1 file changed, 128 insertions(+), 40 deletions(-)

v2: remove unused sed variables, awk from ${AWK} variable,
    curly brace syntax fix after rebase, triple check that
    correct revision of patches are used in testing with
    yocto/bitbake

v1: https://lore.kernel.org/linux-kbuild/20251229114447.45236-1-mikko.rapeli@linaro.org/T/#t

diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
index 79c09b378be8..4cefe3cdfc2f 100755
--- a/scripts/kconfig/merge_config.sh
+++ b/scripts/kconfig/merge_config.sh
@@ -16,8 +16,8 @@
 set -e
 
 clean_up() {
-	rm -f $TMP_FILE
-	rm -f $MERGE_FILE
+	rm -f "${TMP_FILE}"
+	rm -f "${TMP_FILE}.new"
 }
 
 usage() {
@@ -43,6 +43,10 @@ STRICT=false
 CONFIG_PREFIX=${CONFIG_-CONFIG_}
 WARNOVERRIDE=echo
 
+if [ -z "${AWK}" ]; then
+	AWK=awk
+fi
+
 while true; do
 	case $1 in
 	"-n")
@@ -117,11 +121,8 @@ if [ ! -r "$INITFILE" ]; then
 fi
 
 MERGE_LIST=$*
-SED_CONFIG_EXP1="s/^\(${CONFIG_PREFIX}[a-zA-Z0-9_]*\)=.*/\1/p"
-SED_CONFIG_EXP2="s/^# \(${CONFIG_PREFIX}[a-zA-Z0-9_]*\) is not set$/\1/p"
 
 TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX)
-MERGE_FILE=$(mktemp ./.merge_tmp.config.XXXXXXXXXX)
 
 echo "Using $INITFILE as base"
 
@@ -136,42 +137,129 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
 		echo "The merge file '$ORIG_MERGE_FILE' does not exist.  Exit." >&2
 		exit 1
 	fi
-	cat $ORIG_MERGE_FILE > $MERGE_FILE
-	CFG_LIST=$(sed -n -e "$SED_CONFIG_EXP1" -e "$SED_CONFIG_EXP2" $MERGE_FILE)
-
-	for CFG in $CFG_LIST ; do
-		grep -q -w $CFG $TMP_FILE || continue
-		PREV_VAL=$(grep -w $CFG $TMP_FILE)
-		NEW_VAL=$(grep -w $CFG $MERGE_FILE)
-		BUILTIN_FLAG=false
-		if [ "$BUILTIN" = "true" ] && [ "${NEW_VAL#CONFIG_*=}" = "m" ] && [ "${PREV_VAL#CONFIG_*=}" = "y" ]; then
-			${WARNOVERRIDE} Previous  value: $PREV_VAL
-			${WARNOVERRIDE} New value:       $NEW_VAL
-			${WARNOVERRIDE} -y passed, will not demote y to m
-			${WARNOVERRIDE}
-			BUILTIN_FLAG=true
-		elif [ "x$PREV_VAL" != "x$NEW_VAL" ] ; then
-			${WARNOVERRIDE} Value of $CFG is redefined by fragment $ORIG_MERGE_FILE:
-			${WARNOVERRIDE} Previous  value: $PREV_VAL
-			${WARNOVERRIDE} New value:       $NEW_VAL
-			${WARNOVERRIDE}
-			if [ "$STRICT" = "true" ]; then
-				STRICT_MODE_VIOLATED=true
-			fi
-		elif [ "$WARNREDUN" = "true" ]; then
-			${WARNOVERRIDE} Value of $CFG is redundant by fragment $ORIG_MERGE_FILE:
-		fi
-		if [ "$BUILTIN_FLAG" = "false" ]; then
-			sed -i "/$CFG[ =]/d" $TMP_FILE
-		else
-			sed -i "/$CFG[ =]/d" $MERGE_FILE
-		fi
-	done
-	# In case the previous file lacks a new line at the end
-	echo >> $TMP_FILE
-	cat $MERGE_FILE >> $TMP_FILE
-done
+	# Use awk for single-pass processing instead of per-symbol grep/sed
+	if ! "${AWK}" -v prefix="$CONFIG_PREFIX" \
+		-v warnoverride="$WARNOVERRIDE" \
+		-v strict="$STRICT" \
+		-v builtin="$BUILTIN" \
+		-v warnredun="$WARNREDUN" '
+	BEGIN {
+		strict_violated = 0
+		cfg_regex = "^" prefix "[a-zA-Z0-9_]+"
+		notset_regex = "^# " prefix "[a-zA-Z0-9_]+ is not set$"
+	}
+
+	# Extract config name from a line, returns "" if not a config line
+	function get_cfg(line) {
+		if (match(line, cfg_regex)) {
+			return substr(line, RSTART, RLENGTH)
+		} else if (match(line, notset_regex)) {
+			# Extract CONFIG_FOO from "# CONFIG_FOO is not set"
+			sub(/^# /, "", line)
+			sub(/ is not set$/, "", line)
+			return line
+		}
+		return ""
+	}
+
+	# Normalize: strip trailing comments, convert "is not set" to "=n"
+	function normalize(line) {
+		if (line == "") return ""
+		sub(/[[:space:]]+#.*/, "", line)
+		if (line ~ / is not set$/) {
+			sub(/^# /, "", line)
+			sub(/ is not set$/, "=n", line)
+		}
+		return line
+	}
+
+	function warn_builtin(cfg, prev, new) {
+		if (warnoverride == "true") return
+		print cfg ": -y passed, will not demote y to m" > "/dev/stderr"
+		print "  Previous value: " prev > "/dev/stderr"
+		print "  New value:	 " new > "/dev/stderr"
+		print "" > "/dev/stderr"
+	}
+
+	function warn_redefined(cfg, prev, new) {
+		if (warnoverride == "true") return
+		print "Value of " cfg " is redefined by fragment " mergefile ":" > "/dev/stderr"
+		print "  Previous value: " prev > "/dev/stderr"
+		print "  New value:	 " new > "/dev/stderr"
+		print "" > "/dev/stderr"
+	}
+
+	function warn_redundant(cfg) {
+		if (warnredun != "true" || warnoverride == "true") return
+		print "Value of " cfg " is redundant by fragment " mergefile ":" > "/dev/stderr"
+	}
 
+	# First pass: read merge file, store all lines and index
+	FILENAME == ARGV[1] {
+	        mergefile = FILENAME
+		merge_lines[FNR] = $0
+		merge_total = FNR
+		cfg = get_cfg($0)
+		if (cfg != "") {
+			merge_cfg[cfg] = $0
+			merge_cfg_line[cfg] = FNR
+		}
+		next
+	}
+
+	# Second pass: process base file (TMP_FILE)
+	{
+		cfg = get_cfg($0)
+
+		# Not a config or not in merge file - keep it
+		if (cfg == "" || !(cfg in merge_cfg)) {
+			print $0
+			next
+		}
+
+		prev_val = normalize($0)
+		new_val = normalize(merge_cfg[cfg])
+
+		# BUILTIN: do not demote y to m
+		if (builtin == "true" && new_val ~ /=m$/ && prev_val ~ /=y$/) {
+			warn_builtin(cfg, prev_val, new_val)
+			print $0
+			skip_merge[merge_cfg_line[cfg]] = 1
+			next
+		}
+
+		# Values equal - redundant
+		if (prev_val == new_val) {
+			warn_redundant(cfg)
+			next
+		}
+
+		# Values differ - redefined
+		warn_redefined(cfg, prev_val, new_val)
+		if (strict == "true") {
+			strict_violated = 1
+		}
+	}
+
+	END {
+		# Newline in case base file lacks trailing newline
+		print ""
+		# Append merge file, skipping lines marked for builtin preservation
+		for (i = 1; i <= merge_total; i++) {
+			if (!(i in skip_merge)) {
+				print merge_lines[i]
+			}
+		}
+		if (strict_violated) {
+			exit 1
+		}
+	}' \
+	"$ORIG_MERGE_FILE" "$TMP_FILE" > "${TMP_FILE}.new"; then
+		# awk exited non-zero, strict mode was violated
+		STRICT_MODE_VIOLATED=true
+	fi
+	mv "${TMP_FILE}.new" "$TMP_FILE"
+done
 if [ "$STRICT_MODE_VIOLATED" = "true" ]; then
 	echo "The fragment redefined a value and strict mode had been passed."
 	exit 1
-- 
2.34.1

[PATCH v2 2/3] scripts: kconfig: merge_config.sh: use awk in checks too

[Mikko Rapeli]

Converting from shell/sed/grep loop to awk improves runtime
checks of Yocto genericarm64 kernel config from 20 seconds
to under 1 second. Also fix the tests to take into account:

 * kernel config lines with comments
   CONFIG_FOO=m # this is a comment
 * match "# CONFIG_FOO is not set" with "CONFIG_FOO=n"
 * match unset config value to "CONFIG_FOO=n"

Cc: Anders Roxell <anders.roxell@linaro.org>
Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
---
 scripts/kconfig/merge_config.sh | 99 +++++++++++++++++++++++++++++----
 1 file changed, 88 insertions(+), 11 deletions(-)

v2: awk from ${AWK}

v1: https://lore.kernel.org/linux-kbuild/20251229114447.45236-1-mikko.rapeli@linaro.org/T/#t

diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
index 4cefe3cdfc2f..462c791cca12 100755
--- a/scripts/kconfig/merge_config.sh
+++ b/scripts/kconfig/merge_config.sh
@@ -286,16 +286,93 @@ fi
 # allnoconfig: Fills in any missing symbols with # CONFIG_* is not set
 make KCONFIG_ALLCONFIG=$TMP_FILE $OUTPUT_ARG $ALLTARGET
 
+# Check all specified config values took effect (might have missed-dependency issues)
+if ! "${AWK}" -v prefix="$CONFIG_PREFIX" \
+	-v warnoverride="$WARNOVERRIDE" \
+	-v strict="$STRICT" \
+	-v warnredun="$WARNREDUN" '
+BEGIN {
+	strict_violated = 0
+	cfg_regex = "^" prefix "[a-zA-Z0-9_]+"
+	notset_regex = "^# " prefix "[a-zA-Z0-9_]+ is not set$"
+}
 
-# Check all specified config values took (might have missed-dependency issues)
-for CFG in $(sed -n -e "$SED_CONFIG_EXP1" -e "$SED_CONFIG_EXP2" $TMP_FILE); do
+# Extract config name from a line, returns "" if not a config line
+function get_cfg(line) {
+	if (match(line, cfg_regex)) {
+		return substr(line, RSTART, RLENGTH)
+	} else if (match(line, notset_regex)) {
+		# Extract CONFIG_FOO from "# CONFIG_FOO is not set"
+		sub(/^# /, "", line)
+		sub(/ is not set$/, "", line)
+		return line
+	}
+	return ""
+}
 
-	REQUESTED_VAL=$(grep -w -e "$CFG" $TMP_FILE)
-	ACTUAL_VAL=$(grep -w -e "$CFG" "$KCONFIG_CONFIG" || true)
-	if [ "x$REQUESTED_VAL" != "x$ACTUAL_VAL" ] ; then
-		echo "Value requested for $CFG not in final .config"
-		echo "Requested value:  $REQUESTED_VAL"
-		echo "Actual value:     $ACTUAL_VAL"
-		echo ""
-	fi
-done
+# Normalize: strip trailing comments, convert "is not set" to "=n"
+function normalize(line) {
+	if (line == "") return ""
+	sub(/[[:space:]]+#.*/, "", line)
+	if (line ~ / is not set$/) {
+		sub(/^# /, "", line)
+		sub(/ is not set$/, "=n", line)
+	}
+	return line
+}
+
+function warn_mismatch(cfg, merged, final) {
+	if (warnredun == "true") return
+	if (final == "" && !(merged ~ /=n$/)) {
+		print "WARNING: " cfg " not in final config but set in merged config to: " merged > "/dev/stderr"
+	} else if (final == "" && merged ~ /=n$/) {
+		# not set is the same as =n, pass
+	} else if (merged == "" && final != "") {
+		print "WARNING: " cfg " not in merged config but added in final config: " final > "/dev/stderr"
+	} else {
+		print "WARNING: " cfg " differs:" > "/dev/stderr"
+		print "Merged value: " merged > "/dev/stderr"
+		print "Final  value: " final > "/dev/stderr"
+	}
+}
+
+# First pass: read effective config file, store all lines
+FILENAME == ARGV[1] {
+	cfg = get_cfg($0)
+	if (cfg != "") {
+		config_cfg[cfg] = $0
+	}
+	next
+}
+
+# Second pass: process merged config and compare against effective config
+{
+	cfg = get_cfg($0)
+	if (cfg == "") next
+
+	merged_val = normalize($0)
+	final_val = normalize(config_cfg[cfg])
+
+	if (merged_val == final_val) next
+
+	warn_mismatch(cfg, merged_val, final_val)
+
+	if (strict == "true") {
+		strict_violated = 1
+	}
+}
+
+END {
+	if (strict_violated) {
+		exit 1
+	}
+}' \
+"$KCONFIG_CONFIG" "$TMP_FILE"; then
+	# awk exited non-zero, strict mode was violated
+	STRICT_MODE_VIOLATED=true
+fi
+
+if [ "$STRICT" == "true" ] && [ "$STRICT_MODE_VIOLATED" == "true" ]; then
+	echo "Requested and effective config differ"
+	exit 1
+fi
-- 
2.34.1

[PATCH v2 3/3] scripts: kconfig: merge_config.sh: warn on duplicate input files

[Mikko Rapeli]

External scripts like yocto kernel scc may provide
same input config fragment multiple times. This may
be a bug since processing same fragments multiple times
can be time consuming.

Cc: Anders Roxell <anders.roxell@linaro.org>
Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
---
 scripts/kconfig/merge_config.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

v2: no changes

v1: https://lore.kernel.org/linux-kbuild/20251229114447.45236-1-mikko.rapeli@linaro.org/T/#t

diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
index 462c791cca12..cc541a9686b8 100755
--- a/scripts/kconfig/merge_config.sh
+++ b/scripts/kconfig/merge_config.sh
@@ -130,6 +130,8 @@ trap clean_up EXIT
 
 cat $INITFILE > $TMP_FILE
 
+PROCESSED_FILES=""
+
 # Merge files, printing warnings on overridden values
 for ORIG_MERGE_FILE in $MERGE_LIST ; do
 	echo "Merging $ORIG_MERGE_FILE"
@@ -137,6 +139,14 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
 		echo "The merge file '$ORIG_MERGE_FILE' does not exist.  Exit." >&2
 		exit 1
 	fi
+
+	# Check for duplicate input files
+	case " $PROCESSED_FILES " in
+		*" $ORIG_MERGE_FILE "*)
+			${WARNOVERRIDE} "WARNING: Input file provided multiple times: $ORIG_MERGE_FILE"
+			;;
+	esac
+
 	# Use awk for single-pass processing instead of per-symbol grep/sed
 	if ! "${AWK}" -v prefix="$CONFIG_PREFIX" \
 		-v warnoverride="$WARNOVERRIDE" \
@@ -259,6 +269,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
 		STRICT_MODE_VIOLATED=true
 	fi
 	mv "${TMP_FILE}.new" "$TMP_FILE"
+	PROCESSED_FILES="$PROCESSED_FILES $ORIG_MERGE_FILE"
 done
 if [ "$STRICT_MODE_VIOLATED" = "true" ]; then
 	echo "The fragment redefined a value and strict mode had been passed."
-- 
2.34.1

Re: [PATCH v2 1/3] scripts: kconfig: merge_config.sh: refactor from shell/sed/grep to awk

[Nathan Chancellor]

Hi Mikko,

Thanks for the update! Just a heads up, please start a new thread for
newer patch revisions, preferring a link back to the earlier revisions
like b4 does. Sending it as a reply to the previous revision makes it
hard to follow in an mbox, as you might be able to see at the bottom of
the lore thread:

https://lore.kernel.org/linux-kbuild/20251229114447.45236-1-mikko.rapeli@linaro.org/

On Wed, Dec 31, 2025 at 10:40:48AM +0200, Mikko Rapeli wrote:
> From: Anders Roxell <anders.roxell@linaro.org>
> 
> merge_config.sh shell/sed/grep loop scales poorly and is slow.
> With Yocto genericarm64 kernel and around 190 config fragments
> the script takes more than 20 minutes to run on a fast build machine.
> Re-implementation with awk does the same job in 10 seconds.
> Using awk since it is likely available in the build environments
> and using perl, python etc would introduce more complex runtime
> dependencies. awk is good enough and lot better than shell/sed/grep.
> 
> Signed-off-by: Anders Roxell <anders.roxell@linaro.org>
> Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
> ---
>  scripts/kconfig/merge_config.sh | 168 ++++++++++++++++++++++++--------
>  1 file changed, 128 insertions(+), 40 deletions(-)
> 
> v2: remove unused sed variables, awk from ${AWK} variable,
>     curly brace syntax fix after rebase, triple check that
>     correct revision of patches are used in testing with
>     yocto/bitbake
> 
> v1: https://lore.kernel.org/linux-kbuild/20251229114447.45236-1-mikko.rapeli@linaro.org/T/#t
> 
> diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
> index 79c09b378be8..4cefe3cdfc2f 100755
> --- a/scripts/kconfig/merge_config.sh
> +++ b/scripts/kconfig/merge_config.sh
> @@ -16,8 +16,8 @@
>  set -e
>  
>  clean_up() {
> -	rm -f $TMP_FILE
> -	rm -f $MERGE_FILE
> +	rm -f "${TMP_FILE}"
> +	rm -f "${TMP_FILE}.new"
...
> +if [ -z "${AWK}" ]; then

Small nit: This file seems to prefer $VAR over ${VAR} when not using
bash parameter expansion (and that is a little easier for me as a fish
user to read longterm).

> +	# Normalize: strip trailing comments, convert "is not set" to "=n"
> +	function normalize(line) {
> +		if (line == "") return ""
> +		sub(/[[:space:]]+#.*/, "", line)
> +		if (line ~ / is not set$/) {
> +			sub(/^# /, "", line)
> +			sub(/ is not set$/, "=n", line)
> +		}
> +		return line
> +	}

I think this normalization makes it a little harder to read when the
value is changed from "n" to "y".

Prior to this change:

  $ make -j"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux- clean defconfig hardening.config
  ...
  Value of CONFIG_SLAB_FREELIST_HARDENED is redefined by fragment kernel/configs/hardening.config:
  Previous value: # CONFIG_SLAB_FREELIST_HARDENED is not set
  New value: CONFIG_SLAB_FREELIST_HARDENED=y
  ...

After this change:

  $ make -j"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux- clean defconfig hardening.config
  ...
  Value of CONFIG_SLAB_FREELIST_HARDENED is redefined by fragment kernel/configs/hardening.config:
    Previous value: CONFIG_SLAB_FREELIST_HARDENED=n
    New value:     CONFIG_SLAB_FREELIST_HARDENED=y
  ...

Linus has complained about moving from "is not set" to "n" in the past:

https://lore.kernel.org/CAHk-=wgxcu9DFkXAOAFdDtLWwuv6qb5iV1E69yWE-JEVsd-NFg@mail.gmail.com/

I do like the alignment change for the "new value" line but I think
keeping "is not set" would be a little easier to quickly parse than
"=n".

  Value of CONFIG_SLAB_FREELIST_HARDENED is redefined by fragment kernel/configs/hardening.config:
    Previous value: # CONFIG_SLAB_FREELIST_HARDENED is not set
    New value:     CONFIG_SLAB_FREELIST_HARDENED=y

Might just be a personal preference though.

> +	function warn_builtin(cfg, prev, new) {
> +		if (warnoverride == "true") return
> +		print cfg ": -y passed, will not demote y to m" > "/dev/stderr"
> +		print "  Previous value: " prev > "/dev/stderr"
> +		print "  New value:	 " new > "/dev/stderr"
> +		print "" > "/dev/stderr"
> +	}
> +
> +	function warn_redefined(cfg, prev, new) {
> +		if (warnoverride == "true") return
> +		print "Value of " cfg " is redefined by fragment " mergefile ":" > "/dev/stderr"
> +		print "  Previous value: " prev > "/dev/stderr"
> +		print "  New value:	 " new > "/dev/stderr"
> +		print "" > "/dev/stderr"
> +	}
> +
> +	function warn_redundant(cfg) {
> +		if (warnredun != "true" || warnoverride == "true") return
> +		print "Value of " cfg " is redundant by fragment " mergefile ":" > "/dev/stderr"
> +	}

The use of /dev/stderr seems to introduce a change in behavior when
using 'make -s'.

Prior to this change:

  $ make -sj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux- clean defconfig hardening.config

After this change:

  $ make -sj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux- clean defconfig hardening.config
  Value of CONFIG_COMPAT_VDSO is redefined by fragment kernel/configs/hardening.config:
    Previous value: CONFIG_COMPAT_VDSO=y
    New value:     CONFIG_COMPAT_VDSO=n

  Value of CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT is redefined by fragment kernel/configs/hardening.config:
    Previous value: CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT=n
    New value:     CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT=y
  ...

These should probably just be /dev/stdout?

Cheers,
Nathan