[PATCH 1/3] perf test: Support arch-specific shell tests

[PATCH 1/3] perf test: Support arch-specific shell tests

[Namhyung Kim]

This is a preparation for shell tests belong to an arch.

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
 tools/perf/tests/builtin-test.c  |  3 ++
 tools/perf/tests/tests-scripts.c | 73 ++++++++++++++++++++++++++++++++
 tools/perf/tests/tests-scripts.h |  1 +
 3 files changed, 77 insertions(+)

diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index 45d3d8b3317a7c0a..b9a86631b34cac90 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -650,6 +650,7 @@ static struct test_suite **build_suites(void)
 		generic_tests,
 		arch_tests,
 		NULL,
+		NULL,
 	};
 	struct test_suite **result;
 	struct test_suite *t;
@@ -657,6 +658,8 @@ static struct test_suite **build_suites(void)
 
 	if (suites[2] == NULL)
 		suites[2] = create_script_test_suites();
+	if (suites[3] == NULL)
+		suites[3] = create_script_test_suites_arch();
 
 #define for_each_suite(suite)						\
 	for (size_t i = 0, j = 0; i < ARRAY_SIZE(suites); i++, j = 0)	\
diff --git a/tools/perf/tests/tests-scripts.c b/tools/perf/tests/tests-scripts.c
index 1d5759d08141749d..e6e6fab0c3c4bb8c 100644
--- a/tools/perf/tests/tests-scripts.c
+++ b/tools/perf/tests/tests-scripts.c
@@ -24,6 +24,7 @@
 #include "string2.h"
 #include "symbol.h"
 #include "tests.h"
+#include "util/env.h"
 #include "util/rlimit.h"
 #include "util/util.h"
 
@@ -75,6 +76,52 @@ static int shell_tests__dir_fd(void)
 	return open(path, O_PATH);
 }
 
+static int shell_tests__arch_dir_fd(void)
+{
+	struct stat st;
+	const char *arch;
+	char path[PATH_MAX], path2[PATH_MAX], *exec_path;
+	int fd;
+	char *p;
+
+	arch = perf_env__arch(NULL);
+	if (arch == NULL)
+		return -1;
+
+	scnprintf(path, sizeof(path), "./arch/%s/tests/shell", arch);
+	fd = open(path, O_PATH);
+	if (fd >= 0)
+		return fd;
+
+	/* Use directory of executable */
+	if (readlink("/proc/self/exe", path2, sizeof path2) < 0)
+		return -1;
+	/* Follow another level of symlink if there */
+	if (lstat(path2, &st) == 0 && (st.st_mode & S_IFMT) == S_IFLNK) {
+		scnprintf(path, sizeof(path), path2);
+		if (readlink(path, path2, sizeof path2) < 0)
+			return -1;
+	}
+	/* Get directory */
+	p = strrchr(path2, '/');
+	if (p)
+		*p = 0;
+	scnprintf(path, sizeof(path), "%s/arch/%s/tests/shell", path2, arch);
+	fd = open(path, O_PATH);
+	if (fd >= 0)
+		return fd;
+	scnprintf(path, sizeof(path), "%s/source/arch/%s/tests/shell", path2, arch);
+	fd = open(path, O_PATH);
+	if (fd >= 0)
+		return fd;
+
+	/* Then installed path. */
+	exec_path = get_argv_exec_path();
+	scnprintf(path, sizeof(path), "%s/arch/%s/tests/shell", exec_path, arch);
+	free(exec_path);
+	return open(path, O_PATH);
+}
+
 static char *shell_test__description(int dir_fd, const char *name)
 {
 	struct io io;
@@ -291,3 +338,29 @@ struct test_suite **create_script_test_suites(void)
 		close(dir_fd);
 	return result;
 }
+
+struct test_suite **create_script_test_suites_arch(void)
+{
+	struct test_suite **result = NULL, **result_tmp;
+	size_t result_sz = 0;
+	int dir_fd = shell_tests__arch_dir_fd(); /* Walk dir */
+
+	/*
+	 * Append scripts if fd is good, otherwise return a NULL terminated zero
+	 * length array.
+	 */
+	if (dir_fd >= 0)
+		append_scripts_in_dir(dir_fd, &result, &result_sz);
+
+	result_tmp = realloc(result, (result_sz + 1) * sizeof(*result_tmp));
+	if (result_tmp == NULL) {
+		pr_err("Out of memory while building script test suite list\n");
+		abort();
+	}
+	/* NULL terminate the test suite array. */
+	result = result_tmp;
+	result[result_sz] = NULL;
+	if (dir_fd >= 0)
+		close(dir_fd);
+	return result;
+}
diff --git a/tools/perf/tests/tests-scripts.h b/tools/perf/tests/tests-scripts.h
index b553ad26ea17642a..b03ee4a1ee63a25a 100644
--- a/tools/perf/tests/tests-scripts.h
+++ b/tools/perf/tests/tests-scripts.h
@@ -5,5 +5,6 @@
 #include "tests.h"
 
 struct test_suite **create_script_test_suites(void);
+struct test_suite **create_script_test_suites_arch(void);
 
 #endif /* TESTS_SCRIPTS_H */
-- 
2.49.0.1164.gab81da1b16-goog

[PATCH 2/3] perf test: Move some ARM tests to arch/arm64/tests/shell

[Namhyung Kim]

The test_arm_callgraph_fp.sh checks with aarch64 so it should belong to
there.  And IIRC ARM SPE is supported on 64-bit platforms so move the
tests too.  But I'm not sure about coresight so left them.

Also please test it with shellcheck as I couldn't run it actually.

Cc: Leo Yan <leo.yan@arm.com>
Cc: James Clark <james.clark@linaro.org>
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
 tools/perf/arch/arm64/tests/Build                  | 14 ++++++++++++++
 .../arm64}/tests/shell/test_arm_callgraph_fp.sh    |  4 ++--
 .../{ => arch/arm64}/tests/shell/test_arm_spe.sh   |  0
 .../arm64}/tests/shell/test_arm_spe_fork.sh        |  0
 4 files changed, 16 insertions(+), 2 deletions(-)
 rename tools/perf/{ => arch/arm64}/tests/shell/test_arm_callgraph_fp.sh (89%)
 rename tools/perf/{ => arch/arm64}/tests/shell/test_arm_spe.sh (100%)
 rename tools/perf/{ => arch/arm64}/tests/shell/test_arm_spe_fork.sh (100%)

diff --git a/tools/perf/arch/arm64/tests/Build b/tools/perf/arch/arm64/tests/Build
index d44c9de92d425c62..6c73720cb0ffa99d 100644
--- a/tools/perf/arch/arm64/tests/Build
+++ b/tools/perf/arch/arm64/tests/Build
@@ -3,3 +3,17 @@ perf-test-$(CONFIG_DWARF_UNWIND) += dwarf-unwind.o
 
 perf-test-y += arch-tests.o
 perf-test-y += cpuid-match.o
+
+ifdef SHELLCHECK
+  SHELL_TESTS := $(shell find shell -executable -type f -name '*.sh')
+  SHELL_TEST_LOGS := $(SHELL_TESTS:tests/shell/%=shell/%.shellcheck_log)
+else
+  SHELL_TESTS :=
+  SHELL_TEST_LOGS :=
+endif
+
+$(OUTPUT)%.shellcheck_log: %
+	$(call rule_mkdir)
+	$(Q)$(call echo-cmd,test)shellcheck -a -S warning "$<" > $@ || (cat $@ && rm $@ && false)
+
+perf-test-y += $(SHELL_TEST_LOGS)
diff --git a/tools/perf/tests/shell/test_arm_callgraph_fp.sh b/tools/perf/arch/arm64/tests/shell/test_arm_callgraph_fp.sh
similarity index 89%
rename from tools/perf/tests/shell/test_arm_callgraph_fp.sh
rename to tools/perf/arch/arm64/tests/shell/test_arm_callgraph_fp.sh
index 9caa36130175964e..f59ab293d67b9f9c 100755
--- a/tools/perf/tests/shell/test_arm_callgraph_fp.sh
+++ b/tools/perf/arch/arm64/tests/shell/test_arm_callgraph_fp.sh
@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: GPL-2.0
 
 shelldir=$(dirname "$0")
-# shellcheck source=lib/perf_has_symbol.sh
-. "${shelldir}"/lib/perf_has_symbol.sh
+# shellcheck source=../../../../tests/shell/lib/perf_has_symbol.sh
+. "${shelldir}"/../../../../tests/shell/lib/perf_has_symbol.sh
 
 if [ "$(uname -m)" != "aarch64" ]; then
 	exit 2
diff --git a/tools/perf/tests/shell/test_arm_spe.sh b/tools/perf/arch/arm64/tests/shell/test_arm_spe.sh
similarity index 100%
rename from tools/perf/tests/shell/test_arm_spe.sh
rename to tools/perf/arch/arm64/tests/shell/test_arm_spe.sh
diff --git a/tools/perf/tests/shell/test_arm_spe_fork.sh b/tools/perf/arch/arm64/tests/shell/test_arm_spe_fork.sh
similarity index 100%
rename from tools/perf/tests/shell/test_arm_spe_fork.sh
rename to tools/perf/arch/arm64/tests/shell/test_arm_spe_fork.sh
-- 
2.49.0.1164.gab81da1b16-goog

[PATCH 3/3] perf test: Add AMD IBS sw filter test

[Namhyung Kim]

The kernel v6.14 added 'swfilt' to support privilege filtering in
software so that IBS can be used by regular users.  Add a test case in
x86 to verify the behavior.

  $ sudo perf test -vv 'IBS software filter'
  111: AMD IBS software filtering:
  --- start ---
  test child forked, pid 178826
  check availability of IBS swfilt
  run perf record with modifier and swfilt
  [ perf record: Woken up 3 times to write data ]
  [ perf record: Captured and wrote 0.000 MB /dev/null ]
  [ perf record: Woken up 3 times to write data ]
  [ perf record: Captured and wrote 0.000 MB /dev/null ]
  [ perf record: Woken up 3 times to write data ]
  [ perf record: Captured and wrote 0.000 MB /dev/null ]
  [ perf record: Woken up 0 times to write data ]
  [ perf record: Captured and wrote 0.000 MB /dev/null ]
  check number of samples with swfilt
  [ perf record: Woken up 3 times to write data ]
  [ perf record: Captured and wrote 0.037 MB - ]
  [ perf record: Woken up 3 times to write data ]
  [ perf record: Captured and wrote 0.041 MB - ]
  ---- end(0) ----
  111: AMD IBS software filtering                                      : Ok

Cc: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
 tools/perf/arch/x86/tests/Build               |  1 +
 .../arch/x86/tests/shell/amd-ibs-swfilt.sh    | 67 +++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100755 tools/perf/arch/x86/tests/shell/amd-ibs-swfilt.sh

diff --git a/tools/perf/arch/x86/tests/Build b/tools/perf/arch/x86/tests/Build
index 5e00cbfd2d569bb9..317602fbc744316a 100644
--- a/tools/perf/arch/x86/tests/Build
+++ b/tools/perf/arch/x86/tests/Build
@@ -14,6 +14,7 @@ perf-test-y += amd-ibs-period.o
 
 ifdef SHELLCHECK
   SHELL_TESTS := gen-insn-x86-dat.sh
+  SHELL_TESTS += $(shell find shell -executable -type f -name '*.sh')
   SHELL_TEST_LOGS := $(SHELL_TESTS:%=%.shellcheck_log)
 else
   SHELL_TESTS :=
diff --git a/tools/perf/arch/x86/tests/shell/amd-ibs-swfilt.sh b/tools/perf/arch/x86/tests/shell/amd-ibs-swfilt.sh
new file mode 100755
index 0000000000000000..83937aa687ccd859
--- /dev/null
+++ b/tools/perf/arch/x86/tests/shell/amd-ibs-swfilt.sh
@@ -0,0 +1,67 @@
+#!/bin/sh
+# AMD IBS software filtering
+
+echo "check availability of IBS swfilt"
+
+# check if IBS PMU is available
+if [ ! -d /sys/bus/event_source/devices/ibs_op ]; then
+    echo "[SKIP] IBS PMU does not exist"
+    exit 2
+fi
+
+# check if IBS PMU has swfilt format
+if [ ! -f /sys/bus/event_source/devices/ibs_op/format/swfilt ]; then
+    echo "[SKIP] IBS PMU does not have swfilt"
+    exit 2
+fi
+
+echo "run perf record with modifier and swfilt"
+
+# setting any modifiers should fail
+perf record -B -e ibs_op//u -o /dev/null true 2> /dev/null
+if [ $? -eq 0 ]; then
+    echo "[FAIL] IBS PMU should not accept exclude_kernel"
+    exit 1
+fi
+
+# setting it with swfilt should be fine
+perf record -B -e ibs_op/swfilt/u -o /dev/null true
+if [ $? -ne 0 ]; then
+    echo "[FAIL] IBS op PMU cannot handle swfilt for exclude_kernel"
+    exit 1
+fi
+
+# setting it with swfilt=1 should be fine
+perf record -B -e ibs_op/swfilt=1/k -o /dev/null true
+if [ $? -ne 0 ]; then
+    echo "[FAIL] IBS op PMU cannot handle swfilt for exclude_user"
+    exit 1
+fi
+
+# check ibs_fetch PMU as well
+perf record -B -e ibs_fetch/swfilt/u -o /dev/null true
+if [ $? -ne 0 ]; then
+    echo "[FAIL] IBS fetch PMU cannot handle swfilt for exclude_kernel"
+    exit 1
+fi
+
+# check system wide recording
+perf record -aB --synth=no -e ibs_op/swfilt/k -o /dev/null true
+if [ $? -ne 0 ]; then
+    echo "[FAIL] IBS op PMU cannot handle swfilt in system-wide mode"
+    exit 1
+fi
+
+echo "check number of samples with swfilt"
+
+kernel_sample=$(perf record -e ibs_op/swfilt/u -o- true | perf script -i- -F misc | grep -c ^K)
+if [ ${kernel_sample} -ne 0 ]; then
+    echo "[FAIL] unexpected kernel samples: " ${kernel_sample}
+    exit 1
+fi
+
+user_sample=$(perf record -e ibs_fetch/swfilt/k -o- true | perf script -i- -F misc | grep -c ^U)
+if [ ${user_sample} -ne 0 ]; then
+    echo "[FAIL] unexpected user samples: " ${user_sample}
+    exit 1
+fi
-- 
2.49.0.1164.gab81da1b16-goog

Re: [PATCH 1/3] perf test: Support arch-specific shell tests

[Ian Rogers]

On Thu, May 22, 2025 at 10:10 AM Namhyung Kim <namhyung@kernel.org> wrote:
>
> This is a preparation for shell tests belong to an arch.

I keep repeating that I don't like arch and I think ideally we'd be
getting rid of the C arch tests. I just sent out a patch doing this
for 1 test:
https://lore.kernel.org/lkml/20250521165317.713463-2-irogers@google.com/
We should be able to make perf, tests, etc. dependent on a PMU rather
than an architecture. This means that running perf built for ARM will
be able to do things running on an instruction emulator on x86. It
means the tool, the kernel APIs, etc. are generic and new
architectures like RISC-V can test things. It means cross-platform
(record on 1 machine type, report on another) can work without
tripping over load bearing architecture ifdefs. It means that we
benefit from more testing on generic pieces of code on all
architectures - like sample parsing. We can always strcmp the PMU name
or the architecture at runtime.

Structure wise we could have:
tools/perf/pmu/ibs_op/tests/
tools/perf/pmu/ibs_op/tests/shell

It feels noisy compared to just having the shell test in
tools/perf/tests/shell skip when the PMU isn't present. There are also
things like library dependencies that aren't clear when we have >1
directory. I'd prefer if new testing followed the existing model
rather than this.

Thanks,
Ian

Re: [PATCH 1/3] perf test: Support arch-specific shell tests

[James Clark]

On 22/05/2025 6:10 pm, Namhyung Kim wrote:
> This is a preparation for shell tests belong to an arch.
> 
> Signed-off-by: Namhyung Kim <namhyung@kernel.org>
> ---
>   tools/perf/tests/builtin-test.c  |  3 ++
>   tools/perf/tests/tests-scripts.c | 73 ++++++++++++++++++++++++++++++++
>   tools/perf/tests/tests-scripts.h |  1 +
>   3 files changed, 77 insertions(+)
> 
> diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
> index 45d3d8b3317a7c0a..b9a86631b34cac90 100644
> --- a/tools/perf/tests/builtin-test.c
> +++ b/tools/perf/tests/builtin-test.c
> @@ -650,6 +650,7 @@ static struct test_suite **build_suites(void)
>   		generic_tests,
>   		arch_tests,
>   		NULL,
> +		NULL,
>   	};
>   	struct test_suite **result;
>   	struct test_suite *t;
> @@ -657,6 +658,8 @@ static struct test_suite **build_suites(void)
>   
>   	if (suites[2] == NULL)
>   		suites[2] = create_script_test_suites();
> +	if (suites[3] == NULL)
> +		suites[3] = create_script_test_suites_arch();
>   
>   #define for_each_suite(suite)						\
>   	for (size_t i = 0, j = 0; i < ARRAY_SIZE(suites); i++, j = 0)	\
> diff --git a/tools/perf/tests/tests-scripts.c b/tools/perf/tests/tests-scripts.c
> index 1d5759d08141749d..e6e6fab0c3c4bb8c 100644
> --- a/tools/perf/tests/tests-scripts.c
> +++ b/tools/perf/tests/tests-scripts.c
> @@ -24,6 +24,7 @@
>   #include "string2.h"
>   #include "symbol.h"
>   #include "tests.h"
> +#include "util/env.h"
>   #include "util/rlimit.h"
>   #include "util/util.h"
>   
> @@ -75,6 +76,52 @@ static int shell_tests__dir_fd(void)
>   	return open(path, O_PATH);
>   }
>   
> +static int shell_tests__arch_dir_fd(void)
> +{
> +	struct stat st;
> +	const char *arch;
> +	char path[PATH_MAX], path2[PATH_MAX], *exec_path;
> +	int fd;
> +	char *p;
> +
> +	arch = perf_env__arch(NULL);
> +	if (arch == NULL)
> +		return -1;
> +
> +	scnprintf(path, sizeof(path), "./arch/%s/tests/shell", arch);
> +	fd = open(path, O_PATH);
> +	if (fd >= 0)
> +		return fd;
> +
> +	/* Use directory of executable */
> +	if (readlink("/proc/self/exe", path2, sizeof path2) < 0)
> +		return -1;
> +	/* Follow another level of symlink if there */
> +	if (lstat(path2, &st) == 0 && (st.st_mode & S_IFMT) == S_IFLNK) {
> +		scnprintf(path, sizeof(path), path2);
> +		if (readlink(path, path2, sizeof path2) < 0)
> +			return -1;
> +	}
> +	/* Get directory */
> +	p = strrchr(path2, '/');
> +	if (p)
> +		*p = 0;
> +	scnprintf(path, sizeof(path), "%s/arch/%s/tests/shell", path2, arch);
> +	fd = open(path, O_PATH);
> +	if (fd >= 0)
> +		return fd;
> +	scnprintf(path, sizeof(path), "%s/source/arch/%s/tests/shell", path2, arch);
> +	fd = open(path, O_PATH);
> +	if (fd >= 0)
> +		return fd;
> +
> +	/* Then installed path. */
> +	exec_path = get_argv_exec_path();
> +	scnprintf(path, sizeof(path), "%s/arch/%s/tests/shell", exec_path, arch);
> +	free(exec_path);
> +	return open(path, O_PATH);
> +}
> +
>   static char *shell_test__description(int dir_fd, const char *name)
>   {
>   	struct io io;
> @@ -291,3 +338,29 @@ struct test_suite **create_script_test_suites(void)
>   		close(dir_fd);
>   	return result;
>   }
> +
> +struct test_suite **create_script_test_suites_arch(void)
> +{
> +	struct test_suite **result = NULL, **result_tmp;
> +	size_t result_sz = 0;
> +	int dir_fd = shell_tests__arch_dir_fd(); /* Walk dir */
> +
> +	/*
> +	 * Append scripts if fd is good, otherwise return a NULL terminated zero
> +	 * length array.
> +	 */
> +	if (dir_fd >= 0)
> +		append_scripts_in_dir(dir_fd, &result, &result_sz);
> +
> +	result_tmp = realloc(result, (result_sz + 1) * sizeof(*result_tmp));
> +	if (result_tmp == NULL) {
> +		pr_err("Out of memory while building script test suite list\n");
> +		abort();
> +	}
> +	/* NULL terminate the test suite array. */
> +	result = result_tmp;
> +	result[result_sz] = NULL;
> +	if (dir_fd >= 0)
> +		close(dir_fd);
> +	return result;
> +}
> diff --git a/tools/perf/tests/tests-scripts.h b/tools/perf/tests/tests-scripts.h
> index b553ad26ea17642a..b03ee4a1ee63a25a 100644
> --- a/tools/perf/tests/tests-scripts.h
> +++ b/tools/perf/tests/tests-scripts.h
> @@ -5,5 +5,6 @@
>   #include "tests.h"
>   
>   struct test_suite **create_script_test_suites(void);
> +struct test_suite **create_script_test_suites_arch(void);
>   
>   #endif /* TESTS_SCRIPTS_H */

Reviewed-by: James Clark <james.clark@linaro.org>

Re: [PATCH 2/3] perf test: Move some ARM tests to arch/arm64/tests/shell

[James Clark]

On 22/05/2025 6:10 pm, Namhyung Kim wrote:
> The test_arm_callgraph_fp.sh checks with aarch64 so it should belong to
> there.  And IIRC ARM SPE is supported on 64-bit platforms so move the
> tests too.  But I'm not sure about coresight so left them.

Coresight is also supported on arm32 which is why 
tools/perf/arch/arm/util/cs-etm.c is built for both platforms. Having 
said that, I'm not sure if the tests pass on arm32 because we're not 
running them, but they should pass and if someone is running them this 
change could break that.

We could symlink "arch/arm64/tests/shell" to "arch/arm/tests/shell" or 
have a condition in the code that forces one folder for both. Or just 
continue with only arm64. Considering I haven't seen a test failure 
report from there and we've only really been adding tests for arm64 
stuff it could be reasonable.

> 
> Also please test it with shellcheck as I couldn't run it actually.
> 

Shellcheck is working on these scripts for me.

Reviewed-by: James Clark <james.clark@linaro.org>

> Cc: Leo Yan <leo.yan@arm.com>
> Cc: James Clark <james.clark@linaro.org>
> Signed-off-by: Namhyung Kim <namhyung@kernel.org>
> ---
>   tools/perf/arch/arm64/tests/Build                  | 14 ++++++++++++++
>   .../arm64}/tests/shell/test_arm_callgraph_fp.sh    |  4 ++--
>   .../{ => arch/arm64}/tests/shell/test_arm_spe.sh   |  0
>   .../arm64}/tests/shell/test_arm_spe_fork.sh        |  0
>   4 files changed, 16 insertions(+), 2 deletions(-)
>   rename tools/perf/{ => arch/arm64}/tests/shell/test_arm_callgraph_fp.sh (89%)
>   rename tools/perf/{ => arch/arm64}/tests/shell/test_arm_spe.sh (100%)
>   rename tools/perf/{ => arch/arm64}/tests/shell/test_arm_spe_fork.sh (100%)
> 
> diff --git a/tools/perf/arch/arm64/tests/Build b/tools/perf/arch/arm64/tests/Build
> index d44c9de92d425c62..6c73720cb0ffa99d 100644
> --- a/tools/perf/arch/arm64/tests/Build
> +++ b/tools/perf/arch/arm64/tests/Build
> @@ -3,3 +3,17 @@ perf-test-$(CONFIG_DWARF_UNWIND) += dwarf-unwind.o
>   
>   perf-test-y += arch-tests.o
>   perf-test-y += cpuid-match.o
> +
> +ifdef SHELLCHECK
> +  SHELL_TESTS := $(shell find shell -executable -type f -name '*.sh')
> +  SHELL_TEST_LOGS := $(SHELL_TESTS:tests/shell/%=shell/%.shellcheck_log)
> +else
> +  SHELL_TESTS :=
> +  SHELL_TEST_LOGS :=
> +endif
> +
> +$(OUTPUT)%.shellcheck_log: %
> +	$(call rule_mkdir)
> +	$(Q)$(call echo-cmd,test)shellcheck -a -S warning "$<" > $@ || (cat $@ && rm $@ && false)
> +
> +perf-test-y += $(SHELL_TEST_LOGS)
> diff --git a/tools/perf/tests/shell/test_arm_callgraph_fp.sh b/tools/perf/arch/arm64/tests/shell/test_arm_callgraph_fp.sh
> similarity index 89%
> rename from tools/perf/tests/shell/test_arm_callgraph_fp.sh
> rename to tools/perf/arch/arm64/tests/shell/test_arm_callgraph_fp.sh
> index 9caa36130175964e..f59ab293d67b9f9c 100755
> --- a/tools/perf/tests/shell/test_arm_callgraph_fp.sh
> +++ b/tools/perf/arch/arm64/tests/shell/test_arm_callgraph_fp.sh
> @@ -3,8 +3,8 @@
>   # SPDX-License-Identifier: GPL-2.0
>   
>   shelldir=$(dirname "$0")
> -# shellcheck source=lib/perf_has_symbol.sh
> -. "${shelldir}"/lib/perf_has_symbol.sh
> +# shellcheck source=../../../../tests/shell/lib/perf_has_symbol.sh
> +. "${shelldir}"/../../../../tests/shell/lib/perf_has_symbol.sh
>   
>   if [ "$(uname -m)" != "aarch64" ]; then
>   	exit 2
> diff --git a/tools/perf/tests/shell/test_arm_spe.sh b/tools/perf/arch/arm64/tests/shell/test_arm_spe.sh
> similarity index 100%
> rename from tools/perf/tests/shell/test_arm_spe.sh
> rename to tools/perf/arch/arm64/tests/shell/test_arm_spe.sh
> diff --git a/tools/perf/tests/shell/test_arm_spe_fork.sh b/tools/perf/arch/arm64/tests/shell/test_arm_spe_fork.sh
> similarity index 100%
> rename from tools/perf/tests/shell/test_arm_spe_fork.sh
> rename to tools/perf/arch/arm64/tests/shell/test_arm_spe_fork.sh

Re: [PATCH 1/3] perf test: Support arch-specific shell tests

[James Clark]

On 22/05/2025 9:09 pm, Ian Rogers wrote:
> On Thu, May 22, 2025 at 10:10 AM Namhyung Kim <namhyung@kernel.org> wrote:
>>
>> This is a preparation for shell tests belong to an arch.
> 
> I keep repeating that I don't like arch and I think ideally we'd be
> getting rid of the C arch tests. I just sent out a patch doing this
> for 1 test:
> https://lore.kernel.org/lkml/20250521165317.713463-2-irogers@google.com/
> We should be able to make perf, tests, etc. dependent on a PMU rather
> than an architecture. This means that running perf built for ARM will
> be able to do things running on an instruction emulator on x86. It

In this case for Arm SPE and Coresight you can only generate trace by 
running on a full model or a real CPU, so I'm not sure if we could ever 
get close to running on just an emulator.

> means the tool, the kernel APIs, etc. are generic and new
> architectures like RISC-V can test things. It means cross-platform
> (record on 1 machine type, report on another) can work without
> tripping over load bearing architecture ifdefs. It means that we

I have thought about adding some generic decoding side tests for SPE and 
Coresight, but couldn't really get past the fact that you need to put 
the trace dump _and_ the binaries traced into the git repo. Not only 
would this benefit testing on other arches like you say, but it would 
also lock down that decoding of a known file doesn't regress which we 
can't currently do by generating new trace every time the test runs.

If we ever added this they would be separate tests though so they could 
go in the top level folder, where the ones in the arch folder would 
continue to do record and decode. Maybe naming the folders by PMU could 
work, but you could also have both PMU name and arch name folders like:

Recording/requires hardware:

   tools/perf/arch/arm64/tests/shell/cs_etm/

Cross platform decode tests:

   tools/perf/tests/shell/cs_etm/

Which would mirror how the source files are currently laid out:

  tools/perf/arch/arm/util/cs-etm.c
  tools/perf/util/cs-etm.c

Thanks
James

> benefit from more testing on generic pieces of code on all
> architectures - like sample parsing. We can always strcmp the PMU name
> or the architecture at runtime.
> 
> Structure wise we could have:
> tools/perf/pmu/ibs_op/tests/
> tools/perf/pmu/ibs_op/tests/shell
> 
> It feels noisy compared to just having the shell test in
> tools/perf/tests/shell skip when the PMU isn't present. There are also
> things like library dependencies that aren't clear when we have >1
> directory. I'd prefer if new testing followed the existing model
> rather than this.
> 
> Thanks,
> Ian
> 

Re: [PATCH 1/3] perf test: Support arch-specific shell tests

[Arnaldo Carvalho de Melo]

On Fri, May 23, 2025 at 11:48:26AM +0100, James Clark wrote:
> On 22/05/2025 9:09 pm, Ian Rogers wrote:
> > On Thu, May 22, 2025 at 10:10 AM Namhyung Kim <namhyung@kernel.org> wrote:
> > > This is a preparation for shell tests belong to an arch.

> > I keep repeating that I don't like arch and I think ideally we'd be
> > getting rid of the C arch tests. I just sent out a patch doing this
> > for 1 test:
> > https://lore.kernel.org/lkml/20250521165317.713463-2-irogers@google.com/
> > We should be able to make perf, tests, etc. dependent on a PMU rather
> > than an architecture. This means that running perf built for ARM will
> > be able to do things running on an instruction emulator on x86. It
 
> In this case for Arm SPE and Coresight you can only generate trace by
> running on a full model or a real CPU, so I'm not sure if we could ever get
> close to running on just an emulator.
 
> > means the tool, the kernel APIs, etc. are generic and new
> > architectures like RISC-V can test things. It means cross-platform
> > (record on 1 machine type, report on another) can work without
> > tripping over load bearing architecture ifdefs. It means that we
 
> I have thought about adding some generic decoding side tests for SPE and
> Coresight, but couldn't really get past the fact that you need to put the
> trace dump _and_ the binaries traced into the git repo.

So, we could have some .perfconfig setting that states the user wants to
auto-download the tracefiles if not present locally.

If not available locally and not explicitely authorized via
"test.coresight_traces_download=yes" or some more suitable name, then it
would skip the test and show as the reason the lack of needed trace
files, with a hint about how to enable it, something like:

176: Coresight hw trace decoding      : Skip   (Enable fetching via 'perf test test.coresight_trace_download=yes')

We are already auto-downloading debuginfo files (not the whole packages,
just for the needed build-id) in some cases, like:

root@x1:~# pahole --running_kernel_vmlinux
/usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
root@x1:~#
root@x1:~# time perf probe -L icmp_rcv > /dev/null

real	0m2.046s
user	0m1.550s
sys	0m0.486s
root@x1:~#

But if I move that file:

root@x1:~# pahole --running_kernel_vmlinux
/usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
root@x1:~# mv /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux ~
root@x1:~# pahole --running_kernel_vmlinux
vmlinux
root@x1:~#

Oops, it searches the current directory too ;)

root@x1:~# mkdir hideout
root@x1:~# mv vmlinux hideout/
root@x1:~# pahole --running_kernel_vmlinux
pahole: couldn't find a vmlinux that matches the running kernel
HINT: Maybe you're inside a container or missing a debuginfo package?
root@x1:~#

Now:

root@x1:~# time perf probe -L icmp_rcv | head

Takes a while, in other term:

root@x1:~# ps ax|grep 'perf probe' | grep -v grep
1755681 pts/13   S+     0:00 perf probe -L icmp_rcv
root@x1:~# 
root@x1:~# perf trace -p 1755681 --summary sleep 10s

 Summary of events:

 perf (1755681), 4756 events, 100.0%

   syscall            calls  errors  total       min       avg       max       stddev
                                     (msec)    (msec)    (msec)    (msec)        (%)
   --------------- --------  ------ -------- --------- --------- ---------     ------
   poll                 606      0  9645.195     0.000    15.916   757.894     16.65%
   write                580      0    17.904     0.004     0.031     0.237      3.41%
   recvfrom            1191    374     8.380     0.001     0.007     0.083      3.58%
   sendto                 1      0     0.026     0.026     0.026     0.026      0.00%


root@x1:~#

root@x1:~# perf trace -p 1755681 -e write --max-stack 16 --max-events 1
     0.000 ( 0.103 ms): perf/1755681 write(fd: 5, buf: \219*\0\26\230\1\11\136\4\0\0\192\0\5'g\0\0\150\1pid\0p\1487\172h\0\0\1, count: 10157) = 10157
                                       syscall_exit_to_user_mode_prepare ([kernel.kallsyms])
                                       syscall_exit_to_user_mode_prepare ([kernel.kallsyms])
                                       syscall_exit_to_user_mode ([kernel.kallsyms])
                                       do_syscall_64 ([kernel.kallsyms])
                                       entry_SYSCALL_64_after_hwframe ([kernel.kallsyms])
                                       __GI___libc_write (/usr/lib64/libc.so.6)
                                       chop_write.lto_priv.0 (/usr/lib64/libcurl.so.4.8.0)
                                       <invalid> (inlined)
                                       inflate_stream (/usr/lib64/libcurl.so.4.8.0)
                                       <invalid> (inlined)
                                       cw_download_write (/usr/lib64/libcurl.so.4.8.0)
                                       <invalid> (inlined)
                                       <invalid> (inlined)
                                       Curl_readwrite (/usr/lib64/libcurl.so.4.8.0)
                                       multi_runsingle (/usr/lib64/libcurl.so.4.8.0)
                                       curl_multi_perform (/usr/lib64/libcurl.so.4.8.0)
                                       perform_queries (/usr/lib64/libdebuginfod-0.192.so)
                                       debuginfod_query_server_by_buildid (/usr/lib64/libdebuginfod-0.192.so)
                                       open_debuginfo (/home/acme/bin/perf)
                                       __show_line_range (/home/acme/bin/perf)
                                       show_line_range (/home/acme/bin/perf)
                                       __cmd_probe (/home/acme/bin/perf)
root@x1:~#

Those <invalid> ones...

unwind: curl_multi_perform:ip = 0x7f6e7ab33464 (0x4e464)
unwind: access_mem addr 0x7ffe829cec08, val 2b324d40, offset 1480
unwind: access_mem addr 0x7ffe829cec30, val 7ffe829d01f0, offset 1520
unwind: access_mem addr 0x7ffe829cec10, val 2b32b9e4, offset 1488
unwind: access_mem addr 0x7ffe829cec18, val ffffffff, offset 1496
unwind: access_mem addr 0x7ffe829cec20, val 0, offset 1504
unwind: access_mem addr 0x7ffe829cec28, val 0, offset 1512
unwind: access_mem addr 0x7ffe829cec38, val 7f6e7b326c5f, offset 1528
unwind: perform_queries:ip = 0x7f6e7b324b2d (0x5b2d)
unwind: access_mem addr 0x7ffe829d01c8, val 7ffe829d0260, offset 7048
unwind: access_mem addr 0x7ffe829d01f0, val 7ffe829d0320, offset 7088
unwind: access_mem addr 0x7ffe829d01d0, val 0, offset 7056
unwind: access_mem addr 0x7ffe829d01d8, val 0, offset 7064
unwind: access_mem addr 0x7ffe829d01e0, val 2b2db8c0, offset 7072
unwind: access_mem addr 0x7ffe829d01e8, val 2b324d40, offset 7080
unwind: access_mem addr 0x7ffe829d01f8, val 6792ef, offset 7096
unwind: debuginfod_query_server_by_buildid:ip = 0x7f6e7b326c5e (0x7c5e)
unwind: access_mem addr 0x7ffe829d02f8, val 2b2dace0, offset 7352
unwind: access_mem addr 0x7ffe829d0320, val 7ffe829d05b0, offset 7392
unwind: access_mem addr 0x7ffe829d0300, val 0, offset 7360
unwind: access_mem addr 0x7ffe829d0308, val 2b2dace0, offset 7368
unwind: access_mem addr 0x7ffe829d0310, val 0, offset 7376
unwind: access_mem addr 0x7ffe829d0318, val 0, offset 7384
unwind: access_mem addr 0x7ffe829d0328, val 67aaff, offset 7400
unwind: open_debuginfo:ip = 0x6792ee (0x2792ee)
unwind: access_mem addr 0x7ffe829d0588, val 0, offset 8008
unwind: access_mem addr 0x7ffe829d05b0, val 7ffe829d0620, offset 8048
unwind: access_mem addr 0x7ffe829d0590, val 0, offset 8016
unwind: access_mem addr 0x7ffe829d0598, val 2b2dace0, offset 8024
unwind: access_mem addr 0x7ffe829d05a0, val 0, offset 8032
unwind: access_mem addr 0x7ffe829d05a8, val 0, offset 8040
unwind: access_mem addr 0x7ffe829d05b8, val 67b0ea, offset 8056
unwind: __show_line_range:ip = 0x67aafe (0x27aafe)
unwind: access_mem addr 0x7ffe829d05f8, val 0, offset 8120
unwind: access_mem addr 0x7ffe829d0620, val 7ffe829d1180, offset 8160
unwind: access_mem addr 0x7ffe829d0600, val 7ffe829d06a0, offset 8128
unwind: access_mem addr 0x7ffe829d0608, val 8, offset 8136
unwind: access_mem addr 0x7ffe829d0610, val 2b2d74d0, offset 8144
unwind: access_mem addr 0x7ffe829d0618, val 6df140, offset 8152
unwind: access_mem addr 0x7ffe829d0628, val 4979b3, offset 8168
unwind: show_line_range:ip = 0x67b0e9 (0x27b0e9)
unwind: no map for 7ffe829d1158
unwind: access_mem 0x7ffe829d1158 not inside range 0x7ffe829ce640-0x7ffe829d0640
unwind: no map for 7ffe829d1180
unwind: access_mem 0x7ffe829d1180 not inside range 0x7ffe829ce640-0x7ffe829d0640
unwind: no map for 7ffe829d1160
unwind: access_mem 0x7ffe829d1160 not inside range 0x7ffe829ce640-0x7ffe829d0640
unwind: no map for 7ffe829d1168
unwind: access_mem 0x7ffe829d1168 not inside range 0x7ffe829ce640-0x7ffe829d0640
unwind: no map for 7ffe829d1170
unwind: access_mem 0x7ffe829d1170 not inside range 0x7ffe829ce640-0x7ffe829d0640
unwind: no map for 7ffe829d1178
unwind: access_mem 0x7ffe829d1178 not inside range 0x7ffe829ce640-0x7ffe829d0640
unwind: no map for 7ffe829d1188
unwind: access_mem 0x7ffe829d1188 not inside range 0x7ffe829ce640-0x7ffe829d0640
unwind: __cmd_probe:ip = 0x4979b2 (0x979b2)
     0.000 ( 0.102 ms): perf/1755681 write(fd: 5, buf: \197\10\17K\137\0\0\0\9ops\0\197\11#y\4\1\0\8\1\176+E\0\197\12\15]\0\0, count: 16384) = 16384
                                       syscall_exit_to_user_mode_prepare ([kernel.kallsyms])


Anyway, I'm digressing, I need to make this auto-download similar to
what I proposed above for the coresight traces needed by the tests you
think about adding for decoding in all platforms, which I encourage you
to do.

> Not only would this benefit testing on other arches like you say, but
> it would also lock down that decoding of a known file doesn't regress
> which we can't currently do by generating new trace every time the
> test runs.

Right.
 
> If we ever added this they would be separate tests though so they could go
> in the top level folder, where the ones in the arch folder would continue to
> do record and decode. Maybe naming the folders by PMU could work, but you
> could also have both PMU name and arch name folders like:
 
> Recording/requires hardware:
 
>   tools/perf/arch/arm64/tests/shell/cs_etm/
 
> Cross platform decode tests:
 
>   tools/perf/tests/shell/cs_etm/
 
> Which would mirror how the source files are currently laid out:
 
>  tools/perf/arch/arm/util/cs-etm.c
>  tools/perf/util/cs-etm.c

Yeah, I think we can experiment with this and take advantage of the
effort Namhyung already put into this, and then revisit later, after
trying this for a while.

Ah, finally:

root@x1:~# time perf probe -L icmp_rcv | head 
<icmp_rcv@/root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/source-d5d23b89-#usr#src#debug#kernel-6.13.9#linux-6.13.9-100.fc40.x86_64#net#ipv4#icmp.c:0>
      0  int icmp_rcv(struct sk_buff *skb)
         {
      2  	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
         	struct rtable *rt = skb_rtable(skb);
         	struct net *net = dev_net_rcu(rt->dst.dev);
         	struct icmphdr *icmph;
         
         	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
      8  		struct sec_path *sp = skb_sec_path(skb);

real	9m20.630s
user	0m5.427s
sys	0m2.127s
root@x1:~#

And now:

root@x1:~# pahole --running_kernel_vmlinux
/root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo
root@x1:~#

Also:

root@x1:~# file /root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo
/root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, BuildID[sha1]=aa3c82b4a13f9c0e0301bebb20fe958c4db6f362, with debug_info, not stripped
root@x1:~# file hideout/vmlinux
hideout/vmlinux: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, BuildID[sha1]=aa3c82b4a13f9c0e0301bebb20fe958c4db6f362, with debug_info, not stripped
root@x1:~#

root@x1:~# sha256sum /root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo hideout/vmlinux
6e998df9b235ed50ea4d7c6d997450cb7bd6691537e525f002630ae123bc0084  /root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo
6e998df9b235ed50ea4d7c6d997450cb7bd6691537e525f002630ae123bc0084  hideout/vmlinux
root@x1:~#

But:

root@x1:~# rpm -qf /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
kernel-debuginfo-6.13.9-100.fc40.x86_64
root@x1:~# rpm -V kernel-debuginfo-6.13.9-100.fc40.x86_64
missing     /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
root@x1:~#

Lets fix this:

root@x1:~# mv hideout/vmlinux /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
root@x1:~# rpm -V kernel-debuginfo-6.13.9-100.fc40.x86_64
root@x1:~#

Cheers,

- Arnaldo

Re: [PATCH 1/3] perf test: Support arch-specific shell tests

[Ian Rogers]

On Fri, May 23, 2025 at 3:48 AM James Clark <james.clark@linaro.org> wrote:
>
> On 22/05/2025 9:09 pm, Ian Rogers wrote:
> > On Thu, May 22, 2025 at 10:10 AM Namhyung Kim <namhyung@kernel.org> wrote:
> >>
> >> This is a preparation for shell tests belong to an arch.
> >
> > I keep repeating that I don't like arch and I think ideally we'd be
> > getting rid of the C arch tests. I just sent out a patch doing this
> > for 1 test:
> > https://lore.kernel.org/lkml/20250521165317.713463-2-irogers@google.com/
> > We should be able to make perf, tests, etc. dependent on a PMU rather
> > than an architecture. This means that running perf built for ARM will
> > be able to do things running on an instruction emulator on x86. It
>
> In this case for Arm SPE and Coresight you can only generate trace by
> running on a full model or a real CPU, so I'm not sure if we could ever
> get close to running on just an emulator.

Ok, that's the SPE and coresight PMUs, shouldn't the bulk of "perf
test" pass under an emulator and those tests skip when the PMUs aren't
there?

> > means the tool, the kernel APIs, etc. are generic and new
> > architectures like RISC-V can test things. It means cross-platform
> > (record on 1 machine type, report on another) can work without
> > tripping over load bearing architecture ifdefs. It means that we
>
> I have thought about adding some generic decoding side tests for SPE and
> Coresight, but couldn't really get past the fact that you need to put
> the trace dump _and_ the binaries traced into the git repo. Not only
> would this benefit testing on other arches like you say, but it would
> also lock down that decoding of a known file doesn't regress which we
> can't currently do by generating new trace every time the test runs.
>
> If we ever added this they would be separate tests though so they could
> go in the top level folder, where the ones in the arch folder would
> continue to do record and decode. Maybe naming the folders by PMU could
> work, but you could also have both PMU name and arch name folders like:
>
> Recording/requires hardware:
>
>    tools/perf/arch/arm64/tests/shell/cs_etm/
>
> Cross platform decode tests:
>
>    tools/perf/tests/shell/cs_etm/
>
> Which would mirror how the source files are currently laid out:
>
>   tools/perf/arch/arm/util/cs-etm.c
>   tools/perf/util/cs-etm.c

So this a digression, I don't necessarily disagree with it. Back to
adding arch-specific shell tests, we should be working hard to not
target work to a particular architecture. Making code, tests, etc.
specific to an architecture means someone, quite often myself, has to
come along later and undo the damage. Hybrid being a very large and
long case in point where >1 core PMU was hard coded to only work
within the tool for two specific PMUs. To be specific as to why
notions of architecture in the code base are actively harmful (off the
top of my head):

1) architecture isn't a specific enough term as PMUs are always
evolving so we need to support figuring this out, not just hiding the
affected code by an ifdef or build option;

2) lower and more problematic test coverage (will we shellcheck an ARM
test on x86? we're lowering the container coverage or requiring it to
be run on multiple architectures);

3) the tests need to be able to handle being on the architecture with
or without the PMU (such as under a hypervisor);
3.1) the without PMU case is a very frequent source of my patching
tests as most people test on a bare metal development machines and
forget continuous testing environments;
3.2) this kind of support can enable continuous testing efforts that
are run under an emulator. If a cloud has no ARM offering they can at
least emulator test, and SuSE have at least offered this. Google may
or may not run testing through emulators and tests need to handle it,
we have and do carry patches for this;

4) broken cross-platform support, such as with sample parsing where
decoding a sample is missing pieces when not done on x86 (come on,
this is terrible and we don't want more of it);

5) the tests do and should make use of common library code for things
like checking the perf event permissions, this code lives in
tools/perf/tests/shell/lib but the arch dependencies go where? We
shouldn't reinvent the shell testing environment per architecture.


I get that vendors would quite like things not to work on other
architectures, or for event parsing [1], testing, etc. to be in some
way crippled and require architecture specific reinvention. Why should
their engineers invest in infrastructure and testing that could
benefit someone else? Heck sometimes it feels like vendors may be
doing everything in their power via broken PMU names, event names,
etc. to make common infrastructure break when not run in their arch
specific way [2]? I see things differently and indeed when things go
wrong vendors argue that the generic code/behaviors needs to be
changed (I'm thinking of Mark Rutland on Apple PMU support and the
generic behavior changes made at that time that are now not carried
through leading to event parsing and display inconsistencies as well
as RISC-V needing to adopt a different event lookup strategy).

I want the perf tool to have its behavior be generic. The kernel has
an arch directory for understandable reasons, for the perf tool the
key thing is the PMU and we've been building out the infrastructure
for this. Should the perf tool have every event, metric or syscall
number for every architecture? For cross-platform work it doesn't seem
like a completely terrible idea. I'm sensitive to binary size [3] so
I'm not pushing on it.

Anyway, this change is unnecessary as the current skipping behavior
works and is better for the reasons enumerated above. This change gets
a very large nack from me. I'm supportive of more IBS coverage in the
rest of the series, but this change isn't required for that.

Thanks,
Ian

[1] https://lore.kernel.org/lkml/20250416045117.876775-1-irogers@google.com/
[2] https://web.git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git/tree/drivers/perf/arm_cspmu/arm_cspmu.c?h=perf-tools-next#n150
- please call it cpu_cycles to avoid ambiguity with the legacy event.
[3] https://lore.kernel.org/lkml/20250417230740.86048-1-irogers@google.com/



> Thanks
> James
>
> > benefit from more testing on generic pieces of code on all
> > architectures - like sample parsing. We can always strcmp the PMU name
> > or the architecture at runtime.
> >
> > Structure wise we could have:
> > tools/perf/pmu/ibs_op/tests/
> > tools/perf/pmu/ibs_op/tests/shell
> >
> > It feels noisy compared to just having the shell test in
> > tools/perf/tests/shell skip when the PMU isn't present. There are also
> > things like library dependencies that aren't clear when we have >1
> > directory. I'd prefer if new testing followed the existing model
> > rather than this.
> >
> > Thanks,
> > Ian
> >

Re: [PATCH 1/3] perf test: Support arch-specific shell tests

[Namhyung Kim]

Hi Ian,

On Fri, May 23, 2025 at 10:54:02AM -0700, Ian Rogers wrote:
> On Fri, May 23, 2025 at 3:48 AM James Clark <james.clark@linaro.org> wrote:
> >
> > On 22/05/2025 9:09 pm, Ian Rogers wrote:
> > > On Thu, May 22, 2025 at 10:10 AM Namhyung Kim <namhyung@kernel.org> wrote:
> > >>
> > >> This is a preparation for shell tests belong to an arch.
> > >
> > > I keep repeating that I don't like arch and I think ideally we'd be
> > > getting rid of the C arch tests. I just sent out a patch doing this
> > > for 1 test:
> > > https://lore.kernel.org/lkml/20250521165317.713463-2-irogers@google.com/
> > > We should be able to make perf, tests, etc. dependent on a PMU rather
> > > than an architecture. This means that running perf built for ARM will
> > > be able to do things running on an instruction emulator on x86. It
> >
> > In this case for Arm SPE and Coresight you can only generate trace by
> > running on a full model or a real CPU, so I'm not sure if we could ever
> > get close to running on just an emulator.
> 
> Ok, that's the SPE and coresight PMUs, shouldn't the bulk of "perf
> test" pass under an emulator and those tests skip when the PMUs aren't
> there?
> 
> > > means the tool, the kernel APIs, etc. are generic and new
> > > architectures like RISC-V can test things. It means cross-platform
> > > (record on 1 machine type, report on another) can work without
> > > tripping over load bearing architecture ifdefs. It means that we
> >
> > I have thought about adding some generic decoding side tests for SPE and
> > Coresight, but couldn't really get past the fact that you need to put
> > the trace dump _and_ the binaries traced into the git repo. Not only
> > would this benefit testing on other arches like you say, but it would
> > also lock down that decoding of a known file doesn't regress which we
> > can't currently do by generating new trace every time the test runs.
> >
> > If we ever added this they would be separate tests though so they could
> > go in the top level folder, where the ones in the arch folder would
> > continue to do record and decode. Maybe naming the folders by PMU could
> > work, but you could also have both PMU name and arch name folders like:
> >
> > Recording/requires hardware:
> >
> >    tools/perf/arch/arm64/tests/shell/cs_etm/
> >
> > Cross platform decode tests:
> >
> >    tools/perf/tests/shell/cs_etm/
> >
> > Which would mirror how the source files are currently laid out:
> >
> >   tools/perf/arch/arm/util/cs-etm.c
> >   tools/perf/util/cs-etm.c
> 
> So this a digression, I don't necessarily disagree with it. Back to
> adding arch-specific shell tests, we should be working hard to not
> target work to a particular architecture. Making code, tests, etc.
> specific to an architecture means someone, quite often myself, has to
> come along later and undo the damage. Hybrid being a very large and
> long case in point where >1 core PMU was hard coded to only work
> within the tool for two specific PMUs. To be specific as to why
> notions of architecture in the code base are actively harmful (off the
> top of my head):
> 
> 1) architecture isn't a specific enough term as PMUs are always
> evolving so we need to support figuring this out, not just hiding the
> affected code by an ifdef or build option;
> 
> 2) lower and more problematic test coverage (will we shellcheck an ARM
> test on x86? we're lowering the container coverage or requiring it to
> be run on multiple architectures);
> 
> 3) the tests need to be able to handle being on the architecture with
> or without the PMU (such as under a hypervisor);
> 3.1) the without PMU case is a very frequent source of my patching
> tests as most people test on a bare metal development machines and
> forget continuous testing environments;
> 3.2) this kind of support can enable continuous testing efforts that
> are run under an emulator. If a cloud has no ARM offering they can at
> least emulator test, and SuSE have at least offered this. Google may
> or may not run testing through emulators and tests need to handle it,
> we have and do carry patches for this;
> 
> 4) broken cross-platform support, such as with sample parsing where
> decoding a sample is missing pieces when not done on x86 (come on,
> this is terrible and we don't want more of it);
> 
> 5) the tests do and should make use of common library code for things
> like checking the perf event permissions, this code lives in
> tools/perf/tests/shell/lib but the arch dependencies go where? We
> shouldn't reinvent the shell testing environment per architecture.
> 
> 
> I get that vendors would quite like things not to work on other
> architectures, or for event parsing [1], testing, etc. to be in some
> way crippled and require architecture specific reinvention. Why should
> their engineers invest in infrastructure and testing that could
> benefit someone else? Heck sometimes it feels like vendors may be
> doing everything in their power via broken PMU names, event names,
> etc. to make common infrastructure break when not run in their arch
> specific way [2]? I see things differently and indeed when things go
> wrong vendors argue that the generic code/behaviors needs to be
> changed (I'm thinking of Mark Rutland on Apple PMU support and the
> generic behavior changes made at that time that are now not carried
> through leading to event parsing and display inconsistencies as well
> as RISC-V needing to adopt a different event lookup strategy).
> 
> I want the perf tool to have its behavior be generic. The kernel has
> an arch directory for understandable reasons, for the perf tool the
> key thing is the PMU and we've been building out the infrastructure
> for this. Should the perf tool have every event, metric or syscall
> number for every architecture? For cross-platform work it doesn't seem
> like a completely terrible idea. I'm sensitive to binary size [3] so
> I'm not pushing on it.
> 
> Anyway, this change is unnecessary as the current skipping behavior
> works and is better for the reasons enumerated above. This change gets
> a very large nack from me. I'm supportive of more IBS coverage in the
> rest of the series, but this change isn't required for that.

Thanks for the review.  I understand your concern and it'd be nice to
increase the cross-platform coverage.  I don't know if AMD IBS is
available in an emulator but it should be fine to have this test in the
generic place.

Thanks,
Namhyung

> 
> [1] https://lore.kernel.org/lkml/20250416045117.876775-1-irogers@google.com/
> [2] https://web.git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git/tree/drivers/perf/arm_cspmu/arm_cspmu.c?h=perf-tools-next#n150
> - please call it cpu_cycles to avoid ambiguity with the legacy event.
> [3] https://lore.kernel.org/lkml/20250417230740.86048-1-irogers@google.com/
> 
> 
> 
> > Thanks
> > James
> >
> > > benefit from more testing on generic pieces of code on all
> > > architectures - like sample parsing. We can always strcmp the PMU name
> > > or the architecture at runtime.
> > >
> > > Structure wise we could have:
> > > tools/perf/pmu/ibs_op/tests/
> > > tools/perf/pmu/ibs_op/tests/shell
> > >
> > > It feels noisy compared to just having the shell test in
> > > tools/perf/tests/shell skip when the PMU isn't present. There are also
> > > things like library dependencies that aren't clear when we have >1
> > > directory. I'd prefer if new testing followed the existing model
> > > rather than this.
> > >
> > > Thanks,
> > > Ian
> > >

Re: [PATCH 1/3] perf test: Support arch-specific shell tests

[Namhyung Kim]

On Fri, May 23, 2025 at 01:50:57PM -0300, Arnaldo Carvalho de Melo wrote:
> On Fri, May 23, 2025 at 11:48:26AM +0100, James Clark wrote:
> > On 22/05/2025 9:09 pm, Ian Rogers wrote:
> > > On Thu, May 22, 2025 at 10:10 AM Namhyung Kim <namhyung@kernel.org> wrote:
> > > > This is a preparation for shell tests belong to an arch.
> 
> > > I keep repeating that I don't like arch and I think ideally we'd be
> > > getting rid of the C arch tests. I just sent out a patch doing this
> > > for 1 test:
> > > https://lore.kernel.org/lkml/20250521165317.713463-2-irogers@google.com/
> > > We should be able to make perf, tests, etc. dependent on a PMU rather
> > > than an architecture. This means that running perf built for ARM will
> > > be able to do things running on an instruction emulator on x86. It
>  
> > In this case for Arm SPE and Coresight you can only generate trace by
> > running on a full model or a real CPU, so I'm not sure if we could ever get
> > close to running on just an emulator.
>  
> > > means the tool, the kernel APIs, etc. are generic and new
> > > architectures like RISC-V can test things. It means cross-platform
> > > (record on 1 machine type, report on another) can work without
> > > tripping over load bearing architecture ifdefs. It means that we
>  
> > I have thought about adding some generic decoding side tests for SPE and
> > Coresight, but couldn't really get past the fact that you need to put the
> > trace dump _and_ the binaries traced into the git repo.
> 
> So, we could have some .perfconfig setting that states the user wants to
> auto-download the tracefiles if not present locally.
> 
> If not available locally and not explicitely authorized via
> "test.coresight_traces_download=yes" or some more suitable name, then it
> would skip the test and show as the reason the lack of needed trace
> files, with a hint about how to enable it, something like:
> 
> 176: Coresight hw trace decoding      : Skip   (Enable fetching via 'perf test test.coresight_trace_download=yes')
> 
> We are already auto-downloading debuginfo files (not the whole packages,
> just for the needed build-id) in some cases, like:
> 
> root@x1:~# pahole --running_kernel_vmlinux
> /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
> root@x1:~#
> root@x1:~# time perf probe -L icmp_rcv > /dev/null
> 
> real	0m2.046s
> user	0m1.550s
> sys	0m0.486s
> root@x1:~#
> 
> But if I move that file:
> 
> root@x1:~# pahole --running_kernel_vmlinux
> /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
> root@x1:~# mv /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux ~
> root@x1:~# pahole --running_kernel_vmlinux
> vmlinux
> root@x1:~#
> 
> Oops, it searches the current directory too ;)
> 
> root@x1:~# mkdir hideout
> root@x1:~# mv vmlinux hideout/
> root@x1:~# pahole --running_kernel_vmlinux
> pahole: couldn't find a vmlinux that matches the running kernel
> HINT: Maybe you're inside a container or missing a debuginfo package?
> root@x1:~#
> 
> Now:
> 
> root@x1:~# time perf probe -L icmp_rcv | head
> 
> Takes a while, in other term:
> 
> root@x1:~# ps ax|grep 'perf probe' | grep -v grep
> 1755681 pts/13   S+     0:00 perf probe -L icmp_rcv
> root@x1:~# 
> root@x1:~# perf trace -p 1755681 --summary sleep 10s
> 
>  Summary of events:
> 
>  perf (1755681), 4756 events, 100.0%
> 
>    syscall            calls  errors  total       min       avg       max       stddev
>                                      (msec)    (msec)    (msec)    (msec)        (%)
>    --------------- --------  ------ -------- --------- --------- ---------     ------
>    poll                 606      0  9645.195     0.000    15.916   757.894     16.65%
>    write                580      0    17.904     0.004     0.031     0.237      3.41%
>    recvfrom            1191    374     8.380     0.001     0.007     0.083      3.58%
>    sendto                 1      0     0.026     0.026     0.026     0.026      0.00%
> 
> 
> root@x1:~#
> 
> root@x1:~# perf trace -p 1755681 -e write --max-stack 16 --max-events 1
>      0.000 ( 0.103 ms): perf/1755681 write(fd: 5, buf: \219*\0\26\230\1\11\136\4\0\0\192\0\5'g\0\0\150\1pid\0p\1487\172h\0\0\1, count: 10157) = 10157
>                                        syscall_exit_to_user_mode_prepare ([kernel.kallsyms])
>                                        syscall_exit_to_user_mode_prepare ([kernel.kallsyms])
>                                        syscall_exit_to_user_mode ([kernel.kallsyms])
>                                        do_syscall_64 ([kernel.kallsyms])
>                                        entry_SYSCALL_64_after_hwframe ([kernel.kallsyms])
>                                        __GI___libc_write (/usr/lib64/libc.so.6)
>                                        chop_write.lto_priv.0 (/usr/lib64/libcurl.so.4.8.0)
>                                        <invalid> (inlined)
>                                        inflate_stream (/usr/lib64/libcurl.so.4.8.0)
>                                        <invalid> (inlined)
>                                        cw_download_write (/usr/lib64/libcurl.so.4.8.0)
>                                        <invalid> (inlined)
>                                        <invalid> (inlined)
>                                        Curl_readwrite (/usr/lib64/libcurl.so.4.8.0)
>                                        multi_runsingle (/usr/lib64/libcurl.so.4.8.0)
>                                        curl_multi_perform (/usr/lib64/libcurl.so.4.8.0)
>                                        perform_queries (/usr/lib64/libdebuginfod-0.192.so)
>                                        debuginfod_query_server_by_buildid (/usr/lib64/libdebuginfod-0.192.so)
>                                        open_debuginfo (/home/acme/bin/perf)
>                                        __show_line_range (/home/acme/bin/perf)
>                                        show_line_range (/home/acme/bin/perf)
>                                        __cmd_probe (/home/acme/bin/perf)
> root@x1:~#
> 
> Those <invalid> ones...
> 
> unwind: curl_multi_perform:ip = 0x7f6e7ab33464 (0x4e464)
> unwind: access_mem addr 0x7ffe829cec08, val 2b324d40, offset 1480
> unwind: access_mem addr 0x7ffe829cec30, val 7ffe829d01f0, offset 1520
> unwind: access_mem addr 0x7ffe829cec10, val 2b32b9e4, offset 1488
> unwind: access_mem addr 0x7ffe829cec18, val ffffffff, offset 1496
> unwind: access_mem addr 0x7ffe829cec20, val 0, offset 1504
> unwind: access_mem addr 0x7ffe829cec28, val 0, offset 1512
> unwind: access_mem addr 0x7ffe829cec38, val 7f6e7b326c5f, offset 1528
> unwind: perform_queries:ip = 0x7f6e7b324b2d (0x5b2d)
> unwind: access_mem addr 0x7ffe829d01c8, val 7ffe829d0260, offset 7048
> unwind: access_mem addr 0x7ffe829d01f0, val 7ffe829d0320, offset 7088
> unwind: access_mem addr 0x7ffe829d01d0, val 0, offset 7056
> unwind: access_mem addr 0x7ffe829d01d8, val 0, offset 7064
> unwind: access_mem addr 0x7ffe829d01e0, val 2b2db8c0, offset 7072
> unwind: access_mem addr 0x7ffe829d01e8, val 2b324d40, offset 7080
> unwind: access_mem addr 0x7ffe829d01f8, val 6792ef, offset 7096
> unwind: debuginfod_query_server_by_buildid:ip = 0x7f6e7b326c5e (0x7c5e)
> unwind: access_mem addr 0x7ffe829d02f8, val 2b2dace0, offset 7352
> unwind: access_mem addr 0x7ffe829d0320, val 7ffe829d05b0, offset 7392
> unwind: access_mem addr 0x7ffe829d0300, val 0, offset 7360
> unwind: access_mem addr 0x7ffe829d0308, val 2b2dace0, offset 7368
> unwind: access_mem addr 0x7ffe829d0310, val 0, offset 7376
> unwind: access_mem addr 0x7ffe829d0318, val 0, offset 7384
> unwind: access_mem addr 0x7ffe829d0328, val 67aaff, offset 7400
> unwind: open_debuginfo:ip = 0x6792ee (0x2792ee)
> unwind: access_mem addr 0x7ffe829d0588, val 0, offset 8008
> unwind: access_mem addr 0x7ffe829d05b0, val 7ffe829d0620, offset 8048
> unwind: access_mem addr 0x7ffe829d0590, val 0, offset 8016
> unwind: access_mem addr 0x7ffe829d0598, val 2b2dace0, offset 8024
> unwind: access_mem addr 0x7ffe829d05a0, val 0, offset 8032
> unwind: access_mem addr 0x7ffe829d05a8, val 0, offset 8040
> unwind: access_mem addr 0x7ffe829d05b8, val 67b0ea, offset 8056
> unwind: __show_line_range:ip = 0x67aafe (0x27aafe)
> unwind: access_mem addr 0x7ffe829d05f8, val 0, offset 8120
> unwind: access_mem addr 0x7ffe829d0620, val 7ffe829d1180, offset 8160
> unwind: access_mem addr 0x7ffe829d0600, val 7ffe829d06a0, offset 8128
> unwind: access_mem addr 0x7ffe829d0608, val 8, offset 8136
> unwind: access_mem addr 0x7ffe829d0610, val 2b2d74d0, offset 8144
> unwind: access_mem addr 0x7ffe829d0618, val 6df140, offset 8152
> unwind: access_mem addr 0x7ffe829d0628, val 4979b3, offset 8168
> unwind: show_line_range:ip = 0x67b0e9 (0x27b0e9)
> unwind: no map for 7ffe829d1158
> unwind: access_mem 0x7ffe829d1158 not inside range 0x7ffe829ce640-0x7ffe829d0640
> unwind: no map for 7ffe829d1180
> unwind: access_mem 0x7ffe829d1180 not inside range 0x7ffe829ce640-0x7ffe829d0640
> unwind: no map for 7ffe829d1160
> unwind: access_mem 0x7ffe829d1160 not inside range 0x7ffe829ce640-0x7ffe829d0640
> unwind: no map for 7ffe829d1168
> unwind: access_mem 0x7ffe829d1168 not inside range 0x7ffe829ce640-0x7ffe829d0640
> unwind: no map for 7ffe829d1170
> unwind: access_mem 0x7ffe829d1170 not inside range 0x7ffe829ce640-0x7ffe829d0640
> unwind: no map for 7ffe829d1178
> unwind: access_mem 0x7ffe829d1178 not inside range 0x7ffe829ce640-0x7ffe829d0640
> unwind: no map for 7ffe829d1188
> unwind: access_mem 0x7ffe829d1188 not inside range 0x7ffe829ce640-0x7ffe829d0640
> unwind: __cmd_probe:ip = 0x4979b2 (0x979b2)
>      0.000 ( 0.102 ms): perf/1755681 write(fd: 5, buf: \197\10\17K\137\0\0\0\9ops\0\197\11#y\4\1\0\8\1\176+E\0\197\12\15]\0\0, count: 16384) = 16384
>                                        syscall_exit_to_user_mode_prepare ([kernel.kallsyms])
> 
> 
> Anyway, I'm digressing, I need to make this auto-download similar to
> what I proposed above for the coresight traces needed by the tests you
> think about adding for decoding in all platforms, which I encourage you
> to do.
> 
> > Not only would this benefit testing on other arches like you say, but
> > it would also lock down that decoding of a known file doesn't regress
> > which we can't currently do by generating new trace every time the
> > test runs.
> 
> Right.
>  
> > If we ever added this they would be separate tests though so they could go
> > in the top level folder, where the ones in the arch folder would continue to
> > do record and decode. Maybe naming the folders by PMU could work, but you
> > could also have both PMU name and arch name folders like:
>  
> > Recording/requires hardware:
>  
> >   tools/perf/arch/arm64/tests/shell/cs_etm/
>  
> > Cross platform decode tests:
>  
> >   tools/perf/tests/shell/cs_etm/
>  
> > Which would mirror how the source files are currently laid out:
>  
> >  tools/perf/arch/arm/util/cs-etm.c
> >  tools/perf/util/cs-etm.c
> 
> Yeah, I think we can experiment with this and take advantage of the
> effort Namhyung already put into this, and then revisit later, after
> trying this for a while.

I'm ok with sending v2 removing arch shell tests.  Actually it'd be
simpler.  Let me know which way you prefer. :)

Thanks,
Namhyung

> 
> Ah, finally:
> 
> root@x1:~# time perf probe -L icmp_rcv | head 
> <icmp_rcv@/root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/source-d5d23b89-#usr#src#debug#kernel-6.13.9#linux-6.13.9-100.fc40.x86_64#net#ipv4#icmp.c:0>
>       0  int icmp_rcv(struct sk_buff *skb)
>          {
>       2  	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
>          	struct rtable *rt = skb_rtable(skb);
>          	struct net *net = dev_net_rcu(rt->dst.dev);
>          	struct icmphdr *icmph;
>          
>          	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
>       8  		struct sec_path *sp = skb_sec_path(skb);
> 
> real	9m20.630s
> user	0m5.427s
> sys	0m2.127s
> root@x1:~#
> 
> And now:
> 
> root@x1:~# pahole --running_kernel_vmlinux
> /root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo
> root@x1:~#
> 
> Also:
> 
> root@x1:~# file /root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo
> /root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, BuildID[sha1]=aa3c82b4a13f9c0e0301bebb20fe958c4db6f362, with debug_info, not stripped
> root@x1:~# file hideout/vmlinux
> hideout/vmlinux: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, BuildID[sha1]=aa3c82b4a13f9c0e0301bebb20fe958c4db6f362, with debug_info, not stripped
> root@x1:~#
> 
> root@x1:~# sha256sum /root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo hideout/vmlinux
> 6e998df9b235ed50ea4d7c6d997450cb7bd6691537e525f002630ae123bc0084  /root/.cache/debuginfod_client/aa3c82b4a13f9c0e0301bebb20fe958c4db6f362/debuginfo
> 6e998df9b235ed50ea4d7c6d997450cb7bd6691537e525f002630ae123bc0084  hideout/vmlinux
> root@x1:~#
> 
> But:
> 
> root@x1:~# rpm -qf /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
> kernel-debuginfo-6.13.9-100.fc40.x86_64
> root@x1:~# rpm -V kernel-debuginfo-6.13.9-100.fc40.x86_64
> missing     /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
> root@x1:~#
> 
> Lets fix this:
> 
> root@x1:~# mv hideout/vmlinux /usr/lib/debug/lib/modules/6.13.9-100.fc40.x86_64/vmlinux
> root@x1:~# rpm -V kernel-debuginfo-6.13.9-100.fc40.x86_64
> root@x1:~#
> 
> Cheers,
> 
> - Arnaldo