[PATCH bpf-next v1 0/2] Enable vmtest for cross-compile arm64 on x86_64 host, and fix some related issues.

[PATCH bpf-next v1 0/2] Enable vmtest for cross-compile arm64 on x86_64 host, and fix some related issues.

[Lin Yikai]

Enable vmtest for cross-compile arm64 on x86_64 host, and fix some related issues.
I have verified the patch for x86_64 with the target arch of 'x86' or 'arm64'.

v1:
- patch 2:
   - [1/2] Update "vmtest.sh" for cross-compile arm64 on x86_64 host.
   - [2/2] Fix cross-compile issue for some files 
      and a static compile issue for "-lzstd"


Lin Yikai (2):
  selftests/bpf: Update "vmtest.sh" for cross-compile arm64 on x86_64
    host.
  selftests/bpf: Fix cross-compile issue for some files and a static
    compile issue for "-lzstd"

 tools/testing/selftests/bpf/Makefile   | 12 ++++++++-
 tools/testing/selftests/bpf/README.rst | 12 ++++++++-
 tools/testing/selftests/bpf/vmtest.sh  | 37 +++++++++++++++++++++-----
 3 files changed, 53 insertions(+), 8 deletions(-)

-- 
2.34.1

[PATCH bpf-next v1 1/2] selftests/bpf: Update "vmtest.sh" for cross-compile arm64 on x86_64 host.

[Lin Yikai]

Identify "$CROSS_COMPILE" to enable vm_test for cross-compile situation.
Additionally, use "-cpu cortex-a57" flag to accommodate the majority of QEMU CPU lists,
avoiding using "-cpu host," which can cause qemu_system_aarch64 start failure on x86_64 host.

Signed-off-by: Lin Yikai <yikai.lin@vivo.com>
---
 tools/testing/selftests/bpf/README.rst | 12 ++++++++-
 tools/testing/selftests/bpf/vmtest.sh  | 37 +++++++++++++++++++++-----
 2 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst
index 9b974e425af3..c8e6eb8f299d 100644
--- a/tools/testing/selftests/bpf/README.rst
+++ b/tools/testing/selftests/bpf/README.rst
@@ -79,13 +79,23 @@ In case of linker errors when running selftests, try using static linking:
 
   $ LDLIBS=-static PKG_CONFIG='pkg-config --static' vmtest.sh
 
+If you want to make corss-compile, such as compile arm64 on x86_64, you can try:
+
+.. code-block:: console
+
+  $ make headers_install
+  $ export PATH=$PATH:{The corss-compile's path}/bin
+  $ export ARCH=arm64
+  $ export CROSS_COMPILE=aarch64-linux-gnu-
+  $ LDLIBS=-static vmtest.sh
+
 .. note:: Some distros may not support static linking.
 
 .. note:: The script uses pahole and clang based on host environment setting.
           If you want to change pahole and llvm, you can change `PATH` environment
           variable in the beginning of script.
 
-.. note:: The script currently only supports x86_64 and s390x architectures.
+.. note:: The script currently only supports x86_64, s390x and arm64 architectures.
 
 Additional information about selftest failures are
 documented here.
diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh
index 65d14f3bbe30..c7461ed496ab 100755
--- a/tools/testing/selftests/bpf/vmtest.sh
+++ b/tools/testing/selftests/bpf/vmtest.sh
@@ -4,11 +4,11 @@
 set -u
 set -e
 
-# This script currently only works for x86_64 and s390x, as
+# This script currently only works for x86_64, s390x and arm64, as
 # it is based on the VM image used by the BPF CI, which is
 # available only for these architectures.
-ARCH="$(uname -m)"
-case "${ARCH}" in
+HOST_ARCH="$(uname -m)"
+case "${HOST_ARCH}" in
 s390x)
 	QEMU_BINARY=qemu-system-s390x
 	QEMU_CONSOLE="ttyS1"
@@ -32,13 +32,38 @@ aarch64)
 	exit 1
 	;;
 esac
+
+# process CROSS_COMPILE setting to enable cross-compilation
+process_cross_compile() {
+	if [ -z "${CROSS_COMPILE+x}" ]; then
+		return
+	fi
+	case "$1" in
+		x86_64)
+			#Cross-compiling for arm64 on an x86_64 host
+			if [[ $CROSS_COMPILE == *aarch64* ]]; then
+				VM_ARCH=aarch64
+				QEMU_CONSOLE="ttyAMA0,115200"
+				QEMU_BINARY=qemu-system-aarch64
+				QEMU_FLAGS=(-M virt,gic-version=3 -cpu cortex-a57 -smp 8)
+				BZIMAGE="arch/arm64/boot/Image"
+				echo "Setting VM_ARCH from $HOST_ARCH to $VM_ARCH as specified by CROSS_COMPILE"
+			fi
+			;;
+	esac
+}
+
+VM_ARCH=${HOST_ARCH}
+process_cross_compile "$VM_ARCH"
+
+
 DEFAULT_COMMAND="./test_progs"
 MOUNT_DIR="mnt"
 ROOTFS_IMAGE="root.img"
 OUTPUT_DIR="$HOME/.bpf_selftests"
 KCONFIG_REL_PATHS=("tools/testing/selftests/bpf/config"
 	"tools/testing/selftests/bpf/config.vm"
-	"tools/testing/selftests/bpf/config.${ARCH}")
+	"tools/testing/selftests/bpf/config.${VM_ARCH}")
 INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/master/INDEX"
 NUM_COMPILE_JOBS="$(nproc)"
 LOG_FILE_BASE="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S")"
@@ -109,7 +134,7 @@ newest_rootfs_version()
 {
 	{
 	for file in "${!URLS[@]}"; do
-		if [[ $file =~ ^"${ARCH}"/libbpf-vmtest-rootfs-(.*)\.tar\.zst$ ]]; then
+		if [[ $file =~ ^"${VM_ARCH}"/libbpf-vmtest-rootfs-(.*)\.tar\.zst$ ]]; then
 			echo "${BASH_REMATCH[1]}"
 		fi
 	done
@@ -126,7 +151,7 @@ download_rootfs()
 		exit 1
 	fi
 
-	download "${ARCH}/libbpf-vmtest-rootfs-$rootfsversion.tar.zst" |
+	download "${VM_ARCH}/libbpf-vmtest-rootfs-$rootfsversion.tar.zst" |
 		zstd -d | sudo tar -C "$dir" -x
 }
 
-- 
2.34.1

Re: [PATCH bpf-next v1 1/2] selftests/bpf: Update "vmtest.sh" for cross-compile arm64 on x86_64 host.

[Pu Lehui]

On 2024/8/27 21:39, Lin Yikai wrote:
> Identify "$CROSS_COMPILE" to enable vm_test for cross-compile situation.
> Additionally, use "-cpu cortex-a57" flag to accommodate the majority of QEMU CPU lists,
> avoiding using "-cpu host," which can cause qemu_system_aarch64 start failure on x86_64 host.
> 
> Signed-off-by: Lin Yikai <yikai.lin@vivo.com>
> ---
>   tools/testing/selftests/bpf/README.rst | 12 ++++++++-
>   tools/testing/selftests/bpf/vmtest.sh  | 37 +++++++++++++++++++++-----
>   2 files changed, 42 insertions(+), 7 deletions(-)
> 
Hi Yikai,

Your patch reminds me of a previous commit [0], which was able to 
support running vmtest in cross platform for all arch.

Link: 
https://lore.kernel.org/all/20240328124916.293173-2-pulehui@huaweicloud.com/ 
[0]

Re: Re: [PATCH bpf-next v1 1/2] selftests/bpf: Update "vmtest.sh" for cross-compile arm64 on x86_64 host.

[yikai.lin]

On 2024-08-28 at 11:10:38, yikai.lin@vivo.com reply: 





>



>On 2024/8/27 21:39, Lin Yikai wrote:



>> Identify "$CROSS_COMPILE" to enable vm_test for cross-compile situation.



>> Additionally, use "-cpu cortex-a57" flag to accommodate the majority of QEMU CPU lists,



>> avoiding using "-cpu host," which can cause qemu_system_aarch64 start failure on x86_64 host.



>> 



>> Signed-off-by: Lin Yikai <yikai.lin@vivo.com>



>> ---



>>   tools/testing/selftests/bpf/README.rst | 12 ++++++++-



>>   tools/testing/selftests/bpf/vmtest.sh  | 37 +++++++++++++++++++++-----



>>   2 files changed, 42 insertions(+), 7 deletions(-)



>> 



>Hi Yikai,



>



>Your patch reminds me of a previous commit [0], which was able to 



>support running vmtest in cross platform for all arch.



>



>Link: 



>https://lore.kernel.org/all/20240328124916.293173-2-pulehui@huaweicloud.com/ 



>[0]



>
Hi, lehui, thank you for your reply.
I noticed you patch mainly targeted on riscv64 and also extended support to other ARCH.
While these changes are to have not been merged into the mainline yet.

Becuse currently I want to use and extend BPF on Android.
So my focus is primarily on cross-compiling for arm64 on x86.

[PATCH bpf-next v1 2/2] selftests/bpf: Fix cross-compile issue for some files and a static compile issue for "-lzstd"

[Lin Yikai]

1. Fix cross-compile issue for some files:
[Issue]
When cross-compiling bpf selftests for arm64 on x86_64 host, the following error occurs:
progs/loop2.c:20:7: error: incomplete definition of type 'struct user_pt_regs'
   20 |                 if (PT_REGS_RC(ctx) & 1)
      |                     ^~~~~~~~~~~~~~~

There are same error in files: loop1.c, loop2.c, loop3.c, loop6.c ???

[Reason]
On arm64, in file bpf_tracing.h, we use userspace's user_pt_regs,
which is defined in "linux/ptrace.h".
We include the header file by adding "-idirafter /usr/include" for "CLANG_CFLAGS".

However, during cross-compiling, "linux/ptrace.h" is based on x86_64
and has no definition of "struct user_pt_regs".

[Fix]
Thus, to fix this issue, we include the Linux source tree's header file directory.

2. Fix static compile issue for "-lzstd":
[Issue]
By running the command "LDLIBS=-static LDFLAGS=--sysroot=/aarch64-linux-gnu/libc ./vmtest.sh -s -- ./test_progs",
during static cross-compiling, an error occurs:
/aarch64-linux-gnu/bin/ld: aarch64-linux-gnu/libc/usr/lib/libelf.a(elf_compress.o): in function `__libelf_compress':
(.text+0xec): undefined reference to `ZSTD_createCCtx'
/aarch64-linux-gnu/bin/ld: (.text+0xf0): undefined reference to `ZSTD_createCCtx'
...

[Fix]
For static compile, add "LDLIBS += -lzstd".

Signed-off-by: Lin Yikai <yikai.lin@vivo.com>
---
 tools/testing/selftests/bpf/Makefile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index ec7d425c4022..5b725bc890d2 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -48,6 +48,10 @@ CFLAGS += -g $(OPT_FLAGS) -rdynamic					\
 LDFLAGS += $(SAN_LDFLAGS)
 LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread
 
+ifneq (,$(findstring -static,$(LDLIBS)))
+LDLIBS += -lzstd
+endif
+
 LDLIBS += $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null)
 CFLAGS += $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null)
 CFLAGS += $(shell $(PKG_CONFIG) --exists libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1")
@@ -443,13 +447,19 @@ CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))
 endif
 
 CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH))
+CLANG_CFLAGS = $(CLANG_SYS_INCLUDES)
+
 BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN)	\
 	     -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR)			\
 	     -I$(abspath $(OUTPUT)/../usr/include)			\
 	     -Wno-compare-distinct-pointer-types
 # TODO: enable me -Wsign-compare
 
-CLANG_CFLAGS = $(CLANG_SYS_INCLUDES)
+#"make headers_install" at first
+ifneq ($(CROSS_COMPILE),)
+src_uapi_dir := $(srctree)/usr/include
+BPF_CFLAGS += -I$(src_uapi_dir)
+endif
 
 $(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline
 $(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline
-- 
2.34.1

Re: [PATCH bpf-next v1 2/2] selftests/bpf: Fix cross-compile issue for some files and a static compile issue for "-lzstd"

[Andrii Nakryiko]

On Tue, Aug 27, 2024 at 6:40 AM Lin Yikai <yikai.lin@vivo.com> wrote:
>
> 1. Fix cross-compile issue for some files:
> [Issue]
> When cross-compiling bpf selftests for arm64 on x86_64 host, the following error occurs:
> progs/loop2.c:20:7: error: incomplete definition of type 'struct user_pt_regs'
>    20 |                 if (PT_REGS_RC(ctx) & 1)
>       |                     ^~~~~~~~~~~~~~~
>
> There are same error in files: loop1.c, loop2.c, loop3.c, loop6.c ???
>
> [Reason]
> On arm64, in file bpf_tracing.h, we use userspace's user_pt_regs,
> which is defined in "linux/ptrace.h".
> We include the header file by adding "-idirafter /usr/include" for "CLANG_CFLAGS".
>
> However, during cross-compiling, "linux/ptrace.h" is based on x86_64
> and has no definition of "struct user_pt_regs".
>
> [Fix]
> Thus, to fix this issue, we include the Linux source tree's header file directory.
>

Hm.. Not sure that's the right fix. Note -D__TARGET_ARCH_$(SRCARCH) in
BPF_CFLAGS, that __TARGET_ARCH has to match actual target
architecture, so please check that first.

pw-bot: cr

> 2. Fix static compile issue for "-lzstd":
> [Issue]
> By running the command "LDLIBS=-static LDFLAGS=--sysroot=/aarch64-linux-gnu/libc ./vmtest.sh -s -- ./test_progs",
> during static cross-compiling, an error occurs:
> /aarch64-linux-gnu/bin/ld: aarch64-linux-gnu/libc/usr/lib/libelf.a(elf_compress.o): in function `__libelf_compress':
> (.text+0xec): undefined reference to `ZSTD_createCCtx'
> /aarch64-linux-gnu/bin/ld: (.text+0xf0): undefined reference to `ZSTD_createCCtx'
> ...
>
> [Fix]
> For static compile, add "LDLIBS += -lzstd".

we can probably just add it unconditionally, no? But please send it as
a separate change in its own patch

>
> Signed-off-by: Lin Yikai <yikai.lin@vivo.com>
> ---
>  tools/testing/selftests/bpf/Makefile | 12 +++++++++++-
>  1 file changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
> index ec7d425c4022..5b725bc890d2 100644
> --- a/tools/testing/selftests/bpf/Makefile
> +++ b/tools/testing/selftests/bpf/Makefile
> @@ -48,6 +48,10 @@ CFLAGS += -g $(OPT_FLAGS) -rdynamic                                  \
>  LDFLAGS += $(SAN_LDFLAGS)
>  LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread
>
> +ifneq (,$(findstring -static,$(LDLIBS)))
> +LDLIBS += -lzstd
> +endif
> +
>  LDLIBS += $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null)
>  CFLAGS += $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null)
>  CFLAGS += $(shell $(PKG_CONFIG) --exists libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1")
> @@ -443,13 +447,19 @@ CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))
>  endif
>
>  CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH))
> +CLANG_CFLAGS = $(CLANG_SYS_INCLUDES)
> +
>  BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN)    \
>              -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR)                   \
>              -I$(abspath $(OUTPUT)/../usr/include)                      \
>              -Wno-compare-distinct-pointer-types
>  # TODO: enable me -Wsign-compare
>
> -CLANG_CFLAGS = $(CLANG_SYS_INCLUDES)
> +#"make headers_install" at first
> +ifneq ($(CROSS_COMPILE),)
> +src_uapi_dir := $(srctree)/usr/include
> +BPF_CFLAGS += -I$(src_uapi_dir)
> +endif
>
>  $(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline
>  $(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline
> --
> 2.34.1
>

Re: [PATCH bpf-next v1 2/2] selftests/bpf: Fix cross-compile issue for some files and a static compile issue for "-lzstd"

[Yikai Lin]

On 2024/8/28 06:49:18, Lin Yikai reply:

> [Some people who received this message don't often get email from andrii.nakryiko@gmail.com. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]
> 
> On Tue, Aug 27, 2024 at 6:40 AM Lin Yikai <yikai.lin@vivo.com> wrote:
>>
>> 1. Fix cross-compile issue for some files:
>> [Issue]
>> When cross-compiling bpf selftests for arm64 on x86_64 host, the following error occurs:
>> progs/loop2.c:20:7: error: incomplete definition of type 'struct user_pt_regs'
>>     20 |                 if (PT_REGS_RC(ctx) & 1)
>>        |                     ^~~~~~~~~~~~~~~
>>
>> There are same error in files: loop1.c, loop2.c, loop3.c, loop6.c ???
>>
>> [Reason]
>> On arm64, in file bpf_tracing.h, we use userspace's user_pt_regs,
>> which is defined in "linux/ptrace.h".
>> We include the header file by adding "-idirafter /usr/include" for "CLANG_CFLAGS".
>>
>> However, during cross-compiling, "linux/ptrace.h" is based on x86_64
>> and has no definition of "struct user_pt_regs".
>>
>> [Fix]
>> Thus, to fix this issue, we include the Linux source tree's header file directory.
>>
> 
> Hm.. Not sure that's the right fix. Note -D__TARGET_ARCH_$(SRCARCH) in
> BPF_CFLAGS, that __TARGET_ARCH has to match actual target
> architecture, so please check that first.
> 

Hi, Andrii, thanks for your advice.

After searching, I find needed "user_pt_regs" is defined as below:
   bpf-next$ find ./ -name "ptrace.h" | xargs grep -rni "user_pt_regs"
   ./arch/arm64/include/uapi/asm/ptrace.h:88:struct user_pt_regs {

So we can directly add it into include directory
by below approach without "make header_install".

   +ifneq ($(CROSS_COMPILE),)
   +src_uapi_dir := $(srctree)/arch/$(SRCARCH)/include/uapi
   +BPF_CFLAGS += -I$(src_uapi_dir)
   +endif

I'll test it and send a v2 patch.

> pw-bot: cr
> 
>> 2. Fix static compile issue for "-lzstd":
>> [Issue]
>> By running the command "LDLIBS=-static LDFLAGS=--sysroot=/aarch64-linux-gnu/libc ./vmtest.sh -s -- ./test_progs",
>> during static cross-compiling, an error occurs:
>> /aarch64-linux-gnu/bin/ld: aarch64-linux-gnu/libc/usr/lib/libelf.a(elf_compress.o): in function `__libelf_compress':
>> (.text+0xec): undefined reference to `ZSTD_createCCtx'
>> /aarch64-linux-gnu/bin/ld: (.text+0xf0): undefined reference to `ZSTD_createCCtx'
>> ...
>>
>> [Fix]
>> For static compile, add "LDLIBS += -lzstd".
> 
> we can probably just add it unconditionally, no? But please send it as
> a separate change in its own patch
> 
Thanks, I'll try it for v2 patch.

>>
>> Signed-off-by: Lin Yikai <yikai.lin@vivo.com>
>> ---
>>   tools/testing/selftests/bpf/Makefile | 12 +++++++++++-
>>   1 file changed, 11 insertions(+), 1 deletion(-)
>>
>> diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
>> index ec7d425c4022..5b725bc890d2 100644
>> --- a/tools/testing/selftests/bpf/Makefile
>> +++ b/tools/testing/selftests/bpf/Makefile
>> @@ -48,6 +48,10 @@ CFLAGS += -g $(OPT_FLAGS) -rdynamic                                  \
>>   LDFLAGS += $(SAN_LDFLAGS)
>>   LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread
>>
>> +ifneq (,$(findstring -static,$(LDLIBS)))
>> +LDLIBS += -lzstd
>> +endif
>> +
>>   LDLIBS += $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null)
>>   CFLAGS += $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null)
>>   CFLAGS += $(shell $(PKG_CONFIG) --exists libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1")
>> @@ -443,13 +447,19 @@ CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))
>>   endif
>>
>>   CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH))
>> +CLANG_CFLAGS = $(CLANG_SYS_INCLUDES)
>> +
>>   BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN)    \
>>               -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR)                   \
>>               -I$(abspath $(OUTPUT)/../usr/include)                      \
>>               -Wno-compare-distinct-pointer-types
>>   # TODO: enable me -Wsign-compare
>>
>> -CLANG_CFLAGS = $(CLANG_SYS_INCLUDES)
>> +#"make headers_install" at first
>> +ifneq ($(CROSS_COMPILE),)
>> +src_uapi_dir := $(srctree)/usr/include
>> +BPF_CFLAGS += -I$(src_uapi_dir)
>> +endif
>>
>>   $(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline
>>   $(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline
>> --
>> 2.34.1
>>