[PATCH v2] tests/functional: test device passthrough on aarch64

[PATCH v2] tests/functional: test device passthrough on aarch64

[Pierrick Bouvier]

This test allows to document and exercise device passthrough, using a
nested virtual machine setup. Two disks are generated and passed to the
VM, and their content is compared to original images.

Guest and nested guests commands are executed through two scripts, and
init used in both system is configured to trigger a kernel panic in case
any command fails. This is more reliable and readable than executing all
commands through prompt injection and trying to guess what failed.

Initially, this test was supposed to test smmuv3 nested emulation
(combining both stages of translation), but I could not find any setup
(kernel + vmm) able to do the passthrough correctly, despite several
tries.

Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
---
 tests/functional/meson.build                  |   2 +
 .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
 2 files changed, 144 insertions(+)
 create mode 100755 tests/functional/test_aarch64_device_passthrough.py

diff --git a/tests/functional/meson.build b/tests/functional/meson.build
index 3021928a9d4..6cc78abb123 100644
--- a/tests/functional/meson.build
+++ b/tests/functional/meson.build
@@ -13,6 +13,7 @@ endif
 test_timeouts = {
   'aarch64_aspeed_ast2700' : 600,
   'aarch64_aspeed_ast2700fc' : 600,
+  'aarch64_device_passthrough' : 720,
   'aarch64_imx8mp_evk' : 240,
   'aarch64_raspi4' : 480,
   'aarch64_reverse_debug' : 180,
@@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
 tests_aarch64_system_thorough = [
   'aarch64_aspeed_ast2700',
   'aarch64_aspeed_ast2700fc',
+  'aarch64_device_passthrough',
   'aarch64_imx8mp_evk',
   'aarch64_raspi3',
   'aarch64_raspi4',
diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
new file mode 100755
index 00000000000..1f3f158a9ff
--- /dev/null
+++ b/tests/functional/test_aarch64_device_passthrough.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+#
+# Boots a nested guest and compare content of a device (passthrough) to a
+# reference image. Both vfio group and iommufd passthrough methods are tested.
+#
+# Copyright (c) 2025 Linaro Ltd.
+#
+# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import os
+
+from qemu_test import QemuSystemTest, Asset
+from qemu_test import exec_command, wait_for_console_pattern
+from qemu_test import exec_command_and_wait_for_pattern
+from random import randbytes
+
+guest_script = '''
+#!/usr/bin/env bash
+
+set -euo pipefail
+set -x
+
+# find disks from nvme serial
+dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
+dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
+pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
+pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
+
+# bind disks to vfio
+for p in "$pci_vfio" "$pci_iommufd"; do
+    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
+        continue
+    fi
+    echo $p > /sys/bus/pci/drivers/nvme/unbind
+    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
+    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
+done
+
+# boot nested guest and execute /host/nested_guest.sh
+# one disk is passed through vfio group, the other, through iommufd
+qemu-system-aarch64 \
+-M virt \
+-display none \
+-serial stdio \
+-cpu host \
+-enable-kvm \
+-m 1G \
+-kernel /host/Image.gz \
+-drive format=raw,file=/host/guest.ext4,if=virtio \
+-append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \
+-virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \
+-device vfio-pci,host=$pci_vfio \
+-object iommufd,id=iommufd0 \
+-device vfio-pci,host=$pci_iommufd,iommufd=iommufd0
+'''
+
+nested_guest_script = '''
+#!/usr/bin/env bash
+
+set -euo pipefail
+set -x
+
+image_vfio=/host/disk_vfio
+image_iommufd=/host/disk_iommufd
+
+dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
+dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
+
+# compare if devices are identical to original images
+diff $image_vfio /dev/$dev_vfio
+diff $image_iommufd /dev/$dev_iommufd
+
+echo device_passthrough_test_ok
+'''
+
+class Aarch64DevicePassthrough(QemuSystemTest):
+
+    # https://github.com/pbo-linaro/qemu-linux-stack
+    #
+    # Linux kernel is compiled with defconfig +
+    # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD
+    # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde
+    ASSET_DEVICE_PASSTHROUGH_STACK = Asset(
+        ('https://fileserver.linaro.org/s/fx5DXxBYme8dw2G/'
+         'download/device_passthrough.tar.xz'),
+         '812750b664d61c2986f2b149939ae28cafbd60d53e9c7e4b16e97143845e196d')
+
+    # This tests the device passthrough implementation, by booting a VM
+    # supporting it with two nvme disks attached, and launching a nested VM
+    # reading their content.
+    def test_aarch64_device_passthrough(self):
+        self.set_machine('virt')
+        self.require_accelerator('tcg')
+
+        self.vm.set_console()
+
+        stack_path_tar_gz = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch()
+        self.archive_extract(stack_path_tar_gz, format="tar")
+
+        stack = self.scratch_file('out')
+        kernel = os.path.join(stack, 'Image.gz')
+        rootfs_host = os.path.join(stack, 'host.ext4')
+        disk_vfio = os.path.join(stack, 'disk_vfio')
+        disk_iommufd = os.path.join(stack, 'disk_iommufd')
+        guest_cmd = os.path.join(stack, 'guest.sh')
+        nested_guest_cmd = os.path.join(stack, 'nested_guest.sh')
+        # we generate two random disks
+        with open(disk_vfio, "wb") as d: d.write(randbytes(512))
+        with open(disk_iommufd, "wb") as d: d.write(randbytes(1024))
+        with open(guest_cmd, 'w') as s: s.write(guest_script)
+        with open(nested_guest_cmd, 'w') as s: s.write(nested_guest_script)
+
+        self.vm.add_args('-cpu', 'max')
+        self.vm.add_args('-m', '2G')
+        self.vm.add_args('-M', 'virt,'
+                         'virtualization=on,'
+                         'gic-version=max,'
+                         'iommu=smmuv3')
+        self.vm.add_args('-kernel', kernel)
+        self.vm.add_args('-drive', f'format=raw,file={rootfs_host}')
+        self.vm.add_args('-drive',
+                         f'file={disk_vfio},if=none,id=vfio,format=raw')
+        self.vm.add_args('-device', 'nvme,serial=vfio,drive=vfio')
+        self.vm.add_args('-drive',
+                         f'file={disk_iommufd},if=none,id=iommufd,format=raw')
+        self.vm.add_args('-device', 'nvme,serial=iommufd,drive=iommufd')
+        self.vm.add_args('-virtfs',
+                         f'local,path={stack}/,mount_tag=host,'
+                         'security_model=mapped,readonly=off')
+        # boot and execute guest script
+        # init will trigger a kernel panic if script fails
+        self.vm.add_args('-append',
+                         'root=/dev/vda init=/init -- bash /host/guest.sh')
+
+        self.vm.launch()
+        wait_for_console_pattern(self, 'device_passthrough_test_ok',
+                                 failure_message='Kernel panic')
+
+if __name__ == '__main__':
+    QemuSystemTest.main()
-- 
2.47.2

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Pierrick Bouvier]

On 6/27/25 1:02 PM, Pierrick Bouvier wrote:
> This test allows to document and exercise device passthrough, using a
> nested virtual machine setup. Two disks are generated and passed to the
> VM, and their content is compared to original images.
> 
> Guest and nested guests commands are executed through two scripts, and
> init used in both system is configured to trigger a kernel panic in case
> any command fails. This is more reliable and readable than executing all
> commands through prompt injection and trying to guess what failed.
> 
> Initially, this test was supposed to test smmuv3 nested emulation
> (combining both stages of translation), but I could not find any setup
> (kernel + vmm) able to do the passthrough correctly, despite several
> tries.
> 
> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
> ---
>   tests/functional/meson.build                  |   2 +
>   .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>   2 files changed, 144 insertions(+)
>   create mode 100755 tests/functional/test_aarch64_device_passthrough.py
>

v2
--

- remove debug trace left my mistake (" || bash")
- minimized asset size (180MB -> 40MB)

Regards,
Pierrick

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Cédric Le Goater]

Hello,

On 6/27/25 22:02, Pierrick Bouvier wrote:
> This test allows to document and exercise device passthrough, using a
> nested virtual machine setup. Two disks are generated and passed to the
> VM, and their content is compared to original images.
> 
> Guest and nested guests commands are executed through two scripts, and
> init used in both system is configured to trigger a kernel panic in case
> any command fails. This is more reliable and readable than executing all
> commands through prompt injection and trying to guess what failed.
> 
> Initially, this test was supposed to test smmuv3 nested emulation
> (combining both stages of translation), but I could not find any setup
> (kernel + vmm) able to do the passthrough correctly, despite several
> tries.
> 
> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
> ---
>   tests/functional/meson.build                  |   2 +
>   .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>   2 files changed, 144 insertions(+)
>   create mode 100755 tests/functional/test_aarch64_device_passthrough.py
> 
> diff --git a/tests/functional/meson.build b/tests/functional/meson.build
> index 3021928a9d4..6cc78abb123 100644
> --- a/tests/functional/meson.build
> +++ b/tests/functional/meson.build
> @@ -13,6 +13,7 @@ endif
>   test_timeouts = {
>     'aarch64_aspeed_ast2700' : 600,
>     'aarch64_aspeed_ast2700fc' : 600,
> +  'aarch64_device_passthrough' : 720,
>     'aarch64_imx8mp_evk' : 240,
>     'aarch64_raspi4' : 480,
>     'aarch64_reverse_debug' : 180,
> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
>   tests_aarch64_system_thorough = [
>     'aarch64_aspeed_ast2700',
>     'aarch64_aspeed_ast2700fc',
> +  'aarch64_device_passthrough',
>     'aarch64_imx8mp_evk',
>     'aarch64_raspi3',
>     'aarch64_raspi4',
> diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
> new file mode 100755
> index 00000000000..1f3f158a9ff
> --- /dev/null
> +++ b/tests/functional/test_aarch64_device_passthrough.py
> @@ -0,0 +1,142 @@
> +#!/usr/bin/env python3
> +#
> +# Boots a nested guest and compare content of a device (passthrough) to a
> +# reference image. Both vfio group and iommufd passthrough methods are tested.
> +#
> +# Copyright (c) 2025 Linaro Ltd.
> +#
> +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
> +#
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +
> +import os
> +
> +from qemu_test import QemuSystemTest, Asset
> +from qemu_test import exec_command, wait_for_console_pattern
> +from qemu_test import exec_command_and_wait_for_pattern
> +from random import randbytes
> +
> +guest_script = '''
> +#!/usr/bin/env bash
> +
> +set -euo pipefail
> +set -x
> +
> +# find disks from nvme serial
> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
> +
> +# bind disks to vfio
> +for p in "$pci_vfio" "$pci_iommufd"; do
> +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
> +        continue
> +    fi
> +    echo $p > /sys/bus/pci/drivers/nvme/unbind
> +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
> +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
> +done
> +
> +# boot nested guest and execute /host/nested_guest.sh
> +# one disk is passed through vfio group, the other, through iommufd
> +qemu-system-aarch64 \

Is this binary on the host.ext4 image ?

If so, the test is testing a chosen QEMU version compiled in the
L1 guest image but not the current QEMU version, which is the one
running in the L0.

Anyhow this is a very nice test and an excellent base to build on.
As a next step, I’d suggest including tests with NICs using igb
devices and igb virtual functions (VFs).

It would also be great to run the L1 environment using the current
version of QEMU. I haven't found a clean way to achieve that yet :/

Thanks,

C.




> +-M virt \
> +-display none \
> +-serial stdio \
> +-cpu host \
> +-enable-kvm \
> +-m 1G \
> +-kernel /host/Image.gz \
> +-drive format=raw,file=/host/guest.ext4,if=virtio \
> +-append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \
> +-virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \
> +-device vfio-pci,host=$pci_vfio \
> +-object iommufd,id=iommufd0 \
> +-device vfio-pci,host=$pci_iommufd,iommufd=iommufd0
> +'''
> +
> +nested_guest_script = '''
> +#!/usr/bin/env bash
> +
> +set -euo pipefail
> +set -x
> +
> +image_vfio=/host/disk_vfio
> +image_iommufd=/host/disk_iommufd
> +
> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
> +
> +# compare if devices are identical to original images
> +diff $image_vfio /dev/$dev_vfio
> +diff $image_iommufd /dev/$dev_iommufd
> +
> +echo device_passthrough_test_ok
> +'''
> +
> +class Aarch64DevicePassthrough(QemuSystemTest):
> +
> +    # https://github.com/pbo-linaro/qemu-linux-stack
> +    #
> +    # Linux kernel is compiled with defconfig +
> +    # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD
> +    # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde
> +    ASSET_DEVICE_PASSTHROUGH_STACK = Asset(
> +        ('https://fileserver.linaro.org/s/fx5DXxBYme8dw2G/'
> +         'download/device_passthrough.tar.xz'),
> +         '812750b664d61c2986f2b149939ae28cafbd60d53e9c7e4b16e97143845e196d')
> +
> +    # This tests the device passthrough implementation, by booting a VM
> +    # supporting it with two nvme disks attached, and launching a nested VM
> +    # reading their content.
> +    def test_aarch64_device_passthrough(self):
> +        self.set_machine('virt')
> +        self.require_accelerator('tcg')
> +
> +        self.vm.set_console()
> +
> +        stack_path_tar_gz = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch()
> +        self.archive_extract(stack_path_tar_gz, format="tar")
> +
> +        stack = self.scratch_file('out')
> +        kernel = os.path.join(stack, 'Image.gz')
> +        rootfs_host = os.path.join(stack, 'host.ext4')
> +        disk_vfio = os.path.join(stack, 'disk_vfio')
> +        disk_iommufd = os.path.join(stack, 'disk_iommufd')
> +        guest_cmd = os.path.join(stack, 'guest.sh')
> +        nested_guest_cmd = os.path.join(stack, 'nested_guest.sh')
> +        # we generate two random disks
> +        with open(disk_vfio, "wb") as d: d.write(randbytes(512))
> +        with open(disk_iommufd, "wb") as d: d.write(randbytes(1024))
> +        with open(guest_cmd, 'w') as s: s.write(guest_script)
> +        with open(nested_guest_cmd, 'w') as s: s.write(nested_guest_script)
> +
> +        self.vm.add_args('-cpu', 'max')
> +        self.vm.add_args('-m', '2G')
> +        self.vm.add_args('-M', 'virt,'
> +                         'virtualization=on,'
> +                         'gic-version=max,'
> +                         'iommu=smmuv3')
> +        self.vm.add_args('-kernel', kernel)
> +        self.vm.add_args('-drive', f'format=raw,file={rootfs_host}')
> +        self.vm.add_args('-drive',
> +                         f'file={disk_vfio},if=none,id=vfio,format=raw')
> +        self.vm.add_args('-device', 'nvme,serial=vfio,drive=vfio')
> +        self.vm.add_args('-drive',
> +                         f'file={disk_iommufd},if=none,id=iommufd,format=raw')
> +        self.vm.add_args('-device', 'nvme,serial=iommufd,drive=iommufd')
> +        self.vm.add_args('-virtfs',
> +                         f'local,path={stack}/,mount_tag=host,'
> +                         'security_model=mapped,readonly=off')
> +        # boot and execute guest script
> +        # init will trigger a kernel panic if script fails
> +        self.vm.add_args('-append',
> +                         'root=/dev/vda init=/init -- bash /host/guest.sh')
> +
> +        self.vm.launch()
> +        wait_for_console_pattern(self, 'device_passthrough_test_ok',
> +                                 failure_message='Kernel panic')
> +
> +if __name__ == '__main__':
> +    QemuSystemTest.main()

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Daniel P. Berrangé]

On Fri, Jun 27, 2025 at 01:02:22PM -0700, Pierrick Bouvier wrote:
> This test allows to document and exercise device passthrough, using a
> nested virtual machine setup. Two disks are generated and passed to the
> VM, and their content is compared to original images.
> 
> Guest and nested guests commands are executed through two scripts, and
> init used in both system is configured to trigger a kernel panic in case
> any command fails. This is more reliable and readable than executing all
> commands through prompt injection and trying to guess what failed.
> 
> Initially, this test was supposed to test smmuv3 nested emulation
> (combining both stages of translation), but I could not find any setup
> (kernel + vmm) able to do the passthrough correctly, despite several
> tries.
> 
> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
> ---
>  tests/functional/meson.build                  |   2 +
>  .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>  2 files changed, 144 insertions(+)
>  create mode 100755 tests/functional/test_aarch64_device_passthrough.py
> 
> diff --git a/tests/functional/meson.build b/tests/functional/meson.build
> index 3021928a9d4..6cc78abb123 100644
> --- a/tests/functional/meson.build
> +++ b/tests/functional/meson.build
> @@ -13,6 +13,7 @@ endif
>  test_timeouts = {
>    'aarch64_aspeed_ast2700' : 600,
>    'aarch64_aspeed_ast2700fc' : 600,
> +  'aarch64_device_passthrough' : 720,
>    'aarch64_imx8mp_evk' : 240,
>    'aarch64_raspi4' : 480,
>    'aarch64_reverse_debug' : 180,
> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
>  tests_aarch64_system_thorough = [
>    'aarch64_aspeed_ast2700',
>    'aarch64_aspeed_ast2700fc',
> +  'aarch64_device_passthrough',
>    'aarch64_imx8mp_evk',
>    'aarch64_raspi3',
>    'aarch64_raspi4',
> diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
> new file mode 100755
> index 00000000000..1f3f158a9ff
> --- /dev/null
> +++ b/tests/functional/test_aarch64_device_passthrough.py
> @@ -0,0 +1,142 @@
> +#!/usr/bin/env python3
> +#
> +# Boots a nested guest and compare content of a device (passthrough) to a
> +# reference image. Both vfio group and iommufd passthrough methods are tested.
> +#
> +# Copyright (c) 2025 Linaro Ltd.
> +#
> +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
> +#
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +
> +import os
> +
> +from qemu_test import QemuSystemTest, Asset
> +from qemu_test import exec_command, wait_for_console_pattern
> +from qemu_test import exec_command_and_wait_for_pattern
> +from random import randbytes
> +
> +guest_script = '''
> +#!/usr/bin/env bash
> +
> +set -euo pipefail
> +set -x
> +
> +# find disks from nvme serial
> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
> +
> +# bind disks to vfio
> +for p in "$pci_vfio" "$pci_iommufd"; do
> +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
> +        continue
> +    fi
> +    echo $p > /sys/bus/pci/drivers/nvme/unbind
> +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
> +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
> +done
> +
> +# boot nested guest and execute /host/nested_guest.sh
> +# one disk is passed through vfio group, the other, through iommufd
> +qemu-system-aarch64 \
> +-M virt \
> +-display none \
> +-serial stdio \
> +-cpu host \
> +-enable-kvm \
> +-m 1G \
> +-kernel /host/Image.gz \
> +-drive format=raw,file=/host/guest.ext4,if=virtio \
> +-append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \
> +-virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \
> +-device vfio-pci,host=$pci_vfio \
> +-object iommufd,id=iommufd0 \
> +-device vfio-pci,host=$pci_iommufd,iommufd=iommufd0
> +'''
> +
> +nested_guest_script = '''
> +#!/usr/bin/env bash
> +
> +set -euo pipefail
> +set -x
> +
> +image_vfio=/host/disk_vfio
> +image_iommufd=/host/disk_iommufd
> +
> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
> +
> +# compare if devices are identical to original images
> +diff $image_vfio /dev/$dev_vfio
> +diff $image_iommufd /dev/$dev_iommufd
> +
> +echo device_passthrough_test_ok
> +'''
> +
> +class Aarch64DevicePassthrough(QemuSystemTest):
> +
> +    # https://github.com/pbo-linaro/qemu-linux-stack
> +    #
> +    # Linux kernel is compiled with defconfig +
> +    # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD
> +    # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde
> +    ASSET_DEVICE_PASSTHROUGH_STACK = Asset(
> +        ('https://fileserver.linaro.org/s/fx5DXxBYme8dw2G/'
> +         'download/device_passthrough.tar.xz'),
> +         '812750b664d61c2986f2b149939ae28cafbd60d53e9c7e4b16e97143845e196d')
> +
> +    # This tests the device passthrough implementation, by booting a VM
> +    # supporting it with two nvme disks attached, and launching a nested VM
> +    # reading their content.
> +    def test_aarch64_device_passthrough(self):
> +        self.set_machine('virt')
> +        self.require_accelerator('tcg')
> +
> +        self.vm.set_console()
> +
> +        stack_path_tar_gz = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch()
> +        self.archive_extract(stack_path_tar_gz, format="tar")
> +
> +        stack = self.scratch_file('out')
> +        kernel = os.path.join(stack, 'Image.gz')
> +        rootfs_host = os.path.join(stack, 'host.ext4')
> +        disk_vfio = os.path.join(stack, 'disk_vfio')
> +        disk_iommufd = os.path.join(stack, 'disk_iommufd')
> +        guest_cmd = os.path.join(stack, 'guest.sh')
> +        nested_guest_cmd = os.path.join(stack, 'nested_guest.sh')

Don't incrementally create paths like this - use the 
'scratch_file' method for all components

 ie

   kernel = self.scratch_file('out', 'Image.gz')
   rootfs_host =  self.scratch_file('out', 'host.ext4')
   ...etc...

> +        # we generate two random disks
> +        with open(disk_vfio, "wb") as d: d.write(randbytes(512))
> +        with open(disk_iommufd, "wb") as d: d.write(randbytes(1024))
> +        with open(guest_cmd, 'w') as s: s.write(guest_script)
> +        with open(nested_guest_cmd, 'w') as s: s.write(nested_guest_script)
> +
> +        self.vm.add_args('-cpu', 'max')
> +        self.vm.add_args('-m', '2G')
> +        self.vm.add_args('-M', 'virt,'
> +                         'virtualization=on,'
> +                         'gic-version=max,'
> +                         'iommu=smmuv3')
> +        self.vm.add_args('-kernel', kernel)
> +        self.vm.add_args('-drive', f'format=raw,file={rootfs_host}')
> +        self.vm.add_args('-drive',
> +                         f'file={disk_vfio},if=none,id=vfio,format=raw')
> +        self.vm.add_args('-device', 'nvme,serial=vfio,drive=vfio')
> +        self.vm.add_args('-drive',
> +                         f'file={disk_iommufd},if=none,id=iommufd,format=raw')
> +        self.vm.add_args('-device', 'nvme,serial=iommufd,drive=iommufd')
> +        self.vm.add_args('-virtfs',
> +                         f'local,path={stack}/,mount_tag=host,'
> +                         'security_model=mapped,readonly=off')
> +        # boot and execute guest script
> +        # init will trigger a kernel panic if script fails
> +        self.vm.add_args('-append',
> +                         'root=/dev/vda init=/init -- bash /host/guest.sh')
> +
> +        self.vm.launch()
> +        wait_for_console_pattern(self, 'device_passthrough_test_ok',
> +                                 failure_message='Kernel panic')
> +
> +if __name__ == '__main__':
> +    QemuSystemTest.main()
> -- 
> 2.47.2
> 

With regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Alex Bennée]

Cédric Le Goater <clg@kaod.org> writes:

> Hello,
>
> On 6/27/25 22:02, Pierrick Bouvier wrote:
>> This test allows to document and exercise device passthrough, using a
>> nested virtual machine setup. Two disks are generated and passed to the
>> VM, and their content is compared to original images.
>> Guest and nested guests commands are executed through two scripts,
>> and
>> init used in both system is configured to trigger a kernel panic in case
>> any command fails. This is more reliable and readable than executing all
>> commands through prompt injection and trying to guess what failed.
>> Initially, this test was supposed to test smmuv3 nested emulation
>> (combining both stages of translation), but I could not find any setup
>> (kernel + vmm) able to do the passthrough correctly, despite several
>> tries.
>> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>> ---
>>   tests/functional/meson.build                  |   2 +
>>   .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>>   2 files changed, 144 insertions(+)
>>   create mode 100755 tests/functional/test_aarch64_device_passthrough.py
>> diff --git a/tests/functional/meson.build
>> b/tests/functional/meson.build
>> index 3021928a9d4..6cc78abb123 100644
>> --- a/tests/functional/meson.build
>> +++ b/tests/functional/meson.build
>> @@ -13,6 +13,7 @@ endif
>>   test_timeouts = {
>>     'aarch64_aspeed_ast2700' : 600,
>>     'aarch64_aspeed_ast2700fc' : 600,
>> +  'aarch64_device_passthrough' : 720,
>>     'aarch64_imx8mp_evk' : 240,
>>     'aarch64_raspi4' : 480,
>>     'aarch64_reverse_debug' : 180,
>> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
>>   tests_aarch64_system_thorough = [
>>     'aarch64_aspeed_ast2700',
>>     'aarch64_aspeed_ast2700fc',
>> +  'aarch64_device_passthrough',
>>     'aarch64_imx8mp_evk',
>>     'aarch64_raspi3',
>>     'aarch64_raspi4',
>> diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
>> new file mode 100755
>> index 00000000000..1f3f158a9ff
>> --- /dev/null
>> +++ b/tests/functional/test_aarch64_device_passthrough.py
>> @@ -0,0 +1,142 @@
>> +#!/usr/bin/env python3
>> +#
>> +# Boots a nested guest and compare content of a device (passthrough) to a
>> +# reference image. Both vfio group and iommufd passthrough methods are tested.
>> +#
>> +# Copyright (c) 2025 Linaro Ltd.
>> +#
>> +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>> +#
>> +# SPDX-License-Identifier: GPL-2.0-or-later
>> +
>> +import os
>> +
>> +from qemu_test import QemuSystemTest, Asset
>> +from qemu_test import exec_command, wait_for_console_pattern
>> +from qemu_test import exec_command_and_wait_for_pattern
>> +from random import randbytes
>> +
>> +guest_script = '''
>> +#!/usr/bin/env bash
>> +
>> +set -euo pipefail
>> +set -x
>> +
>> +# find disks from nvme serial
>> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
>> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
>> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
>> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
>> +
>> +# bind disks to vfio
>> +for p in "$pci_vfio" "$pci_iommufd"; do
>> +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
>> +        continue
>> +    fi
>> +    echo $p > /sys/bus/pci/drivers/nvme/unbind
>> +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
>> +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
>> +done
>> +
>> +# boot nested guest and execute /host/nested_guest.sh
>> +# one disk is passed through vfio group, the other, through iommufd
>> +qemu-system-aarch64 \
>
> Is this binary on the host.ext4 image ?
>
> If so, the test is testing a chosen QEMU version compiled in the
> L1 guest image but not the current QEMU version, which is the one
> running in the L0.
>
> Anyhow this is a very nice test and an excellent base to build on.
> As a next step, I’d suggest including tests with NICs using igb
> devices and igb virtual functions (VFs).
>
> It would also be great to run the L1 environment using the current
> version of QEMU. I haven't found a clean way to achieve that yet :/

I sometimes boot up with a virtiofsd mapped to $HOME but it gets a
little unstable over time and I haven't had a chance to figure out where
things where going wrong.

We have the containers to reliably build a cross image of QEMU but we
would have to ensure the guest image matches so we don't run into
library skew issues. I have had a static build working but thats not a
very well supported configuration for qemu-system.

-- 
Alex Bennée
Virtualisation Tech Lead @ Linaro

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Cédric Le Goater]

On 7/2/25 15:35, Alex Bennée wrote:
> Cédric Le Goater <clg@kaod.org> writes:
> 
>> Hello,
>>
>> On 6/27/25 22:02, Pierrick Bouvier wrote:
>>> This test allows to document and exercise device passthrough, using a
>>> nested virtual machine setup. Two disks are generated and passed to the
>>> VM, and their content is compared to original images.
>>> Guest and nested guests commands are executed through two scripts,
>>> and
>>> init used in both system is configured to trigger a kernel panic in case
>>> any command fails. This is more reliable and readable than executing all
>>> commands through prompt injection and trying to guess what failed.
>>> Initially, this test was supposed to test smmuv3 nested emulation
>>> (combining both stages of translation), but I could not find any setup
>>> (kernel + vmm) able to do the passthrough correctly, despite several
>>> tries.
>>> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>>> ---
>>>    tests/functional/meson.build                  |   2 +
>>>    .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>>>    2 files changed, 144 insertions(+)
>>>    create mode 100755 tests/functional/test_aarch64_device_passthrough.py
>>> diff --git a/tests/functional/meson.build
>>> b/tests/functional/meson.build
>>> index 3021928a9d4..6cc78abb123 100644
>>> --- a/tests/functional/meson.build
>>> +++ b/tests/functional/meson.build
>>> @@ -13,6 +13,7 @@ endif
>>>    test_timeouts = {
>>>      'aarch64_aspeed_ast2700' : 600,
>>>      'aarch64_aspeed_ast2700fc' : 600,
>>> +  'aarch64_device_passthrough' : 720,
>>>      'aarch64_imx8mp_evk' : 240,
>>>      'aarch64_raspi4' : 480,
>>>      'aarch64_reverse_debug' : 180,
>>> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
>>>    tests_aarch64_system_thorough = [
>>>      'aarch64_aspeed_ast2700',
>>>      'aarch64_aspeed_ast2700fc',
>>> +  'aarch64_device_passthrough',
>>>      'aarch64_imx8mp_evk',
>>>      'aarch64_raspi3',
>>>      'aarch64_raspi4',
>>> diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
>>> new file mode 100755
>>> index 00000000000..1f3f158a9ff
>>> --- /dev/null
>>> +++ b/tests/functional/test_aarch64_device_passthrough.py
>>> @@ -0,0 +1,142 @@
>>> +#!/usr/bin/env python3
>>> +#
>>> +# Boots a nested guest and compare content of a device (passthrough) to a
>>> +# reference image. Both vfio group and iommufd passthrough methods are tested.
>>> +#
>>> +# Copyright (c) 2025 Linaro Ltd.
>>> +#
>>> +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>>> +#
>>> +# SPDX-License-Identifier: GPL-2.0-or-later
>>> +
>>> +import os
>>> +
>>> +from qemu_test import QemuSystemTest, Asset
>>> +from qemu_test import exec_command, wait_for_console_pattern
>>> +from qemu_test import exec_command_and_wait_for_pattern
>>> +from random import randbytes
>>> +
>>> +guest_script = '''
>>> +#!/usr/bin/env bash
>>> +
>>> +set -euo pipefail
>>> +set -x
>>> +
>>> +# find disks from nvme serial
>>> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
>>> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
>>> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
>>> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
>>> +
>>> +# bind disks to vfio
>>> +for p in "$pci_vfio" "$pci_iommufd"; do
>>> +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
>>> +        continue
>>> +    fi
>>> +    echo $p > /sys/bus/pci/drivers/nvme/unbind
>>> +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
>>> +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
>>> +done
>>> +
>>> +# boot nested guest and execute /host/nested_guest.sh
>>> +# one disk is passed through vfio group, the other, through iommufd
>>> +qemu-system-aarch64 \
>>
>> Is this binary on the host.ext4 image ?
>>
>> If so, the test is testing a chosen QEMU version compiled in the
>> L1 guest image but not the current QEMU version, which is the one
>> running in the L0.
>>
>> Anyhow this is a very nice test and an excellent base to build on.
>> As a next step, I’d suggest including tests with NICs using igb
>> devices and igb virtual functions (VFs).
>>
>> It would also be great to run the L1 environment using the current
>> version of QEMU. I haven't found a clean way to achieve that yet :/
> 
> I sometimes boot up with a virtiofsd mapped to $HOME but it gets a
> little unstable over time and I haven't had a chance to figure out where
> things where going wrong.
> 
> We have the containers to reliably build a cross image of QEMU but we
> would have to ensure the guest image matches so we don't run into
> library skew issues. 

and could we generate a disk image from a container build, keep the
artifact and use it in the functional test to boot the L1 guest  ?


Thanks,

C.

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Pierrick Bouvier]

On 7/2/25 5:51 AM, Cédric Le Goater wrote:
> Hello,
> 
> On 6/27/25 22:02, Pierrick Bouvier wrote:
>> This test allows to document and exercise device passthrough, using a
>> nested virtual machine setup. Two disks are generated and passed to the
>> VM, and their content is compared to original images.
>>
>> Guest and nested guests commands are executed through two scripts, and
>> init used in both system is configured to trigger a kernel panic in case
>> any command fails. This is more reliable and readable than executing all
>> commands through prompt injection and trying to guess what failed.
>>
>> Initially, this test was supposed to test smmuv3 nested emulation
>> (combining both stages of translation), but I could not find any setup
>> (kernel + vmm) able to do the passthrough correctly, despite several
>> tries.
>>
>> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>> ---
>>    tests/functional/meson.build                  |   2 +
>>    .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>>    2 files changed, 144 insertions(+)
>>    create mode 100755 tests/functional/test_aarch64_device_passthrough.py
>>
>> diff --git a/tests/functional/meson.build b/tests/functional/meson.build
>> index 3021928a9d4..6cc78abb123 100644
>> --- a/tests/functional/meson.build
>> +++ b/tests/functional/meson.build
>> @@ -13,6 +13,7 @@ endif
>>    test_timeouts = {
>>      'aarch64_aspeed_ast2700' : 600,
>>      'aarch64_aspeed_ast2700fc' : 600,
>> +  'aarch64_device_passthrough' : 720,
>>      'aarch64_imx8mp_evk' : 240,
>>      'aarch64_raspi4' : 480,
>>      'aarch64_reverse_debug' : 180,
>> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
>>    tests_aarch64_system_thorough = [
>>      'aarch64_aspeed_ast2700',
>>      'aarch64_aspeed_ast2700fc',
>> +  'aarch64_device_passthrough',
>>      'aarch64_imx8mp_evk',
>>      'aarch64_raspi3',
>>      'aarch64_raspi4',
>> diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
>> new file mode 100755
>> index 00000000000..1f3f158a9ff
>> --- /dev/null
>> +++ b/tests/functional/test_aarch64_device_passthrough.py
>> @@ -0,0 +1,142 @@
>> +#!/usr/bin/env python3
>> +#
>> +# Boots a nested guest and compare content of a device (passthrough) to a
>> +# reference image. Both vfio group and iommufd passthrough methods are tested.
>> +#
>> +# Copyright (c) 2025 Linaro Ltd.
>> +#
>> +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>> +#
>> +# SPDX-License-Identifier: GPL-2.0-or-later
>> +
>> +import os
>> +
>> +from qemu_test import QemuSystemTest, Asset
>> +from qemu_test import exec_command, wait_for_console_pattern
>> +from qemu_test import exec_command_and_wait_for_pattern
>> +from random import randbytes
>> +
>> +guest_script = '''
>> +#!/usr/bin/env bash
>> +
>> +set -euo pipefail
>> +set -x
>> +
>> +# find disks from nvme serial
>> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
>> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
>> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
>> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
>> +
>> +# bind disks to vfio
>> +for p in "$pci_vfio" "$pci_iommufd"; do
>> +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
>> +        continue
>> +    fi
>> +    echo $p > /sys/bus/pci/drivers/nvme/unbind
>> +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
>> +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
>> +done
>> +
>> +# boot nested guest and execute /host/nested_guest.sh
>> +# one disk is passed through vfio group, the other, through iommufd
>> +qemu-system-aarch64 \
> 
> Is this binary on the host.ext4 image ?
> 

Yes, it is the "vanilla" QEMU package from debian:trixie.
It would be better to cross compile current QEMU revision, and run that, 
but it's not something easily feasible with current CI architecture.

> If so, the test is testing a chosen QEMU version compiled in the
> L1 guest image but not the current QEMU version, which is the one
> running in the L0.
> 

Yes, this limits the scope of the test, I agree.

> Anyhow this is a very nice test and an excellent base to build on.
> As a next step, I’d suggest including tests with NICs using igb
> devices and igb virtual functions (VFs).
>

Sure, it can easily be extended with any scenario.
The key point (where I would like to get review from tests maintainers) 
is to know if using embedded scripts is ok, vs sequential interaction 
with stdio.

> It would also be great to run the L1 environment using the current
> version of QEMU. I haven't found a clean way to achieve that yet :/
>

Any scenario where we start to trigger a compilation will be painfully 
slow unfortunately.
It's clearly a limit of the testing infrastructure we have, and it 
prevents having more nested scenarios.

> Thanks,
> 
> C.
>

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Pierrick Bouvier]

On 7/2/25 6:48 AM, Cédric Le Goater wrote:
> On 7/2/25 15:35, Alex Bennée wrote:
>> Cédric Le Goater <clg@kaod.org> writes:
>>
>>> Hello,
>>>
>>> On 6/27/25 22:02, Pierrick Bouvier wrote:
>>>> This test allows to document and exercise device passthrough, using a
>>>> nested virtual machine setup. Two disks are generated and passed to the
>>>> VM, and their content is compared to original images.
>>>> Guest and nested guests commands are executed through two scripts,
>>>> and
>>>> init used in both system is configured to trigger a kernel panic in case
>>>> any command fails. This is more reliable and readable than executing all
>>>> commands through prompt injection and trying to guess what failed.
>>>> Initially, this test was supposed to test smmuv3 nested emulation
>>>> (combining both stages of translation), but I could not find any setup
>>>> (kernel + vmm) able to do the passthrough correctly, despite several
>>>> tries.
>>>> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>>>> ---
>>>>     tests/functional/meson.build                  |   2 +
>>>>     .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>>>>     2 files changed, 144 insertions(+)
>>>>     create mode 100755 tests/functional/test_aarch64_device_passthrough.py
>>>> diff --git a/tests/functional/meson.build
>>>> b/tests/functional/meson.build
>>>> index 3021928a9d4..6cc78abb123 100644
>>>> --- a/tests/functional/meson.build
>>>> +++ b/tests/functional/meson.build
>>>> @@ -13,6 +13,7 @@ endif
>>>>     test_timeouts = {
>>>>       'aarch64_aspeed_ast2700' : 600,
>>>>       'aarch64_aspeed_ast2700fc' : 600,
>>>> +  'aarch64_device_passthrough' : 720,
>>>>       'aarch64_imx8mp_evk' : 240,
>>>>       'aarch64_raspi4' : 480,
>>>>       'aarch64_reverse_debug' : 180,
>>>> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
>>>>     tests_aarch64_system_thorough = [
>>>>       'aarch64_aspeed_ast2700',
>>>>       'aarch64_aspeed_ast2700fc',
>>>> +  'aarch64_device_passthrough',
>>>>       'aarch64_imx8mp_evk',
>>>>       'aarch64_raspi3',
>>>>       'aarch64_raspi4',
>>>> diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
>>>> new file mode 100755
>>>> index 00000000000..1f3f158a9ff
>>>> --- /dev/null
>>>> +++ b/tests/functional/test_aarch64_device_passthrough.py
>>>> @@ -0,0 +1,142 @@
>>>> +#!/usr/bin/env python3
>>>> +#
>>>> +# Boots a nested guest and compare content of a device (passthrough) to a
>>>> +# reference image. Both vfio group and iommufd passthrough methods are tested.
>>>> +#
>>>> +# Copyright (c) 2025 Linaro Ltd.
>>>> +#
>>>> +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>>>> +#
>>>> +# SPDX-License-Identifier: GPL-2.0-or-later
>>>> +
>>>> +import os
>>>> +
>>>> +from qemu_test import QemuSystemTest, Asset
>>>> +from qemu_test import exec_command, wait_for_console_pattern
>>>> +from qemu_test import exec_command_and_wait_for_pattern
>>>> +from random import randbytes
>>>> +
>>>> +guest_script = '''
>>>> +#!/usr/bin/env bash
>>>> +
>>>> +set -euo pipefail
>>>> +set -x
>>>> +
>>>> +# find disks from nvme serial
>>>> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
>>>> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
>>>> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
>>>> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
>>>> +
>>>> +# bind disks to vfio
>>>> +for p in "$pci_vfio" "$pci_iommufd"; do
>>>> +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
>>>> +        continue
>>>> +    fi
>>>> +    echo $p > /sys/bus/pci/drivers/nvme/unbind
>>>> +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
>>>> +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
>>>> +done
>>>> +
>>>> +# boot nested guest and execute /host/nested_guest.sh
>>>> +# one disk is passed through vfio group, the other, through iommufd
>>>> +qemu-system-aarch64 \
>>>
>>> Is this binary on the host.ext4 image ?
>>>
>>> If so, the test is testing a chosen QEMU version compiled in the
>>> L1 guest image but not the current QEMU version, which is the one
>>> running in the L0.
>>>
>>> Anyhow this is a very nice test and an excellent base to build on.
>>> As a next step, I’d suggest including tests with NICs using igb
>>> devices and igb virtual functions (VFs).
>>>
>>> It would also be great to run the L1 environment using the current
>>> version of QEMU. I haven't found a clean way to achieve that yet :/
>>
>> I sometimes boot up with a virtiofsd mapped to $HOME but it gets a
>> little unstable over time and I haven't had a chance to figure out where
>> things where going wrong.
>>
>> We have the containers to reliably build a cross image of QEMU but we
>> would have to ensure the guest image matches so we don't run into
>> library skew issues.
> 
> and could we generate a disk image from a container build, keep the
> artifact and use it in the functional test to boot the L1 guest  ?
>

It could be an idea. But then it pushes a strong requirement to have 
exactly the same environment running in the VM. For instance, the 
current software stack in v2 is using alpine (while using debian on v1), 
so you'd have to be very careful how you cross compile.

That said, it may be an opportunity to provide a gold/standard test 
setup (based on debian (or any other distro), with full batteries 
setup), and assume this to run several functional tests based on it.
This way, the same "fat" image could be reused with different functional 
tests, and this would allow to provide current QEMU branch as part of 
the container (using virtfs or virtiofsd. I strongly prefer the former, 
which does not rely on any external binary to be built/launched).

> 
> Thanks,
> 
> C.
> 

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Pierrick Bouvier]

On 7/2/25 6:09 AM, Daniel P. Berrangé wrote:
> On Fri, Jun 27, 2025 at 01:02:22PM -0700, Pierrick Bouvier wrote:
>> This test allows to document and exercise device passthrough, using a
>> nested virtual machine setup. Two disks are generated and passed to the
>> VM, and their content is compared to original images.
>>
>> Guest and nested guests commands are executed through two scripts, and
>> init used in both system is configured to trigger a kernel panic in case
>> any command fails. This is more reliable and readable than executing all
>> commands through prompt injection and trying to guess what failed.
>>
>> Initially, this test was supposed to test smmuv3 nested emulation
>> (combining both stages of translation), but I could not find any setup
>> (kernel + vmm) able to do the passthrough correctly, despite several
>> tries.
>>
>> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>> ---
>>   tests/functional/meson.build                  |   2 +
>>   .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>>   2 files changed, 144 insertions(+)
>>   create mode 100755 tests/functional/test_aarch64_device_passthrough.py
>>
>> diff --git a/tests/functional/meson.build b/tests/functional/meson.build
>> index 3021928a9d4..6cc78abb123 100644
>> --- a/tests/functional/meson.build
>> +++ b/tests/functional/meson.build
>> @@ -13,6 +13,7 @@ endif
>>   test_timeouts = {
>>     'aarch64_aspeed_ast2700' : 600,
>>     'aarch64_aspeed_ast2700fc' : 600,
>> +  'aarch64_device_passthrough' : 720,
>>     'aarch64_imx8mp_evk' : 240,
>>     'aarch64_raspi4' : 480,
>>     'aarch64_reverse_debug' : 180,
>> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
>>   tests_aarch64_system_thorough = [
>>     'aarch64_aspeed_ast2700',
>>     'aarch64_aspeed_ast2700fc',
>> +  'aarch64_device_passthrough',
>>     'aarch64_imx8mp_evk',
>>     'aarch64_raspi3',
>>     'aarch64_raspi4',
>> diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
>> new file mode 100755
>> index 00000000000..1f3f158a9ff
>> --- /dev/null
>> +++ b/tests/functional/test_aarch64_device_passthrough.py
>> @@ -0,0 +1,142 @@
>> +#!/usr/bin/env python3
>> +#
>> +# Boots a nested guest and compare content of a device (passthrough) to a
>> +# reference image. Both vfio group and iommufd passthrough methods are tested.
>> +#
>> +# Copyright (c) 2025 Linaro Ltd.
>> +#
>> +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>> +#
>> +# SPDX-License-Identifier: GPL-2.0-or-later
>> +
>> +import os
>> +
>> +from qemu_test import QemuSystemTest, Asset
>> +from qemu_test import exec_command, wait_for_console_pattern
>> +from qemu_test import exec_command_and_wait_for_pattern
>> +from random import randbytes
>> +
>> +guest_script = '''
>> +#!/usr/bin/env bash
>> +
>> +set -euo pipefail
>> +set -x
>> +
>> +# find disks from nvme serial
>> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
>> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
>> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
>> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
>> +
>> +# bind disks to vfio
>> +for p in "$pci_vfio" "$pci_iommufd"; do
>> +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
>> +        continue
>> +    fi
>> +    echo $p > /sys/bus/pci/drivers/nvme/unbind
>> +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
>> +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
>> +done
>> +
>> +# boot nested guest and execute /host/nested_guest.sh
>> +# one disk is passed through vfio group, the other, through iommufd
>> +qemu-system-aarch64 \
>> +-M virt \
>> +-display none \
>> +-serial stdio \
>> +-cpu host \
>> +-enable-kvm \
>> +-m 1G \
>> +-kernel /host/Image.gz \
>> +-drive format=raw,file=/host/guest.ext4,if=virtio \
>> +-append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \
>> +-virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \
>> +-device vfio-pci,host=$pci_vfio \
>> +-object iommufd,id=iommufd0 \
>> +-device vfio-pci,host=$pci_iommufd,iommufd=iommufd0
>> +'''
>> +
>> +nested_guest_script = '''
>> +#!/usr/bin/env bash
>> +
>> +set -euo pipefail
>> +set -x
>> +
>> +image_vfio=/host/disk_vfio
>> +image_iommufd=/host/disk_iommufd
>> +
>> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
>> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
>> +
>> +# compare if devices are identical to original images
>> +diff $image_vfio /dev/$dev_vfio
>> +diff $image_iommufd /dev/$dev_iommufd
>> +
>> +echo device_passthrough_test_ok
>> +'''
>> +
>> +class Aarch64DevicePassthrough(QemuSystemTest):
>> +
>> +    # https://github.com/pbo-linaro/qemu-linux-stack
>> +    #
>> +    # Linux kernel is compiled with defconfig +
>> +    # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD
>> +    # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde
>> +    ASSET_DEVICE_PASSTHROUGH_STACK = Asset(
>> +        ('https://fileserver.linaro.org/s/fx5DXxBYme8dw2G/'
>> +         'download/device_passthrough.tar.xz'),
>> +         '812750b664d61c2986f2b149939ae28cafbd60d53e9c7e4b16e97143845e196d')
>> +
>> +    # This tests the device passthrough implementation, by booting a VM
>> +    # supporting it with two nvme disks attached, and launching a nested VM
>> +    # reading their content.
>> +    def test_aarch64_device_passthrough(self):
>> +        self.set_machine('virt')
>> +        self.require_accelerator('tcg')
>> +
>> +        self.vm.set_console()
>> +
>> +        stack_path_tar_gz = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch()
>> +        self.archive_extract(stack_path_tar_gz, format="tar")
>> +
>> +        stack = self.scratch_file('out')
>> +        kernel = os.path.join(stack, 'Image.gz')
>> +        rootfs_host = os.path.join(stack, 'host.ext4')
>> +        disk_vfio = os.path.join(stack, 'disk_vfio')
>> +        disk_iommufd = os.path.join(stack, 'disk_iommufd')
>> +        guest_cmd = os.path.join(stack, 'guest.sh')
>> +        nested_guest_cmd = os.path.join(stack, 'nested_guest.sh')
> 
> Don't incrementally create paths like this - use the
> 'scratch_file' method for all components
> 
>   ie
> 
>     kernel = self.scratch_file('out', 'Image.gz')
>     rootfs_host =  self.scratch_file('out', 'host.ext4')
>     ...etc...
>

May I ask what's the benefit of this?

It forces you to repeat full path (luckily short in this case), but I 
don't see the difference with simply joining paths.

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Daniel P. Berrangé]

On Wed, Jul 02, 2025 at 01:08:41PM -0700, Pierrick Bouvier wrote:
> On 7/2/25 6:09 AM, Daniel P. Berrangé wrote:
> > On Fri, Jun 27, 2025 at 01:02:22PM -0700, Pierrick Bouvier wrote:
> > > This test allows to document and exercise device passthrough, using a
> > > nested virtual machine setup. Two disks are generated and passed to the
> > > VM, and their content is compared to original images.
> > > 
> > > Guest and nested guests commands are executed through two scripts, and
> > > init used in both system is configured to trigger a kernel panic in case
> > > any command fails. This is more reliable and readable than executing all
> > > commands through prompt injection and trying to guess what failed.
> > > 
> > > Initially, this test was supposed to test smmuv3 nested emulation
> > > (combining both stages of translation), but I could not find any setup
> > > (kernel + vmm) able to do the passthrough correctly, despite several
> > > tries.
> > > 
> > > Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
> > > ---
> > >   tests/functional/meson.build                  |   2 +
> > >   .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
> > >   2 files changed, 144 insertions(+)
> > >   create mode 100755 tests/functional/test_aarch64_device_passthrough.py
> > > 
> > > diff --git a/tests/functional/meson.build b/tests/functional/meson.build
> > > index 3021928a9d4..6cc78abb123 100644
> > > --- a/tests/functional/meson.build
> > > +++ b/tests/functional/meson.build
> > > @@ -13,6 +13,7 @@ endif
> > >   test_timeouts = {
> > >     'aarch64_aspeed_ast2700' : 600,
> > >     'aarch64_aspeed_ast2700fc' : 600,
> > > +  'aarch64_device_passthrough' : 720,
> > >     'aarch64_imx8mp_evk' : 240,
> > >     'aarch64_raspi4' : 480,
> > >     'aarch64_reverse_debug' : 180,
> > > @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
> > >   tests_aarch64_system_thorough = [
> > >     'aarch64_aspeed_ast2700',
> > >     'aarch64_aspeed_ast2700fc',
> > > +  'aarch64_device_passthrough',
> > >     'aarch64_imx8mp_evk',
> > >     'aarch64_raspi3',
> > >     'aarch64_raspi4',
> > > diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
> > > new file mode 100755
> > > index 00000000000..1f3f158a9ff
> > > --- /dev/null
> > > +++ b/tests/functional/test_aarch64_device_passthrough.py
> > > @@ -0,0 +1,142 @@
> > > +#!/usr/bin/env python3
> > > +#
> > > +# Boots a nested guest and compare content of a device (passthrough) to a
> > > +# reference image. Both vfio group and iommufd passthrough methods are tested.
> > > +#
> > > +# Copyright (c) 2025 Linaro Ltd.
> > > +#
> > > +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
> > > +#
> > > +# SPDX-License-Identifier: GPL-2.0-or-later
> > > +
> > > +import os
> > > +
> > > +from qemu_test import QemuSystemTest, Asset
> > > +from qemu_test import exec_command, wait_for_console_pattern
> > > +from qemu_test import exec_command_and_wait_for_pattern
> > > +from random import randbytes
> > > +
> > > +guest_script = '''
> > > +#!/usr/bin/env bash
> > > +
> > > +set -euo pipefail
> > > +set -x
> > > +
> > > +# find disks from nvme serial
> > > +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
> > > +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
> > > +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
> > > +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
> > > +
> > > +# bind disks to vfio
> > > +for p in "$pci_vfio" "$pci_iommufd"; do
> > > +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
> > > +        continue
> > > +    fi
> > > +    echo $p > /sys/bus/pci/drivers/nvme/unbind
> > > +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
> > > +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
> > > +done
> > > +
> > > +# boot nested guest and execute /host/nested_guest.sh
> > > +# one disk is passed through vfio group, the other, through iommufd
> > > +qemu-system-aarch64 \
> > > +-M virt \
> > > +-display none \
> > > +-serial stdio \
> > > +-cpu host \
> > > +-enable-kvm \
> > > +-m 1G \
> > > +-kernel /host/Image.gz \
> > > +-drive format=raw,file=/host/guest.ext4,if=virtio \
> > > +-append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \
> > > +-virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \
> > > +-device vfio-pci,host=$pci_vfio \
> > > +-object iommufd,id=iommufd0 \
> > > +-device vfio-pci,host=$pci_iommufd,iommufd=iommufd0
> > > +'''
> > > +
> > > +nested_guest_script = '''
> > > +#!/usr/bin/env bash
> > > +
> > > +set -euo pipefail
> > > +set -x
> > > +
> > > +image_vfio=/host/disk_vfio
> > > +image_iommufd=/host/disk_iommufd
> > > +
> > > +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
> > > +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
> > > +
> > > +# compare if devices are identical to original images
> > > +diff $image_vfio /dev/$dev_vfio
> > > +diff $image_iommufd /dev/$dev_iommufd
> > > +
> > > +echo device_passthrough_test_ok
> > > +'''
> > > +
> > > +class Aarch64DevicePassthrough(QemuSystemTest):
> > > +
> > > +    # https://github.com/pbo-linaro/qemu-linux-stack
> > > +    #
> > > +    # Linux kernel is compiled with defconfig +
> > > +    # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD
> > > +    # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde
> > > +    ASSET_DEVICE_PASSTHROUGH_STACK = Asset(
> > > +        ('https://fileserver.linaro.org/s/fx5DXxBYme8dw2G/'
> > > +         'download/device_passthrough.tar.xz'),
> > > +         '812750b664d61c2986f2b149939ae28cafbd60d53e9c7e4b16e97143845e196d')
> > > +
> > > +    # This tests the device passthrough implementation, by booting a VM
> > > +    # supporting it with two nvme disks attached, and launching a nested VM
> > > +    # reading their content.
> > > +    def test_aarch64_device_passthrough(self):
> > > +        self.set_machine('virt')
> > > +        self.require_accelerator('tcg')
> > > +
> > > +        self.vm.set_console()
> > > +
> > > +        stack_path_tar_gz = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch()
> > > +        self.archive_extract(stack_path_tar_gz, format="tar")
> > > +
> > > +        stack = self.scratch_file('out')
> > > +        kernel = os.path.join(stack, 'Image.gz')
> > > +        rootfs_host = os.path.join(stack, 'host.ext4')
> > > +        disk_vfio = os.path.join(stack, 'disk_vfio')
> > > +        disk_iommufd = os.path.join(stack, 'disk_iommufd')
> > > +        guest_cmd = os.path.join(stack, 'guest.sh')
> > > +        nested_guest_cmd = os.path.join(stack, 'nested_guest.sh')
> > 
> > Don't incrementally create paths like this - use the
> > 'scratch_file' method for all components
> > 
> >   ie
> > 
> >     kernel = self.scratch_file('out', 'Image.gz')
> >     rootfs_host =  self.scratch_file('out', 'host.ext4')
> >     ...etc...
> > 
> 
> May I ask what's the benefit of this?
> 
> It forces you to repeat full path (luckily short in this case), but I don't
> see the difference with simply joining paths.

The intent is to eliminate use of os.path.* in general, such that instead
of passing around strings, we can eventually pass around pathlib.Path
objects.

With regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

Re: [PATCH v2] tests/functional: test device passthrough on aarch64

[Pierrick Bouvier]

On 7/3/25 2:32 AM, Daniel P. Berrangé wrote:
> On Wed, Jul 02, 2025 at 01:08:41PM -0700, Pierrick Bouvier wrote:
>> On 7/2/25 6:09 AM, Daniel P. Berrangé wrote:
>>> On Fri, Jun 27, 2025 at 01:02:22PM -0700, Pierrick Bouvier wrote:
>>>> This test allows to document and exercise device passthrough, using a
>>>> nested virtual machine setup. Two disks are generated and passed to the
>>>> VM, and their content is compared to original images.
>>>>
>>>> Guest and nested guests commands are executed through two scripts, and
>>>> init used in both system is configured to trigger a kernel panic in case
>>>> any command fails. This is more reliable and readable than executing all
>>>> commands through prompt injection and trying to guess what failed.
>>>>
>>>> Initially, this test was supposed to test smmuv3 nested emulation
>>>> (combining both stages of translation), but I could not find any setup
>>>> (kernel + vmm) able to do the passthrough correctly, despite several
>>>> tries.
>>>>
>>>> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>>>> ---
>>>>    tests/functional/meson.build                  |   2 +
>>>>    .../test_aarch64_device_passthrough.py        | 142 ++++++++++++++++++
>>>>    2 files changed, 144 insertions(+)
>>>>    create mode 100755 tests/functional/test_aarch64_device_passthrough.py
>>>>
>>>> diff --git a/tests/functional/meson.build b/tests/functional/meson.build
>>>> index 3021928a9d4..6cc78abb123 100644
>>>> --- a/tests/functional/meson.build
>>>> +++ b/tests/functional/meson.build
>>>> @@ -13,6 +13,7 @@ endif
>>>>    test_timeouts = {
>>>>      'aarch64_aspeed_ast2700' : 600,
>>>>      'aarch64_aspeed_ast2700fc' : 600,
>>>> +  'aarch64_device_passthrough' : 720,
>>>>      'aarch64_imx8mp_evk' : 240,
>>>>      'aarch64_raspi4' : 480,
>>>>      'aarch64_reverse_debug' : 180,
>>>> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
>>>>    tests_aarch64_system_thorough = [
>>>>      'aarch64_aspeed_ast2700',
>>>>      'aarch64_aspeed_ast2700fc',
>>>> +  'aarch64_device_passthrough',
>>>>      'aarch64_imx8mp_evk',
>>>>      'aarch64_raspi3',
>>>>      'aarch64_raspi4',
>>>> diff --git a/tests/functional/test_aarch64_device_passthrough.py b/tests/functional/test_aarch64_device_passthrough.py
>>>> new file mode 100755
>>>> index 00000000000..1f3f158a9ff
>>>> --- /dev/null
>>>> +++ b/tests/functional/test_aarch64_device_passthrough.py
>>>> @@ -0,0 +1,142 @@
>>>> +#!/usr/bin/env python3
>>>> +#
>>>> +# Boots a nested guest and compare content of a device (passthrough) to a
>>>> +# reference image. Both vfio group and iommufd passthrough methods are tested.
>>>> +#
>>>> +# Copyright (c) 2025 Linaro Ltd.
>>>> +#
>>>> +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org>
>>>> +#
>>>> +# SPDX-License-Identifier: GPL-2.0-or-later
>>>> +
>>>> +import os
>>>> +
>>>> +from qemu_test import QemuSystemTest, Asset
>>>> +from qemu_test import exec_command, wait_for_console_pattern
>>>> +from qemu_test import exec_command_and_wait_for_pattern
>>>> +from random import randbytes
>>>> +
>>>> +guest_script = '''
>>>> +#!/usr/bin/env bash
>>>> +
>>>> +set -euo pipefail
>>>> +set -x
>>>> +
>>>> +# find disks from nvme serial
>>>> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
>>>> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
>>>> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
>>>> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
>>>> +
>>>> +# bind disks to vfio
>>>> +for p in "$pci_vfio" "$pci_iommufd"; do
>>>> +    if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
>>>> +        continue
>>>> +    fi
>>>> +    echo $p > /sys/bus/pci/drivers/nvme/unbind
>>>> +    echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
>>>> +    echo $p > /sys/bus/pci/drivers/vfio-pci/bind
>>>> +done
>>>> +
>>>> +# boot nested guest and execute /host/nested_guest.sh
>>>> +# one disk is passed through vfio group, the other, through iommufd
>>>> +qemu-system-aarch64 \
>>>> +-M virt \
>>>> +-display none \
>>>> +-serial stdio \
>>>> +-cpu host \
>>>> +-enable-kvm \
>>>> +-m 1G \
>>>> +-kernel /host/Image.gz \
>>>> +-drive format=raw,file=/host/guest.ext4,if=virtio \
>>>> +-append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \
>>>> +-virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \
>>>> +-device vfio-pci,host=$pci_vfio \
>>>> +-object iommufd,id=iommufd0 \
>>>> +-device vfio-pci,host=$pci_iommufd,iommufd=iommufd0
>>>> +'''
>>>> +
>>>> +nested_guest_script = '''
>>>> +#!/usr/bin/env bash
>>>> +
>>>> +set -euo pipefail
>>>> +set -x
>>>> +
>>>> +image_vfio=/host/disk_vfio
>>>> +image_iommufd=/host/disk_iommufd
>>>> +
>>>> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
>>>> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
>>>> +
>>>> +# compare if devices are identical to original images
>>>> +diff $image_vfio /dev/$dev_vfio
>>>> +diff $image_iommufd /dev/$dev_iommufd
>>>> +
>>>> +echo device_passthrough_test_ok
>>>> +'''
>>>> +
>>>> +class Aarch64DevicePassthrough(QemuSystemTest):
>>>> +
>>>> +    # https://github.com/pbo-linaro/qemu-linux-stack
>>>> +    #
>>>> +    # Linux kernel is compiled with defconfig +
>>>> +    # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD
>>>> +    # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde
>>>> +    ASSET_DEVICE_PASSTHROUGH_STACK = Asset(
>>>> +        ('https://fileserver.linaro.org/s/fx5DXxBYme8dw2G/'
>>>> +         'download/device_passthrough.tar.xz'),
>>>> +         '812750b664d61c2986f2b149939ae28cafbd60d53e9c7e4b16e97143845e196d')
>>>> +
>>>> +    # This tests the device passthrough implementation, by booting a VM
>>>> +    # supporting it with two nvme disks attached, and launching a nested VM
>>>> +    # reading their content.
>>>> +    def test_aarch64_device_passthrough(self):
>>>> +        self.set_machine('virt')
>>>> +        self.require_accelerator('tcg')
>>>> +
>>>> +        self.vm.set_console()
>>>> +
>>>> +        stack_path_tar_gz = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch()
>>>> +        self.archive_extract(stack_path_tar_gz, format="tar")
>>>> +
>>>> +        stack = self.scratch_file('out')
>>>> +        kernel = os.path.join(stack, 'Image.gz')
>>>> +        rootfs_host = os.path.join(stack, 'host.ext4')
>>>> +        disk_vfio = os.path.join(stack, 'disk_vfio')
>>>> +        disk_iommufd = os.path.join(stack, 'disk_iommufd')
>>>> +        guest_cmd = os.path.join(stack, 'guest.sh')
>>>> +        nested_guest_cmd = os.path.join(stack, 'nested_guest.sh')
>>>
>>> Don't incrementally create paths like this - use the
>>> 'scratch_file' method for all components
>>>
>>>    ie
>>>
>>>      kernel = self.scratch_file('out', 'Image.gz')
>>>      rootfs_host =  self.scratch_file('out', 'host.ext4')
>>>      ...etc...
>>>
>>
>> May I ask what's the benefit of this?
>>
>> It forces you to repeat full path (luckily short in this case), but I don't
>> see the difference with simply joining paths.
> 
> The intent is to eliminate use of os.path.* in general, such that instead
> of passing around strings, we can eventually pass around pathlib.Path
> objects.
>

Ok, thanks.
It seems that this patch as been pulled and merged already.

> With regards,
> Daniel