[PATCH 0/3] Fix NVMe, not only on Raspberry Pi 5

[PATCH 0/3] Fix NVMe, not only on Raspberry Pi 5

[Torsten Duwe]

Hi all,

This simple quick fix has now turned into a small series. It contains
the required changes to translate inbound access addresses, with the
handling of dynamically found devices going into its own patch, content
as suggested by Neil. While at it, I also added the NVMe enablement for
raspberries. It might already work on RPi4 setups (to be tested). RPi5
OTOH will require the other series to enable PCIe on the bcm2712.

Changes from the single patch:
  - the ternary in the macro is gone, explanation updated accordingly
  - instead, device_get_dma_constraints is enhanced to try harder
  - Raspberry Pi NVMe config added

Torsten Duwe (3):
  Skip parent device nodes without a DT reference when looking for
    dma-ranges
  Fix missing address translation for PCIe inbound access
  rpi: enable NVMe

 configs/rpi_arm64_defconfig |  1 +
 drivers/core/device.c       | 14 +++++++++++++-
 drivers/nvme/nvme.c         | 34 +++++++++++++++++++++-------------
 3 files changed, 35 insertions(+), 14 deletions(-)

-- 
2.54.0

[PATCH 1/3] core: Skip parent device nodes without a DT reference when looking for dma-ranges

[Torsten Duwe]

From: Torsten Duwe <duwe@suse.de>

If a device node got created dynamically, there is no guarantee that the
parent node has an associated device tree node which could specify dma
constraints. Especially PCI(e) enumeration adds intermediate "bus nodes",
also dynamically.

Try harder to find the correct configuration by walking up the tree until
a DT association is found.

Suggested-by: Neil Armstrong <neil.armstrong@linaro.org>
Signed-off-by: Torsten Duwe <duwe@suse.de>

---
 drivers/core/device.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/drivers/core/device.c b/drivers/core/device.c
index 779f371b9d5..9de64dbb3c5 100644
--- a/drivers/core/device.c
+++ b/drivers/core/device.c
@@ -459,7 +459,19 @@ static int device_get_dma_constraints(struct udevice *dev)
 	u64 size = 0;
 	int ret;
 
-	if (!CONFIG_IS_ENABLED(DM_DMA) || !parent || !dev_has_ofnode(parent))
+	if (!CONFIG_IS_ENABLED(DM_DMA) || !parent)
+		return 0;
+
+	/* Look for the first node in the parent chain */
+	while (parent) {
+		if (dev_has_ofnode(parent))
+			break;
+
+		parent = dev_get_parent(parent);
+	}
+
+	/* No parents have a node, bail out */
+	if (!parent)
 		return 0;
 
 	/*
-- 
2.54.0

[PATCH 2/3] nvme: Fix missing address translation for PCIe inbound access

[Torsten Duwe]

From: Torsten Duwe <duwe@suse.de>

U-Boot currently does not account for PCIe bridges with a non-zero
inbound access offset when talking NVMe, it only works on platforms
where this offset happens to be zero.

This patch enhances the NVMe driver with the ability to also handle
these cases.

Signed-off-by: Torsten Duwe <duwe@suse.de>

---
 drivers/nvme/nvme.c | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c
index 2b14437f69c..99a47bf25f2 100644
--- a/drivers/nvme/nvme.c
+++ b/drivers/nvme/nvme.c
@@ -12,6 +12,7 @@
 #include <log.h>
 #include <malloc.h>
 #include <memalign.h>
+#include <phys2bus.h>
 #include <time.h>
 #include <dm/device-internal.h>
 #include <linux/compat.h>
@@ -27,6 +28,13 @@
 #define IO_TIMEOUT		30
 #define MAX_PRP_POOL		512
 
+/*
+ * Convert a memory address to the value needed by the PCI device to
+ * access the given location, taking into account inbound window
+ * translations of PCI bridges:
+ */
+#define DEV_ADDR(a)		dev_phys_to_bus(dev->udev, (a))
+
 static int nvme_wait_csts(struct nvme_dev *dev, u32 mask, u32 val)
 {
 	int timeout;
@@ -91,12 +99,12 @@ static int nvme_setup_prps(struct nvme_dev *dev, u64 *prp2,
 	i = 0;
 	while (nprps) {
 		if ((i == (prps_per_page - 1)) && nprps > 1) {
-			*(prp_pool + i) = cpu_to_le64((ulong)prp_pool +
-					page_size);
+			*(prp_pool + i) = cpu_to_le64(DEV_ADDR((ulong)prp_pool +
+								page_size));
 			i = 0;
 			prp_pool += page_size;
 		}
-		*(prp_pool + i++) = cpu_to_le64(dma_addr);
+		*(prp_pool + i++) = cpu_to_le64(DEV_ADDR(dma_addr));
 		dma_addr += page_size;
 		nprps--;
 	}
@@ -393,8 +401,8 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
 	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
 
 	writel(aqa, &dev->bar->aqa);
-	nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
-	nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
+	nvme_writeq(DEV_ADDR((ulong)nvmeq->sq_cmds), &dev->bar->asq);
+	nvme_writeq(DEV_ADDR((ulong)nvmeq->cqes), &dev->bar->acq);
 
 	result = nvme_enable_ctrl(dev);
 	if (result)
@@ -420,7 +428,7 @@ static int nvme_alloc_cq(struct nvme_dev *dev, u16 qid,
 
 	memset(&c, 0, sizeof(c));
 	c.create_cq.opcode = nvme_admin_create_cq;
-	c.create_cq.prp1 = cpu_to_le64((ulong)nvmeq->cqes);
+	c.create_cq.prp1 = cpu_to_le64(DEV_ADDR((ulong)nvmeq->cqes));
 	c.create_cq.cqid = cpu_to_le16(qid);
 	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 	c.create_cq.cq_flags = cpu_to_le16(flags);
@@ -437,7 +445,7 @@ static int nvme_alloc_sq(struct nvme_dev *dev, u16 qid,
 
 	memset(&c, 0, sizeof(c));
 	c.create_sq.opcode = nvme_admin_create_sq;
-	c.create_sq.prp1 = cpu_to_le64((ulong)nvmeq->sq_cmds);
+	c.create_sq.prp1 = cpu_to_le64(DEV_ADDR((ulong)nvmeq->sq_cmds));
 	c.create_sq.sqid = cpu_to_le16(qid);
 	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 	c.create_sq.sq_flags = cpu_to_le16(flags);
@@ -458,14 +466,14 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
 	memset(&c, 0, sizeof(c));
 	c.identify.opcode = nvme_admin_identify;
 	c.identify.nsid = cpu_to_le32(nsid);
-	c.identify.prp1 = cpu_to_le64(dma_addr);
+	c.identify.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
 
 	length -= (page_size - offset);
 	if (length <= 0) {
 		c.identify.prp2 = 0;
 	} else {
 		dma_addr += (page_size - offset);
-		c.identify.prp2 = cpu_to_le64(dma_addr);
+		c.identify.prp2 = cpu_to_le64(DEV_ADDR(dma_addr));
 	}
 
 	c.identify.cns = cpu_to_le32(cns);
@@ -490,7 +498,7 @@ int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
 	memset(&c, 0, sizeof(c));
 	c.features.opcode = nvme_admin_get_features;
 	c.features.nsid = cpu_to_le32(nsid);
-	c.features.prp1 = cpu_to_le64(dma_addr);
+	c.features.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
 	c.features.fid = cpu_to_le32(fid);
 
 	ret = nvme_submit_admin_cmd(dev, &c, result);
@@ -516,7 +524,7 @@ int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
 
 	memset(&c, 0, sizeof(c));
 	c.features.opcode = nvme_admin_set_features;
-	c.features.prp1 = cpu_to_le64(dma_addr);
+	c.features.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
 	c.features.fid = cpu_to_le32(fid);
 	c.features.dword11 = cpu_to_le32(dword11);
 
@@ -785,8 +793,8 @@ static ulong nvme_blk_rw(struct udevice *udev, lbaint_t blknr,
 		c.rw.slba = cpu_to_le64(slba);
 		slba += lbas;
 		c.rw.length = cpu_to_le16(lbas - 1);
-		c.rw.prp1 = cpu_to_le64(temp_buffer);
-		c.rw.prp2 = cpu_to_le64(prp2);
+		c.rw.prp1 = cpu_to_le64(DEV_ADDR(temp_buffer));
+		c.rw.prp2 = cpu_to_le64(DEV_ADDR(prp2));
 		status = nvme_submit_sync_cmd(dev->queues[NVME_IO_Q],
 				&c, NULL, IO_TIMEOUT);
 		if (status)
-- 
2.54.0

[PATCH 3/3] configs: enable NVMe

[Torsten Duwe]

From: Torsten Duwe <duwe@suse.de>

Enable NVMe in the Raspberry Pi 64-Bit default config

Signed-off-by: Torsten Duwe <duwe@suse.de>
---

Thanks to this series, U-Boot now supports the 64GiB address offset
used on the Raspberry Pi 5 for those inbound accesses.

Remember that the other prerequisite on Rpi5 is the PCIe support
for bcm2712.

---
 configs/rpi_arm64_defconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/configs/rpi_arm64_defconfig b/configs/rpi_arm64_defconfig
index 38af5029403..18f8213f3b3 100644
--- a/configs/rpi_arm64_defconfig
+++ b/configs/rpi_arm64_defconfig
@@ -41,6 +41,7 @@ CONFIG_MMC_SDHCI_SDMA=y
 CONFIG_MMC_SDHCI_BCM2835=y
 CONFIG_MMC_SDHCI_BCMSTB=y
 CONFIG_BCMGENET=y
+CONFIG_NVME_PCI=y
 CONFIG_PCI_BRCMSTB=y
 CONFIG_PINCTRL=y
 # CONFIG_PINCTRL_GENERIC is not set
-- 
2.54.0

Re: [PATCH 3/3] configs: enable NVMe

[Peter Robinson]

On Fri, 8 May 2026 at 16:42, Torsten Duwe <duwe@lst.de> wrote:
>
> From: Torsten Duwe <duwe@suse.de>
>
> Enable NVMe in the Raspberry Pi 64-Bit default config
>
> Signed-off-by: Torsten Duwe <duwe@suse.de>
> ---
>
> Thanks to this series, U-Boot now supports the 64GiB address offset
> used on the Raspberry Pi 5 for those inbound accesses.
>
> Remember that the other prerequisite on Rpi5 is the PCIe support
> for bcm2712.

I am in the process of testing this across my devices, awaiting a PCIe
-> M.2 adapter so I can test on the CM4.

One minor nit, I think we should scan for NVME drives:

-CONFIG_PREBOOT="pci enum; usb start;"
+CONFIG_PREBOOT="pci enum; usb start; nvme scan;"

Peter

> ---
>  configs/rpi_arm64_defconfig | 1 +
>  1 file changed, 1 insertion(+)
>
> diff --git a/configs/rpi_arm64_defconfig b/configs/rpi_arm64_defconfig
> index 38af5029403..18f8213f3b3 100644
> --- a/configs/rpi_arm64_defconfig
> +++ b/configs/rpi_arm64_defconfig
> @@ -41,6 +41,7 @@ CONFIG_MMC_SDHCI_SDMA=y
>  CONFIG_MMC_SDHCI_BCM2835=y
>  CONFIG_MMC_SDHCI_BCMSTB=y
>  CONFIG_BCMGENET=y
> +CONFIG_NVME_PCI=y
>  CONFIG_PCI_BRCMSTB=y
>  CONFIG_PINCTRL=y
>  # CONFIG_PINCTRL_GENERIC is not set
> --
> 2.54.0
>

Re: [PATCH 1/3] core: Skip parent device nodes without a DT reference when looking for dma-ranges

[Simon Glass]

Hi Torsten,

On Fri, 8 May 2026 at 09:42, Torsten Duwe <duwe@lst.de> wrote:
>
> From: Torsten Duwe <duwe@suse.de>
>
> If a device node got created dynamically, there is no guarantee that the
> parent node has an associated device tree node which could specify dma
> constraints. Especially PCI(e) enumeration adds intermediate "bus nodes",
> also dynamically.
>
> Try harder to find the correct configuration by walking up the tree until
> a DT association is found.
>
> Suggested-by: Neil Armstrong <neil.armstrong@linaro.org>
> Signed-off-by: Torsten Duwe <duwe@suse.de>
>
> ---
>  drivers/core/device.c | 14 +++++++++++++-
>  1 file changed, 13 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/core/device.c b/drivers/core/device.c
> index 779f371b9d5..9de64dbb3c5 100644
> --- a/drivers/core/device.c
> +++ b/drivers/core/device.c
> @@ -459,7 +459,19 @@ static int device_get_dma_constraints(struct udevice *dev)
>         u64 size = 0;
>         int ret;
>
> -       if (!CONFIG_IS_ENABLED(DM_DMA) || !parent || !dev_has_ofnode(parent))
> +       if (!CONFIG_IS_ENABLED(DM_DMA) || !parent)
> +               return 0;
> +
> +       /* Look for the first node in the parent chain */
> +       while (parent) {
> +               if (dev_has_ofnode(parent))
> +                       break;
> +
> +               parent = dev_get_parent(parent);
> +       }
> +
> +       /* No parents have a node, bail out */
> +       if (!parent)
>                 return 0;
>
>         /*
> --
> 2.54.0
>

There's already a unit test for device_get_dma_constraints() in
test/dm/read.c (dm_test_dma_ranges). Since you are extending it to
walk past nodes without an ofnode, please can you add a case
exercising the new path - a sandbox device whose immediate parent has
no DT reference should do it.

Regards,
Simon

Re: [PATCH 1/3] core: Skip parent device nodes without a DT reference when looking for dma-ranges

[Torsten Duwe]

On Fri, 8 May 2026 12:47:45 -0600
Simon Glass <sjg@chromium.org> wrote:
[...]
> There's already a unit test for device_get_dma_constraints() in
> test/dm/read.c (dm_test_dma_ranges). Since you are extending it to
> walk past nodes without an ofnode, please can you add a case
> exercising the new path - a sandbox device whose immediate parent has
> no DT reference should do it.

I'll have a look.

Talking unit tests, is it just me who experiences reproducible crashes
on RPi5 with today's master branch?

	Torsten

Re: [PATCH 1/3] core: Skip parent device nodes without a DT reference when looking for dma-ranges

[Peter Robinson]

On Fri, 8 May 2026 at 20:42, Torsten Duwe <duwe@lst.de> wrote:
>
> On Fri, 8 May 2026 12:47:45 -0600
> Simon Glass <sjg@chromium.org> wrote:
> [...]
> > There's already a unit test for device_get_dma_constraints() in
> > test/dm/read.c (dm_test_dma_ranges). Since you are extending it to
> > walk past nodes without an ofnode, please can you add a case
> > exercising the new path - a sandbox device whose immediate parent has
> > no DT reference should do it.
>
> I'll have a look.
>
> Talking unit tests, is it just me who experiences reproducible crashes
> on RPi5 with today's master branch?

Not just you, I saw crashes when I tested rc1 but I've not had time to
bisect it and see what other platforms are affected.

Re: [PATCH 1/3] core: Skip parent device nodes without a DT reference when looking for dma-ranges

[Simon Glass]

Hi Peter,

On Fri, 8 May 2026 at 13:56, Peter Robinson <pbrobinson@gmail.com> wrote:
>
> On Fri, 8 May 2026 at 20:42, Torsten Duwe <duwe@lst.de> wrote:
> >
> > On Fri, 8 May 2026 12:47:45 -0600
> > Simon Glass <sjg@chromium.org> wrote:
> > [...]
> > > There's already a unit test for device_get_dma_constraints() in
> > > test/dm/read.c (dm_test_dma_ranges). Since you are extending it to
> > > walk past nodes without an ofnode, please can you add a case
> > > exercising the new path - a sandbox device whose immediate parent has
> > > no DT reference should do it.
> >
> > I'll have a look.
> >
> > Talking unit tests, is it just me who experiences reproducible crashes
> > on RPi5 with today's master branch?
>
> Not just you, I saw crashes when I tested rc1 but I've not had time to
> bisect it and see what other platforms are affected.

I just did a bisect and this is the start and end of it:

HEAD is now at 88dc2788777 Prepare v2026.04
✔ ~/u [v2026.04|✔]
17:08 $ ub-smoke rpi5
.
                    [100%]
1 passed in 29.02s
✔ ~/u [v2026.04|…1]
17:08 $ good
You need to start by "git bisect start"

Do you want me to do it for you [Y/n]?
status: waiting for both good and bad commits
status: waiting for bad commit, 1 good commit known
✔ ~/u [v2026.04|BISECTING|…1]
17:09 $ co us/master
Previous HEAD position was 88dc2788777 Prepare v2026.04
HEAD is now at 3cdd19089f1 Merge tag 'mmc-for-2026.07-rc2' of
https://source.denx.de/u-boot/custodians/u-boot-mmc
✔ ~/u [:3cdd19089f1|BISECTING|…1]
17:09 $ ub-smoke rpi5
E
                    [100%]
============================================= ERRORS
=============================================
_______________________________ ERROR at setup of test_uboot_smoke
_______________________________

request = <SubRequest 'u_boot' for <Function test_uboot_smoke>>
target = Target(name='rpi5',
env=Environment(config_file='/vid/software/devel/ubtest/lab/cfg/kea_env.cfg'),
var_dict=None)
strategy = UBootStrategy(target=Target(name='rpi5',
env=Environment(config_file='/vid/software/devel/ubtest/lab/cfg/kea_env.cfg')...covery_reset=False,
console_needs_power=False, power_on_before_reset=False,
power_on_button=False, power_off_delay=0.0)

    @pytest.fixture
    def u_boot(request, target, strategy):
        if request.config.option.lg_use_running_system:
            strategy.uboot.assume_active()
            target.activate(strategy.uboot)
        else:
            try:
>               strategy.transition("uboot")

/vid/software/devel/ubtest/labgrid/contrib/u-boot/conftest.py:26:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _ _ _ _ _ _ _ _ _ _ _ _ _
/vid/software/devel/ubtest/labgrid/labgrid/strategy/ubootstrategy.py:274:
in transition
    self.target.activate(self.uboot)
/vid/software/devel/ubtest/labgrid/labgrid/target.py:554: in activate
    client.on_activate()
/vid/software/devel/ubtest/labgrid/labgrid/driver/ubootdriver.py:69:
in on_activate
    self._await_prompt()
/vid/software/devel/ubtest/labgrid/labgrid/step.py:220: in wrapper
    _result = func(*_args, **_kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _ _ _ _ _ _ _ _ _ _ _ _ _

self = UBootDriver(target=Target(name='rpi5',
env=Environment(config_file='/vid/software/devel/ubtest/lab/cfg/kea_env.cfg'),
...x version \\d', boot_command='run bootcmd', boot_commands={},
login_timeout=30, boot_timeout=30, strip_timestamp=False)

    @step()
    def _await_prompt(self):
        """Await autoboot line and stop it to get to the prompt, optionally
        enter the password.
        """
        timeout = Timeout(float(self.login_timeout))

        # We call console.expect with a short timeout here to detect if the
        # console is idle, which would result in a timeout without any changes
        # to the before property. So we store the last before value we've seen.
        # Because pexpect keeps any read data in it's buffer when a timeout
        # occours, we can't lose any data this way.
        last_before = None

        expectations = [self.prompt, self.autoboot, self.password_prompt,
                        pattern_u_boot_main_signon, TIMEOUT]
        while True:
            index, before, m, _ = self.console.expect(
                expectations,
                timeout=2
            )
            if index == 0:
                self._status = 1
                break

            elif index == 1:
                self.console.write(self.interrupt.encode('ASCII'))

            elif index == 2:
                if not self.password:
                    raise Exception("Password entry needed but no password set")
                self.console.sendline(self.password)

            elif index == 3:
                self.version = m.group(1).decode('ASCII')
            elif index == 4:
                # expect hit a timeout while waiting for a match
                if before == last_before:
                    # we did not receive anything during the previous
expect cycle
                    # let's assume the target is idle and we can safely issue a
                    # newline to check the state
                    self.console.sendline("")

                if timeout.expired:
                    output = self.console.read_output(False)
                    for line in output.splitlines():
                        print(line.decode('utf-8', errors='replace'))
>                   raise TIMEOUT(
                        f"Timeout of {self.login_timeout} seconds
exceeded during waiting for login"
                    )
E                   pexpect.exceptions.TIMEOUT: Timeout of 30 seconds
exceeded during waiting for login

/vid/software/devel/ubtest/labgrid/labgrid/driver/ubootdriver.py:205: TIMEOUT
------------------------------------- Captured stdout setup
--------------------------------------
Building U-Boot in sourcedir for rpi_arm64
Bootstrapping U-Boot from dir /tmp/b/rpi_arm64
Writing U-Boot using method rpi5

  0.80 RPi: BOOTSYS release VERSION:6fe0b091 DATE: 2024/06/05 TIME: 16:41:49
  0.84 BOOTMODE: 0x06 partition 0 build-ts BUILD_TIMESTAMP=1717602109
serial 648a6bdf boardrev b04170 stc 784131
  0.94 AON_RESET: 00000003 PM_RSTS 00001000
  0.01 RP1_BOOT chip ID: 0x20001927
  0.04 PM_RSTS: 0x00001000
  0.04 part 00000000 reset_info 00000000
  0.08 PMIC reset-event 00000000 rtc 00000000 alarm 00000000 enabled 0
  0.14 uSD voltage 3.3V
  0.33 Initialising SDRAM 'Micron' 16Gb x1 total-size: 16 Gbit 4267
  0.36 DDR 4267 0 0 16 152
  2.01 OTP boardrev b04170 bootrom a a
  2.02 Customer key hash
0000000000000000000000000000000000000000000000000000000000000000
  2.09 VC-JTAG unlocked
  2.32 RP1_BOOT chip ID: 0x20001927

  2.37 RP1_BOOT chip ID: 0x20001927
  2.38 RP1_BOOT: fw size 25992
  3.93 PCI2 init
  3.93 PCI2 reset
  3.37 PCIe scan 00001de4:00000001
  3.37 RP1_CHIP_INFO 20001927

  3.40 RPi: BOOTLOADER release VERSION:6fe0b091 DATE: 2024/06/05 TIME: 16:41:49
  3.47 BOOTMODE: 0x06 partition 0 build-ts BUILD_TIMESTAMP=1717602109
serial 648a6bdf boardrev b04170 stc 3147335
  3.57 AON_RESET: 00000003 PM_RSTS 00001000
  3.61 PCIEx1: PWR 0 DET_WAKE 0
  3.64 M.2 PCIe HAT not detected.
  3.88 usb_pd_init status 3
  3.88 USB_PD CONFIG 0 41
  3.94 XHCI-STOP
  3.94 xHC0 ver: 272 HCS: 03000440 140000f1 07ff000a HCC: 0240fe6d
  3.99 USBSTS 1
  3.01 xHC0 ver: 272 HCS: 03000440 140000f1 07ff000a HCC: 0240fe6d
  3.06 xHC0 ports 3 slots 64 intrs 4
  3.18 XHCI-STOP
  3.18 xHC1 ver: 272 HCS: 03000440 140000f1 07ff000a HCC: 0240fe6d
  3.23 USBSTS 1
  3.25 xHC1 ver: 272 HCS: 03000440 140000f1 07ff000a HCC: 0240fe6d
  3.30 xHC1 ports 3 slots 64 intrs 4
  3.38 Boot mode: SD (01) order f4
  3.28 USB2[2] 000206e1 connected
  3.78 USB2[2] 00200e03 connected enabled
  3.79 USB2 root HUB port 2 init
  3.85 DEV [01:00] 2.16 000000:02 class 0 VID 090c PID 1000
  3.87 MSD device [01:00] 2.16 000000:02 conf 0 iface 0 ep 82#512 01#512
  3.95 MSD [01:00] 2.16 000000:02 register MSD
  3.52 SD HOST: 200000000 CTL0: 0x00800000 BUS: 400000 Hz actual:
390625 HZ div: 512 (256) status: 0x1fff0000 delay: 276
  3.63 SD HOST: 200000000 CTL0: 0x00800f00 BUS: 400000 Hz actual:
390625 HZ div: 512 (256) status: 0x1fff0000 delay: 276
  3.97 OCR c0ff8000 [39]
CID: 0027504853443332476075149214017b
CSD: 400e00325b590000e6e97f800a400000
  3.05 SD: bus-width: 4 spec: 2 SCR: 0x02858083 0x00000000
  3.13 SD HOST: 200000000 CTL0: 0x00800f04 BUS: 50000000 Hz actual:
50000000 HZ div: 4 (2) status: 0x1fff0000 delay: 2
  3.22 MBR: 0x00000800, 1048576 type: 0x0c
  3.24 MBR: 0x00100800,17020996 type: 0x83
  3.28 MBR: 0x00000000,       0 type: 0x00
  3.32 MBR: 0x00000000,       0 type: 0x00
  3.36 Trying partition: 0
  3.38 type: 32 lba: 2048 'mkfs.fat' ' system-boot' clusters 1032412 (1)
  3.44 rsc 32 fat-sectors 8066 root dir cluster 2 sectors 0 entries 0
  3.51 FAT32 clusters 1032412
  3.57 [sdcard] autoboot.txt not found
  3.58 Select partition rsts 0 C(boot_partition) 0 EEPROM config 0 result 0
  3.65 Trying partition: 0
  3.67 type: 32 lba: 2048 'mkfs.fat' ' system-boot' clusters 1032408 (1)
  3.73 rsc 32 fat-sectors 8066 root dir cluster 2 sectors 0 entries 0
  3.80 FAT32 clusters 1032408
  3.87 Read config.txt bytes     1680 hnd 0x2fc7d
  3.90 [sdcard] pieeprom.upd not found
  3.92 usb_max_current_enable default 0 max-current 3000
  4.08 Read bcm2712-rpi-5-b.dtb bytes    80206 hnd 0x352
  4.11 dt-match: compatible: raspberrypi,5-model-b match: brcm,bcm2712
  4.17 dt-match: compatible: brcm,bcm2712 match: brcm,bcm2712
  4.23 Selecting USB low current limit

NOTICE:  BL31: v2.6(release):v2.6-239-g2a9ede0bd
NOTICE:  BL31: Built : 14:26:57, Jun 22 2023


U-Boot 2026.07-rc1-00169-g3cdd19089f1b (May 08 2026 - 17:09:17 -0600)

DRAM:  2 GiB
RPI 5 Model B (0xb04170)
Core:  24 devices, 11 uclasses, devicetree: board
"Synchronous Abort" handler, esr 0x96000045, far 0x7ffff000
elr: 000000000005d11c lr : 000000000005d0f4 (reloc)
elr: 000000003fba511c lr : 000000003fba50f4
x0 : 000000007ffff000 x1 : 60198ddf6491caa3
x2 : 1fe67ddf6491caa2 x3 : 7ffff00000000001
x4 : 0000000000000080 x5 : 0000000000000007
x6 : 0000000000000000 x7 : 000000003f746420
x8 : 0000000000000002 x9 : 000000003f745d88
x10: 0000000040000000 x11: 0000000000000000
x12: 0000000000001000 x13: 000000003fbece00
x14: 0000000000000001 x15: 0000000000000002
x16: 000000003fb8527c x17: 0000000000000000
x18: 000000003f743d90 x19: 0000000000000001
x20: 000000003f72fde8 x21: 000000003fbe4588
x22: 0000000000000000 x23: 0000000000000000
x24: 0000000000000000 x25: 0000000000000000
x26: 0000000000000000 x27: 0000000000000000
x28: 0000000000000000 x29: 000000003f72fd90

Code: f2e3fcc2 ca020061 eb02007f 9a9f1421 (a9000413)
Resetting CPU ...

resetting ...

(lots of error output here)

ERROR ../../../../../../../../vid/software/devel/ubtest/labgrid/contrib/u-boot/test_smoke.py::test_uboot_smoke
- pexpect.exceptions.TIMEOUT: Timeout of 30 seconds exceeded during
waiting for login
1 error in 45.25s
✘-1 ~/u [:3cdd19089f1|BISECTING|…1]
17:09 $ pe
3cdd19089f1 (HEAD, us/master, us/WIP/06May2026, pw-503347-review)
Merge tag 'mmc-for-2026.07-rc2' of
https://source.denx.de/u-boot/custodians/u-boot-mmc
e08877916e2 Merge tag 'mips-fixes-for-v2026.07' of
https://source.denx.de/u-boot/custodians/u-boot-mips
8399b039b03 Merge tag 'efi-2026-07-rc2-2' of
https://source.denx.de/u-boot/custodians/u-boot-efi
980f8a45896 Merge tag 'net-20260506' of
https://source.denx.de/u-boot/custodians/u-boot-net
ffdce9d3fbd Merge tag 'fsl-qoriq-for-2026.07-rc2' of
https://source.denx.de/u-boot/custodians/u-boot-fsl-qoriq
de79075f0f6 mips: mtmips: align MT7621 image blobs to 8-byte boundary
5245bdc98b9 net: phy: mscc: add support for the VSC8572
9717831e293 net: macb: add gigabit implementation for fixed-link
d7fe1f4333a net: macb: do not set user_io when it does not exist
361bb8f827b net: macb: use SA1 for MAC filtering on GEM
✔ ~/u [:3cdd19089f1|BISECTING|…1]
17:10 $ bad
Bisecting: 580 revisions left to test after this (roughly 9 steps)
[6e30330d7ac1b708772b1f5c2afae4bbec7c9678] net: dwc_eth_xgmac: Move
DMA reset and pad calibration after PHY init
✔ ~/u [:6e30330d7ac|BISECTING|…1]
17:10 $ ub-bisect rpi5
Starting bisect on target rpi5 - cherry-pick commit:
running '_ub-bisect-try'
Revision 6e30330d7ac1b708772b1f5c2afae4bbec7c9678, target rpi5
E


CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.15 [sdcard]
autoboot.txt not found␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.16 Select
partition rsts 0 C(boot_partition) 0 EEPROM config 0 result 0␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.23 Trying partition: 0␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.25 type: 32 lba:
2048 'mkfs.fat' ' system-boot' clusters 1032408 (1)␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.32 rsc 32
fat-sectors 8066 root dir cluster 2 sectors 0 entries 0␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.38 FAT32 clusters 1032408␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.45 Read
config.txt bytes     1680 hnd 0x2fc7d␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.48 [sdcard]
pieeprom.upd not found␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.50
usb_max_current_enable default 0 max-current 3000␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.64 Read
bcm2712-rpi-5-b.dtb bytes    80206 hnd 0x352␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.66 dt-match:
compatible: raspberrypi,5-model-b match: brcm,bcm2712␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.72 dt-match:
compatible: brcm,bcm2712 match: brcm,bcm2712␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) <   3.79 Selecting USB
low current limit␍␤
CONSOLE SerialLogger.rp:    SerialDriver(rpi5) < ␍␤
INFO         StepLogger:   ← SerialDriver.expect() result=(4,
b'\r\n\r\nDRAM:  2 GiB\r\nRPI 5 Model B (0xb04170)\r\nCore:  24
devices, 11 uclasses, devicetree… [2.000s]
INFO         StepLogger:   → SerialDriver.expect(pattern=['U-Boot>',
'stop autoboot', 'enter Password:', re.compile(b'(U-Boot( Concept)?
\\d{4}\\.\\d{2}[^\r\n]*\\))'), <class 'pexpect.exceptions.TIMEOUT'>])
INFO         StepLogger:   ← SerialDriver.expect() result=(4,
b'\r\n\r\nDRAM:  2 GiB\r\nRPI 5 Model B (0xb04170)\r\nCore:  24
devices, 11 uclasses, devicetree… [2.000s]
CONSOLE SerialLogger.rp:   SerialDriver(rpi5) > ␤␍␤
INFO         StepLogger:   → SerialDriver.expect(pattern=['U-Boot>',
'stop autoboot', 'enter Password:', re.compile(b'(U-Boot( Concept)?
\\d{4}\\.\\d{2}[^\r\n]*\\))'), <class 'pexpect.exceptions.TIMEOUT'>])
INFO         StepLogger:   ← SerialDriver.expect() result=(4,
b'\r\n\r\nDRAM:  2 GiB\r\nRPI 5 Model B (0xb04170)\r\nCore:  24
devices, 11 uclasses, devicetree… [2.000s]
CONSOLE SerialLogger.rp:   SerialDriver(rpi5) > ␤␍␤
INFO         StepLogger:  ⚠ UBootDriver._await_prompt() [30.715s]
exception=Timeout of 30 seconds exceeded during waiting for login
==================================== short test summary info
=====================================
ERROR test_smoke.py::test_uboot_smoke - pexpect.exceptions.TIMEOUT:
Timeout of 30 seconds exceeded during waiting for login
1 error in 42.37s
a3075db94d49f415658bf7e961e1eae90d9abc33 is the first bad commit
commit a3075db94d49f415658bf7e961e1eae90d9abc33
Author: Marek Vasut <marek.vasut+renesas@mailbox.org>
Date:   Wed Jan 28 00:48:40 2026 +0100

    lmb: Reinstate access to memory above ram_top

    Revert commit eb052cbb896f ("lmb: add and reserve memory above ram_top")
    and commit 1a48b0be93d4 ("lmb: prohibit allocations above ram_top even from
    same bank"). These are based on incorrect premise of the first commit, that
    "U-Boot does not use memory above ram_top". While U-Boot itself indeed does
    not and should not use memory above ram_top, user can perfectly well use
    that memory from the U-Boot shell, for example to load content in there.

    Currently, attempt to use that memory to load large image using TFTP ends
    with "TFTP error: trying to overwrite reserved memory...". With this change
    in place, the memory can be used again.

    Fixes: eb052cbb896f ("lmb: add and reserve memory above ram_top")
    Fixes: 1a48b0be93d4 ("lmb: prohibit allocations above ram_top even
from same bank")
    Reported-by: Yuya Hamamachi <yuya.hamamachi.sx@renesas.com>
    Signed-off-by: Marek Vasut <marek.vasut+renesas@mailbox.org>

 lib/lmb.c | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)
bisect found first bad commit
✔ ~/u [:a3075db94d4|BISECTING|…1]
17:16 $

Actually I recently sent an email about this patch recently but I
forgot to cc you.

Regards,
Simon

Re: [PATCH 3/3] configs: enable NVMe

[Peter Robinson]

On Fri, 8 May 2026 at 19:00, Peter Robinson <pbrobinson@gmail.com> wrote:
>
> On Fri, 8 May 2026 at 16:42, Torsten Duwe <duwe@lst.de> wrote:
> >
> > From: Torsten Duwe <duwe@suse.de>
> >
> > Enable NVMe in the Raspberry Pi 64-Bit default config
> >
> > Signed-off-by: Torsten Duwe <duwe@suse.de>
With the fix below:
Reviewed-by: Peter Robinson <pbrobinson@gmail.com>
Tested-by: Peter Robinson <pbrobinson@gmail.com>

Tested on CM4 with NVME, pci NIC
Tested on RPi5B both SoC revC and devD

> > ---
> >
> > Thanks to this series, U-Boot now supports the 64GiB address offset
> > used on the Raspberry Pi 5 for those inbound accesses.
> >
> > Remember that the other prerequisite on Rpi5 is the PCIe support
> > for bcm2712.
>
> I am in the process of testing this across my devices, awaiting a PCIe
> -> M.2 adapter so I can test on the CM4.
>
> One minor nit, I think we should scan for NVME drives:
>
> -CONFIG_PREBOOT="pci enum; usb start;"
> +CONFIG_PREBOOT="pci enum; usb start; nvme scan;"
>
> Peter
>
> > ---
> >  configs/rpi_arm64_defconfig | 1 +
> >  1 file changed, 1 insertion(+)
> >
> > diff --git a/configs/rpi_arm64_defconfig b/configs/rpi_arm64_defconfig
> > index 38af5029403..18f8213f3b3 100644
> > --- a/configs/rpi_arm64_defconfig
> > +++ b/configs/rpi_arm64_defconfig
> > @@ -41,6 +41,7 @@ CONFIG_MMC_SDHCI_SDMA=y
> >  CONFIG_MMC_SDHCI_BCM2835=y
> >  CONFIG_MMC_SDHCI_BCMSTB=y
> >  CONFIG_BCMGENET=y
> > +CONFIG_NVME_PCI=y
> >  CONFIG_PCI_BRCMSTB=y
> >  CONFIG_PINCTRL=y
> >  # CONFIG_PINCTRL_GENERIC is not set
> > --
> > 2.54.0
> >

Re: [PATCH 2/3] nvme: Fix missing address translation for PCIe inbound access

[Peter Robinson]

On Fri, 8 May 2026 at 16:42, Torsten Duwe <duwe@lst.de> wrote:
>
> From: Torsten Duwe <duwe@suse.de>
>
> U-Boot currently does not account for PCIe bridges with a non-zero
> inbound access offset when talking NVMe, it only works on platforms
> where this offset happens to be zero.
>
> This patch enhances the NVMe driver with the ability to also handle
> these cases.
>
> Signed-off-by: Torsten Duwe <duwe@suse.de>
Reviewed-by: Peter Robinson <pbrobinson@gmail.com>
Tested-by: Peter Robinson <pbrobinson@gmail.com>

>
> ---
>  drivers/nvme/nvme.c | 34 +++++++++++++++++++++-------------
>  1 file changed, 21 insertions(+), 13 deletions(-)
>
> diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c
> index 2b14437f69c..99a47bf25f2 100644
> --- a/drivers/nvme/nvme.c
> +++ b/drivers/nvme/nvme.c
> @@ -12,6 +12,7 @@
>  #include <log.h>
>  #include <malloc.h>
>  #include <memalign.h>
> +#include <phys2bus.h>
>  #include <time.h>
>  #include <dm/device-internal.h>
>  #include <linux/compat.h>
> @@ -27,6 +28,13 @@
>  #define IO_TIMEOUT             30
>  #define MAX_PRP_POOL           512
>
> +/*
> + * Convert a memory address to the value needed by the PCI device to
> + * access the given location, taking into account inbound window
> + * translations of PCI bridges:
> + */
> +#define DEV_ADDR(a)            dev_phys_to_bus(dev->udev, (a))
> +
>  static int nvme_wait_csts(struct nvme_dev *dev, u32 mask, u32 val)
>  {
>         int timeout;
> @@ -91,12 +99,12 @@ static int nvme_setup_prps(struct nvme_dev *dev, u64 *prp2,
>         i = 0;
>         while (nprps) {
>                 if ((i == (prps_per_page - 1)) && nprps > 1) {
> -                       *(prp_pool + i) = cpu_to_le64((ulong)prp_pool +
> -                                       page_size);
> +                       *(prp_pool + i) = cpu_to_le64(DEV_ADDR((ulong)prp_pool +
> +                                                               page_size));
>                         i = 0;
>                         prp_pool += page_size;
>                 }
> -               *(prp_pool + i++) = cpu_to_le64(dma_addr);
> +               *(prp_pool + i++) = cpu_to_le64(DEV_ADDR(dma_addr));
>                 dma_addr += page_size;
>                 nprps--;
>         }
> @@ -393,8 +401,8 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
>         dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
>
>         writel(aqa, &dev->bar->aqa);
> -       nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
> -       nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
> +       nvme_writeq(DEV_ADDR((ulong)nvmeq->sq_cmds), &dev->bar->asq);
> +       nvme_writeq(DEV_ADDR((ulong)nvmeq->cqes), &dev->bar->acq);
>
>         result = nvme_enable_ctrl(dev);
>         if (result)
> @@ -420,7 +428,7 @@ static int nvme_alloc_cq(struct nvme_dev *dev, u16 qid,
>
>         memset(&c, 0, sizeof(c));
>         c.create_cq.opcode = nvme_admin_create_cq;
> -       c.create_cq.prp1 = cpu_to_le64((ulong)nvmeq->cqes);
> +       c.create_cq.prp1 = cpu_to_le64(DEV_ADDR((ulong)nvmeq->cqes));
>         c.create_cq.cqid = cpu_to_le16(qid);
>         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
>         c.create_cq.cq_flags = cpu_to_le16(flags);
> @@ -437,7 +445,7 @@ static int nvme_alloc_sq(struct nvme_dev *dev, u16 qid,
>
>         memset(&c, 0, sizeof(c));
>         c.create_sq.opcode = nvme_admin_create_sq;
> -       c.create_sq.prp1 = cpu_to_le64((ulong)nvmeq->sq_cmds);
> +       c.create_sq.prp1 = cpu_to_le64(DEV_ADDR((ulong)nvmeq->sq_cmds));
>         c.create_sq.sqid = cpu_to_le16(qid);
>         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
>         c.create_sq.sq_flags = cpu_to_le16(flags);
> @@ -458,14 +466,14 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
>         memset(&c, 0, sizeof(c));
>         c.identify.opcode = nvme_admin_identify;
>         c.identify.nsid = cpu_to_le32(nsid);
> -       c.identify.prp1 = cpu_to_le64(dma_addr);
> +       c.identify.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
>
>         length -= (page_size - offset);
>         if (length <= 0) {
>                 c.identify.prp2 = 0;
>         } else {
>                 dma_addr += (page_size - offset);
> -               c.identify.prp2 = cpu_to_le64(dma_addr);
> +               c.identify.prp2 = cpu_to_le64(DEV_ADDR(dma_addr));
>         }
>
>         c.identify.cns = cpu_to_le32(cns);
> @@ -490,7 +498,7 @@ int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
>         memset(&c, 0, sizeof(c));
>         c.features.opcode = nvme_admin_get_features;
>         c.features.nsid = cpu_to_le32(nsid);
> -       c.features.prp1 = cpu_to_le64(dma_addr);
> +       c.features.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
>         c.features.fid = cpu_to_le32(fid);
>
>         ret = nvme_submit_admin_cmd(dev, &c, result);
> @@ -516,7 +524,7 @@ int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
>
>         memset(&c, 0, sizeof(c));
>         c.features.opcode = nvme_admin_set_features;
> -       c.features.prp1 = cpu_to_le64(dma_addr);
> +       c.features.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
>         c.features.fid = cpu_to_le32(fid);
>         c.features.dword11 = cpu_to_le32(dword11);
>
> @@ -785,8 +793,8 @@ static ulong nvme_blk_rw(struct udevice *udev, lbaint_t blknr,
>                 c.rw.slba = cpu_to_le64(slba);
>                 slba += lbas;
>                 c.rw.length = cpu_to_le16(lbas - 1);
> -               c.rw.prp1 = cpu_to_le64(temp_buffer);
> -               c.rw.prp2 = cpu_to_le64(prp2);
> +               c.rw.prp1 = cpu_to_le64(DEV_ADDR(temp_buffer));
> +               c.rw.prp2 = cpu_to_le64(DEV_ADDR(prp2));
>                 status = nvme_submit_sync_cmd(dev->queues[NVME_IO_Q],
>                                 &c, NULL, IO_TIMEOUT);
>                 if (status)
> --
> 2.54.0
>

Re: [PATCH 1/3] core: Skip parent device nodes without a DT reference when looking for dma-ranges

[Peter Robinson]

On Fri, 8 May 2026 at 16:42, Torsten Duwe <duwe@lst.de> wrote:
>
> From: Torsten Duwe <duwe@suse.de>
>
> If a device node got created dynamically, there is no guarantee that the
> parent node has an associated device tree node which could specify dma
> constraints. Especially PCI(e) enumeration adds intermediate "bus nodes",
> also dynamically.
>
> Try harder to find the correct configuration by walking up the tree until
> a DT association is found.
>
> Suggested-by: Neil Armstrong <neil.armstrong@linaro.org>
> Signed-off-by: Torsten Duwe <duwe@suse.de>
Reviewed-by: Peter Robinson <pbrobinson@gmail.com>
Tested-by: Peter Robinson <pbrobinson@gmail.com>

>
> ---
>  drivers/core/device.c | 14 +++++++++++++-
>  1 file changed, 13 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/core/device.c b/drivers/core/device.c
> index 779f371b9d5..9de64dbb3c5 100644
> --- a/drivers/core/device.c
> +++ b/drivers/core/device.c
> @@ -459,7 +459,19 @@ static int device_get_dma_constraints(struct udevice *dev)
>         u64 size = 0;
>         int ret;
>
> -       if (!CONFIG_IS_ENABLED(DM_DMA) || !parent || !dev_has_ofnode(parent))
> +       if (!CONFIG_IS_ENABLED(DM_DMA) || !parent)
> +               return 0;
> +
> +       /* Look for the first node in the parent chain */
> +       while (parent) {
> +               if (dev_has_ofnode(parent))
> +                       break;
> +
> +               parent = dev_get_parent(parent);
> +       }
> +
> +       /* No parents have a node, bail out */
> +       if (!parent)
>                 return 0;
>
>         /*
> --
> 2.54.0
>

Re: [PATCH 0/3] Fix NVMe, not only on Raspberry Pi 5

[Peter Robinson]

On Fri, 8 May 2026 at 16:41, Torsten Duwe <duwe@lst.de> wrote:
>
> Hi all,
>
> This simple quick fix has now turned into a small series. It contains
> the required changes to translate inbound access addresses, with the
> handling of dynamically found devices going into its own patch, content
> as suggested by Neil. While at it, I also added the NVMe enablement for
> raspberries. It might already work on RPi4 setups (to be tested). RPi5
> OTOH will require the other series to enable PCIe on the bcm2712.
>
> Changes from the single patch:
>   - the ternary in the macro is gone, explanation updated accordingly
>   - instead, device_get_dma_constraints is enhanced to try harder
>   - Raspberry Pi NVMe config added

This overall works for me, with the minor nit on patch 3, I tested it
on a RPi400 booted from USB, a CM4 with NVME and a couple of RPi5 with
NVME drives. I'd like to get Neil's review on it too.

> Torsten Duwe (3):
>   Skip parent device nodes without a DT reference when looking for
>     dma-ranges
>   Fix missing address translation for PCIe inbound access
>   rpi: enable NVMe
>
>  configs/rpi_arm64_defconfig |  1 +
>  drivers/core/device.c       | 14 +++++++++++++-
>  drivers/nvme/nvme.c         | 34 +++++++++++++++++++++-------------
>  3 files changed, 35 insertions(+), 14 deletions(-)
>
> --
> 2.54.0
>

Re: [PATCH 2/3] nvme: Fix missing address translation for PCIe inbound access

[Neil Armstrong]

On 5/8/26 17:42, Torsten Duwe wrote:
> From: Torsten Duwe <duwe@suse.de>
> 
> U-Boot currently does not account for PCIe bridges with a non-zero
> inbound access offset when talking NVMe, it only works on platforms
> where this offset happens to be zero.
> 
> This patch enhances the NVMe driver with the ability to also handle
> these cases.
> 
> Signed-off-by: Torsten Duwe <duwe@suse.de>
> 
> ---
>   drivers/nvme/nvme.c | 34 +++++++++++++++++++++-------------
>   1 file changed, 21 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c
> index 2b14437f69c..99a47bf25f2 100644
> --- a/drivers/nvme/nvme.c
> +++ b/drivers/nvme/nvme.c
> @@ -12,6 +12,7 @@
>   #include <log.h>
>   #include <malloc.h>
>   #include <memalign.h>
> +#include <phys2bus.h>
>   #include <time.h>
>   #include <dm/device-internal.h>
>   #include <linux/compat.h>
> @@ -27,6 +28,13 @@
>   #define IO_TIMEOUT		30
>   #define MAX_PRP_POOL		512
>   
> +/*
> + * Convert a memory address to the value needed by the PCI device to
> + * access the given location, taking into account inbound window
> + * translations of PCI bridges:
> + */
> +#define DEV_ADDR(a)		dev_phys_to_bus(dev->udev, (a))

This doesn't look very clean, I would do something closer to linux by precalculating
the DMA addresses after allocation like:
====================><================================
diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c
index 2b14437f69c..1ce06402a90 100644
--- a/drivers/nvme/nvme.c
+++ b/drivers/nvme/nvme.c
@@ -241,6 +241,7 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev,
  	nvmeq->sq_cmds = (void *)memalign(4096, NVME_SQ_SIZE(depth));
  	if (!nvmeq->sq_cmds)
  		goto free_queue;
+	nvmeq->sq_dma_addr = dev_phys_to_bus(dev->udev, nvmeq->sq_cmds);
  	memset((void *)nvmeq->sq_cmds, 0, NVME_SQ_SIZE(depth));

  	nvmeq->dev = dev;
@@ -393,7 +394,7 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
  	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;

  	writel(aqa, &dev->bar->aqa);
-	nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
+	nvme_writeq((ulong)nvmeq->sq_dma_addr, &dev->bar->asq);
  	nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);

  	result = nvme_enable_ctrl(dev);
diff --git a/drivers/nvme/nvme.h b/drivers/nvme/nvme.h
index bc1d612dde4..9c5e57cb586 100644
--- a/drivers/nvme/nvme.h
+++ b/drivers/nvme/nvme.h
@@ -637,6 +637,7 @@ enum nvme_queue_id {
  struct nvme_queue {
  	struct nvme_dev *dev;
  	struct nvme_command *sq_cmds;
+	dma_addr_t sq_dma_addr;
  	struct nvme_completion *cqes;
  	u32 __iomem *q_db;
  	u16 q_depth;
====================><================================

and same for the orther allocated buffers.

Thanks,
Neil

> +
>   static int nvme_wait_csts(struct nvme_dev *dev, u32 mask, u32 val)
>   {
>   	int timeout;
> @@ -91,12 +99,12 @@ static int nvme_setup_prps(struct nvme_dev *dev, u64 *prp2,
>   	i = 0;
>   	while (nprps) {
>   		if ((i == (prps_per_page - 1)) && nprps > 1) {
> -			*(prp_pool + i) = cpu_to_le64((ulong)prp_pool +
> -					page_size);
> +			*(prp_pool + i) = cpu_to_le64(DEV_ADDR((ulong)prp_pool +
> +								page_size));
>   			i = 0;
>   			prp_pool += page_size;
>   		}
> -		*(prp_pool + i++) = cpu_to_le64(dma_addr);
> +		*(prp_pool + i++) = cpu_to_le64(DEV_ADDR(dma_addr));
>   		dma_addr += page_size;
>   		nprps--;
>   	}
> @@ -393,8 +401,8 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
>   	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
>   
>   	writel(aqa, &dev->bar->aqa);
> -	nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
> -	nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
> +	nvme_writeq(DEV_ADDR((ulong)nvmeq->sq_cmds), &dev->bar->asq);
> +	nvme_writeq(DEV_ADDR((ulong)nvmeq->cqes), &dev->bar->acq);
>   
>   	result = nvme_enable_ctrl(dev);
>   	if (result)
> @@ -420,7 +428,7 @@ static int nvme_alloc_cq(struct nvme_dev *dev, u16 qid,
>   
>   	memset(&c, 0, sizeof(c));
>   	c.create_cq.opcode = nvme_admin_create_cq;
> -	c.create_cq.prp1 = cpu_to_le64((ulong)nvmeq->cqes);
> +	c.create_cq.prp1 = cpu_to_le64(DEV_ADDR((ulong)nvmeq->cqes));
>   	c.create_cq.cqid = cpu_to_le16(qid);
>   	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
>   	c.create_cq.cq_flags = cpu_to_le16(flags);
> @@ -437,7 +445,7 @@ static int nvme_alloc_sq(struct nvme_dev *dev, u16 qid,
>   
>   	memset(&c, 0, sizeof(c));
>   	c.create_sq.opcode = nvme_admin_create_sq;
> -	c.create_sq.prp1 = cpu_to_le64((ulong)nvmeq->sq_cmds);
> +	c.create_sq.prp1 = cpu_to_le64(DEV_ADDR((ulong)nvmeq->sq_cmds));
>   	c.create_sq.sqid = cpu_to_le16(qid);
>   	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
>   	c.create_sq.sq_flags = cpu_to_le16(flags);
> @@ -458,14 +466,14 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
>   	memset(&c, 0, sizeof(c));
>   	c.identify.opcode = nvme_admin_identify;
>   	c.identify.nsid = cpu_to_le32(nsid);
> -	c.identify.prp1 = cpu_to_le64(dma_addr);
> +	c.identify.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
>   
>   	length -= (page_size - offset);
>   	if (length <= 0) {
>   		c.identify.prp2 = 0;
>   	} else {
>   		dma_addr += (page_size - offset);
> -		c.identify.prp2 = cpu_to_le64(dma_addr);
> +		c.identify.prp2 = cpu_to_le64(DEV_ADDR(dma_addr));
>   	}
>   
>   	c.identify.cns = cpu_to_le32(cns);
> @@ -490,7 +498,7 @@ int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
>   	memset(&c, 0, sizeof(c));
>   	c.features.opcode = nvme_admin_get_features;
>   	c.features.nsid = cpu_to_le32(nsid);
> -	c.features.prp1 = cpu_to_le64(dma_addr);
> +	c.features.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
>   	c.features.fid = cpu_to_le32(fid);
>   
>   	ret = nvme_submit_admin_cmd(dev, &c, result);
> @@ -516,7 +524,7 @@ int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
>   
>   	memset(&c, 0, sizeof(c));
>   	c.features.opcode = nvme_admin_set_features;
> -	c.features.prp1 = cpu_to_le64(dma_addr);
> +	c.features.prp1 = cpu_to_le64(DEV_ADDR(dma_addr));
>   	c.features.fid = cpu_to_le32(fid);
>   	c.features.dword11 = cpu_to_le32(dword11);
>   
> @@ -785,8 +793,8 @@ static ulong nvme_blk_rw(struct udevice *udev, lbaint_t blknr,
>   		c.rw.slba = cpu_to_le64(slba);
>   		slba += lbas;
>   		c.rw.length = cpu_to_le16(lbas - 1);
> -		c.rw.prp1 = cpu_to_le64(temp_buffer);
> -		c.rw.prp2 = cpu_to_le64(prp2);
> +		c.rw.prp1 = cpu_to_le64(DEV_ADDR(temp_buffer));
> +		c.rw.prp2 = cpu_to_le64(DEV_ADDR(prp2));
>   		status = nvme_submit_sync_cmd(dev->queues[NVME_IO_Q],
>   				&c, NULL, IO_TIMEOUT);
>   		if (status)

Re: [PATCH 2/3] nvme: Fix missing address translation for PCIe inbound access

[Torsten Duwe]

On Mon, 11 May 2026 16:50:48 +0200
Neil Armstrong <neil.armstrong@linaro.org> wrote:

> On 5/8/26 17:42, Torsten Duwe wrote:
[...]
> > +#define DEV_ADDR(a)		dev_phys_to_bus(dev->udev, (a))
> 
> This doesn't look very clean, I would do something closer to linux by precalculating
> the DMA addresses after allocation like:
> ====================><================================
> diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c
> index 2b14437f69c..1ce06402a90 100644
> --- a/drivers/nvme/nvme.c
> +++ b/drivers/nvme/nvme.c
> @@ -241,6 +241,7 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev,
>   	nvmeq->sq_cmds = (void *)memalign(4096, NVME_SQ_SIZE(depth));
>   	if (!nvmeq->sq_cmds)
>   		goto free_queue;
> +	nvmeq->sq_dma_addr = dev_phys_to_bus(dev->udev, nvmeq->sq_cmds);
>   	memset((void *)nvmeq->sq_cmds, 0, NVME_SQ_SIZE(depth));
> 
>   	nvmeq->dev = dev;
> @@ -393,7 +394,7 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
>   	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
> 
>   	writel(aqa, &dev->bar->aqa);
> -	nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
> +	nvme_writeq((ulong)nvmeq->sq_dma_addr, &dev->bar->asq);
>   	nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
> 
>   	result = nvme_enable_ctrl(dev);

Hm, time / space tradeoff. I don't really have a strong preference
regarding this, I'd worry most about U-Boot code size (for which this
doesn't make much of a difference). However, I was under the impression
that during regular NVMe business a lot of those buffers and addresses
get allocated and the translations are required often. That's probably
only true once a full-blown operating system is running.

I'll check the allocations and assignments. Any other opinions?

	Torsten

Re: [PATCH 2/3] nvme: Fix missing address translation for PCIe inbound access

[Neil Armstrong]

On 5/12/26 11:30, Torsten Duwe wrote:
> On Mon, 11 May 2026 16:50:48 +0200
> Neil Armstrong <neil.armstrong@linaro.org> wrote:
> 
>> On 5/8/26 17:42, Torsten Duwe wrote:
> [...]
>>> +#define DEV_ADDR(a)		dev_phys_to_bus(dev->udev, (a))
>>
>> This doesn't look very clean, I would do something closer to linux by precalculating
>> the DMA addresses after allocation like:
>> ====================><================================
>> diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c
>> index 2b14437f69c..1ce06402a90 100644
>> --- a/drivers/nvme/nvme.c
>> +++ b/drivers/nvme/nvme.c
>> @@ -241,6 +241,7 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev,
>>    	nvmeq->sq_cmds = (void *)memalign(4096, NVME_SQ_SIZE(depth));
>>    	if (!nvmeq->sq_cmds)
>>    		goto free_queue;
>> +	nvmeq->sq_dma_addr = dev_phys_to_bus(dev->udev, nvmeq->sq_cmds);
>>    	memset((void *)nvmeq->sq_cmds, 0, NVME_SQ_SIZE(depth));
>>
>>    	nvmeq->dev = dev;
>> @@ -393,7 +394,7 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
>>    	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
>>
>>    	writel(aqa, &dev->bar->aqa);
>> -	nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
>> +	nvme_writeq((ulong)nvmeq->sq_dma_addr, &dev->bar->asq);
>>    	nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
>>
>>    	result = nvme_enable_ctrl(dev);
> 
> Hm, time / space tradeoff. I don't really have a strong preference
> regarding this, I'd worry most about U-Boot code size (for which this
> doesn't make much of a difference). However, I was under the impression
> that during regular NVMe business a lot of those buffers and addresses
> get allocated and the translations are required often. That's probably
> only true once a full-blown operating system is running.
> 
> I'll check the allocations and assignments. Any other opinions?

Other that that, I think it's fine, Linux is basically doing the same.

Thanks,
Neil

> 
> 	Torsten