[PATCH v7 0/3] hw/riscv: clear kernel_entry high bits with 32bit CPUs

[PATCH v7 0/3] hw/riscv: clear kernel_entry high bits with 32bit CPUs

[Daniel Henrique Barboza]

Hi,

In this version I followed Bin Meng's suggestion and reverted patch 1
back from what it was in the v5, acks included, and added a new patch
(3) to fix the problem detected with the Xvisor use case. I believe this
reflects that there is nothing particularly wrong with what we
did in the v5 patch and we're going an extra mile to fix what, at first
glance, is a bug somewhere else.

In patch 3 I also followed Phil's idea and used a translate_fn()
callback to do the bit clearing.


Changes from v6:
- patch 1:
  - restored to the state it was in v5, acks included
- patch 3 (new):
  - clear the higher bits from the result of load_elf_ram_sym() using a
    translate_fn() callback for 32 bit CPUs
v6 link: https://lists.gnu.org/archive/html/qemu-devel/2023-01/msg02630.html

Daniel Henrique Barboza (3):
  hw/riscv/boot.c: consolidate all kernel init in riscv_load_kernel()
  hw/riscv/boot.c: make riscv_load_initrd() static
  hw/riscv: clear kernel_entry higher bits in load_elf_ram_sym()

 hw/riscv/boot.c            | 111 ++++++++++++++++++++++++-------------
 hw/riscv/microchip_pfsoc.c |  12 +---
 hw/riscv/opentitan.c       |   3 +-
 hw/riscv/sifive_e.c        |   4 +-
 hw/riscv/sifive_u.c        |  12 +---
 hw/riscv/spike.c           |  13 +----
 hw/riscv/virt.c            |  12 +---
 include/hw/riscv/boot.h    |   3 +-
 8 files changed, 89 insertions(+), 81 deletions(-)

-- 
2.39.0

[PATCH v7 1/3] hw/riscv/boot.c: consolidate all kernel init in riscv_load_kernel()

[Daniel Henrique Barboza]

The microchip_icicle_kit, sifive_u, spike and virt boards are now doing
the same steps when '-kernel' is used:

- execute load_kernel()
- load init_rd()
- write kernel_cmdline

Let's fold everything inside riscv_load_kernel() to avoid code
repetition. To not change the behavior of boards that aren't calling
riscv_load_init(), add an 'load_initrd' flag to riscv_load_kernel() and
allow these boards to opt out from initrd loading.

Cc: Palmer Dabbelt <palmer@dabbelt.com>
Reviewed-by: Bin Meng <bmeng@tinylab.org>
Reviewed-by: Alistair Francis <alistair.francis@wdc.com>
Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 hw/riscv/boot.c            | 22 +++++++++++++++++++---
 hw/riscv/microchip_pfsoc.c | 12 ++----------
 hw/riscv/opentitan.c       |  2 +-
 hw/riscv/sifive_e.c        |  3 ++-
 hw/riscv/sifive_u.c        | 12 ++----------
 hw/riscv/spike.c           | 11 +----------
 hw/riscv/virt.c            | 12 ++----------
 include/hw/riscv/boot.h    |  1 +
 8 files changed, 30 insertions(+), 45 deletions(-)

diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c
index 2594276223..4888d5c1e0 100644
--- a/hw/riscv/boot.c
+++ b/hw/riscv/boot.c
@@ -175,10 +175,12 @@ target_ulong riscv_load_firmware(const char *firmware_filename,
 
 target_ulong riscv_load_kernel(MachineState *machine,
                                target_ulong kernel_start_addr,
+                               bool load_initrd,
                                symbol_fn_t sym_cb)
 {
     const char *kernel_filename = machine->kernel_filename;
     uint64_t kernel_load_base, kernel_entry;
+    void *fdt = machine->fdt;
 
     g_assert(kernel_filename != NULL);
 
@@ -192,21 +194,35 @@ target_ulong riscv_load_kernel(MachineState *machine,
     if (load_elf_ram_sym(kernel_filename, NULL, NULL, NULL,
                          NULL, &kernel_load_base, NULL, NULL, 0,
                          EM_RISCV, 1, 0, NULL, true, sym_cb) > 0) {
-        return kernel_load_base;
+        kernel_entry = kernel_load_base;
+        goto out;
     }
 
     if (load_uimage_as(kernel_filename, &kernel_entry, NULL, NULL,
                        NULL, NULL, NULL) > 0) {
-        return kernel_entry;
+        goto out;
     }
 
     if (load_image_targphys_as(kernel_filename, kernel_start_addr,
                                current_machine->ram_size, NULL) > 0) {
-        return kernel_start_addr;
+        kernel_entry = kernel_start_addr;
+        goto out;
     }
 
     error_report("could not load kernel '%s'", kernel_filename);
     exit(1);
+
+out:
+    if (load_initrd && machine->initrd_filename) {
+        riscv_load_initrd(machine, kernel_entry);
+    }
+
+    if (fdt && machine->kernel_cmdline && *machine->kernel_cmdline) {
+        qemu_fdt_setprop_string(fdt, "/chosen", "bootargs",
+                                machine->kernel_cmdline);
+    }
+
+    return kernel_entry;
 }
 
 void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry)
diff --git a/hw/riscv/microchip_pfsoc.c b/hw/riscv/microchip_pfsoc.c
index 82ae5e7023..c45023a2b1 100644
--- a/hw/riscv/microchip_pfsoc.c
+++ b/hw/riscv/microchip_pfsoc.c
@@ -629,16 +629,8 @@ static void microchip_icicle_kit_machine_init(MachineState *machine)
         kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc.u_cpus,
                                                          firmware_end_addr);
 
-        kernel_entry = riscv_load_kernel(machine, kernel_start_addr, NULL);
-
-        if (machine->initrd_filename) {
-            riscv_load_initrd(machine, kernel_entry);
-        }
-
-        if (machine->kernel_cmdline && *machine->kernel_cmdline) {
-            qemu_fdt_setprop_string(machine->fdt, "/chosen",
-                                    "bootargs", machine->kernel_cmdline);
-        }
+        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
+                                         true, NULL);
 
         /* Compute the fdt load address in dram */
         fdt_load_addr = riscv_load_fdt(memmap[MICROCHIP_PFSOC_DRAM_LO].base,
diff --git a/hw/riscv/opentitan.c b/hw/riscv/opentitan.c
index 64d5d435b9..f6fd9725a5 100644
--- a/hw/riscv/opentitan.c
+++ b/hw/riscv/opentitan.c
@@ -101,7 +101,7 @@ static void opentitan_board_init(MachineState *machine)
     }
 
     if (machine->kernel_filename) {
-        riscv_load_kernel(machine, memmap[IBEX_DEV_RAM].base, NULL);
+        riscv_load_kernel(machine, memmap[IBEX_DEV_RAM].base, false, NULL);
     }
 }
 
diff --git a/hw/riscv/sifive_e.c b/hw/riscv/sifive_e.c
index 3e3f4b0088..6835d1c807 100644
--- a/hw/riscv/sifive_e.c
+++ b/hw/riscv/sifive_e.c
@@ -114,7 +114,8 @@ static void sifive_e_machine_init(MachineState *machine)
                           memmap[SIFIVE_E_DEV_MROM].base, &address_space_memory);
 
     if (machine->kernel_filename) {
-        riscv_load_kernel(machine, memmap[SIFIVE_E_DEV_DTIM].base, NULL);
+        riscv_load_kernel(machine, memmap[SIFIVE_E_DEV_DTIM].base,
+                          false, NULL);
     }
 }
 
diff --git a/hw/riscv/sifive_u.c b/hw/riscv/sifive_u.c
index bac394c959..9a75d4aa62 100644
--- a/hw/riscv/sifive_u.c
+++ b/hw/riscv/sifive_u.c
@@ -598,16 +598,8 @@ static void sifive_u_machine_init(MachineState *machine)
         kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc.u_cpus,
                                                          firmware_end_addr);
 
-        kernel_entry = riscv_load_kernel(machine, kernel_start_addr, NULL);
-
-        if (machine->initrd_filename) {
-            riscv_load_initrd(machine, kernel_entry);
-        }
-
-        if (machine->kernel_cmdline && *machine->kernel_cmdline) {
-            qemu_fdt_setprop_string(machine->fdt, "/chosen", "bootargs",
-                                    machine->kernel_cmdline);
-        }
+        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
+                                         true, NULL);
     } else {
        /*
         * If dynamic firmware is used, it doesn't know where is the next mode
diff --git a/hw/riscv/spike.c b/hw/riscv/spike.c
index bff9475686..c517885e6e 100644
--- a/hw/riscv/spike.c
+++ b/hw/riscv/spike.c
@@ -308,16 +308,7 @@ static void spike_board_init(MachineState *machine)
                                                          firmware_end_addr);
 
         kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
-                                         htif_symbol_callback);
-
-        if (machine->initrd_filename) {
-            riscv_load_initrd(machine, kernel_entry);
-        }
-
-        if (machine->kernel_cmdline && *machine->kernel_cmdline) {
-            qemu_fdt_setprop_string(machine->fdt, "/chosen", "bootargs",
-                                    machine->kernel_cmdline);
-        }
+                                         true, htif_symbol_callback);
     } else {
        /*
         * If dynamic firmware is used, it doesn't know where is the next mode
diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
index c8e35f861e..a931ed05ab 100644
--- a/hw/riscv/virt.c
+++ b/hw/riscv/virt.c
@@ -1281,16 +1281,8 @@ static void virt_machine_done(Notifier *notifier, void *data)
         kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc[0],
                                                          firmware_end_addr);
 
-        kernel_entry = riscv_load_kernel(machine, kernel_start_addr, NULL);
-
-        if (machine->initrd_filename) {
-            riscv_load_initrd(machine, kernel_entry);
-        }
-
-        if (machine->kernel_cmdline && *machine->kernel_cmdline) {
-            qemu_fdt_setprop_string(machine->fdt, "/chosen", "bootargs",
-                                    machine->kernel_cmdline);
-        }
+        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
+                                         true, NULL);
     } else {
        /*
         * If dynamic firmware is used, it doesn't know where is the next mode
diff --git a/include/hw/riscv/boot.h b/include/hw/riscv/boot.h
index f94653a09b..c3de897371 100644
--- a/include/hw/riscv/boot.h
+++ b/include/hw/riscv/boot.h
@@ -45,6 +45,7 @@ target_ulong riscv_load_firmware(const char *firmware_filename,
                                  symbol_fn_t sym_cb);
 target_ulong riscv_load_kernel(MachineState *machine,
                                target_ulong firmware_end_addr,
+                               bool load_initrd,
                                symbol_fn_t sym_cb);
 void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry);
 uint64_t riscv_load_fdt(hwaddr dram_start, uint64_t dram_size, void *fdt);
-- 
2.39.0

[PATCH v7 2/3] hw/riscv/boot.c: make riscv_load_initrd() static

[Daniel Henrique Barboza]

The only remaining caller is riscv_load_kernel_and_initrd() which
belongs to the same file.

Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Reviewed-by: Bin Meng <bmeng@tinylab.org>
---
 hw/riscv/boot.c         | 80 ++++++++++++++++++++---------------------
 include/hw/riscv/boot.h |  1 -
 2 files changed, 40 insertions(+), 41 deletions(-)

diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c
index 4888d5c1e0..e868fb6ade 100644
--- a/hw/riscv/boot.c
+++ b/hw/riscv/boot.c
@@ -173,6 +173,46 @@ target_ulong riscv_load_firmware(const char *firmware_filename,
     exit(1);
 }
 
+static void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry)
+{
+    const char *filename = machine->initrd_filename;
+    uint64_t mem_size = machine->ram_size;
+    void *fdt = machine->fdt;
+    hwaddr start, end;
+    ssize_t size;
+
+    g_assert(filename != NULL);
+
+    /*
+     * We want to put the initrd far enough into RAM that when the
+     * kernel is uncompressed it will not clobber the initrd. However
+     * on boards without much RAM we must ensure that we still leave
+     * enough room for a decent sized initrd, and on boards with large
+     * amounts of RAM we must avoid the initrd being so far up in RAM
+     * that it is outside lowmem and inaccessible to the kernel.
+     * So for boards with less  than 256MB of RAM we put the initrd
+     * halfway into RAM, and for boards with 256MB of RAM or more we put
+     * the initrd at 128MB.
+     */
+    start = kernel_entry + MIN(mem_size / 2, 128 * MiB);
+
+    size = load_ramdisk(filename, start, mem_size - start);
+    if (size == -1) {
+        size = load_image_targphys(filename, start, mem_size - start);
+        if (size == -1) {
+            error_report("could not load ramdisk '%s'", filename);
+            exit(1);
+        }
+    }
+
+    /* Some RISC-V machines (e.g. opentitan) don't have a fdt. */
+    if (fdt) {
+        end = start + size;
+        qemu_fdt_setprop_cell(fdt, "/chosen", "linux,initrd-start", start);
+        qemu_fdt_setprop_cell(fdt, "/chosen", "linux,initrd-end", end);
+    }
+}
+
 target_ulong riscv_load_kernel(MachineState *machine,
                                target_ulong kernel_start_addr,
                                bool load_initrd,
@@ -225,46 +265,6 @@ out:
     return kernel_entry;
 }
 
-void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry)
-{
-    const char *filename = machine->initrd_filename;
-    uint64_t mem_size = machine->ram_size;
-    void *fdt = machine->fdt;
-    hwaddr start, end;
-    ssize_t size;
-
-    g_assert(filename != NULL);
-
-    /*
-     * We want to put the initrd far enough into RAM that when the
-     * kernel is uncompressed it will not clobber the initrd. However
-     * on boards without much RAM we must ensure that we still leave
-     * enough room for a decent sized initrd, and on boards with large
-     * amounts of RAM we must avoid the initrd being so far up in RAM
-     * that it is outside lowmem and inaccessible to the kernel.
-     * So for boards with less  than 256MB of RAM we put the initrd
-     * halfway into RAM, and for boards with 256MB of RAM or more we put
-     * the initrd at 128MB.
-     */
-    start = kernel_entry + MIN(mem_size / 2, 128 * MiB);
-
-    size = load_ramdisk(filename, start, mem_size - start);
-    if (size == -1) {
-        size = load_image_targphys(filename, start, mem_size - start);
-        if (size == -1) {
-            error_report("could not load ramdisk '%s'", filename);
-            exit(1);
-        }
-    }
-
-    /* Some RISC-V machines (e.g. opentitan) don't have a fdt. */
-    if (fdt) {
-        end = start + size;
-        qemu_fdt_setprop_cell(fdt, "/chosen", "linux,initrd-start", start);
-        qemu_fdt_setprop_cell(fdt, "/chosen", "linux,initrd-end", end);
-    }
-}
-
 uint64_t riscv_load_fdt(hwaddr dram_base, uint64_t mem_size, void *fdt)
 {
     uint64_t temp, fdt_addr;
diff --git a/include/hw/riscv/boot.h b/include/hw/riscv/boot.h
index c3de897371..cbd131bad7 100644
--- a/include/hw/riscv/boot.h
+++ b/include/hw/riscv/boot.h
@@ -47,7 +47,6 @@ target_ulong riscv_load_kernel(MachineState *machine,
                                target_ulong firmware_end_addr,
                                bool load_initrd,
                                symbol_fn_t sym_cb);
-void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry);
 uint64_t riscv_load_fdt(hwaddr dram_start, uint64_t dram_size, void *fdt);
 void riscv_setup_rom_reset_vec(MachineState *machine, RISCVHartArrayState *harts,
                                hwaddr saddr,
-- 
2.39.0

Re: [PATCH v7 2/3] hw/riscv/boot.c: make riscv_load_initrd() static

[Alistair Francis]

On Sat, Jan 14, 2023 at 3:39 AM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> The only remaining caller is riscv_load_kernel_and_initrd() which
> belongs to the same file.
>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> Reviewed-by: Bin Meng <bmeng@tinylab.org>

Reviewed-by: Alistair Francis <alistair.francis@wdc.com>

Alistair

> ---
>  hw/riscv/boot.c         | 80 ++++++++++++++++++++---------------------
>  include/hw/riscv/boot.h |  1 -
>  2 files changed, 40 insertions(+), 41 deletions(-)
>
> diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c
> index 4888d5c1e0..e868fb6ade 100644
> --- a/hw/riscv/boot.c
> +++ b/hw/riscv/boot.c
> @@ -173,6 +173,46 @@ target_ulong riscv_load_firmware(const char *firmware_filename,
>      exit(1);
>  }
>
> +static void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry)
> +{
> +    const char *filename = machine->initrd_filename;
> +    uint64_t mem_size = machine->ram_size;
> +    void *fdt = machine->fdt;
> +    hwaddr start, end;
> +    ssize_t size;
> +
> +    g_assert(filename != NULL);
> +
> +    /*
> +     * We want to put the initrd far enough into RAM that when the
> +     * kernel is uncompressed it will not clobber the initrd. However
> +     * on boards without much RAM we must ensure that we still leave
> +     * enough room for a decent sized initrd, and on boards with large
> +     * amounts of RAM we must avoid the initrd being so far up in RAM
> +     * that it is outside lowmem and inaccessible to the kernel.
> +     * So for boards with less  than 256MB of RAM we put the initrd
> +     * halfway into RAM, and for boards with 256MB of RAM or more we put
> +     * the initrd at 128MB.
> +     */
> +    start = kernel_entry + MIN(mem_size / 2, 128 * MiB);
> +
> +    size = load_ramdisk(filename, start, mem_size - start);
> +    if (size == -1) {
> +        size = load_image_targphys(filename, start, mem_size - start);
> +        if (size == -1) {
> +            error_report("could not load ramdisk '%s'", filename);
> +            exit(1);
> +        }
> +    }
> +
> +    /* Some RISC-V machines (e.g. opentitan) don't have a fdt. */
> +    if (fdt) {
> +        end = start + size;
> +        qemu_fdt_setprop_cell(fdt, "/chosen", "linux,initrd-start", start);
> +        qemu_fdt_setprop_cell(fdt, "/chosen", "linux,initrd-end", end);
> +    }
> +}
> +
>  target_ulong riscv_load_kernel(MachineState *machine,
>                                 target_ulong kernel_start_addr,
>                                 bool load_initrd,
> @@ -225,46 +265,6 @@ out:
>      return kernel_entry;
>  }
>
> -void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry)
> -{
> -    const char *filename = machine->initrd_filename;
> -    uint64_t mem_size = machine->ram_size;
> -    void *fdt = machine->fdt;
> -    hwaddr start, end;
> -    ssize_t size;
> -
> -    g_assert(filename != NULL);
> -
> -    /*
> -     * We want to put the initrd far enough into RAM that when the
> -     * kernel is uncompressed it will not clobber the initrd. However
> -     * on boards without much RAM we must ensure that we still leave
> -     * enough room for a decent sized initrd, and on boards with large
> -     * amounts of RAM we must avoid the initrd being so far up in RAM
> -     * that it is outside lowmem and inaccessible to the kernel.
> -     * So for boards with less  than 256MB of RAM we put the initrd
> -     * halfway into RAM, and for boards with 256MB of RAM or more we put
> -     * the initrd at 128MB.
> -     */
> -    start = kernel_entry + MIN(mem_size / 2, 128 * MiB);
> -
> -    size = load_ramdisk(filename, start, mem_size - start);
> -    if (size == -1) {
> -        size = load_image_targphys(filename, start, mem_size - start);
> -        if (size == -1) {
> -            error_report("could not load ramdisk '%s'", filename);
> -            exit(1);
> -        }
> -    }
> -
> -    /* Some RISC-V machines (e.g. opentitan) don't have a fdt. */
> -    if (fdt) {
> -        end = start + size;
> -        qemu_fdt_setprop_cell(fdt, "/chosen", "linux,initrd-start", start);
> -        qemu_fdt_setprop_cell(fdt, "/chosen", "linux,initrd-end", end);
> -    }
> -}
> -
>  uint64_t riscv_load_fdt(hwaddr dram_base, uint64_t mem_size, void *fdt)
>  {
>      uint64_t temp, fdt_addr;
> diff --git a/include/hw/riscv/boot.h b/include/hw/riscv/boot.h
> index c3de897371..cbd131bad7 100644
> --- a/include/hw/riscv/boot.h
> +++ b/include/hw/riscv/boot.h
> @@ -47,7 +47,6 @@ target_ulong riscv_load_kernel(MachineState *machine,
>                                 target_ulong firmware_end_addr,
>                                 bool load_initrd,
>                                 symbol_fn_t sym_cb);
> -void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry);
>  uint64_t riscv_load_fdt(hwaddr dram_start, uint64_t dram_size, void *fdt);
>  void riscv_setup_rom_reset_vec(MachineState *machine, RISCVHartArrayState *harts,
>                                 hwaddr saddr,
> --
> 2.39.0
>
>

[PATCH v7 3/3] hw/riscv: clear kernel_entry higher bits in load_elf_ram_sym()

[Daniel Henrique Barboza]

Recent hw/risc/boot.c changes caused a regression in an use case with
the Xvisor hypervisor. Running a 32 bit QEMU guest with '-kernel'
stopped working. The reason seems to be that Xvisor is using 64 bit to
encode the 32 bit addresses from the guest, and load_elf_ram_sym() is
sign-extending the result with '1's [1].

This can very well be an issue with Xvisor, but since it's not hard to
amend it in our side we're going for it. Use a translate_fn() callback
to be called by load_elf_ram_sym() and clear the higher bits of the
result if we're running a 32 bit CPU.

[1] https://lists.gnu.org/archive/html/qemu-devel/2023-01/msg02281.html

Suggested-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Suggested-by: Bin Meng <bmeng.cn@gmail.com>
Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 hw/riscv/boot.c            | 23 ++++++++++++++++++++++-
 hw/riscv/microchip_pfsoc.c |  4 ++--
 hw/riscv/opentitan.c       |  3 ++-
 hw/riscv/sifive_e.c        |  3 ++-
 hw/riscv/sifive_u.c        |  4 ++--
 hw/riscv/spike.c           |  2 +-
 hw/riscv/virt.c            |  4 ++--
 include/hw/riscv/boot.h    |  1 +
 8 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c
index e868fb6ade..7f8295bf5e 100644
--- a/hw/riscv/boot.c
+++ b/hw/riscv/boot.c
@@ -213,7 +213,27 @@ static void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry)
     }
 }
 
+static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
+{
+    RISCVHartArrayState *harts = opaque;
+
+    /*
+     * For 32 bit CPUs, kernel_load_base is sign-extended (i.e.
+     * it can be padded with '1's) if the hypervisor, for some
+     * reason, is using 64 bit addresses with 32 bit guests.
+     *
+     * Clear the higher bits to avoid the padding if we're
+     * running a 32 bit CPU.
+     */
+    if (riscv_is_32bit(harts)) {
+        return addr & 0x0fffffff;
+    }
+
+    return addr;
+}
+
 target_ulong riscv_load_kernel(MachineState *machine,
+                               RISCVHartArrayState *harts,
                                target_ulong kernel_start_addr,
                                bool load_initrd,
                                symbol_fn_t sym_cb)
@@ -231,7 +251,8 @@ target_ulong riscv_load_kernel(MachineState *machine,
      * the (expected) load address load address. This allows kernels to have
      * separate SBI and ELF entry points (used by FreeBSD, for example).
      */
-    if (load_elf_ram_sym(kernel_filename, NULL, NULL, NULL,
+    if (load_elf_ram_sym(kernel_filename, NULL,
+                         translate_kernel_address, NULL,
                          NULL, &kernel_load_base, NULL, NULL, 0,
                          EM_RISCV, 1, 0, NULL, true, sym_cb) > 0) {
         kernel_entry = kernel_load_base;
diff --git a/hw/riscv/microchip_pfsoc.c b/hw/riscv/microchip_pfsoc.c
index c45023a2b1..b7e171b605 100644
--- a/hw/riscv/microchip_pfsoc.c
+++ b/hw/riscv/microchip_pfsoc.c
@@ -629,8 +629,8 @@ static void microchip_icicle_kit_machine_init(MachineState *machine)
         kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc.u_cpus,
                                                          firmware_end_addr);
 
-        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
-                                         true, NULL);
+        kernel_entry = riscv_load_kernel(machine, &s->soc.u_cpus,
+                                         kernel_start_addr, true, NULL);
 
         /* Compute the fdt load address in dram */
         fdt_load_addr = riscv_load_fdt(memmap[MICROCHIP_PFSOC_DRAM_LO].base,
diff --git a/hw/riscv/opentitan.c b/hw/riscv/opentitan.c
index f6fd9725a5..1404a52da0 100644
--- a/hw/riscv/opentitan.c
+++ b/hw/riscv/opentitan.c
@@ -101,7 +101,8 @@ static void opentitan_board_init(MachineState *machine)
     }
 
     if (machine->kernel_filename) {
-        riscv_load_kernel(machine, memmap[IBEX_DEV_RAM].base, false, NULL);
+        riscv_load_kernel(machine, &s->soc.cpus,
+                          memmap[IBEX_DEV_RAM].base, false, NULL);
     }
 }
 
diff --git a/hw/riscv/sifive_e.c b/hw/riscv/sifive_e.c
index 6835d1c807..04939b60c3 100644
--- a/hw/riscv/sifive_e.c
+++ b/hw/riscv/sifive_e.c
@@ -114,7 +114,8 @@ static void sifive_e_machine_init(MachineState *machine)
                           memmap[SIFIVE_E_DEV_MROM].base, &address_space_memory);
 
     if (machine->kernel_filename) {
-        riscv_load_kernel(machine, memmap[SIFIVE_E_DEV_DTIM].base,
+        riscv_load_kernel(machine, &s->soc.cpus,
+                          memmap[SIFIVE_E_DEV_DTIM].base,
                           false, NULL);
     }
 }
diff --git a/hw/riscv/sifive_u.c b/hw/riscv/sifive_u.c
index 9a75d4aa62..214430d40c 100644
--- a/hw/riscv/sifive_u.c
+++ b/hw/riscv/sifive_u.c
@@ -598,8 +598,8 @@ static void sifive_u_machine_init(MachineState *machine)
         kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc.u_cpus,
                                                          firmware_end_addr);
 
-        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
-                                         true, NULL);
+        kernel_entry = riscv_load_kernel(machine, &s->soc.u_cpus,
+                                         kernel_start_addr, true, NULL);
     } else {
        /*
         * If dynamic firmware is used, it doesn't know where is the next mode
diff --git a/hw/riscv/spike.c b/hw/riscv/spike.c
index c517885e6e..b3aac2178b 100644
--- a/hw/riscv/spike.c
+++ b/hw/riscv/spike.c
@@ -307,7 +307,7 @@ static void spike_board_init(MachineState *machine)
         kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc[0],
                                                          firmware_end_addr);
 
-        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
+        kernel_entry = riscv_load_kernel(machine, &s->soc[0], kernel_start_addr,
                                          true, htif_symbol_callback);
     } else {
        /*
diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
index a931ed05ab..60c8729b5f 100644
--- a/hw/riscv/virt.c
+++ b/hw/riscv/virt.c
@@ -1281,8 +1281,8 @@ static void virt_machine_done(Notifier *notifier, void *data)
         kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc[0],
                                                          firmware_end_addr);
 
-        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
-                                         true, NULL);
+        kernel_entry = riscv_load_kernel(machine, &s->soc[0],
+                                         kernel_start_addr, true, NULL);
     } else {
        /*
         * If dynamic firmware is used, it doesn't know where is the next mode
diff --git a/include/hw/riscv/boot.h b/include/hw/riscv/boot.h
index cbd131bad7..bc9faed397 100644
--- a/include/hw/riscv/boot.h
+++ b/include/hw/riscv/boot.h
@@ -44,6 +44,7 @@ target_ulong riscv_load_firmware(const char *firmware_filename,
                                  hwaddr firmware_load_addr,
                                  symbol_fn_t sym_cb);
 target_ulong riscv_load_kernel(MachineState *machine,
+                               RISCVHartArrayState *harts,
                                target_ulong firmware_end_addr,
                                bool load_initrd,
                                symbol_fn_t sym_cb);
-- 
2.39.0

Re: [PATCH v7 3/3] hw/riscv: clear kernel_entry higher bits in load_elf_ram_sym()

[Bin Meng]

On Sat, Jan 14, 2023 at 1:18 AM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> Recent hw/risc/boot.c changes caused a regression in an use case with
> the Xvisor hypervisor. Running a 32 bit QEMU guest with '-kernel'
> stopped working. The reason seems to be that Xvisor is using 64 bit to
> encode the 32 bit addresses from the guest, and load_elf_ram_sym() is
> sign-extending the result with '1's [1].

I would say it's not a regression of QEMU but something weird happened
to Alistair's 32-bit Xvisor image.

I just built a 32-bit Xvisor image from the latest Xvisor head
following the instructions provided in its source tree. With the
mainline QEMU only BIN file boots, but ELF does not. My 32-bit Xvisor
image has an address of 0x10000000. Apparently this address is not
correct, and the issue I saw is different from Alistair's. Alistair,
could you investigate why your 32-bit Xvisor ELF image has an address
of 0xffffffff80000000 set to kernel_load_base?

>
> This can very well be an issue with Xvisor, but since it's not hard to
> amend it in our side we're going for it. Use a translate_fn() callback
> to be called by load_elf_ram_sym() and clear the higher bits of the
> result if we're running a 32 bit CPU.
>
> [1] https://lists.gnu.org/archive/html/qemu-devel/2023-01/msg02281.html
>
> Suggested-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> Suggested-by: Bin Meng <bmeng.cn@gmail.com>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
> ---
>  hw/riscv/boot.c            | 23 ++++++++++++++++++++++-
>  hw/riscv/microchip_pfsoc.c |  4 ++--
>  hw/riscv/opentitan.c       |  3 ++-
>  hw/riscv/sifive_e.c        |  3 ++-
>  hw/riscv/sifive_u.c        |  4 ++--
>  hw/riscv/spike.c           |  2 +-
>  hw/riscv/virt.c            |  4 ++--
>  include/hw/riscv/boot.h    |  1 +
>  8 files changed, 34 insertions(+), 10 deletions(-)
>
> diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c
> index e868fb6ade..7f8295bf5e 100644
> --- a/hw/riscv/boot.c
> +++ b/hw/riscv/boot.c
> @@ -213,7 +213,27 @@ static void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry)
>      }
>  }
>
> +static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
> +{
> +    RISCVHartArrayState *harts = opaque;
> +
> +    /*
> +     * For 32 bit CPUs, kernel_load_base is sign-extended (i.e.
> +     * it can be padded with '1's) if the hypervisor, for some
> +     * reason, is using 64 bit addresses with 32 bit guests.
> +     *
> +     * Clear the higher bits to avoid the padding if we're
> +     * running a 32 bit CPU.
> +     */
> +    if (riscv_is_32bit(harts)) {
> +        return addr & 0x0fffffff;
> +    }
> +
> +    return addr;
> +}
> +
>  target_ulong riscv_load_kernel(MachineState *machine,
> +                               RISCVHartArrayState *harts,
>                                 target_ulong kernel_start_addr,
>                                 bool load_initrd,
>                                 symbol_fn_t sym_cb)
> @@ -231,7 +251,8 @@ target_ulong riscv_load_kernel(MachineState *machine,
>       * the (expected) load address load address. This allows kernels to have
>       * separate SBI and ELF entry points (used by FreeBSD, for example).
>       */
> -    if (load_elf_ram_sym(kernel_filename, NULL, NULL, NULL,
> +    if (load_elf_ram_sym(kernel_filename, NULL,
> +                         translate_kernel_address, NULL,
>                           NULL, &kernel_load_base, NULL, NULL, 0,
>                           EM_RISCV, 1, 0, NULL, true, sym_cb) > 0) {
>          kernel_entry = kernel_load_base;
> diff --git a/hw/riscv/microchip_pfsoc.c b/hw/riscv/microchip_pfsoc.c
> index c45023a2b1..b7e171b605 100644
> --- a/hw/riscv/microchip_pfsoc.c
> +++ b/hw/riscv/microchip_pfsoc.c
> @@ -629,8 +629,8 @@ static void microchip_icicle_kit_machine_init(MachineState *machine)
>          kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc.u_cpus,
>                                                           firmware_end_addr);
>
> -        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
> -                                         true, NULL);
> +        kernel_entry = riscv_load_kernel(machine, &s->soc.u_cpus,
> +                                         kernel_start_addr, true, NULL);
>
>          /* Compute the fdt load address in dram */
>          fdt_load_addr = riscv_load_fdt(memmap[MICROCHIP_PFSOC_DRAM_LO].base,
> diff --git a/hw/riscv/opentitan.c b/hw/riscv/opentitan.c
> index f6fd9725a5..1404a52da0 100644
> --- a/hw/riscv/opentitan.c
> +++ b/hw/riscv/opentitan.c
> @@ -101,7 +101,8 @@ static void opentitan_board_init(MachineState *machine)
>      }
>
>      if (machine->kernel_filename) {
> -        riscv_load_kernel(machine, memmap[IBEX_DEV_RAM].base, false, NULL);
> +        riscv_load_kernel(machine, &s->soc.cpus,
> +                          memmap[IBEX_DEV_RAM].base, false, NULL);
>      }
>  }
>
> diff --git a/hw/riscv/sifive_e.c b/hw/riscv/sifive_e.c
> index 6835d1c807..04939b60c3 100644
> --- a/hw/riscv/sifive_e.c
> +++ b/hw/riscv/sifive_e.c
> @@ -114,7 +114,8 @@ static void sifive_e_machine_init(MachineState *machine)
>                            memmap[SIFIVE_E_DEV_MROM].base, &address_space_memory);
>
>      if (machine->kernel_filename) {
> -        riscv_load_kernel(machine, memmap[SIFIVE_E_DEV_DTIM].base,
> +        riscv_load_kernel(machine, &s->soc.cpus,
> +                          memmap[SIFIVE_E_DEV_DTIM].base,
>                            false, NULL);
>      }
>  }
> diff --git a/hw/riscv/sifive_u.c b/hw/riscv/sifive_u.c
> index 9a75d4aa62..214430d40c 100644
> --- a/hw/riscv/sifive_u.c
> +++ b/hw/riscv/sifive_u.c
> @@ -598,8 +598,8 @@ static void sifive_u_machine_init(MachineState *machine)
>          kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc.u_cpus,
>                                                           firmware_end_addr);
>
> -        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
> -                                         true, NULL);
> +        kernel_entry = riscv_load_kernel(machine, &s->soc.u_cpus,
> +                                         kernel_start_addr, true, NULL);
>      } else {
>         /*
>          * If dynamic firmware is used, it doesn't know where is the next mode
> diff --git a/hw/riscv/spike.c b/hw/riscv/spike.c
> index c517885e6e..b3aac2178b 100644
> --- a/hw/riscv/spike.c
> +++ b/hw/riscv/spike.c
> @@ -307,7 +307,7 @@ static void spike_board_init(MachineState *machine)
>          kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc[0],
>                                                           firmware_end_addr);
>
> -        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
> +        kernel_entry = riscv_load_kernel(machine, &s->soc[0], kernel_start_addr,
>                                           true, htif_symbol_callback);
>      } else {
>         /*
> diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> index a931ed05ab..60c8729b5f 100644
> --- a/hw/riscv/virt.c
> +++ b/hw/riscv/virt.c
> @@ -1281,8 +1281,8 @@ static void virt_machine_done(Notifier *notifier, void *data)
>          kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc[0],
>                                                           firmware_end_addr);
>
> -        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
> -                                         true, NULL);
> +        kernel_entry = riscv_load_kernel(machine, &s->soc[0],
> +                                         kernel_start_addr, true, NULL);
>      } else {
>         /*
>          * If dynamic firmware is used, it doesn't know where is the next mode
> diff --git a/include/hw/riscv/boot.h b/include/hw/riscv/boot.h
> index cbd131bad7..bc9faed397 100644
> --- a/include/hw/riscv/boot.h
> +++ b/include/hw/riscv/boot.h
> @@ -44,6 +44,7 @@ target_ulong riscv_load_firmware(const char *firmware_filename,
>                                   hwaddr firmware_load_addr,
>                                   symbol_fn_t sym_cb);
>  target_ulong riscv_load_kernel(MachineState *machine,
> +                               RISCVHartArrayState *harts,
>                                 target_ulong firmware_end_addr,
>                                 bool load_initrd,
>                                 symbol_fn_t sym_cb);

Regards,
Bin

Re: [PATCH v7 3/3] hw/riscv: clear kernel_entry higher bits in load_elf_ram_sym()

[Alistair Francis]

On Sat, Jan 14, 2023 at 11:41 PM Bin Meng <bmeng.cn@gmail.com> wrote:
>
> On Sat, Jan 14, 2023 at 1:18 AM Daniel Henrique Barboza
> <dbarboza@ventanamicro.com> wrote:
> >
> > Recent hw/risc/boot.c changes caused a regression in an use case with
> > the Xvisor hypervisor. Running a 32 bit QEMU guest with '-kernel'
> > stopped working. The reason seems to be that Xvisor is using 64 bit to
> > encode the 32 bit addresses from the guest, and load_elf_ram_sym() is
> > sign-extending the result with '1's [1].
>
> I would say it's not a regression of QEMU but something weird happened
> to Alistair's 32-bit Xvisor image.

I don't think it's a Xvisor issue.

>
> I just built a 32-bit Xvisor image from the latest Xvisor head
> following the instructions provided in its source tree. With the
> mainline QEMU only BIN file boots, but ELF does not. My 32-bit Xvisor
> image has an address of 0x10000000. Apparently this address is not
> correct, and the issue I saw is different from Alistair's. Alistair,
> could you investigate why your 32-bit Xvisor ELF image has an address
> of 0xffffffff80000000 set to kernel_load_base?

Looking in load_elf() in include/hw/elf_ops.h at this line:

    if (lowaddr)
        *lowaddr = (uint64_t)(elf_sword)low;

I can see that `low` is 0x80000000 but lowaddr is set to
0xffffffff80000000. So the address is being sign extended with 1s.

This patch seems to be the correct fix.

Alistair

>
> >
> > This can very well be an issue with Xvisor, but since it's not hard to
> > amend it in our side we're going for it. Use a translate_fn() callback
> > to be called by load_elf_ram_sym() and clear the higher bits of the
> > result if we're running a 32 bit CPU.
> >
> > [1] https://lists.gnu.org/archive/html/qemu-devel/2023-01/msg02281.html
> >
> > Suggested-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> > Suggested-by: Bin Meng <bmeng.cn@gmail.com>
> > Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

Thanks for the patch. This should be the first patch of the series
though, so that we never break guest loading.

> > ---
> >  hw/riscv/boot.c            | 23 ++++++++++++++++++++++-
> >  hw/riscv/microchip_pfsoc.c |  4 ++--
> >  hw/riscv/opentitan.c       |  3 ++-
> >  hw/riscv/sifive_e.c        |  3 ++-
> >  hw/riscv/sifive_u.c        |  4 ++--
> >  hw/riscv/spike.c           |  2 +-
> >  hw/riscv/virt.c            |  4 ++--
> >  include/hw/riscv/boot.h    |  1 +
> >  8 files changed, 34 insertions(+), 10 deletions(-)
> >
> > diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c
> > index e868fb6ade..7f8295bf5e 100644
> > --- a/hw/riscv/boot.c
> > +++ b/hw/riscv/boot.c
> > @@ -213,7 +213,27 @@ static void riscv_load_initrd(MachineState *machine, uint64_t kernel_entry)
> >      }
> >  }
> >
> > +static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
> > +{
> > +    RISCVHartArrayState *harts = opaque;
> > +
> > +    /*
> > +     * For 32 bit CPUs, kernel_load_base is sign-extended (i.e.
> > +     * it can be padded with '1's) if the hypervisor, for some
> > +     * reason, is using 64 bit addresses with 32 bit guests.
> > +     *
> > +     * Clear the higher bits to avoid the padding if we're
> > +     * running a 32 bit CPU.
> > +     */
> > +    if (riscv_is_32bit(harts)) {
> > +        return addr & 0x0fffffff;
> > +    }
> > +
> > +    return addr;
> > +}
> > +
> >  target_ulong riscv_load_kernel(MachineState *machine,
> > +                               RISCVHartArrayState *harts,
> >                                 target_ulong kernel_start_addr,
> >                                 bool load_initrd,
> >                                 symbol_fn_t sym_cb)
> > @@ -231,7 +251,8 @@ target_ulong riscv_load_kernel(MachineState *machine,
> >       * the (expected) load address load address. This allows kernels to have
> >       * separate SBI and ELF entry points (used by FreeBSD, for example).
> >       */
> > -    if (load_elf_ram_sym(kernel_filename, NULL, NULL, NULL,
> > +    if (load_elf_ram_sym(kernel_filename, NULL,
> > +                         translate_kernel_address, NULL,
> >                           NULL, &kernel_load_base, NULL, NULL, 0,
> >                           EM_RISCV, 1, 0, NULL, true, sym_cb) > 0) {
> >          kernel_entry = kernel_load_base;
> > diff --git a/hw/riscv/microchip_pfsoc.c b/hw/riscv/microchip_pfsoc.c
> > index c45023a2b1..b7e171b605 100644
> > --- a/hw/riscv/microchip_pfsoc.c
> > +++ b/hw/riscv/microchip_pfsoc.c
> > @@ -629,8 +629,8 @@ static void microchip_icicle_kit_machine_init(MachineState *machine)
> >          kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc.u_cpus,
> >                                                           firmware_end_addr);
> >
> > -        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
> > -                                         true, NULL);
> > +        kernel_entry = riscv_load_kernel(machine, &s->soc.u_cpus,
> > +                                         kernel_start_addr, true, NULL);
> >
> >          /* Compute the fdt load address in dram */
> >          fdt_load_addr = riscv_load_fdt(memmap[MICROCHIP_PFSOC_DRAM_LO].base,
> > diff --git a/hw/riscv/opentitan.c b/hw/riscv/opentitan.c
> > index f6fd9725a5..1404a52da0 100644
> > --- a/hw/riscv/opentitan.c
> > +++ b/hw/riscv/opentitan.c
> > @@ -101,7 +101,8 @@ static void opentitan_board_init(MachineState *machine)
> >      }
> >
> >      if (machine->kernel_filename) {
> > -        riscv_load_kernel(machine, memmap[IBEX_DEV_RAM].base, false, NULL);
> > +        riscv_load_kernel(machine, &s->soc.cpus,
> > +                          memmap[IBEX_DEV_RAM].base, false, NULL);
> >      }
> >  }
> >
> > diff --git a/hw/riscv/sifive_e.c b/hw/riscv/sifive_e.c
> > index 6835d1c807..04939b60c3 100644
> > --- a/hw/riscv/sifive_e.c
> > +++ b/hw/riscv/sifive_e.c
> > @@ -114,7 +114,8 @@ static void sifive_e_machine_init(MachineState *machine)
> >                            memmap[SIFIVE_E_DEV_MROM].base, &address_space_memory);
> >
> >      if (machine->kernel_filename) {
> > -        riscv_load_kernel(machine, memmap[SIFIVE_E_DEV_DTIM].base,
> > +        riscv_load_kernel(machine, &s->soc.cpus,
> > +                          memmap[SIFIVE_E_DEV_DTIM].base,
> >                            false, NULL);
> >      }
> >  }
> > diff --git a/hw/riscv/sifive_u.c b/hw/riscv/sifive_u.c
> > index 9a75d4aa62..214430d40c 100644
> > --- a/hw/riscv/sifive_u.c
> > +++ b/hw/riscv/sifive_u.c
> > @@ -598,8 +598,8 @@ static void sifive_u_machine_init(MachineState *machine)
> >          kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc.u_cpus,
> >                                                           firmware_end_addr);
> >
> > -        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
> > -                                         true, NULL);
> > +        kernel_entry = riscv_load_kernel(machine, &s->soc.u_cpus,
> > +                                         kernel_start_addr, true, NULL);
> >      } else {
> >         /*
> >          * If dynamic firmware is used, it doesn't know where is the next mode
> > diff --git a/hw/riscv/spike.c b/hw/riscv/spike.c
> > index c517885e6e..b3aac2178b 100644
> > --- a/hw/riscv/spike.c
> > +++ b/hw/riscv/spike.c
> > @@ -307,7 +307,7 @@ static void spike_board_init(MachineState *machine)
> >          kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc[0],
> >                                                           firmware_end_addr);
> >
> > -        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
> > +        kernel_entry = riscv_load_kernel(machine, &s->soc[0], kernel_start_addr,
> >                                           true, htif_symbol_callback);
> >      } else {
> >         /*
> > diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> > index a931ed05ab..60c8729b5f 100644
> > --- a/hw/riscv/virt.c
> > +++ b/hw/riscv/virt.c
> > @@ -1281,8 +1281,8 @@ static void virt_machine_done(Notifier *notifier, void *data)
> >          kernel_start_addr = riscv_calc_kernel_start_addr(&s->soc[0],
> >                                                           firmware_end_addr);
> >
> > -        kernel_entry = riscv_load_kernel(machine, kernel_start_addr,
> > -                                         true, NULL);
> > +        kernel_entry = riscv_load_kernel(machine, &s->soc[0],
> > +                                         kernel_start_addr, true, NULL);
> >      } else {
> >         /*
> >          * If dynamic firmware is used, it doesn't know where is the next mode
> > diff --git a/include/hw/riscv/boot.h b/include/hw/riscv/boot.h
> > index cbd131bad7..bc9faed397 100644
> > --- a/include/hw/riscv/boot.h
> > +++ b/include/hw/riscv/boot.h
> > @@ -44,6 +44,7 @@ target_ulong riscv_load_firmware(const char *firmware_filename,
> >                                   hwaddr firmware_load_addr,
> >                                   symbol_fn_t sym_cb);
> >  target_ulong riscv_load_kernel(MachineState *machine,
> > +                               RISCVHartArrayState *harts,
> >                                 target_ulong firmware_end_addr,
> >                                 bool load_initrd,
> >                                 symbol_fn_t sym_cb);
>
> Regards,
> Bin
>

Re: [PATCH v7 3/3] hw/riscv: clear kernel_entry higher bits in load_elf_ram_sym()

[Bin Meng]

Hi Alistair,

On Mon, Jan 16, 2023 at 12:28 PM Alistair Francis <alistair23@gmail.com> wrote:
>
> On Sat, Jan 14, 2023 at 11:41 PM Bin Meng <bmeng.cn@gmail.com> wrote:
> >
> > On Sat, Jan 14, 2023 at 1:18 AM Daniel Henrique Barboza
> > <dbarboza@ventanamicro.com> wrote:
> > >
> > > Recent hw/risc/boot.c changes caused a regression in an use case with
> > > the Xvisor hypervisor. Running a 32 bit QEMU guest with '-kernel'
> > > stopped working. The reason seems to be that Xvisor is using 64 bit to
> > > encode the 32 bit addresses from the guest, and load_elf_ram_sym() is
> > > sign-extending the result with '1's [1].
> >
> > I would say it's not a regression of QEMU but something weird happened
> > to Alistair's 32-bit Xvisor image.
>
> I don't think it's a Xvisor issue.
>
> >
> > I just built a 32-bit Xvisor image from the latest Xvisor head
> > following the instructions provided in its source tree. With the
> > mainline QEMU only BIN file boots, but ELF does not. My 32-bit Xvisor
> > image has an address of 0x10000000. Apparently this address is not
> > correct, and the issue I saw is different from Alistair's. Alistair,
> > could you investigate why your 32-bit Xvisor ELF image has an address
> > of 0xffffffff80000000 set to kernel_load_base?
>
> Looking in load_elf() in include/hw/elf_ops.h at this line:
>
>     if (lowaddr)
>         *lowaddr = (uint64_t)(elf_sword)low;
>
> I can see that `low` is 0x80000000 but lowaddr is set to
> 0xffffffff80000000. So the address is being sign extended with 1s.
>

I don't understand the sign extension here. This seems intentional as
the codes does the signed extension then casted to unsigned 64-bit.

Do you know why?

> This patch seems to be the correct fix.
>

Regards,
Bin

Re: [PATCH v7 3/3] hw/riscv: clear kernel_entry higher bits in load_elf_ram_sym()

[Alistair Francis]

On Thu, Jan 26, 2023 at 10:07 PM Bin Meng <bmeng.cn@gmail.com> wrote:
>
> Hi Alistair,
>
> On Mon, Jan 16, 2023 at 12:28 PM Alistair Francis <alistair23@gmail.com> wrote:
> >
> > On Sat, Jan 14, 2023 at 11:41 PM Bin Meng <bmeng.cn@gmail.com> wrote:
> > >
> > > On Sat, Jan 14, 2023 at 1:18 AM Daniel Henrique Barboza
> > > <dbarboza@ventanamicro.com> wrote:
> > > >
> > > > Recent hw/risc/boot.c changes caused a regression in an use case with
> > > > the Xvisor hypervisor. Running a 32 bit QEMU guest with '-kernel'
> > > > stopped working. The reason seems to be that Xvisor is using 64 bit to
> > > > encode the 32 bit addresses from the guest, and load_elf_ram_sym() is
> > > > sign-extending the result with '1's [1].
> > >
> > > I would say it's not a regression of QEMU but something weird happened
> > > to Alistair's 32-bit Xvisor image.
> >
> > I don't think it's a Xvisor issue.
> >
> > >
> > > I just built a 32-bit Xvisor image from the latest Xvisor head
> > > following the instructions provided in its source tree. With the
> > > mainline QEMU only BIN file boots, but ELF does not. My 32-bit Xvisor
> > > image has an address of 0x10000000. Apparently this address is not
> > > correct, and the issue I saw is different from Alistair's. Alistair,
> > > could you investigate why your 32-bit Xvisor ELF image has an address
> > > of 0xffffffff80000000 set to kernel_load_base?
> >
> > Looking in load_elf() in include/hw/elf_ops.h at this line:
> >
> >     if (lowaddr)
> >         *lowaddr = (uint64_t)(elf_sword)low;
> >
> > I can see that `low` is 0x80000000 but lowaddr is set to
> > 0xffffffff80000000. So the address is being sign extended with 1s.
> >
>
> I don't understand the sign extension here. This seems intentional as
> the codes does the signed extension then casted to unsigned 64-bit.
>
> Do you know why?

No idea!

Alistair

>
> > This patch seems to be the correct fix.
> >
>
> Regards,
> Bin

Re: [PATCH v7 3/3] hw/riscv: clear kernel_entry higher bits in load_elf_ram_sym()

[Philippe Mathieu-Daudé]

On 13/1/23 18:18, Daniel Henrique Barboza wrote:
> Recent hw/risc/boot.c changes caused a regression in an use case with
> the Xvisor hypervisor. Running a 32 bit QEMU guest with '-kernel'
> stopped working. The reason seems to be that Xvisor is using 64 bit to
> encode the 32 bit addresses from the guest, and load_elf_ram_sym() is
> sign-extending the result with '1's [1].
> 
> This can very well be an issue with Xvisor, but since it's not hard to
> amend it in our side we're going for it. Use a translate_fn() callback
> to be called by load_elf_ram_sym() and clear the higher bits of the
> result if we're running a 32 bit CPU.
> 
> [1] https://lists.gnu.org/archive/html/qemu-devel/2023-01/msg02281.html
> 
> Suggested-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> Suggested-by: Bin Meng <bmeng.cn@gmail.com>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
> ---
>   hw/riscv/boot.c            | 23 ++++++++++++++++++++++-
>   hw/riscv/microchip_pfsoc.c |  4 ++--
>   hw/riscv/opentitan.c       |  3 ++-
>   hw/riscv/sifive_e.c        |  3 ++-
>   hw/riscv/sifive_u.c        |  4 ++--
>   hw/riscv/spike.c           |  2 +-
>   hw/riscv/virt.c            |  4 ++--
>   include/hw/riscv/boot.h    |  1 +
>   8 files changed, 34 insertions(+), 10 deletions(-)

> +static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
> +{
> +    RISCVHartArrayState *harts = opaque;
> +
> +    /*
> +     * For 32 bit CPUs, kernel_load_base is sign-extended (i.e.
> +     * it can be padded with '1's) if the hypervisor, for some
> +     * reason, is using 64 bit addresses with 32 bit guests.
> +     *
> +     * Clear the higher bits to avoid the padding if we're
> +     * running a 32 bit CPU.
> +     */
> +    if (riscv_is_32bit(harts)) {
> +        return addr & 0x0fffffff;

Instead of this magic mask, can we add some architectural definition
in target/riscv/cpu_bits.h and use it as:

            return extract64(addr, 0, xxx_ADDR_BITS);

to make the code self-descriptive?

Otherwise LGTM, thanks!

> +    }
> +
> +    return addr;
> +}