[PATCH v4 0/3] linux-user: Fix static armhf binaries and optmize memory layout

[PATCH v4 0/3] linux-user: Fix static armhf binaries and optmize memory layout

[Helge Deller]

This patch series:
- fixes qemu-arm to run static armhf binaries
- shows address of heap in /proc/pid/maps output for all architectures
- optimizes address layout of loaded executable
- increases free heap for guest apps

NOTE:
- this patch series is for qemu v8.1.0-rc ONLY.
- do not apply on top of v8.0-stable series, as those use
  a different search algorithm for free mmap memory
  and thus will give improper memory layouts.

It can be pulled from here:
https://github.com/hdeller/qemu-hppa/tree/brk-fixes-2

Changes
v4:
- add note that patch series is for v8.1.0-rc only
- changed TASK_UNMAPPED_BASE for 32- on 64-bit userspace

Helge Deller (3):
  linux-user: Show heap address in /proc/pid/maps
  linux-user: Optimize memory layout for static and dynamic executables
  linux-user: Load pie executables at upper memory

 include/exec/cpu_ldst.h |  4 +--
 linux-user/elfload.c    | 59 +++++++++++++----------------------------
 linux-user/loader.h     | 12 +++++++++
 linux-user/main.c       |  1 +
 linux-user/mmap.c       | 14 +---------
 linux-user/qemu.h       |  4 +--
 linux-user/syscall.c    |  8 ++++--
 7 files changed, 43 insertions(+), 59 deletions(-)

--
2.41.0

[PATCH v4 1/3] linux-user: Show heap address in /proc/pid/maps

[Helge Deller]

Show the memory location of the heap in the /proc/pid/maps file inside the
guest. The heap address will be stored in ts->heap_base, so make that
variable visible for all guest architectures, not just architectures for
semihosted binaries (arm, m68k, riscv).

Show 32- and 64-bit pointers with 8 digits and leading zeros (%08x/%08lx).
For 64-bit we could use %16lx, but we mimic the Linux kernel, which shows
it with %08lx too.

Example:

user@machine:/# uname -a
Linux paq 5.15.88+ #47 SMP Sun Jan 15 12:53:11 CET 2023 aarch64 GNU/Linux

user@machine:/# cat /proc/self/maps
00000000-00009000 r-xp 00000000 08:01 2380521                            /usr/bin/cat
00009000-0001f000 ---p 00000000 00:00 0
0001f000-00020000 r--p 0000f000 08:01 2380521                            /usr/bin/cat
00020000-00021000 rw-p 00010000 08:01 2380521                            /usr/bin/cat
00021000-00042000 rw-p 00000000 00:00 0                                  [heap]
5500000000-5500001000 ---p 00000000 00:00 0
5500001000-5500801000 rw-p 00000000 00:00 0                              [stack]
5500801000-5500827000 r-xp 00000000 08:01 2395258                        /usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1
5500827000-550083f000 ---p 00000000 00:00 0
550083f000-5500841000 r--p 0002e000 08:01 2395258                        /usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1
5500841000-5500843000 rw-p 00030000 08:01 2395258                        /usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1
5500843000-5500844000 r-xp 00000000 00:00 0
5500844000-5500846000 rw-p 00000000 00:00 0
5500850000-55009d7000 r-xp 00000000 08:01 2395261                        /usr/lib/aarch64-linux-gnu/libc.so.6
55009d7000-55009ed000 ---p 00187000 08:01 2395261                        /usr/lib/aarch64-linux-gnu/libc.so.6
55009ed000-55009f0000 r--p 0018d000 08:01 2395261                        /usr/lib/aarch64-linux-gnu/libc.so.6
55009f0000-55009f2000 rw-p 00190000 08:01 2395261                        /usr/lib/aarch64-linux-gnu/libc.so.6
55009f2000-55009ff000 rw-p 00000000 00:00 0

Signed-off-by: Helge Deller <deller@gmx.de>
---
 include/exec/cpu_ldst.h | 4 ++--
 linux-user/main.c       | 1 +
 linux-user/qemu.h       | 4 ++--
 linux-user/syscall.c    | 8 ++++++--
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
index 645476f0e5..f1e6f31e88 100644
--- a/include/exec/cpu_ldst.h
+++ b/include/exec/cpu_ldst.h
@@ -72,10 +72,10 @@
  */
 #if TARGET_VIRT_ADDR_SPACE_BITS <= 32
 typedef uint32_t abi_ptr;
-#define TARGET_ABI_FMT_ptr "%x"
+#define TARGET_ABI_FMT_ptr "%08x"
 #else
 typedef uint64_t abi_ptr;
-#define TARGET_ABI_FMT_ptr "%"PRIx64
+#define TARGET_ABI_FMT_ptr "%08"PRIx64
 #endif

 #ifndef TARGET_TAGGED_ADDRESSES
diff --git a/linux-user/main.c b/linux-user/main.c
index dba67ffa36..12f3d8a93e 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -955,6 +955,7 @@ int main(int argc, char **argv, char **envp)
        the real value of GUEST_BASE into account.  */
     tcg_prologue_init(tcg_ctx);

+    ts->heap_base = info->brk;
     target_cpu_copy_regs(env, regs);

     if (gdbstub) {
diff --git a/linux-user/qemu.h b/linux-user/qemu.h
index 802794db63..7a6adac637 100644
--- a/linux-user/qemu.h
+++ b/linux-user/qemu.h
@@ -121,11 +121,11 @@ typedef struct TaskState {
 #ifdef TARGET_M68K
     abi_ulong tp_value;
 #endif
-#if defined(TARGET_ARM) || defined(TARGET_M68K) || defined(TARGET_RISCV)
+
     /* Extra fields for semihosted binaries.  */
     abi_ulong heap_base;
     abi_ulong heap_limit;
-#endif
+
     abi_ulong stack_base;
     int used; /* non zero if used */
     struct image_info *info;
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index 95727a816a..220c4a04b8 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -8131,14 +8131,18 @@ static int open_self_maps_1(CPUArchState *cpu_env, int fd, bool smaps)
                 continue;
             }

+            path = e->path;
+
+            if (ts->heap_base && h2g(min) == ts->heap_base) {
+                path = "[heap]";
+            }
+
 #ifdef TARGET_HPPA
             if (h2g(max) == ts->info->stack_limit) {
 #else
             if (h2g(min) == ts->info->stack_limit) {
 #endif
                 path = "[stack]";
-            } else {
-                path = e->path;
             }

             count = dprintf(fd, TARGET_ABI_FMT_ptr "-" TARGET_ABI_FMT_ptr
--
2.41.0

[PATCH v4 2/3] linux-user: Optimize memory layout for static and dynamic executables

[Helge Deller]

Organize the emulated memory layout in a way which leaves as much memory as
possible for heap for the application.

This patch tries to optize the memory layout by loading pie executables
into lower memory and shared libs into higher memory (at
TASK_UNMAPPED_BASE). This leaves a bigger memory area usable for heap
space which will be located directly after the executable.
Up to now, pie executable and shared libs were loaded directly behind
each other in the area at TASK_UNMAPPED_BASE, which leaves very little
space for heap.

I tested this change on arm64, armhf and hppa (all in chroot on x86-64),
and with a static armhf binary (which is broken without this patch).

This patch temporarily breaks the Thread Sanitizer (TSan) application which
expects specific boundary definitions for memory mappings on different
platforms [1], see commit aab613fb9597 ("linux-user: Update TASK_UNMAPPED_BASE for aarch64")
for aarch64. The follow-up patch fixes it again.

[1] https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/tsan/rtl/tsan_platform.h

Signed-off-by: Helge Deller <deller@gmx.de>
---
 linux-user/elfload.c | 55 +++++++++++++-------------------------------
 linux-user/mmap.c    |  8 ++++---
 2 files changed, 21 insertions(+), 42 deletions(-)

diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index 861ec07abc..47a118e430 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -3023,6 +3023,7 @@ static void load_elf_image(const char *image_name, int image_fd,
     abi_ulong load_addr, load_bias, loaddr, hiaddr, error;
     int i, retval, prot_exec;
     Error *err = NULL;
+    bool is_main_executable;

     /* First of all, some simple consistency checks */
     if (!elf_check_ident(ehdr)) {
@@ -3106,28 +3107,8 @@ static void load_elf_image(const char *image_name, int image_fd,
         }
     }

-    if (pinterp_name != NULL) {
-        /*
-         * This is the main executable.
-         *
-         * Reserve extra space for brk.
-         * We hold on to this space while placing the interpreter
-         * and the stack, lest they be placed immediately after
-         * the data segment and block allocation from the brk.
-         *
-         * 16MB is chosen as "large enough" without being so large as
-         * to allow the result to not fit with a 32-bit guest on a
-         * 32-bit host. However some 64 bit guests (e.g. s390x)
-         * attempt to place their heap further ahead and currently
-         * nothing stops them smashing into QEMUs address space.
-         */
-#if TARGET_LONG_BITS == 64
-        info->reserve_brk = 32 * MiB;
-#else
-        info->reserve_brk = 16 * MiB;
-#endif
-        hiaddr += info->reserve_brk;
-
+    is_main_executable = (pinterp_name != NULL);
+    if (is_main_executable) {
         if (ehdr->e_type == ET_EXEC) {
             /*
              * Make sure that the low address does not conflict with
@@ -3136,7 +3117,7 @@ static void load_elf_image(const char *image_name, int image_fd,
             probe_guest_base(image_name, loaddr, hiaddr);
         } else {
             /*
-             * The binary is dynamic, but we still need to
+             * The binary is dynamic (pie-executabe), but we still need to
              * select guest_base.  In this case we pass a size.
              */
             probe_guest_base(image_name, 0, hiaddr - loaddr);
@@ -3159,7 +3140,7 @@ static void load_elf_image(const char *image_name, int image_fd,
      */
     load_addr = target_mmap(loaddr, (size_t)hiaddr - loaddr + 1, PROT_NONE,
                             MAP_PRIVATE | MAP_ANON | MAP_NORESERVE |
-                            (ehdr->e_type == ET_EXEC ? MAP_FIXED : 0),
+                            (is_main_executable ? MAP_FIXED : 0),
                             -1, 0);
     if (load_addr == -1) {
         goto exit_mmap;
@@ -3194,7 +3175,8 @@ static void load_elf_image(const char *image_name, int image_fd,
     info->end_code = 0;
     info->start_data = -1;
     info->end_data = 0;
-    info->brk = 0;
+    /* possible start for brk is behind all sections of this ELF file. */
+    info->brk = TARGET_PAGE_ALIGN(hiaddr);
     info->elf_flags = ehdr->e_flags;

     prot_exec = PROT_EXEC;
@@ -3288,9 +3270,6 @@ static void load_elf_image(const char *image_name, int image_fd,
                     info->end_data = vaddr_ef;
                 }
             }
-            if (vaddr_em > info->brk) {
-                info->brk = vaddr_em;
-            }
 #ifdef TARGET_MIPS
         } else if (eppnt->p_type == PT_MIPS_ABIFLAGS) {
             Mips_elf_abiflags_v0 abiflags;
@@ -3618,6 +3597,15 @@ int load_elf_binary(struct linux_binprm *bprm, struct image_info *info)

     if (elf_interpreter) {
         load_elf_interp(elf_interpreter, &interp_info, bprm->buf);
+        /*
+	 * Use brk address of interpreter if it was loaded above the
+	 * executable and leaves less than 16 MB for heap.
+	 * This happens e.g. with static binaries on armhf.
+         */
+        if (interp_info.brk > info->brk &&
+            interp_info.load_bias - info->brk < 16 * MiB)  {
+            info->brk = interp_info.brk;
+        }

         /* If the program interpreter is one of these two, then assume
            an iBCS2 image.  Otherwise assume a native linux image.  */
@@ -3672,17 +3660,6 @@ int load_elf_binary(struct linux_binprm *bprm, struct image_info *info)
     bprm->core_dump = &elf_core_dump;
 #endif

-    /*
-     * If we reserved extra space for brk, release it now.
-     * The implementation of do_brk in syscalls.c expects to be able
-     * to mmap pages in this space.
-     */
-    if (info->reserve_brk) {
-        abi_ulong start_brk = HOST_PAGE_ALIGN(info->brk);
-        abi_ulong end_brk = HOST_PAGE_ALIGN(info->brk + info->reserve_brk);
-        target_munmap(start_brk, end_brk - start_brk);
-    }
-
     return 0;
 }

diff --git a/linux-user/mmap.c b/linux-user/mmap.c
index a5dfb56545..848d2fd4bb 100644
--- a/linux-user/mmap.c
+++ b/linux-user/mmap.c
@@ -299,14 +299,16 @@ static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last,
 #ifdef TARGET_AARCH64
 # define TASK_UNMAPPED_BASE  0x5500000000
 #else
-# define TASK_UNMAPPED_BASE  (1ul << 38)
+# define TASK_UNMAPPED_BASE  0x4000000000
 #endif
-#else
+#elif HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 32
 #ifdef TARGET_HPPA
 # define TASK_UNMAPPED_BASE  0xfa000000
 #else
-# define TASK_UNMAPPED_BASE  0x40000000
+# define TASK_UNMAPPED_BASE  0xe0000000
 #endif
+#else /* HOST_LONG_BITS == 32 && TARGET_ABI_BITS == 32 */
+# define TASK_UNMAPPED_BASE  0x40000000
 #endif
 abi_ulong mmap_next_start = TASK_UNMAPPED_BASE;

--
2.41.0

[PATCH v4 3/3] linux-user: Load pie executables at upper memory

[Helge Deller]

Adjust the loader to load dynamic pie executables at around:
~ 0x5500000000  for 64-bit guest binaries on 64-bit host,
- 0x00500000    for 32-bit guest binaries on 64-bit host, and
- 0x00000000    for 32-bit guest binaries on 32-bit host.

This fixes the Thread Sanitizer (TSan) application again, as it was
done in aab613fb9597 ("linux-user: Update TASK_UNMAPPED_BASE for
aarch64"). Additionally it increases the free heap space for
applications.

Signed-off-by: Helge Deller <deller@gmx.de>
---
 linux-user/elfload.c |  6 ++++--
 linux-user/loader.h  | 12 ++++++++++++
 linux-user/mmap.c    | 16 +---------------
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index 47a118e430..8f5a79b537 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -3021,6 +3021,7 @@ static void load_elf_image(const char *image_name, int image_fd,
     struct elfhdr *ehdr = (struct elfhdr *)bprm_buf;
     struct elf_phdr *phdr;
     abi_ulong load_addr, load_bias, loaddr, hiaddr, error;
+    unsigned long load_offset = 0;
     int i, retval, prot_exec;
     Error *err = NULL;
     bool is_main_executable;
@@ -3121,6 +3122,7 @@ static void load_elf_image(const char *image_name, int image_fd,
              * select guest_base.  In this case we pass a size.
              */
             probe_guest_base(image_name, 0, hiaddr - loaddr);
+            load_offset = TASK_UNMAPPED_BASE_PIE;
         }
     }

@@ -3138,7 +3140,7 @@ static void load_elf_image(const char *image_name, int image_fd,
      * In both cases, we will overwrite pages in this range with mappings
      * from the executable.
      */
-    load_addr = target_mmap(loaddr, (size_t)hiaddr - loaddr + 1, PROT_NONE,
+    load_addr = target_mmap(loaddr + load_offset, (size_t)hiaddr - loaddr + 1, PROT_NONE,
                             MAP_PRIVATE | MAP_ANON | MAP_NORESERVE |
                             (is_main_executable ? MAP_FIXED : 0),
                             -1, 0);
@@ -3176,7 +3178,7 @@ static void load_elf_image(const char *image_name, int image_fd,
     info->start_data = -1;
     info->end_data = 0;
     /* possible start for brk is behind all sections of this ELF file. */
-    info->brk = TARGET_PAGE_ALIGN(hiaddr);
+    info->brk = TARGET_PAGE_ALIGN(load_offset + hiaddr);
     info->elf_flags = ehdr->e_flags;

     prot_exec = PROT_EXEC;
diff --git a/linux-user/loader.h b/linux-user/loader.h
index 59cbeacf24..799016cc99 100644
--- a/linux-user/loader.h
+++ b/linux-user/loader.h
@@ -18,6 +18,18 @@
 #ifndef LINUX_USER_LOADER_H
 #define LINUX_USER_LOADER_H

+/* where to map binaries? */
+#if HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 64
+# define TASK_UNMAPPED_BASE_PIE 0x5500000000
+# define TASK_UNMAPPED_BASE	0x7000000000
+#elif HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 32
+# define TASK_UNMAPPED_BASE_PIE	0x00500000
+# define TASK_UNMAPPED_BASE	0xfa000000
+#else /* HOST_LONG_BITS == 32 && TARGET_ABI_BITS == 32 */
+# define TASK_UNMAPPED_BASE_PIE	0x00000000
+# define TASK_UNMAPPED_BASE	0x40000000
+#endif
+
 /*
  * Read a good amount of data initially, to hopefully get all the
  * program headers loaded.
diff --git a/linux-user/mmap.c b/linux-user/mmap.c
index 848d2fd4bb..9434bc805d 100644
--- a/linux-user/mmap.c
+++ b/linux-user/mmap.c
@@ -23,6 +23,7 @@
 #include "user-internals.h"
 #include "user-mmap.h"
 #include "target_mman.h"
+#include "loader.h"

 static pthread_mutex_t mmap_mutex = PTHREAD_MUTEX_INITIALIZER;
 static __thread int mmap_lock_count;
@@ -295,21 +296,6 @@ static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last,
     return true;
 }

-#if HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 64
-#ifdef TARGET_AARCH64
-# define TASK_UNMAPPED_BASE  0x5500000000
-#else
-# define TASK_UNMAPPED_BASE  0x4000000000
-#endif
-#elif HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 32
-#ifdef TARGET_HPPA
-# define TASK_UNMAPPED_BASE  0xfa000000
-#else
-# define TASK_UNMAPPED_BASE  0xe0000000
-#endif
-#else /* HOST_LONG_BITS == 32 && TARGET_ABI_BITS == 32 */
-# define TASK_UNMAPPED_BASE  0x40000000
-#endif
 abi_ulong mmap_next_start = TASK_UNMAPPED_BASE;

 unsigned long last_brk;
--
2.41.0