[PATCH 0/3] gdbstub: Implement follow-fork-mode child

[PATCH 0/3] gdbstub: Implement follow-fork-mode child

[Ilya Leoshkevich]

Based-on: <20240116094411.216665-1-iii@linux.ibm.com>

Hi,

I needed to debug a linux-user crash between fork() and exec() [1] and
realized that gdbstub does not allow this. This series lifts this
restriction (one still cannot debug past exec() though). Patch 1 is a
preliminary refactoring, I can split it if necessary. Patch 2 is the
implementation, and patch 3 is the test.

[1] https://lists.gnu.org/archive/html/qemu-devel/2024-01/msg06424.html

Best regards,
Ilya

Ilya Leoshkevich (3):
  gdbstub: Refactor fork() handling
  gdbstub: Implement follow-fork-mode child
  tests/tcg: Add two follow-fork-mode tests

 bsd-user/freebsd/os-proc.h                    |   6 +-
 bsd-user/main.c                               |   8 +-
 bsd-user/qemu.h                               |   2 +-
 gdbstub/gdbstub.c                             |  29 ++-
 gdbstub/internals.h                           |   3 +
 gdbstub/user.c                                | 225 +++++++++++++++++-
 include/gdbstub/user.h                        |  11 +-
 linux-user/main.c                             |   8 +-
 linux-user/syscall.c                          |   4 +-
 linux-user/user-internals.h                   |   2 +-
 tests/tcg/multiarch/Makefile.target           |  17 +-
 tests/tcg/multiarch/follow-fork-mode.c        |  56 +++++
 .../gdbstub/follow-fork-mode-child.py         |  40 ++++
 .../gdbstub/follow-fork-mode-parent.py        |  16 ++
 14 files changed, 403 insertions(+), 24 deletions(-)
 create mode 100644 tests/tcg/multiarch/follow-fork-mode.c
 create mode 100644 tests/tcg/multiarch/gdbstub/follow-fork-mode-child.py
 create mode 100644 tests/tcg/multiarch/gdbstub/follow-fork-mode-parent.py

-- 
2.43.0

[PATCH 1/3] gdbstub: Refactor fork() handling

[Ilya Leoshkevich]

Prepare for implementing follow-fork-mode child:
* Introduce gdbserver_fork_start(), which for now is a no-op.
* Rename gdbserver_fork() to gdbserver_fork_end(), call it in both
  parent and child processes, and pass the fork()'s return value to it.
* Factor out disable_gdbstub().
* Update ts_tid in the forked child.

Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
---
 bsd-user/freebsd/os-proc.h  |  6 +++---
 bsd-user/main.c             |  8 ++++++--
 bsd-user/qemu.h             |  2 +-
 gdbstub/user.c              | 25 +++++++++++++++++++------
 include/gdbstub/user.h      | 11 ++++++++---
 linux-user/main.c           |  8 ++++++--
 linux-user/syscall.c        |  4 ++--
 linux-user/user-internals.h |  2 +-
 8 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/bsd-user/freebsd/os-proc.h b/bsd-user/freebsd/os-proc.h
index d6418780344..3003c8cb637 100644
--- a/bsd-user/freebsd/os-proc.h
+++ b/bsd-user/freebsd/os-proc.h
@@ -208,7 +208,7 @@ static inline abi_long do_freebsd_fork(void *cpu_env)
      */
     set_second_rval(cpu_env, child_flag);
 
-    fork_end(child_flag);
+    fork_end(ret);
 
     return ret;
 }
@@ -252,7 +252,7 @@ static inline abi_long do_freebsd_rfork(void *cpu_env, abi_long flags)
      * value: 0 for parent process, 1 for child process.
      */
     set_second_rval(cpu_env, child_flag);
-    fork_end(child_flag);
+    fork_end(ret);
 
     return ret;
 
@@ -285,7 +285,7 @@ static inline abi_long do_freebsd_pdfork(void *cpu_env, abi_ulong target_fdp,
      * value: 0 for parent process, 1 for child process.
      */
     set_second_rval(cpu_env, child_flag);
-    fork_end(child_flag);
+    fork_end(ret);
 
     return ret;
 }
diff --git a/bsd-user/main.c b/bsd-user/main.c
index e5efb7b8458..8ecfa395cc5 100644
--- a/bsd-user/main.c
+++ b/bsd-user/main.c
@@ -106,10 +106,13 @@ void fork_start(void)
     start_exclusive();
     cpu_list_lock();
     mmap_fork_start();
+    gdbserver_fork_start();
 }
 
-void fork_end(int child)
+void fork_end(abi_long pid)
 {
+    int child = pid == 0;
+
     if (child) {
         CPUState *cpu, *next_cpu;
         /*
@@ -127,10 +130,11 @@ void fork_end(int child)
          * state, so we don't need to end_exclusive() here.
          */
         qemu_init_cpu_list();
-        gdbserver_fork(thread_cpu);
+        gdbserver_fork_end(pid);
     } else {
         mmap_fork_end(child);
         cpu_list_unlock();
+        gdbserver_fork_end(pid);
         end_exclusive();
     }
 }
diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h
index dc842fffa7d..2414a87559b 100644
--- a/bsd-user/qemu.h
+++ b/bsd-user/qemu.h
@@ -180,7 +180,7 @@ void cpu_loop(CPUArchState *env);
 char *target_strerror(int err);
 int get_osversion(void);
 void fork_start(void);
-void fork_end(int child);
+void fork_end(abi_long pid);
 
 #include "qemu/log.h"
 
diff --git a/gdbstub/user.c b/gdbstub/user.c
index 766f7c08848..120eb7fc117 100644
--- a/gdbstub/user.c
+++ b/gdbstub/user.c
@@ -356,16 +356,29 @@ int gdbserver_start(const char *port_or_path)
     return -1;
 }
 
+void gdbserver_fork_start(void)
+{
+}
+
+static void disable_gdbstub(void)
+{
+    CPUState *cpu;
+
+    close(gdbserver_user_state.fd);
+    gdbserver_user_state.fd = -1;
+    CPU_FOREACH(cpu) {
+        cpu_breakpoint_remove_all(cpu, BP_GDB);
+        /* no cpu_watchpoint_remove_all for user-mode */
+    }
+}
+
 /* Disable gdb stub for child processes.  */
-void gdbserver_fork(CPUState *cpu)
+void gdbserver_fork_end(pid_t pid)
 {
-    if (!gdbserver_state.init || gdbserver_user_state.fd < 0) {
+    if (pid != 0 || !gdbserver_state.init || gdbserver_user_state.fd < 0) {
         return;
     }
-    close(gdbserver_user_state.fd);
-    gdbserver_user_state.fd = -1;
-    cpu_breakpoint_remove_all(cpu, BP_GDB);
-    /* no cpu_watchpoint_remove_all for user-mode */
+    disable_gdbstub();
 }
 
 /*
diff --git a/include/gdbstub/user.h b/include/gdbstub/user.h
index 68b6534130c..1694d4fd330 100644
--- a/include/gdbstub/user.h
+++ b/include/gdbstub/user.h
@@ -46,10 +46,15 @@ static inline int gdb_handlesig(CPUState *cpu, int sig)
 void gdb_signalled(CPUArchState *as, int sig);
 
 /**
- * gdbserver_fork() - disable gdb stub for child processes.
- * @cs: CPU
+ * gdbserver_fork_start() - inform gdb of the upcoming fork()
+ */
+void gdbserver_fork_start(void);
+
+/**
+ * gdbserver_fork_end() - disable gdb stub for child processes.
+ * @pid: 0 if in child process, -1 if fork failed, child process pid otherwise
  */
-void gdbserver_fork(CPUState *cs);
+void gdbserver_fork_end(pid_t pid);
 
 /**
  * gdb_syscall_entry() - inform gdb of syscall entry and yield control to it
diff --git a/linux-user/main.c b/linux-user/main.c
index c9470eeccfc..b42c8f36a1d 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -144,10 +144,13 @@ void fork_start(void)
     mmap_fork_start();
     cpu_list_lock();
     qemu_plugin_user_prefork_lock();
+    gdbserver_fork_start();
 }
 
-void fork_end(int child)
+void fork_end(abi_long pid)
 {
+    int child = pid == 0;
+
     qemu_plugin_user_postfork(child);
     mmap_fork_end(child);
     if (child) {
@@ -160,10 +163,11 @@ void fork_end(int child)
             }
         }
         qemu_init_cpu_list();
-        gdbserver_fork(thread_cpu);
+        ((TaskState *)thread_cpu->opaque)->ts_tid = (pid_t)syscall(SYS_gettid);
     } else {
         cpu_list_unlock();
     }
+    gdbserver_fork_end(pid);
     /*
      * qemu_init_cpu_list() reinitialized the child exclusive state, but we
      * also need to keep current_cpu consistent, so call end_exclusive() for
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index e384e142489..8be0bb57778 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -6669,7 +6669,7 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp,
         if (ret == 0) {
             /* Child Process.  */
             cpu_clone_regs_child(env, newsp, flags);
-            fork_end(1);
+            fork_end(ret);
             /* There is a race condition here.  The parent process could
                theoretically read the TID in the child process before the child
                tid is set.  This would require using either ptrace
@@ -6701,7 +6701,7 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp,
 #endif
                 put_user_u32(pid_fd, parent_tidptr);
                 }
-            fork_end(0);
+            fork_end(ret);
         }
         g_assert(!cpu_in_exclusive_context(cpu));
     }
diff --git a/linux-user/user-internals.h b/linux-user/user-internals.h
index c63ef45fc78..9014014d920 100644
--- a/linux-user/user-internals.h
+++ b/linux-user/user-internals.h
@@ -71,7 +71,7 @@ const char *target_strerror(int err);
 int get_osversion(void);
 void init_qemu_uname_release(void);
 void fork_start(void);
-void fork_end(int child);
+void fork_end(abi_long pid);
 
 /**
  * probe_guest_base:
-- 
2.43.0

[PATCH 2/3] gdbstub: Implement follow-fork-mode child

[Ilya Leoshkevich]

Currently it's not possible to use gdbstub for debugging linux-user
code that runs in a forked child, which is normally done using the `set
follow-fork-mode child` GDB command. Purely on the protocol level, the
missing piece is the fork-events feature.

However, a deeper problem is supporting $Hg switching between different
processes - right now it can do only threads. Implementing this for the
general case would be quite complicated, but, fortunately, for the
follow-fork-mode case there are a few factors that greatly simplify
things: fork() happens in the exclusive section, there are only two
processes involved, and before one of them is resumed, the second one
is detached.

This makes it possible to implement a simplified scheme: the parent and
the child share the gdbserver socket, it's used only by one of them at
any given time, which is coordinated through a separate socketpair. The
processes can read from the gdbserver socket only one byte at a time,
which is not great for performance, but, fortunately, the
follow-fork-mode involves only a few messages.

Add the hooks for the user-specific handling of $qSupported, $Hg, and
$D. Advertise the fork-events support, and remember whether GDB has it
as well. Implement the state machine that is initialized on fork(),
decides the current owner of the gdbserver socket, and is terminated
when one of the two processes is detached. The logic for the parent and
the child is the same, only the initial state is different.

Handle the `stepi` of a syscall corner case by disabling the
single-stepping in detached processes.

Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
---
 gdbstub/gdbstub.c   |  29 ++++--
 gdbstub/internals.h |   3 +
 gdbstub/user.c      | 210 +++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 234 insertions(+), 8 deletions(-)

diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index 7e73e916bdc..46f5dd47e9e 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -991,6 +991,12 @@ static void handle_detach(GArray *params, void *user_ctx)
         pid = get_param(params, 0)->val_ul;
     }
 
+#ifdef CONFIG_USER_ONLY
+    if (gdb_handle_detach_user(pid)) {
+        return;
+    }
+#endif
+
     process = gdb_get_process(pid);
     gdb_process_breakpoint_remove_all(process);
     process->attached = false;
@@ -1066,6 +1072,7 @@ static void handle_cont_with_sig(GArray *params, void *user_ctx)
 
 static void handle_set_thread(GArray *params, void *user_ctx)
 {
+    uint32_t pid, tid;
     CPUState *cpu;
 
     if (params->len != 2) {
@@ -1083,8 +1090,14 @@ static void handle_set_thread(GArray *params, void *user_ctx)
         return;
     }
 
-    cpu = gdb_get_cpu(get_param(params, 1)->thread_id.pid,
-                      get_param(params, 1)->thread_id.tid);
+    pid = get_param(params, 1)->thread_id.pid;
+    tid = get_param(params, 1)->thread_id.tid;
+#ifdef CONFIG_USER_ONLY
+    if (gdb_handle_set_thread_user(pid, tid)) {
+        return;
+    }
+#endif
+    cpu = gdb_get_cpu(pid, tid);
     if (!cpu) {
         gdb_put_packet("E22");
         return;
@@ -1599,6 +1612,7 @@ static void handle_query_thread_extra(GArray *params, void *user_ctx)
 
 static void handle_query_supported(GArray *params, void *user_ctx)
 {
+    const char *gdb_supported;
     CPUClass *cc;
 
     g_string_printf(gdbserver_state.str_buf, "PacketSize=%x", MAX_PACKET_LENGTH);
@@ -1622,9 +1636,14 @@ static void handle_query_supported(GArray *params, void *user_ctx)
     g_string_append(gdbserver_state.str_buf, ";qXfer:exec-file:read+");
 #endif
 
-    if (params->len &&
-        strstr(get_param(params, 0)->data, "multiprocess+")) {
-        gdbserver_state.multiprocess = true;
+    if (params->len) {
+        gdb_supported = get_param(params, 0)->data;
+        if (strstr(gdb_supported, "multiprocess+")) {
+            gdbserver_state.multiprocess = true;
+        }
+#if defined(CONFIG_USER_ONLY)
+        gdb_handle_query_supported_user(gdb_supported);
+#endif
     }
 
     g_string_append(gdbserver_state.str_buf, ";vContSupported+;multiprocess+");
diff --git a/gdbstub/internals.h b/gdbstub/internals.h
index 56b7c13b750..b4724598384 100644
--- a/gdbstub/internals.h
+++ b/gdbstub/internals.h
@@ -196,6 +196,9 @@ void gdb_handle_v_file_pread(GArray *params, void *user_ctx); /* user */
 void gdb_handle_v_file_readlink(GArray *params, void *user_ctx); /* user */
 void gdb_handle_query_xfer_exec_file(GArray *params, void *user_ctx); /* user */
 void gdb_handle_set_catch_syscalls(GArray *params, void *user_ctx); /* user */
+void gdb_handle_query_supported_user(const char *gdb_supported); /* user */
+bool gdb_handle_set_thread_user(uint32_t pid, uint32_t tid); /* user */
+bool gdb_handle_detach_user(uint32_t pid); /* user */
 
 void gdb_handle_query_attached(GArray *params, void *user_ctx); /* both */
 
diff --git a/gdbstub/user.c b/gdbstub/user.c
index 120eb7fc117..962f4cb74e7 100644
--- a/gdbstub/user.c
+++ b/gdbstub/user.c
@@ -10,6 +10,7 @@
  */
 
 #include "qemu/osdep.h"
+#include <sys/syscall.h>
 #include "qemu/bitops.h"
 #include "qemu/cutils.h"
 #include "qemu/sockets.h"
@@ -25,6 +26,41 @@
 #define GDB_NR_SYSCALLS 1024
 typedef unsigned long GDBSyscallsMask[BITS_TO_LONGS(GDB_NR_SYSCALLS)];
 
+/*
+ * Forked child talks to its parent in order to let GDB enforce the
+ * follow-fork-mode. This happens inside a start_exclusive() section, so that
+ * the other threads, which may be forking too, do not interfere. The
+ * implementation relies on GDB not sending $vCont until it has detached
+ * either from the parent (follow-fork-mode child) or from the child
+ * (follow-fork-mode parent).
+ *
+ * The parent and the child share the GDB socket; at any given time only one
+ * of them is allowed to use it, as is reflected in the respective fork_state.
+ * This is negotiated via the fork_sockets pair as a reaction to $Hg.
+ */
+enum GDBForkState {
+    /* Fully owning the GDB socket. */
+    GDB_FORK_ENABLED,
+    /* Working with the GDB socket; the peer is inactive. */
+    GDB_FORK_ACTIVE,
+    /* Handing off the GDB socket to the peer. */
+    GDB_FORK_DEACTIVATING,
+    /* The peer is working with the GDB socket. */
+    GDB_FORK_INACTIVE,
+    /* Asking the peer to close its GDB socket fd. */
+    GDB_FORK_ENABLING,
+    /* Asking the peer to take over, closing our GDB socket fd. */
+    GDB_FORK_DISABLING,
+    /* The peer has taken over, our GDB socket fd is closed. */
+    GDB_FORK_DISABLED,
+};
+
+enum GDBForkMessage {
+    GDB_FORK_ACTIVATE = 'a',
+    GDB_FORK_ENABLE = 'e',
+    GDB_FORK_DISABLE = 'd',
+};
+
 /* User-mode specific state */
 typedef struct {
     int fd;
@@ -36,6 +72,10 @@ typedef struct {
      */
     bool catch_all_syscalls;
     GDBSyscallsMask catch_syscalls_mask;
+    bool fork_events;
+    enum GDBForkState fork_state;
+    int fork_sockets[2];
+    pid_t fork_peer_pid, fork_peer_tid;
 } GDBUserState;
 
 static GDBUserState gdbserver_user_state;
@@ -358,6 +398,18 @@ int gdbserver_start(const char *port_or_path)
 
 void gdbserver_fork_start(void)
 {
+    if (!gdbserver_state.init || gdbserver_user_state.fd < 0) {
+        return;
+    }
+    if (!gdbserver_user_state.fork_events ||
+            qemu_socketpair(AF_UNIX, SOCK_STREAM, 0,
+                            gdbserver_user_state.fork_sockets) < 0) {
+        gdbserver_user_state.fork_state = GDB_FORK_DISABLED;
+        return;
+    }
+    gdbserver_user_state.fork_state = GDB_FORK_INACTIVE;
+    gdbserver_user_state.fork_peer_pid = getpid();
+    gdbserver_user_state.fork_peer_tid = qemu_get_thread_id();
 }
 
 static void disable_gdbstub(void)
@@ -369,16 +421,168 @@ static void disable_gdbstub(void)
     CPU_FOREACH(cpu) {
         cpu_breakpoint_remove_all(cpu, BP_GDB);
         /* no cpu_watchpoint_remove_all for user-mode */
+        cpu_single_step(cpu, 0);
+        tb_flush(cpu);
     }
 }
 
-/* Disable gdb stub for child processes.  */
 void gdbserver_fork_end(pid_t pid)
 {
-    if (pid != 0 || !gdbserver_state.init || gdbserver_user_state.fd < 0) {
+    char b;
+    int fd;
+
+    if (!gdbserver_state.init || gdbserver_user_state.fd < 0) {
+        return;
+    }
+
+    if (pid == -1) {
+        if (gdbserver_user_state.fork_state != GDB_FORK_DISABLED) {
+            g_assert(gdbserver_user_state.fork_state == GDB_FORK_INACTIVE);
+            close(gdbserver_user_state.fork_sockets[0]);
+            close(gdbserver_user_state.fork_sockets[1]);
+        }
         return;
     }
-    disable_gdbstub();
+
+    if (gdbserver_user_state.fork_state == GDB_FORK_DISABLED) {
+        if (pid == 0) {
+            disable_gdbstub();
+        }
+        return;
+    }
+
+    if (pid == 0) {
+        close(gdbserver_user_state.fork_sockets[0]);
+        fd = gdbserver_user_state.fork_sockets[1];
+        g_assert(gdbserver_state.process_num == 1);
+        g_assert(gdbserver_state.processes[0].pid ==
+                     gdbserver_user_state.fork_peer_pid);
+        g_assert(gdbserver_state.processes[0].attached);
+        gdbserver_state.processes[0].pid = getpid();
+    } else {
+        close(gdbserver_user_state.fork_sockets[1]);
+        fd = gdbserver_user_state.fork_sockets[0];
+        gdbserver_user_state.fork_state = GDB_FORK_ACTIVE;
+        gdbserver_user_state.fork_peer_pid = pid;
+        gdbserver_user_state.fork_peer_tid = pid;
+
+        if (!gdbserver_state.allow_stop_reply) {
+            goto fail;
+        }
+        g_string_printf(gdbserver_state.str_buf,
+                        "T%02xfork:p%02x.%02x;thread:p%02x.%02x;",
+                        gdb_target_signal_to_gdb(gdb_target_sigtrap()),
+                        pid, pid, (int)getpid(), qemu_get_thread_id());
+        gdb_put_strbuf();
+    }
+
+    gdbserver_state.state = RS_IDLE;
+    gdbserver_state.allow_stop_reply = false;
+    gdbserver_user_state.running_state = 0;
+    for (;;) {
+        switch (gdbserver_user_state.fork_state) {
+        case GDB_FORK_ENABLED:
+            if (gdbserver_user_state.running_state) {
+                return;
+            }
+            QEMU_FALLTHROUGH;
+        case GDB_FORK_ACTIVE:
+            if (read(gdbserver_user_state.fd, &b, 1) != 1) {
+                goto fail;
+            }
+            gdb_read_byte(b);
+            break;
+        case GDB_FORK_DEACTIVATING:
+            b = GDB_FORK_ACTIVATE;
+            if (write(fd, &b, 1) != 1) {
+                goto fail;
+            }
+            gdbserver_user_state.fork_state = GDB_FORK_INACTIVE;
+            break;
+        case GDB_FORK_INACTIVE:
+            if (read(fd, &b, 1) != 1) {
+                goto fail;
+            }
+            switch (b) {
+            case GDB_FORK_ACTIVATE:
+                gdbserver_user_state.fork_state = GDB_FORK_ACTIVE;
+                break;
+            case GDB_FORK_ENABLE:
+                close(fd);
+                gdbserver_user_state.fork_state = GDB_FORK_ENABLED;
+                break;
+            case GDB_FORK_DISABLE:
+                gdbserver_user_state.fork_state = GDB_FORK_DISABLED;
+                break;
+            default:
+                g_assert_not_reached();
+            }
+            break;
+        case GDB_FORK_ENABLING:
+            b = GDB_FORK_DISABLE;
+            if (write(fd, &b, 1) != 1) {
+                goto fail;
+            }
+            close(fd);
+            gdbserver_user_state.fork_state = GDB_FORK_ENABLED;
+            break;
+        case GDB_FORK_DISABLING:
+            b = GDB_FORK_ENABLE;
+            if (write(fd, &b, 1) != 1) {
+                goto fail;
+            }
+            gdbserver_user_state.fork_state = GDB_FORK_DISABLED;
+            break;
+        case GDB_FORK_DISABLED:
+            close(fd);
+            disable_gdbstub();
+            return;
+        default:
+            g_assert_not_reached();
+        }
+    }
+
+fail:
+    close(fd);
+    if (pid == 0) {
+        disable_gdbstub();
+    }
+}
+
+void gdb_handle_query_supported_user(const char *gdb_supported)
+{
+    if (strstr(gdb_supported, "fork-events+")) {
+        gdbserver_user_state.fork_events = true;
+    }
+    g_string_append(gdbserver_state.str_buf, ";fork-events+");
+}
+
+bool gdb_handle_set_thread_user(uint32_t pid, uint32_t tid)
+{
+    if (gdbserver_user_state.fork_state == GDB_FORK_ACTIVE &&
+            pid == gdbserver_user_state.fork_peer_pid &&
+            tid == gdbserver_user_state.fork_peer_tid) {
+        gdbserver_user_state.fork_state = GDB_FORK_DEACTIVATING;
+        gdb_put_packet("OK");
+        return true;
+    }
+    return false;
+}
+
+bool gdb_handle_detach_user(uint32_t pid)
+{
+    bool enable;
+
+    if (gdbserver_user_state.fork_state == GDB_FORK_ACTIVE) {
+        enable = pid == gdbserver_user_state.fork_peer_pid;
+        if (enable || pid == getpid()) {
+            gdbserver_user_state.fork_state = enable ? GDB_FORK_ENABLING :
+                                                       GDB_FORK_DISABLING;
+            gdb_put_packet("OK");
+            return true;
+        }
+    }
+    return false;
 }
 
 /*
-- 
2.43.0

[PATCH 3/3] tests/tcg: Add two follow-fork-mode tests

[Ilya Leoshkevich]

Add follow-fork-mode child and and follow-fork-mode parent tests.
Check for the obvious pitfalls, such as lingering breakpoints,
catchpoints, and single-step mode.

Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
---
 tests/tcg/multiarch/Makefile.target           | 17 +++++-
 tests/tcg/multiarch/follow-fork-mode.c        | 56 +++++++++++++++++++
 .../gdbstub/follow-fork-mode-child.py         | 40 +++++++++++++
 .../gdbstub/follow-fork-mode-parent.py        | 16 ++++++
 4 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100644 tests/tcg/multiarch/follow-fork-mode.c
 create mode 100644 tests/tcg/multiarch/gdbstub/follow-fork-mode-child.py
 create mode 100644 tests/tcg/multiarch/gdbstub/follow-fork-mode-parent.py

diff --git a/tests/tcg/multiarch/Makefile.target b/tests/tcg/multiarch/Makefile.target
index e10951a8016..b8b70c81860 100644
--- a/tests/tcg/multiarch/Makefile.target
+++ b/tests/tcg/multiarch/Makefile.target
@@ -115,6 +115,20 @@ run-gdbstub-catch-syscalls: catch-syscalls
 		--bin $< --test $(MULTIARCH_SRC)/gdbstub/catch-syscalls.py, \
 	hitting a syscall catchpoint)
 
+run-gdbstub-follow-fork-mode-child: follow-fork-mode
+	$(call run-test, $@, $(GDB_SCRIPT) \
+		--gdb $(GDB) \
+		--qemu $(QEMU) --qargs "$(QEMU_OPTS)" \
+		--bin $< --test $(MULTIARCH_SRC)/gdbstub/follow-fork-mode-child.py, \
+	following children on fork)
+
+run-gdbstub-follow-fork-mode-parent: follow-fork-mode
+	$(call run-test, $@, $(GDB_SCRIPT) \
+		--gdb $(GDB) \
+		--qemu $(QEMU) --qargs "$(QEMU_OPTS)" \
+		--bin $< --test $(MULTIARCH_SRC)/gdbstub/follow-fork-mode-parent.py, \
+	following parents on fork)
+
 else
 run-gdbstub-%:
 	$(call skip-test, "gdbstub test $*", "need working gdb with $(patsubst -%,,$(TARGET_NAME)) support")
@@ -122,7 +136,8 @@ endif
 EXTRA_RUNS += run-gdbstub-sha1 run-gdbstub-qxfer-auxv-read \
 	      run-gdbstub-proc-mappings run-gdbstub-thread-breakpoint \
 	      run-gdbstub-registers run-gdbstub-prot-none \
-	      run-gdbstub-catch-syscalls
+	      run-gdbstub-catch-syscalls run-gdbstub-follow-fork-mode-child \
+	      run-gdbstub-follow-fork-mode-parent
 
 # ARM Compatible Semi Hosting Tests
 #
diff --git a/tests/tcg/multiarch/follow-fork-mode.c b/tests/tcg/multiarch/follow-fork-mode.c
new file mode 100644
index 00000000000..cb6b032b388
--- /dev/null
+++ b/tests/tcg/multiarch/follow-fork-mode.c
@@ -0,0 +1,56 @@
+/*
+ * Test GDB's follow-fork-mode.
+ *
+ * fork() a chain of processes.
+ * Parents sends one byte to their children, and children return their
+ * position in the chain, in order to prove that they survived GDB's fork()
+ * handling.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+#include <assert.h>
+#include <stdlib.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+void break_after_fork(void)
+{
+}
+
+int main(void)
+{
+    int depth = 42, err, i, fd[2], status;
+    pid_t child, pid;
+    ssize_t n;
+    char b;
+
+    for (i = 0; i < depth; i++) {
+        err = pipe(fd);
+        assert(err == 0);
+        child = fork();
+        break_after_fork();
+        assert(child != -1);
+        if (child == 0) {
+            close(fd[1]);
+
+            n = read(fd[0], &b, 1);
+            close(fd[0]);
+            assert(n == 1);
+            assert(b == (char)i);
+        } else {
+            close(fd[0]);
+
+            b = (char)i;
+            n = write(fd[1], &b, 1);
+            close(fd[1]);
+            assert(n == 1);
+
+            pid = waitpid(child, &status, 0);
+            assert(pid == child);
+            assert(WIFEXITED(status));
+            return WEXITSTATUS(status) - 1;
+        }
+    }
+
+    return depth;
+}
diff --git a/tests/tcg/multiarch/gdbstub/follow-fork-mode-child.py b/tests/tcg/multiarch/gdbstub/follow-fork-mode-child.py
new file mode 100644
index 00000000000..72a6e440c08
--- /dev/null
+++ b/tests/tcg/multiarch/gdbstub/follow-fork-mode-child.py
@@ -0,0 +1,40 @@
+"""Test GDB's follow-fork-mode child.
+
+SPDX-License-Identifier: GPL-2.0-or-later
+"""
+from test_gdbstub import main, report
+
+
+def run_test():
+    """Run through the tests one by one"""
+    gdb.execute("set follow-fork-mode child")
+    # Check that the parent breakpoints are unset.
+    gdb.execute("break break_after_fork")
+    # Check that the parent syscall catchpoints are unset.
+    # Skip this check on the architectures that don't have them.
+    have_fork_syscall = False
+    for fork_syscall in ("fork", "clone", "clone2", "clone3"):
+        try:
+            gdb.execute("catch syscall {}".format(fork_syscall))
+        except gdb.error:
+            pass
+        else:
+            have_fork_syscall = True
+    gdb.execute("continue")
+    for i in range(42):
+        if have_fork_syscall:
+            # syscall entry.
+            if i % 2 == 0:
+                # Check that the parent single-stepping is turned off.
+                gdb.execute("si")
+            else:
+                gdb.execute("continue")
+            # syscall exit.
+            gdb.execute("continue")
+        # break_after_fork()
+        gdb.execute("continue")
+    exitcode = int(gdb.parse_and_eval("$_exitcode"))
+    report(exitcode == 42, "{} == 42".format(exitcode))
+
+
+main(run_test)
diff --git a/tests/tcg/multiarch/gdbstub/follow-fork-mode-parent.py b/tests/tcg/multiarch/gdbstub/follow-fork-mode-parent.py
new file mode 100644
index 00000000000..5c2fe722088
--- /dev/null
+++ b/tests/tcg/multiarch/gdbstub/follow-fork-mode-parent.py
@@ -0,0 +1,16 @@
+"""Test GDB's follow-fork-mode parent.
+
+SPDX-License-Identifier: GPL-2.0-or-later
+"""
+from test_gdbstub import main, report
+
+
+def run_test():
+    """Run through the tests one by one"""
+    gdb.execute("set follow-fork-mode parent")
+    gdb.execute("continue")
+    exitcode = int(gdb.parse_and_eval("$_exitcode"))
+    report(exitcode == 0, "{} == 0".format(exitcode))
+
+
+main(run_test)
-- 
2.43.0

Re: [PATCH 0/3] gdbstub: Implement follow-fork-mode child

[Alex Bennée]

Ilya Leoshkevich <iii@linux.ibm.com> writes:

> Based-on: <20240116094411.216665-1-iii@linux.ibm.com>
>
> Hi,
>
> I needed to debug a linux-user crash between fork() and exec() [1] and
> realized that gdbstub does not allow this. This series lifts this
> restriction (one still cannot debug past exec() though). Patch 1 is a
> preliminary refactoring, I can split it if necessary.

Please, I think introduce the api and then one each for adding the hooks
to *-user

> Patch 2 is the
> implementation, and patch 3 is the test.

Always good to see tests ;-)

>
> [1] https://lists.gnu.org/archive/html/qemu-devel/2024-01/msg06424.html
>
> Best regards,
> Ilya
>
> Ilya Leoshkevich (3):
>   gdbstub: Refactor fork() handling
>   gdbstub: Implement follow-fork-mode child
>   tests/tcg: Add two follow-fork-mode tests
>
>  bsd-user/freebsd/os-proc.h                    |   6 +-
>  bsd-user/main.c                               |   8 +-
>  bsd-user/qemu.h                               |   2 +-
>  gdbstub/gdbstub.c                             |  29 ++-
>  gdbstub/internals.h                           |   3 +
>  gdbstub/user.c                                | 225 +++++++++++++++++-
>  include/gdbstub/user.h                        |  11 +-
>  linux-user/main.c                             |   8 +-
>  linux-user/syscall.c                          |   4 +-
>  linux-user/user-internals.h                   |   2 +-
>  tests/tcg/multiarch/Makefile.target           |  17 +-
>  tests/tcg/multiarch/follow-fork-mode.c        |  56 +++++
>  .../gdbstub/follow-fork-mode-child.py         |  40 ++++
>  .../gdbstub/follow-fork-mode-parent.py        |  16 ++
>  14 files changed, 403 insertions(+), 24 deletions(-)
>  create mode 100644 tests/tcg/multiarch/follow-fork-mode.c
>  create mode 100644 tests/tcg/multiarch/gdbstub/follow-fork-mode-child.py
>  create mode 100644 tests/tcg/multiarch/gdbstub/follow-fork-mode-parent.py

-- 
Alex Bennée
Virtualisation Tech Lead @ Linaro

Re: [PATCH 1/3] gdbstub: Refactor fork() handling

[Alex Bennée]

Ilya Leoshkevich <iii@linux.ibm.com> writes:

> Prepare for implementing follow-fork-mode child:
> * Introduce gdbserver_fork_start(), which for now is a no-op.
> * Rename gdbserver_fork() to gdbserver_fork_end(), call it in both
>   parent and child processes, and pass the fork()'s return value to it.
> * Factor out disable_gdbstub().
> * Update ts_tid in the forked child.
>
> Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
> ---
>  bsd-user/freebsd/os-proc.h  |  6 +++---
>  bsd-user/main.c             |  8 ++++++--
>  bsd-user/qemu.h             |  2 +-
>  gdbstub/user.c              | 25 +++++++++++++++++++------
>  include/gdbstub/user.h      | 11 ++++++++---
>  linux-user/main.c           |  8 ++++++--
>  linux-user/syscall.c        |  4 ++--
>  linux-user/user-internals.h |  2 +-
>  8 files changed, 46 insertions(+), 20 deletions(-)
>
<snip>
>  
>  /*
> diff --git a/include/gdbstub/user.h b/include/gdbstub/user.h
> index 68b6534130c..1694d4fd330 100644
> --- a/include/gdbstub/user.h
> +++ b/include/gdbstub/user.h
> @@ -46,10 +46,15 @@ static inline int gdb_handlesig(CPUState *cpu, int sig)
>  void gdb_signalled(CPUArchState *as, int sig);
>  
>  /**
> - * gdbserver_fork() - disable gdb stub for child processes.
> - * @cs: CPU
> + * gdbserver_fork_start() - inform gdb of the upcoming fork()
> + */
> +void gdbserver_fork_start(void);
> +
> +/**
> + * gdbserver_fork_end() - disable gdb stub for child processes.
> + * @pid: 0 if in child process, -1 if fork failed, child process pid otherwise
>   */
> -void gdbserver_fork(CPUState *cs);
> +void gdbserver_fork_end(pid_t pid);
>

This had merge conflicts when I tried to apply to my tree.

-- 
Alex Bennée
Virtualisation Tech Lead @ Linaro

Re: [PATCH 2/3] gdbstub: Implement follow-fork-mode child

[Alex Bennée]

Ilya Leoshkevich <iii@linux.ibm.com> writes:

> Currently it's not possible to use gdbstub for debugging linux-user
> code that runs in a forked child, which is normally done using the `set
> follow-fork-mode child` GDB command. Purely on the protocol level, the
> missing piece is the fork-events feature.
>
> However, a deeper problem is supporting $Hg switching between different
> processes - right now it can do only threads. Implementing this for the
> general case would be quite complicated, but, fortunately, for the
> follow-fork-mode case there are a few factors that greatly simplify
> things: fork() happens in the exclusive section, there are only two
> processes involved, and before one of them is resumed, the second one
> is detached.
>
> This makes it possible to implement a simplified scheme: the parent and
> the child share the gdbserver socket, it's used only by one of them at
> any given time, which is coordinated through a separate socketpair. The
> processes can read from the gdbserver socket only one byte at a time,
> which is not great for performance, but, fortunately, the
> follow-fork-mode involves only a few messages.
>
> Add the hooks for the user-specific handling of $qSupported, $Hg, and
> $D. Advertise the fork-events support, and remember whether GDB has it
> as well. Implement the state machine that is initialized on fork(),
> decides the current owner of the gdbserver socket, and is terminated
> when one of the two processes is detached. The logic for the parent and
> the child is the same, only the initial state is different.
>
> Handle the `stepi` of a syscall corner case by disabling the
> single-stepping in detached processes.
>
> Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
> ---
>  gdbstub/gdbstub.c   |  29 ++++--
>  gdbstub/internals.h |   3 +
>  gdbstub/user.c      | 210 +++++++++++++++++++++++++++++++++++++++++++-
>  3 files changed, 234 insertions(+), 8 deletions(-)
>
> diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
> index 7e73e916bdc..46f5dd47e9e 100644
> --- a/gdbstub/gdbstub.c
> +++ b/gdbstub/gdbstub.c
> @@ -991,6 +991,12 @@ static void handle_detach(GArray *params, void *user_ctx)
>          pid = get_param(params, 0)->val_ul;
>      }
>  
> +#ifdef CONFIG_USER_ONLY
> +    if (gdb_handle_detach_user(pid)) {
> +        return;
> +    }
> +#endif
> +
>      process = gdb_get_process(pid);
>      gdb_process_breakpoint_remove_all(process);
>      process->attached = false;
> @@ -1066,6 +1072,7 @@ static void handle_cont_with_sig(GArray *params, void *user_ctx)
>  
>  static void handle_set_thread(GArray *params, void *user_ctx)
>  {
> +    uint32_t pid, tid;
>      CPUState *cpu;
>  
>      if (params->len != 2) {
> @@ -1083,8 +1090,14 @@ static void handle_set_thread(GArray *params, void *user_ctx)
>          return;
>      }
>  
> -    cpu = gdb_get_cpu(get_param(params, 1)->thread_id.pid,
> -                      get_param(params, 1)->thread_id.tid);
> +    pid = get_param(params, 1)->thread_id.pid;
> +    tid = get_param(params, 1)->thread_id.tid;
> +#ifdef CONFIG_USER_ONLY
> +    if (gdb_handle_set_thread_user(pid, tid)) {
> +        return;
> +    }
> +#endif
> +    cpu = gdb_get_cpu(pid, tid);
>      if (!cpu) {
>          gdb_put_packet("E22");
>          return;
> @@ -1599,6 +1612,7 @@ static void handle_query_thread_extra(GArray *params, void *user_ctx)
>  
>  static void handle_query_supported(GArray *params, void *user_ctx)
>  {
> +    const char *gdb_supported;
>      CPUClass *cc;
>  
>      g_string_printf(gdbserver_state.str_buf, "PacketSize=%x", MAX_PACKET_LENGTH);
> @@ -1622,9 +1636,14 @@ static void handle_query_supported(GArray *params, void *user_ctx)
>      g_string_append(gdbserver_state.str_buf, ";qXfer:exec-file:read+");
>  #endif
>  
> -    if (params->len &&
> -        strstr(get_param(params, 0)->data, "multiprocess+")) {
> -        gdbserver_state.multiprocess = true;
> +    if (params->len) {
> +        gdb_supported = get_param(params, 0)->data;
> +        if (strstr(gdb_supported, "multiprocess+")) {
> +            gdbserver_state.multiprocess = true;
> +        }
> +#if defined(CONFIG_USER_ONLY)
> +        gdb_handle_query_supported_user(gdb_supported);
> +#endif
>      }
>  
>      g_string_append(gdbserver_state.str_buf, ";vContSupported+;multiprocess+");
> diff --git a/gdbstub/internals.h b/gdbstub/internals.h
> index 56b7c13b750..b4724598384 100644
> --- a/gdbstub/internals.h
> +++ b/gdbstub/internals.h
> @@ -196,6 +196,9 @@ void gdb_handle_v_file_pread(GArray *params, void *user_ctx); /* user */
>  void gdb_handle_v_file_readlink(GArray *params, void *user_ctx); /* user */
>  void gdb_handle_query_xfer_exec_file(GArray *params, void *user_ctx); /* user */
>  void gdb_handle_set_catch_syscalls(GArray *params, void *user_ctx); /* user */
> +void gdb_handle_query_supported_user(const char *gdb_supported); /* user */
> +bool gdb_handle_set_thread_user(uint32_t pid, uint32_t tid); /* user */
> +bool gdb_handle_detach_user(uint32_t pid); /* user */
>  
>  void gdb_handle_query_attached(GArray *params, void *user_ctx); /* both */
>  
> diff --git a/gdbstub/user.c b/gdbstub/user.c
> index 120eb7fc117..962f4cb74e7 100644
> --- a/gdbstub/user.c
> +++ b/gdbstub/user.c
> @@ -10,6 +10,7 @@
>   */
>  
>  #include "qemu/osdep.h"
> +#include <sys/syscall.h>
>  #include "qemu/bitops.h"
>  #include "qemu/cutils.h"
>  #include "qemu/sockets.h"
> @@ -25,6 +26,41 @@
>  #define GDB_NR_SYSCALLS 1024
>  typedef unsigned long GDBSyscallsMask[BITS_TO_LONGS(GDB_NR_SYSCALLS)];
>  
> +/*
> + * Forked child talks to its parent in order to let GDB enforce the
> + * follow-fork-mode. This happens inside a start_exclusive() section, so that
> + * the other threads, which may be forking too, do not interfere. The
> + * implementation relies on GDB not sending $vCont until it has detached
> + * either from the parent (follow-fork-mode child) or from the child
> + * (follow-fork-mode parent).
> + *
> + * The parent and the child share the GDB socket; at any given time only one
> + * of them is allowed to use it, as is reflected in the respective fork_state.
> + * This is negotiated via the fork_sockets pair as a reaction to $Hg.
> + */
> +enum GDBForkState {
> +    /* Fully owning the GDB socket. */
> +    GDB_FORK_ENABLED,
> +    /* Working with the GDB socket; the peer is inactive. */
> +    GDB_FORK_ACTIVE,
> +    /* Handing off the GDB socket to the peer. */
> +    GDB_FORK_DEACTIVATING,
> +    /* The peer is working with the GDB socket. */
> +    GDB_FORK_INACTIVE,
> +    /* Asking the peer to close its GDB socket fd. */
> +    GDB_FORK_ENABLING,
> +    /* Asking the peer to take over, closing our GDB socket fd. */
> +    GDB_FORK_DISABLING,
> +    /* The peer has taken over, our GDB socket fd is closed. */
> +    GDB_FORK_DISABLED,
> +};

gulp - thats a potentially fairly complex state diagram. Do we just work
through the states sequentially?

> +
> +enum GDBForkMessage {
> +    GDB_FORK_ACTIVATE = 'a',
> +    GDB_FORK_ENABLE = 'e',
> +    GDB_FORK_DISABLE = 'd',
> +};
> +
>  /* User-mode specific state */
>  typedef struct {
>      int fd;
> @@ -36,6 +72,10 @@ typedef struct {
>       */
>      bool catch_all_syscalls;
>      GDBSyscallsMask catch_syscalls_mask;
> +    bool fork_events;
> +    enum GDBForkState fork_state;
> +    int fork_sockets[2];
> +    pid_t fork_peer_pid, fork_peer_tid;
>  } GDBUserState;
>  
>  static GDBUserState gdbserver_user_state;
> @@ -358,6 +398,18 @@ int gdbserver_start(const char *port_or_path)
>  
>  void gdbserver_fork_start(void)
>  {
> +    if (!gdbserver_state.init || gdbserver_user_state.fd < 0) {
> +        return;
> +    }
> +    if (!gdbserver_user_state.fork_events ||
> +            qemu_socketpair(AF_UNIX, SOCK_STREAM, 0,
> +                            gdbserver_user_state.fork_sockets) < 0) {
> +        gdbserver_user_state.fork_state = GDB_FORK_DISABLED;
> +        return;
> +    }
> +    gdbserver_user_state.fork_state = GDB_FORK_INACTIVE;
> +    gdbserver_user_state.fork_peer_pid = getpid();
> +    gdbserver_user_state.fork_peer_tid = qemu_get_thread_id();
>  }
>  
>  static void disable_gdbstub(void)
> @@ -369,16 +421,168 @@ static void disable_gdbstub(void)
>      CPU_FOREACH(cpu) {
>          cpu_breakpoint_remove_all(cpu, BP_GDB);
>          /* no cpu_watchpoint_remove_all for user-mode */
> +        cpu_single_step(cpu, 0);
> +        tb_flush(cpu);
>      }
>  }
>  
> -/* Disable gdb stub for child processes.  */
>  void gdbserver_fork_end(pid_t pid)
>  {
> -    if (pid != 0 || !gdbserver_state.init || gdbserver_user_state.fd < 0) {
> +    char b;
> +    int fd;
> +
> +    if (!gdbserver_state.init || gdbserver_user_state.fd < 0) {
> +        return;
> +    }
> +
> +    if (pid == -1) {
> +        if (gdbserver_user_state.fork_state != GDB_FORK_DISABLED) {
> +            g_assert(gdbserver_user_state.fork_state == GDB_FORK_INACTIVE);
> +            close(gdbserver_user_state.fork_sockets[0]);
> +            close(gdbserver_user_state.fork_sockets[1]);
> +        }
>          return;
>      }
> -    disable_gdbstub();
> +
> +    if (gdbserver_user_state.fork_state == GDB_FORK_DISABLED) {
> +        if (pid == 0) {
> +            disable_gdbstub();
> +        }
> +        return;
> +    }
> +
> +    if (pid == 0) {
> +        close(gdbserver_user_state.fork_sockets[0]);
> +        fd = gdbserver_user_state.fork_sockets[1];
> +        g_assert(gdbserver_state.process_num == 1);
> +        g_assert(gdbserver_state.processes[0].pid ==
> +                     gdbserver_user_state.fork_peer_pid);
> +        g_assert(gdbserver_state.processes[0].attached);
> +        gdbserver_state.processes[0].pid = getpid();
> +    } else {
> +        close(gdbserver_user_state.fork_sockets[1]);
> +        fd = gdbserver_user_state.fork_sockets[0];
> +        gdbserver_user_state.fork_state = GDB_FORK_ACTIVE;
> +        gdbserver_user_state.fork_peer_pid = pid;
> +        gdbserver_user_state.fork_peer_tid = pid;
> +
> +        if (!gdbserver_state.allow_stop_reply) {
> +            goto fail;
> +        }
> +        g_string_printf(gdbserver_state.str_buf,
> +                        "T%02xfork:p%02x.%02x;thread:p%02x.%02x;",
> +                        gdb_target_signal_to_gdb(gdb_target_sigtrap()),
> +                        pid, pid, (int)getpid(),
> qemu_get_thread_id());

I don't think I messed up the merge but:

../../gdbstub/user.c: In function ‘gdbserver_fork_end’:
../../gdbstub/user.c:461:50: error: implicit declaration of function ‘gdb_target_sigtrap’ [-Werror=implicit-function-declaration]
  461 |                         gdb_target_signal_to_gdb(gdb_target_sigtrap()),
      |                                                  ^~~~~~~~~~~~~~~~~~
../../gdbstub/user.c:461:50: error: nested extern declaration of ‘gdb_target_sigtrap’ [-Werror=nested-externs]
cc1: all warnings being treated as errors

I cant see where gdb_target_sigtrap is from?

-- 
Alex Bennée
Virtualisation Tech Lead @ Linaro

Re: [PATCH 2/3] gdbstub: Implement follow-fork-mode child

[Ilya Leoshkevich]

On Thu, 2024-02-01 at 12:11 +0000, Alex Bennée wrote:
> Ilya Leoshkevich <iii@linux.ibm.com> writes:
> 
> > Currently it's not possible to use gdbstub for debugging linux-user
> > code that runs in a forked child, which is normally done using the
> > `set
> > follow-fork-mode child` GDB command. Purely on the protocol level,
> > the
> > missing piece is the fork-events feature.
> > 
> > However, a deeper problem is supporting $Hg switching between
> > different
> > processes - right now it can do only threads. Implementing this for
> > the
> > general case would be quite complicated, but, fortunately, for the
> > follow-fork-mode case there are a few factors that greatly simplify
> > things: fork() happens in the exclusive section, there are only two
> > processes involved, and before one of them is resumed, the second
> > one
> > is detached.
> > 
> > This makes it possible to implement a simplified scheme: the parent
> > and
> > the child share the gdbserver socket, it's used only by one of them
> > at
> > any given time, which is coordinated through a separate socketpair.
> > The
> > processes can read from the gdbserver socket only one byte at a
> > time,
> > which is not great for performance, but, fortunately, the
> > follow-fork-mode involves only a few messages.
> > 
> > Add the hooks for the user-specific handling of $qSupported, $Hg,
> > and
> > $D. Advertise the fork-events support, and remember whether GDB has
> > it
> > as well. Implement the state machine that is initialized on fork(),
> > decides the current owner of the gdbserver socket, and is
> > terminated
> > when one of the two processes is detached. The logic for the parent
> > and
> > the child is the same, only the initial state is different.
> > 
> > Handle the `stepi` of a syscall corner case by disabling the
> > single-stepping in detached processes.
> > 
> > Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
> > ---
> >  gdbstub/gdbstub.c   |  29 ++++--
> >  gdbstub/internals.h |   3 +
> >  gdbstub/user.c      | 210
> > +++++++++++++++++++++++++++++++++++++++++++-
> >  3 files changed, 234 insertions(+), 8 deletions(-)

[...]
 
> > diff --git a/gdbstub/user.c b/gdbstub/user.c
> > index 120eb7fc117..962f4cb74e7 100644
> > --- a/gdbstub/user.c
> > +++ b/gdbstub/user.c
> > @@ -10,6 +10,7 @@
> >   */
> >  
> >  #include "qemu/osdep.h"
> > +#include <sys/syscall.h>
> >  #include "qemu/bitops.h"
> >  #include "qemu/cutils.h"
> >  #include "qemu/sockets.h"
> > @@ -25,6 +26,41 @@
> >  #define GDB_NR_SYSCALLS 1024
> >  typedef unsigned long
> > GDBSyscallsMask[BITS_TO_LONGS(GDB_NR_SYSCALLS)];
> >  
> > +/*
> > + * Forked child talks to its parent in order to let GDB enforce
> > the
> > + * follow-fork-mode. This happens inside a start_exclusive()
> > section, so that
> > + * the other threads, which may be forking too, do not interfere.
> > The
> > + * implementation relies on GDB not sending $vCont until it has
> > detached
> > + * either from the parent (follow-fork-mode child) or from the
> > child
> > + * (follow-fork-mode parent).
> > + *
> > + * The parent and the child share the GDB socket; at any given
> > time only one
> > + * of them is allowed to use it, as is reflected in the respective
> > fork_state.
> > + * This is negotiated via the fork_sockets pair as a reaction to
> > $Hg.
> > + */
> > +enum GDBForkState {
> > +    /* Fully owning the GDB socket. */
> > +    GDB_FORK_ENABLED,
> > +    /* Working with the GDB socket; the peer is inactive. */
> > +    GDB_FORK_ACTIVE,
> > +    /* Handing off the GDB socket to the peer. */
> > +    GDB_FORK_DEACTIVATING,
> > +    /* The peer is working with the GDB socket. */
> > +    GDB_FORK_INACTIVE,
> > +    /* Asking the peer to close its GDB socket fd. */
> > +    GDB_FORK_ENABLING,
> > +    /* Asking the peer to take over, closing our GDB socket fd. */
> > +    GDB_FORK_DISABLING,
> > +    /* The peer has taken over, our GDB socket fd is closed. */
> > +    GDB_FORK_DISABLED,
> > +};
> 
> gulp - thats a potentially fairly complex state diagram. Do we just
> work
> through the states sequentially?

Unfortunately no. I had less states at some point, but then realized
it was better to have these things laid out explicitly. Let me try to
summarize the possible transitions:

GDB_FORK_ENABLED: Terminal state; GDB follows the current process.
GDB_FORK_DISABLED: Terminal state; GDB follows the other process.
GDB_FORK_ACTIVE -> GDB_FORK_DEACTIVATING: On $Hg.
GDB_FORK_ACTIVE -> GDB_FORK_ENABLING: On $D.
GDB_FORK_ACTIVE -> GDB_FORK_DISABLING: On $D.
GDB_FORK_ACTIVE -> GDB_FORK_DISABLED: On communication error.
GDB_FORK_DEACTIVATING -> GDB_FORK_INACTIVE: On gdb_read_byte() return.
GDB_FORK_DEACTIVATING -> GDB_FORK_DISABLED: On communication error.
GDB_FORK_INACTIVE -> GDB_FORK_ACTIVE: On $Hg in peer.
GDB_FORK_INACTIVE -> GDB_FORK_ENABLE: On $D in peer.
GDB_FORK_INACTIVE -> GDB_FORK_DISABLE: On $D in peer.
GDB_FORK_INACTIVE -> GDB_FORK_DISABLED: On communication error.
GDB_FORK_ENABLING -> GDB_FORK_ENABLED: On gdb_read_byte() return.
GDB_FORK_ENABLING -> GDB_FORK_DISABLED: On communication error.
GDB_FORK_DISABLING -> GDB_FORK_DISABLED: On gdb_read_byte() return.

Some states have only one meaningful transition:

GDB_FORK_DEACTIVATING -> GDB_FORK_INACTIVE
GDB_FORK_ENABLING -> GDB_FORK_ENABLED

and can in theory be squashed, but then the socketpair communication
would have to be moved to the respective user-hook, which would
complicate the error handling.

[...]

> > @@ -369,16 +421,168 @@ static void disable_gdbstub(void)
> >      CPU_FOREACH(cpu) {
> >          cpu_breakpoint_remove_all(cpu, BP_GDB);
> >          /* no cpu_watchpoint_remove_all for user-mode */
> > +        cpu_single_step(cpu, 0);
> > +        tb_flush(cpu);
> >      }
> >  }
> >  
> > -/* Disable gdb stub for child processes.  */
> >  void gdbserver_fork_end(pid_t pid)
> >  {
> > -    if (pid != 0 || !gdbserver_state.init ||
> > gdbserver_user_state.fd < 0) {
> > +    char b;
> > +    int fd;
> > +
> > +    if (!gdbserver_state.init || gdbserver_user_state.fd < 0) {
> > +        return;
> > +    }
> > +
> > +    if (pid == -1) {
> > +        if (gdbserver_user_state.fork_state != GDB_FORK_DISABLED)
> > {
> > +            g_assert(gdbserver_user_state.fork_state ==
> > GDB_FORK_INACTIVE);
> > +            close(gdbserver_user_state.fork_sockets[0]);
> > +            close(gdbserver_user_state.fork_sockets[1]);
> > +        }
> >          return;
> >      }
> > -    disable_gdbstub();
> > +
> > +    if (gdbserver_user_state.fork_state == GDB_FORK_DISABLED) {
> > +        if (pid == 0) {
> > +            disable_gdbstub();
> > +        }
> > +        return;
> > +    }
> > +
> > +    if (pid == 0) {
> > +        close(gdbserver_user_state.fork_sockets[0]);
> > +        fd = gdbserver_user_state.fork_sockets[1];
> > +        g_assert(gdbserver_state.process_num == 1);
> > +        g_assert(gdbserver_state.processes[0].pid ==
> > +                     gdbserver_user_state.fork_peer_pid);
> > +        g_assert(gdbserver_state.processes[0].attached);
> > +        gdbserver_state.processes[0].pid = getpid();
> > +    } else {
> > +        close(gdbserver_user_state.fork_sockets[1]);
> > +        fd = gdbserver_user_state.fork_sockets[0];
> > +        gdbserver_user_state.fork_state = GDB_FORK_ACTIVE;
> > +        gdbserver_user_state.fork_peer_pid = pid;
> > +        gdbserver_user_state.fork_peer_tid = pid;
> > +
> > +        if (!gdbserver_state.allow_stop_reply) {
> > +            goto fail;
> > +        }
> > +        g_string_printf(gdbserver_state.str_buf,
> > +                        "T%02xfork:p%02x.%02x;thread:p%02x.%02x;",
> > +                       
> > gdb_target_signal_to_gdb(gdb_target_sigtrap()),
> > +                        pid, pid, (int)getpid(),
> > qemu_get_thread_id());
> 
> I don't think I messed up the merge but:
> 
> ../../gdbstub/user.c: In function ‘gdbserver_fork_end’:
> ../../gdbstub/user.c:461:50: error: implicit declaration of function
> ‘gdb_target_sigtrap’ [-Werror=implicit-function-declaration]
>   461 |                        
> gdb_target_signal_to_gdb(gdb_target_sigtrap()),
>       |                                                 
> ^~~~~~~~~~~~~~~~~~
> ../../gdbstub/user.c:461:50: error: nested extern declaration of
> ‘gdb_target_sigtrap’ [-Werror=nested-externs]
> cc1: all warnings being treated as errors
> 
> I cant see where gdb_target_sigtrap is from?

This is from [1], which this series is Based-on. I can make one series
with both features if it's more convenient to review.

[1] https://patchew.org/QEMU/20240116094411.216665-1-iii@linux.ibm.com/

Re: [PATCH 2/3] gdbstub: Implement follow-fork-mode child

[Alex Bennée]

Ilya Leoshkevich <iii@linux.ibm.com> writes:

> On Thu, 2024-02-01 at 12:11 +0000, Alex Bennée wrote:
>> Ilya Leoshkevich <iii@linux.ibm.com> writes:
>> 
>> > Currently it's not possible to use gdbstub for debugging linux-user
>> > code that runs in a forked child, which is normally done using the
>> > `set
>> > follow-fork-mode child` GDB command. Purely on the protocol level,
>> > the
>> > missing piece is the fork-events feature.
>> > 
>> > However, a deeper problem is supporting $Hg switching between
>> > different
>> > processes - right now it can do only threads. Implementing this for
>> > the
>> > general case would be quite complicated, but, fortunately, for the
>> > follow-fork-mode case there are a few factors that greatly simplify
>> > things: fork() happens in the exclusive section, there are only two
>> > processes involved, and before one of them is resumed, the second
>> > one
>> > is detached.
>> > 
>> > This makes it possible to implement a simplified scheme: the parent
>> > and
>> > the child share the gdbserver socket, it's used only by one of them
>> > at
>> > any given time, which is coordinated through a separate socketpair.
>> > The
>> > processes can read from the gdbserver socket only one byte at a
>> > time,
>> > which is not great for performance, but, fortunately, the
>> > follow-fork-mode involves only a few messages.
>> > 
>> > Add the hooks for the user-specific handling of $qSupported, $Hg,
>> > and
>> > $D. Advertise the fork-events support, and remember whether GDB has
>> > it
>> > as well. Implement the state machine that is initialized on fork(),
>> > decides the current owner of the gdbserver socket, and is
>> > terminated
>> > when one of the two processes is detached. The logic for the parent
>> > and
>> > the child is the same, only the initial state is different.
>> > 
>> > Handle the `stepi` of a syscall corner case by disabling the
>> > single-stepping in detached processes.
>> > 
>> > Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
>> > ---
>> >  gdbstub/gdbstub.c   |  29 ++++--
>> >  gdbstub/internals.h |   3 +
>> >  gdbstub/user.c      | 210
>> > +++++++++++++++++++++++++++++++++++++++++++-
>> >  3 files changed, 234 insertions(+), 8 deletions(-)
>
> [...]
>  
>> > diff --git a/gdbstub/user.c b/gdbstub/user.c
>> > index 120eb7fc117..962f4cb74e7 100644
>> > --- a/gdbstub/user.c
>> > +++ b/gdbstub/user.c
>> > @@ -10,6 +10,7 @@
>> >   */
>> >  
>> >  #include "qemu/osdep.h"
>> > +#include <sys/syscall.h>
>> >  #include "qemu/bitops.h"
>> >  #include "qemu/cutils.h"
>> >  #include "qemu/sockets.h"
>> > @@ -25,6 +26,41 @@
>> >  #define GDB_NR_SYSCALLS 1024
>> >  typedef unsigned long
>> > GDBSyscallsMask[BITS_TO_LONGS(GDB_NR_SYSCALLS)];
>> >  
>> > +/*
>> > + * Forked child talks to its parent in order to let GDB enforce
>> > the
>> > + * follow-fork-mode. This happens inside a start_exclusive()
>> > section, so that
>> > + * the other threads, which may be forking too, do not interfere.
>> > The
>> > + * implementation relies on GDB not sending $vCont until it has
>> > detached
>> > + * either from the parent (follow-fork-mode child) or from the
>> > child
>> > + * (follow-fork-mode parent).
>> > + *
>> > + * The parent and the child share the GDB socket; at any given
>> > time only one
>> > + * of them is allowed to use it, as is reflected in the respective
>> > fork_state.
>> > + * This is negotiated via the fork_sockets pair as a reaction to
>> > $Hg.
>> > + */
>> > +enum GDBForkState {
>> > +    /* Fully owning the GDB socket. */
>> > +    GDB_FORK_ENABLED,
>> > +    /* Working with the GDB socket; the peer is inactive. */
>> > +    GDB_FORK_ACTIVE,
>> > +    /* Handing off the GDB socket to the peer. */
>> > +    GDB_FORK_DEACTIVATING,
>> > +    /* The peer is working with the GDB socket. */
>> > +    GDB_FORK_INACTIVE,
>> > +    /* Asking the peer to close its GDB socket fd. */
>> > +    GDB_FORK_ENABLING,
>> > +    /* Asking the peer to take over, closing our GDB socket fd. */
>> > +    GDB_FORK_DISABLING,
>> > +    /* The peer has taken over, our GDB socket fd is closed. */
>> > +    GDB_FORK_DISABLED,
>> > +};
>> 
>> gulp - thats a potentially fairly complex state diagram. Do we just
>> work
>> through the states sequentially?
>
> Unfortunately no. I had less states at some point, but then realized
> it was better to have these things laid out explicitly. Let me try to
> summarize the possible transitions:
>
> GDB_FORK_ENABLED: Terminal state; GDB follows the current process.
> GDB_FORK_DISABLED: Terminal state; GDB follows the other process.
> GDB_FORK_ACTIVE -> GDB_FORK_DEACTIVATING: On $Hg.
> GDB_FORK_ACTIVE -> GDB_FORK_ENABLING: On $D.
> GDB_FORK_ACTIVE -> GDB_FORK_DISABLING: On $D.
> GDB_FORK_ACTIVE -> GDB_FORK_DISABLED: On communication error.
> GDB_FORK_DEACTIVATING -> GDB_FORK_INACTIVE: On gdb_read_byte() return.
> GDB_FORK_DEACTIVATING -> GDB_FORK_DISABLED: On communication error.
> GDB_FORK_INACTIVE -> GDB_FORK_ACTIVE: On $Hg in peer.
> GDB_FORK_INACTIVE -> GDB_FORK_ENABLE: On $D in peer.
> GDB_FORK_INACTIVE -> GDB_FORK_DISABLE: On $D in peer.
> GDB_FORK_INACTIVE -> GDB_FORK_DISABLED: On communication error.
> GDB_FORK_ENABLING -> GDB_FORK_ENABLED: On gdb_read_byte() return.
> GDB_FORK_ENABLING -> GDB_FORK_DISABLED: On communication error.
> GDB_FORK_DISABLING -> GDB_FORK_DISABLED: On gdb_read_byte() return.
>
> Some states have only one meaningful transition:
>
> GDB_FORK_DEACTIVATING -> GDB_FORK_INACTIVE
> GDB_FORK_ENABLING -> GDB_FORK_ENABLED
>
> and can in theory be squashed, but then the socketpair communication
> would have to be moved to the respective user-hook, which would
> complicate the error handling.
>
> [...]
>
>> > @@ -369,16 +421,168 @@ static void disable_gdbstub(void)
>> >      CPU_FOREACH(cpu) {
>> >          cpu_breakpoint_remove_all(cpu, BP_GDB);
>> >          /* no cpu_watchpoint_remove_all for user-mode */
>> > +        cpu_single_step(cpu, 0);
>> > +        tb_flush(cpu);
>> >      }
>> >  }
>> >  
>> > -/* Disable gdb stub for child processes.  */
>> >  void gdbserver_fork_end(pid_t pid)
>> >  {
>> > -    if (pid != 0 || !gdbserver_state.init ||
>> > gdbserver_user_state.fd < 0) {
>> > +    char b;
>> > +    int fd;
>> > +
>> > +    if (!gdbserver_state.init || gdbserver_user_state.fd < 0) {
>> > +        return;
>> > +    }
>> > +
>> > +    if (pid == -1) {
>> > +        if (gdbserver_user_state.fork_state != GDB_FORK_DISABLED)
>> > {
>> > +            g_assert(gdbserver_user_state.fork_state ==
>> > GDB_FORK_INACTIVE);
>> > +            close(gdbserver_user_state.fork_sockets[0]);
>> > +            close(gdbserver_user_state.fork_sockets[1]);
>> > +        }
>> >          return;
>> >      }
>> > -    disable_gdbstub();
>> > +
>> > +    if (gdbserver_user_state.fork_state == GDB_FORK_DISABLED) {
>> > +        if (pid == 0) {
>> > +            disable_gdbstub();
>> > +        }
>> > +        return;
>> > +    }
>> > +
>> > +    if (pid == 0) {
>> > +        close(gdbserver_user_state.fork_sockets[0]);
>> > +        fd = gdbserver_user_state.fork_sockets[1];
>> > +        g_assert(gdbserver_state.process_num == 1);
>> > +        g_assert(gdbserver_state.processes[0].pid ==
>> > +                     gdbserver_user_state.fork_peer_pid);
>> > +        g_assert(gdbserver_state.processes[0].attached);
>> > +        gdbserver_state.processes[0].pid = getpid();
>> > +    } else {
>> > +        close(gdbserver_user_state.fork_sockets[1]);
>> > +        fd = gdbserver_user_state.fork_sockets[0];
>> > +        gdbserver_user_state.fork_state = GDB_FORK_ACTIVE;
>> > +        gdbserver_user_state.fork_peer_pid = pid;
>> > +        gdbserver_user_state.fork_peer_tid = pid;
>> > +
>> > +        if (!gdbserver_state.allow_stop_reply) {
>> > +            goto fail;
>> > +        }
>> > +        g_string_printf(gdbserver_state.str_buf,
>> > +                        "T%02xfork:p%02x.%02x;thread:p%02x.%02x;",
>> > +                       
>> > gdb_target_signal_to_gdb(gdb_target_sigtrap()),
>> > +                        pid, pid, (int)getpid(),
>> > qemu_get_thread_id());
>> 
>> I don't think I messed up the merge but:
>> 
>> ../../gdbstub/user.c: In function ‘gdbserver_fork_end’:
>> ../../gdbstub/user.c:461:50: error: implicit declaration of function
>> ‘gdb_target_sigtrap’ [-Werror=implicit-function-declaration]
>>   461 |                        
>> gdb_target_signal_to_gdb(gdb_target_sigtrap()),
>>       |                                                 
>> ^~~~~~~~~~~~~~~~~~
>> ../../gdbstub/user.c:461:50: error: nested extern declaration of
>> ‘gdb_target_sigtrap’ [-Werror=nested-externs]
>> cc1: all warnings being treated as errors
>> 
>> I cant see where gdb_target_sigtrap is from?
>
> This is from [1], which this series is Based-on. I can make one series
> with both features if it's more convenient to review.
>
> [1]
> https://patchew.org/QEMU/20240116094411.216665-1-iii@linux.ibm.com/

Oops missed that one, looking at it now.

-- 
Alex Bennée
Virtualisation Tech Lead @ Linaro