[Qemu-devel] [PATCH 0/3] qemu-gdb: add functionality for inspecting core dumps

[Qemu-devel] [PATCH 0/3] qemu-gdb: add functionality for inspecting core dumps

[Paolo Bonzini]

Currently it is very hard to inspect coroutine in core dumps, because
none of the qemu-gdb functionality works.  Fixing this is not really
possible because "bt" only works on the core dump's stack pointer and
program counter, but the situation would improve noticeably if only
a coroutine's stack pointer and program counter were accessible at all;
that would allow inspecting the coroutine's stack and building a
stack trace from the hex dump of the stack.

The main hurdle is that glibc_pointer_guard() cannot be run on a core
dump, because get_fs_base() uses the arch_prctl system call.  The first
patch modifies that to use the gdb API instead.  The second and third
patch then add the new functions.

Paolo

Paolo Bonzini (3):
  qemu-gdb: allow using glibc_pointer_guard() on core dumps
  qemu-gdb: extract parts of "qemu coroutine" implementation
  qemu-gdb: add $qemu_coroutine_sp and $qemu_coroutine_pc

 scripts/qemu-gdb.py          |  3 ++
 scripts/qemugdb/coroutine.py | 90 +++++++++++++++++++++++++++++---------------
 2 files changed, 62 insertions(+), 31 deletions(-)

-- 
2.5.0

[Qemu-devel] [PATCH 1/3] qemu-gdb: allow using glibc_pointer_guard() on core dumps

[Paolo Bonzini]

get_fs_base() cannot be run on a core dump, because it uses the arch_prctl
system call.  The fs base is the value that is returned by pthread_self(),
and it would be nice to just glean it from the "info threads" output:

* 1    Thread 0x7f16a3fff700 (LWP 33642) pthread_cond_wait@@GLIBC_2.3.2 ()
              ^^^^^^^^^^^^^^

but unfortunately the gdb API does not provide that.  Instead, we can
look for the "arg" argument of the start_thread function if glibc debug
information are available.  If not, fall back to the old mechanism.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 scripts/qemugdb/coroutine.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/scripts/qemugdb/coroutine.py b/scripts/qemugdb/coroutine.py
index 3c54918..b1d4546 100644
--- a/scripts/qemugdb/coroutine.py
+++ b/scripts/qemugdb/coroutine.py
@@ -16,7 +16,8 @@
 import gdb
 
 def get_fs_base():
-    '''Fetch %fs base value using arch_prctl(ARCH_GET_FS)'''
+    '''Fetch %fs base value using arch_prctl(ARCH_GET_FS).  This is
+       pthread_self().'''
     # %rsp - 120 is scratch space according to the SystemV ABI
     old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
     gdb.execute('call arch_prctl(0x1003, $rsp - 120)', False, True)
@@ -24,9 +25,22 @@ def get_fs_base():
     gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True)
     return fs_base
 
+def pthread_self():
+    '''Fetch pthread_self() from the glibc start_thread function.'''
+    f = gdb.newest_frame()
+    while f.name() != 'start_thread':
+        f = f.older()
+        if f is None:
+            return get_fs_base()
+
+    try:
+        return f.read_var("arg")
+    except ValueError:
+        return get_fs_base()
+
 def get_glibc_pointer_guard():
     '''Fetch glibc pointer guard value'''
-    fs_base = get_fs_base()
+    fs_base = pthread_self()
     return gdb.parse_and_eval('*(uint64_t*)((uint64_t)%s + 0x30)' % fs_base)
 
 def glibc_ptr_demangle(val, pointer_guard):
-- 
2.5.0

[Qemu-devel] [PATCH 2/3] qemu-gdb: extract parts of "qemu coroutine" implementation

[Paolo Bonzini]

Provide useful Python functions to reach and decipher a jmpbuf.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 scripts/qemugdb/coroutine.py | 56 +++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/scripts/qemugdb/coroutine.py b/scripts/qemugdb/coroutine.py
index b1d4546..b5c8678 100644
--- a/scripts/qemugdb/coroutine.py
+++ b/scripts/qemugdb/coroutine.py
@@ -47,8 +47,7 @@ def glibc_ptr_demangle(val, pointer_guard):
     '''Undo effect of glibc's PTR_MANGLE()'''
     return gdb.parse_and_eval('(((uint64_t)%s >> 0x11) | ((uint64_t)%s << (64 - 0x11))) ^ (uint64_t)%s' % (val, val, pointer_guard))
 
-def bt_jmpbuf(jmpbuf):
-    '''Backtrace a jmpbuf'''
+def get_jmpbuf_regs(jmpbuf):
     JB_RBX  = 0
     JB_RBP  = 1
     JB_R12  = 2
@@ -58,35 +57,35 @@ def bt_jmpbuf(jmpbuf):
     JB_RSP  = 6
     JB_PC   = 7
 
-    old_rbx = gdb.parse_and_eval('(uint64_t)$rbx')
-    old_rbp = gdb.parse_and_eval('(uint64_t)$rbp')
-    old_rsp = gdb.parse_and_eval('(uint64_t)$rsp')
-    old_r12 = gdb.parse_and_eval('(uint64_t)$r12')
-    old_r13 = gdb.parse_and_eval('(uint64_t)$r13')
-    old_r14 = gdb.parse_and_eval('(uint64_t)$r14')
-    old_r15 = gdb.parse_and_eval('(uint64_t)$r15')
-    old_rip = gdb.parse_and_eval('(uint64_t)$rip')
-
     pointer_guard = get_glibc_pointer_guard()
-    gdb.execute('set $rbx = %s' % jmpbuf[JB_RBX])
-    gdb.execute('set $rbp = %s' % glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard))
-    gdb.execute('set $rsp = %s' % glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard))
-    gdb.execute('set $r12 = %s' % jmpbuf[JB_R12])
-    gdb.execute('set $r13 = %s' % jmpbuf[JB_R13])
-    gdb.execute('set $r14 = %s' % jmpbuf[JB_R14])
-    gdb.execute('set $r15 = %s' % jmpbuf[JB_R15])
-    gdb.execute('set $rip = %s' % glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard))
+    return {'rbx': jmpbuf[JB_RBX],
+        'rbp': glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard),
+        'rsp': glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard),
+        'r12': jmpbuf[JB_R12],
+        'r13': jmpbuf[JB_R13],
+        'r14': jmpbuf[JB_R14],
+        'r15': jmpbuf[JB_R15],
+        'rip': glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard) }
+
+def bt_jmpbuf(jmpbuf):
+    '''Backtrace a jmpbuf'''
+    regs = get_jmpbuf_regs(jmpbuf)
+    old = dict()
+
+    for i in regs:
+        old[i] = gdb.parse_and_eval('(uint64_t)$%s' % i)
+
+    for i in regs:
+        gdb.execute('set $%s = %s' % (i, regs[i]))
 
     gdb.execute('bt')
 
-    gdb.execute('set $rbx = %s' % old_rbx)
-    gdb.execute('set $rbp = %s' % old_rbp)
-    gdb.execute('set $rsp = %s' % old_rsp)
-    gdb.execute('set $r12 = %s' % old_r12)
-    gdb.execute('set $r13 = %s' % old_r13)
-    gdb.execute('set $r14 = %s' % old_r14)
-    gdb.execute('set $r15 = %s' % old_r15)
-    gdb.execute('set $rip = %s' % old_rip)
+    for i in regs:
+        gdb.execute('set $%s = %s' % (i, old[i]))
+
+def coroutine_to_jmpbuf(co):
+    coroutine_pointer = co.cast(gdb.lookup_type('CoroutineUContext').pointer())
+    return coroutine_pointer['env']['__jmpbuf']
 
 
 class CoroutineCommand(gdb.Command):
@@ -101,5 +100,4 @@ class CoroutineCommand(gdb.Command):
             gdb.write('usage: qemu coroutine <coroutine-pointer>\n')
             return
 
-        coroutine_pointer = gdb.parse_and_eval(argv[0]).cast(gdb.lookup_type('CoroutineUContext').pointer())
-        bt_jmpbuf(coroutine_pointer['env']['__jmpbuf'])
+        bt_jmpbuf(coroutine_to_jmpbuf(gdb.parse_and_eval(argv[0])))
-- 
2.5.0

[Qemu-devel] [PATCH 3/3] qemu-gdb: add $qemu_coroutine_sp and $qemu_coroutine_pc

[Paolo Bonzini]

These can be useful to manually get a stack trace of a coroutine inside
a core dump.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 scripts/qemu-gdb.py          |  3 +++
 scripts/qemugdb/coroutine.py | 16 ++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/scripts/qemu-gdb.py b/scripts/qemu-gdb.py
index d6f2e5a..ef2fd19 100644
--- a/scripts/qemu-gdb.py
+++ b/scripts/qemu-gdb.py
@@ -38,6 +38,9 @@ QemuCommand()
 coroutine.CoroutineCommand()
 mtree.MtreeCommand()
 
+coroutine.CoroutineSPFunction()
+coroutine.CoroutinePCFunction()
+
 # Default to silently passing through SIGUSR1, because QEMU sends it
 # to itself a lot.
 gdb.execute('handle SIGUSR1 pass noprint nostop')
diff --git a/scripts/qemugdb/coroutine.py b/scripts/qemugdb/coroutine.py
index b5c8678..ab69979 100644
--- a/scripts/qemugdb/coroutine.py
+++ b/scripts/qemugdb/coroutine.py
@@ -15,6 +15,8 @@
 
 import gdb
 
+VOID_PTR = gdb.lookup_type('void').pointer()
+
 def get_fs_base():
     '''Fetch %fs base value using arch_prctl(ARCH_GET_FS).  This is
        pthread_self().'''
@@ -101,3 +103,17 @@ class CoroutineCommand(gdb.Command):
             return
 
         bt_jmpbuf(coroutine_to_jmpbuf(gdb.parse_and_eval(argv[0])))
+
+class CoroutineSPFunction(gdb.Function):
+    def __init__(self):
+        gdb.Function.__init__(self, 'qemu_coroutine_sp')
+
+    def invoke(self, addr):
+        return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rsp'].cast(VOID_PTR)
+
+class CoroutinePCFunction(gdb.Function):
+    def __init__(self):
+        gdb.Function.__init__(self, 'qemu_coroutine_pc')
+
+    def invoke(self, addr):
+        return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rip'].cast(VOID_PTR)
-- 
2.5.0

Re: [Qemu-devel] [PATCH 0/3] qemu-gdb: add functionality for inspecting core dumps

[Stefan Hajnoczi]

On Mon, Oct 12, 2015 at 10:02:51AM +0200, Paolo Bonzini wrote:
> Currently it is very hard to inspect coroutine in core dumps, because
> none of the qemu-gdb functionality works.  Fixing this is not really
> possible because "bt" only works on the core dump's stack pointer and
> program counter, but the situation would improve noticeably if only
> a coroutine's stack pointer and program counter were accessible at all;
> that would allow inspecting the coroutine's stack and building a
> stack trace from the hex dump of the stack.
> 
> The main hurdle is that glibc_pointer_guard() cannot be run on a core
> dump, because get_fs_base() uses the arch_prctl system call.  The first
> patch modifies that to use the gdb API instead.  The second and third
> patch then add the new functions.
> 
> Paolo
> 
> Paolo Bonzini (3):
>   qemu-gdb: allow using glibc_pointer_guard() on core dumps
>   qemu-gdb: extract parts of "qemu coroutine" implementation
>   qemu-gdb: add $qemu_coroutine_sp and $qemu_coroutine_pc
> 
>  scripts/qemu-gdb.py          |  3 ++
>  scripts/qemugdb/coroutine.py | 90 +++++++++++++++++++++++++++++---------------
>  2 files changed, 62 insertions(+), 31 deletions(-)

Thanks, applied to my block tree:
https://github.com/stefanha/qemu/commits/block

Stefan